Add commit status summary table to reduce query from commit status table (#30223)

lunny · wolfogre · web-flow · commit fc34481d054a · 2024-04-12T09:41:50.000+08:00
This PR adds a new table named commit status summary to reduce queries
from the commit status table. After this change, commit status summary
table will be used for the final result, commit status table will be for
details.

---------

Co-authored-by: Jason Song &lt;i@wolfogre.com&gt;
diff --git a/models/git/commit_status.go b/models/git/commit_status.go
@@ -292,30 +292,27 @@ func GetLatestCommitStatus(ctx context.Context, repoID int64, sha string, listOp
 }
 
 // GetLatestCommitStatusForPairs returns all statuses with a unique context for a given list of repo-sha pairs
-func GetLatestCommitStatusForPairs(ctx context.Context, repoIDsToLatestCommitSHAs map[int64]string, listOptions db.ListOptions) (map[int64][]*CommitStatus, error) {
+func GetLatestCommitStatusForPairs(ctx context.Context, repoSHAs []RepoSHA) (map[int64][]*CommitStatus, error) {
 	type result struct {
 		Index  int64
 		RepoID int64
+		SHA    string
 	}
 
-	results := make([]result, 0, len(repoIDsToLatestCommitSHAs))
+	results := make([]result, 0, len(repoSHAs))
 
 	getBase := func() *xorm.Session {
 		return db.GetEngine(ctx).Table(&CommitStatus{})
 	}
 
 	// Create a disjunction of conditions for each repoID and SHA pair
-	conds := make([]builder.Cond, 0, len(repoIDsToLatestCommitSHAs))
-	for repoID, sha := range repoIDsToLatestCommitSHAs {
-		conds = append(conds, builder.Eq{"repo_id": repoID, "sha": sha})
+	conds := make([]builder.Cond, 0, len(repoSHAs))
+	for _, repoSHA := range repoSHAs {
+		conds = append(conds, builder.Eq{"repo_id": repoSHA.RepoID, "sha": repoSHA.SHA})
 	}
 	sess := getBase().Where(builder.Or(conds...)).
-		Select("max( `index` ) as `index`, repo_id").
-		GroupBy("context_hash, repo_id").OrderBy("max( `index` ) desc")
-
-	if !listOptions.IsListAll() {
-		sess = db.SetSessionPagination(sess, &listOptions)
-	}
+		Select("max( `index` ) as `index`, repo_id, sha").
+		GroupBy("context_hash, repo_id, sha").OrderBy("max( `index` ) desc")
 
 	err := sess.Find(&results)
 	if err != nil {
@@ -332,7 +329,7 @@ func GetLatestCommitStatusForPairs(ctx context.Context, repoIDsToLatestCommitSHA
 			cond := builder.Eq{
 				"`index`": result.Index,
 				"repo_id": result.RepoID,
-				"sha":     repoIDsToLatestCommitSHAs[result.RepoID],
+				"sha":     result.SHA,
 			}
 			conds = append(conds, cond)
 		}
diff --git a/models/git/commit_status_summary.go b/models/git/commit_status_summary.go
@@ -0,0 +1,84 @@
+// Copyright 2024 Gitea. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package git
+
+import (
+	"context"
+
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/modules/setting"
+	api "code.gitea.io/gitea/modules/structs"
+
+	"xorm.io/builder"
+)
+
+// CommitStatusSummary holds the latest commit Status of a single Commit
+type CommitStatusSummary struct {
+	ID     int64                 `xorm:"pk autoincr"`
+	RepoID int64                 `xorm:"INDEX UNIQUE(repo_id_sha)"`
+	SHA    string                `xorm:"VARCHAR(64) NOT NULL INDEX UNIQUE(repo_id_sha)"`
+	State  api.CommitStatusState `xorm:"VARCHAR(7) NOT NULL"`
+}
+
+func init() {
+	db.RegisterModel(new(CommitStatusSummary))
+}
+
+type RepoSHA struct {
+	RepoID int64
+	SHA    string
+}
+
+func GetLatestCommitStatusForRepoAndSHAs(ctx context.Context, repoSHAs []RepoSHA) ([]*CommitStatus, error) {
+	cond := builder.NewCond()
+	for _, rs := range repoSHAs {
+		cond = cond.Or(builder.Eq{"repo_id": rs.RepoID, "sha": rs.SHA})
+	}
+
+	var summaries []CommitStatusSummary
+	if err := db.GetEngine(ctx).Where(cond).Find(&summaries); err != nil {
+		return nil, err
+	}
+
+	commitStatuses := make([]*CommitStatus, 0, len(repoSHAs))
+	for _, summary := range summaries {
+		commitStatuses = append(commitStatuses, &CommitStatus{
+			RepoID: summary.RepoID,
+			SHA:    summary.SHA,
+			State:  summary.State,
+		})
+	}
+	return commitStatuses, nil
+}
+
+func UpdateCommitStatusSummary(ctx context.Context, repoID int64, sha string) error {
+	commitStatuses, _, err := GetLatestCommitStatus(ctx, repoID, sha, db.ListOptionsAll)
+	if err != nil {
+		return err
+	}
+	state := CalcCommitStatus(commitStatuses)
+	// mysql will return 0 when update a record which state hasn't been changed which behaviour is different from other database,
+	// so we need to use insert in on duplicate
+	if setting.Database.Type.IsMySQL() {
+		_, err := db.GetEngine(ctx).Exec("INSERT INTO commit_status_summary (repo_id,sha,state) VALUES (?,?,?) ON DUPLICATE KEY UPDATE state=?",
+			repoID, sha, state.State, state.State)
+		return err
+	}
+
+	if cnt, err := db.GetEngine(ctx).Where("repo_id=? AND sha=?", repoID, sha).
+		Cols("state").
+		Update(&CommitStatusSummary{
+			State: state.State,
+		}); err != nil {
+		return err
+	} else if cnt == 0 {
+		_, err = db.GetEngine(ctx).Insert(&CommitStatusSummary{
+			RepoID: repoID,
+			SHA:    sha,
+			State:  state.State,
+		})
+		return err
+	}
+	return nil
+}
diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go
@@ -576,7 +576,10 @@ var migrations = []Migration{
 
 	// Gitea 1.22.0 ends at 294
 
+	// v294 -> v295
 	NewMigration("Add unique index for project issue table", v1_23.AddUniqueIndexForProjectIssue),
+	// v295 -> v296
+	NewMigration("Add commit status summary table", v1_23.AddCommitStatusSummary),
 }
 
 // GetCurrentDBVersion returns the current db version
diff --git a/models/migrations/v1_23/v295.go b/models/migrations/v1_23/v295.go
@@ -0,0 +1,18 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package v1_23 //nolint
+
+import "xorm.io/xorm"
+
+func AddCommitStatusSummary(x *xorm.Engine) error {
+	type CommitStatusSummary struct {
+		ID     int64  `xorm:"pk autoincr"`
+		RepoID int64  `xorm:"INDEX UNIQUE(repo_id_sha)"`
+		SHA    string `xorm:"VARCHAR(64) NOT NULL INDEX UNIQUE(repo_id_sha)"`
+		State  string `xorm:"VARCHAR(7) NOT NULL"`
+	}
+	// there is no migrations because if there is no data on this table, it will fall back to get data
+	// from commit status
+	return x.Sync2(new(CommitStatusSummary))
+}
diff --git a/services/actions/commit_status.go b/services/actions/commit_status.go
@@ -16,6 +16,7 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	api "code.gitea.io/gitea/modules/structs"
 	webhook_module "code.gitea.io/gitea/modules/webhook"
+	commitstatus_service "code.gitea.io/gitea/services/repository/commitstatus"
 
 	"github.com/nektos/act/pkg/jobparser"
 )
@@ -122,18 +123,13 @@ func createCommitStatus(ctx context.Context, job *actions_model.ActionRunJob) er
 	if err != nil {
 		return fmt.Errorf("HashTypeInterfaceFromHashString: %w", err)
 	}
-	if err := git_model.NewCommitStatus(ctx, git_model.NewCommitStatusOptions{
-		Repo:    repo,
-		SHA:     commitID,
-		Creator: creator,
-		CommitStatus: &git_model.CommitStatus{
-			SHA:         sha,
-			TargetURL:   fmt.Sprintf("%s/jobs/%d", run.Link(), index),
-			Description: description,
-			Context:     ctxname,
-			CreatorID:   creator.ID,
-			State:       state,
-		},
+	if err := commitstatus_service.CreateCommitStatus(ctx, repo, creator, commitID.String(), &git_model.CommitStatus{
+		SHA:         sha,
+		TargetURL:   fmt.Sprintf("%s/jobs/%d", run.Link(), index),
+		Description: description,
+		Context:     ctxname,
+		CreatorID:   creator.ID,
+		State:       state,
 	}); err != nil {
 		return fmt.Errorf("NewCommitStatus: %w", err)
 	}
diff --git a/services/repository/commitstatus/commitstatus.go b/services/repository/commitstatus/commitstatus.go
@@ -7,6 +7,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"fmt"
+	"slices"
 
 	"code.gitea.io/gitea/models/db"
 	git_model "code.gitea.io/gitea/models/git"
@@ -59,13 +60,19 @@ func CreateCommitStatus(ctx context.Context, repo *repo_model.Repository, creato
 		sha = commit.ID.String()
 	}
 
-	if err := git_model.NewCommitStatus(ctx, git_model.NewCommitStatusOptions{
-		Repo:         repo,
-		Creator:      creator,
-		SHA:          commit.ID,
-		CommitStatus: status,
+	if err := db.WithTx(ctx, func(ctx context.Context) error {
+		if err := git_model.NewCommitStatus(ctx, git_model.NewCommitStatusOptions{
+			Repo:         repo,
+			Creator:      creator,
+			SHA:          commit.ID,
+			CommitStatus: status,
+		}); err != nil {
+			return fmt.Errorf("NewCommitStatus[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err)
+		}
+
+		return git_model.UpdateCommitStatusSummary(ctx, repo.ID, commit.ID.String())
 	}); err != nil {
-		return fmt.Errorf("NewCommitStatus[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err)
+		return err
 	}
 
 	defaultBranchCommit, err := gitRepo.GetBranchCommit(repo.DefaultBranch)
@@ -114,8 +121,35 @@ func FindReposLastestCommitStatuses(ctx context.Context, repos []*repo_model.Rep
 		return nil, fmt.Errorf("FindBranchesByRepoAndBranchName: %v", err)
 	}
 
+	var repoSHAs []git_model.RepoSHA
+	for id, sha := range repoIDsToLatestCommitSHAs {
+		repoSHAs = append(repoSHAs, git_model.RepoSHA{RepoID: id, SHA: sha})
+	}
+
+	summaryResults, err := git_model.GetLatestCommitStatusForRepoAndSHAs(ctx, repoSHAs)
+	if err != nil {
+		return nil, fmt.Errorf("GetLatestCommitStatusForRepoAndSHAs: %v", err)
+	}
+
+	for _, summary := range summaryResults {
+		for i, repo := range repos {
+			if repo.ID == summary.RepoID {
+				results[i] = summary
+				_ = slices.DeleteFunc(repoSHAs, func(repoSHA git_model.RepoSHA) bool {
+					return repoSHA.RepoID == repo.ID
+				})
+				if results[i].State != "" {
+					if err := updateCommitStatusCache(ctx, repo.ID, repo.DefaultBranch, results[i].State); err != nil {
+						log.Error("updateCommitStatusCache[%d:%s] failed: %v", repo.ID, repo.DefaultBranch, err)
+					}
+				}
+				break
+			}
+		}
+	}
+
 	// call the database O(1) times to get the commit statuses for all repos
-	repoToItsLatestCommitStatuses, err := git_model.GetLatestCommitStatusForPairs(ctx, repoIDsToLatestCommitSHAs, db.ListOptionsAll)
+	repoToItsLatestCommitStatuses, err := git_model.GetLatestCommitStatusForPairs(ctx, repoSHAs)
 	if err != nil {
 		return nil, fmt.Errorf("GetLatestCommitStatusForPairs: %v", err)
 	}
diff --git a/tests/integration/pull_status_test.go b/tests/integration/pull_status_test.go
@@ -12,6 +12,9 @@ import (
 	"testing"
 
 	auth_model "code.gitea.io/gitea/models/auth"
+	git_model "code.gitea.io/gitea/models/git"
+	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/models/unittest"
 	api "code.gitea.io/gitea/modules/structs"
 
 	"github.com/stretchr/testify/assert"
@@ -90,6 +93,10 @@ func TestPullCreate_CommitStatus(t *testing.T) {
 			assert.True(t, ok)
 			assert.Contains(t, cls, statesIcons[status])
 		}
+
+		repo1 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{OwnerName: "user1", Name: "repo1"})
+		css := unittest.AssertExistsAndLoadBean(t, &git_model.CommitStatusSummary{RepoID: repo1.ID, SHA: commitID})
+		assert.EqualValues(t, api.CommitStatusWarning, css.State)
 	})
 }
 

Original file line number	Diff line number	Diff line change
`@@ -576,7 +576,10 @@ var migrations = []Migration{`
`576`	`576`
`577`	`577`	`// Gitea 1.22.0 ends at 294`
`578`	`578`
	`579`	`+ // v294 -> v295`
`579`	`580`	`NewMigration("Add unique index for project issue table", v1_23.AddUniqueIndexForProjectIssue),`
	`581`	`+ // v295 -> v296`
	`582`	`+ NewMigration("Add commit status summary table", v1_23.AddCommitStatusSummary),`
`580`	`583`	`}`
`581`	`584`
`582`	`585`	`// GetCurrentDBVersion returns the current db version`