diff --git a/cmd/restore_repo.go b/cmd/restore_repo.go index 23932f821c286..90d66a7cc3bf5 100644 --- a/cmd/restore_repo.go +++ b/cmd/restore_repo.go @@ -22,6 +22,15 @@ var CmdRestoreRepository = cli.Command{ Description: "This is a command for restoring the repository data.", Action: runRestoreRepository, Flags: []cli.Flag{ + cli.StringFlag{ + Name: "data_type, d", + Value: "gitea", + Usage: "The data type that will be imported, default is gitea, options are gitea, github", + }, + cli.StringFlag{ + Name: "repo_filepath, f", + Usage: "Repository compressed data path to restore from", + }, cli.StringFlag{ Name: "repo_dir, r", Value: "./data", @@ -61,6 +70,8 @@ func runRestoreRepository(c *cli.Context) error { } statusCode, errStr := private.RestoreRepo( ctx, + c.String("data_type"), + c.String("repo_filepath"), c.String("repo_dir"), c.String("owner_name"), c.String("repo_name"), diff --git a/models/issues/label.go b/models/issues/label.go index dbb7a139effdf..f8c6371ffa0a8 100644 --- a/models/issues/label.go +++ b/models/issues/label.go @@ -300,7 +300,7 @@ func DeleteLabel(id, labelID int64) error { // GetLabelByID returns a label by given ID. func GetLabelByID(ctx context.Context, labelID int64) (*Label, error) { if labelID <= 0 { - return nil, ErrLabelNotExist{labelID} + return nil, ErrLabelNotExist{LabelID: labelID} } l := &Label{} @@ -308,7 +308,7 @@ func GetLabelByID(ctx context.Context, labelID int64) (*Label, error) { if err != nil { return nil, err } else if !has { - return nil, ErrLabelNotExist{l.ID} + return nil, ErrLabelNotExist{LabelID: l.ID} } return l, nil } @@ -434,6 +434,18 @@ func CountLabelsByRepoID(repoID int64) (int64, error) { return db.GetEngine(db.DefaultContext).Where("repo_id = ?", repoID).Count(&Label{}) } +// GetLabelByRepoIDAndName get label from repo +func GetLabelByRepoIDAndName(repoID int64, name string) (*Label, error) { + var label Label + has, err := db.GetEngine(db.DefaultContext).Where("repo_id = ? AND name = ?", repoID, name).Get(&label) + if err != nil { + return nil, err + } else if !has { + return nil, ErrLabelNotExist{} + } + return &label, nil +} + // ________ // \_____ \_______ ____ // / | \_ __ \/ ___\ diff --git a/models/migrate.go b/models/migrate.go index 82cacd4a75b22..ac3d116c220bc 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -83,6 +83,16 @@ func insertIssue(ctx context.Context, issue *issues_model.Issue) error { } } + for _, attach := range issue.Attachments { + attach.IssueID = issue.ID + } + + if len(issue.Attachments) > 0 { + if _, err := sess.Insert(issue.Attachments); err != nil { + return err + } + } + return nil } @@ -116,6 +126,16 @@ func InsertIssueComments(comments []*issues_model.Comment) error { return err } } + + for _, attach := range comment.Attachments { + attach.IssueID = comment.IssueID + attach.CommentID = comment.ID + } + if len(comment.Attachments) > 0 { + if err := db.Insert(ctx, comment.Attachments); err != nil { + return err + } + } } for issueID := range issueIDs { diff --git a/modules/migration/asset.go b/modules/migration/asset.go new file mode 100644 index 0000000000000..162e3ce74d274 --- /dev/null +++ b/modules/migration/asset.go @@ -0,0 +1,24 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migration + +import ( + "io" + "time" +) + +// Asset represents an asset for issue, comment, release +type Asset struct { + ID int64 + Name string + ContentType *string `yaml:"content_type"` + Size *int + DownloadCount *int `yaml:"download_count"` + Created time.Time + Updated time.Time + DownloadURL *string `yaml:"download_url"` + OriginalURL string + // if DownloadURL is nil, the function should be invoked + DownloadFunc func() (io.ReadCloser, error) `yaml:"-"` +} diff --git a/modules/migration/comment.go b/modules/migration/comment.go index f994e972ed469..9f200059e5d8e 100644 --- a/modules/migration/comment.go +++ b/modules/migration/comment.go @@ -13,6 +13,43 @@ type Commentable interface { GetContext() DownloaderContext } +/* +"comment", + "reopen", + "close", + "issue_ref", + "commit_ref", + "comment_ref", + "pull_ref", + "label", + "milestone", + "assignees", + "change_title", + "delete_branch", + "start_tracking", + "stop_tracking", + "add_time_manual", + "cancel_tracking", + "added_deadline", + "modified_deadline", + "removed_deadline", + "add_dependency", + "remove_dependency", + "code", + "review", + "lock", + "unlock", + "change_target_branch", + "delete_time_manual", + "review_request", + "merge_pull", + "pull_push", + "project", + "project_board", + "dismiss_review", + "change_issue_ref", +*/ + // Comment is a standard comment information type Comment struct { IssueIndex int64 `yaml:"issue_index"` @@ -26,6 +63,7 @@ type Comment struct { Content string Reactions []*Reaction Meta map[string]interface{} `yaml:"meta,omitempty"` // see models/issues/comment.go for fields in Comment struct + Assets []*Asset } // GetExternalName ExternalUserMigrated interface diff --git a/modules/migration/downloader.go b/modules/migration/downloader.go index ebd3672d63699..023f961e60cc3 100644 --- a/modules/migration/downloader.go +++ b/modules/migration/downloader.go @@ -10,6 +10,20 @@ import ( "code.gitea.io/gitea/modules/structs" ) +// GetCommentOptions represents an options for get comment +type GetCommentOptions struct { + Commentable + Page int + PageSize int +} + +// GetReviewOptions represents an options for get reviews +type GetReviewOptions struct { + Reviewable + Page int + PageSize int +} + // Downloader downloads the site repo information type Downloader interface { SetContext(context.Context) @@ -19,12 +33,14 @@ type Downloader interface { GetReleases() ([]*Release, error) GetLabels() ([]*Label, error) GetIssues(page, perPage int) ([]*Issue, bool, error) - GetComments(commentable Commentable) ([]*Comment, bool, error) + GetComments(opts GetCommentOptions) ([]*Comment, bool, error) GetAllComments(page, perPage int) ([]*Comment, bool, error) SupportGetRepoComments() bool GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) - GetReviews(reviewable Reviewable) ([]*Review, error) + GetReviews(reviewable Reviewable) ([]*Review, bool, error) + SupportGetRepoReviews() bool FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) + CleanUp() } // DownloaderFactory defines an interface to match a downloader implementation and create a downloader diff --git a/modules/migration/issue.go b/modules/migration/issue.go index 7cb9f84b0d2d9..7bde676083cbb 100644 --- a/modules/migration/issue.go +++ b/modules/migration/issue.go @@ -8,21 +8,22 @@ import "time" // Issue is a standard issue information type Issue struct { - Number int64 `json:"number"` - PosterID int64 `yaml:"poster_id" json:"poster_id"` - PosterName string `yaml:"poster_name" json:"poster_name"` - PosterEmail string `yaml:"poster_email" json:"poster_email"` - Title string `json:"title"` - Content string `json:"content"` - Ref string `json:"ref"` - Milestone string `json:"milestone"` - State string `json:"state"` // closed, open - IsLocked bool `yaml:"is_locked" json:"is_locked"` - Created time.Time `json:"created"` - Updated time.Time `json:"updated"` - Closed *time.Time `json:"closed"` - Labels []*Label `json:"labels"` - Reactions []*Reaction `json:"reactions"` + Number int64 `json:"number"` + PosterID int64 `yaml:"poster_id" json:"poster_id"` + PosterName string `yaml:"poster_name" json:"poster_name"` + PosterEmail string `yaml:"poster_email" json:"poster_email"` + Title string `json:"title"` + Content string `json:"content"` + Ref string `json:"ref"` + Milestone string `json:"milestone"` + State string `json:"state"` // closed, open + IsLocked bool `yaml:"is_locked" json:"is_locked"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` + Closed *time.Time `json:"closed"` + Labels []*Label `json:"labels"` + Reactions []*Reaction `json:"reactions"` + Assets []*Asset Assignees []string `json:"assignees"` ForeignIndex int64 `json:"foreign_id"` Context DownloaderContext `yaml:"-"` diff --git a/modules/migration/null_downloader.go b/modules/migration/null_downloader.go index e5b69331df7af..f6c1e72dcb472 100644 --- a/modules/migration/null_downloader.go +++ b/modules/migration/null_downloader.go @@ -47,7 +47,7 @@ func (n NullDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { } // GetComments returns comments of an issue or PR -func (n NullDownloader) GetComments(commentable Commentable) ([]*Comment, bool, error) { +func (n NullDownloader) GetComments(commentable GetCommentOptions) ([]*Comment, bool, error) { return nil, false, ErrNotSupported{Entity: "Comments"} } @@ -62,8 +62,8 @@ func (n NullDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool } // GetReviews returns pull requests review -func (n NullDownloader) GetReviews(reviewable Reviewable) ([]*Review, error) { - return nil, ErrNotSupported{Entity: "Reviews"} +func (n NullDownloader) GetReviews(reviewable Reviewable) ([]*Review, bool, error) { + return nil, false, ErrNotSupported{Entity: "Reviews"} } // FormatCloneURL add authentication into remote URLs @@ -86,3 +86,12 @@ func (n NullDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr string) ( func (n NullDownloader) SupportGetRepoComments() bool { return false } + +// SupportGetRepoReviews return true if it supports get repo pullrequest reviews +func (n NullDownloader) SupportGetRepoReviews() bool { + return false +} + +// CleanUp clean the downloader temporary resources +func (n NullDownloader) CleanUp() { +} diff --git a/modules/migration/pullrequest.go b/modules/migration/pullrequest.go index 4e7500f0d67f8..a38f10cf6add6 100644 --- a/modules/migration/pullrequest.go +++ b/modules/migration/pullrequest.go @@ -36,7 +36,8 @@ type PullRequest struct { Reactions []*Reaction ForeignIndex int64 Context DownloaderContext `yaml:"-"` - EnsuredSafe bool `yaml:"ensured_safe"` + Assets []*Asset + EnsuredSafe bool `yaml:"ensured_safe"` } func (p *PullRequest) GetLocalIndex() int64 { return p.Number } diff --git a/modules/migration/release.go b/modules/migration/release.go index f92cf25e7bc5d..8bb61d1a5ead7 100644 --- a/modules/migration/release.go +++ b/modules/migration/release.go @@ -4,25 +4,9 @@ package migration import ( - "io" "time" ) -// ReleaseAsset represents a release asset -type ReleaseAsset struct { - ID int64 - Name string - ContentType *string `yaml:"content_type"` - Size *int - DownloadCount *int `yaml:"download_count"` - Created time.Time - Updated time.Time - - DownloadURL *string `yaml:"download_url"` // SECURITY: It is the responsibility of downloader to make sure this is safe - // if DownloadURL is nil, the function should be invoked - DownloadFunc func() (io.ReadCloser, error) `yaml:"-"` // SECURITY: It is the responsibility of downloader to make sure this is safe -} - // Release represents a release type Release struct { TagName string `yaml:"tag_name"` // SECURITY: This must pass git.IsValidRefPattern @@ -34,7 +18,7 @@ type Release struct { PublisherID int64 `yaml:"publisher_id"` PublisherName string `yaml:"publisher_name"` PublisherEmail string `yaml:"publisher_email"` - Assets []*ReleaseAsset + Assets []*Asset Created time.Time Published time.Time } diff --git a/modules/migration/retry_downloader.go b/modules/migration/retry_downloader.go index 1cacf5f37582f..50953068c92b3 100644 --- a/modules/migration/retry_downloader.go +++ b/modules/migration/retry_downloader.go @@ -147,7 +147,7 @@ func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { } // GetComments returns a repository's comments with retry -func (d *RetryDownloader) GetComments(commentable Commentable) ([]*Comment, bool, error) { +func (d *RetryDownloader) GetComments(opts GetCommentOptions) ([]*Comment, bool, error) { var ( comments []*Comment isEnd bool @@ -155,7 +155,7 @@ func (d *RetryDownloader) GetComments(commentable Commentable) ([]*Comment, bool ) err = d.retry(func() error { - comments, isEnd, err = d.Downloader.GetComments(commentable) + comments, isEnd, err = d.Downloader.GetComments(opts) return err }) @@ -179,16 +179,17 @@ func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bo } // GetReviews returns pull requests reviews -func (d *RetryDownloader) GetReviews(reviewable Reviewable) ([]*Review, error) { +func (d *RetryDownloader) GetReviews(reviwable Reviewable) ([]*Review, bool, error) { var ( reviews []*Review + isEnd bool err error ) err = d.retry(func() error { - reviews, err = d.Downloader.GetReviews(reviewable) + reviews, isEnd, err = d.Downloader.GetReviews(reviwable) return err }) - return reviews, err + return reviews, isEnd, err } diff --git a/modules/migration/review.go b/modules/migration/review.go index a420c130c7e29..cc8fc81d58784 100644 --- a/modules/migration/review.go +++ b/modules/migration/review.go @@ -22,16 +22,21 @@ const ( // Review is a standard review information type Review struct { - ID int64 - IssueIndex int64 `yaml:"issue_index"` - ReviewerID int64 `yaml:"reviewer_id"` - ReviewerName string `yaml:"reviewer_name"` - Official bool - CommitID string `yaml:"commit_id"` - Content string - CreatedAt time.Time `yaml:"created_at"` - State string // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT - Comments []*ReviewComment + ID int64 + IssueIndex int64 `yaml:"issue_index"` + ReviewerID int64 `yaml:"reviewer_id"` + ReviewerName string `yaml:"reviewer_name"` + ReviewerEmail string + Official bool + CommitID string `yaml:"commit_id"` + Content string + CreatedAt time.Time `yaml:"created_at"` + State string // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT + Comments []*ReviewComment + ResolvedAt *time.Time `yaml:"resolved_at"` + ResolverID int64 `yaml:"resolver_id"` + ResolverName string `yaml:"resolver_name"` + ResolverEmail string `yaml:"resolver_email"` } // GetExternalName ExternalUserMigrated interface @@ -42,16 +47,18 @@ func (r *Review) GetExternalID() int64 { return r.ReviewerID } // ReviewComment represents a review comment type ReviewComment struct { - ID int64 - InReplyTo int64 `yaml:"in_reply_to"` - Content string - TreePath string `yaml:"tree_path"` - DiffHunk string `yaml:"diff_hunk"` - Position int - Line int - CommitID string `yaml:"commit_id"` - PosterID int64 `yaml:"poster_id"` - Reactions []*Reaction - CreatedAt time.Time `yaml:"created_at"` - UpdatedAt time.Time `yaml:"updated_at"` + ID int64 + InReplyTo int64 `yaml:"in_reply_to"` + Content string + TreePath string `yaml:"tree_path"` + DiffHunk string `yaml:"diff_hunk"` + Position int + Line int + CommitID string `yaml:"commit_id"` + PosterID int64 `yaml:"poster_id"` + PosterName string + PosterEmail string + Reactions []*Reaction + CreatedAt time.Time `yaml:"created_at"` + UpdatedAt time.Time `yaml:"updated_at"` } diff --git a/modules/private/restore_repo.go b/modules/private/restore_repo.go index f40d914a7bb57..3d653f031d411 100644 --- a/modules/private/restore_repo.go +++ b/modules/private/restore_repo.go @@ -16,26 +16,30 @@ import ( // RestoreParams structure holds a data for restore repository type RestoreParams struct { - RepoDir string - OwnerName string - RepoName string - Units []string - Validation bool + Type string + RepoFilePath string + RepoDir string + OwnerName string + RepoName string + Units []string + Validation bool } // RestoreRepo calls the internal RestoreRepo function -func RestoreRepo(ctx context.Context, repoDir, ownerName, repoName string, units []string, validation bool) (int, string) { +func RestoreRepo(ctx context.Context, tp, repoPath, repoDir, ownerName, repoName string, units []string, validation bool) (int, string) { reqURL := setting.LocalURL + "api/internal/restore_repo" req := newInternalRequest(ctx, reqURL, "POST") req.SetTimeout(3*time.Second, 0) // since the request will spend much time, don't timeout req = req.Header("Content-Type", "application/json") jsonBytes, _ := json.Marshal(RestoreParams{ - RepoDir: repoDir, - OwnerName: ownerName, - RepoName: repoName, - Units: units, - Validation: validation, + Type: tp, + RepoFilePath: repoPath, + RepoDir: repoDir, + OwnerName: ownerName, + RepoName: repoName, + Units: units, + Validation: validation, }) req.Body(jsonBytes) resp, err := req.Response() diff --git a/routers/private/restore_repo.go b/routers/private/restore_repo.go index 97ac9a3c5ad95..13605989b8427 100644 --- a/routers/private/restore_repo.go +++ b/routers/private/restore_repo.go @@ -23,11 +23,13 @@ func RestoreRepo(ctx *myCtx.PrivateContext) { return } params := struct { - RepoDir string - OwnerName string - RepoName string - Units []string - Validation bool + Type string + RepoFilePath string + RepoDir string + OwnerName string + RepoName string + Units []string + Validation bool }{} if err = json.Unmarshal(bs, ¶ms); err != nil { ctx.JSON(http.StatusInternalServerError, private.Response{ @@ -36,18 +38,34 @@ func RestoreRepo(ctx *myCtx.PrivateContext) { return } - if err := migrations.RestoreRepository( - ctx, - params.RepoDir, - params.OwnerName, - params.RepoName, - params.Units, - params.Validation, - ); err != nil { - ctx.JSON(http.StatusInternalServerError, private.Response{ - Err: err.Error(), - }) - } else { - ctx.Status(http.StatusOK) + if params.Type == "gitea" { + if err := migrations.RestoreRepository( + ctx, + params.RepoDir, + params.OwnerName, + params.RepoName, + params.Units, + params.Validation, + ); err != nil { + ctx.JSON(http.StatusInternalServerError, private.Response{ + Err: err.Error(), + }) + } else { + ctx.Status(http.StatusOK) + } + } else if params.Type == "github" { + if err := migrations.RestoreFromGithubExportedData( + ctx, + params.RepoFilePath, + params.OwnerName, + params.RepoName, + params.Units, + ); err != nil { + ctx.JSON(http.StatusInternalServerError, private.Response{ + Err: err.Error(), + }) + } else { + ctx.Status(http.StatusOK) + } } } diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index 5bff9e67f340a..4e5ebf0a62a4f 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -1450,11 +1450,13 @@ func ViewIssue(ctx *context.Context) { // check if dependencies can be created across repositories ctx.Data["AllowCrossRepositoryDependencies"] = setting.Service.AllowCrossRepositoryDependencies - if issue.ShowRole, err = roleDescriptor(ctx, repo, issue.Poster, issue); err != nil { - ctx.ServerError("roleDescriptor", err) - return + if issue.OriginalAuthor == "" { + if issue.ShowRole, err = roleDescriptor(ctx, repo, issue.Poster, issue); err != nil { + ctx.ServerError("roleDescriptor", err) + return + } + marked[issue.PosterID] = issue.ShowRole } - marked[issue.PosterID] = issue.ShowRole // Render comments and and fetch participants. participants[0] = issue.Poster @@ -1482,17 +1484,21 @@ func ViewIssue(ctx *context.Context) { ctx.ServerError("RenderString", err) return } - // Check tag. - role, ok = marked[comment.PosterID] - if ok { - comment.ShowRole = role - continue - } + if comment.OriginalAuthor == "" { + // Check tag. + role, ok = marked[comment.PosterID] + if ok { + comment.ShowRole = role + continue + } - comment.ShowRole, err = roleDescriptor(ctx, repo, comment.Poster, issue) - if err != nil { - ctx.ServerError("roleDescriptor", err) - return + comment.ShowRole, err = roleDescriptor(ctx, repo, comment.Poster, issue) + if err != nil { + ctx.ServerError("roleDescriptor", err) + return + } + marked[comment.PosterID] = comment.ShowRole + participants = addParticipant(comment.Poster, participants) } marked[comment.PosterID] = comment.ShowRole participants = addParticipant(comment.Poster, participants) @@ -1581,6 +1587,9 @@ func ViewIssue(ctx *context.Context) { for _, codeComments := range comment.Review.CodeComments { for _, lineComments := range codeComments { for _, c := range lineComments { + if c.OriginalAuthor != "" { + continue + } // Check tag. role, ok = marked[c.PosterID] if ok { diff --git a/routers/web/repo/pull.go b/routers/web/repo/pull.go index 71bc98d13d07c..f2a2e910f64c9 100644 --- a/routers/web/repo/pull.go +++ b/routers/web/repo/pull.go @@ -519,17 +519,16 @@ func PrepareViewPullInfo(ctx *context.Context, issue *issues_model.Issue) *git.C if headBranchExist { if pull.Flow != issues_model.PullRequestFlowGithub { headBranchSha, err = baseGitRepo.GetRefCommitID(pull.GetGitRefName()) - } else { - headBranchSha, err = headGitRepo.GetBranchCommitID(pull.HeadBranch) - } - if err != nil { - ctx.ServerError("GetBranchCommitID", err) - return nil + if err != nil { + ctx.ServerError("GetBranchCommitID", err) + return nil + } } } + headGitRepo.Close() } - if headBranchExist { + if headBranchSha != "" { var err error ctx.Data["UpdateAllowed"], ctx.Data["UpdateByRebaseAllowed"], err = pull_service.IsUserAllowedToUpdate(ctx, pull, ctx.Doer) if err != nil { @@ -587,7 +586,7 @@ func PrepareViewPullInfo(ctx *context.Context, issue *issues_model.Issue) *git.C ctx.Data["HeadBranchCommitID"] = headBranchSha ctx.Data["PullHeadCommitID"] = sha - if pull.HeadRepo == nil || !headBranchExist || headBranchSha != sha { + if pull.HeadRepo == nil || headBranchSha == "" || headBranchSha != sha { ctx.Data["IsPullRequestBroken"] = true if pull.IsSameRepo() { ctx.Data["HeadTarget"] = pull.HeadBranch diff --git a/services/migrations/codebase.go b/services/migrations/codebase.go index c02c8e13a2e3a..170462ad649b0 100644 --- a/services/migrations/codebase.go +++ b/services/migrations/codebase.go @@ -421,10 +421,10 @@ func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, } // GetComments returns comments -func (d *CodebaseDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { - context, ok := commentable.GetContext().(codebaseIssueContext) +func (d *CodebaseDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { + context, ok := opts.Commentable.GetContext().(codebaseIssueContext) if !ok { - return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext()) + return nil, false, fmt.Errorf("unexpected context: %+v", opts.Commentable.GetContext()) } return context.Comments, true, nil @@ -586,6 +586,16 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq return pullRequests, true, nil } +// GetReviews returns pull requests reviews +func (d *CodebaseDownloader) GetReviews(reviwable base.Reviewable) ([]*base.Review, bool, error) { + return []*base.Review{}, true, nil +} + +// GetTopics return repository topics +func (d *CodebaseDownloader) GetTopics() ([]string, error) { + return []string{}, nil +} + func (d *CodebaseDownloader) tryGetUser(userID int64) *codebaseUser { if len(d.userMap) == 0 { var rawUsers struct { diff --git a/services/migrations/codebase_test.go b/services/migrations/codebase_test.go index 68721e06410f1..e77604b94c6e4 100644 --- a/services/migrations/codebase_test.go +++ b/services/migrations/codebase_test.go @@ -106,7 +106,9 @@ func TestCodebaseDownloadRepo(t *testing.T) { }, }, issues) - comments, _, err := downloader.GetComments(issues[0]) + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + Commentable: issues[0], + }) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ { @@ -144,7 +146,9 @@ func TestCodebaseDownloadRepo(t *testing.T) { }, }, prs) - rvs, err := downloader.GetReviews(prs[0]) + rvs, _, err := downloader.GetReviews(base.GetReviewOptions{ + Reviewable: prs[0], + }) assert.NoError(t, err) assert.Empty(t, rvs) } diff --git a/services/migrations/dump.go b/services/migrations/dump.go index cc8518d4a25c6..4f7c55fef855d 100644 --- a/services/migrations/dump.go +++ b/services/migrations/dump.go @@ -650,6 +650,8 @@ func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.Mi if err != nil { return err } + defer downloader.CleanUp() + uploader, err := NewRepositoryDumper(ctx, baseDir, ownerName, opts.RepoName, opts) if err != nil { return err @@ -709,11 +711,13 @@ func RestoreRepository(ctx context.Context, baseDir, ownerName, repoName string, if err != nil { return err } - uploader := NewGiteaLocalUploader(ctx, doer, ownerName, repoName) + downloader, err := NewRepositoryRestorer(ctx, baseDir, ownerName, repoName, validation) if err != nil { return err } + defer downloader.CleanUp() + opts, err := downloader.getRepoOptions() if err != nil { return err @@ -727,6 +731,7 @@ func RestoreRepository(ctx context.Context, baseDir, ownerName, repoName string, return err } + uploader := NewGiteaLocalUploader(ctx, doer, ownerName, repoName) if err = migrateRepository(doer, downloader, uploader, migrateOpts, nil); err != nil { if err1 := uploader.Rollback(); err1 != nil { log.Error("rollback failed: %v", err1) @@ -735,3 +740,32 @@ func RestoreRepository(ctx context.Context, baseDir, ownerName, repoName string, } return updateMigrationPosterIDByGitService(ctx, structs.GitServiceType(tp)) } + +// RestoreFromGithubExportedData restore a repository from the disk directory +func RestoreFromGithubExportedData(ctx context.Context, baseDir, ownerName, repoName string, units []string) error { + doer, err := user_model.GetAdminUser() + if err != nil { + return err + } + downloader, err := NewGithubExportedDataRestorer(ctx, baseDir, ownerName, repoName) + if err != nil { + return err + } + defer downloader.CleanUp() + + migrateOpts := base.MigrateOptions{ + GitServiceType: structs.GithubService, + } + if err := updateOptionsUnits(&migrateOpts, units); err != nil { + return err + } + + uploader := NewGiteaLocalUploader(ctx, doer, ownerName, repoName) + if err = migrateRepository(doer, downloader, uploader, migrateOpts, nil); err != nil { + if err1 := uploader.Rollback(); err1 != nil { + log.Error("rollback failed: %v", err1) + } + return err + } + return updateMigrationPosterIDByGitService(ctx, migrateOpts.GitServiceType) +} diff --git a/services/migrations/gitbucket.go b/services/migrations/gitbucket.go index cc3d4fc93674a..573fe6ecf0f7f 100644 --- a/services/migrations/gitbucket.go +++ b/services/migrations/gitbucket.go @@ -85,3 +85,8 @@ func NewGitBucketDownloader(ctx context.Context, baseURL, userName, password, to func (g *GitBucketDownloader) SupportGetRepoComments() bool { return false } + +// GetReviews is not supported +func (g *GitBucketDownloader) GetReviews(reviwable base.Reviewable) ([]*base.Review, bool, error) { + return nil, true, &base.ErrNotSupported{Entity: "Reviews"} +} diff --git a/services/migrations/gitea_downloader.go b/services/migrations/gitea_downloader.go index 470090b5010a5..fcba1ca84fcf4 100644 --- a/services/migrations/gitea_downloader.go +++ b/services/migrations/gitea_downloader.go @@ -286,7 +286,7 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele for _, asset := range rel.Attachments { size := int(asset.Size) dlCount := int(asset.DownloadCount) - r.Assets = append(r.Assets, &base.ReleaseAsset{ + r.Assets = append(r.Assets, &base.Asset{ ID: asset.ID, Name: asset.Name, Size: &size, @@ -459,7 +459,7 @@ func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, err } // GetComments returns comments according issueNumber -func (g *GiteaDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { +func (g *GiteaDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { allComments := make([]*base.Comment, 0, g.maxPerPage) for i := 1; ; i++ { @@ -470,22 +470,22 @@ func (g *GiteaDownloader) GetComments(commentable base.Commentable) ([]*base.Com default: } - comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, commentable.GetForeignIndex(), gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{ + comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.Commentable.GetForeignIndex(), gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{ PageSize: g.maxPerPage, Page: i, }}) if err != nil { - return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %w", commentable.GetForeignIndex(), err) + return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %w", opts.Commentable.GetForeignIndex(), err) } for _, comment := range comments { reactions, err := g.getCommentReactions(comment.ID) if err != nil { - WarnAndNotice("Unable to load comment reactions during migrating issue #%d for comment %d in %s. Error: %v", commentable.GetForeignIndex(), comment.ID, g, err) + WarnAndNotice("Unable to load comment reactions during migrating issue #%d for comment %d in %s. Error: %v", opts.Commentable.GetForeignIndex(), comment.ID, g, err) } allComments = append(allComments, &base.Comment{ - IssueIndex: commentable.GetLocalIndex(), + IssueIndex: opts.Commentable.GetLocalIndex(), Index: comment.ID, PosterID: comment.Poster.ID, PosterName: comment.Poster.UserName, @@ -625,10 +625,10 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques } // GetReviews returns pull requests review -func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { +func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, bool, error) { if err := g.client.CheckServerVersionConstraint(">=1.12"); err != nil { log.Info("GiteaDownloader: instance to old, skip GetReviews") - return nil, nil + return nil, true, nil } allReviews := make([]*base.Review, 0, g.maxPerPage) @@ -637,7 +637,7 @@ func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review // make sure gitea can shutdown gracefully select { case <-g.ctx.Done(): - return nil, nil + return nil, true, nil default: } @@ -646,7 +646,7 @@ func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review PageSize: g.maxPerPage, }}) if err != nil { - return nil, err + return nil, true, err } for _, pr := range prl { @@ -658,7 +658,7 @@ func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, reviewable.GetForeignIndex(), pr.ID) if err != nil { - return nil, err + return nil, true, err } var reviewComments []*base.ReviewComment for i := range rcl { @@ -700,5 +700,5 @@ func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review break } } - return allReviews, nil + return allReviews, true, nil } diff --git a/services/migrations/gitea_downloader_test.go b/services/migrations/gitea_downloader_test.go index c37c70947e36f..b51acce6b62e7 100644 --- a/services/migrations/gitea_downloader_test.go +++ b/services/migrations/gitea_downloader_test.go @@ -197,7 +197,9 @@ func TestGiteaDownloadRepo(t *testing.T) { }, }, issues) - comments, _, err := downloader.GetComments(&base.Issue{Number: 4, ForeignIndex: 4}) + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + Commentable: &base.Issue{Number: 4, ForeignIndex: 4}, + }) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ { @@ -262,7 +264,9 @@ func TestGiteaDownloadRepo(t *testing.T) { PatchURL: "https://gitea.com/gitea/test_repo/pulls/12.patch", }, prs[1]) - reviews, err := downloader.GetReviews(&base.Issue{Number: 7, ForeignIndex: 7}) + reviews, _, err := downloader.GetReviews(base.GetReviewOptions{ + Reviewable: &base.Issue{Number: 7, ForeignIndex: 7}, + }) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 20370d99f9824..8e3df73edfcdb 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -21,6 +21,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" base "code.gitea.io/gitea/modules/migration" repo_module "code.gitea.io/gitea/modules/repository" @@ -242,6 +243,50 @@ func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error { return nil } +func (g *GiteaLocalUploader) uploadAttachment(asset *base.Asset) (*repo_model.Attachment, error) { + var downloadCnt, size int64 + if asset.DownloadCount != nil { + downloadCnt = int64(*asset.DownloadCount) + } + if asset.Size != nil { + size = int64(*asset.Size) + } + + attach := repo_model.Attachment{ + UUID: uuid.New().String(), + Name: asset.Name, + DownloadCount: downloadCnt, + Size: size, + CreatedUnix: timeutil.TimeStamp(asset.Created.Unix()), + } + + // SECURITY: We cannot check the DownloadURL and DownloadFunc are safe here + // ... we must assume that they are safe and simply download the attachment + // asset.DownloadURL maybe a local file + var rc io.ReadCloser + var err error + if asset.DownloadFunc != nil { + rc, err = asset.DownloadFunc() + if err != nil { + return nil, err + } + } else if asset.DownloadURL != nil { + rc, err = uri.Open(*asset.DownloadURL) + if err != nil { + return nil, err + } + } + if rc == nil { + return nil, nil + } + defer rc.Close() + _, err = storage.Attachments.Save(attach.RelativePath(), rc, int64(*asset.Size)) + if err != nil { + return nil, err + } + return &attach, nil +} + // CreateReleases creates releases func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { rels := make([]*repo_model.Release, 0, len(releases)) @@ -304,43 +349,13 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { asset.Created = release.Created } } - attach := repo_model.Attachment{ - UUID: uuid.New().String(), - Name: asset.Name, - DownloadCount: int64(*asset.DownloadCount), - Size: int64(*asset.Size), - CreatedUnix: timeutil.TimeStamp(asset.Created.Unix()), - } - - // SECURITY: We cannot check the DownloadURL and DownloadFunc are safe here - // ... we must assume that they are safe and simply download the attachment - err := func() error { - // asset.DownloadURL maybe a local file - var rc io.ReadCloser - var err error - if asset.DownloadFunc != nil { - rc, err = asset.DownloadFunc() - if err != nil { - return err - } - } else if asset.DownloadURL != nil { - rc, err = uri.Open(*asset.DownloadURL) - if err != nil { - return err - } - } - if rc == nil { - return nil - } - _, err = storage.Attachments.Save(attach.RelativePath(), rc, int64(*asset.Size)) - rc.Close() - return err - }() + attach, err := g.uploadAttachment(asset) if err != nil { return err } - - rel.Attachments = append(rel.Attachments, &attach) + if attach != nil { + rel.Attachments = append(rel.Attachments, attach) + } } rels = append(rels, &rel) @@ -422,6 +437,27 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { } is.Reactions = append(is.Reactions, &res) } + + for _, asset := range issue.Assets { + if asset.Created.IsZero() { + if !asset.Updated.IsZero() { + asset.Created = asset.Updated + } else { + asset.Created = issue.Updated + } + } + attach, err := g.uploadAttachment(asset) + if err != nil { + return err + } + if attach != nil { + is.Attachments = append(is.Attachments, attach) + if asset.OriginalURL != "" { + is.Content = strings.ReplaceAll(is.Content, asset.OriginalURL, fmt.Sprintf("/attachments/%s", attach.UUID)) + } + } + } + iss = append(iss, &is) } @@ -488,6 +524,79 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { return err } + switch cm.Type { + /*{"Label":"https://github.com/go-gitea/test_repo/labels/duplicate","LabelColor":"cfd3d7","LabelName":"duplicate","LabelTextColor":"000000"}*/ + case issues_model.CommentTypeLabel: + data := make(map[string]string) + if err := json.Unmarshal([]byte(cm.Content), &data); err != nil { + log.Error("unmarshal %s failed: %v", cm.Content, err) + continue + } else { + lb, err := issues_model.GetLabelByRepoIDAndName(issue.RepoID, data["LabelName"]) + if err != nil { + if issues_model.IsErrLabelNotExist(err) { + continue + } + return err + } + cm.LabelID = lb.ID + if data["type"] == "add" { + cm.Content = "1" + } else { + cm.Content = "" + } + } + + /*{"MilestoneTitle":"1.1.0"}*/ + case issues_model.CommentTypeMilestone: + data := make(map[string]string) + if err := json.Unmarshal([]byte(cm.Content), &data); err != nil { + log.Error("unmarshal %s failed: %v", cm.Content, err) + continue + } else { + milestone, err := issues_model.GetMilestoneByRepoIDANDName(issue.RepoID, data["MilestoneTitle"]) + if err != nil { + log.Error("GetMilestoneByRepoIDANDName %d, %s failed: %v", issue.RepoID, data["MilestoneTitle"], err) + continue + } else { + if data["type"] == "add" { + cm.MilestoneID = milestone.ID + } else if data["type"] == "remove" { + cm.OldMilestoneID = milestone.ID + } + } + } + case issues_model.CommentTypeChangeTitle: + data := make(map[string]string) + if err := json.Unmarshal([]byte(cm.Content), &data); err != nil { + log.Error("unmarshal %s failed: %v", cm.Content, err) + continue + } else { + cm.OldTitle = data["OldTitle"] + cm.NewTitle = data["NewTitle"] + } + case issues_model.CommentTypeCommitRef: + data := make(map[string]string) + if err := json.Unmarshal([]byte(cm.Content), &data); err != nil { + log.Error("unmarshal %s failed: %v", cm.Content, err) + } else { + cm.CommitSHA = data["CommitID"] + } + continue + /*{ + "Actor": g.Actor, + "Subject": g.Subject, + }*/ + case issues_model.CommentTypeAssignees: + continue + case issues_model.CommentTypeCommentRef, issues_model.CommentTypeIssueRef, issues_model.CommentTypePullRef: + continue + case issues_model.CommentTypeDeleteBranch: + continue + case issues_model.CommentTypePullRequestPush: + continue + } + // add reactions for _, reaction := range comment.Reactions { res := issues_model.Reaction{ @@ -500,6 +609,26 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { cm.Reactions = append(cm.Reactions, &res) } + for _, asset := range comment.Assets { + if asset.Created.IsZero() { + if !asset.Updated.IsZero() { + asset.Created = asset.Updated + } else { + asset.Created = comment.Updated + } + } + attach, err := g.uploadAttachment(asset) + if err != nil { + return err + } + if attach != nil { + cm.Attachments = append(cm.Attachments, attach) + if asset.OriginalURL != "" { + cm.Content = strings.ReplaceAll(cm.Content, asset.OriginalURL, fmt.Sprintf("/attachments/%s", attach.UUID)) + } + } + } + cms = append(cms, &cm) } @@ -522,6 +651,26 @@ func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error return err } + for _, asset := range pr.Assets { + if asset.Created.IsZero() { + if !asset.Updated.IsZero() { + asset.Created = asset.Updated + } else { + asset.Created = pr.Updated + } + } + attach, err := g.uploadAttachment(asset) + if err != nil { + return err + } + if attach != nil { + gpr.Issue.Attachments = append(gpr.Issue.Attachments, attach) + if asset.OriginalURL != "" { + gpr.Issue.Content = strings.ReplaceAll(gpr.Issue.Content, asset.OriginalURL, fmt.Sprintf("/attachments/%s", attach.UUID)) + } + } + } + gprs = append(gprs, gpr) } if err := models.InsertPullRequests(gprs...); err != nil { @@ -534,6 +683,28 @@ func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error return nil } +func getHeadBranch(pr *base.PullRequest) string { + if pr.IsForkPullRequest() { + return pr.Head.OwnerName + "/" + pr.Head.Ref + } + return pr.Head.Ref +} + +func (g *GiteaLocalUploader) createHeadBranch(pr *base.PullRequest) error { + branchName := getHeadBranch(pr) + headBranch := filepath.Join(g.repo.RepoPath(), "refs", "heads", branchName) + if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil { + return err + } + b, err := os.Create(headBranch) + if err != nil { + return err + } + _, err = b.WriteString(pr.Head.SHA) + b.Close() + return err +} + func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head string, err error) { // SECURITY: this pr must have been must have been ensured safe if !pr.EnsuredSafe { @@ -576,6 +747,33 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head return "", err } + // set head information + pullHead := filepath.Join(g.repo.RepoPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number)) + if err := os.MkdirAll(pullHead, os.ModePerm); err != nil { + return "", err + } + p, err := os.Create(filepath.Join(pullHead, "head")) + if err != nil { + return "", err + } + _, err = p.WriteString(pr.Head.SHA) + p.Close() + if err != nil { + return "", err + } + + // create branch directly if commit exists in the repository + _, err = g.gitRepo.GetCommit(pr.Head.SHA) + if git.IsErrNotExist(err) { + } else if err != nil { + return "", err + } else { + if err := g.createHeadBranch(pr); err != nil { + return "", err + } + return getHeadBranch(pr), nil + } + head = "unknown repository" if pr.IsForkPullRequest() && pr.State != "closed" { // OK we want to fetch the current head as a branch from its CloneURL @@ -822,6 +1020,7 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { IssueID: issue.ID, Content: review.Content, Official: review.Official, + CommitID: review.CommitID, CreatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()), UpdatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()), } @@ -857,7 +1056,9 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { for _, comment := range review.Comments { line := comment.Line if line != 0 { - comment.Position = 1 + if comment.Position == 0 { + comment.Position = 1 + } } else { _, _, line, _ = git.ParseDiffHunkString(comment.DiffHunk) } @@ -865,7 +1066,6 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { // SECURITY: The TreePath must be cleaned! comment.TreePath = path.Clean("/" + comment.TreePath)[1:] - var patch string reader, writer := io.Pipe() defer func() { _ = reader.Close() @@ -879,7 +1079,15 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { _ = writer.Close() }(comment) - patch, _ = git.CutDiffAroundLine(reader, int64((&issues_model.Comment{Line: int64(line + comment.Position - 1)}).UnsignedLine()), line < 0, setting.UI.CodeCommentLines) + unsignedLine := int64((&issues_model.Comment{Line: int64(line + comment.Position - 1)}).UnsignedLine()) + patch, err := git.CutDiffAroundLine(reader, unsignedLine, line < 0, setting.UI.CodeCommentLines) + if err != nil { + log.Warn("CutDiffAroundLine failed when migrating [%s, %d, %v]", g.gitRepo.Path, unsignedLine, line < 0) + } + if patch == "" { + patch = fmt.Sprintf("diff --git a/%s b/%s\n--- a/%s\n+++ b/%s\n%s", comment.TreePath, comment.TreePath, comment.TreePath, comment.TreePath, comment.DiffHunk) + } + _ = reader.Close() if comment.CreatedAt.IsZero() { comment.CreatedAt = review.CreatedAt diff --git a/services/migrations/gitea_uploader_test.go b/services/migrations/gitea_uploader_test.go index 6a942b9b57637..24d10bf132fba 100644 --- a/services/migrations/gitea_uploader_test.go +++ b/services/migrations/gitea_uploader_test.go @@ -44,6 +44,8 @@ func TestGiteaUploadRepo(t *testing.T) { uploader = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName) ) + defer downloader.CleanUp() + err := migrateRepository(user, downloader, uploader, base.MigrateOptions{ CloneAddr: "https://github.com/go-xorm/builder", RepoName: repoName, @@ -342,7 +344,7 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) { }, { name: "fork, invalid Head.Ref", - head: "unknown repository", + head: "user10/INVALID", pr: base.PullRequest{ PatchURL: "", Number: 1, @@ -368,7 +370,7 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) { }, { name: "invalid fork CloneURL", - head: "unknown repository", + head: "WRONG/branch2", pr: base.PullRequest{ PatchURL: "", Number: 1, diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go new file mode 100644 index 0000000000000..27c64b9327fee --- /dev/null +++ b/services/migrations/github_exported_data.go @@ -0,0 +1,1468 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migrations + +import ( + "archive/tar" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "net/url" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + base "code.gitea.io/gitea/modules/migration" + + "github.com/hashicorp/go-version" +) + +/* +{"version":"1.0.1","github_sha":"8de0984858fd99a8dcd2d756cf0f128b9161e3b5"} +*/ +type githubSchema struct { + Version string + GithubSha string `json:"github_sha"` +} + +/* + { + "type": "user", + "url": "https://github.com/sunvim", + "avatar_url": "https://avatars.githubusercontent.com/u/859692?v=4", + "login": "sunvim", + "name": "mobus", + "bio": "code happy ", + "company": "Ankr", + "website": null, + "location": "Shanghai", + "emails": [ + { + "address": "sv0220@163.com", + "primary": true, + "verified": true + } + ], + "billing_plan": null, + "created_at": "2011-06-19T11:25:35Z" + }, +*/ +type githubUser struct { + URL string + AvatarURL string `json:"avatar_url"` + Login string + Name string + Bio string + Company string + Website string + Location string + Emails []struct { + Address string + Primary bool + Verified bool + } + CreatedAt time.Time `json:"created_at"` +} + +func getURLLastField(s string) string { + u, err := url.Parse(s) + if err != nil { + log.Error("parse %s failed: %v", s, err) + return "" + } + fields := strings.Split(u.Path, "/") + if len(fields) == 0 { + return "" + } + return fields[len(fields)-1] +} + +func parseGitHubResID(s string) int64 { + u, err := url.Parse(s) + if err != nil { + log.Error("parse %s failed: %v", s, err) + return 0 + } + fields := strings.Split(u.Path, "/") + if len(fields) == 0 { + return 0 + } + i, _ := strconv.ParseInt(fields[len(fields)-1], 10, 64) + return i +} + +func (g *githubUser) ID() int64 { + return parseGitHubResID(g.AvatarURL) +} + +func (g *githubUser) Email() string { + if len(g.Emails) < 1 { + return "" + } + + for _, e := range g.Emails { + if e.Primary { + return e.Address + } + } + return "" +} + +/* + { + "type": "attachment", + "url": "https://user-images.githubusercontent.com/1595118/2923824-63a167ce-d721-11e3-91b6-74b83dc345bb.png", + "issue": "https://github.com/go-xorm/xorm/issues/205", + "issue_comment": "https://github.com/go-xorm/xorm/issues/115#issuecomment-42628488", + "user": "https://github.com/mintzhao", + "asset_name": "QQ20140509-1.2x.png", + "asset_content_type": "image/png", + "asset_url": "tarball://root/attachments/63a167ce-d721-11e3-91b6-74b83dc345bb/QQ20140509-1.2x.png", + "created_at": "2014-05-09T02:38:54Z" + }, +*/ +type githubAttachment struct { + URL string `json:"url"` + Issue string + IssueComment string `json:"issue_comment"` + User string + AssetName string `json:"asset_name"` + AssetContentType string `json:"asset_content_type"` + AssetURL string `json:"asset_url"` + CreatedAt time.Time `json:"created_at"` +} + +func (g *githubAttachment) GetUserID() int64 { + return parseGitHubResID(g.User) +} + +func (g *githubAttachment) IsIssue() bool { + return len(g.Issue) > 0 +} + +func (g *githubAttachment) IsComment() bool { + return len(g.IssueComment) > 0 +} + +func (g *githubAttachment) IssueID() int64 { + if g.IsIssue() { + return parseGitHubResID(g.Issue) + } + return parseGitHubResID(g.IssueComment) +} + +func (r *GithubExportedDataRestorer) convertAttachments(ls []githubAttachment) []*base.Asset { + res := make([]*base.Asset, 0, len(ls)) + for _, l := range ls { + fPath := strings.TrimPrefix(l.AssetURL, "tarball://root/") + fPath = filepath.Join(r.tmpDir, fPath) + info, err := os.Stat(fPath) + var size int + if err == nil { + size = int(info.Size()) + } + assetURL := "file://" + fPath + res = append(res, &base.Asset{ + Name: l.AssetName, + ContentType: &l.AssetContentType, + Size: &size, + Created: l.CreatedAt, + Updated: l.CreatedAt, + DownloadURL: &assetURL, + OriginalURL: l.URL, + }) + } + return res +} + +/* + { + "user": "https://github.com/mrsdizzie", + "content": "+1", + "subject_type": "Issue", + "created_at": "2019-11-13T04:22:13.000+08:00" + } +*/ +type githubReaction struct { + User string + Content string + SubjectType string `json:"subject_type"` + CreatedAt time.Time `json:"created_at"` +} + +type githubLabel string + +func (l githubLabel) GetName() string { + fields := strings.Split(string(l), "/labels/") + if len(fields) == 0 { + return "" + } + s, err := url.PathUnescape(fields[len(fields)-1]) + if err != nil { + log.Error("url.PathUnescape %s failed: %v", fields[len(fields)-1], err) + return fields[len(fields)-1] + } + return s +} + +// GithubExportedDataRestorer implements an Downloader from the exported data of Github +type GithubExportedDataRestorer struct { + base.NullDownloader + ctx context.Context + tmpDir string // work directory, tar files will be decompress into there + githubDataFilePath string + repoOwner string + repoName string + baseURL string + regMatchIssue *regexp.Regexp + regMatchCommit *regexp.Regexp + labels []*base.Label + users map[string]githubUser + issueAttachments map[string][]githubAttachment + commentAttachments map[string][]githubAttachment + milestones map[string]githubMilestone + attachmentLoaded bool +} + +var _ base.Downloader = &GithubExportedDataRestorer{} + +func decompressFile(targzFile, targetDir string) error { + f, err := os.Open(targzFile) + if err != nil { + return err + } + defer f.Close() + uncompressedStream, err := gzip.NewReader(f) + if err != nil { + return err + } + + tarReader := tar.NewReader(uncompressedStream) + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } else if err != nil { + return err + } + + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(filepath.Join(targetDir, header.Name), os.ModePerm); err != nil { + return err + } + case tar.TypeReg: + outFile, err := os.Create(filepath.Join(targetDir, header.Name)) + if err != nil { + return err + } + if _, err := io.Copy(outFile, tarReader); err != nil { + outFile.Close() + return err + } + outFile.Close() + default: + return fmt.Errorf("decompressFile: uknown type: %d in %s", + header.Typeflag, + header.Name) + } + } + return nil +} + +// NewGithubExportedDataRestorer creates a repository restorer which could restore repository from a github exported data +func NewGithubExportedDataRestorer(ctx context.Context, githubDataFilePath, owner, repoName string) (*GithubExportedDataRestorer, error) { + tmpDir, err := os.MkdirTemp(os.TempDir(), "github_exported_data") + if err != nil { + return nil, err + } + // uncompress the file + if err := decompressFile(githubDataFilePath, tmpDir); err != nil { + return nil, err + } + + restorer := &GithubExportedDataRestorer{ + ctx: ctx, + githubDataFilePath: githubDataFilePath, + tmpDir: tmpDir, + repoOwner: owner, + repoName: repoName, + users: make(map[string]githubUser), + milestones: make(map[string]githubMilestone), + issueAttachments: make(map[string][]githubAttachment), + commentAttachments: make(map[string][]githubAttachment), + } + if err := restorer.readSchema(); err != nil { + return nil, err + } + if err := restorer.getUsers(); err != nil { + return nil, err + } + + return restorer, nil +} + +// CleanUp clean the downloader temporary resources +func (r *GithubExportedDataRestorer) CleanUp() { + if r.tmpDir != "" { + _ = os.RemoveAll(r.tmpDir) + } +} + +// replaceGithubLinks replace #id to new form +// i.e. +// 1) https://github.com/userstyles-world/userstyles.world/commit/b70d545a1cbb5c92ca20f442f59de5d955600408 -> b70d545a1cbb5c92ca20f442f59de5d955600408 +// 2) https://github.com/go-gitea/gitea/issue/1 -> #1 +// 3) https://github.com/go-gitea/gitea/pull/2 -> #2 +func (r *GithubExportedDataRestorer) replaceGithubLinks(content string) string { + c := r.regMatchIssue.ReplaceAllString(content, "#$2") + c = r.regMatchCommit.ReplaceAllString(c, "$1") + return c +} + +// SupportGetRepoComments return true if it can get all comments once +func (r *GithubExportedDataRestorer) SupportGetRepoComments() bool { + return true +} + +// SupportGetRepoComments return true if it can get all comments once +func (r *GithubExportedDataRestorer) SupportGetRepoReviews() bool { + return true +} + +// SetContext set context +func (r *GithubExportedDataRestorer) SetContext(ctx context.Context) { + r.ctx = ctx +} + +func (r *GithubExportedDataRestorer) readSchema() error { + bs, err := os.ReadFile(filepath.Join(r.tmpDir, "schema.json")) + if err != nil { + return err + } + var schema githubSchema + if err := json.Unmarshal(bs, &schema); err != nil { + return err + } + + v, err := version.NewSemver(schema.Version) + if err != nil { + return fmt.Errorf("archive version %s is not semver", schema.Version) + } + s := v.Segments() + if s[0] != 1 { + return fmt.Errorf("archive version is %s, but expected 1.x.x", schema.Version) + } + return nil +} + +// GetRepoInfo returns a repository information +func (r *GithubExportedDataRestorer) GetRepoInfo() (*base.Repository, error) { + type Label struct { + URL string + Name string + Color string + Description string + CreatedAt time.Time `json:"created_at"` + } + type GithubRepo struct { + Name string + URL string + Owner string + Description string + Private bool + Labels []Label + CreatedAt time.Time `json:"created_at"` + DefaultBranch string `json:"default_branch"` + } + + p := filepath.Join(r.tmpDir, "repositories_000001.json") + bs, err := os.ReadFile(p) + if err != nil { + return nil, err + } + + var githubRepositories []GithubRepo + if err := json.Unmarshal(bs, &githubRepositories); err != nil { + return nil, err + } + if len(githubRepositories) == 0 { + return nil, errors.New("no repository found in the json file: repositories_000001.json") + } else if len(githubRepositories) > 1 { + return nil, errors.New("only one repository is supported") + } + + opts := githubRepositories[0] + fields := strings.Split(opts.Owner, "/") + owner := fields[len(fields)-1] + + for _, label := range opts.Labels { + r.labels = append(r.labels, &base.Label{ + Name: label.Name, + Color: label.Color, + Description: label.Description, + }) + } + r.baseURL = opts.URL + r.regMatchIssue, err = regexp.Compile(r.baseURL + "/(issue|pull)/([0-9]+)") + if err != nil { + return nil, err + } + r.regMatchCommit, err = regexp.Compile(r.baseURL + "/commit/([a-z0-9]{7, 40})") + if err != nil { + return nil, err + } + + return &base.Repository{ + Owner: r.repoOwner, + Name: r.repoName, + IsPrivate: opts.Private, + Description: opts.Description, + OriginalURL: opts.URL, + CloneURL: filepath.Join(r.tmpDir, "repositories", owner, opts.Name+".git"), + DefaultBranch: opts.DefaultBranch, + }, nil +} + +// GetTopics return github topics +func (r *GithubExportedDataRestorer) GetTopics() ([]string, error) { + topics := struct { + Topics []string `yaml:"topics"` + }{} + + // FIXME: No topic information provided + + return topics.Topics, nil +} + +func (r *GithubExportedDataRestorer) readJSONFiles(filePrefix string, makeF func() interface{}, f func(content interface{}) error) error { + for i := 1; ; i++ { + p := filepath.Join(r.tmpDir, fmt.Sprintf(filePrefix+"_%06d.json", i)) + _, err := os.Stat(p) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + + bs, err := os.ReadFile(p) + if err != nil { + return err + } + content := makeF() + if err := json.Unmarshal(bs, content); err != nil { + return err + } + + if err := f(content); err != nil { + return err + } + } +} + +func (r *GithubExportedDataRestorer) getUsers() error { + return r.readJSONFiles("users", func() interface{} { + return &[]githubUser{} + }, func(content interface{}) error { + mss := content.(*[]githubUser) + for _, ms := range *mss { + r.users[ms.URL] = ms + } + return nil + }) +} + +func (r *GithubExportedDataRestorer) getAttachments() error { + if r.attachmentLoaded { + return nil + } + + return r.readJSONFiles("attachments", func() interface{} { + r.attachmentLoaded = true + return &[]githubAttachment{} + }, func(content interface{}) error { + mss := content.(*[]githubAttachment) + for _, ms := range *mss { + if ms.IsIssue() { + r.issueAttachments[ms.Issue] = append(r.issueAttachments[ms.Issue], ms) + } else { + r.commentAttachments[ms.IssueComment] = append(r.commentAttachments[ms.IssueComment], ms) + } + } + return nil + }) +} + +/* + { + "type": "milestone", + "url": "https://github.com/go-gitea/test_repo/milestones/1", + "repository": "https://github.com/go-gitea/test_repo", + "user": "https://github.com/mrsdizzie", + "title": "1.0.0", + "description": "Milestone 1.0.0", + "state": "closed", + "due_on": "2019-11-11T00:00:00Z", + "created_at": "2019-11-12T19:37:08Z", + "updated_at": "2019-11-12T21:56:17Z", + "closed_at": "2019-11-12T19:45:49Z" + }, +*/ +type githubMilestone struct { + URL string + User string + Title string + State string + Description string + DueOn time.Time `json:"due_on"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ClosedAt time.Time `json:"closed_at"` +} + +// GetMilestones returns milestones +func (r *GithubExportedDataRestorer) GetMilestones() ([]*base.Milestone, error) { + milestones := make([]*base.Milestone, 0, 10) + if err := r.readJSONFiles("milestones", func() interface{} { + return &[]githubMilestone{} + }, func(content interface{}) error { + mss := content.(*[]githubMilestone) + for _, milestone := range *mss { + r.milestones[milestone.URL] = milestone + milestones = append(milestones, &base.Milestone{ + Title: milestone.Title, + Description: milestone.Description, + Deadline: &milestone.DueOn, + Created: milestone.ClosedAt, + Updated: &milestone.UpdatedAt, + Closed: &milestone.ClosedAt, + State: milestone.State, + }) + } + return nil + }); err != nil { + return nil, err + } + + return milestones, nil +} + +/* + { + "type": "release", + "url": "https://github.com/go-xorm/xorm/releases/tag/v0.3.1", + "repository": "https://github.com/go-xorm/xorm", + "user": "https://github.com/lunny", + "name": "", + "tag_name": "v0.3.1", + "body": "- Features:\n - Support MSSQL DB via ODBC driver ([github.com/lunny/godbc](https://github.com/lunny/godbc));\n - Composite Key, using multiple pk xorm tag \n - Added Row() API as alternative to Iterate() API for traversing result set, provide similar usages to sql.Rows type\n - ORM struct allowed declaration of pointer builtin type as members to allow null DB fields \n - Before and After Event processors\n- Improvements:\n - Allowed int/int32/int64/uint/uint32/uint64/string as Primary Key type\n - Performance improvement for Get()/Find()/Iterate()\n", + "state": "published", + "pending_tag": "v0.3.1", + "prerelease": false, + "target_commitish": "master", + "release_assets": [ + + ], + "published_at": "2014-01-02T09:51:34Z", + "created_at": "2014-01-02T09:48:57Z" + }, +*/ +type githubRelease struct { + User string + Name string + TagName string `json:"tag_name"` + Body string + State string + Prerelease bool + ReleaseAssets []githubAttachment `json:"release_assets"` + TargetCommitish string `json:"target_commitish"` + CreatedAt time.Time `json:"created_at"` + PublishedAt time.Time `json:"published_at"` +} + +func (r *GithubExportedDataRestorer) getUserInfo(u string) (int64, string, string) { + user, ok := r.users[u] + if !ok { + return 0, getURLLastField(u), "" + } + return user.ID(), user.Login, user.Email() +} + +// GetReleases returns releases +func (r *GithubExportedDataRestorer) GetReleases() ([]*base.Release, error) { + releases := make([]*base.Release, 0, 30) + if err := r.readJSONFiles("releases", func() interface{} { + return &[]githubRelease{} + }, func(content interface{}) error { + rss := content.(*[]githubRelease) + for _, rel := range *rss { + id, login, email := r.getUserInfo(rel.User) + releases = append(releases, &base.Release{ + TagName: rel.TagName, + TargetCommitish: rel.TargetCommitish, + Name: rel.Name, + Body: rel.Body, + Draft: rel.State == "draft", + Prerelease: rel.Prerelease, + PublisherID: id, + PublisherName: login, + PublisherEmail: email, + Assets: r.convertAttachments(rel.ReleaseAssets), + Created: rel.CreatedAt, + Published: rel.PublishedAt, + }) + } + return nil + }); err != nil { + return nil, err + } + return releases, nil +} + +// GetLabels returns labels +func (r *GithubExportedDataRestorer) GetLabels() ([]*base.Label, error) { + return r.labels, nil +} + +/* + { + "type": "issue", + "url": "https://github.com/go-xorm/xorm/issues/1", + "repository": "https://github.com/go-xorm/xorm", + "user": "https://github.com/zakzou", + "title": "建表功能已经强大了，不过希望添加上自定义mysql engine和charset", + "body": "如题\n", + "assignee": "https://github.com/lunny", + "assignees": [ + "https://github.com/lunny" + ], + "milestone": null, + "labels": [ + "https://github.com/go-gitea/test_repo/labels/bug", + "https://github.com/go-gitea/test_repo/labels/good%20first%20issue" + ], + "reactions": [ + + ], + "closed_at": "2013-08-08T05:26:00Z", + "created_at": "2013-07-04T08:08:39Z", + "updated_at": "2013-08-08T05:26:00Z" + }, +*/ +type githubIssue struct { + URL string + User string + Title string + Body string + Assignee string + Assignees []string + Milestone string + Labels []githubLabel + Reactions []githubReaction + ClosedAt *time.Time `json:"closed_at"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +func (g *githubIssue) Index() int64 { + fields := strings.Split(g.URL, "/") + i, _ := strconv.ParseInt(fields[len(fields)-1], 10, 64) + return i +} + +func (r *GithubExportedDataRestorer) getLabels(ls []githubLabel) []*base.Label { + res := make([]*base.Label, 0, len(ls)) + for _, l := range ls { + for _, ll := range r.labels { + if l.GetName() == ll.Name { + res = append(res, ll) + break + } + } + } + return res +} + +func (r *GithubExportedDataRestorer) getReactions(ls []githubReaction) []*base.Reaction { + res := make([]*base.Reaction, 0, len(ls)) + for _, l := range ls { + content := l.Content + switch content { + case "thinking_face": + content = "confused" + case "tada": + content = "hooray" + } + + id, login, _ := r.getUserInfo(l.User) + if id == 0 { + log.Warn("Cannot get user %s information, userid will be set 0", l.User) + } else { + res = append(res, &base.Reaction{ + UserID: id, + UserName: login, + Content: content, + }) + } + } + return res +} + +// GetIssues returns issues according start and limit +func (r *GithubExportedDataRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { + if err := r.getAttachments(); err != nil { + return nil, false, err + } + + issues := make([]*base.Issue, 0, 50) + if err := r.readJSONFiles("issues", func() interface{} { + return &[]githubIssue{} + }, func(content interface{}) error { + rss := content.(*[]githubIssue) + for _, issue := range *rss { + id, login, email := r.getUserInfo(issue.User) + var milestone string + if issue.Milestone != "" { + milestone = r.milestones[issue.Milestone].Title + } + + state := "open" + if issue.ClosedAt != nil { + state = "closed" + } + + issue.Body = strings.ReplaceAll(issue.Body, "\u0000", "") + + issues = append(issues, &base.Issue{ + Number: issue.Index(), + Title: issue.Title, + Content: issue.Body, + PosterID: id, + PosterName: login, + PosterEmail: email, + Labels: r.getLabels(issue.Labels), + Reactions: r.getReactions(issue.Reactions), + Assets: r.convertAttachments(r.issueAttachments[issue.URL]), + Milestone: milestone, + Assignees: issue.Assignees, + State: state, + ForeignIndex: issue.Index(), + Closed: issue.ClosedAt, + Created: issue.CreatedAt, + Updated: issue.UpdatedAt, + }) + } + return nil + }); err != nil { + return nil, false, err + } + + return issues, true, nil +} + +type githubComment struct { + URL string `json:"url"` + Issue string + PullRequest string `json:"pull_request"` + User string + Body string + Reactions []githubReaction + CreatedAt time.Time `json:"created_at"` +} + +func getIssueIndex(issue, pullRequest string) int64 { + var c string + if issue != "" { + c = issue + } else { + c = pullRequest + } + + fields := strings.Split(c, "/") + idx, _ := strconv.ParseInt(fields[len(fields)-1], 10, 64) + return idx +} + +func (g *githubComment) GetIssueIndex() int64 { + return getIssueIndex(g.Issue, g.PullRequest) +} + +/* + { + "type": "issue_event", + "url": "https://github.com/go-xorm/xorm/issues/1#event-55275262", + "issue": "https://github.com/go-xorm/xorm/issues/1", + "actor": "https://github.com/lunny", + "event": "assigned", + "created_at": "2013-07-04T14:27:53Z" + }, + { + "type": "issue_event", + "url": "https://github.com/go-xorm/xorm/pull/2#event-56230828", + "pull_request": "https://github.com/go-xorm/xorm/pull/2", + "actor": "https://github.com/lunny", + "event": "referenced", + "commit_id": "1be80583b0fa18e7b478fa12e129c95e9a06a62f", + "commit_repository": "https://github.com/go-xorm/xorm", + "created_at": "2013-07-12T02:10:52Z" + + "label": "https://github.com/go-xorm/xorm/labels/wip", + "label_name": "New Feature", + "label_color": "5319e7", + "label_text_color": "fff", + "milestone_title": "v0.4", + "title_was": "自动读写分离", + "title_is": "Automatical Read/Write seperatelly.", + }, +*/ +type githubIssueEvent struct { + URL string + Issue string + PullRequest string `json:"pull_request"` + Actor string + Event string + CommitID string `json:"commit_id"` + Ref string + BeforeCommitOID string `json:"before_commit_oid"` + AfterCommitOID string `json:"after_commit_oid"` + CommitRepoistory string `json:"commit_repository"` + CreatedAt time.Time `json:"created_at"` + Label string + LabelName string `json:"label_name"` + LabelColor string `json:"label_color"` + LabelTextColor string `json:"label_text_color"` + MilestoneTitle string `json:"milestone_title"` + Subject string + TitleWas string `json:"title_was"` + TitleIs string `json:"title_is"` +} + +// CommentContent returns comment content +func (g *githubIssueEvent) CommentContent() map[string]interface{} { + switch g.Event { + case "closed": + return map[string]interface{}{} + case "head_ref_force_pushed": + return map[string]interface{}{} + case "moved_columns_in_project": + return map[string]interface{}{} + case "connected": + fields := strings.Split(g.Subject, "/") + if len(fields) > 2 { + if fields[len(fields)-2] == "issue" { + return map[string]interface{}{ + "type": "issue_ref", + "subject": fields[len(fields)-1], + } + } else if fields[len(fields)-2] == "pull" { + return map[string]interface{}{ + "type": "pull_ref", + "subject": fields[len(fields)-1], + } + } + } + return map[string]interface{}{} + case "disconnected": + return map[string]interface{}{} + case "referenced": + tp := "commit_ref" + if g.Issue != "" { + tp = "issue_ref" + } else if g.PullRequest != "" { + tp = "pull_ref" + } + return map[string]interface{}{ + "type": tp, + "CommitID": g.CommitID, + } + case "merged": + return map[string]interface{}{} + case "mentioned": + return map[string]interface{}{} + case "subscribed": + return map[string]interface{}{} + case "head_ref_deleted": + return map[string]interface{}{} + case "head_ref_restored": + return map[string]interface{}{} + case "milestoned": + return map[string]interface{}{ + "type": "add", + "MilestoneTitle": g.MilestoneTitle, + } + case "demilestoned": + return map[string]interface{}{ + "type": "remove", + "MilestoneTitle": g.MilestoneTitle, + } + case "labeled": + return map[string]interface{}{ + "type": "add", + "Label": g.Label, + "LabelName": g.LabelName, + "LabelColor": g.LabelColor, + "LabelTextColor": g.LabelTextColor, + } + case "renamed": + return map[string]interface{}{ + "OldTitle": g.TitleWas, + "NewTitle": g.TitleIs, + } + case "ready_for_review": + return map[string]interface{}{} + case "reopened": + return map[string]interface{}{} + case "unlabeled": + return map[string]interface{}{ + "type": "remove", + "Label": g.Label, + "LabelName": g.LabelName, + "LabelColor": g.LabelColor, + "LabelTextColor": g.LabelTextColor, + } + case "assigned": + return map[string]interface{}{ + "Actor": g.Actor, + "Subject": g.Subject, + } + case "added_to_project": + return map[string]interface{}{} + default: + return map[string]interface{}{} + } +} + +// CommentStr returns comment type string +func (g *githubIssueEvent) CommentStr() string { + switch g.Event { + case "assigned": + return "assignees" + case "base_ref_changed": + return "unknown" + case "closed": + return "close" + case "convert_to_draft": + return "unknown" + case "head_ref_force_pushed": + return "pull_push" + case "referenced": + return "commit_ref" + case "moved_columns_in_project": + return "unknown" + case "ready_for_review": + return "unknown" + case "review_requested": + return "unknown" + case "merged": + return "merge_pull" + case "mentioned": + return "unknown" // ignore + case "subscribed": + return "unknown" // ignore + case "head_ref_deleted": + return "delete_branch" + case "head_ref_restored": + return "unknown" + case "added_to_project": + return "unknown" + case "milestoned": + return "milestone" + case "demilestoned": + return "milestone" + case "labeled": + return "label" + case "renamed": + return "change_title" + case "reopened": + return "reopen" + case "unlabeled": + return "label" + + case "pinned": + return "pinned" + case "unpinned": + return "unpinned" + case "connected": + fields := strings.Split(g.Subject, "/") + if len(fields) > 2 { + if fields[len(fields)-2] == "issue" { + return "issue_ref" + } else if fields[len(fields)-2] == "pull" { + return "pull_ref" + } + } + return "unknown" + case "disconnected": + return "unknown" + default: + return "comment" + } +} + +func (g *githubIssueEvent) GetIssueIndex() int64 { + return getIssueIndex(g.Issue, g.PullRequest) +} + +func (r *GithubExportedDataRestorer) getIssueEvents() ([]*base.Comment, error) { + comments := make([]*base.Comment, 0, 10) + if err := r.readJSONFiles("issue_events", func() interface{} { + return &[]githubIssueEvent{} + }, func(content interface{}) error { + rss := content.(*[]githubIssueEvent) + for _, c := range *rss { + id, login, email := r.getUserInfo(c.Actor) + v := c.CommentContent() + bs, err := json.Marshal(v) + if err != nil { + return err + } + + comments = append(comments, &base.Comment{ + IssueIndex: c.GetIssueIndex(), + CommentType: c.CommentStr(), + PosterID: id, + PosterName: login, + PosterEmail: email, + Created: c.CreatedAt, + Updated: c.CreatedAt, // FIXME: + Content: string(bs), + }) + } + return nil + }); err != nil { + return nil, err + } + return comments, nil +} + +func (r *GithubExportedDataRestorer) GetAllComments(page, perPage int) ([]*base.Comment, bool, error) { + return r.GetComments(base.GetCommentOptions{}) +} + +// GetComments returns comments according issueNumber +func (r *GithubExportedDataRestorer) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { + comments := make([]*base.Comment, 0, 10) + if err := r.readJSONFiles("issue_comments", func() interface{} { + return &[]githubComment{} + }, func(content interface{}) error { + rss := content.(*[]githubComment) + for _, c := range *rss { + id, login, email := r.getUserInfo(c.User) + comments = append(comments, &base.Comment{ + IssueIndex: c.GetIssueIndex(), + PosterID: id, + PosterName: login, + PosterEmail: email, + Created: c.CreatedAt, + Updated: c.CreatedAt, // FIXME: + Content: r.replaceGithubLinks(c.Body), + Reactions: r.getReactions(c.Reactions), + Assets: r.convertAttachments(r.commentAttachments[c.URL]), + }) + } + return nil + }); err != nil { + return nil, false, err + } + + comments2, err := r.getIssueEvents() + if err != nil { + return nil, false, err + } + + return append(comments, comments2...), true, nil +} + +/* + { + "type": "pull_request", + "url": "https://github.com/go-xorm/xorm/pull/2", + "user": "https://github.com/airylinus", + "repository": "https://github.com/go-xorm/xorm", + "title": "修正文档中代码示例中的笔误", + "body": "1. 修正变量名错误\n2. 修改查询示例的查询，让代码更易懂\n", + "base": { + "ref": "master", + "sha": "a9eb28a00e4b93817906eac5c8af2a566e8c73af", + "user": "https://github.com/go-xorm", + "repo": "https://github.com/go-xorm/xorm" + }, + "head": { + "ref": "master", + "sha": "c18e4e8d174cd7619333f7645bd9dccd4cbf5168", + "user": "https://github.com/airylinus", + "repo": null + }, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "labels": [ + + ], + "reactions": [ + + ], + "review_requests": [ + + ], + "close_issue_references": [ + + ], + "work_in_progress": false, + "merged_at": "2013-07-12T02:10:52Z", + "closed_at": "2013-07-12T02:10:52Z", + "created_at": "2013-07-12T02:04:44Z" + }, +*/ +type githubPullRequest struct { + URL string + User string + Title string + Body string + Base struct { + Ref string + Sha string + User string + Repo string + } + Head struct { + Ref string + Sha string + User string + Repo string + } + Assignee string + Assignees []string + Milestone string + Labels []githubLabel + Reactions []githubReaction + ReviewRequests []struct{} `json:"review_requests"` + CloseIssueReferences []struct{} `json:"close_issue_references"` + WorkInProgress bool `json:"work_in_progress"` + MergedAt *time.Time `json:"merged_at"` + ClosedAt *time.Time `json:"closed_at"` + CreatedAt time.Time `json:"created_at"` +} + +func (g *githubPullRequest) Index() int64 { + fields := strings.Split(g.URL, "/") + i, _ := strconv.ParseInt(fields[len(fields)-1], 10, 64) + return i +} + +// GetPullRequests returns pull requests according page and perPage +func (r *GithubExportedDataRestorer) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { + pulls := make([]*base.PullRequest, 0, 50) + if err := r.readJSONFiles("pull_requests", func() interface{} { + return &[]githubPullRequest{} + }, func(content interface{}) error { + prs := content.(*[]githubPullRequest) + for _, pr := range *prs { + id, login, email := r.getUserInfo(pr.User) + state := "open" + if pr.MergedAt != nil || pr.ClosedAt != nil { + state = "closed" + } + _, headUser, _ := r.getUserInfo(pr.Head.User) + head := base.PullRequestBranch{ + Ref: pr.Head.Ref, + SHA: pr.Head.Sha, + RepoName: pr.Head.Repo, + OwnerName: headUser, + } + + var milestone string + if pr.Milestone != "" { + milestone = r.milestones[pr.Milestone].Title + } + _, baseUser, _ := r.getUserInfo(pr.Base.User) + pulls = append(pulls, &base.PullRequest{ + Number: pr.Index(), + Title: pr.Title, + Content: pr.Body, + Milestone: milestone, + State: state, + PosterID: id, + PosterName: login, + PosterEmail: email, + ForeignIndex: pr.Index(), + Reactions: r.getReactions(pr.Reactions), + Assets: r.convertAttachments(r.issueAttachments[pr.URL]), + Created: pr.CreatedAt, + Closed: pr.ClosedAt, + Labels: r.getLabels(pr.Labels), + Merged: pr.MergedAt != nil, + MergedTime: pr.MergedAt, + Head: head, + Base: base.PullRequestBranch{ + Ref: pr.Base.Ref, + SHA: pr.Base.Sha, + RepoName: pr.Base.Repo, + OwnerName: baseUser, + }, + Assignees: pr.Assignees, + }) + } + return nil + }); err != nil { + return nil, false, err + } + + return pulls, true, nil +} + +/* + { + "type": "pull_request_review", + "url": "https://github.com/go-gitea/test_repo/pull/3/files#pullrequestreview-315859956", + "pull_request": "https://github.com/go-gitea/test_repo/pull/3", + "user": "https://github.com/jolheiser", + "body": "", + "head_sha": "076160cf0b039f13e5eff19619932d181269414b", + "formatter": "markdown", + "state": 40, + "reactions": [ + + ], + "created_at": "2019-11-12T21:35:24Z", + "submitted_at": "2019-11-12T21:35:24Z" + }, +*/ +type pullrequestReview struct { + URL string + PullRequest string `json:"pull_request"` + User string + Body string + HeadSha string `json:"head_sha"` + State int + Reactions []githubReaction + CreatedAt time.Time `json:"created_at"` + SubmittedAt *time.Time `json:"submitted_at"` +} + +func (p *pullrequestReview) Index() int64 { + fields := strings.Split(p.PullRequest, "/") + idx, _ := strconv.ParseInt(fields[len(fields)-1], 10, 64) + return idx +} + +// GetState return PENDING, APPROVED, REQUEST_CHANGES, or COMMENT +func (p *pullrequestReview) GetState() string { + switch p.State { + case 1: + return base.ReviewStateCommented + case 30: + return base.ReviewStateChangesRequested + case 40: + return base.ReviewStateApproved + } + return fmt.Sprintf("%d", p.State) +} + +/* + { + "type": "pull_request_review_thread", + "url": "https://github.com/go-xorm/xorm/pull/1445/files#pullrequestreviewthread-203253693", + "pull_request": "https://github.com/go-xorm/xorm/pull/1445", + "pull_request_review": "https://github.com/go-xorm/xorm/pull/1445/files#pullrequestreview-295977501", + "diff_hunk": "@@ -245,12 +245,17 @@ func (session *Session) Sync2(beans ...interface{}) error {\n \t\tif err != nil {\n \t\t\treturn err\n \t\t}\n-\t\ttbName := engine.TableName(bean)\n-\t\ttbNameWithSchema := engine.TableName(tbName, true)\n+\t\tvar tbName string\n+\t\tif len(session.statement.AltTableName) > 0 {\n+\t\t\ttbName = session.statement.AltTableName\n+\t\t} else {\n+\t\t\ttbName = engine.TableName(bean)\n+\t\t}\n+\t\ttbNameWithSchema := engine.tbNameWithSchema(tbName)\n \n \t\tvar oriTable *core.Table\n \t\tfor _, tb := range tables {\n-\t\t\tif strings.EqualFold(tb.Name, tbName) {\n+\t\t\tif strings.EqualFold(engine.tbNameWithSchema(tb.Name), engine.tbNameWithSchema(tbName)) {", + "path": "session_schema.go", + "position": 17, + "original_position": 17, + "commit_id": "f6b642c82aab95178a4551a1ff65dc2a631a08cf", + "original_commit_id": "f6b642c82aab95178a4551a1ff65dc2a631a08cf", + "start_line": null, + "line": 258, + "start_side": null, + "side": "right", + "original_start_line": null, + "original_line": 258, + "created_at": "2019-10-02T01:40:41Z", + "resolved_at": null, + "resolver": null + }, +*/ +type pullrequestReviewThread struct { + URL string + PullRequest string `json:"pull_request"` + PullRequestReview string `json:"pull_request_review"` + DiffHunk string `json:"diff_hunk"` + Path string + Position int64 + OriginalPosition int64 `json:"original_position"` + CommitID string `json:"commit_id"` + OriginalCommitID string `json:"original_commit_id"` + Line int64 + Side string + OriginalLine int64 `json:"original_line"` + CreatedAt time.Time `json:"created_at"` + ResolvedAt *time.Time `json:"resolved_at"` + Resolver string +} + +func (p *pullrequestReviewThread) Index() int64 { + fields := strings.Split(p.PullRequest, "/") + idx, _ := strconv.ParseInt(fields[len(fields)-1], 10, 64) + return idx +} + +/* + { + "type": "pull_request_review_comment", + "url": "https://github.com/go-gitea/test_repo/pull/4/files#r363017488", + "pull_request": "https://github.com/go-gitea/test_repo/pull/4", + "pull_request_review": "https://github.com/go-gitea/test_repo/pull/4/files#pullrequestreview-338338740", + "pull_request_review_thread": "https://github.com/go-gitea/test_repo/pull/4/files#pullrequestreviewthread-224172719", + "user": "https://github.com/lunny", + "body": "This is a good pull request.", + "formatter": "markdown", + "diff_hunk": "@@ -1,2 +1,4 @@\n # test_repo\n Test repository for testing migration from github to gitea\n+", + "path": "README.md", + "position": 3, + "original_position": 3, + "commit_id": "2be9101c543658591222acbee3eb799edfc3853d", + "original_commit_id": "2be9101c543658591222acbee3eb799edfc3853d", + "state": 1, + "in_reply_to": null, + "reactions": [ + + ], + "created_at": "2020-01-04T05:33:06Z" + }, +*/ +type pullrequestReviewComment struct { + PullRequest string `json:"pull_request"` + PullRequestReview string `json:"pull_request_review"` + PullRequestReviewThread string `json:"pull_request_review_thread"` + User string + Body string + DiffHunk string `json:"diff_hunk"` + Path string + Position int + OriginalPosition int `json:"original_position"` + CommitID string `json:"commit_id"` + OriginalCommitID string `json:"original_commit_id"` + State int + Reactions []githubReaction + CreatedAt time.Time `json:"created_at"` +} + +func (r *GithubExportedDataRestorer) getReviewComments(thread *pullrequestReviewThread, comments []pullrequestReviewComment) []*base.ReviewComment { + res := make([]*base.ReviewComment, 0, 10) + for _, c := range comments { + id, login, email := r.getUserInfo(c.User) + position := int(thread.Position) + if thread.Side == "right" { + position = int(thread.OriginalPosition) + } + // Line will be parse from diffhunk, so we ignore it here + res = append(res, &base.ReviewComment{ + Content: c.Body, + TreePath: c.Path, + DiffHunk: c.DiffHunk, + Position: position, + CommitID: c.OriginalCommitID, + PosterID: id, + PosterName: login, + PosterEmail: email, + Reactions: r.getReactions(c.Reactions), + CreatedAt: c.CreatedAt, + }) + } + return res +} + +// GetReviews returns pull requests review +func (r *GithubExportedDataRestorer) GetReviews(reviwable base.Reviewable) ([]*base.Review, bool, error) { + comments := make(map[string][]pullrequestReviewComment) + if err := r.readJSONFiles("pull_request_review_comments", func() interface{} { + return &[]pullrequestReviewComment{} + }, func(content interface{}) error { + cs := *content.(*[]pullrequestReviewComment) + for _, c := range cs { + comments[c.PullRequestReviewThread] = append(comments[c.PullRequestReviewThread], c) + } + return nil + }); err != nil { + return nil, true, err + } + + reviews := make(map[string]*base.Review, 10) + if err := r.readJSONFiles("pull_request_reviews", func() interface{} { + return &[]pullrequestReview{} + }, func(content interface{}) error { + prReviews := content.(*[]pullrequestReview) + for _, review := range *prReviews { + id, login, email := r.getUserInfo(review.User) + baseReview := &base.Review{ + IssueIndex: review.Index(), + ReviewerID: id, + ReviewerName: login, + ReviewerEmail: email, + CommitID: review.HeadSha, + Content: review.Body, + CreatedAt: review.CreatedAt, + State: review.GetState(), + } + reviews[review.URL] = baseReview + } + return nil + }); err != nil { + return nil, true, err + } + + if err := r.readJSONFiles("pull_request_review_threads", func() interface{} { + return &[]pullrequestReviewThread{} + }, func(content interface{}) error { + cs := *content.(*[]pullrequestReviewThread) + for _, review := range cs { + reviewComments := comments[review.URL] + if len(reviewComments) == 0 { + continue + } + rr, ok := reviews[review.PullRequestReview] + if !ok { + id, login, email := r.getUserInfo(reviewComments[0].User) + rr = &base.Review{ + IssueIndex: review.Index(), + ReviewerID: id, + ReviewerName: login, + ReviewerEmail: email, + CommitID: review.CommitID, + CreatedAt: review.CreatedAt, + State: base.ReviewStateCommented, + } + reviews[review.URL] = rr + } + + rr.Comments = r.getReviewComments(&review, reviewComments) + rr.ResolvedAt = review.ResolvedAt + if resolver, ok := r.users[review.Resolver]; ok { + rr.ResolverID = resolver.ID() + rr.ResolverName = resolver.Login + rr.ResolverEmail = resolver.Email() + } + } + return nil + }); err != nil { + return nil, true, err + } + + rs := make([]*base.Review, 0, len(reviews)) + for _, review := range reviews { + rs = append(rs, review) + } + + return rs, true, nil +} diff --git a/services/migrations/github_exported_data_test.go b/services/migrations/github_exported_data_test.go new file mode 100644 index 0000000000000..09ab139801a37 --- /dev/null +++ b/services/migrations/github_exported_data_test.go @@ -0,0 +1,106 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migrations + +import ( + "context" + "fmt" + "testing" + + base "code.gitea.io/gitea/modules/migration" + + "github.com/stretchr/testify/assert" +) + +func TestParseGithubExportedData(t *testing.T) { + restorer, err := NewGithubExportedDataRestorer(context.Background(), "../../testdata/github_migration/migration_archive_test_repo.tar.gz", "lunny", "test_repo") + assert.NoError(t, err) + assert.EqualValues(t, 49, len(restorer.users)) + + // repo info + repo, err := restorer.GetRepoInfo() + assert.NoError(t, err) + assert.EqualValues(t, "test_repo", repo.Name) + + // milestones + milestones, err := restorer.GetMilestones() + assert.NoError(t, err) + assert.EqualValues(t, 2, len(milestones)) + + // releases + releases, err := restorer.GetReleases() + assert.NoError(t, err) + assert.EqualValues(t, 1, len(releases)) + assert.EqualValues(t, 0, len(releases[0].Assets)) + + // labels + labels, err := restorer.GetLabels() + assert.NoError(t, err) + assert.EqualValues(t, 9, len(labels)) + + // issues + issues, isEnd, err := restorer.GetIssues(1, 100) + assert.NoError(t, err) + assert.True(t, isEnd) + assert.EqualValues(t, 2, len(issues)) + assert.EqualValues(t, 1, issues[0].ForeignIndex) + assert.EqualValues(t, "Please add an animated gif icon to the merge button", issues[0].Title) + assert.EqualValues(t, "I just want the merge button to hurt my eyes a little. 😝 ", issues[0].Content) + assert.EqualValues(t, "guillep2k", issues[0].PosterName) + assert.EqualValues(t, 2, len(issues[0].Labels), fmt.Sprintf("%#v", issues[0].Labels)) + assert.EqualValues(t, "1.0.0", issues[0].Milestone) + assert.EqualValues(t, 0, len(issues[0].Assets)) + assert.EqualValues(t, 1, len(issues[0].Reactions)) + assert.EqualValues(t, "closed", issues[0].State) + assert.NotNil(t, issues[0].Closed) + assert.NotZero(t, issues[0].Updated) + assert.NotZero(t, issues[0].Created) + + assert.EqualValues(t, 2, issues[1].ForeignIndex) + assert.EqualValues(t, "Test issue", issues[1].Title) + assert.EqualValues(t, "This is test issue 2, do not touch!", issues[1].Content) + assert.EqualValues(t, "mrsdizzie", issues[1].PosterName) + assert.EqualValues(t, 1, len(issues[1].Labels)) + assert.EqualValues(t, "1.1.0", issues[1].Milestone) + assert.EqualValues(t, 0, len(issues[1].Assets)) + assert.EqualValues(t, 6, len(issues[1].Reactions)) + assert.EqualValues(t, "closed", issues[1].State) + assert.NotNil(t, issues[1].Closed) + assert.NotZero(t, issues[1].Updated) + assert.NotZero(t, issues[1].Created) + + // comments + comments, isEnd, err := restorer.GetComments(base.GetCommentOptions{}) + assert.NoError(t, err) + assert.True(t, isEnd) + assert.EqualValues(t, 16, len(comments)) + // first comments are comment type + assert.EqualValues(t, 2, comments[0].IssueIndex) + assert.NotZero(t, comments[0].Created) + + assert.EqualValues(t, 2, comments[1].IssueIndex) + assert.NotZero(t, comments[1].Created) + + // pull requests + prs, isEnd, err := restorer.GetPullRequests(1, 100) + assert.NoError(t, err) + assert.True(t, isEnd) + assert.EqualValues(t, 2, len(prs)) + + assert.EqualValues(t, "Update README.md", prs[0].Title) + assert.EqualValues(t, "add warning to readme", prs[0].Content) + assert.EqualValues(t, 1, len(prs[0].Labels)) + assert.EqualValues(t, "documentation", prs[0].Labels[0].Name) + + assert.EqualValues(t, "Test branch", prs[1].Title) + assert.EqualValues(t, "do not merge this PR", prs[1].Content) + assert.EqualValues(t, 1, len(prs[1].Labels)) + assert.EqualValues(t, "bug", prs[1].Labels[0].Name) + + // reviews + reviews, isEnd, err := restorer.GetReviews(base.GetReviewOptions{}) + assert.NoError(t, err) + assert.True(t, isEnd) + assert.EqualValues(t, 6, len(reviews)) +} diff --git a/services/migrations/github.go b/services/migrations/github_v3.go similarity index 97% rename from services/migrations/github.go rename to services/migrations/github_v3.go index d34ad13b95a71..2d9c82fbb209d 100644 --- a/services/migrations/github.go +++ b/services/migrations/github_v3.go @@ -333,7 +333,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) for _, asset := range rel.Assets { assetID := *asset.ID // Don't optimize this, for closure we need a local variable - r.Assets = append(r.Assets, &base.ReleaseAsset{ + r.Assets = append(r.Assets, &base.Asset{ ID: asset.GetID(), Name: asset.GetName(), ContentType: asset.ContentType, @@ -506,12 +506,12 @@ func (g *GithubDownloaderV3) SupportGetRepoComments() bool { } // GetComments returns comments according issueNumber -func (g *GithubDownloaderV3) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { - comments, err := g.getComments(commentable) +func (g *GithubDownloaderV3) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { + comments, err := g.getComments(opts) return comments, false, err } -func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base.Comment, error) { +func (g *GithubDownloaderV3) getComments(opts base.GetCommentOptions) ([]*base.Comment, error) { var ( allComments = make([]*base.Comment, 0, g.maxPerPage) created = "created" @@ -526,7 +526,7 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base. } for { g.waitAndPickClient() - comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(commentable.GetForeignIndex()), opt) + comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(opts.Commentable.GetForeignIndex()), opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %w", err) } @@ -559,7 +559,7 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base. } allComments = append(allComments, &base.Comment{ - IssueIndex: commentable.GetLocalIndex(), + IssueIndex: opts.Commentable.GetLocalIndex(), Index: comment.GetID(), PosterID: comment.GetUser().GetID(), PosterName: comment.GetUser().GetLogin(), @@ -808,10 +808,10 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques } // GetReviews returns pull requests review -func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { +func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Review, bool, error) { allReviews := make([]*base.Review, 0, g.maxPerPage) if g.SkipReviews { - return allReviews, nil + return allReviews, true, nil } opt := &github.ListOptions{ PerPage: g.maxPerPage, @@ -821,7 +821,7 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev g.waitAndPickClient() reviews, resp, err := g.getClient().PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt) if err != nil { - return nil, fmt.Errorf("error while listing repos: %w", err) + return nil, true, fmt.Errorf("error while listing repos: %w", err) } g.setRate(&resp.Rate) for _, review := range reviews { @@ -835,13 +835,13 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev g.waitAndPickClient() reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), review.GetID(), opt2) if err != nil { - return nil, fmt.Errorf("error while listing repos: %w", err) + return nil, true, fmt.Errorf("error while listing repos: %w", err) } g.setRate(&resp.Rate) cs, err := g.convertGithubReviewComments(reviewComments) if err != nil { - return nil, err + return nil, true, err } r.Comments = append(r.Comments, cs...) if resp.NextPage == 0 { @@ -861,7 +861,7 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev g.waitAndPickClient() reviewers, resp, err := g.getClient().PullRequests.ListReviewers(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt) if err != nil { - return nil, fmt.Errorf("error while listing repos: %w", err) + return nil, false, fmt.Errorf("error while listing repos: %w", err) } g.setRate(&resp.Rate) for _, user := range reviewers.Users { @@ -879,5 +879,5 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev } opt.Page = resp.NextPage } - return allReviews, nil + return allReviews, true, nil } diff --git a/services/migrations/github_test.go b/services/migrations/github_v3_test.go similarity index 96% rename from services/migrations/github_test.go rename to services/migrations/github_v3_test.go index 2b89e6dc0fd16..54a082ca66fce 100644 --- a/services/migrations/github_test.go +++ b/services/migrations/github_v3_test.go @@ -218,7 +218,9 @@ func TestGitHubDownloadRepo(t *testing.T) { }, issues) // downloader.GetComments() - comments, _, err := downloader.GetComments(&base.Issue{Number: 2, ForeignIndex: 2}) + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + Commentable: &base.Issue{Number: 2, ForeignIndex: 2}, + }) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ { @@ -338,7 +340,9 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, prs) - reviews, err := downloader.GetReviews(&base.PullRequest{Number: 3, ForeignIndex: 3}) + reviews, _, err := downloader.GetReviews(base.GetReviewOptions{ + Reviewable: &base.PullRequest{Number: 3, ForeignIndex: 3}, + }) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -370,7 +374,9 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, reviews) - reviews, err = downloader.GetReviews(&base.PullRequest{Number: 4, ForeignIndex: 4}) + reviews, _, err = downloader.GetReviews(base.GetReviewOptions{ + Reviewable: &base.PullRequest{Number: 4, ForeignIndex: 4}, + }) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { diff --git a/services/migrations/gitlab.go b/services/migrations/gitlab.go index 8034869a4ae4b..e4a7b303188ea 100644 --- a/services/migrations/gitlab.go +++ b/services/migrations/gitlab.go @@ -311,7 +311,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea httpClient := NewMigrationHTTPClient() for k, asset := range rel.Assets.Links { - r.Assets = append(r.Assets, &base.ReleaseAsset{ + r.Assets = append(r.Assets, &base.Asset{ ID: int64(asset.ID), Name: asset.Name, ContentType: &rel.Assets.Sources[k].Format, @@ -461,10 +461,10 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er // GetComments returns comments according issueNumber // TODO: figure out how to transfer comment reactions -func (g *GitlabDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { - context, ok := commentable.GetContext().(gitlabIssueContext) +func (g *GitlabDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { + context, ok := opts.Commentable.GetContext().(gitlabIssueContext) if !ok { - return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext()) + return nil, false, fmt.Errorf("unexpected context: %+v", opts.Commentable.GetContext()) } allComments := make([]*base.Comment, 0, g.maxPerPage) @@ -476,12 +476,12 @@ func (g *GitlabDownloader) GetComments(commentable base.Commentable) ([]*base.Co var resp *gitlab.Response var err error if !context.IsMergeRequest { - comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(commentable.GetForeignIndex()), &gitlab.ListIssueDiscussionsOptions{ + comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(opts.Commentable.GetForeignIndex()), &gitlab.ListIssueDiscussionsOptions{ Page: page, PerPage: g.maxPerPage, }, nil, gitlab.WithContext(g.ctx)) } else { - comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(commentable.GetForeignIndex()), &gitlab.ListMergeRequestDiscussionsOptions{ + comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(opts.Commentable.GetForeignIndex()), &gitlab.ListMergeRequestDiscussionsOptions{ Page: page, PerPage: g.maxPerPage, }, nil, gitlab.WithContext(g.ctx)) @@ -495,7 +495,7 @@ func (g *GitlabDownloader) GetComments(commentable base.Commentable) ([]*base.Co if !comment.IndividualNote { for _, note := range comment.Notes { allComments = append(allComments, &base.Comment{ - IssueIndex: commentable.GetLocalIndex(), + IssueIndex: opts.Commentable.GetLocalIndex(), Index: int64(note.ID), PosterID: int64(note.Author.ID), PosterName: note.Author.Username, @@ -507,7 +507,7 @@ func (g *GitlabDownloader) GetComments(commentable base.Commentable) ([]*base.Co } else { c := comment.Notes[0] allComments = append(allComments, &base.Comment{ - IssueIndex: commentable.GetLocalIndex(), + IssueIndex: opts.Commentable.GetLocalIndex(), Index: int64(c.ID), PosterID: int64(c.Author.ID), PosterName: c.Author.Username, @@ -643,14 +643,14 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque } // GetReviews returns pull requests review -func (g *GitlabDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { +func (g *GitlabDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, bool, error) { approvals, resp, err := g.client.MergeRequestApprovals.GetConfiguration(g.repoID, int(reviewable.GetForeignIndex()), gitlab.WithContext(g.ctx)) if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { log.Error(fmt.Sprintf("GitlabDownloader: while migrating a error occurred: '%s'", err.Error())) - return []*base.Review{}, nil + return []*base.Review{}, true, nil } - return nil, err + return nil, true, err } var createdAt time.Time @@ -674,7 +674,7 @@ func (g *GitlabDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Revie }) } - return reviews, nil + return reviews, true, nil } func (g *GitlabDownloader) awardToReaction(award *gitlab.AwardEmoji) *base.Reaction { diff --git a/services/migrations/gitlab_test.go b/services/migrations/gitlab_test.go index 1d8c5989bb534..60c77aeb2eed6 100644 --- a/services/migrations/gitlab_test.go +++ b/services/migrations/gitlab_test.go @@ -213,10 +213,12 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, issues) - comments, _, err := downloader.GetComments(&base.Issue{ - Number: 2, - ForeignIndex: 2, - Context: gitlabIssueContext{IsMergeRequest: false}, + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + Commentable: &base.Issue{ + Number: 2, + ForeignIndex: 2, + Context: gitlabIssueContext{IsMergeRequest: false}, + }, }) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ @@ -303,7 +305,9 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, prs) - rvs, err := downloader.GetReviews(&base.PullRequest{Number: 1, ForeignIndex: 1}) + rvs, _, err := downloader.GetReviews(base.GetReviewOptions{ + Reviewable: &base.PullRequest{Number: 1, ForeignIndex: 1}, + }) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -322,7 +326,9 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, rvs) - rvs, err = downloader.GetReviews(&base.PullRequest{Number: 2, ForeignIndex: 2}) + rvs, _, err = downloader.GetReviews(base.GetReviewOptions{ + Reviewable: &base.PullRequest{Number: 2, ForeignIndex: 2}, + }) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -464,7 +470,9 @@ func TestGitlabGetReviews(t *testing.T) { mux.HandleFunc(fmt.Sprintf("/api/v4/projects/%d/merge_requests/%d/approvals", testCase.repoID, testCase.prID), mock) id := int64(testCase.prID) - rvs, err := downloader.GetReviews(&base.Issue{Number: id, ForeignIndex: id}) + rvs, _, err := downloader.GetReviews(base.GetReviewOptions{ + Reviewable: &base.Issue{Number: id, ForeignIndex: id}, + }) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{&review}, rvs) } diff --git a/services/migrations/gogs.go b/services/migrations/gogs.go index d01934ac6d525..9eddaa98da568 100644 --- a/services/migrations/gogs.go +++ b/services/migrations/gogs.go @@ -236,10 +236,10 @@ func (g *GogsDownloader) getIssues(page int, state string) ([]*base.Issue, bool, } // GetComments returns comments according issueNumber -func (g *GogsDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { +func (g *GogsDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { allComments := make([]*base.Comment, 0, 100) - comments, err := g.client.ListIssueComments(g.repoOwner, g.repoName, commentable.GetForeignIndex()) + comments, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.Commentable.GetForeignIndex()) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %w", err) } @@ -248,7 +248,7 @@ func (g *GogsDownloader) GetComments(commentable base.Commentable) ([]*base.Comm continue } allComments = append(allComments, &base.Comment{ - IssueIndex: commentable.GetLocalIndex(), + IssueIndex: opts.Commentable.GetLocalIndex(), Index: comment.ID, PosterID: comment.Poster.ID, PosterName: comment.Poster.Login, diff --git a/services/migrations/gogs_test.go b/services/migrations/gogs_test.go index 610af183de219..c9fbebda24ff2 100644 --- a/services/migrations/gogs_test.go +++ b/services/migrations/gogs_test.go @@ -110,7 +110,9 @@ func TestGogsDownloadRepo(t *testing.T) { }, issues) // downloader.GetComments() - comments, _, err := downloader.GetComments(&base.Issue{Number: 1, ForeignIndex: 1}) + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + Commentable: &base.Issue{Number: 1, ForeignIndex: 1}, + }) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ { diff --git a/services/migrations/main_test.go b/services/migrations/main_test.go index 30875f6e5ba4c..94240c003283e 100644 --- a/services/migrations/main_test.go +++ b/services/migrations/main_test.go @@ -170,7 +170,7 @@ func assertReactionsEqual(t *testing.T, expected, actual []*base.Reaction) { } } -func assertReleaseAssetEqual(t *testing.T, expected, actual *base.ReleaseAsset) { +func assertReleaseAssetEqual(t *testing.T, expected, actual *base.Asset) { assert.Equal(t, expected.ID, actual.ID) assert.Equal(t, expected.Name, actual.Name) assert.Equal(t, expected.ContentType, actual.ContentType) @@ -181,7 +181,7 @@ func assertReleaseAssetEqual(t *testing.T, expected, actual *base.ReleaseAsset) assert.Equal(t, expected.DownloadURL, actual.DownloadURL) } -func assertReleaseAssetsEqual(t *testing.T, expected, actual []*base.ReleaseAsset) { +func assertReleaseAssetsEqual(t *testing.T, expected, actual []*base.Asset) { if assert.Len(t, actual, len(expected)) { for i := range expected { assertReleaseAssetEqual(t, expected[i], actual[i]) diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index 0ebb3411fdb2f..073a243e537bd 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -123,6 +123,7 @@ func MigrateRepository(ctx context.Context, doer *user_model.User, ownerName str if err != nil { return nil, err } + defer downloader.CleanUp() uploader := NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) uploader.gitServiceType = opts.GitServiceType @@ -347,7 +348,9 @@ func migrateRepository(doer *user_model.User, downloader base.Downloader, upload allComments := make([]*base.Comment, 0, commentBatchSize) for _, issue := range issues { log.Trace("migrating issue %d's comments", issue.Number) - comments, _, err := downloader.GetComments(issue) + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + Commentable: issue, + }) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -379,6 +382,8 @@ func migrateRepository(doer *user_model.User, downloader base.Downloader, upload } } + supportAllReviews := downloader.SupportGetRepoReviews() + if opts.PullRequests { log.Trace("migrating pull requests and comments") messenger("repo.migrate.migrating_pulls") @@ -403,7 +408,9 @@ func migrateRepository(doer *user_model.User, downloader base.Downloader, upload allComments := make([]*base.Comment, 0, commentBatchSize) for _, pr := range prs { log.Trace("migrating pull request %d's comments", pr.Number) - comments, _, err := downloader.GetComments(pr) + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + Commentable: pr, + }) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -427,30 +434,34 @@ func migrateRepository(doer *user_model.User, downloader base.Downloader, upload } } - // migrate reviews - allReviews := make([]*base.Review, 0, reviewBatchSize) - for _, pr := range prs { - reviews, err := downloader.GetReviews(pr) - if err != nil { - if !base.IsErrNotSupported(err) { - return err + if !supportAllComments { + // migrate reviews + allReviews := make([]*base.Review, 0, reviewBatchSize) + for _, pr := range prs { + reviews, _, err := downloader.GetReviews(base.GetReviewOptions{ + Reviewable: pr, + }) + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Warn("migrating reviews is not supported, ignored") + break } - log.Warn("migrating reviews is not supported, ignored") - break - } - allReviews = append(allReviews, reviews...) + allReviews = append(allReviews, reviews...) - if len(allReviews) >= reviewBatchSize { - if err = uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil { - return err + if len(allReviews) >= reviewBatchSize { + if err = uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil { + return err + } + allReviews = allReviews[reviewBatchSize:] } - allReviews = allReviews[reviewBatchSize:] } - } - if len(allReviews) > 0 { - if err = uploader.CreateReviews(allReviews...); err != nil { - return err + if len(allReviews) > 0 { + if err = uploader.CreateReviews(allReviews...); err != nil { + return err + } } } } @@ -479,6 +490,32 @@ func migrateRepository(doer *user_model.User, downloader base.Downloader, upload } } + if supportAllReviews { + log.Trace("migrating reviews") + for i := 1; ; i++ { + // migrate reviews + reviews, isEnd, err := downloader.GetReviews(base.GetReviewOptions{ + Page: i, + PageSize: commentBatchSize, + }) + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Warn("migrating reviews is not supported, ignored") + break + } + + if err = uploader.CreateReviews(reviews...); err != nil { + return err + } + + if isEnd { + break + } + } + } + return uploader.Finish() } diff --git a/services/migrations/onedev.go b/services/migrations/onedev.go index d4b1b73d37c27..582cafe1e024e 100644 --- a/services/migrations/onedev.go +++ b/services/migrations/onedev.go @@ -372,10 +372,10 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er } // GetComments returns comments -func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { - context, ok := commentable.GetContext().(onedevIssueContext) +func (d *OneDevDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { + context, ok := opts.Commentable.GetContext().(onedevIssueContext) if !ok { - return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext()) + return nil, false, fmt.Errorf("unexpected context: %+v", opts.Commentable.GetContext()) } rawComments := make([]struct { @@ -387,9 +387,9 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co var endpoint string if context.IsPullRequest { - endpoint = fmt.Sprintf("/api/pull-requests/%d/comments", commentable.GetForeignIndex()) + endpoint = fmt.Sprintf("/api/pull-requests/%d/comments", opts.Commentable.GetForeignIndex()) } else { - endpoint = fmt.Sprintf("/api/issues/%d/comments", commentable.GetForeignIndex()) + endpoint = fmt.Sprintf("/api/issues/%d/comments", opts.Commentable.GetForeignIndex()) } err := d.callAPI( @@ -408,9 +408,9 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co }, 0, 100) if context.IsPullRequest { - endpoint = fmt.Sprintf("/api/pull-requests/%d/changes", commentable.GetForeignIndex()) + endpoint = fmt.Sprintf("/api/pull-requests/%d/changes", opts.Commentable.GetForeignIndex()) } else { - endpoint = fmt.Sprintf("/api/issues/%d/changes", commentable.GetForeignIndex()) + endpoint = fmt.Sprintf("/api/issues/%d/changes", opts.Commentable.GetForeignIndex()) } err = d.callAPI( @@ -429,7 +429,7 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co } poster := d.tryGetUser(comment.UserID) comments = append(comments, &base.Comment{ - IssueIndex: commentable.GetLocalIndex(), + IssueIndex: opts.Commentable.GetLocalIndex(), Index: comment.ID, PosterID: poster.ID, PosterName: poster.Name, @@ -454,7 +454,7 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co poster := d.tryGetUser(change.UserID) comments = append(comments, &base.Comment{ - IssueIndex: commentable.GetLocalIndex(), + IssueIndex: opts.Commentable.GetLocalIndex(), PosterID: poster.ID, PosterName: poster.Name, PosterEmail: poster.Email, @@ -564,7 +564,7 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque } // GetReviews returns pull requests reviews -func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { +func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, bool, error) { rawReviews := make([]struct { ID int64 `json:"id"` UserID int64 `json:"userId"` @@ -581,7 +581,7 @@ func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Revie &rawReviews, ) if err != nil { - return nil, err + return nil, true, err } reviews := make([]*base.Review, 0, len(rawReviews)) @@ -608,7 +608,7 @@ func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Revie }) } - return reviews, nil + return reviews, true, nil } // GetTopics return repository topics diff --git a/services/migrations/onedev_test.go b/services/migrations/onedev_test.go index 48412fec64abb..76fdefdfbe478 100644 --- a/services/migrations/onedev_test.go +++ b/services/migrations/onedev_test.go @@ -94,10 +94,12 @@ func TestOneDevDownloadRepo(t *testing.T) { }, }, issues) - comments, _, err := downloader.GetComments(&base.Issue{ - Number: 4, - ForeignIndex: 398, - Context: onedevIssueContext{IsPullRequest: false}, + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + Commentable: &base.Issue{ + Number: 4, + ForeignIndex: 398, + Context: onedevIssueContext{IsPullRequest: false}, + }, }) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ @@ -136,7 +138,9 @@ func TestOneDevDownloadRepo(t *testing.T) { }, }, prs) - rvs, err := downloader.GetReviews(&base.PullRequest{Number: 5, ForeignIndex: 186}) + rvs, _, err := downloader.GetReviews(base.GetReviewOptions{ + Reviewable: &base.PullRequest{Number: 5, ForeignIndex: 186}, + }) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { diff --git a/services/migrations/restore.go b/services/migrations/restore.go index fd337b22c7ac2..12fcfd30d867f 100644 --- a/services/migrations/restore.go +++ b/services/migrations/restore.go @@ -196,9 +196,9 @@ func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, } // GetComments returns comments according issueNumber -func (r *RepositoryRestorer) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { +func (r *RepositoryRestorer) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { comments := make([]*base.Comment, 0, 10) - p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", commentable.GetForeignIndex())) + p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", opts.Commentable.GetForeignIndex())) _, err := os.Stat(p) if err != nil { if os.IsNotExist(err) { @@ -248,25 +248,25 @@ func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullReq } // GetReviews returns pull requests review -func (r *RepositoryRestorer) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { +func (r *RepositoryRestorer) GetReviews(reviewable base.Reviewable) ([]*base.Review, bool, error) { reviews := make([]*base.Review, 0, 10) p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", reviewable.GetForeignIndex())) _, err := os.Stat(p) if err != nil { if os.IsNotExist(err) { - return nil, nil + return nil, true, nil } - return nil, err + return nil, true, err } bs, err := os.ReadFile(p) if err != nil { - return nil, err + return nil, true, err } err = yaml.Unmarshal(bs, &reviews) if err != nil { - return nil, err + return nil, true, err } - return reviews, nil + return reviews, true, nil } diff --git a/templates/repo/issue/view_content/comments.tmpl b/templates/repo/issue/view_content/comments.tmpl index 3012b09d5812f..8e0755181282b 100644 --- a/templates/repo/issue/view_content/comments.tmpl +++ b/templates/repo/issue/view_content/comments.tmpl @@ -115,7 +115,19 @@ {{else}} {{$.locale.Tr "repo.issues.closed_at" .EventTag $createdStr | Safe}} {{end}} - + {{else}} + + {{avatar .Poster}} + + + {{.Poster.GetDisplayName}} + {{if .Issue.IsPull}} + {{$.locale.Tr "repo.pulls.closed_at" .EventTag $createdStr | Safe}} + {{else}} + {{$.locale.Tr "repo.issues.closed_at" .EventTag $createdStr | Safe}} + {{end}} + + {{end}} {{else if eq .Type 28}}
@@ -129,7 +141,20 @@ {{else}} {{$.locale.Tr "repo.issues.pull_merged_at" ($link|Escape) (ShortSha $.Issue.PullRequest.MergedCommitID) ($.BaseTarget|Escape) $createdStr | Str2html}} {{end}} - + {{else}} + + {{avatar .Poster}} + + + {{.Poster.GetDisplayName}} + {{$link := printf "%s/commit/%s" $.Repository.HTMLURL ($.Issue.PullRequest.MergedCommitID|PathEscape)}} + {{if eq $.Issue.PullRequest.Status 3}} + {{$.locale.Tr "repo.issues.manually_pull_merged_at" ($link|Escape) (ShortSha $.Issue.PullRequest.MergedCommitID) ($.BaseTarget|Escape) $createdStr | Str2html}} + {{else}} + {{$.locale.Tr "repo.issues.pull_merged_at" ($link|Escape) (ShortSha $.Issue.PullRequest.MergedCommitID) ($.BaseTarget|Escape) $createdStr | Str2html}} + {{end}} + + {{end}}
{{else if eq .Type 3 5 6}} {{$refFrom:= ""}} @@ -186,7 +211,21 @@ {{else}} {{$.locale.Tr "repo.issues.add_remove_labels" (RenderLabels .AddedLabels $.RepoLink) (RenderLabels .RemovedLabels $.RepoLink) $createdStr | Safe}} {{end}} - + {{else}} + + {{avatar .Poster}} + + + {{.Poster.GetDisplayName}} + {{if and .AddedLabels (not .RemovedLabels)}} + {{$.locale.TrN (len .AddedLabels) "repo.issues.add_label" "repo.issues.add_labels" (RenderLabels .AddedLabels) $createdStr | Safe}} + {{else if and (not .AddedLabels) .RemovedLabels}} + {{$.locale.TrN (len .RemovedLabels) "repo.issues.remove_label" "repo.issues.remove_labels" (RenderLabels .RemovedLabels) $createdStr | Safe}} + {{else}} + {{$.locale.Tr "repo.issues.add_remove_labels" (RenderLabels .AddedLabels) (RenderLabels .RemovedLabels) $createdStr | Safe}} + {{end}} + + {{end}} {{end}} {{else if eq .Type 8}} @@ -233,6 +272,7 @@ {{template "shared/user/authorlink" .Poster}} {{$.locale.Tr "repo.issues.change_title_at" (.OldTitle|RenderEmoji) (.NewTitle|RenderEmoji) $createdStr | Safe}} + {{end}} {{else if eq .Type 11}}
@@ -368,9 +408,9 @@
{{if .OriginalAuthor}} {{else}} - - {{avatar .Poster}} - + + {{avatar .Poster}} + {{end}} {{svg (printf "octicon-%s" .Review.Type.Icon)}} @@ -397,6 +437,9 @@ {{if .Review.Dismissed}}
{{$.locale.Tr "repo.issues.review.dismissed_label"}}
{{end}} + {{if $.Repository.OriginalURL}} + ({{$.locale.Tr "repo.migrated_from" ($.Repository.OriginalURL|Escape) ($.Repository.GetOriginalURLHostname|Escape) | Safe}}) + {{end}}
{{if .Content}} diff --git a/testdata/github_migration/migration_archive_test_repo.tar.gz b/testdata/github_migration/migration_archive_test_repo.tar.gz new file mode 100644 index 0000000000000..995e135c3cd8c Binary files /dev/null and b/testdata/github_migration/migration_archive_test_repo.tar.gz differ