From 87f508ea2e65fa47792a4a26d3081580ec562d3d Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 3 Jan 2022 20:23:45 +0800 Subject: [PATCH 01/54] Add support to import repository data from an exported data of Github --- cmd/restore_repo.go | 11 + modules/private/restore_repo.go | 26 +- routers/private/restore_repo.go | 54 +- services/migrations/dump.go | 31 + services/migrations/restore_github.go | 760 ++++++++++++++++++ services/migrations/restore_github_test.go | 55 ++ .../migration_archive_test_repo.tar.gz | Bin 0 -> 24062 bytes 7 files changed, 908 insertions(+), 29 deletions(-) create mode 100644 services/migrations/restore_github.go create mode 100644 services/migrations/restore_github_test.go create mode 100644 testdata/github_migration/migration_archive_test_repo.tar.gz diff --git a/cmd/restore_repo.go b/cmd/restore_repo.go index f0b01e7984c7f..3335d39fa7e53 100644 --- a/cmd/restore_repo.go +++ b/cmd/restore_repo.go @@ -22,6 +22,15 @@ var CmdRestoreRepository = cli.Command{ Description: "This is a command for restoring the repository data.", Action: runRestoreRepository, Flags: []cli.Flag{ + cli.StringFlag{ + Name: "data_type, d", + Value: "gitea", + Usage: "The data type will be imported, default is gitea, options are gitea, github", + }, + cli.StringFlag{ + Name: "repo_filepath, f", + Usage: "Repository compressed data path to restore from", + }, cli.StringFlag{ Name: "repo_dir, r", Value: "./data", @@ -58,6 +67,8 @@ func runRestoreRepository(c *cli.Context) error { statusCode, errStr := private.RestoreRepo( ctx, + c.String("data_type"), + c.String("repo_filepath"), c.String("repo_dir"), c.String("owner_name"), c.String("repo_name"), diff --git a/modules/private/restore_repo.go b/modules/private/restore_repo.go index b1561f392bd28..d2fa64eb1d3bb 100644 --- a/modules/private/restore_repo.go +++ b/modules/private/restore_repo.go @@ -17,26 +17,30 @@ import ( // RestoreParams structure holds a data for restore repository type RestoreParams struct { - RepoDir string - OwnerName string - RepoName string - Units []string - Validation bool + Type string + RepoFilePath string + RepoDir string + OwnerName string + RepoName string + Units []string + Validation bool } // RestoreRepo calls the internal RestoreRepo function -func RestoreRepo(ctx context.Context, repoDir, ownerName, repoName string, units []string, validation bool) (int, string) { +func RestoreRepo(ctx context.Context, tp, repoPath, repoDir, ownerName, repoName string, units []string, validation bool) (int, string) { reqURL := setting.LocalURL + "api/internal/restore_repo" req := newInternalRequest(ctx, reqURL, "POST") req.SetTimeout(3*time.Second, 0) // since the request will spend much time, don't timeout req = req.Header("Content-Type", "application/json") jsonBytes, _ := json.Marshal(RestoreParams{ - RepoDir: repoDir, - OwnerName: ownerName, - RepoName: repoName, - Units: units, - Validation: validation, + Type: tp, + RepoFilePath: repoPath, + RepoDir: repoDir, + OwnerName: ownerName, + RepoName: repoName, + Units: units, + Validation: validation, }) req.Body(jsonBytes) resp, err := req.Response() diff --git a/routers/private/restore_repo.go b/routers/private/restore_repo.go index 34e06e51c29bf..03e9ee6e040da 100644 --- a/routers/private/restore_repo.go +++ b/routers/private/restore_repo.go @@ -24,11 +24,13 @@ func RestoreRepo(ctx *myCtx.PrivateContext) { return } params := struct { - RepoDir string - OwnerName string - RepoName string - Units []string - Validation bool + Type string + RepoFilePath string + RepoDir string + OwnerName string + RepoName string + Units []string + Validation bool }{} if err = json.Unmarshal(bs, ¶ms); err != nil { ctx.JSON(http.StatusInternalServerError, private.Response{ @@ -37,18 +39,34 @@ func RestoreRepo(ctx *myCtx.PrivateContext) { return } - if err := migrations.RestoreRepository( - ctx, - params.RepoDir, - params.OwnerName, - params.RepoName, - params.Units, - params.Validation, - ); err != nil { - ctx.JSON(http.StatusInternalServerError, private.Response{ - Err: err.Error(), - }) - } else { - ctx.Status(http.StatusOK) + if params.Type == "gitea" { + if err := migrations.RestoreRepository( + ctx, + params.RepoDir, + params.OwnerName, + params.RepoName, + params.Units, + params.Validation, + ); err != nil { + ctx.JSON(http.StatusInternalServerError, private.Response{ + Err: err.Error(), + }) + } else { + ctx.Status(http.StatusOK) + } + } else if params.Type == "github" { + if err := migrations.RestoreFromGithubExportedData( + ctx, + params.RepoFilePath, + params.OwnerName, + params.RepoName, + params.Units, + ); err != nil { + ctx.JSON(http.StatusInternalServerError, private.Response{ + Err: err.Error(), + }) + } else { + ctx.Status(http.StatusOK) + } } } diff --git a/services/migrations/dump.go b/services/migrations/dump.go index 6410aa1ee0854..baedf6cdc01d3 100644 --- a/services/migrations/dump.go +++ b/services/migrations/dump.go @@ -636,3 +636,34 @@ func RestoreRepository(ctx context.Context, baseDir, ownerName, repoName string, } return updateMigrationPosterIDByGitService(ctx, structs.GitServiceType(tp)) } + +// RestoreFromGithubExportedData restore a repository from the disk directory +func RestoreFromGithubExportedData(ctx context.Context, baseDir, ownerName, repoName string, units []string) error { + doer, err := user_model.GetAdminUser() + if err != nil { + return err + } + var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, repoName) + downloader, err := NewRepositoryRestorer(ctx, baseDir, ownerName, repoName) + if err != nil { + return err + } + opts, err := downloader.getRepoOptions() + if err != nil { + return err + } + tp, _ := strconv.Atoi(opts["service_type"]) + + var migrateOpts = base.MigrateOptions{ + GitServiceType: structs.GitServiceType(tp), + } + updateOptionsUnits(&migrateOpts, units) + + if err = migrateRepository(downloader, uploader, migrateOpts, nil); err != nil { + if err1 := uploader.Rollback(); err1 != nil { + log.Error("rollback failed: %v", err1) + } + return err + } + return updateMigrationPosterIDByGitService(ctx, structs.GitServiceType(tp)) +} diff --git a/services/migrations/restore_github.go b/services/migrations/restore_github.go new file mode 100644 index 0000000000000..7ed977ac3da43 --- /dev/null +++ b/services/migrations/restore_github.go @@ -0,0 +1,760 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "archive/tar" + "compress/gzip" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + base "code.gitea.io/gitea/modules/migration" +) + +/* +{ + "type": "user", + "url": "https://github.com/sunvim", + "avatar_url": "https://avatars.githubusercontent.com/u/859692?v=4", + "login": "sunvim", + "name": "mobus", + "bio": "code happy ", + "company": "Ankr", + "website": null, + "location": "Shanghai", + "emails": [ + { + "address": "sv0220@163.com", + "primary": true, + "verified": true + } + ], + "billing_plan": null, + "created_at": "2011-06-19T11:25:35Z" + }, +*/ +type GithubUser struct { + URL string + AvatarURL string + Login string + Name string + Bio string + Company string + Website string + Location string + Emails []struct { + Address string + Primary bool + Verified bool + } + CreatedAt time.Time +} + +func (g *GithubUser) ID() int64 { + u, _ := url.Parse(g.AvatarURL) + fields := strings.Split(u.Path, "/") + i, _ := strconv.ParseInt(fields[len(fields)-1], 10, 64) + return i +} + +func (g *GithubUser) Email() string { + if len(g.Emails) < 1 { + return "" + } + + for _, e := range g.Emails { + if e.Primary { + return e.Address + } + } + return "" +} + +/*{ + "type": "attachment", + "url": "https://user-images.githubusercontent.com/1595118/2923824-63a167ce-d721-11e3-91b6-74b83dc345bb.png", + "issue_comment": "https://github.com/go-xorm/xorm/issues/115#issuecomment-42628488", + "user": "https://github.com/mintzhao", + "asset_name": "QQ20140509-1.2x.png", + "asset_content_type": "image/png", + "asset_url": "tarball://root/attachments/63a167ce-d721-11e3-91b6-74b83dc345bb/QQ20140509-1.2x.png", + "created_at": "2014-05-09T02:38:54Z" + }, +*/ +type githubAttachment struct { + IssueComment string + User string + AssetName string + AssetContentType string + AssetURL string + CreatedAt time.Time +} + +/* +{ + "user": "https://github.com/mrsdizzie", + "content": "+1", + "subject_type": "Issue", + "created_at": "2019-11-13T04:22:13.000+08:00" + }*/ +type githubReaction struct { + User string + Content string + SubjectType string + CreatedAt time.Time +} + +type githubLabel string + +func (l githubLabel) GetName() string { + fields := strings.Split(string(l), "/labels/") + return fields[len(fields)-1] +} + +// GithubExportedDataRestorer implements an Downloader from the exported data of Github +type GithubExportedDataRestorer struct { + base.NullDownloader + ctx context.Context + tmpDir string + githubDataFilePath string + repoOwner string + repoName string + labels []*base.Label + users map[string]*GithubUser +} + +func decompressFile(targzFile, targetDir string) error { + f, err := os.Open(targzFile) + if err != nil { + return err + } + defer f.Close() + uncompressedStream, err := gzip.NewReader(f) + if err != nil { + return err + } + + tarReader := tar.NewReader(uncompressedStream) + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } else if err != nil { + return err + } + + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(filepath.Join(targetDir, header.Name), os.ModePerm); err != nil { + return err + } + case tar.TypeReg: + outFile, err := os.Create(filepath.Join(targetDir, header.Name)) + if err != nil { + return err + } + if _, err := io.Copy(outFile, tarReader); err != nil { + outFile.Close() + return err + } + outFile.Close() + default: + return fmt.Errorf("decompressFile: uknown type: %d in %s", + header.Typeflag, + header.Name) + } + } + return nil +} + +// NewGithubExportedDataRestorer creates a repository restorer which could restore repository from a github exported data +func NewGithubExportedDataRestorer(ctx context.Context, githubDataFilePath, owner, repoName string) (*GithubExportedDataRestorer, error) { + tmpDir, err := os.MkdirTemp(os.TempDir(), "github_exported_data") + if err != nil { + return nil, err + } + // uncompress the file + if err := decompressFile(githubDataFilePath, tmpDir); err != nil { + return nil, err + } + + var restorer = &GithubExportedDataRestorer{ + ctx: ctx, + githubDataFilePath: githubDataFilePath, + tmpDir: tmpDir, + repoOwner: owner, + repoName: repoName, + users: make(map[string]*GithubUser), + } + if err := restorer.getUsers(); err != nil { + return nil, err + } + + return restorer, nil +} + +// SetContext set context +func (r *GithubExportedDataRestorer) SetContext(ctx context.Context) { + r.ctx = ctx +} + +// GetRepoInfo returns a repository information +func (r *GithubExportedDataRestorer) GetRepoInfo() (*base.Repository, error) { + type Label struct { + URL string + Name string `json:"name"` + Color string + Description string + CreatedAt time.Time + } + type GithubRepo struct { + Name string `json:"name"` + URL string + Owner string + Description string + Private bool + Labels []Label `json:"labels"` + CreatedAt time.Time + DefaultBranch string + } + + var githubRepositories []GithubRepo + p := filepath.Join(r.tmpDir, "repositories_000001.json") + bs, err := ioutil.ReadFile(p) + if err != nil { + return nil, err + } + + if err := json.Unmarshal(bs, &githubRepositories); err != nil { + return nil, err + } + if len(githubRepositories) <= 0 { + return nil, errors.New("no repository found in the json file: repositories_000001.json") + } else if len(githubRepositories) > 1 { + return nil, errors.New("only one repository is supported") + } + + opts := githubRepositories[0] + fields := strings.Split(opts.Owner, "/") + owner := fields[len(fields)-1] + + for _, label := range opts.Labels { + r.labels = append(r.labels, &base.Label{ + Name: label.Name, + Color: label.Color, + Description: label.Description, + }) + } + + return &base.Repository{ + Owner: r.repoOwner, + Name: r.repoName, + IsPrivate: opts.Private, + Description: opts.Description, + OriginalURL: opts.URL, + CloneURL: filepath.Join(r.tmpDir, "repositories", owner, opts.Name+".git"), + DefaultBranch: opts.DefaultBranch, + }, nil +} + +// GetTopics return github topics +func (r *GithubExportedDataRestorer) GetTopics() ([]string, error) { + var topics = struct { + Topics []string `yaml:"topics"` + }{} + + // FIXME: No topic information provided + + return topics.Topics, nil +} + +func (r *GithubExportedDataRestorer) readJSONFiles(filePrefix string, makeF func() interface{}, f func(content interface{}) error) error { + for i := 1; ; i++ { + p := filepath.Join(r.tmpDir, fmt.Sprintf(filePrefix+"_%06d.json", i)) + _, err := os.Stat(p) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + + bs, err := os.ReadFile(p) + if err != nil { + return err + } + content := makeF() + if err := json.Unmarshal(bs, content); err != nil { + return err + } + + if err := f(content); err != nil { + return err + } + } +} + +func (r *GithubExportedDataRestorer) getUsers() error { + return r.readJSONFiles("users", func() interface{} { + return &[]GithubUser{} + }, func(content interface{}) error { + mss := content.(*[]GithubUser) + for _, ms := range *mss { + r.users[ms.URL] = &ms + } + return nil + }) +} + +// GetMilestones returns milestones +func (r *GithubExportedDataRestorer) GetMilestones() ([]*base.Milestone, error) { + type milestone struct { + Title string + State string + Description string + DueOn time.Time + CreatedAt time.Time + UpdatedAt time.Time + ClosedAt time.Time + } + + var milestones = make([]*base.Milestone, 0, 10) + if err := r.readJSONFiles("milestones", func() interface{} { + return &[]milestone{} + }, func(content interface{}) error { + mss := content.(*[]milestone) + for _, milestone := range *mss { + milestones = append(milestones, &base.Milestone{ + Title: milestone.Title, + Description: milestone.Description, + Deadline: &milestone.DueOn, + Created: milestone.ClosedAt, + Updated: &milestone.UpdatedAt, + Closed: &milestone.ClosedAt, + State: milestone.State, + }) + } + return nil + }); err != nil { + return nil, err + } + + return milestones, nil +} + +// GetReleases returns releases +func (r *GithubExportedDataRestorer) GetReleases() ([]*base.Release, error) { + type release struct { + Name string + TagName string + Body string + State string + Prerelease bool + ReleaseAssets []struct { + } + TargetCommitish string + CreatedAt time.Time + PublishedAt time.Time + } + + var releases = make([]*base.Release, 0, 30) + if err := r.readJSONFiles("releases", func() interface{} { + return &[]release{} + }, func(content interface{}) error { + rss := content.(*[]release) + for _, rel := range *rss { + // TODO + /*for _, asset := range rel.ReleaseAssets { + if asset.DownloadURL != nil { + *asset.DownloadURL = "file://" + filepath.Join(r.baseDir, *asset.DownloadURL) + } + }*/ + + releases = append(releases, &base.Release{ + TagName: rel.TagName, + TargetCommitish: rel.TargetCommitish, + Name: rel.Name, + Body: rel.Body, + Draft: rel.State == "draft", + Prerelease: rel.Prerelease, + //PublisherID : rel. + //PublisherName string `yaml:"publisher_name"` + //PublisherEmail string `yaml:"publisher_email"` + Assets: []*base.ReleaseAsset{}, + Created: rel.CreatedAt, + Published: rel.PublishedAt, + }) + } + return nil + }); err != nil { + return nil, err + } + return releases, nil +} + +// GetLabels returns labels +func (r *GithubExportedDataRestorer) GetLabels() ([]*base.Label, error) { + return r.labels, nil +} + +/* + { + "type": "issue", + "url": "https://github.com/go-xorm/xorm/issues/1", + "repository": "https://github.com/go-xorm/xorm", + "user": "https://github.com/zakzou", + "title": "建表功能已经强大了，不过希望添加上自定义mysql engine和charset", + "body": "如题\n", + "assignee": "https://github.com/lunny", + "assignees": [ + "https://github.com/lunny" + ], + "milestone": null, + "labels": [ + + ], + "reactions": [ + + ], + "closed_at": "2013-08-08T05:26:00Z", + "created_at": "2013-07-04T08:08:39Z", + "updated_at": "2013-08-08T05:26:00Z" + },*/ +type githubIssue struct { + URL string + User string + Title string + Body string + Assignee string + Assignees []string + Milestone string + Lables []githubLabel + Reactions []githubReaction + ClosedAt *time.Time + CreatedAt time.Time + UpdatedAt time.Time +} + +func (g *githubIssue) Index() int64 { + fields := strings.Split(g.URL, "/") + i, _ := strconv.ParseInt(fields[len(fields)-1], 10, 64) + return i +} + +func (r *GithubExportedDataRestorer) getLabels(ls []githubLabel) []*base.Label { + var res = make([]*base.Label, 0, len(ls)) + for _, l := range ls { + for _, ll := range r.labels { + if string(l) == ll.Name { + res = append(res, ll) + break + } + } + } + return res +} + +func (r *GithubExportedDataRestorer) getReactions(ls []githubReaction) []*base.Reaction { + var res = make([]*base.Reaction, 0, len(ls)) + for _, l := range ls { + user := r.users[l.User] + res = append(res, &base.Reaction{ + UserID: user.ID(), + UserName: user.Login, + Content: l.Content, + }) + } + return res +} + +// GetIssues returns issues according start and limit +func (r *GithubExportedDataRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { + var issues = make([]*base.Issue, 0, 50) + if err := r.readJSONFiles("issues", func() interface{} { + return &[]githubIssue{} + }, func(content interface{}) error { + rss := content.(*[]githubIssue) + for _, issue := range *rss { + user := r.users[issue.User] + + var state = "open" + if issue.ClosedAt != nil { + state = "closed" + } + + issues = append(issues, &base.Issue{ + Number: issue.Index(), + Title: issue.Title, + Content: issue.Body, + PosterID: user.ID(), + PosterName: user.Login, + PosterEmail: user.Email(), + Labels: r.getLabels(issue.Lables), + Reactions: r.getReactions(issue.Reactions), + Milestone: issue.Milestone, + Assignees: issue.Assignees, + //Ref: issue. + State: state, + Context: base.BasicIssueContext(issue.Index()), + Closed: issue.ClosedAt, + Created: issue.CreatedAt, + Updated: issue.UpdatedAt, + }) + } + return nil + }); err != nil { + return nil, false, err + } + + return issues, true, nil +} + +// GetComments returns comments according issueNumber +func (r *GithubExportedDataRestorer) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { + type Comment struct { + Issue string + PullRequest string + User string + Body string + Reactions []githubReaction + CreatedAt time.Time + } + + var comments = make([]*base.Comment, 0, 10) + if err := r.readJSONFiles("issue_comments", func() interface{} { + return &[]Comment{} + }, func(content interface{}) error { + rss := content.(*[]Comment) + for _, c := range *rss { + fields := strings.Split(c.Issue, "/") + idx, _ := strconv.ParseInt(fields[len(fields)-1], 10, 64) + u := r.users[c.User] + + comments = append(comments, &base.Comment{ + IssueIndex: idx, + PosterID: u.ID(), + PosterName: u.Login, + PosterEmail: "", + Created: c.CreatedAt, + //Updated:, + Content: c.Body, + Reactions: r.getReactions(c.Reactions), + }) + } + return nil + }); err != nil { + return nil, false, err + } + return comments, true, nil +} + +/* + { + "type": "pull_request", + "url": "https://github.com/go-xorm/xorm/pull/2", + "user": "https://github.com/airylinus", + "repository": "https://github.com/go-xorm/xorm", + "title": "修正文档中代码示例中的笔误", + "body": "1. 修正变量名错误\n2. 修改查询示例的查询，让代码更易懂\n", + "base": { + "ref": "master", + "sha": "a9eb28a00e4b93817906eac5c8af2a566e8c73af", + "user": "https://github.com/go-xorm", + "repo": "https://github.com/go-xorm/xorm" + }, + "head": { + "ref": "master", + "sha": "c18e4e8d174cd7619333f7645bd9dccd4cbf5168", + "user": "https://github.com/airylinus", + "repo": null + }, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "labels": [ + + ], + "reactions": [ + + ], + "review_requests": [ + + ], + "close_issue_references": [ + + ], + "work_in_progress": false, + "merged_at": "2013-07-12T02:10:52Z", + "closed_at": "2013-07-12T02:10:52Z", + "created_at": "2013-07-12T02:04:44Z" + }, +*/ +type githubPullRequest struct { + URL string + User string + Title string + Body string + Base struct { + Ref string + Sha string + User string + Repo string + } + Head struct { + Ref string + Sha string + User string + Repo string + } + Assignee string + Assignees []string + Milestone string + Lables []githubLabel + Reactions []githubReaction + ReviewRequests []struct{} + CloseIssueReferences []struct{} + WorkInProgress bool + MergedAt *time.Time + ClosedAt *time.Time + CreatedAt time.Time +} + +func (g *githubPullRequest) Index() int64 { + fields := strings.Split(g.URL, "/") + i, _ := strconv.ParseInt(fields[len(fields)-1], 10, 64) + return i +} + +// GetPullRequests returns pull requests according page and perPage +func (r *GithubExportedDataRestorer) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { + var pulls = make([]*base.PullRequest, 0, 50) + if err := r.readJSONFiles("pull_requests", func() interface{} { + return &[]githubPullRequest{} + }, func(content interface{}) error { + prs := content.(*[]githubPullRequest) + for _, pr := range *prs { + user := r.users[pr.User] + var state = "open" + if pr.MergedAt != nil { + state = "merged" + } else if pr.ClosedAt != nil { + state = "closed" + } + pulls = append(pulls, &base.PullRequest{ + Number: pr.Index(), + Title: pr.Title, + Content: pr.Body, + Milestone: pr.Milestone, + State: state, + PosterID: user.ID(), + PosterName: user.Login, + PosterEmail: user.Email(), + Context: base.BasicIssueContext(pr.Index()), + Reactions: r.getReactions(pr.Reactions), + Created: pr.CreatedAt, + //Updated: pr., + Closed: pr.ClosedAt, + Labels: r.getLabels(pr.Lables), + //PatchURL : pr. + Merged: pr.MergedAt != nil, + MergedTime: pr.MergedAt, + //MergeCommitSHA : pr.Merge + Head: base.PullRequestBranch{ + Ref: pr.Head.Ref, + SHA: pr.Head.Sha, + // TODO: + }, + Base: base.PullRequestBranch{ + Ref: pr.Base.Ref, + SHA: pr.Base.Sha, + // TODO: + }, + Assignees: pr.Assignees, + }) + } + return nil + }); err != nil { + return nil, false, err + } + + return pulls, true, nil +} + +/* + { + "type": "pull_request_review", + "url": "https://github.com/go-gitea/test_repo/pull/3/files#pullrequestreview-315859956", + "pull_request": "https://github.com/go-gitea/test_repo/pull/3", + "user": "https://github.com/jolheiser", + "body": "", + "head_sha": "076160cf0b039f13e5eff19619932d181269414b", + "formatter": "markdown", + "state": 40, + "reactions": [ + + ], + "created_at": "2019-11-12T21:35:24Z", + "submitted_at": "2019-11-12T21:35:24Z" + }, +*/ +type pullrequestReview struct { + PullRequest string + User string + Body string + HeadSha string + State int + Reactions []githubReaction + CreatedAt time.Time + SubmittedAt *time.Time +} + +func (p *pullrequestReview) Index() int64 { + fields := strings.Split(p.PullRequest, "/") + idx, _ := strconv.ParseInt(fields[len(fields)-1], 10, 64) + return idx +} + +func (p *pullrequestReview) GetState() string { + return fmt.Sprintf("%d", p.State) +} + +// GetReviews returns pull requests review +func (r *GithubExportedDataRestorer) GetReviews(context base.IssueContext) ([]*base.Review, error) { + var reviews = make([]*base.Review, 0, 10) + if err := r.readJSONFiles("pull_request_reviews", func() interface{} { + return &[]pullrequestReview{} + }, func(content interface{}) error { + prReviews := content.(*[]pullrequestReview) + for _, review := range *prReviews { + user := r.users[review.User] + reviews = append(reviews, &base.Review{ + IssueIndex: review.Index(), + ReviewerID: user.ID(), + ReviewerName: user.Login, + CommitID: review.HeadSha, + Content: review.Body, + CreatedAt: review.CreatedAt, + State: review.GetState(), + }) + } + return nil + }); err != nil { + return nil, err + } + + return reviews, nil +} diff --git a/services/migrations/restore_github_test.go b/services/migrations/restore_github_test.go new file mode 100644 index 0000000000000..dfabf115d2440 --- /dev/null +++ b/services/migrations/restore_github_test.go @@ -0,0 +1,55 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "context" + "testing" + + "code.gitea.io/gitea/modules/migration" + base "code.gitea.io/gitea/modules/migration" + "github.com/stretchr/testify/assert" +) + +func TestParseGithubExportedData(t *testing.T) { + restorer, err := NewGithubExportedDataRestorer(context.Background(), "../../testdata/github_migration/migration_archive_test_repo.tar.gz", "lunny", "test_repo") + assert.NoError(t, err) + assert.EqualValues(t, 49, len(restorer.users)) + + repo, err := restorer.GetRepoInfo() + assert.NoError(t, err) + assert.EqualValues(t, "test_repo", repo.Name) + + milestones, err := restorer.GetMilestones() + assert.NoError(t, err) + assert.EqualValues(t, 2, len(milestones)) + + releases, err := restorer.GetReleases() + assert.NoError(t, err) + assert.EqualValues(t, 1, len(releases)) + + labels, err := restorer.GetLabels() + assert.NoError(t, err) + assert.EqualValues(t, 9, len(labels)) + + issues, isEnd, err := restorer.GetIssues(1, 100) + assert.NoError(t, err) + assert.True(t, isEnd) + assert.EqualValues(t, 2, len(issues)) + + comments, isEnd, err := restorer.GetComments(migration.GetCommentOptions{}) + assert.NoError(t, err) + assert.True(t, isEnd) + assert.EqualValues(t, 2, len(comments)) + + prs, isEnd, err := restorer.GetPullRequests(1, 100) + assert.NoError(t, err) + assert.True(t, isEnd) + assert.EqualValues(t, 2, len(prs)) + + reviewers, err := restorer.GetReviews(base.BasicIssueContext(0)) + assert.NoError(t, err) + assert.EqualValues(t, 6, len(reviewers)) +} diff --git a/testdata/github_migration/migration_archive_test_repo.tar.gz b/testdata/github_migration/migration_archive_test_repo.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..995e135c3cd8cd1521640a871b3bf97c6688b5f3 GIT binary patch literal 24062 zcmV)1K+V4&iwFSMvC?4x1MEG^lH)jXbKb8&)I_XjLXYB86cw>yRaHNyyL#GP+8ukQ z*By|El29TE4IiaeZEVCIeelUQpM3GfN8jz!gwOs5`v?39PVgm?qEwPVm9(*~P`641 zm6<>$@`VK0KL6_AF9}|TuE(Dpt?-$=zLGV$qv@(-CV$M09&1^Dci}X%> zk6|D`m3|Ok_1p0eIph&@1L6h|jPP8MO~d@_``>6O7@Li$%ORBEyV!}SyJW4uqv(e) zy*{LjUw(}|f^B<&hQy}io*;nl&>*1h7{OO>fh@d0%thEmY(`wR^x4 z8+akfHNZg7TS}~?C29C$^!fX2Ky4dF?bFQ0oK`oq?Q|Gmhy!AM!U3m3k)&=5-b!a6 zDLqN=>75_X;Kn~T_Z6w>1gIWj>Ju@c(iE+$RSMNAhE%NHV9!Rc2z4LOb?KHw!Y*cb z1QK$M??HOrUtghX;tYw`L`lO9yA9^pBj^PST`OXBatrF?=$0nxJM4~4aOk6J_yfI) z{z|}n2i-!?Kvr##PxyZI8l9u7XXwRJK5}xHVteieb2^&ZA;|e9@o>QZxB=PXR~{2m zpAiuFwneIB^oL*m!}Ezv+$MTSJcr}mO-y6<^b;gU6H!@dDcV3*db-k6WkC^zQFR4X zURRbhy;3?0z*E6|%P*q8rI?lE5v@u^ZXil5k9$GzYA5n6znNfnJi!!%Z$e$XBfq1L zH6q~s$f;<%%p)^<(OwSVF+{*@Njkg-k_P@jhA=s z$$!0LcB;)(89}NG1{3CBA6*t(6G;Hm@%_w47v5+BJO`Hs2k5=Wz?A{35(fTg0t02C zm(K82v)}@8vOGEr0|r)S%)ng&EdX8=ctZRK;30(;3K;lC2%{g!a6)G@TIn3ELAPp# za?Hxh-n-C?jNy1+lM|0=BOSjJ6GB;TDe6F$dTO_)iCQ8H-&4y6Rt3;^<_gMUM>geZ zSzLoB70GA38QrY3uT@WHfAeAR{Az2N$X(r52BN++N_3x^C@3unoD!`ksXa~9BEf8* zOkFnTJR40#B|(FbtXeBoqN&X?6~V8GoOTz739)U~Ukm&QeD88kMCsPxjN)65PJK~^ z>x#6-A1%VqJW?Eb5H4T7D$Vte-ba^z=btZyeP49 zolQ1}>wZ3JK0B5JSs6$s#AiK4R1O>xVB5jLgQzGNilWQaI(-F2Dh<~-2qqN!=q;l| z`!PvCbLLI?S_yNJ!(19LkADNcPk0gm&`e0^NBMv^Aw=jdfv~F0J-ydJDew@+5m6U` z&>$<$VLQ-)CbFm>Cl{dIU%+in3?(AGDu>XYF&hShj$LeLu4hHK;3WOCdH2?>vs@An zc)z10^1J%$z(2h_D%t>SjtVY@A}Eh<-f35F)DNQZLZ_puol0peA*d}hoX_WNW?TMf z!k9e)$FuEJDWH8#&l=EA%4(huXep70p%~&RO2_jp`tU#nH4iY`)QvT&r;I|B-D{G9 zE}5eRTZSlGvX;8Hte??%tQj1$RUJq=&-4gK{GTbPX2JC**)UBAP%C*_gr9kjcr>9F z`Vo8JJFWzS&lnE`(G{Krn7U1L@eqPxx&GbzCc3hD>Nv|tH&J9P+=#WVi|=8ttx05kjG9>lLSTb0jQhdU?M1#T}9I+rBW!% z7}n~Uyt#lMc!ar?B<^B%hT*~?c*JgR38@4oW`DduG5RCy26)&8pXx~R5C=ct^qM_I zR7K4{Q_RL8UREFMTNxy4a<^Jdt>RcG6WfQjbtN$px*JOxPi^*hlN7Y1^c2y=tMeHvC{nZZBIH#OugFSvl5w@aV`ehL0{&lR9vW~#MALi`-cjr}_o4Xe3A|?iP1%)io2PIGtIRx{}oy3$knS5MffT6zoU+`fY3D8w0_C) z&CQRk%d4B4trwy%i9M#Fvm=A=I8E({+Ncq*MnJWeiysBlIwOTb|(+EWW~i007W{j3zDvxC?Z}+D-H}u-aR*{T+lF159Ux zqAPIQ_AJj#`q@bPbYl8nmRgE5kmVjmW*WeizR0@tx; zq3c{*IlGlrQ}SAv5CTHmf?@iBc~&z^ctt6$cg!N~?{1pt1NeFY|FFW(P4s>dOrnxu z3$Xxf3WD(klRhSXRpzRIU`@!8c$I})*Cf)r;2cvzi++kV{N!Hex)( zp5q7D3ihpvjZ)z`G=AbA9Y++n|EcoQM@{Z&UD2g>7kasH316)g69z+)<*vMDFIokb zrANTq(7Qrowkge~Q^ISfxwjd+njga90)4|>$`71*-vU!v6Inpq5c?!q9EDk1xi)&o zhW?0UoXd+-Ok+DHlJ}`S^v>~+g+Y}}eFt95J#dXQqZ%&^MhixVy10ohJ?v9^M-hv$ z_Y2sMAzs{8=fa>gQL?Gz*U^TNR;v2qm@E&uU zbak;K!}C-hH^|YrgCi*eNs5$&=<@1_JalC-ZkzmQ1+%*Ly8!+$1suCCp(O-2BH~WW z0&wp&3S^WTBG^k3U~s%7!L@T)Qx#*4i%~|A9itFmFU?}|_y7GT`kR(4#BjlF#Ne?) zjANSp2@8%nk#+7$%F#qt&7LGWk+B6fbIt@=t#k|nB9-bWrHEmG-e&iBy7Sa_QaX?8uEiWy`1*sY_Y~0;sVTEDFiN!SRy7pakgF|v8O07>+gYDp z)0Ra6FSgMJjebw3p?ind)I;upV|G@l_;G&v->1C^C=5bt2R@53?v^wXUqMNzWA#EPc8B z)wFKlfBy8(4#K=fBdQeOC)2*Wxj>Yc3k)VWK=B`h@DE#hs8{>Bka?azycDn;?!^O%P;UEWbo%sFeZ^FZ%xnJ^4>nWnST_4bfHYJ{5+^_1gDXl9)Sw)h{&}vH}M=z3ve}Y~zUM3KiGh10zw9Qyl z1AWWl0{|*SCIhiSUi<)$rVLngYqNQBUJ>&a6~iK=b<)vkEa9G%x#_0nCUP}((Nfks zW={ys-Ma5#$gb52L3k%2xnG3xUDn3CG#k|esuI0l+qQ|bsxks+H9 z;#Esg8AWE>;5vGnmx=oPQ3 zwwCAXhI5ea@XsC=l)Elp9n#lmuRQ!y zDPBk^Zba0blx8ux*Tavi>qN)Zi6*}-V&4mH``6`e*EIC%)OP{?feshB3Bn+NS+{p> zHO$nuJ?R6YBdutdQBh)tuV{cbD_o{<2flyk4@1`%+|u2VJBr?2b8J-wmh9xofS=rP z-1s)7y^*8ylxDIbMKjstWzH%vJ;_=B>i4@p|D}CSeLp0hpd;KA9^^qlo1gz9$sI{A zKK}{6$el0G|9OhnPpgWeAP`$Pm`Mvbe3CIE(*6nK>v_A+#6LT`77vLXT}ku>G-PPw?Nt2dBb)il zr2Gs51Gs&?{^ttF=SQY49@AIwhi^^usvM6-X`u{sG}G2iAuPGkJQxyt95_$Vkwc?M zgNFm7vDnBzG!Fhqw?pHg?yZM?2w_G+=pB62JYrFgXO0hz2sDCo@ZL9qYCSTy7GLz`aqW+-Busq>gRCPNO;xLooZxc;aNg8vU2(Gn)-ki zpK(wh+~WD@^}(&oGSmXOblPowfLD%itq*kJz{vW*mX0W_4{+&(yZQj<4lt_^Xz>87 z8o5?JaHl@d`C~TfgPK2Ip+2ak{kru5F76Jk4{mwCVU3WB8=Y$8Twza5eQ?VgHfofL zWrcrzKy#JA^#NV2QLPVlp;od!uq$P1^?@$eSJel(RC7}!$+@D6`k?05@@u4ZWf^%B z?C5iCZa&@!s`8^rweF^p(EZb+*t(}aJ8W+BqJ*QA7=6WD=Anu_?U(H5m+a@k(0s{$ z?t|@1_H%!1rS zGm2~E-RhC2^#QLqW3E2n`6JEhgIYdTWfSDhj%KtYnI8}*^D||n_50TLcQo1j&`a>& z&o`{TUt<@Kv&#C&Xr79j89ngv;C^O}Sk^n5^>*dpI5xVPHR4(CYu1Qly|Y;(l5M=r z8qq!CZq|rtgTGlLmTer)8qsZBZ?9YJ?Bdzh;&vWp9rnO$jyo(~ZDX=Ee(w3wie|*+k0nAURTt+^o**++7m`hxp&C1KXcUiRhU!AIDKk_T zj!l`Nx^Qfl8LA80qnV+)5N*s1)rDlc%urp}9;u6~5zXehxLTFkDP+2G9s0|4ISy6k z^ygTI^Rjr5S$%*HvT(i2QtcMrB7aPX%r|H1#tH25S^sLEbLH^KBR_|6Ju2TbQS)xl zNA`beQb#K7|1=~+{<8n)DPBLV1~D51;_=v~Vrw3Ucdnh^Qk(c+dum4o{%YNB7DnXI z?FzICMndwQ^zknVpcqa|RlD#uG-(5}Pr-yAt5p}y7+OnFG#TcDY-Yp0owcmk?K)=6 zLL3iD5M^iaY~0*v(c9ZklhB>=#(p2Qu!56>^2T|aL-%V?8>jbl|c3zF%Wy4e1I z_MQYhiYn>+APV6UR`6I?+lVV*2z}3VSdNec2!sSfA|mL}Gu@LknM{Y7o*W1wBC_ZL z3X00*ul{nV8`lF51QZc3KvX=q6jWAZ0TJ1-!ms$h*K>BK=g4HHJNcdW(d3)%cUHYu zuc}_XDvQQ9IAI0JBFH$wtvrLFm<{Y3G-ak3499WSX61Q;x0)D+=j=AZOc8;$0u{HY z78n-^E-1|RTX0%r-vJg$R0`!Qel*%TEz|DURRYdYvG0rfg<_c`BtC(ca z;9Amk6`OGiqvKu0usyJZp_!V4_uoy9cl|P%1X#dKCIwb7l2cYXs<>o4Qe1-M7LS`) zQZRBs%tuCm?>R*ik=(+Zg0Y!MUe4H@k@<3|;?hz1C1B~p zj0J@Sr4uue38V7iOQfI(ltFTGOACsNkm3=bfuhn9@G}!BEiNey6rNBpK0g!5DJd9V zP&5)5QBpiM6Vz8+Rtn1$%hiCAMfu)(CKQy8LO?NOeDR3V2{|SCFbn)$Ha0EDcIiSdg1vG(KO0&X5u44EZ~uGthA^mQ7-s`Bes>Lw4`J>Xt$@0~ziL7QQy&W}0U042AQInL}~bM4%`E z=ChfgQ4?tepIMXH9<+pyXwfaI6reIRVuyoV%BS?msPOVJF^&VQMvO*ncGQXzj2$C* zinrS_hQ=6%z&Xr};WR^Hq&22-G`)-#X&`~z;YPx`-9ovOSGo7{ztz~vsvtKeiOyy? zj|mhBaUkP>t|CIUmNC}lEe8)aJdjWGp5Q4u7-1Xs@u7Zdtg%nS6F7_42@k+24`cmm z0RejRwE#w-+9j6zyqeirhno-RWL@Q9-2eDIrH8f`Q^p7$g&+eF-lI@r$PH-}xsjz_ ztBZBoD%6yQ6A`B5*=Su4&z}PAdb;LbCvc&+rmhi@aTDm^mJ0%%mCc8XUv0 zX3oapoQa}=v|z_@vz5VUj3=yA#!RD!l1g!;tbs1x!2@PAi7p_c`TqqQt!n+_G)1J= zzaB|L|Kpnwamix-$uv-6%Lmq=SxgqgBtY6~j!@?}cO4r5Zo1^X0Wo!tOZo9QERn}_t&K4;lQjb=Ov%!A5_>x`t#$#0;CUK ziVVNE`+ESZ%G)U6wH#gw8H_^1>RT!hYnCuM`MB@qXS*Q zLm|jeC{48tg19aa#A1OUmJNt&gg|IXfgp`hE!-OJE5A z57{yb+x8_&urpHGRCxCtNkE;6FyqFfE|Eed2BX_!t@~DMD#e<7`v0T`0E}k;j}e$U z{vVOL|Eo_5r~hw}MlHD~zG^r7L0jB$@_{J=E&N$0_$O3B8ite$c0{m=PH65AWB+*# za;+Xo3JPTY)87IFIu%E1S*O$_FDzf-aY;y3J;K-X@LsJ$kU-m6$d8XbvJwf<=(cM_ zni*F~&7%>;2n~&(8_C8q7&@1PLLWv%4V@d#B0g};5cSC21Y_*tOdzL6zOfRw0=Gp< zE1iuqlxFja?yE1YOMx5+o{9Ku=_S!)tGLFXg_f|pApzBATU#SkHcDrsWQ*;#qjh*d z0y~UnlcuD1xaG#?sy?Gs();X&Q#+A;CX(N0iRIWtc9}9KzsnVZvl1GJ*;!jc!A(UE z^B(^8mQU#)%kcmzyP1f(SGAs)=BX5W${JYh5+yJV0ZzU5wGuB2Mv?z;oS;%GWo0rQ5?z+?l`0Ipf=p|K65$zF z*(-KHoELi|CZgs=A@!rbfH+3@4~S9+5i;o@pMz#anD5$6Mr>070)hp>6o`aj-*@ z$87R?Yko>en88*nDmaTMbrO4de=ZI0bnnm!}Cv)cF7|Ju+K?t84Tf z20_t>g+6H2cLoC)VHgY6WdTx_Puf!e-Qf_y1Ph(Y+BpIXBV03TNGL|3+73~pqiqekP(z3+p| z_co07)~JQ1@J84c7Mj9wW6G!D@585IB!YnBx=@5bGcmYFjDsY!1_n1P5e~2Pi;C{E#43P8i7H_$L zsv9eX8gck9y80YrvI&zMFe>rm5q}D3gh-z0e>AOZ|st1 zvQP||J30{-(6OLc6(y-nVwj*~$1u^X@DEADFh*(zM@a&PRhKu~9)iuGj7Jm%d-Ljn z;5eh-#H%lAZUhW1^fp(ckTmp$R}VHtuPQGV{wj)|=w)asq1TI=i{3oZdA%fCZC8md z9`T=u>V0x8x=!EnT%~;YAjI{v2J_TPSn%$;jr9l6WKIvXa zmIg4I{HLM+qtTT7r%TfG{}@pB#6f%l%64dP6s45fvaCrZD& zE~yVsLfYdHSx!};$!&p9yW-}Mb&Y#EA;43;*1kTg1=aCl z6%UP+%Mrm1H)7~~Tu{b?jpjbm%cjDlX*?arAho<^D25w0gSJx*t$`O*LE|dfz|D<- zsRTF>`6$c{r%m|!^@oMZRgMx4S|7E&=x8n}?`<1+%LYIoG(sf8I^c!;6?ar!{Pt0crXEsjAa`;DWP;bqJb6*7y_{E>H3_%|u)2oXE>G-p%t>Na*2sOoPP|K3I8~ zg`x0T--5fE*+lb1gyD|xxTFf+g$PcsjNrqtZ)G~2OLY>yIH9l!Ox(JKp zS^-dTcw5rS%bPjRA94Q*wJyPbD>7=B^`;Ba zmlHJUzYYLcdn@a3fPT0{QBsIH8pz24zk{5oo%J{*{}qUA^|c4UNLGlVmW_8cTdw9W zp`&h_rIN1?&E7g>#H@j$tDJQT(^>B|hhzwVQT#uX1ldIXGm)m0{HI6KkpDs_MC=lv zKc_=17o1+UL|e!k8TwMqga|yj=1}j-b=m^ARK+_ysBXI5-$mqJ=%6(j~ z!Sl9?N~c&0m$t1^N0cj1`}AnhEx9xYm}Cf7v6U5KjR6hkSsh~xXqK%L?69-}kme=U z0mOmnMl?RAh|C>Blo&B=E#6zFw(A6^AQ`bes?0(3Y`V5iV8wC|t5l=v)VSRPTd&{+ z1KOTi?P7V!h)oJ`uMQY0Q~#!2f(i#gT|m7D7_p*xsRHoZl_KvHi~zTdpKitsXm>bZ zj&fNEHDZEuyhE%N4QRQN^HhmW$>}km?XjW@UNv(YK}jn`2g^GpBRIn9a@Jib7(v9< zzC-nFdw6Z75%XKtw+$CHr&yAtXqqAkBWMT)g9&1|iJ^^vvqycbq!$kecmj_@;9W*6 zW%aEJJ=-p4UG)Phg(@S4%}#T%2DA%y$r>Z%S?Cnm!E2hEFlNS z2o}mpH+mqc6jzFE{fn}v@_5<9X7B$visDp?{_ByNhks~H+x$ro2mVbon!>*xsd@OX zW^I-F;GZN?_}3%Ffq(el0NQT0lN?E+IKkNMc8X_s)J76!mN%IQoUxkhI7!)PDofz% zAcDyV{{Lx{3jYMW6`JDz^hm!+#WwJ#U6gF?TKKe&aMxmjpareNVUMJ{J{D?7Wbhe_y-JC)*$Ye(2K72OA47VyEot-2SmY{PX0g zrI#-ma9K|wXUwHP=XS9#edfr-zka*DY(o0TS&xdR`|QCdoOt}zt*gdbcP#oO`}U!) zTUOj~^x`W^>6<6^%y@S5x5w&N9o#y1aPc1|kAC2*ynW1+Rb|~q56-;ynUaGy-P*Hb z<8Qxi+17AHAG+YG*Pc3jtikr~H8Vd<8^8Lqi7S3|Y&313-SER)*F96umK%fn zwtta6wA|9Qx_6(9yIxr__#8I!o?WN;ic?!=A73$fanJmAGb;15c5T^hwQ;9!NMHGD zr`6qAv0tx6wV5aOJUaZ7SG%3;aeUN01HGOlO3mp+Q ze-fZ-7kD-S^bZH`)&kP@inwTOUOVQu#-oScHaFDb%V+hO%&+`)^H=ltJ+x#)vBTZ# z$jRj&@7$1|mY|e1Fl*qjLeW`XC^#zVF`wr}~RxB(2~DQ4FI|G7TwAQhyZ5 zzyHGvVv+Ji``$)V()mA_+vq{2QrM&i{-f~!CXAp}@}J2>r~JR@lE&raj!EY+t-=SlY(Zz1*+R-P#l8ve(yj$Qw2zqoKic6>(?!d%5*9W)_yb-*4T%&7KLh zmJQ4A#gP6pGRB~N?(R8sO8V+0otC_nW_xy+<)%xD`h7jQr}K_&KV4Yesl!KWdMrX3 zI~~sJbLEXaieG)=zwgig`0kR+^S@tob%(#*JFd&~nLYlpr8KYOS@z9G-flOyr1zR|0Scg?b=#%}FD zW%UA2_ctqVD?MIw;^4P075uPemi5$thTWHK{N--@BWKbbJG$=K``zfqGYzN4TzY5P zzFRs|WuHl9Urya$|LK3WmDExr3qJhm+%;u2&piDAzyIlmMfvO3UcLNq(F@z0y{YGx(?uXv~Vnm0_ha8x6*wgnxcX2(pXz^P1xzYFS&G^z(_07QXRr`(}_@LXd z8}|;`v#H0O>+kC^xvJ{Pd&k{!;`rTP9m*d3^69zm8 zo_@vOFa7nxsoOfeJ13{%?`cKb&ADG(J9tr-tz)@UyQh=AzH+@d|GTXpOuOkEKjY;q zo}IL?>co|~S9|udGw%AbVPEaluKsD8KVIGUjlSzMSG{A-8}VHAq~(@lTS^{YxqfGP ze{A4_P3y{coOrMI(hHAPojqh(dhF$n2QT>Xy98Z*2#}|=r{kmb?HM^eb{ivmWIib=lAQEK0kfiYmYMP(q6yq66Ar*x)q)W#=4iI zX?Z7qN&j-N@aCPvh=S|iT7AKwueVQGeju$=A8VI)@XPL)xw~fdni}zc?Pp#n{`Z6J zw)Gu5_My%ld+cyc{r)|2-7SYVH2l%E`tlL;FU}lO_OBOKt?F=p*R{i5>-*2iH{G~y zZRr|*9kLerpRD=T{$u`1=5K%M_@8gyKWFtDJ%{XCao6>CWbJ?WjbA5yOdp*6#2=qt zD6M_uP|oN4#B)z{z4Wd#Z+`tv&$B%5c(-xb zp$n!Q$-ZjglK0N7ZSDT^;6t{?X$N-9xoqbLonC%)&9I>p+6`$OG56== zXFr;^VE@71JExgv)!f%@dzVEUk33)9yQjJA!2hxLq+v~5T|;q8L_kGxEki&S4Iv3h zNQf*Vs3;&JF5rTbWCDSZKo$rnh=OQw2PrC`1+5k>xQkoe5K&REf{21*rQ%i>P!UD# zckfJsYpeB#@AEy6^MlOHz4zR6&pwx#b5hUA_766nxM_&SyzTYzgmM1QM_kM99kWcc z(d5NBseJDN_JvWNeHyLzBXwzKJiklcQk*DqbapDtFZ`~t=Mi}(+8jfzb@e@)8OP&K z&S90S_8G?AytQtQ(~_zkbXMBw=6ef+&Nan54i+n%N<_iNS6;bW28_OAeYif~qVo7| zmnDXIml#Kt^rf>W#IAL%DL!6S*-InK(4@cS?)vTNk)__=qnRU)Wgp!6Tduu#e+$Im zQtJBK<44Y@3&}OopK6{MJ+WncY}*U7s~cugo<{rBce++Q-a%d`8yhuS_+ZtIRoAm4 zPGfUBD}VJW$-pK3v?ALbNIEdNo25O7%d1mU3r|ou`XS~v1!wH; zOgmEK`dgl5rxq!dzI(-qP8MfV=gi+>Rcd#m(N?^G+w0B>6N=z`>fnN|wscD6!h)3k zlpja}k9l5;N*DDF`kplXiOa(It6Pvk3lH6yb@8e9j-t-<3ul-sWFDmULb{#%-y?mSPT3PL*jq9&p zJ|Fq~#{MgEBP6ME<_~>>Chy((m+i7O)Wys87VR`^N$2}XLx*F$=Pbh@r^zcF2b?)n zqbfOYqk3C$d1&{aZs%N0P+!Z@2=rQSa^~=D9+bahTV>F*kPZ<8fG;8y` zrIv~1>bf-tFE)njx3O;ytQ*!;+T)jrH^S2bdn7wo1PZ%srFLExH#c1-8 z9|b-A_eHZdO!ZxOME}${NoYo{Q+C1h)dNtj(uFyD)2KjS#(MD@>ek*3 zw=e&mWebEP9e2xK&*peNEbL@r^2=IpUnj)DYbCnmf#&jV3Zi=a=w4|0W$V~zv(0yl zDt?|ZA?fFwhzkJ|77y<^(LUpeN5zzKY8CByz}V`*0~-qzzdO><-BRc6sM;mD!Af$p z{K>^WTg&njOP-|!y6tLu-F;EoQ15)}BG2t6tP$4_-_G?9RvopjM(s2O*+pSnpZDk; zyz_~hWBrYFvmP~$Ux6LeudSPHvOh+zp)vVsmNLmexSZL^f{7STxP7yChVe=L=izag zvB}l@EmQV49yq_fKB!m_Th;cej1|Axa%5x4pGkcsCGl(TbT->O=0@|GDi&%ia#xWaYNkjhBuloNR??^YNTE)R<} z4Sd*T0(VT;olbpKvU>jHdjY*HJvqZ4vlKU}ret@M7T!I`1d;`?#b68_IU`1fn>dPwqD#-820 z&+1^5e`jfX+M1R<4@&Hgjn>ap{+$>!?2nrGn@8`>?CbHMZcU%ne{)XmXXjZJ9DT-c z)i0o)&ME4m$#G=S>5hyE=f_o2Py1AsJB-EJG~1ic<*fGhDDq)nE{eZpJkxKiA0y?L zmS$DV)9x{thR$DC?C#@LAZZ^|7n1rQ@|N3Xv3q8x{muv44&1%DiGKD~Nv3IH*#rgyTC+WpIB_{8ILz+l$@XpG<`vs2?@sIgh#mfbrf1yMo_PkJC*(pYclFV*^}LZhM&2YxJsGonDzd9@!$i ze{xs$PuXkc#T_ylZNy<*DjsC(9{%1-&B&j zQxLYPu*|=9{L~<`;W_Sg#pdRj@?C!v=kKr;rP5N)ENRIX4Ibe&XZaw>pKC_ndzGrr zOG>}9?a;U2&`l1vPg%m<6HuK{zt<_F-*(OqD}`qi zhCf)$T+pvnJmK$G6`iigrg)?ccWj7A^%!t!hH&Am{^MRfSF6fs$45Ty)^H@TU4NWe zmpl0W>pshbE}nJPQT+-#&rRC2wQPeg693D>Dcm{+a`d6_eix@f zd8u9c^z%DuB$@AaBEltjkSx#Wm{ZXx*RTX7@3KYB$|pue#roe1Lea@rd+sXp-nd6y zS+7?;JL1^I(uxqu63&U=*(X*e4P5*BLi)nun1_B3irgRB#mpKLPg>nxH-HcYHBj;%5ba;x%aOz@gF^isuo zSDWa0gK3RP=`qK5e?Ns|_j<4IM)FegISuW@ydqwncOGlHuD@gLmS?Y*=kBa8a&B8R zS>E@y-mT?ZuT9I_+WviO>tT9z>C>9OPxSY8^&Ea^>uhd&^o4e}rrvkAbqWke_fGG= zqR+2;;#aOYHS6TCIhDO?$Ljlf?Yg_h|Xxm#eTJliK#wf`!+deeo!dhK1Y$*N+(g`?=XJuB~6 z-x^Re@VCjC?#~S+HIXLm=yId8xqqQ%U507nK-IR><{QI*t5WP|ZV!mF z-p)0Qh}{>KiU1TY45QXi^Z%MW!JjQpL*Sie);pLE$M=dTUZ0ulQva1 z4L*>WIVLG0=y#94!F@;@)D#b2t3|^~moAvxV?AxY<&$fHH*!sw$CQ6=>E_e3A+>pJ z!<$wArhb4k<=RwZmnU|cA zC=8QgB1Ei~O5^Oc7fQK81%AjZF~gB9WT8BU2xGBCs3VU<=V5F%9b=)6BBqe(hzi(D zkr;vS96~}u!$LxI*~k@`099gSu_IGJM|mQ#h|WVfbTNm)5i@xVHdpNA#1vv85tqkf zF-1%Q1k4StD-uus+=>pwP;KQ32!Z^$X~o|f$s$IwIcyPwN#}~$VzF2x1Rzk1fieLc z7LS1mcqkiVfrVl3jEsc2g2+xxE{B7Pd2A-ejye{AT~Z#$gHbqa#wKJzgvVvA37Qtf}xs%Wiv^PTl1LDIi-SKJd$RL=}dr z5WJu&k-5*t(gr~l7y`?`!o=D%&~p?_u8l+~Hk+arU~T= zYzCS1=2H$LU2+UC0x)H=3qmK8plyhh1NC#k>l%{xex@HshQ9{~Cv-g@anDa1cmj4pEpw zN+!d|NvTpugepFM9*ZhuAT(1FSKj_3ot?)=VKOG43<9bVG7y>JB7i^WY2DwPsw z5fO428iRqr7BqJtt(3@w#CMc$52qpis47f}L;}GCU<7byL5jlU8E{tot4q0Dg{Q_2 z!*sQJfD^<4Jn}#5L#{B9bU< z?f67L1*THO!SG6Z4O8`pL`8)1$yf|3g;>KH7ZMPljh#5&&)IqEL_V3+(F+Wu=l?L|KlV5{UO$1dMOH?!oosyOvX`Ess+e<_aS76sT_dN-3!4f2}h71 z=_SfHYbVBGB3moD9nJ+3p?1I>Db-RHxNYw`TPT$aBNC{H`)vFY*R+HL2rdB<6k_qt zC@l`QR`@`P2#mzX;6fJy@FxAE$aUbXg$TRKfeA!x;iAN0RD|M60e^7e*Y&k0`qID; z1!4s>St7$@Xkvu3S|#RE@CHJj!EGSSTI{CF2<&kA0A28E@~{g;R0jxX!1zglZ^P0; z(gNwYL77g6tuQ&h6tSX0MiO_3W@0M7j%*zq!olcx2dEe&mnorOMr%4&i4c7#?+P!) zfv;7aQK$rNAHyna;7Ok7EP>TXfUdkVq(fs!pHLV0pl{`dSb*meNDH70YK82L64<)} zb8PPl31aUm!i0cMwjH?JeYi$k3UzCY0;sv*tYz9xA3Ac3k7o!@VC}iL9~H6bSh;;T zVs9^#+eZNuRB<+TWD=AaxM%`^9U{gT?~GVkXxH{hO8{BzEyK+eff%L%;PL7xz!6C0 zu}Cat1Jo1Go)=0~YXX*A3BAD^UxN!w7%B_x@G&3SMMMC67c3Ej7t4baLg|G&F(OO> zP_=|WfU`h_KqaE25@i_f9;jLb)JojxfMFDT1`6P>48!ZfytVrRVJI*mCKJNl9ucO& z*NjQwCJGk8Q_r;cP|v+7NsI31$L84BJm^yI1oaY;BDMJY1b^1#?hd=mwkxT z8@d5tm;$U&4&jKkHGzQh0!|&3$h27upn(A418#|ccz`zB4@pKv=v0JZhhWhNMMxnG z(?)N@9r5w$jE5Vzl=1>OWR0fJxPQt)0{!$p|^p4&+Y;159PU#%e?cIdPK$NBRN zBHk4*ETc7uZ|ojHkPj^#(L<|zZ+a1Zw72;m*eB@9w6@NHj||u8v1`J=TltQ4Av#!Z z*7c8z(sliZb?KTvTpDgEbfK{`zFP`afUsvkasRKqt8Z`G$oBhh{3)i2Zyg8AjvXK+ z5LgIw;ocVd6nMD%6i5|IPIT)?Mv{|yf&J|F{LYz?MzWLGfz57rHBVC#dtT3+Idk48 ziZ#}Emm?lb7jqy0N1eNukD?zG(?jhRJuErsG(yU48~T4&z4vcF_P&5NFJcAq+gm1T zE^XfVKbb5wb704 zW}6?0qc8NldYTOI7_N(Re8FWY*tO`gtlpL<-fz|+lXmCcm`gC3rCXEcs|o7f@|#1A zE&y~D(WLH5cRIHK+N{EeU4;{?o<#clO%7?ioaT+syjiLbi6zIya$U$vi$cQFnA_{B z{4L~3Cm=y!X5Lf<3S~1#jmF-Qevp>&&yVGPvnL?kBHN;2ny7M2y@^m95etyUbl;KS zhiCjI-i+I0J9PVor=dUfWG!8drzM7*xwDPSw)!jY^#z>$ zhSPJ}GMz=YtW}qL65*#RdJr4N_^Eeji|^n$x>#PqN-NME_jB@95!(!JX?7Ds{Ubv_L_2CX-wL1Z?hbXApDp5Dn)l~g&;IXLR?+|BrL)-tr^Y==kCxN@qFko` z-Kzf=B3RpeoKH0hUE{NANx0) z=AWISeB6!^~A7#_#a>XPtm z*PizlL`BG1doA3`tefIg6M0kW5&Y?_z9GMOom8Du8JQ^UAoBf3rh z^6b^YPp@A-J$Uict7k8sAMC$){rnZ(!88w&-S)N`YiGyY4_Y|Q&{6IPmV{(r8zx0d2!#U#*va;`YY^D#o<6>bb0{TGOLg_H*R{`lF? z*2&S0NNCHS$I-&uxpkM4;J&P^dj4_quEhNP>C*E*xHYH$8QiST|MisO{9pC>Gn3}i zfi53?IBL;1jybz`XjVEXVr#amj5A$p!=Tc7& zL_HL>&AxJ7^&I3u`{3zy z5HyG@97=m;P(w@}eK0dcO$TWD`sW`muS>O`38O93&yE3b@o?~zd9p#qJ%%wWr>yx} zgAY@0EK7YBiAy8cWxaK|zAM!;fp9M~5FlYj@OvAF;s-gFXdlp6G$sH?zL&s+3`?SR zrNA%(CZ+C)4L%7*_s#Ed=+p0d-tvY11K$+8l4@DvI6lJ`ooNNcDJ+7*`j4#H2Z?x5 z+8Rj~F`($<9d+J3`^ixouw`JTbafoTC8A9PT*lvXnti7+O~&-gz-csqLwlm2BNRM# zCI21m9$hcg!M&azoc8cgnfj!%M74g|a_@NV=v_;-wp#5*L&Sq5@O;H9CrUcfztKwd z&hdx(SNgO8DZz*#PBX9Vz@?;#8Ign_BFNghdP?A@A^c!WxJjSvHr8P30Uw`tQS@Ev z!*oKuJP*#)Vd6!@;|`yE+G7^HG;mQVfg{nLJs*e+G89a;#&s13G0oU)R_PE9vI{ya zX4mbOdnd7j*Cq=eNP1Zv6qIr&aI&)i{4Cm)!r` z{af2r{SSRw-~ZQ9{=xjeVS2Ac00JtU28f-LBo<4c_Slut~gu(r9 z`&wD#9eH7tW_zg85UPpmILt!NP}yDOhoh0wHQ8WxGRbD*(k~ZJ(a(ZADU+J{@NKAV zgnzIjV%g%waDDi4|JgHMr8rU|$d(s=>JdD(8x*4Ix(#j=;oBZgs0Xe)&$q9pa>(h z>kd8k-E&>+l?#7e`W@dP*qng3_fWZiQ%!^UpAawc!~Vg;AAZ=|$D&=II4kA;{oUKG z7BO09L*)+J?8K%|*N`B0@7~)dFTQU)fkV!&dPsEWL?{;Rzm^CJq+!mP&N6eEL-?dkp5|y7CwdflWAgAZs^1L z8yvcrrs6LE-;aU2_zTnXO&>g8W`|e?*u`tO)?r+503|LKpfM~;P8Kp|nK2siskCj3 z((KVhVwiZZVyf|IM3~Kv=7rZa@Ewlr-NG=(J1YL&wC6m@eDCwkb=^MzGxQ%=R^Kt~d$3;cbG)G&J`?9G?fi|c59ETzXmOCj<#`xS0@)DPrI8VotDU34zCd%8(9Jh%z zKw$4nVqpT`rF+gL9D{;ORVU6kI%Kc2G&71dLLk;e^8V0&32okZW^&eqe&Ese?Z5c( z$7inð|tgYTdH@MN#qHy^!x@%ra|`e4v_`Sf9buesf5lwz=z8+H#px* z&SK-F_r81Cp7y$3sh4{Pw%L|n*ltb# zzI^dWizXz%C>T?u;j1S}?rt`FB?Q=2Z~MJT z3kyDCwGY7;!F?_w#m15oDU4hK%Mh}k{*Yl)8DG_c7%2?v@Zaz1ykzl z8-dUPF!jcbh1G)%hQ2O57HIP{#8AIub}%W zRA&-Iaphps{e}*JFHiL#>1HT%*kba1IP&V8nJb0#-zfSMUf4rOfTlsz^rQAm6R-L zmIR(S*rfpu2ho-k>NVJ}F{N&9(nTGydTHIGtkzc>#G8sXmwg!dV-stvQ_<*#?e22s zPoiq!P`5F;W!Nu!3~S?zrJ3YN6rZgq_&sZxnK2ZPf-JKp*vUC;&wzv_BFV$3GiQX> z#Pu*FQXL{G0F6TvX0>Xgp+DQhs10;41B-37uu87O7&*`yJYLS8W99mV{Z$m@@*Q)M z>Fne%9*0BZDQ0#1ujBnrFW47~^rQMKbA<_GCPsi3RpSg8@cp!@zPs6zlx-36X$>sq;A{elu)!8xjNH}Hj>b+FUr}S&1#3Gma zGXA#3e5Dai^y10fC7>cbsFs5XSlxUzdIyJ$*kTI_GjTc*btngRs|nF1;uW;2)T|xh zyw+$y@CU3XNMlyjY8D2#!TFjr-hu0Y*82br4*C_~e_NVz#AED2XSmGr6)oMOC zr+HBpG%rq_H4kwHuzEI-*5J3!T|QbdNg4Ln!d1cEIfFsktiEXL3x>k|SAo5D|v@B^)HIcLGm#udu`u`;5TuGaz z2Kzl}Zxyrp0-1P7Llt%<;Sc9J_%CbfNO*AKO(YRW@BN1HH@VSPoIYS}gKn<9`>M|o z!Jkz+8Hd?M?`_oU=;`&}@mH`8angiwaH2NyE-QT^(?L@@Ui(FQZVq^T3;Q0t(|MXW zuK61#L#b+7^Z&>Ly7vi6pLJD#x|7PyZ83-i~R#mhGJjJBC>d+;ArY4e!B2J&U3vrifhs`OeVn@ z40~*bk_>JmV7EsC4&1sla>K;4sa_y`PvzzS3zABQj-Gla|Ba6Y=oOqyHI}hH4oL@K zi%}av$R;c?%@P?Ult6v}`3pW#=Zu@v-66pMPGQKvXN{0g21AQNRNRxa*B&Tk6bRNI zPQ$Tp-FcF;U>GNu1=Q)Ryh8mI6~FabZgg>jm&)J$JCa(88IdprrSmM|rLxba3=U zOvQ~^6;@9|?~Q(2b-P^%?iEY}#bhu_!o-|b%7keP|4Ikwh~20w=MN$TAHS4j^w7`I z$C^dr0sXo0a2!y5x{0);qCrLQ6oG`DN9{u`H#{9|RXJT^6)sH!nqop5!Yv~WXg5x7 z?2Sf}&e9l^*}zOodjt(TPM_>ZPC+a{atD%yMgU3_C4{n`F2csf;3B9pK&#wSem1yc z&Zw0ep!onRj-dV#AR=Ukp&KbQI~CrCTQ4}+Bp}&qB+Q!S_n=;Z>;@kR19Z-MSvYXW zN<8HB%oxPRu#D5h%?nycTv=$5n>_269J!8$tmYN<;2<8>KpS+~AA6Iu5E@hHh;Vq? zw09&K9p~{^f`>wC3Q#tF2(nfxX;?h%V53jyZbP#6#IEaJ`lFGaTl6%iXa1f09kuX* zM=kMls2(n?)T?~9ppNw^eumUr{-Ykv*X&t8MR_|{(XMyMQ`|4r)b)Dm;x2m(^<@B9 z0Mgv^jmEWy|!og~q#BCmNn^72r##cQ% z=hW%8=F;ZucS4P^F8m2a3Lg=n)#fJUarU`aP=G3J1ga6U3{o6asIKtP`?|ke0YE|` zzrqE5GMN%RAkqyaUMP^f7ih8;U~vsyxQ56P1a;7b3~s{( z(`+p;jJSHSJbN;O`N?P*x+{099z?Ev*2fqka6Tz)820aS4jQ97l;WLY1!rW3# zT#ZQ`W4s)=AKSlFJ(>~K9C^^4()Tl+tL>1zWURReWu?9^nF-fpt?}Y%e9mIhMh~IOc+Li3GXSR3u~t5-_iL z(VVEqP~%pfUiQ2x$$g|(0pt~=B@7n32eeWReMUr!PG^L!I0;gsVC>O5w6z+7Vo)a7XfhSoH?I*oudR2!C~tcU@kr{jRJnbHID zVCcWxST(m0_l4*$Sr7dpi@D!6H48;du?!6fM-eZ_A4u!sZ|P)inNygDcP|)qeo(Uy zOi8MQV<|s)tA4vNhw@8xVK-Qmyma91Jkbqq(Bk zcIInncYl0D-Lj`npj!P8Eh{wIs_(Ro`NnFdlOmJayb5s4K z-yyRd3i&D_o>K@!z~{b<7Pg6pkwX%q(YjOd^wH>tBMq*GWN`cgMzaNHGwr1Xowvx; z9Vdd6-{u2;{2+-XR?>s8!?E7kWFrjNgGnP>JrcZQIMt3GA2wxv?ZF zDIT%h>}X#H4&s|eZs{VW57Xl~0dpyYt@%;^w!U8U@mWIl@+b=HIF$k^%p%vU{e^O@ zI%-EXOQUv-0n+raHec6Eu&?cv~n$ySRA?wcI-!<#tJaFbgRTjT*h(DJme5C7gKQ>300hC zl_Yb83-)i{a2cMZ@cyzbG_zlTY1Ku{-9fA1+t-?}a9mY*XNXWL*ssM679FO!c3)$y zm0rJS(VH^ghb936r|;beQxmgefBvA|C=n-##Fhk5(+EoTi0>!pCG!`CaOo6>4#pAG zD3f!UT|T+W#rM5dZ~bv%oa=IaSw;V|YTS=0m*xMvU48#=FxaW{zg$oG8u@=Oi~fNU z-DrI}3uA%U8@`AJs)nh|#fmTR7Ul?zRbs^#dJE^VBjt?2|EPSy)rgd=$e6n-OsU+Wp??lfvX1WD^i(Dnp|KZ2$<#~Ekp`M8UR zuU|cVapj;3WbSZ3r0_M;Ar*c0S(7VHKD&RPKhC4047LunRnVVK#^DefJr6AycElS> zII|6%E2J`^mL9mV#!ghUj%gHrQ0~yIh-84?`(M=ByUjlS-}77TIp4a+7tf<`vHNGA z|CRi|rzD?dvBdbs)325P@7DJA-20#X?OOkTEoI64Uz&gPOad>L!#_Af1*i;&0B44q z&#V?PB_tpLV7LS&9ciO=>cJ)svk+a2uF>$B^CjrS2o?A$N>n1yY_nBh(2s za3RUC&!Uu0l#U856u=#EzV-6_Ut)>%Rd~+Rp7@Z%?BOk3NHb1QUb@`3A`+Y zu34+m#_81x(Z4I;JB+o%!};TjrPf5hYBzT2q$8Sk7}aI*+C4YZxz0$=7|%?Gu`HYv zOVPefW#kbPG(!bWHm5<7&=sb`fsLKvFgW&3!+4t9g|j^S4axQFn=3R4*hfq9qgjoM zridw$w_d%3#X9yYWwj3N^C^`2^rzT8hGOg}hZ$%*DX}lipA>oCEN?Uow8pw6Gakv! zEh`q-?KVuVt3LZ9b116C`^+c269G@ZzbBZrVU8FgXu&1LZ2ZRivSMLp6^XUGaWc36 z?QR24=4`af4&=?vinsU+PV|RKvileiF?Xyhxx6jJFJn&2-wHfTbg;yUX@tBZ3Enl# z*m2mL_e(%)p`u2@99`Mz!2;-9Aj$U#;V_Wc?qa59Bw6StAa?F}tLPr1kpRyjM1WmHa`6OR_o zg~O_88wfBVhYI8#P1Fm=3sC1Y6Iv^C@A`t5naOvIJB7K66WqZW(uA`}hCj2%dKslX z=Fa?>BX)EU1xQrkVr9{+!yr2g^yXTAIdb6SP%~2t*3^v?m!F}dZ9*y^ygV=#stnHl zI2ex&a_lozXsnCAb`c@qZ;n`JUuAb>=cz>~Pv>a@h-wE}eBcL@aXiDjpVMe4naLw| zha-B@v#-7GP%x{U{f9fxb2t7T45yiK8!lTBix=rS8kBh@8{D=VT&|ENh%=T<0=c%} z;G~yPZp*DO8FqVQ5yYEh+I1C#6%T_ZNDc!(z@W>ve*}N}oG`{2`U<%P_+bd~dDd@O zBPueLY+B#g&pPP*F&_|QeH39=)9E3RrYT-ta}mk{pF16>AkQ*SusM9MYts@xpXV5> z;WtHc8hO9Xbw5nQAUkISyX!X^7RB3txK^ihk9+VF(n#)TQ=#!E_EZv2kB&`Uk zOlV#YsG9@rH%y4~Ksg{1j@ngEe*EdxUk_gW{N%|20`U)?KKWmhaI7%O1VU2=MXE79 zY4mNU7~{XsPZc(ATWT)=n#t&6fR=~x&qJ??0hfZ|A_(TgR;gBzsL|?oKM%FhOkJ7A z#7x~8tabYOCCYO0pCQ4$D~)kk{QuUR{%@;)yZnpEyVfhc2|CZbTaeQ>ey+>Jj z|NA@JH@E8jzmBrP{)4DqX$*Pe;pVyf-`^Qj_kX8;|L=OrXnSzD<=yc|{?;Au_SWcj z|MqBbr+@3a(az3b82J8ockbLA_ye_cMcm}~=zo9Sy1jGraNv#nZGY#!qV4VX?|iql z{he~RzT2wphW{F6x%1Da^(&8Y75cxuUDN;TC@btgyZ>Bej4SYegPofHyOy%T{)_L> z6~?&2{@?8H)cb!OWx4&oGV90XGX9TFdzJog^=tdzwG=wp9EW3$x5U5I%voLPQkS~a zr7m@;OI_+xm%7xYE_JC(UFuSoy40mEb*W2T>Qa}w)TJ(UsmmXw{69G9+)e;60sw`x Bb4CCF literal 0 HcmV?d00001 From 69ef887b137a42a9d888616aefe812a24edc627e Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 3 Jan 2022 22:07:45 +0800 Subject: [PATCH 02/54] Review comments --- services/migrations/restore_github.go | 74 ++++++++++++++++++++++ services/migrations/restore_github_test.go | 1 + 2 files changed, 75 insertions(+) diff --git a/services/migrations/restore_github.go b/services/migrations/restore_github.go index 7ed977ac3da43..3652ae1f6b881 100644 --- a/services/migrations/restore_github.go +++ b/services/migrations/restore_github.go @@ -712,6 +712,7 @@ func (r *GithubExportedDataRestorer) GetPullRequests(page, perPage int) ([]*base }, */ type pullrequestReview struct { + URL string PullRequest string User string Body string @@ -732,8 +733,80 @@ func (p *pullrequestReview) GetState() string { return fmt.Sprintf("%d", p.State) } +/*{ + "type": "pull_request_review_comment", + "url": "https://github.com/go-gitea/test_repo/pull/4/files#r363017488", + "pull_request": "https://github.com/go-gitea/test_repo/pull/4", + "pull_request_review": "https://github.com/go-gitea/test_repo/pull/4/files#pullrequestreview-338338740", + "pull_request_review_thread": "https://github.com/go-gitea/test_repo/pull/4/files#pullrequestreviewthread-224172719", + "user": "https://github.com/lunny", + "body": "This is a good pull request.", + "formatter": "markdown", + "diff_hunk": "@@ -1,2 +1,4 @@\n # test_repo\n Test repository for testing migration from github to gitea\n+", + "path": "README.md", + "position": 3, + "original_position": 3, + "commit_id": "2be9101c543658591222acbee3eb799edfc3853d", + "original_commit_id": "2be9101c543658591222acbee3eb799edfc3853d", + "state": 1, + "in_reply_to": null, + "reactions": [ + + ], + "created_at": "2020-01-04T05:33:06Z" +},*/ +type pullrequestReviewComment struct { + PullRequest string + PullRequestReview string + PullRequestReviewThread string + User string + Body string + DiffHunk string + Path string + Position int + OriginalPosition int + CommitID string + OriginalCommitID string + State int + //InReplyTo + Reactions []githubReaction + CreatedAt time.Time +} + +func (r *GithubExportedDataRestorer) getReviewComments(comments []pullrequestReviewComment) []*base.ReviewComment { + var res []*base.ReviewComment + for _, c := range comments { + user := r.users[c.User] + res = append(res, &base.ReviewComment{ + //InReplyTo: , + Content: c.Body, + TreePath: c.Path, + DiffHunk: c.DiffHunk, + Position: c.Position, + CommitID: c.CommitID, + PosterID: user.ID(), + Reactions: r.getReactions(c.Reactions), + CreatedAt: c.CreatedAt, + }) + } + return res +} + // GetReviews returns pull requests review func (r *GithubExportedDataRestorer) GetReviews(context base.IssueContext) ([]*base.Review, error) { + var comments = make(map[string][]pullrequestReviewComment) + if err := r.readJSONFiles("pull_request_review_comments", func() interface{} { + return &[]pullrequestReviewComment{} + }, func(content interface{}) error { + cs := *content.(*[]pullrequestReviewComment) + for _, c := range cs { + comments[c.PullRequest] = append(comments[c.PullRequestReview], c) + } + return nil + }); err != nil { + return nil, err + } + var reviews = make([]*base.Review, 0, 10) if err := r.readJSONFiles("pull_request_reviews", func() interface{} { return &[]pullrequestReview{} @@ -749,6 +822,7 @@ func (r *GithubExportedDataRestorer) GetReviews(context base.IssueContext) ([]*b Content: review.Body, CreatedAt: review.CreatedAt, State: review.GetState(), + Comments: r.getReviewComments(comments[review.URL]), }) } return nil diff --git a/services/migrations/restore_github_test.go b/services/migrations/restore_github_test.go index dfabf115d2440..9bc6265ae6a91 100644 --- a/services/migrations/restore_github_test.go +++ b/services/migrations/restore_github_test.go @@ -10,6 +10,7 @@ import ( "code.gitea.io/gitea/modules/migration" base "code.gitea.io/gitea/modules/migration" + "github.com/stretchr/testify/assert" ) From 9baecbf0915affc4b2e0baa774690bb328207326 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 5 Jan 2022 14:20:30 +0800 Subject: [PATCH 03/54] add asset supports --- modules/migration/issue.go | 31 ++-- modules/migration/release.go | 6 +- services/migrations/gitea_downloader.go | 2 +- services/migrations/github.go | 2 +- services/migrations/gitlab.go | 2 +- services/migrations/main_test.go | 4 +- services/migrations/restore_github.go | 157 ++++++++++++++++----- services/migrations/restore_github_test.go | 6 +- 8 files changed, 147 insertions(+), 63 deletions(-) diff --git a/modules/migration/issue.go b/modules/migration/issue.go index 78f648dd2db57..883e01abe78ce 100644 --- a/modules/migration/issue.go +++ b/modules/migration/issue.go @@ -9,21 +9,22 @@ import "time" // Issue is a standard issue information type Issue struct { - Number int64 `json:"number"` - PosterID int64 `yaml:"poster_id" json:"poster_id"` - PosterName string `yaml:"poster_name" json:"poster_name"` - PosterEmail string `yaml:"poster_email" json:"poster_email"` - Title string `json:"title"` - Content string `json:"content"` - Ref string `json:"ref"` - Milestone string `json:"milestone"` - State string `json:"state"` // closed, open - IsLocked bool `yaml:"is_locked" json:"is_locked"` - Created time.Time `json:"created"` - Updated time.Time `json:"updated"` - Closed *time.Time `json:"closed"` - Labels []*Label `json:"labels"` - Reactions []*Reaction `json:"reactions"` + Number int64 `json:"number"` + PosterID int64 `yaml:"poster_id" json:"poster_id"` + PosterName string `yaml:"poster_name" json:"poster_name"` + PosterEmail string `yaml:"poster_email" json:"poster_email"` + Title string `json:"title"` + Content string `json:"content"` + Ref string `json:"ref"` + Milestone string `json:"milestone"` + State string `json:"state"` // closed, open + IsLocked bool `yaml:"is_locked" json:"is_locked"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` + Closed *time.Time `json:"closed"` + Labels []*Label `json:"labels"` + Reactions []*Reaction `json:"reactions"` + Assets []*Asset Assignees []string `json:"assignees"` ForeignIndex int64 `json:"foreign_id"` Context DownloaderContext `yaml:"-"` diff --git a/modules/migration/release.go b/modules/migration/release.go index cbdf01a3ed1d6..030e33af130f4 100644 --- a/modules/migration/release.go +++ b/modules/migration/release.go @@ -9,8 +9,8 @@ import ( "time" ) -// ReleaseAsset represents a release asset -type ReleaseAsset struct { +// Asset represents an asset for issue, comment, release +type Asset struct { ID int64 Name string ContentType *string `yaml:"content_type"` @@ -34,7 +34,7 @@ type Release struct { PublisherID int64 `yaml:"publisher_id"` PublisherName string `yaml:"publisher_name"` PublisherEmail string `yaml:"publisher_email"` - Assets []*ReleaseAsset + Assets []*Asset Created time.Time Published time.Time } diff --git a/services/migrations/gitea_downloader.go b/services/migrations/gitea_downloader.go index 3c02e112ca73e..bb2c4ab2401fd 100644 --- a/services/migrations/gitea_downloader.go +++ b/services/migrations/gitea_downloader.go @@ -271,7 +271,7 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele for _, asset := range rel.Attachments { size := int(asset.Size) dlCount := int(asset.DownloadCount) - r.Assets = append(r.Assets, &base.ReleaseAsset{ + r.Assets = append(r.Assets, &base.Asset{ ID: asset.ID, Name: asset.Name, Size: &size, diff --git a/services/migrations/github.go b/services/migrations/github.go index faf0cf0794179..c796101415246 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -312,7 +312,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) for _, asset := range rel.Assets { assetID := *asset.ID // Don't optimize this, for closure we need a local variable - r.Assets = append(r.Assets, &base.ReleaseAsset{ + r.Assets = append(r.Assets, &base.Asset{ ID: asset.GetID(), Name: asset.GetName(), ContentType: asset.ContentType, diff --git a/services/migrations/gitlab.go b/services/migrations/gitlab.go index 549e3cb659c9a..fcbd30aad4d4e 100644 --- a/services/migrations/gitlab.go +++ b/services/migrations/gitlab.go @@ -295,7 +295,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea httpClient := NewMigrationHTTPClient() for k, asset := range rel.Assets.Links { - r.Assets = append(r.Assets, &base.ReleaseAsset{ + r.Assets = append(r.Assets, &base.Asset{ ID: int64(asset.ID), Name: asset.Name, ContentType: &rel.Assets.Sources[k].Format, diff --git a/services/migrations/main_test.go b/services/migrations/main_test.go index ad9bc9c731eb7..9e4543bb6cf1a 100644 --- a/services/migrations/main_test.go +++ b/services/migrations/main_test.go @@ -171,7 +171,7 @@ func assertReactionsEqual(t *testing.T, expected, actual []*base.Reaction) { } } -func assertReleaseAssetEqual(t *testing.T, expected, actual *base.ReleaseAsset) { +func assertReleaseAssetEqual(t *testing.T, expected, actual *base.Asset) { assert.Equal(t, expected.ID, actual.ID) assert.Equal(t, expected.Name, actual.Name) assert.Equal(t, expected.ContentType, actual.ContentType) @@ -182,7 +182,7 @@ func assertReleaseAssetEqual(t *testing.T, expected, actual *base.ReleaseAsset) assert.Equal(t, expected.DownloadURL, actual.DownloadURL) } -func assertReleaseAssetsEqual(t *testing.T, expected, actual []*base.ReleaseAsset) { +func assertReleaseAssetsEqual(t *testing.T, expected, actual []*base.Asset) { if assert.Len(t, actual, len(expected)) { for i := range expected { assertReleaseAssetEqual(t, expected[i], actual[i]) diff --git a/services/migrations/restore_github.go b/services/migrations/restore_github.go index 3652ae1f6b881..84e423d38a24d 100644 --- a/services/migrations/restore_github.go +++ b/services/migrations/restore_github.go @@ -45,7 +45,7 @@ import ( "created_at": "2011-06-19T11:25:35Z" }, */ -type GithubUser struct { +type githubUser struct { URL string AvatarURL string Login string @@ -62,14 +62,18 @@ type GithubUser struct { CreatedAt time.Time } -func (g *GithubUser) ID() int64 { - u, _ := url.Parse(g.AvatarURL) +func getID(s string) int64 { + u, _ := url.Parse(s) fields := strings.Split(u.Path, "/") i, _ := strconv.ParseInt(fields[len(fields)-1], 10, 64) return i } -func (g *GithubUser) Email() string { +func (g *githubUser) ID() int64 { + return getID(g.AvatarURL) +} + +func (g *githubUser) Email() string { if len(g.Emails) < 1 { return "" } @@ -85,6 +89,7 @@ func (g *GithubUser) Email() string { /*{ "type": "attachment", "url": "https://user-images.githubusercontent.com/1595118/2923824-63a167ce-d721-11e3-91b6-74b83dc345bb.png", + "issue": "https://github.com/go-xorm/xorm/issues/205", "issue_comment": "https://github.com/go-xorm/xorm/issues/115#issuecomment-42628488", "user": "https://github.com/mintzhao", "asset_name": "QQ20140509-1.2x.png", @@ -94,6 +99,7 @@ func (g *GithubUser) Email() string { }, */ type githubAttachment struct { + Issue string IssueComment string User string AssetName string @@ -102,6 +108,38 @@ type githubAttachment struct { CreatedAt time.Time } +func (g *githubAttachment) GetUser() int64 { + return getID(g.User) +} + +func (g *githubAttachment) IsIssue() bool { + return len(g.Issue) > 0 +} + +func (g *githubAttachment) IssueID() int64 { + if g.IsIssue() { + return getID(g.Issue) + } + return getID(g.IssueComment) +} + +func (r *GithubExportedDataRestorer) convertAttachments(ls []*githubAttachment) []*base.Asset { + var res = make([]*base.Asset, 0, len(ls)) + for _, l := range ls { + var assetURL = "file://" + strings.TrimPrefix(l.AssetURL, "tarball://root") + res = append(res, &base.Asset{ + Name: l.AssetName, + ContentType: &l.AssetContentType, + //Size : l.Size, + //DownloadCount *int `yaml:"download_count"` + Created: l.CreatedAt, + //Updated time.Time + DownloadURL: &assetURL, + }) + } + return res +} + /* { "user": "https://github.com/mrsdizzie", @@ -132,7 +170,10 @@ type GithubExportedDataRestorer struct { repoOwner string repoName string labels []*base.Label - users map[string]*GithubUser + users map[string]*githubUser + issueAttachments map[string][]*githubAttachment + commentAttachments map[string][]*githubAttachment + attachmentLoaded bool } func decompressFile(targzFile, targetDir string) error { @@ -196,7 +237,7 @@ func NewGithubExportedDataRestorer(ctx context.Context, githubDataFilePath, owne tmpDir: tmpDir, repoOwner: owner, repoName: repoName, - users: make(map[string]*GithubUser), + users: make(map[string]*githubUser), } if err := restorer.getUsers(); err != nil { return nil, err @@ -240,7 +281,7 @@ func (r *GithubExportedDataRestorer) GetRepoInfo() (*base.Repository, error) { if err := json.Unmarshal(bs, &githubRepositories); err != nil { return nil, err } - if len(githubRepositories) <= 0 { + if len(githubRepositories) == 0 { return nil, errors.New("no repository found in the json file: repositories_000001.json") } else if len(githubRepositories) > 1 { return nil, errors.New("only one repository is supported") @@ -308,9 +349,9 @@ func (r *GithubExportedDataRestorer) readJSONFiles(filePrefix string, makeF func func (r *GithubExportedDataRestorer) getUsers() error { return r.readJSONFiles("users", func() interface{} { - return &[]GithubUser{} + return &[]githubUser{} }, func(content interface{}) error { - mss := content.(*[]GithubUser) + mss := content.(*[]githubUser) for _, ms := range *mss { r.users[ms.URL] = &ms } @@ -318,6 +359,27 @@ func (r *GithubExportedDataRestorer) getUsers() error { }) } +func (r *GithubExportedDataRestorer) getAttachments() error { + if r.attachmentLoaded { + return nil + } + + return r.readJSONFiles("attachments", func() interface{} { + r.attachmentLoaded = true + return &[]githubAttachment{} + }, func(content interface{}) error { + mss := content.(*[]githubAttachment) + for _, ms := range *mss { + if ms.IsIssue() { + r.issueAttachments[ms.Issue] = append(r.issueAttachments[ms.Issue], &ms) + } else { + r.commentAttachments[ms.IssueComment] = append(r.commentAttachments[ms.IssueComment], &ms) + } + } + return nil + }) +} + // GetMilestones returns milestones func (r *GithubExportedDataRestorer) GetMilestones() ([]*base.Milestone, error) { type milestone struct { @@ -354,34 +416,48 @@ func (r *GithubExportedDataRestorer) GetMilestones() ([]*base.Milestone, error) return milestones, nil } +/* +{ + "type": "release", + "url": "https://github.com/go-xorm/xorm/releases/tag/v0.3.1", + "repository": "https://github.com/go-xorm/xorm", + "user": "https://github.com/lunny", + "name": "", + "tag_name": "v0.3.1", + "body": "- Features:\n - Support MSSQL DB via ODBC driver ([github.com/lunny/godbc](https://github.com/lunny/godbc));\n - Composite Key, using multiple pk xorm tag \n - Added Row() API as alternative to Iterate() API for traversing result set, provide similar usages to sql.Rows type\n - ORM struct allowed declaration of pointer builtin type as members to allow null DB fields \n - Before and After Event processors\n- Improvements:\n - Allowed int/int32/int64/uint/uint32/uint64/string as Primary Key type\n - Performance improvement for Get()/Find()/Iterate()\n", + "state": "published", + "pending_tag": "v0.3.1", + "prerelease": false, + "target_commitish": "master", + "release_assets": [ + + ], + "published_at": "2014-01-02T09:51:34Z", + "created_at": "2014-01-02T09:48:57Z" + }, +*/ +type githubRelease struct { + User string + Name string + TagName string + Body string + State string + Prerelease bool + ReleaseAssets []*githubAttachment + TargetCommitish string + CreatedAt time.Time + PublishedAt time.Time +} + // GetReleases returns releases func (r *GithubExportedDataRestorer) GetReleases() ([]*base.Release, error) { - type release struct { - Name string - TagName string - Body string - State string - Prerelease bool - ReleaseAssets []struct { - } - TargetCommitish string - CreatedAt time.Time - PublishedAt time.Time - } - var releases = make([]*base.Release, 0, 30) if err := r.readJSONFiles("releases", func() interface{} { - return &[]release{} + return &[]githubRelease{} }, func(content interface{}) error { - rss := content.(*[]release) + rss := content.(*[]githubRelease) for _, rel := range *rss { - // TODO - /*for _, asset := range rel.ReleaseAssets { - if asset.DownloadURL != nil { - *asset.DownloadURL = "file://" + filepath.Join(r.baseDir, *asset.DownloadURL) - } - }*/ - + user := r.users[rel.User] releases = append(releases, &base.Release{ TagName: rel.TagName, TargetCommitish: rel.TargetCommitish, @@ -389,12 +465,12 @@ func (r *GithubExportedDataRestorer) GetReleases() ([]*base.Release, error) { Body: rel.Body, Draft: rel.State == "draft", Prerelease: rel.Prerelease, - //PublisherID : rel. - //PublisherName string `yaml:"publisher_name"` - //PublisherEmail string `yaml:"publisher_email"` - Assets: []*base.ReleaseAsset{}, - Created: rel.CreatedAt, - Published: rel.PublishedAt, + PublisherID: user.ID(), + PublisherName: user.Login, + PublisherEmail: user.Email(), + Assets: r.convertAttachments(rel.ReleaseAssets), + Created: rel.CreatedAt, + Published: rel.PublishedAt, }) } return nil @@ -481,6 +557,10 @@ func (r *GithubExportedDataRestorer) getReactions(ls []githubReaction) []*base.R // GetIssues returns issues according start and limit func (r *GithubExportedDataRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { + if err := r.getAttachments(); err != nil { + return nil, false, err + } + var issues = make([]*base.Issue, 0, 50) if err := r.readJSONFiles("issues", func() interface{} { return &[]githubIssue{} @@ -503,6 +583,7 @@ func (r *GithubExportedDataRestorer) GetIssues(page, perPage int) ([]*base.Issue PosterEmail: user.Email(), Labels: r.getLabels(issue.Lables), Reactions: r.getReactions(issue.Reactions), + Assets: r.convertAttachments(r.issueAttachments[issue.URL]), Milestone: issue.Milestone, Assignees: issue.Assignees, //Ref: issue. @@ -800,7 +881,7 @@ func (r *GithubExportedDataRestorer) GetReviews(context base.IssueContext) ([]*b }, func(content interface{}) error { cs := *content.(*[]pullrequestReviewComment) for _, c := range cs { - comments[c.PullRequest] = append(comments[c.PullRequestReview], c) + comments[c.PullRequestReview] = append(comments[c.PullRequestReview], c) } return nil }); err != nil { diff --git a/services/migrations/restore_github_test.go b/services/migrations/restore_github_test.go index 9bc6265ae6a91..f385e8cc75579 100644 --- a/services/migrations/restore_github_test.go +++ b/services/migrations/restore_github_test.go @@ -9,7 +9,6 @@ import ( "testing" "code.gitea.io/gitea/modules/migration" - base "code.gitea.io/gitea/modules/migration" "github.com/stretchr/testify/assert" ) @@ -30,6 +29,7 @@ func TestParseGithubExportedData(t *testing.T) { releases, err := restorer.GetReleases() assert.NoError(t, err) assert.EqualValues(t, 1, len(releases)) + assert.EqualValues(t, 1, len(releases[0].Assets)) labels, err := restorer.GetLabels() assert.NoError(t, err) @@ -39,6 +39,8 @@ func TestParseGithubExportedData(t *testing.T) { assert.NoError(t, err) assert.True(t, isEnd) assert.EqualValues(t, 2, len(issues)) + assert.EqualValues(t, 2, issues[0].Assets) + assert.EqualValues(t, 1, issues[1].Assets) comments, isEnd, err := restorer.GetComments(migration.GetCommentOptions{}) assert.NoError(t, err) @@ -50,7 +52,7 @@ func TestParseGithubExportedData(t *testing.T) { assert.True(t, isEnd) assert.EqualValues(t, 2, len(prs)) - reviewers, err := restorer.GetReviews(base.BasicIssueContext(0)) + reviewers, err := restorer.GetReviews(migration.BasicIssueContext(0)) assert.NoError(t, err) assert.EqualValues(t, 6, len(reviewers)) } From ca660956d33645582f19f287b4f61d6b8023bdd3 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 6 Jan 2022 17:34:56 +0800 Subject: [PATCH 04/54] Add more tests --- services/migrations/restore_github_test.go | 23 +++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/services/migrations/restore_github_test.go b/services/migrations/restore_github_test.go index f385e8cc75579..06643980cf6b0 100644 --- a/services/migrations/restore_github_test.go +++ b/services/migrations/restore_github_test.go @@ -29,7 +29,7 @@ func TestParseGithubExportedData(t *testing.T) { releases, err := restorer.GetReleases() assert.NoError(t, err) assert.EqualValues(t, 1, len(releases)) - assert.EqualValues(t, 1, len(releases[0].Assets)) + assert.EqualValues(t, 0, len(releases[0].Assets)) labels, err := restorer.GetLabels() assert.NoError(t, err) @@ -39,8 +39,25 @@ func TestParseGithubExportedData(t *testing.T) { assert.NoError(t, err) assert.True(t, isEnd) assert.EqualValues(t, 2, len(issues)) - assert.EqualValues(t, 2, issues[0].Assets) - assert.EqualValues(t, 1, issues[1].Assets) + assert.EqualValues(t, 1, issues[0].Context.ForeignID()) + assert.EqualValues(t, "Please add an animated gif icon to the merge button", issues[0].Title) + assert.EqualValues(t, "I just want the merge button to hurt my eyes a little. 😝", issues[0].Content) + assert.EqualValues(t, "mrsdizzie", issues[0].PosterName) + assert.EqualValues(t, 1, len(issues[0].Labels)) + assert.EqualValues(t, "1.1.0", len(issues[0].Milestone)) + assert.EqualValues(t, 0, len(issues[0].Assets)) + assert.EqualValues(t, 1, len(issues[0].Reactions)) + assert.True(t, issues[0].State == "closed") + + assert.EqualValues(t, 2, issues[1].Context.ForeignID()) + assert.EqualValues(t, "Please add an animated gif icon to the merge button", issues[1].Title) + assert.EqualValues(t, "I just want the merge button to hurt my eyes a little. 😝", issues[1].Content) + assert.EqualValues(t, "guillep2k", issues[1].PosterName) + assert.EqualValues(t, 2, len(issues[1].Labels)) + assert.EqualValues(t, "1.0.0", len(issues[1].Milestone)) + assert.EqualValues(t, 0, len(issues[1].Assets)) + assert.EqualValues(t, 6, len(issues[1].Reactions)) + assert.True(t, issues[1].State == "closed") comments, isEnd, err := restorer.GetComments(migration.GetCommentOptions{}) assert.NoError(t, err) From e0265233f0ba69f3fd5d1df0e8b0af510ac1cba2 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 7 Jan 2022 11:57:57 +0800 Subject: [PATCH 05/54] some improvements --- cmd/restore_repo.go | 2 +- services/migrations/restore_github.go | 8 ++++---- services/migrations/restore_github_test.go | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/restore_repo.go b/cmd/restore_repo.go index 3335d39fa7e53..a3753810d6884 100644 --- a/cmd/restore_repo.go +++ b/cmd/restore_repo.go @@ -25,7 +25,7 @@ var CmdRestoreRepository = cli.Command{ cli.StringFlag{ Name: "data_type, d", Value: "gitea", - Usage: "The data type will be imported, default is gitea, options are gitea, github", + Usage: "The data type that will be imported, default is gitea, options are gitea, github", }, cli.StringFlag{ Name: "repo_filepath, f", diff --git a/services/migrations/restore_github.go b/services/migrations/restore_github.go index 84e423d38a24d..5d4bed399c85f 100644 --- a/services/migrations/restore_github.go +++ b/services/migrations/restore_github.go @@ -8,7 +8,6 @@ import ( "archive/tar" "compress/gzip" "context" - "encoding/json" "errors" "fmt" "io" @@ -20,6 +19,7 @@ import ( "strings" "time" + "code.gitea.io/gitea/modules/json" base "code.gitea.io/gitea/modules/migration" ) @@ -108,7 +108,7 @@ type githubAttachment struct { CreatedAt time.Time } -func (g *githubAttachment) GetUser() int64 { +func (g *githubAttachment) GetUserID() int64 { return getID(g.User) } @@ -516,7 +516,7 @@ type githubIssue struct { Assignee string Assignees []string Milestone string - Lables []githubLabel + Labels []githubLabel Reactions []githubReaction ClosedAt *time.Time CreatedAt time.Time @@ -581,7 +581,7 @@ func (r *GithubExportedDataRestorer) GetIssues(page, perPage int) ([]*base.Issue PosterID: user.ID(), PosterName: user.Login, PosterEmail: user.Email(), - Labels: r.getLabels(issue.Lables), + Labels: r.getLabels(issue.Labels), Reactions: r.getReactions(issue.Reactions), Assets: r.convertAttachments(r.issueAttachments[issue.URL]), Milestone: issue.Milestone, diff --git a/services/migrations/restore_github_test.go b/services/migrations/restore_github_test.go index 06643980cf6b0..444601f208e5f 100644 --- a/services/migrations/restore_github_test.go +++ b/services/migrations/restore_github_test.go @@ -41,8 +41,8 @@ func TestParseGithubExportedData(t *testing.T) { assert.EqualValues(t, 2, len(issues)) assert.EqualValues(t, 1, issues[0].Context.ForeignID()) assert.EqualValues(t, "Please add an animated gif icon to the merge button", issues[0].Title) - assert.EqualValues(t, "I just want the merge button to hurt my eyes a little. 😝", issues[0].Content) - assert.EqualValues(t, "mrsdizzie", issues[0].PosterName) + assert.EqualValues(t, "I just want the merge button to hurt my eyes a little. 😝 ", issues[0].Content) + assert.EqualValues(t, "justusbunsi", issues[0].PosterName) assert.EqualValues(t, 1, len(issues[0].Labels)) assert.EqualValues(t, "1.1.0", len(issues[0].Milestone)) assert.EqualValues(t, 0, len(issues[0].Assets)) From 4fb0a78041081447ccdc26be9b7ef3a9e3906b4c Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 7 Jan 2022 13:15:19 +0800 Subject: [PATCH 06/54] fix dump --- services/migrations/dump.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/services/migrations/dump.go b/services/migrations/dump.go index baedf6cdc01d3..05e2aa60ef265 100644 --- a/services/migrations/dump.go +++ b/services/migrations/dump.go @@ -644,18 +644,13 @@ func RestoreFromGithubExportedData(ctx context.Context, baseDir, ownerName, repo return err } var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, repoName) - downloader, err := NewRepositoryRestorer(ctx, baseDir, ownerName, repoName) + downloader, err := NewGithubExportedDataRestorer(ctx, baseDir, ownerName, repoName) if err != nil { return err } - opts, err := downloader.getRepoOptions() - if err != nil { - return err - } - tp, _ := strconv.Atoi(opts["service_type"]) var migrateOpts = base.MigrateOptions{ - GitServiceType: structs.GitServiceType(tp), + GitServiceType: structs.GithubService, } updateOptionsUnits(&migrateOpts, units) @@ -665,5 +660,5 @@ func RestoreFromGithubExportedData(ctx context.Context, baseDir, ownerName, repo } return err } - return updateMigrationPosterIDByGitService(ctx, structs.GitServiceType(tp)) + return updateMigrationPosterIDByGitService(ctx, migrateOpts.GitServiceType) } From b41f53320c5a1bab03b60f7040fda6c28adedce4 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 7 Jan 2022 13:33:10 +0800 Subject: [PATCH 07/54] Add version check --- services/migrations/restore_github.go | 34 +++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/services/migrations/restore_github.go b/services/migrations/restore_github.go index 5d4bed399c85f..8b0014c3876c4 100644 --- a/services/migrations/restore_github.go +++ b/services/migrations/restore_github.go @@ -21,8 +21,18 @@ import ( "code.gitea.io/gitea/modules/json" base "code.gitea.io/gitea/modules/migration" + + "github.com/hashicorp/go-version" ) +/* + {"version":"1.0.1","github_sha":"8de0984858fd99a8dcd2d756cf0f128b9161e3b5"} +*/ +type githubSchema struct { + Version string + GithubSha string +} + /* { "type": "user", @@ -239,6 +249,9 @@ func NewGithubExportedDataRestorer(ctx context.Context, githubDataFilePath, owne repoName: repoName, users: make(map[string]*githubUser), } + if err := restorer.readSchema(); err != nil { + return nil, err + } if err := restorer.getUsers(); err != nil { return nil, err } @@ -251,6 +264,27 @@ func (r *GithubExportedDataRestorer) SetContext(ctx context.Context) { r.ctx = ctx } +func (r *GithubExportedDataRestorer) readSchema() error { + bs, err := os.ReadFile(filepath.Join(r.tmpDir, "schema.json")) + if err != nil { + return err + } + var schema githubSchema + if err := json.Unmarshal(bs, &schema); err != nil { + return err + } + + v, err := version.NewSemver(schema.Version) + if err != nil { + return fmt.Errorf("archive version %s is not semver", schema.Version) + } + s := v.Segments() + if s[0] != 1 { + return fmt.Errorf("archive version is %s, but expected 1.x.x", schema.Version) + } + return nil +} + // GetRepoInfo returns a repository information func (r *GithubExportedDataRestorer) GetRepoInfo() (*base.Repository, error) { type Label struct { From 7d318292eb0ba6adaa57256e3686ea10f3caf4d1 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 8 Jan 2022 21:14:29 +0800 Subject: [PATCH 08/54] more tests --- modules/migration/comment.go | 40 +++- services/migrations/restore_github.go | 247 +++++++++++++++++---- services/migrations/restore_github_test.go | 44 +++- 3 files changed, 270 insertions(+), 61 deletions(-) diff --git a/modules/migration/comment.go b/modules/migration/comment.go index 0447689b74cd6..48ced0e221172 100644 --- a/modules/migration/comment.go +++ b/modules/migration/comment.go @@ -14,9 +14,47 @@ type Commentable interface { GetContext() DownloaderContext } +/* +"comment", + "reopen", + "close", + "issue_ref", + "commit_ref", + "comment_ref", + "pull_ref", + "label", + "milestone", + "assignees", + "change_title", + "delete_branch", + "start_tracking", + "stop_tracking", + "add_time_manual", + "cancel_tracking", + "added_deadline", + "modified_deadline", + "removed_deadline", + "add_dependency", + "remove_dependency", + "code", + "review", + "lock", + "unlock", + "change_target_branch", + "delete_time_manual", + "review_request", + "merge_pull", + "pull_push", + "project", + "project_board", + "dismiss_review", + "change_issue_ref", +*/ + // Comment is a standard comment information type Comment struct { - IssueIndex int64 `yaml:"issue_index"` + Type string // empty means comment + IssueIndex int64 `yaml:"issue_index"` Index int64 PosterID int64 `yaml:"poster_id"` PosterName string `yaml:"poster_name"` diff --git a/services/migrations/restore_github.go b/services/migrations/restore_github.go index 8b0014c3876c4..e5e5acf033104 100644 --- a/services/migrations/restore_github.go +++ b/services/migrations/restore_github.go @@ -20,6 +20,7 @@ import ( "time" "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" base "code.gitea.io/gitea/modules/migration" "github.com/hashicorp/go-version" @@ -133,7 +134,7 @@ func (g *githubAttachment) IssueID() int64 { return getID(g.IssueComment) } -func (r *GithubExportedDataRestorer) convertAttachments(ls []*githubAttachment) []*base.Asset { +func (r *GithubExportedDataRestorer) convertAttachments(ls []githubAttachment) []*base.Asset { var res = make([]*base.Asset, 0, len(ls)) for _, l := range ls { var assetURL = "file://" + strings.TrimPrefix(l.AssetURL, "tarball://root") @@ -168,7 +169,12 @@ type githubLabel string func (l githubLabel) GetName() string { fields := strings.Split(string(l), "/labels/") - return fields[len(fields)-1] + s, err := url.PathUnescape(fields[len(fields)-1]) + if err != nil { + log.Error("url.PathUnescape %s failed: %v", fields[len(fields)-1], err) + return fields[len(fields)-1] + } + return s } // GithubExportedDataRestorer implements an Downloader from the exported data of Github @@ -180,9 +186,10 @@ type GithubExportedDataRestorer struct { repoOwner string repoName string labels []*base.Label - users map[string]*githubUser - issueAttachments map[string][]*githubAttachment - commentAttachments map[string][]*githubAttachment + users map[string]githubUser + issueAttachments map[string][]githubAttachment + commentAttachments map[string][]githubAttachment + milestones map[string]githubMilestone attachmentLoaded bool } @@ -247,7 +254,8 @@ func NewGithubExportedDataRestorer(ctx context.Context, githubDataFilePath, owne tmpDir: tmpDir, repoOwner: owner, repoName: repoName, - users: make(map[string]*githubUser), + users: make(map[string]githubUser), + milestones: make(map[string]githubMilestone), } if err := restorer.readSchema(); err != nil { return nil, err @@ -387,7 +395,7 @@ func (r *GithubExportedDataRestorer) getUsers() error { }, func(content interface{}) error { mss := content.(*[]githubUser) for _, ms := range *mss { - r.users[ms.URL] = &ms + r.users[ms.URL] = ms } return nil }) @@ -405,33 +413,49 @@ func (r *GithubExportedDataRestorer) getAttachments() error { mss := content.(*[]githubAttachment) for _, ms := range *mss { if ms.IsIssue() { - r.issueAttachments[ms.Issue] = append(r.issueAttachments[ms.Issue], &ms) + r.issueAttachments[ms.Issue] = append(r.issueAttachments[ms.Issue], ms) } else { - r.commentAttachments[ms.IssueComment] = append(r.commentAttachments[ms.IssueComment], &ms) + r.commentAttachments[ms.IssueComment] = append(r.commentAttachments[ms.IssueComment], ms) } } return nil }) } +/* { + "type": "milestone", + "url": "https://github.com/go-gitea/test_repo/milestones/1", + "repository": "https://github.com/go-gitea/test_repo", + "user": "https://github.com/mrsdizzie", + "title": "1.0.0", + "description": "Milestone 1.0.0", + "state": "closed", + "due_on": "2019-11-11T00:00:00Z", + "created_at": "2019-11-12T19:37:08Z", + "updated_at": "2019-11-12T21:56:17Z", + "closed_at": "2019-11-12T19:45:49Z" + },*/ +type githubMilestone struct { + URL string + User string + Title string + State string + Description string + DueOn time.Time + CreatedAt time.Time + UpdatedAt time.Time + ClosedAt time.Time +} + // GetMilestones returns milestones func (r *GithubExportedDataRestorer) GetMilestones() ([]*base.Milestone, error) { - type milestone struct { - Title string - State string - Description string - DueOn time.Time - CreatedAt time.Time - UpdatedAt time.Time - ClosedAt time.Time - } - var milestones = make([]*base.Milestone, 0, 10) if err := r.readJSONFiles("milestones", func() interface{} { - return &[]milestone{} + return &[]githubMilestone{} }, func(content interface{}) error { - mss := content.(*[]milestone) + mss := content.(*[]githubMilestone) for _, milestone := range *mss { + r.milestones[milestone.URL] = milestone milestones = append(milestones, &base.Milestone{ Title: milestone.Title, Description: milestone.Description, @@ -477,7 +501,7 @@ type githubRelease struct { Body string State string Prerelease bool - ReleaseAssets []*githubAttachment + ReleaseAssets []githubAttachment TargetCommitish string CreatedAt time.Time PublishedAt time.Time @@ -533,8 +557,9 @@ func (r *GithubExportedDataRestorer) GetLabels() ([]*base.Label, error) { ], "milestone": null, "labels": [ - - ], + "https://github.com/go-gitea/test_repo/labels/bug", + "https://github.com/go-gitea/test_repo/labels/good%20first%20issue" + ], "reactions": [ ], @@ -552,9 +577,9 @@ type githubIssue struct { Milestone string Labels []githubLabel Reactions []githubReaction - ClosedAt *time.Time - CreatedAt time.Time - UpdatedAt time.Time + ClosedAt *time.Time `json:"closed_at"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` } func (g *githubIssue) Index() int64 { @@ -567,7 +592,7 @@ func (r *GithubExportedDataRestorer) getLabels(ls []githubLabel) []*base.Label { var res = make([]*base.Label, 0, len(ls)) for _, l := range ls { for _, ll := range r.labels { - if string(l) == ll.Name { + if l.GetName() == ll.Name { res = append(res, ll) break } @@ -602,6 +627,10 @@ func (r *GithubExportedDataRestorer) GetIssues(page, perPage int) ([]*base.Issue rss := content.(*[]githubIssue) for _, issue := range *rss { user := r.users[issue.User] + var milestone string + if issue.Milestone != "" { + milestone = r.milestones[issue.Milestone].Title + } var state = "open" if issue.ClosedAt != nil { @@ -618,7 +647,7 @@ func (r *GithubExportedDataRestorer) GetIssues(page, perPage int) ([]*base.Issue Labels: r.getLabels(issue.Labels), Reactions: r.getReactions(issue.Reactions), Assets: r.convertAttachments(r.issueAttachments[issue.URL]), - Milestone: issue.Milestone, + Milestone: milestone, Assignees: issue.Assignees, //Ref: issue. State: state, @@ -636,43 +665,165 @@ func (r *GithubExportedDataRestorer) GetIssues(page, perPage int) ([]*base.Issue return issues, true, nil } -// GetComments returns comments according issueNumber -func (r *GithubExportedDataRestorer) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { - type Comment struct { - Issue string - PullRequest string - User string - Body string - Reactions []githubReaction - CreatedAt time.Time +type githubComment struct { + Issue string + PullRequest string `json:"pull_request"` + User string + Body string + Reactions []githubReaction + CreatedAt time.Time `json:"created_at"` +} + +func getIssueIndex(issue, pullRequest string) int64 { + var c string + if issue != "" { + c = issue + } else { + c = pullRequest } + fields := strings.Split(c, "/") + idx, _ := strconv.ParseInt(fields[len(fields)-1], 10, 64) + return idx +} + +func (g *githubComment) GetIssueIndex() int64 { + return getIssueIndex(g.Issue, g.PullRequest) +} + +/* +{ + "type": "issue_event", + "url": "https://github.com/go-xorm/xorm/issues/1#event-55275262", + "issue": "https://github.com/go-xorm/xorm/issues/1", + "actor": "https://github.com/lunny", + "event": "assigned", + "created_at": "2013-07-04T14:27:53Z" + }, + { + "type": "issue_event", + "url": "https://github.com/go-xorm/xorm/pull/2#event-56230828", + "pull_request": "https://github.com/go-xorm/xorm/pull/2", + "actor": "https://github.com/lunny", + "event": "referenced", + "commit_id": "1be80583b0fa18e7b478fa12e129c95e9a06a62f", + "commit_repository": "https://github.com/go-xorm/xorm", + "created_at": "2013-07-12T02:10:52Z" + + "label": "https://github.com/go-xorm/xorm/labels/wip", + "label_name": "New Feature", + "label_color": "5319e7", + "label_text_color": "fff", + "milestone_title": "v0.4", + "title_was": "自动读写分离", + "title_is": "Automatical Read/Write seperatelly.", + },*/ +type githubIssueEvent struct { + URL string + Issue string + PullRequest string `json:"pull_request"` + Actor string + Event string + CommitID string `json:"commit_id"` + CommitRepoistory string `json:"commit_repository"` + + CreatedAt time.Time `json:"created_at"` +} + +func (g *githubIssueEvent) CommentStr() string { + switch g.Event { + case "closed": + return "close" + case "referenced": + return "commit_ref" + case "merged": + return "merge_pull" + case "mentioned": + return "" // ignore + case "subscribed": + return "" // ignore + case "head_ref_deleted": + return "delete_branch" + case "milestoned": + return "milestone" + case "labeled": + return "label" + case "renamed": + return "change_title" + case "reopened": + return "reopen" + case "unlabeled": + return "label" + case "assigned": + return "assignees" + default: + return "comment" + } +} + +func (g *githubIssueEvent) GetIssueIndex() int64 { + return getIssueIndex(g.Issue, g.PullRequest) +} + +func (r *GithubExportedDataRestorer) getIssueEvents() ([]*base.Comment, error) { + var comments []*base.Comment + if err := r.readJSONFiles("issue_events", func() interface{} { + return &[]githubIssueEvent{} + }, func(content interface{}) error { + rss := content.(*[]githubIssueEvent) + for _, c := range *rss { + var u = r.users[c.Actor] + //var tp string + + comments = append(comments, &base.Comment{ + Type: c.CommentStr(), + IssueIndex: c.GetIssueIndex(), + PosterID: u.ID(), + PosterName: u.Login, + PosterEmail: u.Email(), + Created: c.CreatedAt, + Updated: c.CreatedAt, // FIXME: + }) + } + return nil + }); err != nil { + return nil, err + } + return comments, nil +} + +// GetComments returns comments according issueNumber +func (r *GithubExportedDataRestorer) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { var comments = make([]*base.Comment, 0, 10) if err := r.readJSONFiles("issue_comments", func() interface{} { - return &[]Comment{} + return &[]githubComment{} }, func(content interface{}) error { - rss := content.(*[]Comment) + rss := content.(*[]githubComment) for _, c := range *rss { - fields := strings.Split(c.Issue, "/") - idx, _ := strconv.ParseInt(fields[len(fields)-1], 10, 64) u := r.users[c.User] comments = append(comments, &base.Comment{ - IssueIndex: idx, + IssueIndex: c.GetIssueIndex(), PosterID: u.ID(), PosterName: u.Login, - PosterEmail: "", + PosterEmail: u.Email(), Created: c.CreatedAt, - //Updated:, - Content: c.Body, - Reactions: r.getReactions(c.Reactions), + Updated: c.CreatedAt, // FIXME: + Content: c.Body, + Reactions: r.getReactions(c.Reactions), }) } return nil }); err != nil { return nil, false, err } - return comments, true, nil + + comments2, err := r.getIssueEvents() + if err != nil { + return nil, false, err + } + + return append(comments, comments2...), true, nil } /* diff --git a/services/migrations/restore_github_test.go b/services/migrations/restore_github_test.go index 444601f208e5f..52529fe97cb1d 100644 --- a/services/migrations/restore_github_test.go +++ b/services/migrations/restore_github_test.go @@ -6,6 +6,7 @@ package migrations import ( "context" + "fmt" "testing" "code.gitea.io/gitea/modules/migration" @@ -18,23 +19,28 @@ func TestParseGithubExportedData(t *testing.T) { assert.NoError(t, err) assert.EqualValues(t, 49, len(restorer.users)) + // repo info repo, err := restorer.GetRepoInfo() assert.NoError(t, err) assert.EqualValues(t, "test_repo", repo.Name) + // milestones milestones, err := restorer.GetMilestones() assert.NoError(t, err) assert.EqualValues(t, 2, len(milestones)) + // releases releases, err := restorer.GetReleases() assert.NoError(t, err) assert.EqualValues(t, 1, len(releases)) assert.EqualValues(t, 0, len(releases[0].Assets)) + // labels labels, err := restorer.GetLabels() assert.NoError(t, err) assert.EqualValues(t, 9, len(labels)) + // issues issues, isEnd, err := restorer.GetIssues(1, 100) assert.NoError(t, err) assert.True(t, isEnd) @@ -42,34 +48,48 @@ func TestParseGithubExportedData(t *testing.T) { assert.EqualValues(t, 1, issues[0].Context.ForeignID()) assert.EqualValues(t, "Please add an animated gif icon to the merge button", issues[0].Title) assert.EqualValues(t, "I just want the merge button to hurt my eyes a little. 😝 ", issues[0].Content) - assert.EqualValues(t, "justusbunsi", issues[0].PosterName) - assert.EqualValues(t, 1, len(issues[0].Labels)) - assert.EqualValues(t, "1.1.0", len(issues[0].Milestone)) + assert.EqualValues(t, "guillep2k", issues[0].PosterName) + assert.EqualValues(t, 2, len(issues[0].Labels), fmt.Sprintf("%#v", issues[0].Labels)) + assert.EqualValues(t, "1.0.0", issues[0].Milestone) assert.EqualValues(t, 0, len(issues[0].Assets)) assert.EqualValues(t, 1, len(issues[0].Reactions)) - assert.True(t, issues[0].State == "closed") + assert.EqualValues(t, "closed", issues[0].State) + assert.NotNil(t, issues[0].Closed) + assert.NotZero(t, issues[0].Updated) + assert.NotZero(t, issues[0].Created) assert.EqualValues(t, 2, issues[1].Context.ForeignID()) - assert.EqualValues(t, "Please add an animated gif icon to the merge button", issues[1].Title) - assert.EqualValues(t, "I just want the merge button to hurt my eyes a little. 😝", issues[1].Content) - assert.EqualValues(t, "guillep2k", issues[1].PosterName) - assert.EqualValues(t, 2, len(issues[1].Labels)) - assert.EqualValues(t, "1.0.0", len(issues[1].Milestone)) + assert.EqualValues(t, "Test issue", issues[1].Title) + assert.EqualValues(t, "This is test issue 2, do not touch!", issues[1].Content) + assert.EqualValues(t, "mrsdizzie", issues[1].PosterName) + assert.EqualValues(t, 1, len(issues[1].Labels)) + assert.EqualValues(t, "1.1.0", issues[1].Milestone) assert.EqualValues(t, 0, len(issues[1].Assets)) assert.EqualValues(t, 6, len(issues[1].Reactions)) - assert.True(t, issues[1].State == "closed") + assert.EqualValues(t, "closed", issues[1].State) + assert.NotNil(t, issues[1].Closed) + assert.NotZero(t, issues[1].Updated) + assert.NotZero(t, issues[1].Created) + // comments comments, isEnd, err := restorer.GetComments(migration.GetCommentOptions{}) assert.NoError(t, err) assert.True(t, isEnd) assert.EqualValues(t, 2, len(comments)) + assert.EqualValues(t, 2, comments[0].IssueIndex) + assert.NotZero(t, comments[0].Created) + assert.EqualValues(t, 2, comments[1].IssueIndex) + assert.NotZero(t, comments[1].Created) + + // pull requests prs, isEnd, err := restorer.GetPullRequests(1, 100) assert.NoError(t, err) assert.True(t, isEnd) assert.EqualValues(t, 2, len(prs)) - reviewers, err := restorer.GetReviews(migration.BasicIssueContext(0)) + // reviews + reviews, err := restorer.GetReviews(migration.BasicIssueContext(0)) assert.NoError(t, err) - assert.EqualValues(t, 6, len(reviewers)) + assert.EqualValues(t, 6, len(reviews)) } From 14a96775b48f5558b7824057ec4e719b820ce8e2 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 9 Jan 2022 00:26:57 +0800 Subject: [PATCH 09/54] Fix some bugs --- models/issue_comment.go | 10 +++ services/migrations/gitea_uploader.go | 6 +- services/migrations/restore_github.go | 98 +++++++++++----------- services/migrations/restore_github_test.go | 3 +- 4 files changed, 68 insertions(+), 49 deletions(-) diff --git a/models/issue_comment.go b/models/issue_comment.go index 90c95afa4e4fb..4c717c938ef4f 100644 --- a/models/issue_comment.go +++ b/models/issue_comment.go @@ -159,6 +159,16 @@ func (t CommentType) String() string { return commentStrings[t] } +// GetCommentTypeByString return the comment type by given comment type string +func GetCommentTypeByString(s string) CommentType { + for i, cs := range commentStrings { + if cs == s { + return CommentType(i) + } + } + return -1 +} + // RoleDescriptor defines comment tag type type RoleDescriptor int diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index fec1fc8c7dac3..77b18635b9b65 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -425,6 +425,10 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { cms := make([]*models.Comment, 0, len(comments)) for _, comment := range comments { + if comment.Type == "" { // ignore unexpected comments + continue + } + var issue *models.Issue issue, ok := g.issues[comment.IssueIndex] if !ok { @@ -440,7 +444,7 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { cm := models.Comment{ IssueID: issue.ID, - Type: models.CommentTypeComment, + Type: models.GetCommentTypeByString(comment.Type), Content: comment.Content, CreatedUnix: timeutil.TimeStamp(comment.Created.Unix()), UpdatedUnix: timeutil.TimeStamp(comment.Updated.Unix()), diff --git a/services/migrations/restore_github.go b/services/migrations/restore_github.go index e5e5acf033104..998a2453d82b3 100644 --- a/services/migrations/restore_github.go +++ b/services/migrations/restore_github.go @@ -31,7 +31,7 @@ import ( */ type githubSchema struct { Version string - GithubSha string + GithubSha string `json:"github_sha"` } /* @@ -58,7 +58,7 @@ type githubSchema struct { */ type githubUser struct { URL string - AvatarURL string + AvatarURL string `json:"avatar_url"` Login string Name string Bio string @@ -70,7 +70,7 @@ type githubUser struct { Primary bool Verified bool } - CreatedAt time.Time + CreatedAt time.Time `json:"created_at"` } func getID(s string) int64 { @@ -111,12 +111,12 @@ func (g *githubUser) Email() string { */ type githubAttachment struct { Issue string - IssueComment string + IssueComment string `json:"issue_comment"` User string - AssetName string - AssetContentType string - AssetURL string - CreatedAt time.Time + AssetName string `json:"asset_name"` + AssetContentType string `json:"asset_content_type"` + AssetURL string `json:"asset_url"` + CreatedAt time.Time `json:"created_at"` } func (g *githubAttachment) GetUserID() int64 { @@ -161,8 +161,8 @@ func (r *GithubExportedDataRestorer) convertAttachments(ls []githubAttachment) [ type githubReaction struct { User string Content string - SubjectType string - CreatedAt time.Time + SubjectType string `json:"subject_type"` + CreatedAt time.Time `json:"created_at"` } type githubLabel string @@ -267,6 +267,11 @@ func NewGithubExportedDataRestorer(ctx context.Context, githubDataFilePath, owne return restorer, nil } +// SupportGetRepoComments return true if it can get all comments once +func (r *GithubExportedDataRestorer) SupportGetRepoComments() bool { + return true +} + // SetContext set context func (r *GithubExportedDataRestorer) SetContext(ctx context.Context) { r.ctx = ctx @@ -297,20 +302,20 @@ func (r *GithubExportedDataRestorer) readSchema() error { func (r *GithubExportedDataRestorer) GetRepoInfo() (*base.Repository, error) { type Label struct { URL string - Name string `json:"name"` + Name string Color string Description string - CreatedAt time.Time + CreatedAt time.Time `json:"created_at"` } type GithubRepo struct { - Name string `json:"name"` + Name string URL string Owner string Description string Private bool - Labels []Label `json:"labels"` - CreatedAt time.Time - DefaultBranch string + Labels []Label + CreatedAt time.Time `json:"created_at"` + DefaultBranch string `json:"default_branch"` } var githubRepositories []GithubRepo @@ -441,10 +446,10 @@ type githubMilestone struct { Title string State string Description string - DueOn time.Time - CreatedAt time.Time - UpdatedAt time.Time - ClosedAt time.Time + DueOn time.Time `json:"due_on"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ClosedAt time.Time `json:"closed_at"` } // GetMilestones returns milestones @@ -497,14 +502,14 @@ func (r *GithubExportedDataRestorer) GetMilestones() ([]*base.Milestone, error) type githubRelease struct { User string Name string - TagName string + TagName string `json:"tag_name"` Body string State string Prerelease bool - ReleaseAssets []githubAttachment - TargetCommitish string - CreatedAt time.Time - PublishedAt time.Time + ReleaseAssets []githubAttachment `json:"release_assets"` + TargetCommitish string `json:"target_commitish"` + CreatedAt time.Time `json:"created_at"` + PublishedAt time.Time `json:"published_at"` } // GetReleases returns releases @@ -724,10 +729,9 @@ type githubIssueEvent struct { PullRequest string `json:"pull_request"` Actor string Event string - CommitID string `json:"commit_id"` - CommitRepoistory string `json:"commit_repository"` - - CreatedAt time.Time `json:"created_at"` + CommitID string `json:"commit_id"` + CommitRepoistory string `json:"commit_repository"` + CreatedAt time.Time `json:"created_at"` } func (g *githubIssueEvent) CommentStr() string { @@ -891,12 +895,12 @@ type githubPullRequest struct { Milestone string Lables []githubLabel Reactions []githubReaction - ReviewRequests []struct{} - CloseIssueReferences []struct{} - WorkInProgress bool - MergedAt *time.Time - ClosedAt *time.Time - CreatedAt time.Time + ReviewRequests []struct{} `json:"review_requests"` + CloseIssueReferences []struct{} `json:"close_issue_references"` + WorkInProgress bool `json:"work_in_progress"` + MergedAt *time.Time `json:"merged_at"` + ClosedAt *time.Time `json:"closed_at"` + CreatedAt time.Time `json:"created_at"` } func (g *githubPullRequest) Index() int64 { @@ -979,14 +983,14 @@ func (r *GithubExportedDataRestorer) GetPullRequests(page, perPage int) ([]*base */ type pullrequestReview struct { URL string - PullRequest string + PullRequest string `json:"pull_request"` User string Body string - HeadSha string + HeadSha string `json:"head_sha"` State int Reactions []githubReaction - CreatedAt time.Time - SubmittedAt *time.Time + CreatedAt time.Time `json:"created_at"` + SubmittedAt *time.Time `json:"submitted_at"` } func (p *pullrequestReview) Index() int64 { @@ -1022,21 +1026,21 @@ func (p *pullrequestReview) GetState() string { "created_at": "2020-01-04T05:33:06Z" },*/ type pullrequestReviewComment struct { - PullRequest string - PullRequestReview string - PullRequestReviewThread string + PullRequest string `json:"pull_request"` + PullRequestReview string `json:"pull_request_review"` + PullRequestReviewThread string `json:"pull_request_review_thread"` User string Body string - DiffHunk string + DiffHunk string `json:"diff_hunk"` Path string Position int - OriginalPosition int - CommitID string - OriginalCommitID string + OriginalPosition int `json:"original_position"` + CommitID string `json:"commit_id"` + OriginalCommitID string `json:"original_commit_id"` State int //InReplyTo Reactions []githubReaction - CreatedAt time.Time + CreatedAt time.Time `json:"created_at"` } func (r *GithubExportedDataRestorer) getReviewComments(comments []pullrequestReviewComment) []*base.ReviewComment { diff --git a/services/migrations/restore_github_test.go b/services/migrations/restore_github_test.go index 52529fe97cb1d..22924d24079f2 100644 --- a/services/migrations/restore_github_test.go +++ b/services/migrations/restore_github_test.go @@ -75,7 +75,8 @@ func TestParseGithubExportedData(t *testing.T) { comments, isEnd, err := restorer.GetComments(migration.GetCommentOptions{}) assert.NoError(t, err) assert.True(t, isEnd) - assert.EqualValues(t, 2, len(comments)) + assert.EqualValues(t, 16, len(comments)) + // first comments are comment type assert.EqualValues(t, 2, comments[0].IssueIndex) assert.NotZero(t, comments[0].Created) From 8ddde320304be290102bc63fab3ac4fb5ce516a5 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 9 Jan 2022 00:41:39 +0800 Subject: [PATCH 10/54] Fix bug --- services/migrations/restore_github.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/services/migrations/restore_github.go b/services/migrations/restore_github.go index 998a2453d82b3..6751c5455731e 100644 --- a/services/migrations/restore_github.go +++ b/services/migrations/restore_github.go @@ -893,7 +893,7 @@ type githubPullRequest struct { Assignee string Assignees []string Milestone string - Lables []githubLabel + Labels []githubLabel Reactions []githubReaction ReviewRequests []struct{} `json:"review_requests"` CloseIssueReferences []struct{} `json:"close_issue_references"` @@ -919,9 +919,7 @@ func (r *GithubExportedDataRestorer) GetPullRequests(page, perPage int) ([]*base for _, pr := range *prs { user := r.users[pr.User] var state = "open" - if pr.MergedAt != nil { - state = "merged" - } else if pr.ClosedAt != nil { + if pr.MergedAt != nil || pr.ClosedAt != nil { state = "closed" } pulls = append(pulls, &base.PullRequest{ @@ -938,7 +936,7 @@ func (r *GithubExportedDataRestorer) GetPullRequests(page, perPage int) ([]*base Created: pr.CreatedAt, //Updated: pr., Closed: pr.ClosedAt, - Labels: r.getLabels(pr.Lables), + Labels: r.getLabels(pr.Labels), //PatchURL : pr. Merged: pr.MergedAt != nil, MergedTime: pr.MergedAt, From 6adb90fbe0ba03bc516598a40ecb7259fb3a5736 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 10 Jan 2022 11:41:47 +0800 Subject: [PATCH 11/54] improve get reviews --- modules/migration/downloader.go | 16 ++++- modules/migration/null_downloader.go | 9 ++- modules/migration/retry_downloader.go | 7 +- services/migrations/codebase.go | 10 +++ services/migrations/codebase_test.go | 2 +- services/migrations/gitbucket.go | 5 ++ services/migrations/gitea_downloader.go | 13 ++-- services/migrations/gitea_downloader_test.go | 2 +- services/migrations/gitea_uploader.go | 2 + ...tore_github.go => github_exported_data.go} | 67 ++++++++++++++++-- ...b_test.go => github_exported_data_test.go} | 14 +++- .../migrations/{github.go => github_v3.go} | 10 +-- .../{github_test.go => github_v3_test.go} | 4 +- services/migrations/gitlab.go | 8 +-- services/migrations/gitlab_test.go | 4 +- services/migrations/migrate.go | 68 +++++++++++++------ services/migrations/onedev.go | 6 +- services/migrations/onedev_test.go | 2 +- services/migrations/restore.go | 12 ++-- 19 files changed, 198 insertions(+), 63 deletions(-) rename services/migrations/{restore_github.go => github_exported_data.go} (94%) rename services/migrations/{restore_github_test.go => github_exported_data_test.go} (84%) rename services/migrations/{github.go => github_v3.go} (98%) rename services/migrations/{github_test.go => github_v3_test.go} (98%) diff --git a/modules/migration/downloader.go b/modules/migration/downloader.go index 7759c96056a92..72896bd3dc164 100644 --- a/modules/migration/downloader.go +++ b/modules/migration/downloader.go @@ -11,6 +11,20 @@ import ( "code.gitea.io/gitea/modules/structs" ) +// GetCommentOptions represents an options for get comment +type GetCommentOptions struct { + Context DownloaderContext + Page int + PageSize int +} + +// GetReviewOptions represents an options for get reviews +type GetReviewOptions struct { + Context DownloaderContext + Page int + PageSize int +} + // Downloader downloads the site repo information type Downloader interface { SetContext(context.Context) @@ -24,7 +38,7 @@ type Downloader interface { GetAllComments(page, perPage int) ([]*Comment, bool, error) SupportGetRepoComments() bool GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) - GetReviews(reviewable Reviewable) ([]*Review, error) + GetReviews(reviewable Reviewable) ([]*Review, bool, error) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) } diff --git a/modules/migration/null_downloader.go b/modules/migration/null_downloader.go index 32da720f1601f..6855de87d1a3c 100644 --- a/modules/migration/null_downloader.go +++ b/modules/migration/null_downloader.go @@ -63,8 +63,8 @@ func (n NullDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool } // GetReviews returns pull requests review -func (n NullDownloader) GetReviews(reviewable Reviewable) ([]*Review, error) { - return nil, &ErrNotSupported{Entity: "Reviews"} +func (n NullDownloader) GetReviews(reviewable Reviewable) ([]*Review, bool, error) { + return nil, false, &ErrNotSupported{Entity: "Reviews"} } // FormatCloneURL add authentication into remote URLs @@ -87,3 +87,8 @@ func (n NullDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr string) ( func (n NullDownloader) SupportGetRepoComments() bool { return false } + +// SupportGetRepoReviews return true if it supports get repo pullrequest reviews +func (n NullDownloader) SupportGetRepoReviews() bool { + return false +} diff --git a/modules/migration/retry_downloader.go b/modules/migration/retry_downloader.go index 2e40c102bea88..36640cc1eeaa5 100644 --- a/modules/migration/retry_downloader.go +++ b/modules/migration/retry_downloader.go @@ -180,16 +180,17 @@ func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bo } // GetReviews returns pull requests reviews -func (d *RetryDownloader) GetReviews(reviewable Reviewable) ([]*Review, error) { +func (d *RetryDownloader) GetReviews(reviewable Reviewable) ([]*Review, bool, error) { var ( reviews []*Review + isEnd bool err error ) err = d.retry(func() error { - reviews, err = d.Downloader.GetReviews(reviewable) + reviews, isEnd, err = d.Downloader.GetReviews(reviewable) return err }) - return reviews, err + return reviews, isEnd, err } diff --git a/services/migrations/codebase.go b/services/migrations/codebase.go index bb74c0a49d1dd..780d17f774a92 100644 --- a/services/migrations/codebase.go +++ b/services/migrations/codebase.go @@ -569,6 +569,16 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq return pullRequests, true, nil } +// GetReviews returns pull requests reviews +func (d *CodebaseDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, bool, error) { + return []*base.Review{}, true, nil +} + +// GetTopics return repository topics +func (d *CodebaseDownloader) GetTopics() ([]string, error) { + return []string{}, nil +} + func (d *CodebaseDownloader) tryGetUser(userID int64) *codebaseUser { if len(d.userMap) == 0 { var rawUsers struct { diff --git a/services/migrations/codebase_test.go b/services/migrations/codebase_test.go index 03b5946d715c9..21d56d17751d1 100644 --- a/services/migrations/codebase_test.go +++ b/services/migrations/codebase_test.go @@ -145,7 +145,7 @@ func TestCodebaseDownloadRepo(t *testing.T) { }, }, prs) - rvs, err := downloader.GetReviews(prs[0]) + rvs, _, err := downloader.GetReviews(prs[0]) assert.NoError(t, err) assert.Empty(t, rvs) } diff --git a/services/migrations/gitbucket.go b/services/migrations/gitbucket.go index 92b6cac73806b..39c70c0e042ab 100644 --- a/services/migrations/gitbucket.go +++ b/services/migrations/gitbucket.go @@ -64,3 +64,8 @@ func NewGitBucketDownloader(ctx context.Context, baseURL, userName, password, to func (g *GitBucketDownloader) SupportGetRepoComments() bool { return false } + +// GetReviews is not supported +func (g *GitBucketDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, bool, error) { + return nil, true, &base.ErrNotSupported{Entity: "Reviews"} +} diff --git a/services/migrations/gitea_downloader.go b/services/migrations/gitea_downloader.go index bb2c4ab2401fd..6868987403186 100644 --- a/services/migrations/gitea_downloader.go +++ b/services/migrations/gitea_downloader.go @@ -614,10 +614,10 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques } // GetReviews returns pull requests review -func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { +func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, bool, error) { if err := g.client.CheckServerVersionConstraint(">=1.12"); err != nil { log.Info("GiteaDownloader: instance to old, skip GetReviews") - return nil, nil + return nil, true, nil } allReviews := make([]*base.Review, 0, g.maxPerPage) @@ -626,7 +626,7 @@ func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review // make sure gitea can shutdown gracefully select { case <-g.ctx.Done(): - return nil, nil + return nil, true, nil default: } @@ -635,14 +635,13 @@ func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review PageSize: g.maxPerPage, }}) if err != nil { - return nil, err + return nil, true, err } for _, pr := range prl { - rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, reviewable.GetForeignIndex(), pr.ID) if err != nil { - return nil, err + return nil, true, err } var reviewComments []*base.ReviewComment for i := range rcl { @@ -682,5 +681,5 @@ func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review break } } - return allReviews, nil + return allReviews, true, nil } diff --git a/services/migrations/gitea_downloader_test.go b/services/migrations/gitea_downloader_test.go index 601b0a7c79331..e8129b151ca4d 100644 --- a/services/migrations/gitea_downloader_test.go +++ b/services/migrations/gitea_downloader_test.go @@ -263,7 +263,7 @@ func TestGiteaDownloadRepo(t *testing.T) { PatchURL: "https://gitea.com/gitea/test_repo/pulls/12.patch", }, prs[1]) - reviews, err := downloader.GetReviews(&base.Issue{Number: 7, ForeignIndex: 7}) + reviews, _, err := downloader.GetReviews(&base.Issue{Number: 7, ForeignIndex: 7}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 77b18635b9b65..5afa466f0b170 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -633,6 +633,8 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR pr.Updated = pr.Created } + fmt.Printf("===== %v %#v\n", pr.Number, labels) + issue := models.Issue{ RepoID: g.repo.ID, Repo: g.repo, diff --git a/services/migrations/restore_github.go b/services/migrations/github_exported_data.go similarity index 94% rename from services/migrations/restore_github.go rename to services/migrations/github_exported_data.go index 6751c5455731e..9bec08ba8f4f1 100644 --- a/services/migrations/restore_github.go +++ b/services/migrations/github_exported_data.go @@ -272,6 +272,11 @@ func (r *GithubExportedDataRestorer) SupportGetRepoComments() bool { return true } +// SupportGetRepoComments return true if it can get all comments once +func (r *GithubExportedDataRestorer) SupportGetRepoReviews() bool { + return true +} + // SetContext set context func (r *GithubExportedDataRestorer) SetContext(ctx context.Context) { r.ctx = ctx @@ -732,8 +737,55 @@ type githubIssueEvent struct { CommitID string `json:"commit_id"` CommitRepoistory string `json:"commit_repository"` CreatedAt time.Time `json:"created_at"` + Label string + LabelName string `json:"label_name` + LabelColor string `json:"label_color` + LabelTextColor string `json:"label_text_color"` + MilestoneTitle string `json:"milestone_title"` + TitleWas string `json:"title_was"` + TitleIs string `json:"title_is"` +} + +// CommentContent returns comment content +func (g *githubIssueEvent) CommentContent() map[string]interface{} { + switch g.Event { + case "closed": + return map[string]interface{}{} + case "referenced": + return map[string]interface{}{} + case "merged": + return map[string]interface{}{} + case "mentioned": + return map[string]interface{}{} + case "subscribed": + return map[string]interface{}{} + case "head_ref_deleted": + return map[string]interface{}{} + case "milestoned": + return map[string]interface{}{ + "MilestoneTitle": g.MilestoneTitle, + } + case "labeled": + return map[string]interface{}{ + "Label": g.Label, + "LabelName": g.LabelName, + "LabelColor": g.LabelColor, + "LabelTextColor": g.LabelTextColor, + } + case "renamed": + return map[string]interface{}{} + case "reopened": + return map[string]interface{}{} + case "unlabeled": + return map[string]interface{}{} + case "assigned": + return map[string]interface{}{} + default: + return map[string]interface{}{} + } } +// CommentStr returns comment type string func (g *githubIssueEvent) CommentStr() string { switch g.Event { case "closed": @@ -777,7 +829,11 @@ func (r *GithubExportedDataRestorer) getIssueEvents() ([]*base.Comment, error) { rss := content.(*[]githubIssueEvent) for _, c := range *rss { var u = r.users[c.Actor] - //var tp string + v := c.CommentContent() + bs, err := json.Marshal(v) + if err != nil { + return err + } comments = append(comments, &base.Comment{ Type: c.CommentStr(), @@ -787,6 +843,7 @@ func (r *GithubExportedDataRestorer) getIssueEvents() ([]*base.Comment, error) { PosterEmail: u.Email(), Created: c.CreatedAt, Updated: c.CreatedAt, // FIXME: + Content: string(bs), }) } return nil @@ -1061,7 +1118,7 @@ func (r *GithubExportedDataRestorer) getReviewComments(comments []pullrequestRev } // GetReviews returns pull requests review -func (r *GithubExportedDataRestorer) GetReviews(context base.IssueContext) ([]*base.Review, error) { +func (r *GithubExportedDataRestorer) GetReviews(opts base.GetReviewOptions) ([]*base.Review, bool, error) { var comments = make(map[string][]pullrequestReviewComment) if err := r.readJSONFiles("pull_request_review_comments", func() interface{} { return &[]pullrequestReviewComment{} @@ -1072,7 +1129,7 @@ func (r *GithubExportedDataRestorer) GetReviews(context base.IssueContext) ([]*b } return nil }); err != nil { - return nil, err + return nil, true, err } var reviews = make([]*base.Review, 0, 10) @@ -1095,8 +1152,8 @@ func (r *GithubExportedDataRestorer) GetReviews(context base.IssueContext) ([]*b } return nil }); err != nil { - return nil, err + return nil, true, err } - return reviews, nil + return reviews, true, nil } diff --git a/services/migrations/restore_github_test.go b/services/migrations/github_exported_data_test.go similarity index 84% rename from services/migrations/restore_github_test.go rename to services/migrations/github_exported_data_test.go index 22924d24079f2..abb267da20bbb 100644 --- a/services/migrations/restore_github_test.go +++ b/services/migrations/github_exported_data_test.go @@ -10,6 +10,7 @@ import ( "testing" "code.gitea.io/gitea/modules/migration" + base "code.gitea.io/gitea/modules/migration" "github.com/stretchr/testify/assert" ) @@ -89,8 +90,19 @@ func TestParseGithubExportedData(t *testing.T) { assert.True(t, isEnd) assert.EqualValues(t, 2, len(prs)) + assert.EqualValues(t, "Update README.md", prs[0].Title) + assert.EqualValues(t, "add warning to readme", prs[0].Content) + assert.EqualValues(t, 1, len(prs[0].Labels)) + assert.EqualValues(t, "documentation", prs[0].Labels[0].Name) + + assert.EqualValues(t, "Test branch", prs[1].Title) + assert.EqualValues(t, "do not merge this PR", prs[1].Content) + assert.EqualValues(t, 1, len(prs[1].Labels)) + assert.EqualValues(t, "bug", prs[1].Labels[0].Name) + // reviews - reviews, err := restorer.GetReviews(migration.BasicIssueContext(0)) + reviews, isEnd, err := restorer.GetReviews(base.GetReviewOptions{}) assert.NoError(t, err) + assert.True(t, isEnd) assert.EqualValues(t, 6, len(reviews)) } diff --git a/services/migrations/github.go b/services/migrations/github_v3.go similarity index 98% rename from services/migrations/github.go rename to services/migrations/github_v3.go index c796101415246..d872d74384b77 100644 --- a/services/migrations/github.go +++ b/services/migrations/github_v3.go @@ -773,7 +773,7 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques } // GetReviews returns pull requests review -func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { +func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Review, bool, error) { allReviews := make([]*base.Review, 0, g.maxPerPage) opt := &github.ListOptions{ PerPage: g.maxPerPage, @@ -782,7 +782,7 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev g.waitAndPickClient() reviews, resp, err := g.getClient().PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt) if err != nil { - return nil, fmt.Errorf("error while listing repos: %v", err) + return nil, true, fmt.Errorf("error while listing repos: %v", err) } g.setRate(&resp.Rate) for _, review := range reviews { @@ -796,13 +796,13 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev g.waitAndPickClient() reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), review.GetID(), opt2) if err != nil { - return nil, fmt.Errorf("error while listing repos: %v", err) + return nil, true, fmt.Errorf("error while listing repos: %v", err) } g.setRate(&resp.Rate) cs, err := g.convertGithubReviewComments(reviewComments) if err != nil { - return nil, err + return nil, true, err } r.Comments = append(r.Comments, cs...) if resp.NextPage == 0 { @@ -817,5 +817,5 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev } opt.Page = resp.NextPage } - return allReviews, nil + return allReviews, true, nil } diff --git a/services/migrations/github_test.go b/services/migrations/github_v3_test.go similarity index 98% rename from services/migrations/github_test.go rename to services/migrations/github_v3_test.go index 90c1fcaef5b6b..d0d4661d70e4f 100644 --- a/services/migrations/github_test.go +++ b/services/migrations/github_v3_test.go @@ -335,7 +335,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, prs) - reviews, err := downloader.GetReviews(&base.PullRequest{Number: 3, ForeignIndex: 3}) + reviews, _, err := downloader.GetReviews(&base.PullRequest{Number: 3, ForeignIndex: 3}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -367,7 +367,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, reviews) - reviews, err = downloader.GetReviews(&base.PullRequest{Number: 4, ForeignIndex: 4}) + reviews, _, err = downloader.GetReviews(&base.PullRequest{Number: 4, ForeignIndex: 4}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { diff --git a/services/migrations/gitlab.go b/services/migrations/gitlab.go index fcbd30aad4d4e..b1e79c68a3d62 100644 --- a/services/migrations/gitlab.go +++ b/services/migrations/gitlab.go @@ -616,14 +616,14 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque } // GetReviews returns pull requests review -func (g *GitlabDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { +func (g *GitlabDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, bool, error) { approvals, resp, err := g.client.MergeRequestApprovals.GetConfiguration(g.repoID, int(reviewable.GetForeignIndex()), gitlab.WithContext(g.ctx)) if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { log.Error(fmt.Sprintf("GitlabDownloader: while migrating a error occurred: '%s'", err.Error())) - return []*base.Review{}, nil + return []*base.Review{}, true, nil } - return nil, err + return nil, true, err } var createdAt time.Time @@ -647,7 +647,7 @@ func (g *GitlabDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Revie }) } - return reviews, nil + return reviews, true, nil } func (g *GitlabDownloader) awardToReaction(award *gitlab.AwardEmoji) *base.Reaction { diff --git a/services/migrations/gitlab_test.go b/services/migrations/gitlab_test.go index 829964b384e5b..d004056fe8788 100644 --- a/services/migrations/gitlab_test.go +++ b/services/migrations/gitlab_test.go @@ -304,7 +304,7 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, prs) - rvs, err := downloader.GetReviews(&base.PullRequest{Number: 1, ForeignIndex: 1}) + rvs, _, err := downloader.GetReviews(&base.PullRequest{Number: 1, ForeignIndex: 1}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -323,7 +323,7 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, rvs) - rvs, err = downloader.GetReviews(&base.PullRequest{Number: 2, ForeignIndex: 2}) + rvs, _, err = downloader.GetReviews(&base.PullRequest{Number: 2, ForeignIndex: 2}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index 700f06af35dc0..ed0dced7160a5 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -356,6 +356,8 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts } } + supportAllReviews := downloader.SupportGetRepoReviews() + if opts.PullRequests { log.Trace("migrating pull requests and comments") messenger("repo.migrate.migrating_pulls") @@ -404,30 +406,32 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts } } - // migrate reviews - allReviews := make([]*base.Review, 0, reviewBatchSize) - for _, pr := range prs { - reviews, err := downloader.GetReviews(pr) - if err != nil { - if !base.IsErrNotSupported(err) { - return err + if !supportAllComments { + // migrate reviews + allReviews := make([]*base.Review, 0, reviewBatchSize) + for _, pr := range prs { + reviews, _, err := downloader.GetReviews(pr) + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Warn("migrating reviews is not supported, ignored") + break } - log.Warn("migrating reviews is not supported, ignored") - break - } - allReviews = append(allReviews, reviews...) + allReviews = append(allReviews, reviews...) - if len(allReviews) >= reviewBatchSize { - if err = uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil { - return err + if len(allReviews) >= reviewBatchSize { + if err = uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil { + return err + } + allReviews = allReviews[reviewBatchSize:] } - allReviews = allReviews[reviewBatchSize:] } - } - if len(allReviews) > 0 { - if err = uploader.CreateReviews(allReviews...); err != nil { - return err + if len(allReviews) > 0 { + if err = uploader.CreateReviews(allReviews...); err != nil { + return err + } } } } @@ -456,6 +460,32 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts } } + if supportAllReviews { + log.Trace("migrating reviews") + for i := 1; ; i++ { + // migrate reviews + reviews, isEnd, err := downloader.GetReviews(base.GetReviewOptions{ + Page: i, + PageSize: commentBatchSize, + }) + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Warn("migrating reviews is not supported, ignored") + break + } + + if err = uploader.CreateReviews(reviews...); err != nil { + return err + } + + if isEnd { + break + } + } + } + return uploader.Finish() } diff --git a/services/migrations/onedev.go b/services/migrations/onedev.go index d4b30939ce954..76b30df1bb6f7 100644 --- a/services/migrations/onedev.go +++ b/services/migrations/onedev.go @@ -548,7 +548,7 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque } // GetReviews returns pull requests reviews -func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { +func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, bool, error) { rawReviews := make([]struct { ID int64 `json:"id"` UserID int64 `json:"userId"` @@ -565,7 +565,7 @@ func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Revie &rawReviews, ) if err != nil { - return nil, err + return nil, true, err } reviews := make([]*base.Review, 0, len(rawReviews)) @@ -592,7 +592,7 @@ func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Revie }) } - return reviews, nil + return reviews, true, nil } // GetTopics return repository topics diff --git a/services/migrations/onedev_test.go b/services/migrations/onedev_test.go index 6a17eb334bae8..30cb326247e1c 100644 --- a/services/migrations/onedev_test.go +++ b/services/migrations/onedev_test.go @@ -137,7 +137,7 @@ func TestOneDevDownloadRepo(t *testing.T) { }, }, prs) - rvs, err := downloader.GetReviews(&base.PullRequest{Number: 5, ForeignIndex: 186}) + rvs, _, err := downloader.GetReviews(&base.PullRequest{Number: 5, ForeignIndex: 186}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { diff --git a/services/migrations/restore.go b/services/migrations/restore.go index 8c9654a7e3c2b..c9b7f438cf491 100644 --- a/services/migrations/restore.go +++ b/services/migrations/restore.go @@ -248,25 +248,25 @@ func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullReq } // GetReviews returns pull requests review -func (r *RepositoryRestorer) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { +func (r *RepositoryRestorer) GetReviews(reviewable base.Reviewable) ([]*base.Review, bool, error) { reviews := make([]*base.Review, 0, 10) p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", reviewable.GetForeignIndex())) _, err := os.Stat(p) if err != nil { if os.IsNotExist(err) { - return nil, nil + return nil, true, nil } - return nil, err + return nil, true, err } bs, err := os.ReadFile(p) if err != nil { - return nil, err + return nil, true, err } err = yaml.Unmarshal(bs, &reviews) if err != nil { - return nil, err + return nil, true, err } - return reviews, nil + return reviews, true, nil } From 3e7f20d5734d834daad781692010e95659d09e0d Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 10 Jan 2022 12:05:46 +0800 Subject: [PATCH 12/54] Fix lint --- services/migrations/github_exported_data.go | 4 ++-- services/migrations/github_exported_data_test.go | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index 9bec08ba8f4f1..309e70b2e14d5 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -738,8 +738,8 @@ type githubIssueEvent struct { CommitRepoistory string `json:"commit_repository"` CreatedAt time.Time `json:"created_at"` Label string - LabelName string `json:"label_name` - LabelColor string `json:"label_color` + LabelName string `json:"label_name"` + LabelColor string `json:"label_color"` LabelTextColor string `json:"label_text_color"` MilestoneTitle string `json:"milestone_title"` TitleWas string `json:"title_was"` diff --git a/services/migrations/github_exported_data_test.go b/services/migrations/github_exported_data_test.go index abb267da20bbb..60b90a761cd4b 100644 --- a/services/migrations/github_exported_data_test.go +++ b/services/migrations/github_exported_data_test.go @@ -9,7 +9,6 @@ import ( "fmt" "testing" - "code.gitea.io/gitea/modules/migration" base "code.gitea.io/gitea/modules/migration" "github.com/stretchr/testify/assert" @@ -73,7 +72,7 @@ func TestParseGithubExportedData(t *testing.T) { assert.NotZero(t, issues[1].Created) // comments - comments, isEnd, err := restorer.GetComments(migration.GetCommentOptions{}) + comments, isEnd, err := restorer.GetComments(base.GetCommentOptions{}) assert.NoError(t, err) assert.True(t, isEnd) assert.EqualValues(t, 16, len(comments)) From acf3985d5e9a63e677b17de3d3d32d0a132ea55e Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 10 Jan 2022 13:36:09 +0800 Subject: [PATCH 13/54] Fix review --- modules/migration/review.go | 47 +++++++++++---------- services/migrations/github_exported_data.go | 44 ++++++++++++------- 2 files changed, 53 insertions(+), 38 deletions(-) diff --git a/modules/migration/review.go b/modules/migration/review.go index e4db33d98fd38..5e2f60318ccdd 100644 --- a/modules/migration/review.go +++ b/modules/migration/review.go @@ -22,16 +22,17 @@ const ( // Review is a standard review information type Review struct { - ID int64 - IssueIndex int64 `yaml:"issue_index"` - ReviewerID int64 `yaml:"reviewer_id"` - ReviewerName string `yaml:"reviewer_name"` - Official bool - CommitID string `yaml:"commit_id"` - Content string - CreatedAt time.Time `yaml:"created_at"` - State string // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT - Comments []*ReviewComment + ID int64 + IssueIndex int64 `yaml:"issue_index"` + ReviewerID int64 `yaml:"reviewer_id"` + ReviewerName string `yaml:"reviewer_name"` + ReviewerEmail string + Official bool + CommitID string `yaml:"commit_id"` + Content string + CreatedAt time.Time `yaml:"created_at"` + State string // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT + Comments []*ReviewComment } // GetExternalName ExternalUserMigrated interface @@ -42,16 +43,18 @@ func (r *Review) GetExternalID() int64 { return r.ReviewerID } // ReviewComment represents a review comment type ReviewComment struct { - ID int64 - InReplyTo int64 `yaml:"in_reply_to"` - Content string - TreePath string `yaml:"tree_path"` - DiffHunk string `yaml:"diff_hunk"` - Position int - Line int - CommitID string `yaml:"commit_id"` - PosterID int64 `yaml:"poster_id"` - Reactions []*Reaction - CreatedAt time.Time `yaml:"created_at"` - UpdatedAt time.Time `yaml:"updated_at"` + ID int64 + InReplyTo int64 `yaml:"in_reply_to"` + Content string + TreePath string `yaml:"tree_path"` + DiffHunk string `yaml:"diff_hunk"` + Position int + Line int + CommitID string `yaml:"commit_id"` + PosterID int64 `yaml:"poster_id"` + PosterName string + PosterEmail string + Reactions []*Reaction + CreatedAt time.Time `yaml:"created_at"` + UpdatedAt time.Time `yaml:"updated_at"` } diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index 309e70b2e14d5..f0a329dd30fb5 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -1054,7 +1054,16 @@ func (p *pullrequestReview) Index() int64 { return idx } +// GetState return PENDING, APPROVED, REQUEST_CHANGES, or COMMENT func (p *pullrequestReview) GetState() string { + switch p.State { + case 1: + return "COMMENT" + case 30: + return "REQUEST_CHANGES" + case 40: + return "APPROVED" + } return fmt.Sprintf("%d", p.State) } @@ -1104,14 +1113,16 @@ func (r *GithubExportedDataRestorer) getReviewComments(comments []pullrequestRev user := r.users[c.User] res = append(res, &base.ReviewComment{ //InReplyTo: , - Content: c.Body, - TreePath: c.Path, - DiffHunk: c.DiffHunk, - Position: c.Position, - CommitID: c.CommitID, - PosterID: user.ID(), - Reactions: r.getReactions(c.Reactions), - CreatedAt: c.CreatedAt, + Content: c.Body, + TreePath: c.Path, + DiffHunk: c.DiffHunk, + Position: c.Position, + CommitID: c.CommitID, + PosterID: user.ID(), + PosterName: user.Login, + PosterEmail: user.Email(), + Reactions: r.getReactions(c.Reactions), + CreatedAt: c.CreatedAt, }) } return res @@ -1140,14 +1151,15 @@ func (r *GithubExportedDataRestorer) GetReviews(opts base.GetReviewOptions) ([]* for _, review := range *prReviews { user := r.users[review.User] reviews = append(reviews, &base.Review{ - IssueIndex: review.Index(), - ReviewerID: user.ID(), - ReviewerName: user.Login, - CommitID: review.HeadSha, - Content: review.Body, - CreatedAt: review.CreatedAt, - State: review.GetState(), - Comments: r.getReviewComments(comments[review.URL]), + IssueIndex: review.Index(), + ReviewerID: user.ID(), + ReviewerName: user.Login, + ReviewerEmail: user.Email(), + CommitID: review.HeadSha, + Content: review.Body, + CreatedAt: review.CreatedAt, + State: review.GetState(), + Comments: r.getReviewComments(comments[review.URL]), }) } return nil From e5726ed3dae204728b6ad7814675bcc5cda3f081 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 10 Jan 2022 17:45:37 +0800 Subject: [PATCH 14/54] Improve some comments --- services/migrations/gitea_uploader.go | 10 +-- services/migrations/github_exported_data.go | 14 +++- .../repo/issue/view_content/comments.tmpl | 68 ++++++++++++++----- 3 files changed, 68 insertions(+), 24 deletions(-) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 5afa466f0b170..a471e6895cb59 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -425,7 +425,11 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { cms := make([]*models.Comment, 0, len(comments)) for _, comment := range comments { - if comment.Type == "" { // ignore unexpected comments + if comment.Type == "" { + comment.Type = "comment" + } + cmType := models.GetCommentTypeByString(comment.Type) + if cmType == -1 { // ignore continue } @@ -444,7 +448,7 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { cm := models.Comment{ IssueID: issue.ID, - Type: models.GetCommentTypeByString(comment.Type), + Type: cmType, Content: comment.Content, CreatedUnix: timeutil.TimeStamp(comment.Created.Unix()), UpdatedUnix: timeutil.TimeStamp(comment.Updated.Unix()), @@ -633,8 +637,6 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR pr.Updated = pr.Created } - fmt.Printf("===== %v %#v\n", pr.Number, labels) - issue := models.Issue{ RepoID: g.repo.ID, Repo: g.repo, diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index f0a329dd30fb5..9db7740df8801 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -614,11 +614,19 @@ func (r *GithubExportedDataRestorer) getLabels(ls []githubLabel) []*base.Label { func (r *GithubExportedDataRestorer) getReactions(ls []githubReaction) []*base.Reaction { var res = make([]*base.Reaction, 0, len(ls)) for _, l := range ls { + var content = l.Content + switch content { + case "thinking_face": + content = "confused" + case "tada": + content = "hooray" + } + user := r.users[l.User] res = append(res, &base.Reaction{ UserID: user.ID(), UserName: user.Login, - Content: l.Content, + Content: content, }) } return res @@ -795,9 +803,9 @@ func (g *githubIssueEvent) CommentStr() string { case "merged": return "merge_pull" case "mentioned": - return "" // ignore + return "unknown" // ignore case "subscribed": - return "" // ignore + return "unknown" // ignore case "head_ref_deleted": return "delete_branch" case "milestoned": diff --git a/templates/repo/issue/view_content/comments.tmpl b/templates/repo/issue/view_content/comments.tmpl index 235f4c8fc2662..7dd20699453c1 100644 --- a/templates/repo/issue/view_content/comments.tmpl +++ b/templates/repo/issue/view_content/comments.tmpl @@ -111,17 +111,36 @@ {{else if eq .Type 2}}
{{svg "octicon-circle-slash"}} - - {{avatar .Poster}} - - - {{.Poster.GetDisplayName}} - {{if .Issue.IsPull }} - {{$.i18n.Tr "repo.pulls.closed_at" .EventTag $createdStr | Safe}} - {{else}} - {{$.i18n.Tr "repo.issues.closed_at" .EventTag $createdStr | Safe}} + {{if .OriginalAuthor }} + + {{svg (MigrationIcon $.Repository.GetOriginalURLHostname)}} + {{ .OriginalAuthor }} + + + {{if .Issue.IsPull }} + {{$.i18n.Tr "repo.pulls.closed_at" .EventTag $createdStr | Safe}} + {{else}} + {{$.i18n.Tr "repo.issues.closed_at" .EventTag $createdStr | Safe}} + {{end}} + + {{if $.Repository.OriginalURL}} + + ({{$.i18n.Tr "repo.migrated_from" ($.Repository.OriginalURL|Escape) ($.Repository.GetOriginalURLHostname|Escape) | Safe }}) + {{end}} - + {{else}} + + {{avatar .Poster}} + + + {{.Poster.GetDisplayName}} + {{if .Issue.IsPull }} + {{$.i18n.Tr "repo.pulls.closed_at" .EventTag $createdStr | Safe}} + {{else}} + {{$.i18n.Tr "repo.issues.closed_at" .EventTag $createdStr | Safe}} + {{end}} + + {{end}}
{{else if eq .Type 28}}
@@ -206,13 +225,28 @@ {{else if eq .Type 8}}
{{svg "octicon-milestone"}} - - {{avatar .Poster}} - - - {{.Poster.GetDisplayName}} - {{if gt .OldMilestoneID 0}}{{if gt .MilestoneID 0}}{{$.i18n.Tr "repo.issues.change_milestone_at" (.OldMilestone.Name|Escape) (.Milestone.Name|Escape) $createdStr | Safe}}{{else}}{{$.i18n.Tr "repo.issues.remove_milestone_at" (.OldMilestone.Name|Escape) $createdStr | Safe}}{{end}}{{else if gt .MilestoneID 0}}{{$.i18n.Tr "repo.issues.add_milestone_at" (.Milestone.Name|Escape) $createdStr | Safe}}{{end}} - + {{if .OriginalAuthor }} + + {{svg (MigrationIcon $.Repository.GetOriginalURLHostname)}} + {{ .OriginalAuthor }} + + + {{if gt .OldMilestoneID 0}}{{if gt .MilestoneID 0}}{{$.i18n.Tr "repo.issues.change_milestone_at" (.OldMilestone.Name|Escape) (.Milestone.Name|Escape) $createdStr | Safe}}{{else}}{{$.i18n.Tr "repo.issues.remove_milestone_at" (.OldMilestone.Name|Escape) $createdStr | Safe}}{{end}}{{else if gt .MilestoneID 0}}{{$.i18n.Tr "repo.issues.add_milestone_at" (.Milestone.Name|Escape) $createdStr | Safe}}{{end}} + + {{if $.Repository.OriginalURL}} + + ({{$.i18n.Tr "repo.migrated_from" ($.Repository.OriginalURL|Escape) ($.Repository.GetOriginalURLHostname|Escape) | Safe }}) + + {{end}} + {{else}} + + {{avatar .Poster}} + + + {{.Poster.GetDisplayName}} + {{if gt .OldMilestoneID 0}}{{if gt .MilestoneID 0}}{{$.i18n.Tr "repo.issues.change_milestone_at" (.OldMilestone.Name|Escape) (.Milestone.Name|Escape) $createdStr | Safe}}{{else}}{{$.i18n.Tr "repo.issues.remove_milestone_at" (.OldMilestone.Name|Escape) $createdStr | Safe}}{{end}}{{else if gt .MilestoneID 0}}{{$.i18n.Tr "repo.issues.add_milestone_at" (.Milestone.Name|Escape) $createdStr | Safe}}{{end}} + + {{end}}
{{else if eq .Type 9}}
From 1c77daf0729c68a89109fca30c27f160f0e58613 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 10 Jan 2022 21:54:49 +0800 Subject: [PATCH 15/54] Fix label and milestone comments --- models/error.go | 7 ++++- models/issue_label.go | 16 ++++++++-- services/migrations/gitea_uploader.go | 35 +++++++++++++++++++++ services/migrations/github_exported_data.go | 9 +++++- 4 files changed, 63 insertions(+), 4 deletions(-) diff --git a/models/error.go b/models/error.go index 0dc14c3e318c5..e46466adc38c1 100644 --- a/models/error.go +++ b/models/error.go @@ -1017,6 +1017,8 @@ func (err ErrOrgLabelNotExist) Error() string { // ErrLabelNotExist represents a "LabelNotExist" kind of error. type ErrLabelNotExist struct { LabelID int64 + RepoID int64 + Name string } // IsErrLabelNotExist checks if an error is a ErrLabelNotExist. @@ -1026,7 +1028,10 @@ func IsErrLabelNotExist(err error) bool { } func (err ErrLabelNotExist) Error() string { - return fmt.Sprintf("label does not exist [label_id: %d]", err.LabelID) + if err.LabelID > 0 { + return fmt.Sprintf("label does not exist [label_id: %d]", err.LabelID) + } + return fmt.Sprintf("label does not exist [repo_id: %d, name: %s]", err.RepoID, err.Name) } // ____ ___ .__ .___ diff --git a/models/issue_label.go b/models/issue_label.go index 48a48dbb7cbcd..070d588e1a608 100644 --- a/models/issue_label.go +++ b/models/issue_label.go @@ -240,7 +240,7 @@ func DeleteLabel(id, labelID int64) error { // GetLabelByID returns a label by given ID. func GetLabelByID(ctx context.Context, labelID int64) (*Label, error) { if labelID <= 0 { - return nil, ErrLabelNotExist{labelID} + return nil, ErrLabelNotExist{LabelID: labelID} } l := &Label{} @@ -248,7 +248,7 @@ func GetLabelByID(ctx context.Context, labelID int64) (*Label, error) { if err != nil { return nil, err } else if !has { - return nil, ErrLabelNotExist{l.ID} + return nil, ErrLabelNotExist{LabelID: l.ID} } return l, nil } @@ -374,6 +374,18 @@ func CountLabelsByRepoID(repoID int64) (int64, error) { return db.GetEngine(db.DefaultContext).Where("repo_id = ?", repoID).Count(&Label{}) } +// GetLabelByRepoIDAndName get label from repo +func GetLabelByRepoIDAndName(repoID int64, name string) (*Label, error) { + var label Label + has, err := db.GetEngine(db.DefaultContext).Where("repo_id = ? AND name = ?", repoID, name).Get(&label) + if err != nil { + return nil, err + } else if !has { + return nil, ErrLabelNotExist{} + } + return &label, nil +} + // ________ // \_____ \_______ ____ // / | \_ __ \/ ___\ diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index a471e6895cb59..e72109278814b 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -23,6 +23,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" base "code.gitea.io/gitea/modules/migration" repo_module "code.gitea.io/gitea/modules/repository" @@ -458,6 +459,40 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { return err } + switch cmType { + /*{"Label":"https://github.com/go-gitea/test_repo/labels/duplicate","LabelColor":"cfd3d7","LabelName":"duplicate","LabelTextColor":"000000"}*/ + case models.CommentTypeLabel: + data := make(map[string]string) + if err := json.Unmarshal([]byte(cm.Content), &data); err != nil { + log.Error("unmarshal %s failed: %v", cm.Content, err) + } else { + lb, err := models.GetLabelByRepoIDAndName(issue.RepoID, data["LabelName"]) + if err != nil { + return err + } + if data["type"] == "add" { + cm.LabelID = lb.ID + } else { + cm.LabelID = -lb.ID + } + cm.Content = "1" + } + + /*{"MilestoneTitle":"1.1.0"}*/ + case models.CommentTypeMilestone: + data := make(map[string]string) + if err := json.Unmarshal([]byte(cm.Content), &data); err != nil { + log.Error("unmarshal %s failed: %v", cm.Content, err) + } else { + milestone, err := models.GetMilestoneByRepoIDANDName(issue.RepoID, data["MilestoneTitle"]) + if err != nil { + log.Error("GetMilestoneByRepoIDANDName %d, %s failed: %v", issue.RepoID, data["MilestoneTitle"], err) + } else { + cm.MilestoneID = milestone.ID + } + } + } + // add reactions for _, reaction := range comment.Reactions { res := issues_model.Reaction{ diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index 9db7740df8801..24ca7d52872b5 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -775,6 +775,7 @@ func (g *githubIssueEvent) CommentContent() map[string]interface{} { } case "labeled": return map[string]interface{}{ + "type": "add", "Label": g.Label, "LabelName": g.LabelName, "LabelColor": g.LabelColor, @@ -785,7 +786,13 @@ func (g *githubIssueEvent) CommentContent() map[string]interface{} { case "reopened": return map[string]interface{}{} case "unlabeled": - return map[string]interface{}{} + return map[string]interface{}{ + "type": "remove", + "Label": g.Label, + "LabelName": g.LabelName, + "LabelColor": g.LabelColor, + "LabelTextColor": g.LabelTextColor, + } case "assigned": return map[string]interface{}{} default: From 5d7577e6fce10f4a9e0eafb40bd38ad2cbdcf6aa Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 10 Jan 2022 23:45:09 +0800 Subject: [PATCH 16/54] Fix close comment --- services/migrations/github_exported_data.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index 24ca7d52872b5..883f425673ba3 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -1073,11 +1073,11 @@ func (p *pullrequestReview) Index() int64 { func (p *pullrequestReview) GetState() string { switch p.State { case 1: - return "COMMENT" + return base.ReviewStateCommented case 30: - return "REQUEST_CHANGES" + return base.ReviewStateChangesRequested case 40: - return "APPROVED" + return base.ReviewStateApproved } return fmt.Sprintf("%d", p.State) } @@ -1131,7 +1131,8 @@ func (r *GithubExportedDataRestorer) getReviewComments(comments []pullrequestRev Content: c.Body, TreePath: c.Path, DiffHunk: c.DiffHunk, - Position: c.Position, + Position: c.OriginalPosition, + Line: c.Position, CommitID: c.CommitID, PosterID: user.ID(), PosterName: user.Login, From 392f1f1e4e0f57b11b94b9a66f70a3e80637816c Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 12 Jan 2022 18:32:27 +0800 Subject: [PATCH 17/54] Fix bug --- services/migrations/github_exported_data.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index 883f425673ba3..38bbe57ad70ee 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -256,6 +256,8 @@ func NewGithubExportedDataRestorer(ctx context.Context, githubDataFilePath, owne repoName: repoName, users: make(map[string]githubUser), milestones: make(map[string]githubMilestone), + issueAttachments: make(map[string][]githubAttachment), + commentAttachments: make(map[string][]githubAttachment), } if err := restorer.readSchema(); err != nil { return nil, err From 712145fa8a0bd670d85ce050d2ebcc3d53b33fe8 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 15 Jan 2022 20:20:59 +0800 Subject: [PATCH 18/54] Fix bugs when importing --- services/migrations/gitea_uploader.go | 80 ++++++++++++--------- services/migrations/github_exported_data.go | 18 +++-- 2 files changed, 57 insertions(+), 41 deletions(-) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index e72109278814b..0a213f3328ae3 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -468,6 +468,9 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { } else { lb, err := models.GetLabelByRepoIDAndName(issue.RepoID, data["LabelName"]) if err != nil { + if models.IsErrLabelNotExist(err) { + continue + } return err } if data["type"] == "add" { @@ -540,46 +543,53 @@ func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error } func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head string, err error) { - // download patch file - err = func() error { - if pr.PatchURL == "" { - return nil - } - // pr.PatchURL maybe a local file - ret, err := uri.Open(pr.PatchURL) - if err != nil { - return err - } - defer ret.Close() - pullDir := filepath.Join(g.repo.RepoPath(), "pulls") - if err = os.MkdirAll(pullDir, os.ModePerm); err != nil { - return err - } - f, err := os.Create(filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number))) - if err != nil { - return err - } - defer f.Close() - _, err = io.Copy(f, ret) - return err - }() + refs, err := g.gitRepo.GetRefsFiltered(fmt.Sprintf("refs/pull/%d/head", pr.Number)) if err != nil { return "", err } - // set head information pullHead := filepath.Join(g.repo.RepoPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number)) - if err := os.MkdirAll(pullHead, os.ModePerm); err != nil { - return "", err - } - p, err := os.Create(filepath.Join(pullHead, "head")) - if err != nil { - return "", err - } - _, err = p.WriteString(pr.Head.SHA) - p.Close() - if err != nil { - return "", err + + if len(refs) == 0 { + // download patch file + if err = func() error { + if pr.PatchURL == "" { + return nil + } + // pr.PatchURL maybe a local file + ret, err := uri.Open(pr.PatchURL) + if err != nil { + return err + } + defer ret.Close() + pullDir := filepath.Join(g.repo.RepoPath(), "pulls") + if err = os.MkdirAll(pullDir, os.ModePerm); err != nil { + return err + } + f, err := os.Create(filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number))) + if err != nil { + return err + } + defer f.Close() + _, err = io.Copy(f, ret) + return err + }(); err != nil { + return "", err + } + + // set head information + if err := os.MkdirAll(pullHead, os.ModePerm); err != nil { + return "", err + } + p, err := os.Create(filepath.Join(pullHead, "head")) + if err != nil { + return "", err + } + _, err = p.WriteString(pr.Head.SHA) + p.Close() + if err != nil { + return "", err + } } head = "unknown repository" diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index 38bbe57ad70ee..c640f02be198d 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -624,12 +624,16 @@ func (r *GithubExportedDataRestorer) getReactions(ls []githubReaction) []*base.R content = "hooray" } - user := r.users[l.User] - res = append(res, &base.Reaction{ - UserID: user.ID(), - UserName: user.Login, - Content: content, - }) + user, ok := r.users[l.User] + if !ok { + log.Warn("Cannot get user %s infomation, ignored", l.User) + } else { + res = append(res, &base.Reaction{ + UserID: user.ID(), + UserName: user.Login, + Content: content, + }) + } } return res } @@ -657,6 +661,8 @@ func (r *GithubExportedDataRestorer) GetIssues(page, perPage int) ([]*base.Issue state = "closed" } + issue.Body = strings.ReplaceAll(issue.Body, "\u0000", "") + issues = append(issues, &base.Issue{ Number: issue.Index(), Title: issue.Title, From daca6eec7fe7d5be55d0497bc94cfb8802e49213 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 15 Jan 2022 23:31:11 +0800 Subject: [PATCH 19/54] Fix bugs when importing --- services/migrations/gitea_uploader.go | 31 ++++++++++++++++++++- services/migrations/github_exported_data.go | 25 ++++++++++++++--- 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 0a213f3328ae3..0ba43fc1ff73f 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -465,6 +465,7 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { data := make(map[string]string) if err := json.Unmarshal([]byte(cm.Content), &data); err != nil { log.Error("unmarshal %s failed: %v", cm.Content, err) + continue } else { lb, err := models.GetLabelByRepoIDAndName(issue.RepoID, data["LabelName"]) if err != nil { @@ -486,14 +487,42 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { data := make(map[string]string) if err := json.Unmarshal([]byte(cm.Content), &data); err != nil { log.Error("unmarshal %s failed: %v", cm.Content, err) + continue } else { milestone, err := models.GetMilestoneByRepoIDANDName(issue.RepoID, data["MilestoneTitle"]) if err != nil { log.Error("GetMilestoneByRepoIDANDName %d, %s failed: %v", issue.RepoID, data["MilestoneTitle"], err) } else { - cm.MilestoneID = milestone.ID + if data["type"] == "add" { + cm.MilestoneID = milestone.ID + } else if data["type"] == "remove" { + cm.OldMilestoneID = milestone.ID + } } } + case models.CommentTypeChangeTitle: + var data = make(map[string]string) + if err := json.Unmarshal([]byte(cm.Content), &data); err != nil { + log.Error("unmarshal %s failed: %v", cm.Content, err) + continue + } else { + cm.OldTitle = data["OldTitle"] + cm.NewTitle = data["NewTitle"] + } + case models.CommentTypeCommitRef: + var data = make(map[string]string) + if err := json.Unmarshal([]byte(cm.Content), &data); err != nil { + log.Error("unmarshal %s failed: %v", cm.Content, err) + continue + } else { + cm.CommitSHA = data["CommitID"] + } + /*{ + "Actor": g.Actor, + "Subject": g.Subject, + }*/ + case models.CommentTypeAssignees: + continue } // add reactions diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index c640f02be198d..cbb306fdf2fa9 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -626,7 +626,7 @@ func (r *GithubExportedDataRestorer) getReactions(ls []githubReaction) []*base.R user, ok := r.users[l.User] if !ok { - log.Warn("Cannot get user %s infomation, ignored", l.User) + log.Warn("Cannot get user %s information, ignored", l.User) } else { res = append(res, &base.Reaction{ UserID: user.ID(), @@ -758,6 +758,7 @@ type githubIssueEvent struct { LabelColor string `json:"label_color"` LabelTextColor string `json:"label_text_color"` MilestoneTitle string `json:"milestone_title"` + Subject string TitleWas string `json:"title_was"` TitleIs string `json:"title_is"` } @@ -768,7 +769,9 @@ func (g *githubIssueEvent) CommentContent() map[string]interface{} { case "closed": return map[string]interface{}{} case "referenced": - return map[string]interface{}{} + return map[string]interface{}{ + "CommitID": g.CommitID, + } case "merged": return map[string]interface{}{} case "mentioned": @@ -779,6 +782,12 @@ func (g *githubIssueEvent) CommentContent() map[string]interface{} { return map[string]interface{}{} case "milestoned": return map[string]interface{}{ + "type": "add", + "MilestoneTitle": g.MilestoneTitle, + } + case "demilestoned": + return map[string]interface{}{ + "type": "remove", "MilestoneTitle": g.MilestoneTitle, } case "labeled": @@ -790,7 +799,10 @@ func (g *githubIssueEvent) CommentContent() map[string]interface{} { "LabelTextColor": g.LabelTextColor, } case "renamed": - return map[string]interface{}{} + return map[string]interface{}{ + "OldTitle": g.TitleWas, + "NewTitle": g.TitleIs, + } case "reopened": return map[string]interface{}{} case "unlabeled": @@ -802,7 +814,10 @@ func (g *githubIssueEvent) CommentContent() map[string]interface{} { "LabelTextColor": g.LabelTextColor, } case "assigned": - return map[string]interface{}{} + return map[string]interface{}{ + "Actor": g.Actor, + "Subject": g.Subject, + } default: return map[string]interface{}{} } @@ -825,6 +840,8 @@ func (g *githubIssueEvent) CommentStr() string { return "delete_branch" case "milestoned": return "milestone" + case "demilestoned": + return "milestone" case "labeled": return "label" case "renamed": From e87444418f4c2f2468478c821e94dae35b713a4a Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 17 Jan 2022 00:01:50 +0800 Subject: [PATCH 20/54] Fix opened pull request --- services/migrations/gitea_uploader.go | 90 ++++++++++++--------- services/migrations/github_exported_data.go | 33 ++++++-- 2 files changed, 81 insertions(+), 42 deletions(-) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 0ba43fc1ff73f..f99a44a50ffdd 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -474,12 +474,12 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { } return err } + cm.LabelID = lb.ID if data["type"] == "add" { - cm.LabelID = lb.ID + cm.Content = "1" } else { - cm.LabelID = -lb.ID + cm.Content = "" } - cm.Content = "1" } /*{"MilestoneTitle":"1.1.0"}*/ @@ -492,6 +492,7 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { milestone, err := models.GetMilestoneByRepoIDANDName(issue.RepoID, data["MilestoneTitle"]) if err != nil { log.Error("GetMilestoneByRepoIDANDName %d, %s failed: %v", issue.RepoID, data["MilestoneTitle"], err) + continue } else { if data["type"] == "add" { cm.MilestoneID = milestone.ID @@ -501,7 +502,7 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { } } case models.CommentTypeChangeTitle: - var data = make(map[string]string) + data := make(map[string]string) if err := json.Unmarshal([]byte(cm.Content), &data); err != nil { log.Error("unmarshal %s failed: %v", cm.Content, err) continue @@ -510,19 +511,25 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { cm.NewTitle = data["NewTitle"] } case models.CommentTypeCommitRef: - var data = make(map[string]string) + data := make(map[string]string) if err := json.Unmarshal([]byte(cm.Content), &data); err != nil { log.Error("unmarshal %s failed: %v", cm.Content, err) continue } else { cm.CommitSHA = data["CommitID"] + // cm.Content = fmt.Sprintf(`%s`, html.EscapeString(repo.Link()), html.EscapeString(url.PathEscape(c.Sha1)), html.EscapeString(strings.SplitN(c.Message, "\n", 2)[0])) } + continue /*{ "Actor": g.Actor, "Subject": g.Subject, }*/ case models.CommentTypeAssignees: continue + case models.CommentTypeCommentRef, models.CommentTypeIssueRef, models.CommentTypePullRef: + continue + case models.CommentTypeDeleteBranch: + continue } // add reactions @@ -571,7 +578,7 @@ func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error return nil } -func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head string, err error) { +func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (string, error) { refs, err := g.gitRepo.GetRefsFiltered(fmt.Sprintf("refs/pull/%d/head", pr.Number)) if err != nil { return "", err @@ -621,41 +628,50 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head } } - head = "unknown repository" - if pr.IsForkPullRequest() && pr.State != "closed" { - if pr.Head.OwnerName != "" { - remote := pr.Head.OwnerName - _, ok := g.prHeadCache[remote] - if !ok { - // git remote add - err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true) - if err != nil { - log.Error("AddRemote failed: %s", err) - } else { - g.prHeadCache[remote] = struct{}{} - ok = true - } - } - - if ok { - _, _, err = git.NewCommand(g.ctx, "fetch", "--no-tags", "--", remote, pr.Head.Ref).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()}) - if err != nil { - log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err) - } else { - headBranch := filepath.Join(g.repo.RepoPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref) - if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil { - return "", err - } - b, err := os.Create(headBranch) + head := "unknown repository" + if pr.State != "closed" { + if pr.IsForkPullRequest() { + if pr.Head.OwnerName != "" { + remote := pr.Head.OwnerName + _, ok := g.prHeadCache[remote] + if !ok { + // git remote add + err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true) if err != nil { - return "", err + log.Error("AddRemote failed: %s", err) + } else { + g.prHeadCache[remote] = struct{}{} + ok = true } - _, err = b.WriteString(pr.Head.SHA) - b.Close() + } + + if ok { + _, _, err = git.NewCommand(g.ctx, "fetch", "--no-tags", "--", remote, pr.Head.Ref).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()}) if err != nil { - return "", err + log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err) + } else { + headBranch := filepath.Join(g.repo.RepoPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref) + if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil { + return "", err + } + b, err := os.Create(headBranch) + if err != nil { + return "", err + } + _, err = b.WriteString(pr.Head.SHA) + b.Close() + if err != nil { + return "", err + } + head = pr.Head.OwnerName + "/" + pr.Head.Ref } - head = pr.Head.OwnerName + "/" + pr.Head.Ref + } + } + } else { + head = pr.Head.Ref + if !g.gitRepo.IsBranchExist(pr.Head.Ref) && pr.Head.SHA != "" { + if err := g.gitRepo.CreateBranch(pr.Head.Ref, pr.Head.SHA); err != nil { + return "", err } } } diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index cbb306fdf2fa9..567231427d6c6 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -769,7 +769,14 @@ func (g *githubIssueEvent) CommentContent() map[string]interface{} { case "closed": return map[string]interface{}{} case "referenced": + var tp = "commit_ref" + if g.Issue != "" { + tp = "issue_ref" + } else if g.PullRequest != "" { + tp = "pull_ref" + } return map[string]interface{}{ + "type": tp, "CommitID": g.CommitID, } case "merged": @@ -852,6 +859,10 @@ func (g *githubIssueEvent) CommentStr() string { return "label" case "assigned": return "assignees" + case "pinned": + return "pinned" + case "unpinned": + return "unpinned" default: return "comment" } @@ -1019,6 +1030,22 @@ func (r *GithubExportedDataRestorer) GetPullRequests(page, perPage int) ([]*base if pr.MergedAt != nil || pr.ClosedAt != nil { state = "closed" } + var isFork = !(pr.Head.User == pr.Base.User && pr.Head.Repo == pr.Base.Repo) + var head = base.PullRequestBranch{ + Ref: pr.Head.Ref, + SHA: pr.Head.Sha, + } + if isFork { + if pr.Head.User != "" { + fields := strings.Split(pr.Head.User, "/") + if pr.Head.Ref == "" { + pr.Head.Ref = fmt.Sprintf("%d", pr.Index()) + } + head.Ref = fmt.Sprintf("%s/%s", fields[len(fields)-1], pr.Head.Ref) + } else { + head.Ref = fmt.Sprintf("pr/%d", pr.Index()) + } + } pulls = append(pulls, &base.PullRequest{ Number: pr.Index(), Title: pr.Title, @@ -1038,11 +1065,7 @@ func (r *GithubExportedDataRestorer) GetPullRequests(page, perPage int) ([]*base Merged: pr.MergedAt != nil, MergedTime: pr.MergedAt, //MergeCommitSHA : pr.Merge - Head: base.PullRequestBranch{ - Ref: pr.Head.Ref, - SHA: pr.Head.Sha, - // TODO: - }, + Head: head, Base: base.PullRequestBranch{ Ref: pr.Base.Ref, SHA: pr.Base.Sha, From d2bce40db9fea4c3344f187479f5c335e12fe78a Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 17 Jan 2022 01:07:08 +0800 Subject: [PATCH 21/54] Ignore reference comments --- services/migrations/gitea_uploader.go | 2 ++ services/migrations/github_exported_data.go | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index f99a44a50ffdd..4bc85b82f32bc 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -530,6 +530,8 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { continue case models.CommentTypeDeleteBranch: continue + case models.CommentTypePullPush: + continue } // add reactions diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index 567231427d6c6..4a69c53381659 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -750,7 +750,10 @@ type githubIssueEvent struct { PullRequest string `json:"pull_request"` Actor string Event string - CommitID string `json:"commit_id"` + CommitID string `json:"commit_id"` + Ref string + BeforeCommitOID string `json:"before_commit_oid"` + AfterCommitOID string `json:"after_commit_oid"` CommitRepoistory string `json:"commit_repository"` CreatedAt time.Time `json:"created_at"` Label string @@ -768,6 +771,8 @@ func (g *githubIssueEvent) CommentContent() map[string]interface{} { switch g.Event { case "closed": return map[string]interface{}{} + case "head_ref_force_pushed": + return map[string]interface{}{} case "referenced": var tp = "commit_ref" if g.Issue != "" { @@ -835,6 +840,8 @@ func (g *githubIssueEvent) CommentStr() string { switch g.Event { case "closed": return "close" + case "head_ref_force_pushed": + return "pull_push" case "referenced": return "commit_ref" case "merged": From 0d6a1ea5895808fc899b74f0eda04a4c573321be Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 17 Jan 2022 17:37:41 +0800 Subject: [PATCH 22/54] improve pull view --- routers/web/repo/pull.go | 30 ++++++++++++++++-------------- services/pull/pull.go | 15 +++++---------- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/routers/web/repo/pull.go b/routers/web/repo/pull.go index 3f24be33d6623..a86cd6bbc2162 100644 --- a/routers/web/repo/pull.go +++ b/routers/web/repo/pull.go @@ -468,7 +468,6 @@ func PrepareViewPullInfo(ctx *context.Context, issue *models.Issue) *git.Compare return compareInfo } - var headBranchExist bool var headBranchSha string // HeadRepo may be missing if pull.HeadRepo != nil { @@ -480,25 +479,28 @@ func PrepareViewPullInfo(ctx *context.Context, issue *models.Issue) *git.Compare defer headGitRepo.Close() if pull.Flow == models.PullRequestFlowGithub { - headBranchExist = headGitRepo.IsBranchExist(pull.HeadBranch) - } else { - headBranchExist = git.IsReferenceExist(ctx, baseGitRepo.Path, pull.GetGitRefName()) + if headGitRepo.IsBranchExist(pull.HeadBranch) { + headBranchSha, err = headGitRepo.GetBranchCommitID(pull.HeadBranch) + if err != nil { + ctx.ServerError("GetBranchCommitID", err) + return nil + } + } } - if headBranchExist { - if pull.Flow != models.PullRequestFlowGithub { + if headBranchSha == "" { + if git.IsReferenceExist(ctx, baseGitRepo.Path, pull.GetGitRefName()) { headBranchSha, err = baseGitRepo.GetRefCommitID(pull.GetGitRefName()) - } else { - headBranchSha, err = headGitRepo.GetBranchCommitID(pull.HeadBranch) - } - if err != nil { - ctx.ServerError("GetBranchCommitID", err) - return nil + if err != nil { + ctx.ServerError("GetBranchCommitID", err) + return nil + } } } + headGitRepo.Close() } - if headBranchExist { + if headBranchSha != "" { var err error ctx.Data["UpdateAllowed"], ctx.Data["UpdateByRebaseAllowed"], err = pull_service.IsUserAllowedToUpdate(ctx, pull, ctx.Doer) if err != nil { @@ -554,7 +556,7 @@ func PrepareViewPullInfo(ctx *context.Context, issue *models.Issue) *git.Compare ctx.Data["HeadBranchCommitID"] = headBranchSha ctx.Data["PullHeadCommitID"] = sha - if pull.HeadRepo == nil || !headBranchExist || headBranchSha != sha { + if pull.HeadRepo == nil || headBranchSha == "" || headBranchSha != sha { ctx.Data["IsPullRequestBroken"] = true if pull.IsSameRepo() { ctx.Data["HeadTarget"] = pull.HeadBranch diff --git a/services/pull/pull.go b/services/pull/pull.go index efac3f019eff0..817bae0925d81 100644 --- a/services/pull/pull.go +++ b/services/pull/pull.go @@ -606,17 +606,12 @@ func GetSquashMergeCommitMessages(ctx context.Context, pr *models.PullRequest) s } defer closer.Close() - var headCommit *git.Commit - if pr.Flow == models.PullRequestFlowGithub { - headCommit, err = gitRepo.GetBranchCommit(pr.HeadBranch) - } else { - pr.HeadCommitID, err = gitRepo.GetRefCommitID(pr.GetGitRefName()) - if err != nil { - log.Error("Unable to get head commit: %s Error: %v", pr.GetGitRefName(), err) - return "" - } - headCommit, err = gitRepo.GetCommit(pr.HeadCommitID) + pr.HeadCommitID, err = gitRepo.GetRefCommitID(pr.GetGitRefName()) + if err != nil { + log.Error("Unable to get head commit: %s Error: %v", pr.GetGitRefName(), err) + return "" } + headCommit, err := gitRepo.GetCommit(pr.HeadCommitID) if err != nil { log.Error("Unable to get head commit: %s Error: %v", pr.HeadBranch, err) return "" From 426effbc493869987e0f620344753ff733c3a6ce Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 18 Jan 2022 13:49:33 +0800 Subject: [PATCH 23/54] Fix bug --- services/migrations/github_exported_data.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index 4a69c53381659..2625ce822cffe 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -1186,8 +1186,8 @@ func (r *GithubExportedDataRestorer) getReviewComments(comments []pullrequestRev Content: c.Body, TreePath: c.Path, DiffHunk: c.DiffHunk, - Position: c.OriginalPosition, - Line: c.Position, + Position: c.Position, + Line: c.OriginalPosition, CommitID: c.CommitID, PosterID: user.ID(), PosterName: user.Login, From 81fe0568de62838ca7a2f80208f06d80f2f9cf34 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 20 Jan 2022 17:37:19 +0800 Subject: [PATCH 24/54] Fix bug --- modules/migration/review.go | 4 + services/migrations/github_exported_data.go | 94 ++++++++++++++++++++- 2 files changed, 94 insertions(+), 4 deletions(-) diff --git a/modules/migration/review.go b/modules/migration/review.go index 5e2f60318ccdd..85f73eded00f2 100644 --- a/modules/migration/review.go +++ b/modules/migration/review.go @@ -33,6 +33,10 @@ type Review struct { CreatedAt time.Time `yaml:"created_at"` State string // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT Comments []*ReviewComment + ResolvedAt *time.Time `yaml:"resolved_at"` + ResolverID int64 `yaml:"resolver_id"` + ResolverName string `yaml:"resolver_name"` + ResolverEmail string `yaml:"resolver_email"` } // GetExternalName ExternalUserMigrated interface diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index 2625ce822cffe..7d9190d4d7ad1 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -1137,6 +1137,55 @@ func (p *pullrequestReview) GetState() string { return fmt.Sprintf("%d", p.State) } +/* +{ + "type": "pull_request_review_thread", + "url": "https://github.com/go-xorm/xorm/pull/1445/files#pullrequestreviewthread-203253693", + "pull_request": "https://github.com/go-xorm/xorm/pull/1445", + "pull_request_review": "https://github.com/go-xorm/xorm/pull/1445/files#pullrequestreview-295977501", + "diff_hunk": "@@ -245,12 +245,17 @@ func (session *Session) Sync2(beans ...interface{}) error {\n \t\tif err != nil {\n \t\t\treturn err\n \t\t}\n-\t\ttbName := engine.TableName(bean)\n-\t\ttbNameWithSchema := engine.TableName(tbName, true)\n+\t\tvar tbName string\n+\t\tif len(session.statement.AltTableName) > 0 {\n+\t\t\ttbName = session.statement.AltTableName\n+\t\t} else {\n+\t\t\ttbName = engine.TableName(bean)\n+\t\t}\n+\t\ttbNameWithSchema := engine.tbNameWithSchema(tbName)\n \n \t\tvar oriTable *core.Table\n \t\tfor _, tb := range tables {\n-\t\t\tif strings.EqualFold(tb.Name, tbName) {\n+\t\t\tif strings.EqualFold(engine.tbNameWithSchema(tb.Name), engine.tbNameWithSchema(tbName)) {", + "path": "session_schema.go", + "position": 17, + "original_position": 17, + "commit_id": "f6b642c82aab95178a4551a1ff65dc2a631a08cf", + "original_commit_id": "f6b642c82aab95178a4551a1ff65dc2a631a08cf", + "start_line": null, + "line": 258, + "start_side": null, + "side": "right", + "original_start_line": null, + "original_line": 258, + "created_at": "2019-10-02T01:40:41Z", + "resolved_at": null, + "resolver": null + },*/ +type pullrequestReviewThread struct { + URL string + PullRequest string `json:"pull_request"` + PullRequestReview string `json:"pull_request_review"` + DiffHunk string `json:"diff_hunk"` + Path string + Position int64 + OriginalPosition int64 `json:"original_position"` + CommitID string `json:"commit_id"` + OriginalCommitID string `json:"original_commit_id"` + //StartLine + Line int64 + //StartSide + Side string + //OriginalStartLine string `json:"head_sha"` + OriginalLine int64 + CreatedAt time.Time `json:"created_at"` + ResolvedAt *time.Time `json:"resolved_at"` + Resolver string +} + +func (p *pullrequestReviewThread) Index() int64 { + fields := strings.Split(p.PullRequest, "/") + idx, _ := strconv.ParseInt(fields[len(fields)-1], 10, 64) + return idx +} + /*{ "type": "pull_request_review_comment", "url": "https://github.com/go-gitea/test_repo/pull/4/files#r363017488", @@ -1207,7 +1256,7 @@ func (r *GithubExportedDataRestorer) GetReviews(opts base.GetReviewOptions) ([]* }, func(content interface{}) error { cs := *content.(*[]pullrequestReviewComment) for _, c := range cs { - comments[c.PullRequestReview] = append(comments[c.PullRequestReview], c) + comments[c.PullRequestReviewThread] = append(comments[c.PullRequestReviewThread], c) } return nil }); err != nil { @@ -1220,8 +1269,11 @@ func (r *GithubExportedDataRestorer) GetReviews(opts base.GetReviewOptions) ([]* }, func(content interface{}) error { prReviews := content.(*[]pullrequestReview) for _, review := range *prReviews { + if review.State == 1 { + continue + } user := r.users[review.User] - reviews = append(reviews, &base.Review{ + baseReview := &base.Review{ IssueIndex: review.Index(), ReviewerID: user.ID(), ReviewerName: user.Login, @@ -1230,8 +1282,42 @@ func (r *GithubExportedDataRestorer) GetReviews(opts base.GetReviewOptions) ([]* Content: review.Body, CreatedAt: review.CreatedAt, State: review.GetState(), - Comments: r.getReviewComments(comments[review.URL]), - }) + } + reviews = append(reviews, baseReview) + } + return nil + }); err != nil { + return nil, true, err + } + + if err := r.readJSONFiles("pull_request_review_threads", func() interface{} { + return &[]pullrequestReviewThread{} + }, func(content interface{}) error { + cs := *content.(*[]pullrequestReviewThread) + for _, review := range cs { + reviewComments := comments[review.URL] + if len(reviewComments) == 0 { + continue + } + user := r.users[reviewComments[0].User] + baseReview := &base.Review{ + IssueIndex: review.Index(), + ReviewerID: user.ID(), + ReviewerName: user.Login, + ReviewerEmail: user.Email(), + CommitID: reviewComments[0].CommitID, + CreatedAt: review.CreatedAt, + State: base.ReviewStateCommented, + Comments: r.getReviewComments(reviewComments), + ResolvedAt: review.ResolvedAt, + } + + if resolver, ok := r.users[review.Resolver]; ok { + baseReview.ResolverID = resolver.ID() + baseReview.ResolverName = resolver.Login + baseReview.ResolverEmail = resolver.Email() + } + reviews = append(reviews, baseReview) } return nil }); err != nil { From ae8fe0cafdfefb034cace9146eccb4b86e7256ee Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 22 Jan 2022 00:14:38 +0800 Subject: [PATCH 25/54] add trace log --- services/migrations/gitea_uploader.go | 5 +++++ services/migrations/github_exported_data.go | 6 ++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 4bc85b82f32bc..196318b228bd2 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -819,6 +819,7 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { IssueID: issue.ID, Content: review.Content, Official: review.Official, + CommitID: review.CommitID, CreatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()), UpdatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()), } @@ -839,13 +840,16 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { } for _, comment := range review.Comments { + log.Info("444444=========== %#v", comment) line := comment.Line if line != 0 { comment.Position = 1 } else { _, _, line, _ = git.ParseDiffHunkString(comment.DiffHunk) + log.Info("5555555 %v", line) } headCommitID, err := g.gitRepo.GetRefCommitID(pr.GetGitRefName()) + log.Info("66666666 %v,,,,,%v", headCommitID, pr.MergeBase) if err != nil { log.Warn("GetRefCommitID[%s]: %v, the review comment will be ignored", pr.GetGitRefName(), err) continue @@ -866,6 +870,7 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { }(comment) patch, _ = git.CutDiffAroundLine(reader, int64((&models.Comment{Line: int64(line + comment.Position - 1)}).UnsignedLine()), line < 0, setting.UI.CodeCommentLines) + log.Info("7777777 patch:%s, line: %d, headCommitID: %s", patch, line+comment.Position-1, headCommitID) if comment.CreatedAt.IsZero() { comment.CreatedAt = review.CreatedAt diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index 7d9190d4d7ad1..a3ddb9eae9e38 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -1237,7 +1237,7 @@ func (r *GithubExportedDataRestorer) getReviewComments(comments []pullrequestRev DiffHunk: c.DiffHunk, Position: c.Position, Line: c.OriginalPosition, - CommitID: c.CommitID, + CommitID: c.OriginalCommitID, PosterID: user.ID(), PosterName: user.Login, PosterEmail: user.Email(), @@ -1256,6 +1256,7 @@ func (r *GithubExportedDataRestorer) GetReviews(opts base.GetReviewOptions) ([]* }, func(content interface{}) error { cs := *content.(*[]pullrequestReviewComment) for _, c := range cs { + log.Info("111------%#v,,,,%#v, %#v", c.PullRequestReview, c.CommitID, c.OriginalCommitID) comments[c.PullRequestReviewThread] = append(comments[c.PullRequestReviewThread], c) } return nil @@ -1299,13 +1300,14 @@ func (r *GithubExportedDataRestorer) GetReviews(opts base.GetReviewOptions) ([]* if len(reviewComments) == 0 { continue } + log.Info("2222------- %#v", reviewComments[0]) user := r.users[reviewComments[0].User] baseReview := &base.Review{ IssueIndex: review.Index(), ReviewerID: user.ID(), ReviewerName: user.Login, ReviewerEmail: user.Email(), - CommitID: reviewComments[0].CommitID, + CommitID: reviewComments[0].OriginalCommitID, CreatedAt: review.CreatedAt, State: base.ReviewStateCommented, Comments: r.getReviewComments(reviewComments), From 0ca2dab60d9d75c5abd28fdc1e2f2d18956904f3 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 22 Jan 2022 17:37:15 +0800 Subject: [PATCH 26/54] Improve some missed review --- modules/git/diff.go | 1 + services/migrations/gitea_uploader.go | 19 ++++--- services/migrations/github_exported_data.go | 59 ++++++++++----------- 3 files changed, 42 insertions(+), 37 deletions(-) diff --git a/modules/git/diff.go b/modules/git/diff.go index c9d68bb130fe9..447d4bf579c99 100644 --- a/modules/git/diff.go +++ b/modules/git/diff.go @@ -52,6 +52,7 @@ func GetRepoRawDiffForFile(repo *Repository, startCommit, endCommit string, diff if err != nil { return err } + fileArgs := make([]string, 0) if len(file) > 0 { fileArgs = append(fileArgs, "--", file) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 196318b228bd2..bc568bda20332 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -840,22 +840,20 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { } for _, comment := range review.Comments { - log.Info("444444=========== %#v", comment) line := comment.Line if line != 0 { - comment.Position = 1 + if comment.Position == 0 { + comment.Position = 1 + } } else { _, _, line, _ = git.ParseDiffHunkString(comment.DiffHunk) - log.Info("5555555 %v", line) } headCommitID, err := g.gitRepo.GetRefCommitID(pr.GetGitRefName()) - log.Info("66666666 %v,,,,,%v", headCommitID, pr.MergeBase) if err != nil { log.Warn("GetRefCommitID[%s]: %v, the review comment will be ignored", pr.GetGitRefName(), err) continue } - var patch string reader, writer := io.Pipe() defer func() { _ = reader.Close() @@ -869,8 +867,15 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { _ = writer.Close() }(comment) - patch, _ = git.CutDiffAroundLine(reader, int64((&models.Comment{Line: int64(line + comment.Position - 1)}).UnsignedLine()), line < 0, setting.UI.CodeCommentLines) - log.Info("7777777 patch:%s, line: %d, headCommitID: %s", patch, line+comment.Position-1, headCommitID) + unsignedLine := int64((&models.Comment{Line: int64(line + comment.Position - 1)}).UnsignedLine()) + patch, err := git.CutDiffAroundLine(reader, unsignedLine, line < 0, setting.UI.CodeCommentLines) + if err != nil { + log.Warn("CutDiffAroundLine failed when migrating [%s, %d, %v]", g.gitRepo.Path, unsignedLine, line < 0) + } + if patch == "" { + patch = fmt.Sprintf("diff --git a/%s b/%s\n--- a/%s\n+++ b/%s\n%s", comment.TreePath, comment.TreePath, comment.TreePath, comment.TreePath, comment.DiffHunk) + } + _ = reader.Close() if comment.CreatedAt.IsZero() { comment.CreatedAt = review.CreatedAt diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index a3ddb9eae9e38..722a0f53a3f57 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -1174,7 +1174,7 @@ type pullrequestReviewThread struct { //StartSide Side string //OriginalStartLine string `json:"head_sha"` - OriginalLine int64 + OriginalLine int64 `json:"original_line"` CreatedAt time.Time `json:"created_at"` ResolvedAt *time.Time `json:"resolved_at"` Resolver string @@ -1226,17 +1226,23 @@ type pullrequestReviewComment struct { CreatedAt time.Time `json:"created_at"` } -func (r *GithubExportedDataRestorer) getReviewComments(comments []pullrequestReviewComment) []*base.ReviewComment { +func (r *GithubExportedDataRestorer) getReviewComments(thread *pullrequestReviewThread, comments []pullrequestReviewComment) []*base.ReviewComment { var res []*base.ReviewComment for _, c := range comments { user := r.users[c.User] + position := int(thread.Position) + //line := -int(thread.Line) + if thread.Side == "right" { + position = int(thread.OriginalPosition) + //line = int(thread.OriginalLine) + } res = append(res, &base.ReviewComment{ //InReplyTo: , - Content: c.Body, - TreePath: c.Path, - DiffHunk: c.DiffHunk, - Position: c.Position, - Line: c.OriginalPosition, + Content: c.Body, + TreePath: c.Path, + DiffHunk: c.DiffHunk, + Position: position, + //Line: line, line will be parse from diffhunk CommitID: c.OriginalCommitID, PosterID: user.ID(), PosterName: user.Login, @@ -1256,7 +1262,6 @@ func (r *GithubExportedDataRestorer) GetReviews(opts base.GetReviewOptions) ([]* }, func(content interface{}) error { cs := *content.(*[]pullrequestReviewComment) for _, c := range cs { - log.Info("111------%#v,,,,%#v, %#v", c.PullRequestReview, c.CommitID, c.OriginalCommitID) comments[c.PullRequestReviewThread] = append(comments[c.PullRequestReviewThread], c) } return nil @@ -1264,15 +1269,12 @@ func (r *GithubExportedDataRestorer) GetReviews(opts base.GetReviewOptions) ([]* return nil, true, err } - var reviews = make([]*base.Review, 0, 10) + var reviews = make(map[string]*base.Review, 10) if err := r.readJSONFiles("pull_request_reviews", func() interface{} { return &[]pullrequestReview{} }, func(content interface{}) error { prReviews := content.(*[]pullrequestReview) for _, review := range *prReviews { - if review.State == 1 { - continue - } user := r.users[review.User] baseReview := &base.Review{ IssueIndex: review.Index(), @@ -1284,7 +1286,7 @@ func (r *GithubExportedDataRestorer) GetReviews(opts base.GetReviewOptions) ([]* CreatedAt: review.CreatedAt, State: review.GetState(), } - reviews = append(reviews, baseReview) + reviews[review.URL] = baseReview } return nil }); err != nil { @@ -1300,31 +1302,28 @@ func (r *GithubExportedDataRestorer) GetReviews(opts base.GetReviewOptions) ([]* if len(reviewComments) == 0 { continue } - log.Info("2222------- %#v", reviewComments[0]) - user := r.users[reviewComments[0].User] - baseReview := &base.Review{ - IssueIndex: review.Index(), - ReviewerID: user.ID(), - ReviewerName: user.Login, - ReviewerEmail: user.Email(), - CommitID: reviewComments[0].OriginalCommitID, - CreatedAt: review.CreatedAt, - State: base.ReviewStateCommented, - Comments: r.getReviewComments(reviewComments), - ResolvedAt: review.ResolvedAt, + rr, ok := reviews[review.PullRequestReview] + if !ok { + return fmt.Errorf("cannot find review thread %s's review", review.PullRequestReview) } + rr.Comments = r.getReviewComments(&review, reviewComments) + rr.ResolvedAt = review.ResolvedAt if resolver, ok := r.users[review.Resolver]; ok { - baseReview.ResolverID = resolver.ID() - baseReview.ResolverName = resolver.Login - baseReview.ResolverEmail = resolver.Email() + rr.ResolverID = resolver.ID() + rr.ResolverName = resolver.Login + rr.ResolverEmail = resolver.Email() } - reviews = append(reviews, baseReview) } return nil }); err != nil { return nil, true, err } - return reviews, true, nil + var rs = make([]*base.Review, 0, len(reviews)) + for _, review := range reviews { + rs = append(rs, review) + } + + return rs, true, nil } From 95aa21c6f2259339f025ea1c9bac3bf4d98bbed4 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 22 Jan 2022 18:30:18 +0800 Subject: [PATCH 27/54] Fix build --- services/migrations/gitea_uploader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index bc568bda20332..3c40a0a7b2ef1 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -530,7 +530,7 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { continue case models.CommentTypeDeleteBranch: continue - case models.CommentTypePullPush: + case models.CommentTypePullRequestPush: continue } From f70f7d297a38629ca6afe85bba5c44ce6115f383 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 23 Jan 2022 00:51:12 +0800 Subject: [PATCH 28/54] Fix lint --- services/migrations/dump.go | 4 +- services/migrations/gitea_uploader.go | 2 - services/migrations/github_exported_data.go | 118 +++++++++----------- 3 files changed, 57 insertions(+), 67 deletions(-) diff --git a/services/migrations/dump.go b/services/migrations/dump.go index 05e2aa60ef265..a237d00d5ae06 100644 --- a/services/migrations/dump.go +++ b/services/migrations/dump.go @@ -643,13 +643,13 @@ func RestoreFromGithubExportedData(ctx context.Context, baseDir, ownerName, repo if err != nil { return err } - var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, repoName) + uploader := NewGiteaLocalUploader(ctx, doer, ownerName, repoName) downloader, err := NewGithubExportedDataRestorer(ctx, baseDir, ownerName, repoName) if err != nil { return err } - var migrateOpts = base.MigrateOptions{ + migrateOpts := base.MigrateOptions{ GitServiceType: structs.GithubService, } updateOptionsUnits(&migrateOpts, units) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 3c40a0a7b2ef1..623f5aaa7c590 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -514,10 +514,8 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { data := make(map[string]string) if err := json.Unmarshal([]byte(cm.Content), &data); err != nil { log.Error("unmarshal %s failed: %v", cm.Content, err) - continue } else { cm.CommitSHA = data["CommitID"] - // cm.Content = fmt.Sprintf(`%s`, html.EscapeString(repo.Link()), html.EscapeString(url.PathEscape(c.Sha1)), html.EscapeString(strings.SplitN(c.Message, "\n", 2)[0])) } continue /*{ diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index 722a0f53a3f57..71f9d2b51a679 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -135,16 +135,20 @@ func (g *githubAttachment) IssueID() int64 { } func (r *GithubExportedDataRestorer) convertAttachments(ls []githubAttachment) []*base.Asset { - var res = make([]*base.Asset, 0, len(ls)) + res := make([]*base.Asset, 0, len(ls)) for _, l := range ls { - var assetURL = "file://" + strings.TrimPrefix(l.AssetURL, "tarball://root") + fPath := strings.TrimPrefix(l.AssetURL, "tarball://root") + info, err := os.Stat(fPath) + var size int + if err == nil { + size = int(info.Size()) + } + assetURL := "file://" + fPath res = append(res, &base.Asset{ Name: l.AssetName, ContentType: &l.AssetContentType, - //Size : l.Size, - //DownloadCount *int `yaml:"download_count"` - Created: l.CreatedAt, - //Updated time.Time + Size: &size, + Created: l.CreatedAt, DownloadURL: &assetURL, }) } @@ -248,7 +252,7 @@ func NewGithubExportedDataRestorer(ctx context.Context, githubDataFilePath, owne return nil, err } - var restorer = &GithubExportedDataRestorer{ + restorer := &GithubExportedDataRestorer{ ctx: ctx, githubDataFilePath: githubDataFilePath, tmpDir: tmpDir, @@ -325,13 +329,13 @@ func (r *GithubExportedDataRestorer) GetRepoInfo() (*base.Repository, error) { DefaultBranch string `json:"default_branch"` } - var githubRepositories []GithubRepo p := filepath.Join(r.tmpDir, "repositories_000001.json") bs, err := ioutil.ReadFile(p) if err != nil { return nil, err } + var githubRepositories []GithubRepo if err := json.Unmarshal(bs, &githubRepositories); err != nil { return nil, err } @@ -366,7 +370,7 @@ func (r *GithubExportedDataRestorer) GetRepoInfo() (*base.Repository, error) { // GetTopics return github topics func (r *GithubExportedDataRestorer) GetTopics() ([]string, error) { - var topics = struct { + topics := struct { Topics []string `yaml:"topics"` }{} @@ -461,7 +465,7 @@ type githubMilestone struct { // GetMilestones returns milestones func (r *GithubExportedDataRestorer) GetMilestones() ([]*base.Milestone, error) { - var milestones = make([]*base.Milestone, 0, 10) + milestones := make([]*base.Milestone, 0, 10) if err := r.readJSONFiles("milestones", func() interface{} { return &[]githubMilestone{} }, func(content interface{}) error { @@ -521,7 +525,7 @@ type githubRelease struct { // GetReleases returns releases func (r *GithubExportedDataRestorer) GetReleases() ([]*base.Release, error) { - var releases = make([]*base.Release, 0, 30) + releases := make([]*base.Release, 0, 30) if err := r.readJSONFiles("releases", func() interface{} { return &[]githubRelease{} }, func(content interface{}) error { @@ -601,7 +605,7 @@ func (g *githubIssue) Index() int64 { } func (r *GithubExportedDataRestorer) getLabels(ls []githubLabel) []*base.Label { - var res = make([]*base.Label, 0, len(ls)) + res := make([]*base.Label, 0, len(ls)) for _, l := range ls { for _, ll := range r.labels { if l.GetName() == ll.Name { @@ -614,9 +618,9 @@ func (r *GithubExportedDataRestorer) getLabels(ls []githubLabel) []*base.Label { } func (r *GithubExportedDataRestorer) getReactions(ls []githubReaction) []*base.Reaction { - var res = make([]*base.Reaction, 0, len(ls)) + res := make([]*base.Reaction, 0, len(ls)) for _, l := range ls { - var content = l.Content + content := l.Content switch content { case "thinking_face": content = "confused" @@ -644,7 +648,7 @@ func (r *GithubExportedDataRestorer) GetIssues(page, perPage int) ([]*base.Issue return nil, false, err } - var issues = make([]*base.Issue, 0, 50) + issues := make([]*base.Issue, 0, 50) if err := r.readJSONFiles("issues", func() interface{} { return &[]githubIssue{} }, func(content interface{}) error { @@ -656,7 +660,7 @@ func (r *GithubExportedDataRestorer) GetIssues(page, perPage int) ([]*base.Issue milestone = r.milestones[issue.Milestone].Title } - var state = "open" + state := "open" if issue.ClosedAt != nil { state = "closed" } @@ -675,12 +679,11 @@ func (r *GithubExportedDataRestorer) GetIssues(page, perPage int) ([]*base.Issue Assets: r.convertAttachments(r.issueAttachments[issue.URL]), Milestone: milestone, Assignees: issue.Assignees, - //Ref: issue. - State: state, - Context: base.BasicIssueContext(issue.Index()), - Closed: issue.ClosedAt, - Created: issue.CreatedAt, - Updated: issue.UpdatedAt, + State: state, + Context: base.BasicIssueContext(issue.Index()), + Closed: issue.ClosedAt, + Created: issue.CreatedAt, + Updated: issue.UpdatedAt, }) } return nil @@ -774,7 +777,7 @@ func (g *githubIssueEvent) CommentContent() map[string]interface{} { case "head_ref_force_pushed": return map[string]interface{}{} case "referenced": - var tp = "commit_ref" + tp := "commit_ref" if g.Issue != "" { tp = "issue_ref" } else if g.PullRequest != "" { @@ -880,13 +883,13 @@ func (g *githubIssueEvent) GetIssueIndex() int64 { } func (r *GithubExportedDataRestorer) getIssueEvents() ([]*base.Comment, error) { - var comments []*base.Comment + comments := make([]*base.Comment, 0, 10) if err := r.readJSONFiles("issue_events", func() interface{} { return &[]githubIssueEvent{} }, func(content interface{}) error { rss := content.(*[]githubIssueEvent) for _, c := range *rss { - var u = r.users[c.Actor] + u := r.users[c.Actor] v := c.CommentContent() bs, err := json.Marshal(v) if err != nil { @@ -913,7 +916,7 @@ func (r *GithubExportedDataRestorer) getIssueEvents() ([]*base.Comment, error) { // GetComments returns comments according issueNumber func (r *GithubExportedDataRestorer) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { - var comments = make([]*base.Comment, 0, 10) + comments := make([]*base.Comment, 0, 10) if err := r.readJSONFiles("issue_comments", func() interface{} { return &[]githubComment{} }, func(content interface{}) error { @@ -1026,19 +1029,19 @@ func (g *githubPullRequest) Index() int64 { // GetPullRequests returns pull requests according page and perPage func (r *GithubExportedDataRestorer) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { - var pulls = make([]*base.PullRequest, 0, 50) + pulls := make([]*base.PullRequest, 0, 50) if err := r.readJSONFiles("pull_requests", func() interface{} { return &[]githubPullRequest{} }, func(content interface{}) error { prs := content.(*[]githubPullRequest) for _, pr := range *prs { user := r.users[pr.User] - var state = "open" + state := "open" if pr.MergedAt != nil || pr.ClosedAt != nil { state = "closed" } - var isFork = !(pr.Head.User == pr.Base.User && pr.Head.Repo == pr.Base.Repo) - var head = base.PullRequestBranch{ + isFork := !(pr.Head.User == pr.Base.User && pr.Head.Repo == pr.Base.Repo) + head := base.PullRequestBranch{ Ref: pr.Head.Ref, SHA: pr.Head.Sha, } @@ -1065,18 +1068,14 @@ func (r *GithubExportedDataRestorer) GetPullRequests(page, perPage int) ([]*base Context: base.BasicIssueContext(pr.Index()), Reactions: r.getReactions(pr.Reactions), Created: pr.CreatedAt, - //Updated: pr., - Closed: pr.ClosedAt, - Labels: r.getLabels(pr.Labels), - //PatchURL : pr. - Merged: pr.MergedAt != nil, - MergedTime: pr.MergedAt, - //MergeCommitSHA : pr.Merge - Head: head, + Closed: pr.ClosedAt, + Labels: r.getLabels(pr.Labels), + Merged: pr.MergedAt != nil, + MergedTime: pr.MergedAt, + Head: head, Base: base.PullRequestBranch{ Ref: pr.Base.Ref, SHA: pr.Base.Sha, - // TODO: }, Assignees: pr.Assignees, }) @@ -1169,15 +1168,12 @@ type pullrequestReviewThread struct { OriginalPosition int64 `json:"original_position"` CommitID string `json:"commit_id"` OriginalCommitID string `json:"original_commit_id"` - //StartLine - Line int64 - //StartSide - Side string - //OriginalStartLine string `json:"head_sha"` - OriginalLine int64 `json:"original_line"` - CreatedAt time.Time `json:"created_at"` - ResolvedAt *time.Time `json:"resolved_at"` - Resolver string + Line int64 + Side string + OriginalLine int64 `json:"original_line"` + CreatedAt time.Time `json:"created_at"` + ResolvedAt *time.Time `json:"resolved_at"` + Resolver string } func (p *pullrequestReviewThread) Index() int64 { @@ -1221,28 +1217,24 @@ type pullrequestReviewComment struct { CommitID string `json:"commit_id"` OriginalCommitID string `json:"original_commit_id"` State int - //InReplyTo - Reactions []githubReaction - CreatedAt time.Time `json:"created_at"` + Reactions []githubReaction + CreatedAt time.Time `json:"created_at"` } func (r *GithubExportedDataRestorer) getReviewComments(thread *pullrequestReviewThread, comments []pullrequestReviewComment) []*base.ReviewComment { - var res []*base.ReviewComment + res := make([]*base.ReviewComment, 0, 10) for _, c := range comments { user := r.users[c.User] position := int(thread.Position) - //line := -int(thread.Line) if thread.Side == "right" { position = int(thread.OriginalPosition) - //line = int(thread.OriginalLine) } + // Line will be parse from diffhunk, so we ignore it here res = append(res, &base.ReviewComment{ - //InReplyTo: , - Content: c.Body, - TreePath: c.Path, - DiffHunk: c.DiffHunk, - Position: position, - //Line: line, line will be parse from diffhunk + Content: c.Body, + TreePath: c.Path, + DiffHunk: c.DiffHunk, + Position: position, CommitID: c.OriginalCommitID, PosterID: user.ID(), PosterName: user.Login, @@ -1256,7 +1248,7 @@ func (r *GithubExportedDataRestorer) getReviewComments(thread *pullrequestReview // GetReviews returns pull requests review func (r *GithubExportedDataRestorer) GetReviews(opts base.GetReviewOptions) ([]*base.Review, bool, error) { - var comments = make(map[string][]pullrequestReviewComment) + comments := make(map[string][]pullrequestReviewComment) if err := r.readJSONFiles("pull_request_review_comments", func() interface{} { return &[]pullrequestReviewComment{} }, func(content interface{}) error { @@ -1269,7 +1261,7 @@ func (r *GithubExportedDataRestorer) GetReviews(opts base.GetReviewOptions) ([]* return nil, true, err } - var reviews = make(map[string]*base.Review, 10) + reviews := make(map[string]*base.Review, 10) if err := r.readJSONFiles("pull_request_reviews", func() interface{} { return &[]pullrequestReview{} }, func(content interface{}) error { @@ -1320,7 +1312,7 @@ func (r *GithubExportedDataRestorer) GetReviews(opts base.GetReviewOptions) ([]* return nil, true, err } - var rs = make([]*base.Review, 0, len(reviews)) + rs := make([]*base.Review, 0, len(reviews)) for _, review := range reviews { rs = append(rs, review) } From 1241095aa70e7cc69e4026b7ec41f4b7d626a0c0 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 23 Jan 2022 01:13:31 +0800 Subject: [PATCH 29/54] Fix bug --- services/migrations/github_exported_data.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index 71f9d2b51a679..acc5784531a3f 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -1056,11 +1056,15 @@ func (r *GithubExportedDataRestorer) GetPullRequests(page, perPage int) ([]*base head.Ref = fmt.Sprintf("pr/%d", pr.Index()) } } + var milestone string + if pr.Milestone != "" { + milestone = r.milestones[pr.Milestone].Title + } pulls = append(pulls, &base.PullRequest{ Number: pr.Index(), Title: pr.Title, Content: pr.Body, - Milestone: pr.Milestone, + Milestone: milestone, State: state, PosterID: user.ID(), PosterName: user.Login, From d52c8f14ef4fe0c6f9636a7f301e61b61668d407 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 23 Jan 2022 13:23:00 +0800 Subject: [PATCH 30/54] Fix bug --- services/migrations/github_exported_data.go | 12 +- .../repo/issue/view_content/comments.tmpl | 138 ++++++++++++------ 2 files changed, 108 insertions(+), 42 deletions(-) diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index acc5784531a3f..5c229b9e7eb48 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -1300,7 +1300,17 @@ func (r *GithubExportedDataRestorer) GetReviews(opts base.GetReviewOptions) ([]* } rr, ok := reviews[review.PullRequestReview] if !ok { - return fmt.Errorf("cannot find review thread %s's review", review.PullRequestReview) + user := r.users[reviewComments[0].User] + rr = &base.Review{ + IssueIndex: review.Index(), + ReviewerID: user.ID(), + ReviewerName: user.Login, + ReviewerEmail: user.Email(), + CommitID: review.CommitID, + CreatedAt: review.CreatedAt, + State: base.ReviewStateCommented, + } + reviews[review.URL] = rr } rr.Comments = r.getReviewComments(&review, reviewComments) diff --git a/templates/repo/issue/view_content/comments.tmpl b/templates/repo/issue/view_content/comments.tmpl index 7dd20699453c1..b674af4fa4c14 100644 --- a/templates/repo/issue/view_content/comments.tmpl +++ b/templates/repo/issue/view_content/comments.tmpl @@ -96,17 +96,31 @@ {{else if eq .Type 1}}
{{svg "octicon-dot-fill"}} - - {{avatar .Poster}} - - - {{.Poster.GetDisplayName}} - {{if .Issue.IsPull }} - {{$.i18n.Tr "repo.pulls.reopened_at" .EventTag $createdStr | Safe}} - {{else}} - {{$.i18n.Tr "repo.issues.reopened_at" .EventTag $createdStr | Safe}} - {{end}} - + {{if .OriginalAuthor }} + + {{svg (MigrationIcon $.Repository.GetOriginalURLHostname)}} + {{ .OriginalAuthor }} + + + {{if .Issue.IsPull }} + {{$.i18n.Tr "repo.pulls.reopened_at" .EventTag $createdStr | Safe}} + {{else}} + {{$.i18n.Tr "repo.issues.reopened_at" .EventTag $createdStr | Safe}} + {{end}} + + {{else}} + + {{avatar .Poster}} + + + {{.Poster.GetDisplayName}} + {{if .Issue.IsPull }} + {{$.i18n.Tr "repo.pulls.reopened_at" .EventTag $createdStr | Safe}} + {{else}} + {{$.i18n.Tr "repo.issues.reopened_at" .EventTag $createdStr | Safe}} + {{end}} + + {{end}}
{{else if eq .Type 2}}
@@ -145,18 +159,38 @@ {{else if eq .Type 28}}
{{svg "octicon-git-merge"}} - - {{avatar .Poster}} - - - {{.Poster.GetDisplayName}} - {{$link := printf "%s/commit/%s" $.Repository.HTMLURL ($.Issue.PullRequest.MergedCommitID|PathEscape)}} - {{if eq $.Issue.PullRequest.Status 3}} - {{$.i18n.Tr "repo.issues.manually_pull_merged_at" ($link|Escape) (ShortSha $.Issue.PullRequest.MergedCommitID) ($.BaseTarget|Escape) $createdStr | Str2html}} - {{else}} - {{$.i18n.Tr "repo.issues.pull_merged_at" ($link|Escape) (ShortSha $.Issue.PullRequest.MergedCommitID) ($.BaseTarget|Escape) $createdStr | Str2html}} + {{if .OriginalAuthor }} + + {{svg (MigrationIcon $.Repository.GetOriginalURLHostname)}} + {{ .OriginalAuthor }} + + + {{$link := printf "%s/commit/%s" $.Repository.HTMLURL ($.Issue.PullRequest.MergedCommitID|PathEscape)}} + {{if eq $.Issue.PullRequest.Status 3}} + {{$.i18n.Tr "repo.issues.manually_pull_merged_at" ($link|Escape) (ShortSha $.Issue.PullRequest.MergedCommitID) ($.BaseTarget|Escape) $createdStr | Str2html}} + {{else}} + {{$.i18n.Tr "repo.issues.pull_merged_at" ($link|Escape) (ShortSha $.Issue.PullRequest.MergedCommitID) ($.BaseTarget|Escape) $createdStr | Str2html}} + {{end}} + + {{if $.Repository.OriginalURL}} + + ({{$.i18n.Tr "repo.migrated_from" ($.Repository.OriginalURL|Escape) ($.Repository.GetOriginalURLHostname|Escape) | Safe }}) + {{end}} - + {{else}} + + {{avatar .Poster}} + + + {{.Poster.GetDisplayName}} + {{$link := printf "%s/commit/%s" $.Repository.HTMLURL ($.Issue.PullRequest.MergedCommitID|PathEscape)}} + {{if eq $.Issue.PullRequest.Status 3}} + {{$.i18n.Tr "repo.issues.manually_pull_merged_at" ($link|Escape) (ShortSha $.Issue.PullRequest.MergedCommitID) ($.BaseTarget|Escape) $createdStr | Str2html}} + {{else}} + {{$.i18n.Tr "repo.issues.pull_merged_at" ($link|Escape) (ShortSha $.Issue.PullRequest.MergedCommitID) ($.BaseTarget|Escape) $createdStr | Str2html}} + {{end}} + + {{end}}
{{else if eq .Type 3 5 6}} {{ $refFrom:= "" }} @@ -207,19 +241,40 @@ {{if or .AddedLabels .RemovedLabels}}
{{svg "octicon-tag"}} - - {{avatar .Poster}} - - - {{.Poster.GetDisplayName}} - {{if and .AddedLabels (not .RemovedLabels)}} - {{$.i18n.TrN (len .AddedLabels) "repo.issues.add_label" "repo.issues.add_labels" (RenderLabels .AddedLabels) $createdStr | Safe}} - {{else if and (not .AddedLabels) .RemovedLabels}} - {{$.i18n.TrN (len .RemovedLabels) "repo.issues.remove_label" "repo.issues.remove_labels" (RenderLabels .RemovedLabels) $createdStr | Safe}} - {{else}} - {{$.i18n.Tr "repo.issues.add_remove_labels" (RenderLabels .AddedLabels) (RenderLabels .RemovedLabels) $createdStr | Safe}} + {{if .OriginalAuthor }} + + {{svg (MigrationIcon $.Repository.GetOriginalURLHostname)}} + {{ .OriginalAuthor }} + + + {{if and .AddedLabels (not .RemovedLabels)}} + {{$.i18n.TrN (len .AddedLabels) "repo.issues.add_label" "repo.issues.add_labels" (RenderLabels .AddedLabels) $createdStr | Safe}} + {{else if and (not .AddedLabels) .RemovedLabels}} + {{$.i18n.TrN (len .RemovedLabels) "repo.issues.remove_label" "repo.issues.remove_labels" (RenderLabels .RemovedLabels) $createdStr | Safe}} + {{else}} + {{$.i18n.Tr "repo.issues.add_remove_labels" (RenderLabels .AddedLabels) (RenderLabels .RemovedLabels) $createdStr | Safe}} + {{end}} + + {{if $.Repository.OriginalURL}} + + ({{$.i18n.Tr "repo.migrated_from" ($.Repository.OriginalURL|Escape) ($.Repository.GetOriginalURLHostname|Escape) | Safe }}) + {{end}} - + {{else}} + + {{avatar .Poster}} + + + {{.Poster.GetDisplayName}} + {{if and .AddedLabels (not .RemovedLabels)}} + {{$.i18n.TrN (len .AddedLabels) "repo.issues.add_label" "repo.issues.add_labels" (RenderLabels .AddedLabels) $createdStr | Safe}} + {{else if and (not .AddedLabels) .RemovedLabels}} + {{$.i18n.TrN (len .RemovedLabels) "repo.issues.remove_label" "repo.issues.remove_labels" (RenderLabels .RemovedLabels) $createdStr | Safe}} + {{else}} + {{$.i18n.Tr "repo.issues.add_remove_labels" (RenderLabels .AddedLabels) (RenderLabels .RemovedLabels) $createdStr | Safe}} + {{end}} + + {{end}}
{{end}} {{else if eq .Type 8}} @@ -441,21 +496,19 @@ {{else if eq .Type 22}}
- {{if .OriginalAuthor }} + {{if .OriginalAuthor}} {{else}} - - {{avatar .Poster}} - + + {{avatar .Poster}} + {{end}} {{svg (printf "octicon-%s" .Review.Type.Icon)}} - {{if .OriginalAuthor }} + {{if .OriginalAuthor}} {{svg (MigrationIcon $.Repository.GetOriginalURLHostname)}} {{ .OriginalAuthor }} - {{if $.Repository.OriginalURL}} - ({{$.i18n.Tr "repo.migrated_from" ($.Repository.OriginalURL|Escape) ($.Repository.GetOriginalURLHostname|Escape) | Safe }}){{end}} {{else}} {{.Poster.GetDisplayName}} {{end}} @@ -472,6 +525,9 @@ {{if .Review.Dismissed}}
{{$.i18n.Tr "repo.issues.review.dismissed_label"}}
{{end}} + {{if $.Repository.OriginalURL}} + ({{$.i18n.Tr "repo.migrated_from" ($.Repository.OriginalURL|Escape) ($.Repository.GetOriginalURLHostname|Escape) | Safe }}) + {{end}}
{{if .Content}} From 8ee93ede3b9634be76598b679842f0781dc4ea11 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 23 Jan 2022 15:42:40 +0800 Subject: [PATCH 31/54] Fix bug --- routers/web/repo/issue.go | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index 079ccbf6cf968..1f0398eb6ac3c 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -1348,11 +1348,13 @@ func ViewIssue(ctx *context.Context) { // check if dependencies can be created across repositories ctx.Data["AllowCrossRepositoryDependencies"] = setting.Service.AllowCrossRepositoryDependencies - if issue.ShowRole, err = roleDescriptor(ctx, repo, issue.Poster, issue); err != nil { - ctx.ServerError("roleDescriptor", err) - return + if issue.OriginalAuthor == "" { + if issue.ShowRole, err = roleDescriptor(ctx, repo, issue.Poster, issue); err != nil { + ctx.ServerError("roleDescriptor", err) + return + } + marked[issue.PosterID] = issue.ShowRole } - marked[issue.PosterID] = issue.ShowRole // Render comments and and fetch participants. participants[0] = issue.Poster @@ -1380,20 +1382,22 @@ func ViewIssue(ctx *context.Context) { ctx.ServerError("RenderString", err) return } - // Check tag. - role, ok = marked[comment.PosterID] - if ok { - comment.ShowRole = role - continue - } + if comment.OriginalAuthor == "" { + // Check tag. + role, ok = marked[comment.PosterID] + if ok { + comment.ShowRole = role + continue + } - comment.ShowRole, err = roleDescriptor(ctx, repo, comment.Poster, issue) - if err != nil { - ctx.ServerError("roleDescriptor", err) - return + comment.ShowRole, err = roleDescriptor(ctx, repo, comment.Poster, issue) + if err != nil { + ctx.ServerError("roleDescriptor", err) + return + } + marked[comment.PosterID] = comment.ShowRole + participants = addParticipant(comment.Poster, participants) } - marked[comment.PosterID] = comment.ShowRole - participants = addParticipant(comment.Poster, participants) } else if comment.Type == models.CommentTypeLabel { if err = comment.LoadLabel(); err != nil { ctx.ServerError("LoadLabel", err) @@ -1479,6 +1483,9 @@ func ViewIssue(ctx *context.Context) { for _, codeComments := range comment.Review.CodeComments { for _, lineComments := range codeComments { for _, c := range lineComments { + if c.OriginalAuthor != "" { + continue + } // Check tag. role, ok = marked[c.PosterID] if ok { From c201d755cda68f74c124aac64957d32c76540ef1 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 23 Jan 2022 15:54:22 +0800 Subject: [PATCH 32/54] Fix comment --- templates/repo/issue/view_content/comments.tmpl | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/templates/repo/issue/view_content/comments.tmpl b/templates/repo/issue/view_content/comments.tmpl index b674af4fa4c14..fc61494bb9b74 100644 --- a/templates/repo/issue/view_content/comments.tmpl +++ b/templates/repo/issue/view_content/comments.tmpl @@ -669,12 +669,18 @@ {{svg (MigrationIcon $.Repository.GetOriginalURLHostname)}} {{ .OriginalAuthor }} - {{if $.Repository.OriginalURL}} - ({{$.i18n.Tr "repo.migrated_from" ($.Repository.OriginalURL|Escape) ($.Repository.GetOriginalURLHostname|Escape) | Safe }}){{end}} + + {{$.i18n.Tr "repo.issues.commented_at" (.HashTag|Escape) $createdSubStr | Safe}} + {{if $.Repository.OriginalURL}} + ({{$.i18n.Tr "repo.migrated_from" ($.Repository.OriginalURL|Escape) ($.Repository.GetOriginalURLHostname|Escape) | Safe }}) + {{end}} + {{else}} - {{.Poster.GetDisplayName}} + + {{.Poster.GetDisplayName}} + {{$.i18n.Tr "repo.issues.commented_at" (.HashTag|Escape) $createdSubStr | Safe}} + {{end}} - {{$.i18n.Tr "repo.issues.commented_at" (.HashTag|Escape) $createdSubStr | Safe}}
From 473ca7cac60443054cdc919fd33978ef65ae2f95 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 25 Jan 2022 19:56:49 +0800 Subject: [PATCH 33/54] Some improvements --- modules/migration/downloader.go | 1 + modules/migration/null_downloader.go | 4 +++ services/migrations/dump.go | 10 ++++++-- services/migrations/gitea_uploader_test.go | 2 ++ services/migrations/github_exported_data.go | 27 ++++++++++++++++++++- services/migrations/migrate.go | 1 + 6 files changed, 42 insertions(+), 3 deletions(-) diff --git a/modules/migration/downloader.go b/modules/migration/downloader.go index 72896bd3dc164..67f18a655500e 100644 --- a/modules/migration/downloader.go +++ b/modules/migration/downloader.go @@ -40,6 +40,7 @@ type Downloader interface { GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) GetReviews(reviewable Reviewable) ([]*Review, bool, error) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) + CleanUp() } // DownloaderFactory defines an interface to match a downloader implementation and create a downloader diff --git a/modules/migration/null_downloader.go b/modules/migration/null_downloader.go index 6855de87d1a3c..23b21f0a18e02 100644 --- a/modules/migration/null_downloader.go +++ b/modules/migration/null_downloader.go @@ -92,3 +92,7 @@ func (n NullDownloader) SupportGetRepoComments() bool { func (n NullDownloader) SupportGetRepoReviews() bool { return false } + +// CleanUp clean the downloader temporary resources +func (n NullDownloader) CleanUp() { +} diff --git a/services/migrations/dump.go b/services/migrations/dump.go index a237d00d5ae06..bbbc4416b14f4 100644 --- a/services/migrations/dump.go +++ b/services/migrations/dump.go @@ -558,6 +558,8 @@ func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.Mi if err != nil { return err } + defer downloader.CleanUp() + uploader, err := NewRepositoryDumper(ctx, baseDir, ownerName, opts.RepoName, opts) if err != nil { return err @@ -612,11 +614,13 @@ func RestoreRepository(ctx context.Context, baseDir, ownerName, repoName string, if err != nil { return err } - uploader := NewGiteaLocalUploader(ctx, doer, ownerName, repoName) + downloader, err := NewRepositoryRestorer(ctx, baseDir, ownerName, repoName, validation) if err != nil { return err } + defer downloader.CleanUp() + opts, err := downloader.getRepoOptions() if err != nil { return err @@ -628,6 +632,7 @@ func RestoreRepository(ctx context.Context, baseDir, ownerName, repoName string, } updateOptionsUnits(&migrateOpts, units) + uploader := NewGiteaLocalUploader(ctx, doer, ownerName, repoName) if err = migrateRepository(downloader, uploader, migrateOpts, nil); err != nil { if err1 := uploader.Rollback(); err1 != nil { log.Error("rollback failed: %v", err1) @@ -643,17 +648,18 @@ func RestoreFromGithubExportedData(ctx context.Context, baseDir, ownerName, repo if err != nil { return err } - uploader := NewGiteaLocalUploader(ctx, doer, ownerName, repoName) downloader, err := NewGithubExportedDataRestorer(ctx, baseDir, ownerName, repoName) if err != nil { return err } + defer downloader.CleanUp() migrateOpts := base.MigrateOptions{ GitServiceType: structs.GithubService, } updateOptionsUnits(&migrateOpts, units) + uploader := NewGiteaLocalUploader(ctx, doer, ownerName, repoName) if err = migrateRepository(downloader, uploader, migrateOpts, nil); err != nil { if err1 := uploader.Rollback(); err1 != nil { log.Error("rollback failed: %v", err1) diff --git a/services/migrations/gitea_uploader_test.go b/services/migrations/gitea_uploader_test.go index bd7c6e0657ef3..9ba09f7b7f81b 100644 --- a/services/migrations/gitea_uploader_test.go +++ b/services/migrations/gitea_uploader_test.go @@ -46,6 +46,8 @@ func TestGiteaUploadRepo(t *testing.T) { uploader = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName) ) + defer downloader.CleanUp() + err := migrateRepository(downloader, uploader, base.MigrateOptions{ CloneAddr: "https://github.com/go-xorm/builder", RepoName: repoName, diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index 5c229b9e7eb48..872ee8a632e23 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -74,8 +74,15 @@ type githubUser struct { } func getID(s string) int64 { - u, _ := url.Parse(s) + u, err := url.Parse(s) + if err != nil { + log.Error("parse %s failed: %v", s, err) + return 0 + } fields := strings.Split(u.Path, "/") + if len(fields) == 0 { + return 0 + } i, _ := strconv.ParseInt(fields[len(fields)-1], 10, 64) return i } @@ -173,6 +180,9 @@ type githubLabel string func (l githubLabel) GetName() string { fields := strings.Split(string(l), "/labels/") + if len(fields) == 0 { + return "" + } s, err := url.PathUnescape(fields[len(fields)-1]) if err != nil { log.Error("url.PathUnescape %s failed: %v", fields[len(fields)-1], err) @@ -197,6 +207,8 @@ type GithubExportedDataRestorer struct { attachmentLoaded bool } +var _ base.Downloader = &GithubExportedDataRestorer{} + func decompressFile(targzFile, targetDir string) error { f, err := os.Open(targzFile) if err != nil { @@ -273,6 +285,19 @@ func NewGithubExportedDataRestorer(ctx context.Context, githubDataFilePath, owne return restorer, nil } +// CleanUp clean the downloader temporary resources +func (r *GithubExportedDataRestorer) CleanUp() { + if r.tmpDir != "" { + _ = os.RemoveAll(r.tmpDir) + } +} + +// replaceComment replace #id to new form +// i.e. https://github.com/userstyles-world/userstyles.world/commit/b70d545a1cbb5c92ca20f442f59de5d955600408 -> +func (r *GithubExportedDataRestorer) replaceComment(content string) string { + return "" +} + // SupportGetRepoComments return true if it can get all comments once func (r *GithubExportedDataRestorer) SupportGetRepoComments() bool { return true diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index ed0dced7160a5..f97856231e814 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -121,6 +121,7 @@ func MigrateRepository(ctx context.Context, doer *user_model.User, ownerName str if err != nil { return nil, err } + defer downloader.CleanUp() uploader := NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) uploader.gitServiceType = opts.GitServiceType From 4a7505ee3b17b321368af809d8efe7f339b3de46 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 25 Jan 2022 20:30:58 +0800 Subject: [PATCH 34/54] rename getID to parseGitHubResID --- services/migrations/github_exported_data.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index 872ee8a632e23..c00059332abd3 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -73,7 +73,7 @@ type githubUser struct { CreatedAt time.Time `json:"created_at"` } -func getID(s string) int64 { +func parseGitHubResID(s string) int64 { u, err := url.Parse(s) if err != nil { log.Error("parse %s failed: %v", s, err) @@ -88,7 +88,7 @@ func getID(s string) int64 { } func (g *githubUser) ID() int64 { - return getID(g.AvatarURL) + return parseGitHubResID(g.AvatarURL) } func (g *githubUser) Email() string { @@ -127,7 +127,7 @@ type githubAttachment struct { } func (g *githubAttachment) GetUserID() int64 { - return getID(g.User) + return parseGitHubResID(g.User) } func (g *githubAttachment) IsIssue() bool { @@ -136,9 +136,9 @@ func (g *githubAttachment) IsIssue() bool { func (g *githubAttachment) IssueID() int64 { if g.IsIssue() { - return getID(g.Issue) + return parseGitHubResID(g.Issue) } - return getID(g.IssueComment) + return parseGitHubResID(g.IssueComment) } func (r *GithubExportedDataRestorer) convertAttachments(ls []githubAttachment) []*base.Asset { From 549230c4edf4fe080f26ea419324de272e5cf1fb Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 26 Jan 2022 21:01:35 +0800 Subject: [PATCH 35/54] Some improvements --- services/migrations/github_exported_data.go | 136 ++++++++++++------ .../repo/issue/view_content/comments.tmpl | 15 ++ 2 files changed, 109 insertions(+), 42 deletions(-) diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index c00059332abd3..ab1fbc73a80d3 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -15,6 +15,7 @@ import ( "net/url" "os" "path/filepath" + "regexp" "strconv" "strings" "time" @@ -73,6 +74,19 @@ type githubUser struct { CreatedAt time.Time `json:"created_at"` } +func getURLLastField(s string) string { + u, err := url.Parse(s) + if err != nil { + log.Error("parse %s failed: %v", s, err) + return "" + } + fields := strings.Split(u.Path, "/") + if len(fields) == 0 { + return "" + } + return fields[len(fields)-1] +} + func parseGitHubResID(s string) int64 { u, err := url.Parse(s) if err != nil { @@ -199,6 +213,8 @@ type GithubExportedDataRestorer struct { githubDataFilePath string repoOwner string repoName string + baseURL string + regMatchIssue *regexp.Regexp labels []*base.Label users map[string]githubUser issueAttachments map[string][]githubAttachment @@ -293,9 +309,15 @@ func (r *GithubExportedDataRestorer) CleanUp() { } // replaceComment replace #id to new form -// i.e. https://github.com/userstyles-world/userstyles.world/commit/b70d545a1cbb5c92ca20f442f59de5d955600408 -> -func (r *GithubExportedDataRestorer) replaceComment(content string) string { - return "" +// i.e. +// 1) https://github.com/userstyles-world/userstyles.world/commit/b70d545a1cbb5c92ca20f442f59de5d955600408 -> +// 2) https://github.com/go-gitea/gitea/issue/1 -> #1 +// 3) https://github.com/go-gitea/gitea/pull/2 -> #2 +func (r *GithubExportedDataRestorer) replaceGithubLinks(content string) string { + c := strings.ReplaceAll(content, r.baseURL+"/issue/", "#") + c = strings.ReplaceAll(c, r.baseURL+"/pull/", "#") + c = strings.ReplaceAll(c, r.baseURL+"/commit/", "") + return c } // SupportGetRepoComments return true if it can get all comments once @@ -381,6 +403,11 @@ func (r *GithubExportedDataRestorer) GetRepoInfo() (*base.Repository, error) { Description: label.Description, }) } + r.baseURL = opts.URL + r.regMatchIssue, err = regexp.Compile(r.baseURL + "/[issue|pull]/([0-9]+).*") + if err != nil { + return nil, err + } return &base.Repository{ Owner: r.repoOwner, @@ -548,6 +575,14 @@ type githubRelease struct { PublishedAt time.Time `json:"published_at"` } +func (r *GithubExportedDataRestorer) getUserInfo(u string) (int64, string, string) { + user, ok := r.users[u] + if !ok { + return 0, getURLLastField(u), "" + } + return user.ID(), user.Login, user.Email() +} + // GetReleases returns releases func (r *GithubExportedDataRestorer) GetReleases() ([]*base.Release, error) { releases := make([]*base.Release, 0, 30) @@ -556,7 +591,7 @@ func (r *GithubExportedDataRestorer) GetReleases() ([]*base.Release, error) { }, func(content interface{}) error { rss := content.(*[]githubRelease) for _, rel := range *rss { - user := r.users[rel.User] + id, login, email := r.getUserInfo(rel.User) releases = append(releases, &base.Release{ TagName: rel.TagName, TargetCommitish: rel.TargetCommitish, @@ -564,9 +599,9 @@ func (r *GithubExportedDataRestorer) GetReleases() ([]*base.Release, error) { Body: rel.Body, Draft: rel.State == "draft", Prerelease: rel.Prerelease, - PublisherID: user.ID(), - PublisherName: user.Login, - PublisherEmail: user.Email(), + PublisherID: id, + PublisherName: login, + PublisherEmail: email, Assets: r.convertAttachments(rel.ReleaseAssets), Created: rel.CreatedAt, Published: rel.PublishedAt, @@ -653,13 +688,13 @@ func (r *GithubExportedDataRestorer) getReactions(ls []githubReaction) []*base.R content = "hooray" } - user, ok := r.users[l.User] - if !ok { - log.Warn("Cannot get user %s information, ignored", l.User) + id, login, _ := r.getUserInfo(l.User) + if id == 0 { + log.Warn("Cannot get user %s information, userid will be set 0", l.User) } else { res = append(res, &base.Reaction{ - UserID: user.ID(), - UserName: user.Login, + UserID: id, + UserName: login, Content: content, }) } @@ -679,7 +714,7 @@ func (r *GithubExportedDataRestorer) GetIssues(page, perPage int) ([]*base.Issue }, func(content interface{}) error { rss := content.(*[]githubIssue) for _, issue := range *rss { - user := r.users[issue.User] + id, login, email := r.getUserInfo(issue.User) var milestone string if issue.Milestone != "" { milestone = r.milestones[issue.Milestone].Title @@ -696,9 +731,9 @@ func (r *GithubExportedDataRestorer) GetIssues(page, perPage int) ([]*base.Issue Number: issue.Index(), Title: issue.Title, Content: issue.Body, - PosterID: user.ID(), - PosterName: user.Login, - PosterEmail: user.Email(), + PosterID: id, + PosterName: login, + PosterEmail: email, Labels: r.getLabels(issue.Labels), Reactions: r.getReactions(issue.Reactions), Assets: r.convertAttachments(r.issueAttachments[issue.URL]), @@ -801,6 +836,8 @@ func (g *githubIssueEvent) CommentContent() map[string]interface{} { return map[string]interface{}{} case "head_ref_force_pushed": return map[string]interface{}{} + case "moved_columns_in_project": + return map[string]interface{}{} case "referenced": tp := "commit_ref" if g.Issue != "" { @@ -820,6 +857,8 @@ func (g *githubIssueEvent) CommentContent() map[string]interface{} { return map[string]interface{}{} case "head_ref_deleted": return map[string]interface{}{} + case "head_ref_restored": + return map[string]interface{}{} case "milestoned": return map[string]interface{}{ "type": "add", @@ -843,6 +882,8 @@ func (g *githubIssueEvent) CommentContent() map[string]interface{} { "OldTitle": g.TitleWas, "NewTitle": g.TitleIs, } + case "ready_for_review": + return map[string]interface{}{} case "reopened": return map[string]interface{}{} case "unlabeled": @@ -858,6 +899,8 @@ func (g *githubIssueEvent) CommentContent() map[string]interface{} { "Actor": g.Actor, "Subject": g.Subject, } + case "added_to_project": + return map[string]interface{}{} default: return map[string]interface{}{} } @@ -872,6 +915,12 @@ func (g *githubIssueEvent) CommentStr() string { return "pull_push" case "referenced": return "commit_ref" + case "moved_columns_in_project": + return "unknown" + case "convert_to_draft": + return "unknown" + case "ready_for_review": + return "unknown" case "merged": return "merge_pull" case "mentioned": @@ -880,6 +929,10 @@ func (g *githubIssueEvent) CommentStr() string { return "unknown" // ignore case "head_ref_deleted": return "delete_branch" + case "head_ref_restored": + return "unknown" + case "added_to_project": + return "unknown" case "milestoned": return "milestone" case "demilestoned": @@ -914,7 +967,7 @@ func (r *GithubExportedDataRestorer) getIssueEvents() ([]*base.Comment, error) { }, func(content interface{}) error { rss := content.(*[]githubIssueEvent) for _, c := range *rss { - u := r.users[c.Actor] + id, login, email := r.getUserInfo(c.Actor) v := c.CommentContent() bs, err := json.Marshal(v) if err != nil { @@ -924,9 +977,9 @@ func (r *GithubExportedDataRestorer) getIssueEvents() ([]*base.Comment, error) { comments = append(comments, &base.Comment{ Type: c.CommentStr(), IssueIndex: c.GetIssueIndex(), - PosterID: u.ID(), - PosterName: u.Login, - PosterEmail: u.Email(), + PosterID: id, + PosterName: login, + PosterEmail: email, Created: c.CreatedAt, Updated: c.CreatedAt, // FIXME: Content: string(bs), @@ -947,16 +1000,15 @@ func (r *GithubExportedDataRestorer) GetComments(opts base.GetCommentOptions) ([ }, func(content interface{}) error { rss := content.(*[]githubComment) for _, c := range *rss { - u := r.users[c.User] - + id, login, email := r.getUserInfo(c.User) comments = append(comments, &base.Comment{ IssueIndex: c.GetIssueIndex(), - PosterID: u.ID(), - PosterName: u.Login, - PosterEmail: u.Email(), + PosterID: id, + PosterName: login, + PosterEmail: email, Created: c.CreatedAt, Updated: c.CreatedAt, // FIXME: - Content: c.Body, + Content: r.replaceGithubLinks(c.Body), Reactions: r.getReactions(c.Reactions), }) } @@ -1060,7 +1112,7 @@ func (r *GithubExportedDataRestorer) GetPullRequests(page, perPage int) ([]*base }, func(content interface{}) error { prs := content.(*[]githubPullRequest) for _, pr := range *prs { - user := r.users[pr.User] + id, login, email := r.getUserInfo(pr.User) state := "open" if pr.MergedAt != nil || pr.ClosedAt != nil { state = "closed" @@ -1091,9 +1143,9 @@ func (r *GithubExportedDataRestorer) GetPullRequests(page, perPage int) ([]*base Content: pr.Body, Milestone: milestone, State: state, - PosterID: user.ID(), - PosterName: user.Login, - PosterEmail: user.Email(), + PosterID: id, + PosterName: login, + PosterEmail: email, Context: base.BasicIssueContext(pr.Index()), Reactions: r.getReactions(pr.Reactions), Created: pr.CreatedAt, @@ -1253,7 +1305,7 @@ type pullrequestReviewComment struct { func (r *GithubExportedDataRestorer) getReviewComments(thread *pullrequestReviewThread, comments []pullrequestReviewComment) []*base.ReviewComment { res := make([]*base.ReviewComment, 0, 10) for _, c := range comments { - user := r.users[c.User] + id, login, email := r.getUserInfo(c.User) position := int(thread.Position) if thread.Side == "right" { position = int(thread.OriginalPosition) @@ -1265,9 +1317,9 @@ func (r *GithubExportedDataRestorer) getReviewComments(thread *pullrequestReview DiffHunk: c.DiffHunk, Position: position, CommitID: c.OriginalCommitID, - PosterID: user.ID(), - PosterName: user.Login, - PosterEmail: user.Email(), + PosterID: id, + PosterName: login, + PosterEmail: email, Reactions: r.getReactions(c.Reactions), CreatedAt: c.CreatedAt, }) @@ -1296,12 +1348,12 @@ func (r *GithubExportedDataRestorer) GetReviews(opts base.GetReviewOptions) ([]* }, func(content interface{}) error { prReviews := content.(*[]pullrequestReview) for _, review := range *prReviews { - user := r.users[review.User] + id, login, email := r.getUserInfo(review.User) baseReview := &base.Review{ IssueIndex: review.Index(), - ReviewerID: user.ID(), - ReviewerName: user.Login, - ReviewerEmail: user.Email(), + ReviewerID: id, + ReviewerName: login, + ReviewerEmail: email, CommitID: review.HeadSha, Content: review.Body, CreatedAt: review.CreatedAt, @@ -1325,12 +1377,12 @@ func (r *GithubExportedDataRestorer) GetReviews(opts base.GetReviewOptions) ([]* } rr, ok := reviews[review.PullRequestReview] if !ok { - user := r.users[reviewComments[0].User] + id, login, email := r.getUserInfo(reviewComments[0].User) rr = &base.Review{ IssueIndex: review.Index(), - ReviewerID: user.ID(), - ReviewerName: user.Login, - ReviewerEmail: user.Email(), + ReviewerID: id, + ReviewerName: login, + ReviewerEmail: email, CommitID: review.CommitID, CreatedAt: review.CreatedAt, State: base.ReviewStateCommented, diff --git a/templates/repo/issue/view_content/comments.tmpl b/templates/repo/issue/view_content/comments.tmpl index fc61494bb9b74..40185e97422c3 100644 --- a/templates/repo/issue/view_content/comments.tmpl +++ b/templates/repo/issue/view_content/comments.tmpl @@ -337,6 +337,20 @@ {{else if eq .Type 10}}
{{svg "octicon-pencil"}} + {{if .OriginalAuthor }} + + {{svg (MigrationIcon $.Repository.GetOriginalURLHostname)}} + {{ .OriginalAuthor }} + + + {{$.i18n.Tr "repo.issues.change_title_at" (.OldTitle|RenderEmoji) (.NewTitle|RenderEmoji) $createdStr | Safe}} + + {{if $.Repository.OriginalURL}} + + ({{$.i18n.Tr "repo.migrated_from" ($.Repository.OriginalURL|Escape) ($.Repository.GetOriginalURLHostname|Escape) | Safe }}) + + {{end}} + {{else}} {{avatar .Poster}} @@ -344,6 +358,7 @@ {{.Poster.GetDisplayName}} {{$.i18n.Tr "repo.issues.change_title_at" (.OldTitle|RenderEmoji) (.NewTitle|RenderEmoji) $createdStr | Safe}} + {{end}}
{{else if eq .Type 11}}
From 1c9945545581a105568b10c87cee2808cff7cf62 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 27 Jan 2022 00:51:53 +0800 Subject: [PATCH 36/54] Use regex to replace commit hash and issue/pull --- services/migrations/github_exported_data.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index ab1fbc73a80d3..2035158dec0a8 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -215,6 +215,7 @@ type GithubExportedDataRestorer struct { repoName string baseURL string regMatchIssue *regexp.Regexp + regMatchCommit *regexp.Regexp labels []*base.Label users map[string]githubUser issueAttachments map[string][]githubAttachment @@ -310,13 +311,12 @@ func (r *GithubExportedDataRestorer) CleanUp() { // replaceComment replace #id to new form // i.e. -// 1) https://github.com/userstyles-world/userstyles.world/commit/b70d545a1cbb5c92ca20f442f59de5d955600408 -> +// 1) https://github.com/userstyles-world/userstyles.world/commit/b70d545a1cbb5c92ca20f442f59de5d955600408 -> b70d545a1cbb5c92ca20f442f59de5d955600408 // 2) https://github.com/go-gitea/gitea/issue/1 -> #1 // 3) https://github.com/go-gitea/gitea/pull/2 -> #2 func (r *GithubExportedDataRestorer) replaceGithubLinks(content string) string { - c := strings.ReplaceAll(content, r.baseURL+"/issue/", "#") - c = strings.ReplaceAll(c, r.baseURL+"/pull/", "#") - c = strings.ReplaceAll(c, r.baseURL+"/commit/", "") + c := r.regMatchIssue.ReplaceAllString(content, "#$2") + c = r.regMatchCommit.ReplaceAllString(c, "$1") return c } @@ -404,7 +404,11 @@ func (r *GithubExportedDataRestorer) GetRepoInfo() (*base.Repository, error) { }) } r.baseURL = opts.URL - r.regMatchIssue, err = regexp.Compile(r.baseURL + "/[issue|pull]/([0-9]+).*") + r.regMatchIssue, err = regexp.Compile(r.baseURL + "/(issue|pull)/([0-9]+)") + if err != nil { + return nil, err + } + r.regMatchCommit, err = regexp.Compile(r.baseURL + "/commit/([a-z0-9]{7, 40})") if err != nil { return nil, err } From 8e7d9a3289afeb1bc663b58ea063ca33d95edecf Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 3 Feb 2022 11:41:32 +0800 Subject: [PATCH 37/54] upload assets and replace the links in issue/comment/pullrequest --- models/migrate.go | 20 +++ modules/migration/asset.go | 25 ++++ modules/migration/comment.go | 1 + modules/migration/pullrequest.go | 1 + modules/migration/release.go | 15 --- services/migrations/gitea_uploader.go | 132 +++++++++++++++----- services/migrations/github_exported_data.go | 17 ++- 7 files changed, 162 insertions(+), 49 deletions(-) create mode 100644 modules/migration/asset.go diff --git a/models/migrate.go b/models/migrate.go index 7b12bc9c935aa..b7d1802decada 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -91,6 +91,16 @@ func insertIssue(ctx context.Context, issue *Issue) error { } } + for _, attach := range issue.Attachments { + attach.IssueID = issue.ID + } + + if len(issue.Attachments) > 0 { + if _, err := sess.Insert(issue.Attachments); err != nil { + return err + } + } + return nil } @@ -124,6 +134,16 @@ func InsertIssueComments(comments []*Comment) error { return err } } + + for _, attach := range comment.Attachments { + attach.IssueID = comment.IssueID + attach.CommentID = comment.ID + } + if len(comment.Attachments) > 0 { + if err := db.Insert(ctx, comment.Attachments); err != nil { + return err + } + } } for issueID := range issueIDs { diff --git a/modules/migration/asset.go b/modules/migration/asset.go new file mode 100644 index 0000000000000..464aa4a0dff34 --- /dev/null +++ b/modules/migration/asset.go @@ -0,0 +1,25 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migration + +import ( + "io" + "time" +) + +// Asset represents an asset for issue, comment, release +type Asset struct { + ID int64 + Name string + ContentType *string `yaml:"content_type"` + Size *int + DownloadCount *int `yaml:"download_count"` + Created time.Time + Updated time.Time + DownloadURL *string `yaml:"download_url"` + OriginalURL string + // if DownloadURL is nil, the function should be invoked + DownloadFunc func() (io.ReadCloser, error) `yaml:"-"` +} diff --git a/modules/migration/comment.go b/modules/migration/comment.go index 48ced0e221172..a342107148e53 100644 --- a/modules/migration/comment.go +++ b/modules/migration/comment.go @@ -63,6 +63,7 @@ type Comment struct { Updated time.Time Content string Reactions []*Reaction + Assets []*Asset } // GetExternalName ExternalUserMigrated interface diff --git a/modules/migration/pullrequest.go b/modules/migration/pullrequest.go index eaa0dd45e2c4b..248988baeb66b 100644 --- a/modules/migration/pullrequest.go +++ b/modules/migration/pullrequest.go @@ -37,6 +37,7 @@ type PullRequest struct { Reactions []*Reaction ForeignIndex int64 Context DownloaderContext `yaml:"-"` + Assets []*Asset } func (p *PullRequest) GetLocalIndex() int64 { return p.Number } diff --git a/modules/migration/release.go b/modules/migration/release.go index 030e33af130f4..7a38b7e0e9ac7 100644 --- a/modules/migration/release.go +++ b/modules/migration/release.go @@ -5,24 +5,9 @@ package migration import ( - "io" "time" ) -// Asset represents an asset for issue, comment, release -type Asset struct { - ID int64 - Name string - ContentType *string `yaml:"content_type"` - Size *int - DownloadCount *int `yaml:"download_count"` - Created time.Time - Updated time.Time - DownloadURL *string `yaml:"download_url"` - // if DownloadURL is nil, the function should be invoked - DownloadFunc func() (io.ReadCloser, error) `yaml:"-"` -} - // Release represents a release type Release struct { TagName string `yaml:"tag_name"` diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 623f5aaa7c590..d6571f410189e 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -237,6 +237,40 @@ func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error { return nil } +func (g *GiteaLocalUploader) uploadAttachment(asset *base.Asset) (*repo_model.Attachment, error) { + attach := repo_model.Attachment{ + UUID: gouuid.New().String(), + Name: asset.Name, + DownloadCount: int64(*asset.DownloadCount), + Size: int64(*asset.Size), + CreatedUnix: timeutil.TimeStamp(asset.Created.Unix()), + } + + // asset.DownloadURL maybe a local file + var rc io.ReadCloser + var err error + if asset.DownloadFunc != nil { + rc, err = asset.DownloadFunc() + if err != nil { + return nil, err + } + } else if asset.DownloadURL != nil { + rc, err = uri.Open(*asset.DownloadURL) + if err != nil { + return nil, err + } + } + if rc == nil { + return nil, nil + } + defer rc.Close() + _, err = storage.Attachments.Save(attach.RelativePath(), rc, int64(*asset.Size)) + if err != nil { + return nil, err + } + return &attach, nil +} + // CreateReleases creates releases func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { rels := make([]*models.Release, 0, len(releases)) @@ -289,42 +323,13 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { asset.Created = release.Created } } - attach := repo_model.Attachment{ - UUID: gouuid.New().String(), - Name: asset.Name, - DownloadCount: int64(*asset.DownloadCount), - Size: int64(*asset.Size), - CreatedUnix: timeutil.TimeStamp(asset.Created.Unix()), - } - - // download attachment - err := func() error { - // asset.DownloadURL maybe a local file - var rc io.ReadCloser - var err error - if asset.DownloadFunc != nil { - rc, err = asset.DownloadFunc() - if err != nil { - return err - } - } else if asset.DownloadURL != nil { - rc, err = uri.Open(*asset.DownloadURL) - if err != nil { - return err - } - } - if rc == nil { - return nil - } - _, err = storage.Attachments.Save(attach.RelativePath(), rc, int64(*asset.Size)) - rc.Close() - return err - }() + attach, err := g.uploadAttachment(asset) if err != nil { return err } - - rel.Attachments = append(rel.Attachments, &attach) + if attach != nil { + rel.Attachments = append(rel.Attachments, attach) + } } rels = append(rels, &rel) @@ -406,6 +411,27 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { } is.Reactions = append(is.Reactions, &res) } + + for _, asset := range issue.Assets { + if asset.Created.IsZero() { + if !asset.Updated.IsZero() { + asset.Created = asset.Updated + } else { + asset.Created = issue.Updated + } + } + attach, err := g.uploadAttachment(asset) + if err != nil { + return err + } + if attach != nil { + is.Attachments = append(is.Attachments, attach) + if asset.OriginalURL != "" { + is.Content = strings.ReplaceAll(is.Content, asset.OriginalURL, fmt.Sprintf("/attachments/%s", attach.UUID)) + } + } + } + iss = append(iss, &is) } @@ -544,6 +570,26 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { cm.Reactions = append(cm.Reactions, &res) } + for _, asset := range comment.Assets { + if asset.Created.IsZero() { + if !asset.Updated.IsZero() { + asset.Created = asset.Updated + } else { + asset.Created = comment.Updated + } + } + attach, err := g.uploadAttachment(asset) + if err != nil { + return err + } + if attach != nil { + cm.Attachments = append(cm.Attachments, attach) + if asset.OriginalURL != "" { + cm.Content = strings.ReplaceAll(cm.Content, asset.OriginalURL, fmt.Sprintf("/attachments/%s", attach.UUID)) + } + } + } + cms = append(cms, &cm) } @@ -566,6 +612,26 @@ func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error return err } + for _, asset := range pr.Assets { + if asset.Created.IsZero() { + if !asset.Updated.IsZero() { + asset.Created = asset.Updated + } else { + asset.Created = pr.Updated + } + } + attach, err := g.uploadAttachment(asset) + if err != nil { + return err + } + if attach != nil { + gpr.Issue.Attachments = append(gpr.Issue.Attachments, attach) + if asset.OriginalURL != "" { + gpr.Issue.Content = strings.ReplaceAll(gpr.Issue.Content, asset.OriginalURL, fmt.Sprintf("/attachments/%s", attach.UUID)) + } + } + } + gprs = append(gprs, gpr) } if err := models.InsertPullRequests(gprs...); err != nil { diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index 2035158dec0a8..1bcc9fb70a676 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -131,6 +131,7 @@ func (g *githubUser) Email() string { }, */ type githubAttachment struct { + URL string `json:"url"` Issue string IssueComment string `json:"issue_comment"` User string @@ -148,6 +149,10 @@ func (g *githubAttachment) IsIssue() bool { return len(g.Issue) > 0 } +func (g *githubAttachment) IsComment() bool { + return len(g.IssueComment) > 0 +} + func (g *githubAttachment) IssueID() int64 { if g.IsIssue() { return parseGitHubResID(g.Issue) @@ -170,7 +175,9 @@ func (r *GithubExportedDataRestorer) convertAttachments(ls []githubAttachment) [ ContentType: &l.AssetContentType, Size: &size, Created: l.CreatedAt, + Updated: l.CreatedAt, DownloadURL: &assetURL, + OriginalURL: l.URL, }) } return res @@ -309,11 +316,16 @@ func (r *GithubExportedDataRestorer) CleanUp() { } } -// replaceComment replace #id to new form +var ( + regMatchAttachment = regexp.MustCompile("https://user-images.githubusercontent.com/([0-9]+)/(.*)") +) + +// replaceGithubLinks replace #id to new form // i.e. // 1) https://github.com/userstyles-world/userstyles.world/commit/b70d545a1cbb5c92ca20f442f59de5d955600408 -> b70d545a1cbb5c92ca20f442f59de5d955600408 // 2) https://github.com/go-gitea/gitea/issue/1 -> #1 // 3) https://github.com/go-gitea/gitea/pull/2 -> #2 +// 4) https://user-images.githubusercontent.com/1824502/146297011-8c01ea20-276e-421b-98a9-e39fe76ac046.png -> attachment link func (r *GithubExportedDataRestorer) replaceGithubLinks(content string) string { c := r.regMatchIssue.ReplaceAllString(content, "#$2") c = r.regMatchCommit.ReplaceAllString(c, "$1") @@ -759,6 +771,7 @@ func (r *GithubExportedDataRestorer) GetIssues(page, perPage int) ([]*base.Issue } type githubComment struct { + URL string `json:"url"` Issue string PullRequest string `json:"pull_request"` User string @@ -1014,6 +1027,7 @@ func (r *GithubExportedDataRestorer) GetComments(opts base.GetCommentOptions) ([ Updated: c.CreatedAt, // FIXME: Content: r.replaceGithubLinks(c.Body), Reactions: r.getReactions(c.Reactions), + Assets: r.convertAttachments(r.issueAttachments[c.URL]), }) } return nil @@ -1152,6 +1166,7 @@ func (r *GithubExportedDataRestorer) GetPullRequests(page, perPage int) ([]*base PosterEmail: email, Context: base.BasicIssueContext(pr.Index()), Reactions: r.getReactions(pr.Reactions), + Assets: r.convertAttachments(r.issueAttachments[pr.URL]), Created: pr.CreatedAt, Closed: pr.ClosedAt, Labels: r.getLabels(pr.Labels), From 20f30f1c651c05794c5fa12b594c5083338f3b66 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 3 Feb 2022 11:49:38 +0800 Subject: [PATCH 38/54] Fix lint --- services/migrations/gitlab_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/migrations/gitlab_test.go b/services/migrations/gitlab_test.go index d004056fe8788..03ce361a8d476 100644 --- a/services/migrations/gitlab_test.go +++ b/services/migrations/gitlab_test.go @@ -465,7 +465,7 @@ func TestGitlabGetReviews(t *testing.T) { mux.HandleFunc(fmt.Sprintf("/api/v4/projects/%d/merge_requests/%d/approvals", testCase.repoID, testCase.prID), mock) id := int64(testCase.prID) - rvs, err := downloader.GetReviews(&base.Issue{Number: id, ForeignIndex: id}) + rvs, _, err := downloader.GetReviews(&base.Issue{Number: id, ForeignIndex: id}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{&review}, rvs) } From cc964e729f258971672e5c5822d2965a715be958 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 3 Feb 2022 15:31:36 +0800 Subject: [PATCH 39/54] fix upload attachment --- services/migrations/gitea_uploader.go | 12 ++++++++++-- services/migrations/github_exported_data.go | 5 +++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index d6571f410189e..06473fc8c5acf 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -238,11 +238,19 @@ func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error { } func (g *GiteaLocalUploader) uploadAttachment(asset *base.Asset) (*repo_model.Attachment, error) { + var downloadCnt, size int64 + if asset.DownloadCount != nil { + downloadCnt = int64(*asset.DownloadCount) + } + if asset.Size != nil { + size = int64(*asset.Size) + } + attach := repo_model.Attachment{ UUID: gouuid.New().String(), Name: asset.Name, - DownloadCount: int64(*asset.DownloadCount), - Size: int64(*asset.Size), + DownloadCount: downloadCnt, + Size: size, CreatedUnix: timeutil.TimeStamp(asset.Created.Unix()), } diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index 1bcc9fb70a676..9b353ea17f021 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -163,7 +163,8 @@ func (g *githubAttachment) IssueID() int64 { func (r *GithubExportedDataRestorer) convertAttachments(ls []githubAttachment) []*base.Asset { res := make([]*base.Asset, 0, len(ls)) for _, l := range ls { - fPath := strings.TrimPrefix(l.AssetURL, "tarball://root") + fPath := strings.TrimPrefix(l.AssetURL, "tarball://root/") + fPath = filepath.Join(r.tmpDir, fPath) info, err := os.Stat(fPath) var size int if err == nil { @@ -216,7 +217,7 @@ func (l githubLabel) GetName() string { type GithubExportedDataRestorer struct { base.NullDownloader ctx context.Context - tmpDir string + tmpDir string // work directory, tar files will be decompress into there githubDataFilePath string repoOwner string repoName string From 2e516b5327435372ad03914865a4d5a27024bb06 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 3 Feb 2022 15:33:36 +0800 Subject: [PATCH 40/54] Fix lint --- services/migrations/github_exported_data.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index 9b353ea17f021..63f7c6147f100 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -317,16 +317,11 @@ func (r *GithubExportedDataRestorer) CleanUp() { } } -var ( - regMatchAttachment = regexp.MustCompile("https://user-images.githubusercontent.com/([0-9]+)/(.*)") -) - // replaceGithubLinks replace #id to new form // i.e. // 1) https://github.com/userstyles-world/userstyles.world/commit/b70d545a1cbb5c92ca20f442f59de5d955600408 -> b70d545a1cbb5c92ca20f442f59de5d955600408 // 2) https://github.com/go-gitea/gitea/issue/1 -> #1 // 3) https://github.com/go-gitea/gitea/pull/2 -> #2 -// 4) https://user-images.githubusercontent.com/1824502/146297011-8c01ea20-276e-421b-98a9-e39fe76ac046.png -> attachment link func (r *GithubExportedDataRestorer) replaceGithubLinks(content string) string { c := r.regMatchIssue.ReplaceAllString(content, "#$2") c = r.regMatchCommit.ReplaceAllString(c, "$1") From cc2d527043894fdba10cc94556a3c858eaac710b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 3 Feb 2022 20:49:10 +0800 Subject: [PATCH 41/54] Fix bug --- services/migrations/github_exported_data.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index 63f7c6147f100..e2e9d9927ac92 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -1023,7 +1023,7 @@ func (r *GithubExportedDataRestorer) GetComments(opts base.GetCommentOptions) ([ Updated: c.CreatedAt, // FIXME: Content: r.replaceGithubLinks(c.Body), Reactions: r.getReactions(c.Reactions), - Assets: r.convertAttachments(r.issueAttachments[c.URL]), + Assets: r.convertAttachments(r.commentAttachments[c.URL]), }) } return nil From d76ff4fb222dbe5b4cd56aecf685dd05bf9db66b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 3 Feb 2022 21:29:23 +0800 Subject: [PATCH 42/54] Fix bug --- services/migrations/github_exported_data.go | 30 +++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index e2e9d9927ac92..ae40276f4c79c 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -851,6 +851,24 @@ func (g *githubIssueEvent) CommentContent() map[string]interface{} { return map[string]interface{}{} case "moved_columns_in_project": return map[string]interface{}{} + case "connected": + fields := strings.Split(g.Subject, "/") + if len(fields) > 2 { + if fields[len(fields)-2] == "issue" { + return map[string]interface{}{ + "type": "issue_ref", + "subject": fields[len(fields)-1], + } + } else if fields[len(fields)-2] == "pull" { + return map[string]interface{}{ + "type": "pull_ref", + "subject": fields[len(fields)-1], + } + } + } + return map[string]interface{}{} + case "disconnected": + return map[string]interface{}{} case "referenced": tp := "commit_ref" if g.Issue != "" { @@ -964,6 +982,18 @@ func (g *githubIssueEvent) CommentStr() string { return "pinned" case "unpinned": return "unpinned" + case "connected": + fields := strings.Split(g.Subject, "/") + if len(fields) > 2 { + if fields[len(fields)-2] == "issue" { + return "issue_ref" + } else if fields[len(fields)-2] == "pull" { + return "pull_ref" + } + } + return "unknown" + case "disconnected": + return "unknown" default: return "comment" } From e806f198cc26c6ca99ab7005c730f4733efe6875 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 6 Feb 2022 19:59:18 +0800 Subject: [PATCH 43/54] Ignore unsupported event --- services/migrations/github_exported_data.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index ae40276f4c79c..edc18d8de06e1 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -940,18 +940,24 @@ func (g *githubIssueEvent) CommentContent() map[string]interface{} { // CommentStr returns comment type string func (g *githubIssueEvent) CommentStr() string { switch g.Event { + case "assigned": + return "assignees" + case "base_ref_changed": + return "unknown" case "closed": return "close" + case "convert_to_draft": + return "unknown" case "head_ref_force_pushed": return "pull_push" case "referenced": return "commit_ref" case "moved_columns_in_project": return "unknown" - case "convert_to_draft": - return "unknown" case "ready_for_review": return "unknown" + case "review_requested": + return "unknown" case "merged": return "merge_pull" case "mentioned": @@ -976,8 +982,7 @@ func (g *githubIssueEvent) CommentStr() string { return "reopen" case "unlabeled": return "label" - case "assigned": - return "assignees" + case "pinned": return "pinned" case "unpinned": From 46d8251eb6c5c1b2f77a0bdd7d88a70cf4f6938a Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 21 Mar 2022 08:44:42 +0800 Subject: [PATCH 44/54] Merge --- modules/migration/downloader.go | 9 +-- modules/migration/issue.go | 5 +- modules/migration/null_downloader.go | 4 +- modules/migration/retry_downloader.go | 8 +-- services/migrations/codebase.go | 8 +-- services/migrations/gitbucket.go | 2 +- services/migrations/gitea_downloader.go | 20 +++---- services/migrations/github_exported_data.go | 66 ++++++++++----------- services/migrations/github_v3.go | 18 +++--- services/migrations/gitlab.go | 20 +++---- services/migrations/gogs.go | 6 +- services/migrations/migrate.go | 12 +++- services/migrations/onedev.go | 24 ++++---- services/migrations/restore.go | 8 +-- 14 files changed, 109 insertions(+), 101 deletions(-) diff --git a/modules/migration/downloader.go b/modules/migration/downloader.go index 67f18a655500e..9b3a347307a1c 100644 --- a/modules/migration/downloader.go +++ b/modules/migration/downloader.go @@ -13,14 +13,14 @@ import ( // GetCommentOptions represents an options for get comment type GetCommentOptions struct { - Context DownloaderContext + Commentable Page int PageSize int } // GetReviewOptions represents an options for get reviews type GetReviewOptions struct { - Context DownloaderContext + Reviewable Page int PageSize int } @@ -34,11 +34,12 @@ type Downloader interface { GetReleases() ([]*Release, error) GetLabels() ([]*Label, error) GetIssues(page, perPage int) ([]*Issue, bool, error) - GetComments(commentable Commentable) ([]*Comment, bool, error) + GetComments(opts GetCommentOptions) ([]*Comment, bool, error) GetAllComments(page, perPage int) ([]*Comment, bool, error) SupportGetRepoComments() bool GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) - GetReviews(reviewable Reviewable) ([]*Review, bool, error) + GetReviews(opts GetReviewOptions) ([]*Review, bool, error) + SupportGetRepoReviews() bool FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) CleanUp() } diff --git a/modules/migration/issue.go b/modules/migration/issue.go index 883e01abe78ce..06327aa11b4c8 100644 --- a/modules/migration/issue.go +++ b/modules/migration/issue.go @@ -36,6 +36,7 @@ func (i *Issue) GetExternalName() string { return i.PosterName } // GetExternalID ExternalUserMigrated interface func (i *Issue) GetExternalID() int64 { return i.PosterID } -func (i *Issue) GetLocalIndex() int64 { return i.Number } -func (i *Issue) GetForeignIndex() int64 { return i.ForeignIndex } +func (i *Issue) GetLocalIndex() int64 { return i.Number } +func (i *Issue) GetForeignIndex() int64 { return i.ForeignIndex } + func (i *Issue) GetContext() DownloaderContext { return i.Context } diff --git a/modules/migration/null_downloader.go b/modules/migration/null_downloader.go index 23b21f0a18e02..41bb1f7714397 100644 --- a/modules/migration/null_downloader.go +++ b/modules/migration/null_downloader.go @@ -48,7 +48,7 @@ func (n NullDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { } // GetComments returns comments of an issue or PR -func (n NullDownloader) GetComments(commentable Commentable) ([]*Comment, bool, error) { +func (n NullDownloader) GetComments(opts GetCommentOptions) ([]*Comment, bool, error) { return nil, false, &ErrNotSupported{Entity: "Comments"} } @@ -63,7 +63,7 @@ func (n NullDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool } // GetReviews returns pull requests review -func (n NullDownloader) GetReviews(reviewable Reviewable) ([]*Review, bool, error) { +func (n NullDownloader) GetReviews(opts GetReviewOptions) ([]*Review, bool, error) { return nil, false, &ErrNotSupported{Entity: "Reviews"} } diff --git a/modules/migration/retry_downloader.go b/modules/migration/retry_downloader.go index 36640cc1eeaa5..7fb023a6d6cc8 100644 --- a/modules/migration/retry_downloader.go +++ b/modules/migration/retry_downloader.go @@ -148,7 +148,7 @@ func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { } // GetComments returns a repository's comments with retry -func (d *RetryDownloader) GetComments(commentable Commentable) ([]*Comment, bool, error) { +func (d *RetryDownloader) GetComments(opts GetCommentOptions) ([]*Comment, bool, error) { var ( comments []*Comment isEnd bool @@ -156,7 +156,7 @@ func (d *RetryDownloader) GetComments(commentable Commentable) ([]*Comment, bool ) err = d.retry(func() error { - comments, isEnd, err = d.Downloader.GetComments(commentable) + comments, isEnd, err = d.Downloader.GetComments(opts) return err }) @@ -180,7 +180,7 @@ func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bo } // GetReviews returns pull requests reviews -func (d *RetryDownloader) GetReviews(reviewable Reviewable) ([]*Review, bool, error) { +func (d *RetryDownloader) GetReviews(opts GetReviewOptions) ([]*Review, bool, error) { var ( reviews []*Review isEnd bool @@ -188,7 +188,7 @@ func (d *RetryDownloader) GetReviews(reviewable Reviewable) ([]*Review, bool, er ) err = d.retry(func() error { - reviews, isEnd, err = d.Downloader.GetReviews(reviewable) + reviews, isEnd, err = d.Downloader.GetReviews(opts) return err }) diff --git a/services/migrations/codebase.go b/services/migrations/codebase.go index 780d17f774a92..ec17980411cee 100644 --- a/services/migrations/codebase.go +++ b/services/migrations/codebase.go @@ -407,10 +407,10 @@ func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, } // GetComments returns comments -func (d *CodebaseDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { - context, ok := commentable.GetContext().(codebaseIssueContext) +func (d *CodebaseDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { + context, ok := opts.Commentable.GetContext().(codebaseIssueContext) if !ok { - return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext()) + return nil, false, fmt.Errorf("unexpected context: %+v", opts.Commentable.GetContext()) } return context.Comments, true, nil @@ -570,7 +570,7 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq } // GetReviews returns pull requests reviews -func (d *CodebaseDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, bool, error) { +func (d *CodebaseDownloader) GetReviews(opts base.GetReviewOptions) ([]*base.Review, bool, error) { return []*base.Review{}, true, nil } diff --git a/services/migrations/gitbucket.go b/services/migrations/gitbucket.go index 39c70c0e042ab..f7dae53ea9a2e 100644 --- a/services/migrations/gitbucket.go +++ b/services/migrations/gitbucket.go @@ -66,6 +66,6 @@ func (g *GitBucketDownloader) SupportGetRepoComments() bool { } // GetReviews is not supported -func (g *GitBucketDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, bool, error) { +func (g *GitBucketDownloader) GetReviews(reviewable base.GetReviewOptions) ([]*base.Review, bool, error) { return nil, true, &base.ErrNotSupported{Entity: "Reviews"} } diff --git a/services/migrations/gitea_downloader.go b/services/migrations/gitea_downloader.go index 6868987403186..ddc115fab4d81 100644 --- a/services/migrations/gitea_downloader.go +++ b/services/migrations/gitea_downloader.go @@ -442,7 +442,7 @@ func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, err } // GetComments returns comments according issueNumber -func (g *GiteaDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { +func (g *GiteaDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { allComments := make([]*base.Comment, 0, g.maxPerPage) for i := 1; ; i++ { @@ -453,26 +453,26 @@ func (g *GiteaDownloader) GetComments(commentable base.Commentable) ([]*base.Com default: } - comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, commentable.GetForeignIndex(), gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{ + comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.Commentable.GetForeignIndex(), gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{ PageSize: g.maxPerPage, Page: i, }}) if err != nil { - return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %v", commentable.GetForeignIndex(), err) + return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %v", opts.Commentable.GetForeignIndex(), err) } for _, comment := range comments { reactions, err := g.getCommentReactions(comment.ID) if err != nil { - log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", commentable.GetForeignIndex(), comment.ID, g.repoOwner, g.repoName, err) + log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.Commentable.GetForeignIndex(), comment.ID, g.repoOwner, g.repoName, err) if err2 := admin_model.CreateRepositoryNotice( - fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", commentable.GetForeignIndex(), comment.ID, g.repoOwner, g.repoName, err)); err2 != nil { + fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.Commentable.GetForeignIndex(), comment.ID, g.repoOwner, g.repoName, err)); err2 != nil { log.Error("create repository notice failed: ", err2) } } allComments = append(allComments, &base.Comment{ - IssueIndex: commentable.GetLocalIndex(), + IssueIndex: opts.Commentable.GetLocalIndex(), Index: comment.ID, PosterID: comment.Poster.ID, PosterName: comment.Poster.UserName, @@ -614,7 +614,7 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques } // GetReviews returns pull requests review -func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, bool, error) { +func (g *GiteaDownloader) GetReviews(opts base.GetReviewOptions) ([]*base.Review, bool, error) { if err := g.client.CheckServerVersionConstraint(">=1.12"); err != nil { log.Info("GiteaDownloader: instance to old, skip GetReviews") return nil, true, nil @@ -630,7 +630,7 @@ func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review default: } - prl, _, err := g.client.ListPullReviews(g.repoOwner, g.repoName, reviewable.GetForeignIndex(), gitea_sdk.ListPullReviewsOptions{ListOptions: gitea_sdk.ListOptions{ + prl, _, err := g.client.ListPullReviews(g.repoOwner, g.repoName, opts.Reviewable.GetForeignIndex(), gitea_sdk.ListPullReviewsOptions{ListOptions: gitea_sdk.ListOptions{ Page: i, PageSize: g.maxPerPage, }}) @@ -639,7 +639,7 @@ func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review } for _, pr := range prl { - rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, reviewable.GetForeignIndex(), pr.ID) + rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, opts.Reviewable.GetForeignIndex(), pr.ID) if err != nil { return nil, true, err } @@ -665,7 +665,7 @@ func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review allReviews = append(allReviews, &base.Review{ ID: pr.ID, - IssueIndex: reviewable.GetLocalIndex(), + IssueIndex: opts.Reviewable.GetLocalIndex(), ReviewerID: pr.Reviewer.ID, ReviewerName: pr.Reviewer.UserName, Official: pr.Official, diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index edc18d8de06e1..8082b1d172838 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -740,22 +740,22 @@ func (r *GithubExportedDataRestorer) GetIssues(page, perPage int) ([]*base.Issue issue.Body = strings.ReplaceAll(issue.Body, "\u0000", "") issues = append(issues, &base.Issue{ - Number: issue.Index(), - Title: issue.Title, - Content: issue.Body, - PosterID: id, - PosterName: login, - PosterEmail: email, - Labels: r.getLabels(issue.Labels), - Reactions: r.getReactions(issue.Reactions), - Assets: r.convertAttachments(r.issueAttachments[issue.URL]), - Milestone: milestone, - Assignees: issue.Assignees, - State: state, - Context: base.BasicIssueContext(issue.Index()), - Closed: issue.ClosedAt, - Created: issue.CreatedAt, - Updated: issue.UpdatedAt, + Number: issue.Index(), + Title: issue.Title, + Content: issue.Body, + PosterID: id, + PosterName: login, + PosterEmail: email, + Labels: r.getLabels(issue.Labels), + Reactions: r.getReactions(issue.Reactions), + Assets: r.convertAttachments(r.issueAttachments[issue.URL]), + Milestone: milestone, + Assignees: issue.Assignees, + State: state, + ForeignIndex: issue.Index(), + Closed: issue.ClosedAt, + Created: issue.CreatedAt, + Updated: issue.UpdatedAt, }) } return nil @@ -1187,23 +1187,23 @@ func (r *GithubExportedDataRestorer) GetPullRequests(page, perPage int) ([]*base milestone = r.milestones[pr.Milestone].Title } pulls = append(pulls, &base.PullRequest{ - Number: pr.Index(), - Title: pr.Title, - Content: pr.Body, - Milestone: milestone, - State: state, - PosterID: id, - PosterName: login, - PosterEmail: email, - Context: base.BasicIssueContext(pr.Index()), - Reactions: r.getReactions(pr.Reactions), - Assets: r.convertAttachments(r.issueAttachments[pr.URL]), - Created: pr.CreatedAt, - Closed: pr.ClosedAt, - Labels: r.getLabels(pr.Labels), - Merged: pr.MergedAt != nil, - MergedTime: pr.MergedAt, - Head: head, + Number: pr.Index(), + Title: pr.Title, + Content: pr.Body, + Milestone: milestone, + State: state, + PosterID: id, + PosterName: login, + PosterEmail: email, + ForeignIndex: pr.Index(), + Reactions: r.getReactions(pr.Reactions), + Assets: r.convertAttachments(r.issueAttachments[pr.URL]), + Created: pr.CreatedAt, + Closed: pr.ClosedAt, + Labels: r.getLabels(pr.Labels), + Merged: pr.MergedAt != nil, + MergedTime: pr.MergedAt, + Head: head, Base: base.PullRequestBranch{ Ref: pr.Base.Ref, SHA: pr.Base.Sha, diff --git a/services/migrations/github_v3.go b/services/migrations/github_v3.go index d872d74384b77..0c770027172b9 100644 --- a/services/migrations/github_v3.go +++ b/services/migrations/github_v3.go @@ -474,12 +474,12 @@ func (g *GithubDownloaderV3) SupportGetRepoComments() bool { } // GetComments returns comments according issueNumber -func (g *GithubDownloaderV3) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { - comments, err := g.getComments(commentable) +func (g *GithubDownloaderV3) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { + comments, err := g.getComments(opts) return comments, false, err } -func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base.Comment, error) { +func (g *GithubDownloaderV3) getComments(opts base.GetCommentOptions) ([]*base.Comment, error) { var ( allComments = make([]*base.Comment, 0, g.maxPerPage) created = "created" @@ -494,7 +494,7 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base. } for { g.waitAndPickClient() - comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(commentable.GetForeignIndex()), opt) + comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(opts.Commentable.GetForeignIndex()), opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } @@ -527,7 +527,7 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base. } allComments = append(allComments, &base.Comment{ - IssueIndex: commentable.GetLocalIndex(), + IssueIndex: opts.Commentable.GetLocalIndex(), Index: comment.GetID(), PosterID: comment.GetUser().GetID(), PosterName: comment.GetUser().GetLogin(), @@ -773,28 +773,28 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques } // GetReviews returns pull requests review -func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Review, bool, error) { +func (g *GithubDownloaderV3) GetReviews(opts base.GetReviewOptions) ([]*base.Review, bool, error) { allReviews := make([]*base.Review, 0, g.maxPerPage) opt := &github.ListOptions{ PerPage: g.maxPerPage, } for { g.waitAndPickClient() - reviews, resp, err := g.getClient().PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt) + reviews, resp, err := g.getClient().PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(opts.Reviewable.GetForeignIndex()), opt) if err != nil { return nil, true, fmt.Errorf("error while listing repos: %v", err) } g.setRate(&resp.Rate) for _, review := range reviews { r := convertGithubReview(review) - r.IssueIndex = reviewable.GetLocalIndex() + r.IssueIndex = opts.Reviewable.GetLocalIndex() // retrieve all review comments opt2 := &github.ListOptions{ PerPage: g.maxPerPage, } for { g.waitAndPickClient() - reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), review.GetID(), opt2) + reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(opts.Reviewable.GetForeignIndex()), review.GetID(), opt2) if err != nil { return nil, true, fmt.Errorf("error while listing repos: %v", err) } diff --git a/services/migrations/gitlab.go b/services/migrations/gitlab.go index b1e79c68a3d62..ef1cf2ef2f78e 100644 --- a/services/migrations/gitlab.go +++ b/services/migrations/gitlab.go @@ -437,10 +437,10 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er // GetComments returns comments according issueNumber // TODO: figure out how to transfer comment reactions -func (g *GitlabDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { - context, ok := commentable.GetContext().(gitlabIssueContext) +func (g *GitlabDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { + context, ok := opts.Commentable.GetContext().(gitlabIssueContext) if !ok { - return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext()) + return nil, false, fmt.Errorf("unexpected context: %+v", opts.Commentable.GetContext()) } allComments := make([]*base.Comment, 0, g.maxPerPage) @@ -452,12 +452,12 @@ func (g *GitlabDownloader) GetComments(commentable base.Commentable) ([]*base.Co var resp *gitlab.Response var err error if !context.IsMergeRequest { - comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(commentable.GetForeignIndex()), &gitlab.ListIssueDiscussionsOptions{ + comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(opts.Commentable.GetForeignIndex()), &gitlab.ListIssueDiscussionsOptions{ Page: page, PerPage: g.maxPerPage, }, nil, gitlab.WithContext(g.ctx)) } else { - comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(commentable.GetForeignIndex()), &gitlab.ListMergeRequestDiscussionsOptions{ + comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(opts.Commentable.GetForeignIndex()), &gitlab.ListMergeRequestDiscussionsOptions{ Page: page, PerPage: g.maxPerPage, }, nil, gitlab.WithContext(g.ctx)) @@ -471,7 +471,7 @@ func (g *GitlabDownloader) GetComments(commentable base.Commentable) ([]*base.Co if !comment.IndividualNote { for _, note := range comment.Notes { allComments = append(allComments, &base.Comment{ - IssueIndex: commentable.GetLocalIndex(), + IssueIndex: opts.Commentable.GetLocalIndex(), Index: int64(note.ID), PosterID: int64(note.Author.ID), PosterName: note.Author.Username, @@ -483,7 +483,7 @@ func (g *GitlabDownloader) GetComments(commentable base.Commentable) ([]*base.Co } else { c := comment.Notes[0] allComments = append(allComments, &base.Comment{ - IssueIndex: commentable.GetLocalIndex(), + IssueIndex: opts.Commentable.GetLocalIndex(), Index: int64(c.ID), PosterID: int64(c.Author.ID), PosterName: c.Author.Username, @@ -616,8 +616,8 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque } // GetReviews returns pull requests review -func (g *GitlabDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, bool, error) { - approvals, resp, err := g.client.MergeRequestApprovals.GetConfiguration(g.repoID, int(reviewable.GetForeignIndex()), gitlab.WithContext(g.ctx)) +func (g *GitlabDownloader) GetReviews(opts base.GetReviewOptions) ([]*base.Review, bool, error) { + approvals, resp, err := g.client.MergeRequestApprovals.GetConfiguration(g.repoID, int(opts.Reviewable.GetForeignIndex()), gitlab.WithContext(g.ctx)) if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { log.Error(fmt.Sprintf("GitlabDownloader: while migrating a error occurred: '%s'", err.Error())) @@ -638,7 +638,7 @@ func (g *GitlabDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Revie reviews := make([]*base.Review, 0, len(approvals.ApprovedBy)) for _, user := range approvals.ApprovedBy { reviews = append(reviews, &base.Review{ - IssueIndex: reviewable.GetLocalIndex(), + IssueIndex: opts.Reviewable.GetLocalIndex(), ReviewerID: int64(user.User.ID), ReviewerName: user.User.Username, CreatedAt: createdAt, diff --git a/services/migrations/gogs.go b/services/migrations/gogs.go index a28033218eac5..85e05b42033f5 100644 --- a/services/migrations/gogs.go +++ b/services/migrations/gogs.go @@ -223,10 +223,10 @@ func (g *GogsDownloader) getIssues(page int, state string) ([]*base.Issue, bool, } // GetComments returns comments according issueNumber -func (g *GogsDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { +func (g *GogsDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { allComments := make([]*base.Comment, 0, 100) - comments, err := g.client.ListIssueComments(g.repoOwner, g.repoName, commentable.GetForeignIndex()) + comments, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.Commentable.GetForeignIndex()) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %v", err) } @@ -235,7 +235,7 @@ func (g *GogsDownloader) GetComments(commentable base.Commentable) ([]*base.Comm continue } allComments = append(allComments, &base.Comment{ - IssueIndex: commentable.GetLocalIndex(), + IssueIndex: opts.Commentable.GetLocalIndex(), Index: comment.ID, PosterID: comment.Poster.ID, PosterName: comment.Poster.Login, diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index f97856231e814..2497828474196 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -325,7 +325,9 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts allComments := make([]*base.Comment, 0, commentBatchSize) for _, issue := range issues { log.Trace("migrating issue %d's comments", issue.Number) - comments, _, err := downloader.GetComments(issue) + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + Commentable: issue, + }) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -383,7 +385,9 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts allComments := make([]*base.Comment, 0, commentBatchSize) for _, pr := range prs { log.Trace("migrating pull request %d's comments", pr.Number) - comments, _, err := downloader.GetComments(pr) + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + Commentable: pr, + }) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -411,7 +415,9 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts // migrate reviews allReviews := make([]*base.Review, 0, reviewBatchSize) for _, pr := range prs { - reviews, _, err := downloader.GetReviews(pr) + reviews, _, err := downloader.GetReviews(base.GetReviewOptions{ + Reviewable: pr, + }) if err != nil { if !base.IsErrNotSupported(err) { return err diff --git a/services/migrations/onedev.go b/services/migrations/onedev.go index 76b30df1bb6f7..4cc095146f5bf 100644 --- a/services/migrations/onedev.go +++ b/services/migrations/onedev.go @@ -359,10 +359,10 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er } // GetComments returns comments -func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { - context, ok := commentable.GetContext().(onedevIssueContext) +func (d *OneDevDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { + context, ok := opts.Commentable.GetContext().(onedevIssueContext) if !ok { - return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext()) + return nil, false, fmt.Errorf("unexpected context: %+v", opts.Commentable.GetContext()) } rawComments := make([]struct { @@ -374,9 +374,9 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co var endpoint string if context.IsPullRequest { - endpoint = fmt.Sprintf("/api/pull-requests/%d/comments", commentable.GetForeignIndex()) + endpoint = fmt.Sprintf("/api/pull-requests/%d/comments", opts.Commentable.GetForeignIndex()) } else { - endpoint = fmt.Sprintf("/api/issues/%d/comments", commentable.GetForeignIndex()) + endpoint = fmt.Sprintf("/api/issues/%d/comments", opts.Commentable.GetForeignIndex()) } err := d.callAPI( @@ -395,9 +395,9 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co }, 0, 100) if context.IsPullRequest { - endpoint = fmt.Sprintf("/api/pull-requests/%d/changes", commentable.GetForeignIndex()) + endpoint = fmt.Sprintf("/api/pull-requests/%d/changes", opts.Commentable.GetForeignIndex()) } else { - endpoint = fmt.Sprintf("/api/issues/%d/changes", commentable.GetForeignIndex()) + endpoint = fmt.Sprintf("/api/issues/%d/changes", opts.Commentable.GetForeignIndex()) } err = d.callAPI( @@ -416,7 +416,7 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co } poster := d.tryGetUser(comment.UserID) comments = append(comments, &base.Comment{ - IssueIndex: commentable.GetLocalIndex(), + IssueIndex: opts.Commentable.GetLocalIndex(), Index: comment.ID, PosterID: poster.ID, PosterName: poster.Name, @@ -441,7 +441,7 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co poster := d.tryGetUser(change.UserID) comments = append(comments, &base.Comment{ - IssueIndex: commentable.GetLocalIndex(), + IssueIndex: opts.Commentable.GetLocalIndex(), PosterID: poster.ID, PosterName: poster.Name, PosterEmail: poster.Email, @@ -548,7 +548,7 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque } // GetReviews returns pull requests reviews -func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, bool, error) { +func (d *OneDevDownloader) GetReviews(opts base.GetReviewOptions) ([]*base.Review, bool, error) { rawReviews := make([]struct { ID int64 `json:"id"` UserID int64 `json:"userId"` @@ -560,7 +560,7 @@ func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Revie }, 0, 100) err := d.callAPI( - fmt.Sprintf("/api/pull-requests/%d/reviews", reviewable.GetForeignIndex()), + fmt.Sprintf("/api/pull-requests/%d/reviews", opts.Reviewable.GetForeignIndex()), nil, &rawReviews, ) @@ -584,7 +584,7 @@ func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Revie poster := d.tryGetUser(review.UserID) reviews = append(reviews, &base.Review{ - IssueIndex: reviewable.GetLocalIndex(), + IssueIndex: opts.Reviewable.GetLocalIndex(), ReviewerID: poster.ID, ReviewerName: poster.Name, Content: content, diff --git a/services/migrations/restore.go b/services/migrations/restore.go index c9b7f438cf491..562285bf81917 100644 --- a/services/migrations/restore.go +++ b/services/migrations/restore.go @@ -197,9 +197,9 @@ func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, } // GetComments returns comments according issueNumber -func (r *RepositoryRestorer) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { +func (r *RepositoryRestorer) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { comments := make([]*base.Comment, 0, 10) - p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", commentable.GetForeignIndex())) + p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", opts.Commentable.GetForeignIndex())) _, err := os.Stat(p) if err != nil { if os.IsNotExist(err) { @@ -248,9 +248,9 @@ func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullReq } // GetReviews returns pull requests review -func (r *RepositoryRestorer) GetReviews(reviewable base.Reviewable) ([]*base.Review, bool, error) { +func (r *RepositoryRestorer) GetReviews(opts base.GetReviewOptions) ([]*base.Review, bool, error) { reviews := make([]*base.Review, 0, 10) - p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", reviewable.GetForeignIndex())) + p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", opts.GetForeignIndex())) _, err := os.Stat(p) if err != nil { if os.IsNotExist(err) { From f10b3de7fbc3eb5d78177aaf00a965d633bbd1fc Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 7 Apr 2022 09:35:54 +0800 Subject: [PATCH 45/54] Fix lint --- services/migrations/codebase_test.go | 8 +++++-- services/migrations/gitea_downloader_test.go | 8 +++++-- services/migrations/github_exported_data.go | 3 +-- .../migrations/github_exported_data_test.go | 4 ++-- services/migrations/github_v3_test.go | 12 +++++++--- services/migrations/gitlab_test.go | 22 +++++++++++++------ services/migrations/gogs_test.go | 4 +++- services/migrations/onedev_test.go | 14 +++++++----- 8 files changed, 51 insertions(+), 24 deletions(-) diff --git a/services/migrations/codebase_test.go b/services/migrations/codebase_test.go index 21d56d17751d1..8f300f384250f 100644 --- a/services/migrations/codebase_test.go +++ b/services/migrations/codebase_test.go @@ -107,7 +107,9 @@ func TestCodebaseDownloadRepo(t *testing.T) { }, }, issues) - comments, _, err := downloader.GetComments(issues[0]) + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + Commentable: issues[0], + }) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ { @@ -145,7 +147,9 @@ func TestCodebaseDownloadRepo(t *testing.T) { }, }, prs) - rvs, _, err := downloader.GetReviews(prs[0]) + rvs, _, err := downloader.GetReviews(base.GetReviewOptions{ + Reviewable: prs[0], + }) assert.NoError(t, err) assert.Empty(t, rvs) } diff --git a/services/migrations/gitea_downloader_test.go b/services/migrations/gitea_downloader_test.go index e8129b151ca4d..2c12bd2a24ba9 100644 --- a/services/migrations/gitea_downloader_test.go +++ b/services/migrations/gitea_downloader_test.go @@ -198,7 +198,9 @@ func TestGiteaDownloadRepo(t *testing.T) { }, }, issues) - comments, _, err := downloader.GetComments(&base.Issue{Number: 4, ForeignIndex: 4}) + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + Commentable: &base.Issue{Number: 4, ForeignIndex: 4}, + }) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ { @@ -263,7 +265,9 @@ func TestGiteaDownloadRepo(t *testing.T) { PatchURL: "https://gitea.com/gitea/test_repo/pulls/12.patch", }, prs[1]) - reviews, _, err := downloader.GetReviews(&base.Issue{Number: 7, ForeignIndex: 7}) + reviews, _, err := downloader.GetReviews(base.GetReviewOptions{ + Reviewable: &base.Issue{Number: 7, ForeignIndex: 7}, + }) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index 8082b1d172838..00fd973b9627f 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -11,7 +11,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/url" "os" "path/filepath" @@ -385,7 +384,7 @@ func (r *GithubExportedDataRestorer) GetRepoInfo() (*base.Repository, error) { } p := filepath.Join(r.tmpDir, "repositories_000001.json") - bs, err := ioutil.ReadFile(p) + bs, err := os.ReadFile(p) if err != nil { return nil, err } diff --git a/services/migrations/github_exported_data_test.go b/services/migrations/github_exported_data_test.go index 60b90a761cd4b..92335899dfe0a 100644 --- a/services/migrations/github_exported_data_test.go +++ b/services/migrations/github_exported_data_test.go @@ -45,7 +45,7 @@ func TestParseGithubExportedData(t *testing.T) { assert.NoError(t, err) assert.True(t, isEnd) assert.EqualValues(t, 2, len(issues)) - assert.EqualValues(t, 1, issues[0].Context.ForeignID()) + assert.EqualValues(t, 1, issues[0].ForeignIndex) assert.EqualValues(t, "Please add an animated gif icon to the merge button", issues[0].Title) assert.EqualValues(t, "I just want the merge button to hurt my eyes a little. 😝 ", issues[0].Content) assert.EqualValues(t, "guillep2k", issues[0].PosterName) @@ -58,7 +58,7 @@ func TestParseGithubExportedData(t *testing.T) { assert.NotZero(t, issues[0].Updated) assert.NotZero(t, issues[0].Created) - assert.EqualValues(t, 2, issues[1].Context.ForeignID()) + assert.EqualValues(t, 2, issues[1].ForeignIndex) assert.EqualValues(t, "Test issue", issues[1].Title) assert.EqualValues(t, "This is test issue 2, do not touch!", issues[1].Content) assert.EqualValues(t, "mrsdizzie", issues[1].PosterName) diff --git a/services/migrations/github_v3_test.go b/services/migrations/github_v3_test.go index d0d4661d70e4f..9f4fefaff182f 100644 --- a/services/migrations/github_v3_test.go +++ b/services/migrations/github_v3_test.go @@ -215,7 +215,9 @@ func TestGitHubDownloadRepo(t *testing.T) { }, issues) // downloader.GetComments() - comments, _, err := downloader.GetComments(&base.Issue{Number: 2, ForeignIndex: 2}) + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + Commentable: &base.Issue{Number: 2, ForeignIndex: 2}, + }) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ { @@ -335,7 +337,9 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, prs) - reviews, _, err := downloader.GetReviews(&base.PullRequest{Number: 3, ForeignIndex: 3}) + reviews, _, err := downloader.GetReviews(base.GetReviewOptions{ + Reviewable: &base.PullRequest{Number: 3, ForeignIndex: 3}, + }) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -367,7 +371,9 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, reviews) - reviews, _, err = downloader.GetReviews(&base.PullRequest{Number: 4, ForeignIndex: 4}) + reviews, _, err = downloader.GetReviews(base.GetReviewOptions{ + Reviewable: &base.PullRequest{Number: 4, ForeignIndex: 4}, + }) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { diff --git a/services/migrations/gitlab_test.go b/services/migrations/gitlab_test.go index 03ce361a8d476..c9d3c1a2f2521 100644 --- a/services/migrations/gitlab_test.go +++ b/services/migrations/gitlab_test.go @@ -214,10 +214,12 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, issues) - comments, _, err := downloader.GetComments(&base.Issue{ - Number: 2, - ForeignIndex: 2, - Context: gitlabIssueContext{IsMergeRequest: false}, + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + Commentable: &base.Issue{ + Number: 2, + ForeignIndex: 2, + Context: gitlabIssueContext{IsMergeRequest: false}, + }, }) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ @@ -304,7 +306,9 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, prs) - rvs, _, err := downloader.GetReviews(&base.PullRequest{Number: 1, ForeignIndex: 1}) + rvs, _, err := downloader.GetReviews(base.GetReviewOptions{ + Reviewable: &base.PullRequest{Number: 1, ForeignIndex: 1}, + }) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -323,7 +327,9 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, rvs) - rvs, _, err = downloader.GetReviews(&base.PullRequest{Number: 2, ForeignIndex: 2}) + rvs, _, err = downloader.GetReviews(base.GetReviewOptions{ + Reviewable: &base.PullRequest{Number: 2, ForeignIndex: 2}, + }) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -465,7 +471,9 @@ func TestGitlabGetReviews(t *testing.T) { mux.HandleFunc(fmt.Sprintf("/api/v4/projects/%d/merge_requests/%d/approvals", testCase.repoID, testCase.prID), mock) id := int64(testCase.prID) - rvs, _, err := downloader.GetReviews(&base.Issue{Number: id, ForeignIndex: id}) + rvs, _, err := downloader.GetReviews(base.GetReviewOptions{ + Reviewable: &base.Issue{Number: id, ForeignIndex: id}, + }) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{&review}, rvs) } diff --git a/services/migrations/gogs_test.go b/services/migrations/gogs_test.go index 501161b610dad..e23e1d3b223e3 100644 --- a/services/migrations/gogs_test.go +++ b/services/migrations/gogs_test.go @@ -111,7 +111,9 @@ func TestGogsDownloadRepo(t *testing.T) { }, issues) // downloader.GetComments() - comments, _, err := downloader.GetComments(&base.Issue{Number: 1, ForeignIndex: 1}) + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + Commentable: &base.Issue{Number: 1, ForeignIndex: 1}, + }) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ { diff --git a/services/migrations/onedev_test.go b/services/migrations/onedev_test.go index 30cb326247e1c..87804ac2d8897 100644 --- a/services/migrations/onedev_test.go +++ b/services/migrations/onedev_test.go @@ -95,10 +95,12 @@ func TestOneDevDownloadRepo(t *testing.T) { }, }, issues) - comments, _, err := downloader.GetComments(&base.Issue{ - Number: 4, - ForeignIndex: 398, - Context: onedevIssueContext{IsPullRequest: false}, + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + Commentable: &base.Issue{ + Number: 4, + ForeignIndex: 398, + Context: onedevIssueContext{IsPullRequest: false}, + }, }) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ @@ -137,7 +139,9 @@ func TestOneDevDownloadRepo(t *testing.T) { }, }, prs) - rvs, _, err := downloader.GetReviews(&base.PullRequest{Number: 5, ForeignIndex: 186}) + rvs, _, err := downloader.GetReviews(base.GetReviewOptions{ + Reviewable: &base.PullRequest{Number: 5, ForeignIndex: 186}, + }) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { From eef99f6bd0eec24af9f6247e902f920658121d0e Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 24 Apr 2022 16:33:51 +0800 Subject: [PATCH 46/54] Merge --- services/migrations/gitea_uploader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 06473fc8c5acf..ba646f4df37f6 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -523,7 +523,7 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { log.Error("unmarshal %s failed: %v", cm.Content, err) continue } else { - milestone, err := models.GetMilestoneByRepoIDANDName(issue.RepoID, data["MilestoneTitle"]) + milestone, err := issues_model.GetMilestoneByRepoIDANDName(issue.RepoID, data["MilestoneTitle"]) if err != nil { log.Error("GetMilestoneByRepoIDANDName %d, %s failed: %v", issue.RepoID, data["MilestoneTitle"], err) continue From b78ad205022b24f214f857a39372a42d8e7975b9 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 23 May 2022 19:24:15 +0800 Subject: [PATCH 47/54] Revert changes on updateGitForPullRequest --- services/migrations/gitea_uploader.go | 148 ++++++++++++-------------- 1 file changed, 66 insertions(+), 82 deletions(-) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index ba646f4df37f6..121cc243b1ca3 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -652,100 +652,84 @@ func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error return nil } -func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (string, error) { - refs, err := g.gitRepo.GetRefsFiltered(fmt.Sprintf("refs/pull/%d/head", pr.Number)) +func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head string, err error) { + // download patch file + err = func() error { + if pr.PatchURL == "" { + return nil + } + // pr.PatchURL maybe a local file + ret, err := uri.Open(pr.PatchURL) + if err != nil { + return err + } + defer ret.Close() + pullDir := filepath.Join(g.repo.RepoPath(), "pulls") + if err = os.MkdirAll(pullDir, os.ModePerm); err != nil { + return err + } + f, err := os.Create(filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number))) + if err != nil { + return err + } + defer f.Close() + _, err = io.Copy(f, ret) + return err + }() if err != nil { return "", err } + // set head information pullHead := filepath.Join(g.repo.RepoPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number)) + if err := os.MkdirAll(pullHead, os.ModePerm); err != nil { + return "", err + } + p, err := os.Create(filepath.Join(pullHead, "head")) + if err != nil { + return "", err + } + _, err = p.WriteString(pr.Head.SHA) + p.Close() + if err != nil { + return "", err + } - if len(refs) == 0 { - // download patch file - if err = func() error { - if pr.PatchURL == "" { - return nil - } - // pr.PatchURL maybe a local file - ret, err := uri.Open(pr.PatchURL) - if err != nil { - return err - } - defer ret.Close() - pullDir := filepath.Join(g.repo.RepoPath(), "pulls") - if err = os.MkdirAll(pullDir, os.ModePerm); err != nil { - return err - } - f, err := os.Create(filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number))) - if err != nil { - return err + head = "unknown repository" + if pr.IsForkPullRequest() && pr.State != "closed" { + if pr.Head.OwnerName != "" { + remote := pr.Head.OwnerName + _, ok := g.prHeadCache[remote] + if !ok { + // git remote add + err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true) + if err != nil { + log.Error("AddRemote failed: %s", err) + } else { + g.prHeadCache[remote] = struct{}{} + ok = true + } } - defer f.Close() - _, err = io.Copy(f, ret) - return err - }(); err != nil { - return "", err - } - // set head information - if err := os.MkdirAll(pullHead, os.ModePerm); err != nil { - return "", err - } - p, err := os.Create(filepath.Join(pullHead, "head")) - if err != nil { - return "", err - } - _, err = p.WriteString(pr.Head.SHA) - p.Close() - if err != nil { - return "", err - } - } - - head := "unknown repository" - if pr.State != "closed" { - if pr.IsForkPullRequest() { - if pr.Head.OwnerName != "" { - remote := pr.Head.OwnerName - _, ok := g.prHeadCache[remote] - if !ok { - // git remote add - err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true) + if ok { + _, _, err = git.NewCommand(g.ctx, "fetch", "--no-tags", "--", remote, pr.Head.Ref).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()}) + if err != nil { + log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err) + } else { + headBranch := filepath.Join(g.repo.RepoPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref) + if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil { + return "", err + } + b, err := os.Create(headBranch) if err != nil { - log.Error("AddRemote failed: %s", err) - } else { - g.prHeadCache[remote] = struct{}{} - ok = true + return "", err } - } - - if ok { - _, _, err = git.NewCommand(g.ctx, "fetch", "--no-tags", "--", remote, pr.Head.Ref).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()}) + _, err = b.WriteString(pr.Head.SHA) + b.Close() if err != nil { - log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err) - } else { - headBranch := filepath.Join(g.repo.RepoPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref) - if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil { - return "", err - } - b, err := os.Create(headBranch) - if err != nil { - return "", err - } - _, err = b.WriteString(pr.Head.SHA) - b.Close() - if err != nil { - return "", err - } - head = pr.Head.OwnerName + "/" + pr.Head.Ref + return "", err } - } - } - } else { - head = pr.Head.Ref - if !g.gitRepo.IsBranchExist(pr.Head.Ref) && pr.Head.SHA != "" { - if err := g.gitRepo.CreateBranch(pr.Head.Ref, pr.Head.SHA); err != nil { - return "", err + head = pr.Head.OwnerName + "/" + pr.Head.Ref } } } From 0ab78e931f393f1d2f7a4e397d0febed7e1febab Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 24 May 2022 17:36:09 +0800 Subject: [PATCH 48/54] Improve pr check --- services/migrations/gitea_uploader.go | 96 ++++++++++++++------- services/migrations/github_exported_data.go | 31 ++++--- 2 files changed, 81 insertions(+), 46 deletions(-) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 121cc243b1ca3..c658ba5c6a982 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -652,6 +652,28 @@ func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error return nil } +func getHeadBranch(pr *base.PullRequest) string { + if pr.IsForkPullRequest() { + return pr.Head.OwnerName + "/" + pr.Head.Ref + } + return pr.Head.Ref +} + +func (g *GiteaLocalUploader) createHeadBranch(pr *base.PullRequest) error { + branchName := getHeadBranch(pr) + headBranch := filepath.Join(g.repo.RepoPath(), "refs", "heads", branchName) + if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil { + return err + } + b, err := os.Create(headBranch) + if err != nil { + return err + } + _, err = b.WriteString(pr.Head.SHA) + b.Close() + return err +} + func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head string, err error) { // download patch file err = func() error { @@ -695,41 +717,55 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head return "", err } - head = "unknown repository" - if pr.IsForkPullRequest() && pr.State != "closed" { - if pr.Head.OwnerName != "" { - remote := pr.Head.OwnerName - _, ok := g.prHeadCache[remote] - if !ok { - // git remote add - err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true) - if err != nil { - log.Error("AddRemote failed: %s", err) - } else { - g.prHeadCache[remote] = struct{}{} - ok = true - } - } + // create branch directly if commit exists in the repository + _, err = g.gitRepo.GetCommit(pr.Head.SHA) + if git.IsErrNotExist(err) { + } else if err != nil { + return "", err + } else { + if err := g.createHeadBranch(pr); err != nil { + return "", err + } + return getHeadBranch(pr), nil + } - if ok { - _, _, err = git.NewCommand(g.ctx, "fetch", "--no-tags", "--", remote, pr.Head.Ref).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()}) - if err != nil { - log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err) - } else { - headBranch := filepath.Join(g.repo.RepoPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref) - if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil { - return "", err - } - b, err := os.Create(headBranch) + head = pullHead + if pr.IsForkPullRequest() { + if pr.State != "closed" { + if pr.Head.OwnerName != "" { + remote := pr.Head.OwnerName + _, ok := g.prHeadCache[remote] + if !ok { + // git remote add + err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true) if err != nil { - return "", err + log.Error("AddRemote[%s] failed: %s", pr.Head.CloneURL, err) + } else { + g.prHeadCache[remote] = struct{}{} + ok = true } - _, err = b.WriteString(pr.Head.SHA) - b.Close() + } + + if ok { + _, _, err = git.NewCommand(g.ctx, "fetch", "--no-tags", "--", remote, pr.Head.Ref).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()}) if err != nil { - return "", err + log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err) + } else { + headBranch := filepath.Join(g.repo.RepoPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref) + if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil { + return "", err + } + b, err := os.Create(headBranch) + if err != nil { + return "", err + } + _, err = b.WriteString(pr.Head.SHA) + b.Close() + if err != nil { + return "", err + } + head = pr.Head.OwnerName + "/" + pr.Head.Ref } - head = pr.Head.OwnerName + "/" + pr.Head.Ref } } } diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index 00fd973b9627f..17871bf483dc1 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -1039,6 +1039,10 @@ func (r *GithubExportedDataRestorer) getIssueEvents() ([]*base.Comment, error) { return comments, nil } +func (r *GithubExportedDataRestorer) GetAllComments(page, perPage int) ([]*base.Comment, bool, error) { + return r.GetComments(base.GetCommentOptions{}) +} + // GetComments returns comments according issueNumber func (r *GithubExportedDataRestorer) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { comments := make([]*base.Comment, 0, 10) @@ -1165,26 +1169,19 @@ func (r *GithubExportedDataRestorer) GetPullRequests(page, perPage int) ([]*base if pr.MergedAt != nil || pr.ClosedAt != nil { state = "closed" } - isFork := !(pr.Head.User == pr.Base.User && pr.Head.Repo == pr.Base.Repo) + _, headUser, _ := r.getUserInfo(pr.Head.User) head := base.PullRequestBranch{ - Ref: pr.Head.Ref, - SHA: pr.Head.Sha, - } - if isFork { - if pr.Head.User != "" { - fields := strings.Split(pr.Head.User, "/") - if pr.Head.Ref == "" { - pr.Head.Ref = fmt.Sprintf("%d", pr.Index()) - } - head.Ref = fmt.Sprintf("%s/%s", fields[len(fields)-1], pr.Head.Ref) - } else { - head.Ref = fmt.Sprintf("pr/%d", pr.Index()) - } + Ref: pr.Head.Ref, + SHA: pr.Head.Sha, + RepoName: pr.Head.Repo, + OwnerName: headUser, } + var milestone string if pr.Milestone != "" { milestone = r.milestones[pr.Milestone].Title } + _, baseUser, _ := r.getUserInfo(pr.Base.User) pulls = append(pulls, &base.PullRequest{ Number: pr.Index(), Title: pr.Title, @@ -1204,8 +1201,10 @@ func (r *GithubExportedDataRestorer) GetPullRequests(page, perPage int) ([]*base MergedTime: pr.MergedAt, Head: head, Base: base.PullRequestBranch{ - Ref: pr.Base.Ref, - SHA: pr.Base.Sha, + Ref: pr.Base.Ref, + SHA: pr.Base.Sha, + RepoName: pr.Base.Repo, + OwnerName: baseUser, }, Assignees: pr.Assignees, }) From d52fac055164c15d2bcc690bbe8f6058d9e2ef67 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 24 May 2022 19:28:32 +0800 Subject: [PATCH 49/54] Fix test --- services/migrations/gitea_uploader_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/migrations/gitea_uploader_test.go b/services/migrations/gitea_uploader_test.go index 9ba09f7b7f81b..d40feb6468f98 100644 --- a/services/migrations/gitea_uploader_test.go +++ b/services/migrations/gitea_uploader_test.go @@ -346,7 +346,7 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) { }, { name: "fork, invalid Head.Ref", - head: "unknown repository", + head: "user10/INVALID", pr: base.PullRequest{ PatchURL: "", Number: 1, From 98065f8f48c650fd6325d833a01c226ebe06a19a Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 24 May 2022 21:30:14 +0800 Subject: [PATCH 50/54] Fix test --- services/migrations/gitea_uploader_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/migrations/gitea_uploader_test.go b/services/migrations/gitea_uploader_test.go index d40feb6468f98..9e3888b626750 100644 --- a/services/migrations/gitea_uploader_test.go +++ b/services/migrations/gitea_uploader_test.go @@ -372,7 +372,7 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) { }, { name: "invalid fork CloneURL", - head: "unknown repository", + head: "WRONG/branch2", pr: base.PullRequest{ PatchURL: "", Number: 1, From a15d33fd45be4aab3df5e58445c1be4e259df5b4 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 18 Jun 2022 15:46:23 +0800 Subject: [PATCH 51/54] Fix lint --- services/migrations/dump.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/migrations/dump.go b/services/migrations/dump.go index bf0da90f8a863..7aa247f1f1f0d 100644 --- a/services/migrations/dump.go +++ b/services/migrations/dump.go @@ -670,7 +670,9 @@ func RestoreFromGithubExportedData(ctx context.Context, baseDir, ownerName, repo migrateOpts := base.MigrateOptions{ GitServiceType: structs.GithubService, } - updateOptionsUnits(&migrateOpts, units) + if err := updateOptionsUnits(&migrateOpts, units); err != nil { + return err + } uploader := NewGiteaLocalUploader(ctx, doer, ownerName, repoName) if err = migrateRepository(downloader, uploader, migrateOpts, nil); err != nil { From 468fe45bdf9e98e9edbfb7e664b491712244ed0f Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 20 Oct 2022 22:28:25 +0800 Subject: [PATCH 52/54] Fix check --- modules/migration/pullrequest.go | 2 +- services/migrations/github_exported_data.go | 331 +++++++++--------- .../repo/issue/view_content/comments.tmpl | 4 +- 3 files changed, 173 insertions(+), 164 deletions(-) diff --git a/modules/migration/pullrequest.go b/modules/migration/pullrequest.go index 93eaf7ab53c9f..082495d2981c9 100644 --- a/modules/migration/pullrequest.go +++ b/modules/migration/pullrequest.go @@ -38,7 +38,7 @@ type PullRequest struct { ForeignIndex int64 Context DownloaderContext `yaml:"-"` Assets []*Asset - EnsuredSafe bool `yaml:"ensured_safe"` + EnsuredSafe bool `yaml:"ensured_safe"` } func (p *PullRequest) GetLocalIndex() int64 { return p.Number } diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index 2fd636628522f..3baaea43b840d 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -27,7 +27,7 @@ import ( ) /* - {"version":"1.0.1","github_sha":"8de0984858fd99a8dcd2d756cf0f128b9161e3b5"} +{"version":"1.0.1","github_sha":"8de0984858fd99a8dcd2d756cf0f128b9161e3b5"} */ type githubSchema struct { Version string @@ -35,26 +35,26 @@ type githubSchema struct { } /* -{ - "type": "user", - "url": "https://github.com/sunvim", - "avatar_url": "https://avatars.githubusercontent.com/u/859692?v=4", - "login": "sunvim", - "name": "mobus", - "bio": "code happy ", - "company": "Ankr", - "website": null, - "location": "Shanghai", - "emails": [ - { - "address": "sv0220@163.com", - "primary": true, - "verified": true - } - ], - "billing_plan": null, - "created_at": "2011-06-19T11:25:35Z" - }, + { + "type": "user", + "url": "https://github.com/sunvim", + "avatar_url": "https://avatars.githubusercontent.com/u/859692?v=4", + "login": "sunvim", + "name": "mobus", + "bio": "code happy ", + "company": "Ankr", + "website": null, + "location": "Shanghai", + "emails": [ + { + "address": "sv0220@163.com", + "primary": true, + "verified": true + } + ], + "billing_plan": null, + "created_at": "2011-06-19T11:25:35Z" + }, */ type githubUser struct { URL string @@ -117,17 +117,18 @@ func (g *githubUser) Email() string { return "" } -/*{ - "type": "attachment", - "url": "https://user-images.githubusercontent.com/1595118/2923824-63a167ce-d721-11e3-91b6-74b83dc345bb.png", - "issue": "https://github.com/go-xorm/xorm/issues/205", - "issue_comment": "https://github.com/go-xorm/xorm/issues/115#issuecomment-42628488", - "user": "https://github.com/mintzhao", - "asset_name": "QQ20140509-1.2x.png", - "asset_content_type": "image/png", - "asset_url": "tarball://root/attachments/63a167ce-d721-11e3-91b6-74b83dc345bb/QQ20140509-1.2x.png", - "created_at": "2014-05-09T02:38:54Z" - }, +/* + { + "type": "attachment", + "url": "https://user-images.githubusercontent.com/1595118/2923824-63a167ce-d721-11e3-91b6-74b83dc345bb.png", + "issue": "https://github.com/go-xorm/xorm/issues/205", + "issue_comment": "https://github.com/go-xorm/xorm/issues/115#issuecomment-42628488", + "user": "https://github.com/mintzhao", + "asset_name": "QQ20140509-1.2x.png", + "asset_content_type": "image/png", + "asset_url": "tarball://root/attachments/63a167ce-d721-11e3-91b6-74b83dc345bb/QQ20140509-1.2x.png", + "created_at": "2014-05-09T02:38:54Z" + }, */ type githubAttachment struct { URL string `json:"url"` @@ -184,12 +185,13 @@ func (r *GithubExportedDataRestorer) convertAttachments(ls []githubAttachment) [ } /* -{ - "user": "https://github.com/mrsdizzie", - "content": "+1", - "subject_type": "Issue", - "created_at": "2019-11-13T04:22:13.000+08:00" - }*/ + { + "user": "https://github.com/mrsdizzie", + "content": "+1", + "subject_type": "Issue", + "created_at": "2019-11-13T04:22:13.000+08:00" + } +*/ type githubReaction struct { User string Content string @@ -501,19 +503,21 @@ func (r *GithubExportedDataRestorer) getAttachments() error { }) } -/* { - "type": "milestone", - "url": "https://github.com/go-gitea/test_repo/milestones/1", - "repository": "https://github.com/go-gitea/test_repo", - "user": "https://github.com/mrsdizzie", - "title": "1.0.0", - "description": "Milestone 1.0.0", - "state": "closed", - "due_on": "2019-11-11T00:00:00Z", - "created_at": "2019-11-12T19:37:08Z", - "updated_at": "2019-11-12T21:56:17Z", - "closed_at": "2019-11-12T19:45:49Z" - },*/ +/* + { + "type": "milestone", + "url": "https://github.com/go-gitea/test_repo/milestones/1", + "repository": "https://github.com/go-gitea/test_repo", + "user": "https://github.com/mrsdizzie", + "title": "1.0.0", + "description": "Milestone 1.0.0", + "state": "closed", + "due_on": "2019-11-11T00:00:00Z", + "created_at": "2019-11-12T19:37:08Z", + "updated_at": "2019-11-12T21:56:17Z", + "closed_at": "2019-11-12T19:45:49Z" + }, +*/ type githubMilestone struct { URL string User string @@ -554,24 +558,24 @@ func (r *GithubExportedDataRestorer) GetMilestones() ([]*base.Milestone, error) } /* -{ - "type": "release", - "url": "https://github.com/go-xorm/xorm/releases/tag/v0.3.1", - "repository": "https://github.com/go-xorm/xorm", - "user": "https://github.com/lunny", - "name": "", - "tag_name": "v0.3.1", - "body": "- Features:\n - Support MSSQL DB via ODBC driver ([github.com/lunny/godbc](https://github.com/lunny/godbc));\n - Composite Key, using multiple pk xorm tag \n - Added Row() API as alternative to Iterate() API for traversing result set, provide similar usages to sql.Rows type\n - ORM struct allowed declaration of pointer builtin type as members to allow null DB fields \n - Before and After Event processors\n- Improvements:\n - Allowed int/int32/int64/uint/uint32/uint64/string as Primary Key type\n - Performance improvement for Get()/Find()/Iterate()\n", - "state": "published", - "pending_tag": "v0.3.1", - "prerelease": false, - "target_commitish": "master", - "release_assets": [ - - ], - "published_at": "2014-01-02T09:51:34Z", - "created_at": "2014-01-02T09:48:57Z" - }, + { + "type": "release", + "url": "https://github.com/go-xorm/xorm/releases/tag/v0.3.1", + "repository": "https://github.com/go-xorm/xorm", + "user": "https://github.com/lunny", + "name": "", + "tag_name": "v0.3.1", + "body": "- Features:\n - Support MSSQL DB via ODBC driver ([github.com/lunny/godbc](https://github.com/lunny/godbc));\n - Composite Key, using multiple pk xorm tag \n - Added Row() API as alternative to Iterate() API for traversing result set, provide similar usages to sql.Rows type\n - ORM struct allowed declaration of pointer builtin type as members to allow null DB fields \n - Before and After Event processors\n- Improvements:\n - Allowed int/int32/int64/uint/uint32/uint64/string as Primary Key type\n - Performance improvement for Get()/Find()/Iterate()\n", + "state": "published", + "pending_tag": "v0.3.1", + "prerelease": false, + "target_commitish": "master", + "release_assets": [ + + ], + "published_at": "2014-01-02T09:51:34Z", + "created_at": "2014-01-02T09:48:57Z" + }, */ type githubRelease struct { User string @@ -631,29 +635,30 @@ func (r *GithubExportedDataRestorer) GetLabels() ([]*base.Label, error) { } /* - { - "type": "issue", - "url": "https://github.com/go-xorm/xorm/issues/1", - "repository": "https://github.com/go-xorm/xorm", - "user": "https://github.com/zakzou", - "title": "建表功能已经强大了，不过希望添加上自定义mysql engine和charset", - "body": "如题\n", - "assignee": "https://github.com/lunny", - "assignees": [ - "https://github.com/lunny" - ], - "milestone": null, - "labels": [ - "https://github.com/go-gitea/test_repo/labels/bug", - "https://github.com/go-gitea/test_repo/labels/good%20first%20issue" - ], - "reactions": [ - - ], - "closed_at": "2013-08-08T05:26:00Z", - "created_at": "2013-07-04T08:08:39Z", - "updated_at": "2013-08-08T05:26:00Z" - },*/ + { + "type": "issue", + "url": "https://github.com/go-xorm/xorm/issues/1", + "repository": "https://github.com/go-xorm/xorm", + "user": "https://github.com/zakzou", + "title": "建表功能已经强大了，不过希望添加上自定义mysql engine和charset", + "body": "如题\n", + "assignee": "https://github.com/lunny", + "assignees": [ + "https://github.com/lunny" + ], + "milestone": null, + "labels": [ + "https://github.com/go-gitea/test_repo/labels/bug", + "https://github.com/go-gitea/test_repo/labels/good%20first%20issue" + ], + "reactions": [ + + ], + "closed_at": "2013-08-08T05:26:00Z", + "created_at": "2013-07-04T08:08:39Z", + "updated_at": "2013-08-08T05:26:00Z" + }, +*/ type githubIssue struct { URL string User string @@ -793,32 +798,33 @@ func (g *githubComment) GetIssueIndex() int64 { } /* -{ - "type": "issue_event", - "url": "https://github.com/go-xorm/xorm/issues/1#event-55275262", - "issue": "https://github.com/go-xorm/xorm/issues/1", - "actor": "https://github.com/lunny", - "event": "assigned", - "created_at": "2013-07-04T14:27:53Z" - }, - { - "type": "issue_event", - "url": "https://github.com/go-xorm/xorm/pull/2#event-56230828", - "pull_request": "https://github.com/go-xorm/xorm/pull/2", - "actor": "https://github.com/lunny", - "event": "referenced", - "commit_id": "1be80583b0fa18e7b478fa12e129c95e9a06a62f", - "commit_repository": "https://github.com/go-xorm/xorm", - "created_at": "2013-07-12T02:10:52Z" - - "label": "https://github.com/go-xorm/xorm/labels/wip", - "label_name": "New Feature", - "label_color": "5319e7", - "label_text_color": "fff", - "milestone_title": "v0.4", - "title_was": "自动读写分离", - "title_is": "Automatical Read/Write seperatelly.", - },*/ + { + "type": "issue_event", + "url": "https://github.com/go-xorm/xorm/issues/1#event-55275262", + "issue": "https://github.com/go-xorm/xorm/issues/1", + "actor": "https://github.com/lunny", + "event": "assigned", + "created_at": "2013-07-04T14:27:53Z" + }, + { + "type": "issue_event", + "url": "https://github.com/go-xorm/xorm/pull/2#event-56230828", + "pull_request": "https://github.com/go-xorm/xorm/pull/2", + "actor": "https://github.com/lunny", + "event": "referenced", + "commit_id": "1be80583b0fa18e7b478fa12e129c95e9a06a62f", + "commit_repository": "https://github.com/go-xorm/xorm", + "created_at": "2013-07-12T02:10:52Z" + + "label": "https://github.com/go-xorm/xorm/labels/wip", + "label_name": "New Feature", + "label_color": "5319e7", + "label_text_color": "fff", + "milestone_title": "v0.4", + "title_was": "自动读写分离", + "title_is": "Automatical Read/Write seperatelly.", + }, +*/ type githubIssueEvent struct { URL string Issue string @@ -1266,27 +1272,28 @@ func (p *pullrequestReview) GetState() string { } /* -{ - "type": "pull_request_review_thread", - "url": "https://github.com/go-xorm/xorm/pull/1445/files#pullrequestreviewthread-203253693", - "pull_request": "https://github.com/go-xorm/xorm/pull/1445", - "pull_request_review": "https://github.com/go-xorm/xorm/pull/1445/files#pullrequestreview-295977501", - "diff_hunk": "@@ -245,12 +245,17 @@ func (session *Session) Sync2(beans ...interface{}) error {\n \t\tif err != nil {\n \t\t\treturn err\n \t\t}\n-\t\ttbName := engine.TableName(bean)\n-\t\ttbNameWithSchema := engine.TableName(tbName, true)\n+\t\tvar tbName string\n+\t\tif len(session.statement.AltTableName) > 0 {\n+\t\t\ttbName = session.statement.AltTableName\n+\t\t} else {\n+\t\t\ttbName = engine.TableName(bean)\n+\t\t}\n+\t\ttbNameWithSchema := engine.tbNameWithSchema(tbName)\n \n \t\tvar oriTable *core.Table\n \t\tfor _, tb := range tables {\n-\t\t\tif strings.EqualFold(tb.Name, tbName) {\n+\t\t\tif strings.EqualFold(engine.tbNameWithSchema(tb.Name), engine.tbNameWithSchema(tbName)) {", - "path": "session_schema.go", - "position": 17, - "original_position": 17, - "commit_id": "f6b642c82aab95178a4551a1ff65dc2a631a08cf", - "original_commit_id": "f6b642c82aab95178a4551a1ff65dc2a631a08cf", - "start_line": null, - "line": 258, - "start_side": null, - "side": "right", - "original_start_line": null, - "original_line": 258, - "created_at": "2019-10-02T01:40:41Z", - "resolved_at": null, - "resolver": null - },*/ + { + "type": "pull_request_review_thread", + "url": "https://github.com/go-xorm/xorm/pull/1445/files#pullrequestreviewthread-203253693", + "pull_request": "https://github.com/go-xorm/xorm/pull/1445", + "pull_request_review": "https://github.com/go-xorm/xorm/pull/1445/files#pullrequestreview-295977501", + "diff_hunk": "@@ -245,12 +245,17 @@ func (session *Session) Sync2(beans ...interface{}) error {\n \t\tif err != nil {\n \t\t\treturn err\n \t\t}\n-\t\ttbName := engine.TableName(bean)\n-\t\ttbNameWithSchema := engine.TableName(tbName, true)\n+\t\tvar tbName string\n+\t\tif len(session.statement.AltTableName) > 0 {\n+\t\t\ttbName = session.statement.AltTableName\n+\t\t} else {\n+\t\t\ttbName = engine.TableName(bean)\n+\t\t}\n+\t\ttbNameWithSchema := engine.tbNameWithSchema(tbName)\n \n \t\tvar oriTable *core.Table\n \t\tfor _, tb := range tables {\n-\t\t\tif strings.EqualFold(tb.Name, tbName) {\n+\t\t\tif strings.EqualFold(engine.tbNameWithSchema(tb.Name), engine.tbNameWithSchema(tbName)) {", + "path": "session_schema.go", + "position": 17, + "original_position": 17, + "commit_id": "f6b642c82aab95178a4551a1ff65dc2a631a08cf", + "original_commit_id": "f6b642c82aab95178a4551a1ff65dc2a631a08cf", + "start_line": null, + "line": 258, + "start_side": null, + "side": "right", + "original_start_line": null, + "original_line": 258, + "created_at": "2019-10-02T01:40:41Z", + "resolved_at": null, + "resolver": null + }, +*/ type pullrequestReviewThread struct { URL string PullRequest string `json:"pull_request"` @@ -1311,28 +1318,30 @@ func (p *pullrequestReviewThread) Index() int64 { return idx } -/*{ - "type": "pull_request_review_comment", - "url": "https://github.com/go-gitea/test_repo/pull/4/files#r363017488", - "pull_request": "https://github.com/go-gitea/test_repo/pull/4", - "pull_request_review": "https://github.com/go-gitea/test_repo/pull/4/files#pullrequestreview-338338740", - "pull_request_review_thread": "https://github.com/go-gitea/test_repo/pull/4/files#pullrequestreviewthread-224172719", - "user": "https://github.com/lunny", - "body": "This is a good pull request.", - "formatter": "markdown", - "diff_hunk": "@@ -1,2 +1,4 @@\n # test_repo\n Test repository for testing migration from github to gitea\n+", - "path": "README.md", - "position": 3, - "original_position": 3, - "commit_id": "2be9101c543658591222acbee3eb799edfc3853d", - "original_commit_id": "2be9101c543658591222acbee3eb799edfc3853d", - "state": 1, - "in_reply_to": null, - "reactions": [ - - ], - "created_at": "2020-01-04T05:33:06Z" -},*/ +/* + { + "type": "pull_request_review_comment", + "url": "https://github.com/go-gitea/test_repo/pull/4/files#r363017488", + "pull_request": "https://github.com/go-gitea/test_repo/pull/4", + "pull_request_review": "https://github.com/go-gitea/test_repo/pull/4/files#pullrequestreview-338338740", + "pull_request_review_thread": "https://github.com/go-gitea/test_repo/pull/4/files#pullrequestreviewthread-224172719", + "user": "https://github.com/lunny", + "body": "This is a good pull request.", + "formatter": "markdown", + "diff_hunk": "@@ -1,2 +1,4 @@\n # test_repo\n Test repository for testing migration from github to gitea\n+", + "path": "README.md", + "position": 3, + "original_position": 3, + "commit_id": "2be9101c543658591222acbee3eb799edfc3853d", + "original_commit_id": "2be9101c543658591222acbee3eb799edfc3853d", + "state": 1, + "in_reply_to": null, + "reactions": [ + + ], + "created_at": "2020-01-04T05:33:06Z" + }, +*/ type pullrequestReviewComment struct { PullRequest string `json:"pull_request"` PullRequestReview string `json:"pull_request_review"` diff --git a/templates/repo/issue/view_content/comments.tmpl b/templates/repo/issue/view_content/comments.tmpl index c1db5719c3856..6900ace0eb1e1 100644 --- a/templates/repo/issue/view_content/comments.tmpl +++ b/templates/repo/issue/view_content/comments.tmpl @@ -123,7 +123,7 @@ {{.Poster.GetDisplayName}} - {{if .Issue.IsPull }} + {{if .Issue.IsPull}} {{$.locale.Tr "repo.pulls.closed_at" .EventTag $createdStr | Safe}} {{else}} {{$.locale.Tr "repo.issues.closed_at" .EventTag $createdStr | Safe}} @@ -440,7 +440,7 @@
{{$.locale.Tr "repo.issues.review.dismissed_label"}}
{{end}} {{if $.Repository.OriginalURL}} - ({{$.locale.Tr "repo.migrated_from" ($.Repository.OriginalURL|Escape) ($.Repository.GetOriginalURLHostname|Escape) | Safe }}) + ({{$.locale.Tr "repo.migrated_from" ($.Repository.OriginalURL|Escape) ($.Repository.GetOriginalURLHostname|Escape) | Safe}}) {{end}}
From f2ed8b1cb80958f2252a9bec3a5ff1324ed2a0b2 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Wed, 1 Feb 2023 16:48:48 +0800 Subject: [PATCH 53/54] chore: lint codes --- modules/migration/asset.go | 3 +-- services/migrations/github_exported_data.go | 5 ++--- services/migrations/github_exported_data_test.go | 3 +-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/modules/migration/asset.go b/modules/migration/asset.go index 464aa4a0dff34..162e3ce74d274 100644 --- a/modules/migration/asset.go +++ b/modules/migration/asset.go @@ -1,6 +1,5 @@ // Copyright 2022 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT package migration diff --git a/services/migrations/github_exported_data.go b/services/migrations/github_exported_data.go index 3baaea43b840d..27c64b9327fee 100644 --- a/services/migrations/github_exported_data.go +++ b/services/migrations/github_exported_data.go @@ -1,6 +1,5 @@ // Copyright 2022 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT package migrations @@ -1028,8 +1027,8 @@ func (r *GithubExportedDataRestorer) getIssueEvents() ([]*base.Comment, error) { } comments = append(comments, &base.Comment{ - Type: c.CommentStr(), IssueIndex: c.GetIssueIndex(), + CommentType: c.CommentStr(), PosterID: id, PosterName: login, PosterEmail: email, diff --git a/services/migrations/github_exported_data_test.go b/services/migrations/github_exported_data_test.go index 92335899dfe0a..09ab139801a37 100644 --- a/services/migrations/github_exported_data_test.go +++ b/services/migrations/github_exported_data_test.go @@ -1,6 +1,5 @@ // Copyright 2022 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT package migrations From 61c640311e1495ab0e11c76895ddbc82873b07cc Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 9 Feb 2023 11:55:21 +0800 Subject: [PATCH 54/54] chore: revert intended rm --- templates/repo/issue/view_content/comments.tmpl | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/repo/issue/view_content/comments.tmpl b/templates/repo/issue/view_content/comments.tmpl index 3f85ee8796693..8e0755181282b 100644 --- a/templates/repo/issue/view_content/comments.tmpl +++ b/templates/repo/issue/view_content/comments.tmpl @@ -586,6 +586,7 @@ {{else}} {{template "shared/user/authorlink" .Poster}} {{end}} + {{$.locale.Tr "repo.issues.commented_at" (.HashTag|Escape) $createdSubStr | Safe}}