From 7798250b87a533a9d4e4b36d6cbe67b7c41cc8a5 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 14 Jul 2020 11:46:15 +0800 Subject: [PATCH 01/47] Dump github/gitlab repository data to a local directory --- cmd/dump_repo.go | 66 +++++ main.go | 1 + modules/migrations/dump.go | 501 ++++++++++++++++++++++++++++++++++ modules/migrations/migrate.go | 26 +- 4 files changed, 584 insertions(+), 10 deletions(-) create mode 100644 cmd/dump_repo.go create mode 100644 modules/migrations/dump.go diff --git a/cmd/dump_repo.go b/cmd/dump_repo.go new file mode 100644 index 0000000000000..f886691d641ab --- /dev/null +++ b/cmd/dump_repo.go @@ -0,0 +1,66 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "context" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migrations" + "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/setting" + + "github.com/urfave/cli" +) + +// CmdDumpRepository represents the available dump repository sub-command. +var CmdDumpRepository = cli.Command{ + Name: "dump-repo", + Usage: "Dump the repository from github/gitlab", + Description: "This is a command for dumping the repository data.", + Action: runDumpRepository, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "repo_dir, r", + Value: "./data", + Usage: "Repository dir path", + }, + }, +} + +func runDumpRepository(ctx *cli.Context) error { + if err := initDB(); err != nil { + return err + } + + log.Trace("AppPath: %s", setting.AppPath) + log.Trace("AppWorkPath: %s", setting.AppWorkPath) + log.Trace("Custom path: %s", setting.CustomPath) + log.Trace("Log path: %s", setting.LogRootPath) + setting.InitDBConfig() + + if err := migrations.DumpRepository( + context.Background(), + ctx.String("repo_dir"), + ctx.String("owner_name"), + base.MigrateOptions{ + CloneAddr: ctx.String("clone_addr"), + AuthUsername: ctx.String("auth_username"), + AuthPassword: ctx.String("auth_password"), + RepoName: ctx.String("repo_name"), + Wiki: true, + Issues: true, + Milestones: true, + Labels: true, + Releases: true, + Comments: true, + PullRequests: true, + }); err != nil { + log.Fatal("Failed to dump repository: %v", err) + return err + } + + return nil +} diff --git a/main.go b/main.go index 8ee6ffa92ccdc..29eb21d535f33 100644 --- a/main.go +++ b/main.go @@ -72,6 +72,7 @@ arguments - which can alternatively be run by running the subcommand web.` cmd.Cmdembedded, cmd.CmdMigrateStorage, cmd.CmdDocs, + cmd.CmdDumpRepository, } // Now adjust these commands to add our global configuration options diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go new file mode 100644 index 0000000000000..ca293bedc3381 --- /dev/null +++ b/modules/migrations/dump.go @@ -0,0 +1,501 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/structs" + + "gopkg.in/yaml.v2" +) + +var ( + _ base.Uploader = &RepositoryDumper{} +) + +// RepositoryDumper implements an Uploader to the local directory +type RepositoryDumper struct { + ctx context.Context + baseDir string + repoOwner string + repoName string + milestoneFile *os.File + labelFile *os.File + releaseFile *os.File + issueFile *os.File + commentFile *os.File + pullrequestFile *os.File + reviewFile *os.File + + gitRepo *git.Repository + prHeadCache map[string]struct{} + userMap map[int64]int64 // external user id mapping to user id + prCache map[int64]*models.PullRequest + gitServiceType structs.GitServiceType +} + +// NewRepositoryDumper creates an gitea Uploader +func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName string) *RepositoryDumper { + return &RepositoryDumper{ + ctx: ctx, + baseDir: baseDir, + repoOwner: repoOwner, + repoName: repoName, + prHeadCache: make(map[string]struct{}), + userMap: make(map[int64]int64), + prCache: make(map[int64]*models.PullRequest), + } +} + +// MaxBatchInsertSize returns the table's max batch insert size +func (g *RepositoryDumper) MaxBatchInsertSize(tp string) int { + return 1000 +} + +func (g *RepositoryDumper) repoPath() string { + return filepath.Join(g.baseDir, "git") +} + +func (g *RepositoryDumper) wikiPath() string { + return filepath.Join(g.baseDir, "wiki") +} + +func (g *RepositoryDumper) topicDir() string { + return filepath.Join(g.baseDir, "topic") +} + +func (g *RepositoryDumper) milestoneDir() string { + return filepath.Join(g.baseDir, "milestone") +} + +func (g *RepositoryDumper) labelDir() string { + return filepath.Join(g.baseDir, "label") +} + +func (g *RepositoryDumper) releaseDir() string { + return filepath.Join(g.baseDir, "release") +} + +func (g *RepositoryDumper) issueDir() string { + return filepath.Join(g.baseDir, "issue") +} + +func (g *RepositoryDumper) commentDir() string { + return filepath.Join(g.baseDir, "comment") +} + +func (g *RepositoryDumper) pullrequestDir() string { + return filepath.Join(g.baseDir, "pullrequest") +} + +func (g *RepositoryDumper) reviewDir() string { + return filepath.Join(g.baseDir, "review") +} + +// CreateRepo creates a repository +func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error { + repoPath := g.repoPath() + if err := os.MkdirAll(repoPath, os.ModePerm); err != nil { + return err + } + + migrateTimeout := 2 * time.Hour + + err := git.Clone(opts.CloneAddr, repoPath, git.CloneRepoOptions{ + Mirror: true, + Quiet: true, + Timeout: migrateTimeout, + }) + if err != nil { + return fmt.Errorf("Clone: %v", err) + } + + if opts.Wiki { + wikiPath := g.wikiPath() + wikiRemotePath := repository.WikiRemoteURL(opts.CloneAddr) + if len(wikiRemotePath) > 0 { + if err := os.MkdirAll(wikiPath, os.ModePerm); err != nil { + return fmt.Errorf("Failed to remove %s: %v", wikiPath, err) + } + + if err := git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{ + Mirror: true, + Quiet: true, + Timeout: migrateTimeout, + Branch: "master", + }); err != nil { + log.Warn("Clone wiki: %v", err) + if err := os.RemoveAll(wikiPath); err != nil { + return fmt.Errorf("Failed to remove %s: %v", wikiPath, err) + } + } + } + } + + g.gitRepo, err = git.OpenRepository(g.repoPath()) + return err +} + +// Close closes this uploader +func (g *RepositoryDumper) Close() { + if g.gitRepo != nil { + g.gitRepo.Close() + } + if g.milestoneFile != nil { + g.milestoneFile.Close() + } + if g.labelFile != nil { + g.labelFile.Close() + } + if g.releaseFile != nil { + g.releaseFile.Close() + } + if g.issueFile != nil { + g.issueFile.Close() + } + if g.pullrequestFile != nil { + g.pullrequestFile.Close() + } + if g.reviewFile != nil { + g.reviewFile.Close() + } +} + +// CreateTopics creates topics +func (g *RepositoryDumper) CreateTopics(topics ...string) error { + if err := os.MkdirAll(g.topicDir(), os.ModePerm); err != nil { + return err + } + f, err := os.Create(filepath.Join(g.topicDir(), "data.yml")) + if err != nil { + return err + } + defer f.Close() + + bs, err := yaml.Marshal(map[string]interface{}{ + "topics": topics, + }) + if err != nil { + return err + } + + if _, err := f.Write(bs); err != nil { + return err + } + + return nil +} + +// CreateMilestones creates milestones +func (g *RepositoryDumper) CreateMilestones(milestones ...*base.Milestone) error { + var err error + if g.milestoneFile == nil { + if err := os.MkdirAll(g.milestoneDir(), os.ModePerm); err != nil { + return err + } + g.milestoneFile, err = os.Create(filepath.Join(g.milestoneDir(), "data.yml")) + if err != nil { + return err + } + } + + bs, err := yaml.Marshal(milestones) + if err != nil { + return err + } + + if _, err := g.milestoneFile.Write(bs); err != nil { + return err + } + + return nil +} + +// CreateLabels creates labels +func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error { + var err error + if g.labelFile == nil { + if err := os.MkdirAll(g.labelDir(), os.ModePerm); err != nil { + return err + } + g.labelFile, err = os.Create(filepath.Join(g.labelDir(), "data.yml")) + if err != nil { + return err + } + } + + bs, err := yaml.Marshal(labels) + if err != nil { + return err + } + + if _, err := g.labelFile.Write(bs); err != nil { + return err + } + + return nil +} + +// CreateReleases creates releases +func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { + for _, release := range releases { + attachDir := filepath.Join(g.releaseDir(), "assets", release.Name) + if err := os.MkdirAll(attachDir, os.ModePerm); err != nil { + return err + } + for _, asset := range release.Assets { + attachLocalPath := filepath.Join(attachDir, asset.Name) + // download attachment + err := func(attachLocalPath string) error { + resp, err := http.Get(asset.URL) + if err != nil { + return err + } + defer resp.Body.Close() + + fw, err := os.Create(attachLocalPath) + if err != nil { + return fmt.Errorf("Create: %v", err) + } + defer fw.Close() + + _, err = io.Copy(fw, resp.Body) + return err + }(attachLocalPath) + if err != nil { + return err + } + } + } + + var err error + if g.releaseFile == nil { + if err := os.MkdirAll(g.releaseDir(), os.ModePerm); err != nil { + return err + } + g.releaseFile, err = os.Create(filepath.Join(g.releaseDir(), "data.yml")) + if err != nil { + return err + } + } + + bs, err := yaml.Marshal(releases) + if err != nil { + return err + } + + if _, err := g.releaseFile.Write(bs); err != nil { + return err + } + + return nil +} + +// SyncTags syncs releases with tags in the database +func (g *RepositoryDumper) SyncTags() error { + return nil +} + +// CreateIssues creates issues +func (g *RepositoryDumper) CreateIssues(issues ...*base.Issue) error { + var err error + if g.issueFile == nil { + if err := os.MkdirAll(g.issueDir(), os.ModePerm); err != nil { + return err + } + g.issueFile, err = os.Create(filepath.Join(g.issueDir(), "data.yml")) + if err != nil { + return err + } + } + + bs, err := yaml.Marshal(issues) + if err != nil { + return err + } + + if _, err := g.issueFile.Write(bs); err != nil { + return err + } + + return nil +} + +// CreateComments creates comments of issues +func (g *RepositoryDumper) CreateComments(comments ...*base.Comment) error { + var err error + if g.commentFile == nil { + if err := os.MkdirAll(g.commentDir(), os.ModePerm); err != nil { + return err + } + g.commentFile, err = os.Create(filepath.Join(g.commentDir(), "data.yml")) + if err != nil { + return err + } + } + + bs, err := yaml.Marshal(comments) + if err != nil { + return err + } + + if _, err := g.commentFile.Write(bs); err != nil { + return err + } + + return nil +} + +// CreatePullRequests creates pull requests +func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { + for _, pr := range prs { + // download patch file + err := func() error { + resp, err := http.Get(pr.PatchURL) + if err != nil { + return err + } + defer resp.Body.Close() + pullDir := filepath.Join(g.repoPath(), "pulls") + if err = os.MkdirAll(pullDir, os.ModePerm); err != nil { + return err + } + f, err := os.Create(filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number))) + if err != nil { + return err + } + defer f.Close() + _, err = io.Copy(f, resp.Body) + return err + }() + if err != nil { + return err + } + + // set head information + pullHead := filepath.Join(g.repoPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number)) + if err := os.MkdirAll(pullHead, os.ModePerm); err != nil { + return err + } + p, err := os.Create(filepath.Join(pullHead, "head")) + if err != nil { + return err + } + _, err = p.WriteString(pr.Head.SHA) + p.Close() + if err != nil { + return err + } + + if pr.IsForkPullRequest() && pr.State != "closed" { + if pr.Head.OwnerName != "" { + remote := pr.Head.OwnerName + _, ok := g.prHeadCache[remote] + if !ok { + // git remote add + err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true) + if err != nil { + log.Error("AddRemote failed: %s", err) + } else { + g.prHeadCache[remote] = struct{}{} + ok = true + } + } + + if ok { + _, err = git.NewCommand("fetch", remote, pr.Head.Ref).RunInDir(g.repoPath()) + if err != nil { + log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err) + } else { + headBranch := filepath.Join(g.repoPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref) + if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil { + return err + } + b, err := os.Create(headBranch) + if err != nil { + return err + } + _, err = b.WriteString(pr.Head.SHA) + b.Close() + if err != nil { + return err + } + } + } + } + } + } + + var err error + if g.pullrequestFile == nil { + if err := os.MkdirAll(g.pullrequestDir(), os.ModePerm); err != nil { + return err + } + g.pullrequestFile, err = os.Create(filepath.Join(g.pullrequestDir(), "data.yml")) + if err != nil { + return err + } + } + + bs, err := yaml.Marshal(prs) + if err != nil { + return err + } + + if _, err := g.pullrequestFile.Write(bs); err != nil { + return err + } + + return nil +} + +// CreateReviews create pull request reviews +func (g *RepositoryDumper) CreateReviews(reviews ...*base.Review) error { + var err error + if g.reviewFile == nil { + if err := os.MkdirAll(g.reviewDir(), os.ModePerm); err != nil { + return err + } + g.reviewFile, err = os.Create(filepath.Join(g.reviewDir(), "data.yml")) + if err != nil { + return err + } + } + + bs, err := yaml.Marshal(reviews) + if err != nil { + return err + } + + if _, err := g.reviewFile.Write(bs); err != nil { + return err + } + + return nil +} + +// Rollback when migrating failed, this will rollback all the changes. +func (g *RepositoryDumper) Rollback() error { + g.Close() + return os.RemoveAll(g.baseDir) +} + +// DumpRepository dump repository according MigrateOptions to a local directory +func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.MigrateOptions) error { + var uploader = NewRepositoryDumper(ctx, baseDir, ownerName, opts.RepoName) + return MigrateRepositoryWithUploader(ctx, ownerName, opts, uploader) +} diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index b3ecb8114a402..bf693cb0e45e5 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -73,17 +73,29 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, if err != nil { return nil, err } + var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) + uploader.gitServiceType = opts.GitServiceType + if err := MigrateRepositoryWithUploader(ctx, ownerName, opts, uploader); err != nil { + if err2 := models.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil { + log.Error("create respotiry notice failed: ", err2) + } + return nil, err + } + return uploader.repo, nil +} +// MigrateRepositoryWithUploader migrate repository according MigrateOptions and uploader +func MigrateRepositoryWithUploader(ctx context.Context, ownerName string, opts base.MigrateOptions, uploader base.Uploader) error { var ( downloader base.Downloader - uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) + err error ) for _, factory := range factories { if factory.GitServiceType() == opts.GitServiceType { downloader, err = factory.New(ctx, opts) if err != nil { - return nil, err + return err } break } @@ -101,8 +113,6 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, log.Trace("Will migrate from git: %s", opts.OriginalURL) } - uploader.gitServiceType = opts.GitServiceType - if setting.Migrations.MaxAttempts > 1 { downloader = base.NewRetryDownloader(ctx, downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff) } @@ -111,14 +121,10 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, if err1 := uploader.Rollback(); err1 != nil { log.Error("rollback failed: %v", err1) } - - if err2 := models.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil { - log.Error("create repository notice failed: ", err2) - } - return nil, err + return err } - return uploader.repo, nil + return nil } // migrateRepository will download information and then upload it to Uploader, this is a simple From 9fa7df47141a2671e843721978b9eb50f4baf06f Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 14 Jul 2020 11:54:46 +0800 Subject: [PATCH 02/47] Fix lint --- modules/migrations/dump.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index ca293bedc3381..6760d132636ec 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -13,12 +13,10 @@ import ( "path/filepath" "time" - "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/repository" - "code.gitea.io/gitea/modules/structs" "gopkg.in/yaml.v2" ) @@ -41,11 +39,8 @@ type RepositoryDumper struct { pullrequestFile *os.File reviewFile *os.File - gitRepo *git.Repository - prHeadCache map[string]struct{} - userMap map[int64]int64 // external user id mapping to user id - prCache map[int64]*models.PullRequest - gitServiceType structs.GitServiceType + gitRepo *git.Repository + prHeadCache map[string]struct{} } // NewRepositoryDumper creates an gitea Uploader @@ -56,8 +51,6 @@ func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName strin repoOwner: repoOwner, repoName: repoName, prHeadCache: make(map[string]struct{}), - userMap: make(map[int64]int64), - prCache: make(map[int64]*models.PullRequest), } } From 13d4b3af6ee1b50510ccb5b8d3f6d333f1cc052f Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 18 Jul 2020 15:56:47 +0800 Subject: [PATCH 03/47] Adjust directory structure --- cmd/dump_repo.go | 25 +++++++++++++ modules/migrations/base/comment.go | 8 ++-- modules/migrations/base/issue.go | 8 ++-- modules/migrations/base/pullrequest.go | 22 +++++------ modules/migrations/base/reaction.go | 4 +- modules/migrations/base/release.go | 14 +++---- modules/migrations/base/repo.go | 10 +++-- modules/migrations/base/review.go | 26 ++++++------- modules/migrations/dump.go | 52 ++++++++++++++------------ 9 files changed, 100 insertions(+), 69 deletions(-) diff --git a/cmd/dump_repo.go b/cmd/dump_repo.go index f886691d641ab..c0618b834534d 100644 --- a/cmd/dump_repo.go +++ b/cmd/dump_repo.go @@ -27,6 +27,31 @@ var CmdDumpRepository = cli.Command{ Value: "./data", Usage: "Repository dir path", }, + cli.StringFlag{ + Name: "clone_addr", + Value: "", + Usage: "Clone URL", + }, + cli.StringFlag{ + Name: "auth_username", + Value: "", + Usage: "", + }, + cli.StringFlag{ + Name: "auth_password", + Value: "", + Usage: "", + }, + cli.StringFlag{ + Name: "owner_name", + Value: "", + Usage: "", + }, + cli.StringFlag{ + Name: "repo_name", + Value: "", + Usage: "", + }, }, } diff --git a/modules/migrations/base/comment.go b/modules/migrations/base/comment.go index 4a653e474b0f5..3c32e63b82939 100644 --- a/modules/migrations/base/comment.go +++ b/modules/migrations/base/comment.go @@ -9,10 +9,10 @@ import "time" // Comment is a standard comment information type Comment struct { - IssueIndex int64 - PosterID int64 - PosterName string - PosterEmail string + IssueIndex int64 `yaml:"issue_index"` + PosterID int64 `yaml:"poster_id"` + PosterName string `yaml:"poster_name"` + PosterEmail string `yaml:"poster_email"` Created time.Time Updated time.Time Content string diff --git a/modules/migrations/base/issue.go b/modules/migrations/base/issue.go index f9dc8b93feecd..8b1b4612444d7 100644 --- a/modules/migrations/base/issue.go +++ b/modules/migrations/base/issue.go @@ -10,15 +10,15 @@ import "time" // Issue is a standard issue information type Issue struct { Number int64 - PosterID int64 - PosterName string - PosterEmail string + PosterID int64 `yaml:"poster_id"` + PosterName string `yaml:"poster_name"` + PosterEmail string `yaml:"poster_email"` Title string Content string Ref string Milestone string State string // closed, open - IsLocked bool + IsLocked bool `yaml:"is_locked"` Created time.Time Updated time.Time Closed *time.Time diff --git a/modules/migrations/base/pullrequest.go b/modules/migrations/base/pullrequest.go index ee612fbb8e836..6411137d0aa33 100644 --- a/modules/migrations/base/pullrequest.go +++ b/modules/migrations/base/pullrequest.go @@ -13,11 +13,11 @@ import ( // PullRequest defines a standard pull request information type PullRequest struct { Number int64 - OriginalNumber int64 + OriginalNumber int64 `yaml:"original_number"` Title string - PosterName string - PosterID int64 - PosterEmail string + PosterName string `yaml:"poster_name"` + PosterID int64 `yaml:"poster_id"` + PosterEmail string `yaml:"poster_email"` Content string Milestone string State string @@ -25,14 +25,14 @@ type PullRequest struct { Updated time.Time Closed *time.Time Labels []*Label - PatchURL string + PatchURL string `yaml:"patch_url"` Merged bool - MergedTime *time.Time - MergeCommitSHA string + MergedTime *time.Time `yaml:"merged_time"` + MergeCommitSHA string `yaml:"merge_commit_sha"` Head PullRequestBranch Base PullRequestBranch Assignees []string - IsLocked bool + IsLocked bool `yaml:"is_locked"` Reactions []*Reaction } @@ -43,11 +43,11 @@ func (p *PullRequest) IsForkPullRequest() bool { // PullRequestBranch represents a pull request branch type PullRequestBranch struct { - CloneURL string + CloneURL string `yaml:"clone_url"` Ref string SHA string - RepoName string - OwnerName string + RepoName string `yaml:"repo_name"` + OwnerName string `yaml:"owner_name"` } // RepoPath returns pull request repo path diff --git a/modules/migrations/base/reaction.go b/modules/migrations/base/reaction.go index b79223d4cd8e7..1519499134759 100644 --- a/modules/migrations/base/reaction.go +++ b/modules/migrations/base/reaction.go @@ -6,7 +6,7 @@ package base // Reaction represents a reaction to an issue/pr/comment. type Reaction struct { - UserID int64 - UserName string + UserID int64 `yaml:"user_id"` + UserName string `yaml:"user_name"` Content string } diff --git a/modules/migrations/base/release.go b/modules/migrations/base/release.go index c9b26ab1dae03..777b3a3fe0fff 100644 --- a/modules/migrations/base/release.go +++ b/modules/migrations/base/release.go @@ -10,9 +10,9 @@ import "time" type ReleaseAsset struct { ID int64 Name string - ContentType *string + ContentType *string `yaml:"content_type"` Size *int - DownloadCount *int + DownloadCount *int `yaml:"download_count"` Created time.Time Updated time.Time DownloadURL *string @@ -20,15 +20,15 @@ type ReleaseAsset struct { // Release represents a release type Release struct { - TagName string - TargetCommitish string + TagName string `yaml:"tag_name"` + TargetCommitish string `yaml:"target_commitish"` Name string Body string Draft bool Prerelease bool - PublisherID int64 - PublisherName string - PublisherEmail string + PublisherID int64 `yaml:"publisher_id"` + PublisherName string `yaml:"publisher_name"` + PublisherEmail string `yaml:"publisher_email"` Assets []ReleaseAsset Created time.Time Published time.Time diff --git a/modules/migrations/base/repo.go b/modules/migrations/base/repo.go index d26a9118545d2..aca50c645a080 100644 --- a/modules/migrations/base/repo.go +++ b/modules/migrations/base/repo.go @@ -9,10 +9,12 @@ package base type Repository struct { Name string Owner string - IsPrivate bool - IsMirror bool + IsPrivate bool `yaml:"is_private"` + IsMirror bool `yaml:"is_mirror"` Description string - CloneURL string - OriginalURL string + AuthUsername string `yaml:"auth_username"` + AuthPassword string `yaml:"auth_password"` + CloneURL string `yaml:"clone_url"` + OriginalURL string `yaml:"original_url"` DefaultBranch string } diff --git a/modules/migrations/base/review.go b/modules/migrations/base/review.go index 0a9d03dae9025..6344f0384d647 100644 --- a/modules/migrations/base/review.go +++ b/modules/migrations/base/review.go @@ -17,29 +17,29 @@ const ( // Review is a standard review information type Review struct { ID int64 - IssueIndex int64 - ReviewerID int64 - ReviewerName string + IssueIndex int64 `yaml:"issue_index"` + ReviewerID int64 `yaml:"reviewer_id"` + ReviewerName string `yaml:"reviewer_name"` Official bool - CommitID string + CommitID string `yaml:"commit_id"` Content string - CreatedAt time.Time - State string // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT + CreatedAt time.Time `yaml:"created_at"` + State string // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT Comments []*ReviewComment } // ReviewComment represents a review comment type ReviewComment struct { ID int64 - InReplyTo int64 + InReplyTo int64 `yaml:"in_reply_to"` Content string - TreePath string - DiffHunk string + TreePath string `yaml:"tree_path"` + DiffHunk string `yaml:"diff_hunk"` Position int Line int - CommitID string - PosterID int64 + CommitID string `yaml:"commit_id"` + PosterID int64 `yaml:"poster_id"` Reactions []*Reaction - CreatedAt time.Time - UpdatedAt time.Time + CreatedAt time.Time `yaml:"created_at"` + UpdatedAt time.Time `yaml:"updated_at"` } diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index 6760d132636ec..99dfc2a2c356a 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -45,6 +45,7 @@ type RepositoryDumper struct { // NewRepositoryDumper creates an gitea Uploader func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName string) *RepositoryDumper { + baseDir = filepath.Join(baseDir, repoOwner, repoName) return &RepositoryDumper{ ctx: ctx, baseDir: baseDir, @@ -59,7 +60,7 @@ func (g *RepositoryDumper) MaxBatchInsertSize(tp string) int { return 1000 } -func (g *RepositoryDumper) repoPath() string { +func (g *RepositoryDumper) gitPath() string { return filepath.Join(g.baseDir, "git") } @@ -68,40 +69,43 @@ func (g *RepositoryDumper) wikiPath() string { } func (g *RepositoryDumper) topicDir() string { - return filepath.Join(g.baseDir, "topic") + return filepath.Join(g.baseDir) } func (g *RepositoryDumper) milestoneDir() string { - return filepath.Join(g.baseDir, "milestone") + return filepath.Join(g.baseDir) } func (g *RepositoryDumper) labelDir() string { - return filepath.Join(g.baseDir, "label") + return filepath.Join(g.baseDir) } func (g *RepositoryDumper) releaseDir() string { - return filepath.Join(g.baseDir, "release") + return filepath.Join(g.baseDir) } func (g *RepositoryDumper) issueDir() string { - return filepath.Join(g.baseDir, "issue") + return filepath.Join(g.baseDir) } func (g *RepositoryDumper) commentDir() string { - return filepath.Join(g.baseDir, "comment") + return filepath.Join(g.baseDir) } func (g *RepositoryDumper) pullrequestDir() string { - return filepath.Join(g.baseDir, "pullrequest") + return filepath.Join(g.baseDir) } func (g *RepositoryDumper) reviewDir() string { - return filepath.Join(g.baseDir, "review") + return filepath.Join(g.baseDir) } // CreateRepo creates a repository func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error { - repoPath := g.repoPath() + if err := os.MkdirAll(g.baseDir, os.ModePerm); err != nil { + return err + } + repoPath := g.gitPath() if err := os.MkdirAll(repoPath, os.ModePerm); err != nil { return err } @@ -139,7 +143,7 @@ func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOp } } - g.gitRepo, err = git.OpenRepository(g.repoPath()) + g.gitRepo, err = git.OpenRepository(g.gitPath()) return err } @@ -173,7 +177,7 @@ func (g *RepositoryDumper) CreateTopics(topics ...string) error { if err := os.MkdirAll(g.topicDir(), os.ModePerm); err != nil { return err } - f, err := os.Create(filepath.Join(g.topicDir(), "data.yml")) + f, err := os.Create(filepath.Join(g.topicDir(), "topic.yml")) if err != nil { return err } @@ -200,7 +204,7 @@ func (g *RepositoryDumper) CreateMilestones(milestones ...*base.Milestone) error if err := os.MkdirAll(g.milestoneDir(), os.ModePerm); err != nil { return err } - g.milestoneFile, err = os.Create(filepath.Join(g.milestoneDir(), "data.yml")) + g.milestoneFile, err = os.Create(filepath.Join(g.milestoneDir(), "milestone.yml")) if err != nil { return err } @@ -225,7 +229,7 @@ func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error { if err := os.MkdirAll(g.labelDir(), os.ModePerm); err != nil { return err } - g.labelFile, err = os.Create(filepath.Join(g.labelDir(), "data.yml")) + g.labelFile, err = os.Create(filepath.Join(g.labelDir(), "label.yml")) if err != nil { return err } @@ -246,7 +250,7 @@ func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error { // CreateReleases creates releases func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { for _, release := range releases { - attachDir := filepath.Join(g.releaseDir(), "assets", release.Name) + attachDir := filepath.Join(g.releaseDir(), "release_assets", release.Name) if err := os.MkdirAll(attachDir, os.ModePerm); err != nil { return err } @@ -280,7 +284,7 @@ func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { if err := os.MkdirAll(g.releaseDir(), os.ModePerm); err != nil { return err } - g.releaseFile, err = os.Create(filepath.Join(g.releaseDir(), "data.yml")) + g.releaseFile, err = os.Create(filepath.Join(g.releaseDir(), "release.yml")) if err != nil { return err } @@ -310,7 +314,7 @@ func (g *RepositoryDumper) CreateIssues(issues ...*base.Issue) error { if err := os.MkdirAll(g.issueDir(), os.ModePerm); err != nil { return err } - g.issueFile, err = os.Create(filepath.Join(g.issueDir(), "data.yml")) + g.issueFile, err = os.Create(filepath.Join(g.issueDir(), "issue.yml")) if err != nil { return err } @@ -335,7 +339,7 @@ func (g *RepositoryDumper) CreateComments(comments ...*base.Comment) error { if err := os.MkdirAll(g.commentDir(), os.ModePerm); err != nil { return err } - g.commentFile, err = os.Create(filepath.Join(g.commentDir(), "data.yml")) + g.commentFile, err = os.Create(filepath.Join(g.commentDir(), "comment.yml")) if err != nil { return err } @@ -363,7 +367,7 @@ func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { return err } defer resp.Body.Close() - pullDir := filepath.Join(g.repoPath(), "pulls") + pullDir := filepath.Join(g.gitPath(), "pulls") if err = os.MkdirAll(pullDir, os.ModePerm); err != nil { return err } @@ -380,7 +384,7 @@ func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { } // set head information - pullHead := filepath.Join(g.repoPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number)) + pullHead := filepath.Join(g.gitPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number)) if err := os.MkdirAll(pullHead, os.ModePerm); err != nil { return err } @@ -410,11 +414,11 @@ func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { } if ok { - _, err = git.NewCommand("fetch", remote, pr.Head.Ref).RunInDir(g.repoPath()) + _, err = git.NewCommand("fetch", remote, pr.Head.Ref).RunInDir(g.gitPath()) if err != nil { log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err) } else { - headBranch := filepath.Join(g.repoPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref) + headBranch := filepath.Join(g.gitPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref) if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil { return err } @@ -438,7 +442,7 @@ func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { if err := os.MkdirAll(g.pullrequestDir(), os.ModePerm); err != nil { return err } - g.pullrequestFile, err = os.Create(filepath.Join(g.pullrequestDir(), "data.yml")) + g.pullrequestFile, err = os.Create(filepath.Join(g.pullrequestDir(), "pull_request.yml")) if err != nil { return err } @@ -463,7 +467,7 @@ func (g *RepositoryDumper) CreateReviews(reviews ...*base.Review) error { if err := os.MkdirAll(g.reviewDir(), os.ModePerm); err != nil { return err } - g.reviewFile, err = os.Create(filepath.Join(g.reviewDir(), "data.yml")) + g.reviewFile, err = os.Create(filepath.Join(g.reviewDir(), "review.yml")) if err != nil { return err } From 8c3f3db9259b15575bae19daf3018c11eb83de52 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 18 Jul 2020 21:00:04 +0800 Subject: [PATCH 04/47] Allow migration special units --- cmd/dump_repo.go | 69 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/cmd/dump_repo.go b/cmd/dump_repo.go index c0618b834534d..8622a821bd3d2 100644 --- a/cmd/dump_repo.go +++ b/cmd/dump_repo.go @@ -6,6 +6,7 @@ package cmd import ( "context" + "strings" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations" @@ -30,27 +31,33 @@ var CmdDumpRepository = cli.Command{ cli.StringFlag{ Name: "clone_addr", Value: "", - Usage: "Clone URL", + Usage: "The URL will be clone, currently could be a github or gitlab http/https URL", }, cli.StringFlag{ Name: "auth_username", Value: "", - Usage: "", + Usage: "The username or personal token to visit the clone_addr, it's required", }, cli.StringFlag{ Name: "auth_password", Value: "", - Usage: "", + Usage: "The password to visit the clone_addr if auth_username is a real user name", }, cli.StringFlag{ Name: "owner_name", Value: "", - Usage: "", + Usage: "The data will be stored on a directory with owner name if not empty", }, cli.StringFlag{ Name: "repo_name", Value: "", - Usage: "", + Usage: "The data will be stored on a directory with repository name if not empty", + }, + cli.StringFlag{ + Name: "units", + Value: "", + Usage: `Which items will be migrated, one or more units should be seperated as comma. +wiki, issues, labels, releases, milestones, pull_requests, comments are allowed. Empty means all units.`, }, }, } @@ -66,23 +73,49 @@ func runDumpRepository(ctx *cli.Context) error { log.Trace("Log path: %s", setting.LogRootPath) setting.InitDBConfig() + var opts = base.MigrateOptions{ + CloneAddr: ctx.String("clone_addr"), + AuthUsername: ctx.String("auth_username"), + AuthPassword: ctx.String("auth_password"), + RepoName: ctx.String("repo_name"), + } + + if len(ctx.String("units")) == 0 { + opts.Wiki = true + opts.Issues = true + opts.Milestones = true + opts.Labels = true + opts.Releases = true + opts.Comments = true + opts.PullRequests = true + } else { + units := strings.Split(ctx.String("units"), ",") + for _, unit := range units { + switch strings.ToLower(unit) { + case "wiki": + opts.Wiki = true + case "issues": + opts.Issues = true + case "milestones": + opts.Milestones = true + case "labels": + opts.Labels = true + case "releases": + opts.Releases = true + case "comments": + opts.Comments = true + case "pull_requests": + opts.PullRequests = true + } + } + } + if err := migrations.DumpRepository( context.Background(), ctx.String("repo_dir"), ctx.String("owner_name"), - base.MigrateOptions{ - CloneAddr: ctx.String("clone_addr"), - AuthUsername: ctx.String("auth_username"), - AuthPassword: ctx.String("auth_password"), - RepoName: ctx.String("repo_name"), - Wiki: true, - Issues: true, - Milestones: true, - Labels: true, - Releases: true, - Comments: true, - PullRequests: true, - }); err != nil { + opts, + ); err != nil { log.Fatal("Failed to dump repository: %v", err) return err } From 9e7915c47d3735e640b524480ccc46fd2dde2f78 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 18 Jul 2020 21:38:44 +0800 Subject: [PATCH 05/47] Allow migration ignore release assets --- cmd/dump_repo.go | 5 ++- modules/migrations/dump.go | 86 ++++++++++++++++++++------------------ modules/structs/repo.go | 27 ++++++++++++ 3 files changed, 76 insertions(+), 42 deletions(-) diff --git a/cmd/dump_repo.go b/cmd/dump_repo.go index 8622a821bd3d2..bc911611d56a2 100644 --- a/cmd/dump_repo.go +++ b/cmd/dump_repo.go @@ -57,7 +57,7 @@ var CmdDumpRepository = cli.Command{ Name: "units", Value: "", Usage: `Which items will be migrated, one or more units should be seperated as comma. -wiki, issues, labels, releases, milestones, pull_requests, comments are allowed. Empty means all units.`, +wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.`, }, }, } @@ -88,6 +88,7 @@ func runDumpRepository(ctx *cli.Context) error { opts.Releases = true opts.Comments = true opts.PullRequests = true + opts.ReleaseAssets = true } else { units := strings.Split(ctx.String("units"), ",") for _, unit := range units { @@ -102,6 +103,8 @@ func runDumpRepository(ctx *cli.Context) error { opts.Labels = true case "releases": opts.Releases = true + case "release_assets": + opts.ReleaseAssets = true case "comments": opts.Comments = true case "pull_requests": diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index 99dfc2a2c356a..4e0c751a2c6c7 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -27,31 +27,33 @@ var ( // RepositoryDumper implements an Uploader to the local directory type RepositoryDumper struct { - ctx context.Context - baseDir string - repoOwner string - repoName string - milestoneFile *os.File - labelFile *os.File - releaseFile *os.File - issueFile *os.File - commentFile *os.File - pullrequestFile *os.File - reviewFile *os.File + ctx context.Context + baseDir string + repoOwner string + repoName string + milestoneFile *os.File + labelFile *os.File + releaseFile *os.File + issueFile *os.File + commentFile *os.File + pullrequestFile *os.File + reviewFile *os.File + migrateReleaseAssets bool gitRepo *git.Repository prHeadCache map[string]struct{} } // NewRepositoryDumper creates an gitea Uploader -func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName string) *RepositoryDumper { +func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName string, migrateReleaseAssets bool) *RepositoryDumper { baseDir = filepath.Join(baseDir, repoOwner, repoName) return &RepositoryDumper{ - ctx: ctx, - baseDir: baseDir, - repoOwner: repoOwner, - repoName: repoName, - prHeadCache: make(map[string]struct{}), + ctx: ctx, + baseDir: baseDir, + repoOwner: repoOwner, + repoName: repoName, + prHeadCache: make(map[string]struct{}), + migrateReleaseAssets: migrateReleaseAssets, } } @@ -249,32 +251,34 @@ func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error { // CreateReleases creates releases func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { - for _, release := range releases { - attachDir := filepath.Join(g.releaseDir(), "release_assets", release.Name) - if err := os.MkdirAll(attachDir, os.ModePerm); err != nil { - return err - } - for _, asset := range release.Assets { - attachLocalPath := filepath.Join(attachDir, asset.Name) - // download attachment - err := func(attachLocalPath string) error { - resp, err := http.Get(asset.URL) - if err != nil { - return err - } - defer resp.Body.Close() + if g.migrateReleaseAssets { + for _, release := range releases { + attachDir := filepath.Join(g.releaseDir(), "release_assets", release.Name) + if err := os.MkdirAll(attachDir, os.ModePerm); err != nil { + return err + } + for _, asset := range release.Assets { + attachLocalPath := filepath.Join(attachDir, asset.Name) + // download attachment + err := func(attachLocalPath string) error { + resp, err := http.Get(asset.URL) + if err != nil { + return err + } + defer resp.Body.Close() - fw, err := os.Create(attachLocalPath) + fw, err := os.Create(attachLocalPath) + if err != nil { + return fmt.Errorf("Create: %v", err) + } + defer fw.Close() + + _, err = io.Copy(fw, resp.Body) + return err + }(attachLocalPath) if err != nil { - return fmt.Errorf("Create: %v", err) + return err } - defer fw.Close() - - _, err = io.Copy(fw, resp.Body) - return err - }(attachLocalPath) - if err != nil { - return err } } } @@ -493,6 +497,6 @@ func (g *RepositoryDumper) Rollback() error { // DumpRepository dump repository according MigrateOptions to a local directory func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.MigrateOptions) error { - var uploader = NewRepositoryDumper(ctx, baseDir, ownerName, opts.RepoName) + var uploader = NewRepositoryDumper(ctx, baseDir, ownerName, opts.RepoName, opts.ReleaseAssets) return MigrateRepositoryWithUploader(ctx, ownerName, opts, uploader) } diff --git a/modules/structs/repo.go b/modules/structs/repo.go index c12f8e1c18e11..805aa03cabb9b 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -278,3 +278,30 @@ var ( GiteaService, } ) + +// MigrateRepoOption options for migrating a repository from an external service +type MigrateRepoOption struct { + // required: true + CloneAddr string `json:"clone_addr" binding:"Required"` + AuthUsername string `json:"auth_username"` + AuthPassword string `json:"auth_password"` + AuthToken string `json:"auth_token"` + // required: true + UID int `json:"uid" binding:"Required"` + // required: true + RepoName string `json:"repo_name" binding:"Required"` + Mirror bool `json:"mirror"` + Private bool `json:"private"` + Description string `json:"description"` + OriginalURL string + GitServiceType GitServiceType + Wiki bool + Issues bool + Milestones bool + Labels bool + Releases bool + ReleaseAssets bool + Comments bool + PullRequests bool + MigrateToRepoID int64 +} From ed4e3762d7994535b66b3b6f6d2405579493ef97 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 19 Jul 2020 15:09:18 +0800 Subject: [PATCH 06/47] Fix lint --- cmd/dump_repo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/dump_repo.go b/cmd/dump_repo.go index bc911611d56a2..b551bf626b65d 100644 --- a/cmd/dump_repo.go +++ b/cmd/dump_repo.go @@ -56,7 +56,7 @@ var CmdDumpRepository = cli.Command{ cli.StringFlag{ Name: "units", Value: "", - Usage: `Which items will be migrated, one or more units should be seperated as comma. + Usage: `Which items will be migrated, one or more units should be separated as comma. wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.`, }, }, From c49020bbd17878b513c43b1ccb5d28eab84b6677 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 22 Aug 2020 23:27:28 +0800 Subject: [PATCH 07/47] Add restore repository --- cmd/restore_repo.go | 122 ++++++++++++++++++++++++++++++++++ models/admin.go | 13 ++++ modules/migrations/dump.go | 12 ++++ modules/migrations/migrate.go | 6 +- modules/migrations/restore.go | 81 ++++++++++++++++++++++ 5 files changed, 233 insertions(+), 1 deletion(-) create mode 100644 cmd/restore_repo.go create mode 100644 modules/migrations/restore.go diff --git a/cmd/restore_repo.go b/cmd/restore_repo.go new file mode 100644 index 0000000000000..e4d1b45112ec3 --- /dev/null +++ b/cmd/restore_repo.go @@ -0,0 +1,122 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "context" + "strings" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migrations" + "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/setting" + + "github.com/urfave/cli" +) + +// CmdRestoreRepository represents the available restore a repository sub-command. +var CmdRestoreRepository = cli.Command{ + Name: "restore-repo", + Usage: "Restore the repository from disk", + Description: "This is a command for restoring the repository data.", + Action: runRestoreRepository, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "repo_dir, r", + Value: "./data", + Usage: "Repository dir path", + }, + cli.StringFlag{ + Name: "clone_addr", + Value: "", + Usage: "The URL will be clone, currently could be a github or gitlab http/https URL", + }, + cli.StringFlag{ + Name: "auth_username", + Value: "", + Usage: "The username or personal token to visit the clone_addr, it's required", + }, + cli.StringFlag{ + Name: "auth_password", + Value: "", + Usage: "The password to visit the clone_addr if auth_username is a real user name", + }, + cli.StringFlag{ + Name: "owner_name", + Value: "", + Usage: "The data will be stored on a directory with owner name if not empty", + }, + cli.StringFlag{ + Name: "repo_name", + Value: "", + Usage: "The data will be stored on a directory with repository name if not empty", + }, + cli.StringFlag{ + Name: "units", + Value: "", + Usage: `Which items will be migrated, one or more units should be separated as comma. +wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.`, + }, + }, +} + +func runRestoreRepository(ctx *cli.Context) error { + if err := initDB(); err != nil { + return err + } + + log.Trace("AppPath: %s", setting.AppPath) + log.Trace("AppWorkPath: %s", setting.AppWorkPath) + log.Trace("Custom path: %s", setting.CustomPath) + log.Trace("Log path: %s", setting.LogRootPath) + setting.InitDBConfig() + + var opts = base.MigrateOptions{} + + if len(ctx.String("units")) == 0 { + opts.Wiki = true + opts.Issues = true + opts.Milestones = true + opts.Labels = true + opts.Releases = true + opts.Comments = true + opts.PullRequests = true + opts.ReleaseAssets = true + } else { + units := strings.Split(ctx.String("units"), ",") + for _, unit := range units { + switch strings.ToLower(unit) { + case "wiki": + opts.Wiki = true + case "issues": + opts.Issues = true + case "milestones": + opts.Milestones = true + case "labels": + opts.Labels = true + case "releases": + opts.Releases = true + case "release_assets": + opts.ReleaseAssets = true + case "comments": + opts.Comments = true + case "pull_requests": + opts.PullRequests = true + } + } + } + + if err := migrations.RestoreRepository( + context.Background(), + ctx.String("repo_dir"), + ctx.String("owner_name"), + ctx.String("repo_name"), + ); err != nil { + log.Fatal("Failed to dump repository: %v", err) + return err + } + + return nil +} diff --git a/models/admin.go b/models/admin.go index 420adbcda9991..4635676d0cbc8 100644 --- a/models/admin.go +++ b/models/admin.go @@ -132,3 +132,16 @@ func DeleteNoticesByIDs(ids []int64) error { Delete(new(Notice)) return err } + +// GetAdminUser returns the first administrator +func GetAdminUser() (*User, error) { + var admin User + has, err := x.Where("is_admin=?", true).Get(&admin) + if err != nil { + return nil, err + } else if !has { + return nil, ErrUserNotExist{} + } + + return &admin, nil +} diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index 4e0c751a2c6c7..c77b692bbe024 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -13,6 +13,7 @@ import ( "path/filepath" "time" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" @@ -500,3 +501,14 @@ func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.Mi var uploader = NewRepositoryDumper(ctx, baseDir, ownerName, opts.RepoName, opts.ReleaseAssets) return MigrateRepositoryWithUploader(ctx, ownerName, opts, uploader) } + +// RestoreRepository restore a repository from the disk direcotry +func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName string) error { + doer, err := models.GetAdminUser() + if err != nil { + return err + } + var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, repoName) + var downloader = NewRepositoryRestorer(ctx, baseDir, ownerName, repoName) + return DoMigrateRepository(downloader, uploader, base.MigrateOptions{}) +} diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index bf693cb0e45e5..6327815bfcbe0 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -127,9 +127,13 @@ func MigrateRepositoryWithUploader(ctx context.Context, ownerName string, opts b return nil } -// migrateRepository will download information and then upload it to Uploader, this is a simple +// DoMigrateRepository will download information and then upload it to Uploader, this is a simple // process for small repository. For a big repository, save all the data to disk // before upload is better +func DoMigrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions) error { + return migrateRepository(downloader, uploader, opts) +} + func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions) error { repo, err := downloader.GetRepoInfo() if err != nil { diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go new file mode 100644 index 0000000000000..d9aab0596ed9e --- /dev/null +++ b/modules/migrations/restore.go @@ -0,0 +1,81 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "context" + "os" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/migrations/base" +) + +// RepositoryRestorer implements an Downloader from the local directory +type RepositoryRestorer struct { + ctx context.Context + baseDir string + repoOwner string + repoName string + milestoneFile *os.File + labelFile *os.File + releaseFile *os.File + issueFile *os.File + commentFile *os.File + pullrequestFile *os.File + reviewFile *os.File + migrateReleaseAssets bool + + gitRepo *git.Repository + prHeadCache map[string]struct{} +} + +func NewRepositoryRestorer(ctx context.Context, baseDir string, owner, repoName string) *RepositoryRestorer { + return &RepositoryRestorer{ + ctx: ctx, + baseDir: baseDir, + repoOwner: owner, + repoName: repoName, + } +} + +func (r *RepositoryRestorer) SetContext(ctx context.Context) { + r.ctx = ctx +} + +func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) { + +} + +func (r *RepositoryRestorer) GetTopics() ([]string, error) { + +} + +func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) { + +} + +func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) { + +} + +func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) { + +} + +func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { + +} + +func (r *RepositoryRestorer) GetComments(issueNumber int64) ([]*base.Comment, error) { + +} + +func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullRequest, error) { + +} + +func (r *RepositoryRestorer) GetReviews(pullRequestNumber int64) ([]*base.Review, error) { + +} From c13e7450d2645b8ebffb71398f6ceb0035f01e74 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 1 Sep 2020 14:04:52 +0800 Subject: [PATCH 08/47] stage the changes --- modules/migrations/dump.go | 9 +++++---- modules/migrations/restore.go | 21 +++++++++++++-------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index c77b692bbe024..88c0326477427 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -251,7 +251,7 @@ func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error { } // CreateReleases creates releases -func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { +func (g *RepositoryDumper) CreateReleases(downloader base.Downloader, releases ...*base.Release) error { if g.migrateReleaseAssets { for _, release := range releases { attachDir := filepath.Join(g.releaseDir(), "release_assets", release.Name) @@ -261,12 +261,13 @@ func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { for _, asset := range release.Assets { attachLocalPath := filepath.Join(attachDir, asset.Name) // download attachment + err := func(attachLocalPath string) error { - resp, err := http.Get(asset.URL) + rc, err := downloader.GetAsset(release.TagName, asset.ID) if err != nil { return err } - defer resp.Body.Close() + defer rc.Close() fw, err := os.Create(attachLocalPath) if err != nil { @@ -274,7 +275,7 @@ func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { } defer fw.Close() - _, err = io.Copy(fw, resp.Body) + _, err = io.Copy(fw, rc) return err }(attachLocalPath) if err != nil { diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go index d9aab0596ed9e..6b38ca99026d9 100644 --- a/modules/migrations/restore.go +++ b/modules/migrations/restore.go @@ -6,6 +6,7 @@ package migrations import ( "context" + "io" "os" "code.gitea.io/gitea/modules/git" @@ -45,37 +46,41 @@ func (r *RepositoryRestorer) SetContext(ctx context.Context) { } func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) { - + return nil, nil } func (r *RepositoryRestorer) GetTopics() ([]string, error) { - + return nil, nil } func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) { - + return nil, nil } func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) { + return nil, nil +} +func (r *RepositoryRestorer) GetAsset(tagName string, id int) (io.ReadCloser, error) { + return nil, nil } func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) { - + return nil, nil } func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { - + return nil, false, nil } func (r *RepositoryRestorer) GetComments(issueNumber int64) ([]*base.Comment, error) { - + return nil, nil } func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullRequest, error) { - + return nil, nil } func (r *RepositoryRestorer) GetReviews(pullRequestNumber int64) ([]*base.Review, error) { - + return nil, nil } From aa5ffb3c14bff18065e024a31291c1f02517e5e9 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 11 Sep 2020 21:09:49 +0800 Subject: [PATCH 09/47] Merge --- modules/migrations/base/options.go | 1 + modules/migrations/restore.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/migrations/base/options.go b/modules/migrations/base/options.go index dbc40b138aadc..3c9b2c22fcab9 100644 --- a/modules/migrations/base/options.go +++ b/modules/migrations/base/options.go @@ -31,5 +31,6 @@ type MigrateOptions struct { Releases bool Comments bool PullRequests bool + ReleaseAssets bool MigrateToRepoID int64 } diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go index 6b38ca99026d9..5300286505359 100644 --- a/modules/migrations/restore.go +++ b/modules/migrations/restore.go @@ -61,7 +61,7 @@ func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) { return nil, nil } -func (r *RepositoryRestorer) GetAsset(tagName string, id int) (io.ReadCloser, error) { +func (r *RepositoryRestorer) GetAsset(tagName string, id int64) (io.ReadCloser, error) { return nil, nil } From 92fbde74c94bd90e666699bfd1efa91e83b5cf3b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 11 Sep 2020 21:42:20 +0800 Subject: [PATCH 10/47] Fix lint --- modules/migrations/dump.go | 2 +- modules/migrations/restore.go | 21 ++++----------------- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index 88c0326477427..094d229f37656 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -503,7 +503,7 @@ func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.Mi return MigrateRepositoryWithUploader(ctx, ownerName, opts, uploader) } -// RestoreRepository restore a repository from the disk direcotry +// RestoreRepository restore a repository from the disk directory func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName string) error { doer, err := models.GetAdminUser() if err != nil { diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go index 5300286505359..517681ba05c83 100644 --- a/modules/migrations/restore.go +++ b/modules/migrations/restore.go @@ -7,29 +7,16 @@ package migrations import ( "context" "io" - "os" - "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/migrations/base" ) // RepositoryRestorer implements an Downloader from the local directory type RepositoryRestorer struct { - ctx context.Context - baseDir string - repoOwner string - repoName string - milestoneFile *os.File - labelFile *os.File - releaseFile *os.File - issueFile *os.File - commentFile *os.File - pullrequestFile *os.File - reviewFile *os.File - migrateReleaseAssets bool - - gitRepo *git.Repository - prHeadCache map[string]struct{} + ctx context.Context + baseDir string + repoOwner string + repoName string } func NewRepositoryRestorer(ctx context.Context, baseDir string, owner, repoName string) *RepositoryRestorer { From 399901021f984e268ecc5ea3f57e9b3f765ba250 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 18 Oct 2020 11:16:22 +0800 Subject: [PATCH 11/47] Update the interface --- modules/migrations/dump.go | 3 ++- modules/migrations/restore.go | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index 094d229f37656..e399c8507a3ce 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -263,7 +263,8 @@ func (g *RepositoryDumper) CreateReleases(downloader base.Downloader, releases . // download attachment err := func(attachLocalPath string) error { - rc, err := downloader.GetAsset(release.TagName, asset.ID) + // FIXME: release ID + rc, err := downloader.GetAsset(release.TagName, 0, asset.ID) if err != nil { return err } diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go index 517681ba05c83..91424e3dd1023 100644 --- a/modules/migrations/restore.go +++ b/modules/migrations/restore.go @@ -48,7 +48,7 @@ func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) { return nil, nil } -func (r *RepositoryRestorer) GetAsset(tagName string, id int64) (io.ReadCloser, error) { +func (r *RepositoryRestorer) GetAsset(tagName string, relID, assetID int64) (io.ReadCloser, error) { return nil, nil } @@ -64,8 +64,8 @@ func (r *RepositoryRestorer) GetComments(issueNumber int64) ([]*base.Comment, er return nil, nil } -func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullRequest, error) { - return nil, nil +func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { + return nil, false, nil } func (r *RepositoryRestorer) GetReviews(pullRequestNumber int64) ([]*base.Review, error) { From 6d633efdb057976a4620d250f08a5cd0868236f3 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 20 Oct 2020 23:51:15 +0800 Subject: [PATCH 12/47] Add some restore methods --- cmd/dump_repo.go | 9 +++ cmd/restore_repo.go | 19 +----- modules/migrations/dump.go | 2 +- modules/migrations/restore.go | 113 ++++++++++++++++++++++++++++++++-- 4 files changed, 120 insertions(+), 23 deletions(-) diff --git a/cmd/dump_repo.go b/cmd/dump_repo.go index b551bf626b65d..3116bca34b4cc 100644 --- a/cmd/dump_repo.go +++ b/cmd/dump_repo.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" "github.com/urfave/cli" ) @@ -23,6 +24,11 @@ var CmdDumpRepository = cli.Command{ Description: "This is a command for dumping the repository data.", Action: runDumpRepository, Flags: []cli.Flag{ + cli.IntFlag{ + Name: "git_service", + Value: 1, + Usage: "Git service, 1 plain git, 2 github, 3 gitea, 4 gitlab", + }, cli.StringFlag{ Name: "repo_dir, r", Value: "./data", @@ -74,6 +80,7 @@ func runDumpRepository(ctx *cli.Context) error { setting.InitDBConfig() var opts = base.MigrateOptions{ + GitServiceType: structs.GitServiceType(ctx.Int("git_service")), CloneAddr: ctx.String("clone_addr"), AuthUsername: ctx.String("auth_username"), AuthPassword: ctx.String("auth_password"), @@ -123,5 +130,7 @@ func runDumpRepository(ctx *cli.Context) error { return err } + log.Trace("Dump finished!!!") + return nil } diff --git a/cmd/restore_repo.go b/cmd/restore_repo.go index e4d1b45112ec3..684b768e2ae73 100644 --- a/cmd/restore_repo.go +++ b/cmd/restore_repo.go @@ -28,21 +28,6 @@ var CmdRestoreRepository = cli.Command{ Value: "./data", Usage: "Repository dir path", }, - cli.StringFlag{ - Name: "clone_addr", - Value: "", - Usage: "The URL will be clone, currently could be a github or gitlab http/https URL", - }, - cli.StringFlag{ - Name: "auth_username", - Value: "", - Usage: "The username or personal token to visit the clone_addr, it's required", - }, - cli.StringFlag{ - Name: "auth_password", - Value: "", - Usage: "The password to visit the clone_addr if auth_username is a real user name", - }, cli.StringFlag{ Name: "owner_name", Value: "", @@ -73,7 +58,9 @@ func runRestoreRepository(ctx *cli.Context) error { log.Trace("Log path: %s", setting.LogRootPath) setting.InitDBConfig() - var opts = base.MigrateOptions{} + var opts = base.MigrateOptions{ + RepoName: ctx.String("repo_name"), + } if len(ctx.String("units")) == 0 { opts.Wiki = true diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index e399c8507a3ce..748e27f439034 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -254,7 +254,7 @@ func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error { func (g *RepositoryDumper) CreateReleases(downloader base.Downloader, releases ...*base.Release) error { if g.migrateReleaseAssets { for _, release := range releases { - attachDir := filepath.Join(g.releaseDir(), "release_assets", release.Name) + attachDir := filepath.Join(g.releaseDir(), "release_assets", release.TagName) if err := os.MkdirAll(attachDir, os.ModePerm); err != nil { return err } diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go index 91424e3dd1023..a738bdda4091a 100644 --- a/modules/migrations/restore.go +++ b/modules/migrations/restore.go @@ -28,6 +28,46 @@ func NewRepositoryRestorer(ctx context.Context, baseDir string, owner, repoName } } +func (g *RepositoryRestorer) gitPath() string { + return filepath.Join(g.baseDir, "git") +} + +func (g *RepositoryRestorer) wikiPath() string { + return filepath.Join(g.baseDir, "wiki") +} + +func (g *RepositoryRestorer) topicDir() string { + return filepath.Join(g.baseDir) +} + +func (g *RepositoryRestorer) milestoneDir() string { + return filepath.Join(g.baseDir) +} + +func (g *RepositoryRestorer) labelDir() string { + return filepath.Join(g.baseDir) +} + +func (g *RepositoryRestorer) releaseDir() string { + return filepath.Join(g.baseDir) +} + +func (g *RepositoryRestorer) issueDir() string { + return filepath.Join(g.baseDir) +} + +func (g *RepositoryRestorer) commentDir() string { + return filepath.Join(g.baseDir) +} + +func (g *RepositoryRestorer) pullrequestDir() string { + return filepath.Join(g.baseDir) +} + +func (g *RepositoryRestorer) reviewDir() string { + return filepath.Join(g.baseDir) +} + func (r *RepositoryRestorer) SetContext(ctx context.Context) { r.ctx = ctx } @@ -37,27 +77,88 @@ func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) { } func (r *RepositoryRestorer) GetTopics() ([]string, error) { - return nil, nil + p := filepath.Join(g.topicDir(), "topic.yml")) + + var topics = struct { + Topics []string `yaml:"topics"` + } + + bs, err := ioutil.ReadAll(p) + if err != nil { + return err + } + + err := yaml.Unmarshal(bs, &topics) + if err != nil { + return err + } + return topics.Topics, nil } func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) { - return nil, nil + var milestones = make([]*base.Milestone) + p := filepath.Join(g.milestoneDir(), "milestone.yml") + bs, err := ioutil.ReadAll(p) + if err != nil { + return err + } + + err := yaml.Unmarshal(bs, &milestones) + if err != nil { + return err + } + return milestones, nil } func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) { - return nil, nil + var releases = make([]*base.Release) + p := filepath.Join(g.releaseDir(), "release.yml") + bs, err := ioutil.ReadAll(p) + if err != nil { + return err + } + + err := yaml.Unmarshal(bs, &releases) + if err != nil { + return err + } + return releases, nil } func (r *RepositoryRestorer) GetAsset(tagName string, relID, assetID int64) (io.ReadCloser, error) { - return nil, nil + attachDir := filepath.Join(g.releaseDir(), "release_assets", tagName) + attachLocalPath := filepath.Join(attachDir, asset.ID) + return os.Open(attachLocalPath) } func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) { - return nil, nil + var labels = make([]*base.Label) + p := filepath.Join(g.labelDir(), "label.yml") + bs, err := ioutil.ReadAll(p) + if err != nil { + return err + } + + err := yaml.Unmarshal(bs, &labels) + if err != nil { + return err + } + return labels, nil } func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { - return nil, false, nil + var issues = make([]*base.Issue) + p := filepath.Join(g.issuesDir(), "issue.yml") + bs, err := ioutil.ReadAll(p) + if err != nil { + return nil, false, err + } + + err := yaml.Unmarshal(bs, &issues) + if err != nil { + return err + } + return issues, false, nil } func (r *RepositoryRestorer) GetComments(issueNumber int64) ([]*base.Comment, error) { From 7006ba23277c497c92be670b139438d0e5cf906b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 21 Oct 2020 12:48:39 +0800 Subject: [PATCH 13/47] Finish restore --- modules/migrations/dump.go | 89 ++++++++++++++++---------- modules/migrations/restore.go | 113 +++++++++++++++++++++++----------- 2 files changed, 133 insertions(+), 69 deletions(-) diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index 748e27f439034..de8ee2dfefb8f 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -36,9 +36,9 @@ type RepositoryDumper struct { labelFile *os.File releaseFile *os.File issueFile *os.File - commentFile *os.File + commentFiles map[int64]*os.File pullrequestFile *os.File - reviewFile *os.File + reviewFiles map[int64]*os.File migrateReleaseAssets bool gitRepo *git.Repository @@ -54,6 +54,8 @@ func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName strin repoOwner: repoOwner, repoName: repoName, prHeadCache: make(map[string]struct{}), + commentFiles: make(map[int64]*os.File), + reviewFiles: make(map[int64]*os.File), migrateReleaseAssets: migrateReleaseAssets, } } @@ -92,7 +94,7 @@ func (g *RepositoryDumper) issueDir() string { } func (g *RepositoryDumper) commentDir() string { - return filepath.Join(g.baseDir) + return filepath.Join(g.baseDir, "comments") } func (g *RepositoryDumper) pullrequestDir() string { @@ -100,7 +102,7 @@ func (g *RepositoryDumper) pullrequestDir() string { } func (g *RepositoryDumper) reviewDir() string { - return filepath.Join(g.baseDir) + return filepath.Join(g.baseDir, "reviews") } // CreateRepo creates a repository @@ -167,11 +169,14 @@ func (g *RepositoryDumper) Close() { if g.issueFile != nil { g.issueFile.Close() } + for _, f := range g.commentFiles { + f.Close() + } if g.pullrequestFile != nil { g.pullrequestFile.Close() } - if g.reviewFile != nil { - g.reviewFile.Close() + for _, f := range g.reviewFiles { + f.Close() } } @@ -341,24 +346,34 @@ func (g *RepositoryDumper) CreateIssues(issues ...*base.Issue) error { // CreateComments creates comments of issues func (g *RepositoryDumper) CreateComments(comments ...*base.Comment) error { - var err error - if g.commentFile == nil { - if err := os.MkdirAll(g.commentDir(), os.ModePerm); err != nil { - return err - } - g.commentFile, err = os.Create(filepath.Join(g.commentDir(), "comment.yml")) - if err != nil { - return err - } + var commentsMap = make(map[int64][]*base.Comment, len(comments)) + for _, comment := range comments { + commentsMap[comment.IssueIndex] = append(commentsMap[comment.IssueIndex], comment) } - bs, err := yaml.Marshal(comments) - if err != nil { + if err := os.MkdirAll(g.commentDir(), os.ModePerm); err != nil { return err } - if _, err := g.commentFile.Write(bs); err != nil { - return err + for issueNumber, comments := range commentsMap { + var err error + commentFile := g.commentFiles[issueNumber] + if commentFile == nil { + commentFile, err = os.Create(filepath.Join(g.commentDir(), fmt.Sprintf("%d.yml", issueNumber))) + if err != nil { + return err + } + g.commentFiles[issueNumber] = commentFile + } + + bs, err := yaml.Marshal(comments) + if err != nil { + return err + } + + if _, err := commentFile.Write(bs); err != nil { + return err + } } return nil @@ -469,24 +484,34 @@ func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { // CreateReviews create pull request reviews func (g *RepositoryDumper) CreateReviews(reviews ...*base.Review) error { - var err error - if g.reviewFile == nil { - if err := os.MkdirAll(g.reviewDir(), os.ModePerm); err != nil { - return err - } - g.reviewFile, err = os.Create(filepath.Join(g.reviewDir(), "review.yml")) - if err != nil { - return err - } + var reviewsMap = make(map[int64][]*base.Review, len(reviews)) + for _, review := range reviews { + reviewsMap[review.ID] = append(reviewsMap[review.ID], review) } - bs, err := yaml.Marshal(reviews) - if err != nil { + if err := os.MkdirAll(g.reviewDir(), os.ModePerm); err != nil { return err } - if _, err := g.reviewFile.Write(bs); err != nil { - return err + for prNumber, reviews := range reviewsMap { + var err error + reviewFile := g.reviewFiles[prNumber] + if reviewFile == nil { + reviewFile, err = os.Create(filepath.Join(g.reviewDir(), fmt.Sprintf("%d.yml", prNumber))) + if err != nil { + return err + } + g.reviewFiles[prNumber] = reviewFile + } + + bs, err := yaml.Marshal(reviews) + if err != nil { + return err + } + + if _, err := reviewFile.Write(bs); err != nil { + return err + } } return nil diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go index a738bdda4091a..01808993de839 100644 --- a/modules/migrations/restore.go +++ b/modules/migrations/restore.go @@ -6,9 +6,15 @@ package migrations import ( "context" + "fmt" "io" + "io/ioutil" + "os" + "path/filepath" "code.gitea.io/gitea/modules/migrations/base" + + "gopkg.in/yaml.v2" ) // RepositoryRestorer implements an Downloader from the local directory @@ -57,7 +63,7 @@ func (g *RepositoryRestorer) issueDir() string { } func (g *RepositoryRestorer) commentDir() string { - return filepath.Join(g.baseDir) + return filepath.Join(g.baseDir, "comments") } func (g *RepositoryRestorer) pullrequestDir() string { @@ -65,7 +71,7 @@ func (g *RepositoryRestorer) pullrequestDir() string { } func (g *RepositoryRestorer) reviewDir() string { - return filepath.Join(g.baseDir) + return filepath.Join(g.baseDir, "reviews") } func (r *RepositoryRestorer) SetContext(ctx context.Context) { @@ -77,98 +83,131 @@ func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) { } func (r *RepositoryRestorer) GetTopics() ([]string, error) { - p := filepath.Join(g.topicDir(), "topic.yml")) + p := filepath.Join(r.topicDir(), "topic.yml") var topics = struct { Topics []string `yaml:"topics"` - } + }{} - bs, err := ioutil.ReadAll(p) + bs, err := ioutil.ReadFile(p) if err != nil { - return err + return nil, err } - err := yaml.Unmarshal(bs, &topics) + err = yaml.Unmarshal(bs, &topics) if err != nil { - return err + return nil, err } return topics.Topics, nil } func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) { - var milestones = make([]*base.Milestone) - p := filepath.Join(g.milestoneDir(), "milestone.yml") - bs, err := ioutil.ReadAll(p) + var milestones = make([]*base.Milestone, 0, 10) + p := filepath.Join(r.milestoneDir(), "milestone.yml") + bs, err := ioutil.ReadFile(p) if err != nil { - return err + return nil, err } - err := yaml.Unmarshal(bs, &milestones) + err = yaml.Unmarshal(bs, &milestones) if err != nil { - return err + return nil, err } return milestones, nil } func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) { - var releases = make([]*base.Release) - p := filepath.Join(g.releaseDir(), "release.yml") - bs, err := ioutil.ReadAll(p) + var releases = make([]*base.Release, 0, 10) + p := filepath.Join(r.releaseDir(), "release.yml") + bs, err := ioutil.ReadFile(p) if err != nil { - return err + return nil, err } - err := yaml.Unmarshal(bs, &releases) + err = yaml.Unmarshal(bs, &releases) if err != nil { - return err + return nil, err } return releases, nil } func (r *RepositoryRestorer) GetAsset(tagName string, relID, assetID int64) (io.ReadCloser, error) { - attachDir := filepath.Join(g.releaseDir(), "release_assets", tagName) - attachLocalPath := filepath.Join(attachDir, asset.ID) + attachDir := filepath.Join(r.releaseDir(), "release_assets", tagName) + attachLocalPath := filepath.Join(attachDir, fmt.Sprintf("%d", assetID)) return os.Open(attachLocalPath) } func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) { - var labels = make([]*base.Label) - p := filepath.Join(g.labelDir(), "label.yml") - bs, err := ioutil.ReadAll(p) + var labels = make([]*base.Label, 0, 10) + p := filepath.Join(r.labelDir(), "label.yml") + bs, err := ioutil.ReadFile(p) if err != nil { - return err + return nil, err } - err := yaml.Unmarshal(bs, &labels) + err = yaml.Unmarshal(bs, &labels) if err != nil { - return err + return nil, err } return labels, nil } func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { - var issues = make([]*base.Issue) - p := filepath.Join(g.issuesDir(), "issue.yml") - bs, err := ioutil.ReadAll(p) + var issues = make([]*base.Issue, 0, 10) + p := filepath.Join(r.issueDir(), "issue.yml") + bs, err := ioutil.ReadFile(p) if err != nil { return nil, false, err } - err := yaml.Unmarshal(bs, &issues) + err = yaml.Unmarshal(bs, &issues) if err != nil { - return err + return nil, false, err } - return issues, false, nil + return issues, true, nil } func (r *RepositoryRestorer) GetComments(issueNumber int64) ([]*base.Comment, error) { - return nil, nil + var comments = make([]*base.Comment, 0, 10) + p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", issueNumber)) + bs, err := ioutil.ReadFile(p) + if err != nil { + return nil, err + } + + err = yaml.Unmarshal(bs, &comments) + if err != nil { + return nil, err + } + return comments, nil } func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { - return nil, false, nil + var pulls = make([]*base.PullRequest, 0, 10) + p := filepath.Join(r.pullrequestDir(), "pull_request.yml") + bs, err := ioutil.ReadFile(p) + if err != nil { + return nil, false, err + } + + err = yaml.Unmarshal(bs, &pulls) + if err != nil { + return nil, false, err + } + return pulls, true, nil } func (r *RepositoryRestorer) GetReviews(pullRequestNumber int64) ([]*base.Review, error) { - return nil, nil + var reviews = make([]*base.Review, 0, 10) + p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", pullRequestNumber)) + bs, err := ioutil.ReadFile(p) + if err != nil { + return nil, err + } + + err = yaml.Unmarshal(bs, &reviews) + if err != nil { + return nil, err + } + return reviews, nil } From d1746bcc72d3ab65c95f1d02e58e35cee825f0d8 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 21 Oct 2020 13:03:54 +0800 Subject: [PATCH 14/47] Add comments --- modules/migrations/restore.go | 66 +++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go index 01808993de839..dac45126c7a9d 100644 --- a/modules/migrations/restore.go +++ b/modules/migrations/restore.go @@ -25,6 +25,7 @@ type RepositoryRestorer struct { repoName string } +// NewRepositoryRestorer creates a repository restorer which could restore repository from a dumped folder func NewRepositoryRestorer(ctx context.Context, baseDir string, owner, repoName string) *RepositoryRestorer { return &RepositoryRestorer{ ctx: ctx, @@ -34,54 +35,65 @@ func NewRepositoryRestorer(ctx context.Context, baseDir string, owner, repoName } } -func (g *RepositoryRestorer) gitPath() string { - return filepath.Join(g.baseDir, "git") +func (r *RepositoryRestorer) gitPath() string { + return filepath.Join(r.baseDir, "git") } -func (g *RepositoryRestorer) wikiPath() string { - return filepath.Join(g.baseDir, "wiki") +func (r *RepositoryRestorer) wikiPath() string { + return filepath.Join(r.baseDir, "wiki") } -func (g *RepositoryRestorer) topicDir() string { - return filepath.Join(g.baseDir) +func (r *RepositoryRestorer) topicDir() string { + return filepath.Join(r.baseDir) } -func (g *RepositoryRestorer) milestoneDir() string { - return filepath.Join(g.baseDir) +func (r *RepositoryRestorer) milestoneDir() string { + return filepath.Join(r.baseDir) } -func (g *RepositoryRestorer) labelDir() string { - return filepath.Join(g.baseDir) +func (r *RepositoryRestorer) labelDir() string { + return filepath.Join(r.baseDir) } -func (g *RepositoryRestorer) releaseDir() string { - return filepath.Join(g.baseDir) +func (r *RepositoryRestorer) releaseDir() string { + return filepath.Join(r.baseDir) } -func (g *RepositoryRestorer) issueDir() string { - return filepath.Join(g.baseDir) +func (r *RepositoryRestorer) issueDir() string { + return filepath.Join(r.baseDir) } -func (g *RepositoryRestorer) commentDir() string { - return filepath.Join(g.baseDir, "comments") +func (r *RepositoryRestorer) commentDir() string { + return filepath.Join(r.baseDir, "comments") } -func (g *RepositoryRestorer) pullrequestDir() string { - return filepath.Join(g.baseDir) +func (r *RepositoryRestorer) pullrequestDir() string { + return filepath.Join(r.baseDir) } -func (g *RepositoryRestorer) reviewDir() string { - return filepath.Join(g.baseDir, "reviews") +func (r *RepositoryRestorer) reviewDir() string { + return filepath.Join(r.baseDir, "reviews") } +// SetContext set context func (r *RepositoryRestorer) SetContext(ctx context.Context) { r.ctx = ctx } +// GetRepoInfo returns a repository information func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) { - return nil, nil -} - + return &base.Repository{ + Owner: r.repoOwner, + Name: r.repoName, + IsPrivate: true, + //Description: gr.GetDescription(), // FIXME + //OriginalURL: gr.GetHTMLURL(), // FIXME + //CloneURL: gr.GetCloneURL(), // FIXME + DefaultBranch: "master", // FIXME + }, nil +} + +// GetTopics return github topics func (r *RepositoryRestorer) GetTopics() ([]string, error) { p := filepath.Join(r.topicDir(), "topic.yml") @@ -101,6 +113,7 @@ func (r *RepositoryRestorer) GetTopics() ([]string, error) { return topics.Topics, nil } +// GetMilestones returns milestones func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) { var milestones = make([]*base.Milestone, 0, 10) p := filepath.Join(r.milestoneDir(), "milestone.yml") @@ -116,6 +129,7 @@ func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) { return milestones, nil } +// GetReleases returns releases func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) { var releases = make([]*base.Release, 0, 10) p := filepath.Join(r.releaseDir(), "release.yml") @@ -131,12 +145,14 @@ func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) { return releases, nil } +// GetAsset returns an asset func (r *RepositoryRestorer) GetAsset(tagName string, relID, assetID int64) (io.ReadCloser, error) { attachDir := filepath.Join(r.releaseDir(), "release_assets", tagName) attachLocalPath := filepath.Join(attachDir, fmt.Sprintf("%d", assetID)) return os.Open(attachLocalPath) } +// GetLabels returns labels func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) { var labels = make([]*base.Label, 0, 10) p := filepath.Join(r.labelDir(), "label.yml") @@ -152,6 +168,7 @@ func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) { return labels, nil } +// GetIssues returns issues according start and limit func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { var issues = make([]*base.Issue, 0, 10) p := filepath.Join(r.issueDir(), "issue.yml") @@ -167,6 +184,7 @@ func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, return issues, true, nil } +// GetComments returns comments according issueNumber func (r *RepositoryRestorer) GetComments(issueNumber int64) ([]*base.Comment, error) { var comments = make([]*base.Comment, 0, 10) p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", issueNumber)) @@ -182,6 +200,7 @@ func (r *RepositoryRestorer) GetComments(issueNumber int64) ([]*base.Comment, er return comments, nil } +// GetPullRequests returns pull requests according page and perPage func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { var pulls = make([]*base.PullRequest, 0, 10) p := filepath.Join(r.pullrequestDir(), "pull_request.yml") @@ -197,6 +216,7 @@ func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullReq return pulls, true, nil } +// GetReviews returns pull requests review func (r *RepositoryRestorer) GetReviews(pullRequestNumber int64) ([]*base.Review, error) { var reviews = make([]*base.Review, 0, 10) p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", pullRequestNumber)) From 08e7770121b6f7ee6459dd70ae90952086d22a5d Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 21 Oct 2020 16:07:13 +0800 Subject: [PATCH 15/47] Fix restore --- cmd/restore_repo.go | 2 +- main.go | 1 + models/task.go | 4 --- modules/migrations/base/uploader.go | 1 + modules/migrations/dump.go | 19 ++++++++-- modules/migrations/error.go | 5 ++- modules/migrations/gitea_uploader.go | 10 ++++++ modules/migrations/gitea_uploader_test.go | 5 ++- modules/migrations/migrate.go | 4 ++- modules/migrations/restore.go | 42 ++++++++++++++++++++++- routers/api/v1/repo/migrate.go | 9 ----- 11 files changed, 81 insertions(+), 21 deletions(-) diff --git a/cmd/restore_repo.go b/cmd/restore_repo.go index 684b768e2ae73..2904fff726608 100644 --- a/cmd/restore_repo.go +++ b/cmd/restore_repo.go @@ -101,7 +101,7 @@ func runRestoreRepository(ctx *cli.Context) error { ctx.String("owner_name"), ctx.String("repo_name"), ); err != nil { - log.Fatal("Failed to dump repository: %v", err) + log.Fatal("Failed to restore repository: %v", err) return err } diff --git a/main.go b/main.go index 29eb21d535f33..6cbdc244018b4 100644 --- a/main.go +++ b/main.go @@ -73,6 +73,7 @@ arguments - which can alternatively be run by running the subcommand web.` cmd.CmdMigrateStorage, cmd.CmdDocs, cmd.CmdDumpRepository, + cmd.CmdRestoreRepository, } // Now adjust these commands to add our global configuration options diff --git a/models/task.go b/models/task.go index b86314b44938a..b729bb8632a64 100644 --- a/models/task.go +++ b/models/task.go @@ -211,10 +211,6 @@ func FinishMigrateTask(task *Task) error { if _, err := sess.ID(task.ID).Cols("status", "end_time").Update(task); err != nil { return err } - task.Repo.Status = RepositoryReady - if _, err := sess.ID(task.RepoID).Cols("status").Update(task.Repo); err != nil { - return err - } return sess.Commit() } diff --git a/modules/migrations/base/uploader.go b/modules/migrations/base/uploader.go index 07c2bb0d42396..63cb085a4856f 100644 --- a/modules/migrations/base/uploader.go +++ b/modules/migrations/base/uploader.go @@ -19,5 +19,6 @@ type Uploader interface { CreatePullRequests(prs ...*PullRequest) error CreateReviews(reviews ...*Review) error Rollback() error + Finish() error Close() } diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index de8ee2dfefb8f..6dea5a4cda194 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -486,7 +486,7 @@ func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { func (g *RepositoryDumper) CreateReviews(reviews ...*base.Review) error { var reviewsMap = make(map[int64][]*base.Review, len(reviews)) for _, review := range reviews { - reviewsMap[review.ID] = append(reviewsMap[review.ID], review) + reviewsMap[review.IssueIndex] = append(reviewsMap[review.IssueIndex], review) } if err := os.MkdirAll(g.reviewDir(), os.ModePerm); err != nil { @@ -523,6 +523,10 @@ func (g *RepositoryDumper) Rollback() error { return os.RemoveAll(g.baseDir) } +func (g *RepositoryDumper) Finish() error { + return nil +} + // DumpRepository dump repository according MigrateOptions to a local directory func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.MigrateOptions) error { var uploader = NewRepositoryDumper(ctx, baseDir, ownerName, opts.RepoName, opts.ReleaseAssets) @@ -537,5 +541,14 @@ func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName } var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, repoName) var downloader = NewRepositoryRestorer(ctx, baseDir, ownerName, repoName) - return DoMigrateRepository(downloader, uploader, base.MigrateOptions{}) -} + return DoMigrateRepository(downloader, uploader, base.MigrateOptions{ + Wiki: true, + Issues: true, + Milestones: true, + Labels: true, + Releases: true, + Comments: true, + PullRequests: true, + ReleaseAssets: true, + }) +} \ No newline at end of file diff --git a/modules/migrations/error.go b/modules/migrations/error.go index b2e2315fc879d..a3fd1a630a790 100644 --- a/modules/migrations/error.go +++ b/modules/migrations/error.go @@ -14,6 +14,9 @@ import ( var ( // ErrNotSupported returns the error not supported ErrNotSupported = errors.New("not supported") + + // ErrRepoNotCreated returns the error that repository not created + ErrRepoNotCreated = errors.New("repository is not created yet") ) // IsRateLimitError returns true if the err is github.RateLimitError @@ -26,4 +29,4 @@ func IsRateLimitError(err error) bool { func IsTwoFactorAuthError(err error) bool { _, ok := err.(*github.TwoFactorAuthError) return ok -} +} \ No newline at end of file diff --git a/modules/migrations/gitea_uploader.go b/modules/migrations/gitea_uploader.go index 91ddda9c3958c..d73cc550d511f 100644 --- a/modules/migrations/gitea_uploader.go +++ b/modules/migrations/gitea_uploader.go @@ -859,3 +859,13 @@ func (g *GiteaLocalUploader) Rollback() error { } return nil } + +// Finish when migrating success, this will do some status update things. +func (g *GiteaLocalUploader) Finish() error { + if g.repo == nil || g.repo.ID <= 0{ + return ErrRepoNotCreated + } + + g.repo.Status = models.RepositoryReady + return models.UpdateRepositoryCols(g.repo, "status") +} \ No newline at end of file diff --git a/modules/migrations/gitea_uploader_test.go b/modules/migrations/gitea_uploader_test.go index 8432a1eecdadd..21aa722f8c121 100644 --- a/modules/migrations/gitea_uploader_test.go +++ b/modules/migrations/gitea_uploader_test.go @@ -33,7 +33,7 @@ func TestGiteaUploadRepo(t *testing.T) { uploader = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName) ) - err := migrateRepository(downloader, uploader, base.MigrateOptions{ + migratedRepo, err := migrateRepository(downloader, uploader, base.MigrateOptions{ CloneAddr: "https://github.com/go-xorm/builder", RepoName: repoName, AuthUsername: "", @@ -49,9 +49,12 @@ func TestGiteaUploadRepo(t *testing.T) { Mirror: false, }) assert.NoError(t, err) + assert.NotNil(t, migratedRepo) + assert.EqualValues(t, models.RepositoryReady, migratedRepo.Status) repo := models.AssertExistsAndLoadBean(t, &models.Repository{OwnerID: user.ID, Name: repoName}).(*models.Repository) assert.True(t, repo.HasWiki()) + assert.EqualValues(t, migratedRepo.ID, repo.ID) milestones, err := models.GetMilestones(models.GetMilestonesOption{ RepoID: repo.ID, diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 6327815bfcbe0..e95895331d4f9 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -251,6 +251,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts var allComments = make([]*base.Comment, 0, commentBatchSize) for _, issue := range issues { + log.Trace("migrating issue %d's comments", issue.Number) comments, err := downloader.GetComments(issue.Number) if err != nil { return err @@ -299,6 +300,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts // plain comments var allComments = make([]*base.Comment, 0, commentBatchSize) for _, pr := range prs { + log.Trace("migrating issue %d's comments", pr.Number) comments, err := downloader.GetComments(pr.Number) if err != nil { return err @@ -360,7 +362,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts } } - return nil + return uploader.Finish() } // Init migrations service diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go index dac45126c7a9d..81669c1a7cd11 100644 --- a/modules/migrations/restore.go +++ b/modules/migrations/restore.go @@ -88,7 +88,7 @@ func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) { IsPrivate: true, //Description: gr.GetDescription(), // FIXME //OriginalURL: gr.GetHTMLURL(), // FIXME - //CloneURL: gr.GetCloneURL(), // FIXME + CloneURL: r.gitPath(), // FIXME DefaultBranch: "master", // FIXME }, nil } @@ -156,6 +156,14 @@ func (r *RepositoryRestorer) GetAsset(tagName string, relID, assetID int64) (io. func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) { var labels = make([]*base.Label, 0, 10) p := filepath.Join(r.labelDir(), "label.yml") + _, err := os.Stat(p) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + bs, err := ioutil.ReadFile(p) if err != nil { return nil, err @@ -172,6 +180,14 @@ func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) { func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { var issues = make([]*base.Issue, 0, 10) p := filepath.Join(r.issueDir(), "issue.yml") + _, err := os.Stat(p) + if err != nil { + if os.IsNotExist(err) { + return nil, true, nil + } + return nil, false, err + } + bs, err := ioutil.ReadFile(p) if err != nil { return nil, false, err @@ -188,6 +204,14 @@ func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, func (r *RepositoryRestorer) GetComments(issueNumber int64) ([]*base.Comment, error) { var comments = make([]*base.Comment, 0, 10) p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", issueNumber)) + _, err := os.Stat(p) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + bs, err := ioutil.ReadFile(p) if err != nil { return nil, err @@ -204,6 +228,14 @@ func (r *RepositoryRestorer) GetComments(issueNumber int64) ([]*base.Comment, er func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { var pulls = make([]*base.PullRequest, 0, 10) p := filepath.Join(r.pullrequestDir(), "pull_request.yml") + _, err := os.Stat(p) + if err != nil { + if os.IsNotExist(err) { + return nil, true, nil + } + return nil, false, err + } + bs, err := ioutil.ReadFile(p) if err != nil { return nil, false, err @@ -220,6 +252,14 @@ func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullReq func (r *RepositoryRestorer) GetReviews(pullRequestNumber int64) ([]*base.Review, error) { var reviews = make([]*base.Review, 0, 10) p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", pullRequestNumber)) + _, err := os.Stat(p) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + bs, err := ioutil.ReadFile(p) if err != nil { return nil, err diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go index ab480c29aaf2e..b97608efa23e9 100644 --- a/routers/api/v1/repo/migrate.go +++ b/routers/api/v1/repo/migrate.go @@ -18,7 +18,6 @@ import ( "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations" - "code.gitea.io/gitea/modules/notification" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" @@ -175,14 +174,6 @@ func Migrate(ctx *context.APIContext, form api.MigrateRepoOptions) { err = errors.New(buf.String()) } - if err == nil { - repo.Status = models.RepositoryReady - if err := models.UpdateRepositoryCols(repo, "status"); err == nil { - notification.NotifyMigrateRepository(ctx.User, repoOwner, repo) - return - } - } - if repo != nil { if errDelete := models.DeleteRepository(ctx.User, repoOwner.ID, repo.ID); errDelete != nil { log.Error("DeleteRepository: %v", errDelete) From 3c7b2d20271082adecf4448e77216b1e8f8dfca5 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 21 Oct 2020 19:28:57 +0800 Subject: [PATCH 16/47] Add a token flag --- cmd/dump_repo.go | 10 +++++-- modules/migrations/dump.go | 53 +++++++++++++++++++++++++++++++++-- modules/migrations/restore.go | 27 ++++++++++++++---- 3 files changed, 81 insertions(+), 9 deletions(-) diff --git a/cmd/dump_repo.go b/cmd/dump_repo.go index 3116bca34b4cc..fe7bd02122a0a 100644 --- a/cmd/dump_repo.go +++ b/cmd/dump_repo.go @@ -42,12 +42,17 @@ var CmdDumpRepository = cli.Command{ cli.StringFlag{ Name: "auth_username", Value: "", - Usage: "The username or personal token to visit the clone_addr, it's required", + Usage: "The username to visit the clone_addr", }, cli.StringFlag{ Name: "auth_password", Value: "", - Usage: "The password to visit the clone_addr if auth_username is a real user name", + Usage: "The password to visit the clone_addr", + }, + cli.StringFlag{ + Name: "auth_token", + Value: "", + Usage: "The personal token to visit the clone_addr", }, cli.StringFlag{ Name: "owner_name", @@ -84,6 +89,7 @@ func runDumpRepository(ctx *cli.Context) error { CloneAddr: ctx.String("clone_addr"), AuthUsername: ctx.String("auth_username"), AuthPassword: ctx.String("auth_password"), + AuthToken: ctx.String("auth_token"), RepoName: ctx.String("repo_name"), } diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index 6dea5a4cda194..be0a70dd20073 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "net/http" + "net/url" "os" "path/filepath" "time" @@ -110,6 +111,54 @@ func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOp if err := os.MkdirAll(g.baseDir, os.ModePerm); err != nil { return err } + + var remoteAddr = repo.CloneURL + if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 { + u, err := url.Parse(repo.CloneURL) + if err != nil { + return err + } + u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword) + if len(opts.AuthToken) > 0 { + u.User = url.UserPassword("oauth2", opts.AuthToken) + } + remoteAddr = u.String() + } + + f, err := os.Create(filepath.Join(g.baseDir, "repo.yml")) + if err != nil { + return err + } + defer f.Close() + + bs, err := yaml.Marshal(map[string]interface{}{ + "name": repo.Name, + "owner": repo.Owner, + "description": repo.Description, + "clone_addr": remoteAddr, + "original_url": repo.OriginalURL, + "is_private": opts.Private, + "auth_username": opts.AuthUsername, + "auth_password": opts.AuthPassword, + "auth_token": opts.AuthToken, + "service_type": opts.GitServiceType, + "wiki": opts.Wiki, + "issues": opts.Issues, + "milestones": opts.Milestones, + "labels": opts.Labels, + "releases": opts.Releases, + "comments": opts.Comments, + "pulls": opts.PullRequests, + "assets": opts.ReleaseAssets, + }) + if err != nil { + return err + } + + if _, err := f.Write(bs); err != nil { + return err + } + repoPath := g.gitPath() if err := os.MkdirAll(repoPath, os.ModePerm); err != nil { return err @@ -117,7 +166,7 @@ func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOp migrateTimeout := 2 * time.Hour - err := git.Clone(opts.CloneAddr, repoPath, git.CloneRepoOptions{ + err = git.Clone(remoteAddr, repoPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, Timeout: migrateTimeout, @@ -128,7 +177,7 @@ func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOp if opts.Wiki { wikiPath := g.wikiPath() - wikiRemotePath := repository.WikiRemoteURL(opts.CloneAddr) + wikiRemotePath := repository.WikiRemoteURL(remoteAddr) if len(wikiRemotePath) > 0 { if err := os.MkdirAll(wikiPath, os.ModePerm); err != nil { return fmt.Errorf("Failed to remove %s: %v", wikiPath, err) diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go index 81669c1a7cd11..1e4cab1e09200 100644 --- a/modules/migrations/restore.go +++ b/modules/migrations/restore.go @@ -11,6 +11,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strconv" "code.gitea.io/gitea/modules/migrations/base" @@ -82,14 +83,30 @@ func (r *RepositoryRestorer) SetContext(ctx context.Context) { // GetRepoInfo returns a repository information func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) { + p := filepath.Join(r.baseDir, "repo.yml") + bs, err := ioutil.ReadFile(p) + if err != nil { + return nil, err + } + + var opts = make(map[string]string) + err = yaml.Unmarshal(bs, &opts) + if err != nil { + return nil, err + } + + isPrivate, _ := strconv.ParseBool(opts["is_private"]) + return &base.Repository{ Owner: r.repoOwner, Name: r.repoName, - IsPrivate: true, - //Description: gr.GetDescription(), // FIXME - //OriginalURL: gr.GetHTMLURL(), // FIXME - CloneURL: r.gitPath(), // FIXME - DefaultBranch: "master", // FIXME + IsPrivate: isPrivate, + Description: opts["description"], + OriginalURL: opts["original_url"], + CloneURL: opts["clone_addr"], + DefaultBranch: opts["default_branch"], + AuthUsername: opts["auth_username"], + AuthPassword: opts["auth_passwd"], }, nil } From 770aac9eaedfc3f6c0bff2ccd72bef0f1cd46af2 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 21 Oct 2020 23:06:35 +0800 Subject: [PATCH 17/47] Fix bug --- modules/migrations/dump.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index be0a70dd20073..754aa1050f5a9 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -112,19 +112,6 @@ func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOp return err } - var remoteAddr = repo.CloneURL - if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 { - u, err := url.Parse(repo.CloneURL) - if err != nil { - return err - } - u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword) - if len(opts.AuthToken) > 0 { - u.User = url.UserPassword("oauth2", opts.AuthToken) - } - remoteAddr = u.String() - } - f, err := os.Create(filepath.Join(g.baseDir, "repo.yml")) if err != nil { return err @@ -135,7 +122,7 @@ func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOp "name": repo.Name, "owner": repo.Owner, "description": repo.Description, - "clone_addr": remoteAddr, + "clone_addr": opts.CloneAddr, "original_url": repo.OriginalURL, "is_private": opts.Private, "auth_username": opts.AuthUsername, @@ -166,6 +153,19 @@ func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOp migrateTimeout := 2 * time.Hour + var remoteAddr = repo.CloneURL + if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 { + u, err := url.Parse(repo.CloneURL) + if err != nil { + return err + } + u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword) + if len(opts.AuthToken) > 0 { + u.User = url.UserPassword("oauth2", opts.AuthToken) + } + remoteAddr = u.String() + } + err = git.Clone(remoteAddr, repoPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, From d367872c842edfd26f0014a8c9073c51ce9da09f Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 21 Oct 2020 23:10:18 +0800 Subject: [PATCH 18/47] Fix test --- modules/migrations/gitea_uploader_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/migrations/gitea_uploader_test.go b/modules/migrations/gitea_uploader_test.go index 21aa722f8c121..32f9dc91acd46 100644 --- a/modules/migrations/gitea_uploader_test.go +++ b/modules/migrations/gitea_uploader_test.go @@ -33,7 +33,7 @@ func TestGiteaUploadRepo(t *testing.T) { uploader = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName) ) - migratedRepo, err := migrateRepository(downloader, uploader, base.MigrateOptions{ + err := migrateRepository(downloader, uploader, base.MigrateOptions{ CloneAddr: "https://github.com/go-xorm/builder", RepoName: repoName, AuthUsername: "", @@ -49,12 +49,11 @@ func TestGiteaUploadRepo(t *testing.T) { Mirror: false, }) assert.NoError(t, err) - assert.NotNil(t, migratedRepo) - assert.EqualValues(t, models.RepositoryReady, migratedRepo.Status) repo := models.AssertExistsAndLoadBean(t, &models.Repository{OwnerID: user.ID, Name: repoName}).(*models.Repository) assert.True(t, repo.HasWiki()) assert.EqualValues(t, migratedRepo.ID, repo.ID) + assert.EqualValues(t, models.RepositoryReady, repo.Status) milestones, err := models.GetMilestones(models.GetMilestonesOption{ RepoID: repo.ID, From bbc289c506514e23dd72d40c29adeac58097b319 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 22 Oct 2020 21:08:38 +0800 Subject: [PATCH 19/47] Fix test --- modules/migrations/gitea_uploader_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/migrations/gitea_uploader_test.go b/modules/migrations/gitea_uploader_test.go index 32f9dc91acd46..3c7def467582e 100644 --- a/modules/migrations/gitea_uploader_test.go +++ b/modules/migrations/gitea_uploader_test.go @@ -52,7 +52,6 @@ func TestGiteaUploadRepo(t *testing.T) { repo := models.AssertExistsAndLoadBean(t, &models.Repository{OwnerID: user.ID, Name: repoName}).(*models.Repository) assert.True(t, repo.HasWiki()) - assert.EqualValues(t, migratedRepo.ID, repo.ID) assert.EqualValues(t, models.RepositoryReady, repo.Status) milestones, err := models.GetMilestones(models.GetMilestonesOption{ From 4f43c445847dd62d36b578385283fd7ed7d92ffa Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 23 Oct 2020 10:40:31 +0800 Subject: [PATCH 20/47] Fix bug --- modules/migrations/dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index 754aa1050f5a9..9168478e73b07 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -313,7 +313,7 @@ func (g *RepositoryDumper) CreateReleases(downloader base.Downloader, releases . return err } for _, asset := range release.Assets { - attachLocalPath := filepath.Join(attachDir, asset.Name) + attachLocalPath := filepath.Join(attachDir, asset.ID) // download attachment err := func(attachLocalPath string) error { From 0b7786f13b278caeffd4e1210da2b0e552df9060 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 23 Oct 2020 11:28:52 +0800 Subject: [PATCH 21/47] Fix bug --- modules/migrations/dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index 9168478e73b07..cba1d79cd6f82 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -313,7 +313,7 @@ func (g *RepositoryDumper) CreateReleases(downloader base.Downloader, releases . return err } for _, asset := range release.Assets { - attachLocalPath := filepath.Join(attachDir, asset.ID) + attachLocalPath := filepath.Join(attachDir, fmt.Sprintf("%d", asset.ID)) // download attachment err := func(attachLocalPath string) error { From 7cfd9d857721b4692ee8cd3dde088f81169aaa30 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 23 Oct 2020 12:12:42 +0800 Subject: [PATCH 22/47] Fix lint --- modules/migrations/dump.go | 115 +++++++++++---------------- modules/migrations/error.go | 2 +- modules/migrations/gitea_uploader.go | 4 +- modules/migrations/restore.go | 8 -- 4 files changed, 50 insertions(+), 79 deletions(-) diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index cba1d79cd6f82..7d5df25704c6a 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -55,8 +55,8 @@ func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName strin repoOwner: repoOwner, repoName: repoName, prHeadCache: make(map[string]struct{}), - commentFiles: make(map[int64]*os.File), - reviewFiles: make(map[int64]*os.File), + commentFiles: make(map[int64]*os.File), + reviewFiles: make(map[int64]*os.File), migrateReleaseAssets: migrateReleaseAssets, } } @@ -119,24 +119,24 @@ func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOp defer f.Close() bs, err := yaml.Marshal(map[string]interface{}{ - "name": repo.Name, - "owner": repo.Owner, - "description": repo.Description, - "clone_addr": opts.CloneAddr, - "original_url": repo.OriginalURL, - "is_private": opts.Private, + "name": repo.Name, + "owner": repo.Owner, + "description": repo.Description, + "clone_addr": opts.CloneAddr, + "original_url": repo.OriginalURL, + "is_private": opts.Private, "auth_username": opts.AuthUsername, "auth_password": opts.AuthPassword, - "auth_token": opts.AuthToken, - "service_type": opts.GitServiceType, - "wiki": opts.Wiki, - "issues": opts.Issues, - "milestones": opts.Milestones, - "labels": opts.Labels, - "releases": opts.Releases, - "comments": opts.Comments, - "pulls": opts.PullRequests, - "assets": opts.ReleaseAssets, + "auth_token": opts.AuthToken, + "service_type": opts.GitServiceType, + "wiki": opts.Wiki, + "issues": opts.Issues, + "milestones": opts.Milestones, + "labels": opts.Labels, + "releases": opts.Releases, + "comments": opts.Comments, + "pulls": opts.PullRequests, + "assets": opts.ReleaseAssets, }) if err != nil { return err @@ -393,34 +393,28 @@ func (g *RepositoryDumper) CreateIssues(issues ...*base.Issue) error { return nil } -// CreateComments creates comments of issues -func (g *RepositoryDumper) CreateComments(comments ...*base.Comment) error { - var commentsMap = make(map[int64][]*base.Comment, len(comments)) - for _, comment := range comments { - commentsMap[comment.IssueIndex] = append(commentsMap[comment.IssueIndex], comment) - } - - if err := os.MkdirAll(g.commentDir(), os.ModePerm); err != nil { +func (g *RepositoryDumper) createItems(dir string, itemFiles map[int64]*os.File, itemsMap map[int64][]interface{}) error { + if err := os.MkdirAll(dir, os.ModePerm); err != nil { return err } - for issueNumber, comments := range commentsMap { + for number, items := range itemsMap { var err error - commentFile := g.commentFiles[issueNumber] - if commentFile == nil { - commentFile, err = os.Create(filepath.Join(g.commentDir(), fmt.Sprintf("%d.yml", issueNumber))) + itemFile := itemFiles[number] + if itemFile == nil { + itemFile, err = os.Create(filepath.Join(dir, fmt.Sprintf("%d.yml", number))) if err != nil { return err } - g.commentFiles[issueNumber] = commentFile + itemFiles[number] = itemFile } - bs, err := yaml.Marshal(comments) + bs, err := yaml.Marshal(items) if err != nil { return err } - if _, err := commentFile.Write(bs); err != nil { + if _, err := itemFile.Write(bs); err != nil { return err } } @@ -428,6 +422,16 @@ func (g *RepositoryDumper) CreateComments(comments ...*base.Comment) error { return nil } +// CreateComments creates comments of issues +func (g *RepositoryDumper) CreateComments(comments ...*base.Comment) error { + var commentsMap = make(map[int64][]interface{}, len(comments)) + for _, comment := range comments { + commentsMap[comment.IssueIndex] = append(commentsMap[comment.IssueIndex], comment) + } + + return g.createItems(g.commentDir(), g.commentFiles, commentsMap) +} + // CreatePullRequests creates pull requests func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { for _, pr := range prs { @@ -533,37 +537,12 @@ func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { // CreateReviews create pull request reviews func (g *RepositoryDumper) CreateReviews(reviews ...*base.Review) error { - var reviewsMap = make(map[int64][]*base.Review, len(reviews)) + var reviewsMap = make(map[int64][]interface{}, len(reviews)) for _, review := range reviews { reviewsMap[review.IssueIndex] = append(reviewsMap[review.IssueIndex], review) } - if err := os.MkdirAll(g.reviewDir(), os.ModePerm); err != nil { - return err - } - - for prNumber, reviews := range reviewsMap { - var err error - reviewFile := g.reviewFiles[prNumber] - if reviewFile == nil { - reviewFile, err = os.Create(filepath.Join(g.reviewDir(), fmt.Sprintf("%d.yml", prNumber))) - if err != nil { - return err - } - g.reviewFiles[prNumber] = reviewFile - } - - bs, err := yaml.Marshal(reviews) - if err != nil { - return err - } - - if _, err := reviewFile.Write(bs); err != nil { - return err - } - } - - return nil + return g.createItems(g.reviewDir(), g.reviewFiles, reviewsMap) } // Rollback when migrating failed, this will rollback all the changes. @@ -591,13 +570,13 @@ func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, repoName) var downloader = NewRepositoryRestorer(ctx, baseDir, ownerName, repoName) return DoMigrateRepository(downloader, uploader, base.MigrateOptions{ - Wiki: true, - Issues: true, - Milestones: true, - Labels: true, - Releases: true, - Comments: true, - PullRequests: true, + Wiki: true, + Issues: true, + Milestones: true, + Labels: true, + Releases: true, + Comments: true, + PullRequests: true, ReleaseAssets: true, }) -} \ No newline at end of file +} diff --git a/modules/migrations/error.go b/modules/migrations/error.go index a3fd1a630a790..462ba29026664 100644 --- a/modules/migrations/error.go +++ b/modules/migrations/error.go @@ -29,4 +29,4 @@ func IsRateLimitError(err error) bool { func IsTwoFactorAuthError(err error) bool { _, ok := err.(*github.TwoFactorAuthError) return ok -} \ No newline at end of file +} diff --git a/modules/migrations/gitea_uploader.go b/modules/migrations/gitea_uploader.go index d73cc550d511f..2860993a7027f 100644 --- a/modules/migrations/gitea_uploader.go +++ b/modules/migrations/gitea_uploader.go @@ -862,10 +862,10 @@ func (g *GiteaLocalUploader) Rollback() error { // Finish when migrating success, this will do some status update things. func (g *GiteaLocalUploader) Finish() error { - if g.repo == nil || g.repo.ID <= 0{ + if g.repo == nil || g.repo.ID <= 0 { return ErrRepoNotCreated } g.repo.Status = models.RepositoryReady return models.UpdateRepositoryCols(g.repo, "status") -} \ No newline at end of file +} diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go index 1e4cab1e09200..434508ec0627e 100644 --- a/modules/migrations/restore.go +++ b/modules/migrations/restore.go @@ -36,14 +36,6 @@ func NewRepositoryRestorer(ctx context.Context, baseDir string, owner, repoName } } -func (r *RepositoryRestorer) gitPath() string { - return filepath.Join(r.baseDir, "git") -} - -func (r *RepositoryRestorer) wikiPath() string { - return filepath.Join(r.baseDir, "wiki") -} - func (r *RepositoryRestorer) topicDir() string { return filepath.Join(r.baseDir) } From c43966ba2f2bce513c6f309949e826c837bae2c2 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 23 Oct 2020 19:04:23 +0800 Subject: [PATCH 23/47] Fix restore --- cmd/restore_repo.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/restore_repo.go b/cmd/restore_repo.go index 2904fff726608..0b30c10f0a2c8 100644 --- a/cmd/restore_repo.go +++ b/cmd/restore_repo.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" "github.com/urfave/cli" ) @@ -58,8 +59,12 @@ func runRestoreRepository(ctx *cli.Context) error { log.Trace("Log path: %s", setting.LogRootPath) setting.InitDBConfig() + if err := storage.Init(); err != nil { + return err + } + var opts = base.MigrateOptions{ - RepoName: ctx.String("repo_name"), + RepoName: ctx.String("repo_name"), } if len(ctx.String("units")) == 0 { From a8bdf4808f919639717689a1ca68c79dc7bfe7fd Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 23 Oct 2020 21:12:32 +0800 Subject: [PATCH 24/47] refactor downloader --- modules/migrations/dump.go | 24 ++++- modules/migrations/migrate.go | 162 ++++++++++++++++------------------ 2 files changed, 98 insertions(+), 88 deletions(-) diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index 7d5df25704c6a..003ad1746e9a5 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -551,14 +551,26 @@ func (g *RepositoryDumper) Rollback() error { return os.RemoveAll(g.baseDir) } +// Finish when migrating succeed, this will update something. func (g *RepositoryDumper) Finish() error { return nil } // DumpRepository dump repository according MigrateOptions to a local directory func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.MigrateOptions) error { + downloader, err := newDownloader(ctx, ownerName, opts) + if err != nil { + return err + } var uploader = NewRepositoryDumper(ctx, baseDir, ownerName, opts.RepoName, opts.ReleaseAssets) - return MigrateRepositoryWithUploader(ctx, ownerName, opts, uploader) + + if err := migrateRepository(downloader, uploader, opts); err != nil { + if err1 := uploader.Rollback(); err1 != nil { + log.Error("rollback failed: %v", err1) + } + return err + } + return nil } // RestoreRepository restore a repository from the disk directory @@ -569,7 +581,7 @@ func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName } var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, repoName) var downloader = NewRepositoryRestorer(ctx, baseDir, ownerName, repoName) - return DoMigrateRepository(downloader, uploader, base.MigrateOptions{ + if err = migrateRepository(downloader, uploader, base.MigrateOptions{ Wiki: true, Issues: true, Milestones: true, @@ -578,5 +590,11 @@ func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName Comments: true, PullRequests: true, ReleaseAssets: true, - }) + }); err != nil { + if err1 := uploader.Rollback(); err1 != nil { + log.Error("rollback failed: %v", err1) + } + return err + } + return nil } diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index e95895331d4f9..4d061c76fc28f 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -73,9 +73,18 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, if err != nil { return nil, err } + downloader, err := newDownloader(ctx, ownerName, opts) + if err != nil { + return nil, err + } + var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) uploader.gitServiceType = opts.GitServiceType - if err := MigrateRepositoryWithUploader(ctx, ownerName, opts, uploader); err != nil { + + if err := migrateRepository(downloader, uploader, opts); err != nil { + if err1 := uploader.Rollback(); err1 != nil { + log.Error("rollback failed: %v", err1) + } if err2 := models.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil { log.Error("create respotiry notice failed: ", err2) } @@ -84,8 +93,7 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, return uploader.repo, nil } -// MigrateRepositoryWithUploader migrate repository according MigrateOptions and uploader -func MigrateRepositoryWithUploader(ctx context.Context, ownerName string, opts base.MigrateOptions, uploader base.Uploader) error { +func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptions) (base.Downloader, error) { var ( downloader base.Downloader err error @@ -95,7 +103,7 @@ func MigrateRepositoryWithUploader(ctx context.Context, ownerName string, opts b if factory.GitServiceType() == opts.GitServiceType { downloader, err = factory.New(ctx, opts) if err != nil { - return err + return nil, err } break } @@ -116,24 +124,12 @@ func MigrateRepositoryWithUploader(ctx context.Context, ownerName string, opts b if setting.Migrations.MaxAttempts > 1 { downloader = base.NewRetryDownloader(ctx, downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff) } - - if err := migrateRepository(downloader, uploader, opts); err != nil { - if err1 := uploader.Rollback(); err1 != nil { - log.Error("rollback failed: %v", err1) - } - return err - } - - return nil + return downloader, nil } -// DoMigrateRepository will download information and then upload it to Uploader, this is a simple +// migrateRepository will download information and then upload it to Uploader, this is a simple // process for small repository. For a big repository, save all the data to disk // before upload is better -func DoMigrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions) error { - return migrateRepository(downloader, uploader, opts) -} - func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions) error { repo, err := downloader.GetRepoInfo() if err != nil { @@ -245,32 +241,30 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts return err } - if !opts.Comments { - continue - } + if opts.Comments { + var allComments = make([]*base.Comment, 0, commentBatchSize) + for _, issue := range issues { + log.Trace("migrating issue %d's comments", issue.Number) + comments, err := downloader.GetComments(issue.Number) + if err != nil { + return err + } - var allComments = make([]*base.Comment, 0, commentBatchSize) - for _, issue := range issues { - log.Trace("migrating issue %d's comments", issue.Number) - comments, err := downloader.GetComments(issue.Number) - if err != nil { - return err - } + allComments = append(allComments, comments...) - allComments = append(allComments, comments...) + if len(allComments) >= commentBatchSize { + if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil { + return err + } - if len(allComments) >= commentBatchSize { - if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil { - return err + allComments = allComments[commentBatchSize:] } - - allComments = allComments[commentBatchSize:] } - } - if len(allComments) > 0 { - if err := uploader.CreateComments(allComments...); err != nil { - return err + if len(allComments) > 0 { + if err := uploader.CreateComments(allComments...); err != nil { + return err + } } } @@ -293,66 +287,64 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts return err } - if !opts.Comments { - continue - } - - // plain comments - var allComments = make([]*base.Comment, 0, commentBatchSize) - for _, pr := range prs { - log.Trace("migrating issue %d's comments", pr.Number) - comments, err := downloader.GetComments(pr.Number) - if err != nil { - return err - } + if opts.Comments { + // plain comments + var allComments = make([]*base.Comment, 0, commentBatchSize) + for _, pr := range prs { + log.Trace("migrating issue %d's comments", pr.Number) + comments, err := downloader.GetComments(pr.Number) + if err != nil { + return err + } - allComments = append(allComments, comments...) + allComments = append(allComments, comments...) - if len(allComments) >= commentBatchSize { - if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil { - return err + if len(allComments) >= commentBatchSize { + if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil { + return err + } + allComments = allComments[commentBatchSize:] } - allComments = allComments[commentBatchSize:] } - } - if len(allComments) > 0 { - if err := uploader.CreateComments(allComments...); err != nil { - return err + if len(allComments) > 0 { + if err := uploader.CreateComments(allComments...); err != nil { + return err + } } - } - // migrate reviews - var allReviews = make([]*base.Review, 0, reviewBatchSize) - for _, pr := range prs { - number := pr.Number + // migrate reviews + var allReviews = make([]*base.Review, 0, reviewBatchSize) + for _, pr := range prs { + number := pr.Number - // on gitlab migrations pull number change - if pr.OriginalNumber > 0 { - number = pr.OriginalNumber - } + // on gitlab migrations pull number change + if pr.OriginalNumber > 0 { + number = pr.OriginalNumber + } - reviews, err := downloader.GetReviews(number) - if pr.OriginalNumber > 0 { - for i := range reviews { - reviews[i].IssueIndex = pr.Number + reviews, err := downloader.GetReviews(number) + if pr.OriginalNumber > 0 { + for i := range reviews { + reviews[i].IssueIndex = pr.Number + } + } + if err != nil { + return err } - } - if err != nil { - return err - } - allReviews = append(allReviews, reviews...) + allReviews = append(allReviews, reviews...) - if len(allReviews) >= reviewBatchSize { - if err := uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil { - return err + if len(allReviews) >= reviewBatchSize { + if err := uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil { + return err + } + allReviews = allReviews[reviewBatchSize:] } - allReviews = allReviews[reviewBatchSize:] } - } - if len(allReviews) > 0 { - if err := uploader.CreateReviews(allReviews...); err != nil { - return err + if len(allReviews) > 0 { + if err := uploader.CreateReviews(allReviews...); err != nil { + return err + } } } From 20cbc6411d44bd225cb46ef617ad691879c8eba4 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 23 Oct 2020 21:13:47 +0800 Subject: [PATCH 25/47] fmt --- cmd/dump_repo.go | 10 +++++----- modules/migrations/restore.go | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/dump_repo.go b/cmd/dump_repo.go index fe7bd02122a0a..85b628993f86e 100644 --- a/cmd/dump_repo.go +++ b/cmd/dump_repo.go @@ -86,11 +86,11 @@ func runDumpRepository(ctx *cli.Context) error { var opts = base.MigrateOptions{ GitServiceType: structs.GitServiceType(ctx.Int("git_service")), - CloneAddr: ctx.String("clone_addr"), - AuthUsername: ctx.String("auth_username"), - AuthPassword: ctx.String("auth_password"), - AuthToken: ctx.String("auth_token"), - RepoName: ctx.String("repo_name"), + CloneAddr: ctx.String("clone_addr"), + AuthUsername: ctx.String("auth_username"), + AuthPassword: ctx.String("auth_password"), + AuthToken: ctx.String("auth_token"), + RepoName: ctx.String("repo_name"), } if len(ctx.String("units")) == 0 { diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go index 434508ec0627e..be8fc65a02d1e 100644 --- a/modules/migrations/restore.go +++ b/modules/migrations/restore.go @@ -97,15 +97,15 @@ func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) { OriginalURL: opts["original_url"], CloneURL: opts["clone_addr"], DefaultBranch: opts["default_branch"], - AuthUsername: opts["auth_username"], - AuthPassword: opts["auth_passwd"], + AuthUsername: opts["auth_username"], + AuthPassword: opts["auth_passwd"], }, nil } // GetTopics return github topics func (r *RepositoryRestorer) GetTopics() ([]string, error) { p := filepath.Join(r.topicDir(), "topic.yml") - + var topics = struct { Topics []string `yaml:"topics"` }{} From 79f0a4a6da92c130fa4c79cdd808fd12e270c8bc Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 24 Oct 2020 11:14:29 +0800 Subject: [PATCH 26/47] Fix bug isEnd detection on getIssues --- modules/migrations/base/downloader.go | 1 + modules/migrations/git.go | 5 +++++ modules/migrations/gitea_downloader.go | 5 +++++ modules/migrations/github.go | 6 ++++++ modules/migrations/gitlab.go | 5 +++++ modules/migrations/migrate.go | 13 +++++++++++-- modules/migrations/restore.go | 5 +++++ 7 files changed, 38 insertions(+), 2 deletions(-) diff --git a/modules/migrations/base/downloader.go b/modules/migrations/base/downloader.go index 5c47ed53052c6..9b8d026cdd727 100644 --- a/modules/migrations/base/downloader.go +++ b/modules/migrations/base/downloader.go @@ -31,6 +31,7 @@ type Downloader interface { GetComments(issueNumber int64) ([]*Comment, error) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) GetReviews(pullRequestNumber int64) ([]*Review, error) + GetMaxPerPage() int } // DownloaderFactory defines an interface to match a downloader implementation and create a downloader diff --git a/modules/migrations/git.go b/modules/migrations/git.go index 0aad8dbef5bb7..2c0a6d1d72d0d 100644 --- a/modules/migrations/git.go +++ b/modules/migrations/git.go @@ -89,3 +89,8 @@ func (g *PlainGitDownloader) GetPullRequests(start, limit int) ([]*base.PullRequ func (g *PlainGitDownloader) GetReviews(issueNumber int64) ([]*base.Review, error) { return nil, ErrNotSupported } + +// GetMaxPerPage returns the max per page size +func (g *PlainGitDownloader) GetMaxPerPage() int { + return 100 +} diff --git a/modules/migrations/gitea_downloader.go b/modules/migrations/gitea_downloader.go index 0509c708bff5c..c408b755dc3c7 100644 --- a/modules/migrations/gitea_downloader.go +++ b/modules/migrations/gitea_downloader.go @@ -683,3 +683,8 @@ func (g *GiteaDownloader) GetReviews(index int64) ([]*base.Review, error) { } return allReviews, nil } + +// GetMaxPerPage returns the max per page size +func (g *GiteaDownloader) GetMaxPerPage() int { + return g.maxPerPage +} diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 7aa1e57274cd8..e265e235e0d76 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -342,6 +342,11 @@ func (g *GithubDownloaderV3) GetAsset(_ string, _, id int64) (io.ReadCloser, err return asset, nil } +// GetMaxPerPage returns the max per page size +func (g *GithubDownloaderV3) GetMaxPerPage() int { + return 100 +} + // GetIssues returns issues according start and limit func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { if perPage > g.maxPerPage { @@ -363,6 +368,7 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, if err != nil { return nil, false, fmt.Errorf("error while listing repos: %v", err) } + log.Trace("Request get issues %d/%d, but in fact get %d", perPage, page, len(issues)) g.rate = &resp.Rate for _, issue := range issues { if issue.IsPullRequest() { diff --git a/modules/migrations/gitlab.go b/modules/migrations/gitlab.go index b1027c4f64f6b..9c75283707cdd 100644 --- a/modules/migrations/gitlab.go +++ b/modules/migrations/gitlab.go @@ -648,3 +648,8 @@ func (g *GitlabDownloader) awardToReaction(award *gitlab.AwardEmoji) *base.React Content: award.Name, } } + +// GetMaxPerPage returns the max per page size +func (g *GitlabDownloader) GetMaxPerPage() int { + return 100 +} diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 4d061c76fc28f..ad186ac56d17e 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -127,6 +127,13 @@ func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptio return downloader, nil } +func min(a, b int) int { + if a <= b { + return a + } + return b +} + // migrateRepository will download information and then upload it to Uploader, this is a simple // process for small repository. For a big repository, save all the data to disk // before upload is better @@ -157,6 +164,8 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts } } + maxPerpage := downloader.GetMaxPerPage() + if opts.Milestones { log.Trace("migrating milestones") milestones, err := downloader.GetMilestones() @@ -232,7 +241,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts var issueBatchSize = uploader.MaxBatchInsertSize("issue") for i := 1; ; i++ { - issues, isEnd, err := downloader.GetIssues(i, issueBatchSize) + issues, isEnd, err := downloader.GetIssues(i, min(issueBatchSize, maxPerpage)) if err != nil { return err } @@ -278,7 +287,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts log.Trace("migrating pull requests and comments") var prBatchSize = uploader.MaxBatchInsertSize("pullrequest") for i := 1; ; i++ { - prs, isEnd, err := downloader.GetPullRequests(i, prBatchSize) + prs, isEnd, err := downloader.GetPullRequests(i, min(prBatchSize, maxPerpage)) if err != nil { return err } diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go index be8fc65a02d1e..df97dac8dd6ef 100644 --- a/modules/migrations/restore.go +++ b/modules/migrations/restore.go @@ -280,3 +280,8 @@ func (r *RepositoryRestorer) GetReviews(pullRequestNumber int64) ([]*base.Review } return reviews, nil } + +// GetMaxPerPage returns per page +func (r *RepositoryRestorer) GetMaxPerPage() int { + return 10000 +} From ad60586b6f51f438ed0d756f7f82cac5e3d7aeed Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 24 Oct 2020 11:28:40 +0800 Subject: [PATCH 27/47] Refactor maxPerPage --- modules/migrations/base/downloader.go | 1 - modules/migrations/git.go | 5 ----- modules/migrations/gitea_downloader.go | 5 ----- modules/migrations/github.go | 5 ----- modules/migrations/gitlab.go | 5 ----- modules/migrations/migrate.go | 13 ++----------- modules/migrations/restore.go | 5 ----- 7 files changed, 2 insertions(+), 37 deletions(-) diff --git a/modules/migrations/base/downloader.go b/modules/migrations/base/downloader.go index 9b8d026cdd727..5c47ed53052c6 100644 --- a/modules/migrations/base/downloader.go +++ b/modules/migrations/base/downloader.go @@ -31,7 +31,6 @@ type Downloader interface { GetComments(issueNumber int64) ([]*Comment, error) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) GetReviews(pullRequestNumber int64) ([]*Review, error) - GetMaxPerPage() int } // DownloaderFactory defines an interface to match a downloader implementation and create a downloader diff --git a/modules/migrations/git.go b/modules/migrations/git.go index 2c0a6d1d72d0d..0aad8dbef5bb7 100644 --- a/modules/migrations/git.go +++ b/modules/migrations/git.go @@ -89,8 +89,3 @@ func (g *PlainGitDownloader) GetPullRequests(start, limit int) ([]*base.PullRequ func (g *PlainGitDownloader) GetReviews(issueNumber int64) ([]*base.Review, error) { return nil, ErrNotSupported } - -// GetMaxPerPage returns the max per page size -func (g *PlainGitDownloader) GetMaxPerPage() int { - return 100 -} diff --git a/modules/migrations/gitea_downloader.go b/modules/migrations/gitea_downloader.go index c408b755dc3c7..0509c708bff5c 100644 --- a/modules/migrations/gitea_downloader.go +++ b/modules/migrations/gitea_downloader.go @@ -683,8 +683,3 @@ func (g *GiteaDownloader) GetReviews(index int64) ([]*base.Review, error) { } return allReviews, nil } - -// GetMaxPerPage returns the max per page size -func (g *GiteaDownloader) GetMaxPerPage() int { - return g.maxPerPage -} diff --git a/modules/migrations/github.go b/modules/migrations/github.go index e265e235e0d76..1de626a66a26b 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -342,11 +342,6 @@ func (g *GithubDownloaderV3) GetAsset(_ string, _, id int64) (io.ReadCloser, err return asset, nil } -// GetMaxPerPage returns the max per page size -func (g *GithubDownloaderV3) GetMaxPerPage() int { - return 100 -} - // GetIssues returns issues according start and limit func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { if perPage > g.maxPerPage { diff --git a/modules/migrations/gitlab.go b/modules/migrations/gitlab.go index 9c75283707cdd..b1027c4f64f6b 100644 --- a/modules/migrations/gitlab.go +++ b/modules/migrations/gitlab.go @@ -648,8 +648,3 @@ func (g *GitlabDownloader) awardToReaction(award *gitlab.AwardEmoji) *base.React Content: award.Name, } } - -// GetMaxPerPage returns the max per page size -func (g *GitlabDownloader) GetMaxPerPage() int { - return 100 -} diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index ad186ac56d17e..4d061c76fc28f 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -127,13 +127,6 @@ func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptio return downloader, nil } -func min(a, b int) int { - if a <= b { - return a - } - return b -} - // migrateRepository will download information and then upload it to Uploader, this is a simple // process for small repository. For a big repository, save all the data to disk // before upload is better @@ -164,8 +157,6 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts } } - maxPerpage := downloader.GetMaxPerPage() - if opts.Milestones { log.Trace("migrating milestones") milestones, err := downloader.GetMilestones() @@ -241,7 +232,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts var issueBatchSize = uploader.MaxBatchInsertSize("issue") for i := 1; ; i++ { - issues, isEnd, err := downloader.GetIssues(i, min(issueBatchSize, maxPerpage)) + issues, isEnd, err := downloader.GetIssues(i, issueBatchSize) if err != nil { return err } @@ -287,7 +278,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts log.Trace("migrating pull requests and comments") var prBatchSize = uploader.MaxBatchInsertSize("pullrequest") for i := 1; ; i++ { - prs, isEnd, err := downloader.GetPullRequests(i, min(prBatchSize, maxPerpage)) + prs, isEnd, err := downloader.GetPullRequests(i, prBatchSize) if err != nil { return err } diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go index df97dac8dd6ef..be8fc65a02d1e 100644 --- a/modules/migrations/restore.go +++ b/modules/migrations/restore.go @@ -280,8 +280,3 @@ func (r *RepositoryRestorer) GetReviews(pullRequestNumber int64) ([]*base.Review } return reviews, nil } - -// GetMaxPerPage returns per page -func (r *RepositoryRestorer) GetMaxPerPage() int { - return 10000 -} From d33175d28ebc0e79eea059152faf72823d555b16 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 24 Oct 2020 11:46:18 +0800 Subject: [PATCH 28/47] Remove unused codes --- modules/migrations/base/repo.go | 2 -- modules/migrations/restore.go | 2 -- 2 files changed, 4 deletions(-) diff --git a/modules/migrations/base/repo.go b/modules/migrations/base/repo.go index aca50c645a080..693a96314dc1f 100644 --- a/modules/migrations/base/repo.go +++ b/modules/migrations/base/repo.go @@ -12,8 +12,6 @@ type Repository struct { IsPrivate bool `yaml:"is_private"` IsMirror bool `yaml:"is_mirror"` Description string - AuthUsername string `yaml:"auth_username"` - AuthPassword string `yaml:"auth_password"` CloneURL string `yaml:"clone_url"` OriginalURL string `yaml:"original_url"` DefaultBranch string diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go index be8fc65a02d1e..a66e44ac442dd 100644 --- a/modules/migrations/restore.go +++ b/modules/migrations/restore.go @@ -97,8 +97,6 @@ func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) { OriginalURL: opts["original_url"], CloneURL: opts["clone_addr"], DefaultBranch: opts["default_branch"], - AuthUsername: opts["auth_username"], - AuthPassword: opts["auth_passwd"], }, nil } From 04bebd636b229858f9434b5ffee4c13c7744196d Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 24 Oct 2020 12:06:19 +0800 Subject: [PATCH 29/47] Remove unused codes --- modules/structs/repo.go | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/modules/structs/repo.go b/modules/structs/repo.go index 805aa03cabb9b..c12f8e1c18e11 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -278,30 +278,3 @@ var ( GiteaService, } ) - -// MigrateRepoOption options for migrating a repository from an external service -type MigrateRepoOption struct { - // required: true - CloneAddr string `json:"clone_addr" binding:"Required"` - AuthUsername string `json:"auth_username"` - AuthPassword string `json:"auth_password"` - AuthToken string `json:"auth_token"` - // required: true - UID int `json:"uid" binding:"Required"` - // required: true - RepoName string `json:"repo_name" binding:"Required"` - Mirror bool `json:"mirror"` - Private bool `json:"private"` - Description string `json:"description"` - OriginalURL string - GitServiceType GitServiceType - Wiki bool - Issues bool - Milestones bool - Labels bool - Releases bool - ReleaseAssets bool - Comments bool - PullRequests bool - MigrateToRepoID int64 -} From af0be0b24324ccd09e63d6ccb4f729a4007b662c Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 27 Oct 2020 12:37:45 +0800 Subject: [PATCH 30/47] Fix bug --- routers/api/v1/repo/migrate.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go index b97608efa23e9..3fd93009042a9 100644 --- a/routers/api/v1/repo/migrate.go +++ b/routers/api/v1/repo/migrate.go @@ -18,6 +18,7 @@ import ( "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations" + "code.gitea.io/gitea/modules/notification" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" @@ -174,6 +175,11 @@ func Migrate(ctx *context.APIContext, form api.MigrateRepoOptions) { err = errors.New(buf.String()) } + if err == nil { + notification.NotifyMigrateRepository(ctx.User, repoOwner, repo) + return + } + if repo != nil { if errDelete := models.DeleteRepository(ctx.User, repoOwner.ID, repo.ID); errDelete != nil { log.Error("DeleteRepository: %v", errDelete) From e615c19101d615177e6321a1414a93919c5a9911 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 29 Oct 2020 10:18:02 +0800 Subject: [PATCH 31/47] Fix restore --- modules/migrations/dump.go | 33 ++++++++++++-------------- modules/migrations/gitea_uploader.go | 35 ++++++++++++++++++---------- modules/migrations/restore.go | 16 +++++++++++++ 3 files changed, 54 insertions(+), 30 deletions(-) diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index 003ad1746e9a5..a9be2e1b630c3 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -119,24 +119,21 @@ func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOp defer f.Close() bs, err := yaml.Marshal(map[string]interface{}{ - "name": repo.Name, - "owner": repo.Owner, - "description": repo.Description, - "clone_addr": opts.CloneAddr, - "original_url": repo.OriginalURL, - "is_private": opts.Private, - "auth_username": opts.AuthUsername, - "auth_password": opts.AuthPassword, - "auth_token": opts.AuthToken, - "service_type": opts.GitServiceType, - "wiki": opts.Wiki, - "issues": opts.Issues, - "milestones": opts.Milestones, - "labels": opts.Labels, - "releases": opts.Releases, - "comments": opts.Comments, - "pulls": opts.PullRequests, - "assets": opts.ReleaseAssets, + "name": repo.Name, + "owner": repo.Owner, + "description": repo.Description, + "clone_addr": opts.CloneAddr, + "original_url": repo.OriginalURL, + "is_private": opts.Private, + "service_type": opts.GitServiceType, + "wiki": opts.Wiki, + "issues": opts.Issues, + "milestones": opts.Milestones, + "labels": opts.Labels, + "releases": opts.Releases, + "comments": opts.Comments, + "pulls": opts.PullRequests, + "assets": opts.ReleaseAssets, }) if err != nil { return err diff --git a/modules/migrations/gitea_uploader.go b/modules/migrations/gitea_uploader.go index 2860993a7027f..3467a61b8cd75 100644 --- a/modules/migrations/gitea_uploader.go +++ b/modules/migrations/gitea_uploader.go @@ -86,26 +86,33 @@ func (g *GiteaLocalUploader) MaxBatchInsertSize(tp string) int { return 10 } -// CreateRepo creates a repository -func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error { - owner, err := models.GetUserByName(g.repoOwner) - if err != nil { - return err - } - - var remoteAddr = repo.CloneURL +func fullURL(opts base.MigrateOptions, remoteAddr string) (string, error) { + var fullRemoteAddr = remoteAddr if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 { - u, err := url.Parse(repo.CloneURL) + u, err := url.Parse(remoteAddr) if err != nil { - return err + return "", err } u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword) if len(opts.AuthToken) > 0 { u.User = url.UserPassword("oauth2", opts.AuthToken) } - remoteAddr = u.String() + fullRemoteAddr = u.String() + } + return fullRemoteAddr, nil +} + +// CreateRepo creates a repository +func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error { + owner, err := models.GetUserByName(g.repoOwner) + if err != nil { + return err } + remoteAddr, err := fullURL(opts, repo.CloneURL) + if err != nil { + return err + } var r *models.Repository if opts.MigrateToRepoID <= 0 { r, err = repo_module.CreateRepository(g.doer, owner, models.CreateRepoOptions{ @@ -290,7 +297,11 @@ func (g *GiteaLocalUploader) CreateReleases(downloader base.Downloader, releases return err } } else { - resp, err := http.Get(*asset.DownloadURL) + remoteAddr, err := fullURL(opts, *asset.DownloadURL) + if err != nil { + return err + } + resp, err := http.Get(remoteAddr) if err != nil { return err } diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go index a66e44ac442dd..5780bb63f2215 100644 --- a/modules/migrations/restore.go +++ b/modules/migrations/restore.go @@ -124,6 +124,14 @@ func (r *RepositoryRestorer) GetTopics() ([]string, error) { func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) { var milestones = make([]*base.Milestone, 0, 10) p := filepath.Join(r.milestoneDir(), "milestone.yml") + _, err := os.Stat(p) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + bs, err := ioutil.ReadFile(p) if err != nil { return nil, err @@ -140,6 +148,14 @@ func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) { func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) { var releases = make([]*base.Release, 0, 10) p := filepath.Join(r.releaseDir(), "release.yml") + _, err := os.Stat(p) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + bs, err := ioutil.ReadFile(p) if err != nil { return nil, err From ab53820eacc06bb5cd54d7c78a60ecbdc48aef52 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 3 Nov 2020 16:27:40 +0800 Subject: [PATCH 32/47] Fix dump --- modules/migrations/dump.go | 58 +++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index a9be2e1b630c3..28ce7e5b3abd5 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -33,6 +33,7 @@ type RepositoryDumper struct { baseDir string repoOwner string repoName string + opts base.MigrateOptions milestoneFile *os.File labelFile *os.File releaseFile *os.File @@ -47,17 +48,17 @@ type RepositoryDumper struct { } // NewRepositoryDumper creates an gitea Uploader -func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName string, migrateReleaseAssets bool) *RepositoryDumper { +func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName string, opts base.MigrateOptions) *RepositoryDumper { baseDir = filepath.Join(baseDir, repoOwner, repoName) return &RepositoryDumper{ - ctx: ctx, - baseDir: baseDir, - repoOwner: repoOwner, - repoName: repoName, - prHeadCache: make(map[string]struct{}), - commentFiles: make(map[int64]*os.File), - reviewFiles: make(map[int64]*os.File), - migrateReleaseAssets: migrateReleaseAssets, + ctx: ctx, + opts: opts, + baseDir: baseDir, + repoOwner: repoOwner, + repoName: repoName, + prHeadCache: make(map[string]struct{}), + commentFiles: make(map[int64]*os.File), + reviewFiles: make(map[int64]*os.File), } } @@ -106,6 +107,22 @@ func (g *RepositoryDumper) reviewDir() string { return filepath.Join(g.baseDir, "reviews") } +func (g *RepositoryDumper) setURLToken(remoteAddr string) (string, error) { + if len(g.opts.AuthToken) > 0 || len(g.opts.AuthUsername) > 0 { + u, err := url.Parse(remoteAddr) + if err != nil { + return "", err + } + u.User = url.UserPassword(g.opts.AuthUsername, g.opts.AuthPassword) + if len(g.opts.AuthToken) > 0 { + u.User = url.UserPassword("oauth2", g.opts.AuthToken) + } + remoteAddr = u.String() + } + + return remoteAddr, nil +} + // CreateRepo creates a repository func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error { if err := os.MkdirAll(g.baseDir, os.ModePerm); err != nil { @@ -150,17 +167,9 @@ func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOp migrateTimeout := 2 * time.Hour - var remoteAddr = repo.CloneURL - if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 { - u, err := url.Parse(repo.CloneURL) - if err != nil { - return err - } - u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword) - if len(opts.AuthToken) > 0 { - u.User = url.UserPassword("oauth2", opts.AuthToken) - } - remoteAddr = u.String() + remoteAddr, err := g.setURLToken(repo.CloneURL) + if err != nil { + return err } err = git.Clone(remoteAddr, repoPath, git.CloneRepoOptions{ @@ -434,7 +443,11 @@ func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { for _, pr := range prs { // download patch file err := func() error { - resp, err := http.Get(pr.PatchURL) + u, err := g.setURLToken(pr.PatchURL) + if err != nil { + return err + } + resp, err := http.Get(u) if err != nil { return err } @@ -476,6 +489,7 @@ func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { _, ok := g.prHeadCache[remote] if !ok { // git remote add + // TODO: how to handle private CloneURL? err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true) if err != nil { log.Error("AddRemote failed: %s", err) @@ -559,7 +573,7 @@ func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.Mi if err != nil { return err } - var uploader = NewRepositoryDumper(ctx, baseDir, ownerName, opts.RepoName, opts.ReleaseAssets) + var uploader = NewRepositoryDumper(ctx, baseDir, ownerName, opts.RepoName, opts) if err := migrateRepository(downloader, uploader, opts); err != nil { if err1 := uploader.Rollback(); err1 != nil { From 96ca8e41ef7d3eaa84bc6783166a7d61e74c765d Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 24 Nov 2020 07:57:00 +0800 Subject: [PATCH 33/47] Uploader should not depend downloader --- modules/migrations/base/downloader.go | 7 ----- modules/migrations/base/uploader.go | 2 +- modules/migrations/dump.go | 18 ++++++++---- modules/migrations/git.go | 6 ---- modules/migrations/gitea_downloader.go | 1 + modules/migrations/gitea_uploader.go | 32 +++++++-------------- modules/migrations/migrate.go | 2 +- modules/uri/uri.go | 40 ++++++++++++++++++++++++++ modules/uri/uri_test.go | 20 +++++++++++++ 9 files changed, 86 insertions(+), 42 deletions(-) create mode 100644 modules/uri/uri.go create mode 100644 modules/uri/uri_test.go diff --git a/modules/migrations/base/downloader.go b/modules/migrations/base/downloader.go index 5c47ed53052c6..afa99105c9e6c 100644 --- a/modules/migrations/base/downloader.go +++ b/modules/migrations/base/downloader.go @@ -7,20 +7,13 @@ package base import ( "context" - "io" "time" "code.gitea.io/gitea/modules/structs" ) -// AssetDownloader downloads an asset (attachment) for a release -type AssetDownloader interface { - GetAsset(relTag string, relID, id int64) (io.ReadCloser, error) -} - // Downloader downloads the site repo informations type Downloader interface { - AssetDownloader SetContext(context.Context) GetRepoInfo() (*Repository, error) GetTopics() ([]string, error) diff --git a/modules/migrations/base/uploader.go b/modules/migrations/base/uploader.go index 63cb085a4856f..dfcf81d052240 100644 --- a/modules/migrations/base/uploader.go +++ b/modules/migrations/base/uploader.go @@ -11,7 +11,7 @@ type Uploader interface { CreateRepo(repo *Repository, opts MigrateOptions) error CreateTopics(topic ...string) error CreateMilestones(milestones ...*Milestone) error - CreateReleases(downloader Downloader, releases ...*Release) error + CreateReleases(releases ...*Release) error SyncTags() error CreateLabels(labels ...*Label) error CreateIssues(issues ...*Issue) error diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index 28ce7e5b3abd5..f2c37837925eb 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/uri" "gopkg.in/yaml.v2" ) @@ -311,7 +312,7 @@ func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error { } // CreateReleases creates releases -func (g *RepositoryDumper) CreateReleases(downloader base.Downloader, releases ...*base.Release) error { +func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { if g.migrateReleaseAssets { for _, release := range releases { attachDir := filepath.Join(g.releaseDir(), "release_assets", release.TagName) @@ -323,8 +324,7 @@ func (g *RepositoryDumper) CreateReleases(downloader base.Downloader, releases . // download attachment err := func(attachLocalPath string) error { - // FIXME: release ID - rc, err := downloader.GetAsset(release.TagName, 0, asset.ID) + rc, err := uri.Open(*asset.DownloadURL) if err != nil { return err } @@ -456,13 +456,19 @@ func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { if err = os.MkdirAll(pullDir, os.ModePerm); err != nil { return err } - f, err := os.Create(filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number))) + fPath := filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number)) + f, err := os.Create(fPath) if err != nil { return err } defer f.Close() - _, err = io.Copy(f, resp.Body) - return err + if _, err = io.Copy(f, resp.Body); err != nil { + return err + } + fAbsPath, _ := filepath.Abs(fPath) + pr.PatchURL = "file://" + fAbsPath + + return nil }() if err != nil { return err diff --git a/modules/migrations/git.go b/modules/migrations/git.go index 0aad8dbef5bb7..88222086e4a8e 100644 --- a/modules/migrations/git.go +++ b/modules/migrations/git.go @@ -6,7 +6,6 @@ package migrations import ( "context" - "io" "code.gitea.io/gitea/modules/migrations/base" ) @@ -65,11 +64,6 @@ func (g *PlainGitDownloader) GetReleases() ([]*base.Release, error) { return nil, ErrNotSupported } -// GetAsset returns an asset -func (g *PlainGitDownloader) GetAsset(_ string, _, _ int64) (io.ReadCloser, error) { - return nil, ErrNotSupported -} - // GetIssues returns issues according page and perPage func (g *PlainGitDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { return nil, false, ErrNotSupported diff --git a/modules/migrations/gitea_downloader.go b/modules/migrations/gitea_downloader.go index 0509c708bff5c..d1b482e3e3dbc 100644 --- a/modules/migrations/gitea_downloader.go +++ b/modules/migrations/gitea_downloader.go @@ -316,6 +316,7 @@ func (g *GiteaDownloader) GetAsset(_ string, relID, id int64) (io.ReadCloser, er if err != nil { return nil, err } + // FIXME: for a private download? resp, err := http.Get(asset.DownloadURL) if err != nil { return nil, err diff --git a/modules/migrations/gitea_uploader.go b/modules/migrations/gitea_uploader.go index 3467a61b8cd75..64503e8c132ab 100644 --- a/modules/migrations/gitea_uploader.go +++ b/modules/migrations/gitea_uploader.go @@ -10,7 +10,6 @@ import ( "context" "fmt" "io" - "net/http" "net/url" "os" "path/filepath" @@ -28,6 +27,7 @@ import ( "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/uri" "code.gitea.io/gitea/services/pull" gouuid "github.com/google/uuid" @@ -231,7 +231,7 @@ func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error { } // CreateReleases creates releases -func (g *GiteaLocalUploader) CreateReleases(downloader base.Downloader, releases ...*base.Release) error { +func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { var rels = make([]*models.Release, 0, len(releases)) for _, release := range releases { var rel = models.Release{ @@ -290,23 +290,12 @@ func (g *GiteaLocalUploader) CreateReleases(downloader base.Downloader, releases // download attachment err = func() error { - var rc io.ReadCloser - if asset.DownloadURL == nil { - rc, err = downloader.GetAsset(rel.TagName, rel.ID, asset.ID) - if err != nil { - return err - } - } else { - remoteAddr, err := fullURL(opts, *asset.DownloadURL) - if err != nil { - return err - } - resp, err := http.Get(remoteAddr) - if err != nil { - return err - } - rc = resp.Body + // asset.DownloadURL maybe a local file + rc, err := uri.Open(*asset.DownloadURL) + if err != nil { + return err } + defer rc.Close() _, err = storage.Attachments.Save(attach.RelativePath(), rc) return err }() @@ -570,11 +559,12 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR // download patch file err := func() error { - resp, err := http.Get(pr.PatchURL) + // pr.PatchURL maybe a local file + ret, err := uri.Open(pr.PatchURL) if err != nil { return err } - defer resp.Body.Close() + defer ret.Close() pullDir := filepath.Join(g.repo.RepoPath(), "pulls") if err = os.MkdirAll(pullDir, os.ModePerm); err != nil { return err @@ -584,7 +574,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR return err } defer f.Close() - _, err = io.Copy(f, resp.Body) + _, err = io.Copy(f, ret) return err }() if err != nil { diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 4d061c76fc28f..cd98055a2079a 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -210,7 +210,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts relBatchSize = len(releases) } - if err := uploader.CreateReleases(downloader, releases[:relBatchSize]...); err != nil { + if err := uploader.CreateReleases(releases[:relBatchSize]...); err != nil { return err } releases = releases[relBatchSize:] diff --git a/modules/uri/uri.go b/modules/uri/uri.go new file mode 100644 index 0000000000000..0967a0802ff8c --- /dev/null +++ b/modules/uri/uri.go @@ -0,0 +1,40 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package uri + +import ( + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" +) + +// ErrURISchemeNotSupported represents a scheme error +type ErrURISchemeNotSupported struct { + Scheme string +} + +func (e ErrURISchemeNotSupported) Error() string { + return fmt.Sprintf("Unsupported scheme: %v", e.Scheme) +} + +// Open open a local file or a remote file +func Open(uriStr string) (io.ReadCloser, error) { + u, err := url.Parse(uriStr) + if err != nil { + return nil, err + } + switch strings.ToLower(u.Scheme) { + case "http", "https": + f, err := http.Get(uriStr) + return f.Body, err + case "file": + return os.Open(u.Path) + default: + return nil, ErrURISchemeNotSupported{Scheme: u.Scheme} + } +} diff --git a/modules/uri/uri_test.go b/modules/uri/uri_test.go new file mode 100644 index 0000000000000..8cadd6b918f86 --- /dev/null +++ b/modules/uri/uri_test.go @@ -0,0 +1,20 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package uri + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestReadURI(t *testing.T) { + p, err := filepath.Abs("./uri.go") + assert.NoError(t, err) + f, err := Open("file://" + p) + assert.NoError(t, err) + defer f.Close() +} From 3075168aac8ad92d5ae74a4b75e7f77a0944e8d1 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 21 Dec 2020 16:13:58 +0800 Subject: [PATCH 34/47] use release attachment name but not id --- modules/migrations/dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index f2c37837925eb..db28f65911106 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -320,7 +320,7 @@ func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { return err } for _, asset := range release.Assets { - attachLocalPath := filepath.Join(attachDir, fmt.Sprintf("%d", asset.ID)) + attachLocalPath := filepath.Join(attachDir, fmt.Sprintf("%s", asset.Name)) // download attachment err := func(attachLocalPath string) error { From 97cc82c66e78ddd0af981a3a96669bf94640b084 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 22 Dec 2020 20:02:49 +0800 Subject: [PATCH 35/47] Fix restore bug --- cmd/restore_repo.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/restore_repo.go b/cmd/restore_repo.go index 0b30c10f0a2c8..8569c43365c49 100644 --- a/cmd/restore_repo.go +++ b/cmd/restore_repo.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" + pull_service "code.gitea.io/gitea/services/pull" "github.com/urfave/cli" ) @@ -63,6 +64,10 @@ func runRestoreRepository(ctx *cli.Context) error { return err } + if err := pull_service.Init(); err != nil { + return err + } + var opts = base.MigrateOptions{ RepoName: ctx.String("repo_name"), } From 6c5044dc21987bfb123684aa96a6b03745431d6d Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 22 Dec 2020 21:44:46 +0800 Subject: [PATCH 36/47] Fix lint --- modules/migrations/dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index db28f65911106..a5efd46d5b580 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -320,7 +320,7 @@ func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { return err } for _, asset := range release.Assets { - attachLocalPath := filepath.Join(attachDir, fmt.Sprintf("%s", asset.Name)) + attachLocalPath := filepath.Join(attachDir, asset.Name) // download attachment err := func(attachLocalPath string) error { From c09c7ed2b2c5b1bf962f92e519d679d1de82992b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 26 Dec 2020 11:28:36 +0800 Subject: [PATCH 37/47] Fix restore bug --- modules/migrations/dump.go | 28 ++++++++++++++-------------- modules/migrations/gitea_uploader.go | 22 ++++++++++++---------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index a5efd46d5b580..4a90d5f47e8e5 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -30,19 +30,18 @@ var ( // RepositoryDumper implements an Uploader to the local directory type RepositoryDumper struct { - ctx context.Context - baseDir string - repoOwner string - repoName string - opts base.MigrateOptions - milestoneFile *os.File - labelFile *os.File - releaseFile *os.File - issueFile *os.File - commentFiles map[int64]*os.File - pullrequestFile *os.File - reviewFiles map[int64]*os.File - migrateReleaseAssets bool + ctx context.Context + baseDir string + repoOwner string + repoName string + opts base.MigrateOptions + milestoneFile *os.File + labelFile *os.File + releaseFile *os.File + issueFile *os.File + commentFiles map[int64]*os.File + pullrequestFile *os.File + reviewFiles map[int64]*os.File gitRepo *git.Repository prHeadCache map[string]struct{} @@ -313,7 +312,7 @@ func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error { // CreateReleases creates releases func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { - if g.migrateReleaseAssets { + if g.opts.ReleaseAssets { for _, release := range releases { attachDir := filepath.Join(g.releaseDir(), "release_assets", release.TagName) if err := os.MkdirAll(attachDir, os.ModePerm); err != nil { @@ -342,6 +341,7 @@ func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { if err != nil { return err } + asset.DownloadURL = &attachLocalPath } } } diff --git a/modules/migrations/gitea_uploader.go b/modules/migrations/gitea_uploader.go index 64503e8c132ab..292970851f8c8 100644 --- a/modules/migrations/gitea_uploader.go +++ b/modules/migrations/gitea_uploader.go @@ -288,19 +288,21 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { CreatedUnix: timeutil.TimeStamp(asset.Created.Unix()), } - // download attachment - err = func() error { - // asset.DownloadURL maybe a local file - rc, err := uri.Open(*asset.DownloadURL) + if asset.DownloadURL != nil { + // download attachment + err = func() error { + // asset.DownloadURL maybe a local file + rc, err := uri.Open(*asset.DownloadURL) + if err != nil { + return err + } + defer rc.Close() + _, err = storage.Attachments.Save(attach.RelativePath(), rc) + return err + }() if err != nil { return err } - defer rc.Close() - _, err = storage.Attachments.Save(attach.RelativePath(), rc) - return err - }() - if err != nil { - return err } rel.Attachments = append(rel.Attachments, &attach) } From 8f4e74144f9c1ff949309f12c9e87636432112d8 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 26 Dec 2020 13:51:16 +0800 Subject: [PATCH 38/47] Add a method of DownloadFunc for base.Release to make uploader not depend on downloader --- modules/migrations/base/release.go | 7 ++++- modules/migrations/dump.go | 20 +++++++++--- modules/migrations/gitea_downloader.go | 30 +++++++++--------- modules/migrations/gitea_uploader.go | 31 ++++++++++++------- modules/migrations/github.go | 22 ++++++-------- modules/migrations/gitlab.go | 42 ++++++++++++-------------- 6 files changed, 85 insertions(+), 67 deletions(-) diff --git a/modules/migrations/base/release.go b/modules/migrations/base/release.go index 777b3a3fe0fff..a142cfd6782a0 100644 --- a/modules/migrations/base/release.go +++ b/modules/migrations/base/release.go @@ -4,7 +4,10 @@ package base -import "time" +import ( + "io" + "time" +) // ReleaseAsset represents a release asset type ReleaseAsset struct { @@ -16,6 +19,8 @@ type ReleaseAsset struct { Created time.Time Updated time.Time DownloadURL *string + // if DownloadURL is nil, the function should be invoked + DownloadFunc func() (io.ReadCloser, error) } // Release represents a release diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index 4a90d5f47e8e5..96a48ebac9a37 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -19,7 +19,6 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/repository" - "code.gitea.io/gitea/modules/uri" "gopkg.in/yaml.v2" ) @@ -323,9 +322,19 @@ func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { // download attachment err := func(attachLocalPath string) error { - rc, err := uri.Open(*asset.DownloadURL) - if err != nil { - return err + var rc io.ReadCloser + var err error + if asset.DownloadURL == nil { + rc, err = asset.DownloadFunc() + if err != nil { + return err + } + } else { + resp, err := http.Get(*asset.DownloadURL) + if err != nil { + return err + } + rc = resp.Body } defer rc.Close() @@ -341,7 +350,8 @@ func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { if err != nil { return err } - asset.DownloadURL = &attachLocalPath + attachLocalPath = "file://" + attachLocalPath + asset.DownloadURL = &attachLocalPath // to save the filepath on the yml file, change the source } } } diff --git a/modules/migrations/gitea_downloader.go b/modules/migrations/gitea_downloader.go index d1b482e3e3dbc..c96c4f672baf4 100644 --- a/modules/migrations/gitea_downloader.go +++ b/modules/migrations/gitea_downloader.go @@ -275,6 +275,20 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele DownloadCount: &dlCount, Created: asset.Created, DownloadURL: &asset.DownloadURL, + DownloadFunc: func() (io.ReadCloser, error) { + asset, _, err := g.client.GetReleaseAttachment(g.repoOwner, g.repoName, rel.ID, asset.ID) + if err != nil { + return nil, err + } + // FIXME: for a private download? + resp, err := http.Get(asset.DownloadURL) + if err != nil { + return nil, err + } + + // resp.Body is closed by the uploader + return resp.Body, nil + }, }) } return r @@ -310,22 +324,6 @@ func (g *GiteaDownloader) GetReleases() ([]*base.Release, error) { return releases, nil } -// GetAsset returns an asset -func (g *GiteaDownloader) GetAsset(_ string, relID, id int64) (io.ReadCloser, error) { - asset, _, err := g.client.GetReleaseAttachment(g.repoOwner, g.repoName, relID, id) - if err != nil { - return nil, err - } - // FIXME: for a private download? - resp, err := http.Get(asset.DownloadURL) - if err != nil { - return nil, err - } - - // resp.Body is closed by the uploader - return resp.Body, nil -} - func (g *GiteaDownloader) getIssueReactions(index int64) ([]*base.Reaction, error) { var reactions []*base.Reaction if err := g.client.CheckServerVersionConstraint(">=1.11"); err != nil { diff --git a/modules/migrations/gitea_uploader.go b/modules/migrations/gitea_uploader.go index 292970851f8c8..78a68867d6af5 100644 --- a/modules/migrations/gitea_uploader.go +++ b/modules/migrations/gitea_uploader.go @@ -10,6 +10,7 @@ import ( "context" "fmt" "io" + "net/http" "net/url" "os" "path/filepath" @@ -288,22 +289,30 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { CreatedUnix: timeutil.TimeStamp(asset.Created.Unix()), } - if asset.DownloadURL != nil { - // download attachment - err = func() error { - // asset.DownloadURL maybe a local file - rc, err := uri.Open(*asset.DownloadURL) + // download attachment + err = func() error { + // asset.DownloadURL maybe a local file + var rc io.ReadCloser + if asset.DownloadURL == nil { + rc, err = asset.DownloadFunc() if err != nil { return err } - defer rc.Close() - _, err = storage.Attachments.Save(attach.RelativePath(), rc) - return err - }() - if err != nil { - return err + } else { + resp, err := http.Get(*asset.DownloadURL) + if err != nil { + return err + } + rc = resp.Body } + defer rc.Close() + _, err = storage.Attachments.Save(attach.RelativePath(), rc) + return err + }() + if err != nil { + return err } + rel.Attachments = append(rel.Attachments, &attach) } diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 1de626a66a26b..8c082089991c8 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -299,6 +299,16 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) DownloadCount: asset.DownloadCount, Created: asset.CreatedAt.Time, Updated: asset.UpdatedAt.Time, + DownloadFunc: func() (io.ReadCloser, error) { + asset, redir, err := g.client.Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, *asset.ID, http.DefaultClient) + if err != nil { + return nil, err + } + if asset == nil { + return ioutil.NopCloser(bytes.NewBufferString(redir)), nil + } + return asset, nil + }, }) } return r @@ -330,18 +340,6 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) { return releases, nil } -// GetAsset returns an asset -func (g *GithubDownloaderV3) GetAsset(_ string, _, id int64) (io.ReadCloser, error) { - asset, redir, err := g.client.Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, id, http.DefaultClient) - if err != nil { - return nil, err - } - if asset == nil { - return ioutil.NopCloser(bytes.NewBufferString(redir)), nil - } - return asset, nil -} - // GetIssues returns issues according start and limit func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { if perPage > g.maxPerPage { diff --git a/modules/migrations/gitlab.go b/modules/migrations/gitlab.go index b1027c4f64f6b..86ea4e347da31 100644 --- a/modules/migrations/gitlab.go +++ b/modules/migrations/gitlab.go @@ -301,6 +301,26 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea ContentType: &rel.Assets.Sources[k].Format, Size: &zero, DownloadCount: &zero, + DownloadFunc: func() (io.ReadCloser, error) { + link, _, err := g.client.ReleaseLinks.GetReleaseLink(g.repoID, rel.TagName, asset.ID, gitlab.WithContext(g.ctx)) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("GET", link.URL, nil) + if err != nil { + return nil, err + } + req = req.WithContext(g.ctx) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + + // resp.Body is closed by the uploader + return resp.Body, nil + }, }) } return r @@ -329,28 +349,6 @@ func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) { return releases, nil } -// GetAsset returns an asset -func (g *GitlabDownloader) GetAsset(tag string, _, id int64) (io.ReadCloser, error) { - link, _, err := g.client.ReleaseLinks.GetReleaseLink(g.repoID, tag, int(id), gitlab.WithContext(g.ctx)) - if err != nil { - return nil, err - } - - req, err := http.NewRequest("GET", link.URL, nil) - if err != nil { - return nil, err - } - req = req.WithContext(g.ctx) - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - - // resp.Body is closed by the uploader - return resp.Body, nil -} - // GetIssues returns issues according start and limit // Note: issue label description and colors are not supported by the go-gitlab library at this time func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { From 2124a616adacf7cb6ab256306b661d8362505e85 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 26 Dec 2020 14:09:49 +0800 Subject: [PATCH 39/47] fix Release yml marshal --- modules/migrations/base/release.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/migrations/base/release.go b/modules/migrations/base/release.go index a142cfd6782a0..e89a12abd43ad 100644 --- a/modules/migrations/base/release.go +++ b/modules/migrations/base/release.go @@ -20,7 +20,7 @@ type ReleaseAsset struct { Updated time.Time DownloadURL *string // if DownloadURL is nil, the function should be invoked - DownloadFunc func() (io.ReadCloser, error) + DownloadFunc func() (io.ReadCloser, error) `yaml:"-"` } // Release represents a release From 2e90a9f1b29690e9fd4d8d3bd643cfc8859005d8 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 26 Dec 2020 14:24:51 +0800 Subject: [PATCH 40/47] Fix trace information --- modules/migrations/migrate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index cd98055a2079a..4c15626e57949 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -291,7 +291,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts // plain comments var allComments = make([]*base.Comment, 0, commentBatchSize) for _, pr := range prs { - log.Trace("migrating issue %d's comments", pr.Number) + log.Trace("migrating pull request %d's comments", pr.Number) comments, err := downloader.GetComments(pr.Number) if err != nil { return err From 2b0fe432e915e0102f1ea89947aed3d2d250818c Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 26 Dec 2020 14:53:24 +0800 Subject: [PATCH 41/47] Fix bug when dump & restore --- modules/migrations/base/release.go | 4 ++-- modules/migrations/gitea_downloader.go | 2 +- modules/migrations/gitea_uploader.go | 4 +--- modules/migrations/github.go | 2 +- modules/migrations/gitlab.go | 2 +- modules/migrations/restore.go | 8 -------- 6 files changed, 6 insertions(+), 16 deletions(-) diff --git a/modules/migrations/base/release.go b/modules/migrations/base/release.go index e89a12abd43ad..8b4339928bf63 100644 --- a/modules/migrations/base/release.go +++ b/modules/migrations/base/release.go @@ -18,7 +18,7 @@ type ReleaseAsset struct { DownloadCount *int `yaml:"download_count"` Created time.Time Updated time.Time - DownloadURL *string + DownloadURL *string `yaml:"download_url"` // if DownloadURL is nil, the function should be invoked DownloadFunc func() (io.ReadCloser, error) `yaml:"-"` } @@ -34,7 +34,7 @@ type Release struct { PublisherID int64 `yaml:"publisher_id"` PublisherName string `yaml:"publisher_name"` PublisherEmail string `yaml:"publisher_email"` - Assets []ReleaseAsset + Assets []*ReleaseAsset Created time.Time Published time.Time } diff --git a/modules/migrations/gitea_downloader.go b/modules/migrations/gitea_downloader.go index c96c4f672baf4..0c690464fa96e 100644 --- a/modules/migrations/gitea_downloader.go +++ b/modules/migrations/gitea_downloader.go @@ -268,7 +268,7 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele for _, asset := range rel.Attachments { size := int(asset.Size) dlCount := int(asset.DownloadCount) - r.Assets = append(r.Assets, base.ReleaseAsset{ + r.Assets = append(r.Assets, &base.ReleaseAsset{ ID: asset.ID, Name: asset.Name, Size: &size, diff --git a/modules/migrations/gitea_uploader.go b/modules/migrations/gitea_uploader.go index 78a68867d6af5..6118b3b5c1fb4 100644 --- a/modules/migrations/gitea_uploader.go +++ b/modules/migrations/gitea_uploader.go @@ -10,7 +10,6 @@ import ( "context" "fmt" "io" - "net/http" "net/url" "os" "path/filepath" @@ -299,11 +298,10 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { return err } } else { - resp, err := http.Get(*asset.DownloadURL) + rc, err = uri.Open(*asset.DownloadURL) if err != nil { return err } - rc = resp.Body } defer rc.Close() _, err = storage.Attachments.Save(attach.RelativePath(), rc) diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 8c082089991c8..178517ba42f2e 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -291,7 +291,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) } for _, asset := range rel.Assets { - r.Assets = append(r.Assets, base.ReleaseAsset{ + r.Assets = append(r.Assets, &base.ReleaseAsset{ ID: *asset.ID, Name: *asset.Name, ContentType: asset.ContentType, diff --git a/modules/migrations/gitlab.go b/modules/migrations/gitlab.go index 86ea4e347da31..e3fa956758d38 100644 --- a/modules/migrations/gitlab.go +++ b/modules/migrations/gitlab.go @@ -295,7 +295,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea } for k, asset := range rel.Assets.Links { - r.Assets = append(r.Assets, base.ReleaseAsset{ + r.Assets = append(r.Assets, &base.ReleaseAsset{ ID: int64(asset.ID), Name: asset.Name, ContentType: &rel.Assets.Sources[k].Format, diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go index 5780bb63f2215..be2cf4f3deabc 100644 --- a/modules/migrations/restore.go +++ b/modules/migrations/restore.go @@ -7,7 +7,6 @@ package migrations import ( "context" "fmt" - "io" "io/ioutil" "os" "path/filepath" @@ -168,13 +167,6 @@ func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) { return releases, nil } -// GetAsset returns an asset -func (r *RepositoryRestorer) GetAsset(tagName string, relID, assetID int64) (io.ReadCloser, error) { - attachDir := filepath.Join(r.releaseDir(), "release_assets", tagName) - attachLocalPath := filepath.Join(attachDir, fmt.Sprintf("%d", assetID)) - return os.Open(attachLocalPath) -} - // GetLabels returns labels func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) { var labels = make([]*base.Label, 0, 10) From f91ba44fa8bcbb431e78ea0c9f2b078cda08e6b0 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 26 Dec 2020 20:46:18 +0800 Subject: [PATCH 42/47] Save relative path on yml file --- modules/migrations/dump.go | 83 ++++++++++------------------------- modules/migrations/restore.go | 8 ++++ 2 files changed, 30 insertions(+), 61 deletions(-) diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index 96a48ebac9a37..c63bcef981d76 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -47,8 +47,11 @@ type RepositoryDumper struct { } // NewRepositoryDumper creates an gitea Uploader -func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName string, opts base.MigrateOptions) *RepositoryDumper { +func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName string, opts base.MigrateOptions) (*RepositoryDumper, error) { baseDir = filepath.Join(baseDir, repoOwner, repoName) + if err := os.MkdirAll(baseDir, os.ModePerm); err != nil { + return nil, err + } return &RepositoryDumper{ ctx: ctx, opts: opts, @@ -58,7 +61,7 @@ func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName strin prHeadCache: make(map[string]struct{}), commentFiles: make(map[int64]*os.File), reviewFiles: make(map[int64]*os.File), - } + }, nil } // MaxBatchInsertSize returns the table's max batch insert size @@ -74,34 +77,10 @@ func (g *RepositoryDumper) wikiPath() string { return filepath.Join(g.baseDir, "wiki") } -func (g *RepositoryDumper) topicDir() string { - return filepath.Join(g.baseDir) -} - -func (g *RepositoryDumper) milestoneDir() string { - return filepath.Join(g.baseDir) -} - -func (g *RepositoryDumper) labelDir() string { - return filepath.Join(g.baseDir) -} - -func (g *RepositoryDumper) releaseDir() string { - return filepath.Join(g.baseDir) -} - -func (g *RepositoryDumper) issueDir() string { - return filepath.Join(g.baseDir) -} - func (g *RepositoryDumper) commentDir() string { return filepath.Join(g.baseDir, "comments") } -func (g *RepositoryDumper) pullrequestDir() string { - return filepath.Join(g.baseDir) -} - func (g *RepositoryDumper) reviewDir() string { return filepath.Join(g.baseDir, "reviews") } @@ -124,10 +103,6 @@ func (g *RepositoryDumper) setURLToken(remoteAddr string) (string, error) { // CreateRepo creates a repository func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error { - if err := os.MkdirAll(g.baseDir, os.ModePerm); err != nil { - return err - } - f, err := os.Create(filepath.Join(g.baseDir, "repo.yml")) if err != nil { return err @@ -236,10 +211,7 @@ func (g *RepositoryDumper) Close() { // CreateTopics creates topics func (g *RepositoryDumper) CreateTopics(topics ...string) error { - if err := os.MkdirAll(g.topicDir(), os.ModePerm); err != nil { - return err - } - f, err := os.Create(filepath.Join(g.topicDir(), "topic.yml")) + f, err := os.Create(filepath.Join(g.baseDir, "topic.yml")) if err != nil { return err } @@ -263,10 +235,7 @@ func (g *RepositoryDumper) CreateTopics(topics ...string) error { func (g *RepositoryDumper) CreateMilestones(milestones ...*base.Milestone) error { var err error if g.milestoneFile == nil { - if err := os.MkdirAll(g.milestoneDir(), os.ModePerm); err != nil { - return err - } - g.milestoneFile, err = os.Create(filepath.Join(g.milestoneDir(), "milestone.yml")) + g.milestoneFile, err = os.Create(filepath.Join(g.baseDir, "milestone.yml")) if err != nil { return err } @@ -288,10 +257,7 @@ func (g *RepositoryDumper) CreateMilestones(milestones ...*base.Milestone) error func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error { var err error if g.labelFile == nil { - if err := os.MkdirAll(g.labelDir(), os.ModePerm); err != nil { - return err - } - g.labelFile, err = os.Create(filepath.Join(g.labelDir(), "label.yml")) + g.labelFile, err = os.Create(filepath.Join(g.baseDir, "label.yml")) if err != nil { return err } @@ -313,15 +279,15 @@ func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error { func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { if g.opts.ReleaseAssets { for _, release := range releases { - attachDir := filepath.Join(g.releaseDir(), "release_assets", release.TagName) - if err := os.MkdirAll(attachDir, os.ModePerm); err != nil { + attachDir := filepath.Join("release_assets", release.TagName) + if err := os.MkdirAll(filepath.Join(g.baseDir, attachDir), os.ModePerm); err != nil { return err } for _, asset := range release.Assets { attachLocalPath := filepath.Join(attachDir, asset.Name) // download attachment - err := func(attachLocalPath string) error { + err := func(attachPath string) error { var rc io.ReadCloser var err error if asset.DownloadURL == nil { @@ -338,7 +304,7 @@ func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { } defer rc.Close() - fw, err := os.Create(attachLocalPath) + fw, err := os.Create(attachPath) if err != nil { return fmt.Errorf("Create: %v", err) } @@ -346,11 +312,10 @@ func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { _, err = io.Copy(fw, rc) return err - }(attachLocalPath) + }(filepath.Join(g.baseDir, attachLocalPath)) if err != nil { return err } - attachLocalPath = "file://" + attachLocalPath asset.DownloadURL = &attachLocalPath // to save the filepath on the yml file, change the source } } @@ -358,10 +323,7 @@ func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { var err error if g.releaseFile == nil { - if err := os.MkdirAll(g.releaseDir(), os.ModePerm); err != nil { - return err - } - g.releaseFile, err = os.Create(filepath.Join(g.releaseDir(), "release.yml")) + g.releaseFile, err = os.Create(filepath.Join(g.baseDir, "release.yml")) if err != nil { return err } @@ -388,10 +350,7 @@ func (g *RepositoryDumper) SyncTags() error { func (g *RepositoryDumper) CreateIssues(issues ...*base.Issue) error { var err error if g.issueFile == nil { - if err := os.MkdirAll(g.issueDir(), os.ModePerm); err != nil { - return err - } - g.issueFile, err = os.Create(filepath.Join(g.issueDir(), "issue.yml")) + g.issueFile, err = os.Create(filepath.Join(g.baseDir, "issue.yml")) if err != nil { return err } @@ -475,8 +434,7 @@ func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { if _, err = io.Copy(f, resp.Body); err != nil { return err } - fAbsPath, _ := filepath.Abs(fPath) - pr.PatchURL = "file://" + fAbsPath + pr.PatchURL = "./git/pulls/" + fmt.Sprintf("%d.patch", pr.Number) return nil }() @@ -541,10 +499,10 @@ func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { var err error if g.pullrequestFile == nil { - if err := os.MkdirAll(g.pullrequestDir(), os.ModePerm); err != nil { + if err := os.MkdirAll(g.baseDir, os.ModePerm); err != nil { return err } - g.pullrequestFile, err = os.Create(filepath.Join(g.pullrequestDir(), "pull_request.yml")) + g.pullrequestFile, err = os.Create(filepath.Join(g.baseDir, "pull_request.yml")) if err != nil { return err } @@ -589,7 +547,10 @@ func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.Mi if err != nil { return err } - var uploader = NewRepositoryDumper(ctx, baseDir, ownerName, opts.RepoName, opts) + uploader, err := NewRepositoryDumper(ctx, baseDir, ownerName, opts.RepoName, opts) + if err != nil { + return err + } if err := migrateRepository(downloader, uploader, opts); err != nil { if err1 := uploader.Rollback(); err1 != nil { diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go index be2cf4f3deabc..0b5c7d1a44510 100644 --- a/modules/migrations/restore.go +++ b/modules/migrations/restore.go @@ -164,6 +164,11 @@ func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) { if err != nil { return nil, err } + for _, rel := range releases { + for _, asset := range rel.Assets { + *asset.DownloadURL = "file://" + filepath.Join(r.baseDir, *asset.DownloadURL) + } + } return releases, nil } @@ -260,6 +265,9 @@ func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullReq if err != nil { return nil, false, err } + for _, pr := range pulls { + pr.PatchURL = filepath.Join(r.baseDir, pr.PatchURL) + } return pulls, true, nil } From eb5da17db9123e706bb8600da6e89cffc8b38360 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 26 Dec 2020 21:19:57 +0800 Subject: [PATCH 43/47] Fix bug --- modules/migrations/dump.go | 5 +++- modules/migrations/restore.go | 46 ++++++++++------------------------- 2 files changed, 17 insertions(+), 34 deletions(-) diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index c63bcef981d76..16659fcb08b6c 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -568,7 +568,10 @@ func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName return err } var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, repoName) - var downloader = NewRepositoryRestorer(ctx, baseDir, ownerName, repoName) + downloader, err := NewRepositoryRestorer(ctx, baseDir, ownerName, repoName) + if err != nil { + return err + } if err = migrateRepository(downloader, uploader, base.MigrateOptions{ Wiki: true, Issues: true, diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go index 0b5c7d1a44510..5550aaeb03a31 100644 --- a/modules/migrations/restore.go +++ b/modules/migrations/restore.go @@ -26,43 +26,23 @@ type RepositoryRestorer struct { } // NewRepositoryRestorer creates a repository restorer which could restore repository from a dumped folder -func NewRepositoryRestorer(ctx context.Context, baseDir string, owner, repoName string) *RepositoryRestorer { +func NewRepositoryRestorer(ctx context.Context, baseDir string, owner, repoName string) (*RepositoryRestorer, error) { + baseDir, err := filepath.Abs(baseDir) + if err != nil { + return nil, err + } return &RepositoryRestorer{ ctx: ctx, baseDir: baseDir, repoOwner: owner, repoName: repoName, - } -} - -func (r *RepositoryRestorer) topicDir() string { - return filepath.Join(r.baseDir) -} - -func (r *RepositoryRestorer) milestoneDir() string { - return filepath.Join(r.baseDir) -} - -func (r *RepositoryRestorer) labelDir() string { - return filepath.Join(r.baseDir) -} - -func (r *RepositoryRestorer) releaseDir() string { - return filepath.Join(r.baseDir) -} - -func (r *RepositoryRestorer) issueDir() string { - return filepath.Join(r.baseDir) + }, nil } func (r *RepositoryRestorer) commentDir() string { return filepath.Join(r.baseDir, "comments") } -func (r *RepositoryRestorer) pullrequestDir() string { - return filepath.Join(r.baseDir) -} - func (r *RepositoryRestorer) reviewDir() string { return filepath.Join(r.baseDir, "reviews") } @@ -101,7 +81,7 @@ func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) { // GetTopics return github topics func (r *RepositoryRestorer) GetTopics() ([]string, error) { - p := filepath.Join(r.topicDir(), "topic.yml") + p := filepath.Join(r.baseDir, "topic.yml") var topics = struct { Topics []string `yaml:"topics"` @@ -122,7 +102,7 @@ func (r *RepositoryRestorer) GetTopics() ([]string, error) { // GetMilestones returns milestones func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) { var milestones = make([]*base.Milestone, 0, 10) - p := filepath.Join(r.milestoneDir(), "milestone.yml") + p := filepath.Join(r.baseDir, "milestone.yml") _, err := os.Stat(p) if err != nil { if os.IsNotExist(err) { @@ -146,7 +126,7 @@ func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) { // GetReleases returns releases func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) { var releases = make([]*base.Release, 0, 10) - p := filepath.Join(r.releaseDir(), "release.yml") + p := filepath.Join(r.baseDir, "release.yml") _, err := os.Stat(p) if err != nil { if os.IsNotExist(err) { @@ -175,7 +155,7 @@ func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) { // GetLabels returns labels func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) { var labels = make([]*base.Label, 0, 10) - p := filepath.Join(r.labelDir(), "label.yml") + p := filepath.Join(r.baseDir, "label.yml") _, err := os.Stat(p) if err != nil { if os.IsNotExist(err) { @@ -199,7 +179,7 @@ func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) { // GetIssues returns issues according start and limit func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { var issues = make([]*base.Issue, 0, 10) - p := filepath.Join(r.issueDir(), "issue.yml") + p := filepath.Join(r.baseDir, "issue.yml") _, err := os.Stat(p) if err != nil { if os.IsNotExist(err) { @@ -247,7 +227,7 @@ func (r *RepositoryRestorer) GetComments(issueNumber int64) ([]*base.Comment, er // GetPullRequests returns pull requests according page and perPage func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { var pulls = make([]*base.PullRequest, 0, 10) - p := filepath.Join(r.pullrequestDir(), "pull_request.yml") + p := filepath.Join(r.baseDir, "pull_request.yml") _, err := os.Stat(p) if err != nil { if os.IsNotExist(err) { @@ -266,7 +246,7 @@ func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullReq return nil, false, err } for _, pr := range pulls { - pr.PatchURL = filepath.Join(r.baseDir, pr.PatchURL) + pr.PatchURL = "file://" + filepath.Join(r.baseDir, pr.PatchURL) } return pulls, true, nil } From 3c5b2e11611e5efcf9c66b800519d7a129f710fc Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 26 Dec 2020 21:29:16 +0800 Subject: [PATCH 44/47] Use relative path --- modules/migrations/dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index 16659fcb08b6c..3c3b9a1753903 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -434,7 +434,7 @@ func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { if _, err = io.Copy(f, resp.Body); err != nil { return err } - pr.PatchURL = "./git/pulls/" + fmt.Sprintf("%d.patch", pr.Number) + pr.PatchURL = "git/pulls/" + fmt.Sprintf("%d.patch", pr.Number) return nil }() From 80b5c9a1a268c735321b49cc472c8f5dc669d313 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 26 Dec 2020 23:34:04 +0800 Subject: [PATCH 45/47] Update docs --- cmd/dump_repo.go | 6 ++--- cmd/restore_repo.go | 8 +++---- docs/content/doc/usage/command-line.en-us.md | 25 ++++++++++++++++++++ 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/cmd/dump_repo.go b/cmd/dump_repo.go index 85b628993f86e..aa3bfd84ac4b0 100644 --- a/cmd/dump_repo.go +++ b/cmd/dump_repo.go @@ -20,7 +20,7 @@ import ( // CmdDumpRepository represents the available dump repository sub-command. var CmdDumpRepository = cli.Command{ Name: "dump-repo", - Usage: "Dump the repository from github/gitlab", + Usage: "Dump the repository from git/github/gitea/gitlab", Description: "This is a command for dumping the repository data.", Action: runDumpRepository, Flags: []cli.Flag{ @@ -32,12 +32,12 @@ var CmdDumpRepository = cli.Command{ cli.StringFlag{ Name: "repo_dir, r", Value: "./data", - Usage: "Repository dir path", + Usage: "Repository dir path to store the data", }, cli.StringFlag{ Name: "clone_addr", Value: "", - Usage: "The URL will be clone, currently could be a github or gitlab http/https URL", + Usage: "The URL will be clone, currently could be a git/github/gitea/gitlab http/https URL", }, cli.StringFlag{ Name: "auth_username", diff --git a/cmd/restore_repo.go b/cmd/restore_repo.go index 8569c43365c49..541995879bf4c 100644 --- a/cmd/restore_repo.go +++ b/cmd/restore_repo.go @@ -28,22 +28,22 @@ var CmdRestoreRepository = cli.Command{ cli.StringFlag{ Name: "repo_dir, r", Value: "./data", - Usage: "Repository dir path", + Usage: "Repository dir path to restore from", }, cli.StringFlag{ Name: "owner_name", Value: "", - Usage: "The data will be stored on a directory with owner name if not empty", + Usage: "Restore destination owner name", }, cli.StringFlag{ Name: "repo_name", Value: "", - Usage: "The data will be stored on a directory with repository name if not empty", + Usage: "Restore destination repository name", }, cli.StringFlag{ Name: "units", Value: "", - Usage: `Which items will be migrated, one or more units should be separated as comma. + Usage: `Which items will be restored, one or more units should be separated as comma. wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.`, }, }, diff --git a/docs/content/doc/usage/command-line.en-us.md b/docs/content/doc/usage/command-line.en-us.md index a09d5dde7377f..b26f695e6be78 100644 --- a/docs/content/doc/usage/command-line.en-us.md +++ b/docs/content/doc/usage/command-line.en-us.md @@ -441,3 +441,28 @@ Manage running server operations: - `--host value`, `-H value`: Mail server host (defaults to: 127.0.0.1:25) - `--send-to value`, `-s value`: Email address(es) to send to - `--subject value`, `-S value`: Subject header of sent emails + +### dump-repo + +Dump-repo dumps repository data from git/github/gitea/gitlab: + +- Options: + - `--git_service service` : Git service, 1 plain git, 2 github, 3 gitea, 4 gitlab + - `--repo_dir dir`, `-r dir`: Repository dir path to store the data + - `--clone_addr addr`: The URL will be clone, currently could be a git/github/gitea/gitlab http/https URL. i.e. https://github.com/lunny/tango.git + - `--auth_username lunny`: The username to visit the clone_addr + - `--auth_password `: The password to visit the clone_addr + - `--auth_token `: The personal token to visit the clone_addr + - `--owner_name lunny`: The data will be stored on a directory with owner name if not empty + - `--repo_name tango`: The data will be stored on a directory with repository name if not empty + - `--units `: Which items will be migrated, one or more units should be separated as comma. wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units. + +### restore-repo + +Restore-repo restore repository data from disk dir: + +- Options: + - `--repo_dir dir`, `-r dir`: Repository dir path to restore from + - `--owner_name lunny`: Restore destination owner name + - `--repo_name tango`: Restore destination repository name + - `--units `: Which items will be restored, one or more units should be separated as comma. wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units. \ No newline at end of file From ceefd988b109161d045c08588470d6850ede101b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 27 Dec 2020 09:40:01 +0800 Subject: [PATCH 46/47] Use git service string but not int --- cmd/dump_repo.go | 10 +++++----- docs/content/doc/usage/command-line.en-us.md | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/dump_repo.go b/cmd/dump_repo.go index aa3bfd84ac4b0..ee038b6392ee8 100644 --- a/cmd/dump_repo.go +++ b/cmd/dump_repo.go @@ -8,11 +8,11 @@ import ( "context" "strings" + "code.gitea.io/gitea/modules/convert" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/structs" "github.com/urfave/cli" ) @@ -24,10 +24,10 @@ var CmdDumpRepository = cli.Command{ Description: "This is a command for dumping the repository data.", Action: runDumpRepository, Flags: []cli.Flag{ - cli.IntFlag{ + cli.StringFlag{ Name: "git_service", - Value: 1, - Usage: "Git service, 1 plain git, 2 github, 3 gitea, 4 gitlab", + Value: "", + Usage: "Git service, git, github, gitea, gitlab", }, cli.StringFlag{ Name: "repo_dir, r", @@ -85,7 +85,7 @@ func runDumpRepository(ctx *cli.Context) error { setting.InitDBConfig() var opts = base.MigrateOptions{ - GitServiceType: structs.GitServiceType(ctx.Int("git_service")), + GitServiceType: convert.ToGitServiceType(ctx.String("git_service")), CloneAddr: ctx.String("clone_addr"), AuthUsername: ctx.String("auth_username"), AuthPassword: ctx.String("auth_password"), diff --git a/docs/content/doc/usage/command-line.en-us.md b/docs/content/doc/usage/command-line.en-us.md index b26f695e6be78..05b8dee6e5e4c 100644 --- a/docs/content/doc/usage/command-line.en-us.md +++ b/docs/content/doc/usage/command-line.en-us.md @@ -447,7 +447,7 @@ Manage running server operations: Dump-repo dumps repository data from git/github/gitea/gitlab: - Options: - - `--git_service service` : Git service, 1 plain git, 2 github, 3 gitea, 4 gitlab + - `--git_service service` : Git service, it could be `git`, `github`, `gitea`, `gitlab` - `--repo_dir dir`, `-r dir`: Repository dir path to store the data - `--clone_addr addr`: The URL will be clone, currently could be a git/github/gitea/gitlab http/https URL. i.e. https://github.com/lunny/tango.git - `--auth_username lunny`: The username to visit the clone_addr From 695d23e56a9626adccf3f0a369dc5472757c49c7 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 27 Dec 2020 10:15:12 +0800 Subject: [PATCH 47/47] Recognize clone addr to service type --- cmd/dump_repo.go | 26 +++++++++++++++++--- docs/content/doc/usage/command-line.en-us.md | 2 +- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/cmd/dump_repo.go b/cmd/dump_repo.go index ee038b6392ee8..cea640b53438d 100644 --- a/cmd/dump_repo.go +++ b/cmd/dump_repo.go @@ -6,6 +6,7 @@ package cmd import ( "context" + "errors" "strings" "code.gitea.io/gitea/modules/convert" @@ -13,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" "github.com/urfave/cli" ) @@ -27,7 +29,7 @@ var CmdDumpRepository = cli.Command{ cli.StringFlag{ Name: "git_service", Value: "", - Usage: "Git service, git, github, gitea, gitlab", + Usage: "Git service, git, github, gitea, gitlab. If clone_addr could be recognized, this could be ignored.", }, cli.StringFlag{ Name: "repo_dir, r", @@ -84,9 +86,27 @@ func runDumpRepository(ctx *cli.Context) error { log.Trace("Log path: %s", setting.LogRootPath) setting.InitDBConfig() + var ( + serviceType structs.GitServiceType + cloneAddr = ctx.String("clone_addr") + serviceStr = ctx.String("git_service") + ) + + if strings.HasPrefix(strings.ToLower(cloneAddr), "https://github.com/") { + serviceStr = "github" + } else if strings.HasPrefix(strings.ToLower(cloneAddr), "https://gitlab.com/") { + serviceStr = "gitlab" + } else if strings.HasPrefix(strings.ToLower(cloneAddr), "https://gitea.com/") { + serviceStr = "gitea" + } + if serviceStr == "" { + return errors.New("git_service missed or clone_addr cannot be recognized") + } + serviceType = convert.ToGitServiceType(serviceStr) + var opts = base.MigrateOptions{ - GitServiceType: convert.ToGitServiceType(ctx.String("git_service")), - CloneAddr: ctx.String("clone_addr"), + GitServiceType: serviceType, + CloneAddr: cloneAddr, AuthUsername: ctx.String("auth_username"), AuthPassword: ctx.String("auth_password"), AuthToken: ctx.String("auth_token"), diff --git a/docs/content/doc/usage/command-line.en-us.md b/docs/content/doc/usage/command-line.en-us.md index 05b8dee6e5e4c..98d047fb489e6 100644 --- a/docs/content/doc/usage/command-line.en-us.md +++ b/docs/content/doc/usage/command-line.en-us.md @@ -447,7 +447,7 @@ Manage running server operations: Dump-repo dumps repository data from git/github/gitea/gitlab: - Options: - - `--git_service service` : Git service, it could be `git`, `github`, `gitea`, `gitlab` + - `--git_service service` : Git service, it could be `git`, `github`, `gitea`, `gitlab`, If clone_addr could be recognized, this could be ignored. - `--repo_dir dir`, `-r dir`: Repository dir path to store the data - `--clone_addr addr`: The URL will be clone, currently could be a git/github/gitea/gitlab http/https URL. i.e. https://github.com/lunny/tango.git - `--auth_username lunny`: The username to visit the clone_addr