Skip to content

Commit c03e488

Browse files
KN4CK3Rzeripath
andauthored
Add LFS Migration and Mirror (#14726)
* Implemented LFS client. * Implemented scanning for pointer files. * Implemented downloading of lfs files. * Moved model-dependent code into services. * Removed models dependency. Added TryReadPointerFromBuffer. * Migrated code from service to module. * Centralised storage creation. * Removed dependency from models. * Moved ContentStore into modules. * Share structs between server and client. * Moved method to services. * Implemented lfs download on clone. * Implemented LFS sync on clone and mirror update. * Added form fields. * Updated templates. * Fixed condition. * Use alternate endpoint. * Added missing methods. * Fixed typo and make linter happy. * Detached pointer parser from gogit dependency. * Fixed TestGetLFSRange test. * Added context to support cancellation. * Use ReadFull to probably read more data. * Removed duplicated code from models. * Moved scan implementation into pointer_scanner_nogogit. * Changed method name. * Added comments. * Added more/specific log/error messages. * Embedded lfs.Pointer into models.LFSMetaObject. * Moved code from models to module. * Moved code from models to module. * Moved code from models to module. * Reduced pointer usage. * Embedded type. * Use promoted fields. * Fixed unexpected eof. * Added unit tests. * Implemented migration of local file paths. * Show an error on invalid LFS endpoints. * Hide settings if not used. * Added LFS info to mirror struct. * Fixed comment. * Check LFS endpoint. * Manage LFS settings from mirror page. * Fixed selector. * Adjusted selector. * Added more tests. * Added local filesystem migration test. * Fixed typo. * Reset settings. * Added special windows path handling. * Added unit test for HTTPClient. * Added unit test for BasicTransferAdapter. * Moved into util package. * Test if LFS endpoint is allowed. * Added support for git:// * Just use a static placeholder as the displayed url may be invalid. * Reverted to original code. * Added "Advanced Settings". * Updated wording. * Added discovery info link. * Implemented suggestion. * Fixed missing format parameter. * Added Pointer.IsValid(). * Always remove model on error. * Added suggestions. * Use channel instead of array. * Update routers/repo/migrate.go * fmt Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: zeripath <art27@cantab.net>
1 parent f544414 commit c03e488

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

75 files changed

+2158
-710
lines changed

cmd/serv.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,11 @@ import (
1717
"time"
1818

1919
"code.gitea.io/gitea/models"
20-
"code.gitea.io/gitea/modules/lfs"
2120
"code.gitea.io/gitea/modules/log"
2221
"code.gitea.io/gitea/modules/pprof"
2322
"code.gitea.io/gitea/modules/private"
2423
"code.gitea.io/gitea/modules/setting"
24+
"code.gitea.io/gitea/services/lfs"
2525

2626
"github.com/dgrijalva/jwt-go"
2727
jsoniter "github.com/json-iterator/go"
+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
// Copyright 2021 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package integrations
6+
7+
import (
8+
"net/http"
9+
"path"
10+
"testing"
11+
12+
"code.gitea.io/gitea/models"
13+
"code.gitea.io/gitea/modules/lfs"
14+
"code.gitea.io/gitea/modules/setting"
15+
api "code.gitea.io/gitea/modules/structs"
16+
17+
"github.com/stretchr/testify/assert"
18+
)
19+
20+
func TestAPIRepoLFSMigrateLocal(t *testing.T) {
21+
defer prepareTestEnv(t)()
22+
23+
oldImportLocalPaths := setting.ImportLocalPaths
24+
oldAllowLocalNetworks := setting.Migrations.AllowLocalNetworks
25+
setting.ImportLocalPaths = true
26+
setting.Migrations.AllowLocalNetworks = true
27+
28+
user := models.AssertExistsAndLoadBean(t, &models.User{ID: 1}).(*models.User)
29+
session := loginUser(t, user.Name)
30+
token := getTokenForLoggedInUser(t, session)
31+
32+
req := NewRequestWithJSON(t, "POST", "/api/v1/repos/migrate?token="+token, &api.MigrateRepoOptions{
33+
CloneAddr: path.Join(setting.RepoRootPath, "migration/lfs-test.git"),
34+
RepoOwnerID: user.ID,
35+
RepoName: "lfs-test-local",
36+
LFS: true,
37+
})
38+
resp := MakeRequest(t, req, NoExpectedStatus)
39+
assert.EqualValues(t, http.StatusCreated, resp.Code)
40+
41+
store := lfs.NewContentStore()
42+
ok, _ := store.Verify(lfs.Pointer{Oid: "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041", Size: 6})
43+
assert.True(t, ok)
44+
ok, _ = store.Verify(lfs.Pointer{Oid: "d6f175817f886ec6fbbc1515326465fa96c3bfd54a4ea06cfd6dbbd8340e0152", Size: 6})
45+
assert.True(t, ok)
46+
47+
setting.ImportLocalPaths = oldImportLocalPaths
48+
setting.Migrations.AllowLocalNetworks = oldAllowLocalNetworks
49+
}

integrations/git_test.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818

1919
"code.gitea.io/gitea/models"
2020
"code.gitea.io/gitea/modules/git"
21+
"code.gitea.io/gitea/modules/lfs"
2122
"code.gitea.io/gitea/modules/setting"
2223
api "code.gitea.io/gitea/modules/structs"
2324
"code.gitea.io/gitea/modules/util"
@@ -218,7 +219,7 @@ func rawTest(t *testing.T, ctx *APITestContext, little, big, littleLFS, bigLFS s
218219
assert.NotEqual(t, littleSize, resp.Body.Len())
219220
assert.LessOrEqual(t, resp.Body.Len(), 1024)
220221
if resp.Body.Len() != littleSize && resp.Body.Len() <= 1024 {
221-
assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier)
222+
assert.Contains(t, resp.Body.String(), lfs.MetaFileIdentifier)
222223
}
223224
}
224225

@@ -232,7 +233,7 @@ func rawTest(t *testing.T, ctx *APITestContext, little, big, littleLFS, bigLFS s
232233
resp := session.MakeRequest(t, req, http.StatusOK)
233234
assert.NotEqual(t, bigSize, resp.Body.Len())
234235
if resp.Body.Len() != bigSize && resp.Body.Len() <= 1024 {
235-
assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier)
236+
assert.Contains(t, resp.Body.String(), lfs.MetaFileIdentifier)
236237
}
237238
}
238239
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
ref: refs/heads/master
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
[core]
2+
bare = false
3+
repositoryformatversion = 0
4+
filemode = false
5+
symlinks = false
6+
ignorecase = true
7+
logallrefupdates = true
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Unnamed repository; edit this file 'description' to name the repository.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/sh
2+
command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-checkout.\n"; exit 2; }
3+
git lfs post-checkout "$@"
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/sh
2+
command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-commit.\n"; exit 2; }
3+
git lfs post-commit "$@"
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/sh
2+
command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-merge.\n"; exit 2; }
3+
git lfs post-merge "$@"
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/sh
2+
command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/pre-push.\n"; exit 2; }
3+
git lfs pre-push "$@"
Binary file not shown.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
dummy2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
dummy1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
546244003622c64b2fc3c2cd544d7a29882c8383

integrations/lfs_getobject_test.go

+7-20
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,6 @@ package integrations
77
import (
88
"archive/zip"
99
"bytes"
10-
"crypto/sha256"
11-
"encoding/hex"
12-
"io"
1310
"io/ioutil"
1411
"net/http"
1512
"net/http/httptest"
@@ -18,46 +15,36 @@ import (
1815
"code.gitea.io/gitea/models"
1916
"code.gitea.io/gitea/modules/lfs"
2017
"code.gitea.io/gitea/modules/setting"
21-
"code.gitea.io/gitea/modules/storage"
2218
"code.gitea.io/gitea/routers/routes"
2319

2420
gzipp "github.com/klauspost/compress/gzip"
2521
"github.com/stretchr/testify/assert"
2622
)
2723

28-
func GenerateLFSOid(content io.Reader) (string, error) {
29-
h := sha256.New()
30-
if _, err := io.Copy(h, content); err != nil {
31-
return "", err
32-
}
33-
sum := h.Sum(nil)
34-
return hex.EncodeToString(sum), nil
35-
}
36-
3724
var lfsID = int64(20000)
3825

3926
func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string {
40-
oid, err := GenerateLFSOid(bytes.NewReader(*content))
27+
pointer, err := lfs.GeneratePointer(bytes.NewReader(*content))
4128
assert.NoError(t, err)
4229
var lfsMetaObject *models.LFSMetaObject
4330

4431
if setting.Database.UsePostgreSQL {
45-
lfsMetaObject = &models.LFSMetaObject{ID: lfsID, Oid: oid, Size: int64(len(*content)), RepositoryID: repositoryID}
32+
lfsMetaObject = &models.LFSMetaObject{ID: lfsID, Pointer: pointer, RepositoryID: repositoryID}
4633
} else {
47-
lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: int64(len(*content)), RepositoryID: repositoryID}
34+
lfsMetaObject = &models.LFSMetaObject{Pointer: pointer, RepositoryID: repositoryID}
4835
}
4936

5037
lfsID++
5138
lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject)
5239
assert.NoError(t, err)
53-
contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS}
54-
exist, err := contentStore.Exists(lfsMetaObject)
40+
contentStore := lfs.NewContentStore()
41+
exist, err := contentStore.Exists(pointer)
5542
assert.NoError(t, err)
5643
if !exist {
57-
err := contentStore.Put(lfsMetaObject, bytes.NewReader(*content))
44+
err := contentStore.Put(pointer, bytes.NewReader(*content))
5845
assert.NoError(t, err)
5946
}
60-
return oid
47+
return pointer.Oid
6148
}
6249

6350
func storeAndGetLfs(t *testing.T, content *[]byte, extraHeader *http.Header, expectedStatus int) *httptest.ResponseRecorder {
+117
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
// Copyright 2021 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package integrations
6+
7+
import (
8+
"fmt"
9+
"io/ioutil"
10+
"net/url"
11+
"os"
12+
"path/filepath"
13+
"testing"
14+
15+
"code.gitea.io/gitea/modules/lfs"
16+
17+
"github.com/stretchr/testify/assert"
18+
)
19+
20+
func str2url(raw string) *url.URL {
21+
u, _ := url.Parse(raw)
22+
return u
23+
}
24+
25+
func TestDetermineLocalEndpoint(t *testing.T) {
26+
defer prepareTestEnv(t)()
27+
28+
root, _ := ioutil.TempDir("", "lfs_test")
29+
defer os.RemoveAll(root)
30+
31+
rootdotgit, _ := ioutil.TempDir("", "lfs_test")
32+
defer os.RemoveAll(rootdotgit)
33+
os.Mkdir(filepath.Join(rootdotgit, ".git"), 0700)
34+
35+
lfsroot, _ := ioutil.TempDir("", "lfs_test")
36+
defer os.RemoveAll(lfsroot)
37+
38+
// Test cases
39+
var cases = []struct {
40+
cloneurl string
41+
lfsurl string
42+
expected *url.URL
43+
}{
44+
// case 0
45+
{
46+
cloneurl: root,
47+
lfsurl: "",
48+
expected: str2url(fmt.Sprintf("file://%s", root)),
49+
},
50+
// case 1
51+
{
52+
cloneurl: root,
53+
lfsurl: lfsroot,
54+
expected: str2url(fmt.Sprintf("file://%s", lfsroot)),
55+
},
56+
// case 2
57+
{
58+
cloneurl: "https://git.com/repo.git",
59+
lfsurl: lfsroot,
60+
expected: str2url(fmt.Sprintf("file://%s", lfsroot)),
61+
},
62+
// case 3
63+
{
64+
cloneurl: rootdotgit,
65+
lfsurl: "",
66+
expected: str2url(fmt.Sprintf("file://%s", filepath.Join(rootdotgit, ".git"))),
67+
},
68+
// case 4
69+
{
70+
cloneurl: "",
71+
lfsurl: rootdotgit,
72+
expected: str2url(fmt.Sprintf("file://%s", filepath.Join(rootdotgit, ".git"))),
73+
},
74+
// case 5
75+
{
76+
cloneurl: rootdotgit,
77+
lfsurl: rootdotgit,
78+
expected: str2url(fmt.Sprintf("file://%s", filepath.Join(rootdotgit, ".git"))),
79+
},
80+
// case 6
81+
{
82+
cloneurl: fmt.Sprintf("file://%s", root),
83+
lfsurl: "",
84+
expected: str2url(fmt.Sprintf("file://%s", root)),
85+
},
86+
// case 7
87+
{
88+
cloneurl: fmt.Sprintf("file://%s", root),
89+
lfsurl: fmt.Sprintf("file://%s", lfsroot),
90+
expected: str2url(fmt.Sprintf("file://%s", lfsroot)),
91+
},
92+
// case 8
93+
{
94+
cloneurl: root,
95+
lfsurl: fmt.Sprintf("file://%s", lfsroot),
96+
expected: str2url(fmt.Sprintf("file://%s", lfsroot)),
97+
},
98+
// case 9
99+
{
100+
cloneurl: "",
101+
lfsurl: "/does/not/exist",
102+
expected: nil,
103+
},
104+
// case 10
105+
{
106+
cloneurl: "",
107+
lfsurl: "file:///does/not/exist",
108+
expected: str2url("file:///does/not/exist"),
109+
},
110+
}
111+
112+
for n, c := range cases {
113+
ep := lfs.DetermineEndpoint(c.cloneurl, c.lfsurl)
114+
115+
assert.Equal(t, c.expected, ep, "case %d: error should match", n)
116+
}
117+
}

0 commit comments

Comments
 (0)