Skip to content

Commit 9e5d0a0

Browse files
lunnylunny
authored
Global code search support (#3664)
* add global code search on explore * fix bug when no anyone public repos * change the icon * fix typo and add UnitTypeCode check for login non-admin user * fix ui description when no match
1 parent 4163cdf commit 9e5d0a0

File tree

10 files changed

+238
-9
lines changed

10 files changed

+238
-9
lines changed

‎models/repo.go

+6
Original file line numberDiff line numberDiff line change
@@ -1945,6 +1945,12 @@ func GetRepositoryByID(id int64) (*Repository, error) {
19451945
return getRepositoryByID(x, id)
19461946
}
19471947

1948+
// GetRepositoriesMapByIDs returns the repositories by given id slice.
1949+
func GetRepositoriesMapByIDs(ids []int64) (map[int64]*Repository, error) {
1950+
var repos = make(map[int64]*Repository, len(ids))
1951+
return repos, x.In("id", ids).Find(&repos)
1952+
}
1953+
19481954
// GetUserRepositories returns a list of repositories of given user.
19491955
func GetUserRepositories(userID int64, private bool, page, pageSize int, orderBy string) ([]*Repository, error) {
19501956
if len(orderBy) == 0 {

‎models/repo_list.go

+25
Original file line numberDiff line numberDiff line change
@@ -249,3 +249,28 @@ func SearchRepositoryByName(opts *SearchRepoOptions) (RepositoryList, int64, err
249249

250250
return repos, count, nil
251251
}
252+
253+
// FindUserAccessibleRepoIDs find all accessible repositories' ID by user's id
254+
func FindUserAccessibleRepoIDs(userID int64) ([]int64, error) {
255+
var accessCond builder.Cond = builder.Eq{"is_private": false}
256+
257+
if userID > 0 {
258+
accessCond = accessCond.Or(
259+
builder.Eq{"owner_id": userID},
260+
builder.And(
261+
builder.Expr("id IN (SELECT repo_id FROM `access` WHERE access.user_id = ?)", userID),
262+
builder.Neq{"owner_id": userID},
263+
),
264+
)
265+
}
266+
267+
repoIDs := make([]int64, 0, 10)
268+
if err := x.
269+
Table("repository").
270+
Cols("id").
271+
Where(accessCond).
272+
Find(&repoIDs); err != nil {
273+
return nil, fmt.Errorf("FindUserAccesibleRepoIDs: %v", err)
274+
}
275+
return repoIDs, nil
276+
}

‎modules/indexer/repo.go

+21-6
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"github.com/blevesearch/bleve/analysis/token/lowercase"
1717
"github.com/blevesearch/bleve/analysis/token/unique"
1818
"github.com/blevesearch/bleve/analysis/tokenizer/unicode"
19+
"github.com/blevesearch/bleve/search/query"
1920
"github.com/ethantkoenig/rupture"
2021
)
2122

@@ -158,6 +159,7 @@ func DeleteRepoFromIndexer(repoID int64) error {
158159

159160
// RepoSearchResult result of performing a search in a repo
160161
type RepoSearchResult struct {
162+
RepoID int64
161163
StartIndex int
162164
EndIndex int
163165
Filename string
@@ -166,17 +168,29 @@ type RepoSearchResult struct {
166168

167169
// SearchRepoByKeyword searches for files in the specified repo.
168170
// Returns the matching file-paths
169-
func SearchRepoByKeyword(repoID int64, keyword string, page, pageSize int) (int64, []*RepoSearchResult, error) {
171+
func SearchRepoByKeyword(repoIDs []int64, keyword string, page, pageSize int) (int64, []*RepoSearchResult, error) {
170172
phraseQuery := bleve.NewMatchPhraseQuery(keyword)
171173
phraseQuery.FieldVal = "Content"
172174
phraseQuery.Analyzer = repoIndexerAnalyzer
173-
indexerQuery := bleve.NewConjunctionQuery(
174-
numericEqualityQuery(repoID, "RepoID"),
175-
phraseQuery,
176-
)
175+
176+
var indexerQuery query.Query
177+
if len(repoIDs) > 0 {
178+
var repoQueries = make([]query.Query, 0, len(repoIDs))
179+
for _, repoID := range repoIDs {
180+
repoQueries = append(repoQueries, numericEqualityQuery(repoID, "RepoID"))
181+
}
182+
183+
indexerQuery = bleve.NewConjunctionQuery(
184+
bleve.NewDisjunctionQuery(repoQueries...),
185+
phraseQuery,
186+
)
187+
} else {
188+
indexerQuery = phraseQuery
189+
}
190+
177191
from := (page - 1) * pageSize
178192
searchRequest := bleve.NewSearchRequestOptions(indexerQuery, pageSize, from, false)
179-
searchRequest.Fields = []string{"Content"}
193+
searchRequest.Fields = []string{"Content", "RepoID"}
180194
searchRequest.IncludeLocations = true
181195

182196
result, err := repoIndexer.Search(searchRequest)
@@ -199,6 +213,7 @@ func SearchRepoByKeyword(repoID int64, keyword string, page, pageSize int) (int6
199213
}
200214
}
201215
searchResults[i] = &RepoSearchResult{
216+
RepoID: int64(hit.Fields["RepoID"].(float64)),
202217
StartIndex: startIndex,
203218
EndIndex: endIndex,
204219
Filename: filenameOfIndexerID(hit.ID),

‎modules/search/search.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717

1818
// Result a search result to display
1919
type Result struct {
20+
RepoID int64
2021
Filename string
2122
HighlightClass string
2223
LineNumbers []int
@@ -98,6 +99,7 @@ func searchResult(result *indexer.RepoSearchResult, startIndex, endIndex int) (*
9899
index += len(line)
99100
}
100101
return &Result{
102+
RepoID: result.RepoID,
101103
Filename: result.Filename,
102104
HighlightClass: highlight.FileNameToHighlightClass(result.Filename),
103105
LineNumbers: lineNumbers,
@@ -106,12 +108,12 @@ func searchResult(result *indexer.RepoSearchResult, startIndex, endIndex int) (*
106108
}
107109

108110
// PerformSearch perform a search on a repository
109-
func PerformSearch(repoID int64, keyword string, page, pageSize int) (int, []*Result, error) {
111+
func PerformSearch(repoIDs []int64, keyword string, page, pageSize int) (int, []*Result, error) {
110112
if len(keyword) == 0 {
111113
return 0, nil, nil
112114
}
113115

114-
total, results, err := indexer.SearchRepoByKeyword(repoID, keyword, page, pageSize)
116+
total, results, err := indexer.SearchRepoByKeyword(repoIDs, keyword, page, pageSize)
115117
if err != nil {
116118
return 0, nil, err
117119
}

‎options/locale/locale_en-US.ini

+3
Original file line numberDiff line numberDiff line change
@@ -169,9 +169,12 @@ repos = Repositories
169169
users = Users
170170
organizations = Organizations
171171
search = Search
172+
code = Code
172173
repo_no_results = No matching repositories have been found.
173174
user_no_results = No matching users have been found.
174175
org_no_results = No matching organizations have been found.
176+
code_no_results = No matching codes have been found.
177+
code_search_results = Search results for "%s"
175178

176179
[auth]
177180
create_new_account = Create Account

‎routers/home.go

+116
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"code.gitea.io/gitea/models"
1212
"code.gitea.io/gitea/modules/base"
1313
"code.gitea.io/gitea/modules/context"
14+
"code.gitea.io/gitea/modules/search"
1415
"code.gitea.io/gitea/modules/setting"
1516
"code.gitea.io/gitea/modules/util"
1617
"code.gitea.io/gitea/routers/user"
@@ -27,6 +28,8 @@ const (
2728
tplExploreUsers base.TplName = "explore/users"
2829
// tplExploreOrganizations explore organizations page template
2930
tplExploreOrganizations base.TplName = "explore/organizations"
31+
// tplExploreCode explore code page template
32+
tplExploreCode base.TplName = "explore/code"
3033
)
3134

3235
// Home render home page
@@ -49,6 +52,7 @@ func Home(ctx *context.Context) {
4952
}
5053

5154
ctx.Data["PageIsHome"] = true
55+
ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
5256
ctx.HTML(200, tplHome)
5357
}
5458

@@ -124,6 +128,7 @@ func RenderRepoSearch(ctx *context.Context, opts *RepoSearchOptions) {
124128
ctx.Data["Total"] = count
125129
ctx.Data["Page"] = paginater.New(int(count), opts.PageSize, page, 5)
126130
ctx.Data["Repos"] = repos
131+
ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
127132

128133
ctx.HTML(200, opts.TplName)
129134
}
@@ -133,6 +138,7 @@ func ExploreRepos(ctx *context.Context) {
133138
ctx.Data["Title"] = ctx.Tr("explore")
134139
ctx.Data["PageIsExplore"] = true
135140
ctx.Data["PageIsExploreRepositories"] = true
141+
ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
136142

137143
var ownerID int64
138144
if ctx.User != nil && !ctx.User.IsAdmin {
@@ -194,6 +200,7 @@ func RenderUserSearch(ctx *context.Context, opts *models.SearchUserOptions, tplN
194200
ctx.Data["Page"] = paginater.New(int(count), opts.PageSize, opts.Page, 5)
195201
ctx.Data["Users"] = users
196202
ctx.Data["ShowUserEmail"] = setting.UI.ShowUserEmail
203+
ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
197204

198205
ctx.HTML(200, tplName)
199206
}
@@ -203,6 +210,7 @@ func ExploreUsers(ctx *context.Context) {
203210
ctx.Data["Title"] = ctx.Tr("explore")
204211
ctx.Data["PageIsExplore"] = true
205212
ctx.Data["PageIsExploreUsers"] = true
213+
ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
206214

207215
RenderUserSearch(ctx, &models.SearchUserOptions{
208216
Type: models.UserTypeIndividual,
@@ -216,13 +224,121 @@ func ExploreOrganizations(ctx *context.Context) {
216224
ctx.Data["Title"] = ctx.Tr("explore")
217225
ctx.Data["PageIsExplore"] = true
218226
ctx.Data["PageIsExploreOrganizations"] = true
227+
ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
219228

220229
RenderUserSearch(ctx, &models.SearchUserOptions{
221230
Type: models.UserTypeOrganization,
222231
PageSize: setting.UI.ExplorePagingNum,
223232
}, tplExploreOrganizations)
224233
}
225234

235+
// ExploreCode render explore code page
236+
func ExploreCode(ctx *context.Context) {
237+
if !setting.Indexer.RepoIndexerEnabled {
238+
ctx.Redirect("/explore", 302)
239+
return
240+
}
241+
242+
ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
243+
ctx.Data["Title"] = ctx.Tr("explore")
244+
ctx.Data["PageIsExplore"] = true
245+
ctx.Data["PageIsExploreCode"] = true
246+
247+
keyword := strings.TrimSpace(ctx.Query("q"))
248+
page := ctx.QueryInt("page")
249+
if page <= 0 {
250+
page = 1
251+
}
252+
253+
var (
254+
repoIDs []int64
255+
err error
256+
isAdmin bool
257+
userID int64
258+
)
259+
if ctx.User != nil {
260+
userID = ctx.User.ID
261+
isAdmin = ctx.User.IsAdmin
262+
}
263+
264+
// guest user or non-admin user
265+
if ctx.User == nil || !isAdmin {
266+
repoIDs, err = models.FindUserAccessibleRepoIDs(userID)
267+
if err != nil {
268+
ctx.ServerError("SearchResults", err)
269+
return
270+
}
271+
}
272+
273+
var (
274+
total int
275+
searchResults []*search.Result
276+
)
277+
278+
// if non-admin login user, we need check UnitTypeCode at first
279+
if ctx.User != nil && len(repoIDs) > 0 {
280+
repoMaps, err := models.GetRepositoriesMapByIDs(repoIDs)
281+
if err != nil {
282+
ctx.ServerError("SearchResults", err)
283+
return
284+
}
285+
286+
var rightRepoMap = make(map[int64]*models.Repository, len(repoMaps))
287+
repoIDs = make([]int64, 0, len(repoMaps))
288+
for id, repo := range repoMaps {
289+
if repo.CheckUnitUser(userID, isAdmin, models.UnitTypeCode) {
290+
rightRepoMap[id] = repo
291+
repoIDs = append(repoIDs, id)
292+
}
293+
}
294+
295+
ctx.Data["RepoMaps"] = rightRepoMap
296+
297+
total, searchResults, err = search.PerformSearch(repoIDs, keyword, page, setting.UI.RepoSearchPagingNum)
298+
if err != nil {
299+
ctx.ServerError("SearchResults", err)
300+
return
301+
}
302+
// if non-login user or isAdmin, no need to check UnitTypeCode
303+
} else if (ctx.User == nil && len(repoIDs) > 0) || isAdmin {
304+
total, searchResults, err = search.PerformSearch(repoIDs, keyword, page, setting.UI.RepoSearchPagingNum)
305+
if err != nil {
306+
ctx.ServerError("SearchResults", err)
307+
return
308+
}
309+
310+
var loadRepoIDs = make([]int64, 0, len(searchResults))
311+
for _, result := range searchResults {
312+
var find bool
313+
for _, id := range loadRepoIDs {
314+
if id == result.RepoID {
315+
find = true
316+
break
317+
}
318+
}
319+
if !find {
320+
loadRepoIDs = append(loadRepoIDs, result.RepoID)
321+
}
322+
}
323+
324+
repoMaps, err := models.GetRepositoriesMapByIDs(loadRepoIDs)
325+
if err != nil {
326+
ctx.ServerError("SearchResults", err)
327+
return
328+
}
329+
330+
ctx.Data["RepoMaps"] = repoMaps
331+
}
332+
333+
ctx.Data["Keyword"] = keyword
334+
pager := paginater.New(total, setting.UI.RepoSearchPagingNum, page, 5)
335+
ctx.Data["Page"] = pager
336+
ctx.Data["SearchResults"] = searchResults
337+
ctx.Data["RequireHighlightJS"] = true
338+
ctx.Data["PageIsViewCode"] = true
339+
ctx.HTML(200, tplExploreCode)
340+
}
341+
226342
// NotFound render 404 page
227343
func NotFound(ctx *context.Context) {
228344
ctx.Data["Title"] = "Page Not Found"

‎routers/repo/search.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@ func Search(ctx *context.Context) {
2929
if page <= 0 {
3030
page = 1
3131
}
32-
total, searchResults, err := search.PerformSearch(ctx.Repo.Repository.ID, keyword, page, setting.UI.RepoSearchPagingNum)
32+
total, searchResults, err := search.PerformSearch([]int64{ctx.Repo.Repository.ID},
33+
keyword, page, setting.UI.RepoSearchPagingNum)
3334
if err != nil {
3435
ctx.ServerError("SearchResults", err)
3536
return

‎routers/routes/routes.go

+1
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,7 @@ func RegisterRoutes(m *macaron.Macaron) {
170170
m.Get("/repos", routers.ExploreRepos)
171171
m.Get("/users", routers.ExploreUsers)
172172
m.Get("/organizations", routers.ExploreOrganizations)
173+
m.Get("/code", routers.ExploreCode)
173174
}, ignSignIn)
174175
m.Combo("/install", routers.InstallInit).Get(routers.Install).
175176
Post(bindIgnErr(auth.InstallForm{}), routers.InstallPost)

‎templates/explore/code.tmpl

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
{{template "base/head" .}}
2+
<div class="explore users">
3+
{{template "explore/navbar" .}}
4+
<div class="ui container">
5+
<form class="ui form" style="max-width: 100%">
6+
<div class="ui fluid action input">
7+
<input name="q" value="{{.Keyword}}" placeholder="{{.i18n.Tr "explore.search"}}..." autofocus>
8+
<input type="hidden" name="tab" value="{{$.TabName}}">
9+
<button class="ui blue button">{{.i18n.Tr "explore.search"}}</button>
10+
</div>
11+
</form>
12+
<div class="ui divider"></div>
13+
14+
<div class="ui user list">
15+
{{if .SearchResults}}
16+
<h3>
17+
{{.i18n.Tr "explore.code_search_results" (.Keyword|Escape) | Str2html }}
18+
</h3>
19+
<div class="repository search">
20+
{{range $result := .SearchResults}}
21+
{{$repo := (index $.RepoMaps .RepoID)}}
22+
<div class="diff-file-box diff-box file-content non-diff-file-content repo-search-result">
23+
<h4 class="ui top attached normal header">
24+
<span class="file"><a rel="nofollow" href="{{EscapePound $repo.HTMLURL}}">{{$repo.FullName}}</a> - {{.Filename}}</span>
25+
<a class="ui basic grey tiny button" rel="nofollow" href="{{EscapePound $repo.HTMLURL}}/src/branch/{{$repo.DefaultBranch}}/{{EscapePound .Filename}}">{{$.i18n.Tr "repo.diff.view_file"}}</a>
26+
</h4>
27+
<div class="ui attached table segment">
28+
<div class="file-body file-code code-view">
29+
<table>
30+
<tbody>
31+
<tr>
32+
<td class="lines-num">
33+
{{range .LineNumbers}}
34+
<a href="{{EscapePound $repo.HTMLURL}}/src/branch/{{$repo.DefaultBranch}}/{{EscapePound $result.Filename}}#L{{.}}"><span>{{.}}</span></a>
35+
{{end}}
36+
</td>
37+
<td class="lines-code"><pre><code class="{{.HighlightClass}}"><ol class="linenums">{{.FormattedLines}}</ol></code></pre></td>
38+
</tr>
39+
</tbody>
40+
</table>
41+
</div>
42+
</div>
43+
</div>
44+
{{end}}
45+
</div>
46+
{{else}}
47+
<div>{{$.i18n.Tr "explore.code_no_results"}}</div>
48+
{{end}}
49+
</div>
50+
51+
{{template "base/paginate" .}}
52+
</div>
53+
</div>
54+
{{template "base/footer" .}}
55+

0 commit comments

Comments
 (0)