From c10503afeccd5172ace7613094dd5fe1e0770c55 Mon Sep 17 00:00:00 2001
From: Jui-Nan Lin <jnlinn@gmail.com>
Date: Wed, 27 Jan 2021 18:00:35 +0800
Subject: [PATCH] [Feature] add precise search type for Elastic Search (#12869)
* feat: add type query parameters for specifying precise search
* feat: add select dropdown in search box
Co-authored-by: Lauris BH <lauris@nix.lv>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>
---
modules/context/pagination.go | 1 +
modules/indexer/code/bleve.go | 25 ++++++++++++++++++-------
modules/indexer/code/elastic_search.go | 13 +++++++++++--
modules/indexer/code/indexer.go | 2 +-
modules/indexer/code/indexer_test.go | 2 +-
modules/indexer/code/search.go | 4 ++--
modules/indexer/code/wrapped.go | 4 ++--
options/locale/locale_en-US.ini | 4 ++++
routers/home.go | 8 ++++++--
routers/repo/search.go | 6 +++++-
templates/explore/code.tmpl | 13 +++++++++++--
templates/repo/search.tmpl | 20 +++++++++++++++-----
12 files changed, 77 insertions(+), 25 deletions(-)
diff --git a/modules/context/pagination.go b/modules/context/pagination.go
index a6638f4086..b678fccd15 100644
--- a/modules/context/pagination.go
+++ b/modules/context/pagination.go
@@ -53,4 +53,5 @@ func (p *Pagination) SetDefaultParams(ctx *Context) {
p.AddParam(ctx, "sort", "SortType")
p.AddParam(ctx, "q", "Keyword")
p.AddParam(ctx, "tab", "TabName")
+ p.AddParam(ctx, "t", "queryType")
}
diff --git a/modules/indexer/code/bleve.go b/modules/indexer/code/bleve.go
index b0822ad222..826efde4c1 100644
--- a/modules/indexer/code/bleve.go
+++ b/modules/indexer/code/bleve.go
@@ -280,12 +280,23 @@ func (b *BleveIndexer) Delete(repoID int64) error {
// Search searches for files in the specified repo.
// Returns the matching file-paths
-func (b *BleveIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int) (int64, []*SearchResult, []*SearchResultLanguages, error) {
- phraseQuery := bleve.NewMatchPhraseQuery(keyword)
- phraseQuery.FieldVal = "Content"
- phraseQuery.Analyzer = repoIndexerAnalyzer
+func (b *BleveIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
+ var (
+ indexerQuery query.Query
+ keywordQuery query.Query
+ )
+
+ if isMatch {
+ prefixQuery := bleve.NewPrefixQuery(keyword)
+ prefixQuery.FieldVal = "Content"
+ keywordQuery = prefixQuery
+ } else {
+ phraseQuery := bleve.NewMatchPhraseQuery(keyword)
+ phraseQuery.FieldVal = "Content"
+ phraseQuery.Analyzer = repoIndexerAnalyzer
+ keywordQuery = phraseQuery
+ }
- var indexerQuery query.Query
if len(repoIDs) > 0 {
var repoQueries = make([]query.Query, 0, len(repoIDs))
for _, repoID := range repoIDs {
@@ -294,10 +305,10 @@ func (b *BleveIndexer) Search(repoIDs []int64, language, keyword string, page, p
indexerQuery = bleve.NewConjunctionQuery(
bleve.NewDisjunctionQuery(repoQueries...),
- phraseQuery,
+ keywordQuery,
)
} else {
- indexerQuery = phraseQuery
+ indexerQuery = keywordQuery
}
// Save for reuse without language filter
diff --git a/modules/indexer/code/elastic_search.go b/modules/indexer/code/elastic_search.go
index 0f61c4e592..f81dbb34d4 100644
--- a/modules/indexer/code/elastic_search.go
+++ b/modules/indexer/code/elastic_search.go
@@ -27,6 +27,10 @@ import (
const (
esRepoIndexerLatestVersion = 1
+ // multi-match-types, currently only 2 types are used
+ // Reference: https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-multi-match-query.html#multi-match-types
+ esMultiMatchTypeBestFields = "best_fields"
+ esMultiMatchTypePhrasePrefix = "phrase_prefix"
)
var (
@@ -330,8 +334,13 @@ func extractAggs(searchResult *elastic.SearchResult) []*SearchResultLanguages {
}
// Search searches for codes and language stats by given conditions.
-func (b *ElasticSearchIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int) (int64, []*SearchResult, []*SearchResultLanguages, error) {
- kwQuery := elastic.NewMultiMatchQuery(keyword, "content")
+func (b *ElasticSearchIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
+ searchType := esMultiMatchTypeBestFields
+ if isMatch {
+ searchType = esMultiMatchTypePhrasePrefix
+ }
+
+ kwQuery := elastic.NewMultiMatchQuery(keyword, "content").Type(searchType)
query := elastic.NewBoolQuery()
query = query.Must(kwQuery)
if len(repoIDs) > 0 {
diff --git a/modules/indexer/code/indexer.go b/modules/indexer/code/indexer.go
index 35c298a548..a7d78e9fdc 100644
--- a/modules/indexer/code/indexer.go
+++ b/modules/indexer/code/indexer.go
@@ -43,7 +43,7 @@ type SearchResultLanguages struct {
type Indexer interface {
Index(repo *models.Repository, sha string, changes *repoChanges) error
Delete(repoID int64) error
- Search(repoIDs []int64, language, keyword string, page, pageSize int) (int64, []*SearchResult, []*SearchResultLanguages, error)
+ Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error)
Close()
}
diff --git a/modules/indexer/code/indexer_test.go b/modules/indexer/code/indexer_test.go
index 0b4851a48a..8fcb7a0e8a 100644
--- a/modules/indexer/code/indexer_test.go
+++ b/modules/indexer/code/indexer_test.go
@@ -64,7 +64,7 @@ func testIndexer(name string, t *testing.T, indexer Indexer) {
for _, kw := range keywords {
t.Run(kw.Keyword, func(t *testing.T) {
- total, res, langs, err := indexer.Search(kw.RepoIDs, "", kw.Keyword, 1, 10)
+ total, res, langs, err := indexer.Search(kw.RepoIDs, "", kw.Keyword, 1, 10, false)
assert.NoError(t, err)
assert.EqualValues(t, len(kw.IDs), total)
assert.EqualValues(t, kw.Langs, len(langs))
diff --git a/modules/indexer/code/search.go b/modules/indexer/code/search.go
index 29ed416541..51b7c9427d 100644
--- a/modules/indexer/code/search.go
+++ b/modules/indexer/code/search.go
@@ -106,12 +106,12 @@ func searchResult(result *SearchResult, startIndex, endIndex int) (*Result, erro
}
// PerformSearch perform a search on a repository
-func PerformSearch(repoIDs []int64, language, keyword string, page, pageSize int) (int, []*Result, []*SearchResultLanguages, error) {
+func PerformSearch(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int, []*Result, []*SearchResultLanguages, error) {
if len(keyword) == 0 {
return 0, nil, nil, nil
}
- total, results, resultLanguages, err := indexer.Search(repoIDs, language, keyword, page, pageSize)
+ total, results, resultLanguages, err := indexer.Search(repoIDs, language, keyword, page, pageSize, isMatch)
if err != nil {
return 0, nil, nil, err
}
diff --git a/modules/indexer/code/wrapped.go b/modules/indexer/code/wrapped.go
index d839544874..5b19f9c625 100644
--- a/modules/indexer/code/wrapped.go
+++ b/modules/indexer/code/wrapped.go
@@ -73,12 +73,12 @@ func (w *wrappedIndexer) Delete(repoID int64) error {
return indexer.Delete(repoID)
}
-func (w *wrappedIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int) (int64, []*SearchResult, []*SearchResultLanguages, error) {
+func (w *wrappedIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
indexer, err := w.get()
if err != nil {
return 0, nil, nil, err
}
- return indexer.Search(repoIDs, language, keyword, page, pageSize)
+ return indexer.Search(repoIDs, language, keyword, page, pageSize, isMatch)
}
diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini
index 1fe43ce29b..4083f71147 100644
--- a/options/locale/locale_en-US.ini
+++ b/options/locale/locale_en-US.ini
@@ -237,6 +237,8 @@ users = Users
organizations = Organizations
search = Search
code = Code
+search.fuzzy = Fuzzy
+search.match = Match
repo_no_results = No matching repositories found.
user_no_results = No matching users found.
org_no_results = No matching organizations found.
@@ -1462,6 +1464,8 @@ activity.git_stats_deletion_n = %d deletions
search = Search
search.search_repo = Search repository
+search.fuzzy = Fuzzy
+search.match = Match
search.results = Search results for "%s" in <a href="%s">%s</a>
settings = Settings
diff --git a/routers/home.go b/routers/home.go
index d37bf8e31b..f82ee9808b 100644
--- a/routers/home.go
+++ b/routers/home.go
@@ -299,6 +299,9 @@ func ExploreCode(ctx *context.Context) {
page = 1
}
+ queryType := strings.TrimSpace(ctx.Query("t"))
+ isMatch := queryType == "match"
+
var (
repoIDs []int64
err error
@@ -342,14 +345,14 @@ func ExploreCode(ctx *context.Context) {
ctx.Data["RepoMaps"] = rightRepoMap
- total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum)
+ total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
if err != nil {
ctx.ServerError("SearchResults", err)
return
}
// if non-login user or isAdmin, no need to check UnitTypeCode
} else if (ctx.User == nil && len(repoIDs) > 0) || isAdmin {
- total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum)
+ total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
if err != nil {
ctx.ServerError("SearchResults", err)
return
@@ -380,6 +383,7 @@ func ExploreCode(ctx *context.Context) {
ctx.Data["Keyword"] = keyword
ctx.Data["Language"] = language
+ ctx.Data["queryType"] = queryType
ctx.Data["SearchResults"] = searchResults
ctx.Data["SearchResultLanguages"] = searchResultLanguages
ctx.Data["RequireHighlightJS"] = true
diff --git a/routers/repo/search.go b/routers/repo/search.go
index e110ae2a72..42fe3d7584 100644
--- a/routers/repo/search.go
+++ b/routers/repo/search.go
@@ -28,14 +28,18 @@ func Search(ctx *context.Context) {
if page <= 0 {
page = 1
}
+ queryType := strings.TrimSpace(ctx.Query("t"))
+ isMatch := queryType == "match"
+
total, searchResults, searchResultLanguages, err := code_indexer.PerformSearch([]int64{ctx.Repo.Repository.ID},
- language, keyword, page, setting.UI.RepoSearchPagingNum)
+ language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
if err != nil {
ctx.ServerError("SearchResults", err)
return
}
ctx.Data["Keyword"] = keyword
ctx.Data["Language"] = language
+ ctx.Data["queryType"] = queryType
ctx.Data["SourcePath"] = setting.AppSubURL + "/" +
path.Join(ctx.Repo.Repository.Owner.Name, ctx.Repo.Repository.Name)
ctx.Data["SearchResults"] = searchResults
diff --git a/templates/explore/code.tmpl b/templates/explore/code.tmpl
index 1c41dd843f..2465663a60 100644
--- a/templates/explore/code.tmpl
+++ b/templates/explore/code.tmpl
@@ -5,9 +5,19 @@
<form class="ui form ignore-dirty" style="max-width: 100%">
<input type="hidden" name="tab" value="{{$.TabName}}">
<div class="ui fluid action input">
+ <div class="twelve wide field">
<input name="q" value="{{.Keyword}}" placeholder="{{.i18n.Tr "explore.search"}}..." autofocus>
+ </div>
+ <div class="two wide field">
+ <select name="t">
+ <option value="">{{.i18n.Tr "explore.search.fuzzy"}}</option>
+ <option value="match" {{if eq .queryType "match"}}selected{{end}}>{{.i18n.Tr "explore.search.match"}}</option>
+ </select>
+ </div>
+ <div class="three field">
<button class="ui blue button">{{.i18n.Tr "explore.search"}}</button>
</div>
+ </div>
</form>
<div class="ui divider"></div>
@@ -18,7 +28,7 @@
</h3>
<div class="df ac fw">
{{range $term := .SearchResultLanguages}}
- <a class="ui text-label df ac mr-1 my-1 {{if eq $.Language $term.Language}}primary {{end}}basic label" href="{{AppSubUrl}}/explore/code?q={{$.Keyword}}{{if ne $.Language $term.Language}}&l={{$term.Language}}{{end}}">
+ <a class="ui text-label df ac mr-1 my-1 {{if eq $.Language $term.Language}}primary {{end}}basic label" href="{{AppSubUrl}}/explore/code?q={{$.Keyword}}{{if ne $.Language $term.Language}}&l={{$term.Language}}{{end}}{{if ne $.queryType ""}}&t={{$.queryType}}{{end}}">
<i class="color-icon mr-3" style="background-color: {{$term.Color}}"></i>
{{$term.Language}}
<div class="detail">{{$term.Count}}</div>
@@ -62,4 +72,3 @@
</div>
</div>
{{template "base/footer" .}}
-
diff --git a/templates/repo/search.tmpl b/templates/repo/search.tmpl
index b66391a5ac..ab9e9be2d6 100644
--- a/templates/repo/search.tmpl
+++ b/templates/repo/search.tmpl
@@ -5,10 +5,20 @@
<div class="ui repo-search">
<form class="ui form ignore-dirty" method="get">
<div class="ui fluid action input">
- <input name="q" value="{{.Keyword}}" placeholder="{{.i18n.Tr "repo.search.search_repo"}}">
- <button class="ui button" type="submit">
- <i class="icon df ac jc">{{svg "octicon-search" 16}}</i>
- </button>
+ <div class="twelve wide field">
+ <input name="q" value="{{.Keyword}}" placeholder="{{.i18n.Tr "repo.search.search_repo"}}">
+ </div>
+ <div class="two wide field">
+ <select name="t">
+ <option value="">{{.i18n.Tr "repo.search.fuzzy"}}</option>
+ <option value="match" {{if eq .queryType "match"}}selected{{end}}>{{.i18n.Tr "repo.search.match"}}</option>
+ </select>
+ </div>
+ <div class="three field">
+ <button class="ui button" type="submit">
+ <i class="icon df ac jc">{{svg "octicon-search" 16}}</i>
+ </button>
+ </div>
</div>
</form>
</div>
@@ -18,7 +28,7 @@
</h3>
<div class="df ac fw">
{{range $term := .SearchResultLanguages}}
- <a class="ui text-label df ac mr-1 my-1 {{if eq $.Language $term.Language}}primary {{end}}basic label" href="{{EscapePound $.SourcePath}}/search?q={{$.Keyword}}{{if ne $.Language $term.Language}}&l={{$term.Language}}{{end}}">
+ <a class="ui text-label df ac mr-1 my-1 {{if eq $.Language $term.Language}}primary {{end}}basic label" href="{{EscapePound $.SourcePath}}/search?q={{$.Keyword}}{{if ne $.Language $term.Language}}&l={{$term.Language}}{{end}}{{if ne $.queryType ""}}&t={{$.queryType}}{{end}}">
<i class="color-icon mr-3" style="background-color: {{$term.Color}}"></i>
{{$term.Language}}
<div class="detail">{{$term.Count}}</div>