From 52c2e82813c2519cd08c9ecd66ef7b0815682770 Mon Sep 17 00:00:00 2001
From: Sandro Santilli <strk@kbt.io>
Date: Fri, 10 Jun 2022 07:39:53 +0200
Subject: [PATCH] Custom regexp external issues (#17624)
* Implement custom regular expression for external issue tracking.
Signed-off-by: Alexander Beyn <malex@fatelectrons.org>
* Fix syntax/style
* Update repo.go
* Set metas['regexp']
* gofmt
* fix some tests
* fix more tests
* refactor frontend
* use LRU cache for regexp
* Update modules/markup/html_internal_test.go
Co-authored-by: Alexander Beyn <malex@fatelectrons.org>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>
Co-authored-by: Lauris BH <lauris@nix.lv>
Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
---
models/repo/repo.go | 3 ++
models/repo/repo_unit.go | 7 ++--
models/repo_test.go | 3 ++
modules/markup/html.go | 40 +++++++++++++++------
modules/markup/html_internal_test.go | 52 +++++++++++++++++++++++++---
modules/references/references.go | 20 ++++++++++-
modules/regexplru/regexplru.go | 45 ++++++++++++++++++++++++
modules/regexplru/regexplru_test.go | 27 +++++++++++++++
options/locale/locale_en-US.ini | 3 ++
routers/web/repo/setting.go | 7 ++--
services/forms/repo_form.go | 1 +
templates/repo/settings/options.tmpl | 19 +++++++---
web_src/js/features/repo-legacy.js | 5 +++
13 files changed, 206 insertions(+), 26 deletions(-)
create mode 100644 modules/regexplru/regexplru.go
create mode 100644 modules/regexplru/regexplru_test.go
diff --git a/models/repo/repo.go b/models/repo/repo.go
index 3fd6b94eb1..57d85435eb 100644
--- a/models/repo/repo.go
+++ b/models/repo/repo.go
@@ -414,6 +414,9 @@ func (repo *Repository) ComposeMetas() map[string]string {
switch unit.ExternalTrackerConfig().ExternalTrackerStyle {
case markup.IssueNameStyleAlphanumeric:
metas["style"] = markup.IssueNameStyleAlphanumeric
+ case markup.IssueNameStyleRegexp:
+ metas["style"] = markup.IssueNameStyleRegexp
+ metas["regexp"] = unit.ExternalTrackerConfig().ExternalTrackerRegexpPattern
default:
metas["style"] = markup.IssueNameStyleNumeric
}
diff --git a/models/repo/repo_unit.go b/models/repo/repo_unit.go
index 8c17d6138c..da3e19dece 100644
--- a/models/repo/repo_unit.go
+++ b/models/repo/repo_unit.go
@@ -76,9 +76,10 @@ func (cfg *ExternalWikiConfig) ToDB() ([]byte, error) {
// ExternalTrackerConfig describes external tracker config
type ExternalTrackerConfig struct {
- ExternalTrackerURL string
- ExternalTrackerFormat string
- ExternalTrackerStyle string
+ ExternalTrackerURL string
+ ExternalTrackerFormat string
+ ExternalTrackerStyle string
+ ExternalTrackerRegexpPattern string
}
// FromDB fills up a ExternalTrackerConfig from serialized format.
diff --git a/models/repo_test.go b/models/repo_test.go
index c9e66398d1..f554ff16a6 100644
--- a/models/repo_test.go
+++ b/models/repo_test.go
@@ -74,6 +74,9 @@ func TestMetas(t *testing.T) {
externalTracker.ExternalTrackerConfig().ExternalTrackerStyle = markup.IssueNameStyleNumeric
testSuccess(markup.IssueNameStyleNumeric)
+ externalTracker.ExternalTrackerConfig().ExternalTrackerStyle = markup.IssueNameStyleRegexp
+ testSuccess(markup.IssueNameStyleRegexp)
+
repo, err := repo_model.GetRepositoryByID(3)
assert.NoError(t, err)
diff --git a/modules/markup/html.go b/modules/markup/html.go
index c5d36e701f..69d9ba3ef2 100644
--- a/modules/markup/html.go
+++ b/modules/markup/html.go
@@ -20,6 +20,7 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/markup/common"
"code.gitea.io/gitea/modules/references"
+ "code.gitea.io/gitea/modules/regexplru"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/templates/vars"
"code.gitea.io/gitea/modules/util"
@@ -33,6 +34,7 @@ import (
const (
IssueNameStyleNumeric = "numeric"
IssueNameStyleAlphanumeric = "alphanumeric"
+ IssueNameStyleRegexp = "regexp"
)
var (
@@ -815,19 +817,35 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) {
)
next := node.NextSibling
+
for node != nil && node != next {
- _, exttrack := ctx.Metas["format"]
- alphanum := ctx.Metas["style"] == IssueNameStyleAlphanumeric
+ _, hasExtTrackFormat := ctx.Metas["format"]
// Repos with external issue trackers might still need to reference local PRs
// We need to concern with the first one that shows up in the text, whichever it is
- found, ref = references.FindRenderizableReferenceNumeric(node.Data, exttrack && alphanum)
- if exttrack && alphanum {
- if found2, ref2 := references.FindRenderizableReferenceAlphanumeric(node.Data); found2 {
- if !found || ref2.RefLocation.Start < ref.RefLocation.Start {
- found = true
- ref = ref2
- }
+ isNumericStyle := ctx.Metas["style"] == "" || ctx.Metas["style"] == IssueNameStyleNumeric
+ foundNumeric, refNumeric := references.FindRenderizableReferenceNumeric(node.Data, hasExtTrackFormat && !isNumericStyle)
+
+ switch ctx.Metas["style"] {
+ case "", IssueNameStyleNumeric:
+ found, ref = foundNumeric, refNumeric
+ case IssueNameStyleAlphanumeric:
+ found, ref = references.FindRenderizableReferenceAlphanumeric(node.Data)
+ case IssueNameStyleRegexp:
+ pattern, err := regexplru.GetCompiled(ctx.Metas["regexp"])
+ if err != nil {
+ return
+ }
+ found, ref = references.FindRenderizableReferenceRegexp(node.Data, pattern)
+ }
+
+ // Repos with external issue trackers might still need to reference local PRs
+ // We need to concern with the first one that shows up in the text, whichever it is
+ if hasExtTrackFormat && !isNumericStyle {
+ // If numeric (PR) was found, and it was BEFORE the non-numeric pattern, use that
+ if foundNumeric && refNumeric.RefLocation.Start < ref.RefLocation.Start {
+ found = foundNumeric
+ ref = refNumeric
}
}
if !found {
@@ -836,7 +854,7 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) {
var link *html.Node
reftext := node.Data[ref.RefLocation.Start:ref.RefLocation.End]
- if exttrack && !ref.IsPull {
+ if hasExtTrackFormat && !ref.IsPull {
ctx.Metas["index"] = ref.Issue
res, err := vars.Expand(ctx.Metas["format"], ctx.Metas)
@@ -869,7 +887,7 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) {
// Decorate action keywords if actionable
var keyword *html.Node
- if references.IsXrefActionable(ref, exttrack, alphanum) {
+ if references.IsXrefActionable(ref, hasExtTrackFormat) {
keyword = createKeyword(node.Data[ref.ActionLocation.Start:ref.ActionLocation.End])
} else {
keyword = &html.Node{
diff --git a/modules/markup/html_internal_test.go b/modules/markup/html_internal_test.go
index f0eb3253e1..25b0f7b7a5 100644
--- a/modules/markup/html_internal_test.go
+++ b/modules/markup/html_internal_test.go
@@ -21,8 +21,8 @@ const (
TestRepoURL = TestAppURL + TestOrgRepo + "/"
)
-// alphanumLink an HTML link to an alphanumeric-style issue
-func alphanumIssueLink(baseURL, class, name string) string {
+// externalIssueLink an HTML link to an alphanumeric-style issue
+func externalIssueLink(baseURL, class, name string) string {
return link(util.URLJoin(baseURL, name), class, name)
}
@@ -54,6 +54,13 @@ var alphanumericMetas = map[string]string{
"style": IssueNameStyleAlphanumeric,
}
+var regexpMetas = map[string]string{
+ "format": "https://someurl.com/{user}/{repo}/{index}",
+ "user": "someUser",
+ "repo": "someRepo",
+ "style": IssueNameStyleRegexp,
+}
+
// these values should match the TestOrgRepo const above
var localMetas = map[string]string{
"user": "gogits",
@@ -184,7 +191,7 @@ func TestRender_IssueIndexPattern4(t *testing.T) {
test := func(s, expectedFmt string, names ...string) {
links := make([]interface{}, len(names))
for i, name := range names {
- links[i] = alphanumIssueLink("https://someurl.com/someUser/someRepo/", "ref-issue ref-external-issue", name)
+ links[i] = externalIssueLink("https://someurl.com/someUser/someRepo/", "ref-issue ref-external-issue", name)
}
expected := fmt.Sprintf(expectedFmt, links...)
testRenderIssueIndexPattern(t, s, expected, &RenderContext{Metas: alphanumericMetas})
@@ -194,6 +201,43 @@ func TestRender_IssueIndexPattern4(t *testing.T) {
test("test issue ABCDEFGHIJ-1234567890", "test issue %s", "ABCDEFGHIJ-1234567890")
}
+func TestRender_IssueIndexPattern5(t *testing.T) {
+ setting.AppURL = TestAppURL
+
+ // regexp: render inputs without valid mentions
+ test := func(s, expectedFmt, pattern string, ids, names []string) {
+ metas := regexpMetas
+ metas["regexp"] = pattern
+ links := make([]interface{}, len(ids))
+ for i, id := range ids {
+ links[i] = link(util.URLJoin("https://someurl.com/someUser/someRepo/", id), "ref-issue ref-external-issue", names[i])
+ }
+
+ expected := fmt.Sprintf(expectedFmt, links...)
+ testRenderIssueIndexPattern(t, s, expected, &RenderContext{Metas: metas})
+ }
+
+ test("abc ISSUE-123 def", "abc %s def",
+ "ISSUE-(\\d+)",
+ []string{"123"},
+ []string{"ISSUE-123"},
+ )
+
+ test("abc (ISSUE 123) def", "abc %s def",
+ "\\(ISSUE (\\d+)\\)",
+ []string{"123"},
+ []string{"(ISSUE 123)"},
+ )
+
+ test("abc ISSUE-123 def", "abc %s def",
+ "(ISSUE-(\\d+))",
+ []string{"ISSUE-123"},
+ []string{"ISSUE-123"},
+ )
+
+ testRenderIssueIndexPattern(t, "will not match", "will not match", &RenderContext{Metas: regexpMetas})
+}
+
func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *RenderContext) {
if ctx.URLPrefix == "" {
ctx.URLPrefix = TestAppURL
@@ -202,7 +246,7 @@ func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *Rend
var buf strings.Builder
err := postProcess(ctx, []processor{issueIndexPatternProcessor}, strings.NewReader(input), &buf)
assert.NoError(t, err)
- assert.Equal(t, expected, buf.String())
+ assert.Equal(t, expected, buf.String(), "input=%q", input)
}
func TestRender_AutoLink(t *testing.T) {
diff --git a/modules/references/references.go b/modules/references/references.go
index 630e621043..7f5086d093 100644
--- a/modules/references/references.go
+++ b/modules/references/references.go
@@ -351,6 +351,24 @@ func FindRenderizableReferenceNumeric(content string, prOnly bool) (bool, *Rende
}
}
+// FindRenderizableReferenceRegexp returns the first regexp unvalidated references found in a string.
+func FindRenderizableReferenceRegexp(content string, pattern *regexp.Regexp) (bool, *RenderizableReference) {
+ match := pattern.FindStringSubmatchIndex(content)
+ if len(match) < 4 {
+ return false, nil
+ }
+
+ action, location := findActionKeywords([]byte(content), match[2])
+
+ return true, &RenderizableReference{
+ Issue: content[match[2]:match[3]],
+ RefLocation: &RefSpan{Start: match[0], End: match[1]},
+ Action: action,
+ ActionLocation: location,
+ IsPull: false,
+ }
+}
+
// FindRenderizableReferenceAlphanumeric returns the first alphanumeric unvalidated references found in a string.
func FindRenderizableReferenceAlphanumeric(content string) (bool, *RenderizableReference) {
match := issueAlphanumericPattern.FindStringSubmatchIndex(content)
@@ -547,7 +565,7 @@ func findActionKeywords(content []byte, start int) (XRefAction, *RefSpan) {
}
// IsXrefActionable returns true if the xref action is actionable (i.e. produces a result when resolved)
-func IsXrefActionable(ref *RenderizableReference, extTracker, alphaNum bool) bool {
+func IsXrefActionable(ref *RenderizableReference, extTracker bool) bool {
if extTracker {
// External issues cannot be automatically closed
return false
diff --git a/modules/regexplru/regexplru.go b/modules/regexplru/regexplru.go
new file mode 100644
index 0000000000..97c7cff4c1
--- /dev/null
+++ b/modules/regexplru/regexplru.go
@@ -0,0 +1,45 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package regexplru
+
+import (
+ "regexp"
+
+ "code.gitea.io/gitea/modules/log"
+
+ lru "github.com/hashicorp/golang-lru"
+)
+
+var lruCache *lru.Cache
+
+func init() {
+ var err error
+ lruCache, err = lru.New(1000)
+ if err != nil {
+ log.Fatal("failed to new LRU cache, err: %v", err)
+ }
+}
+
+// GetCompiled works like regexp.Compile, the compiled expr or error is stored in LRU cache
+func GetCompiled(expr string) (r *regexp.Regexp, err error) {
+ v, ok := lruCache.Get(expr)
+ if !ok {
+ r, err = regexp.Compile(expr)
+ if err != nil {
+ lruCache.Add(expr, err)
+ return nil, err
+ }
+ lruCache.Add(expr, r)
+ } else {
+ r, ok = v.(*regexp.Regexp)
+ if !ok {
+ if err, ok = v.(error); ok {
+ return nil, err
+ }
+ panic("impossible")
+ }
+ }
+ return r, nil
+}
diff --git a/modules/regexplru/regexplru_test.go b/modules/regexplru/regexplru_test.go
new file mode 100644
index 0000000000..041f0dcfb9
--- /dev/null
+++ b/modules/regexplru/regexplru_test.go
@@ -0,0 +1,27 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package regexplru
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestRegexpLru(t *testing.T) {
+ r, err := GetCompiled("a")
+ assert.NoError(t, err)
+ assert.True(t, r.MatchString("a"))
+
+ r, err = GetCompiled("a")
+ assert.NoError(t, err)
+ assert.True(t, r.MatchString("a"))
+
+ assert.EqualValues(t, 1, lruCache.Len())
+
+ _, err = GetCompiled("(")
+ assert.Error(t, err)
+ assert.EqualValues(t, 2, lruCache.Len())
+}
diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini
index b9ba6e1136..c4ad714717 100644
--- a/options/locale/locale_en-US.ini
+++ b/options/locale/locale_en-US.ini
@@ -1811,6 +1811,9 @@ settings.tracker_url_format_error = The external issue tracker URL format is not
settings.tracker_issue_style = External Issue Tracker Number Format
settings.tracker_issue_style.numeric = Numeric
settings.tracker_issue_style.alphanumeric = Alphanumeric
+settings.tracker_issue_style.regexp = Regular Expression
+settings.tracker_issue_style.regexp_pattern = Regular Expression Pattern
+settings.tracker_issue_style.regexp_pattern_desc = The first captured group will be used in place of <code>{index}</code>.
settings.tracker_url_format_desc = Use the placeholders <code>{user}</code>, <code>{repo}</code> and <code>{index}</code> for the username, repository name and issue index.
settings.enable_timetracker = Enable Time Tracking
settings.allow_only_contributors_to_track_time = Let Only Contributors Track Time
diff --git a/routers/web/repo/setting.go b/routers/web/repo/setting.go
index 1a7a41ae91..f49ef6e85d 100644
--- a/routers/web/repo/setting.go
+++ b/routers/web/repo/setting.go
@@ -434,9 +434,10 @@ func SettingsPost(ctx *context.Context) {
RepoID: repo.ID,
Type: unit_model.TypeExternalTracker,
Config: &repo_model.ExternalTrackerConfig{
- ExternalTrackerURL: form.ExternalTrackerURL,
- ExternalTrackerFormat: form.TrackerURLFormat,
- ExternalTrackerStyle: form.TrackerIssueStyle,
+ ExternalTrackerURL: form.ExternalTrackerURL,
+ ExternalTrackerFormat: form.TrackerURLFormat,
+ ExternalTrackerStyle: form.TrackerIssueStyle,
+ ExternalTrackerRegexpPattern: form.ExternalTrackerRegexpPattern,
},
})
deleteUnitTypes = append(deleteUnitTypes, unit_model.TypeIssues)
diff --git a/services/forms/repo_form.go b/services/forms/repo_form.go
index 2bcb91f8c3..738a77d2bb 100644
--- a/services/forms/repo_form.go
+++ b/services/forms/repo_form.go
@@ -141,6 +141,7 @@ type RepoSettingForm struct {
ExternalTrackerURL string
TrackerURLFormat string
TrackerIssueStyle string
+ ExternalTrackerRegexpPattern string
EnableCloseIssuesViaCommitInAnyBranch bool
EnableProjects bool
EnablePackages bool
diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl
index f44d9c98ad..67a98aff43 100644
--- a/templates/repo/settings/options.tmpl
+++ b/templates/repo/settings/options.tmpl
@@ -361,16 +361,27 @@
<div class="ui radio checkbox">
{{$externalTracker := (.Repository.MustGetUnit $.UnitTypeExternalTracker)}}
{{$externalTrackerStyle := $externalTracker.ExternalTrackerConfig.ExternalTrackerStyle}}
- <input class="hidden" tabindex="0" name="tracker_issue_style" type="radio" value="numeric" {{if $externalTrackerStyle}}{{if eq $externalTrackerStyle "numeric"}}checked=""{{end}}{{end}}/>
- <label>{{.i18n.Tr "repo.settings.tracker_issue_style.numeric"}} <span class="ui light grey text">(#1234)</span></label>
+ <input class="js-tracker-issue-style" name="tracker_issue_style" type="radio" value="numeric" {{if eq $externalTrackerStyle "numeric"}}checked{{end}}>
+ <label>{{.i18n.Tr "repo.settings.tracker_issue_style.numeric"}} <span class="ui light grey text">#1234</span></label>
</div>
</div>
<div class="field">
<div class="ui radio checkbox">
- <input class="hidden" tabindex="0" name="tracker_issue_style" type="radio" value="alphanumeric" {{if $externalTrackerStyle}}{{if eq $externalTracker.ExternalTrackerConfig.ExternalTrackerStyle "alphanumeric"}}checked=""{{end}}{{end}} />
- <label>{{.i18n.Tr "repo.settings.tracker_issue_style.alphanumeric"}} <span class="ui light grey text">(ABC-123, DEFG-234)</span></label>
+ <input class="js-tracker-issue-style" name="tracker_issue_style" type="radio" value="alphanumeric" {{if eq $externalTrackerStyle "alphanumeric"}}checked{{end}}>
+ <label>{{.i18n.Tr "repo.settings.tracker_issue_style.alphanumeric"}} <span class="ui light grey text">ABC-123 , DEFG-234</span></label>
</div>
</div>
+ <div class="field">
+ <div class="ui radio checkbox">
+ <input class="js-tracker-issue-style" name="tracker_issue_style" type="radio" value="regexp" {{if eq $externalTrackerStyle "regexp"}}checked{{end}}>
+ <label>{{.i18n.Tr "repo.settings.tracker_issue_style.regexp"}} <span class="ui light grey text">(ISSUE-\d+) , ISSUE-(\d+)</span></label>
+ </div>
+ </div>
+ </div>
+ <div class="field {{if ne $externalTrackerStyle "regexp"}}disabled{{end}}" id="tracker-issue-style-regex-box">
+ <label for="external_tracker_regexp_pattern">{{.i18n.Tr "repo.settings.tracker_issue_style.regexp_pattern"}}</label>
+ <input id="external_tracker_regexp_pattern" name="external_tracker_regexp_pattern" value="{{(.Repository.MustGetUnit $.UnitTypeExternalTracker).ExternalTrackerConfig.ExternalTrackerRegexpPattern}}">
+ <p class="help">{{.i18n.Tr "repo.settings.tracker_issue_style.regexp_pattern_desc" | Str2html}}</p>
</div>
</div>
</div>
diff --git a/web_src/js/features/repo-legacy.js b/web_src/js/features/repo-legacy.js
index 53471b30cf..6cdde6a1e4 100644
--- a/web_src/js/features/repo-legacy.js
+++ b/web_src/js/features/repo-legacy.js
@@ -462,6 +462,11 @@ export function initRepository() {
if (typeof $(this).data('context') !== 'undefined') $($(this).data('context')).addClass('disabled');
}
});
+ const $trackerIssueStyleRadios = $('.js-tracker-issue-style');
+ $trackerIssueStyleRadios.on('change input', () => {
+ const checkedVal = $trackerIssueStyleRadios.filter(':checked').val();
+ $('#tracker-issue-style-regex-box').toggleClass('disabled', checkedVal !== 'regexp');
+ });
}
// Labels