Icycoide/RSCC-Forge
1
0
Fork 0
forked from kevadesu/forgejo
Code Pull requests Activity

Merge branch 'forgejo' into bugfix-arch

Browse source
This commit is contained in:
Exploding Dragon 2024-08-27 02:34:54 +00:00
commit bc4849a904
71 changed files with 1044 additions and 365 deletions
Download patch file Download diff file Expand all files Collapse all files

46
modules/git/batch.go Normal file
View file

@ -0,0 +1,46 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package git
import (
"bufio"
"context"
)
type Batch struct {
cancel context.CancelFunc
Reader *bufio.Reader
Writer WriteCloserError
}
func (repo *Repository) NewBatch(ctx context.Context) (*Batch, error) {
// Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first!
if err := ensureValidGitRepository(ctx, repo.Path); err != nil {
return nil, err
}
var batch Batch
batch.Writer, batch.Reader, batch.cancel = catFileBatch(ctx, repo.Path)
return &batch, nil
}
func (repo *Repository) NewBatchCheck(ctx context.Context) (*Batch, error) {
// Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first!
if err := ensureValidGitRepository(ctx, repo.Path); err != nil {
return nil, err
}
var check Batch
check.Writer, check.Reader, check.cancel = catFileBatchCheck(ctx, repo.Path)
return &check, nil
}
func (b *Batch) Close() {
if b.cancel != nil {
b.cancel()
b.Reader = nil
b.Writer = nil
b.cancel = nil
}
}

12
modules/git/batch_reader.go
View file

@ -26,10 +26,10 @@ type WriteCloserError interface {
CloseWithError(err error) error
}
// EnsureValidGitRepository runs git rev-parse in the repository path - thus ensuring that the repository is a valid repository.
// ensureValidGitRepository runs git rev-parse in the repository path - thus ensuring that the repository is a valid repository.
// Run before opening git cat-file.
// This is needed otherwise the git cat-file will hang for invalid repositories.
func EnsureValidGitRepository(ctx context.Context, repoPath string) error {
func ensureValidGitRepository(ctx context.Context, repoPath string) error {
stderr := strings.Builder{}
err := NewCommand(ctx, "rev-parse").
SetDescription(fmt.Sprintf("%s rev-parse [repo_path: %s]", GitExecutable, repoPath)).
@ -43,8 +43,8 @@ func EnsureValidGitRepository(ctx context.Context, repoPath string) error {
return nil
}
// CatFileBatchCheck opens git cat-file --batch-check in the provided repo and returns a stdin pipe, a stdout reader and cancel function
func CatFileBatchCheck(ctx context.Context, repoPath string) (WriteCloserError, *bufio.Reader, func()) {
// catFileBatchCheck opens git cat-file --batch-check in the provided repo and returns a stdin pipe, a stdout reader and cancel function
func catFileBatchCheck(ctx context.Context, repoPath string) (WriteCloserError, *bufio.Reader, func()) {
batchStdinReader, batchStdinWriter := io.Pipe()
batchStdoutReader, batchStdoutWriter := io.Pipe()
ctx, ctxCancel := context.WithCancel(ctx)
@ -93,8 +93,8 @@ func CatFileBatchCheck(ctx context.Context, repoPath string) (WriteCloserError,
return batchStdinWriter, batchReader, cancel
}
// CatFileBatch opens git cat-file --batch in the provided repo and returns a stdin pipe, a stdout reader and cancel function
func CatFileBatch(ctx context.Context, repoPath string) (WriteCloserError, *bufio.Reader, func()) {
// catFileBatch opens git cat-file --batch in the provided repo and returns a stdin pipe, a stdout reader and cancel function
func catFileBatch(ctx context.Context, repoPath string) (WriteCloserError, *bufio.Reader, func()) {
// We often want to feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary.
// so let's create a batch stdin and stdout
batchStdinReader, batchStdinWriter := io.Pipe()

15
modules/git/blob.go
View file

@ -28,9 +28,12 @@ type Blob struct {
// DataAsync gets a ReadCloser for the contents of a blob without reading it all.
// Calling the Close function on the result will discard all unread output.
func (b *Blob) DataAsync() (io.ReadCloser, error) {
wr, rd, cancel := b.repo.CatFileBatch(b.repo.Ctx)
wr, rd, cancel, err := b.repo.CatFileBatch(b.repo.Ctx)
if err != nil {
return nil, err
}
_, err := wr.Write([]byte(b.ID.String() + "\n"))
_, err = wr.Write([]byte(b.ID.String() + "\n"))
if err != nil {
cancel()
return nil, err
@ -66,9 +69,13 @@ func (b *Blob) Size() int64 {
return b.size
}
wr, rd, cancel := b.repo.CatFileBatchCheck(b.repo.Ctx)
wr, rd, cancel, err := b.repo.CatFileBatchCheck(b.repo.Ctx)
if err != nil {
log.Debug("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repo.Path, err)
return 0
}
defer cancel()
_, err := wr.Write([]byte(b.ID.String() + "\n"))
_, err = wr.Write([]byte(b.ID.String() + "\n"))
if err != nil {
log.Debug("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repo.Path, err)
return 0

5
modules/git/commit_info.go
View file

@ -129,7 +129,10 @@ func GetLastCommitForPaths(ctx context.Context, commit *Commit, treePath string,
return nil, err
}
batchStdinWriter, batchReader, cancel := commit.repo.CatFileBatch(ctx)
batchStdinWriter, batchReader, cancel, err := commit.repo.CatFileBatch(ctx)
if err != nil {
return nil, err
}
defer cancel()
commitsMap := map[string]*Commit{}

6
modules/git/git.go
View file

@ -38,6 +38,8 @@ var (
InvertedGitFlushEnv bool // 2.43.1
SupportCheckAttrOnBare bool // >= 2.40
HasSSHExecutable bool
gitVersion *version.Version
)
@ -203,6 +205,10 @@ func InitFull(ctx context.Context) (err error) {
globalCommandArgs = append(globalCommandArgs, "-c", "filter.lfs.required=", "-c", "filter.lfs.smudge=", "-c", "filter.lfs.clean=")
}
// Detect the presence of the ssh executable in $PATH.
_, err = exec.LookPath("ssh")
HasSSHExecutable = err == nil
return syncGitConfig()
}

5
modules/git/pipeline/lfs.go
View file

@ -67,7 +67,10 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err
// Next feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary.
// so let's create a batch stdin and stdout
batchStdinWriter, batchReader, cancel := repo.CatFileBatch(repo.Ctx)
batchStdinWriter, batchReader, cancel, err := repo.CatFileBatch(repo.Ctx)
if err != nil {
return nil, err
}
defer cancel()
// We'll use a scanner for the revList because it's simpler than a bufio.Reader

96
modules/git/repo_base.go
View file

@ -21,15 +21,11 @@ type Repository struct {
gpgSettings *GPGSettings
batchInUse bool
batchCancel context.CancelFunc
batchReader *bufio.Reader
batchWriter WriteCloserError
batchInUse bool
batch *Batch
checkInUse bool
checkCancel context.CancelFunc
checkReader *bufio.Reader
checkWriter WriteCloserError
checkInUse bool
check *Batch
Ctx context.Context
LastCommitCache *LastCommitCache
@ -51,63 +47,75 @@ func OpenRepository(ctx context.Context, repoPath string) (*Repository, error) {
return nil, errors.New("no such file or directory")
}
// Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first!
if err := EnsureValidGitRepository(ctx, repoPath); err != nil {
return nil, err
}
repo := &Repository{
return &Repository{
Path: repoPath,
tagCache: newObjectCache(),
Ctx: ctx,
}
repo.batchWriter, repo.batchReader, repo.batchCancel = CatFileBatch(ctx, repoPath)
repo.checkWriter, repo.checkReader, repo.checkCancel = CatFileBatchCheck(ctx, repoPath)
return repo, nil
}, nil
}
// CatFileBatch obtains a CatFileBatch for this repository
func (repo *Repository) CatFileBatch(ctx context.Context) (WriteCloserError, *bufio.Reader, func()) {
if repo.batchCancel == nil || repo.batchInUse {
log.Debug("Opening temporary cat file batch for: %s", repo.Path)
return CatFileBatch(ctx, repo.Path)
func (repo *Repository) CatFileBatch(ctx context.Context) (WriteCloserError, *bufio.Reader, func(), error) {
if repo.batch == nil {
var err error
repo.batch, err = repo.NewBatch(ctx)
if err != nil {
return nil, nil, nil, err
}
}
repo.batchInUse = true
return repo.batchWriter, repo.batchReader, func() {
repo.batchInUse = false
if !repo.batchInUse {
repo.batchInUse = true
return repo.batch.Writer, repo.batch.Reader, func() {
repo.batchInUse = false
}, nil
}
log.Debug("Opening temporary cat file batch for: %s", repo.Path)
tempBatch, err := repo.NewBatch(ctx)
if err != nil {
return nil, nil, nil, err
}
return tempBatch.Writer, tempBatch.Reader, tempBatch.Close, nil
}
// CatFileBatchCheck obtains a CatFileBatchCheck for this repository
func (repo *Repository) CatFileBatchCheck(ctx context.Context) (WriteCloserError, *bufio.Reader, func()) {
if repo.checkCancel == nil || repo.checkInUse {
log.Debug("Opening temporary cat file batch-check for: %s", repo.Path)
return CatFileBatchCheck(ctx, repo.Path)
func (repo *Repository) CatFileBatchCheck(ctx context.Context) (WriteCloserError, *bufio.Reader, func(), error) {
if repo.check == nil {
var err error
repo.check, err = repo.NewBatchCheck(ctx)
if err != nil {
return nil, nil, nil, err
}
}
repo.checkInUse = true
return repo.checkWriter, repo.checkReader, func() {
repo.checkInUse = false
if !repo.checkInUse {
repo.checkInUse = true
return repo.check.Writer, repo.check.Reader, func() {
repo.checkInUse = false
}, nil
}
log.Debug("Opening temporary cat file batch-check for: %s", repo.Path)
tempBatchCheck, err := repo.NewBatchCheck(ctx)
if err != nil {
return nil, nil, nil, err
}
return tempBatchCheck.Writer, tempBatchCheck.Reader, tempBatchCheck.Close, nil
}
func (repo *Repository) Close() error {
if repo == nil {
return nil
}
if repo.batchCancel != nil {
repo.batchCancel()
repo.batchReader = nil
repo.batchWriter = nil
repo.batchCancel = nil
if repo.batch != nil {
repo.batch.Close()
repo.batch = nil
repo.batchInUse = false
}
if repo.checkCancel != nil {
repo.checkCancel()
repo.checkCancel = nil
repo.checkReader = nil
repo.checkWriter = nil
if repo.check != nil {
repo.check.Close()
repo.check = nil
repo.checkInUse = false
}
repo.LastCommitCache = nil

163
modules/git/repo_base_test.go Normal file
View file

@ -0,0 +1,163 @@
// Copyright 2024 The Forgejo Authors. All rights reserved.
// SPDX-License-Identifier: GPL-3.0-or-later
package git
import (
"bufio"
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// This unit test relies on the implementation detail of CatFileBatch.
func TestCatFileBatch(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
repo, err := OpenRepository(ctx, "./tests/repos/repo1_bare")
require.NoError(t, err)
defer repo.Close()
var wr WriteCloserError
var r *bufio.Reader
var cancel1 func()
t.Run("Request cat file batch", func(t *testing.T) {
assert.Nil(t, repo.batch)
wr, r, cancel1, err = repo.CatFileBatch(ctx)
require.NoError(t, err)
assert.NotNil(t, repo.batch)
assert.Equal(t, repo.batch.Writer, wr)
assert.True(t, repo.batchInUse)
})
t.Run("Request temporary cat file batch", func(t *testing.T) {
wr, r, cancel, err := repo.CatFileBatch(ctx)
require.NoError(t, err)
assert.NotEqual(t, repo.batch.Writer, wr)
t.Run("Check temporary cat file batch", func(t *testing.T) {
_, err = wr.Write([]byte("95bb4d39648ee7e325106df01a621c530863a653" + "\n"))
require.NoError(t, err)
sha, typ, size, err := ReadBatchLine(r)
require.NoError(t, err)
assert.Equal(t, "commit", typ)
assert.EqualValues(t, []byte("95bb4d39648ee7e325106df01a621c530863a653"), sha)
assert.EqualValues(t, 144, size)
})
cancel()
assert.True(t, repo.batchInUse)
})
t.Run("Check cached cat file batch", func(t *testing.T) {
_, err = wr.Write([]byte("95bb4d39648ee7e325106df01a621c530863a653" + "\n"))
require.NoError(t, err)
sha, typ, size, err := ReadBatchLine(r)
require.NoError(t, err)
assert.Equal(t, "commit", typ)
assert.EqualValues(t, []byte("95bb4d39648ee7e325106df01a621c530863a653"), sha)
assert.EqualValues(t, 144, size)
})
t.Run("Cancel cached cat file batch", func(t *testing.T) {
cancel1()
assert.False(t, repo.batchInUse)
assert.NotNil(t, repo.batch)
})
t.Run("Request cached cat file batch", func(t *testing.T) {
wr, _, _, err := repo.CatFileBatch(ctx)
require.NoError(t, err)
assert.NotNil(t, repo.batch)
assert.Equal(t, repo.batch.Writer, wr)
assert.True(t, repo.batchInUse)
t.Run("Close git repo", func(t *testing.T) {
require.NoError(t, repo.Close())
assert.Nil(t, repo.batch)
})
_, err = wr.Write([]byte("95bb4d39648ee7e325106df01a621c530863a653" + "\n"))
require.Error(t, err)
})
}
// This unit test relies on the implementation detail of CatFileBatchCheck.
func TestCatFileBatchCheck(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
repo, err := OpenRepository(ctx, "./tests/repos/repo1_bare")
require.NoError(t, err)
defer repo.Close()
var wr WriteCloserError
var r *bufio.Reader
var cancel1 func()
t.Run("Request cat file batch check", func(t *testing.T) {
assert.Nil(t, repo.check)
wr, r, cancel1, err = repo.CatFileBatchCheck(ctx)
require.NoError(t, err)
assert.NotNil(t, repo.check)
assert.Equal(t, repo.check.Writer, wr)
assert.True(t, repo.checkInUse)
})
t.Run("Request temporary cat file batch check", func(t *testing.T) {
wr, r, cancel, err := repo.CatFileBatchCheck(ctx)
require.NoError(t, err)
assert.NotEqual(t, repo.check.Writer, wr)
t.Run("Check temporary cat file batch check", func(t *testing.T) {
_, err = wr.Write([]byte("test" + "\n"))
require.NoError(t, err)
sha, typ, size, err := ReadBatchLine(r)
require.NoError(t, err)
assert.Equal(t, "tag", typ)
assert.EqualValues(t, []byte("3ad28a9149a2864384548f3d17ed7f38014c9e8a"), sha)
assert.EqualValues(t, 807, size)
})
cancel()
assert.True(t, repo.checkInUse)
})
t.Run("Check cached cat file batch check", func(t *testing.T) {
_, err = wr.Write([]byte("test" + "\n"))
require.NoError(t, err)
sha, typ, size, err := ReadBatchLine(r)
require.NoError(t, err)
assert.Equal(t, "tag", typ)
assert.EqualValues(t, []byte("3ad28a9149a2864384548f3d17ed7f38014c9e8a"), sha)
assert.EqualValues(t, 807, size)
})
t.Run("Cancel cached cat file batch check", func(t *testing.T) {
cancel1()
assert.False(t, repo.checkInUse)
assert.NotNil(t, repo.check)
})
t.Run("Request cached cat file batch check", func(t *testing.T) {
wr, _, _, err := repo.CatFileBatchCheck(ctx)
require.NoError(t, err)
assert.NotNil(t, repo.check)
assert.Equal(t, repo.check.Writer, wr)
assert.True(t, repo.checkInUse)
t.Run("Close git repo", func(t *testing.T) {
require.NoError(t, repo.Close())
assert.Nil(t, repo.check)
})
_, err = wr.Write([]byte("test" + "\n"))
require.Error(t, err)
})
}

16
modules/git/repo_branch.go
View file

@ -169,9 +169,13 @@ func (repo *Repository) IsObjectExist(name string) bool {
return false
}
wr, rd, cancel := repo.CatFileBatchCheck(repo.Ctx)
wr, rd, cancel, err := repo.CatFileBatchCheck(repo.Ctx)
if err != nil {
log.Debug("Error writing to CatFileBatchCheck %v", err)
return false
}
defer cancel()
_, err := wr.Write([]byte(name + "\n"))
_, err = wr.Write([]byte(name + "\n"))
if err != nil {
log.Debug("Error writing to CatFileBatchCheck %v", err)
return false
@ -186,9 +190,13 @@ func (repo *Repository) IsReferenceExist(name string) bool {
return false
}
wr, rd, cancel := repo.CatFileBatchCheck(repo.Ctx)
wr, rd, cancel, err := repo.CatFileBatchCheck(repo.Ctx)
if err != nil {
log.Debug("Error writing to CatFileBatchCheck %v", err)
return false
}
defer cancel()
_, err := wr.Write([]byte(name + "\n"))
_, err = wr.Write([]byte(name + "\n"))
if err != nil {
log.Debug("Error writing to CatFileBatchCheck %v", err)
return false

21
modules/git/repo_commit.go
View file

@ -536,9 +536,12 @@ func (repo *Repository) ResolveReference(name string) (string, error) {
// GetRefCommitID returns the last commit ID string of given reference (branch or tag).
func (repo *Repository) GetRefCommitID(name string) (string, error) {
wr, rd, cancel := repo.CatFileBatchCheck(repo.Ctx)
wr, rd, cancel, err := repo.CatFileBatchCheck(repo.Ctx)
if err != nil {
return "", err
}
defer cancel()
_, err := wr.Write([]byte(name + "\n"))
_, err = wr.Write([]byte(name + "\n"))
if err != nil {
return "", err
}
@ -564,12 +567,19 @@ func (repo *Repository) RemoveReference(name string) error {
// IsCommitExist returns true if given commit exists in current repository.
func (repo *Repository) IsCommitExist(name string) bool {
if err := ensureValidGitRepository(repo.Ctx, repo.Path); err != nil {
log.Error("IsCommitExist: %v", err)
return false
}
_, _, err := NewCommand(repo.Ctx, "cat-file", "-e").AddDynamicArguments(name).RunStdString(&RunOpts{Dir: repo.Path})
return err == nil
}
func (repo *Repository) getCommit(id ObjectID) (*Commit, error) {
wr, rd, cancel := repo.CatFileBatch(repo.Ctx)
wr, rd, cancel, err := repo.CatFileBatch(repo.Ctx)
if err != nil {
return nil, err
}
defer cancel()
_, _ = wr.Write([]byte(id.String() + "\n"))
@ -646,7 +656,10 @@ func (repo *Repository) ConvertToGitID(commitID string) (ObjectID, error) {
}
}
wr, rd, cancel := repo.CatFileBatchCheck(repo.Ctx)
wr, rd, cancel, err := repo.CatFileBatchCheck(repo.Ctx)
if err != nil {
return nil, err
}
defer cancel()
_, err = wr.Write([]byte(commitID + "\n"))
if err != nil {

5
modules/git/repo_language_stats.go
View file

@ -60,7 +60,10 @@ func mergeLanguageStats(stats map[string]int64) map[string]int64 {
func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) {
// We will feed the commit IDs in order into cat-file --batch, followed by blobs as necessary.
// so let's create a batch stdin and stdout
batchStdinWriter, batchReader, cancel := repo.CatFileBatch(repo.Ctx)
batchStdinWriter, batchReader, cancel, err := repo.CatFileBatch(repo.Ctx)
if err != nil {
return nil, err
}
defer cancel()
writeID := func(id string) error {

12
modules/git/repo_tag.go
View file

@ -257,9 +257,12 @@ func (repo *Repository) GetTags(skip, limit int) (tags []string, err error) {
// GetTagType gets the type of the tag, either commit (simple) or tag (annotated)
func (repo *Repository) GetTagType(id ObjectID) (string, error) {
wr, rd, cancel := repo.CatFileBatchCheck(repo.Ctx)
wr, rd, cancel, err := repo.CatFileBatchCheck(repo.Ctx)
if err != nil {
return "", err
}
defer cancel()
_, err := wr.Write([]byte(id.String() + "\n"))
_, err = wr.Write([]byte(id.String() + "\n"))
if err != nil {
return "", err
}
@ -315,7 +318,10 @@ func (repo *Repository) getTag(tagID ObjectID, name string) (*Tag, error) {
}
// The tag is an annotated tag with a message.
wr, rd, cancel := repo.CatFileBatch(repo.Ctx)
wr, rd, cancel, err := repo.CatFileBatch(repo.Ctx)
if err != nil {
return nil, err
}
defer cancel()
if _, err := wr.Write([]byte(tagID.String() + "\n")); err != nil {

5
modules/git/repo_tree.go
View file

@ -68,7 +68,10 @@ func (repo *Repository) CommitTree(author, committer *Signature, tree *Tree, opt
}
func (repo *Repository) getTree(id ObjectID) (*Tree, error) {
wr, rd, cancel := repo.CatFileBatch(repo.Ctx)
wr, rd, cancel, err := repo.CatFileBatch(repo.Ctx)
if err != nil {
return nil, err
}
defer cancel()
_, _ = wr.Write([]byte(id.String() + "\n"))

5
modules/git/tree.go
View file

@ -41,7 +41,10 @@ func (t *Tree) ListEntries() (Entries, error) {
}
if t.repo != nil {
wr, rd, cancel := t.repo.CatFileBatch(t.repo.Ctx)
wr, rd, cancel, err := t.repo.CatFileBatch(t.repo.Ctx)
if err != nil {
return nil, err
}
defer cancel()
_, _ = wr.Write([]byte(t.ID.String() + "\n"))

8
modules/git/tree_entry.go
View file

@ -47,9 +47,13 @@ func (te *TreeEntry) Size() int64 {
return te.size
}
wr, rd, cancel := te.ptree.repo.CatFileBatchCheck(te.ptree.repo.Ctx)
wr, rd, cancel, err := te.ptree.repo.CatFileBatchCheck(te.ptree.repo.Ctx)
if err != nil {
log.Debug("error whilst reading size for %s in %s. Error: %v", te.ID.String(), te.ptree.repo.Path, err)
return 0
}
defer cancel()
_, err := wr.Write([]byte(te.ID.String() + "\n"))
_, err = wr.Write([]byte(te.ID.String() + "\n"))
if err != nil {
log.Debug("error whilst reading size for %s in %s. Error: %v", te.ID.String(), te.ptree.repo.Path, err)
return 0

20
modules/indexer/code/bleve/bleve.go
View file

@ -16,10 +16,10 @@ import (
"code.gitea.io/gitea/modules/analyze"
"code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/indexer/code/internal"
indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
inner_bleve "code.gitea.io/gitea/modules/indexer/internal/bleve"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/typesniffer"
@ -193,21 +193,23 @@ func (b *Indexer) addDelete(filename string, repo *repo_model.Repository, batch
func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error {
batch := inner_bleve.NewFlushingBatch(b.inner.Indexer, maxBatchSize)
if len(changes.Updates) > 0 {
// Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first!
if err := git.EnsureValidGitRepository(ctx, repo.RepoPath()); err != nil {
log.Error("Unable to open git repo: %s for %-v: %v", repo.RepoPath(), repo, err)
r, err := gitrepo.OpenRepository(ctx, repo)
if err != nil {
return err
}
batchWriter, batchReader, cancel := git.CatFileBatch(ctx, repo.RepoPath())
defer cancel()
defer r.Close()
gitBatch, err := r.NewBatch(ctx)
if err != nil {
return err
}
defer gitBatch.Close()
for _, update := range changes.Updates {
if err := b.addUpdate(ctx, batchWriter, batchReader, sha, update, repo, batch); err != nil {
if err := b.addUpdate(ctx, gitBatch.Writer, gitBatch.Reader, sha, update, repo, batch); err != nil {
return err
}
}
cancel()
gitBatch.Close()
}
for _, filename := range changes.RemovedFilenames {
if err := b.addDelete(filename, repo, batch); err != nil {

20
modules/indexer/code/elasticsearch/elasticsearch.go
View file

@ -15,11 +15,11 @@ import (
"code.gitea.io/gitea/modules/analyze"
"code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/indexer/code/internal"
indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
inner_elasticsearch "code.gitea.io/gitea/modules/indexer/internal/elasticsearch"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/typesniffer"
@ -154,17 +154,19 @@ func (b *Indexer) addDelete(filename string, repo *repo_model.Repository) elasti
func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error {
reqs := make([]elastic.BulkableRequest, 0)
if len(changes.Updates) > 0 {
// Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first!
if err := git.EnsureValidGitRepository(ctx, repo.RepoPath()); err != nil {
log.Error("Unable to open git repo: %s for %-v: %v", repo.RepoPath(), repo, err)
r, err := gitrepo.OpenRepository(ctx, repo)
if err != nil {
return err
}
batchWriter, batchReader, cancel := git.CatFileBatch(ctx, repo.RepoPath())
defer cancel()
defer r.Close()
batch, err := r.NewBatch(ctx)
if err != nil {
return err
}
defer batch.Close()
for _, update := range changes.Updates {
updateReqs, err := b.addUpdate(ctx, batchWriter, batchReader, sha, update, repo)
updateReqs, err := b.addUpdate(ctx, batch.Writer, batch.Reader, sha, update, repo)
if err != nil {
return err
}
@ -172,7 +174,7 @@ func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha st
reqs = append(reqs, updateReqs...)
}
}
cancel()
batch.Close()
}
for _, filename := range changes.RemovedFilenames {

261
modules/queue/mock/redisuniversalclient.go
View file

@ -2540,6 +2540,101 @@ func (mr *MockUniversalClientMockRecorder) HExists(arg0, arg1, arg2 any) *gomock
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HExists", reflect.TypeOf((*MockUniversalClient)(nil).HExists), arg0, arg1, arg2)
}
// HExpire mocks base method.
func (m *MockUniversalClient) HExpire(arg0 context.Context, arg1 string, arg2 time.Duration, arg3 ...string) *redis.IntSliceCmd {
m.ctrl.T.Helper()
varargs := []any{arg0, arg1, arg2}
for _, a := range arg3 {
varargs = append(varargs, a)
}
ret := m.ctrl.Call(m, "HExpire", varargs...)
ret0, _ := ret[0].(*redis.IntSliceCmd)
return ret0
}
// HExpire indicates an expected call of HExpire.
func (mr *MockUniversalClientMockRecorder) HExpire(arg0, arg1, arg2 any, arg3 ...any) *gomock.Call {
mr.mock.ctrl.T.Helper()
varargs := append([]any{arg0, arg1, arg2}, arg3...)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HExpire", reflect.TypeOf((*MockUniversalClient)(nil).HExpire), varargs...)
}
// HExpireAt mocks base method.
func (m *MockUniversalClient) HExpireAt(arg0 context.Context, arg1 string, arg2 time.Time, arg3 ...string) *redis.IntSliceCmd {
m.ctrl.T.Helper()
varargs := []any{arg0, arg1, arg2}
for _, a := range arg3 {
varargs = append(varargs, a)
}
ret := m.ctrl.Call(m, "HExpireAt", varargs...)
ret0, _ := ret[0].(*redis.IntSliceCmd)
return ret0
}
// HExpireAt indicates an expected call of HExpireAt.
func (mr *MockUniversalClientMockRecorder) HExpireAt(arg0, arg1, arg2 any, arg3 ...any) *gomock.Call {
mr.mock.ctrl.T.Helper()
varargs := append([]any{arg0, arg1, arg2}, arg3...)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HExpireAt", reflect.TypeOf((*MockUniversalClient)(nil).HExpireAt), varargs...)
}
// HExpireAtWithArgs mocks base method.
func (m *MockUniversalClient) HExpireAtWithArgs(arg0 context.Context, arg1 string, arg2 time.Time, arg3 redis.HExpireArgs, arg4 ...string) *redis.IntSliceCmd {
m.ctrl.T.Helper()
varargs := []any{arg0, arg1, arg2, arg3}
for _, a := range arg4 {
varargs = append(varargs, a)
}
ret := m.ctrl.Call(m, "HExpireAtWithArgs", varargs...)
ret0, _ := ret[0].(*redis.IntSliceCmd)
return ret0
}
// HExpireAtWithArgs indicates an expected call of HExpireAtWithArgs.
func (mr *MockUniversalClientMockRecorder) HExpireAtWithArgs(arg0, arg1, arg2, arg3 any, arg4 ...any) *gomock.Call {
mr.mock.ctrl.T.Helper()
varargs := append([]any{arg0, arg1, arg2, arg3}, arg4...)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HExpireAtWithArgs", reflect.TypeOf((*MockUniversalClient)(nil).HExpireAtWithArgs), varargs...)
}
// HExpireTime mocks base method.
func (m *MockUniversalClient) HExpireTime(arg0 context.Context, arg1 string, arg2 ...string) *redis.IntSliceCmd {
m.ctrl.T.Helper()
varargs := []any{arg0, arg1}
for _, a := range arg2 {
varargs = append(varargs, a)
}
ret := m.ctrl.Call(m, "HExpireTime", varargs...)
ret0, _ := ret[0].(*redis.IntSliceCmd)
return ret0
}
// HExpireTime indicates an expected call of HExpireTime.
func (mr *MockUniversalClientMockRecorder) HExpireTime(arg0, arg1 any, arg2 ...any) *gomock.Call {
mr.mock.ctrl.T.Helper()
varargs := append([]any{arg0, arg1}, arg2...)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HExpireTime", reflect.TypeOf((*MockUniversalClient)(nil).HExpireTime), varargs...)
}
// HExpireWithArgs mocks base method.
func (m *MockUniversalClient) HExpireWithArgs(arg0 context.Context, arg1 string, arg2 time.Duration, arg3 redis.HExpireArgs, arg4 ...string) *redis.IntSliceCmd {
m.ctrl.T.Helper()
varargs := []any{arg0, arg1, arg2, arg3}
for _, a := range arg4 {
varargs = append(varargs, a)
}
ret := m.ctrl.Call(m, "HExpireWithArgs", varargs...)
ret0, _ := ret[0].(*redis.IntSliceCmd)
return ret0
}
// HExpireWithArgs indicates an expected call of HExpireWithArgs.
func (mr *MockUniversalClientMockRecorder) HExpireWithArgs(arg0, arg1, arg2, arg3 any, arg4 ...any) *gomock.Call {
mr.mock.ctrl.T.Helper()
varargs := append([]any{arg0, arg1, arg2, arg3}, arg4...)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HExpireWithArgs", reflect.TypeOf((*MockUniversalClient)(nil).HExpireWithArgs), varargs...)
}
// HGet mocks base method.
func (m *MockUniversalClient) HGet(arg0 context.Context, arg1, arg2 string) *redis.StringCmd {
m.ctrl.T.Helper()
@ -2662,6 +2757,139 @@ func (mr *MockUniversalClientMockRecorder) HMSet(arg0, arg1 any, arg2 ...any) *g
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HMSet", reflect.TypeOf((*MockUniversalClient)(nil).HMSet), varargs...)
}
// HPExpire mocks base method.
func (m *MockUniversalClient) HPExpire(arg0 context.Context, arg1 string, arg2 time.Duration, arg3 ...string) *redis.IntSliceCmd {
m.ctrl.T.Helper()
varargs := []any{arg0, arg1, arg2}
for _, a := range arg3 {
varargs = append(varargs, a)
}
ret := m.ctrl.Call(m, "HPExpire", varargs...)
ret0, _ := ret[0].(*redis.IntSliceCmd)
return ret0
}
// HPExpire indicates an expected call of HPExpire.
func (mr *MockUniversalClientMockRecorder) HPExpire(arg0, arg1, arg2 any, arg3 ...any) *gomock.Call {
mr.mock.ctrl.T.Helper()
varargs := append([]any{arg0, arg1, arg2}, arg3...)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HPExpire", reflect.TypeOf((*MockUniversalClient)(nil).HPExpire), varargs...)
}
// HPExpireAt mocks base method.
func (m *MockUniversalClient) HPExpireAt(arg0 context.Context, arg1 string, arg2 time.Time, arg3 ...string) *redis.IntSliceCmd {
m.ctrl.T.Helper()
varargs := []any{arg0, arg1, arg2}
for _, a := range arg3 {
varargs = append(varargs, a)
}
ret := m.ctrl.Call(m, "HPExpireAt", varargs...)
ret0, _ := ret[0].(*redis.IntSliceCmd)
return ret0
}
// HPExpireAt indicates an expected call of HPExpireAt.
func (mr *MockUniversalClientMockRecorder) HPExpireAt(arg0, arg1, arg2 any, arg3 ...any) *gomock.Call {
mr.mock.ctrl.T.Helper()
varargs := append([]any{arg0, arg1, arg2}, arg3...)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HPExpireAt", reflect.TypeOf((*MockUniversalClient)(nil).HPExpireAt), varargs...)
}
// HPExpireAtWithArgs mocks base method.
func (m *MockUniversalClient) HPExpireAtWithArgs(arg0 context.Context, arg1 string, arg2 time.Time, arg3 redis.HExpireArgs, arg4 ...string) *redis.IntSliceCmd {
m.ctrl.T.Helper()
varargs := []any{arg0, arg1, arg2, arg3}
for _, a := range arg4 {
varargs = append(varargs, a)
}
ret := m.ctrl.Call(m, "HPExpireAtWithArgs", varargs...)
ret0, _ := ret[0].(*redis.IntSliceCmd)
return ret0
}
// HPExpireAtWithArgs indicates an expected call of HPExpireAtWithArgs.
func (mr *MockUniversalClientMockRecorder) HPExpireAtWithArgs(arg0, arg1, arg2, arg3 any, arg4 ...any) *gomock.Call {
mr.mock.ctrl.T.Helper()
varargs := append([]any{arg0, arg1, arg2, arg3}, arg4...)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HPExpireAtWithArgs", reflect.TypeOf((*MockUniversalClient)(nil).HPExpireAtWithArgs), varargs...)
}
// HPExpireTime mocks base method.
func (m *MockUniversalClient) HPExpireTime(arg0 context.Context, arg1 string, arg2 ...string) *redis.IntSliceCmd {
m.ctrl.T.Helper()
varargs := []any{arg0, arg1}
for _, a := range arg2 {
varargs = append(varargs, a)
}
ret := m.ctrl.Call(m, "HPExpireTime", varargs...)
ret0, _ := ret[0].(*redis.IntSliceCmd)
return ret0
}
// HPExpireTime indicates an expected call of HPExpireTime.
func (mr *MockUniversalClientMockRecorder) HPExpireTime(arg0, arg1 any, arg2 ...any) *gomock.Call {
mr.mock.ctrl.T.Helper()
varargs := append([]any{arg0, arg1}, arg2...)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HPExpireTime", reflect.TypeOf((*MockUniversalClient)(nil).HPExpireTime), varargs...)
}
// HPExpireWithArgs mocks base method.
func (m *MockUniversalClient) HPExpireWithArgs(arg0 context.Context, arg1 string, arg2 time.Duration, arg3 redis.HExpireArgs, arg4 ...string) *redis.IntSliceCmd {
m.ctrl.T.Helper()
varargs := []any{arg0, arg1, arg2, arg3}
for _, a := range arg4 {
varargs = append(varargs, a)
}
ret := m.ctrl.Call(m, "HPExpireWithArgs", varargs...)
ret0, _ := ret[0].(*redis.IntSliceCmd)
return ret0
}
// HPExpireWithArgs indicates an expected call of HPExpireWithArgs.
func (mr *MockUniversalClientMockRecorder) HPExpireWithArgs(arg0, arg1, arg2, arg3 any, arg4 ...any) *gomock.Call {
mr.mock.ctrl.T.Helper()
varargs := append([]any{arg0, arg1, arg2, arg3}, arg4...)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HPExpireWithArgs", reflect.TypeOf((*MockUniversalClient)(nil).HPExpireWithArgs), varargs...)
}
// HPTTL mocks base method.
func (m *MockUniversalClient) HPTTL(arg0 context.Context, arg1 string, arg2 ...string) *redis.IntSliceCmd {
m.ctrl.T.Helper()
varargs := []any{arg0, arg1}
for _, a := range arg2 {
varargs = append(varargs, a)
}
ret := m.ctrl.Call(m, "HPTTL", varargs...)
ret0, _ := ret[0].(*redis.IntSliceCmd)
return ret0
}
// HPTTL indicates an expected call of HPTTL.
func (mr *MockUniversalClientMockRecorder) HPTTL(arg0, arg1 any, arg2 ...any) *gomock.Call {
mr.mock.ctrl.T.Helper()
varargs := append([]any{arg0, arg1}, arg2...)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HPTTL", reflect.TypeOf((*MockUniversalClient)(nil).HPTTL), varargs...)
}
// HPersist mocks base method.
func (m *MockUniversalClient) HPersist(arg0 context.Context, arg1 string, arg2 ...string) *redis.IntSliceCmd {
m.ctrl.T.Helper()
varargs := []any{arg0, arg1}
for _, a := range arg2 {
varargs = append(varargs, a)
}
ret := m.ctrl.Call(m, "HPersist", varargs...)
ret0, _ := ret[0].(*redis.IntSliceCmd)
return ret0
}
// HPersist indicates an expected call of HPersist.
func (mr *MockUniversalClientMockRecorder) HPersist(arg0, arg1 any, arg2 ...any) *gomock.Call {
mr.mock.ctrl.T.Helper()
varargs := append([]any{arg0, arg1}, arg2...)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HPersist", reflect.TypeOf((*MockUniversalClient)(nil).HPersist), varargs...)
}
// HRandField mocks base method.
func (m *MockUniversalClient) HRandField(arg0 context.Context, arg1 string, arg2 int) *redis.StringSliceCmd {
m.ctrl.T.Helper()
@ -2704,6 +2932,20 @@ func (mr *MockUniversalClientMockRecorder) HScan(arg0, arg1, arg2, arg3, arg4 an
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HScan", reflect.TypeOf((*MockUniversalClient)(nil).HScan), arg0, arg1, arg2, arg3, arg4)
}
// HScanNoValues mocks base method.
func (m *MockUniversalClient) HScanNoValues(arg0 context.Context, arg1 string, arg2 uint64, arg3 string, arg4 int64) *redis.ScanCmd {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "HScanNoValues", arg0, arg1, arg2, arg3, arg4)
ret0, _ := ret[0].(*redis.ScanCmd)
return ret0
}
// HScanNoValues indicates an expected call of HScanNoValues.
func (mr *MockUniversalClientMockRecorder) HScanNoValues(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HScanNoValues", reflect.TypeOf((*MockUniversalClient)(nil).HScanNoValues), arg0, arg1, arg2, arg3, arg4)
}
// HSet mocks base method.
func (m *MockUniversalClient) HSet(arg0 context.Context, arg1 string, arg2 ...any) *redis.IntCmd {
m.ctrl.T.Helper()
@ -2737,6 +2979,25 @@ func (mr *MockUniversalClientMockRecorder) HSetNX(arg0, arg1, arg2, arg3 any) *g
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HSetNX", reflect.TypeOf((*MockUniversalClient)(nil).HSetNX), arg0, arg1, arg2, arg3)
}
// HTTL mocks base method.
func (m *MockUniversalClient) HTTL(arg0 context.Context, arg1 string, arg2 ...string) *redis.IntSliceCmd {
m.ctrl.T.Helper()
varargs := []any{arg0, arg1}
for _, a := range arg2 {
varargs = append(varargs, a)
}
ret := m.ctrl.Call(m, "HTTL", varargs...)
ret0, _ := ret[0].(*redis.IntSliceCmd)
return ret0
}
// HTTL indicates an expected call of HTTL.
func (mr *MockUniversalClientMockRecorder) HTTL(arg0, arg1 any, arg2 ...any) *gomock.Call {
mr.mock.ctrl.T.Helper()
varargs := append([]any{arg0, arg1}, arg2...)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HTTL", reflect.TypeOf((*MockUniversalClient)(nil).HTTL), varargs...)
}
// HVals mocks base method.
func (m *MockUniversalClient) HVals(arg0 context.Context, arg1 string) *redis.StringSliceCmd {
m.ctrl.T.Helper()

2
modules/session/db.go
View file

@ -11,7 +11,7 @@ import (
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/modules/timeutil"
"gitea.com/go-chi/session"
"code.forgejo.org/go-chi/session"
)
// DBStore represents a session store implementation based on the DB.

2
modules/session/redis.go
View file

@ -25,7 +25,7 @@ import (
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/nosql"
"gitea.com/go-chi/session"
"code.forgejo.org/go-chi/session"
"github.com/redis/go-redis/v9"
)

2
modules/session/store.go
View file

@ -6,7 +6,7 @@ package session
import (
"net/http"
"gitea.com/go-chi/session"
"code.forgejo.org/go-chi/session"
)
// Store represents a session store

15
modules/session/virtual.go
View file

@ -8,12 +8,12 @@ import (
"sync"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
"gitea.com/go-chi/session"
couchbase "gitea.com/go-chi/session/couchbase"
memcache "gitea.com/go-chi/session/memcache"
mysql "gitea.com/go-chi/session/mysql"
postgres "gitea.com/go-chi/session/postgres"
"code.forgejo.org/go-chi/session"
memcache "code.forgejo.org/go-chi/session/memcache"
mysql "code.forgejo.org/go-chi/session/mysql"
postgres "code.forgejo.org/go-chi/session/postgres"
)
// VirtualSessionProvider represents a shadowed session provider implementation.
@ -35,6 +35,9 @@ func (o *VirtualSessionProvider) Init(gclifetime int64, config string) error {
switch opts.Provider {
case "memory":
o.provider = &session.MemProvider{}
case "couchbase":
log.Warn("Couchbase as session provider is no longer supported, falling back to file as session provider")
fallthrough
case "file":
o.provider = &session.FileProvider{}
case "redis":
@ -45,8 +48,6 @@ func (o *VirtualSessionProvider) Init(gclifetime int64, config string) error {
o.provider = &mysql.MysqlProvider{}
case "postgres":
o.provider = &postgres.PostgresProvider{}
case "couchbase":
o.provider = &couchbase.CouchbaseProvider{}
case "memcache":
o.provider = &memcache.MemcacheProvider{}
default:

9
modules/setting/quota.go
View file

@ -7,9 +7,18 @@ package setting
var Quota = struct {
Enabled bool `ini:"ENABLED"`
DefaultGroups []string `ini:"DEFAULT_GROUPS"`
Default struct {
Total int64
} `ini:"quota.default"`
}{
Enabled: false,
DefaultGroups: []string{},
Default: struct {
Total int64
}{
Total: -1,
},
}
func loadQuotaFrom(rootCfg ConfigProvider) {