diff --git a/cmd/admin.go b/cmd/admin.go index 1e41a0af6c..cd083a29e8 100644 --- a/cmd/admin.go +++ b/cmd/admin.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" pwd "code.gitea.io/gitea/modules/password" + "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "github.com/urfave/cli" @@ -373,7 +374,7 @@ func runRepoSyncReleases(c *cli.Context) error { } log.Trace(" currentNumReleases is %d, running SyncReleasesWithTags", oldnum) - if err = models.SyncReleasesWithTags(repo, gitRepo); err != nil { + if err = repository.SyncReleasesWithTags(repo, gitRepo); err != nil { log.Warn(" SyncReleasesWithTags: %v", err) gitRepo.Close() continue diff --git a/models/migrations/v39.go b/models/migrations/v39.go index dc5f6ee091..e0b84b969c 100644 --- a/models/migrations/v39.go +++ b/models/migrations/v39.go @@ -10,6 +10,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/repository" "xorm.io/xorm" ) @@ -44,7 +45,7 @@ func releaseAddColumnIsTagAndSyncTags(x *xorm.Engine) error { continue } - if err = models.SyncReleasesWithTags(repo, gitRepo); err != nil { + if err = repository.SyncReleasesWithTags(repo, gitRepo); err != nil { log.Warn("SyncReleasesWithTags: %v", err) } gitRepo.Close() diff --git a/models/release.go b/models/release.go index a0f0621ab0..46f9e88752 100644 --- a/models/release.go +++ b/models/release.go @@ -10,7 +10,6 @@ import ( "sort" "strings" - "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" api "code.gitea.io/gitea/modules/structs" @@ -318,49 +317,6 @@ func DeleteReleaseByID(id int64) error { return err } -// SyncReleasesWithTags synchronizes release table with repository tags -func SyncReleasesWithTags(repo *Repository, gitRepo *git.Repository) error { - existingRelTags := make(map[string]struct{}) - opts := FindReleasesOptions{IncludeDrafts: true, IncludeTags: true} - for page := 1; ; page++ { - rels, err := GetReleasesByRepoID(repo.ID, opts, page, 100) - if err != nil { - return fmt.Errorf("GetReleasesByRepoID: %v", err) - } - if len(rels) == 0 { - break - } - for _, rel := range rels { - if rel.IsDraft { - continue - } - commitID, err := gitRepo.GetTagCommitID(rel.TagName) - if err != nil && !git.IsErrNotExist(err) { - return fmt.Errorf("GetTagCommitID: %v", err) - } - if git.IsErrNotExist(err) || commitID != rel.Sha1 { - if err := PushUpdateDeleteTag(repo, rel.TagName); err != nil { - return fmt.Errorf("PushUpdateDeleteTag: %v", err) - } - } else { - existingRelTags[strings.ToLower(rel.TagName)] = struct{}{} - } - } - } - tags, err := gitRepo.GetTags() - if err != nil { - return fmt.Errorf("GetTags: %v", err) - } - for _, tagName := range tags { - if _, ok := existingRelTags[strings.ToLower(tagName)]; !ok { - if err := PushUpdateAddTag(repo, gitRepo, tagName); err != nil { - return fmt.Errorf("pushUpdateAddTag: %v", err) - } - } - } - return nil -} - // UpdateReleasesMigrationsByType updates all migrated repositories' releases from gitServiceType to replace originalAuthorID to posterID func UpdateReleasesMigrationsByType(gitServiceType structs.GitServiceType, originalAuthorID string, posterID int64) error { _, err := x.Table("release"). diff --git a/models/repo.go b/models/repo.go index 09735bd2c2..f4ac75b8f0 100644 --- a/models/repo.go +++ b/models/repo.go @@ -39,7 +39,6 @@ import ( "github.com/mcuadros/go-version" "github.com/unknwon/com" - ini "gopkg.in/ini.v1" "xorm.io/builder" ) @@ -941,25 +940,6 @@ func (repo *Repository) CloneLink() (cl *CloneLink) { return repo.cloneLink(x, false) } -/* - GitHub, GitLab, Gogs: *.wiki.git - BitBucket: *.git/wiki -*/ -var commonWikiURLSuffixes = []string{".wiki.git", ".git/wiki"} - -// wikiRemoteURL returns accessible repository URL for wiki if exists. -// Otherwise, it returns an empty string. -func wikiRemoteURL(remote string) string { - remote = strings.TrimSuffix(remote, ".git") - for _, suffix := range commonWikiURLSuffixes { - wikiURL := remote + suffix - if git.IsRepoURLAccessible(wikiURL) { - return wikiURL - } - } - return "" -} - // CheckCreateRepository check if could created a repository func CheckCreateRepository(doer, u *User, name string) error { if !doer.CanCreateRepo() { @@ -979,118 +959,9 @@ func CheckCreateRepository(doer, u *User, name string) error { return nil } -// MigrateRepositoryGitData starts migrating git related data after created migrating repository -func MigrateRepositoryGitData(doer, u *User, repo *Repository, opts api.MigrateRepoOption) (*Repository, error) { - repoPath := RepoPath(u.Name, opts.RepoName) - - if u.IsOrganization() { - t, err := u.GetOwnerTeam() - if err != nil { - return nil, err - } - repo.NumWatches = t.NumMembers - } else { - repo.NumWatches = 1 - } - - migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second - - var err error - if err = os.RemoveAll(repoPath); err != nil { - return repo, fmt.Errorf("Failed to remove %s: %v", repoPath, err) - } - - if err = git.Clone(opts.CloneAddr, repoPath, git.CloneRepoOptions{ - Mirror: true, - Quiet: true, - Timeout: migrateTimeout, - }); err != nil { - return repo, fmt.Errorf("Clone: %v", err) - } - - if opts.Wiki { - wikiPath := WikiPath(u.Name, opts.RepoName) - wikiRemotePath := wikiRemoteURL(opts.CloneAddr) - if len(wikiRemotePath) > 0 { - if err := os.RemoveAll(wikiPath); err != nil { - return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) - } - - if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{ - Mirror: true, - Quiet: true, - Timeout: migrateTimeout, - Branch: "master", - }); err != nil { - log.Warn("Clone wiki: %v", err) - if err := os.RemoveAll(wikiPath); err != nil { - return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) - } - } - } - } - - gitRepo, err := git.OpenRepository(repoPath) - if err != nil { - return repo, fmt.Errorf("OpenRepository: %v", err) - } - defer gitRepo.Close() - - repo.IsEmpty, err = gitRepo.IsEmpty() - if err != nil { - return repo, fmt.Errorf("git.IsEmpty: %v", err) - } - - if !opts.Releases && !repo.IsEmpty { - // Try to get HEAD branch and set it as default branch. - headBranch, err := gitRepo.GetHEADBranch() - if err != nil { - return repo, fmt.Errorf("GetHEADBranch: %v", err) - } - if headBranch != nil { - repo.DefaultBranch = headBranch.Name - } - - if err = SyncReleasesWithTags(repo, gitRepo); err != nil { - log.Error("Failed to synchronize tags to releases for repository: %v", err) - } - } - - if err = repo.UpdateSize(); err != nil { - log.Error("Failed to update size for repository: %v", err) - } - - if opts.Mirror { - if _, err = x.InsertOne(&Mirror{ - RepoID: repo.ID, - Interval: setting.Mirror.DefaultInterval, - EnablePrune: true, - NextUpdateUnix: timeutil.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval), - }); err != nil { - return repo, fmt.Errorf("InsertOne: %v", err) - } - - repo.IsMirror = true - err = UpdateRepository(repo, false) - } else { - repo, err = CleanUpMigrateInfo(repo) - } - - return repo, err -} - -// cleanUpMigrateGitConfig removes mirror info which prevents "push --all". -// This also removes possible user credentials. -func cleanUpMigrateGitConfig(configPath string) error { - cfg, err := ini.Load(configPath) - if err != nil { - return fmt.Errorf("open config file: %v", err) - } - cfg.DeleteSection("remote \"origin\"") - if err = cfg.SaveToIndent(configPath, "\t"); err != nil { - return fmt.Errorf("save config file: %v", err) - } - return nil +// CreateDelegateHooks creates all the hooks scripts for the repo +func CreateDelegateHooks(repoPath string) error { + return createDelegateHooks(repoPath) } // createDelegateHooks creates all the hooks scripts for the repo @@ -1132,32 +1003,6 @@ func createDelegateHooks(repoPath string) (err error) { return nil } -// CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors. -func CleanUpMigrateInfo(repo *Repository) (*Repository, error) { - repoPath := repo.RepoPath() - if err := createDelegateHooks(repoPath); err != nil { - return repo, fmt.Errorf("createDelegateHooks: %v", err) - } - if repo.HasWiki() { - if err := createDelegateHooks(repo.WikiPath()); err != nil { - return repo, fmt.Errorf("createDelegateHooks.(wiki): %v", err) - } - } - - _, err := git.NewCommand("remote", "rm", "origin").RunInDir(repoPath) - if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { - return repo, fmt.Errorf("CleanUpMigrateInfo: %v", err) - } - - if repo.HasWiki() { - if err := cleanUpMigrateGitConfig(path.Join(repo.WikiPath(), "config")); err != nil { - return repo, fmt.Errorf("cleanUpMigrateGitConfig (wiki): %v", err) - } - } - - return repo, UpdateRepository(repo, false) -} - // initRepoCommit temporarily changes with work directory. func initRepoCommit(tmpPath string, u *User) (err error) { commitTimeStr := time.Now().Format(time.RFC3339) diff --git a/models/repo_mirror.go b/models/repo_mirror.go index aa0ec26808..10b0a7b139 100644 --- a/models/repo_mirror.go +++ b/models/repo_mirror.go @@ -97,3 +97,9 @@ func MirrorsIterate(f func(idx int, bean interface{}) error) error { And("next_update_unix!=0"). Iterate(new(Mirror), f) } + +// InsertMirror inserts a mirror to database +func InsertMirror(mirror *Mirror) error { + _, err := x.Insert(mirror) + return err +} diff --git a/modules/migrations/gitea.go b/modules/migrations/gitea.go index 18f4da7584..db2143fe7e 100644 --- a/modules/migrations/gitea.go +++ b/modules/migrations/gitea.go @@ -21,6 +21,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" @@ -111,7 +112,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate return err } - r, err = models.MigrateRepositoryGitData(g.doer, owner, r, structs.MigrateRepoOption{ + r, err = repository.MigrateRepositoryGitData(g.doer, owner, r, structs.MigrateRepoOption{ RepoName: g.repoName, Description: repo.Description, OriginalURL: repo.OriginalURL, @@ -294,7 +295,7 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { // SyncTags syncs releases with tags in the database func (g *GiteaLocalUploader) SyncTags() error { - return models.SyncReleasesWithTags(g.repo, g.gitRepo) + return repository.SyncReleasesWithTags(g.repo, g.gitRepo) } // CreateIssues creates issues diff --git a/modules/repository/repo.go b/modules/repository/repo.go new file mode 100644 index 0000000000..ea526a1e30 --- /dev/null +++ b/modules/repository/repo.go @@ -0,0 +1,223 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repository + +import ( + "fmt" + "os" + "path" + "strings" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/timeutil" + "gopkg.in/ini.v1" +) + +/* + GitHub, GitLab, Gogs: *.wiki.git + BitBucket: *.git/wiki +*/ +var commonWikiURLSuffixes = []string{".wiki.git", ".git/wiki"} + +// wikiRemoteURL returns accessible repository URL for wiki if exists. +// Otherwise, it returns an empty string. +func wikiRemoteURL(remote string) string { + remote = strings.TrimSuffix(remote, ".git") + for _, suffix := range commonWikiURLSuffixes { + wikiURL := remote + suffix + if git.IsRepoURLAccessible(wikiURL) { + return wikiURL + } + } + return "" +} + +// MigrateRepositoryGitData starts migrating git related data after created migrating repository +func MigrateRepositoryGitData(doer, u *models.User, repo *models.Repository, opts api.MigrateRepoOption) (*models.Repository, error) { + repoPath := models.RepoPath(u.Name, opts.RepoName) + + if u.IsOrganization() { + t, err := u.GetOwnerTeam() + if err != nil { + return nil, err + } + repo.NumWatches = t.NumMembers + } else { + repo.NumWatches = 1 + } + + migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second + + var err error + if err = os.RemoveAll(repoPath); err != nil { + return repo, fmt.Errorf("Failed to remove %s: %v", repoPath, err) + } + + if err = git.Clone(opts.CloneAddr, repoPath, git.CloneRepoOptions{ + Mirror: true, + Quiet: true, + Timeout: migrateTimeout, + }); err != nil { + return repo, fmt.Errorf("Clone: %v", err) + } + + if opts.Wiki { + wikiPath := models.WikiPath(u.Name, opts.RepoName) + wikiRemotePath := wikiRemoteURL(opts.CloneAddr) + if len(wikiRemotePath) > 0 { + if err := os.RemoveAll(wikiPath); err != nil { + return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) + } + + if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{ + Mirror: true, + Quiet: true, + Timeout: migrateTimeout, + Branch: "master", + }); err != nil { + log.Warn("Clone wiki: %v", err) + if err := os.RemoveAll(wikiPath); err != nil { + return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) + } + } + } + } + + gitRepo, err := git.OpenRepository(repoPath) + if err != nil { + return repo, fmt.Errorf("OpenRepository: %v", err) + } + defer gitRepo.Close() + + repo.IsEmpty, err = gitRepo.IsEmpty() + if err != nil { + return repo, fmt.Errorf("git.IsEmpty: %v", err) + } + + if !opts.Releases && !repo.IsEmpty { + // Try to get HEAD branch and set it as default branch. + headBranch, err := gitRepo.GetHEADBranch() + if err != nil { + return repo, fmt.Errorf("GetHEADBranch: %v", err) + } + if headBranch != nil { + repo.DefaultBranch = headBranch.Name + } + + if err = SyncReleasesWithTags(repo, gitRepo); err != nil { + log.Error("Failed to synchronize tags to releases for repository: %v", err) + } + } + + if err = repo.UpdateSize(); err != nil { + log.Error("Failed to update size for repository: %v", err) + } + + if opts.Mirror { + if err = models.InsertMirror(&models.Mirror{ + RepoID: repo.ID, + Interval: setting.Mirror.DefaultInterval, + EnablePrune: true, + NextUpdateUnix: timeutil.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval), + }); err != nil { + return repo, fmt.Errorf("InsertOne: %v", err) + } + + repo.IsMirror = true + err = models.UpdateRepository(repo, false) + } else { + repo, err = CleanUpMigrateInfo(repo) + } + + return repo, err +} + +// cleanUpMigrateGitConfig removes mirror info which prevents "push --all". +// This also removes possible user credentials. +func cleanUpMigrateGitConfig(configPath string) error { + cfg, err := ini.Load(configPath) + if err != nil { + return fmt.Errorf("open config file: %v", err) + } + cfg.DeleteSection("remote \"origin\"") + if err = cfg.SaveToIndent(configPath, "\t"); err != nil { + return fmt.Errorf("save config file: %v", err) + } + return nil +} + +// CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors. +func CleanUpMigrateInfo(repo *models.Repository) (*models.Repository, error) { + repoPath := repo.RepoPath() + if err := models.CreateDelegateHooks(repoPath); err != nil { + return repo, fmt.Errorf("createDelegateHooks: %v", err) + } + if repo.HasWiki() { + if err := models.CreateDelegateHooks(repo.WikiPath()); err != nil { + return repo, fmt.Errorf("createDelegateHooks.(wiki): %v", err) + } + } + + _, err := git.NewCommand("remote", "rm", "origin").RunInDir(repoPath) + if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { + return repo, fmt.Errorf("CleanUpMigrateInfo: %v", err) + } + + if repo.HasWiki() { + if err := cleanUpMigrateGitConfig(path.Join(repo.WikiPath(), "config")); err != nil { + return repo, fmt.Errorf("cleanUpMigrateGitConfig (wiki): %v", err) + } + } + + return repo, models.UpdateRepository(repo, false) +} + +// SyncReleasesWithTags synchronizes release table with repository tags +func SyncReleasesWithTags(repo *models.Repository, gitRepo *git.Repository) error { + existingRelTags := make(map[string]struct{}) + opts := models.FindReleasesOptions{IncludeDrafts: true, IncludeTags: true} + for page := 1; ; page++ { + rels, err := models.GetReleasesByRepoID(repo.ID, opts, page, 100) + if err != nil { + return fmt.Errorf("GetReleasesByRepoID: %v", err) + } + if len(rels) == 0 { + break + } + for _, rel := range rels { + if rel.IsDraft { + continue + } + commitID, err := gitRepo.GetTagCommitID(rel.TagName) + if err != nil && !git.IsErrNotExist(err) { + return fmt.Errorf("GetTagCommitID: %v", err) + } + if git.IsErrNotExist(err) || commitID != rel.Sha1 { + if err := models.PushUpdateDeleteTag(repo, rel.TagName); err != nil { + return fmt.Errorf("PushUpdateDeleteTag: %v", err) + } + } else { + existingRelTags[strings.ToLower(rel.TagName)] = struct{}{} + } + } + } + tags, err := gitRepo.GetTags() + if err != nil { + return fmt.Errorf("GetTags: %v", err) + } + for _, tagName := range tags { + if _, ok := existingRelTags[strings.ToLower(tagName)]; !ok { + if err := models.PushUpdateAddTag(repo, gitRepo, tagName); err != nil { + return fmt.Errorf("pushUpdateAddTag: %v", err) + } + } + } + return nil +} diff --git a/routers/repo/setting.go b/routers/repo/setting.go index a3650b1901..e5c2a9dbc6 100644 --- a/routers/repo/setting.go +++ b/routers/repo/setting.go @@ -20,6 +20,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/validation" @@ -338,7 +339,7 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) { } repo.IsMirror = false - if _, err := models.CleanUpMigrateInfo(repo); err != nil { + if _, err := repository.CleanUpMigrateInfo(repo); err != nil { ctx.ServerError("CleanUpMigrateInfo", err) return } else if err = models.DeleteMirrorByRepoID(ctx.Repo.Repository.ID); err != nil { diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index 9c52f1723b..1ad9448b6b 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification" + "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/sync" "code.gitea.io/gitea/modules/timeutil" @@ -207,7 +208,7 @@ func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) { log.Error("OpenRepository: %v", err) return nil, false } - if err = models.SyncReleasesWithTags(m.Repo, gitRepo); err != nil { + if err = repository.SyncReleasesWithTags(m.Repo, gitRepo); err != nil { gitRepo.Close() log.Error("Failed to synchronize tags to releases for repository: %v", err) } diff --git a/services/mirror/mirror_test.go b/services/mirror/mirror_test.go index 81811ffe4a..816ef230fd 100644 --- a/services/mirror/mirror_test.go +++ b/services/mirror/mirror_test.go @@ -10,6 +10,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/structs" release_service "code.gitea.io/gitea/services/release" @@ -46,7 +47,7 @@ func TestRelease_MirrorDelete(t *testing.T) { }) assert.NoError(t, err) - mirror, err := models.MigrateRepositoryGitData(user, user, mirrorRepo, opts) + mirror, err := repository.MigrateRepositoryGitData(user, user, mirrorRepo, opts) assert.NoError(t, err) gitRepo, err := git.OpenRepository(repoPath)