Move migrations into services and base into modules/migration (#17663)

* Move migrtions into services and base into modules/migration

* Fix imports

* Fix lint

modules/migration/comment.go

@@ -0,0 +1,20 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import "time"
+
+// Comment is a standard comment information
+type Comment struct {
+	IssueIndex  int64  `yaml:"issue_index"`
+	PosterID    int64  `yaml:"poster_id"`
+	PosterName  string `yaml:"poster_name"`
+	PosterEmail string `yaml:"poster_email"`
+	Created     time.Time
+	Updated     time.Time
+	Content     string
+	Reactions   []*Reaction
+}

modules/migration/downloader.go

@@ -0,0 +1,41 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import (
+	"context"
+
+	"code.gitea.io/gitea/modules/structs"
+)
+
+// GetCommentOptions represents an options for get comment
+type GetCommentOptions struct {
+	Context  IssueContext
+	Page     int
+	PageSize int
+}
+
+// Downloader downloads the site repo information
+type Downloader interface {
+	SetContext(context.Context)
+	GetRepoInfo() (*Repository, error)
+	GetTopics() ([]string, error)
+	GetMilestones() ([]*Milestone, error)
+	GetReleases() ([]*Release, error)
+	GetLabels() ([]*Label, error)
+	GetIssues(page, perPage int) ([]*Issue, bool, error)
+	GetComments(opts GetCommentOptions) ([]*Comment, bool, error)
+	SupportGetRepoComments() bool
+	GetPullRequests(page, perPage int) ([]*PullRequest, bool, error)
+	GetReviews(pullRequestContext IssueContext) ([]*Review, error)
+	FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error)
+}
+
+// DownloaderFactory defines an interface to match a downloader implementation and create a downloader
+type DownloaderFactory interface {
+	New(ctx context.Context, opts MigrateOptions) (Downloader, error)
+	GitServiceType() structs.GitServiceType
+}

modules/migration/error.go

@@ -0,0 +1,26 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import "fmt"
+
+// ErrNotSupported represents status if a downloader do not supported something.
+type ErrNotSupported struct {
+	Entity string
+}
+
+// IsErrNotSupported checks if an error is an ErrNotSupported
+func IsErrNotSupported(err error) bool {
+	_, ok := err.(ErrNotSupported)
+	return ok
+}
+
+// Error return error message
+func (err ErrNotSupported) Error() string {
+	if len(err.Entity) != 0 {
+		return fmt.Sprintf("'%s' not supported", err.Entity)
+	}
+	return "not supported"
+}

modules/migration/issue.go

@@ -0,0 +1,48 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import "time"
+
+// IssueContext is used to map between local and foreign issue/PR ids.
+type IssueContext interface {
+	LocalID() int64
+	ForeignID() int64
+}
+
+// BasicIssueContext is a 1:1 mapping between local and foreign ids.
+type BasicIssueContext int64
+
+// LocalID gets the local id.
+func (c BasicIssueContext) LocalID() int64 {
+	return int64(c)
+}
+
+// ForeignID gets the foreign id.
+func (c BasicIssueContext) ForeignID() int64 {
+	return int64(c)
+}
+
+// Issue is a standard issue information
+type Issue struct {
+	Number      int64
+	PosterID    int64  `yaml:"poster_id"`
+	PosterName  string `yaml:"poster_name"`
+	PosterEmail string `yaml:"poster_email"`
+	Title       string
+	Content     string
+	Ref         string
+	Milestone   string
+	State       string // closed, open
+	IsLocked    bool   `yaml:"is_locked"`
+	Created     time.Time
+	Updated     time.Time
+	Closed      *time.Time
+	Labels      []*Label
+	Reactions   []*Reaction
+	Assignees   []string
+	Context     IssueContext `yaml:"-"`
+}

modules/migration/label.go

@@ -0,0 +1,13 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+// Label defines a standard label information
+type Label struct {
+	Name        string
+	Color       string
+	Description string
+}

modules/migration/messenger.go

@@ -0,0 +1,11 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+// Messenger is a formatting function similar to i18n.Tr
+type Messenger func(key string, args ...interface{})
+
+// NilMessenger represents an empty formatting function
+func NilMessenger(string, ...interface{}) {}

modules/migration/milestone.go

@@ -0,0 +1,19 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import "time"
+
+// Milestone defines a standard milestone
+type Milestone struct {
+	Title       string
+	Description string
+	Deadline    *time.Time
+	Created     time.Time
+	Updated     *time.Time
+	Closed      *time.Time
+	State       string // open, closed
+}

modules/migration/null_downloader.go

@@ -0,0 +1,87 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import (
+	"context"
+	"net/url"
+)
+
+// NullDownloader implements a blank downloader
+type NullDownloader struct {
+}
+
+var (
+	_ Downloader = &NullDownloader{}
+)
+
+// SetContext set context
+func (n NullDownloader) SetContext(_ context.Context) {}
+
+// GetRepoInfo returns a repository information
+func (n NullDownloader) GetRepoInfo() (*Repository, error) {
+	return nil, &ErrNotSupported{Entity: "RepoInfo"}
+}
+
+// GetTopics return repository topics
+func (n NullDownloader) GetTopics() ([]string, error) {
+	return nil, &ErrNotSupported{Entity: "Topics"}
+}
+
+// GetMilestones returns milestones
+func (n NullDownloader) GetMilestones() ([]*Milestone, error) {
+	return nil, &ErrNotSupported{Entity: "Milestones"}
+}
+
+// GetReleases returns releases
+func (n NullDownloader) GetReleases() ([]*Release, error) {
+	return nil, &ErrNotSupported{Entity: "Releases"}
+}
+
+// GetLabels returns labels
+func (n NullDownloader) GetLabels() ([]*Label, error) {
+	return nil, &ErrNotSupported{Entity: "Labels"}
+}
+
+// GetIssues returns issues according start and limit
+func (n NullDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
+	return nil, false, &ErrNotSupported{Entity: "Issues"}
+}
+
+// GetComments returns comments according the options
+func (n NullDownloader) GetComments(GetCommentOptions) ([]*Comment, bool, error) {
+	return nil, false, &ErrNotSupported{Entity: "Comments"}
+}
+
+// GetPullRequests returns pull requests according page and perPage
+func (n NullDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) {
+	return nil, false, &ErrNotSupported{Entity: "PullRequests"}
+}
+
+// GetReviews returns pull requests review
+func (n NullDownloader) GetReviews(pullRequestContext IssueContext) ([]*Review, error) {
+	return nil, &ErrNotSupported{Entity: "Reviews"}
+}
+
+// FormatCloneURL add authentification into remote URLs
+func (n NullDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) {
+	if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
+		u, err := url.Parse(remoteAddr)
+		if err != nil {
+			return "", err
+		}
+		u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
+		if len(opts.AuthToken) > 0 {
+			u.User = url.UserPassword("oauth2", opts.AuthToken)
+		}
+		return u.String(), nil
+	}
+	return remoteAddr, nil
+}
+
+// SupportGetRepoComments return true if it supports get repo comments
+func (n NullDownloader) SupportGetRepoComments() bool {
+	return false
+}

modules/migration/options.go

@@ -0,0 +1,42 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import "code.gitea.io/gitea/modules/structs"
+
+// MigrateOptions defines the way a repository gets migrated
+// this is for internal usage by migrations module and func who interact with it
+type MigrateOptions struct {
+	// required: true
+	CloneAddr             string `json:"clone_addr" binding:"Required"`
+	CloneAddrEncrypted    string `json:"clone_addr_encrypted,omitempty"`
+	AuthUsername          string `json:"auth_username"`
+	AuthPassword          string `json:"-"`
+	AuthPasswordEncrypted string `json:"auth_password_encrypted,omitempty"`
+	AuthToken             string `json:"-"`
+	AuthTokenEncrypted    string `json:"auth_token_encrypted,omitempty"`
+	// required: true
+	UID int `json:"uid" binding:"Required"`
+	// required: true
+	RepoName        string `json:"repo_name" binding:"Required"`
+	Mirror          bool   `json:"mirror"`
+	LFS             bool   `json:"lfs"`
+	LFSEndpoint     string `json:"lfs_endpoint"`
+	Private         bool   `json:"private"`
+	Description     string `json:"description"`
+	OriginalURL     string
+	GitServiceType  structs.GitServiceType
+	Wiki            bool
+	Issues          bool
+	Milestones      bool
+	Labels          bool
+	Releases        bool
+	Comments        bool
+	PullRequests    bool
+	ReleaseAssets   bool
+	MigrateToRepoID int64
+	MirrorInterval  string `json:"mirror_interval"`
+}

modules/migration/pullrequest.go

@@ -0,0 +1,61 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import (
+	"fmt"
+	"time"
+)
+
+// PullRequest defines a standard pull request information
+type PullRequest struct {
+	Number         int64
+	Title          string
+	PosterName     string `yaml:"poster_name"`
+	PosterID       int64  `yaml:"poster_id"`
+	PosterEmail    string `yaml:"poster_email"`
+	Content        string
+	Milestone      string
+	State          string
+	Created        time.Time
+	Updated        time.Time
+	Closed         *time.Time
+	Labels         []*Label
+	PatchURL       string `yaml:"patch_url"`
+	Merged         bool
+	MergedTime     *time.Time `yaml:"merged_time"`
+	MergeCommitSHA string     `yaml:"merge_commit_sha"`
+	Head           PullRequestBranch
+	Base           PullRequestBranch
+	Assignees      []string
+	IsLocked       bool `yaml:"is_locked"`
+	Reactions      []*Reaction
+	Context        IssueContext `yaml:"-"`
+}
+
+// IsForkPullRequest returns true if the pull request from a forked repository but not the same repository
+func (p *PullRequest) IsForkPullRequest() bool {
+	return p.Head.RepoPath() != p.Base.RepoPath()
+}
+
+// GetGitRefName returns pull request relative path to head
+func (p PullRequest) GetGitRefName() string {
+	return fmt.Sprintf("refs/pull/%d/head", p.Number)
+}
+
+// PullRequestBranch represents a pull request branch
+type PullRequestBranch struct {
+	CloneURL  string `yaml:"clone_url"`
+	Ref       string
+	SHA       string
+	RepoName  string `yaml:"repo_name"`
+	OwnerName string `yaml:"owner_name"`
+}
+
+// RepoPath returns pull request repo path
+func (p PullRequestBranch) RepoPath() string {
+	return fmt.Sprintf("%s/%s", p.OwnerName, p.RepoName)
+}

modules/migration/reaction.go

@@ -0,0 +1,12 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+// Reaction represents a reaction to an issue/pr/comment.
+type Reaction struct {
+	UserID   int64  `yaml:"user_id"`
+	UserName string `yaml:"user_name"`
+	Content  string
+}

modules/migration/release.go

@@ -0,0 +1,40 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import (
+	"io"
+	"time"
+)
+
+// ReleaseAsset represents a release asset
+type ReleaseAsset struct {
+	ID            int64
+	Name          string
+	ContentType   *string `yaml:"content_type"`
+	Size          *int
+	DownloadCount *int `yaml:"download_count"`
+	Created       time.Time
+	Updated       time.Time
+	DownloadURL   *string `yaml:"download_url"`
+	// if DownloadURL is nil, the function should be invoked
+	DownloadFunc func() (io.ReadCloser, error) `yaml:"-"`
+}
+
+// Release represents a release
+type Release struct {
+	TagName         string `yaml:"tag_name"`
+	TargetCommitish string `yaml:"target_commitish"`
+	Name            string
+	Body            string
+	Draft           bool
+	Prerelease      bool
+	PublisherID     int64  `yaml:"publisher_id"`
+	PublisherName   string `yaml:"publisher_name"`
+	PublisherEmail  string `yaml:"publisher_email"`
+	Assets          []*ReleaseAsset
+	Created         time.Time
+	Published       time.Time
+}

modules/migration/repo.go

@@ -0,0 +1,18 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+// Repository defines a standard repository information
+type Repository struct {
+	Name          string
+	Owner         string
+	IsPrivate     bool `yaml:"is_private"`
+	IsMirror      bool `yaml:"is_mirror"`
+	Description   string
+	CloneURL      string `yaml:"clone_url"`
+	OriginalURL   string `yaml:"original_url"`
+	DefaultBranch string
+}

modules/migration/retry_downloader.go

@@ -0,0 +1,197 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import (
+	"context"
+	"time"
+)
+
+var (
+	_ Downloader = &RetryDownloader{}
+)
+
+// RetryDownloader retry the downloads
+type RetryDownloader struct {
+	Downloader
+	ctx        context.Context
+	RetryTimes int // the total execute times
+	RetryDelay int // time to delay seconds
+}
+
+// NewRetryDownloader creates a retry downloader
+func NewRetryDownloader(ctx context.Context, downloader Downloader, retryTimes, retryDelay int) *RetryDownloader {
+	return &RetryDownloader{
+		Downloader: downloader,
+		ctx:        ctx,
+		RetryTimes: retryTimes,
+		RetryDelay: retryDelay,
+	}
+}
+
+func (d *RetryDownloader) retry(work func() error) error {
+	var (
+		times = d.RetryTimes
+		err   error
+	)
+	for ; times > 0; times-- {
+		if err = work(); err == nil {
+			return nil
+		}
+		if IsErrNotSupported(err) {
+			return err
+		}
+		select {
+		case <-d.ctx.Done():
+			return d.ctx.Err()
+		case <-time.After(time.Second * time.Duration(d.RetryDelay)):
+		}
+	}
+	return err
+}
+
+// SetContext set context
+func (d *RetryDownloader) SetContext(ctx context.Context) {
+	d.ctx = ctx
+	d.Downloader.SetContext(ctx)
+}
+
+// GetRepoInfo returns a repository information with retry
+func (d *RetryDownloader) GetRepoInfo() (*Repository, error) {
+	var (
+		repo *Repository
+		err  error
+	)
+
+	err = d.retry(func() error {
+		repo, err = d.Downloader.GetRepoInfo()
+		return err
+	})
+
+	return repo, err
+}
+
+// GetTopics returns a repository's topics with retry
+func (d *RetryDownloader) GetTopics() ([]string, error) {
+	var (
+		topics []string
+		err    error
+	)
+
+	err = d.retry(func() error {
+		topics, err = d.Downloader.GetTopics()
+		return err
+	})
+
+	return topics, err
+}
+
+// GetMilestones returns a repository's milestones with retry
+func (d *RetryDownloader) GetMilestones() ([]*Milestone, error) {
+	var (
+		milestones []*Milestone
+		err        error
+	)
+
+	err = d.retry(func() error {
+		milestones, err = d.Downloader.GetMilestones()
+		return err
+	})
+
+	return milestones, err
+}
+
+// GetReleases returns a repository's releases with retry
+func (d *RetryDownloader) GetReleases() ([]*Release, error) {
+	var (
+		releases []*Release
+		err      error
+	)
+
+	err = d.retry(func() error {
+		releases, err = d.Downloader.GetReleases()
+		return err
+	})
+
+	return releases, err
+}
+
+// GetLabels returns a repository's labels with retry
+func (d *RetryDownloader) GetLabels() ([]*Label, error) {
+	var (
+		labels []*Label
+		err    error
+	)
+
+	err = d.retry(func() error {
+		labels, err = d.Downloader.GetLabels()
+		return err
+	})
+
+	return labels, err
+}
+
+// GetIssues returns a repository's issues with retry
+func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
+	var (
+		issues []*Issue
+		isEnd  bool
+		err    error
+	)
+
+	err = d.retry(func() error {
+		issues, isEnd, err = d.Downloader.GetIssues(page, perPage)
+		return err
+	})
+
+	return issues, isEnd, err
+}
+
+// GetComments returns a repository's comments with retry
+func (d *RetryDownloader) GetComments(opts GetCommentOptions) ([]*Comment, bool, error) {
+	var (
+		comments []*Comment
+		isEnd    bool
+		err      error
+	)
+
+	err = d.retry(func() error {
+		comments, isEnd, err = d.Downloader.GetComments(opts)
+		return err
+	})
+
+	return comments, isEnd, err
+}
+
+// GetPullRequests returns a repository's pull requests with retry
+func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) {
+	var (
+		prs   []*PullRequest
+		err   error
+		isEnd bool
+	)
+
+	err = d.retry(func() error {
+		prs, isEnd, err = d.Downloader.GetPullRequests(page, perPage)
+		return err
+	})
+
+	return prs, isEnd, err
+}
+
+// GetReviews returns pull requests reviews
+func (d *RetryDownloader) GetReviews(pullRequestContext IssueContext) ([]*Review, error) {
+	var (
+		reviews []*Review
+		err     error
+	)
+
+	err = d.retry(func() error {
+		reviews, err = d.Downloader.GetReviews(pullRequestContext)
+		return err
+	})
+
+	return reviews, err
+}

modules/migration/review.go

@@ -0,0 +1,45 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import "time"
+
+// enumerate all review states
+const (
+	ReviewStatePending          = "PENDING"
+	ReviewStateApproved         = "APPROVED"
+	ReviewStateChangesRequested = "CHANGES_REQUESTED"
+	ReviewStateCommented        = "COMMENTED"
+)
+
+// Review is a standard review information
+type Review struct {
+	ID           int64
+	IssueIndex   int64  `yaml:"issue_index"`
+	ReviewerID   int64  `yaml:"reviewer_id"`
+	ReviewerName string `yaml:"reviewer_name"`
+	Official     bool
+	CommitID     string `yaml:"commit_id"`
+	Content      string
+	CreatedAt    time.Time `yaml:"created_at"`
+	State        string    // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT
+	Comments     []*ReviewComment
+}
+
+// ReviewComment represents a review comment
+type ReviewComment struct {
+	ID        int64
+	InReplyTo int64 `yaml:"in_reply_to"`
+	Content   string
+	TreePath  string `yaml:"tree_path"`
+	DiffHunk  string `yaml:"diff_hunk"`
+	Position  int
+	Line      int
+	CommitID  string `yaml:"commit_id"`
+	PosterID  int64  `yaml:"poster_id"`
+	Reactions []*Reaction
+	CreatedAt time.Time `yaml:"created_at"`
+	UpdatedAt time.Time `yaml:"updated_at"`
+}

modules/migration/uploader.go

@@ -0,0 +1,24 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+// Uploader uploads all the information of one repository
+type Uploader interface {
+	MaxBatchInsertSize(tp string) int
+	CreateRepo(repo *Repository, opts MigrateOptions) error
+	CreateTopics(topic ...string) error
+	CreateMilestones(milestones ...*Milestone) error
+	CreateReleases(releases ...*Release) error
+	SyncTags() error
+	CreateLabels(labels ...*Label) error
+	CreateIssues(issues ...*Issue) error
+	CreateComments(comments ...*Comment) error
+	CreatePullRequests(prs ...*PullRequest) error
+	CreateReviews(reviews ...*Review) error
+	Rollback() error
+	Finish() error
+	Close()
+}