Dump github/gitlab/gitea repository data to a local directory and restore to gitea (#12244)

* Dump github/gitlab repository data to a local directory

* Fix lint

* Adjust directory structure

* Allow migration special units

* Allow migration ignore release assets

* Fix lint

* Add restore repository

* stage the changes

* Merge

* Fix lint

* Update the interface

* Add some restore methods

* Finish restore

* Add comments

* Fix restore

* Add a token flag

* Fix bug

* Fix test

* Fix test

* Fix bug

* Fix bug

* Fix lint

* Fix restore

* refactor downloader

* fmt

* Fix bug isEnd detection on getIssues

* Refactor maxPerPage

* Remove unused codes

* Remove unused codes

* Fix bug

* Fix restore

* Fix dump

* Uploader should not depend downloader

* use release attachment name but not id

* Fix restore bug

* Fix lint

* Fix restore bug

* Add a method of DownloadFunc for base.Release to make uploader not depend on downloader

* fix Release yml marshal

* Fix trace information

* Fix bug when dump & restore

* Save relative path on yml file

* Fix bug

* Use relative path

* Update docs

* Use git service string but not int

* Recognize clone addr to service type

modules/uri/uri.go

@@ -0,0 +1,40 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package uri
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+)
+
+// ErrURISchemeNotSupported represents a scheme error
+type ErrURISchemeNotSupported struct {
+	Scheme string
+}
+
+func (e ErrURISchemeNotSupported) Error() string {
+	return fmt.Sprintf("Unsupported scheme: %v", e.Scheme)
+}
+
+// Open open a local file or a remote file
+func Open(uriStr string) (io.ReadCloser, error) {
+	u, err := url.Parse(uriStr)
+	if err != nil {
+		return nil, err
+	}
+	switch strings.ToLower(u.Scheme) {
+	case "http", "https":
+		f, err := http.Get(uriStr)
+		return f.Body, err
+	case "file":
+		return os.Open(u.Path)
+	default:
+		return nil, ErrURISchemeNotSupported{Scheme: u.Scheme}
+	}
+}

modules/uri/uri_test.go

@@ -0,0 +1,20 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package uri
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReadURI(t *testing.T) {
+	p, err := filepath.Abs("./uri.go")
+	assert.NoError(t, err)
+	f, err := Open("file://" + p)
+	assert.NoError(t, err)
+	defer f.Close()
+}