Provide self-registering storage system (#12978)

* Provide self-registering storage system

Signed-off-by: Andrew Thornton <art27@cantab.net>

* More simplification

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Remove old strings from setting

Signed-off-by: Andrew Thornton <art27@cantab.net>

* oops attachments not attachment

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>

modules/storage/helper.go

@@ -0,0 +1,65 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package storage
+
+import (
+	"encoding/json"
+	"reflect"
+)
+
+// Mappable represents an interface that can MapTo another interface
+type Mappable interface {
+	MapTo(v interface{}) error
+}
+
+// toConfig will attempt to convert a given configuration cfg into the provided exemplar type.
+//
+// It will tolerate the cfg being passed as a []byte or string of a json representation of the
+// exemplar or the correct type of the exemplar itself
+func toConfig(exemplar, cfg interface{}) (interface{}, error) {
+
+	// First of all check if we've got the same type as the exemplar - if so it's all fine.
+	if reflect.TypeOf(cfg).AssignableTo(reflect.TypeOf(exemplar)) {
+		return cfg, nil
+	}
+
+	// Now if not - does it provide a MapTo function we can try?
+	if mappable, ok := cfg.(Mappable); ok {
+		newVal := reflect.New(reflect.TypeOf(exemplar))
+		if err := mappable.MapTo(newVal.Interface()); err == nil {
+			return newVal.Elem().Interface(), nil
+		}
+		// MapTo has failed us ... let's try the json route ...
+	}
+
+	// OK we've been passed a byte array right?
+	configBytes, ok := cfg.([]byte)
+	if !ok {
+		// oh ... it's a string then?
+		var configStr string
+
+		configStr, ok = cfg.(string)
+		configBytes = []byte(configStr)
+	}
+	if !ok {
+		// hmm ... can we marshal it to json?
+		var err error
+
+		configBytes, err = json.Marshal(cfg)
+		ok = (err == nil)
+	}
+	if !ok {
+		// no ... we've tried hard enough at this point - throw an error!
+		return nil, ErrInvalidConfiguration{cfg: cfg}
+	}
+
+	// OK unmarshal the byte array into a new copy of the exemplar
+	newVal := reflect.New(reflect.TypeOf(exemplar))
+	if err := json.Unmarshal(configBytes, newVal.Interface()); err != nil {
+		// If we can't unmarshal it then return an error!
+		return nil, ErrInvalidConfiguration{cfg: cfg, err: err}
+	}
+	return newVal.Elem().Interface(), nil
+}

modules/storage/local.go

@@ -5,6 +5,7 @@
 package storage
 
 import (
+	"context"
 	"io"
 	"net/url"
 	"os"
@@ -17,19 +18,35 @@ var (
 	_ ObjectStorage = &LocalStorage{}
 )
 
+// LocalStorageType is the type descriptor for local storage
+const LocalStorageType Type = "local"
+
+// LocalStorageConfig represents the configuration for a local storage
+type LocalStorageConfig struct {
+	Path string `ini:"PATH"`
+}
+
 // LocalStorage represents a local files storage
 type LocalStorage struct {
+	ctx context.Context
 	dir string
 }
 
 // NewLocalStorage returns a local files
-func NewLocalStorage(bucket string) (*LocalStorage, error) {
-	if err := os.MkdirAll(bucket, os.ModePerm); err != nil {
+func NewLocalStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error) {
+	configInterface, err := toConfig(LocalStorageConfig{}, cfg)
+	if err != nil {
+		return nil, err
+	}
+	config := configInterface.(LocalStorageConfig)
+
+	if err := os.MkdirAll(config.Path, os.ModePerm); err != nil {
 		return nil, err
 	}
 
 	return &LocalStorage{
-		dir: bucket,
+		ctx: ctx,
+		dir: config.Path,
 	}, nil
 }
 
@@ -80,6 +97,11 @@ func (l *LocalStorage) IterateObjects(fn func(path string, obj Object) error) er
 		if err != nil {
 			return err
 		}
+		select {
+		case <-l.ctx.Done():
+			return l.ctx.Err()
+		default:
+		}
 		if path == l.dir {
 			return nil
 		}
@@ -98,3 +120,7 @@ func (l *LocalStorage) IterateObjects(fn func(path string, obj Object) error) er
 		return fn(relPath, obj)
 	})
 }
+
+func init() {
+	RegisterStorageType(LocalStorageType, NewLocalStorage)
+}

modules/storage/minio.go

@@ -18,8 +18,9 @@ import (
 )
 
 var (
-	_            ObjectStorage = &MinioStorage{}
-	quoteEscaper               = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
+	_ ObjectStorage = &MinioStorage{}
+
+	quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
 )
 
 type minioObject struct {
@@ -35,6 +36,20 @@ func (m *minioObject) Stat() (os.FileInfo, error) {
 	return &minioFileInfo{oi}, nil
 }
 
+// MinioStorageType is the type descriptor for minio storage
+const MinioStorageType Type = "minio"
+
+// MinioStorageConfig represents the configuration for a minio storage
+type MinioStorageConfig struct {
+	Endpoint        string `ini:"MINIO_ENDPOINT"`
+	AccessKeyID     string `ini:"MINIO_ACCESS_KEY_ID"`
+	SecretAccessKey string `ini:"MINIO_SECRET_ACCESS_KEY"`
+	Bucket          string `ini:"MINIO_BUCKET"`
+	Location        string `ini:"MINIO_LOCATION"`
+	BasePath        string `ini:"MINIO_BASE_PATH"`
+	UseSSL          bool   `ini:"MINIO_USE_SSL"`
+}
+
 // MinioStorage returns a minio bucket storage
 type MinioStorage struct {
 	ctx      context.Context
@@ -44,20 +59,26 @@ type MinioStorage struct {
 }
 
 // NewMinioStorage returns a minio storage
-func NewMinioStorage(ctx context.Context, endpoint, accessKeyID, secretAccessKey, bucket, location, basePath string, useSSL bool) (*MinioStorage, error) {
-	minioClient, err := minio.New(endpoint, &minio.Options{
-		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
-		Secure: useSSL,
+func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error) {
+	configInterface, err := toConfig(MinioStorageConfig{}, cfg)
+	if err != nil {
+		return nil, err
+	}
+	config := configInterface.(MinioStorageConfig)
+
+	minioClient, err := minio.New(config.Endpoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, ""),
+		Secure: config.UseSSL,
 	})
 	if err != nil {
 		return nil, err
 	}
 
-	if err := minioClient.MakeBucket(ctx, bucket, minio.MakeBucketOptions{
-		Region: location,
+	if err := minioClient.MakeBucket(ctx, config.Bucket, minio.MakeBucketOptions{
+		Region: config.Location,
 	}); err != nil {
 		// Check to see if we already own this bucket (which happens if you run this twice)
-		exists, errBucketExists := minioClient.BucketExists(ctx, bucket)
+		exists, errBucketExists := minioClient.BucketExists(ctx, config.Bucket)
 		if !exists || errBucketExists != nil {
 			return nil, err
 		}
@@ -66,8 +87,8 @@ func NewMinioStorage(ctx context.Context, endpoint, accessKeyID, secretAccessKey
 	return &MinioStorage{
 		ctx:      ctx,
 		client:   minioClient,
-		bucket:   bucket,
-		basePath: basePath,
+		bucket:   config.Bucket,
+		basePath: config.BasePath,
 	}, nil
 }
 
@@ -183,3 +204,7 @@ func (m *MinioStorage) IterateObjects(fn func(path string, obj Object) error) er
 	}
 	return nil
 }
+
+func init() {
+	RegisterStorageType(MinioStorageType, NewMinioStorage)
+}

modules/storage/storage.go

@@ -22,6 +22,38 @@ var (
 	ErrIterateObjectsNotSupported = errors.New("iterateObjects method not supported")
 )
 
+// ErrInvalidConfiguration is called when there is invalid configuration for a storage
+type ErrInvalidConfiguration struct {
+	cfg interface{}
+	err error
+}
+
+func (err ErrInvalidConfiguration) Error() string {
+	if err.err != nil {
+		return fmt.Sprintf("Invalid Configuration Argument: %v: Error: %v", err.cfg, err.err)
+	}
+	return fmt.Sprintf("Invalid Configuration Argument: %v", err.cfg)
+}
+
+// IsErrInvalidConfiguration checks if an error is an ErrInvalidConfiguration
+func IsErrInvalidConfiguration(err error) bool {
+	_, ok := err.(ErrInvalidConfiguration)
+	return ok
+}
+
+// Type is a type of Storage
+type Type string
+
+// NewStorageFunc is a function that creates a storage
+type NewStorageFunc func(ctx context.Context, cfg interface{}) (ObjectStorage, error)
+
+var storageMap = map[Type]NewStorageFunc{}
+
+// RegisterStorageType registers a provided storage type with a function to create it
+func RegisterStorageType(typ Type, fn func(ctx context.Context, cfg interface{}) (ObjectStorage, error)) {
+	storageMap[typ] = fn
+}
+
 // Object represents the object on the storage
 type Object interface {
 	io.ReadCloser
@@ -67,41 +99,25 @@ func Init() error {
 	return initLFS()
 }
 
-func initStorage(storageCfg setting.Storage) (ObjectStorage, error) {
-	var err error
-	var s ObjectStorage
-	switch storageCfg.Type {
-	case setting.LocalStorageType:
-		s, err = NewLocalStorage(storageCfg.Path)
-	case setting.MinioStorageType:
-		minio := storageCfg.Minio
-		s, err = NewMinioStorage(
-			context.Background(),
-			minio.Endpoint,
-			minio.AccessKeyID,
-			minio.SecretAccessKey,
-			minio.Bucket,
-			minio.Location,
-			minio.BasePath,
-			minio.UseSSL,
-		)
-	default:
-		return nil, fmt.Errorf("Unsupported attachment store type: %s", storageCfg.Type)
+// NewStorage takes a storage type and some config and returns an ObjectStorage or an error
+func NewStorage(typStr string, cfg interface{}) (ObjectStorage, error) {
+	if len(typStr) == 0 {
+		typStr = string(LocalStorageType)
+	}
+	fn, ok := storageMap[Type(typStr)]
+	if !ok {
+		return nil, fmt.Errorf("Unsupported storage type: %s", typStr)
 	}
 
-	if err != nil {
-		return nil, err
-	}
-
-	return s, nil
+	return fn(context.Background(), cfg)
 }
 
 func initAttachments() (err error) {
-	Attachments, err = initStorage(setting.Attachment.Storage)
+	Attachments, err = NewStorage(setting.Attachment.Storage.Type, setting.Attachment.Storage)
 	return
 }
 
 func initLFS() (err error) {
-	LFS, err = initStorage(setting.LFS.Storage)
+	LFS, err = NewStorage(setting.LFS.Storage.Type, setting.LFS.Storage)
 	return
 }