From 846a522ecc5fcdfff1e875e3d006ea68f26137dd Mon Sep 17 00:00:00 2001
From: Earl Warren <contact@earl-warren.org>
Date: Sat, 1 Jul 2023 13:29:50 +0200
Subject: [PATCH] [F3] GetLocalMatchingRemote for user
(cherry picked from commit e73cb837f57be0d6c65d6ecb13da621a362351da)
(cherry picked from commit a24bc0b85e1702917a6b39282a869b26654b1aa0)
---
.golangci.yml | 4 +-
cmd/f3.go | 29 ++++++++++-
go.mod | 2 +-
go.sum | 4 +-
services/f3/driver/driver.go | 7 ++-
services/f3/driver/user.go | 49 +++++++++++++++++--
services/f3/util/util.go | 9 ++--
tests/integration/f3_test.go | 95 ++++++++++++++++++++++++++++++++++--
8 files changed, 181 insertions(+), 18 deletions(-)
diff --git a/.golangci.yml b/.golangci.yml
index 0248d54494..52f36c5614 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -106,9 +106,9 @@ issues:
- gosec
- unparam
- staticcheck
- - path: services/f3/driver/driver.go
+ - path: services/f3/driver/base.go
linters:
- - typecheck
+ - gosimple
- path: models/migrations/v
linters:
- gocyclo
diff --git a/cmd/f3.go b/cmd/f3.go
index 91163671e3..de58702bcf 100644
--- a/cmd/f3.go
+++ b/cmd/f3.go
@@ -6,6 +6,7 @@ import (
"context"
"fmt"
+ auth_model "code.gitea.io/gitea/models/auth"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/services/f3/util"
@@ -37,6 +38,11 @@ var CmdF3 = cli.Command{
Value: "",
Usage: "The name of the repository",
},
+ cli.StringFlag{
+ Name: "authentication-source",
+ Value: "",
+ Usage: "The name of the authentication source matching the forge of origin",
+ },
cli.BoolFlag{
Name: "no-pull-request",
Usage: "Do not dump pull requests",
@@ -67,6 +73,17 @@ func runF3(ctx *cli.Context) error {
return RunF3(stdCtx, ctx)
}
+func getAuthenticationSource(ctx context.Context, authenticationSource string) (*auth_model.Source, error) {
+ source, err := auth_model.GetSourceByName(ctx, authenticationSource)
+ if err != nil {
+ if auth_model.IsErrSourceNotExist(err) {
+ return nil, nil
+ }
+ return nil, err
+ }
+ return source, nil
+}
+
func RunF3(stdCtx context.Context, ctx *cli.Context) error {
doer, err := user_model.GetAdminUser()
if err != nil {
@@ -78,7 +95,17 @@ func RunF3(stdCtx context.Context, ctx *cli.Context) error {
features.PullRequests = false
}
- forgejo := util.ForgejoForgeRoot(features, doer)
+ var sourceID int64
+ sourceName := ctx.String("authentication-source")
+ source, err := getAuthenticationSource(stdCtx, sourceName)
+ if err != nil {
+ return fmt.Errorf("error retrieving the authentication-source %s %v", sourceName, err)
+ }
+ if source != nil {
+ sourceID = source.ID
+ }
+
+ forgejo := util.ForgejoForgeRoot(features, doer, sourceID)
f3 := util.F3ForgeRoot(features, ctx.String("directory"))
if ctx.Bool("export") {
diff --git a/go.mod b/go.mod
index dd44293336..07471edd5a 100644
--- a/go.mod
+++ b/go.mod
@@ -119,7 +119,7 @@ require (
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
gopkg.in/ini.v1 v1.67.0
gopkg.in/yaml.v3 v3.0.1
- lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230601123105-50a6e740ac04
+ lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230701182935-ce3394d54c1e
mvdan.cc/xurls/v2 v2.4.0
strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251
xorm.io/builder v0.3.12
diff --git a/go.sum b/go.sum
index ccc3ff194d..dd447d554e 100644
--- a/go.sum
+++ b/go.sum
@@ -1806,8 +1806,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230601123105-50a6e740ac04 h1:JdNHyMEVNixsOvNw3XqrkWi/RqVLN+wjrdeL6NVk2jE=
-lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230601123105-50a6e740ac04/go.mod h1:yIlQydnn+pym6OH20iQ7fbe2TjLfnlOTtEOqvjFaC70=
+lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230701182935-ce3394d54c1e h1:dcD+UGLSrgeHYyEWZ+1mZvxQ2BXKyEhjhHcB7a6XgeA=
+lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230701182935-ce3394d54c1e/go.mod h1:yIlQydnn+pym6OH20iQ7fbe2TjLfnlOTtEOqvjFaC70=
lukechampine.com/uint128 v1.1.1 h1:pnxCASz787iMf+02ssImqk6OLt+Z5QHMoZyUXR4z6JU=
lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
modernc.org/cc/v3 v3.33.6/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
diff --git a/services/f3/driver/driver.go b/services/f3/driver/driver.go
index 4f58e5bc5a..34088d751d 100644
--- a/services/f3/driver/driver.go
+++ b/services/f3/driver/driver.go
@@ -18,7 +18,8 @@ import (
type Options struct {
gof3.Options
- Doer *user_model.User
+ AuthenticationSource int64
+ Doer *user_model.User
}
type Forgejo struct {
@@ -59,6 +60,10 @@ func (o *Forgejo) GetDoer() *user_model.User {
return o.options.Doer
}
+func (o *Forgejo) GetAuthenticationSource() int64 {
+ return o.options.AuthenticationSource
+}
+
func (o *Forgejo) GetNewMigrationHTTPClient() gof3.NewMigrationHTTPClientFun {
return migrations.NewMigrationHTTPClient
}
diff --git a/services/f3/driver/user.go b/services/f3/driver/user.go
index 0d1a7719e4..bef7906ed2 100644
--- a/services/f3/driver/user.go
+++ b/services/f3/driver/user.go
@@ -6,11 +6,13 @@ import (
"context"
"fmt"
+ auth_model "code.gitea.io/gitea/models/auth"
"code.gitea.io/gitea/models/db"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/util"
user_service "code.gitea.io/gitea/services/user"
+ "lab.forgefriends.org/friendlyforgeformat/gof3/forges/common"
"lab.forgefriends.org/friendlyforgeformat/gof3/format"
f3_util "lab.forgefriends.org/friendlyforgeformat/gof3/util"
)
@@ -46,7 +48,7 @@ func (o *User) IsNil() bool {
}
func (o *User) Equals(other *User) bool {
- return (o.Name == other.Name)
+ return (o.ID == other.ID)
}
func (o *User) ToFormatInterface() format.Interface {
@@ -79,6 +81,36 @@ type UserProvider struct {
BaseProvider
}
+func getLocalMatchingRemote(ctx context.Context, authenticationSource int64, id string) *user_model.User {
+ u := &user_model.User{
+ LoginName: id,
+ LoginSource: authenticationSource,
+ LoginType: auth_model.OAuth2,
+ Type: user_model.UserTypeIndividual,
+ }
+ has, err := db.GetEngine(ctx).Get(u)
+ if err != nil {
+ panic(err)
+ } else if !has {
+ return nil
+ }
+ return u
+}
+
+func (o *UserProvider) GetLocalMatchingRemote(ctx context.Context, format format.Interface, parents ...common.ContainerObjectInterface) (string, bool) {
+ authenticationSource := o.g.GetAuthenticationSource()
+ if authenticationSource == 0 {
+ return "", false
+ }
+ user := getLocalMatchingRemote(ctx, authenticationSource, format.GetIDString())
+ if user != nil {
+ o.g.GetLogger().Debug("found existing user %d with a matching authentication source for %s", user.ID, format.GetIDString())
+ return fmt.Sprintf("%d", user.ID), true
+ }
+ o.g.GetLogger().Debug("no pre-existing local user for %s", format.GetIDString())
+ return "", false
+}
+
func (o *UserProvider) ToFormat(ctx context.Context, user *User) *format.User {
return user.ToFormat()
}
@@ -105,19 +137,21 @@ func (o *UserProvider) ProcessObject(ctx context.Context, user *User) {
}
func (o *UserProvider) Get(ctx context.Context, exemplar *User) *User {
+ o.g.GetLogger().Debug("%+v", exemplar)
var user *user_model.User
var err error
if exemplar.GetID() > 0 {
user, err = user_model.GetUserByID(ctx, exemplar.GetID())
+ o.g.GetLogger().Debug("GetUserByID: %+v %v", user, err)
} else if exemplar.Name != "" {
user, err = user_model.GetUserByName(ctx, exemplar.Name)
} else {
panic("GetID() == 0 and UserName == \"\"")
}
- if user_model.IsErrUserNotExist(err) {
- return &User{}
- }
if err != nil {
+ if user_model.IsErrUserNotExist(err) {
+ return &User{}
+ }
panic(fmt.Errorf("user %v %w", exemplar, err))
}
return UserConverter(user)
@@ -127,7 +161,12 @@ func (o *UserProvider) Put(ctx context.Context, user *User) *User {
overwriteDefault := &user_model.CreateUserOverwriteOptions{
IsActive: util.OptionalBoolTrue,
}
- u := user.User
+ u := user_model.User{
+ Name: user.Name,
+ FullName: user.FullName,
+ Email: user.Email,
+ Passwd: user.Passwd,
+ }
err := user_model.CreateUser(&u, overwriteDefault)
if err != nil {
panic(err)
diff --git a/services/f3/util/util.go b/services/f3/util/util.go
index b6f3ae3e4a..fd07611615 100644
--- a/services/f3/util/util.go
+++ b/services/f3/util/util.go
@@ -15,7 +15,9 @@ import (
func ToF3Logger(messenger base.Messenger) gof3.Logger {
if messenger == nil {
- messenger = func(string, ...interface{}) {}
+ messenger = func(message string, args ...interface{}) {
+ log.Info("Message: "+message, args...)
+ }
}
return gof3.Logger{
Message: messenger,
@@ -29,13 +31,14 @@ func ToF3Logger(messenger base.Messenger) gof3.Logger {
}
}
-func ForgejoForgeRoot(features gof3.Features, doer *user_model.User) *f3_forges.ForgeRoot {
+func ForgejoForgeRoot(features gof3.Features, doer *user_model.User, authenticationSource int64) *f3_forges.ForgeRoot {
forgeRoot := f3_forges.NewForgeRootFromDriver(&driver.Forgejo{}, &driver.Options{
Options: gof3.Options{
Features: features,
Logger: ToF3Logger(nil),
},
- Doer: doer,
+ Doer: doer,
+ AuthenticationSource: authenticationSource,
})
return forgeRoot
}
diff --git a/tests/integration/f3_test.go b/tests/integration/f3_test.go
index 2a179d9d9f..1c3ab2b02b 100644
--- a/tests/integration/f3_test.go
+++ b/tests/integration/f3_test.go
@@ -12,6 +12,7 @@ import (
auth_model "code.gitea.io/gitea/models/auth"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
+ "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/test"
"code.gitea.io/gitea/services/f3/util"
@@ -69,7 +70,7 @@ func TestF3(t *testing.T) {
fixture.NewAsset()
fixture.NewIssueComment(nil)
fixture.NewPullRequestComment()
- fixture.NewReview()
+ // fixture.NewReview()
fixture.NewIssueReaction()
fixture.NewCommentReaction()
@@ -78,7 +79,7 @@ func TestF3(t *testing.T) {
//
doer, err := user_model.GetAdminUser()
assert.NoError(t, err)
- forgejoLocal := util.ForgejoForgeRoot(gof3.AllFeatures, doer)
+ forgejoLocal := util.ForgejoForgeRoot(gof3.AllFeatures, doer, 0)
options := f3_common.NewMirrorOptionsRecurse()
forgejoLocal.Forge.Mirror(context.Background(), fixture.Forge, options)
@@ -117,7 +118,7 @@ func TestF3(t *testing.T) {
assert.Contains(t, files, "/release/")
assert.Contains(t, files, "/asset/")
assert.Contains(t, files, "/comment/")
- assert.Contains(t, files, "/review/")
+ // assert.Contains(t, files, "/review/")
assert.Contains(t, files, "/reaction/")
// f3_util.Command(context.Background(), "cp", "-a", f3.GetDirectory(), "abc")
})
@@ -179,3 +180,91 @@ func TestMaybePromoteF3User(t *testing.T) {
assert.Equal(t, userBeforeSignIn.Email, "")
assert.Equal(t, userAfterSignIn.Email, gitlabEmail)
}
+
+func TestF3UserMapping(t *testing.T) {
+ onGiteaRun(t, func(t *testing.T, u *url.URL) {
+ AllowLocalNetworks := setting.Migrations.AllowLocalNetworks
+ setting.F3.Enabled = true
+ setting.Migrations.AllowLocalNetworks = true
+ AppVer := setting.AppVer
+ // Gitea SDK (go-sdk) need to parse the AppVer from server response, so we must set it to a valid version string.
+ setting.AppVer = "1.16.0"
+ defer func() {
+ setting.Migrations.AllowLocalNetworks = AllowLocalNetworks
+ setting.AppVer = AppVer
+ }()
+
+ log.Debug("Step 1: create a fixture")
+ fixtureNewF3Forge := func(t f3_tests.TestingT, user *format.User, tmpDir string) *f3_forges.ForgeRoot {
+ root := f3_forges.NewForgeRoot(&f3_f3.Options{
+ Options: gof3.Options{
+ Configuration: gof3.Configuration{
+ Directory: tmpDir,
+ },
+ Features: gof3.AllFeatures,
+ Logger: util.ToF3Logger(nil),
+ },
+ Remap: true,
+ })
+ return root
+ }
+ fixture := f3_forges.NewFixture(t, f3_forges.FixtureForgeFactory{Fun: fixtureNewF3Forge, AdminRequired: false})
+ userID := int64(5432)
+ fixture.NewUser(userID)
+ // fixture.NewProject()
+
+ log.Debug("Step 2: mirror the fixture into Forgejo")
+ //
+ // OAuth2 authentication source GitLab
+ //
+ gitlabName := "gitlab"
+ gitlab := addAuthSource(t, authSourcePayloadGitLabCustom(gitlabName))
+ //
+ // Create a user as if it had been previously been created by the F3
+ // authentication source.
+ //
+ gitlabUserID := fmt.Sprintf("%d", userID)
+ gitlabUser := &user_model.User{
+ Name: "gitlabuser",
+ Email: "gitlabuser@example.com",
+ LoginType: auth_model.OAuth2,
+ LoginSource: gitlab.ID,
+ LoginName: gitlabUserID,
+ }
+ defer createUser(context.Background(), t, gitlabUser)()
+
+ doer, err := user_model.GetAdminUser()
+ assert.NoError(t, err)
+ forgejoLocal := util.ForgejoForgeRoot(gof3.AllFeatures, doer, gitlab.ID)
+ options := f3_common.NewMirrorOptionsRecurse()
+ forgejoLocal.Forge.Mirror(context.Background(), fixture.Forge, options)
+
+ log.Debug("Step 3: mirror Forgejo into F3")
+ adminUsername := "user1"
+ forgejoAPI := f3_forges.NewForgeRootFromDriver(&f3_forgejo.Forgejo{}, &f3_forgejo.Options{
+ Options: gof3.Options{
+ Configuration: gof3.Configuration{
+ URL: setting.AppURL,
+ Directory: t.TempDir(),
+ },
+ Features: gof3.AllFeatures,
+ Logger: util.ToF3Logger(nil),
+ },
+ AuthToken: getUserToken(t, adminUsername, auth_model.AccessTokenScopeWriteAdmin, auth_model.AccessTokenScopeAll),
+ })
+
+ f3 := f3_forges.FixtureNewF3Forge(t, nil, t.TempDir())
+ apiForge := forgejoAPI.Forge
+ apiUser := apiForge.Users.GetFromFormat(context.Background(), &format.User{UserName: gitlabUser.Name})
+ // apiProject := apiUser.Projects.GetFromFormat(context.Background(), &format.Project{Name: fixture.ProjectFormat.Name})
+ // options = f3_common.NewMirrorOptionsRecurse(apiUser, apiProject)
+ options = f3_common.NewMirrorOptionsRecurse(apiUser)
+ f3.Forge.Mirror(context.Background(), apiForge, options)
+
+ //
+ // Step 4: verify the fixture and F3 are equivalent
+ //
+ files := f3_util.Command(context.Background(), "find", f3.GetDirectory())
+ assert.Contains(t, files, fmt.Sprintf("/user/%d", gitlabUser.ID))
+ })
+}