Merge remote-tracking branch 'giteaofficial/main'

* giteaofficial/main:
  auth/reverseproxy: Add support for full name (go-gitea#20776)
  Add migrate repo archiver and packages storage support on command line (go-gitea#20757)
  Refactor AssertExistsAndLoadBean to use generics (go-gitea#20797)
  [skip ci] Updated translations via Crowdin
  Fix the commit diff route (go-gitea#20796)
  Fix push mirror address backend get error Address cause setting page display error (go-gitea#20593)

Author: zjjhot

cmd/main_test.go

@@ -0,0 +1,23 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cmd
+
+import (
+	"testing"
+
+	"code.gitea.io/gitea/models/unittest"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+func init() {
+	setting.SetCustomPathAndConf("", "", "")
+	setting.LoadForTest()
+}
+
+func TestMain(m *testing.M) {
+	unittest.MainTest(m, &unittest.TestOptions{
+		GiteaRootPath: "..",
+	})
+}

cmd/migrate_storage.go

@@ -12,9 +12,11 @@ import (
 	"code.gitea.io/gitea/models/db"
 	git_model "code.gitea.io/gitea/models/git"
 	"code.gitea.io/gitea/models/migrations"
+	packages_model "code.gitea.io/gitea/models/packages"
 	repo_model "code.gitea.io/gitea/models/repo"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/log"
+	packages_module "code.gitea.io/gitea/modules/packages"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/storage"
 
@@ -25,13 +27,13 @@ import (
 var CmdMigrateStorage = cli.Command{
 	Name:        "migrate-storage",
 	Usage:       "Migrate the storage",
-	Description: "This is a command for migrating storage.",
+	Description: "Copies stored files from storage configured in app.ini to parameter-configured storage",
 	Action:      runMigrateStorage,
 	Flags: []cli.Flag{
 		cli.StringFlag{
 			Name:  "type, t",
 			Value: "",
-			Usage: "Kinds of files to migrate, currently only 'attachments' is supported",
+			Usage: "Type of stored files to copy.  Allowed types: 'attachments', 'lfs', 'avatars', 'repo-avatars', 'repo-archivers', 'packages'",
 		},
 		cli.StringFlag{
 			Name:  "storage, s",
@@ -80,34 +82,53 @@ var CmdMigrateStorage = cli.Command{
 	},
 }
 
-func migrateAttachments(dstStorage storage.ObjectStorage) error {
-	return repo_model.IterateAttachment(func(attach *repo_model.Attachment) error {
+func migrateAttachments(ctx context.Context, dstStorage storage.ObjectStorage) error {
+	return db.IterateObjects(ctx, func(attach *repo_model.Attachment) error {
 		_, err := storage.Copy(dstStorage, attach.RelativePath(), storage.Attachments, attach.RelativePath())
 		return err
 	})
 }
 
-func migrateLFS(dstStorage storage.ObjectStorage) error {
-	return git_model.IterateLFS(func(mo *git_model.LFSMetaObject) error {
+func migrateLFS(ctx context.Context, dstStorage storage.ObjectStorage) error {
+	return db.IterateObjects(ctx, func(mo *git_model.LFSMetaObject) error {
 		_, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath())
 		return err
 	})
 }
 
-func migrateAvatars(dstStorage storage.ObjectStorage) error {
-	return user_model.IterateUser(func(user *user_model.User) error {
+func migrateAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
+	return db.IterateObjects(ctx, func(user *user_model.User) error {
 		_, err := storage.Copy(dstStorage, user.CustomAvatarRelativePath(), storage.Avatars, user.CustomAvatarRelativePath())
 		return err
 	})
 }
 
-func migrateRepoAvatars(dstStorage storage.ObjectStorage) error {
-	return repo_model.IterateRepository(func(repo *repo_model.Repository) error {
+func migrateRepoAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
+	return db.IterateObjects(ctx, func(repo *repo_model.Repository) error {
 		_, err := storage.Copy(dstStorage, repo.CustomAvatarRelativePath(), storage.RepoAvatars, repo.CustomAvatarRelativePath())
 		return err
 	})
 }
 
+func migrateRepoArchivers(ctx context.Context, dstStorage storage.ObjectStorage) error {
+	return db.IterateObjects(ctx, func(archiver *repo_model.RepoArchiver) error {
+		p, err := archiver.RelativePath()
+		if err != nil {
+			return err
+		}
+		_, err = storage.Copy(dstStorage, p, storage.RepoArchives, p)
+		return err
+	})
+}
+
+func migratePackages(ctx context.Context, dstStorage storage.ObjectStorage) error {
+	return db.IterateObjects(ctx, func(pb *packages_model.PackageBlob) error {
+		p := packages_module.KeyToRelativePath(packages_module.BlobHash256Key(pb.HashSHA256))
+		_, err := storage.Copy(dstStorage, p, storage.Packages, p)
+		return err
+	})
+}
+
 func runMigrateStorage(ctx *cli.Context) error {
 	stdCtx, cancel := installSignals()
 	defer cancel()
@@ -127,8 +148,6 @@ func runMigrateStorage(ctx *cli.Context) error {
 		return err
 	}
 
-	goCtx := context.Background()
-
 	if err := storage.Init(); err != nil {
 		return err
 	}
@@ -145,13 +164,13 @@ func runMigrateStorage(ctx *cli.Context) error {
 			return nil
 		}
 		dstStorage, err = storage.NewLocalStorage(
-			goCtx,
+			stdCtx,
 			storage.LocalStorageConfig{
 				Path: p,
 			})
 	case string(storage.MinioStorageType):
 		dstStorage, err = storage.NewMinioStorage(
-			goCtx,
+			stdCtx,
 			storage.MinioStorageConfig{
 				Endpoint:        ctx.String("minio-endpoint"),
 				AccessKeyID:     ctx.String("minio-access-key-id"),
@@ -162,35 +181,29 @@ func runMigrateStorage(ctx *cli.Context) error {
 				UseSSL:          ctx.Bool("minio-use-ssl"),
 			})
 	default:
-		return fmt.Errorf("Unsupported storage type: %s", ctx.String("storage"))
+		return fmt.Errorf("unsupported storage type: %s", ctx.String("storage"))
 	}
 	if err != nil {
 		return err
 	}
 
+	migratedMethods := map[string]func(context.Context, storage.ObjectStorage) error{
+		"attachments":    migrateAttachments,
+		"lfs":            migrateLFS,
+		"avatars":        migrateAvatars,
+		"repo-avatars":   migrateRepoAvatars,
+		"repo-archivers": migrateRepoArchivers,
+		"packages":       migratePackages,
+	}
+
 	tp := strings.ToLower(ctx.String("type"))
-	switch tp {
-	case "attachments":
-		if err := migrateAttachments(dstStorage); err != nil {
-			return err
-		}
-	case "lfs":
-		if err := migrateLFS(dstStorage); err != nil {
-			return err
-		}
-	case "avatars":
-		if err := migrateAvatars(dstStorage); err != nil {
-			return err
-		}
-	case "repo-avatars":
-		if err := migrateRepoAvatars(dstStorage); err != nil {
+	if m, ok := migratedMethods[tp]; ok {
+		if err := m(stdCtx, dstStorage); err != nil {
 			return err
 		}
-	default:
-		return fmt.Errorf("Unsupported storage: %s", ctx.String("type"))
+		log.Info("%s files have successfully been copied to the new storage.", tp)
+		return nil
 	}
 
-	log.Warn("All files have been copied to the new placement but old files are still on the original placement.")
-
-	return nil
+	return fmt.Errorf("unsupported storage: %s", ctx.String("type"))
 }

cmd/migrate_storage_test.go

@@ -0,0 +1,74 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cmd
+
+import (
+	"context"
+	"os"
+	"strings"
+	"testing"
+
+	"code.gitea.io/gitea/models/packages"
+	"code.gitea.io/gitea/models/unittest"
+	user_model "code.gitea.io/gitea/models/user"
+	packages_module "code.gitea.io/gitea/modules/packages"
+	"code.gitea.io/gitea/modules/storage"
+	packages_service "code.gitea.io/gitea/services/packages"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMigratePackages(t *testing.T) {
+	assert.NoError(t, unittest.PrepareTestDatabase())
+
+	creator := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
+
+	content := "package main\n\nfunc main() {\nfmt.Println(\"hi\")\n}\n"
+	buf, err := packages_module.CreateHashedBufferFromReader(strings.NewReader(content), 1024)
+	assert.NoError(t, err)
+	defer buf.Close()
+
+	v, f, err := packages_service.CreatePackageAndAddFile(&packages_service.PackageCreationInfo{
+		PackageInfo: packages_service.PackageInfo{
+			Owner:       creator,
+			PackageType: packages.TypeGeneric,
+			Name:        "test",
+			Version:     "1.0.0",
+		},
+		Creator:           creator,
+		SemverCompatible:  true,
+		VersionProperties: map[string]string{},
+	}, &packages_service.PackageFileCreationInfo{
+		PackageFileInfo: packages_service.PackageFileInfo{
+			Filename: "a.go",
+		},
+		Data:   buf,
+		IsLead: true,
+	})
+	assert.NoError(t, err)
+	assert.NotNil(t, v)
+	assert.NotNil(t, f)
+
+	ctx := context.Background()
+
+	p, err := os.MkdirTemp(os.TempDir(), "migrated_packages")
+	assert.NoError(t, err)
+
+	dstStorage, err := storage.NewLocalStorage(
+		ctx,
+		storage.LocalStorageConfig{
+			Path: p,
+		})
+	assert.NoError(t, err)
+
+	err = migratePackages(ctx, dstStorage)
+	assert.NoError(t, err)
+
+	entries, err := os.ReadDir(p)
+	assert.NoError(t, err)
+	assert.EqualValues(t, 2, len(entries))
+	assert.EqualValues(t, "01", entries[0].Name())
+	assert.EqualValues(t, "tmp", entries[1].Name())
+}

custom/conf/app.example.ini

@@ -377,9 +377,10 @@ INTERNAL_TOKEN=
 ;; Name of cookie used to store authentication information.
 ;COOKIE_REMEMBER_NAME = gitea_incredible
 ;;
-;; Reverse proxy authentication header name of user name and email
+;; Reverse proxy authentication header name of user name, email, and full name
 ;REVERSE_PROXY_AUTHENTICATION_USER = X-WEBAUTH-USER
 ;REVERSE_PROXY_AUTHENTICATION_EMAIL = X-WEBAUTH-EMAIL
+;REVERSE_PROXY_AUTHENTICATION_FULL_NAME = X-WEBAUTH-FULLNAME
 ;;
 ;; Interpret X-Forwarded-For header or the X-Real-IP header and set this as the remote IP for the request
 ;REVERSE_PROXY_LIMIT = 1
@@ -694,6 +695,7 @@ ROUTER = console
 ;ENABLE_REVERSE_PROXY_AUTHENTICATION = false
 ;ENABLE_REVERSE_PROXY_AUTO_REGISTRATION = false
 ;ENABLE_REVERSE_PROXY_EMAIL = false
+;ENABLE_REVERSE_PROXY_FULL_NAME = false
 ;;
 ;; Enable captcha validation for registration
 ;ENABLE_CAPTCHA = false

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -492,6 +492,8 @@ Certain queues have defaults that override the defaults set in `[queue]` (this o
    authentication.
 - `REVERSE_PROXY_AUTHENTICATION_EMAIL`: **X-WEBAUTH-EMAIL**: Header name for reverse proxy
    authentication provided email.
+- `REVERSE_PROXY_AUTHENTICATION_FULL_NAME`: **X-WEBAUTH-FULLNAME**: Header name for reverse proxy
+   authentication provided full name.
 - `REVERSE_PROXY_LIMIT`: **1**: Interpret X-Forwarded-For header or the X-Real-IP header and set this as the remote IP for the request.
    Number of trusted proxy count. Set to zero to not use these headers.
 - `REVERSE_PROXY_TRUSTED_PROXIES`: **127.0.0.0/8,::1/128**: List of IP addresses and networks separated by comma of trusted proxy servers. Use `*` to trust all.
@@ -577,6 +579,8 @@ Certain queues have defaults that override the defaults set in `[queue]` (this o
    for reverse authentication.
 - `ENABLE_REVERSE_PROXY_EMAIL`: **false**: Enable this to allow to auto-registration with a
    provided email rather than a generated email.
+- `ENABLE_REVERSE_PROXY_FULL_NAME`: **false**: Enable this to allow to auto-registration with a
+   provided full name for the user.
 - `ENABLE_CAPTCHA`: **false**: Enable this to use captcha validation for registration.
 - `REQUIRE_EXTERNAL_REGISTRATION_CAPTCHA`: **false**: Enable this to force captcha validation
    even for External Accounts (i.e. GitHub, OpenID Connect, etc). You also must enable `ENABLE_CAPTCHA`.

integrations/admin_user_test.go

@@ -61,7 +61,7 @@ func makeRequest(t *testing.T, formData user_model.User, headerCode int) {
 	})
 
 	session.MakeRequest(t, req, headerCode)
-	user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: formData.ID}).(*user_model.User)
+	user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: formData.ID})
 	assert.Equal(t, formData.Name, user.Name)
 	assert.Equal(t, formData.LoginName, user.LoginName)
 	assert.Equal(t, formData.Email, user.Email)

integrations/api_admin_test.go

@@ -22,7 +22,7 @@ func TestAPIAdminCreateAndDeleteSSHKey(t *testing.T) {
 	defer prepareTestEnv(t)()
 	// user1 is an admin user
 	session := loginUser(t, "user1")
-	keyOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: "user2"}).(*user_model.User)
+	keyOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: "user2"})
 
 	token := getTokenForLoggedInUser(t, session)
 	urlStr := fmt.Sprintf("/api/v1/admin/users/%s/keys?token=%s", keyOwner.Name, token)
@@ -194,7 +194,7 @@ func TestAPIEditUser(t *testing.T) {
 	json.Unmarshal(resp.Body.Bytes(), &errMap)
 	assert.EqualValues(t, "email is not allowed to be empty string", errMap["message"].(string))
 
-	user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{LoginName: "user2"}).(*user_model.User)
+	user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{LoginName: "user2"})
 	assert.False(t, user2.IsRestricted)
 	bTrue := true
 	req = NewRequestWithJSON(t, "PATCH", urlStr, api.EditUserOption{
@@ -205,6 +205,6 @@ func TestAPIEditUser(t *testing.T) {
 		Restricted: &bTrue,
 	})
 	session.MakeRequest(t, req, http.StatusOK)
-	user2 = unittest.AssertExistsAndLoadBean(t, &user_model.User{LoginName: "user2"}).(*user_model.User)
+	user2 = unittest.AssertExistsAndLoadBean(t, &user_model.User{LoginName: "user2"})
 	assert.True(t, user2.IsRestricted)
 }