Add migrate repo archiver and packages storage support on command line

cmd/migrate_storage.go

@@ -12,9 +12,11 @@ import (
 	"code.gitea.io/gitea/models/db"
 	git_model "code.gitea.io/gitea/models/git"
 	"code.gitea.io/gitea/models/migrations"
+	packages_model "code.gitea.io/gitea/models/packages"
 	repo_model "code.gitea.io/gitea/models/repo"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/log"
+	packages_module "code.gitea.io/gitea/modules/packages"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/storage"
 
@@ -81,33 +83,58 @@ var CmdMigrateStorage = cli.Command{
 }
 
 func migrateAttachments(dstStorage storage.ObjectStorage) error {
-	return repo_model.IterateAttachment(func(attach *repo_model.Attachment) error {
+	return db.IterateObjects(db.DefaultContext, func(attach *repo_model.Attachment) error {
 		_, err := storage.Copy(dstStorage, attach.RelativePath(), storage.Attachments, attach.RelativePath())
 		return err
 	})
 }
 
 func migrateLFS(dstStorage storage.ObjectStorage) error {
-	return git_model.IterateLFS(func(mo *git_model.LFSMetaObject) error {
+	return db.IterateObjects(db.DefaultContext, func(mo *git_model.LFSMetaObject) error {
 		_, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath())
 		return err
 	})
 }
 
 func migrateAvatars(dstStorage storage.ObjectStorage) error {
-	return user_model.IterateUser(func(user *user_model.User) error {
+	return db.IterateObjects(db.DefaultContext, func(user *user_model.User) error {
 		_, err := storage.Copy(dstStorage, user.CustomAvatarRelativePath(), storage.Avatars, user.CustomAvatarRelativePath())
 		return err
 	})
 }
 
 func migrateRepoAvatars(dstStorage storage.ObjectStorage) error {
-	return repo_model.IterateRepository(func(repo *repo_model.Repository) error {
+	return db.IterateObjects(db.DefaultContext, func(repo *repo_model.Repository) error {
 		_, err := storage.Copy(dstStorage, repo.CustomAvatarRelativePath(), storage.RepoAvatars, repo.CustomAvatarRelativePath())
 		return err
 	})
 }
 
+func migrateRepoArchivers(dstStorage storage.ObjectStorage) error {
+	return db.IterateObjects(db.DefaultContext, func(archiver *repo_model.RepoArchiver) error {
+		p, err := archiver.RelativePath()
+		if err != nil {
+			return err
+		}
+		_, err = storage.Copy(dstStorage, p, storage.RepoArchives, p)
+		return err
+	})
+}
+
+func migratePackages(dstStorage storage.ObjectStorage) error {
+	ctx := db.DefaultContext
+	return db.IterateObjects(ctx, func(pf *packages_model.PackageFile) error {
+		pb, err := packages_model.GetBlobByID(ctx, pf.BlobID)
+		if err != nil {
+			return err
+		}
+
+		p := packages_module.KeyToRelativePath(packages_module.BlobHash256Key(pb.HashSHA256))
+		_, err = storage.Copy(dstStorage, p, storage.RepoAvatars, p)
+		return err
+	})
+}
+
 func runMigrateStorage(ctx *cli.Context) error {
 	stdCtx, cancel := installSignals()
 	defer cancel()
@@ -168,25 +195,21 @@ func runMigrateStorage(ctx *cli.Context) error {
 		return err
 	}
 
+	migratedMethods := map[string]func(storage.ObjectStorage) error{
+		"attachments":    migrateAttachments,
+		"lfs":            migrateLFS,
+		"avatars":        migrateAvatars,
+		"repo-avatars":   migrateRepoAvatars,
+		"repo-archivers": migrateRepoArchivers,
+		"packages":       migratePackages,
+	}
+
 	tp := strings.ToLower(ctx.String("type"))
-	switch tp {
-	case "attachments":
-		if err := migrateAttachments(dstStorage); err != nil {
-			return err
-		}
-	case "lfs":
-		if err := migrateLFS(dstStorage); err != nil {
+	if m, ok := migratedMethods[tp]; ok {
+		if err := m(dstStorage); err != nil {
 			return err
 		}
-	case "avatars":
-		if err := migrateAvatars(dstStorage); err != nil {
-			return err
-		}
-	case "repo-avatars":
-		if err := migrateRepoAvatars(dstStorage); err != nil {
-			return err
-		}
-	default:
+	} else {
 		return fmt.Errorf("Unsupported storage: %s", ctx.String("type"))
 	}
 

models/db/iterate.go

@@ -0,0 +1,34 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package db
+
+import (
+	"context"
+
+	"code.gitea.io/gitea/modules/setting"
+)
+
+// IterateObjects iterate all the Bean object
+func IterateObjects[Object any](ctx context.Context, f func(repo *Object) error) error {
+	var start int
+	batchSize := setting.Database.IterateBufferSize
+	sess := GetEngine(ctx)
+	for {
+		repos := make([]*Object, 0, batchSize)
+		if err := sess.Limit(batchSize, start).Find(&repos); err != nil {
+			return err
+		}
+		if len(repos) == 0 {
+			return nil
+		}
+		start += len(repos)
+
+		for _, repo := range repos {
+			if err := f(repo); err != nil {
+				return err
+			}
+		}
+	}
+}

models/git/lfs.go

@@ -278,29 +278,6 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *user_model.User, repoID int6
 	return committer.Commit()
 }
 
-// IterateLFS iterates lfs object
-func IterateLFS(f func(mo *LFSMetaObject) error) error {
-	var start int
-	const batchSize = 100
-	e := db.GetEngine(db.DefaultContext)
-	for {
-		mos := make([]*LFSMetaObject, 0, batchSize)
-		if err := e.Limit(batchSize, start).Find(&mos); err != nil {
-			return err
-		}
-		if len(mos) == 0 {
-			return nil
-		}
-		start += len(mos)
-
-		for _, mo := range mos {
-			if err := f(mo); err != nil {
-				return err
-			}
-		}
-	}
-}
-
 // CopyLFS copies LFS data from one repo to another
 func CopyLFS(ctx context.Context, newRepo, oldRepo *repo_model.Repository) error {
 	var lfsObjects []*LFSMetaObject

models/repo/attachment.go

@@ -226,28 +226,6 @@ func DeleteAttachmentsByRelease(releaseID int64) error {
 	return err
 }
 
-// IterateAttachment iterates attachments; it should not be used when Gitea is servicing users.
-func IterateAttachment(f func(attach *Attachment) error) error {
-	var start int
-	const batchSize = 100
-	for {
-		attachments := make([]*Attachment, 0, batchSize)
-		if err := db.GetEngine(db.DefaultContext).Limit(batchSize, start).Find(&attachments); err != nil {
-			return err
-		}
-		if len(attachments) == 0 {
-			return nil
-		}
-		start += len(attachments)
-
-		for _, attach := range attachments {
-			if err := f(attach); err != nil {
-				return err
-			}
-		}
-	}
-}
-
 // CountOrphanedAttachments returns the number of bad attachments
 func CountOrphanedAttachments() (int64, error) {
 	return db.GetEngine(db.DefaultContext).Where("(issue_id > 0 and issue_id not in (select id from issue)) or (release_id > 0 and release_id not in (select id from `release`))").

models/repo/repo_list.go

@@ -15,36 +15,12 @@ import (
 	"code.gitea.io/gitea/models/unit"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/container"
-	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/util"
 
 	"xorm.io/builder"
 )
 
-// IterateRepository iterate repositories
-func IterateRepository(f func(repo *Repository) error) error {
-	var start int
-	batchSize := setting.Database.IterateBufferSize
-	sess := db.GetEngine(db.DefaultContext)
-	for {
-		repos := make([]*Repository, 0, batchSize)
-		if err := sess.Limit(batchSize, start).Find(&repos); err != nil {
-			return err
-		}
-		if len(repos) == 0 {
-			return nil
-		}
-		start += len(repos)
-
-		for _, repo := range repos {
-			if err := f(repo); err != nil {
-				return err
-			}
-		}
-	}
-}
-
 // FindReposMapByIDs find repos as map
 func FindReposMapByIDs(repoIDs []int64, res map[int64]*Repository) error {
 	return db.GetEngine(db.DefaultContext).In("id", repoIDs).Find(&res)

models/user/search.go

@@ -9,7 +9,6 @@ import (
 	"strings"
 
 	"code.gitea.io/gitea/models/db"
-	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/util"
 
@@ -125,28 +124,6 @@ func SearchUsers(opts *SearchUserOptions) (users []*User, _ int64, _ error) {
 	return users, count, sessQuery.Find(&users)
 }
 
-// IterateUser iterate users
-func IterateUser(f func(user *User) error) error {
-	var start int
-	batchSize := setting.Database.IterateBufferSize
-	for {
-		users := make([]*User, 0, batchSize)
-		if err := db.GetEngine(db.DefaultContext).Limit(batchSize, start).Find(&users); err != nil {
-			return err
-		}
-		if len(users) == 0 {
-			return nil
-		}
-		start += len(users)
-
-		for _, user := range users {
-			if err := f(user); err != nil {
-				return err
-			}
-		}
-	}
-}
-
 // BuildCanSeeUserCondition creates a condition which can be used to restrict results to users/orgs the actor can see
 func BuildCanSeeUserCondition(actor *User) builder.Cond {
 	if actor != nil {

modules/packages/content_store.go

@@ -27,21 +27,21 @@ func NewContentStore() *ContentStore {
 
 // Get gets a package blob
 func (s *ContentStore) Get(key BlobHash256Key) (storage.Object, error) {
-	return s.store.Open(keyToRelativePath(key))
+	return s.store.Open(KeyToRelativePath(key))
 }
 
 // Save stores a package blob
 func (s *ContentStore) Save(key BlobHash256Key, r io.Reader, size int64) error {
-	_, err := s.store.Save(keyToRelativePath(key), r, size)
+	_, err := s.store.Save(KeyToRelativePath(key), r, size)
 	return err
 }
 
 // Delete deletes a package blob
 func (s *ContentStore) Delete(key BlobHash256Key) error {
-	return s.store.Delete(keyToRelativePath(key))
+	return s.store.Delete(KeyToRelativePath(key))
 }
 
-// keyToRelativePath converts the sha256 key aabb000000... to aa/bb/aabb000000...
-func keyToRelativePath(key BlobHash256Key) string {
+// KeyToRelativePath converts the sha256 key aabb000000... to aa/bb/aabb000000...
+func KeyToRelativePath(key BlobHash256Key) string {
 	return path.Join(string(key)[0:2], string(key)[2:4], string(key))
 }

routers/private/mail.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"strconv"
 
+	"code.gitea.io/gitea/models/db"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/json"
@@ -59,7 +60,7 @@ func SendEmail(ctx *context.PrivateContext) {
 			}
 		}
 	} else {
-		err := user_model.IterateUser(func(user *user_model.User) error {
+		err := db.IterateObjects(ctx, func(user *user_model.User) error {
 			if len(user.Email) > 0 && user.IsActive {
 				emails = append(emails, user.Email)
 			}

services/repository/avatar.go

@@ -96,7 +96,7 @@ func DeleteAvatar(repo *repo_model.Repository) error {
 
 // RemoveRandomAvatars removes the randomly generated avatars that were created for repositories
 func RemoveRandomAvatars(ctx context.Context) error {
-	return repo_model.IterateRepository(func(repository *repo_model.Repository) error {
+	return db.IterateObjects(ctx, func(repository *repo_model.Repository) error {
 		select {
 		case <-ctx.Done():
 			return db.ErrCancelledf("before random avatars removed for %s", repository.FullName())