From bc7912c698c2454176d0b3f3f78d8390ec637d93 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 5 Nov 2019 20:19:20 +0000 Subject: [PATCH 1/5] Migrate from git count-objects to a raw directory size --- models/migrations/v28.go | 4 ++-- models/repo.go | 7 ++++--- modules/git/repo.go | 4 ++-- modules/util/path.go | 20 +++++++++++++++++++- 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/models/migrations/v28.go b/models/migrations/v28.go index 587e944ce64e5..a849fea3c23cf 100644 --- a/models/migrations/v28.go +++ b/models/migrations/v28.go @@ -60,9 +60,9 @@ func addRepoSize(x *xorm.Engine) (err error) { } repoPath := filepath.Join(setting.RepoRootPath, strings.ToLower(user.Name), strings.ToLower(repo.Name)) + ".git" - countObject, err := git.GetRepoSize(repoPath) + countObject, err := git.CountObjects(repoPath) if err != nil { - log.Warn("GetRepoSize: %v", err) + log.Warn("CountObjects: %v", err) continue } diff --git a/models/repo.go b/models/repo.go index 7945cb309d341..0183c52f70733 100644 --- a/models/repo.go +++ b/models/repo.go @@ -36,6 +36,7 @@ import ( api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/sync" "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" "github.com/mcuadros/go-version" "github.com/unknwon/com" @@ -708,17 +709,17 @@ func (repo *Repository) IsOwnedBy(userID int64) bool { } func (repo *Repository) updateSize(e Engine) error { - repoInfoSize, err := git.GetRepoSize(repo.repoPath(e)) + size, err := util.GetDirectorySize(repo.repoPath(e)) if err != nil { return fmt.Errorf("UpdateSize: %v", err) } - repo.Size = repoInfoSize.Size + repoInfoSize.SizePack + repo.Size = size _, err = e.ID(repo.ID).Cols("size").Update(repo) return err } -// UpdateSize updates the repository size, calculating it using git.GetRepoSize +// UpdateSize updates the repository size, calculating it using git.CountObjects func (repo *Repository) UpdateSize() error { return repo.updateSize(x) } diff --git a/modules/git/repo.go b/modules/git/repo.go index e1d75ca4aaa52..f1e68c1a2b221 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -299,8 +299,8 @@ const ( statSizeGarbage = "size-garbage: " ) -// GetRepoSize returns disk consumption for repo in path -func GetRepoSize(repoPath string) (*CountObject, error) { +// CountObjects returns the results of git count-objects on the repoPath +func CountObjects(repoPath string) (*CountObject, error) { cmd := NewCommand("count-objects", "-v") stdout, err := cmd.RunInDir(repoPath) if err != nil { diff --git a/modules/util/path.go b/modules/util/path.go index f79334209cd4f..ca245abd919db 100644 --- a/modules/util/path.go +++ b/modules/util/path.go @@ -4,7 +4,10 @@ package util -import "path/filepath" +import ( + "os" + "path/filepath" +) // EnsureAbsolutePath ensure that a path is absolute, making it // relative to absoluteBase if necessary @@ -14,3 +17,18 @@ func EnsureAbsolutePath(path string, absoluteBase string) string { } return filepath.Join(absoluteBase, path) } + +// GetDirectorySize returns the dumb disk consumption for a given path +func GetDirectorySize(path string) (int64, error) { + var size int64 + err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + size += info.Size() + } + return err + }) + return size, err +} From 97683d080798650893e50d89659ffec274376137 Mon Sep 17 00:00:00 2001 From: zeripath Date: Tue, 5 Nov 2019 20:42:40 +0000 Subject: [PATCH 2/5] Update models/repo.go --- models/repo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/repo.go b/models/repo.go index 0183c52f70733..4a8159ad8c6f8 100644 --- a/models/repo.go +++ b/models/repo.go @@ -719,7 +719,7 @@ func (repo *Repository) updateSize(e Engine) error { return err } -// UpdateSize updates the repository size, calculating it using git.CountObjects +// UpdateSize updates the repository size, calculating it using util.GetDirectorySize func (repo *Repository) UpdateSize() error { return repo.updateSize(x) } From c8a212fde1fbcc009effad1cfe6d0b8317e2ac72 Mon Sep 17 00:00:00 2001 From: zeripath Date: Tue, 5 Nov 2019 23:55:27 +0000 Subject: [PATCH 3/5] Update modules/util/path.go Oops --- modules/util/path.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/util/path.go b/modules/util/path.go index ca245abd919db..8ca8b1a655702 100644 --- a/modules/util/path.go +++ b/modules/util/path.go @@ -25,7 +25,7 @@ func GetDirectorySize(path string) (int64, error) { if err != nil { return err } - if !info.IsDir() { + if info != nil && !info.IsDir() { size += info.Size() } return err From b12ddf7d1911a9d157fa6dd3810ba9527142f409 Mon Sep 17 00:00:00 2001 From: zeripath Date: Wed, 6 Nov 2019 01:42:52 +0000 Subject: [PATCH 4/5] Update path.go --- modules/util/path.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/util/path.go b/modules/util/path.go index 8ca8b1a655702..d28e9257ef736 100644 --- a/modules/util/path.go +++ b/modules/util/path.go @@ -22,9 +22,6 @@ func EnsureAbsolutePath(path string, absoluteBase string) string { func GetDirectorySize(path string) (int64, error) { var size int64 err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { - if err != nil { - return err - } if info != nil && !info.IsDir() { size += info.Size() } From 7f95e9202ab96f388275e669d9eb09a841934565 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 10 Nov 2019 18:13:15 +0000 Subject: [PATCH 5/5] As per @guillep2k ignore unusual files --- modules/util/path.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/util/path.go b/modules/util/path.go index ca245abd919db..1b35a1a38bf78 100644 --- a/modules/util/path.go +++ b/modules/util/path.go @@ -18,6 +18,8 @@ func EnsureAbsolutePath(path string, absoluteBase string) string { return filepath.Join(absoluteBase, path) } +const notRegularFileMode os.FileMode = os.ModeDir | os.ModeSymlink | os.ModeNamedPipe | os.ModeSocket | os.ModeDevice | os.ModeCharDevice | os.ModeIrregular + // GetDirectorySize returns the dumb disk consumption for a given path func GetDirectorySize(path string) (int64, error) { var size int64 @@ -25,7 +27,7 @@ func GetDirectorySize(path string) (int64, error) { if err != nil { return err } - if !info.IsDir() { + if info != nil && (info.Mode()¬RegularFileMode) == 0 { size += info.Size() } return err