diff --git a/pkg/downloader/downloader.go b/pkg/downloader/downloader.go index 440b6bbc..b61d25f6 100644 --- a/pkg/downloader/downloader.go +++ b/pkg/downloader/downloader.go @@ -377,40 +377,102 @@ func (d *OciDownloader) Download(opts DownloadOptions) error { } func (d *GitDownloader) Download(opts DownloadOptions) error { - var msg string - if len(opts.Source.Git.Tag) != 0 { - msg = fmt.Sprintf("with tag '%s'", opts.Source.Git.Tag) - } - - if len(opts.Source.Git.Commit) != 0 { - msg = fmt.Sprintf("with commit '%s'", opts.Source.Git.Commit) - } - - if len(opts.Source.Git.Branch) != 0 { - msg = fmt.Sprintf("with branch '%s'", opts.Source.Git.Branch) - } - - reporter.ReportMsgTo( - fmt.Sprintf("cloning '%s' %s", opts.Source.Git.Url, msg), - opts.LogWriter, - ) - // download the package from the git repo gitSource := opts.Source.Git if gitSource == nil { return errors.New("git source is nil") } - - _, err := git.CloneWithOpts( + cloneOpts := []git.CloneOption{ git.WithCommit(gitSource.Commit), git.WithBranch(gitSource.Branch), git.WithTag(gitSource.Tag), - git.WithRepoURL(gitSource.Url), - git.WithLocalPath(opts.LocalPath), - ) - - if err != nil { - return err } + if ok, err := features.Enabled(features.SupportNewStorage); err == nil && ok { + if opts.EnableCache { + hash, err := gitSource.Hash() + if err != nil { + return err + } + + cacheFullPath := filepath.Join(opts.CachePath, hash) + localFullPath := filepath.Join(opts.LocalPath, hash) + // Check if the package is already downloaded, if so, skip the download. + if utils.DirExists(localFullPath) && + utils.DirExists(filepath.Join(localFullPath, constants.KCL_MOD)) { + return nil + } else { + // Try to clone the bare repository from the cache path. + _, err := git.CloneWithOpts( + append( + cloneOpts, + git.WithRepoURL(cacheFullPath), + git.WithLocalPath(localFullPath), + )..., + ) + // If failed to clone the bare repository from the cache path, + // clone the bare repository from the remote git repository, update the cache. + if err != nil { + _, err := git.CloneWithOpts( + append( + cloneOpts, + git.WithRepoURL(gitSource.Url), + git.WithLocalPath(cacheFullPath), + git.WithBare(true), + )..., + ) + if err != nil { + return err + } + } + // After cloning the bare repository, + // Clone the repository from the cache path to the local path. + _, err = git.CloneWithOpts( + append( + cloneOpts, + git.WithRepoURL(cacheFullPath), + git.WithLocalPath(localFullPath), + )..., + ) + if err != nil { + return err + } + } + } + } else { + var msg string + if len(opts.Source.Git.Tag) != 0 { + msg = fmt.Sprintf("with tag '%s'", opts.Source.Git.Tag) + } + + if len(opts.Source.Git.Commit) != 0 { + msg = fmt.Sprintf("with commit '%s'", opts.Source.Git.Commit) + } + + if len(opts.Source.Git.Branch) != 0 { + msg = fmt.Sprintf("with branch '%s'", opts.Source.Git.Branch) + } + + reporter.ReportMsgTo( + fmt.Sprintf("cloning '%s' %s", opts.Source.Git.Url, msg), + opts.LogWriter, + ) + // download the package from the git repo + gitSource := opts.Source.Git + if gitSource == nil { + return errors.New("git source is nil") + } + + _, err := git.CloneWithOpts( + git.WithCommit(gitSource.Commit), + git.WithBranch(gitSource.Branch), + git.WithTag(gitSource.Tag), + git.WithRepoURL(gitSource.Url), + git.WithLocalPath(opts.LocalPath), + ) + + if err != nil { + return err + } + } return nil } diff --git a/pkg/downloader/downloader_test.go b/pkg/downloader/downloader_test.go index c93a5533..d81527c3 100644 --- a/pkg/downloader/downloader_test.go +++ b/pkg/downloader/downloader_test.go @@ -1,11 +1,16 @@ package downloader import ( + "fmt" "os" "path/filepath" "testing" "gotest.tools/v3/assert" + "kcl-lang.io/kpm/pkg/features" + "kcl-lang.io/kpm/pkg/git" + "kcl-lang.io/kpm/pkg/test" + "kcl-lang.io/kpm/pkg/utils" ) const testDataDir = "test_data" @@ -18,7 +23,7 @@ func getTestDir(subDir string) string { return testDir } -func TestOciDownloader(t *testing.T) { +func testOciDownloader(t *testing.T) { path_oci := getTestDir("test_oci") if err := os.MkdirAll(path_oci, os.ModePerm); err != nil { t.Fatal(err) @@ -44,9 +49,12 @@ func TestOciDownloader(t *testing.T) { )) assert.Equal(t, err, nil) +} - path_git := getTestDir("test_git") - if err := os.MkdirAll(path_oci, os.ModePerm); err != nil { +func testGitDownloader(t *testing.T) { + features.Enable(features.SupportNewStorage) + path_git := getTestDir("test_git_bare_repo") + if err := os.MkdirAll(path_git, os.ModePerm); err != nil { t.Fatal(err) } @@ -55,16 +63,30 @@ func TestOciDownloader(t *testing.T) { }() gitDownloader := GitDownloader{} + gitSource := Source{ + Git: &Git{ + Url: "https://github.com/kcl-lang/flask-demo-kcl-manifests.git", + Commit: "ade147b", + }, + } - err = gitDownloader.Download(*NewDownloadOptions( - WithSource(Source{ - Git: &Git{ - Url: "https://github.com/kcl-lang/flask-demo-kcl-manifests.git", - Commit: "ade147b", - }, - }), - WithLocalPath(path_git), + err := gitDownloader.Download(*NewDownloadOptions( + WithSource(gitSource), + WithLocalPath(filepath.Join(path_git, "git", "checkout")), + WithCachePath(filepath.Join(path_git, "git", "db")), + WithEnableCache(true), )) + fmt.Printf("err: %v\n", err) + assert.Equal(t, err, nil) + gitHash, err := gitSource.Hash() assert.Equal(t, err, nil) + assert.Equal(t, git.IsGitBareRepo(filepath.Join(path_git, "git", "db", gitHash)), true) + assert.Equal(t, utils.DirExists(filepath.Join(path_git, "git", "checkout", gitHash)), true) + assert.Equal(t, utils.DirExists(filepath.Join(path_git, "git", "checkout", gitHash, "kcl.mod")), true) +} + +func TestWithGlobalLock(t *testing.T) { + test.RunTestWithGlobalLock(t, "TestOciDownloader", testOciDownloader) + test.RunTestWithGlobalLock(t, "TestGitDownloader", testGitDownloader) } diff --git a/pkg/downloader/source.go b/pkg/downloader/source.go index c0e14775..77943038 100644 --- a/pkg/downloader/source.go +++ b/pkg/downloader/source.go @@ -566,7 +566,24 @@ func (s *Source) Hash() (string, error) { } func (g *Git) Hash() (string, error) { - return utils.ShortHash(g.Url) + gitURL := strings.TrimSuffix(g.Url, filepath.Ext(g.Url)) + packageFilename := filepath.Base(gitURL) + filenamePattern := "%s_%s" + + if g.Tag != "" { + packageFilename = fmt.Sprintf(filenamePattern, packageFilename, g.Tag) + } else if g.Commit != "" { + packageFilename = fmt.Sprintf(filenamePattern, packageFilename, g.Commit) + } else if g.Branch != "" { + packageFilename = fmt.Sprintf(filenamePattern, packageFilename, g.Branch) + } + + hash, err := utils.ShortHash(filepath.Dir(gitURL)) + if err != nil { + return "", err + } + + return filepath.Join(hash, packageFilename), nil } func (o *Oci) Hash() (string, error) { diff --git a/pkg/git/getter.go b/pkg/git/getter.go index e9472fce..d957354d 100644 --- a/pkg/git/getter.go +++ b/pkg/git/getter.go @@ -3,6 +3,7 @@ package git import ( "fmt" + "net/url" "github.com/hashicorp/go-getter" "kcl-lang.io/kpm/pkg/constants" @@ -27,6 +28,19 @@ func (cloneOpts *CloneOptions) ForceGitUrl() (string, error) { return "", nil } + repoUrl, err := url.Parse(cloneOpts.RepoURL) + if err != nil { + return "", err + } + + // If the Git URL is a file path, which is a local bare repo, + // we need to force the protocol to "file://" + if repoUrl.Scheme == "" { + repoUrl.Scheme = "file" + } + + cloneOpts.RepoURL = repoUrl.String() + var attributes = []string{cloneOpts.Branch, cloneOpts.Commit, cloneOpts.Tag} for _, attr := range attributes { if attr != "" { diff --git a/pkg/git/git.go b/pkg/git/git.go index 6ece5128..10a731cd 100644 --- a/pkg/git/git.go +++ b/pkg/git/git.go @@ -8,6 +8,7 @@ import ( "net/http" "os/exec" "regexp" + "strings" "time" "github.com/go-git/go-git/v5" @@ -301,3 +302,13 @@ func GetAllGithubReleases(url string) ([]string, error) { return releaseTags, nil } + +// IsGitBareRepo checks if a directory is a bare git repository +func IsGitBareRepo(dir string) bool { + cmd := exec.Command("git", "-C", dir, "rev-parse", "--is-bare-repository") + output, err := cmd.Output() + if err != nil { + return false + } + return strings.TrimSpace(string(output)) == "true" +} diff --git a/pkg/git/git_test.go b/pkg/git/git_test.go index 7625b409..d9ce61d0 100644 --- a/pkg/git/git_test.go +++ b/pkg/git/git_test.go @@ -2,7 +2,6 @@ package git import ( "bytes" - "fmt" "os" "os/exec" "path/filepath" @@ -161,9 +160,6 @@ func TestCloneWithOptions(t *testing.T) { err = cmd.Run() assert.Equal(t, err, nil) - // Construct the file URL for the local bare repository - bareRepoURL := fmt.Sprintf("file://%s", bareRepoPath) - // Clone the local bare repository as a normal repository and checkout a commit tmpdir, err := os.MkdirTemp("", "clone_non_bare_repo") assert.Equal(t, err, nil) @@ -173,7 +169,7 @@ func TestCloneWithOptions(t *testing.T) { }() repo, err := CloneWithOpts( - WithRepoURL(bareRepoURL), + WithRepoURL(bareRepoPath), WithCommit("4e59d5852cd76542f9f0ec65e5773ca9f4e02462"), WithWriter(&buf), WithLocalPath(tmpdir),