Merge pull request #505 from zong-zhe/git-downloader-cache

feat: GitDownloader supports cache

pkg/downloader/downloader.go

@@ -377,40 +377,102 @@ func (d *OciDownloader) Download(opts DownloadOptions) error {
 }
 
 func (d *GitDownloader) Download(opts DownloadOptions) error {
-	var msg string
-	if len(opts.Source.Git.Tag) != 0 {
-		msg = fmt.Sprintf("with tag '%s'", opts.Source.Git.Tag)
-	}
-
-	if len(opts.Source.Git.Commit) != 0 {
-		msg = fmt.Sprintf("with commit '%s'", opts.Source.Git.Commit)
-	}
-
-	if len(opts.Source.Git.Branch) != 0 {
-		msg = fmt.Sprintf("with branch '%s'", opts.Source.Git.Branch)
-	}
-
-	reporter.ReportMsgTo(
-		fmt.Sprintf("cloning '%s' %s", opts.Source.Git.Url, msg),
-		opts.LogWriter,
-	)
-	// download the package from the git repo
 	gitSource := opts.Source.Git
 	if gitSource == nil {
 		return errors.New("git source is nil")
 	}
-
-	_, err := git.CloneWithOpts(
+	cloneOpts := []git.CloneOption{
 		git.WithCommit(gitSource.Commit),
 		git.WithBranch(gitSource.Branch),
 		git.WithTag(gitSource.Tag),
-		git.WithRepoURL(gitSource.Url),
-		git.WithLocalPath(opts.LocalPath),
-	)
-
-	if err != nil {
-		return err
 	}
 
+	if ok, err := features.Enabled(features.SupportNewStorage); err == nil && ok {
+		if opts.EnableCache {
+			hash, err := gitSource.Hash()
+			if err != nil {
+				return err
+			}
+
+			cacheFullPath := filepath.Join(opts.CachePath, hash)
+			localFullPath := filepath.Join(opts.LocalPath, hash)
+			// Check if the package is already downloaded, if so, skip the download.
+			if utils.DirExists(localFullPath) &&
+				utils.DirExists(filepath.Join(localFullPath, constants.KCL_MOD)) {
+				return nil
+			} else {
+				// Try to clone the bare repository from the cache path.
+				_, err := git.CloneWithOpts(
+					append(
+						cloneOpts,
+						git.WithRepoURL(cacheFullPath),
+						git.WithLocalPath(localFullPath),
+					)...,
+				)
+				// If failed to clone the bare repository from the cache path,
+				// clone the bare repository from the remote git repository, update the cache.
+				if err != nil {
+					_, err := git.CloneWithOpts(
+						append(
+							cloneOpts,
+							git.WithRepoURL(gitSource.Url),
+							git.WithLocalPath(cacheFullPath),
+							git.WithBare(true),
+						)...,
+					)
+					if err != nil {
+						return err
+					}
+				}
+				// After cloning the bare repository,
+				// Clone the repository from the cache path to the local path.
+				_, err = git.CloneWithOpts(
+					append(
+						cloneOpts,
+						git.WithRepoURL(cacheFullPath),
+						git.WithLocalPath(localFullPath),
+					)...,
+				)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	} else {
+		var msg string
+		if len(opts.Source.Git.Tag) != 0 {
+			msg = fmt.Sprintf("with tag '%s'", opts.Source.Git.Tag)
+		}
+
+		if len(opts.Source.Git.Commit) != 0 {
+			msg = fmt.Sprintf("with commit '%s'", opts.Source.Git.Commit)
+		}
+
+		if len(opts.Source.Git.Branch) != 0 {
+			msg = fmt.Sprintf("with branch '%s'", opts.Source.Git.Branch)
+		}
+
+		reporter.ReportMsgTo(
+			fmt.Sprintf("cloning '%s' %s", opts.Source.Git.Url, msg),
+			opts.LogWriter,
+		)
+		// download the package from the git repo
+		gitSource := opts.Source.Git
+		if gitSource == nil {
+			return errors.New("git source is nil")
+		}
+
+		_, err := git.CloneWithOpts(
+			git.WithCommit(gitSource.Commit),
+			git.WithBranch(gitSource.Branch),
+			git.WithTag(gitSource.Tag),
+			git.WithRepoURL(gitSource.Url),
+			git.WithLocalPath(opts.LocalPath),
+		)
+
+		if err != nil {
+			return err
+		}
+	}
 	return nil
 }

pkg/downloader/downloader_test.go

@@ -1,11 +1,16 @@
 package downloader
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"testing"
 
 	"gotest.tools/v3/assert"
+	"kcl-lang.io/kpm/pkg/features"
+	"kcl-lang.io/kpm/pkg/git"
+	"kcl-lang.io/kpm/pkg/test"
+	"kcl-lang.io/kpm/pkg/utils"
 )
 
 const testDataDir = "test_data"
@@ -18,7 +23,7 @@ func getTestDir(subDir string) string {
 	return testDir
 }
 
-func TestOciDownloader(t *testing.T) {
+func testOciDownloader(t *testing.T) {
 	path_oci := getTestDir("test_oci")
 	if err := os.MkdirAll(path_oci, os.ModePerm); err != nil {
 		t.Fatal(err)
@@ -44,9 +49,12 @@ func TestOciDownloader(t *testing.T) {
 	))
 
 	assert.Equal(t, err, nil)
+}
 
-	path_git := getTestDir("test_git")
-	if err := os.MkdirAll(path_oci, os.ModePerm); err != nil {
+func testGitDownloader(t *testing.T) {
+	features.Enable(features.SupportNewStorage)
+	path_git := getTestDir("test_git_bare_repo")
+	if err := os.MkdirAll(path_git, os.ModePerm); err != nil {
 		t.Fatal(err)
 	}
 
@@ -55,16 +63,30 @@ func TestOciDownloader(t *testing.T) {
 	}()
 
 	gitDownloader := GitDownloader{}
+	gitSource := Source{
+		Git: &Git{
+			Url:    "https://github.com/kcl-lang/flask-demo-kcl-manifests.git",
+			Commit: "ade147b",
+		},
+	}
 
-	err = gitDownloader.Download(*NewDownloadOptions(
-		WithSource(Source{
-			Git: &Git{
-				Url:    "https://github.com/kcl-lang/flask-demo-kcl-manifests.git",
-				Commit: "ade147b",
-			},
-		}),
-		WithLocalPath(path_git),
+	err := gitDownloader.Download(*NewDownloadOptions(
+		WithSource(gitSource),
+		WithLocalPath(filepath.Join(path_git, "git", "checkout")),
+		WithCachePath(filepath.Join(path_git, "git", "db")),
+		WithEnableCache(true),
 	))
 
+	fmt.Printf("err: %v\n", err)
+	assert.Equal(t, err, nil)
+	gitHash, err := gitSource.Hash()
 	assert.Equal(t, err, nil)
+	assert.Equal(t, git.IsGitBareRepo(filepath.Join(path_git, "git", "db", gitHash)), true)
+	assert.Equal(t, utils.DirExists(filepath.Join(path_git, "git", "checkout", gitHash)), true)
+	assert.Equal(t, utils.DirExists(filepath.Join(path_git, "git", "checkout", gitHash, "kcl.mod")), true)
+}
+
+func TestWithGlobalLock(t *testing.T) {
+	test.RunTestWithGlobalLock(t, "TestOciDownloader", testOciDownloader)
+	test.RunTestWithGlobalLock(t, "TestGitDownloader", testGitDownloader)
 }

pkg/downloader/source.go

@@ -566,7 +566,24 @@ func (s *Source) Hash() (string, error) {
 }
 
 func (g *Git) Hash() (string, error) {
-	return utils.ShortHash(g.Url)
+	gitURL := strings.TrimSuffix(g.Url, filepath.Ext(g.Url))
+	packageFilename := filepath.Base(gitURL)
+	filenamePattern := "%s_%s"
+
+	if g.Tag != "" {
+		packageFilename = fmt.Sprintf(filenamePattern, packageFilename, g.Tag)
+	} else if g.Commit != "" {
+		packageFilename = fmt.Sprintf(filenamePattern, packageFilename, g.Commit)
+	} else if g.Branch != "" {
+		packageFilename = fmt.Sprintf(filenamePattern, packageFilename, g.Branch)
+	}
+
+	hash, err := utils.ShortHash(filepath.Dir(gitURL))
+	if err != nil {
+		return "", err
+	}
+
+	return filepath.Join(hash, packageFilename), nil
 }
 
 func (o *Oci) Hash() (string, error) {

pkg/git/getter.go

@@ -3,6 +3,7 @@ package git
 
 import (
 	"fmt"
+	"net/url"
 
 	"github.com/hashicorp/go-getter"
 	"kcl-lang.io/kpm/pkg/constants"
@@ -27,6 +28,19 @@ func (cloneOpts *CloneOptions) ForceGitUrl() (string, error) {
 		return "", nil
 	}
 
+	repoUrl, err := url.Parse(cloneOpts.RepoURL)
+	if err != nil {
+		return "", err
+	}
+
+	// If the Git URL is a file path, which is a local bare repo,
+	// we need to force the protocol to "file://"
+	if repoUrl.Scheme == "" {
+		repoUrl.Scheme = "file"
+	}
+
+	cloneOpts.RepoURL = repoUrl.String()
+
 	var attributes = []string{cloneOpts.Branch, cloneOpts.Commit, cloneOpts.Tag}
 	for _, attr := range attributes {
 		if attr != "" {

pkg/git/git.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"os/exec"
 	"regexp"
+	"strings"
 	"time"
 
 	"github.com/go-git/go-git/v5"
@@ -301,3 +302,13 @@ func GetAllGithubReleases(url string) ([]string, error) {
 
 	return releaseTags, nil
 }
+
+// IsGitBareRepo checks if a directory is a bare git repository
+func IsGitBareRepo(dir string) bool {
+	cmd := exec.Command("git", "-C", dir, "rev-parse", "--is-bare-repository")
+	output, err := cmd.Output()
+	if err != nil {
+		return false
+	}
+	return strings.TrimSpace(string(output)) == "true"
+}

pkg/git/git_test.go

@@ -2,7 +2,6 @@ package git
 
 import (
 	"bytes"
-	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -161,9 +160,6 @@ func TestCloneWithOptions(t *testing.T) {
 		err = cmd.Run()
 		assert.Equal(t, err, nil)
 
-		// Construct the file URL for the local bare repository
-		bareRepoURL := fmt.Sprintf("file://%s", bareRepoPath)
-
 		// Clone the local bare repository as a normal repository and checkout a commit
 		tmpdir, err := os.MkdirTemp("", "clone_non_bare_repo")
 		assert.Equal(t, err, nil)
@@ -173,7 +169,7 @@ func TestCloneWithOptions(t *testing.T) {
 		}()
 
 		repo, err := CloneWithOpts(
-			WithRepoURL(bareRepoURL),
+			WithRepoURL(bareRepoPath),
 			WithCommit("4e59d5852cd76542f9f0ec65e5773ca9f4e02462"),
 			WithWriter(&buf),
 			WithLocalPath(tmpdir),