Sparse checkout feature - Download subdirectory

pkg/client/client.go

@@ -628,6 +628,7 @@ func (c *KpmClient) CompileGitPkg(gitOpts *git.CloneOptions, compileOpts *opt.Co
 		git.WithCommit(gitOpts.Commit),
 		git.WithBranch(gitOpts.Branch),
 		git.WithTag(gitOpts.Tag),
+		git.WithSubPackage(gitOpts.SubPackage),
 		git.WithRepoURL(gitOpts.RepoURL),
 		git.WithLocalPath(tmpDir),
 	)
@@ -1203,6 +1204,10 @@ func (c *KpmClient) DownloadFromGit(dep *downloader.Git, localPath string) (stri
 		msg = fmt.Sprintf("with commit '%s'", dep.Commit)
 	}
 
+	if len(dep.SubPackage) != 0 {
+		msg = fmt.Sprintf("with subpackage '%s'", dep.SubPackage)
+	}
+
 	if len(dep.Branch) != 0 {
 		msg = fmt.Sprintf("with branch '%s'", dep.Branch)
 	}
@@ -1216,6 +1221,7 @@ func (c *KpmClient) DownloadFromGit(dep *downloader.Git, localPath string) (stri
 		git.WithCommit(dep.Commit),
 		git.WithTag(dep.Tag),
 		git.WithRepoURL(dep.Url),
+		git.WithSubPackage(dep.SubPackage),
 		git.WithLocalPath(localPath),
 		git.WithWriter(c.logWriter),
 	)

pkg/client/client_test.go

@@ -1010,7 +1010,7 @@ func TestRunGit(t *testing.T) {
 	testPath := getTestDir("test_run_git")
 
 	opts := opt.DefaultCompileOptions()
-	gitOpts := git.NewCloneOptions("https://github.com/KusionStack/catalog", "", "0.1.2", "", filepath.Join(testPath, "catalog"), nil)
+	gitOpts := git.NewCloneOptions("https://github.com/KusionStack/catalog", "", "0.1.2", "", "", filepath.Join(testPath, "catalog"), nil)
 	defer func() {
 		_ = os.RemoveAll(filepath.Join(testPath, "catalog"))
 	}()
@@ -1400,7 +1400,7 @@ func TestRunGitWithLocalDep(t *testing.T) {
 
 		expectPath := filepath.Join(testPath, tc.expectFile)
 		opts := opt.DefaultCompileOptions()
-		gitOpts := git.NewCloneOptions("https://github.com/kcl-lang/flask-demo-kcl-manifests.git", tc.ref, "", "", "", nil)
+		gitOpts := git.NewCloneOptions("https://github.com/kcl-lang/flask-demo-kcl-manifests.git", tc.ref, "", "", "", "", nil)
 
 		result, err := kpmcli.CompileGitPkg(gitOpts, opts)
 		assert.Equal(t, err, nil)

pkg/cmd/cmd_add.go

@@ -45,6 +45,10 @@ func NewAddCmd(kpmcli *client.KpmClient) *cli.Command {
 				Name:  "rename",
 				Usage: "rename the package name in kcl.mod.lock",
 			},
+			&cli.StringFlag{
+				Name:  "sub-package",
+				Usage: "package name inside a monorepo when using git flag",
+			},
 		},
 
 		Action: func(c *cli.Context) error {
@@ -198,6 +202,12 @@ func parseGitRegistryOptions(c *cli.Context) (*opt.RegistryOptions, *reporter.Kp
 		return nil, err
 	}
 
+	gitSubPackage, err := onlyOnceOption(c, "sub-package")
+
+	if err != (*reporter.KpmEvent)(nil) {
+		return nil, err
+	}
+
 	if gitUrl == "" {
 		return nil, reporter.NewErrorEvent(reporter.InvalidGitUrl, fmt.Errorf("the argument 'git' is required"))
 	}
@@ -208,9 +218,10 @@ func parseGitRegistryOptions(c *cli.Context) (*opt.RegistryOptions, *reporter.Kp
 
 	return &opt.RegistryOptions{
 		Git: &opt.GitOptions{
-			Url:    gitUrl,
-			Tag:    gitTag,
-			Commit: gitCommit,
+			Url:        gitUrl,
+			Tag:        gitTag,
+			Commit:     gitCommit,
+			SubPackage: gitSubPackage,
 		},
 	}, nil
 }

pkg/cmd/cmd_run.go

@@ -143,7 +143,7 @@ func KpmRun(c *cli.Context, kpmcli *client.KpmClient) error {
 			// 'kpm run' compile the package from the kcl package tar.
 			compileResult, err = kpmcli.CompileTarPkg(runEntry.PackageSource(), kclOpts)
 		} else if runEntry.IsGit() {
-			gitOpts := git.NewCloneOptions(runEntry.PackageSource(), "", c.String(FLAG_TAG), "", "", nil)
+			gitOpts := git.NewCloneOptions(runEntry.PackageSource(), "", c.String(FLAG_TAG), "", "", "", nil)
 			// 'kpm run' compile the package from the git url
 			compileResult, err = kpmcli.CompileGitPkg(gitOpts, kclOpts)
 		} else {

pkg/downloader/downloader.go

@@ -203,6 +203,10 @@ func (d *GitDownloader) Download(opts DownloadOptions) error {
 		msg = fmt.Sprintf("with branch '%s'", opts.Source.Git.Branch)
 	}
 
+	if len(opts.Source.Git.SubPackage) != 0 {
+		msg = fmt.Sprintf("with sub-package '%s'", opts.Source.Git.SubPackage)
+	}
+
 	reporter.ReportMsgTo(
 		fmt.Sprintf("cloning '%s' %s", opts.Source.Git.Url, msg),
 		opts.LogWriter,
@@ -213,10 +217,13 @@ func (d *GitDownloader) Download(opts DownloadOptions) error {
 		return errors.New("git source is nil")
 	}
 
+	fmt.Println("===>", gitSource.SubPackage)
+
 	_, err := git.CloneWithOpts(
 		git.WithCommit(gitSource.Commit),
 		git.WithBranch(gitSource.Branch),
 		git.WithTag(gitSource.Tag),
+		git.WithSubPackage(gitSource.SubPackage),
 		git.WithRepoURL(gitSource.Url),
 		git.WithLocalPath(opts.LocalPath),
 	)

pkg/downloader/source.go

@@ -34,11 +34,12 @@ type Oci struct {
 
 // Git is the package source from git registry.
 type Git struct {
-	Url     string `toml:"url,omitempty"`
-	Branch  string `toml:"branch,omitempty"`
-	Commit  string `toml:"commit,omitempty"`
-	Tag     string `toml:"git_tag,omitempty"`
-	Version string `toml:"version,omitempty"`
+	Url        string `toml:"url,omitempty"`
+	Branch     string `toml:"branch,omitempty"`
+	Commit     string `toml:"commit,omitempty"`
+	SubPackage string `toml:"sub-package,omitempty"`
+	Tag        string `toml:"git_tag,omitempty"`
+	Version    string `toml:"version,omitempty"`
 }
 
 type Registry struct {

pkg/downloader/toml.go

@@ -58,6 +58,7 @@ func (registry *Registry) MarshalTOML() string {
 const GIT_URL_PATTERN = "git = \"%s\""
 const TAG_PATTERN = "tag = \"%s\""
 const GIT_COMMIT_PATTERN = "commit = \"%s\""
+const GIT_SUB_PACKAGE_PATTERN = "sub-package = \"%s\""
 const GIT_BRANCH_PATTERN = "branch = \"%s\""
 const VERSION_PATTERN = "version = \"%s\""
 const SEPARATOR = ", "
@@ -76,6 +77,11 @@ func (git *Git) MarshalTOML() string {
 		sb.WriteString(fmt.Sprintf(GIT_COMMIT_PATTERN, git.Commit))
 	}
 
+	if len(git.SubPackage) != 0 {
+		sb.WriteString(SEPARATOR)
+		sb.WriteString(fmt.Sprintf(GIT_SUB_PACKAGE_PATTERN, git.SubPackage))
+	}
+
 	if len(git.Branch) != 0 {
 		sb.WriteString(SEPARATOR)
 		sb.WriteString(fmt.Sprintf(GIT_BRANCH_PATTERN, git.Branch))
@@ -165,6 +171,7 @@ func (source *Source) UnmarshalModTOML(data interface{}) error {
 const GIT_URL_FLAG = "git"
 const TAG_FLAG = "tag"
 const GIT_COMMIT_FLAG = "commit"
+const GIT_SUB_PACKAGE_flag = "sub-package"
 const GIT_BRANCH_FLAG = "branch"
 
 func (git *Git) UnmarshalModTOML(data interface{}) error {
@@ -185,6 +192,10 @@ func (git *Git) UnmarshalModTOML(data interface{}) error {
 		git.Commit = v
 	}
 
+	if v, ok := meta[GIT_SUB_PACKAGE_PATTERN].(string); ok {
+		git.SubPackage = v
+	}
+
 	if v, ok := meta[GIT_BRANCH_FLAG].(string); ok {
 		git.Branch = v
 	}

pkg/git/getter.go

@@ -20,22 +20,27 @@ func ForceProtocol(url, protocol string) string {
 	return protocol + url
 }
 
-// ForceGitUrl will add the branch, tag or commit to the git URL and force it to the git protocol
-// `<URL>` will return `Git::<URL>?ref=<branch|tag|commit>`
+// ForceGitUrl will add the subpackage and branch, tag or commit to the git URL and force it to the git protocol
+// `<URL>` will return `Git::<URL>//<SubPackage>?ref=<branch|tag|commit>`
 func (cloneOpts *CloneOptions) ForceGitUrl() (string, error) {
 	if err := cloneOpts.Validate(); err != nil {
 		return "", nil
 	}
 
+	newRepoUrl := ""
+	if cloneOpts.SubPackage != "" {
+		newRepoUrl = cloneOpts.RepoURL + "//" + cloneOpts.SubPackage
+	}
+
 	var attributes = []string{cloneOpts.Branch, cloneOpts.Commit, cloneOpts.Tag}
 	for _, attr := range attributes {
 		if attr != "" {
 			return ForceProtocol(
-				cloneOpts.RepoURL+fmt.Sprintf(constants.GIT_PROTOCOL_URL_PATTERN, attr),
+				newRepoUrl+fmt.Sprintf(constants.GIT_PROTOCOL_URL_PATTERN, attr),
 				GIT_PROTOCOL,
 			), nil
 		}
 	}
 
-	return ForceProtocol(cloneOpts.RepoURL, GIT_PROTOCOL), nil
+	return ForceProtocol(newRepoUrl, GIT_PROTOCOL), nil
 }

pkg/git/git.go

@@ -17,25 +17,27 @@ import (
 
 // CloneOptions is a struct for specifying options for cloning a git repository
 type CloneOptions struct {
-	RepoURL   string
-	Commit    string
-	Tag       string
-	Branch    string
-	LocalPath string
-	Writer    io.Writer
+	RepoURL    string
+	Commit     string
+	Tag        string
+	SubPackage string
+	Branch     string
+	LocalPath  string
+	Writer     io.Writer
 }
 
 // CloneOption is a function that modifies CloneOptions
 type CloneOption func(*CloneOptions)
 
-func NewCloneOptions(repoUrl, commit, tag, branch, localpath string, Writer io.Writer) *CloneOptions {
+func NewCloneOptions(repoUrl, commit, tag, subpackage, branch, localpath string, Writer io.Writer) *CloneOptions {
 	return &CloneOptions{
-		RepoURL:   repoUrl,
-		Commit:    commit,
-		Tag:       tag,
-		Branch:    branch,
-		LocalPath: localpath,
-		Writer:    Writer,
+		RepoURL:    repoUrl,
+		Commit:     commit,
+		Tag:        tag,
+		SubPackage: subpackage,
+		Branch:     branch,
+		LocalPath:  localpath,
+		Writer:     Writer,
 	}
 }
 
@@ -60,6 +62,13 @@ func WithCommit(commit string) CloneOption {
 	}
 }
 
+// WithSubPackage sets the subpackage for CloneOptions
+func WithSubPackage(subpackage string) CloneOption {
+	return func(o *CloneOptions) {
+		o.SubPackage = subpackage
+	}
+}
+
 // WithTag sets the tag for CloneOptions
 func WithTag(tag string) CloneOption {
 	return func(o *CloneOptions) {
@@ -84,6 +93,7 @@ func WithWriter(writer io.Writer) CloneOption {
 // Validate checks if the CloneOptions are valid
 func (cloneOpts *CloneOptions) Validate() error {
 	onlyOneAllowed := 0
+	onlyOnePackageAllowed := 0
 	if cloneOpts.Branch != "" {
 		onlyOneAllowed++
 	}
@@ -93,10 +103,16 @@ func (cloneOpts *CloneOptions) Validate() error {
 	if cloneOpts.Commit != "" {
 		onlyOneAllowed++
 	}
+	if cloneOpts.SubPackage != "" {
+		onlyOnePackageAllowed++
+	}
 
 	if onlyOneAllowed > 1 {
 		return errors.New("only one of branch, tag or commit is allowed")
 	}
+	if onlyOnePackageAllowed > 1 {
+		return errors.New("only one subpackage is allowed")
+	}
 
 	return nil
 }
@@ -112,6 +128,9 @@ func (cloneOpts *CloneOptions) Clone() (*git.Repository, error) {
 		return nil, err
 	}
 
+	fmt.Println("===>", cloneOpts.SubPackage)
+	fmt.Println(url)
+
 	client := &getter.Client{
 		Src:       url,
 		Dst:       cloneOpts.LocalPath,

pkg/git/git_test.go

@@ -26,10 +26,11 @@ func TestWithGitOptions(t *testing.T) {
 }
 
 func TestNewCloneOptions(t *testing.T) {
-	cloneOpts := NewCloneOptions("https://github.com/kcl-lang/kcl", "", "v1.0.0", "", "", nil)
+	cloneOpts := NewCloneOptions("https://github.com/kcl-lang/kcl", "", "v1.0.0", "", "", "", nil)
 	assert.Equal(t, cloneOpts.RepoURL, "https://github.com/kcl-lang/kcl")
 	assert.Equal(t, cloneOpts.Tag, "v1.0.0")
 	assert.Equal(t, cloneOpts.Commit, "")
+	assert.Equal(t, cloneOpts.SubPackage, "")
 	assert.Equal(t, cloneOpts.Branch, "")
 	assert.Equal(t, cloneOpts.LocalPath, "")
 	assert.Equal(t, cloneOpts.Writer, nil)

pkg/opt/opt.go

@@ -382,6 +382,7 @@ type GitOptions struct {
 	Branch string
 	Commit string
 	Tag    string
+	SubPackage string
 }
 
 func (opts *GitOptions) Validate() error {

pkg/package/modfile.go

@@ -7,8 +7,8 @@ import (
 	"path/filepath"
 	"strings"
 
-	orderedmap "github.com/elliotchance/orderedmap/v2"
 	"github.com/BurntSushi/toml"
+	orderedmap "github.com/elliotchance/orderedmap/v2"
 
 	"kcl-lang.io/kcl-go/pkg/kcl"
 
@@ -459,10 +459,11 @@ func (deps *Dependencies) loadLockFile(filepath string) error {
 func ParseOpt(opt *opt.RegistryOptions) (*Dependency, error) {
 	if opt.Git != nil {
 		gitSource := downloader.Git{
-			Url:    opt.Git.Url,
-			Branch: opt.Git.Branch,
-			Commit: opt.Git.Commit,
-			Tag:    opt.Git.Tag,
+			Url:        opt.Git.Url,
+			Branch:     opt.Git.Branch,
+			Commit:     opt.Git.Commit,
+			Tag:        opt.Git.Tag,
+			SubPackage: opt.Git.SubPackage,
 		}
 
 		gitRef, err := gitSource.GetValidGitReference()