diff --git a/pkg/client/client.go b/pkg/client/client.go index ea3bd0fd..946ddf51 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -283,17 +283,17 @@ const PKG_NAME_PATTERN = "%s_%s" // 2. in the vendor subdirectory of the current package. // 3. the dependency is from the local path. func (c *KpmClient) getDepStorePath(search_path string, d *pkg.Dependency, isVendor bool) string { - storePkgName := d.GenPathSuffix() - if d.IsFromLocal() { return d.GetLocalFullPath(search_path) } else { + path := "" if isVendor { - return filepath.Join(search_path, "vendor", storePkgName) + path = filepath.Join(search_path, "vendor", storePkgName) } else { - return filepath.Join(c.homePath, storePkgName) + path = filepath.Join(c.homePath, storePkgName) } + return path } } @@ -747,7 +747,7 @@ func (c *KpmClient) AddDepWithOpts(kclPkg *pkg.KclPkg, opt *opt.AddOptions) (*pk c.noSumCheck = opt.NoSumCheck kclPkg.NoSumCheck = opt.NoSumCheck - // 1. get the name and version of the repository from the input arguments. + // 1. get the name and version of the repository/package from the input arguments. d, err := pkg.ParseOpt(&opt.RegistryOpts) if err != nil { return nil, err @@ -1101,7 +1101,16 @@ func (c *KpmClient) Download(dep *pkg.Dependency, homePath, localPath string) (* return nil, err } - dep.LocalFullPath = localPath + if dep.GetPackage() != "" { + localFullPath, err := utils.FindPackage(localPath, dep.GetPackage()) + if err != nil { + return nil, err + } + dep.LocalFullPath = localFullPath + dep.Name = dep.GetPackage() + } else { + dep.LocalFullPath = localPath + } // Creating symbolic links in a global cache is not an optimal solution. // This allows kclvm to locate the package by default. // This feature is unstable and will be removed soon. @@ -1111,7 +1120,7 @@ func (c *KpmClient) Download(dep *pkg.Dependency, homePath, localPath string) (* // } dep.FullName = dep.GenDepFullName() - modFile, err := c.LoadModFile(localPath) + modFile, err := c.LoadModFile(dep.LocalFullPath) if err != nil { return nil, err } diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index b47161f5..9b4bac72 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -9,6 +9,7 @@ import ( "log" "os" "path/filepath" + "regexp" "runtime" "sort" "strings" @@ -139,6 +140,129 @@ func TestDownloadLatestOci(t *testing.T) { assert.Equal(t, utils.DirExists(filepath.Join(getTestDir("download"), "helloworld")), false) } +func TestDownloadGitWithPackage(t *testing.T) { + testPath := filepath.Join(getTestDir("download"), "a_random_name") + + defer func() { + err := os.RemoveAll(getTestDir("download")) + if err != nil { + t.Errorf("Failed to remove directory: %v", err) + } + }() + + err := os.MkdirAll(testPath, 0755) + assert.Equal(t, err, nil) + + depFromGit := pkg.Dependency{ + Name: "k8s", + Version: "", + Source: downloader.Source{ + Git: &downloader.Git{ + Url: "https://github.com/kcl-lang/modules.git", + Commit: "bdd4d00a88bc3534ae50affa8328df2927fd2171", + Package: "add-ndots", + }, + }, + } + + kpmcli, err := NewKpmClient() + assert.Equal(t, err, nil) + + dep, err := kpmcli.Download(&depFromGit, "", testPath) + + assert.Equal(t, err, nil) + assert.Equal(t, dep.Source.Git.Package, "add-ndots") +} + +func TestModandLockFilesWithGitPackageDownload(t *testing.T) { + testPkgPath := getTestDir("test_mod_file_package") + + if runtime.GOOS == "windows" { + testPkgPath = filepath.Join(testPkgPath, "test_pkg_win") + } else { + testPkgPath = filepath.Join(testPkgPath, "test_pkg") + } + + kpmcli, err := NewKpmClient() + assert.Equal(t, err, nil) + + kclPkg, err := kpmcli.LoadPkgFromPath(testPkgPath) + assert.Equal(t, err, nil) + + opts := opt.AddOptions{ + LocalPath: testPkgPath, + RegistryOpts: opt.RegistryOptions{ + Git: &opt.GitOptions{ + Url: "https://github.com/kcl-lang/modules.git", + Commit: "ee03122b5f45b09eb48694422fc99a0772f6bba8", + Package: "agent", + }, + }, + } + + _, err = kpmcli.AddDepWithOpts(kclPkg, &opts) + assert.Equal(t, err, nil) + + testPkgPathMod := filepath.Join(testPkgPath, "kcl.mod") + testPkgPathModExpect := filepath.Join(testPkgPath, "expect.mod") + testPkgPathModLock := filepath.Join(testPkgPath, "kcl.mod.lock") + testPkgPathModLockExpect := filepath.Join(testPkgPath, "expect.mod.lock") + + modContent, err := os.ReadFile(testPkgPathMod) + assert.Equal(t, err, nil) + + modExpectContent, err := os.ReadFile(testPkgPathModExpect) + assert.Equal(t, err, nil) + + modContentStr := string(modContent) + modExpectContentStr := string(modExpectContent) + + for _, str := range []*string{&modContentStr, &modExpectContentStr} { + *str = strings.ReplaceAll(*str, " ", "") + *str = strings.ReplaceAll(*str, "\r\n", "") + *str = strings.ReplaceAll(*str, "\n", "") + + sumRegex := regexp.MustCompile(`sum\s*=\s*"[^"]+"`) + *str = sumRegex.ReplaceAllString(*str, "") + + *str = strings.TrimRight(*str, ", \t\r\n") + } + + assert.Equal(t, modExpectContentStr, modContentStr) + + modLockContent, err := os.ReadFile(testPkgPathModLock) + assert.Equal(t, err, nil) + + modLockExpectContent, err := os.ReadFile(testPkgPathModLockExpect) + assert.Equal(t, err, nil) + + modLockContentStr := string(modLockContent) + modLockExpectContentStr := string(modLockExpectContent) + + for _, str := range []*string{&modLockContentStr, &modLockExpectContentStr} { + *str = strings.ReplaceAll(*str, " ", "") + *str = strings.ReplaceAll(*str, "\r\n", "") + *str = strings.ReplaceAll(*str, "\n", "") + + sumRegex := regexp.MustCompile(`sum\s*=\s*"[^"]+"`) + *str = sumRegex.ReplaceAllString(*str, "") + + *str = strings.TrimRight(*str, ", \t\r\n") + } + + fmt.Println(modLockContentStr) + + assert.Equal(t, modLockExpectContentStr, modLockContentStr) + + defer func() { + err = os.Truncate(testPkgPathMod, 0) + assert.Equal(t, err, nil) + + err = os.Truncate(testPkgPathModLock, 0) + assert.Equal(t, err, nil) + }() +} + func TestDependencyGraph(t *testing.T) { testDir := getTestDir("test_dependency_graph") assert.Equal(t, utils.DirExists(filepath.Join(testDir, "kcl.mod.lock")), false) @@ -889,7 +1013,7 @@ func TestUpdateWithKclModlock(t *testing.T) { err = kpmcli.UpdateDeps(kclPkg) assert.Equal(t, err, nil) got_lock_file := filepath.Join(dest_testDir, "kcl.mod.lock") - got_content, err := os.ReadFile(got_lock_file) + got_content, err := os.ReadFile(got_lock_file) // help assert.Equal(t, err, nil) expected_path := filepath.Join(dest_testDir, "expected") diff --git a/pkg/client/test_data/test_mod_file_package/test_pkg/expect.mod b/pkg/client/test_data/test_mod_file_package/test_pkg/expect.mod new file mode 100644 index 00000000..18fcce1e --- /dev/null +++ b/pkg/client/test_data/test_mod_file_package/test_pkg/expect.mod @@ -0,0 +1,4 @@ +[package] + +[dependencies] +agent = { git = "https://github.com/kcl-lang/modules.git", commit = "ee03122b5f45b09eb48694422fc99a0772f6bba8", package = "agent" } \ No newline at end of file diff --git a/pkg/client/test_data/test_mod_file_package/test_pkg/expect.mod.lock b/pkg/client/test_data/test_mod_file_package/test_pkg/expect.mod.lock new file mode 100644 index 00000000..f7821a7d --- /dev/null +++ b/pkg/client/test_data/test_mod_file_package/test_pkg/expect.mod.lock @@ -0,0 +1,12 @@ +[dependencies] + [dependencies.agent] + name = "agent" + full_name = "agent_ee03122b5f45b09eb48694422fc99a0772f6bba8" + version = "0.1.0" + url = "https://github.com/kcl-lang/modules.git" + commit = "ee03122b5f45b09eb48694422fc99a0772f6bba8" + package = "agent" + [dependencies.k8s] + name = "k8s" + full_name = "k8s_1.28" + version = "1.28" diff --git a/pkg/client/test_data/test_mod_file_package/test_pkg/kcl.mod b/pkg/client/test_data/test_mod_file_package/test_pkg/kcl.mod new file mode 100644 index 00000000..e69de29b diff --git a/pkg/client/test_data/test_mod_file_package/test_pkg/kcl.mod.lock b/pkg/client/test_data/test_mod_file_package/test_pkg/kcl.mod.lock new file mode 100644 index 00000000..e69de29b diff --git a/pkg/client/test_data/test_mod_file_package/test_pkg_win/expect.mod b/pkg/client/test_data/test_mod_file_package/test_pkg_win/expect.mod new file mode 100644 index 00000000..18fcce1e --- /dev/null +++ b/pkg/client/test_data/test_mod_file_package/test_pkg_win/expect.mod @@ -0,0 +1,4 @@ +[package] + +[dependencies] +agent = { git = "https://github.com/kcl-lang/modules.git", commit = "ee03122b5f45b09eb48694422fc99a0772f6bba8", package = "agent" } \ No newline at end of file diff --git a/pkg/client/test_data/test_mod_file_package/test_pkg_win/expect.mod.lock b/pkg/client/test_data/test_mod_file_package/test_pkg_win/expect.mod.lock new file mode 100644 index 00000000..f7821a7d --- /dev/null +++ b/pkg/client/test_data/test_mod_file_package/test_pkg_win/expect.mod.lock @@ -0,0 +1,12 @@ +[dependencies] + [dependencies.agent] + name = "agent" + full_name = "agent_ee03122b5f45b09eb48694422fc99a0772f6bba8" + version = "0.1.0" + url = "https://github.com/kcl-lang/modules.git" + commit = "ee03122b5f45b09eb48694422fc99a0772f6bba8" + package = "agent" + [dependencies.k8s] + name = "k8s" + full_name = "k8s_1.28" + version = "1.28" diff --git a/pkg/client/test_data/test_mod_file_package/test_pkg_win/kcl.mod b/pkg/client/test_data/test_mod_file_package/test_pkg_win/kcl.mod new file mode 100644 index 00000000..e69de29b diff --git a/pkg/client/test_data/test_mod_file_package/test_pkg_win/kcl.mod.lock b/pkg/client/test_data/test_mod_file_package/test_pkg_win/kcl.mod.lock new file mode 100644 index 00000000..e69de29b diff --git a/pkg/cmd/cmd_add.go b/pkg/cmd/cmd_add.go index f0786759..30761bdf 100644 --- a/pkg/cmd/cmd_add.go +++ b/pkg/cmd/cmd_add.go @@ -45,6 +45,10 @@ func NewAddCmd(kpmcli *client.KpmClient) *cli.Command { Name: "rename", Usage: "rename the package name in kcl.mod.lock", }, + &cli.StringSliceFlag{ + Name: "package", + Usage: "package name to use in case of git", + }, }, Action: func(c *cli.Context) error { @@ -198,6 +202,12 @@ func parseGitRegistryOptions(c *cli.Context) (*opt.RegistryOptions, *reporter.Kp return nil, err } + gitPackage, err := onlyOnceOption(c, "package") + + if err != (*reporter.KpmEvent)(nil) { + return nil, err + } + if gitUrl == "" { return nil, reporter.NewErrorEvent(reporter.InvalidGitUrl, fmt.Errorf("the argument 'git' is required")) } @@ -208,9 +218,10 @@ func parseGitRegistryOptions(c *cli.Context) (*opt.RegistryOptions, *reporter.Kp return &opt.RegistryOptions{ Git: &opt.GitOptions{ - Url: gitUrl, - Tag: gitTag, - Commit: gitCommit, + Url: gitUrl, + Tag: gitTag, + Commit: gitCommit, + Package: gitPackage, }, }, nil } diff --git a/pkg/downloader/source.go b/pkg/downloader/source.go index 5cec444e..2576bbee 100644 --- a/pkg/downloader/source.go +++ b/pkg/downloader/source.go @@ -39,6 +39,7 @@ type Git struct { Commit string `toml:"commit,omitempty"` Tag string `toml:"git_tag,omitempty"` Version string `toml:"version,omitempty"` + Package string `toml:"package,omitempty"` } type Registry struct { @@ -211,6 +212,13 @@ func (git *Git) ToFilePath() (string, error) { ), nil } +func (git *Git) GetPackage() string { + if(git == nil) { + return "" + } + return git.Package +} + func (oci *Oci) ToFilePath() (string, error) { if oci == nil { return "", fmt.Errorf("oci source is nil") diff --git a/pkg/downloader/toml.go b/pkg/downloader/toml.go index 5c4d3f6c..565c4b34 100644 --- a/pkg/downloader/toml.go +++ b/pkg/downloader/toml.go @@ -69,6 +69,7 @@ const TAG_PATTERN = "tag = \"%s\"" const GIT_COMMIT_PATTERN = "commit = \"%s\"" const GIT_BRANCH_PATTERN = "branch = \"%s\"" const VERSION_PATTERN = "version = \"%s\"" +const GIT_PACKAGE = "package = \"%s\"" const SEPARATOR = ", " func (git *Git) MarshalTOML() string { @@ -94,6 +95,12 @@ func (git *Git) MarshalTOML() string { sb.WriteString(SEPARATOR) sb.WriteString(fmt.Sprintf(VERSION_PATTERN, git.Version)) } + + if len(git.Package) != 0 { + sb.WriteString(SEPARATOR) + sb.WriteString(fmt.Sprintf(GIT_PACKAGE, git.Package)) + } + return sb.String() } @@ -175,6 +182,7 @@ const GIT_URL_FLAG = "git" const TAG_FLAG = "tag" const GIT_COMMIT_FLAG = "commit" const GIT_BRANCH_FLAG = "branch" +const GIT_PACKAGE_FLAG = "package" func (git *Git) UnmarshalModTOML(data interface{}) error { meta, ok := data.(map[string]interface{}) @@ -198,6 +206,10 @@ func (git *Git) UnmarshalModTOML(data interface{}) error { git.Branch = v } + if v, ok := meta[GIT_PACKAGE_FLAG].(string); ok { + git.Package = v + } + return nil } diff --git a/pkg/opt/opt.go b/pkg/opt/opt.go index 9c863ca4..02309a55 100644 --- a/pkg/opt/opt.go +++ b/pkg/opt/opt.go @@ -378,10 +378,11 @@ func ParseLocalPathOptions(localPath string) (*LocalOptions, *reporter.KpmEvent) } type GitOptions struct { - Url string - Branch string - Commit string - Tag string + Url string + Branch string + Commit string + Tag string + Package string } func (opts *GitOptions) Validate() error { diff --git a/pkg/package/modfile.go b/pkg/package/modfile.go index b4fb3b49..0f72cf32 100644 --- a/pkg/package/modfile.go +++ b/pkg/package/modfile.go @@ -482,10 +482,11 @@ func (deps *Dependencies) loadLockFile(filepath string) error { func ParseOpt(opt *opt.RegistryOptions) (*Dependency, error) { if opt.Git != nil { gitSource := downloader.Git{ - Url: opt.Git.Url, - Branch: opt.Git.Branch, - Commit: opt.Git.Commit, - Tag: opt.Git.Tag, + Url: opt.Git.Url, + Branch: opt.Git.Branch, + Commit: opt.Git.Commit, + Tag: opt.Git.Tag, + Package: opt.Git.Package, } gitRef, err := gitSource.GetValidGitReference() @@ -566,18 +567,29 @@ func ParseOpt(opt *opt.RegistryOptions) (*Dependency, error) { const PKG_NAME_PATTERN = "%s_%s" // ParseRepoFullNameFromGitSource will extract the kcl package name from the git url. +// If the package flag is passed then it will be used as the package name. func ParseRepoFullNameFromGitSource(gitSrc downloader.Git) (string, error) { ref, err := gitSrc.GetValidGitReference() if err != nil { return "", err } if len(ref) != 0 { + if len(gitSrc.Package) != 0 { + return fmt.Sprintf(PKG_NAME_PATTERN, gitSrc.Package, ref), nil + } return fmt.Sprintf(PKG_NAME_PATTERN, utils.ParseRepoNameFromGitUrl(gitSrc.Url), ref), nil } + if len(gitSrc.Package) != 0 { + return gitSrc.Package, nil + } return utils.ParseRepoNameFromGitUrl(gitSrc.Url), nil } // ParseRepoNameFromGitSource will extract the kcl package name from the git url. +// If the package flag is passed then it will be used func ParseRepoNameFromGitSource(gitSrc downloader.Git) string { + if gitSrc.Package != "" { + return gitSrc.Package + } return utils.ParseRepoNameFromGitUrl(gitSrc.Url) } diff --git a/pkg/package/modfile_test.go b/pkg/package/modfile_test.go index 8d3425d1..d340c183 100644 --- a/pkg/package/modfile_test.go +++ b/pkg/package/modfile_test.go @@ -146,6 +146,20 @@ func TestParseOpt(t *testing.T) { assert.Equal(t, dep.Branch, "") assert.Equal(t, dep.Commit, "") assert.Equal(t, dep.Git.Tag, "test_tag") + + dep, _ = ParseOpt(&opt.RegistryOptions{ + Git: &opt.GitOptions{ + Url: "test.git", + Branch: "", + Commit: "", + Tag: "test_tag", + Package: "k8s", + }, + }) + + assert.Equal(t, dep.Name, "k8s") + assert.Equal(t, dep.FullName, "k8s_test_tag") + assert.Equal(t, dep.Git.Package, "k8s") } func TestLoadModFileNotExist(t *testing.T) { diff --git a/pkg/utils/test_data/test_find_package/test_1/kcl.mod b/pkg/utils/test_data/test_find_package/test_1/kcl.mod new file mode 100644 index 00000000..6b1923fc --- /dev/null +++ b/pkg/utils/test_data/test_find_package/test_1/kcl.mod @@ -0,0 +1,4 @@ +[package] +name = "test_find_package_false" +edition = "v0.1.0" +version = "0.0.1" \ No newline at end of file diff --git a/pkg/utils/test_data/test_find_package/test_2/kcl.mod b/pkg/utils/test_data/test_find_package/test_2/kcl.mod new file mode 100644 index 00000000..78e4cf8a --- /dev/null +++ b/pkg/utils/test_data/test_find_package/test_2/kcl.mod @@ -0,0 +1,4 @@ +[package] +name = "test_find_package" +edition = "v0.1.0" +version = "0.0.1" \ No newline at end of file diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 14bc9b06..6c7cdff2 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -17,6 +17,7 @@ import ( "runtime" "strings" + "github.com/BurntSushi/toml" "github.com/distribution/reference" "github.com/moby/term" "github.com/otiai10/copy" @@ -599,3 +600,50 @@ func AbsTarPath(tarPath string) (string, error) { return absTarPath, nil } + +// FindPackage finds the package with the package name 'targetPackage' under the 'root' directory kcl.mod file. +func FindPackage(root, targetPackage string) (string, error) { + var result string + err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + kclModPath := filepath.Join(path, constants.KCL_MOD) + if _, err := os.Stat(kclModPath); err == nil { + if matchesPackageName(kclModPath, targetPackage) { + result = path + return filepath.SkipAll + } + } + } + return nil + }) + + if err != nil { + return "", err + } + if result == "" { + return "", fmt.Errorf("kcl.mod with package '%s' not found", targetPackage) + } + return result, nil +} + +// MatchesPackageName checks whether the package name in the kcl.mod file under 'kclModPath' is equal to 'targetPackage'. +func matchesPackageName(kclModPath, targetPackage string) bool { + type Package struct { + Name string `toml:"name"` + } + type ModFile struct { + Package Package `toml:"package"` + } + + var modFile ModFile + _, err := toml.DecodeFile(kclModPath, &modFile) + if err != nil { + fmt.Printf("Error parsing kcl.mod file: %v\n", err) + return false + } + + return modFile.Package.Name == targetPackage +} diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go index 92d1d1d2..fc36bde2 100644 --- a/pkg/utils/utils_test.go +++ b/pkg/utils/utils_test.go @@ -302,3 +302,16 @@ func TestIsModRelativePath(t *testing.T) { assert.Equal(t, IsModRelativePath("${helloworld:KCL_MOD}/aaa"), true) assert.Equal(t, IsModRelativePath("xxx/xxx/xxx"), false) } + + +func TestFindPackage(t *testing.T) { + testDir := getTestDir("test_find_package") + correctAddress := filepath.Join(testDir, "test_2") + foundAddress, _ := FindPackage(testDir, "test_find_package") + assert.Equal(t, foundAddress, correctAddress) +} + +func TestMatchesPackageName(t *testing.T) { + address := filepath.Join(getTestDir("test_find_package"), "test_2", "kcl.mod") + assert.Equal(t, matchesPackageName(address, "test_find_package"), true) +} \ No newline at end of file