feat: add ModSpec for dependency source to specify one module from …

…source (#507) * feat: add `PkgSpec` for dependency source to specify the package Signed-off-by: zongz <zongzhe1024@163.com> * fix: rm field `SpecOnly` in source Signed-off-by: zongz <zongzhe1024@163.com> * fix: rm irrelevant modifications Signed-off-by: zongz <zongzhe1024@163.com> * fix: fix typo 'TestAddWithDiffVersionNoSumCheck' -> 'testAddWithDiffVersionNoSumCheck' Signed-off-by: zongz <zongzhe1024@163.com> --------- Signed-off-by: zongz <zongzhe1024@163.com>
kcl-lang · Oct 29, 2024 · ab68fb5 · ab68fb5
1 parent f48b8b4
commit ab68fb5
Show file tree

Hide file tree

Showing 24 changed files with 260 additions and 218 deletions.
diff --git a/go.mod b/go.mod
@@ -157,6 +157,7 @@ require (
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-getter v1.7.6
 	github.com/hashicorp/go-version v1.7.0
+	github.com/jinzhu/copier v0.4.0
 	github.com/kubescape/go-git-url v0.0.30
 	github.com/moby/term v0.5.0
 	github.com/onsi/ginkgo/v2 v2.20.2

diff --git a/go.sum b/go.sum
@@ -524,6 +524,8 @@ github.com/invopop/yaml v0.3.1 h1:f0+ZpmhfBSS4MhG+4HYseMdJhoeeopbSKbq5Rpeelso=
 github.com/invopop/yaml v0.3.1/go.mod h1:PMOp3nn4/12yEZUFfmOuNHJsZToEEOwoWsT+D81KkeA=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
+github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
+github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=

diff --git a/pkg/api/kpm_pkg_test.go b/pkg/api/kpm_pkg_test.go
@@ -45,9 +45,9 @@ func testPackageApi(t *testing.T) {
 	assert.Equal(t, dep.Name, "helloworld")
 	assert.Equal(t, dep.FullName, "helloworld_0.1.3")
 	assert.Equal(t, dep.Version, "0.1.3")
-	assert.Equal(t, dep.Source.Registry.Oci.Reg, "ghcr.io")
-	assert.Equal(t, dep.Source.Registry.Oci.Repo, "kcl-lang/helloworld")
-	assert.Equal(t, dep.Source.Registry.Oci.Tag, "0.1.3")
+	assert.Equal(t, dep.Source.Oci.Reg, "ghcr.io")
+	assert.Equal(t, dep.Source.Oci.Repo, "kcl-lang/helloworld")
+	assert.Equal(t, dep.Source.Oci.Tag, "0.1.3")
 
 	assert.Equal(t, dep.GetLocalFullPath(""), filepath.Join(kcl_pkg_path, "helloworld_0.1.3"))
 
@@ -83,9 +83,9 @@ func TestApiGetDependenciesInModFile(t *testing.T) {
 	assert.Equal(t, dep.Name, "k8s")
 	assert.Equal(t, dep.FullName, "k8s_1.27")
 	assert.Equal(t, dep.Version, "1.27")
-	assert.Equal(t, dep.Source.Registry.Oci.Reg, "ghcr.io")
-	assert.Equal(t, dep.Source.Registry.Oci.Repo, "kcl-lang/k8s")
-	assert.Equal(t, dep.Source.Registry.Oci.Tag, "1.27")
+	assert.Equal(t, dep.Source.Oci.Reg, "ghcr.io")
+	assert.Equal(t, dep.Source.Oci.Repo, "kcl-lang/k8s")
+	assert.Equal(t, dep.Source.Oci.Tag, "1.27")
 }
 
 func testGetAllSchemaTypesMappingNamed(t *testing.T) {

diff --git a/pkg/client/client.go b/pkg/client/client.go
@@ -425,18 +425,10 @@ func (c *KpmClient) UpdateDeps(kclPkg *pkg.KclPkg) error {
 		}
 	} else {
 		// update kcl.mod
-		err = kclPkg.ModFile.StoreModFile()
+		err = kclPkg.UpdateModAndLockFile()
 		if err != nil {
 			return err
 		}
-
-		// Generate file kcl.mod.lock.
-		if !kclPkg.NoSumCheck {
-			err := kclPkg.LockDepsVersion()
-			if err != nil {
-				return err
-			}
-		}
 	}
 
 	return nil
@@ -537,6 +529,20 @@ func (c *KpmClient) AddDepWithOpts(kclPkg *pkg.KclPkg, opt *opt.AddOptions) (*pk
 		return nil, err
 	}
 
+	// Backup the dependency used in kcl.mod
+	if opt.RegistryOpts.Registry != nil {
+		kclPkg.BackupDepUI(d.Name, &pkg.Dependency{
+			Name:    d.Name,
+			Version: d.Version,
+			Source: downloader.Source{
+				ModSpec: &downloader.ModSpec{
+					Name:    d.Name,
+					Version: d.Version,
+				},
+			},
+		})
+	}
+
 	reporter.ReportMsgTo(
 		fmt.Sprintf("adding dependency '%s'", d.Name),
 		c.logWriter,
@@ -684,18 +690,6 @@ func (c *KpmClient) FillDepInfo(dep *pkg.Dependency, homepath string) error {
 			dep.Source.Oci.Repo = urlpath
 		}
 	}
-	if dep.Source.Registry != nil {
-		if len(dep.Source.Registry.Reg) == 0 {
-			dep.Source.Registry.Reg = c.GetSettings().DefaultOciRegistry()
-		}
-
-		if len(dep.Source.Registry.Repo) == 0 {
-			urlpath := utils.JoinPath(c.GetSettings().DefaultOciRepo(), dep.Name)
-			dep.Source.Registry.Repo = urlpath
-		}
-
-		dep.Version = dep.Source.Registry.Version
-	}
 	if dep.Source.Git != nil && dep.Source.Git.GetPackage() != "" {
 		name := utils.ParseRepoNameFromGitUrl(dep.Source.Git.Url)
 		if len(dep.Source.Git.Tag) != 0 {
@@ -846,12 +840,10 @@ func (c *KpmClient) Download(dep *pkg.Dependency, homePath, localPath string) (*
 		dep.Version = modFile.Pkg.Version
 	}
 
-	if dep.Source.Oci != nil || dep.Source.Registry != nil {
+	if dep.Source.Oci != nil {
 		var ociSource *downloader.Oci
 		if dep.Source.Oci != nil {
 			ociSource = dep.Source.Oci
-		} else if dep.Source.Registry != nil {
-			ociSource = dep.Source.Registry.Oci
 		}
 		// Select the latest tag, if the tag, the user inputed, is empty.
 		if ociSource.Tag == "" || ociSource.Tag == constants.LATEST {
@@ -860,10 +852,8 @@ func (c *KpmClient) Download(dep *pkg.Dependency, homePath, localPath string) (*
 				return nil, err
 			}
 			ociSource.Tag = latestTag
-
-			if dep.Source.Registry != nil {
-				dep.Source.Registry.Tag = latestTag
-				dep.Source.Registry.Version = latestTag
+			if dep.Source.ModSpec != nil {
+				dep.Source.ModSpec.Version = latestTag
 			}
 
 			// Complete some information that the local three dependencies depend on.

diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go
@@ -57,7 +57,7 @@ func initTestDir(subDir string) string {
 func TestWithGlobalLock(t *testing.T) {
 	test.RunTestWithGlobalLock(t, "TestUpdateWithKclMod", testUpdateWithKclMod)
 	test.RunTestWithGlobalLock(t, "TestUpdateWithKclModlock", testUpdateWithKclModlock)
-	test.RunTestWithGlobalLock(t, "TestUpdateWithNoSumCheck", TestUpdateWithNoSumCheck)
+	test.RunTestWithGlobalLock(t, "TestUpdateWithNoSumCheck", testUpdateWithNoSumCheck)
 	test.RunTestWithGlobalLock(t, "TestAddWithDiffVersionNoSumCheck", testAddWithDiffVersionNoSumCheck)
 	test.RunTestWithGlobalLock(t, "TestAddWithDiffVersionWithSumCheck", testAddWithDiffVersionWithSumCheck)
 	test.RunTestWithGlobalLock(t, "TestDownloadOci", testDownloadOci)
@@ -1190,7 +1190,7 @@ func testAddWithNoSumCheck(t *testing.T) {
 	}()
 }
 
-func TestUpdateWithNoSumCheck(t *testing.T) {
+func testUpdateWithNoSumCheck(t *testing.T) {
 	pkgPath := getTestDir("test_update_no_sum_check")
 	kpmcli, err := NewKpmClient()
 	assert.Equal(t, err, nil)

diff --git a/pkg/client/test_data/run_with_default_dep/kcl.mod.lock.expect b/pkg/client/test_data/run_with_default_dep/kcl.mod.lock.expect
@@ -3,3 +3,6 @@
     name = "helloworld"
     full_name = "helloworld_0.1.2"
     version = "0.1.2"
+    reg = "ghcr.io"
+    repo = "kcl-lang/helloworld"
+    oci_tag = "0.1.2"
diff --git a/pkg/client/test_data/test_add_diff_version/with_sum_check/kcl.mod.lock.expect b/pkg/client/test_data/test_add_diff_version/with_sum_check/kcl.mod.lock.expect
@@ -3,3 +3,6 @@
     name = "helloworld"
     full_name = "helloworld_0.1.2"
     version = "0.1.2"
+    reg = "ghcr.io"
+    repo = "kcl-lang/helloworld"
+    oci_tag = "0.1.2"
diff --git a/pkg/client/test_data/test_mod_file_package/test_pkg/expect.mod b/pkg/client/test_data/test_mod_file_package/test_pkg/expect.mod
@@ -4,4 +4,4 @@ edition = "v0.10.0"
 version = "0.0.1"
 
 [dependencies]
-cc = { git = "https://github.com/kcl-lang/flask-demo-kcl-manifests.git", commit = "8308200", package = "cc" }
+cc = { git = "https://github.com/kcl-lang/flask-demo-kcl-manifests.git", commit = "8308200", package = "cc"}
diff --git a/pkg/client/test_data/test_update/test_update_kcl_mod/expected b/pkg/client/test_data/test_update/test_update_kcl_mod/expected
@@ -3,3 +3,6 @@
     name = "helloworld"
     full_name = "helloworld_0.1.2"
     version = "0.1.2"
+    reg = "ghcr.io"
+    repo = "kcl-lang/helloworld"
+    oci_tag = "0.1.2"
diff --git a/pkg/client/test_data/test_update_with_mvs/update_0/pkg/kcl.mod.lock b/pkg/client/test_data/test_update_with_mvs/update_0/pkg/kcl.mod.lock
@@ -15,3 +15,6 @@
     name = "helloworld"
     full_name = "helloworld_0.1.1"
     version = "0.1.1"
+    reg = "ghcr.io"
+    repo = "kcl-lang/helloworld"
+    oci_tag = "0.1.1"
diff --git a/pkg/client/test_data/test_update_with_mvs/update_0/pkg/kcl.mod.lock.expect b/pkg/client/test_data/test_update_with_mvs/update_0/pkg/kcl.mod.lock.expect
@@ -15,3 +15,6 @@
     name = "helloworld"
     full_name = "helloworld_0.1.1"
     version = "0.1.1"
+    reg = "ghcr.io"
+    repo = "kcl-lang/helloworld"
+    oci_tag = "0.1.1"
diff --git a/pkg/client/test_data/test_update_with_mvs/update_1/pkg/kcl.mod.lock b/pkg/client/test_data/test_update_with_mvs/update_1/pkg/kcl.mod.lock
@@ -15,3 +15,6 @@
     name = "helloworld"
     full_name = "helloworld_0.1.1"
     version = "0.1.1"
+    reg = "ghcr.io"
+    repo = "kcl-lang/helloworld"
+    oci_tag = "0.1.1"
diff --git a/pkg/client/test_data/test_update_with_mvs/update_1/pkg/kcl.mod.lock.expect b/pkg/client/test_data/test_update_with_mvs/update_1/pkg/kcl.mod.lock.expect
@@ -15,3 +15,6 @@
     name = "helloworld"
     full_name = "helloworld_0.1.1"
     version = "0.1.1"
+    reg = "ghcr.io"
+    repo = "kcl-lang/helloworld"
+    oci_tag = "0.1.1"
diff --git a/pkg/client/test_data/update_with_default_dep/kcl.mod.lock.expect b/pkg/client/test_data/update_with_default_dep/kcl.mod.lock.expect
@@ -3,3 +3,7 @@
     name = "helloworld"
     full_name = "helloworld_0.1.2"
     version = "0.1.2"
+    reg = "ghcr.io"
+    repo = "kcl-lang/helloworld"
+    oci_tag = "0.1.2"
+
diff --git a/pkg/downloader/downloader.go b/pkg/downloader/downloader.go
@@ -142,14 +142,11 @@ func (d *DepDownloader) Download(opts DownloadOptions) error {
 
 	localPath := opts.LocalPath
 	cacheFullPath := opts.CachePath
-	if opts.EnableCache {
+	if ok, err := features.Enabled(features.SupportNewStorage); err == nil && !ok && opts.EnableCache {
 		// TODO: After the new local storage structure is complete,
 		// this section should be replaced with the new storage structure instead of the cache path according to the <Cache Path>/<Package Name>.
 		//  https://github.com/kcl-lang/kpm/issues/384
 		var pkgFullName string
-		if opts.Source.Registry != nil && len(opts.Source.Registry.Version) != 0 {
-			pkgFullName = fmt.Sprintf("%s_%s", filepath.Base(opts.Source.Registry.Oci.Repo), opts.Source.Registry.Version)
-		}
 		if opts.Source.Oci != nil && len(opts.Source.Oci.Tag) != 0 {
 			pkgFullName = fmt.Sprintf("%s_%s", filepath.Base(opts.Source.Oci.Repo), opts.Source.Oci.Tag)
 		}
@@ -167,7 +164,7 @@ func (d *DepDownloader) Download(opts DownloadOptions) error {
 			pkgFullName = fmt.Sprintf("%s_%s", filepath.Base(gitUrl), opts.Source.Git.Commit)
 		}
 
-		cacheFullPath := filepath.Join(opts.CachePath, pkgFullName)
+		cacheFullPath = filepath.Join(opts.CachePath, pkgFullName)
 
 		if utils.DirExists(cacheFullPath) && utils.DirExists(filepath.Join(cacheFullPath, constants.KCL_MOD)) {
 			// copy the cache to the local path
@@ -188,10 +185,7 @@ func (d *DepDownloader) Download(opts DownloadOptions) error {
 
 	opts.LocalPath = tmpDir
 	// Dispatch the download to the specific downloader by package source.
-	if opts.Source.Oci != nil || opts.Source.Registry != nil {
-		if opts.Source.Registry != nil {
-			opts.Source.Oci = opts.Source.Registry.Oci
-		}
+	if opts.Source.Oci != nil {
 		if d.OciDownloader == nil {
 			d.OciDownloader = &OciDownloader{}
 		}