From ab68fb584cd8c5b1c20d426cbc120a61520e10f3 Mon Sep 17 00:00:00 2001 From: zongz <68977949+zong-zhe@users.noreply.github.com> Date: Tue, 29 Oct 2024 20:19:25 +0800 Subject: [PATCH] feat: add `ModSpec` for dependency source to specify one module from source (#507) * feat: add `PkgSpec` for dependency source to specify the package Signed-off-by: zongz * fix: rm field `SpecOnly` in source Signed-off-by: zongz * fix: rm irrelevant modifications Signed-off-by: zongz * fix: fix typo 'TestAddWithDiffVersionNoSumCheck' -> 'testAddWithDiffVersionNoSumCheck' Signed-off-by: zongz --------- Signed-off-by: zongz --- go.mod | 1 + go.sum | 2 + pkg/api/kpm_pkg_test.go | 12 +- pkg/client/client.go | 46 +++---- pkg/client/client_test.go | 4 +- .../run_with_default_dep/kcl.mod.lock.expect | 3 + .../with_sum_check/kcl.mod.lock.expect | 3 + .../test_mod_file_package/test_pkg/expect.mod | 2 +- .../test_update/test_update_kcl_mod/expected | 3 + .../update_0/pkg/kcl.mod.lock | 3 + .../update_0/pkg/kcl.mod.lock.expect | 3 + .../update_1/pkg/kcl.mod.lock | 3 + .../update_1/pkg/kcl.mod.lock.expect | 3 + .../kcl.mod.lock.expect | 4 + pkg/downloader/downloader.go | 12 +- pkg/downloader/source.go | 106 +++++++--------- pkg/downloader/toml.go | 113 +++++++++--------- pkg/package/modfile.go | 11 +- pkg/package/modfile_test.go | 2 +- pkg/package/package.go | 81 +++++++++---- pkg/package/package_test.go | 14 +-- .../test_data/test_data_toml/expected.toml | 2 +- pkg/package/toml.go | 17 ++- pkg/package/toml_test.go | 28 +++-- 24 files changed, 260 insertions(+), 218 deletions(-) diff --git a/go.mod b/go.mod index 7c4f5d78..3a204494 100644 --- a/go.mod +++ b/go.mod @@ -157,6 +157,7 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-getter v1.7.6 github.com/hashicorp/go-version v1.7.0 + github.com/jinzhu/copier v0.4.0 github.com/kubescape/go-git-url v0.0.30 github.com/moby/term v0.5.0 github.com/onsi/ginkgo/v2 v2.20.2 diff --git a/go.sum b/go.sum index 6ac3e304..1c49598a 100644 --- a/go.sum +++ b/go.sum @@ -524,6 +524,8 @@ github.com/invopop/yaml v0.3.1 h1:f0+ZpmhfBSS4MhG+4HYseMdJhoeeopbSKbq5Rpeelso= github.com/invopop/yaml v0.3.1/go.mod h1:PMOp3nn4/12yEZUFfmOuNHJsZToEEOwoWsT+D81KkeA= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= +github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= diff --git a/pkg/api/kpm_pkg_test.go b/pkg/api/kpm_pkg_test.go index f06a315b..2e5834bd 100644 --- a/pkg/api/kpm_pkg_test.go +++ b/pkg/api/kpm_pkg_test.go @@ -45,9 +45,9 @@ func testPackageApi(t *testing.T) { assert.Equal(t, dep.Name, "helloworld") assert.Equal(t, dep.FullName, "helloworld_0.1.3") assert.Equal(t, dep.Version, "0.1.3") - assert.Equal(t, dep.Source.Registry.Oci.Reg, "ghcr.io") - assert.Equal(t, dep.Source.Registry.Oci.Repo, "kcl-lang/helloworld") - assert.Equal(t, dep.Source.Registry.Oci.Tag, "0.1.3") + assert.Equal(t, dep.Source.Oci.Reg, "ghcr.io") + assert.Equal(t, dep.Source.Oci.Repo, "kcl-lang/helloworld") + assert.Equal(t, dep.Source.Oci.Tag, "0.1.3") assert.Equal(t, dep.GetLocalFullPath(""), filepath.Join(kcl_pkg_path, "helloworld_0.1.3")) @@ -83,9 +83,9 @@ func TestApiGetDependenciesInModFile(t *testing.T) { assert.Equal(t, dep.Name, "k8s") assert.Equal(t, dep.FullName, "k8s_1.27") assert.Equal(t, dep.Version, "1.27") - assert.Equal(t, dep.Source.Registry.Oci.Reg, "ghcr.io") - assert.Equal(t, dep.Source.Registry.Oci.Repo, "kcl-lang/k8s") - assert.Equal(t, dep.Source.Registry.Oci.Tag, "1.27") + assert.Equal(t, dep.Source.Oci.Reg, "ghcr.io") + assert.Equal(t, dep.Source.Oci.Repo, "kcl-lang/k8s") + assert.Equal(t, dep.Source.Oci.Tag, "1.27") } func testGetAllSchemaTypesMappingNamed(t *testing.T) { diff --git a/pkg/client/client.go b/pkg/client/client.go index c002e071..5d000786 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -425,18 +425,10 @@ func (c *KpmClient) UpdateDeps(kclPkg *pkg.KclPkg) error { } } else { // update kcl.mod - err = kclPkg.ModFile.StoreModFile() + err = kclPkg.UpdateModAndLockFile() if err != nil { return err } - - // Generate file kcl.mod.lock. - if !kclPkg.NoSumCheck { - err := kclPkg.LockDepsVersion() - if err != nil { - return err - } - } } return nil @@ -537,6 +529,20 @@ func (c *KpmClient) AddDepWithOpts(kclPkg *pkg.KclPkg, opt *opt.AddOptions) (*pk return nil, err } + // Backup the dependency used in kcl.mod + if opt.RegistryOpts.Registry != nil { + kclPkg.BackupDepUI(d.Name, &pkg.Dependency{ + Name: d.Name, + Version: d.Version, + Source: downloader.Source{ + ModSpec: &downloader.ModSpec{ + Name: d.Name, + Version: d.Version, + }, + }, + }) + } + reporter.ReportMsgTo( fmt.Sprintf("adding dependency '%s'", d.Name), c.logWriter, @@ -684,18 +690,6 @@ func (c *KpmClient) FillDepInfo(dep *pkg.Dependency, homepath string) error { dep.Source.Oci.Repo = urlpath } } - if dep.Source.Registry != nil { - if len(dep.Source.Registry.Reg) == 0 { - dep.Source.Registry.Reg = c.GetSettings().DefaultOciRegistry() - } - - if len(dep.Source.Registry.Repo) == 0 { - urlpath := utils.JoinPath(c.GetSettings().DefaultOciRepo(), dep.Name) - dep.Source.Registry.Repo = urlpath - } - - dep.Version = dep.Source.Registry.Version - } if dep.Source.Git != nil && dep.Source.Git.GetPackage() != "" { name := utils.ParseRepoNameFromGitUrl(dep.Source.Git.Url) if len(dep.Source.Git.Tag) != 0 { @@ -846,12 +840,10 @@ func (c *KpmClient) Download(dep *pkg.Dependency, homePath, localPath string) (* dep.Version = modFile.Pkg.Version } - if dep.Source.Oci != nil || dep.Source.Registry != nil { + if dep.Source.Oci != nil { var ociSource *downloader.Oci if dep.Source.Oci != nil { ociSource = dep.Source.Oci - } else if dep.Source.Registry != nil { - ociSource = dep.Source.Registry.Oci } // Select the latest tag, if the tag, the user inputed, is empty. if ociSource.Tag == "" || ociSource.Tag == constants.LATEST { @@ -860,10 +852,8 @@ func (c *KpmClient) Download(dep *pkg.Dependency, homePath, localPath string) (* return nil, err } ociSource.Tag = latestTag - - if dep.Source.Registry != nil { - dep.Source.Registry.Tag = latestTag - dep.Source.Registry.Version = latestTag + if dep.Source.ModSpec != nil { + dep.Source.ModSpec.Version = latestTag } // Complete some information that the local three dependencies depend on. diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 449fff59..1223cca3 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -57,7 +57,7 @@ func initTestDir(subDir string) string { func TestWithGlobalLock(t *testing.T) { test.RunTestWithGlobalLock(t, "TestUpdateWithKclMod", testUpdateWithKclMod) test.RunTestWithGlobalLock(t, "TestUpdateWithKclModlock", testUpdateWithKclModlock) - test.RunTestWithGlobalLock(t, "TestUpdateWithNoSumCheck", TestUpdateWithNoSumCheck) + test.RunTestWithGlobalLock(t, "TestUpdateWithNoSumCheck", testUpdateWithNoSumCheck) test.RunTestWithGlobalLock(t, "TestAddWithDiffVersionNoSumCheck", testAddWithDiffVersionNoSumCheck) test.RunTestWithGlobalLock(t, "TestAddWithDiffVersionWithSumCheck", testAddWithDiffVersionWithSumCheck) test.RunTestWithGlobalLock(t, "TestDownloadOci", testDownloadOci) @@ -1190,7 +1190,7 @@ func testAddWithNoSumCheck(t *testing.T) { }() } -func TestUpdateWithNoSumCheck(t *testing.T) { +func testUpdateWithNoSumCheck(t *testing.T) { pkgPath := getTestDir("test_update_no_sum_check") kpmcli, err := NewKpmClient() assert.Equal(t, err, nil) diff --git a/pkg/client/test_data/run_with_default_dep/kcl.mod.lock.expect b/pkg/client/test_data/run_with_default_dep/kcl.mod.lock.expect index e64da0b4..c1bebf8c 100644 --- a/pkg/client/test_data/run_with_default_dep/kcl.mod.lock.expect +++ b/pkg/client/test_data/run_with_default_dep/kcl.mod.lock.expect @@ -3,3 +3,6 @@ name = "helloworld" full_name = "helloworld_0.1.2" version = "0.1.2" + reg = "ghcr.io" + repo = "kcl-lang/helloworld" + oci_tag = "0.1.2" diff --git a/pkg/client/test_data/test_add_diff_version/with_sum_check/kcl.mod.lock.expect b/pkg/client/test_data/test_add_diff_version/with_sum_check/kcl.mod.lock.expect index e64da0b4..c1bebf8c 100644 --- a/pkg/client/test_data/test_add_diff_version/with_sum_check/kcl.mod.lock.expect +++ b/pkg/client/test_data/test_add_diff_version/with_sum_check/kcl.mod.lock.expect @@ -3,3 +3,6 @@ name = "helloworld" full_name = "helloworld_0.1.2" version = "0.1.2" + reg = "ghcr.io" + repo = "kcl-lang/helloworld" + oci_tag = "0.1.2" diff --git a/pkg/client/test_data/test_mod_file_package/test_pkg/expect.mod b/pkg/client/test_data/test_mod_file_package/test_pkg/expect.mod index b9c6d8ef..fc5f43d5 100644 --- a/pkg/client/test_data/test_mod_file_package/test_pkg/expect.mod +++ b/pkg/client/test_data/test_mod_file_package/test_pkg/expect.mod @@ -4,4 +4,4 @@ edition = "v0.10.0" version = "0.0.1" [dependencies] -cc = { git = "https://github.com/kcl-lang/flask-demo-kcl-manifests.git", commit = "8308200", package = "cc" } \ No newline at end of file +cc = { git = "https://github.com/kcl-lang/flask-demo-kcl-manifests.git", commit = "8308200", package = "cc"} \ No newline at end of file diff --git a/pkg/client/test_data/test_update/test_update_kcl_mod/expected b/pkg/client/test_data/test_update/test_update_kcl_mod/expected index e64da0b4..c1bebf8c 100644 --- a/pkg/client/test_data/test_update/test_update_kcl_mod/expected +++ b/pkg/client/test_data/test_update/test_update_kcl_mod/expected @@ -3,3 +3,6 @@ name = "helloworld" full_name = "helloworld_0.1.2" version = "0.1.2" + reg = "ghcr.io" + repo = "kcl-lang/helloworld" + oci_tag = "0.1.2" diff --git a/pkg/client/test_data/test_update_with_mvs/update_0/pkg/kcl.mod.lock b/pkg/client/test_data/test_update_with_mvs/update_0/pkg/kcl.mod.lock index e60448ad..f3d86095 100644 --- a/pkg/client/test_data/test_update_with_mvs/update_0/pkg/kcl.mod.lock +++ b/pkg/client/test_data/test_update_with_mvs/update_0/pkg/kcl.mod.lock @@ -15,3 +15,6 @@ name = "helloworld" full_name = "helloworld_0.1.1" version = "0.1.1" + reg = "ghcr.io" + repo = "kcl-lang/helloworld" + oci_tag = "0.1.1" diff --git a/pkg/client/test_data/test_update_with_mvs/update_0/pkg/kcl.mod.lock.expect b/pkg/client/test_data/test_update_with_mvs/update_0/pkg/kcl.mod.lock.expect index e60448ad..f3d86095 100644 --- a/pkg/client/test_data/test_update_with_mvs/update_0/pkg/kcl.mod.lock.expect +++ b/pkg/client/test_data/test_update_with_mvs/update_0/pkg/kcl.mod.lock.expect @@ -15,3 +15,6 @@ name = "helloworld" full_name = "helloworld_0.1.1" version = "0.1.1" + reg = "ghcr.io" + repo = "kcl-lang/helloworld" + oci_tag = "0.1.1" diff --git a/pkg/client/test_data/test_update_with_mvs/update_1/pkg/kcl.mod.lock b/pkg/client/test_data/test_update_with_mvs/update_1/pkg/kcl.mod.lock index e60448ad..f3d86095 100644 --- a/pkg/client/test_data/test_update_with_mvs/update_1/pkg/kcl.mod.lock +++ b/pkg/client/test_data/test_update_with_mvs/update_1/pkg/kcl.mod.lock @@ -15,3 +15,6 @@ name = "helloworld" full_name = "helloworld_0.1.1" version = "0.1.1" + reg = "ghcr.io" + repo = "kcl-lang/helloworld" + oci_tag = "0.1.1" diff --git a/pkg/client/test_data/test_update_with_mvs/update_1/pkg/kcl.mod.lock.expect b/pkg/client/test_data/test_update_with_mvs/update_1/pkg/kcl.mod.lock.expect index e60448ad..f3d86095 100644 --- a/pkg/client/test_data/test_update_with_mvs/update_1/pkg/kcl.mod.lock.expect +++ b/pkg/client/test_data/test_update_with_mvs/update_1/pkg/kcl.mod.lock.expect @@ -15,3 +15,6 @@ name = "helloworld" full_name = "helloworld_0.1.1" version = "0.1.1" + reg = "ghcr.io" + repo = "kcl-lang/helloworld" + oci_tag = "0.1.1" diff --git a/pkg/client/test_data/update_with_default_dep/kcl.mod.lock.expect b/pkg/client/test_data/update_with_default_dep/kcl.mod.lock.expect index e64da0b4..358da422 100644 --- a/pkg/client/test_data/update_with_default_dep/kcl.mod.lock.expect +++ b/pkg/client/test_data/update_with_default_dep/kcl.mod.lock.expect @@ -3,3 +3,7 @@ name = "helloworld" full_name = "helloworld_0.1.2" version = "0.1.2" + reg = "ghcr.io" + repo = "kcl-lang/helloworld" + oci_tag = "0.1.2" + diff --git a/pkg/downloader/downloader.go b/pkg/downloader/downloader.go index b61d25f6..a5e277e7 100644 --- a/pkg/downloader/downloader.go +++ b/pkg/downloader/downloader.go @@ -142,14 +142,11 @@ func (d *DepDownloader) Download(opts DownloadOptions) error { localPath := opts.LocalPath cacheFullPath := opts.CachePath - if opts.EnableCache { + if ok, err := features.Enabled(features.SupportNewStorage); err == nil && !ok && opts.EnableCache { // TODO: After the new local storage structure is complete, // this section should be replaced with the new storage structure instead of the cache path according to the /. // https://github.com/kcl-lang/kpm/issues/384 var pkgFullName string - if opts.Source.Registry != nil && len(opts.Source.Registry.Version) != 0 { - pkgFullName = fmt.Sprintf("%s_%s", filepath.Base(opts.Source.Registry.Oci.Repo), opts.Source.Registry.Version) - } if opts.Source.Oci != nil && len(opts.Source.Oci.Tag) != 0 { pkgFullName = fmt.Sprintf("%s_%s", filepath.Base(opts.Source.Oci.Repo), opts.Source.Oci.Tag) } @@ -167,7 +164,7 @@ func (d *DepDownloader) Download(opts DownloadOptions) error { pkgFullName = fmt.Sprintf("%s_%s", filepath.Base(gitUrl), opts.Source.Git.Commit) } - cacheFullPath := filepath.Join(opts.CachePath, pkgFullName) + cacheFullPath = filepath.Join(opts.CachePath, pkgFullName) if utils.DirExists(cacheFullPath) && utils.DirExists(filepath.Join(cacheFullPath, constants.KCL_MOD)) { // copy the cache to the local path @@ -188,10 +185,7 @@ func (d *DepDownloader) Download(opts DownloadOptions) error { opts.LocalPath = tmpDir // Dispatch the download to the specific downloader by package source. - if opts.Source.Oci != nil || opts.Source.Registry != nil { - if opts.Source.Registry != nil { - opts.Source.Oci = opts.Source.Registry.Oci - } + if opts.Source.Oci != nil { if d.OciDownloader == nil { d.OciDownloader = &OciDownloader{} } diff --git a/pkg/downloader/source.go b/pkg/downloader/source.go index 77943038..81a6f44c 100644 --- a/pkg/downloader/source.go +++ b/pkg/downloader/source.go @@ -8,20 +8,40 @@ import ( "path/filepath" "strings" + "github.com/hashicorp/go-version" "kcl-lang.io/kpm/pkg/constants" "kcl-lang.io/kpm/pkg/opt" "kcl-lang.io/kpm/pkg/settings" "kcl-lang.io/kpm/pkg/utils" ) -// Source is the package source from registry. +// The KCL module. +type ModSpec struct { + Name string + Version string +} + +// IsNil returns true if the ModSpec is nil. +func (p *ModSpec) IsNil() bool { + return p == nil || (p.Name == "" && p.Version == "") +} + +// Source is the module source. +// It can be from git, oci, local path. +// `ModSpec` is used to represent the module in the source. +// If there are more than one module from the source, use `ModSpec` to specify the module. +// If the `ModSpec` is nil, it means the source is one module. type Source struct { - *Registry + ModSpec *ModSpec `toml:"-"` *Git *Oci *Local `toml:"-"` } +func (s *Source) SpecOnly() bool { + return !s.ModSpec.IsNil() && s.Git == nil && s.Oci == nil && s.Local == nil +} + type Local struct { Path string `toml:"path,omitempty"` } @@ -42,12 +62,6 @@ type Git struct { Package string `toml:"package,omitempty"` } -type Registry struct { - *Oci `toml:"-"` - Name string `toml:"-"` - Version string `toml:"-"` -} - func NewSourceFromStr(sourceStr string) (*Source, error) { source := &Source{} err := source.FromString(sourceStr) @@ -58,7 +72,7 @@ func NewSourceFromStr(sourceStr string) (*Source, error) { } func (source *Source) IsNilSource() bool { - return source == nil || (source.Git == nil && source.Oci == nil && source.Local == nil && source.Registry == nil) + return source == nil || (source.Git == nil && source.Oci == nil && source.Local == nil && source.ModSpec.IsNil()) } func (source *Source) IsLocalPath() bool { @@ -74,11 +88,11 @@ func (source *Source) IsLocalTgzPath() bool { } func (source *Source) IsRemote() bool { - return source.Git != nil || source.Oci != nil || source.Registry != nil + return source.Git != nil || source.Oci != nil || !source.ModSpec.IsNil() } func (source *Source) IsPackaged() bool { - return source.IsLocalTarPath() || source.Git != nil || source.Oci != nil || source.Registry != nil + return source.IsLocalTarPath() || source.Git != nil || source.Oci != nil || !source.ModSpec.IsNil() } // If the source is a local path, check if it is a real local package(a directory with kcl.mod file). @@ -102,9 +116,6 @@ func (source *Source) FindRootPath() (string, error) { if source.Local != nil { return source.Local.FindRootPath() } - if source.Registry != nil { - return source.Registry.ToFilePath() - } return "", fmt.Errorf("source is nil") } @@ -198,9 +209,6 @@ func (source *Source) ToFilePath() (string, error) { if source.Local != nil { return source.Local.ToFilePath() } - if source.Registry != nil { - return source.Registry.ToFilePath() - } return "", fmt.Errorf("source is nil") } @@ -252,18 +260,6 @@ func (local *Local) ToFilePath() (string, error) { return local.ToString() } -func (registry *Registry) ToFilePath() (string, error) { - if registry == nil { - return "", fmt.Errorf("registry is nil") - } - - ociPath, err := registry.Oci.ToFilePath() - if err != nil { - return "", err - } - return ociPath, nil -} - func (source *Source) ToString() (string, error) { if source == nil { return "", fmt.Errorf("source is nil") @@ -277,9 +273,6 @@ func (source *Source) ToString() (string, error) { if source.Local != nil { return source.Local.ToString() } - if source.Registry != nil { - return source.Registry.ToString() - } return "", fmt.Errorf("source is nil") } @@ -343,15 +336,6 @@ func (local *Local) ToString() (string, error) { return pathUrl.String(), nil } -func (registry *Registry) ToString() (string, error) { - ociStr, err := registry.Oci.ToString() - if err != nil { - return "", err - } - - return ociStr, nil -} - func (source *Source) FromString(sourceStr string) error { if source == nil { return fmt.Errorf("source is nil") @@ -369,8 +353,8 @@ func (source *Source) FromString(sourceStr string) error { source.Oci = &Oci{} source.Oci.FromString(sourceStr) } else if sourceUrl.Scheme == constants.DefaultOciScheme { - source.Registry = &Registry{} - source.Registry.FromString(sourceStr) + source.ModSpec = &ModSpec{} + source.ModSpec.FromString(sourceStr) } else { source.Local = &Local{} source.Local.FromString(sourceStr) @@ -435,32 +419,31 @@ func (local *Local) FromString(localStr string) error { return nil } -// default::oci://ghcr.io/kcl-lang/k8s?name=k8s?version=0.1.1 -func (registry *Registry) FromString(registryStr string) error { - if registry == nil { +func (ps *ModSpec) FromString(registryStr string) error { + if ps == nil { return fmt.Errorf("registry is nil") } + parts := strings.Split(registryStr, ":") + if len(parts) == 1 { + return nil + } - registryUrl, err := url.Parse(registryStr) - if err != nil { - return err + if len(parts) > 2 { + return errors.New("invalid package reference") } - if registryUrl.Scheme != constants.DefaultOciScheme { - return fmt.Errorf("invalid registry url with schema: %s", registryUrl.Scheme) + if parts[1] == "" { + return errors.New("invalid package reference") } - oci := &Oci{} - registryUrl.Scheme = constants.OciScheme - err = oci.FromString(registryUrl.String()) + ps.Name = parts[0] + ps.Version = parts[1] + + _, err := version.NewVersion(ps.Version) if err != nil { return err } - registry.Name = registryUrl.Query().Get("name") - registry.Version = registryUrl.Query().Get("version") - registry.Oci = oci - return nil } @@ -559,9 +542,6 @@ func (s *Source) Hash() (string, error) { if s.Local != nil { return s.Local.Hash() } - if s.Registry != nil { - return s.Registry.Hash() - } return "", nil } @@ -605,7 +585,3 @@ func (o *Oci) Hash() (string, error) { func (l *Local) Hash() (string, error) { return utils.ShortHash(l.Path) } - -func (r *Registry) Hash() (string, error) { - return r.Oci.Hash() -} diff --git a/pkg/downloader/toml.go b/pkg/downloader/toml.go index 565c4b34..c2201382 100644 --- a/pkg/downloader/toml.go +++ b/pkg/downloader/toml.go @@ -10,55 +10,58 @@ import ( const NEWLINE = "\n" const SOURCE_PATTERN = "{ %s }" +const DEP_PATTERN = "%s = %s" +const SPEC_PATTERN = "%s = %q" -func (source *Source) MarshalTOML() string { +func (ps *ModSpec) MarshalTOML() string { var sb strings.Builder - - if source.Registry != nil { - registryToml := source.Registry.MarshalTOML() - if len(registryToml) != 0 { - sb.WriteString(fmt.Sprintf("%q", registryToml)) - } + if ps != nil && len(ps.Version) != 0 && len(ps.Name) != 0 { + sb.WriteString(fmt.Sprintf(SPEC_PATTERN, ps.Name, ps.Version)) + return sb.String() } - if source.Git != nil { - gitToml := source.Git.MarshalTOML() - if len(gitToml) != 0 { - sb.WriteString(fmt.Sprintf(SOURCE_PATTERN, gitToml)) + return sb.String() +} + +func (source *Source) MarshalTOML() string { + var sb strings.Builder + if source.SpecOnly() { + return source.ModSpec.MarshalTOML() + } else { + var pkgVersion string + var tomlStr string + if source.ModSpec != nil && len(source.ModSpec.Version) > 0 { + pkgVersion = fmt.Sprintf(", version = %q", source.ModSpec.Version) } - } - if source.Oci != nil { - ociToml := source.Oci.MarshalTOML() - if len(ociToml) != 0 { - if len(source.Oci.Reg) != 0 && len(source.Oci.Repo) != 0 { - sb.WriteString(fmt.Sprintf(SOURCE_PATTERN, ociToml)) - } else { - sb.WriteString(ociToml) + if source.Git != nil { + tomlStr = source.Git.MarshalTOML() + if len(tomlStr) != 0 { + tomlStr = fmt.Sprintf(SOURCE_PATTERN, tomlStr+pkgVersion) } } - } - if source.Local != nil { - localPathToml := source.Local.MarshalTOML() - if len(localPathToml) != 0 { - sb.WriteString(fmt.Sprintf(SOURCE_PATTERN, localPathToml)) + if source.Oci != nil { + tomlStr = source.Oci.MarshalTOML() + if len(tomlStr) != 0 { + if len(source.Oci.Reg) != 0 && len(source.Oci.Repo) != 0 { + tomlStr = fmt.Sprintf(SOURCE_PATTERN, tomlStr+pkgVersion) + } + } } - } - - return sb.String() -} -func (registry *Registry) MarshalTOML() string { - var sb strings.Builder - if len(registry.Version) != 0 { - sb.WriteString(registry.Version) - return sb.String() - } + if source.Local != nil { + tomlStr = source.Local.MarshalTOML() + if len(tomlStr) != 0 { + tomlStr = fmt.Sprintf(SOURCE_PATTERN, tomlStr+pkgVersion) + } + } - if len(registry.Oci.Tag) != 0 { - sb.WriteString(registry.Oci.Tag) - return sb.String() + if source.ModSpec != nil && len(source.ModSpec.Name) != 0 { + sb.WriteString(fmt.Sprintf(DEP_PATTERN, source.ModSpec.Name, tomlStr)) + } else { + sb.WriteString(tomlStr) + } } return sb.String() @@ -155,24 +158,35 @@ func (source *Source) UnmarshalModTOML(data interface{}) error { return err } source.Oci = &oci - } else { - reg := Registry{} - err := reg.UnmarshalModTOML(data) + } + + if v, ok := meta["version"].(string); ok { + pSpec := ModSpec{} + err := pSpec.UnmarshalModTOML(v) if err != nil { return err } - source.Registry = ® + source.ModSpec = &pSpec } } _, ok = data.(string) if ok { - registry := Registry{} - err := registry.UnmarshalModTOML(data) + pSpec := ModSpec{} + err := pSpec.UnmarshalModTOML(data) if err != nil { return err } - source.Registry = ®istry + source.ModSpec = &pSpec + } + + return nil +} + +func (ps *ModSpec) UnmarshalModTOML(data interface{}) error { + version, ok := data.(string) + if ok { + ps.Version = version } return nil @@ -244,14 +258,3 @@ func (local *Local) UnmarshalModTOML(data interface{}) error { return nil } - -func (reg *Registry) UnmarshalModTOML(data interface{}) error { - version, ok := data.(string) - if ok { - reg.Version = version - reg.Oci = &Oci{} - reg.Oci.Tag = version - } - - return nil -} diff --git a/pkg/package/modfile.go b/pkg/package/modfile.go index 49494e8e..212302c5 100644 --- a/pkg/package/modfile.go +++ b/pkg/package/modfile.go @@ -283,8 +283,6 @@ func (d *Dependency) GenPathSuffix() string { } else { storePkgName = fmt.Sprintf(PKG_NAME_PATTERN, name, d.Source.Git.Branch) } - } else if d.Source.Registry != nil { - storePkgName = fmt.Sprintf(PKG_NAME_PATTERN, name, d.Source.Registry.Version) } else { storePkgName = fmt.Sprintf(PKG_NAME_PATTERN, d.Name, d.Version) } @@ -348,9 +346,6 @@ func (dep *Dependency) GetDownloadPath() string { if dep.Source.Oci != nil { return dep.Source.Oci.IntoOciUrl() } - if dep.Source.Registry != nil { - return dep.Source.Registry.Oci.IntoOciUrl() - } return "" } @@ -384,7 +379,7 @@ func (dep *Dependency) GetSourceType() string { if dep.Source.Git != nil { return GIT } - if dep.Source.Oci != nil || dep.Source.Registry != nil { + if dep.Source.Oci != nil { return OCI } if dep.Source.Local != nil { @@ -572,11 +567,11 @@ func ParseOpt(opt *opt.RegistryOptions) (*Dependency, error) { Name: opt.Registry.Ref, FullName: opt.Registry.Ref + "_" + opt.Registry.Tag, Source: downloader.Source{ - Registry: &downloader.Registry{ - Oci: &ociSource, + ModSpec: &downloader.ModSpec{ Version: opt.Registry.Tag, Name: opt.Registry.Ref, }, + Oci: &ociSource, }, Version: opt.Registry.Tag, }, nil diff --git a/pkg/package/modfile_test.go b/pkg/package/modfile_test.go index 68e9ab73..b42a910e 100644 --- a/pkg/package/modfile_test.go +++ b/pkg/package/modfile_test.go @@ -192,7 +192,7 @@ func TestLoadModFile(t *testing.T) { assert.Equal(t, modFile.Dependencies.Deps.GetOrDefault("oci_name", TestPkgDependency).Name, "oci_name") assert.Equal(t, modFile.Dependencies.Deps.GetOrDefault("oci_name", TestPkgDependency).Version, "oci_tag") - assert.Equal(t, modFile.Dependencies.Deps.GetOrDefault("oci_name", TestPkgDependency).Source.Registry.Tag, "oci_tag") + assert.Equal(t, modFile.Dependencies.Deps.GetOrDefault("oci_name", TestPkgDependency).Source.ModSpec.Version, "oci_tag") assert.Equal(t, err, nil) assert.Equal(t, modFile.Dependencies.Deps.GetOrDefault("helloworld", TestPkgDependency).Name, "helloworld") diff --git a/pkg/package/package.go b/pkg/package/package.go index 94a70a4d..b16dce67 100644 --- a/pkg/package/package.go +++ b/pkg/package/package.go @@ -7,6 +7,7 @@ import ( "strings" orderedmap "github.com/elliotchance/orderedmap/v2" + "github.com/jinzhu/copier" "kcl-lang.io/kcl-go/pkg/kcl" "kcl-lang.io/kpm/pkg/constants" "kcl-lang.io/kpm/pkg/downloader" @@ -32,6 +33,13 @@ type KclPkg struct { Dependencies // The flag 'NoSumCheck' is true if the checksum of the current kcl package is not checked. NoSumCheck bool + // A snapshot of the dependencies in kcl.mod + // readonly and user can't modify it. + depUI DependenciesUI +} + +func (p *KclPkg) BackupDepUI(name string, dep *Dependency) { + p.depUI.Deps[name] = *dep } type LoadOptions struct { @@ -80,6 +88,25 @@ func LoadKclPkgWithOpts(options ...LoadOption) (*KclPkg, error) { return nil, fmt.Errorf("could not load 'kcl.mod' in '%s'\n%w", pkgPath, err) } + // Save snapshot of the dependencies in kcl.mod. + depsUI := DependenciesUI{ + Deps: make(map[string]Dependency), + } + + for _, name := range modFile.Deps.Keys() { + dep, ok := modFile.Deps.Get(name) + if !ok { + return nil, fmt.Errorf("could not load 'kcl.mod' in '%s'\n%w", pkgPath, err) + } + depSnap := Dependency{} + err := copier.Copy(&depSnap, &dep) + if err != nil { + fmt.Printf("failed to copy dependency: %v\n", err) + continue + } + depsUI.Deps[name] = depSnap + } + // pre-process the package. // 1. Transform the local path to the absolute path. err = convertDepsLocalPathToAbsPath(&modFile.Dependencies, pkgPath) @@ -103,8 +130,11 @@ func LoadKclPkgWithOpts(options ...LoadOption) (*KclPkg, error) { } else { // If there is no source in the lock file, fill the default oci registry. if lockDep.Source.IsNilSource() { - lockDep.Source.Registry = &downloader.Registry{ - Name: lockDep.Name, + lockDep.Source = downloader.Source{ + ModSpec: &downloader.ModSpec{ + Name: lockDep.Name, + Version: lockDep.Version, + }, Oci: &downloader.Oci{ Reg: opts.Settings.DefaultOciRegistry(), Repo: utils.JoinPath(opts.Settings.DefaultOciRepo(), lockDep.Name), @@ -120,6 +150,7 @@ func LoadKclPkgWithOpts(options ...LoadOption) (*KclPkg, error) { ModFile: *modFile, HomePath: pkgPath, Dependencies: *deps, + depUI: depsUI, }, nil } @@ -202,27 +233,11 @@ func fillDepsInfoWithSettings(deps *Dependencies, settings *settings.Settings) e dep.Source.Oci.Repo = urlpath } } - if dep.Source.Registry != nil { - if len(dep.Source.Registry.Reg) == 0 { - dep.Source.Registry.Reg = settings.DefaultOciRegistry() - } - - if len(dep.Source.Registry.Repo) == 0 { - urlpath := utils.JoinPath(settings.DefaultOciRepo(), dep.Name) - dep.Source.Registry.Repo = urlpath - } - - dep.Version = dep.Source.Registry.Version - } - if dep.Source.IsNilSource() { - dep.Source.Registry = &downloader.Registry{ - Name: dep.Name, - Version: dep.Version, - Oci: &downloader.Oci{ - Reg: settings.DefaultOciRegistry(), - Repo: utils.JoinPath(settings.DefaultOciRepo(), dep.Name), - Tag: dep.Version, - }, + if dep.Source.SpecOnly() { + dep.Source.Oci = &downloader.Oci{ + Reg: settings.DefaultOciRegistry(), + Repo: utils.JoinPath(settings.DefaultOciRepo(), dep.ModSpec.Name), + Tag: dep.ModSpec.Version, } } dep.FullName = dep.GenDepFullName() @@ -369,6 +384,26 @@ func (kclPkg *KclPkg) LocalVendorPath() string { // updateModAndLockFile will update kcl.mod and kcl.mod.lock func (kclPkg *KclPkg) UpdateModAndLockFile() error { + + // Load kcl.mod SnapShot. + depSnapShot := kclPkg.depUI + + for _, name := range kclPkg.ModFile.Deps.Keys() { + modDep, ok := kclPkg.ModFile.Deps.Get(name) + if !ok { + return fmt.Errorf("failed to get dependency %s", name) + } + + if existDep, ok := depSnapShot.Deps[name]; ok { + if existDep.Source.SpecOnly() { + existDep.Source.ModSpec = modDep.ModSpec + } else { + existDep.Source = modDep.Source + } + kclPkg.ModFile.Deps.Set(name, existDep) + } + } + // Generate file kcl.mod. err := kclPkg.ModFile.StoreModFile() if err != nil { diff --git a/pkg/package/package_test.go b/pkg/package/package_test.go index 9b7d056c..b5a51846 100644 --- a/pkg/package/package_test.go +++ b/pkg/package/package_test.go @@ -113,7 +113,7 @@ func TestLoadKclPkgFromTar(t *testing.T) { assert.Equal(t, kclPkg.ModFile.Deps.GetOrDefault("oci_konfig", TestPkgDependency).Name, "oci_konfig") assert.Equal(t, kclPkg.ModFile.Deps.GetOrDefault("oci_konfig", TestPkgDependency).FullName, "oci_konfig_0.0.1") - assert.Equal(t, kclPkg.ModFile.Deps.GetOrDefault("oci_konfig", TestPkgDependency).Registry.Tag, "0.0.1") + assert.Equal(t, kclPkg.ModFile.Deps.GetOrDefault("oci_konfig", TestPkgDependency).ModSpec.Version, "0.0.1") assert.Equal(t, kclPkg.Deps.Len(), 2) assert.Equal(t, kclPkg.Deps.GetOrDefault("konfig", TestPkgDependency).Name, "konfig") @@ -124,9 +124,9 @@ func TestLoadKclPkgFromTar(t *testing.T) { assert.Equal(t, kclPkg.Deps.GetOrDefault("oci_konfig", TestPkgDependency).Name, "oci_konfig") assert.Equal(t, kclPkg.Deps.GetOrDefault("oci_konfig", TestPkgDependency).FullName, "oci_konfig_0.0.1") - assert.Equal(t, kclPkg.Deps.GetOrDefault("oci_konfig", TestPkgDependency).Registry.Oci.Reg, "ghcr.io") - assert.Equal(t, kclPkg.Deps.GetOrDefault("oci_konfig", TestPkgDependency).Registry.Oci.Repo, "kcl-lang/oci_konfig") - assert.Equal(t, kclPkg.Deps.GetOrDefault("oci_konfig", TestPkgDependency).Registry.Oci.Tag, "0.0.1") + assert.Equal(t, kclPkg.Deps.GetOrDefault("oci_konfig", TestPkgDependency).Oci.Reg, "ghcr.io") + assert.Equal(t, kclPkg.Deps.GetOrDefault("oci_konfig", TestPkgDependency).Oci.Repo, "kcl-lang/oci_konfig") + assert.Equal(t, kclPkg.Deps.GetOrDefault("oci_konfig", TestPkgDependency).Oci.Tag, "0.0.1") assert.Equal(t, kclPkg.Deps.GetOrDefault("oci_konfig", TestPkgDependency).Sum, "sLr3e6W4RPrXYyswdOSiKqkHes1QHX2tk6SwxAPDqqo=") assert.Equal(t, kclPkg.GetPkgTag(), "0.0.3") @@ -151,7 +151,7 @@ func TestLoadPkgFromLock(t *testing.T) { assert.Equal(t, kpkg.Dependencies.Deps.Len(), 1) assert.Equal(t, kpkg.Dependencies.Deps.GetOrDefault("helloworld", TestPkgDependency).Name, "helloworld") assert.Equal(t, kpkg.Dependencies.Deps.GetOrDefault("helloworld", TestPkgDependency).FullName, "helloworld_0.1.2") - assert.Equal(t, kpkg.Dependencies.Deps.GetOrDefault("helloworld", TestPkgDependency).Source.Registry.Oci.Reg, "ghcr.io") - assert.Equal(t, kpkg.Dependencies.Deps.GetOrDefault("helloworld", TestPkgDependency).Source.Registry.Oci.Repo, "kcl-lang/helloworld") - assert.Equal(t, kpkg.Dependencies.Deps.GetOrDefault("helloworld", TestPkgDependency).Source.Registry.Oci.Tag, "0.1.2") + assert.Equal(t, kpkg.Dependencies.Deps.GetOrDefault("helloworld", TestPkgDependency).Source.Oci.Reg, "ghcr.io") + assert.Equal(t, kpkg.Dependencies.Deps.GetOrDefault("helloworld", TestPkgDependency).Source.Oci.Repo, "kcl-lang/helloworld") + assert.Equal(t, kpkg.Dependencies.Deps.GetOrDefault("helloworld", TestPkgDependency).Source.Oci.Tag, "0.1.2") } diff --git a/pkg/package/test_data/test_data_toml/expected.toml b/pkg/package/test_data/test_data_toml/expected.toml index 7bb599e9..d455d208 100644 --- a/pkg/package/test_data/test_data_toml/expected.toml +++ b/pkg/package/test_data/test_data_toml/expected.toml @@ -7,4 +7,4 @@ exclude = ["target/", ".git/", "*.log"] [dependencies] MyOciKcl1 = "0.0.1" -MyKcl1 = { git = "https://github.com/test/MyKcl1.git", tag = "v0.0.2" } +MyKcl1 = { git = "https://github.com/test/MyKcl1.git", tag = "v0.0.2", version = "0.0.2" } diff --git a/pkg/package/toml.go b/pkg/package/toml.go index 730297f4..f58bcf89 100644 --- a/pkg/package/toml.go +++ b/pkg/package/toml.go @@ -87,11 +87,14 @@ func (dep *Dependencies) MarshalTOML() string { const DEP_PATTERN = "%s = %s" func (dep *Dependency) MarshalTOML() string { - source := dep.Source.MarshalTOML() var sb strings.Builder - if len(source) != 0 { - sb.WriteString(fmt.Sprintf(DEP_PATTERN, dep.Name, source)) + + if dep.Source.ModSpec != nil && dep.Source.ModSpec.Version != "" && dep.Source.ModSpec.Name != "" { + sb.WriteString(dep.Source.MarshalTOML()) + } else { + sb.WriteString(fmt.Sprintf(DEP_PATTERN, dep.Name, dep.Source.MarshalTOML())) } + return sb.String() } @@ -252,12 +255,16 @@ func (dep *Dependency) UnmarshalModTOML(data interface{}) error { if source.Oci != nil { version = source.Oci.Tag } - if source.Registry != nil { - version = source.Registry.Version + + if source.ModSpec != nil { + version = source.ModSpec.Version } dep.FullName = fmt.Sprintf(PKG_NAME_PATTERN, dep.Name, version) dep.Version = version + if dep.Source.ModSpec != nil && dep.Source.ModSpec.Name == "" { + dep.Source.ModSpec.Name = dep.Name + } return nil } diff --git a/pkg/package/toml_test.go b/pkg/package/toml_test.go index 54cf225a..545b4300 100644 --- a/pkg/package/toml_test.go +++ b/pkg/package/toml_test.go @@ -34,6 +34,10 @@ func TestMarshalTOML(t *testing.T) { Name: "MyKcl1", FullName: "MyKcl1_v0.0.2", Source: downloader.Source{ + ModSpec: &downloader.ModSpec{ + Name: "MyKcl1", + Version: "0.0.2", + }, Git: &downloader.Git{ Url: "https://github.com/test/MyKcl1.git", Tag: "v0.0.2", @@ -46,8 +50,9 @@ func TestMarshalTOML(t *testing.T) { FullName: "MyOciKcl1_0.0.1", Version: "0.0.1", Source: downloader.Source{ - Oci: &downloader.Oci{ - Tag: "0.0.1", + ModSpec: &downloader.ModSpec{ + Name: "MyOciKcl1", + Version: "0.0.1", }, }, } @@ -81,7 +86,7 @@ func TestUnMarshalTOML(t *testing.T) { assert.Equal(t, modfile.Dependencies.Deps.Len(), 2) assert.NotEqual(t, modfile.Dependencies.Deps.GetOrDefault("MyKcl1", TestPkgDependency), nil) assert.Equal(t, modfile.Dependencies.Deps.GetOrDefault("MyKcl1", TestPkgDependency).Name, "MyKcl1") - assert.Equal(t, modfile.Dependencies.Deps.GetOrDefault("MyKcl1", TestPkgDependency).FullName, "MyKcl1_v0.0.2") + assert.Equal(t, modfile.Dependencies.Deps.GetOrDefault("MyKcl1", TestPkgDependency).FullName, "MyKcl1_0.0.2") assert.NotEqual(t, modfile.Dependencies.Deps.GetOrDefault("MyKcl1", TestPkgDependency).Source.Git, nil) assert.Equal(t, modfile.Dependencies.Deps.GetOrDefault("MyKcl1", TestPkgDependency).Source.Git.Url, "https://github.com/test/MyKcl1.git") assert.Equal(t, modfile.Dependencies.Deps.GetOrDefault("MyKcl1", TestPkgDependency).Source.Git.Tag, "v0.0.2") @@ -89,8 +94,8 @@ func TestUnMarshalTOML(t *testing.T) { assert.NotEqual(t, modfile.Dependencies.Deps.GetOrDefault("MyOciKcl1", TestPkgDependency), nil) assert.Equal(t, modfile.Dependencies.Deps.GetOrDefault("MyOciKcl1", TestPkgDependency).Name, "MyOciKcl1") assert.Equal(t, modfile.Dependencies.Deps.GetOrDefault("MyOciKcl1", TestPkgDependency).FullName, "MyOciKcl1_0.0.1") - assert.NotEqual(t, modfile.Dependencies.Deps.GetOrDefault("MyOciKcl1", TestPkgDependency).Source.Registry, nil) - assert.Equal(t, modfile.Dependencies.Deps.GetOrDefault("MyOciKcl1", TestPkgDependency).Source.Registry.Tag, "0.0.1") + assert.NotEqual(t, modfile.Dependencies.Deps.GetOrDefault("MyOciKcl1", TestPkgDependency).Source.ModSpec, nil) + assert.Equal(t, modfile.Dependencies.Deps.GetOrDefault("MyOciKcl1", TestPkgDependency).Source.ModSpec.Version, "0.0.1") } func TestMarshalLockTOML(t *testing.T) { @@ -237,6 +242,10 @@ func TestMarshalOciUrl(t *testing.T) { FullName: "oci_pkg_0.0.1", Version: "0.0.1", Source: downloader.Source{ + ModSpec: &downloader.ModSpec{ + Name: "oci_pkg", + Version: "0.0.1", + }, Oci: &downloader.Oci{ Reg: "ghcr.io", Repo: "kcl-lang/oci_pkg", @@ -306,6 +315,10 @@ func TestInitEmptyPkg(t *testing.T) { Name: "MyKcl1", FullName: "MyKcl1_v0.0.2", Source: downloader.Source{ + ModSpec: &downloader.ModSpec{ + Name: "MyKcl1", + Version: "0.0.2", + }, Git: &downloader.Git{ Url: "https://github.com/test/MyKcl1.git", Tag: "v0.0.2", @@ -318,8 +331,9 @@ func TestInitEmptyPkg(t *testing.T) { FullName: "MyOciKcl1_0.0.1", Version: "0.0.1", Source: downloader.Source{ - Oci: &downloader.Oci{ - Tag: "0.0.1", + ModSpec: &downloader.ModSpec{ + Name: "MyOciKcl1", + Version: "0.0.1", }, }, }