feat: make 'add' supports ModSpec (#514)

Signed-off-by: zongz <zongzhe1024@163.com>

pkg/client/add.go

@@ -0,0 +1,181 @@
+package client
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"kcl-lang.io/kpm/pkg/downloader"
+	pkg "kcl-lang.io/kpm/pkg/package"
+	"kcl-lang.io/kpm/pkg/utils"
+	"kcl-lang.io/kpm/pkg/visitor"
+)
+
+type AddOptions struct {
+	// Source is the source of the package to be pulled.
+	// Including git, oci, local.
+	Sources []*downloader.Source
+	KclPkg  *pkg.KclPkg
+}
+
+type AddOption func(*AddOptions) error
+
+func WithAddSource(source *downloader.Source) AddOption {
+	return func(opts *AddOptions) error {
+		if opts.Sources == nil {
+			opts.Sources = make([]*downloader.Source, 0)
+		}
+		opts.Sources = append(opts.Sources, source)
+		return nil
+	}
+}
+
+func WithAddSources(sources []*downloader.Source) AddOption {
+	return func(ro *AddOptions) error {
+		ro.Sources = sources
+		return nil
+	}
+}
+
+func WithAddSourceUrl(sourceUrl string) AddOption {
+	return func(opts *AddOptions) error {
+		if opts.Sources == nil {
+			opts.Sources = make([]*downloader.Source, 0)
+		}
+		source, err := downloader.NewSourceFromStr(sourceUrl)
+		if err != nil {
+			return err
+		}
+		opts.Sources = append(opts.Sources, source)
+		return nil
+	}
+}
+
+func WithAddSourceUrls(sourceUrls []string) AddOption {
+	return func(opts *AddOptions) error {
+		var sources []*downloader.Source
+		for _, sourceUrl := range sourceUrls {
+			source, err := downloader.NewSourceFromStr(sourceUrl)
+			if err != nil {
+				return err
+			}
+			sources = append(sources, source)
+		}
+		opts.Sources = sources
+		return nil
+	}
+}
+
+func WithAddKclPkg(kclPkg *pkg.KclPkg) AddOption {
+	return func(opts *AddOptions) error {
+		opts.KclPkg = kclPkg
+		return nil
+	}
+}
+
+func NewAddOptions(opts ...AddOption) *AddOptions {
+	ao := &AddOptions{}
+	for _, opt := range opts {
+		opt(ao)
+	}
+	return ao
+}
+
+func (c *KpmClient) Add(options ...AddOption) error {
+	opts := &AddOptions{}
+	for _, option := range options {
+		if err := option(opts); err != nil {
+			return err
+		}
+	}
+	addedPkg := opts.KclPkg
+
+	visitorSelector := func(source *downloader.Source) (visitor.Visitor, error) {
+		pkgVisitor := &visitor.PkgVisitor{
+			Settings:  &c.settings,
+			LogWriter: c.logWriter,
+		}
+
+		if source.IsRemote() {
+			return &visitor.RemoteVisitor{
+				PkgVisitor:            pkgVisitor,
+				Downloader:            c.DepDownloader,
+				InsecureSkipTLSverify: c.insecureSkipTLSverify,
+				EnableCache:           true,
+				CachePath:             c.homePath,
+				VisitedSpace:          c.homePath,
+			}, nil
+		} else if source.IsLocalTarPath() || source.IsLocalTgzPath() {
+			return visitor.NewArchiveVisitor(pkgVisitor), nil
+		} else if source.IsLocalPath() {
+			return pkgVisitor, nil
+		} else {
+			return nil, fmt.Errorf("unsupported source")
+		}
+	}
+
+	for _, depSource := range opts.Sources {
+		// Set the default OCI registry and repo if the source is nil and the package spec is not nil.
+		if depSource.IsNilSource() && !depSource.ModSpec.IsNil() {
+			depSource.Oci = &downloader.Oci{
+				Reg:  c.GetSettings().Conf.DefaultOciRegistry,
+				Repo: utils.JoinPath(c.GetSettings().Conf.DefaultOciRepo, depSource.ModSpec.Name),
+				Tag:  depSource.ModSpec.Version,
+			}
+		}
+
+		var fullSouce *downloader.Source
+		// Transform the relative path to the full path.
+		if depSource.IsLocalPath() && !filepath.IsAbs(depSource.Path) {
+			fullSouce = &downloader.Source{
+				ModSpec: depSource.ModSpec,
+				Local: &downloader.Local{
+					Path: filepath.Join(addedPkg.HomePath, depSource.Path),
+				},
+			}
+		} else {
+			fullSouce = depSource
+		}
+
+		visitor, err := visitorSelector(fullSouce)
+		if err != nil {
+			return err
+		}
+
+		// Visit the dependency source
+		// If the dependency is remote, the visitor will download it to the local.
+		// If the dependency is already in local cache, the visitor will not download it again.
+		err = visitor.Visit(fullSouce, func(depPkg *pkg.KclPkg) error {
+			dep := pkg.Dependency{
+				Name:          depPkg.ModFile.Pkg.Name,
+				FullName:      depPkg.GetPkgFullName(),
+				Version:       depPkg.ModFile.Pkg.Version,
+				LocalFullPath: depPkg.HomePath,
+				Source:        *depSource,
+			}
+
+			// Add the dependency to the kcl.mod file.
+			if modExistDep, ok := addedPkg.ModFile.Dependencies.Deps.Get(dep.Name); ok {
+				if less, err := modExistDep.VersionLessThan(&dep); less && err == nil {
+					addedPkg.ModFile.Dependencies.Deps.Set(dep.Name, dep)
+				}
+			} else {
+				addedPkg.ModFile.Dependencies.Deps.Set(dep.Name, dep)
+			}
+
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	// Iterate the dependencies and update the kcl.mod and kcl.mod.lock respectively.
+	_, err := c.Update(
+		WithUpdatedKclPkg(addedPkg),
+	)
+
+	if err != nil {
+		return err
+	}
+	return nil
+}

pkg/client/add_test.go

@@ -0,0 +1,117 @@
+package client
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/otiai10/copy"
+	"github.com/stretchr/testify/assert"
+	pkg "kcl-lang.io/kpm/pkg/package"
+	"kcl-lang.io/kpm/pkg/utils"
+)
+
+func testAddWithModSpec(t *testing.T) {
+	tests := []struct {
+		name       string
+		pkgSubPath string
+		sourceUrl  string
+	}{
+		{
+			name:       "TestAddOciWithModSpec",
+			pkgSubPath: "oci",
+			sourceUrl:  "oci://ghcr.io/kcl-lang/helloworld?tag=0.1.4&mod=subhelloworld:0.0.1",
+		},
+		{
+			name:       "TestAddGitWithModSpec",
+			pkgSubPath: "git",
+			sourceUrl:  "git://github.com/kcl-lang/flask-demo-kcl-manifests.git?commit=8308200&mod=cc:0.0.1",
+		},
+		{
+			name:       "TestAddLocalWithModSpec",
+			pkgSubPath: filepath.Join("local", "pkg"),
+			sourceUrl:  "../dep?mod=sub:0.0.1",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			testDir := getTestDir("add_with_mod_spec")
+			pkgPath := filepath.Join(testDir, tt.pkgSubPath)
+
+			modbkPath := filepath.Join(pkgPath, "kcl.mod.bk")
+			modPath := filepath.Join(pkgPath, "kcl.mod")
+			modExpect := filepath.Join(pkgPath, "kcl.mod.expect")
+			lockbkPath := filepath.Join(pkgPath, "kcl.mod.lock.bk")
+			lockPath := filepath.Join(pkgPath, "kcl.mod.lock")
+			lockExpect := filepath.Join(pkgPath, "kcl.mod.lock.expect")
+
+			err := copy.Copy(modbkPath, modPath)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			err = copy.Copy(lockbkPath, lockPath)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			defer func() {
+				// remove the copied files
+				err := os.RemoveAll(modPath)
+				if err != nil {
+					t.Fatal(err)
+				}
+				err = os.RemoveAll(lockPath)
+				if err != nil {
+					t.Fatal(err)
+				}
+			}()
+
+			kpmcli, err := NewKpmClient()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			kpkg, err := pkg.LoadKclPkgWithOpts(
+				pkg.WithPath(pkgPath),
+				pkg.WithSettings(kpmcli.GetSettings()),
+			)
+
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			err = kpmcli.Add(
+				WithAddKclPkg(kpkg),
+				WithAddSourceUrl(tt.sourceUrl),
+			)
+
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			expectedMod, err := os.ReadFile(modExpect)
+			if err != nil {
+				t.Fatal(err)
+			}
+			gotMod, err := os.ReadFile(modPath)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			expectedLock, err := os.ReadFile(lockExpect)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			gotLock, err := os.ReadFile(lockPath)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			assert.Equal(t, utils.RmNewline(string(expectedMod)), utils.RmNewline(string(gotMod)))
+			assert.Equal(t, utils.RmNewline(string(expectedLock)), utils.RmNewline(string(gotLock)))
+		})
+	}
+}

pkg/client/client_test.go

@@ -82,6 +82,7 @@ func TestWithGlobalLock(t *testing.T) {
 	test.RunTestWithGlobalLock(t, "TestDownloadGitWithPackage", testDownloadGitWithPackage)
 	test.RunTestWithGlobalLock(t, "TestModandLockFilesWithGitPackageDownload", testModandLockFilesWithGitPackageDownload)
 	test.RunTestWithGlobalLock(t, "TestDependencyGraph", testDependencyGraph)
+	test.RunTestWithGlobalLock(t, "testAddWithModSpec", testAddWithModSpec)
 }
 
 // TestDownloadOci test download from oci registry.

pkg/client/test_data/add_with_mod_spec/git/kcl.mod.bk

@@ -0,0 +1,4 @@
+[package]
+name = "git"
+edition = "v0.10.0"
+version = "0.0.1"

pkg/client/test_data/add_with_mod_spec/git/kcl.mod.expect

@@ -0,0 +1,7 @@
+[package]
+name = "git"
+edition = "v0.10.0"
+version = "0.0.1"
+
+[dependencies]
+cc = { git = "https://github.com/kcl-lang/flask-demo-kcl-manifests.git", commit = "8308200", version = "0.0.1" }

pkg/client/test_data/add_with_mod_spec/git/kcl.mod.lock.expect

@@ -0,0 +1,7 @@
+[dependencies]
+  [dependencies.cc]
+    name = "cc"
+    full_name = "cc_0.0.1"
+    version = "0.0.1"
+    url = "https://github.com/kcl-lang/flask-demo-kcl-manifests.git"
+    commit = "8308200"

pkg/client/test_data/add_with_mod_spec/local/dep/kcl.mod

@@ -0,0 +1,4 @@
+[package]
+name = "dep"
+edition = "v0.10.0"
+version = "0.0.1"

pkg/client/test_data/add_with_mod_spec/local/dep/main.k

@@ -0,0 +1 @@
+The_first_kcl_program = 'Hello World!'

pkg/client/test_data/add_with_mod_spec/local/dep/sub/kcl.mod

@@ -0,0 +1,4 @@
+[package]
+name = "sub"
+edition = "v0.10.0"
+version = "0.0.1"

pkg/client/test_data/add_with_mod_spec/local/dep/sub/main.k

@@ -0,0 +1 @@
+The_first_kcl_program = 'Hello World!'

pkg/client/test_data/add_with_mod_spec/local/pkg/kcl.mod.bk

@@ -0,0 +1,4 @@
+[package]
+name = "pkg"
+edition = "v0.10.0"
+version = "0.0.1"

pkg/client/test_data/add_with_mod_spec/local/pkg/kcl.mod.expect

@@ -0,0 +1,7 @@
+[package]
+name = "pkg"
+edition = "v0.10.0"
+version = "0.0.1"
+
+[dependencies]
+sub = { path = "../dep", version = "0.0.1" }

pkg/client/test_data/add_with_mod_spec/local/pkg/kcl.mod.lock.expect

@@ -0,0 +1,5 @@
+[dependencies]
+  [dependencies.sub]
+    name = "sub"
+    full_name = "sub_0.0.1"
+    version = "0.0.1"

pkg/client/test_data/add_with_mod_spec/local/pkg/main.k

@@ -0,0 +1 @@
+The_first_kcl_program = 'Hello World!'

pkg/client/test_data/add_with_mod_spec/oci/kcl.mod.bk

@@ -0,0 +1,6 @@
+[package]
+name = "oci"
+edition = "v0.10.0"
+version = "0.0.1"
+
+[dependencies]

pkg/client/test_data/add_with_mod_spec/oci/kcl.mod.expect

@@ -0,0 +1,7 @@
+[package]
+name = "oci"
+edition = "v0.10.0"
+version = "0.0.1"
+
+[dependencies]
+subhelloworld = { oci = "oci://ghcr.io/kcl-lang/helloworld", tag = "0.1.4", version = "0.0.1" }

pkg/client/test_data/add_with_mod_spec/oci/kcl.mod.lock.expect

@@ -0,0 +1,8 @@
+[dependencies]
+  [dependencies.subhelloworld]
+    name = "subhelloworld"
+    full_name = "subhelloworld_0.0.1"
+    version = "0.0.1"
+    reg = "ghcr.io"
+    repo = "kcl-lang/helloworld"
+    oci_tag = "0.1.4"

pkg/client/test_data/add_with_mod_spec/oci/main.k

@@ -0,0 +1 @@
+The_first_kcl_program = 'Hello World!'