feat: add checksum in kpm workflow (#520)

* integrate checksum in kpm workflow

Signed-off-by: Nishant Bansal <nishant.bansal.282003@gmail.com>

* added feature flag

Signed-off-by: Nishant Bansal <nishant.bansal.282003@gmail.com>

---------

Signed-off-by: Nishant Bansal <nishant.bansal.282003@gmail.com>

pkg/client/client.go

@@ -19,6 +19,7 @@ import (
 	"oras.land/oras-go/v2"
 	remoteauth "oras.land/oras-go/v2/registry/remote/auth"
 
+	"kcl-lang.io/kpm/pkg/checker"
 	"kcl-lang.io/kpm/pkg/constants"
 	"kcl-lang.io/kpm/pkg/downloader"
 	"kcl-lang.io/kpm/pkg/env"
@@ -47,6 +48,8 @@ type KpmClient struct {
 	homePath string
 	// The settings of kpm loaded from the global configuration file.
 	settings settings.Settings
+	// The checker to validate dependencies
+	DepChecker *checker.DepChecker
 	// The flag of whether to check the checksum of the package and update kcl.mod.lock.
 	noSumCheck bool
 	// The flag of whether to skip the verification of TLS.
@@ -66,10 +69,16 @@ func NewKpmClient() (*KpmClient, error) {
 		return nil, err
 	}
 
+	depChecker := checker.NewDepChecker(
+		checker.WithCheckers(checker.NewIdentChecker(), checker.NewVersionChecker(), checker.NewSumChecker(
+			checker.WithSettings(*settings))),
+	)
+
 	return &KpmClient{
 		logWriter:     os.Stdout,
 		settings:      *settings,
 		homePath:      homePath,
+		DepChecker:    depChecker,
 		DepDownloader: &downloader.DepDownloader{},
 	}, nil
 }
@@ -897,16 +906,11 @@ func (c *KpmClient) Download(dep *pkg.Dependency, homePath, localPath string) (*
 		}
 
 		dep.FromKclPkg(dpkg)
-		dep.Sum, err = c.AcquireDepSum(*dep)
+
+		dep.Sum, err = utils.HashDir(localPath)
 		if err != nil {
 			return nil, err
 		}
-		if dep.Sum == "" {
-			dep.Sum, err = utils.HashDir(localPath)
-			if err != nil {
-				return nil, err
-			}
-		}
 
 		if dep.LocalFullPath == "" {
 			dep.LocalFullPath = localPath
@@ -926,6 +930,9 @@ func (c *KpmClient) Download(dep *pkg.Dependency, homePath, localPath string) (*
 		// if err != nil {
 		//     return nil, err
 		// }
+		if err := c.ValidateDependency(dep); err != nil {
+			return nil, err
+		}
 	}
 
 	if dep.Source.Local != nil {
@@ -939,6 +946,26 @@ func (c *KpmClient) Download(dep *pkg.Dependency, homePath, localPath string) (*
 	return dep, nil
 }
 
+func (c *KpmClient) ValidateDependency(dep *pkg.Dependency) error {
+	if ok, err := features.Enabled(features.SupportCheckSum); err == nil && ok {
+		tmpKclPkg := pkg.KclPkg{
+			HomePath: dep.LocalFullPath,
+			Dependencies: pkg.Dependencies{Deps: func() *orderedmap.OrderedMap[string, pkg.Dependency] {
+				m := orderedmap.NewOrderedMap[string, pkg.Dependency]()
+				m.Set(dep.Name, *dep)
+				return m
+			}()},
+			NoSumCheck: c.GetNoSumCheck(),
+		}
+
+		if err := c.DepChecker.Check(tmpKclPkg); err != nil {
+			return reporter.NewErrorEvent(reporter.InvalidKclPkg, err, fmt.Sprintf("%s package does not match the original kcl package", dep.FullName))
+		}
+	}
+
+	return nil
+}
+
 // DownloadFromGit will download the dependency from the git repository.
 func (c *KpmClient) DownloadFromGit(dep *downloader.Git, localPath string) (string, error) {
 	var msg string
@@ -1148,13 +1175,57 @@ func (c *KpmClient) PullFromOci(localPath, source, tag string) error {
 		)
 	}
 
+	if err := c.ValidatePkgPullFromOci(ociOpts, storagePath); err != nil {
+		return reporter.NewErrorEvent(
+			reporter.InvalidKclPkg,
+			err,
+			fmt.Sprintf("failed to validate kclPkg at %s", storagePath),
+		)
+	}
+
 	reporter.ReportMsgTo(
 		fmt.Sprintf("pulled '%s' in '%s' successfully", source, storagePath),
 		c.logWriter,
 	)
 	return nil
 }
 
+func (c *KpmClient) ValidatePkgPullFromOci(ociOpts *opt.OciOptions, storagePath string) error {
+	kclPkg, err := c.LoadPkgFromPath(storagePath)
+	if err != nil {
+		return reporter.NewErrorEvent(
+			reporter.FailedGetPkg,
+			err,
+			fmt.Sprintf("failed to load kclPkg at %v", storagePath),
+		)
+	}
+
+	dep := &pkg.Dependency{
+		Name: kclPkg.GetPkgName(),
+		Source: downloader.Source{
+			Oci: &downloader.Oci{
+				Reg:  ociOpts.Reg,
+				Repo: ociOpts.Repo,
+				Tag:  ociOpts.Tag,
+			},
+		},
+	}
+
+	dep.FromKclPkg(kclPkg)
+	dep.Sum, err = utils.HashDir(storagePath)
+	if err != nil {
+		return reporter.NewErrorEvent(
+			reporter.FailedHashPkg,
+			err,
+			fmt.Sprintf("failed to hash kclPkg - %s", dep.Name),
+		)
+	}
+	if err := c.ValidateDependency(dep); err != nil {
+		return err
+	}
+	return nil
+}
+
 // PushToOci will push a kcl package to oci registry.
 func (c *KpmClient) PushToOci(localPath string, ociOpts *opt.OciOptions) error {
 	repoPath := utils.JoinPath(ociOpts.Reg, ociOpts.Repo)
@@ -1525,6 +1596,8 @@ func (c *KpmClient) pullTarFromOci(localPath string, ociOpts *opt.OciOptions) er
 		tagSelected = ociOpts.Tag
 	}
 
+	ociOpts.Tag = tagSelected
+
 	full_repo := utils.JoinPath(ociOpts.Reg, ociOpts.Repo)
 	reporter.ReportMsgTo(
 		fmt.Sprintf("pulling '%s:%s' from '%s'", ociOpts.Repo, tagSelected, full_repo),

pkg/client/client_test.go

@@ -27,6 +27,7 @@ import (
 	"kcl-lang.io/kcl-go/pkg/kcl"
 	"kcl-lang.io/kpm/pkg/downloader"
 	"kcl-lang.io/kpm/pkg/env"
+	"kcl-lang.io/kpm/pkg/features"
 	"kcl-lang.io/kpm/pkg/opt"
 	pkg "kcl-lang.io/kpm/pkg/package"
 	"kcl-lang.io/kpm/pkg/reporter"
@@ -2263,3 +2264,46 @@ func testPushWithInsecureSkipTLSverify(t *testing.T) {
 
 	assert.Equal(t, buf.String(), "Called Success\n")
 }
+
+func TestValidateDependency(t *testing.T) {
+	features.Enable(features.SupportCheckSum)
+	defer features.Disable(features.SupportCheckSum)
+
+	kpmcli, err := NewKpmClient()
+	assert.Equal(t, err, nil)
+
+	dep1 := pkg.Dependency{
+		Name:          "helloworld",
+		FullName:      "helloworld_0.1.2",
+		Version:       "0.1.2",
+		Sum:           "PN0OMEV9M8VGFn1CtA/T3bcgZmMJmOo+RkBrLKIWYeQ=",
+		LocalFullPath: "path/to/kcl/package",
+		Source: downloader.Source{
+			Oci: &downloader.Oci{
+				Reg:  "ghcr.io",
+				Repo: "kcl-lang/helloworld",
+				Tag:  "0.1.2",
+			},
+		},
+	}
+	err = kpmcli.ValidateDependency(&dep1)
+	assert.Equal(t, err, nil)
+
+	dep2 := pkg.Dependency{
+		Name:          "helloworld",
+		FullName:      "helloworld_0.1.2",
+		Version:       "0.1.2",
+		Sum:           "fail-to-validate-dependency",
+		LocalFullPath: "path/to/kcl/package",
+		Source: downloader.Source{
+			Oci: &downloader.Oci{
+				Reg:  "ghcr.io",
+				Repo: "kcl-lang/helloworld",
+				Tag:  "0.1.2",
+			},
+		},
+	}
+
+	err = kpmcli.ValidateDependency(&dep2)
+	assert.Error(t, err)
+}

pkg/features/features.go

@@ -13,12 +13,15 @@ const (
 	SupportMVS = "SupportMVS"
 	// SupportNewStorage is the feature gate for enabling the support for the new storage structure.
 	SupportNewStorage = "SupportNewStorage"
+	// SupportCheckSum is the feature gate for enabling the support for the checksum verification.
+	SupportCheckSum = "SupportCheckSum"
 )
 
 var (
 	features = map[string]bool{
 		SupportMVS:        false,
 		SupportNewStorage: false,
+		SupportCheckSum:   false,
 	}
 	mu sync.Mutex
 )

test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url/test_suite.stdout

@@ -1,4 +1,4 @@
 start to pull 'oci://localhost:5001/test/k8s'
 the latest version '1.31.2' will be pulled
 pulling '/test/k8s:1.31.2' from 'localhost:5001/test/k8s'
-pulled 'oci://localhost:5001/test/k8s' in '<workspace>/localhost:5001/test/k8s' successfully
+pulled 'oci://localhost:5001/test/k8s' in '<workspace>/localhost:5001/test/k8s/1.31.2' successfully

test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_pkg_name/test_suite.stdout

@@ -1,4 +1,4 @@
 start to pull 'k8s'
 the latest version '1.31.2' will be pulled
 pulling 'test/k8s:1.31.2' from 'localhost:5001/test/k8s'
-pulled 'k8s' in '<workspace>/localhost:5001/test/k8s' successfully
+pulled 'k8s' in '<workspace>/localhost:5001/test/k8s/1.31.2' successfully

test/e2e/test_suites/kpm/exec_outside_pkg/pull_private_pkg/test_suite.stdout

@@ -1,4 +1,4 @@
 start to pull 'oci://localhost:5001/test/helloworld'
 the latest version '0.1.2' will be pulled
 pulling '/test/helloworld:0.1.2' from 'localhost:5001/test/helloworld'
-pulled 'oci://localhost:5001/test/helloworld' in '<workspace>/localhost:5001/test/helloworld' successfully
+pulled 'oci://localhost:5001/test/helloworld' in '<workspace>/localhost:5001/test/helloworld/0.1.2' successfully