From 50ac936ce3d94f750acf264744d2efc108ca74da Mon Sep 17 00:00:00 2001 From: zongz Date: Tue, 27 Aug 2024 16:38:41 +0800 Subject: [PATCH] feat: add resolver for dependency resolving Signed-off-by: zongz --- pkg/client/client.go | 2 + pkg/client/resolver.go | 179 ++++++++++++++++++ pkg/client/resolver_test.go | 58 ++++++ .../test_data/test_resolve_graph/dep1/kcl.mod | 7 + .../test_resolve_graph/dep1/kcl.mod.lock | 6 + .../test_data/test_resolve_graph/dep1/main.k | 1 + .../test_data/test_resolve_graph/pkg/kcl.mod | 8 + .../test_resolve_graph/pkg/kcl.mod.lock | 9 + .../test_data/test_resolve_graph/pkg/main.k | 1 + pkg/client/visitor.go | 4 + pkg/downloader/downloader.go | 64 +++++++ 11 files changed, 339 insertions(+) create mode 100644 pkg/client/resolver.go create mode 100644 pkg/client/resolver_test.go create mode 100644 pkg/client/test_data/test_resolve_graph/dep1/kcl.mod create mode 100644 pkg/client/test_data/test_resolve_graph/dep1/kcl.mod.lock create mode 100644 pkg/client/test_data/test_resolve_graph/dep1/main.k create mode 100644 pkg/client/test_data/test_resolve_graph/pkg/kcl.mod create mode 100644 pkg/client/test_data/test_resolve_graph/pkg/kcl.mod.lock create mode 100644 pkg/client/test_data/test_resolve_graph/pkg/main.k diff --git a/pkg/client/client.go b/pkg/client/client.go index 73915360..12328ad4 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -44,6 +44,8 @@ type KpmClient struct { logWriter io.Writer // The downloader of the dependencies. DepDownloader *downloader.DepDownloader + // The dependency resolver. + DepsResolver *DepsResolver // credential store credsClient *downloader.CredClient // The home path of kpm for global configuration file and kcl package storage path. diff --git a/pkg/client/resolver.go b/pkg/client/resolver.go new file mode 100644 index 00000000..667d8fce --- /dev/null +++ b/pkg/client/resolver.go @@ -0,0 +1,179 @@ +package client + +import ( + "path/filepath" + + "kcl-lang.io/kpm/pkg/downloader" + pkg "kcl-lang.io/kpm/pkg/package" +) + +// ResolveOption is the option for resolving dependencies. +type ResolveOption func(*ResolveOptions) error + +// resolveFunc is the function for resolving each dependency when traversing the dependency graph. +// currentPkg is the current package to be resolved and parentPkg is the parent package of the current package. +type resolveFunc func(currentPkg, parentPkg *pkg.KclPkg) error + +type ResolveOptions struct { + // Source is the source of the package to be pulled. + // Including git, oci, local. + Source *downloader.Source + // EnableCache is the flag to enable the cache during the resolving the remote package. + EnableCache bool + // CachePath is the path of the cache. + CachePath string +} + +// WithEnableCache sets the flag to enable the cache during the resolving the remote package. +func WithEnableCache(enableCache bool) ResolveOption { + return func(opts *ResolveOptions) error { + opts.EnableCache = enableCache + return nil + } +} + +// WithCachePath sets the path of the cache. +func WithCachePath(cachePath string) ResolveOption { + return func(opts *ResolveOptions) error { + opts.CachePath = cachePath + return nil + } +} + +// WithPkgSource sets the source of the package to be resolved. +func WithPkgSource(source *downloader.Source) ResolveOption { + return func(opts *ResolveOptions) error { + opts.Source = source + return nil + } +} + +// WithPkgSourceUrl sets the source of the package to be resolved by the source url. +func WithPkgSourceUrl(sourceUrl string) ResolveOption { + return func(opts *ResolveOptions) error { + source, err := downloader.NewSourceFromStr(sourceUrl) + if err != nil { + return err + } + opts.Source = source + return nil + } +} + +// DepsResolver is the resolver for resolving dependencies. +type DepsResolver struct { + kpmClient *KpmClient + resolveFuncs []resolveFunc +} + +// NewDepsResolver creates a new DepsResolver. +func NewDepsResolver(kpmClient *KpmClient) *DepsResolver { + return &DepsResolver{ + kpmClient: kpmClient, + resolveFuncs: []resolveFunc{}, + } +} + +// AddResolveFunc adds a resolve function to the DepsResolver. +func (dr *DepsResolver) AddResolveFunc(rf resolveFunc) { + if dr.resolveFuncs == nil { + dr.resolveFuncs = []resolveFunc{} + } + + dr.resolveFuncs = append(dr.resolveFuncs, rf) +} + +// Resolve resolves the dependencies of the package. +func (dr *DepsResolver) Resolve(options ...ResolveOption) error { + opts := &ResolveOptions{} + for _, option := range options { + if err := option(opts); err != nil { + return err + } + } + + // visitorSelectorFunc selects the visitor for the source. + // For remote source, it will use the RemoteVisitor and enable the cache. + // For local source, it will use the PkgVisitor. + visitorSelectorFunc := func(source *downloader.Source) (Visitor, error) { + if source.IsRemote() { + PkgVisitor := NewRemoteVisitor(NewPkgVisitor(dr.kpmClient)) + PkgVisitor.EnableCache = opts.EnableCache + if opts.CachePath == "" { + PkgVisitor.CachePath = dr.kpmClient.homePath + } else { + PkgVisitor.CachePath = opts.CachePath + } + return PkgVisitor, nil + } else { + return NewVisitor(*opts.Source, dr.kpmClient), nil + } + } + + // visitFunc is the function for visiting the package. + // It will traverse the dependency graph and visit each dependency by source. + visitFunc := func(kclPkg *pkg.KclPkg) error { + // Traverse the all dependencies of the package. + for _, depKey := range kclPkg.ModFile.Deps.Keys() { + dep, ok := kclPkg.ModFile.Deps.Get(depKey) + if !ok { + break + } + + // Get the dependency source. + var depSource downloader.Source + // If the dependency source is a local path and the path is not absolute, transform the path to absolute path. + if dep.Source.IsLocalPath() && !filepath.IsAbs(dep.Source.Path) { + depSource = downloader.Source{ + Local: &downloader.Local{ + Path: filepath.Join(kclPkg.HomePath, dep.Source.Path), + }, + } + } else { + depSource = dep.Source + } + + // Get the visitor for the dependency source. + visitor, err := visitorSelectorFunc(&depSource) + if err != nil { + return err + } + + // Visit this dependency and current package as the parent package. + err = visitor.Visit(&depSource, + func(childPkg *pkg.KclPkg) error { + for _, resolveFunc := range dr.resolveFuncs { + err := resolveFunc(childPkg, kclPkg) + if err != nil { + return err + } + } + return nil + }, + ) + + if err != nil { + return err + } + + // Recursively resolve the dependencies of the dependency. + err = dr.Resolve( + WithPkgSource(&depSource), + WithEnableCache(opts.EnableCache), + WithCachePath(opts.CachePath), + ) + if err != nil { + return err + } + } + + return nil + } + + visitor, err := visitorSelectorFunc(opts.Source) + if err != nil { + return err + } + + return visitor.Visit(opts.Source, visitFunc) +} diff --git a/pkg/client/resolver_test.go b/pkg/client/resolver_test.go new file mode 100644 index 00000000..37559b3e --- /dev/null +++ b/pkg/client/resolver_test.go @@ -0,0 +1,58 @@ +package client + +import ( + "bytes" + "fmt" + "path/filepath" + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "kcl-lang.io/kpm/pkg/downloader" + pkg "kcl-lang.io/kpm/pkg/package" +) + +func TestResolver(t *testing.T) { + kpmcli, err := NewKpmClient() + if err != nil { + t.Fatal(err) + } + + resolve_path := getTestDir("test_resolve_graph") + pkgPath := filepath.Join(resolve_path, "pkg") + + pkgSource, err := downloader.NewSourceFromStr(pkgPath) + if err != nil { + t.Fatal(err) + } + + var res []string + var buf bytes.Buffer + + kpmcli.SetLogWriter(&buf) + resolver := NewDepsResolver(kpmcli) + resolver.AddResolveFunc(func(currentPkg, parentPkg *pkg.KclPkg) error { + res = append(res, fmt.Sprintf("%s -> %s", parentPkg.GetPkgName(), currentPkg.GetPkgName())) + return nil + }) + + err = resolver.Resolve( + WithEnableCache(true), + WithPkgSource(pkgSource), + ) + + if err != nil { + t.Fatal(err) + } + + expected := []string{ + "dep1 -> helloworld", + "pkg -> dep1", + "pkg -> helloworld", + } + + sort.Strings(res) + assert.Equal(t, len(res), 3) + assert.Equal(t, res, expected) + assert.Equal(t, buf.String(), "") +} diff --git a/pkg/client/test_data/test_resolve_graph/dep1/kcl.mod b/pkg/client/test_data/test_resolve_graph/dep1/kcl.mod new file mode 100644 index 00000000..81a79664 --- /dev/null +++ b/pkg/client/test_data/test_resolve_graph/dep1/kcl.mod @@ -0,0 +1,7 @@ +[package] +name = "dep1" +edition = "v0.10.0" +version = "0.0.1" + +[dependencies] +helloworld = "0.1.1" diff --git a/pkg/client/test_data/test_resolve_graph/dep1/kcl.mod.lock b/pkg/client/test_data/test_resolve_graph/dep1/kcl.mod.lock new file mode 100644 index 00000000..667cdff0 --- /dev/null +++ b/pkg/client/test_data/test_resolve_graph/dep1/kcl.mod.lock @@ -0,0 +1,6 @@ +[dependencies] + [dependencies.helloworld] + name = "helloworld" + full_name = "helloworld_0.1.1" + version = "0.1.1" + sum = "7OO4YK2QuRWPq9C7KTzcWcti5yUnueCjptT3OXiPVeQ=" diff --git a/pkg/client/test_data/test_resolve_graph/dep1/main.k b/pkg/client/test_data/test_resolve_graph/dep1/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/test_resolve_graph/dep1/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/client/test_data/test_resolve_graph/pkg/kcl.mod b/pkg/client/test_data/test_resolve_graph/pkg/kcl.mod new file mode 100644 index 00000000..a4a26b14 --- /dev/null +++ b/pkg/client/test_data/test_resolve_graph/pkg/kcl.mod @@ -0,0 +1,8 @@ +[package] +name = "pkg" +edition = "v0.10.0" +version = "0.0.1" + +[dependencies] +dep1 = { path = "../dep1" } +helloworld = "0.1.2" diff --git a/pkg/client/test_data/test_resolve_graph/pkg/kcl.mod.lock b/pkg/client/test_data/test_resolve_graph/pkg/kcl.mod.lock new file mode 100644 index 00000000..12335071 --- /dev/null +++ b/pkg/client/test_data/test_resolve_graph/pkg/kcl.mod.lock @@ -0,0 +1,9 @@ +[dependencies] + [dependencies.dep1] + name = "dep1" + full_name = "dep1_0.0.1" + version = "0.0.1" + [dependencies.helloworld] + name = "helloworld" + full_name = "helloworld_0.1.2" + version = "0.1.2" diff --git a/pkg/client/test_data/test_resolve_graph/pkg/main.k b/pkg/client/test_data/test_resolve_graph/pkg/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/test_resolve_graph/pkg/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/client/visitor.go b/pkg/client/visitor.go index 91772cc7..1a0f498f 100644 --- a/pkg/client/visitor.go +++ b/pkg/client/visitor.go @@ -90,6 +90,8 @@ func (vpv *VirtualPkgVisitor) Visit(s *downloader.Source, v visitFunc) error { // RemoteVisitor is the visitor for visiting a remote package. type RemoteVisitor struct { *PkgVisitor + EnableCache bool + CachePath string } // NewRemoteVisitor creates a new RemoteVisitor. @@ -128,6 +130,8 @@ func (rv *RemoteVisitor) Visit(s *downloader.Source, v visitFunc) error { downloader.WithLogWriter(rv.kpmcli.GetLogWriter()), downloader.WithSettings(*rv.kpmcli.GetSettings()), downloader.WithCredsClient(credCli), + downloader.WithCachePath(rv.CachePath), + downloader.WithEnableCache(rv.EnableCache), )) if err != nil { diff --git a/pkg/downloader/downloader.go b/pkg/downloader/downloader.go index 2369e3d0..346f6737 100644 --- a/pkg/downloader/downloader.go +++ b/pkg/downloader/downloader.go @@ -8,6 +8,8 @@ import ( "path/filepath" v1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/otiai10/copy" + "kcl-lang.io/kpm/pkg/constants" "kcl-lang.io/kpm/pkg/git" "kcl-lang.io/kpm/pkg/oci" "kcl-lang.io/kpm/pkg/reporter" @@ -20,6 +22,11 @@ import ( type DownloadOptions struct { // LocalPath is the local path to download the package. LocalPath string + // CachePath is the cache path to download the package. + CachePath string + // EnableCache is the flag to enable the cache. + // If `EnableCache` is false, this will not result in increasing disk usage. + EnableCache bool // Source is the source of the package. including git, oci, local. Source Source // Settings is the default settings and authrization information. @@ -32,6 +39,18 @@ type DownloadOptions struct { type Option func(*DownloadOptions) +func WithCachePath(cachePath string) Option { + return func(do *DownloadOptions) { + do.CachePath = cachePath + } +} + +func WithEnableCache(enableCache bool) Option { + return func(do *DownloadOptions) { + do.EnableCache = enableCache + } +} + func WithCredsClient(credsClient *CredClient) Option { return func(do *DownloadOptions) { do.credsClient = credsClient @@ -99,6 +118,51 @@ func NewOciDownloader(platform string) *DepDownloader { } func (d *DepDownloader) Download(opts DownloadOptions) error { + + var localPath string + if opts.EnableCache { + // TODO: After the new local storage structure is complete, + // this section should be replaced with the new storage structure instead of the cache path according to the /. + var pkgFullName string + if opts.Source.Registry != nil && len(opts.Source.Registry.Version) != 0 { + pkgFullName = fmt.Sprintf("%s_%s", filepath.Base(opts.Source.Registry.Oci.Repo), opts.Source.Registry.Version) + } + if opts.Source.Oci != nil && len(opts.Source.Oci.Tag) != 0 { + pkgFullName = fmt.Sprintf("%s_%s", filepath.Base(opts.Source.Oci.Repo), opts.Source.Oci.Tag) + } + if opts.Source.Git != nil && len(opts.Source.Git.Tag) != 0 { + pkgFullName = fmt.Sprintf("%s_%s", filepath.Base(opts.Source.Git.Url), opts.Source.Git.Tag) + } + if opts.Source.Git != nil && len(opts.Source.Git.Branch) != 0 { + pkgFullName = fmt.Sprintf("%s_%s", filepath.Base(opts.Source.Git.Url), opts.Source.Git.Branch) + } + if opts.Source.Git != nil && len(opts.Source.Git.Commit) != 0 { + pkgFullName = fmt.Sprintf("%s_%s", filepath.Base(opts.Source.Git.Url), opts.Source.Git.Commit) + } + + cacheFullPath := filepath.Join(opts.CachePath, pkgFullName) + + if utils.DirExists(cacheFullPath) && utils.DirExists(filepath.Join(cacheFullPath, constants.KCL_MOD)) { + // copy the cache to the local path + if cacheFullPath != opts.LocalPath { + err := copy.Copy(cacheFullPath, opts.LocalPath) + if err != nil { + return err + } + } + return nil + } else { + err := os.MkdirAll(cacheFullPath, 0755) + if err != nil { + return err + } + localPath = cacheFullPath + } + } else { + localPath = opts.LocalPath + } + + opts.LocalPath = localPath // Dispatch the download to the specific downloader by package source. if opts.Source.Oci != nil || opts.Source.Registry != nil { if opts.Source.Registry != nil {