From 255d871a9fd0eb07e48b8d5f8f33053a27ea6846 Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Fri, 29 Mar 2024 00:48:20 +0530 Subject: [PATCH 01/17] feat: add already existing mvs library in go and use module package as graph key value type Signed-off-by: Akash Kumar --- pkg/3rdparty/gover/gover.go | 75 ++++ pkg/3rdparty/gover/internal/gover.go | 223 ++++++++++++ pkg/3rdparty/gover/mod.go | 127 +++++++ pkg/3rdparty/gover/toolchain.go | 90 +++++ pkg/3rdparty/mvs/errors.go | 105 ++++++ pkg/3rdparty/mvs/graph.go | 226 +++++++++++++ pkg/3rdparty/mvs/mvs.go | 488 +++++++++++++++++++++++++++ pkg/3rdparty/par/queue.go | 88 +++++ pkg/3rdparty/par/work.go | 223 ++++++++++++ pkg/client/client.go | 15 +- pkg/cmd/cmd_graph.go | 15 +- pkg/graph/graph.go | 24 ++ pkg/mvs/mvs.go | 38 +++ pkg/package/package.go | 4 + 14 files changed, 1732 insertions(+), 9 deletions(-) create mode 100644 pkg/3rdparty/gover/gover.go create mode 100644 pkg/3rdparty/gover/internal/gover.go create mode 100644 pkg/3rdparty/gover/mod.go create mode 100644 pkg/3rdparty/gover/toolchain.go create mode 100644 pkg/3rdparty/mvs/errors.go create mode 100644 pkg/3rdparty/mvs/graph.go create mode 100644 pkg/3rdparty/mvs/mvs.go create mode 100644 pkg/3rdparty/par/queue.go create mode 100644 pkg/3rdparty/par/work.go create mode 100644 pkg/graph/graph.go create mode 100644 pkg/mvs/mvs.go diff --git a/pkg/3rdparty/gover/gover.go b/pkg/3rdparty/gover/gover.go new file mode 100644 index 00000000..06c29459 --- /dev/null +++ b/pkg/3rdparty/gover/gover.go @@ -0,0 +1,75 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package gover implements support for Go toolchain versions like 1.21.0 and 1.21rc1. +// (For historical reasons, Go does not use semver for its toolchains.) +// This package provides the same basic analysis that golang.org/x/mod/semver does for semver. +// It also provides some helpers for extracting versions from go.mod files +// and for dealing with module.Versions that may use Go versions or semver +// depending on the module path. +package gover + +import ( + "kcl-lang.io/kpm/pkg/3rdparty/gover/internal" +) + +// Compare returns -1, 0, or +1 depending on whether +// x < y, x == y, or x > y, interpreted as toolchain versions. +// The versions x and y must not begin with a "go" prefix: just "1.21" not "go1.21". +// Malformed versions compare less than well-formed versions and equal to each other. +// The language version "1.21" compares less than the release candidate and eventual releases "1.21rc1" and "1.21.0". +func Compare(x, y string) int { + return gover.Compare(x, y) +} + +// Max returns the maximum of x and y interpreted as toolchain versions, +// compared using Compare. +// If x and y compare equal, Max returns x. +func Max(x, y string) string { + return gover.Max(x, y) +} + +// IsLang reports whether v denotes the overall Go language version +// and not a specific release. Starting with the Go 1.21 release, "1.x" denotes +// the overall language version; the first release is "1.x.0". +// The distinction is important because the relative ordering is +// +// 1.21 < 1.21rc1 < 1.21.0 +// +// meaning that Go 1.21rc1 and Go 1.21.0 will both handle go.mod files that +// say "go 1.21", but Go 1.21rc1 will not handle files that say "go 1.21.0". +func IsLang(x string) bool { + return gover.IsLang(x) +} + +// Lang returns the Go language version. For example, Lang("1.2.3") == "1.2". +func Lang(x string) string { + return gover.Lang(x) +} + +// IsPrerelease reports whether v denotes a Go prerelease version. +func IsPrerelease(x string) bool { + return gover.Parse(x).Kind != "" +} + +// Prev returns the Go major release immediately preceding v, +// or v itself if v is the first Go major release (1.0) or not a supported +// Go version. +// +// Examples: +// +// Prev("1.2") = "1.1" +// Prev("1.3rc4") = "1.2" +func Prev(x string) string { + v := gover.Parse(x) + if gover.CmpInt(v.Minor, "1") <= 0 { + return v.Major + } + return v.Major + "." + gover.DecInt(v.Minor) +} + +// IsValid reports whether the version x is valid. +func IsValid(x string) bool { + return gover.IsValid(x) +} diff --git a/pkg/3rdparty/gover/internal/gover.go b/pkg/3rdparty/gover/internal/gover.go new file mode 100644 index 00000000..2ad06846 --- /dev/null +++ b/pkg/3rdparty/gover/internal/gover.go @@ -0,0 +1,223 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package gover implements support for Go toolchain versions like 1.21.0 and 1.21rc1. +// (For historical reasons, Go does not use semver for its toolchains.) +// This package provides the same basic analysis that golang.org/x/mod/semver does for semver. +// +// The go/version package should be imported instead of this one when possible. +// Note that this package works on "1.21" while go/version works on "go1.21". +package gover + +import ( + "cmp" +) + +// A Version is a parsed Go version: major[.Minor[.Patch]][kind[pre]] +// The numbers are the original decimal strings to avoid integer overflows +// and since there is very little actual math. (Probably overflow doesn't matter in practice, +// but at the time this code was written, there was an existing test that used +// go1.99999999999, which does not fit in an int on 32-bit platforms. +// The "big decimal" representation avoids the problem entirely.) +type Version struct { + Major string // decimal + Minor string // decimal or "" + Patch string // decimal or "" + Kind string // "", "alpha", "beta", "rc" + Pre string // decimal or "" +} + +// Compare returns -1, 0, or +1 depending on whether +// x < y, x == y, or x > y, interpreted as toolchain versions. +// The versions x and y must not begin with a "go" prefix: just "1.21" not "go1.21". +// Malformed versions compare less than well-formed versions and equal to each other. +// The language version "1.21" compares less than the release candidate and eventual releases "1.21rc1" and "1.21.0". +func Compare(x, y string) int { + vx := Parse(x) + vy := Parse(y) + + if c := CmpInt(vx.Major, vy.Major); c != 0 { + return c + } + if c := CmpInt(vx.Minor, vy.Minor); c != 0 { + return c + } + if c := CmpInt(vx.Patch, vy.Patch); c != 0 { + return c + } + if c := cmp.Compare(vx.Kind, vy.Kind); c != 0 { // "" < alpha < beta < rc + return c + } + if c := CmpInt(vx.Pre, vy.Pre); c != 0 { + return c + } + return 0 +} + +// Max returns the maximum of x and y interpreted as toolchain versions, +// compared using Compare. +// If x and y compare equal, Max returns x. +func Max(x, y string) string { + if Compare(x, y) < 0 { + return y + } + return x +} + +// IsLang reports whether v denotes the overall Go language version +// and not a specific release. Starting with the Go 1.21 release, "1.x" denotes +// the overall language version; the first release is "1.x.0". +// The distinction is important because the relative ordering is +// +// 1.21 < 1.21rc1 < 1.21.0 +// +// meaning that Go 1.21rc1 and Go 1.21.0 will both handle go.mod files that +// say "go 1.21", but Go 1.21rc1 will not handle files that say "go 1.21.0". +func IsLang(x string) bool { + v := Parse(x) + return v != Version{} && v.Patch == "" && v.Kind == "" && v.Pre == "" +} + +// Lang returns the Go language version. For example, Lang("1.2.3") == "1.2". +func Lang(x string) string { + v := Parse(x) + if v.Minor == "" || v.Major == "1" && v.Minor == "0" { + return v.Major + } + return v.Major + "." + v.Minor +} + +// IsValid reports whether the version x is valid. +func IsValid(x string) bool { + return Parse(x) != Version{} +} + +// Parse parses the Go version string x into a version. +// It returns the zero version if x is malformed. +func Parse(x string) Version { + var v Version + + // Parse major version. + var ok bool + v.Major, x, ok = cutInt(x) + if !ok { + return Version{} + } + if x == "" { + // Interpret "1" as "1.0.0". + v.Minor = "0" + v.Patch = "0" + return v + } + + // Parse . before minor version. + if x[0] != '.' { + return Version{} + } + + // Parse minor version. + v.Minor, x, ok = cutInt(x[1:]) + if !ok { + return Version{} + } + if x == "" { + // Patch missing is same as "0" for older versions. + // Starting in Go 1.21, patch missing is different from explicit .0. + if CmpInt(v.Minor, "21") < 0 { + v.Patch = "0" + } + return v + } + + // Parse patch if present. + if x[0] == '.' { + v.Patch, x, ok = cutInt(x[1:]) + if !ok || x != "" { + // Note that we are disallowing prereleases (alpha, beta, rc) for patch releases here (x != ""). + // Allowing them would be a bit confusing because we already have: + // 1.21 < 1.21rc1 + // But a prerelease of a patch would have the opposite effect: + // 1.21.3rc1 < 1.21.3 + // We've never needed them before, so let's not start now. + return Version{} + } + return v + } + + // Parse prerelease. + i := 0 + for i < len(x) && (x[i] < '0' || '9' < x[i]) { + if x[i] < 'a' || 'z' < x[i] { + return Version{} + } + i++ + } + if i == 0 { + return Version{} + } + v.Kind, x = x[:i], x[i:] + if x == "" { + return v + } + v.Pre, x, ok = cutInt(x) + if !ok || x != "" { + return Version{} + } + + return v +} + +// cutInt scans the leading decimal number at the start of x to an integer +// and returns that value and the rest of the string. +func cutInt(x string) (n, rest string, ok bool) { + i := 0 + for i < len(x) && '0' <= x[i] && x[i] <= '9' { + i++ + } + if i == 0 || x[0] == '0' && i != 1 { // no digits or unnecessary leading zero + return "", "", false + } + return x[:i], x[i:], true +} + +// CmpInt returns cmp.Compare(x, y) interpreting x and y as decimal numbers. +// (Copied from golang.org/x/mod/semver's compareInt.) +func CmpInt(x, y string) int { + if x == y { + return 0 + } + if len(x) < len(y) { + return -1 + } + if len(x) > len(y) { + return +1 + } + if x < y { + return -1 + } else { + return +1 + } +} + +// DecInt returns the decimal string decremented by 1, or the empty string +// if the decimal is all zeroes. +// (Copied from golang.org/x/mod/module's decDecimal.) +func DecInt(decimal string) string { + // Scan right to left turning 0s to 9s until you find a digit to decrement. + digits := []byte(decimal) + i := len(digits) - 1 + for ; i >= 0 && digits[i] == '0'; i-- { + digits[i] = '9' + } + if i < 0 { + // decimal is all zeros + return "" + } + if i == 0 && digits[i] == '1' && len(digits) > 1 { + digits = digits[1:] + } else { + digits[i]-- + } + return string(digits) +} diff --git a/pkg/3rdparty/gover/mod.go b/pkg/3rdparty/gover/mod.go new file mode 100644 index 00000000..d3cc1706 --- /dev/null +++ b/pkg/3rdparty/gover/mod.go @@ -0,0 +1,127 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gover + +import ( + "sort" + "strings" + + "golang.org/x/mod/module" + "golang.org/x/mod/semver" +) + +// IsToolchain reports whether the module path corresponds to the +// virtual, non-downloadable module tracking go or toolchain directives in the go.mod file. +// +// Note that IsToolchain only matches "go" and "toolchain", not the +// real, downloadable module "golang.org/toolchain" containing toolchain files. +// +// IsToolchain("go") = true +// IsToolchain("toolchain") = true +// IsToolchain("golang.org/x/tools") = false +// IsToolchain("golang.org/toolchain") = false +func IsToolchain(path string) bool { + return path == "go" || path == "toolchain" +} + +// ModCompare returns the result of comparing the versions x and y +// for the module with the given path. +// The path is necessary because the "go" and "toolchain" modules +// use a different version syntax and semantics (gover, this package) +// than most modules (semver). +func ModCompare(path string, x, y string) int { + if path == "go" { + return Compare(x, y) + } + if path == "toolchain" { + return Compare(maybeToolchainVersion(x), maybeToolchainVersion(y)) + } + return semver.Compare(x, y) +} + +// ModSort is like module.Sort but understands the "go" and "toolchain" +// modules and their version ordering. +func ModSort(list []module.Version) { + sort.Slice(list, func(i, j int) bool { + mi := list[i] + mj := list[j] + if mi.Path != mj.Path { + return mi.Path < mj.Path + } + // To help go.sum formatting, allow version/file. + // Compare semver prefix by semver rules, + // file by string order. + vi := mi.Version + vj := mj.Version + var fi, fj string + if k := strings.Index(vi, "/"); k >= 0 { + vi, fi = vi[:k], vi[k:] + } + if k := strings.Index(vj, "/"); k >= 0 { + vj, fj = vj[:k], vj[k:] + } + if vi != vj { + return ModCompare(mi.Path, vi, vj) < 0 + } + return fi < fj + }) +} + +// ModIsValid reports whether vers is a valid version syntax for the module with the given path. +func ModIsValid(path, vers string) bool { + if IsToolchain(path) { + if path == "toolchain" { + return IsValid(FromToolchain(vers)) + } + return IsValid(vers) + } + return semver.IsValid(vers) +} + +// ModIsPrefix reports whether v is a valid version syntax prefix for the module with the given path. +// The caller is assumed to have checked that ModIsValid(path, vers) is true. +func ModIsPrefix(path, vers string) bool { + if IsToolchain(path) { + if path == "toolchain" { + return IsLang(FromToolchain(vers)) + } + return IsLang(vers) + } + // Semver + dots := 0 + for i := 0; i < len(vers); i++ { + switch vers[i] { + case '-', '+': + return false + case '.': + dots++ + if dots >= 2 { + return false + } + } + } + return true +} + +// ModIsPrerelease reports whether v is a prerelease version for the module with the given path. +// The caller is assumed to have checked that ModIsValid(path, vers) is true. +func ModIsPrerelease(path, vers string) bool { + if IsToolchain(path) { + return IsPrerelease(vers) + } + return semver.Prerelease(vers) != "" +} + +// ModMajorMinor returns the "major.minor" truncation of the version v, +// for use as a prefix in "@patch" queries. +func ModMajorMinor(path, vers string) string { + if IsToolchain(path) { + if path == "toolchain" { + return "go" + Lang(FromToolchain(vers)) + } + return Lang(vers) + } + return semver.MajorMinor(vers) +} diff --git a/pkg/3rdparty/gover/toolchain.go b/pkg/3rdparty/gover/toolchain.go new file mode 100644 index 00000000..fda95adc --- /dev/null +++ b/pkg/3rdparty/gover/toolchain.go @@ -0,0 +1,90 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gover + +import ( + "context" + "errors" + "strings" +) + +// FromToolchain returns the Go version for the named toolchain, +// derived from the name itself (not by running the toolchain). +// A toolchain is named "goVERSION". +// A suffix after the VERSION introduced by a -, space, or tab is removed. +// Examples: +// +// FromToolchain("go1.2.3") == "1.2.3" +// FromToolchain("go1.2.3-bigcorp") == "1.2.3" +// FromToolchain("invalid") == "" +func FromToolchain(name string) string { + if strings.ContainsAny(name, "\\/") { + // The suffix must not include a path separator, since that would cause + // exec.LookPath to resolve it from a relative directory instead of from + // $PATH. + return "" + } + + var v string + if strings.HasPrefix(name, "go") { + v = name[2:] + } else { + return "" + } + // Some builds use custom suffixes; strip them. + if i := strings.IndexAny(v, " \t-"); i >= 0 { + v = v[:i] + } + if !IsValid(v) { + return "" + } + return v +} + +func maybeToolchainVersion(name string) string { + if IsValid(name) { + return name + } + return FromToolchain(name) +} + +// ToolchainMax returns the maximum of x and y interpreted as toolchain names, +// compared using Compare(FromToolchain(x), FromToolchain(y)). +// If x and y compare equal, Max returns x. +func ToolchainMax(x, y string) string { + if Compare(FromToolchain(x), FromToolchain(y)) < 0 { + return y + } + return x +} + +// Startup records the information that went into the startup-time version switch. +// It is initialized by switchGoToolchain. +var Startup struct { + GOTOOLCHAIN string // $GOTOOLCHAIN setting + AutoFile string // go.mod or go.work file consulted + AutoGoVersion string // go line found in file + AutoToolchain string // toolchain line found in file +} + +// A TooNewError explains that a module is too new for this version of Go. +type TooNewError struct { + What string + GoVersion string + Toolchain string // for callers if they want to use it, but not printed +} + +var ErrTooNew = errors.New("module too new") + +func (e *TooNewError) Is(err error) bool { + return err == ErrTooNew +} + +// A Switcher provides the ability to switch to a new toolchain in response to TooNewErrors. +// See [cmd/go/internal/toolchain.Switcher] for documentation. +type Switcher interface { + Error(err error) + Switch(ctx context.Context) +} diff --git a/pkg/3rdparty/mvs/errors.go b/pkg/3rdparty/mvs/errors.go new file mode 100644 index 00000000..8db65d65 --- /dev/null +++ b/pkg/3rdparty/mvs/errors.go @@ -0,0 +1,105 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mvs + +import ( + "fmt" + "strings" + + "golang.org/x/mod/module" +) + +// BuildListError decorates an error that occurred gathering requirements +// while constructing a build list. BuildListError prints the chain +// of requirements to the module where the error occurred. +type BuildListError struct { + Err error + stack []buildListErrorElem +} + +type buildListErrorElem struct { + m module.Version + + // nextReason is the reason this module depends on the next module in the + // stack. Typically either "requires", or "updating to". + nextReason string +} + +// NewBuildListError returns a new BuildListError wrapping an error that +// occurred at a module found along the given path of requirements and/or +// upgrades, which must be non-empty. +// +// The isVersionChange function reports whether a path step is due to an +// explicit upgrade or downgrade (as opposed to an existing requirement in a +// go.mod file). A nil isVersionChange function indicates that none of the path +// steps are due to explicit version changes. +func NewBuildListError(err error, path []module.Version, isVersionChange func(from, to module.Version) bool) *BuildListError { + stack := make([]buildListErrorElem, 0, len(path)) + for len(path) > 1 { + reason := "requires" + if isVersionChange != nil && isVersionChange(path[0], path[1]) { + reason = "updating to" + } + stack = append(stack, buildListErrorElem{ + m: path[0], + nextReason: reason, + }) + path = path[1:] + } + stack = append(stack, buildListErrorElem{m: path[0]}) + + return &BuildListError{ + Err: err, + stack: stack, + } +} + +// Module returns the module where the error occurred. If the module stack +// is empty, this returns a zero value. +func (e *BuildListError) Module() module.Version { + if len(e.stack) == 0 { + return module.Version{} + } + return e.stack[len(e.stack)-1].m +} + +func (e *BuildListError) Error() string { + b := &strings.Builder{} + stack := e.stack + + // Don't print modules at the beginning of the chain without a + // version. These always seem to be the main module or a + // synthetic module ("target@"). + for len(stack) > 0 && stack[0].m.Version == "" { + stack = stack[1:] + } + + if len(stack) == 0 { + b.WriteString(e.Err.Error()) + } else { + for _, elem := range stack[:len(stack)-1] { + fmt.Fprintf(b, "%s %s\n\t", elem.m, elem.nextReason) + } + // Ensure that the final module path and version are included as part of the + // error message. + m := stack[len(stack)-1].m + if mErr, ok := e.Err.(*module.ModuleError); ok { + actual := module.Version{Path: mErr.Path, Version: mErr.Version} + if v, ok := mErr.Err.(*module.InvalidVersionError); ok { + actual.Version = v.Version + } + if actual == m { + fmt.Fprintf(b, "%v", e.Err) + } else { + fmt.Fprintf(b, "%s (replaced by %s): %v", m, actual, mErr.Err) + } + } else { + fmt.Fprintf(b, "%v", module.VersionError(m, e.Err)) + } + } + return b.String() +} + +func (e *BuildListError) Unwrap() error { return e.Err } diff --git a/pkg/3rdparty/mvs/graph.go b/pkg/3rdparty/mvs/graph.go new file mode 100644 index 00000000..85489e86 --- /dev/null +++ b/pkg/3rdparty/mvs/graph.go @@ -0,0 +1,226 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mvs + +import ( + "fmt" + "slices" + + "kcl-lang.io/kpm/pkg/3rdparty/gover" + + "golang.org/x/mod/module" +) + +// Graph implements an incremental version of the MVS algorithm, with the +// requirements pushed by the caller instead of pulled by the MVS traversal. +type Graph struct { + cmp func(p, v1, v2 string) int + roots []module.Version + + required map[module.Version][]module.Version + + isRoot map[module.Version]bool // contains true for roots and false for reachable non-roots + selected map[string]string // path → version +} + +// NewGraph returns an incremental MVS graph containing only a set of root +// dependencies and using the given max function for version strings. +// +// The caller must ensure that the root slice is not modified while the Graph +// may be in use. +func NewGraph(cmp func(p, v1, v2 string) int, roots []module.Version) *Graph { + g := &Graph{ + cmp: cmp, + roots: slices.Clip(roots), + required: make(map[module.Version][]module.Version), + isRoot: make(map[module.Version]bool), + selected: make(map[string]string), + } + + for _, m := range roots { + g.isRoot[m] = true + if g.cmp(m.Path, g.Selected(m.Path), m.Version) < 0 { + g.selected[m.Path] = m.Version + } + } + + return g +} + +// Require adds the information that module m requires all modules in reqs. +// The reqs slice must not be modified after it is passed to Require. +// +// m must be reachable by some existing chain of requirements from g's target, +// and Require must not have been called for it already. +// +// If any of the modules in reqs has the same path as g's target, +// the target must have higher precedence than the version in req. +func (g *Graph) Require(m module.Version, reqs []module.Version) { + // To help catch disconnected-graph bugs, enforce that all required versions + // are actually reachable from the roots (and therefore should affect the + // selected versions of the modules they name). + if _, reachable := g.isRoot[m]; !reachable { + panic(fmt.Sprintf("%v is not reachable from any root", m)) + } + + // Truncate reqs to its capacity to avoid aliasing bugs if it is later + // returned from RequiredBy and appended to. + reqs = slices.Clip(reqs) + + if _, dup := g.required[m]; dup { + panic(fmt.Sprintf("requirements of %v have already been set", m)) + } + g.required[m] = reqs + + for _, dep := range reqs { + // Mark dep reachable, regardless of whether it is selected. + if _, ok := g.isRoot[dep]; !ok { + g.isRoot[dep] = false + } + + if g.cmp(dep.Path, g.Selected(dep.Path), dep.Version) < 0 { + g.selected[dep.Path] = dep.Version + } + } +} + +// RequiredBy returns the slice of requirements passed to Require for m, if any, +// with its capacity reduced to its length. +// If Require has not been called for m, RequiredBy(m) returns ok=false. +// +// The caller must not modify the returned slice, but may safely append to it +// and may rely on it not to be modified. +func (g *Graph) RequiredBy(m module.Version) (reqs []module.Version, ok bool) { + reqs, ok = g.required[m] + return reqs, ok +} + +// Selected returns the selected version of the given module path. +// +// If no version is selected, Selected returns version "none". +func (g *Graph) Selected(path string) (version string) { + v, ok := g.selected[path] + if !ok { + return "none" + } + return v +} + +// BuildList returns the selected versions of all modules present in the Graph, +// beginning with the selected versions of each module path in the roots of g. +// +// The order of the remaining elements in the list is deterministic +// but arbitrary. +func (g *Graph) BuildList() []module.Version { + seenRoot := make(map[string]bool, len(g.roots)) + + var list []module.Version + for _, r := range g.roots { + if seenRoot[r.Path] { + // Multiple copies of the same root, with the same or different versions, + // are a bit of a degenerate case: we will take the transitive + // requirements of both roots into account, but only the higher one can + // possibly be selected. However — especially given that we need the + // seenRoot map for later anyway — it is simpler to support this + // degenerate case than to forbid it. + continue + } + + if v := g.Selected(r.Path); v != "none" { + list = append(list, module.Version{Path: r.Path, Version: v}) + } + seenRoot[r.Path] = true + } + uniqueRoots := list + + for path, version := range g.selected { + if !seenRoot[path] { + list = append(list, module.Version{Path: path, Version: version}) + } + } + gover.ModSort(list[len(uniqueRoots):]) + + return list +} + +// WalkBreadthFirst invokes f once, in breadth-first order, for each module +// version other than "none" that appears in the graph, regardless of whether +// that version is selected. +func (g *Graph) WalkBreadthFirst(f func(m module.Version)) { + var queue []module.Version + enqueued := make(map[module.Version]bool) + for _, m := range g.roots { + if m.Version != "none" { + queue = append(queue, m) + enqueued[m] = true + } + } + + for len(queue) > 0 { + m := queue[0] + queue = queue[1:] + + f(m) + + reqs, _ := g.RequiredBy(m) + for _, r := range reqs { + if !enqueued[r] && r.Version != "none" { + queue = append(queue, r) + enqueued[r] = true + } + } + } +} + +// FindPath reports a shortest requirement path starting at one of the roots of +// the graph and ending at a module version m for which f(m) returns true, or +// nil if no such path exists. +func (g *Graph) FindPath(f func(module.Version) bool) []module.Version { + // firstRequires[a] = b means that in a breadth-first traversal of the + // requirement graph, the module version a was first required by b. + firstRequires := make(map[module.Version]module.Version) + + queue := g.roots + for _, m := range g.roots { + firstRequires[m] = module.Version{} + } + + for len(queue) > 0 { + m := queue[0] + queue = queue[1:] + + if f(m) { + // Construct the path reversed (because we're starting from the far + // endpoint), then reverse it. + path := []module.Version{m} + for { + m = firstRequires[m] + if m.Path == "" { + break + } + path = append(path, m) + } + + i, j := 0, len(path)-1 + for i < j { + path[i], path[j] = path[j], path[i] + i++ + j-- + } + + return path + } + + reqs, _ := g.RequiredBy(m) + for _, r := range reqs { + if _, seen := firstRequires[r]; !seen { + queue = append(queue, r) + firstRequires[r] = m + } + } + } + + return nil +} diff --git a/pkg/3rdparty/mvs/mvs.go b/pkg/3rdparty/mvs/mvs.go new file mode 100644 index 00000000..08df38a1 --- /dev/null +++ b/pkg/3rdparty/mvs/mvs.go @@ -0,0 +1,488 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package mvs implements Minimal Version Selection. +// See https://research.swtch.com/vgo-mvs. +package mvs + +import ( + "fmt" + "reflect" + "sort" + "sync" + + "kcl-lang.io/kpm/pkg/3rdparty/par" + + "golang.org/x/mod/module" +) + +// A Reqs is the requirement graph on which Minimal Version Selection (MVS) operates. +// +// The version strings are opaque except for the special version "none" +// (see the documentation for module.Version). In particular, MVS does not +// assume that the version strings are semantic versions; instead, the Max method +// gives access to the comparison operation. +// +// It must be safe to call methods on a Reqs from multiple goroutines simultaneously. +// Because a Reqs may read the underlying graph from the network on demand, +// the MVS algorithms parallelize the traversal to overlap network delays. +type Reqs interface { + // Required returns the module versions explicitly required by m itself. + // The caller must not modify the returned list. + Required(m module.Version) ([]module.Version, error) + + // Max returns the maximum of v1 and v2 (it returns either v1 or v2) + // in the module with path p. + // + // For all versions v, Max(v, "none") must be v, + // and for the target passed as the first argument to MVS functions, + // Max(target, v) must be target. + // + // Note that v1 < v2 can be written Max(v1, v2) != v1 + // and similarly v1 <= v2 can be written Max(v1, v2) == v2. + Max(p, v1, v2 string) string +} + +// An UpgradeReqs is a Reqs that can also identify available upgrades. +type UpgradeReqs interface { + Reqs + + // Upgrade returns the upgraded version of m, + // for use during an UpgradeAll operation. + // If m should be kept as is, Upgrade returns m. + // If m is not yet used in the build, then m.Version will be "none". + // More typically, m.Version will be the version required + // by some other module in the build. + // + // If no module version is available for the given path, + // Upgrade returns a non-nil error. + // TODO(rsc): Upgrade must be able to return errors, + // but should "no latest version" just return m instead? + Upgrade(m module.Version) (module.Version, error) +} + +// A DowngradeReqs is a Reqs that can also identify available downgrades. +type DowngradeReqs interface { + Reqs + + // Previous returns the version of m.Path immediately prior to m.Version, + // or "none" if no such version is known. + Previous(m module.Version) (module.Version, error) +} + +// BuildList returns the build list for the target module. +// +// target is the root vertex of a module requirement graph. For cmd/go, this is +// typically the main module, but note that this algorithm is not intended to +// be Go-specific: module paths and versions are treated as opaque values. +// +// reqs describes the module requirement graph and provides an opaque method +// for comparing versions. +// +// BuildList traverses the graph and returns a list containing the highest +// version for each visited module. The first element of the returned list is +// target itself; reqs.Max requires target.Version to compare higher than all +// other versions, so no other version can be selected. The remaining elements +// of the list are sorted by path. +// +// See https://research.swtch.com/vgo-mvs for details. +func BuildList(targets []module.Version, reqs Reqs) ([]module.Version, error) { + return buildList(targets, reqs, nil) +} + +func buildList(targets []module.Version, reqs Reqs, upgrade func(module.Version) (module.Version, error)) ([]module.Version, error) { + cmp := func(p, v1, v2 string) int { + if reqs.Max(p, v1, v2) != v1 { + return -1 + } + if reqs.Max(p, v2, v1) != v2 { + return 1 + } + return 0 + } + + var ( + mu sync.Mutex + g = NewGraph(cmp, targets) + upgrades = map[module.Version]module.Version{} + errs = map[module.Version]error{} // (non-nil errors only) + ) + + // Explore work graph in parallel in case reqs.Required + // does high-latency network operations. + var work par.Work[module.Version] + for _, target := range targets { + work.Add(target) + } + work.Do(10, func(m module.Version) { + + var required []module.Version + var err error + if m.Version != "none" { + required, err = reqs.Required(m) + } + + u := m + if upgrade != nil { + upgradeTo, upErr := upgrade(m) + if upErr == nil { + u = upgradeTo + } else if err == nil { + err = upErr + } + } + + mu.Lock() + if err != nil { + errs[m] = err + } + if u != m { + upgrades[m] = u + required = append([]module.Version{u}, required...) + } + g.Require(m, required) + mu.Unlock() + + for _, r := range required { + work.Add(r) + } + }) + + // If there was an error, find the shortest path from the target to the + // node where the error occurred so we can report a useful error message. + if len(errs) > 0 { + errPath := g.FindPath(func(m module.Version) bool { + return errs[m] != nil + }) + if len(errPath) == 0 { + panic("internal error: could not reconstruct path to module with error") + } + + err := errs[errPath[len(errPath)-1]] + isUpgrade := func(from, to module.Version) bool { + if u, ok := upgrades[from]; ok { + return u == to + } + return false + } + return nil, NewBuildListError(err, errPath, isUpgrade) + } + + // The final list is the minimum version of each module found in the graph. + list := g.BuildList() + if vs := list[:len(targets)]; !reflect.DeepEqual(vs, targets) { + // target.Version will be "" for modload, the main client of MVS. + // "" denotes the main module, which has no version. However, MVS treats + // version strings as opaque, so "" is not a special value here. + // See golang.org/issue/31491, golang.org/issue/29773. + panic(fmt.Sprintf("mistake: chose versions %+v instead of targets %+v", vs, targets)) + } + return list, nil +} + +// Req returns the minimal requirement list for the target module, +// with the constraint that all module paths listed in base must +// appear in the returned list. +func Req(mainModule module.Version, base []string, reqs Reqs) ([]module.Version, error) { + list, err := BuildList([]module.Version{mainModule}, reqs) + if err != nil { + return nil, err + } + + // Note: Not running in parallel because we assume + // that list came from a previous operation that paged + // in all the requirements, so there's no I/O to overlap now. + + max := map[string]string{} + for _, m := range list { + max[m.Path] = m.Version + } + + // Compute postorder, cache requirements. + var postorder []module.Version + reqCache := map[module.Version][]module.Version{} + reqCache[mainModule] = nil + + var walk func(module.Version) error + walk = func(m module.Version) error { + _, ok := reqCache[m] + if ok { + return nil + } + required, err := reqs.Required(m) + if err != nil { + return err + } + reqCache[m] = required + for _, m1 := range required { + if err := walk(m1); err != nil { + return err + } + } + postorder = append(postorder, m) + return nil + } + for _, m := range list { + if err := walk(m); err != nil { + return nil, err + } + } + + // Walk modules in reverse post-order, only adding those not implied already. + have := map[module.Version]bool{} + walk = func(m module.Version) error { + if have[m] { + return nil + } + have[m] = true + for _, m1 := range reqCache[m] { + walk(m1) + } + return nil + } + // First walk the base modules that must be listed. + var min []module.Version + haveBase := map[string]bool{} + for _, path := range base { + if haveBase[path] { + continue + } + m := module.Version{Path: path, Version: max[path]} + min = append(min, m) + walk(m) + haveBase[path] = true + } + // Now the reverse postorder to bring in anything else. + for i := len(postorder) - 1; i >= 0; i-- { + m := postorder[i] + if max[m.Path] != m.Version { + // Older version. + continue + } + if !have[m] { + min = append(min, m) + walk(m) + } + } + sort.Slice(min, func(i, j int) bool { + return min[i].Path < min[j].Path + }) + return min, nil +} + +// UpgradeAll returns a build list for the target module +// in which every module is upgraded to its latest version. +func UpgradeAll(target module.Version, reqs UpgradeReqs) ([]module.Version, error) { + return buildList([]module.Version{target}, reqs, func(m module.Version) (module.Version, error) { + if m.Path == target.Path { + return target, nil + } + + return reqs.Upgrade(m) + }) +} + +// Upgrade returns a build list for the target module +// in which the given additional modules are upgraded. +func Upgrade(target module.Version, reqs UpgradeReqs, upgrade ...module.Version) ([]module.Version, error) { + list, err := reqs.Required(target) + if err != nil { + return nil, err + } + + pathInList := make(map[string]bool, len(list)) + for _, m := range list { + pathInList[m.Path] = true + } + list = append([]module.Version(nil), list...) + + upgradeTo := make(map[string]string, len(upgrade)) + for _, u := range upgrade { + if !pathInList[u.Path] { + list = append(list, module.Version{Path: u.Path, Version: "none"}) + } + if prev, dup := upgradeTo[u.Path]; dup { + upgradeTo[u.Path] = reqs.Max(u.Path, prev, u.Version) + } else { + upgradeTo[u.Path] = u.Version + } + } + + return buildList([]module.Version{target}, &override{target, list, reqs}, func(m module.Version) (module.Version, error) { + if v, ok := upgradeTo[m.Path]; ok { + return module.Version{Path: m.Path, Version: v}, nil + } + return m, nil + }) +} + +// Downgrade returns a build list for the target module +// in which the given additional modules are downgraded, +// potentially overriding the requirements of the target. +// +// The versions to be downgraded may be unreachable from reqs.Latest and +// reqs.Previous, but the methods of reqs must otherwise handle such versions +// correctly. +func Downgrade(target module.Version, reqs DowngradeReqs, downgrade ...module.Version) ([]module.Version, error) { + // Per https://research.swtch.com/vgo-mvs#algorithm_4: + // “To avoid an unnecessary downgrade to E 1.1, we must also add a new + // requirement on E 1.2. We can apply Algorithm R to find the minimal set of + // new requirements to write to go.mod.” + // + // In order to generate those new requirements, we need to identify versions + // for every module in the build list — not just reqs.Required(target). + list, err := BuildList([]module.Version{target}, reqs) + if err != nil { + return nil, err + } + list = list[1:] // remove target + + max := make(map[string]string) + for _, r := range list { + max[r.Path] = r.Version + } + for _, d := range downgrade { + if v, ok := max[d.Path]; !ok || reqs.Max(d.Path, v, d.Version) != d.Version { + max[d.Path] = d.Version + } + } + + var ( + added = make(map[module.Version]bool) + rdeps = make(map[module.Version][]module.Version) + excluded = make(map[module.Version]bool) + ) + var exclude func(module.Version) + exclude = func(m module.Version) { + if excluded[m] { + return + } + excluded[m] = true + for _, p := range rdeps[m] { + exclude(p) + } + } + var add func(module.Version) + add = func(m module.Version) { + if added[m] { + return + } + added[m] = true + if v, ok := max[m.Path]; ok && reqs.Max(m.Path, m.Version, v) != v { + // m would upgrade an existing dependency — it is not a strict downgrade, + // and because it was already present as a dependency, it could affect the + // behavior of other relevant packages. + exclude(m) + return + } + list, err := reqs.Required(m) + if err != nil { + // If we can't load the requirements, we couldn't load the go.mod file. + // There are a number of reasons this can happen, but this usually + // means an older version of the module had a missing or invalid + // go.mod file. For example, if example.com/mod released v2.0.0 before + // migrating to modules (v2.0.0+incompatible), then added a valid go.mod + // in v2.0.1, downgrading from v2.0.1 would cause this error. + // + // TODO(golang.org/issue/31730, golang.org/issue/30134): if the error + // is transient (we couldn't download go.mod), return the error from + // Downgrade. Currently, we can't tell what kind of error it is. + exclude(m) + return + } + for _, r := range list { + add(r) + if excluded[r] { + exclude(m) + return + } + rdeps[r] = append(rdeps[r], m) + } + } + + downgraded := make([]module.Version, 0, len(list)+1) + downgraded = append(downgraded, target) +List: + for _, r := range list { + add(r) + for excluded[r] { + p, err := reqs.Previous(r) + if err != nil { + // This is likely a transient error reaching the repository, + // rather than a permanent error with the retrieved version. + // + // TODO(golang.org/issue/31730, golang.org/issue/30134): + // decode what to do based on the actual error. + return nil, err + } + // If the target version is a pseudo-version, it may not be + // included when iterating over prior versions using reqs.Previous. + // Insert it into the right place in the iteration. + // If v is excluded, p should be returned again by reqs.Previous on the next iteration. + if v := max[r.Path]; reqs.Max(r.Path, v, r.Version) != v && reqs.Max(r.Path, p.Version, v) != p.Version { + p.Version = v + } + if p.Version == "none" { + continue List + } + add(p) + r = p + } + downgraded = append(downgraded, r) + } + + // The downgrades we computed above only downgrade to versions enumerated by + // reqs.Previous. However, reqs.Previous omits some versions — such as + // pseudo-versions and retracted versions — that may be selected as transitive + // requirements of other modules. + // + // If one of those requirements pulls the version back up above the version + // identified by reqs.Previous, then the transitive dependencies of that that + // initially-downgraded version should no longer matter — in particular, we + // should not add new dependencies on module paths that nothing else in the + // updated module graph even requires. + // + // In order to eliminate those spurious dependencies, we recompute the build + // list with the actual versions of the downgraded modules as selected by MVS, + // instead of our initial downgrades. + // (See the downhiddenartifact and downhiddencross test cases). + actual, err := BuildList([]module.Version{target}, &override{ + target: target, + list: downgraded, + Reqs: reqs, + }) + if err != nil { + return nil, err + } + actualVersion := make(map[string]string, len(actual)) + for _, m := range actual { + actualVersion[m.Path] = m.Version + } + + downgraded = downgraded[:0] + for _, m := range list { + if v, ok := actualVersion[m.Path]; ok { + downgraded = append(downgraded, module.Version{Path: m.Path, Version: v}) + } + } + + return BuildList([]module.Version{target}, &override{ + target: target, + list: downgraded, + Reqs: reqs, + }) +} + +type override struct { + target module.Version + list []module.Version + Reqs +} + +func (r *override) Required(m module.Version) ([]module.Version, error) { + if m == r.target { + return r.list, nil + } + return r.Reqs.Required(m) +} diff --git a/pkg/3rdparty/par/queue.go b/pkg/3rdparty/par/queue.go new file mode 100644 index 00000000..180bc75e --- /dev/null +++ b/pkg/3rdparty/par/queue.go @@ -0,0 +1,88 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package par + +import "fmt" + +// Queue manages a set of work items to be executed in parallel. The number of +// active work items is limited, and excess items are queued sequentially. +type Queue struct { + maxActive int + st chan queueState +} + +type queueState struct { + active int // number of goroutines processing work; always nonzero when len(backlog) > 0 + backlog []func() + idle chan struct{} // if non-nil, closed when active becomes 0 +} + +// NewQueue returns a Queue that executes up to maxActive items in parallel. +// +// maxActive must be positive. +func NewQueue(maxActive int) *Queue { + if maxActive < 1 { + panic(fmt.Sprintf("par.NewQueue called with nonpositive limit (%d)", maxActive)) + } + + q := &Queue{ + maxActive: maxActive, + st: make(chan queueState, 1), + } + q.st <- queueState{} + return q +} + +// Add adds f as a work item in the queue. +// +// Add returns immediately, but the queue will be marked as non-idle until after +// f (and any subsequently-added work) has completed. +func (q *Queue) Add(f func()) { + st := <-q.st + if st.active == q.maxActive { + st.backlog = append(st.backlog, f) + q.st <- st + return + } + if st.active == 0 { + // Mark q as non-idle. + st.idle = nil + } + st.active++ + q.st <- st + + go func() { + for { + f() + + st := <-q.st + if len(st.backlog) == 0 { + if st.active--; st.active == 0 && st.idle != nil { + close(st.idle) + } + q.st <- st + return + } + f, st.backlog = st.backlog[0], st.backlog[1:] + q.st <- st + } + }() +} + +// Idle returns a channel that will be closed when q has no (active or enqueued) +// work outstanding. +func (q *Queue) Idle() <-chan struct{} { + st := <-q.st + defer func() { q.st <- st }() + + if st.idle == nil { + st.idle = make(chan struct{}) + if st.active == 0 { + close(st.idle) + } + } + + return st.idle +} diff --git a/pkg/3rdparty/par/work.go b/pkg/3rdparty/par/work.go new file mode 100644 index 00000000..5b6de942 --- /dev/null +++ b/pkg/3rdparty/par/work.go @@ -0,0 +1,223 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package par implements parallel execution helpers. +package par + +import ( + "errors" + "math/rand" + "sync" + "sync/atomic" +) + +// Work manages a set of work items to be executed in parallel, at most once each. +// The items in the set must all be valid map keys. +type Work[T comparable] struct { + f func(T) // function to run for each item + running int // total number of runners + + mu sync.Mutex + added map[T]bool // items added to set + todo []T // items yet to be run + wait sync.Cond // wait when todo is empty + waiting int // number of runners waiting for todo +} + +func (w *Work[T]) init() { + if w.added == nil { + w.added = make(map[T]bool) + } +} + +// Add adds item to the work set, if it hasn't already been added. +func (w *Work[T]) Add(item T) { + w.mu.Lock() + w.init() + if !w.added[item] { + w.added[item] = true + w.todo = append(w.todo, item) + if w.waiting > 0 { + w.wait.Signal() + } + } + w.mu.Unlock() +} + +// Do runs f in parallel on items from the work set, +// with at most n invocations of f running at a time. +// It returns when everything added to the work set has been processed. +// At least one item should have been added to the work set +// before calling Do (or else Do returns immediately), +// but it is allowed for f(item) to add new items to the set. +// Do should only be used once on a given Work. +func (w *Work[T]) Do(n int, f func(item T)) { + if n < 1 { + panic("par.Work.Do: n < 1") + } + if w.running >= 1 { + panic("par.Work.Do: already called Do") + } + + w.running = n + w.f = f + w.wait.L = &w.mu + + for i := 0; i < n-1; i++ { + go w.runner() + } + w.runner() +} + +// runner executes work in w until both nothing is left to do +// and all the runners are waiting for work. +// (Then all the runners return.) +func (w *Work[T]) runner() { + for { + // Wait for something to do. + w.mu.Lock() + for len(w.todo) == 0 { + w.waiting++ + if w.waiting == w.running { + // All done. + w.wait.Broadcast() + w.mu.Unlock() + return + } + w.wait.Wait() + w.waiting-- + } + + // Pick something to do at random, + // to eliminate pathological contention + // in case items added at about the same time + // are most likely to contend. + i := rand.Intn(len(w.todo)) + item := w.todo[i] + w.todo[i] = w.todo[len(w.todo)-1] + w.todo = w.todo[:len(w.todo)-1] + w.mu.Unlock() + + w.f(item) + } +} + +// ErrCache is like Cache except that it also stores +// an error value alongside the cached value V. +type ErrCache[K comparable, V any] struct { + Cache[K, errValue[V]] +} + +type errValue[V any] struct { + v V + err error +} + +func (c *ErrCache[K, V]) Do(key K, f func() (V, error)) (V, error) { + v := c.Cache.Do(key, func() errValue[V] { + v, err := f() + return errValue[V]{v, err} + }) + return v.v, v.err +} + +var ErrCacheEntryNotFound = errors.New("cache entry not found") + +// Get returns the cached result associated with key. +// It returns ErrCacheEntryNotFound if there is no such result. +func (c *ErrCache[K, V]) Get(key K) (V, error) { + v, ok := c.Cache.Get(key) + if !ok { + v.err = ErrCacheEntryNotFound + } + return v.v, v.err +} + +// Cache runs an action once per key and caches the result. +type Cache[K comparable, V any] struct { + m sync.Map +} + +type cacheEntry[V any] struct { + done atomic.Bool + mu sync.Mutex + result V +} + +// Do calls the function f if and only if Do is being called for the first time with this key. +// No call to Do with a given key returns until the one call to f returns. +// Do returns the value returned by the one call to f. +func (c *Cache[K, V]) Do(key K, f func() V) V { + entryIface, ok := c.m.Load(key) + if !ok { + entryIface, _ = c.m.LoadOrStore(key, new(cacheEntry[V])) + } + e := entryIface.(*cacheEntry[V]) + if !e.done.Load() { + e.mu.Lock() + if !e.done.Load() { + e.result = f() + e.done.Store(true) + } + e.mu.Unlock() + } + return e.result +} + +// Get returns the cached result associated with key +// and reports whether there is such a result. +// +// If the result for key is being computed, Get does not wait for the computation to finish. +func (c *Cache[K, V]) Get(key K) (V, bool) { + entryIface, ok := c.m.Load(key) + if !ok { + return *new(V), false + } + e := entryIface.(*cacheEntry[V]) + if !e.done.Load() { + return *new(V), false + } + return e.result, true +} + +// Clear removes all entries in the cache. +// +// Concurrent calls to Get may return old values. Concurrent calls to Do +// may return old values or store results in entries that have been deleted. +// +// TODO(jayconrod): Delete this after the package cache clearing functions +// in internal/load have been removed. +func (c *Cache[K, V]) Clear() { + c.m.Range(func(key, value any) bool { + c.m.Delete(key) + return true + }) +} + +// Delete removes an entry from the map. It is safe to call Delete for an +// entry that does not exist. Delete will return quickly, even if the result +// for a key is still being computed; the computation will finish, but the +// result won't be accessible through the cache. +// +// TODO(jayconrod): Delete this after the package cache clearing functions +// in internal/load have been removed. +func (c *Cache[K, V]) Delete(key K) { + c.m.Delete(key) +} + +// DeleteIf calls pred for each key in the map. If pred returns true for a key, +// DeleteIf removes the corresponding entry. If the result for a key is +// still being computed, DeleteIf will remove the entry without waiting for +// the computation to finish. The result won't be accessible through the cache. +// +// TODO(jayconrod): Delete this after the package cache clearing functions +// in internal/load have been removed. +func (c *Cache[K, V]) DeleteIf(pred func(key K) bool) { + c.m.Range(func(key, _ any) bool { + if key := key.(K); pred(key) { + c.Delete(key) + } + return true + }) +} diff --git a/pkg/client/client.go b/pkg/client/client.go index b762fcc0..94571792 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -14,6 +14,7 @@ import ( "github.com/dominikbraun/graph" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/otiai10/copy" + "golang.org/x/mod/module" "kcl-lang.io/kcl-go/pkg/kcl" "oras.land/oras-go/v2" @@ -1191,12 +1192,15 @@ func (c *KpmClient) ParseOciOptionFromString(oci string, tag string) (*opt.OciOp } // InitGraphAndDownloadDeps initializes a dependency graph and call downloadDeps function. -func (c *KpmClient) InitGraphAndDownloadDeps(kclPkg *pkg.KclPkg) (*pkg.Dependencies, graph.Graph[string, string], error) { +func (c *KpmClient) InitGraphAndDownloadDeps(kclPkg *pkg.KclPkg) (*pkg.Dependencies, graph.Graph[module.Version, module.Version], error) { - depGraph := graph.New(graph.StringHash, graph.Directed(), graph.PreventCycles()) + moduleHash := func(m module.Version) module.Version { + return m + } + depGraph := graph.New(moduleHash, graph.Directed(), graph.PreventCycles()) // add the root vertex(package name) to the dependency graph. - root := fmt.Sprintf("%s@%s", kclPkg.GetPkgName(), kclPkg.GetPkgVersion()) + root := module.Version{Path: kclPkg.GetPkgName(), Version: kclPkg.GetPkgVersion()} err := depGraph.AddVertex(root) if err != nil { return nil, nil, err @@ -1234,7 +1238,7 @@ func (c *KpmClient) dependencyExists(dep *pkg.Dependency, lockDeps *pkg.Dependen } // downloadDeps will download all the dependencies of the current kcl package. -func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencies, depGraph graph.Graph[string, string], parent string) (*pkg.Dependencies, error) { +func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencies, depGraph graph.Graph[module.Version, module.Version], parent module.Version) (*pkg.Dependencies, error) { newDeps := pkg.Dependencies{ Deps: make(map[string]pkg.Dependency), } @@ -1305,8 +1309,7 @@ func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencie } return nil, err } - source := fmt.Sprintf("%s@%s", d.Name, d.Version) - source = strings.TrimRight(source, "@") + source := module.Version{Path: d.Name, Version: d.Version} err = depGraph.AddVertex(source) if err != nil && err != graph.ErrVertexAlreadyExists { return nil, err diff --git a/pkg/cmd/cmd_graph.go b/pkg/cmd/cmd_graph.go index 3d1ec0f1..8bb4409a 100644 --- a/pkg/cmd/cmd_graph.go +++ b/pkg/cmd/cmd_graph.go @@ -8,6 +8,7 @@ import ( "github.com/dominikbraun/graph" "github.com/urfave/cli/v2" + "golang.org/x/mod/module" "kcl-lang.io/kpm/pkg/client" "kcl-lang.io/kpm/pkg/env" pkg "kcl-lang.io/kpm/pkg/package" @@ -72,12 +73,20 @@ func KpmGraph(c *cli.Context, kpmcli *client.KpmClient) error { return err } + format := func(m module.Version) string { + formattedMsg := m.Path + if m.Version != "" { + formattedMsg += "@" + m.Version + } + return formattedMsg + } + // print the dependency graph to stdout. - root := fmt.Sprintf("%s@%s", kclPkg.GetPkgName(), kclPkg.GetPkgVersion()) - err = graph.BFS(depGraph, root, func(source string) bool { + root := module.Version{Path: kclPkg.GetPkgName(), Version: kclPkg.GetPkgVersion()} + err = graph.BFS(depGraph, root, func(source module.Version) bool { for target := range adjMap[source] { reporter.ReportMsgTo( - fmt.Sprint(source, " ", target), + fmt.Sprint(format(source), " ", format(target)), kpmcli.GetLogWriter(), ) } diff --git a/pkg/graph/graph.go b/pkg/graph/graph.go new file mode 100644 index 00000000..cb0fca1f --- /dev/null +++ b/pkg/graph/graph.go @@ -0,0 +1,24 @@ +package graph + +import ( + "fmt" + + "github.com/dominikbraun/graph" + "golang.org/x/mod/module" +) + +// ToAdjacencyList converts graph to adjacency list +func ToAdjacencyList(g graph.Graph[module.Version, module.Version]) (map[module.Version][]module.Version, error) { + AdjacencyMap, err := g.AdjacencyMap() + if err != nil { + return nil, fmt.Errorf("failed to get adjacency map: %w", err) + } + + adjList := make(map[module.Version][]module.Version) + for from, v := range AdjacencyMap { + for to := range v { + adjList[from] = append(adjList[from], to) + } + } + return adjList, nil +} diff --git a/pkg/mvs/mvs.go b/pkg/mvs/mvs.go new file mode 100644 index 00000000..291b5213 --- /dev/null +++ b/pkg/mvs/mvs.go @@ -0,0 +1,38 @@ +package mvs + +import ( + "fmt" + + "golang.org/x/mod/module" +) + +type ReqsMap map[module.Version][]module.Version + +func (r ReqsMap) Max(_, v1, v2 string) string { + if v1 == "none" || v2 == "" { + return v2 + } + if v2 == "none" || v1 == "" { + return v1 + } + if v1 < v2 { + return v2 + } + return v1 +} + +func (r ReqsMap) Upgrade(m module.Version) (module.Version, error) { + panic("unimplemented") +} + +func (r ReqsMap) Previous(m module.Version) (module.Version, error) { + panic("unimplemented") +} + +func (r ReqsMap) Required(m module.Version) ([]module.Version, error) { + rr, ok := r[m] + if !ok { + return nil, fmt.Errorf("missing module: %v", m) + } + return rr, nil +} diff --git a/pkg/package/package.go b/pkg/package/package.go index 01058e25..15906c02 100644 --- a/pkg/package/package.go +++ b/pkg/package/package.go @@ -6,6 +6,8 @@ import ( "path/filepath" "strings" + "golang.org/x/mod/module" + "kcl-lang.io/kcl-go/pkg/kcl" "kcl-lang.io/kpm/pkg/constants" errors "kcl-lang.io/kpm/pkg/errors" @@ -20,6 +22,8 @@ type KclPkg struct { // The dependencies in the current kcl package are the dependencies of kcl.mod.lock, // not the dependencies in kcl.mod. Dependencies + // minimal build list for the current kcl package. + BuildList []module.Version // The flag 'NoSumCheck' is true if the checksum of the current kcl package is not checked. NoSumCheck bool } From ec6f1f21961673cbbdcfdfbc6fba152eff3d61b6 Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Sat, 30 Mar 2024 21:16:52 +0530 Subject: [PATCH 02/17] fix: build error Signed-off-by: Akash Kumar --- pkg/client/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index c4dca86d..20c285c1 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -1287,7 +1287,7 @@ func (c *KpmClient) dependencyExists(dep *pkg.Dependency, lockDeps *pkg.Dependen } // downloadDeps will download all the dependencies of the current kcl package. -func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencies, depGraph graph.Graph[module.Version, module.Version], pkghome, parent module.Version) (*pkg.Dependencies, error) { +func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencies, depGraph graph.Graph[module.Version, module.Version], pkghome string, parent module.Version) (*pkg.Dependencies, error) { newDeps := pkg.Dependencies{ Deps: make(map[string]pkg.Dependency), From 62ff3e6cf3731f36a745e49f103b13b7dd292c44 Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Fri, 5 Apr 2024 00:45:23 +0530 Subject: [PATCH 03/17] fix: Supports adding third-party dependencies from git repo with the version field Signed-off-by: Akash Kumar --- pkg/client/client.go | 7 +++-- pkg/client/client_test.go | 29 +++++++++++-------- .../test_pkg/kcl.mod.expect | 2 +- .../test_pkg/kcl.mod.lock.expect | 3 +- pkg/package/modfile.go | 9 +++--- pkg/package/toml.go | 5 ++++ 6 files changed, 34 insertions(+), 21 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index 20c285c1..aef48855 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -827,12 +827,13 @@ func (c *KpmClient) Download(dep *pkg.Dependency, homePath, localPath string) (* return nil, err } dep.FullName = dep.GenDepFullName() - // If the dependency is from git commit, the version is the commit id. - // If the dependency is from git tag, the version is the tag. - dep.Version, err = dep.Source.Git.GetValidGitReference() + + modFile, err := c.LoadModFile(localPath) if err != nil { return nil, err } + dep.Version = modFile.Pkg.Version + dep.Source.Git.Version = modFile.Pkg.Version } if dep.Source.Oci != nil { diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 83102f12..afd189ce 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -16,6 +16,7 @@ import ( "github.com/dominikbraun/graph" "github.com/otiai10/copy" "github.com/stretchr/testify/assert" + "golang.org/x/mod/module" "kcl-lang.io/kcl-go/pkg/kcl" "kcl-lang.io/kpm/pkg/env" "kcl-lang.io/kpm/pkg/git" @@ -181,28 +182,32 @@ func TestDependencyGraph(t *testing.T) { adjMap, err := depGraph.AdjacencyMap() assert.Equal(t, err, nil) + m := func(Path, Version string) module.Version { + return module.Version{Path, Version} + } + edgeProp := graph.EdgeProperties{ Attributes: map[string]string{}, Weight: 0, Data: nil, } assert.Equal(t, adjMap, - map[string]map[string]graph.Edge[string]{ - "dependency_graph@0.0.1": { - "teleport@0.1.0": {Source: "dependency_graph@0.0.1", Target: "teleport@0.1.0", Properties: edgeProp}, - "rabbitmq@0.0.1": {Source: "dependency_graph@0.0.1", Target: "rabbitmq@0.0.1", Properties: edgeProp}, - "agent@0.1.0": {Source: "dependency_graph@0.0.1", Target: "agent@0.1.0", Properties: edgeProp}, + map[module.Version]map[module.Version]graph.Edge[module.Version]{ + m("dependency_graph", "0.0.1"): { + m("teleport", "0.1.0"): {Source: m("dependency_graph", "0.0.1"), Target: m("teleport", "0.1.0"), Properties: edgeProp}, + m("rabbitmq", "0.0.1"): {Source: m("dependency_graph", "0.0.1"), Target: m("rabbitmq", "0.0.1"), Properties: edgeProp}, + m("agent", "0.1.0"): {Source: m("dependency_graph", "0.0.1"), Target: m("agent", "0.1.0"), Properties: edgeProp}, }, - "teleport@0.1.0": { - "k8s@1.28": {Source: "teleport@0.1.0", Target: "k8s@1.28", Properties: edgeProp}, + m("teleport", "0.1.0"): { + m("k8s", "1.28"): {Source: m("teleport", "0.1.0"), Target: m("k8s", "1.28"), Properties: edgeProp}, }, - "rabbitmq@0.0.1": { - "k8s@1.28": {Source: "rabbitmq@0.0.1", Target: "k8s@1.28", Properties: edgeProp}, + m("rabbitmq", "0.0.1"): { + m("k8s", "1.28"): {Source: m("rabbitmq", "0.0.1"), Target: m("k8s", "1.28"), Properties: edgeProp}, }, - "agent@0.1.0": { - "k8s@1.28": {Source: "agent@0.1.0", Target: "k8s@1.28", Properties: edgeProp}, + m("agent", "0.1.0"): { + m("k8s", "1.28"): {Source: m("agent", "0.1.0"), Target: m("k8s", "1.28"), Properties: edgeProp}, }, - "k8s@1.28": {}, + m("k8s", "1.28"): {}, }, ) } diff --git a/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.expect b/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.expect index 475d36f8..112fdd08 100644 --- a/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.expect +++ b/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.expect @@ -4,4 +4,4 @@ edition = "0.0.1" version = "0.0.1" [dependencies] -catalog = { git = "https://github.com/KusionStack/catalog.git", commit = "a29e3db" } +catalog = { git = "https://github.com/KusionStack/catalog.git", commit = "a29e3db", version = "0.1.0" } diff --git a/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.lock.expect b/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.lock.expect index ccc62bba..f8a58f5b 100644 --- a/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.lock.expect +++ b/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.lock.expect @@ -2,7 +2,8 @@ [dependencies.catalog] name = "catalog" full_name = "catalog_a29e3db" - version = "a29e3db" + version = "0.1.0" sum = "kFmlrYJbJUFFTEXjC9cquc80WB+UpZ/6oMPKrfgyeks=" url = "https://github.com/KusionStack/catalog.git" commit = "a29e3db" + version = "0.1.0" diff --git a/pkg/package/modfile.go b/pkg/package/modfile.go index c92a4275..baab5471 100644 --- a/pkg/package/modfile.go +++ b/pkg/package/modfile.go @@ -298,10 +298,11 @@ func (oci *Oci) FromString(ociUrl string) (*Oci, error) { // Git is the package source from git registry. type Git struct { - Url string `toml:"url,omitempty"` - Branch string `toml:"branch,omitempty"` - Commit string `toml:"commit,omitempty"` - Tag string `toml:"git_tag,omitempty"` + Url string `toml:"url,omitempty"` + Branch string `toml:"branch,omitempty"` + Commit string `toml:"commit,omitempty"` + Tag string `toml:"git_tag,omitempty"` + Version string `toml:"version,omitempty"` } // GetValidGitReference will get the valid git reference from git source. diff --git a/pkg/package/toml.go b/pkg/package/toml.go index 6bc8a34e..8a9a3204 100644 --- a/pkg/package/toml.go +++ b/pkg/package/toml.go @@ -118,6 +118,7 @@ func (source *Source) MarshalTOML() string { const GIT_URL_PATTERN = "git = \"%s\"" const TAG_PATTERN = "tag = \"%s\"" const GIT_COMMIT_PATTERN = "commit = \"%s\"" +const VERSION_PATTERN = "version = \"%s\"" const SEPARATOR = ", " func (git *Git) MarshalTOML() string { @@ -133,6 +134,10 @@ func (git *Git) MarshalTOML() string { sb.WriteString(SEPARATOR) sb.WriteString(fmt.Sprintf(GIT_COMMIT_PATTERN, git.Commit)) } + if len(git.Version) != 0 { + sb.WriteString(SEPARATOR) + sb.WriteString(fmt.Sprintf(VERSION_PATTERN, git.Version)) + } return sb.String() } From 6cf8df8fc726931894c0d6970c7d518ba2f36370 Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Fri, 12 Apr 2024 19:25:21 +0530 Subject: [PATCH 04/17] address review comments Signed-off-by: Akash Kumar --- .../test_data/add_with_git_commit/test_pkg/kcl.mod.lock.expect | 1 - pkg/package/package.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.lock.expect b/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.lock.expect index f8a58f5b..fed06b76 100644 --- a/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.lock.expect +++ b/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.lock.expect @@ -2,7 +2,6 @@ [dependencies.catalog] name = "catalog" full_name = "catalog_a29e3db" - version = "0.1.0" sum = "kFmlrYJbJUFFTEXjC9cquc80WB+UpZ/6oMPKrfgyeks=" url = "https://github.com/KusionStack/catalog.git" commit = "a29e3db" diff --git a/pkg/package/package.go b/pkg/package/package.go index 814bd95f..28401265 100644 --- a/pkg/package/package.go +++ b/pkg/package/package.go @@ -22,7 +22,7 @@ type KclPkg struct { // The dependencies in the current kcl package are the dependencies of kcl.mod.lock, // not the dependencies in kcl.mod. Dependencies - // minimal build list for the current kcl package. + // BuildList denotes the minimal build list for the current kcl package. BuildList []module.Version // The flag 'NoSumCheck' is true if the checksum of the current kcl package is not checked. NoSumCheck bool From 3fd05e7bf6add75cf8ec0e1b523b381123d057c0 Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Sun, 12 May 2024 00:00:57 +0530 Subject: [PATCH 05/17] feat: add implementation of upgrade, previous, required, max Signed-off-by: Akash Kumar --- go.mod | 7 + go.sum | 9 + pkg/3rdparty/mvs/mvs.go | 3 +- pkg/3rdparty/mvs/mvs_test.go | 635 ++++++++++++++++++ pkg/client/client.go | 8 +- pkg/client/client_test.go | 2 +- pkg/errors/errors.go | 3 + pkg/git/git.go | 85 +++ pkg/graph/graph.go | 33 +- pkg/mvs/mvs.go | 112 ++- pkg/mvs/mvs_test.go | 192 ++++++ .../test_data/test_with_external_deps/kcl.mod | 10 + .../test_with_internal_deps/aaa/kcl.mod | 8 + .../test_with_internal_deps/bbb/kcl.mod | 5 + .../test_with_internal_deps/ccc/kcl.mod | 5 + pkg/oci/oci.go | 18 + pkg/package/modfile.go | 29 + pkg/package/package.go | 4 - pkg/semver/semver.go | 19 + pkg/semver/semver_test.go | 32 + 20 files changed, 1184 insertions(+), 35 deletions(-) create mode 100644 pkg/3rdparty/mvs/mvs_test.go create mode 100644 pkg/mvs/mvs_test.go create mode 100644 pkg/mvs/test_data/test_with_external_deps/kcl.mod create mode 100644 pkg/mvs/test_data/test_with_internal_deps/aaa/kcl.mod create mode 100644 pkg/mvs/test_data/test_with_internal_deps/bbb/kcl.mod create mode 100644 pkg/mvs/test_data/test_with_internal_deps/ccc/kcl.mod diff --git a/go.mod b/go.mod index acd7eaae..29997e6d 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chai2010/jsonv v1.1.3 // indirect github.com/chai2010/protorpc v1.1.4 // indirect + github.com/chainguard-dev/git-urls v1.0.2 // indirect github.com/cloudflare/circl v1.3.7 // indirect github.com/containerd/containerd v1.7.0 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect @@ -45,10 +46,12 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/fatih/color v1.10.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/getkin/kin-openapi v0.123.0 // indirect + github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/go-logr/logr v1.3.0 // indirect @@ -128,11 +131,14 @@ require ( google.golang.org/grpc v1.62.0 // indirect google.golang.org/protobuf v1.32.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 // indirect kcl-lang.io/lib v0.8.0 // indirect ) require ( + github.com/containers/image v3.0.2+incompatible github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/go-git/go-git/v5 v5.11.0 github.com/gofrs/flock v0.8.1 @@ -140,6 +146,7 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-getter v1.7.3 github.com/hashicorp/go-version v1.6.0 + github.com/kubescape/go-git-url v0.0.30 github.com/moby/term v0.0.0-20221205130635-1aeaba878587 github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 diff --git a/go.sum b/go.sum index 6b62aa77..7162af86 100644 --- a/go.sum +++ b/go.sum @@ -246,6 +246,8 @@ github.com/chai2010/jsonv v1.1.3 h1:gBIHXn/5mdEPTuWZfjC54fn/yUSRR8OGobXobcc6now= github.com/chai2010/jsonv v1.1.3/go.mod h1:mEoT1dQ9qVF4oP9peVTl0UymTmJwXoTDOh+sNA6+XII= github.com/chai2010/protorpc v1.1.4 h1:CTtFUhzXRoeuR7FtgQ2b2vdT/KgWVpCM+sIus8zJjHs= github.com/chai2010/protorpc v1.1.4/go.mod h1:/wO0kiyVdu7ug8dCMrA2yDr2vLfyhsLEuzLa9J2HJ+I= +github.com/chainguard-dev/git-urls v1.0.2 h1:pSpT7ifrpc5X55n4aTTm7FFUE+ZQHKiqpiwNkJrVcKQ= +github.com/chainguard-dev/git-urls v1.0.2/go.mod h1:rbGgj10OS7UgZlbzdUQIQpT0k/D4+An04HJY7Ol+Y/o= github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -271,6 +273,8 @@ github.com/containerd/containerd v1.7.0 h1:G/ZQr3gMZs6ZT0qPUZ15znx5QSdQdASW11nXT github.com/containerd/containerd v1.7.0/go.mod h1:QfR7Efgb/6X2BDpTPJRvPTYDE9rsF0FsXX9J8sIs/sc= github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= +github.com/containers/image v3.0.2+incompatible h1:B1lqAE8MUPCrsBLE86J0gnXleeRq8zJnQryhiiGQNyE= +github.com/containers/image v3.0.2+incompatible/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= @@ -325,6 +329,7 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/getkin/kin-openapi v0.123.0 h1:zIik0mRwFNLyvtXK274Q6ut+dPh6nlxBp0x7mNrPhs8= github.com/getkin/kin-openapi v0.123.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= @@ -545,6 +550,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubescape/go-git-url v0.0.30 h1:PIbg86ae0ftee/p/Tu/6CA1ju6VoJ51G3sQWNHOm6wg= +github.com/kubescape/go-git-url v0.0.30/go.mod h1:3ddc1HEflms1vMhD9owt/3FBES070UaYTUarcjx8jDk= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -1309,6 +1316,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk= +k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kcl-lang.io/kcl-go v0.8.0 h1:ep+r4QMiAOeTkOJl9then58D+W8OpX5WHKuL+Tf8+po= kcl-lang.io/kcl-go v0.8.0/go.mod h1:Z+bJWXe5X1Xra7AUOvDpCb4WBcCGNo0sqQG/bNjz6+k= kcl-lang.io/lib v0.8.0 h1:bzMzPpaXaAxWO9JP0B7eI2ZFOYfojdEYUMtNGlUrPx4= diff --git a/pkg/3rdparty/mvs/mvs.go b/pkg/3rdparty/mvs/mvs.go index 08df38a1..4376c9d1 100644 --- a/pkg/3rdparty/mvs/mvs.go +++ b/pkg/3rdparty/mvs/mvs.go @@ -12,9 +12,8 @@ import ( "sort" "sync" - "kcl-lang.io/kpm/pkg/3rdparty/par" - "golang.org/x/mod/module" + "kcl-lang.io/kpm/pkg/3rdparty/par" ) // A Reqs is the requirement graph on which Minimal Version Selection (MVS) operates. diff --git a/pkg/3rdparty/mvs/mvs_test.go b/pkg/3rdparty/mvs/mvs_test.go new file mode 100644 index 00000000..b18ee2de --- /dev/null +++ b/pkg/3rdparty/mvs/mvs_test.go @@ -0,0 +1,635 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mvs + +import ( + "fmt" + "reflect" + "strings" + "testing" + + "golang.org/x/mod/module" +) + +var tests = ` +# Scenario from blog. +name: blog +A: B1 C2 +B1: D3 +C1: D2 +C2: D4 +C3: D5 +C4: G1 +D2: E1 +D3: E2 +D4: E2 F1 +D5: E2 +G1: C4 +A2: B1 C4 D4 +build A: A B1 C2 D4 E2 F1 +upgrade* A: A B1 C4 D5 E2 F1 G1 +upgrade A C4: A B1 C4 D4 E2 F1 G1 +build A2: A2 B1 C4 D4 E2 F1 G1 +downgrade A2 D2: A2 C4 D2 E2 F1 G1 + +name: trim +A: B1 C2 +B1: D3 +C2: B2 +B2: +build A: A B2 C2 D3 + +# Cross-dependency between D and E. +# No matter how it arises, should get result of merging all build lists via max, +# which leads to including both D2 and E2. + +name: cross1 +A: B C +B: D1 +C: D2 +D1: E2 +D2: E1 +build A: A B C D2 E2 + +name: cross1V +A: B2 C D2 E1 +B1: +B2: D1 +C: D2 +D1: E2 +D2: E1 +build A: A B2 C D2 E2 + +name: cross1U +A: B1 C +B1: +B2: D1 +C: D2 +D1: E2 +D2: E1 +build A: A B1 C D2 E1 +upgrade A B2: A B2 C D2 E2 + +name: cross1R +A: B C +B: D2 +C: D1 +D1: E2 +D2: E1 +build A: A B C D2 E2 + +name: cross1X +A: B C +B: D1 E2 +C: D2 +D1: E2 +D2: E1 +build A: A B C D2 E2 + +name: cross2 +A: B D2 +B: D1 +D1: E2 +D2: E1 +build A: A B D2 E2 + +name: cross2X +A: B D2 +B: D1 E2 +C: D2 +D1: E2 +D2: E1 +build A: A B D2 E2 + +name: cross3 +A: B D2 E1 +B: D1 +D1: E2 +D2: E1 +build A: A B D2 E2 + +name: cross3X +A: B D2 E1 +B: D1 E2 +D1: E2 +D2: E1 +build A: A B D2 E2 + +# Should not get E2 here, because B has been updated +# not to depend on D1 anymore. +name: cross4 +A1: B1 D2 +A2: B2 D2 +B1: D1 +B2: D2 +D1: E2 +D2: E1 +build A1: A1 B1 D2 E2 +build A2: A2 B2 D2 E1 + +# But the upgrade from A1 preserves the E2 dep explicitly. +upgrade A1 B2: A1 B2 D2 E2 +upgradereq A1 B2: B2 E2 + +name: cross5 +A: D1 +D1: E2 +D2: E1 +build A: A D1 E2 +upgrade* A: A D2 E2 +upgrade A D2: A D2 E2 +upgradereq A D2: D2 E2 + +name: cross6 +A: D2 +D1: E2 +D2: E1 +build A: A D2 E1 +upgrade* A: A D2 E2 +upgrade A E2: A D2 E2 + +name: cross7 +A: B C +B: D1 +C: E1 +D1: E2 +E1: D2 +build A: A B C D2 E2 + +# golang.org/issue/31248: +# Even though we select X2, the requirement on I1 +# via X1 should be preserved. +name: cross8 +M: A1 B1 +A1: X1 +B1: X2 +X1: I1 +X2: +build M: M A1 B1 I1 X2 + +# Upgrade from B1 to B2 should not drop the transitive dep on D. +name: drop +A: B1 C1 +B1: D1 +B2: +C2: +D2: +build A: A B1 C1 D1 +upgrade* A: A B2 C2 D2 + +name: simplify +A: B1 C1 +B1: C2 +C1: D1 +C2: +build A: A B1 C2 D1 + +name: up1 +A: B1 C1 +B1: +B2: +B3: +B4: +B5.hidden: +C2: +C3: +build A: A B1 C1 +upgrade* A: A B4 C3 + +name: up2 +A: B5.hidden C1 +B1: +B2: +B3: +B4: +B5.hidden: +C2: +C3: +build A: A B5.hidden C1 +upgrade* A: A B5.hidden C3 + +name: down1 +A: B2 +B1: C1 +B2: C2 +build A: A B2 C2 +downgrade A C1: A B1 C1 + +name: down2 +A: B2 E2 +B1: +B2: C2 F2 +C1: +D1: +C2: D2 E2 +D2: B2 +E2: D2 +E1: +F1: +build A: A B2 C2 D2 E2 F2 +downgrade A F1: A B1 C1 D1 E1 F1 + +# https://research.swtch.com/vgo-mvs#algorithm_4: +# “[D]owngrades are constrained to only downgrade packages, not also upgrade +# them; if an upgrade before downgrade is needed, the user must ask for it +# explicitly.” +# +# Here, downgrading B2 to B1 upgrades C1 to C2, and C2 does not depend on D2. +# However, C2 would be an upgrade — not a downgrade — so B1 must also be +# rejected. +name: downcross1 +A: B2 C1 +B1: C2 +B2: C1 +C1: D2 +C2: +D1: +D2: +build A: A B2 C1 D2 +downgrade A D1: A D1 + +# https://research.swtch.com/vgo-mvs#algorithm_4: +# “Unlike upgrades, downgrades must work by removing requirements, not adding +# them.” +# +# However, downgrading a requirement may introduce a new requirement on a +# previously-unrequired module. If each dependency's requirements are complete +# (“tidy”), that can't change the behavior of any other package whose version is +# not also being downgraded, so we should allow it. +name: downcross2 +A: B2 +B1: C1 +B2: D2 +C1: +D1: +D2: +build A: A B2 D2 +downgrade A D1: A B1 C1 D1 + +name: downcycle +A: A B2 +B2: A +B1: +build A: A B2 +downgrade A B1: A B1 + +# Both B3 and C2 require D2. +# If we downgrade D to D1, then in isolation B3 would downgrade to B1, +# because B2 is hidden — B1 is the next-highest version that is not hidden. +# However, if we downgrade D, we will also downgrade C to C1. +# And C1 requires B2.hidden, and B2.hidden also meets our requirements: +# it is compatible with D1 and a strict downgrade from B3. +# +# Since neither the initial nor the final build list includes B1, +# and the nothing in the final downgraded build list requires E at all, +# no dependency on E1 (required by only B1) should be introduced. +# +name: downhiddenartifact +A: B3 C2 +A1: B3 +B1: E1 +B2.hidden: +B3: D2 +C1: B2.hidden +C2: D2 +D1: +D2: +build A1: A1 B3 D2 +downgrade A1 D1: A1 B1 D1 E1 +build A: A B3 C2 D2 +downgrade A D1: A B2.hidden C1 D1 + +# Both B3 and C3 require D2. +# If we downgrade D to D1, then in isolation B3 would downgrade to B1, +# and C3 would downgrade to C1. +# But C1 requires B2.hidden, and B1 requires C2.hidden, so we can't +# downgrade to either of those without pulling the other back up a little. +# +# B2.hidden and C2.hidden are both compatible with D1, so that still +# meets our requirements — but then we're in an odd state in which +# B and C have both been downgraded to hidden versions, without any +# remaining requirements to explain how those hidden versions got there. +# +# TODO(bcmills): Would it be better to force downgrades to land on non-hidden +# versions? +# In this case, that would remove the dependencies on B and C entirely. +# +name: downhiddencross +A: B3 C3 +B1: C2.hidden +B2.hidden: +B3: D2 +C1: B2.hidden +C2.hidden: +C3: D2 +D1: +D2: +build A: A B3 C3 D2 +downgrade A D1: A B2.hidden C2.hidden D1 + +# golang.org/issue/25542. +name: noprev1 +A: B4 C2 +B2.hidden: +C2: +build A: A B4 C2 +downgrade A B2.hidden: A B2.hidden C2 + +name: noprev2 +A: B4 C2 +B2.hidden: +B1: +C2: +build A: A B4 C2 +downgrade A B2.hidden: A B2.hidden C2 + +name: noprev3 +A: B4 C2 +B3: +B2.hidden: +C2: +build A: A B4 C2 +downgrade A B2.hidden: A B2.hidden C2 + +# Cycles involving the target. + +# The target must be the newest version of itself. +name: cycle1 +A: B1 +B1: A1 +B2: A2 +B3: A3 +build A: A B1 +upgrade A B2: A B2 +upgrade* A: A B3 + +# golang.org/issue/29773: +# Requirements of older versions of the target +# must be carried over. +name: cycle2 +A: B1 +A1: C1 +A2: D1 +B1: A1 +B2: A2 +C1: A2 +C2: +D2: +build A: A B1 C1 D1 +upgrade* A: A B2 C2 D2 + +# Cycles with multiple possible solutions. +# (golang.org/issue/34086) +name: cycle3 +M: A1 C2 +A1: B1 +B1: C1 +B2: C2 +C1: +C2: B2 +build M: M A1 B2 C2 +req M: A1 B2 +req M A: A1 B2 +req M C: A1 C2 + +# Requirement minimization. + +name: req1 +A: B1 C1 D1 E1 F1 +B1: C1 E1 F1 +req A: B1 D1 +req A C: B1 C1 D1 + +name: req2 +A: G1 H1 +G1: H1 +H1: G1 +req A: G1 +req A G: G1 +req A H: H1 + +name: req3 +M: A1 B1 +A1: X1 +B1: X2 +X1: I1 +X2: +req M: A1 B1 + +name: reqnone +M: Anone B1 D1 E1 +B1: Cnone D1 +E1: Fnone +build M: M B1 D1 E1 +req M: B1 E1 + +name: reqdup +M: A1 B1 +A1: B1 +B1: +req M A A: A1 + +name: reqcross +M: A1 B1 C1 +A1: B1 C1 +B1: C1 +C1: +req M A B: A1 B1 +` + +func Test(t *testing.T) { + var ( + name string + reqs reqsMap + fns []func(*testing.T) + ) + flush := func() { + if name != "" { + t.Run(name, func(t *testing.T) { + for _, fn := range fns { + fn(t) + } + if len(fns) == 0 { + t.Errorf("no functions tested") + } + }) + } + } + m := func(s string) module.Version { + return module.Version{Path: s[:1], Version: s[1:]} + } + ms := func(list []string) []module.Version { + var mlist []module.Version + for _, s := range list { + mlist = append(mlist, m(s)) + } + return mlist + } + checkList := func(t *testing.T, desc string, list []module.Version, err error, val string) { + if err != nil { + t.Fatalf("%s: %v", desc, err) + } + vs := ms(strings.Fields(val)) + if !reflect.DeepEqual(list, vs) { + t.Errorf("%s = %v, want %v", desc, list, vs) + } + } + + for _, line := range strings.Split(tests, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "#") || line == "" { + continue + } + i := strings.Index(line, ":") + if i < 0 { + t.Fatalf("missing colon: %q", line) + } + key := strings.TrimSpace(line[:i]) + val := strings.TrimSpace(line[i+1:]) + if key == "" { + t.Fatalf("missing key: %q", line) + } + kf := strings.Fields(key) + switch kf[0] { + case "name": + if len(kf) != 1 { + t.Fatalf("name takes no arguments: %q", line) + } + flush() + reqs = make(reqsMap) + fns = nil + name = val + continue + case "build": + if len(kf) != 2 { + t.Fatalf("build takes one argument: %q", line) + } + fns = append(fns, func(t *testing.T) { + list, err := BuildList([]module.Version{m(kf[1])}, reqs) + checkList(t, key, list, err, val) + }) + continue + case "upgrade*": + if len(kf) != 2 { + t.Fatalf("upgrade* takes one argument: %q", line) + } + fns = append(fns, func(t *testing.T) { + list, err := UpgradeAll(m(kf[1]), reqs) + checkList(t, key, list, err, val) + }) + continue + case "upgradereq": + if len(kf) < 2 { + t.Fatalf("upgrade takes at least one argument: %q", line) + } + fns = append(fns, func(t *testing.T) { + list, err := Upgrade(m(kf[1]), reqs, ms(kf[2:])...) + if err == nil { + // Copy the reqs map, but substitute the upgraded requirements in + // place of the target's original requirements. + upReqs := make(reqsMap, len(reqs)) + for m, r := range reqs { + upReqs[m] = r + } + upReqs[m(kf[1])] = list + + list, err = Req(m(kf[1]), nil, upReqs) + } + checkList(t, key, list, err, val) + }) + continue + case "upgrade": + if len(kf) < 2 { + t.Fatalf("upgrade takes at least one argument: %q", line) + } + fns = append(fns, func(t *testing.T) { + list, err := Upgrade(m(kf[1]), reqs, ms(kf[2:])...) + checkList(t, key, list, err, val) + }) + continue + case "downgrade": + if len(kf) < 2 { + t.Fatalf("downgrade takes at least one argument: %q", line) + } + fns = append(fns, func(t *testing.T) { + list, err := Downgrade(m(kf[1]), reqs, ms(kf[1:])...) + checkList(t, key, list, err, val) + }) + continue + case "req": + if len(kf) < 2 { + t.Fatalf("req takes at least one argument: %q", line) + } + fns = append(fns, func(t *testing.T) { + list, err := Req(m(kf[1]), kf[2:], reqs) + checkList(t, key, list, err, val) + }) + continue + } + if len(kf) == 1 && 'A' <= key[0] && key[0] <= 'Z' { + var rs []module.Version + for _, f := range strings.Fields(val) { + r := m(f) + if reqs[r] == nil { + reqs[r] = []module.Version{} + } + rs = append(rs, r) + } + reqs[m(key)] = rs + continue + } + t.Fatalf("bad line: %q", line) + } + flush() +} + +type reqsMap map[module.Version][]module.Version + +func (r reqsMap) Max(_, v1, v2 string) string { + if v1 == "none" || v2 == "" { + return v2 + } + if v2 == "none" || v1 == "" { + return v1 + } + if v1 < v2 { + return v2 + } + return v1 +} + +func (r reqsMap) Upgrade(m module.Version) (module.Version, error) { + u := module.Version{Version: "none"} + for k := range r { + if k.Path == m.Path && r.Max(k.Path, u.Version, k.Version) == k.Version && !strings.HasSuffix(k.Version, ".hidden") { + u = k + } + } + if u.Path == "" { + return module.Version{}, fmt.Errorf("missing module: %v", module.Version{Path: m.Path}) + } + return u, nil +} + +func (r reqsMap) Previous(m module.Version) (module.Version, error) { + var p module.Version + for k := range r { + if k.Path == m.Path && p.Version < k.Version && k.Version < m.Version && !strings.HasSuffix(k.Version, ".hidden") { + p = k + } + } + if p.Path == "" { + return module.Version{Path: m.Path, Version: "none"}, nil + } + return p, nil +} + +func (r reqsMap) Required(m module.Version) ([]module.Version, error) { + rr, ok := r[m] + if !ok { + return nil, fmt.Errorf("missing module: %v", m) + } + return rr, nil +} diff --git a/pkg/client/client.go b/pkg/client/client.go index 7384b78f..e7bb1d4d 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -1255,7 +1255,7 @@ func (c *KpmClient) InitGraphAndDownloadDeps(kclPkg *pkg.KclPkg) (*pkg.Dependenc return nil, nil, err } - changedDeps, err := c.downloadDeps(&kclPkg.ModFile.Dependencies, &kclPkg.Dependencies, depGraph, kclPkg.HomePath, root) + changedDeps, err := c.downloadDeps(kclPkg.ModFile.Dependencies, kclPkg.Dependencies, depGraph, kclPkg.HomePath, root) if err != nil { return nil, nil, err } @@ -1299,7 +1299,7 @@ func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencie return nil, errors.InvalidDependency } - existDep := c.dependencyExists(&d, lockDeps) + existDep := c.dependencyExists(&d, &lockDeps) if existDep != nil { newDeps.Deps[d.Name] = *existDep continue @@ -1366,7 +1366,7 @@ func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencie source := module.Version{Path: d.Name, Version: d.Version} - err = depGraph.AddVertex(source) + err = depGraph.AddVertex(source, graph.VertexAttribute(d.GetSourceType(), d.GetDownloadPath())) if err != nil && err != graph.ErrVertexAlreadyExists { return nil, err } @@ -1384,7 +1384,7 @@ func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencie } // Download the indirect dependencies. - nested, err := c.downloadDeps(&deppkg.ModFile.Dependencies, lockDeps, depGraph, deppkg.HomePath, source) + nested, err := c.downloadDeps(deppkg.ModFile.Dependencies, lockDeps, depGraph, deppkg.HomePath, source) if err != nil { return nil, err } diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 9f38ae1f..81a055b3 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -184,7 +184,7 @@ func TestDependencyGraph(t *testing.T) { assert.Equal(t, err, nil) m := func(Path, Version string) module.Version { - return module.Version{Path, Version} + return module.Version{Path: Path, Version: Version} } edgeProp := graph.EdgeProperties{ diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go index 5ae56535..5ec109c4 100644 --- a/pkg/errors/errors.go +++ b/pkg/errors/errors.go @@ -24,6 +24,9 @@ var InvalidAddOptionsInvalidOciRef = errors.New("invalid 'kpm add' argument, you var InvalidAddOptionsInvalidOciReg = errors.New("invalid 'kpm add' argument, you must provide a Reg for the package.") var InvalidAddOptionsInvalidOciRepo = errors.New("invalid 'kpm add' argument, you must provide a Repo for the package.") +// Invalid 'kpm update' +var MultipleSources = errors.New("multiple sources found, there must be a single source.") + // Invalid 'kpm run' var InvalidRunOptionsWithoutEntryFiles = errors.New("invalid 'kpm run' argument, you must provide an entry file.") var EntryFileNotFound = errors.New("entry file cannot be found, please make sure the '--input' entry file can be found") diff --git a/pkg/git/git.go b/pkg/git/git.go index 0af8db18..667b4338 100644 --- a/pkg/git/git.go +++ b/pkg/git/git.go @@ -1,12 +1,18 @@ package git import ( + "encoding/json" "errors" + "fmt" "io" + "net/http" + "regexp" + "time" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/hashicorp/go-getter" + giturl "github.com/kubescape/go-git-url" ) // CloneOptions is a struct for specifying options for cloning a git repository @@ -153,3 +159,82 @@ func Clone(repoURL string, tagName string, localPath string, writer io.Writer) ( }) return repo, err } + +type GitHubRelease struct { + TagName string `json:"tag_name"` +} + +// parseNextPageURL extracts the 'next' page URL from the 'Link' header +func parseNextPageURL(linkHeader string) (string, error) { + // Regex to extract 'next' page URL from the link header + r := regexp.MustCompile(`<([^>]+)>;\s*rel="next"`) + matches := r.FindStringSubmatch(linkHeader) + + if len(matches) < 2 { + return "", errors.New("next page URL not found") + } + return matches[1], nil +} + +// GetAllGithubReleases fetches all releases from a GitHub repository +func GetAllGithubReleases(url string) ([]string, error) { + // Initialize and parse the URL to extract owner and repo names + gitURL, err := giturl.NewGitURL(url) + if err != nil { + return nil, err + } + + // Construct initial API URL for the first page + apiBase := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases", gitURL.GetOwnerName(), gitURL.GetRepoName()) + apiURL := fmt.Sprintf("%s?per_page=100&page=1", apiBase) + + client := http.Client{ + Timeout: 10 * time.Second, + } + + var releaseTags []string + + for apiURL != "" { + req, err := http.NewRequest("GET", apiURL, nil) + if err != nil { + return nil, err + } + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to fetch tags, status code: %d", resp.StatusCode) + } + + // Decode the JSON response into a slice of releases + var releases []GitHubRelease + if err := json.NewDecoder(resp.Body).Decode(&releases); err != nil { + return nil, err + } + + // Extract tag names from the releases + for _, release := range releases { + releaseTags = append(releaseTags, release.TagName) + } + + // Read the `Link` header to get the next page URL, if available + linkHeader := resp.Header.Get("Link") + if linkHeader != "" { + nextURL, err := parseNextPageURL(linkHeader) + if err != nil { + apiURL = "" + } else { + apiURL = nextURL + } + } else { + apiURL = "" + } + fmt.Println(apiURL) + } + + return releaseTags, nil +} diff --git a/pkg/graph/graph.go b/pkg/graph/graph.go index cb0fca1f..e22d40e2 100644 --- a/pkg/graph/graph.go +++ b/pkg/graph/graph.go @@ -5,20 +5,39 @@ import ( "github.com/dominikbraun/graph" "golang.org/x/mod/module" + pkg "kcl-lang.io/kpm/pkg/package" ) -// ToAdjacencyList converts graph to adjacency list -func ToAdjacencyList(g graph.Graph[module.Version, module.Version]) (map[module.Version][]module.Version, error) { +func ChangeGraphType(g graph.Graph[pkg.Dependency, pkg.Dependency]) (graph.Graph[module.Version, module.Version], error) { AdjacencyMap, err := g.AdjacencyMap() if err != nil { return nil, fmt.Errorf("failed to get adjacency map: %w", err) } - adjList := make(map[module.Version][]module.Version) - for from, v := range AdjacencyMap { - for to := range v { - adjList[from] = append(adjList[from], to) + m := func(dep pkg.Dependency) module.Version { + return module.Version{Path: dep.Name, Version: dep.Version} + } + + moduleHash := func(m module.Version) module.Version { + return m + } + + depGraph := graph.New(moduleHash, graph.Directed(), graph.PreventCycles()) + for node, edges := range AdjacencyMap { + err := depGraph.AddVertex(m(node)) + if err != nil && err != graph.ErrVertexAlreadyExists { + return nil, fmt.Errorf("failed to add vertex: %w", err) + } + for edge := range edges { + err := depGraph.AddVertex(m(edge)) + if err != nil && err != graph.ErrVertexAlreadyExists { + return nil, fmt.Errorf("failed to add vertex: %w", err) + } + err = depGraph.AddEdge(m(node), m(edge)) + if err != nil { + return nil, fmt.Errorf("failed to add edge: %w", err) + } } } - return adjList, nil + return depGraph, nil } diff --git a/pkg/mvs/mvs.go b/pkg/mvs/mvs.go index 291b5213..be89bdec 100644 --- a/pkg/mvs/mvs.go +++ b/pkg/mvs/mvs.go @@ -3,36 +3,114 @@ package mvs import ( "fmt" + "github.com/dominikbraun/graph" + "github.com/hashicorp/go-version" "golang.org/x/mod/module" + "kcl-lang.io/kpm/pkg/errors" + "kcl-lang.io/kpm/pkg/git" + "kcl-lang.io/kpm/pkg/oci" + pkg "kcl-lang.io/kpm/pkg/package" + "kcl-lang.io/kpm/pkg/reporter" + "kcl-lang.io/kpm/pkg/semver" ) -type ReqsMap map[module.Version][]module.Version +type ReqsGraph struct { + graph.Graph[module.Version, module.Version] +} -func (r ReqsMap) Max(_, v1, v2 string) string { - if v1 == "none" || v2 == "" { - return v2 +func (r ReqsGraph) Max(_, v1, v2 string) string { + version1, err := version.NewVersion(v1) + if err != nil { + reporter.NewErrorEvent(reporter.FailedParseVersion, err, fmt.Sprintf("failed to parse version %s", v1)) + return "" + } + version2, err := version.NewVersion(v2) + if err != nil { + reporter.NewErrorEvent(reporter.FailedParseVersion, err, fmt.Sprintf("failed to parse version %s", v2)) + return "" } - if v2 == "none" || v1 == "" { + if version1.GreaterThan(version2) { return v1 } - if v1 < v2 { - return v2 + return v2 +} + +func (r ReqsGraph) Upgrade(m module.Version) (module.Version, error) { + _, properties, err := r.VertexWithProperties(m) + if err != nil { + return module.Version{}, err + } + + releases, err := getReleasesFromSource(properties) + if err != nil { + return module.Version{}, err + } + + if releases == nil { + return m, nil + } + + m.Version, err = semver.LatestVersion(releases) + if err != nil { + return module.Version{}, err } - return v1 + return m, nil } -func (r ReqsMap) Upgrade(m module.Version) (module.Version, error) { - panic("unimplemented") +func (r ReqsGraph) Previous(m module.Version) (module.Version, error) { + _, properties, err := r.VertexWithProperties(m) + if err != nil { + return module.Version{}, err + } + + releases, err := getReleasesFromSource(properties) + if err != nil { + return module.Version{}, err + } + + if releases == nil { + return m, nil + } + + m.Version, err = semver.OldestVersion(releases) + if err != nil { + return module.Version{}, err + } + return m, nil } -func (r ReqsMap) Previous(m module.Version) (module.Version, error) { - panic("unimplemented") +func (r ReqsGraph) Required(m module.Version) ([]module.Version, error) { + adjMap, err := r.AdjacencyMap() + if err != nil { + return nil, err + } + var reqs []module.Version + for v := range adjMap[m] { + reqs = append(reqs, v) + } + return reqs, nil } -func (r ReqsMap) Required(m module.Version) ([]module.Version, error) { - rr, ok := r[m] - if !ok { - return nil, fmt.Errorf("missing module: %v", m) +func getReleasesFromSource(properties graph.VertexProperties) ([]string, error) { + var releases []string + var err error + + // there must be only one property depending on the download source type + if len(properties.Attributes) != 1 { + return nil, errors.MultipleSources } - return rr, nil + + for k, v := range properties.Attributes { + switch k { + case pkg.GIT: + releases, err = git.GetAllGithubReleases(v) + case pkg.OCI: + releases, err = oci.GetAllImageTags(v) + } + if err != nil { + return nil, err + } + } + + return releases, nil } diff --git a/pkg/mvs/mvs_test.go b/pkg/mvs/mvs_test.go new file mode 100644 index 00000000..b9cb8006 --- /dev/null +++ b/pkg/mvs/mvs_test.go @@ -0,0 +1,192 @@ +package mvs + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "golang.org/x/mod/module" + "kcl-lang.io/kpm/pkg/3rdparty/mvs" + "kcl-lang.io/kpm/pkg/client" + "kcl-lang.io/kpm/pkg/utils" +) + +const testDataDir = "test_data" + +func getTestDir(subDir string) string { + pwd, _ := os.Getwd() + testDir := filepath.Join(pwd, testDataDir) + testDir = filepath.Join(testDir, subDir) + + return testDir +} + +func TestMax(t *testing.T) { + reqs := ReqsGraph{} + assert.Equal(t, reqs.Max("", "1.0.0", "2.0.0"), "2.0.0") + assert.Equal(t, reqs.Max("", "1.2", "2.0"), "2.0") + assert.Equal(t, reqs.Max("", "2.5.0", "2.6"), "2.6") + assert.Equal(t, reqs.Max("", "2.0.0", "v3.0"), "v3.0") +} + +func TestRequired(t *testing.T) { + pkg_path := filepath.Join(getTestDir("test_with_internal_deps"), "aaa") + assert.Equal(t, utils.DirExists(filepath.Join(pkg_path, "kcl.mod")), true) + kpmcli, err := client.NewKpmClient() + assert.Equal(t, err, nil) + kclPkg, err := kpmcli.LoadPkgFromPath(pkg_path) + assert.Equal(t, err, nil) + + _, depGraph, err := kpmcli.InitGraphAndDownloadDeps(kclPkg) + assert.Equal(t, err, nil) + + reqs := ReqsGraph{ + depGraph, + } + + req, err := reqs.Required(module.Version{Path: "aaa", Version: "0.0.1"}) + assert.Equal(t, err, nil) + assert.Equal(t, len(req), 2) + + expectedReqs := []module.Version{ + {Path: "bbb", Version: "0.0.1"}, + {Path: "ccc", Version: "0.0.1"}, + } + assert.Equal(t, req, expectedReqs) +} + +func TestMinBuildList(t *testing.T) { + pkg_path := getTestDir("test_with_external_deps") + assert.Equal(t, utils.DirExists(filepath.Join(pkg_path, "kcl.mod")), true) + kpmcli, err := client.NewKpmClient() + assert.Equal(t, err, nil) + kclPkg, err := kpmcli.LoadPkgFromPath(pkg_path) + assert.Equal(t, err, nil) + + _, depGraph, err := kpmcli.InitGraphAndDownloadDeps(kclPkg) + assert.Equal(t, err, nil) + + reqs := ReqsGraph{ + depGraph, + } + + target := module.Version{Path: kclPkg.GetPkgName(), Version: kclPkg.GetPkgVersion()} + + req, err := mvs.BuildList([]module.Version{target}, reqs) + assert.Equal(t, err, nil) + + expectedReqs := []module.Version{ + {Path: "test_with_external_deps", Version: "0.0.1"}, + {Path: "argo-cd-order", Version: "0.1.2"}, + {Path: "helloworld", Version: "0.1.0"}, + {Path: "json_merge_patch", Version: "0.1.0"}, + {Path: "k8s", Version: "1.29"}, + {Path: "podinfo", Version: "0.1.1"}, + } + assert.Equal(t, req, expectedReqs) + + base := []string{target.Path} + for depName := range kclPkg.Dependencies.Deps { + base = append(base, depName) + } + req, err = mvs.Req(target, base, reqs) + assert.Equal(t, err, nil) + + expectedReqs = []module.Version{ + {Path: "argo-cd-order", Version: "0.1.2"}, + {Path: "helloworld", Version: "0.1.0"}, + {Path: "json_merge_patch", Version: "0.1.0"}, + {Path: "k8s", Version: "1.29"}, + {Path: "podinfo", Version: "0.1.1"}, + {Path: "test_with_external_deps", Version: "0.0.1"}, + } + assert.Equal(t, req, expectedReqs) +} + +func TestUpgradeToLatest(t *testing.T) { + pkg_path := getTestDir("test_with_external_deps") + assert.Equal(t, utils.DirExists(filepath.Join(pkg_path, "kcl.mod")), true) + kpmcli, err := client.NewKpmClient() + assert.Equal(t, err, nil) + kclPkg, err := kpmcli.LoadPkgFromPath(pkg_path) + assert.Equal(t, err, nil) + + _, depGraph, err := kpmcli.InitGraphAndDownloadDeps(kclPkg) + assert.Equal(t, err, nil) + + reqs := ReqsGraph{ + depGraph, + } + + upgrade, err := reqs.Upgrade(module.Version{Path: "k8s", Version: "1.27"}) + assert.Equal(t, err, nil) + assert.Equal(t, upgrade, module.Version{Path: "k8s", Version: "1.29"}) +} + +func TestUpgradeAllToLatest(t *testing.T) { + pkg_path := getTestDir("test_with_external_deps") + assert.Equal(t, utils.DirExists(filepath.Join(pkg_path, "kcl.mod")), true) + kpmcli, err := client.NewKpmClient() + assert.Equal(t, err, nil) + kclPkg, err := kpmcli.LoadPkgFromPath(pkg_path) + assert.Equal(t, err, nil) + + _, depGraph, err := kpmcli.InitGraphAndDownloadDeps(kclPkg) + assert.Equal(t, err, nil) + + reqs := ReqsGraph{ + depGraph, + } + + target := module.Version{Path: kclPkg.GetPkgName(), Version: kclPkg.GetPkgVersion()} + fmt.Println(target) + upgrade, err := mvs.UpgradeAll(target, reqs) + assert.Equal(t, err, nil) + assert.Equal(t, upgrade, module.Version{Path: "k8s", Version: "1.29"}) +} + +func TestPrevious(t *testing.T) { + pkg_path := getTestDir("test_with_external_deps") + assert.Equal(t, utils.DirExists(filepath.Join(pkg_path, "kcl.mod")), true) + kpmcli, err := client.NewKpmClient() + assert.Equal(t, err, nil) + kclPkg, err := kpmcli.LoadPkgFromPath(pkg_path) + assert.Equal(t, err, nil) + + _, depGraph, err := kpmcli.InitGraphAndDownloadDeps(kclPkg) + assert.Equal(t, err, nil) + + reqs := ReqsGraph{ + depGraph, + } + + downgrade, err := reqs.Previous(module.Version{Path: "k8s", Version: "1.27"}) + assert.Equal(t, err, nil) + assert.Equal(t, downgrade, module.Version{Path: "k8s", Version: "1.14"}) +} + +func TestUpgradePreviousOfLocalDependency(t *testing.T) { + pkg_path := filepath.Join(getTestDir("test_with_internal_deps"), "aaa") + assert.Equal(t, utils.DirExists(filepath.Join(pkg_path, "kcl.mod")), true) + kpmcli, err := client.NewKpmClient() + assert.Equal(t, err, nil) + kclPkg, err := kpmcli.LoadPkgFromPath(pkg_path) + assert.Equal(t, err, nil) + + _, depGraph, err := kpmcli.InitGraphAndDownloadDeps(kclPkg) + assert.Equal(t, err, nil) + + reqs := ReqsGraph{ + depGraph, + } + + upgrade, err := reqs.Upgrade(module.Version{Path: "bbb", Version: "0.0.1"}) + assert.Equal(t, err, nil) + assert.Equal(t, upgrade, module.Version{Path: "bbb", Version: "0.0.1"}) + + downgrade, err := reqs.Previous(module.Version{Path: "bbb", Version: "0.0.1"}) + assert.Equal(t, err, nil) + assert.Equal(t, downgrade, module.Version{Path: "bbb", Version: "0.0.1"}) +} diff --git a/pkg/mvs/test_data/test_with_external_deps/kcl.mod b/pkg/mvs/test_data/test_with_external_deps/kcl.mod new file mode 100644 index 00000000..67278fa8 --- /dev/null +++ b/pkg/mvs/test_data/test_with_external_deps/kcl.mod @@ -0,0 +1,10 @@ +[package] +name = "test_with_external_deps" +edition = "0.0.1" +version = "0.0.1" + +[dependencies] +k8s = "1.27" +helloworld = "0.1.0" +argo-cd-order = "0.1.2" +podinfo = "0.1.1" \ No newline at end of file diff --git a/pkg/mvs/test_data/test_with_internal_deps/aaa/kcl.mod b/pkg/mvs/test_data/test_with_internal_deps/aaa/kcl.mod new file mode 100644 index 00000000..fdd2927f --- /dev/null +++ b/pkg/mvs/test_data/test_with_internal_deps/aaa/kcl.mod @@ -0,0 +1,8 @@ +[package] +name = "aaa" +edition = "0.0.1" +version = "0.0.1" + +[dependencies] +bbb = { path = "../bbb" } +ccc = { path = "../ccc" } diff --git a/pkg/mvs/test_data/test_with_internal_deps/bbb/kcl.mod b/pkg/mvs/test_data/test_with_internal_deps/bbb/kcl.mod new file mode 100644 index 00000000..e9ea10a5 --- /dev/null +++ b/pkg/mvs/test_data/test_with_internal_deps/bbb/kcl.mod @@ -0,0 +1,5 @@ +[package] +name = "bbb" +edition = "0.0.1" +version = "0.0.1" + diff --git a/pkg/mvs/test_data/test_with_internal_deps/ccc/kcl.mod b/pkg/mvs/test_data/test_with_internal_deps/ccc/kcl.mod new file mode 100644 index 00000000..9a762a4f --- /dev/null +++ b/pkg/mvs/test_data/test_with_internal_deps/ccc/kcl.mod @@ -0,0 +1,5 @@ +[package] +name = "ccc" +edition = "0.0.1" +version = "0.0.1" + diff --git a/pkg/oci/oci.go b/pkg/oci/oci.go index 989ac88c..c51da2bb 100644 --- a/pkg/oci/oci.go +++ b/pkg/oci/oci.go @@ -4,9 +4,13 @@ import ( "context" "fmt" "io" + "log" "os" "path/filepath" + "strings" + "github.com/containers/image/docker" + "github.com/containers/image/types" v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/thoas/go-funk" "oras.land/oras-go/pkg/auth" @@ -385,3 +389,17 @@ func GenOciManifestFromPkg(kclPkg *pkg.KclPkg) (map[string]string, error) { res[constants.DEFAULT_KCL_OCI_MANIFEST_SUM] = sum return res, nil } + +func GetAllImageTags(imageName string) ([]string, error) { + sysCtx := &types.SystemContext{} + ref, err := docker.ParseReference("//" + strings.TrimPrefix(imageName, "oci://")) + if err != nil { + log.Fatalf("Error parsing reference: %v", err) + } + + tags, err := docker.GetRepositoryTags(context.Background(), sysCtx, ref) + if err != nil { + log.Fatalf("Error getting tags: %v", err) + } + return tags, nil +} diff --git a/pkg/package/modfile.go b/pkg/package/modfile.go index 65657ed8..55e6cb2a 100644 --- a/pkg/package/modfile.go +++ b/pkg/package/modfile.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/BurntSushi/toml" + "kcl-lang.io/kcl-go/pkg/kcl" "oras.land/oras-go/v2/registry" @@ -24,6 +25,9 @@ import ( const ( MOD_FILE = "kcl.mod" MOD_LOCK_FILE = "kcl.mod.lock" + GIT = "git" + OCI = "oci" + LOCAL = "local" ) // 'Package' is the kcl package section of 'kcl.mod'. @@ -244,6 +248,31 @@ func (dep *Dependency) GenDepFullName() string { return dep.FullName } +// GetDownloadPath will get the download path of a dependency. +func (dep *Dependency) GetDownloadPath() string { + if dep.Source.Git != nil { + return dep.Source.Git.Url + } + if dep.Source.Oci != nil { + return dep.Source.Oci.IntoOciUrl() + } + return "" +} + +// GetSourceType will get the source type of a dependency. +func (dep *Dependency) GetSourceType() string { + if dep.Source.Git != nil { + return GIT + } + if dep.Source.Oci != nil { + return OCI + } + if dep.Source.Local != nil { + return LOCAL + } + return "" +} + // Source is the package source from registry. type Source struct { *Git diff --git a/pkg/package/package.go b/pkg/package/package.go index 28401265..91dd547d 100644 --- a/pkg/package/package.go +++ b/pkg/package/package.go @@ -6,8 +6,6 @@ import ( "path/filepath" "strings" - "golang.org/x/mod/module" - "kcl-lang.io/kcl-go/pkg/kcl" "kcl-lang.io/kpm/pkg/constants" errors "kcl-lang.io/kpm/pkg/errors" @@ -22,8 +20,6 @@ type KclPkg struct { // The dependencies in the current kcl package are the dependencies of kcl.mod.lock, // not the dependencies in kcl.mod. Dependencies - // BuildList denotes the minimal build list for the current kcl package. - BuildList []module.Version // The flag 'NoSumCheck' is true if the checksum of the current kcl package is not checked. NoSumCheck bool } diff --git a/pkg/semver/semver.go b/pkg/semver/semver.go index 450bf8f9..53e0dd21 100644 --- a/pkg/semver/semver.go +++ b/pkg/semver/semver.go @@ -26,3 +26,22 @@ func LatestVersion(versions []string) (string, error) { return latest.Original(), nil } + +func OldestVersion(versions []string) (string, error) { + var oldest *version.Version + for _, v := range versions { + ver, err := version.NewVersion(v) + if err != nil { + return "", reporter.NewErrorEvent(reporter.FailedParseVersion, err, fmt.Sprintf("failed to parse version %s", v)) + } + if oldest == nil || ver.LessThan(oldest) { + oldest = ver + } + } + + if oldest == nil { + return "", errors.InvalidVersionFormat + } + + return oldest.Original(), nil +} \ No newline at end of file diff --git a/pkg/semver/semver_test.go b/pkg/semver/semver_test.go index 7e211eea..560b4d99 100644 --- a/pkg/semver/semver_test.go +++ b/pkg/semver/semver_test.go @@ -38,3 +38,35 @@ func TestTheLatestTagWithMissingVersion(t *testing.T) { assert.Equal(t, err, nil) assert.Equal(t, latest, "5.5") } + +func TestOldestVersion(t *testing.T) { + oldest, err := OldestVersion([]string{"1.2.3", "1.4.0", "1.3.5", "1.0.0"}) + assert.Equal(t, err, nil) + assert.Equal(t, oldest, "1.0.0") + + oldest, err = OldestVersion([]string{}) + assert.Equal(t, err, errors.InvalidVersionFormat) + assert.Equal(t, oldest, "") + + oldest, err = OldestVersion([]string{"invalid_version"}) + assert.Equal(t, err.Error(), "failed to parse version invalid_version\nMalformed version: invalid_version\n") + assert.Equal(t, oldest, "") + + oldest, err = OldestVersion([]string{"1.2.3", "1.4.0", "1.3.5", "invalid_version"}) + assert.Equal(t, err.Error(), "failed to parse version invalid_version\nMalformed version: invalid_version\n") + assert.Equal(t, oldest, "") +} + +func TestOldestVersionWithVariousFormats(t *testing.T) { + oldest, err := OldestVersion([]string{"2.2", "2.4.5", "2.3.9", "2.1.0", "2.0"}) + assert.Equal(t, err, nil) + assert.Equal(t, oldest, "2.0") + + oldest, err = OldestVersion([]string{"0.1", "0.1.1", "0.1.2-beta", "0.0.9"}) + assert.Equal(t, err, nil) + assert.Equal(t, oldest, "0.0.9") + + oldest, err = OldestVersion([]string{"3.3.3", "3.2", "3.1", "3.0.0"}) + assert.Equal(t, err, nil) + assert.Equal(t, oldest, "3.0.0") +} From cfd80ed3d881296ced139f2bc4017e0f39b85e6f Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Sun, 12 May 2024 23:46:07 +0530 Subject: [PATCH 06/17] add tests Signed-off-by: Akash Kumar --- pkg/3rdparty/mvs/mvs_test.go | 2 +- pkg/client/client.go | 26 +++++----- pkg/mvs/mvs.go | 54 ++++++++++++++++++++- pkg/mvs/mvs_test.go | 93 ++++++++++++++++++++++++++---------- pkg/package/modfile.go | 20 ++++++++ pkg/package/modfile_test.go | 13 +++++ pkg/semver/semver.go | 58 +++++++++++++++++++--- pkg/semver/semver_test.go | 40 +++++++--------- 8 files changed, 235 insertions(+), 71 deletions(-) diff --git a/pkg/3rdparty/mvs/mvs_test.go b/pkg/3rdparty/mvs/mvs_test.go index b18ee2de..92b6c837 100644 --- a/pkg/3rdparty/mvs/mvs_test.go +++ b/pkg/3rdparty/mvs/mvs_test.go @@ -212,7 +212,7 @@ upgrade* A: A B5.hidden C3 name: down1 A: B2 -B1: C1 +B1: C2 B2: C2 build A: A B2 C2 downgrade A C1: A B1 C1 diff --git a/pkg/client/client.go b/pkg/client/client.go index e7bb1d4d..9655c727 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -1255,7 +1255,7 @@ func (c *KpmClient) InitGraphAndDownloadDeps(kclPkg *pkg.KclPkg) (*pkg.Dependenc return nil, nil, err } - changedDeps, err := c.downloadDeps(kclPkg.ModFile.Dependencies, kclPkg.Dependencies, depGraph, kclPkg.HomePath, root) + changedDeps, err := c.DownloadDeps(kclPkg.ModFile.Dependencies, kclPkg.Dependencies, depGraph, kclPkg.HomePath, root) if err != nil { return nil, nil, err } @@ -1287,7 +1287,7 @@ func (c *KpmClient) dependencyExists(dep *pkg.Dependency, lockDeps *pkg.Dependen } // downloadDeps will download all the dependencies of the current kcl package. -func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencies, depGraph graph.Graph[module.Version, module.Version], pkghome string, parent module.Version) (*pkg.Dependencies, error) { +func (c *KpmClient) DownloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencies, depGraph graph.Graph[module.Version, module.Version], pkghome string, parent module.Version) (*pkg.Dependencies, error) { newDeps := pkg.Dependencies{ Deps: make(map[string]pkg.Dependency), @@ -1371,20 +1371,22 @@ func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencie return nil, err } - err = depGraph.AddEdge(parent, source) - if err != nil { - if err == graph.ErrEdgeCreatesCycle { - return nil, reporter.NewErrorEvent( - reporter.CircularDependencyExist, - nil, - fmt.Sprintf("adding %s as a dependency results in a cycle", source), - ) + if parent != (module.Version{}) { + err = depGraph.AddEdge(parent, source) + if err != nil { + if err == graph.ErrEdgeCreatesCycle { + return nil, reporter.NewErrorEvent( + reporter.CircularDependencyExist, + nil, + fmt.Sprintf("adding %s as a dependency results in a cycle", source), + ) + } + return nil, err } - return nil, err } // Download the indirect dependencies. - nested, err := c.downloadDeps(deppkg.ModFile.Dependencies, lockDeps, depGraph, deppkg.HomePath, source) + nested, err := c.DownloadDeps(deppkg.ModFile.Dependencies, lockDeps, depGraph, deppkg.HomePath, source) if err != nil { return nil, err } diff --git a/pkg/mvs/mvs.go b/pkg/mvs/mvs.go index be89bdec..307cfb67 100644 --- a/pkg/mvs/mvs.go +++ b/pkg/mvs/mvs.go @@ -6,6 +6,7 @@ import ( "github.com/dominikbraun/graph" "github.com/hashicorp/go-version" "golang.org/x/mod/module" + "kcl-lang.io/kpm/pkg/client" "kcl-lang.io/kpm/pkg/errors" "kcl-lang.io/kpm/pkg/git" "kcl-lang.io/kpm/pkg/oci" @@ -16,6 +17,8 @@ import ( type ReqsGraph struct { graph.Graph[module.Version, module.Version] + kpmClient *client.KpmClient + kpmPkg *pkg.KclPkg } func (r ReqsGraph) Max(_, v1, v2 string) string { @@ -50,10 +53,33 @@ func (r ReqsGraph) Upgrade(m module.Version) (module.Version, error) { return m, nil } - m.Version, err = semver.LatestVersion(releases) + m.Version, err = semver.LatestCompatibleVersion(releases, m.Version) if err != nil { return module.Version{}, err } + _, err = r.Vertex(m) + if err == graph.ErrVertexNotFound { + d := pkg.Dependency{ + Name: m.Path, + Version: m.Version, + } + d.FullName = d.GenDepFullName() + for sourceType, uri := range properties.Attributes { + d.Source, err = pkg.GenSource(sourceType, uri, m.Version) + if err != nil { + return module.Version{}, err + } + } + deps := pkg.Dependencies{ + Deps: map[string]pkg.Dependency{ + m.Path: d, + }, + } + lockDeps := pkg.Dependencies{ + Deps: make(map[string]pkg.Dependency), + } + r.kpmClient.DownloadDeps(deps, lockDeps, r.Graph, r.kpmPkg.HomePath, module.Version{}) + } return m, nil } @@ -72,10 +98,34 @@ func (r ReqsGraph) Previous(m module.Version) (module.Version, error) { return m, nil } - m.Version, err = semver.OldestVersion(releases) + m.Version, err = semver.LeastOldCompatibleVersion(releases, m.Version) if err != nil { return module.Version{}, err } + + _, err = r.Vertex(m) + if err == graph.ErrVertexNotFound { + d := pkg.Dependency{ + Name: m.Path, + Version: m.Version, + } + d.FullName = d.GenDepFullName() + for sourceType, uri := range properties.Attributes { + d.Source, err = pkg.GenSource(sourceType, uri, m.Version) + if err != nil { + return module.Version{}, err + } + } + deps := pkg.Dependencies{ + Deps: map[string]pkg.Dependency{ + m.Path: d, + }, + } + lockDeps := pkg.Dependencies{ + Deps: make(map[string]pkg.Dependency), + } + r.kpmClient.DownloadDeps(deps, lockDeps, r.Graph, r.kpmPkg.HomePath, module.Version{}) + } return m, nil } diff --git a/pkg/mvs/mvs_test.go b/pkg/mvs/mvs_test.go index b9cb8006..3dce50cb 100644 --- a/pkg/mvs/mvs_test.go +++ b/pkg/mvs/mvs_test.go @@ -1,7 +1,6 @@ package mvs import ( - "fmt" "os" "path/filepath" "testing" @@ -44,6 +43,8 @@ func TestRequired(t *testing.T) { reqs := ReqsGraph{ depGraph, + kpmcli, + kclPkg, } req, err := reqs.Required(module.Version{Path: "aaa", Version: "0.0.1"}) @@ -57,7 +58,7 @@ func TestRequired(t *testing.T) { assert.Equal(t, req, expectedReqs) } -func TestMinBuildList(t *testing.T) { +func TestUpgrade(t *testing.T) { pkg_path := getTestDir("test_with_external_deps") assert.Equal(t, utils.DirExists(filepath.Join(pkg_path, "kcl.mod")), true) kpmcli, err := client.NewKpmClient() @@ -70,39 +71,27 @@ func TestMinBuildList(t *testing.T) { reqs := ReqsGraph{ depGraph, + kpmcli, + kclPkg, } target := module.Version{Path: kclPkg.GetPkgName(), Version: kclPkg.GetPkgVersion()} - - req, err := mvs.BuildList([]module.Version{target}, reqs) + upgradeList := []module.Version{ + {Path: "argo-cd-order", Version: "0.2.0"}, + {Path: "helloworld", Version: "0.1.1"}, + } + upgrade, err := mvs.Upgrade(target, reqs, upgradeList...) assert.Equal(t, err, nil) expectedReqs := []module.Version{ {Path: "test_with_external_deps", Version: "0.0.1"}, - {Path: "argo-cd-order", Version: "0.1.2"}, - {Path: "helloworld", Version: "0.1.0"}, - {Path: "json_merge_patch", Version: "0.1.0"}, - {Path: "k8s", Version: "1.29"}, - {Path: "podinfo", Version: "0.1.1"}, - } - assert.Equal(t, req, expectedReqs) - - base := []string{target.Path} - for depName := range kclPkg.Dependencies.Deps { - base = append(base, depName) - } - req, err = mvs.Req(target, base, reqs) - assert.Equal(t, err, nil) - - expectedReqs = []module.Version{ - {Path: "argo-cd-order", Version: "0.1.2"}, - {Path: "helloworld", Version: "0.1.0"}, + {Path: "argo-cd-order", Version: "0.2.0"}, + {Path: "helloworld", Version: "0.1.1"}, {Path: "json_merge_patch", Version: "0.1.0"}, {Path: "k8s", Version: "1.29"}, {Path: "podinfo", Version: "0.1.1"}, - {Path: "test_with_external_deps", Version: "0.0.1"}, } - assert.Equal(t, req, expectedReqs) + assert.Equal(t, upgrade, expectedReqs) } func TestUpgradeToLatest(t *testing.T) { @@ -118,6 +107,8 @@ func TestUpgradeToLatest(t *testing.T) { reqs := ReqsGraph{ depGraph, + kpmcli, + kclPkg, } upgrade, err := reqs.Upgrade(module.Version{Path: "k8s", Version: "1.27"}) @@ -138,13 +129,24 @@ func TestUpgradeAllToLatest(t *testing.T) { reqs := ReqsGraph{ depGraph, + kpmcli, + kclPkg, } target := module.Version{Path: kclPkg.GetPkgName(), Version: kclPkg.GetPkgVersion()} - fmt.Println(target) + upgrade, err := mvs.UpgradeAll(target, reqs) assert.Equal(t, err, nil) - assert.Equal(t, upgrade, module.Version{Path: "k8s", Version: "1.29"}) + + expectedReqs := []module.Version{ + {Path: "test_with_external_deps", Version: "0.0.1"}, + {Path: "argo-cd-order", Version: "0.2.0"}, + {Path: "helloworld", Version: "0.1.2"}, + {Path: "json_merge_patch", Version: "0.1.1"}, + {Path: "k8s", Version: "1.29"}, + {Path: "podinfo", Version: "0.1.1"}, + } + assert.Equal(t, upgrade, expectedReqs) } func TestPrevious(t *testing.T) { @@ -160,6 +162,8 @@ func TestPrevious(t *testing.T) { reqs := ReqsGraph{ depGraph, + kpmcli, + kclPkg, } downgrade, err := reqs.Previous(module.Version{Path: "k8s", Version: "1.27"}) @@ -180,6 +184,8 @@ func TestUpgradePreviousOfLocalDependency(t *testing.T) { reqs := ReqsGraph{ depGraph, + kpmcli, + kclPkg, } upgrade, err := reqs.Upgrade(module.Version{Path: "bbb", Version: "0.0.1"}) @@ -190,3 +196,38 @@ func TestUpgradePreviousOfLocalDependency(t *testing.T) { assert.Equal(t, err, nil) assert.Equal(t, downgrade, module.Version{Path: "bbb", Version: "0.0.1"}) } + +func TestDowngrade(t *testing.T) { + pkg_path := getTestDir("test_with_external_deps") + assert.Equal(t, utils.DirExists(filepath.Join(pkg_path, "kcl.mod")), true) + kpmcli, err := client.NewKpmClient() + assert.Equal(t, err, nil) + kclPkg, err := kpmcli.LoadPkgFromPath(pkg_path) + assert.Equal(t, err, nil) + + _, depGraph, err := kpmcli.InitGraphAndDownloadDeps(kclPkg) + assert.Equal(t, err, nil) + + reqs := ReqsGraph{ + depGraph, + kpmcli, + kclPkg, + } + + target := module.Version{Path: kclPkg.GetPkgName(), Version: kclPkg.GetPkgVersion()} + downgradeList := []module.Version{ + {Path: "k8s", Version: "1.17"}, + } + downgrade, err := mvs.Downgrade(target, reqs, downgradeList...) + assert.Equal(t, err, nil) + + expectedReqs := []module.Version{ + {Path: "test_with_external_deps", Version: "0.0.1"}, + {Path: "argo-cd-order", Version: "0.2.0"}, + {Path: "helloworld", Version: "0.1.1"}, + {Path: "json_merge_patch", Version: "0.1.0"}, + {Path: "k8s", Version: "1.27"}, + {Path: "podinfo", Version: "0.1.1"}, + } + assert.Equal(t, downgrade, expectedReqs) +} diff --git a/pkg/package/modfile.go b/pkg/package/modfile.go index 55e6cb2a..ef68d7b1 100644 --- a/pkg/package/modfile.go +++ b/pkg/package/modfile.go @@ -259,6 +259,26 @@ func (dep *Dependency) GetDownloadPath() string { return "" } +func GenSource(sourceType string, uri string, tagName string) (Source, error) { + source := Source{} + if sourceType == GIT { + source.Git = &Git{ + Url: uri, + Tag: tagName, + } + return source, nil + } + if sourceType == OCI { + oci := Oci{} + _, err := oci.FromString(uri + ":" + tagName) + if err != nil { + return Source{}, err + } + source.Oci = &oci + } + return source, nil +} + // GetSourceType will get the source type of a dependency. func (dep *Dependency) GetSourceType() string { if dep.Source.Git != nil { diff --git a/pkg/package/modfile_test.go b/pkg/package/modfile_test.go index f3f05e2e..ebc3bd0d 100644 --- a/pkg/package/modfile_test.go +++ b/pkg/package/modfile_test.go @@ -235,3 +235,16 @@ func TestGetFilePath(t *testing.T) { assert.Equal(t, mfile.GetModFilePath(), filepath.Join(testPath, MOD_FILE)) assert.Equal(t, mfile.GetModLockFilePath(), filepath.Join(testPath, MOD_LOCK_FILE)) } + +func TestGenSource(t *testing.T) { + src, err := GenSource("git", "https://github.com/kcl-lang/kcl", "0.8.7") + assert.Equal(t, err, nil) + assert.Equal(t, src.Git.Url, "https://github.com/kcl-lang/kcl") + assert.Equal(t, src.Git.Tag, "0.8.7") + + src, err = GenSource("oci", "oci://ghcr.io/kcl-lang/k8s", "1.24") + assert.Equal(t, err, nil) + assert.Equal(t, src.Oci.Reg, "ghcr.io") + assert.Equal(t, src.Oci.Repo, "kcl-lang/k8s") + assert.Equal(t, src.Oci.Tag, "1.24") +} \ No newline at end of file diff --git a/pkg/semver/semver.go b/pkg/semver/semver.go index 53e0dd21..0e4f449e 100644 --- a/pkg/semver/semver.go +++ b/pkg/semver/semver.go @@ -27,21 +27,65 @@ func LatestVersion(versions []string) (string, error) { return latest.Original(), nil } -func OldestVersion(versions []string) (string, error) { - var oldest *version.Version +// LeastOldVersion returns the version that is most recent and less than the base version. +func LeastOldVersion(versions []string, baseVersion string) (string, error) { + base, err := version.NewVersion(baseVersion) + if err != nil { + return "", fmt.Errorf("invalid base version: %v", err) + } + + var leastOld *version.Version for _, v := range versions { ver, err := version.NewVersion(v) if err != nil { return "", reporter.NewErrorEvent(reporter.FailedParseVersion, err, fmt.Sprintf("failed to parse version %s", v)) } - if oldest == nil || ver.LessThan(oldest) { - oldest = ver + + // Only consider versions less than the base version + if ver.LessThan(base) { + if leastOld == nil || ver.GreaterThan(leastOld) { + leastOld = ver + } } } - if oldest == nil { + if leastOld == nil { return "", errors.InvalidVersionFormat } - return oldest.Original(), nil -} \ No newline at end of file + return leastOld.Original(), nil +} + +func filterCompatibleVersions(versions []string, baseVersion string) ([]string, error) { + base, err := version.NewVersion(baseVersion) + if err != nil { + return nil, fmt.Errorf("invalid base version: %v", err) + } + var compatibleVersions []string + for _, v := range versions { + ver, err := version.NewVersion(v) + if err != nil { + continue // skip versions that fail to parse + } + if ver.Segments()[0] == base.Segments()[0] && ver.Prerelease() == "" { + compatibleVersions = append(compatibleVersions, ver.Original()) + } + } + return compatibleVersions, nil +} + +func LatestCompatibleVersion(versions []string, baseVersion string) (string, error) { + compatibleVersions, err := filterCompatibleVersions(versions, baseVersion) + if err != nil { + return "", err + } + return LatestVersion(compatibleVersions) +} + +func LeastOldCompatibleVersion(versions []string, baseVersion string) (string, error) { + compatibleVersions, err := filterCompatibleVersions(versions, baseVersion) + if err != nil { + return "", err + } + return LeastOldVersion(compatibleVersions, baseVersion) +} diff --git a/pkg/semver/semver_test.go b/pkg/semver/semver_test.go index 560b4d99..4ab8adfd 100644 --- a/pkg/semver/semver_test.go +++ b/pkg/semver/semver_test.go @@ -39,34 +39,28 @@ func TestTheLatestTagWithMissingVersion(t *testing.T) { assert.Equal(t, latest, "5.5") } -func TestOldestVersion(t *testing.T) { - oldest, err := OldestVersion([]string{"1.2.3", "1.4.0", "1.3.5", "1.0.0"}) +func TestLeastOldVersion(t *testing.T) { + leastOld, err := LeastOldVersion([]string{"1.2.3", "1.4.0", "2.0.0", "1.3.5", "1.0.0"}, "1.2.0") assert.Equal(t, err, nil) - assert.Equal(t, oldest, "1.0.0") + assert.Equal(t, leastOld, "1.0.0") - oldest, err = OldestVersion([]string{}) - assert.Equal(t, err, errors.InvalidVersionFormat) - assert.Equal(t, oldest, "") - - oldest, err = OldestVersion([]string{"invalid_version"}) - assert.Equal(t, err.Error(), "failed to parse version invalid_version\nMalformed version: invalid_version\n") - assert.Equal(t, oldest, "") - - oldest, err = OldestVersion([]string{"1.2.3", "1.4.0", "1.3.5", "invalid_version"}) - assert.Equal(t, err.Error(), "failed to parse version invalid_version\nMalformed version: invalid_version\n") - assert.Equal(t, oldest, "") -} - -func TestOldestVersionWithVariousFormats(t *testing.T) { - oldest, err := OldestVersion([]string{"2.2", "2.4.5", "2.3.9", "2.1.0", "2.0"}) + leastOld, err = LeastOldVersion([]string{"2.2.0", "2.4.0", "3.0.0", "2.3.5"}, "2.5.0") assert.Equal(t, err, nil) - assert.Equal(t, oldest, "2.0") + assert.Equal(t, leastOld, "2.4.0") +} - oldest, err = OldestVersion([]string{"0.1", "0.1.1", "0.1.2-beta", "0.0.9"}) +func TestFilterCompatibleVersions(t *testing.T) { + compatible, err := filterCompatibleVersions([]string{"1.2.3", "1.4.0", "2.0.0", "1.3.5", "1.0.0"}, "1.2.0") assert.Equal(t, err, nil) - assert.Equal(t, oldest, "0.0.9") + expCompatible := []string{"1.2.3", "1.4.0", "1.3.5", "1.0.0"} + for i, v := range compatible { + assert.Equal(t, v, expCompatible[i]) + } - oldest, err = OldestVersion([]string{"3.3.3", "3.2", "3.1", "3.0.0"}) + compatible, err = filterCompatibleVersions([]string{"2.2.0", "2.4.0", "3.0.0", "2.3.5"}, "2.0.0") assert.Equal(t, err, nil) - assert.Equal(t, oldest, "3.0.0") + expCompatible = []string{"2.2.0", "2.4.0", "2.3.5"} + for i, v := range compatible { + assert.Equal(t, v, expCompatible[i]) + } } From 6cfd4637736cf77f5bf93c6be2a3281d00edda6a Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Mon, 13 May 2024 00:48:28 +0530 Subject: [PATCH 07/17] fix: downgrade issue Signed-off-by: Akash Kumar --- pkg/mvs/mvs.go | 13 ++++++++++--- pkg/mvs/mvs_test.go | 11 +++++------ pkg/semver/semver.go | 24 +++++++----------------- pkg/semver/semver_test.go | 10 +++++----- 4 files changed, 27 insertions(+), 31 deletions(-) diff --git a/pkg/mvs/mvs.go b/pkg/mvs/mvs.go index 307cfb67..668eb854 100644 --- a/pkg/mvs/mvs.go +++ b/pkg/mvs/mvs.go @@ -7,7 +7,7 @@ import ( "github.com/hashicorp/go-version" "golang.org/x/mod/module" "kcl-lang.io/kpm/pkg/client" - "kcl-lang.io/kpm/pkg/errors" + errInt "kcl-lang.io/kpm/pkg/errors" "kcl-lang.io/kpm/pkg/git" "kcl-lang.io/kpm/pkg/oci" pkg "kcl-lang.io/kpm/pkg/package" @@ -98,11 +98,18 @@ func (r ReqsGraph) Previous(m module.Version) (module.Version, error) { return m, nil } + // copy the version to compare it later + v := m.Version + m.Version, err = semver.LeastOldCompatibleVersion(releases, m.Version) - if err != nil { + if err != nil && err != errInt.InvalidVersionFormat { return module.Version{}, err } + if v == m.Version { + return module.Version{Path: m.Path, Version: "none"}, nil + } + _, err = r.Vertex(m) if err == graph.ErrVertexNotFound { d := pkg.Dependency{ @@ -147,7 +154,7 @@ func getReleasesFromSource(properties graph.VertexProperties) ([]string, error) // there must be only one property depending on the download source type if len(properties.Attributes) != 1 { - return nil, errors.MultipleSources + return nil, errInt.MultipleSources } for k, v := range properties.Attributes { diff --git a/pkg/mvs/mvs_test.go b/pkg/mvs/mvs_test.go index 3dce50cb..1e15a8ef 100644 --- a/pkg/mvs/mvs_test.go +++ b/pkg/mvs/mvs_test.go @@ -222,12 +222,11 @@ func TestDowngrade(t *testing.T) { assert.Equal(t, err, nil) expectedReqs := []module.Version{ - {Path: "test_with_external_deps", Version: "0.0.1"}, - {Path: "argo-cd-order", Version: "0.2.0"}, - {Path: "helloworld", Version: "0.1.1"}, - {Path: "json_merge_patch", Version: "0.1.0"}, - {Path: "k8s", Version: "1.27"}, - {Path: "podinfo", Version: "0.1.1"}, + {Path:"test_with_external_deps", Version:"0.0.1"}, + {Path:"argo-cd-order", Version:"0.1.2"}, + {Path:"helloworld", Version:"0.1.0"}, + {Path:"json_merge_patch", Version:"0.1.0"}, + {Path:"k8s", Version:"1.17"}, } assert.Equal(t, downgrade, expectedReqs) } diff --git a/pkg/semver/semver.go b/pkg/semver/semver.go index 0e4f449e..6026a023 100644 --- a/pkg/semver/semver.go +++ b/pkg/semver/semver.go @@ -27,33 +27,23 @@ func LatestVersion(versions []string) (string, error) { return latest.Original(), nil } -// LeastOldVersion returns the version that is most recent and less than the base version. -func LeastOldVersion(versions []string, baseVersion string) (string, error) { - base, err := version.NewVersion(baseVersion) - if err != nil { - return "", fmt.Errorf("invalid base version: %v", err) - } - - var leastOld *version.Version +func OldestVersion(versions []string) (string, error) { + var oldest *version.Version for _, v := range versions { ver, err := version.NewVersion(v) if err != nil { return "", reporter.NewErrorEvent(reporter.FailedParseVersion, err, fmt.Sprintf("failed to parse version %s", v)) } - - // Only consider versions less than the base version - if ver.LessThan(base) { - if leastOld == nil || ver.GreaterThan(leastOld) { - leastOld = ver - } + if oldest == nil || ver.LessThan(oldest) { + oldest = ver } } - if leastOld == nil { + if oldest == nil { return "", errors.InvalidVersionFormat } - return leastOld.Original(), nil + return oldest.Original(), nil } func filterCompatibleVersions(versions []string, baseVersion string) ([]string, error) { @@ -87,5 +77,5 @@ func LeastOldCompatibleVersion(versions []string, baseVersion string) (string, e if err != nil { return "", err } - return LeastOldVersion(compatibleVersions, baseVersion) + return OldestVersion(compatibleVersions) } diff --git a/pkg/semver/semver_test.go b/pkg/semver/semver_test.go index 4ab8adfd..d03bf631 100644 --- a/pkg/semver/semver_test.go +++ b/pkg/semver/semver_test.go @@ -39,14 +39,14 @@ func TestTheLatestTagWithMissingVersion(t *testing.T) { assert.Equal(t, latest, "5.5") } -func TestLeastOldVersion(t *testing.T) { - leastOld, err := LeastOldVersion([]string{"1.2.3", "1.4.0", "2.0.0", "1.3.5", "1.0.0"}, "1.2.0") +func TestOldestVersion(t *testing.T) { + oldest, err := OldestVersion([]string{"1.2.3", "1.4.0", "2.0.0", "1.3.5", "1.0.0"}) assert.Equal(t, err, nil) - assert.Equal(t, leastOld, "1.0.0") + assert.Equal(t, oldest, "1.0.0") - leastOld, err = LeastOldVersion([]string{"2.2.0", "2.4.0", "3.0.0", "2.3.5"}, "2.5.0") + oldest, err = OldestVersion([]string{"2.2.0", "2.4.0", "3.0.0", "2.3.5"}) assert.Equal(t, err, nil) - assert.Equal(t, leastOld, "2.4.0") + assert.Equal(t, oldest, "2.2.0") } func TestFilterCompatibleVersions(t *testing.T) { From 444d9da0688440815979f022c636673277a3e647 Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Tue, 14 May 2024 18:14:34 +0530 Subject: [PATCH 08/17] fix unit tests Signed-off-by: Akash Kumar --- go.mod | 1 - go.sum | 6 ++++-- pkg/3rdparty/mvs/mvs_test.go | 4 ++-- pkg/mvs/mvs_test.go | 4 ++-- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 4d591c13..ee5dd98d 100644 --- a/go.mod +++ b/go.mod @@ -135,7 +135,6 @@ require ( google.golang.org/protobuf v1.33.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 // indirect kcl-lang.io/lib v0.8.0 // indirect ) diff --git a/go.sum b/go.sum index a4e6b2f5..9a7993f4 100644 --- a/go.sum +++ b/go.sum @@ -271,12 +271,14 @@ github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaD github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/containerd v1.7.0 h1:G/ZQr3gMZs6ZT0qPUZ15znx5QSdQdASW11nXTLTM2Pg= github.com/containerd/containerd v1.7.0/go.mod h1:QfR7Efgb/6X2BDpTPJRvPTYDE9rsF0FsXX9J8sIs/sc= +github.com/containerd/containerd v1.7.11 h1:lfGKw3eU35sjV0aG2eYZTiwFEY1pCzxdzicHP3SZILw= +github.com/containerd/containerd v1.7.11/go.mod h1:5UluHxHTX2rdvYuZ5OJTC5m/KJNs0Zs9wVoJm9zf5ZE= github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= -github.com/containers/image v3.0.2+incompatible h1:B1lqAE8MUPCrsBLE86J0gnXleeRq8zJnQryhiiGQNyE= -github.com/containers/image v3.0.2+incompatible/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containers/image v3.0.2+incompatible h1:B1lqAE8MUPCrsBLE86J0gnXleeRq8zJnQryhiiGQNyE= +github.com/containers/image v3.0.2+incompatible/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= diff --git a/pkg/3rdparty/mvs/mvs_test.go b/pkg/3rdparty/mvs/mvs_test.go index 92b6c837..6e1e71cd 100644 --- a/pkg/3rdparty/mvs/mvs_test.go +++ b/pkg/3rdparty/mvs/mvs_test.go @@ -212,7 +212,7 @@ upgrade* A: A B5.hidden C3 name: down1 A: B2 -B1: C2 +B1: C1 B2: C2 build A: A B2 C2 downgrade A C1: A B1 C1 @@ -606,7 +606,7 @@ func (r reqsMap) Upgrade(m module.Version) (module.Version, error) { if k.Path == m.Path && r.Max(k.Path, u.Version, k.Version) == k.Version && !strings.HasSuffix(k.Version, ".hidden") { u = k } - } + } if u.Path == "" { return module.Version{}, fmt.Errorf("missing module: %v", module.Version{Path: m.Path}) } diff --git a/pkg/mvs/mvs_test.go b/pkg/mvs/mvs_test.go index 1e15a8ef..f18f3973 100644 --- a/pkg/mvs/mvs_test.go +++ b/pkg/mvs/mvs_test.go @@ -52,8 +52,8 @@ func TestRequired(t *testing.T) { assert.Equal(t, len(req), 2) expectedReqs := []module.Version{ - {Path: "bbb", Version: "0.0.1"}, - {Path: "ccc", Version: "0.0.1"}, + {Path:"ccc", Version:"0.0.1"}, + {Path:"bbb", Version:"0.0.1"}, } assert.Equal(t, req, expectedReqs) } From 1502ec91c18edebcb0a73a997cec17ba60f69553 Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Tue, 14 May 2024 18:24:01 +0530 Subject: [PATCH 09/17] fix golint ci failure Signed-off-by: Akash Kumar --- pkg/mvs/mvs.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pkg/mvs/mvs.go b/pkg/mvs/mvs.go index 668eb854..0babc1ec 100644 --- a/pkg/mvs/mvs.go +++ b/pkg/mvs/mvs.go @@ -21,15 +21,15 @@ type ReqsGraph struct { kpmPkg *pkg.KclPkg } -func (r ReqsGraph) Max(_, v1, v2 string) string { +func (r ReqsGraph) Max(path, v1, v2 string) string { version1, err := version.NewVersion(v1) if err != nil { - reporter.NewErrorEvent(reporter.FailedParseVersion, err, fmt.Sprintf("failed to parse version %s", v1)) + reporter.Fatal(reporter.FailedParseVersion, err, fmt.Sprintf("failed to parse version %s for module %s", v1, path)) return "" } version2, err := version.NewVersion(v2) if err != nil { - reporter.NewErrorEvent(reporter.FailedParseVersion, err, fmt.Sprintf("failed to parse version %s", v2)) + reporter.Fatal(reporter.FailedParseVersion, err, fmt.Sprintf("failed to parse version %s for module %s", v2, path)) return "" } if version1.GreaterThan(version2) { @@ -78,7 +78,10 @@ func (r ReqsGraph) Upgrade(m module.Version) (module.Version, error) { lockDeps := pkg.Dependencies{ Deps: make(map[string]pkg.Dependency), } - r.kpmClient.DownloadDeps(deps, lockDeps, r.Graph, r.kpmPkg.HomePath, module.Version{}) + _, err = r.kpmClient.DownloadDeps(deps, lockDeps, r.Graph, r.kpmPkg.HomePath, module.Version{}) + if err != nil { + return module.Version{}, err + } } return m, nil } @@ -131,7 +134,10 @@ func (r ReqsGraph) Previous(m module.Version) (module.Version, error) { lockDeps := pkg.Dependencies{ Deps: make(map[string]pkg.Dependency), } - r.kpmClient.DownloadDeps(deps, lockDeps, r.Graph, r.kpmPkg.HomePath, module.Version{}) + _, err = r.kpmClient.DownloadDeps(deps, lockDeps, r.Graph, r.kpmPkg.HomePath, module.Version{}) + if err != nil { + return module.Version{}, err + } } return m, nil } From 1e5f228f88af2df950bceab1dba51461ffe8cbd0 Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Tue, 14 May 2024 18:27:56 +0530 Subject: [PATCH 10/17] add check to return err if git host name is other than github Signed-off-by: Akash Kumar --- pkg/git/git.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/git/git.go b/pkg/git/git.go index 667b4338..8d244ac9 100644 --- a/pkg/git/git.go +++ b/pkg/git/git.go @@ -184,6 +184,10 @@ func GetAllGithubReleases(url string) ([]string, error) { return nil, err } + if gitURL.GetHostName() != "github.com" { + return nil, errors.New("only GitHub repositories are currently supported") + } + // Construct initial API URL for the first page apiBase := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases", gitURL.GetOwnerName(), gitURL.GetRepoName()) apiURL := fmt.Sprintf("%s?per_page=100&page=1", apiBase) From bc623007167b4fa26f86f7d1ac81407e552034c0 Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Tue, 14 May 2024 21:42:14 +0530 Subject: [PATCH 11/17] fix unit tests Signed-off-by: Akash Kumar --- pkg/mvs/mvs.go | 6 ++++++ pkg/mvs/mvs_test.go | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/mvs/mvs.go b/pkg/mvs/mvs.go index 0babc1ec..ea53475f 100644 --- a/pkg/mvs/mvs.go +++ b/pkg/mvs/mvs.go @@ -22,6 +22,12 @@ type ReqsGraph struct { } func (r ReqsGraph) Max(path, v1, v2 string) string { + if v1 == "none" || v2 == "" { + return v2 + } + if v2 == "none" || v1 == "" { + return v1 + } version1, err := version.NewVersion(v1) if err != nil { reporter.Fatal(reporter.FailedParseVersion, err, fmt.Sprintf("failed to parse version %s for module %s", v1, path)) diff --git a/pkg/mvs/mvs_test.go b/pkg/mvs/mvs_test.go index f18f3973..b90da6d5 100644 --- a/pkg/mvs/mvs_test.go +++ b/pkg/mvs/mvs_test.go @@ -3,6 +3,7 @@ package mvs import ( "os" "path/filepath" + "sort" "testing" "github.com/stretchr/testify/assert" @@ -52,9 +53,12 @@ func TestRequired(t *testing.T) { assert.Equal(t, len(req), 2) expectedReqs := []module.Version{ - {Path:"ccc", Version:"0.0.1"}, {Path:"bbb", Version:"0.0.1"}, + {Path:"ccc", Version:"0.0.1"}, } + sort.Slice(req, func(i, j int) bool { + return req[i].Path < req[j].Path + }) assert.Equal(t, req, expectedReqs) } From 030d434cac1463f7cf0e6c4b278282a0d0eedc21 Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Tue, 14 May 2024 21:46:06 +0530 Subject: [PATCH 12/17] delete unused function Signed-off-by: Akash Kumar --- pkg/client/client_test.go | 1 + pkg/cmd/cmd_tidy.go | 85 +++++++++++++++++++++++++++++++++++++++ pkg/git/git.go | 1 + pkg/graph/graph.go | 43 -------------------- 4 files changed, 87 insertions(+), 43 deletions(-) create mode 100644 pkg/cmd/cmd_tidy.go delete mode 100644 pkg/graph/graph.go diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 09f9e15b..79ee1665 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -1469,3 +1469,4 @@ func testRunWithOciDownloader(t *testing.T) { assert.Equal(t, buf.String(), "downloading 'zong-zhe/helloworld:0.0.3' from 'ghcr.io/zong-zhe/helloworld:0.0.3'\n") assert.Equal(t, res.GetRawYamlResult(), "The_first_kcl_program: Hello World!") } + diff --git a/pkg/cmd/cmd_tidy.go b/pkg/cmd/cmd_tidy.go new file mode 100644 index 00000000..a1f3f387 --- /dev/null +++ b/pkg/cmd/cmd_tidy.go @@ -0,0 +1,85 @@ +// Copyright 2024 The KCL Authors. All rights reserved. + +package cmd + +import ( + "fmt" + "os" + + "github.com/dominikbraun/graph" + "github.com/urfave/cli/v2" + "golang.org/x/mod/module" + "kcl-lang.io/kpm/pkg/client" + "kcl-lang.io/kpm/pkg/env" + pkg "kcl-lang.io/kpm/pkg/package" + "kcl-lang.io/kpm/pkg/reporter" +) + +// NewTidyCmd new a Command for `kpm graph`. +func NewTidyCmd(kpmcli *client.KpmClient) *cli.Command { + return &cli.Command{ + Hidden: false, + Name: "graph", + Usage: "prints the module dependency graph", + Action: func(c *cli.Context) error { + return KpmTidy(c, kpmcli) + }, + } +} + +func KpmTidy(c *cli.Context, kpmcli *client.KpmClient) error { + pwd, err := os.Getwd() + + if err != nil { + return reporter.NewErrorEvent(reporter.Bug, err, "internal bugs, please contact us to fix it.") + } + + globalPkgPath, err := env.GetAbsPkgPath() + if err != nil { + return err + } + + kclPkg, err := pkg.LoadKclPkg(pwd) + if err != nil { + return err + } + + err = kclPkg.ValidateKpmHome(globalPkgPath) + if err != (*reporter.KpmEvent)(nil) { + return err + } + + _, depGraph, err := kpmcli.InitGraphAndDownloadDeps(kclPkg) + if err != nil { + return err + } + + adjMap, err := depGraph.AdjacencyMap() + if err != nil { + return err + } + + format := func(m module.Version) string { + formattedMsg := m.Path + if m.Version != "" { + formattedMsg += "@" + m.Version + } + return formattedMsg + } + + // print the dependency graph to stdout. + root := module.Version{Path: kclPkg.GetPkgName(), Version: kclPkg.GetPkgVersion()} + err = graph.BFS(depGraph, root, func(source module.Version) bool { + for target := range adjMap[source] { + reporter.ReportMsgTo( + fmt.Sprint(format(source), " ", format(target)), + kpmcli.GetLogWriter(), + ) + } + return false + }) + if err != nil { + return err + } + return nil +} diff --git a/pkg/git/git.go b/pkg/git/git.go index 8d244ac9..9e8466d0 100644 --- a/pkg/git/git.go +++ b/pkg/git/git.go @@ -146,6 +146,7 @@ func CloneWithOpts(opts ...CloneOption) (*git.Repository, error) { return nil, err } + return cloneOpts.Clone() } diff --git a/pkg/graph/graph.go b/pkg/graph/graph.go deleted file mode 100644 index e22d40e2..00000000 --- a/pkg/graph/graph.go +++ /dev/null @@ -1,43 +0,0 @@ -package graph - -import ( - "fmt" - - "github.com/dominikbraun/graph" - "golang.org/x/mod/module" - pkg "kcl-lang.io/kpm/pkg/package" -) - -func ChangeGraphType(g graph.Graph[pkg.Dependency, pkg.Dependency]) (graph.Graph[module.Version, module.Version], error) { - AdjacencyMap, err := g.AdjacencyMap() - if err != nil { - return nil, fmt.Errorf("failed to get adjacency map: %w", err) - } - - m := func(dep pkg.Dependency) module.Version { - return module.Version{Path: dep.Name, Version: dep.Version} - } - - moduleHash := func(m module.Version) module.Version { - return m - } - - depGraph := graph.New(moduleHash, graph.Directed(), graph.PreventCycles()) - for node, edges := range AdjacencyMap { - err := depGraph.AddVertex(m(node)) - if err != nil && err != graph.ErrVertexAlreadyExists { - return nil, fmt.Errorf("failed to add vertex: %w", err) - } - for edge := range edges { - err := depGraph.AddVertex(m(edge)) - if err != nil && err != graph.ErrVertexAlreadyExists { - return nil, fmt.Errorf("failed to add vertex: %w", err) - } - err = depGraph.AddEdge(m(node), m(edge)) - if err != nil { - return nil, fmt.Errorf("failed to add edge: %w", err) - } - } - } - return depGraph, nil -} From c9675cdb0acf0aa493694e3a7926b0f80af2e8cd Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Tue, 14 May 2024 22:07:34 +0530 Subject: [PATCH 13/17] remove not needed changes Signed-off-by: Akash Kumar --- pkg/client/client.go | 8 ++++---- pkg/mvs/mvs.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index 4b71948f..4a1741ee 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -1293,7 +1293,7 @@ func (c *KpmClient) InitGraphAndDownloadDeps(kclPkg *pkg.KclPkg) (*pkg.Dependenc return nil, nil, err } - changedDeps, err := c.DownloadDeps(kclPkg.ModFile.Dependencies, kclPkg.Dependencies, depGraph, kclPkg.HomePath, root) + changedDeps, err := c.DownloadDeps(&kclPkg.ModFile.Dependencies, &kclPkg.Dependencies, depGraph, kclPkg.HomePath, root) if err != nil { return nil, nil, err } @@ -1325,7 +1325,7 @@ func (c *KpmClient) dependencyExists(dep *pkg.Dependency, lockDeps *pkg.Dependen } // downloadDeps will download all the dependencies of the current kcl package. -func (c *KpmClient) DownloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencies, depGraph graph.Graph[module.Version, module.Version], pkghome string, parent module.Version) (*pkg.Dependencies, error) { +func (c *KpmClient) DownloadDeps(deps *pkg.Dependencies, lockDeps *pkg.Dependencies, depGraph graph.Graph[module.Version, module.Version], pkghome string, parent module.Version) (*pkg.Dependencies, error) { newDeps := pkg.Dependencies{ Deps: make(map[string]pkg.Dependency), @@ -1337,7 +1337,7 @@ func (c *KpmClient) DownloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencie return nil, errors.InvalidDependency } - existDep := c.dependencyExists(&d, &lockDeps) + existDep := c.dependencyExists(&d, lockDeps) if existDep != nil { newDeps.Deps[d.Name] = *existDep continue @@ -1424,7 +1424,7 @@ func (c *KpmClient) DownloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencie } // Download the indirect dependencies. - nested, err := c.DownloadDeps(deppkg.ModFile.Dependencies, lockDeps, depGraph, deppkg.HomePath, source) + nested, err := c.DownloadDeps(&deppkg.ModFile.Dependencies, lockDeps, depGraph, deppkg.HomePath, source) if err != nil { return nil, err } diff --git a/pkg/mvs/mvs.go b/pkg/mvs/mvs.go index ea53475f..fbd817c7 100644 --- a/pkg/mvs/mvs.go +++ b/pkg/mvs/mvs.go @@ -84,7 +84,7 @@ func (r ReqsGraph) Upgrade(m module.Version) (module.Version, error) { lockDeps := pkg.Dependencies{ Deps: make(map[string]pkg.Dependency), } - _, err = r.kpmClient.DownloadDeps(deps, lockDeps, r.Graph, r.kpmPkg.HomePath, module.Version{}) + _, err = r.kpmClient.DownloadDeps(&deps, &lockDeps, r.Graph, r.kpmPkg.HomePath, module.Version{}) if err != nil { return module.Version{}, err } @@ -140,7 +140,7 @@ func (r ReqsGraph) Previous(m module.Version) (module.Version, error) { lockDeps := pkg.Dependencies{ Deps: make(map[string]pkg.Dependency), } - _, err = r.kpmClient.DownloadDeps(deps, lockDeps, r.Graph, r.kpmPkg.HomePath, module.Version{}) + _, err = r.kpmClient.DownloadDeps(&deps, &lockDeps, r.Graph, r.kpmPkg.HomePath, module.Version{}) if err != nil { return module.Version{}, err } From 3e9cae6e8eadaa3d475510de745dd850b20b97b6 Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Tue, 14 May 2024 22:34:17 +0530 Subject: [PATCH 14/17] fix unit tests Signed-off-by: Akash Kumar --- pkg/client/client.go | 1 - .../test_data/add_with_git_commit/test_pkg/kcl.mod.expect | 2 +- .../add_with_git_commit/test_pkg/kcl.mod.lock.expect | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index 4a1741ee..0732784e 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -847,7 +847,6 @@ func (c *KpmClient) Download(dep *pkg.Dependency, homePath, localPath string) (* return nil, err } dep.Version = modFile.Pkg.Version - dep.Source.Git.Version = modFile.Pkg.Version } if dep.Source.Oci != nil { diff --git a/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.expect b/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.expect index 112fdd08..475d36f8 100644 --- a/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.expect +++ b/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.expect @@ -4,4 +4,4 @@ edition = "0.0.1" version = "0.0.1" [dependencies] -catalog = { git = "https://github.com/KusionStack/catalog.git", commit = "a29e3db", version = "0.1.0" } +catalog = { git = "https://github.com/KusionStack/catalog.git", commit = "a29e3db" } diff --git a/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.lock.expect b/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.lock.expect index fed06b76..dbb61a69 100644 --- a/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.lock.expect +++ b/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.lock.expect @@ -2,7 +2,7 @@ [dependencies.catalog] name = "catalog" full_name = "catalog_a29e3db" + version = "0.1.0" sum = "kFmlrYJbJUFFTEXjC9cquc80WB+UpZ/6oMPKrfgyeks=" url = "https://github.com/KusionStack/catalog.git" - commit = "a29e3db" - version = "0.1.0" + commit = "a29e3db" \ No newline at end of file From 1a30352ebef1d6465079ff86f4c40905d284fd7a Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Wed, 15 May 2024 08:36:32 +0530 Subject: [PATCH 15/17] remove tidy command Signed-off-by: Akash Kumar --- pkg/cmd/cmd_tidy.go | 85 --------------------------------------------- 1 file changed, 85 deletions(-) delete mode 100644 pkg/cmd/cmd_tidy.go diff --git a/pkg/cmd/cmd_tidy.go b/pkg/cmd/cmd_tidy.go deleted file mode 100644 index a1f3f387..00000000 --- a/pkg/cmd/cmd_tidy.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2024 The KCL Authors. All rights reserved. - -package cmd - -import ( - "fmt" - "os" - - "github.com/dominikbraun/graph" - "github.com/urfave/cli/v2" - "golang.org/x/mod/module" - "kcl-lang.io/kpm/pkg/client" - "kcl-lang.io/kpm/pkg/env" - pkg "kcl-lang.io/kpm/pkg/package" - "kcl-lang.io/kpm/pkg/reporter" -) - -// NewTidyCmd new a Command for `kpm graph`. -func NewTidyCmd(kpmcli *client.KpmClient) *cli.Command { - return &cli.Command{ - Hidden: false, - Name: "graph", - Usage: "prints the module dependency graph", - Action: func(c *cli.Context) error { - return KpmTidy(c, kpmcli) - }, - } -} - -func KpmTidy(c *cli.Context, kpmcli *client.KpmClient) error { - pwd, err := os.Getwd() - - if err != nil { - return reporter.NewErrorEvent(reporter.Bug, err, "internal bugs, please contact us to fix it.") - } - - globalPkgPath, err := env.GetAbsPkgPath() - if err != nil { - return err - } - - kclPkg, err := pkg.LoadKclPkg(pwd) - if err != nil { - return err - } - - err = kclPkg.ValidateKpmHome(globalPkgPath) - if err != (*reporter.KpmEvent)(nil) { - return err - } - - _, depGraph, err := kpmcli.InitGraphAndDownloadDeps(kclPkg) - if err != nil { - return err - } - - adjMap, err := depGraph.AdjacencyMap() - if err != nil { - return err - } - - format := func(m module.Version) string { - formattedMsg := m.Path - if m.Version != "" { - formattedMsg += "@" + m.Version - } - return formattedMsg - } - - // print the dependency graph to stdout. - root := module.Version{Path: kclPkg.GetPkgName(), Version: kclPkg.GetPkgVersion()} - err = graph.BFS(depGraph, root, func(source module.Version) bool { - for target := range adjMap[source] { - reporter.ReportMsgTo( - fmt.Sprint(format(source), " ", format(target)), - kpmcli.GetLogWriter(), - ) - } - return false - }) - if err != nil { - return err - } - return nil -} From 6724705ff5b8a57f3dd2a14a17177e180262abc5 Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Wed, 15 May 2024 15:23:39 +0530 Subject: [PATCH 16/17] fix windows ci failure Signed-off-by: Akash Kumar --- .../add_with_git_commit/test_pkg_win/kcl.mod.lock.expect | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/client/test_data/add_with_git_commit/test_pkg_win/kcl.mod.lock.expect b/pkg/client/test_data/add_with_git_commit/test_pkg_win/kcl.mod.lock.expect index 67893893..dbb61a69 100644 --- a/pkg/client/test_data/add_with_git_commit/test_pkg_win/kcl.mod.lock.expect +++ b/pkg/client/test_data/add_with_git_commit/test_pkg_win/kcl.mod.lock.expect @@ -2,7 +2,7 @@ [dependencies.catalog] name = "catalog" full_name = "catalog_a29e3db" - version = "a29e3db" - sum = "zhh1yHk5TrNi9apHUQF3hPOlwi5Kc75cNHjcVmGv+Qo=" + version = "0.1.0" + sum = "kFmlrYJbJUFFTEXjC9cquc80WB+UpZ/6oMPKrfgyeks=" url = "https://github.com/KusionStack/catalog.git" - commit = "a29e3db" + commit = "a29e3db" \ No newline at end of file From 1a8c8a433f7efccd9bbc93eb06a8d5d4bb53b072 Mon Sep 17 00:00:00 2001 From: Akash Kumar Date: Wed, 15 May 2024 15:30:44 +0530 Subject: [PATCH 17/17] fix windows ci Signed-off-by: Akash Kumar --- .../add_with_git_commit/test_pkg_win/kcl.mod.lock.expect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/client/test_data/add_with_git_commit/test_pkg_win/kcl.mod.lock.expect b/pkg/client/test_data/add_with_git_commit/test_pkg_win/kcl.mod.lock.expect index dbb61a69..cda1fc63 100644 --- a/pkg/client/test_data/add_with_git_commit/test_pkg_win/kcl.mod.lock.expect +++ b/pkg/client/test_data/add_with_git_commit/test_pkg_win/kcl.mod.lock.expect @@ -3,6 +3,6 @@ name = "catalog" full_name = "catalog_a29e3db" version = "0.1.0" - sum = "kFmlrYJbJUFFTEXjC9cquc80WB+UpZ/6oMPKrfgyeks=" + sum = "zhh1yHk5TrNi9apHUQF3hPOlwi5Kc75cNHjcVmGv+Qo=" url = "https://github.com/KusionStack/catalog.git" commit = "a29e3db" \ No newline at end of file