diff --git a/.github/scripts/external-packages-license-check.go b/.github/scripts/external-packages-license-check.go new file mode 100644 index 00000000..77745697 --- /dev/null +++ b/.github/scripts/external-packages-license-check.go @@ -0,0 +1,89 @@ +package main + +import ( + "bufio" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" +) + +var licenseRegexp = regexp.MustCompile("// SPDX-License-Identifier: ([^\\s]*)$") + +func fileLicense(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + scanner := bufio.NewScanner(file) + + for scanner.Scan() { + line := scanner.Text() + matches := licenseRegexp.FindStringSubmatch(line) + if len(matches) > 0 { + return matches[1], nil + } + } + + return "", nil +} + +func checkDirLicense(path string, valid string) error { + return filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error { + if !strings.HasSuffix(path, ".go") { + return nil + } + license, err := fileLicense(path) + if err != nil { + return err + } + if license == "" { + return fmt.Errorf("cannot find a valid license in %q", path) + } + if license != valid { + return fmt.Errorf("expected %q to be %q, got %q", path, valid, license) + } + return nil + }) +} + +func run() error { + // Check external packages licenses. + err := checkDirLicense("public", "Apache-2.0") + if err != nil { + return fmt.Errorf("invalid license in exported package: %s", err) + } + + // Check the internal dependencies of the external packages. + output, err := exec.Command("sh", "-c", "go list -deps -test ./public/*").Output() + if err != nil { + return err + } + lines := strings.Split(string(output), "\n") + var internalPkgs []string + for _, line := range lines { + if strings.Contains(line, "github.com/canonical/chisel/internal") { + internalPkgs = append(internalPkgs, strings.TrimPrefix(line, "github.com/canonical/chisel/")) + } + } + for _, pkg := range internalPkgs { + err := checkDirLicense(pkg, "Apache-2.0") + if err != nil { + return fmt.Errorf("invalid license in depedency %q: %s", pkg, err) + } + } + + return nil +} + +func main() { + err := run() + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } +} diff --git a/.github/workflows/license.yaml b/.github/workflows/license.yaml new file mode 100644 index 00000000..0298da71 --- /dev/null +++ b/.github/workflows/license.yaml @@ -0,0 +1,22 @@ +name: License check + +on: + workflow_dispatch: + push: + pull_request: + branches: [main] + +jobs: + external-packages: + runs-on: ubuntu-22.04 + name: External packages license check + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-go@v3 + with: + go-version-file: 'go.mod' + + - name: Run license check + run: | + go run .github/scripts/external-packages-license-check.go diff --git a/internal/apachetestutil/manifest.go b/internal/apachetestutil/manifest.go new file mode 100644 index 00000000..cb3006f8 --- /dev/null +++ b/internal/apachetestutil/manifest.go @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: Apache-2.0 + +package apachetestutil + +import ( + "gopkg.in/check.v1" + + "github.com/canonical/chisel/public/manifest" +) + +type ManifestContents struct { + Paths []*manifest.Path + Packages []*manifest.Package + Slices []*manifest.Slice + Contents []*manifest.Content +} + +func DumpManifestContents(c *check.C, mfest *manifest.Manifest) *ManifestContents { + var slices []*manifest.Slice + err := mfest.IterateSlices("", func(slice *manifest.Slice) error { + slices = append(slices, slice) + return nil + }) + c.Assert(err, check.IsNil) + + var pkgs []*manifest.Package + err = mfest.IteratePackages(func(pkg *manifest.Package) error { + pkgs = append(pkgs, pkg) + return nil + }) + c.Assert(err, check.IsNil) + + var paths []*manifest.Path + err = mfest.IteratePaths("", func(path *manifest.Path) error { + paths = append(paths, path) + return nil + }) + c.Assert(err, check.IsNil) + + var contents []*manifest.Content + err = mfest.IterateContents("", func(content *manifest.Content) error { + contents = append(contents, content) + return nil + }) + c.Assert(err, check.IsNil) + + mc := ManifestContents{ + Paths: paths, + Packages: pkgs, + Slices: slices, + Contents: contents, + } + return &mc +} diff --git a/internal/apacheutil/log.go b/internal/apacheutil/log.go new file mode 100644 index 00000000..c4404181 --- /dev/null +++ b/internal/apacheutil/log.go @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: Apache-2.0 + +package apacheutil + +import ( + "fmt" + "sync" +) + +// Avoid importing the log type information unnecessarily. There's a small cost +// associated with using an interface rather than the type. Depending on how +// often the logger is plugged in, it would be worth using the type instead. +type log_Logger interface { + Output(calldepth int, s string) error +} + +var globalLoggerLock sync.Mutex +var globalLogger log_Logger +var globalDebug bool + +// Specify the *log.Logger object where log messages should be sent to. +func SetLogger(logger log_Logger) { + globalLoggerLock.Lock() + globalLogger = logger + globalLoggerLock.Unlock() +} + +// Enable the delivery of debug messages to the logger. Only meaningful +// if a logger is also set. +func SetDebug(debug bool) { + globalLoggerLock.Lock() + globalDebug = debug + globalLoggerLock.Unlock() +} + +// logf sends to the logger registered via SetLogger the string resulting +// from running format and args through Sprintf. +func logf(format string, args ...interface{}) { + globalLoggerLock.Lock() + defer globalLoggerLock.Unlock() + if globalLogger != nil { + globalLogger.Output(2, fmt.Sprintf(format, args...)) + } +} + +// debugf sends to the logger registered via SetLogger the string resulting +// from running format and args through Sprintf, but only if debugging was +// enabled via SetDebug. +func debugf(format string, args ...interface{}) { + globalLoggerLock.Lock() + defer globalLoggerLock.Unlock() + if globalDebug && globalLogger != nil { + globalLogger.Output(2, fmt.Sprintf(format, args...)) + } +} diff --git a/internal/apacheutil/suite_test.go b/internal/apacheutil/suite_test.go new file mode 100644 index 00000000..50c664f5 --- /dev/null +++ b/internal/apacheutil/suite_test.go @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: Apache-2.0 + +package apacheutil_test + +import ( + "testing" + + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/apacheutil" +) + +func Test(t *testing.T) { TestingT(t) } + +type S struct{} + +var _ = Suite(&S{}) + +func (s *S) SetUpTest(c *C) { + apacheutil.SetDebug(true) + apacheutil.SetLogger(c) +} + +func (s *S) TearDownTest(c *C) { + apacheutil.SetDebug(false) + apacheutil.SetLogger(nil) +} diff --git a/internal/apacheutil/util.go b/internal/apacheutil/util.go new file mode 100644 index 00000000..7e0f8e6d --- /dev/null +++ b/internal/apacheutil/util.go @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: Apache-2.0 + +package apacheutil + +import ( + "fmt" + "regexp" +) + +type SliceKey struct { + Package string + Slice string +} + +func (s SliceKey) String() string { return s.Package + "_" + s.Slice } + +// FnameExp matches the slice definition file basename. +var FnameExp = regexp.MustCompile(`^([a-z0-9](?:-?[.a-z0-9+]){1,})\.yaml$`) + +// SnameExp matches only the slice name, without the leading package name. +var SnameExp = regexp.MustCompile(`^([a-z](?:-?[a-z0-9]){2,})$`) + +// knameExp matches the slice full name in pkg_slice format. +var knameExp = regexp.MustCompile(`^([a-z0-9](?:-?[.a-z0-9+]){1,})_([a-z](?:-?[a-z0-9]){2,})$`) + +func ParseSliceKey(sliceKey string) (SliceKey, error) { + match := knameExp.FindStringSubmatch(sliceKey) + if match == nil { + return SliceKey{}, fmt.Errorf("invalid slice reference: %q", sliceKey) + } + return SliceKey{match[1], match[2]}, nil +} diff --git a/internal/apacheutil/util_test.go b/internal/apacheutil/util_test.go new file mode 100644 index 00000000..6289473c --- /dev/null +++ b/internal/apacheutil/util_test.go @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: Apache-2.0 + +package apacheutil_test + +import ( + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/apacheutil" +) + +var sliceKeyTests = []struct { + input string + expected apacheutil.SliceKey + err string +}{{ + input: "foo_bar", + expected: apacheutil.SliceKey{Package: "foo", Slice: "bar"}, +}, { + input: "fo_bar", + expected: apacheutil.SliceKey{Package: "fo", Slice: "bar"}, +}, { + input: "1234_bar", + expected: apacheutil.SliceKey{Package: "1234", Slice: "bar"}, +}, { + input: "foo1.1-2-3_bar", + expected: apacheutil.SliceKey{Package: "foo1.1-2-3", Slice: "bar"}, +}, { + input: "foo-pkg_dashed-slice-name", + expected: apacheutil.SliceKey{Package: "foo-pkg", Slice: "dashed-slice-name"}, +}, { + input: "foo+_bar", + expected: apacheutil.SliceKey{Package: "foo+", Slice: "bar"}, +}, { + input: "foo_slice123", + expected: apacheutil.SliceKey{Package: "foo", Slice: "slice123"}, +}, { + input: "g++_bins", + expected: apacheutil.SliceKey{Package: "g++", Slice: "bins"}, +}, { + input: "a+_bar", + expected: apacheutil.SliceKey{Package: "a+", Slice: "bar"}, +}, { + input: "a._bar", + expected: apacheutil.SliceKey{Package: "a.", Slice: "bar"}, +}, { + input: "foo_ba", + err: `invalid slice reference: "foo_ba"`, +}, { + input: "f_bar", + err: `invalid slice reference: "f_bar"`, +}, { + input: "1234_789", + err: `invalid slice reference: "1234_789"`, +}, { + input: "foo_bar.x.y", + err: `invalid slice reference: "foo_bar.x.y"`, +}, { + input: "foo-_-bar", + err: `invalid slice reference: "foo-_-bar"`, +}, { + input: "foo_bar-", + err: `invalid slice reference: "foo_bar-"`, +}, { + input: "foo-_bar", + err: `invalid slice reference: "foo-_bar"`, +}, { + input: "-foo_bar", + err: `invalid slice reference: "-foo_bar"`, +}, { + input: "foo_bar_baz", + err: `invalid slice reference: "foo_bar_baz"`, +}, { + input: "a-_bar", + err: `invalid slice reference: "a-_bar"`, +}, { + input: "+++_bar", + err: `invalid slice reference: "\+\+\+_bar"`, +}, { + input: "..._bar", + err: `invalid slice reference: "\.\.\._bar"`, +}, { + input: "white space_no-whitespace", + err: `invalid slice reference: "white space_no-whitespace"`, +}} + +func (s *S) TestParseSliceKey(c *C) { + for _, test := range sliceKeyTests { + key, err := apacheutil.ParseSliceKey(test.input) + if test.err != "" { + c.Assert(err, ErrorMatches, test.err) + continue + } + c.Assert(err, IsNil) + c.Assert(key, DeepEquals, test.expected) + } +} diff --git a/internal/jsonwall/log.go b/internal/manifestutil/log.go similarity index 98% rename from internal/jsonwall/log.go rename to internal/manifestutil/log.go index 3e27f66c..19396d9d 100644 --- a/internal/jsonwall/log.go +++ b/internal/manifestutil/log.go @@ -1,4 +1,4 @@ -package jsonwall +package manifestutil import ( "fmt" diff --git a/internal/manifest/manifest.go b/internal/manifestutil/manifestutil.go similarity index 71% rename from internal/manifest/manifest.go rename to internal/manifestutil/manifestutil.go index aeb40df7..b791130d 100644 --- a/internal/manifest/manifest.go +++ b/internal/manifestutil/manifestutil.go @@ -1,4 +1,4 @@ -package manifest +package manifestutil import ( "fmt" @@ -9,162 +9,15 @@ import ( "sort" "strings" + "github.com/canonical/chisel/internal/apacheutil" "github.com/canonical/chisel/internal/archive" - "github.com/canonical/chisel/internal/jsonwall" "github.com/canonical/chisel/internal/setup" + "github.com/canonical/chisel/public/jsonwall" + "github.com/canonical/chisel/public/manifest" ) -const Schema = "1.0" const DefaultFilename = "manifest.wall" -type Package struct { - Kind string `json:"kind"` - Name string `json:"name,omitempty"` - Version string `json:"version,omitempty"` - Digest string `json:"sha256,omitempty"` - Arch string `json:"arch,omitempty"` -} - -type Slice struct { - Kind string `json:"kind"` - Name string `json:"name,omitempty"` -} - -type Path struct { - Kind string `json:"kind"` - Path string `json:"path,omitempty"` - Mode string `json:"mode,omitempty"` - Slices []string `json:"slices,omitempty"` - SHA256 string `json:"sha256,omitempty"` - FinalSHA256 string `json:"final_sha256,omitempty"` - Size uint64 `json:"size,omitempty"` - Link string `json:"link,omitempty"` - Inode uint64 `json:"inode,omitempty"` -} - -type Content struct { - Kind string `json:"kind"` - Slice string `json:"slice,omitempty"` - Path string `json:"path,omitempty"` -} - -type Manifest struct { - db *jsonwall.DB -} - -// Read loads a Manifest without performing any validation. The data is assumed -// to be both valid jsonwall and a valid Manifest (see Validate). -func Read(reader io.Reader) (manifest *Manifest, err error) { - defer func() { - if err != nil { - err = fmt.Errorf("cannot read manifest: %s", err) - } - }() - - db, err := jsonwall.ReadDB(reader) - if err != nil { - return nil, err - } - mfestSchema := db.Schema() - if mfestSchema != Schema { - return nil, fmt.Errorf("unknown schema version %q", mfestSchema) - } - - manifest = &Manifest{db: db} - return manifest, nil -} - -func (manifest *Manifest) IteratePaths(pathPrefix string, onMatch func(*Path) error) (err error) { - return iteratePrefix(manifest, &Path{Kind: "path", Path: pathPrefix}, onMatch) -} - -func (manifest *Manifest) IteratePackages(onMatch func(*Package) error) (err error) { - return iteratePrefix(manifest, &Package{Kind: "package"}, onMatch) -} - -func (manifest *Manifest) IterateSlices(pkgName string, onMatch func(*Slice) error) (err error) { - return iteratePrefix(manifest, &Slice{Kind: "slice", Name: pkgName}, onMatch) -} - -func (manifest *Manifest) IterateContents(slice string, onMatch func(*Content) error) (err error) { - return iteratePrefix(manifest, &Content{Kind: "content", Slice: slice}, onMatch) -} - -// Validate checks that the Manifest is valid. Note that to do that it has to -// load practically the whole manifest into memory and unmarshall all the -// entries. -func Validate(manifest *Manifest) (err error) { - defer func() { - if err != nil { - err = fmt.Errorf("invalid manifest: %s", err) - } - }() - - pkgExist := map[string]bool{} - err = manifest.IteratePackages(func(pkg *Package) error { - pkgExist[pkg.Name] = true - return nil - }) - if err != nil { - return err - } - - sliceExist := map[string]bool{} - err = manifest.IterateSlices("", func(slice *Slice) error { - sk, err := setup.ParseSliceKey(slice.Name) - if err != nil { - return err - } - if !pkgExist[sk.Package] { - return fmt.Errorf("slice %s refers to missing package %q", slice.Name, sk.Package) - } - sliceExist[slice.Name] = true - return nil - }) - if err != nil { - return err - } - - pathToSlices := map[string][]string{} - err = manifest.IterateContents("", func(content *Content) error { - if !sliceExist[content.Slice] { - return fmt.Errorf("content path %q refers to missing slice %s", content.Path, content.Slice) - } - if !slices.Contains(pathToSlices[content.Path], content.Slice) { - pathToSlices[content.Path] = append(pathToSlices[content.Path], content.Slice) - } - return nil - }) - if err != nil { - return err - } - - done := map[string]bool{} - err = manifest.IteratePaths("", func(path *Path) error { - pathSlices, ok := pathToSlices[path.Path] - if !ok { - return fmt.Errorf("path %s has no matching entry in contents", path.Path) - } - slices.Sort(pathSlices) - slices.Sort(path.Slices) - if !slices.Equal(pathSlices, path.Slices) { - return fmt.Errorf("path %s and content have diverging slices: %q != %q", path.Path, path.Slices, pathSlices) - } - done[path.Path] = true - return nil - }) - if err != nil { - return err - } - - if len(done) != len(pathToSlices) { - for path := range pathToSlices { - return fmt.Errorf("content path %s has no matching entry in paths", path) - } - } - return nil -} - // FindPaths finds the paths marked with "generate:manifest" and // returns a map from the manifest path to all the slices that declare it. func FindPaths(slices []*setup.Slice) map[string][]*setup.Slice { @@ -189,7 +42,7 @@ type WriteOptions struct { func Write(options *WriteOptions, writer io.Writer) error { dbw := jsonwall.NewDBWriter(&jsonwall.DBWriterOptions{ - Schema: Schema, + Schema: manifest.Schema, }) err := fastValidate(options) @@ -216,32 +69,9 @@ func Write(options *WriteOptions, writer io.Writer) error { return err } -type prefixable interface { - Path | Content | Package | Slice -} - -func iteratePrefix[T prefixable](manifest *Manifest, prefix *T, onMatch func(*T) error) error { - iter, err := manifest.db.IteratePrefix(prefix) - if err != nil { - return err - } - for iter.Next() { - var val T - err := iter.Get(&val) - if err != nil { - return fmt.Errorf("cannot read manifest: %s", err) - } - err = onMatch(&val) - if err != nil { - return err - } - } - return nil -} - func manifestAddPackages(dbw *jsonwall.DBWriter, infos []*archive.PackageInfo) error { for _, info := range infos { - err := dbw.Add(&Package{ + err := dbw.Add(&manifest.Package{ Kind: "package", Name: info.Name, Version: info.Version, @@ -257,7 +87,7 @@ func manifestAddPackages(dbw *jsonwall.DBWriter, infos []*archive.PackageInfo) e func manifestAddSlices(dbw *jsonwall.DBWriter, slices []*setup.Slice) error { for _, slice := range slices { - err := dbw.Add(&Slice{ + err := dbw.Add(&manifest.Slice{ Kind: "slice", Name: slice.String(), }) @@ -272,7 +102,7 @@ func manifestAddReport(dbw *jsonwall.DBWriter, report *Report) error { for _, entry := range report.Entries { sliceNames := []string{} for slice := range entry.Slices { - err := dbw.Add(&Content{ + err := dbw.Add(&manifest.Content{ Kind: "content", Slice: slice.String(), Path: entry.Path, @@ -283,7 +113,7 @@ func manifestAddReport(dbw *jsonwall.DBWriter, report *Report) error { sliceNames = append(sliceNames, slice.String()) } sort.Strings(sliceNames) - err := dbw.Add(&Path{ + err := dbw.Add(&manifest.Path{ Kind: "path", Path: entry.Path, Mode: fmt.Sprintf("0%o", unixPerm(entry.Mode)), @@ -437,3 +267,78 @@ func validatePackage(pkg *archive.PackageInfo) (err error) { } return nil } + +// Validate checks that the Manifest is valid. Note that to do that it has to +// load practically the whole manifest into memory and unmarshall all the +// entries. +func Validate(mfest *manifest.Manifest) (err error) { + defer func() { + if err != nil { + err = fmt.Errorf("invalid manifest: %s", err) + } + }() + + pkgExist := map[string]bool{} + err = mfest.IteratePackages(func(pkg *manifest.Package) error { + pkgExist[pkg.Name] = true + return nil + }) + if err != nil { + return err + } + + sliceExist := map[string]bool{} + err = mfest.IterateSlices("", func(slice *manifest.Slice) error { + sk, err := apacheutil.ParseSliceKey(slice.Name) + if err != nil { + return err + } + if !pkgExist[sk.Package] { + return fmt.Errorf("slice %s refers to missing package %q", slice.Name, sk.Package) + } + sliceExist[slice.Name] = true + return nil + }) + if err != nil { + return err + } + + pathToSlices := map[string][]string{} + err = mfest.IterateContents("", func(content *manifest.Content) error { + if !sliceExist[content.Slice] { + return fmt.Errorf("content path %q refers to missing slice %s", content.Path, content.Slice) + } + if !slices.Contains(pathToSlices[content.Path], content.Slice) { + pathToSlices[content.Path] = append(pathToSlices[content.Path], content.Slice) + } + return nil + }) + if err != nil { + return err + } + + done := map[string]bool{} + err = mfest.IteratePaths("", func(path *manifest.Path) error { + pathSlices, ok := pathToSlices[path.Path] + if !ok { + return fmt.Errorf("path %s has no matching entry in contents", path.Path) + } + slices.Sort(pathSlices) + slices.Sort(path.Slices) + if !slices.Equal(pathSlices, path.Slices) { + return fmt.Errorf("path %s and content have diverging slices: %q != %q", path.Path, path.Slices, pathSlices) + } + done[path.Path] = true + return nil + }) + if err != nil { + return err + } + + if len(done) != len(pathToSlices) { + for path := range pathToSlices { + return fmt.Errorf("content path %s has no matching entry in paths", path) + } + } + return nil +} diff --git a/internal/manifest/manifest_test.go b/internal/manifestutil/manifestutil_test.go similarity index 65% rename from internal/manifest/manifest_test.go rename to internal/manifestutil/manifestutil_test.go index de2a71c8..2bab0a68 100644 --- a/internal/manifest/manifest_test.go +++ b/internal/manifestutil/manifestutil_test.go @@ -1,4 +1,4 @@ -package manifest_test +package manifestutil_test import ( "bytes" @@ -11,177 +11,13 @@ import ( . "gopkg.in/check.v1" + "github.com/canonical/chisel/internal/apachetestutil" "github.com/canonical/chisel/internal/archive" - "github.com/canonical/chisel/internal/manifest" + "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/setup" + "github.com/canonical/chisel/public/manifest" ) -type manifestContents struct { - Paths []*manifest.Path - Packages []*manifest.Package - Slices []*manifest.Slice - Contents []*manifest.Content -} - -var readManifestTests = []struct { - summary string - input string - mfest *manifestContents - valError string - readError string -}{{ - summary: "All types", - input: ` - {"jsonwall":"1.0","schema":"1.0","count":13} - {"kind":"content","slice":"pkg1_manifest","path":"/manifest/manifest.wall"} - {"kind":"content","slice":"pkg1_myslice","path":"/dir/file"} - {"kind":"content","slice":"pkg1_myslice","path":"/dir/file2"} - {"kind":"content","slice":"pkg1_myslice","path":"/dir/foo/bar/"} - {"kind":"content","slice":"pkg1_myslice","path":"/dir/hardlink"} - {"kind":"content","slice":"pkg1_myslice","path":"/dir/link/file"} - {"kind":"content","slice":"pkg2_myotherslice","path":"/dir/foo/bar/"} - {"kind":"package","name":"pkg1","version":"v1","sha256":"hash1","arch":"arch1"} - {"kind":"package","name":"pkg2","version":"v2","sha256":"hash2","arch":"arch2"} - {"kind":"path","path":"/dir/file","mode":"0644","slices":["pkg1_myslice"],"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_sha256":"8067926c032c090867013d14fb0eb21ae858344f62ad07086fd32375845c91a6","size":21} - {"kind":"path","path":"/dir/file2","mode":"0644","slices":["pkg1_myslice"],"sha256":"b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c","size":3,"inode":1} - {"kind":"path","path":"/dir/foo/bar/","mode":"01777","slices":["pkg2_myotherslice","pkg1_myslice"]} - {"kind":"path","path":"/dir/hardlink","mode":"0644","slices":["pkg1_myslice"],"sha256":"b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c","size":3,"inode":1} - {"kind":"path","path":"/dir/link/file","mode":"0644","slices":["pkg1_myslice"],"link":"/dir/file"} - {"kind":"path","path":"/manifest/manifest.wall","mode":"0644","slices":["pkg1_manifest"]} - {"kind":"slice","name":"pkg1_manifest"} - {"kind":"slice","name":"pkg1_myslice"} - {"kind":"slice","name":"pkg2_myotherslice"} - `, - mfest: &manifestContents{ - Paths: []*manifest.Path{ - {Kind: "path", Path: "/dir/file", Mode: "0644", Slices: []string{"pkg1_myslice"}, SHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", FinalSHA256: "8067926c032c090867013d14fb0eb21ae858344f62ad07086fd32375845c91a6", Size: 0x15, Link: ""}, - {Kind: "path", Path: "/dir/file2", Mode: "0644", Slices: []string{"pkg1_myslice"}, SHA256: "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c", Size: 0x03, Link: "", Inode: 0x01}, - {Kind: "path", Path: "/dir/foo/bar/", Mode: "01777", Slices: []string{"pkg2_myotherslice", "pkg1_myslice"}, SHA256: "", FinalSHA256: "", Size: 0x0, Link: ""}, - {Kind: "path", Path: "/dir/hardlink", Mode: "0644", Slices: []string{"pkg1_myslice"}, SHA256: "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c", Size: 0x03, Link: "", Inode: 0x01}, - {Kind: "path", Path: "/dir/link/file", Mode: "0644", Slices: []string{"pkg1_myslice"}, SHA256: "", FinalSHA256: "", Size: 0x0, Link: "/dir/file"}, - {Kind: "path", Path: "/manifest/manifest.wall", Mode: "0644", Slices: []string{"pkg1_manifest"}, SHA256: "", FinalSHA256: "", Size: 0x0, Link: ""}, - }, - Packages: []*manifest.Package{ - {Kind: "package", Name: "pkg1", Version: "v1", Digest: "hash1", Arch: "arch1"}, - {Kind: "package", Name: "pkg2", Version: "v2", Digest: "hash2", Arch: "arch2"}, - }, - Slices: []*manifest.Slice{ - {Kind: "slice", Name: "pkg1_manifest"}, - {Kind: "slice", Name: "pkg1_myslice"}, - {Kind: "slice", Name: "pkg2_myotherslice"}, - }, - Contents: []*manifest.Content{ - {Kind: "content", Slice: "pkg1_manifest", Path: "/manifest/manifest.wall"}, - {Kind: "content", Slice: "pkg1_myslice", Path: "/dir/file"}, - {Kind: "content", Slice: "pkg1_myslice", Path: "/dir/file2"}, - {Kind: "content", Slice: "pkg1_myslice", Path: "/dir/foo/bar/"}, - {Kind: "content", Slice: "pkg1_myslice", Path: "/dir/hardlink"}, - {Kind: "content", Slice: "pkg1_myslice", Path: "/dir/link/file"}, - {Kind: "content", Slice: "pkg2_myotherslice", Path: "/dir/foo/bar/"}, - }, - }, -}, { - summary: "Slice not found", - input: ` - {"jsonwall":"1.0","schema":"1.0","count":1} - {"kind":"content","slice":"pkg1_manifest","path":"/manifest/manifest.wall"} - `, - valError: `invalid manifest: content path "/manifest/manifest.wall" refers to missing slice pkg1_manifest`, -}, { - summary: "Package not found", - input: ` - {"jsonwall":"1.0","schema":"1.0","count":1} - {"kind":"slice","name":"pkg1_manifest"} - `, - valError: `invalid manifest: slice pkg1_manifest refers to missing package "pkg1"`, -}, { - summary: "Path not found in contents", - input: ` - {"jsonwall":"1.0","schema":"1.0","count":1} - {"kind":"path","path":"/dir/","mode":"01777","slices":["pkg1_myslice"]} - `, - valError: `invalid manifest: path /dir/ has no matching entry in contents`, -}, { - summary: "Content and path have different slices", - input: ` - {"jsonwall":"1.0","schema":"1.0","count":3} - {"kind":"content","slice":"pkg1_myotherslice","path":"/dir/"} - {"kind":"package","name":"pkg1","version":"v1","sha256":"hash1","arch":"arch1"} - {"kind":"path","path":"/dir/","mode":"01777","slices":["pkg1_myslice"]} - {"kind":"slice","name":"pkg1_myotherslice"} - `, - valError: `invalid manifest: path /dir/ and content have diverging slices: \["pkg1_myslice"\] != \["pkg1_myotherslice"\]`, -}, { - summary: "Content not found in paths", - input: ` - {"jsonwall":"1.0","schema":"1.0","count":3} - {"kind":"content","slice":"pkg1_myslice","path":"/dir/"} - {"kind":"package","name":"pkg1","version":"v1","sha256":"hash1","arch":"arch1"} - {"kind":"slice","name":"pkg1_myslice"} - `, - valError: `invalid manifest: content path /dir/ has no matching entry in paths`, -}, { - summary: "Malformed jsonwall", - input: ` - {"jsonwall":"1.0","schema":"1.0","count":1} - {"kind":"content", "not valid json" - `, - valError: `invalid manifest: cannot read manifest: unexpected end of JSON input`, -}, { - summary: "Unknown schema", - input: ` - {"jsonwall":"1.0","schema":"2.0","count":1} - {"kind":"package","name":"pkg1","version":"v1","sha256":"hash1","arch":"arch1"} - `, - readError: `cannot read manifest: unknown schema version "2.0"`, -}} - -func (s *S) TestManifestReadValidate(c *C) { - for _, test := range readManifestTests { - c.Logf("Summary: %s", test.summary) - - // Reindent the jsonwall to remove leading tabs in each line. - lines := strings.Split(strings.TrimSpace(test.input), "\n") - trimmedLines := make([]string, 0, len(lines)) - for _, line := range lines { - trimmedLines = append(trimmedLines, strings.TrimLeft(line, "\t")) - } - test.input = strings.Join(trimmedLines, "\n") - // Assert that the jsonwall is valid, for the test to be meaningful. - slices.Sort(trimmedLines) - orderedInput := strings.Join(trimmedLines, "\n") - c.Assert(test.input, DeepEquals, orderedInput, Commentf("input jsonwall lines should be ordered")) - - tmpDir := c.MkDir() - manifestPath := path.Join(tmpDir, "manifest.wall") - w, err := os.OpenFile(manifestPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) - c.Assert(err, IsNil) - _, err = w.Write([]byte(test.input)) - c.Assert(err, IsNil) - w.Close() - - r, err := os.OpenFile(manifestPath, os.O_RDONLY, 0644) - c.Assert(err, IsNil) - defer r.Close() - - mfest, err := manifest.Read(r) - if test.readError != "" { - c.Assert(err, ErrorMatches, test.readError) - continue - } - c.Assert(err, IsNil) - err = manifest.Validate(mfest) - if test.valError != "" { - c.Assert(err, ErrorMatches, test.valError) - continue - } - c.Assert(err, IsNil) - if test.mfest != nil { - c.Assert(dumpManifestContents(c, mfest), DeepEquals, test.mfest) - } - } -} - var findPathsTests = []struct { summary string slices []*setup.Slice @@ -255,7 +91,7 @@ func (s *S) TestFindPaths(c *C) { for _, test := range findPathsTests { c.Logf("Summary: %s", test.summary) - manifestSlices := manifest.FindPaths(test.slices) + manifestSlices := manifestutil.FindPaths(test.slices) slicesByName := map[string]*setup.Slice{} for _, slice := range test.slices { @@ -285,17 +121,17 @@ var slice2 = &setup.Slice{ var generateManifestTests = []struct { summary string - report *manifest.Report + report *manifestutil.Report packageInfo []*archive.PackageInfo selection []*setup.Slice - expected *manifestContents + expected *apachetestutil.ManifestContents error string }{{ summary: "Basic", selection: []*setup.Slice{slice1, slice2}, - report: &manifest.Report{ + report: &manifestutil.Report{ Root: "/", - Entries: map[string]manifest.ReportEntry{ + Entries: map[string]manifestutil.ReportEntry{ "/file": { Path: "/file", Mode: 0456, @@ -323,7 +159,7 @@ var generateManifestTests = []struct { Arch: "a2", SHA256: "s2", }}, - expected: &manifestContents{ + expected: &apachetestutil.ManifestContents{ Paths: []*manifest.Path{{ Kind: "path", Path: "/file", @@ -375,9 +211,9 @@ var generateManifestTests = []struct { }, }, { summary: "Missing slice", - report: &manifest.Report{ + report: &manifestutil.Report{ Root: "/", - Entries: map[string]manifest.ReportEntry{ + Entries: map[string]manifestutil.ReportEntry{ "/file": { Path: "/file", Mode: 0456, @@ -392,9 +228,9 @@ var generateManifestTests = []struct { error: `internal error: invalid manifest: path "/file" refers to missing slice package1_slice1`, }, { summary: "Missing package", - report: &manifest.Report{ + report: &manifestutil.Report{ Root: "/", - Entries: map[string]manifest.ReportEntry{ + Entries: map[string]manifestutil.ReportEntry{ "/file": { Path: "/file", Mode: 0456, @@ -409,9 +245,9 @@ var generateManifestTests = []struct { error: `internal error: invalid manifest: slice package1_slice1 refers to missing package "package1"`, }, { summary: "Invalid path: slices is empty", - report: &manifest.Report{ + report: &manifestutil.Report{ Root: "/", - Entries: map[string]manifest.ReportEntry{ + Entries: map[string]manifestutil.ReportEntry{ "/file": { Path: "/file", Mode: 0456, @@ -421,9 +257,9 @@ var generateManifestTests = []struct { error: `internal error: invalid manifest: path "/file" has invalid options: slices is empty`, }, { summary: "Invalid path: link set for symlink", - report: &manifest.Report{ + report: &manifestutil.Report{ Root: "/", - Entries: map[string]manifest.ReportEntry{ + Entries: map[string]manifestutil.ReportEntry{ "/link": { Path: "/link", Mode: 0456 | fs.ModeSymlink, @@ -434,9 +270,9 @@ var generateManifestTests = []struct { error: `internal error: invalid manifest: path "/link" has invalid options: link not set for symlink`, }, { summary: "Invalid path: sha256 set for symlink", - report: &manifest.Report{ + report: &manifestutil.Report{ Root: "/", - Entries: map[string]manifest.ReportEntry{ + Entries: map[string]manifestutil.ReportEntry{ "/link": { Path: "/link", Mode: 0456 | fs.ModeSymlink, @@ -449,9 +285,9 @@ var generateManifestTests = []struct { error: `internal error: invalid manifest: path "/link" has invalid options: sha256 set for symlink`, }, { summary: "Invalid path: final_sha256 set for symlink", - report: &manifest.Report{ + report: &manifestutil.Report{ Root: "/", - Entries: map[string]manifest.ReportEntry{ + Entries: map[string]manifestutil.ReportEntry{ "/link": { Path: "/link", Mode: 0456 | fs.ModeSymlink, @@ -464,9 +300,9 @@ var generateManifestTests = []struct { error: `internal error: invalid manifest: path "/link" has invalid options: final_sha256 set for symlink`, }, { summary: "Invalid path: size set for symlink", - report: &manifest.Report{ + report: &manifestutil.Report{ Root: "/", - Entries: map[string]manifest.ReportEntry{ + Entries: map[string]manifestutil.ReportEntry{ "/link": { Path: "/link", Mode: 0456 | fs.ModeSymlink, @@ -479,9 +315,9 @@ var generateManifestTests = []struct { error: `internal error: invalid manifest: path "/link" has invalid options: size set for symlink`, }, { summary: "Invalid path: link set for directory", - report: &manifest.Report{ + report: &manifestutil.Report{ Root: "/", - Entries: map[string]manifest.ReportEntry{ + Entries: map[string]manifestutil.ReportEntry{ "/dir": { Path: "/dir", Mode: 0456 | fs.ModeDir, @@ -493,9 +329,9 @@ var generateManifestTests = []struct { error: `internal error: invalid manifest: path "/dir" has invalid options: link set for directory`, }, { summary: "Invalid path: sha256 set for directory", - report: &manifest.Report{ + report: &manifestutil.Report{ Root: "/", - Entries: map[string]manifest.ReportEntry{ + Entries: map[string]manifestutil.ReportEntry{ "/dir": { Path: "/dir", Mode: 0456 | fs.ModeDir, @@ -507,9 +343,9 @@ var generateManifestTests = []struct { error: `internal error: invalid manifest: path "/dir" has invalid options: sha256 set for directory`, }, { summary: "Invalid path: final_sha256 set for directory", - report: &manifest.Report{ + report: &manifestutil.Report{ Root: "/", - Entries: map[string]manifest.ReportEntry{ + Entries: map[string]manifestutil.ReportEntry{ "/dir": { Path: "/dir", Mode: 0456 | fs.ModeDir, @@ -521,9 +357,9 @@ var generateManifestTests = []struct { error: `internal error: invalid manifest: path "/dir" has invalid options: final_sha256 set for directory`, }, { summary: "Invalid path: size set for directory", - report: &manifest.Report{ + report: &manifestutil.Report{ Root: "/", - Entries: map[string]manifest.ReportEntry{ + Entries: map[string]manifestutil.ReportEntry{ "/dir": { Path: "/dir", Mode: 0456 | fs.ModeDir, @@ -536,9 +372,9 @@ var generateManifestTests = []struct { }, { summary: "Basic hard link", selection: []*setup.Slice{slice1}, - report: &manifest.Report{ + report: &manifestutil.Report{ Root: "/", - Entries: map[string]manifest.ReportEntry{ + Entries: map[string]manifestutil.ReportEntry{ "/file": { Path: "/file", Mode: 0456, @@ -565,7 +401,7 @@ var generateManifestTests = []struct { Arch: "a1", SHA256: "s1", }}, - expected: &manifestContents{ + expected: &apachetestutil.ManifestContents{ Paths: []*manifest.Path{{ Kind: "path", Path: "/file", @@ -608,9 +444,9 @@ var generateManifestTests = []struct { }, }, { summary: "Skipped hard link id", - report: &manifest.Report{ + report: &manifestutil.Report{ Root: "/", - Entries: map[string]manifest.ReportEntry{ + Entries: map[string]manifestutil.ReportEntry{ "/file": { Path: "/file", Slices: map[*setup.Slice]bool{slice1: true}, @@ -621,9 +457,9 @@ var generateManifestTests = []struct { error: `internal error: invalid manifest: cannot find hard link id 1`, }, { summary: "Hard link group has only one path", - report: &manifest.Report{ + report: &manifestutil.Report{ Root: "/", - Entries: map[string]manifest.ReportEntry{ + Entries: map[string]manifestutil.ReportEntry{ "/file": { Path: "/file", Slices: map[*setup.Slice]bool{slice1: true}, @@ -634,9 +470,9 @@ var generateManifestTests = []struct { error: `internal error: invalid manifest: hard link group 1 has only one path: /file`, }, { summary: "Hard linked paths differ", - report: &manifest.Report{ + report: &manifestutil.Report{ Root: "/", - Entries: map[string]manifest.ReportEntry{ + Entries: map[string]manifestutil.ReportEntry{ "/file": { Path: "/file", Mode: 0456, @@ -705,13 +541,13 @@ func (s *S) TestGenerateManifests(c *C) { }} } - options := &manifest.WriteOptions{ + options := &manifestutil.WriteOptions{ PackageInfo: test.packageInfo, Selection: test.selection, Report: test.report, } var buffer bytes.Buffer - err := manifest.Write(options, &buffer) + err := manifestutil.Write(options, &buffer) if test.error != "" { c.Assert(err, ErrorMatches, test.error) continue @@ -719,21 +555,21 @@ func (s *S) TestGenerateManifests(c *C) { c.Assert(err, IsNil) mfest, err := manifest.Read(&buffer) c.Assert(err, IsNil) - err = manifest.Validate(mfest) + err = manifestutil.Validate(mfest) c.Assert(err, IsNil) - contents := dumpManifestContents(c, mfest) + contents := apachetestutil.DumpManifestContents(c, mfest) c.Assert(contents, DeepEquals, test.expected) } } func (s *S) TestGenerateNoManifests(c *C) { - report, err := manifest.NewReport("/") + report, err := manifestutil.NewReport("/") c.Assert(err, IsNil) - options := &manifest.WriteOptions{ + options := &manifestutil.WriteOptions{ Report: report, } var buffer bytes.Buffer - err = manifest.Write(options, &buffer) + err = manifestutil.Write(options, &buffer) c.Assert(err, IsNil) var reader io.Reader = &buffer @@ -743,40 +579,98 @@ func (s *S) TestGenerateNoManifests(c *C) { c.Assert(n, Equals, 0) } -func dumpManifestContents(c *C, mfest *manifest.Manifest) *manifestContents { - var slices []*manifest.Slice - err := mfest.IterateSlices("", func(slice *manifest.Slice) error { - slices = append(slices, slice) - return nil - }) - c.Assert(err, IsNil) +var validateManifestTests = []struct { + summary string + input string + mfest *apachetestutil.ManifestContents + error string +}{{ + summary: "Slice not found", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":1} + {"kind":"content","slice":"pkg1_manifest","path":"/manifest/manifest.wall"} + `, + error: `invalid manifest: content path "/manifest/manifest.wall" refers to missing slice pkg1_manifest`, +}, { + summary: "Package not found", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":1} + {"kind":"slice","name":"pkg1_manifest"} + `, + error: `invalid manifest: slice pkg1_manifest refers to missing package "pkg1"`, +}, { + summary: "Path not found in contents", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":1} + {"kind":"path","path":"/dir/","mode":"01777","slices":["pkg1_myslice"]} + `, + error: `invalid manifest: path /dir/ has no matching entry in contents`, +}, { + summary: "Content and path have different slices", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":3} + {"kind":"content","slice":"pkg1_myotherslice","path":"/dir/"} + {"kind":"package","name":"pkg1","version":"v1","sha256":"hash1","arch":"arch1"} + {"kind":"path","path":"/dir/","mode":"01777","slices":["pkg1_myslice"]} + {"kind":"slice","name":"pkg1_myotherslice"} + `, + error: `invalid manifest: path /dir/ and content have diverging slices: \["pkg1_myslice"\] != \["pkg1_myotherslice"\]`, +}, { + summary: "Content not found in paths", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":3} + {"kind":"content","slice":"pkg1_myslice","path":"/dir/"} + {"kind":"package","name":"pkg1","version":"v1","sha256":"hash1","arch":"arch1"} + {"kind":"slice","name":"pkg1_myslice"} + `, + error: `invalid manifest: content path /dir/ has no matching entry in paths`, +}, { + summary: "Malformed jsonwall", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":1} + {"kind":"content", "not valid json" + `, + error: `invalid manifest: cannot read manifest: unexpected end of JSON input`, +}} - var pkgs []*manifest.Package - err = mfest.IteratePackages(func(pkg *manifest.Package) error { - pkgs = append(pkgs, pkg) - return nil - }) - c.Assert(err, IsNil) +func (s *S) TestManifestValidate(c *C) { + for _, test := range validateManifestTests { + c.Logf("Summary: %s", test.summary) - var paths []*manifest.Path - err = mfest.IteratePaths("", func(path *manifest.Path) error { - paths = append(paths, path) - return nil - }) - c.Assert(err, IsNil) + // Reindent the jsonwall to remove leading tabs in each line. + lines := strings.Split(strings.TrimSpace(test.input), "\n") + trimmedLines := make([]string, 0, len(lines)) + for _, line := range lines { + trimmedLines = append(trimmedLines, strings.TrimLeft(line, "\t")) + } + test.input = strings.Join(trimmedLines, "\n") + // Assert that the jsonwall is valid, for the test to be meaningful. + slices.Sort(trimmedLines) + orderedInput := strings.Join(trimmedLines, "\n") + c.Assert(test.input, DeepEquals, orderedInput, Commentf("input jsonwall lines should be ordered")) - var contents []*manifest.Content - err = mfest.IterateContents("", func(content *manifest.Content) error { - contents = append(contents, content) - return nil - }) - c.Assert(err, IsNil) + tmpDir := c.MkDir() + manifestPath := path.Join(tmpDir, "manifest.wall") + w, err := os.OpenFile(manifestPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + c.Assert(err, IsNil) + _, err = w.Write([]byte(test.input)) + c.Assert(err, IsNil) + w.Close() + + r, err := os.OpenFile(manifestPath, os.O_RDONLY, 0644) + c.Assert(err, IsNil) + defer r.Close() - mc := manifestContents{ - Paths: paths, - Packages: pkgs, - Slices: slices, - Contents: contents, + mfest, err := manifest.Read(r) + c.Assert(err, IsNil) + err = manifestutil.Validate(mfest) + if test.error != "" { + c.Assert(err, ErrorMatches, test.error) + continue + } + c.Assert(err, IsNil) + if test.mfest != nil { + c.Assert(apachetestutil.DumpManifestContents(c, mfest), DeepEquals, test.mfest) + } } - return &mc } diff --git a/internal/manifest/report.go b/internal/manifestutil/report.go similarity index 99% rename from internal/manifest/report.go rename to internal/manifestutil/report.go index 4fe9915f..c294b1d2 100644 --- a/internal/manifest/report.go +++ b/internal/manifestutil/report.go @@ -1,4 +1,4 @@ -package manifest +package manifestutil import ( "fmt" diff --git a/internal/manifest/report_test.go b/internal/manifestutil/report_test.go similarity index 92% rename from internal/manifest/report_test.go rename to internal/manifestutil/report_test.go index 34e7d5e4..d56acddb 100644 --- a/internal/manifest/report_test.go +++ b/internal/manifestutil/report_test.go @@ -1,4 +1,4 @@ -package manifest_test +package manifestutil_test import ( "io/fs" @@ -6,7 +6,7 @@ import ( . "gopkg.in/check.v1" "github.com/canonical/chisel/internal/fsutil" - "github.com/canonical/chisel/internal/manifest" + "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/setup" ) @@ -70,13 +70,13 @@ var reportTests = []struct { add []sliceAndEntry mutate []*fsutil.Entry // indexed by path. - expected map[string]manifest.ReportEntry + expected map[string]manifestutil.ReportEntry // error after adding the last [sliceAndEntry]. err string }{{ summary: "Regular directory", add: []sliceAndEntry{{entry: sampleDir, slice: oneSlice}}, - expected: map[string]manifest.ReportEntry{ + expected: map[string]manifestutil.ReportEntry{ "/example-dir/": { Path: "/example-dir/", Mode: fs.ModeDir | 0654, @@ -89,7 +89,7 @@ var reportTests = []struct { {entry: sampleDir, slice: oneSlice}, {entry: sampleDir, slice: otherSlice}, }, - expected: map[string]manifest.ReportEntry{ + expected: map[string]manifestutil.ReportEntry{ "/example-dir/": { Path: "/example-dir/", Mode: fs.ModeDir | 0654, @@ -99,7 +99,7 @@ var reportTests = []struct { }, { summary: "Regular file", add: []sliceAndEntry{{entry: sampleFile, slice: oneSlice}}, - expected: map[string]manifest.ReportEntry{ + expected: map[string]manifestutil.ReportEntry{ "/example-file": { Path: "/example-file", Mode: 0777, @@ -111,7 +111,7 @@ var reportTests = []struct { }, { summary: "Regular file link", add: []sliceAndEntry{{entry: sampleSymlink, slice: oneSlice}}, - expected: map[string]manifest.ReportEntry{ + expected: map[string]manifestutil.ReportEntry{ "/example-link": { Path: "/example-link", Mode: fs.ModeSymlink | 0777, @@ -126,7 +126,7 @@ var reportTests = []struct { {entry: sampleDir, slice: oneSlice}, {entry: sampleFile, slice: otherSlice}, }, - expected: map[string]manifest.ReportEntry{ + expected: map[string]manifestutil.ReportEntry{ "/example-dir/": { Path: "/example-dir/", Mode: fs.ModeDir | 0654, @@ -147,7 +147,7 @@ var reportTests = []struct { {entry: sampleFile, slice: oneSlice}, {entry: sampleFile, slice: oneSlice}, }, - expected: map[string]manifest.ReportEntry{ + expected: map[string]manifestutil.ReportEntry{ "/example-file": { Path: "/example-file", Mode: 0777, @@ -231,7 +231,7 @@ var reportTests = []struct { {entry: sampleDir, slice: oneSlice}, }, mutate: []*fsutil.Entry{&sampleFileMutated}, - expected: map[string]manifest.ReportEntry{ + expected: map[string]manifestutil.ReportEntry{ "/example-dir/": { Path: "/example-dir/", Mode: fs.ModeDir | 0654, @@ -253,7 +253,7 @@ var reportTests = []struct { {entry: sampleFile, slice: oneSlice}, }, mutate: []*fsutil.Entry{&sampleFile}, - expected: map[string]manifest.ReportEntry{ + expected: map[string]manifestutil.ReportEntry{ "/example-file": { Path: "/example-file", Mode: 0777, @@ -278,7 +278,7 @@ var reportTests = []struct { add: []sliceAndEntry{ {entry: sampleFile, slice: oneSlice}, {entry: sampleHardLink, slice: oneSlice}}, - expected: map[string]manifest.ReportEntry{ + expected: map[string]manifestutil.ReportEntry{ "/example-file": { Path: "/example-file", Mode: 0777, @@ -320,7 +320,7 @@ var reportTests = []struct { }, slice: otherSlice, }}, - expected: map[string]manifest.ReportEntry{ + expected: map[string]manifestutil.ReportEntry{ "/example-file": { Path: "/example-file", Mode: 0777, @@ -359,7 +359,7 @@ var reportTests = []struct { func (s *S) TestReport(c *C) { for _, test := range reportTests { var err error - report, err := manifest.NewReport("/base/") + report, err := manifestutil.NewReport("/base/") c.Assert(err, IsNil) for _, si := range test.add { err = report.Add(si.slice, &si.entry) @@ -377,12 +377,12 @@ func (s *S) TestReport(c *C) { } func (s *S) TestRootRelativePath(c *C) { - _, err := manifest.NewReport("../base/") + _, err := manifestutil.NewReport("../base/") c.Assert(err, ErrorMatches, `cannot use relative path for report root: "../base/"`) } func (s *S) TestRootOnlySlash(c *C) { - report, err := manifest.NewReport("/") + report, err := manifestutil.NewReport("/") c.Assert(err, IsNil) c.Assert(report.Root, Equals, "/") } diff --git a/internal/manifestutil/suite_test.go b/internal/manifestutil/suite_test.go new file mode 100644 index 00000000..b0df3185 --- /dev/null +++ b/internal/manifestutil/suite_test.go @@ -0,0 +1,25 @@ +package manifestutil_test + +import ( + "testing" + + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/manifestutil" +) + +func Test(t *testing.T) { TestingT(t) } + +type S struct{} + +var _ = Suite(&S{}) + +func (s *S) SetUpTest(c *C) { + manifestutil.SetDebug(true) + manifestutil.SetLogger(c) +} + +func (s *S) TearDownTest(c *C) { + manifestutil.SetDebug(false) + manifestutil.SetLogger(nil) +} diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 66a3d56d..59e6a2ab 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -4,11 +4,11 @@ import ( "fmt" "os" "path/filepath" - "regexp" "strings" "golang.org/x/crypto/openpgp/packet" + "github.com/canonical/chisel/internal/apacheutil" "github.com/canonical/chisel/internal/strdist" ) @@ -103,13 +103,13 @@ func (pi *PathInfo) SameContent(other *PathInfo) bool { pi.Generate == other.Generate) } -type SliceKey struct { - Package string - Slice string +type SliceKey = apacheutil.SliceKey + +func ParseSliceKey(sliceKey string) (SliceKey, error) { + return apacheutil.ParseSliceKey(sliceKey) } -func (s *Slice) String() string { return s.Package + "_" + s.Name } -func (s SliceKey) String() string { return s.Package + "_" + s.Slice } +func (s *Slice) String() string { return s.Package + "_" + s.Name } // Selection holds the required configuration to create a Build for a selection // of slices from a Release. It's still an abstract proposal in the sense that @@ -294,23 +294,6 @@ func order(pkgs map[string]*Package, keys []SliceKey) ([]SliceKey, error) { return order, nil } -// fnameExp matches the slice definition file basename. -var fnameExp = regexp.MustCompile(`^([a-z0-9](?:-?[.a-z0-9+]){1,})\.yaml$`) - -// snameExp matches only the slice name, without the leading package name. -var snameExp = regexp.MustCompile(`^([a-z](?:-?[a-z0-9]){2,})$`) - -// knameExp matches the slice full name in pkg_slice format. -var knameExp = regexp.MustCompile(`^([a-z0-9](?:-?[.a-z0-9+]){1,})_([a-z](?:-?[a-z0-9]){2,})$`) - -func ParseSliceKey(sliceKey string) (SliceKey, error) { - match := knameExp.FindStringSubmatch(sliceKey) - if match == nil { - return SliceKey{}, fmt.Errorf("invalid slice reference: %q", sliceKey) - } - return SliceKey{match[1], match[2]}, nil -} - func readRelease(baseDir string) (*Release, error) { baseDir = filepath.Clean(baseDir) filePath := filepath.Join(baseDir, "chisel.yaml") @@ -346,7 +329,7 @@ func readSlices(release *Release, baseDir, dirName string) error { if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".yaml") { continue } - match := fnameExp.FindStringSubmatch(entry.Name()) + match := apacheutil.FnameExp.FindStringSubmatch(entry.Name()) if match == nil { return fmt.Errorf("invalid slice definition filename: %q", entry.Name()) } diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 2131038f..51b5b253 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -2261,93 +2261,6 @@ func (s *S) TestPackageYAMLFormat(c *C) { } } -var sliceKeyTests = []struct { - input string - expected setup.SliceKey - err string -}{{ - input: "foo_bar", - expected: setup.SliceKey{Package: "foo", Slice: "bar"}, -}, { - input: "fo_bar", - expected: setup.SliceKey{Package: "fo", Slice: "bar"}, -}, { - input: "1234_bar", - expected: setup.SliceKey{Package: "1234", Slice: "bar"}, -}, { - input: "foo1.1-2-3_bar", - expected: setup.SliceKey{Package: "foo1.1-2-3", Slice: "bar"}, -}, { - input: "foo-pkg_dashed-slice-name", - expected: setup.SliceKey{Package: "foo-pkg", Slice: "dashed-slice-name"}, -}, { - input: "foo+_bar", - expected: setup.SliceKey{Package: "foo+", Slice: "bar"}, -}, { - input: "foo_slice123", - expected: setup.SliceKey{Package: "foo", Slice: "slice123"}, -}, { - input: "g++_bins", - expected: setup.SliceKey{Package: "g++", Slice: "bins"}, -}, { - input: "a+_bar", - expected: setup.SliceKey{Package: "a+", Slice: "bar"}, -}, { - input: "a._bar", - expected: setup.SliceKey{Package: "a.", Slice: "bar"}, -}, { - input: "foo_ba", - err: `invalid slice reference: "foo_ba"`, -}, { - input: "f_bar", - err: `invalid slice reference: "f_bar"`, -}, { - input: "1234_789", - err: `invalid slice reference: "1234_789"`, -}, { - input: "foo_bar.x.y", - err: `invalid slice reference: "foo_bar.x.y"`, -}, { - input: "foo-_-bar", - err: `invalid slice reference: "foo-_-bar"`, -}, { - input: "foo_bar-", - err: `invalid slice reference: "foo_bar-"`, -}, { - input: "foo-_bar", - err: `invalid slice reference: "foo-_bar"`, -}, { - input: "-foo_bar", - err: `invalid slice reference: "-foo_bar"`, -}, { - input: "foo_bar_baz", - err: `invalid slice reference: "foo_bar_baz"`, -}, { - input: "a-_bar", - err: `invalid slice reference: "a-_bar"`, -}, { - input: "+++_bar", - err: `invalid slice reference: "\+\+\+_bar"`, -}, { - input: "..._bar", - err: `invalid slice reference: "\.\.\._bar"`, -}, { - input: "white space_no-whitespace", - err: `invalid slice reference: "white space_no-whitespace"`, -}} - -func (s *S) TestParseSliceKey(c *C) { - for _, test := range sliceKeyTests { - key, err := setup.ParseSliceKey(test.input) - if test.err != "" { - c.Assert(err, ErrorMatches, test.err) - continue - } - c.Assert(err, IsNil) - c.Assert(key, DeepEquals, test.expected) - } -} - // This is an awkward test because right now the fact Generate is considered // by SameContent is irrelevant to the implementation, because the code path // happens to not touch it. More important than this test, there's an entry diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go index ced00f2e..8ae3b3b3 100644 --- a/internal/setup/yaml.go +++ b/internal/setup/yaml.go @@ -10,6 +10,7 @@ import ( "golang.org/x/crypto/openpgp/packet" "gopkg.in/yaml.v3" + "github.com/canonical/chisel/internal/apacheutil" "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/pgputil" @@ -303,7 +304,7 @@ func parsePackage(baseDir, pkgName, pkgPath string, data []byte) (*Package, erro zeroPath := yamlPath{} for sliceName, yamlSlice := range yamlPkg.Slices { - match := snameExp.FindStringSubmatch(sliceName) + match := apacheutil.SnameExp.FindStringSubmatch(sliceName) if match == nil { return nil, fmt.Errorf("invalid slice name %q in %s", sliceName, pkgPath) } diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 39f4fc4a..dca25a0b 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -18,7 +18,7 @@ import ( "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/fsutil" - "github.com/canonical/chisel/internal/manifest" + "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/scripts" "github.com/canonical/chisel/internal/setup" ) @@ -157,7 +157,7 @@ func Run(options *RunOptions) error { // listed as until: mutate in all the slices that reference them. knownPaths := map[string]pathData{} addKnownPath(knownPaths, "/", pathData{}) - report, err := manifest.NewReport(targetDir) + report, err := manifestutil.NewReport(targetDir) if err != nil { return fmt.Errorf("internal error: cannot create report: %w", err) } @@ -325,8 +325,8 @@ func Run(options *RunOptions) error { } func generateManifests(targetDir string, selection *setup.Selection, - report *manifest.Report, pkgInfos []*archive.PackageInfo) error { - manifestSlices := manifest.FindPaths(selection.Slices) + report *manifestutil.Report, pkgInfos []*archive.PackageInfo) error { + manifestSlices := manifestutil.FindPaths(selection.Slices) if len(manifestSlices) == 0 { // Nothing to do. return nil @@ -358,12 +358,12 @@ func generateManifests(targetDir string, selection *setup.Selection, return err } defer w.Close() - writeOptions := &manifest.WriteOptions{ + writeOptions := &manifestutil.WriteOptions{ PackageInfo: pkgInfos, Selection: selection.Slices, Report: report, } - err = manifest.Write(writeOptions, w) + err = manifestutil.Write(writeOptions, w) return err } diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index df4d1c05..e8ffecac 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -15,10 +15,11 @@ import ( . "gopkg.in/check.v1" "github.com/canonical/chisel/internal/archive" - "github.com/canonical/chisel/internal/manifest" + "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" "github.com/canonical/chisel/internal/testutil" + "github.com/canonical/chisel/public/manifest" ) var ( @@ -1971,7 +1972,7 @@ func readManifest(c *C, targetDir, manifestPath string) *manifest.Manifest { defer r.Close() mfest, err := manifest.Read(r) c.Assert(err, IsNil) - err = manifest.Validate(mfest) + err = manifestutil.Validate(mfest) c.Assert(err, IsNil) // Assert that the mode of the manifest.wall file matches the one recorded diff --git a/internal/jsonwall/jsonwall.go b/public/jsonwall/jsonwall.go similarity index 99% rename from internal/jsonwall/jsonwall.go rename to public/jsonwall/jsonwall.go index b5c94043..97717070 100644 --- a/internal/jsonwall/jsonwall.go +++ b/public/jsonwall/jsonwall.go @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 + // Package jsonwall provides an interface to work with database files in the simple // "jsonwall" format, which consists of a text file with one JSON object per line, // where both the individual JSON fields and the lines themselves are sorted to diff --git a/internal/jsonwall/jsonwall_test.go b/public/jsonwall/jsonwall_test.go similarity index 98% rename from internal/jsonwall/jsonwall_test.go rename to public/jsonwall/jsonwall_test.go index 9e436ed1..57bd1c69 100644 --- a/internal/jsonwall/jsonwall_test.go +++ b/public/jsonwall/jsonwall_test.go @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 + package jsonwall_test import ( @@ -5,7 +7,7 @@ import ( "bytes" - "github.com/canonical/chisel/internal/jsonwall" + "github.com/canonical/chisel/public/jsonwall" ) type DataType struct { diff --git a/public/jsonwall/log.go b/public/jsonwall/log.go new file mode 100644 index 00000000..5a3d1d4d --- /dev/null +++ b/public/jsonwall/log.go @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: Apache-2.0 + +package jsonwall + +import ( + "fmt" + "sync" +) + +// Avoid importing the log type information unnecessarily. There's a small cost +// associated with using an interface rather than the type. Depending on how +// often the logger is plugged in, it would be worth using the type instead. +type log_Logger interface { + Output(calldepth int, s string) error +} + +var globalLoggerLock sync.Mutex +var globalLogger log_Logger +var globalDebug bool + +// Specify the *log.Logger object where log messages should be sent to. +func SetLogger(logger log_Logger) { + globalLoggerLock.Lock() + globalLogger = logger + globalLoggerLock.Unlock() +} + +// Enable the delivery of debug messages to the logger. Only meaningful +// if a logger is also set. +func SetDebug(debug bool) { + globalLoggerLock.Lock() + globalDebug = debug + globalLoggerLock.Unlock() +} + +// logf sends to the logger registered via SetLogger the string resulting +// from running format and args through Sprintf. +func logf(format string, args ...interface{}) { + globalLoggerLock.Lock() + defer globalLoggerLock.Unlock() + if globalLogger != nil { + globalLogger.Output(2, fmt.Sprintf(format, args...)) + } +} + +// debugf sends to the logger registered via SetLogger the string resulting +// from running format and args through Sprintf, but only if debugging was +// enabled via SetDebug. +func debugf(format string, args ...interface{}) { + globalLoggerLock.Lock() + defer globalLoggerLock.Unlock() + if globalDebug && globalLogger != nil { + globalLogger.Output(2, fmt.Sprintf(format, args...)) + } +} diff --git a/internal/jsonwall/suite_test.go b/public/jsonwall/suite_test.go similarity index 78% rename from internal/jsonwall/suite_test.go rename to public/jsonwall/suite_test.go index 3f7fd9fe..b4f1edf3 100644 --- a/internal/jsonwall/suite_test.go +++ b/public/jsonwall/suite_test.go @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 + package jsonwall_test import ( @@ -5,7 +7,7 @@ import ( . "gopkg.in/check.v1" - "github.com/canonical/chisel/internal/jsonwall" + "github.com/canonical/chisel/public/jsonwall" ) func Test(t *testing.T) { TestingT(t) } diff --git a/internal/manifest/log.go b/public/manifest/log.go similarity index 97% rename from internal/manifest/log.go rename to public/manifest/log.go index 7fe0f91c..36641bc4 100644 --- a/internal/manifest/log.go +++ b/public/manifest/log.go @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 + package manifest import ( diff --git a/public/manifest/manifest.go b/public/manifest/manifest.go new file mode 100644 index 00000000..1e4809b8 --- /dev/null +++ b/public/manifest/manifest.go @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: Apache-2.0 + +package manifest + +import ( + "fmt" + "io" + + "github.com/canonical/chisel/public/jsonwall" +) + +const Schema = "1.0" + +type Package struct { + Kind string `json:"kind"` + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` + Digest string `json:"sha256,omitempty"` + Arch string `json:"arch,omitempty"` +} + +type Slice struct { + Kind string `json:"kind"` + Name string `json:"name,omitempty"` +} + +type Path struct { + Kind string `json:"kind"` + Path string `json:"path,omitempty"` + Mode string `json:"mode,omitempty"` + Slices []string `json:"slices,omitempty"` + SHA256 string `json:"sha256,omitempty"` + FinalSHA256 string `json:"final_sha256,omitempty"` + Size uint64 `json:"size,omitempty"` + Link string `json:"link,omitempty"` + Inode uint64 `json:"inode,omitempty"` +} + +type Content struct { + Kind string `json:"kind"` + Slice string `json:"slice,omitempty"` + Path string `json:"path,omitempty"` +} + +type Manifest struct { + db *jsonwall.DB +} + +// Read loads a Manifest without performing any validation. The data is assumed +// to be both valid jsonwall and a valid Manifest (see Validate). +func Read(reader io.Reader) (manifest *Manifest, err error) { + defer func() { + if err != nil { + err = fmt.Errorf("cannot read manifest: %s", err) + } + }() + + db, err := jsonwall.ReadDB(reader) + if err != nil { + return nil, err + } + mfestSchema := db.Schema() + if mfestSchema != Schema { + return nil, fmt.Errorf("unknown schema version %q", mfestSchema) + } + + manifest = &Manifest{db: db} + return manifest, nil +} + +func (manifest *Manifest) IteratePaths(pathPrefix string, onMatch func(*Path) error) (err error) { + return iteratePrefix(manifest, &Path{Kind: "path", Path: pathPrefix}, onMatch) +} + +func (manifest *Manifest) IteratePackages(onMatch func(*Package) error) (err error) { + return iteratePrefix(manifest, &Package{Kind: "package"}, onMatch) +} + +func (manifest *Manifest) IterateSlices(pkgName string, onMatch func(*Slice) error) (err error) { + return iteratePrefix(manifest, &Slice{Kind: "slice", Name: pkgName}, onMatch) +} + +func (manifest *Manifest) IterateContents(slice string, onMatch func(*Content) error) (err error) { + return iteratePrefix(manifest, &Content{Kind: "content", Slice: slice}, onMatch) +} + +type prefixable interface { + Path | Content | Package | Slice +} + +func iteratePrefix[T prefixable](manifest *Manifest, prefix *T, onMatch func(*T) error) error { + iter, err := manifest.db.IteratePrefix(prefix) + if err != nil { + return err + } + for iter.Next() { + var val T + err := iter.Get(&val) + if err != nil { + return fmt.Errorf("cannot read manifest: %s", err) + } + err = onMatch(&val) + if err != nil { + return err + } + } + return nil +} diff --git a/public/manifest/manifest_test.go b/public/manifest/manifest_test.go new file mode 100644 index 00000000..d710e121 --- /dev/null +++ b/public/manifest/manifest_test.go @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: Apache-2.0 + +package manifest_test + +import ( + "os" + "path" + "slices" + "strings" + + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/apachetestutil" + "github.com/canonical/chisel/public/manifest" +) + +var readManifestTests = []struct { + summary string + input string + mfest *apachetestutil.ManifestContents + error string +}{{ + summary: "All types", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":13} + {"kind":"content","slice":"pkg1_manifest","path":"/manifest/manifest.wall"} + {"kind":"content","slice":"pkg1_myslice","path":"/dir/file"} + {"kind":"content","slice":"pkg1_myslice","path":"/dir/file2"} + {"kind":"content","slice":"pkg1_myslice","path":"/dir/foo/bar/"} + {"kind":"content","slice":"pkg1_myslice","path":"/dir/hardlink"} + {"kind":"content","slice":"pkg1_myslice","path":"/dir/link/file"} + {"kind":"content","slice":"pkg2_myotherslice","path":"/dir/foo/bar/"} + {"kind":"package","name":"pkg1","version":"v1","sha256":"hash1","arch":"arch1"} + {"kind":"package","name":"pkg2","version":"v2","sha256":"hash2","arch":"arch2"} + {"kind":"path","path":"/dir/file","mode":"0644","slices":["pkg1_myslice"],"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_sha256":"8067926c032c090867013d14fb0eb21ae858344f62ad07086fd32375845c91a6","size":21} + {"kind":"path","path":"/dir/file2","mode":"0644","slices":["pkg1_myslice"],"sha256":"b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c","size":3,"inode":1} + {"kind":"path","path":"/dir/foo/bar/","mode":"01777","slices":["pkg2_myotherslice","pkg1_myslice"]} + {"kind":"path","path":"/dir/hardlink","mode":"0644","slices":["pkg1_myslice"],"sha256":"b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c","size":3,"inode":1} + {"kind":"path","path":"/dir/link/file","mode":"0644","slices":["pkg1_myslice"],"link":"/dir/file"} + {"kind":"path","path":"/manifest/manifest.wall","mode":"0644","slices":["pkg1_manifest"]} + {"kind":"slice","name":"pkg1_manifest"} + {"kind":"slice","name":"pkg1_myslice"} + {"kind":"slice","name":"pkg2_myotherslice"} + `, + mfest: &apachetestutil.ManifestContents{ + Paths: []*manifest.Path{ + {Kind: "path", Path: "/dir/file", Mode: "0644", Slices: []string{"pkg1_myslice"}, SHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", FinalSHA256: "8067926c032c090867013d14fb0eb21ae858344f62ad07086fd32375845c91a6", Size: 0x15, Link: ""}, + {Kind: "path", Path: "/dir/file2", Mode: "0644", Slices: []string{"pkg1_myslice"}, SHA256: "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c", Size: 0x03, Link: "", Inode: 0x01}, + {Kind: "path", Path: "/dir/foo/bar/", Mode: "01777", Slices: []string{"pkg2_myotherslice", "pkg1_myslice"}, SHA256: "", FinalSHA256: "", Size: 0x0, Link: ""}, + {Kind: "path", Path: "/dir/hardlink", Mode: "0644", Slices: []string{"pkg1_myslice"}, SHA256: "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c", Size: 0x03, Link: "", Inode: 0x01}, + {Kind: "path", Path: "/dir/link/file", Mode: "0644", Slices: []string{"pkg1_myslice"}, SHA256: "", FinalSHA256: "", Size: 0x0, Link: "/dir/file"}, + {Kind: "path", Path: "/manifest/manifest.wall", Mode: "0644", Slices: []string{"pkg1_manifest"}, SHA256: "", FinalSHA256: "", Size: 0x0, Link: ""}, + }, + Packages: []*manifest.Package{ + {Kind: "package", Name: "pkg1", Version: "v1", Digest: "hash1", Arch: "arch1"}, + {Kind: "package", Name: "pkg2", Version: "v2", Digest: "hash2", Arch: "arch2"}, + }, + Slices: []*manifest.Slice{ + {Kind: "slice", Name: "pkg1_manifest"}, + {Kind: "slice", Name: "pkg1_myslice"}, + {Kind: "slice", Name: "pkg2_myotherslice"}, + }, + Contents: []*manifest.Content{ + {Kind: "content", Slice: "pkg1_manifest", Path: "/manifest/manifest.wall"}, + {Kind: "content", Slice: "pkg1_myslice", Path: "/dir/file"}, + {Kind: "content", Slice: "pkg1_myslice", Path: "/dir/file2"}, + {Kind: "content", Slice: "pkg1_myslice", Path: "/dir/foo/bar/"}, + {Kind: "content", Slice: "pkg1_myslice", Path: "/dir/hardlink"}, + {Kind: "content", Slice: "pkg1_myslice", Path: "/dir/link/file"}, + {Kind: "content", Slice: "pkg2_myotherslice", Path: "/dir/foo/bar/"}, + }, + }, +}, { + summary: "Unknown schema", + input: ` + {"jsonwall":"1.0","schema":"2.0","count":1} + {"kind":"package","name":"pkg1","version":"v1","sha256":"hash1","arch":"arch1"} + `, + error: `cannot read manifest: unknown schema version "2.0"`, +}} + +func (s *S) TestManifestRead(c *C) { + for _, test := range readManifestTests { + c.Logf("Summary: %s", test.summary) + + // Reindent the jsonwall to remove leading tabs in each line. + lines := strings.Split(strings.TrimSpace(test.input), "\n") + trimmedLines := make([]string, 0, len(lines)) + for _, line := range lines { + trimmedLines = append(trimmedLines, strings.TrimLeft(line, "\t")) + } + test.input = strings.Join(trimmedLines, "\n") + // Assert that the jsonwall is valid, for the test to be meaningful. + slices.Sort(trimmedLines) + orderedInput := strings.Join(trimmedLines, "\n") + c.Assert(test.input, DeepEquals, orderedInput, Commentf("input jsonwall lines should be ordered")) + + tmpDir := c.MkDir() + manifestPath := path.Join(tmpDir, "manifest.wall") + w, err := os.OpenFile(manifestPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + c.Assert(err, IsNil) + _, err = w.Write([]byte(test.input)) + c.Assert(err, IsNil) + w.Close() + + r, err := os.OpenFile(manifestPath, os.O_RDONLY, 0644) + c.Assert(err, IsNil) + defer r.Close() + + mfest, err := manifest.Read(r) + if test.error != "" { + c.Assert(err, ErrorMatches, test.error) + continue + } + c.Assert(err, IsNil) + if test.mfest != nil { + c.Assert(apachetestutil.DumpManifestContents(c, mfest), DeepEquals, test.mfest) + } + } +} diff --git a/internal/manifest/suite_test.go b/public/manifest/suite_test.go similarity index 53% rename from internal/manifest/suite_test.go rename to public/manifest/suite_test.go index 9450841d..0e0f3ea4 100644 --- a/internal/manifest/suite_test.go +++ b/public/manifest/suite_test.go @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 + package manifest_test import ( @@ -5,7 +7,7 @@ import ( . "gopkg.in/check.v1" - "github.com/canonical/chisel/internal/slicer" + "github.com/canonical/chisel/public/manifest" ) func Test(t *testing.T) { TestingT(t) } @@ -15,11 +17,11 @@ type S struct{} var _ = Suite(&S{}) func (s *S) SetUpTest(c *C) { - slicer.SetDebug(true) - slicer.SetLogger(c) + manifest.SetDebug(true) + manifest.SetLogger(c) } func (s *S) TearDownTest(c *C) { - slicer.SetDebug(false) - slicer.SetLogger(nil) + manifest.SetDebug(false) + manifest.SetLogger(nil) }