feat: public manifest reading and jsonwall Go packages (#182)

.github/scripts/external-packages-license-check.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"io/fs"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+var licenseRegexp = regexp.MustCompile("// SPDX-License-Identifier: ([^\\s]*)$")
+
+func fileLicense(path string) (string, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+	scanner := bufio.NewScanner(file)
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		matches := licenseRegexp.FindStringSubmatch(line)
+		if len(matches) > 0 {
+			return matches[1], nil
+		}
+	}
+
+	return "", nil
+}
+
+func checkDirLicense(path string, valid string) error {
+	return filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error {
+		if !strings.HasSuffix(path, ".go") {
+			return nil
+		}
+		license, err := fileLicense(path)
+		if err != nil {
+			return err
+		}
+		if license == "" {
+			return fmt.Errorf("cannot find a valid license in %q", path)
+		}
+		if license != valid {
+			return fmt.Errorf("expected %q to be %q, got %q", path, valid, license)
+		}
+		return nil
+	})
+}
+
+func run() error {
+	// Check external packages licenses.
+	err := checkDirLicense("public", "Apache-2.0")
+	if err != nil {
+		return fmt.Errorf("invalid license in exported package: %s", err)
+	}
+
+	// Check the internal dependencies of the external packages.
+	output, err := exec.Command("sh", "-c", "go list -deps -test ./public/*").Output()
+	if err != nil {
+		return err
+	}
+	lines := strings.Split(string(output), "\n")
+	var internalPkgs []string
+	for _, line := range lines {
+		if strings.Contains(line, "github.com/canonical/chisel/internal") {
+			internalPkgs = append(internalPkgs, strings.TrimPrefix(line, "github.com/canonical/chisel/"))
+		}
+	}
+	for _, pkg := range internalPkgs {
+		err := checkDirLicense(pkg, "Apache-2.0")
+		if err != nil {
+			return fmt.Errorf("invalid license in depedency %q: %s", pkg, err)
+		}
+	}
+
+	return nil
+}
+
+func main() {
+	err := run()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err)
+		os.Exit(1)
+	}
+}

.github/workflows/license.yaml

@@ -0,0 +1,22 @@
+name: License check
+
+on:
+  workflow_dispatch:
+  push:
+  pull_request:
+    branches: [main]
+
+jobs:
+  external-packages:
+    runs-on: ubuntu-22.04
+    name: External packages license check
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Run license check
+        run: |
+          go run .github/scripts/external-packages-license-check.go

internal/apachetestutil/manifest.go

@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package apachetestutil
+
+import (
+	"gopkg.in/check.v1"
+
+	"github.com/canonical/chisel/public/manifest"
+)
+
+type ManifestContents struct {
+	Paths    []*manifest.Path
+	Packages []*manifest.Package
+	Slices   []*manifest.Slice
+	Contents []*manifest.Content
+}
+
+func DumpManifestContents(c *check.C, mfest *manifest.Manifest) *ManifestContents {
+	var slices []*manifest.Slice
+	err := mfest.IterateSlices("", func(slice *manifest.Slice) error {
+		slices = append(slices, slice)
+		return nil
+	})
+	c.Assert(err, check.IsNil)
+
+	var pkgs []*manifest.Package
+	err = mfest.IteratePackages(func(pkg *manifest.Package) error {
+		pkgs = append(pkgs, pkg)
+		return nil
+	})
+	c.Assert(err, check.IsNil)
+
+	var paths []*manifest.Path
+	err = mfest.IteratePaths("", func(path *manifest.Path) error {
+		paths = append(paths, path)
+		return nil
+	})
+	c.Assert(err, check.IsNil)
+
+	var contents []*manifest.Content
+	err = mfest.IterateContents("", func(content *manifest.Content) error {
+		contents = append(contents, content)
+		return nil
+	})
+	c.Assert(err, check.IsNil)
+
+	mc := ManifestContents{
+		Paths:    paths,
+		Packages: pkgs,
+		Slices:   slices,
+		Contents: contents,
+	}
+	return &mc
+}

internal/apacheutil/log.go

@@ -0,0 +1,55 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package apacheutil
+
+import (
+	"fmt"
+	"sync"
+)
+
+// Avoid importing the log type information unnecessarily.  There's a small cost
+// associated with using an interface rather than the type.  Depending on how
+// often the logger is plugged in, it would be worth using the type instead.
+type log_Logger interface {
+	Output(calldepth int, s string) error
+}
+
+var globalLoggerLock sync.Mutex
+var globalLogger log_Logger
+var globalDebug bool
+
+// Specify the *log.Logger object where log messages should be sent to.
+func SetLogger(logger log_Logger) {
+	globalLoggerLock.Lock()
+	globalLogger = logger
+	globalLoggerLock.Unlock()
+}
+
+// Enable the delivery of debug messages to the logger.  Only meaningful
+// if a logger is also set.
+func SetDebug(debug bool) {
+	globalLoggerLock.Lock()
+	globalDebug = debug
+	globalLoggerLock.Unlock()
+}
+
+// logf sends to the logger registered via SetLogger the string resulting
+// from running format and args through Sprintf.
+func logf(format string, args ...interface{}) {
+	globalLoggerLock.Lock()
+	defer globalLoggerLock.Unlock()
+	if globalLogger != nil {
+		globalLogger.Output(2, fmt.Sprintf(format, args...))
+	}
+}
+
+// debugf sends to the logger registered via SetLogger the string resulting
+// from running format and args through Sprintf, but only if debugging was
+// enabled via SetDebug.
+func debugf(format string, args ...interface{}) {
+	globalLoggerLock.Lock()
+	defer globalLoggerLock.Unlock()
+	if globalDebug && globalLogger != nil {
+		globalLogger.Output(2, fmt.Sprintf(format, args...))
+	}
+}

internal/apacheutil/suite_test.go

@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package apacheutil_test
+
+import (
+	"testing"
+
+	. "gopkg.in/check.v1"
+
+	"github.com/canonical/chisel/internal/apacheutil"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type S struct{}
+
+var _ = Suite(&S{})
+
+func (s *S) SetUpTest(c *C) {
+	apacheutil.SetDebug(true)
+	apacheutil.SetLogger(c)
+}
+
+func (s *S) TearDownTest(c *C) {
+	apacheutil.SetDebug(false)
+	apacheutil.SetLogger(nil)
+}

internal/apacheutil/util.go

@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package apacheutil
+
+import (
+	"fmt"
+	"regexp"
+)
+
+type SliceKey struct {
+	Package string
+	Slice   string
+}
+
+func (s SliceKey) String() string { return s.Package + "_" + s.Slice }
+
+// FnameExp matches the slice definition file basename.
+var FnameExp = regexp.MustCompile(`^([a-z0-9](?:-?[.a-z0-9+]){1,})\.yaml$`)
+
+// SnameExp matches only the slice name, without the leading package name.
+var SnameExp = regexp.MustCompile(`^([a-z](?:-?[a-z0-9]){2,})$`)
+
+// knameExp matches the slice full name in pkg_slice format.
+var knameExp = regexp.MustCompile(`^([a-z0-9](?:-?[.a-z0-9+]){1,})_([a-z](?:-?[a-z0-9]){2,})$`)
+
+func ParseSliceKey(sliceKey string) (SliceKey, error) {
+	match := knameExp.FindStringSubmatch(sliceKey)
+	if match == nil {
+		return SliceKey{}, fmt.Errorf("invalid slice reference: %q", sliceKey)
+	}
+	return SliceKey{match[1], match[2]}, nil
+}

internal/apacheutil/util_test.go

@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package apacheutil_test
+
+import (
+	. "gopkg.in/check.v1"
+
+	"github.com/canonical/chisel/internal/apacheutil"
+)
+
+var sliceKeyTests = []struct {
+	input    string
+	expected apacheutil.SliceKey
+	err      string
+}{{
+	input:    "foo_bar",
+	expected: apacheutil.SliceKey{Package: "foo", Slice: "bar"},
+}, {
+	input:    "fo_bar",
+	expected: apacheutil.SliceKey{Package: "fo", Slice: "bar"},
+}, {
+	input:    "1234_bar",
+	expected: apacheutil.SliceKey{Package: "1234", Slice: "bar"},
+}, {
+	input:    "foo1.1-2-3_bar",
+	expected: apacheutil.SliceKey{Package: "foo1.1-2-3", Slice: "bar"},
+}, {
+	input:    "foo-pkg_dashed-slice-name",
+	expected: apacheutil.SliceKey{Package: "foo-pkg", Slice: "dashed-slice-name"},
+}, {
+	input:    "foo+_bar",
+	expected: apacheutil.SliceKey{Package: "foo+", Slice: "bar"},
+}, {
+	input:    "foo_slice123",
+	expected: apacheutil.SliceKey{Package: "foo", Slice: "slice123"},
+}, {
+	input:    "g++_bins",
+	expected: apacheutil.SliceKey{Package: "g++", Slice: "bins"},
+}, {
+	input:    "a+_bar",
+	expected: apacheutil.SliceKey{Package: "a+", Slice: "bar"},
+}, {
+	input:    "a._bar",
+	expected: apacheutil.SliceKey{Package: "a.", Slice: "bar"},
+}, {
+	input: "foo_ba",
+	err:   `invalid slice reference: "foo_ba"`,
+}, {
+	input: "f_bar",
+	err:   `invalid slice reference: "f_bar"`,
+}, {
+	input: "1234_789",
+	err:   `invalid slice reference: "1234_789"`,
+}, {
+	input: "foo_bar.x.y",
+	err:   `invalid slice reference: "foo_bar.x.y"`,
+}, {
+	input: "foo-_-bar",
+	err:   `invalid slice reference: "foo-_-bar"`,
+}, {
+	input: "foo_bar-",
+	err:   `invalid slice reference: "foo_bar-"`,
+}, {
+	input: "foo-_bar",
+	err:   `invalid slice reference: "foo-_bar"`,
+}, {
+	input: "-foo_bar",
+	err:   `invalid slice reference: "-foo_bar"`,
+}, {
+	input: "foo_bar_baz",
+	err:   `invalid slice reference: "foo_bar_baz"`,
+}, {
+	input: "a-_bar",
+	err:   `invalid slice reference: "a-_bar"`,
+}, {
+	input: "+++_bar",
+	err:   `invalid slice reference: "\+\+\+_bar"`,
+}, {
+	input: "..._bar",
+	err:   `invalid slice reference: "\.\.\._bar"`,
+}, {
+	input: "white space_no-whitespace",
+	err:   `invalid slice reference: "white space_no-whitespace"`,
+}}
+
+func (s *S) TestParseSliceKey(c *C) {
+	for _, test := range sliceKeyTests {
+		key, err := apacheutil.ParseSliceKey(test.input)
+		if test.err != "" {
+			c.Assert(err, ErrorMatches, test.err)
+			continue
+		}
+		c.Assert(err, IsNil)
+		c.Assert(key, DeepEquals, test.expected)
+	}
+}

internal/jsonwall/log.go → internal/manifestutil/log.go

@@ -1,4 +1,4 @@
-package jsonwall
+package manifestutil
 
 import (
 	"fmt"