From 11a324ceca40c373ae5298ff6b93986340a945d7 Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Mon, 5 Feb 2024 15:36:34 +0100 Subject: [PATCH] feat: creating files logs information for use in chisel.db (#105) When creating files as part of the cut command, use a proxy that logs information about the hash, path, mode, etc. This will then be ingrated into a report which will be part of chisel.db. --- internal/archive/archive_test.go | 2 ++ internal/deb/extract.go | 5 +-- internal/deb/extract_test.go | 2 ++ internal/fsutil/create.go | 62 ++++++++++++++++++++++++++++++-- internal/fsutil/create_test.go | 52 ++++++++++++++++++++++++--- internal/setup/fetch.go | 2 +- internal/slicer/slicer.go | 5 ++- 7 files changed, 119 insertions(+), 11 deletions(-) diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index af741dbe..6d7aead2 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -19,6 +19,7 @@ import ( "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/archive/testarchive" "github.com/canonical/chisel/internal/deb" + "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/testutil" ) @@ -500,6 +501,7 @@ func (s *S) testOpenArchiveArch(c *C, release ubuntuRelease, arch string) { {Path: "/hostname"}, }, }, + Creator: fsutil.NewCreator(), }) c.Assert(err, IsNil) diff --git a/internal/deb/extract.go b/internal/deb/extract.go index 5de2f403..e9748424 100644 --- a/internal/deb/extract.go +++ b/internal/deb/extract.go @@ -25,6 +25,7 @@ type ExtractOptions struct { TargetDir string Extract map[string][]ExtractInfo Globbed map[string][]string + Creator *fsutil.Creator } type ExtractInfo struct { @@ -185,7 +186,7 @@ func extractData(dataReader io.Reader, options *ExtractOptions) error { // Base directory for extracted content. Relevant mainly to preserve // the metadata, since the extracted content itself will also create // any missing directories unaccounted for in the options. - err := fsutil.Create(&fsutil.CreateOptions{ + err := options.Creator.Create(&fsutil.CreateOptions{ Path: filepath.Join(options.TargetDir, sourcePath), Mode: tarHeader.FileInfo().Mode(), MakeParents: true, @@ -226,7 +227,7 @@ func extractData(dataReader io.Reader, options *ExtractOptions) error { if extractInfo.Mode != 0 { tarHeader.Mode = int64(extractInfo.Mode) } - err := fsutil.Create(&fsutil.CreateOptions{ + err := options.Creator.Create(&fsutil.CreateOptions{ Path: targetPath, Mode: tarHeader.FileInfo().Mode(), Data: pathReader, diff --git a/internal/deb/extract_test.go b/internal/deb/extract_test.go index 3147dc36..14898d42 100644 --- a/internal/deb/extract_test.go +++ b/internal/deb/extract_test.go @@ -6,6 +6,7 @@ import ( . "gopkg.in/check.v1" "github.com/canonical/chisel/internal/deb" + "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/testutil" ) @@ -290,6 +291,7 @@ func (s *S) TestExtract(c *C) { options := test.options options.Package = "base-files" options.TargetDir = dir + options.Creator = fsutil.NewCreator() if test.globbed != nil { options.Globbed = make(map[string][]string) diff --git a/internal/fsutil/create.go b/internal/fsutil/create.go index b39b3f39..01f7e821 100644 --- a/internal/fsutil/create.go +++ b/internal/fsutil/create.go @@ -1,7 +1,10 @@ package fsutil import ( + "crypto/sha256" + "encoding/hex" "fmt" + "hash" "io" "io/fs" "os" @@ -18,8 +21,32 @@ type CreateOptions struct { MakeParents bool } -// Creates a filesystem entry according to the provided options. -func Create(o *CreateOptions) error { +type Info struct { + Path string + Mode fs.FileMode + Hash string + Size int + Link string +} + +type Creator struct { + // Created keeps track of information about the filesystem entries created. + // If an entry is created several times it only tracks the latest one. + Created map[string]Info +} + +func NewCreator() *Creator { + return &Creator{Created: make(map[string]Info)} +} + +// Create creates a filesystem entry according to the provided options. +func (c *Creator) Create(options *CreateOptions) error { + rp := &readerProxy{inner: options.Data, h: sha256.New()} + // Use the proxy instead of the raw Reader. + optsCopy := *options + optsCopy.Data = rp + o := &optsCopy + var err error if o.MakeParents { if err := os.MkdirAll(filepath.Dir(o.Path), 0755); err != nil { @@ -36,7 +63,19 @@ func Create(o *CreateOptions) error { default: err = fmt.Errorf("unsupported file type: %s", o.Path) } - return err + if err != nil { + return err + } + + info := Info{ + Path: o.Path, + Mode: o.Mode, + Hash: hex.EncodeToString(rp.h.Sum(nil)), + Size: rp.size, + Link: o.Link, + } + c.Created[o.Path] = info + return nil } func createDir(o *CreateOptions) error { @@ -84,3 +123,20 @@ func createSymlink(o *CreateOptions) error { } return os.Symlink(o.Link, o.Path) } + +// readerProxy implements the io.Reader interface proxying the calls to its +// inner io.Reader. On each read, the proxy keeps track of the file size and hash. +type readerProxy struct { + inner io.Reader + h hash.Hash + size int +} + +var _ io.Reader = (*readerProxy)(nil) + +func (rp *readerProxy) Read(p []byte) (n int, err error) { + n, err = rp.inner.Read(p) + rp.h.Write(p[:n]) + rp.size += n + return n, err +} diff --git a/internal/fsutil/create_test.go b/internal/fsutil/create_test.go index 67c5a94f..0f2f8290 100644 --- a/internal/fsutil/create_test.go +++ b/internal/fsutil/create_test.go @@ -2,8 +2,10 @@ package fsutil_test import ( "bytes" + "fmt" "io/fs" "path/filepath" + "strings" "syscall" . "gopkg.in/check.v1" @@ -73,18 +75,60 @@ func (s *S) TestCreate(c *C) { }() for _, test := range createTests { + if test.result == nil { + // Empty map for no files created. + test.result = make(map[string]string) + } c.Logf("Options: %v", test.options) dir := c.MkDir() options := test.options options.Path = filepath.Join(dir, options.Path) - err := fsutil.Create(&options) + creator := fsutil.NewCreator() + err := creator.Create(&options) if test.error != "" { c.Assert(err, ErrorMatches, test.error) - continue } else { c.Assert(err, IsNil) } - result := testutil.TreeDump(dir) - c.Assert(result, DeepEquals, test.result) + c.Assert(testutil.TreeDump(dir), DeepEquals, test.result) + if test.options.MakeParents { + // The creator does not record the parent directories created + // implicitly. + for path, info := range treeDumpFSCreator(creator, dir) { + c.Assert(info, Equals, test.result[path]) + } + } else { + c.Assert(treeDumpFSCreator(creator, dir), DeepEquals, test.result) + } + } +} + +// treeDumpFSCreator dumps the contents stored in Creator about the filesystem +// entries created using the same format as [testutil.TreeDump]. +func treeDumpFSCreator(cr *fsutil.Creator, root string) map[string]string { + result := make(map[string]string) + for _, file := range cr.Created { + path := strings.TrimPrefix(file.Path, root) + fperm := file.Mode.Perm() + if file.Mode&fs.ModeSticky != 0 { + fperm |= 01000 + } + switch file.Mode.Type() { + case fs.ModeDir: + result[path+"/"] = fmt.Sprintf("dir %#o", fperm) + case fs.ModeSymlink: + result[path] = fmt.Sprintf("symlink %s", file.Link) + case 0: // Regular + var entry string + if file.Size == 0 { + entry = fmt.Sprintf("file %#o empty", file.Mode.Perm()) + } else { + entry = fmt.Sprintf("file %#o %s", fperm, file.Hash[:8]) + } + result[path] = entry + default: + panic(fmt.Errorf("unknown file type %d: %s", file.Mode.Type(), path)) + } } + return result } diff --git a/internal/setup/fetch.go b/internal/setup/fetch.go index 6446baad..b34584a5 100644 --- a/internal/setup/fetch.go +++ b/internal/setup/fetch.go @@ -137,7 +137,7 @@ func extractTar(dataReader io.Reader, targetDir string) error { //debugf("Extracting header: %#v", tarHeader) - err = fsutil.Create(&fsutil.CreateOptions{ + err = fsutil.NewCreator().Create(&fsutil.CreateOptions{ Path: filepath.Join(targetDir, sourcePath), Mode: tarHeader.FileInfo().Mode(), Data: tarReader, diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 2b684b5d..e0c14a8e 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -153,11 +153,13 @@ func Run(options *RunOptions) error { if reader == nil { continue } + creator := fsutil.NewCreator() err := deb.Extract(reader, &deb.ExtractOptions{ Package: slice.Package, Extract: extract[slice.Package], TargetDir: targetDir, Globbed: globbedPaths, + Creator: creator, }) reader.Close() packages[slice.Package] = nil @@ -211,7 +213,8 @@ func Run(options *RunOptions) error { return fmt.Errorf("internal error: cannot extract path of kind %q", pathInfo.Kind) } - err := fsutil.Create(&fsutil.CreateOptions{ + creator := fsutil.NewCreator() + err := creator.Create(&fsutil.CreateOptions{ Path: targetPath, Mode: tarHeader.FileInfo().Mode(), Data: fileContent,