From f804e05360c00ae09ca9852415d54c4ff314099c Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Mon, 12 Aug 2024 19:55:40 -0700 Subject: [PATCH 1/5] feat: Support UnixFS mode and modification times in ipld dag and mfs Adds support for storing and retrieving file mode and last modification time. Support added to: Support added to: - Files - LinkFiles - Webfiles - Directories Tar archives are supported by the parent branch. --- ipld/unixfs/file/unixfile.go | 47 ++-- .../unixfs/importer/balanced/balanced_test.go | 48 ++++ ipld/unixfs/importer/balanced/builder.go | 29 +- ipld/unixfs/importer/helpers/dagbuilder.go | 97 +++++-- ipld/unixfs/importer/trickle/trickle_test.go | 109 +++++++- ipld/unixfs/importer/trickle/trickledag.go | 22 +- ipld/unixfs/io/dagreader.go | 25 +- ipld/unixfs/mod/dagmodifier.go | 17 +- ipld/unixfs/pb/unixfs.pb.go | 112 ++++++-- ipld/unixfs/pb/unixfs.proto | 17 ++ ipld/unixfs/unixfs.go | 141 +++++++++- ipld/unixfs/unixfs_test.go | 250 ++++++++++++++++++ mfs/dir.go | 77 +++++- mfs/file.go | 90 +++++++ mfs/mfs_test.go | 155 ++++++++++- mfs/ops.go | 23 +- mfs/root.go | 3 + 17 files changed, 1168 insertions(+), 94 deletions(-) diff --git a/ipld/unixfs/file/unixfile.go b/ipld/unixfs/file/unixfile.go index 0a923e937..0cf7616c1 100644 --- a/ipld/unixfs/file/unixfile.go +++ b/ipld/unixfs/file/unixfile.go @@ -23,7 +23,8 @@ type ufsDirectory struct { dserv ipld.DAGService dir uio.Directory size int64 - stat os.FileInfo + mode os.FileMode + mtime time.Time } type ufsIterator struct { @@ -122,17 +123,11 @@ func (d *ufsDirectory) Entries() files.DirIterator { } func (d *ufsDirectory) Mode() os.FileMode { - if d.stat == nil { - return 0 - } - return d.stat.Mode() + return d.mode } func (d *ufsDirectory) ModTime() time.Time { - if d.stat == nil { - return time.Time{} - } - return d.stat.ModTime() + return d.mtime } func (d *ufsDirectory) Size() (int64, error) { @@ -141,28 +136,21 @@ func (d *ufsDirectory) Size() (int64, error) { type ufsFile struct { uio.DagReader - stat os.FileInfo } func (f *ufsFile) Mode() os.FileMode { - if f.stat == nil { - return 0 - } - return f.stat.Mode() + return f.DagReader.Mode() } func (f *ufsFile) ModTime() time.Time { - if f.stat == nil { - return time.Time{} - } - return f.stat.ModTime() + return f.DagReader.ModTime() } func (f *ufsFile) Size() (int64, error) { return int64(f.DagReader.Size()), nil } -func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd *dag.ProtoNode, stat os.FileInfo) (files.Directory, error) { +func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd *dag.ProtoNode) (files.Directory, error) { dir, err := uio.NewDirectoryFromNode(dserv, nd) if err != nil { return nil, err @@ -173,32 +161,35 @@ func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd *dag.ProtoNode, return nil, err } + fsn, err := ft.FSNodeFromBytes(nd.Data()) + if err != nil { + return nil, err + } + return &ufsDirectory{ ctx: ctx, dserv: dserv, - dir: dir, - size: int64(size), - stat: stat, + dir: dir, + size: int64(size), + mode: fsn.Mode(), + mtime: fsn.ModTime(), }, nil } func NewUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node) (files.Node, error) { - return NewUnixfsFileWithStat(ctx, dserv, nd, nil) -} - -func NewUnixfsFileWithStat(ctx context.Context, dserv ipld.DAGService, nd ipld.Node, stat os.FileInfo) (files.Node, error) { switch dn := nd.(type) { case *dag.ProtoNode: fsn, err := ft.FSNodeFromBytes(dn.Data()) if err != nil { return nil, err } + if fsn.IsDir() { - return newUnixfsDir(ctx, dserv, dn, stat) + return newUnixfsDir(ctx, dserv, dn) } if fsn.Type() == ft.TSymlink { - return files.NewLinkFile(string(fsn.Data()), stat), nil + return files.NewSymlinkFile(string(fsn.Data()), fsn.ModTime()), nil } case *dag.RawNode: diff --git a/ipld/unixfs/importer/balanced/balanced_test.go b/ipld/unixfs/importer/balanced/balanced_test.go index 5a5dcf9ad..4ea4cb8a9 100644 --- a/ipld/unixfs/importer/balanced/balanced_test.go +++ b/ipld/unixfs/importer/balanced/balanced_test.go @@ -7,6 +7,7 @@ import ( "io" mrand "math/rand" "testing" + "time" h "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" uio "github.com/ipfs/boxo/ipld/unixfs/io" @@ -26,6 +27,10 @@ func buildTestDag(ds ipld.DAGService, spl chunker.Splitter) (*dag.ProtoNode, err Maxlinks: h.DefaultLinksPerBlock, } + return buildTestDagWithParams(spl, dbp) +} + +func buildTestDagWithParams(spl chunker.Splitter, dbp h.DagBuilderParams) (*dag.ProtoNode, error) { db, err := dbp.New(spl) if err != nil { return nil, err @@ -335,3 +340,46 @@ func TestSeekingConsistency(t *testing.T) { t.Fatal(err) } } + +func TestMetadataNoData(t *testing.T) { + testMetadata(t, new(bytes.Buffer)) +} + +func TestMetadata(t *testing.T) { + nbytes := 3 * chunker.DefaultBlockSize + buf := new(bytes.Buffer) + _, err := io.CopyN(buf, random.NewRand(), nbytes) + if err != nil { + t.Fatal(err) + } + + testMetadata(t, buf) +} + +func testMetadata(t *testing.T, buf *bytes.Buffer) { + dagserv := mdtest.Mock() + dbp := h.DagBuilderParams{ + Dagserv: dagserv, + Maxlinks: h.DefaultLinksPerBlock, + FileMode: 0522, + FileModTime: time.Unix(1638111600, 76552), + } + + nd, err := buildTestDagWithParams(chunker.DefaultSplitter(buf), dbp) + if err != nil { + t.Fatal(err) + } + + dr, err := uio.NewDagReader(context.Background(), nd, dagserv) + if err != nil { + t.Fatal(err) + } + + if !dr.ModTime().Equal(dbp.FileModTime) { + t.Errorf("got modtime %v, wanted %v", dr.ModTime(), dbp.FileModTime) + } + + if dr.Mode() != dbp.FileMode { + t.Errorf("got filemode %o, wanted %o", dr.Mode(), dbp.FileMode) + } +} diff --git a/ipld/unixfs/importer/balanced/builder.go b/ipld/unixfs/importer/balanced/builder.go index 0fdb0fd28..915d0a439 100644 --- a/ipld/unixfs/importer/balanced/builder.go +++ b/ipld/unixfs/importer/balanced/builder.go @@ -130,18 +130,33 @@ import ( // | Chunk 1 | | Chunk 2 | | Chunk 3 | // +=========+ +=========+ + - - - - + func Layout(db *h.DagBuilderHelper) (ipld.Node, error) { + var root ipld.Node + var err error + if db.Done() { - // No data, return just an empty node. - root, err := db.NewLeafNode(nil, ft.TFile) - if err != nil { - return nil, err - } + // No data, just create an empty node. + root, err = db.NewLeafNode(nil, ft.TFile) // This works without Filestore support (`ProcessFileStore`). // TODO: Why? Is there a test case missing? + } else { + root, err = layoutData(db) + } + + if err != nil { + return nil, err + } - return root, db.Add(root) + if db.HasFileAttributes() { + err = db.SetFileAttributes(root) + if err != nil { + return nil, err + } } + return root, db.Add(root) +} + +func layoutData(db *h.DagBuilderHelper) (ipld.Node, error) { // The first `root` will be a single leaf node with data // (corner case), after that subsequent `root` nodes will // always be internal nodes (with a depth > 0) that can @@ -172,7 +187,7 @@ func Layout(db *h.DagBuilderHelper) (ipld.Node, error) { } } - return root, db.Add(root) + return root, nil } // fillNodeRec will "fill" the given internal (non-leaf) `node` with data by diff --git a/ipld/unixfs/importer/helpers/dagbuilder.go b/ipld/unixfs/importer/helpers/dagbuilder.go index 25514d795..aefffad15 100644 --- a/ipld/unixfs/importer/helpers/dagbuilder.go +++ b/ipld/unixfs/importer/helpers/dagbuilder.go @@ -5,6 +5,7 @@ import ( "errors" "io" "os" + "time" dag "github.com/ipfs/boxo/ipld/merkledag" @@ -23,13 +24,15 @@ var ErrMissingFsRef = errors.New("missing file path or URL, can't create filesto // DagBuilderHelper wraps together a bunch of objects needed to // efficiently create unixfs dag trees type DagBuilderHelper struct { - dserv ipld.DAGService - spl chunker.Splitter - recvdErr error - rawLeaves bool - nextData []byte // the next item to return. - maxlinks int - cidBuilder cid.Builder + dserv ipld.DAGService + spl chunker.Splitter + recvdErr error + rawLeaves bool + nextData []byte // the next item to return. + maxlinks int + cidBuilder cid.Builder + fileMode os.FileMode + fileModTime time.Time // Filestore support variables. // ---------------------------- @@ -62,6 +65,12 @@ type DagBuilderParams struct { // DAGService to write blocks to (required) Dagserv ipld.DAGService + // The unixfs file mode + FileMode os.FileMode + + // The unixfs last modified time + FileModTime time.Time + // NoCopy signals to the chunker that it should track fileinfo for // filestore adds NoCopy bool @@ -71,11 +80,13 @@ type DagBuilderParams struct { // chunker.Splitter as data source. func (dbp *DagBuilderParams) New(spl chunker.Splitter) (*DagBuilderHelper, error) { db := &DagBuilderHelper{ - dserv: dbp.Dagserv, - spl: spl, - rawLeaves: dbp.RawLeaves, - cidBuilder: dbp.CidBuilder, - maxlinks: dbp.Maxlinks, + dserv: dbp.Dagserv, + spl: spl, + rawLeaves: dbp.RawLeaves, + cidBuilder: dbp.CidBuilder, + maxlinks: dbp.Maxlinks, + fileMode: dbp.FileMode, + fileModTime: dbp.FileModTime, } if fi, ok := spl.Reader().(files.FileInfo); dbp.NoCopy && ok { db.fullPath = fi.AbsPath() @@ -138,9 +149,9 @@ func (db *DagBuilderHelper) GetCidBuilder() cid.Builder { return db.cidBuilder } -// NewLeafNode creates a leaf node filled with data. If rawLeaves is -// defined then a raw leaf will be returned. Otherwise, it will create -// and return `FSNodeOverDag` with `fsNodeType`. +// NewLeafNode creates a leaf node filled with data. If rawLeaves is defined +// then a raw leaf will be returned. Otherwise, it will create and return +// `FSNodeOverDag` with `fsNodeType`. func (db *DagBuilderHelper) NewLeafNode(data []byte, fsNodeType pb.Data_DataType) (ipld.Node, error) { if len(data) > BlockSizeLimit { return nil, ErrSizeLimitExceeded @@ -161,6 +172,7 @@ func (db *DagBuilderHelper) NewLeafNode(data []byte, fsNodeType pb.Data_DataType // Encapsulate the data in UnixFS node (instead of a raw node). fsNodeOverDag := db.NewFSNodeOverDag(fsNodeType) fsNodeOverDag.SetFileData(data) + node, err := fsNodeOverDag.Commit() if err != nil { return nil, err @@ -172,9 +184,10 @@ func (db *DagBuilderHelper) NewLeafNode(data []byte, fsNodeType pb.Data_DataType return node, nil } -// FillNodeLayer will add datanodes as children to the give node until +// FillNodeLayer will add data-nodes as children to the given node until // it is full in this layer or no more data. -// NOTE: This function creates raw data nodes so it only works +// +// NOTE: This function creates raw data nodes, so it only works // for the `trickle.Layout`. func (db *DagBuilderHelper) FillNodeLayer(node *FSNodeOverDag) error { // while we have room AND we're not done @@ -265,6 +278,34 @@ func (db *DagBuilderHelper) Maxlinks() int { return db.maxlinks } +// HasFileAttributes will return false if Filestore is being used, +// otherwise returns true if a file mode or last modification time is set. +func (db *DagBuilderHelper) HasFileAttributes() bool { + return db.fullPath == "" && (db.fileMode != 0 || !db.fileModTime.IsZero()) +} + +// SetFileAttributes stores file attributes present in the `DagBuilderHelper` +// into the associated `ft.FSNode`. +func (db *DagBuilderHelper) SetFileAttributes(n ipld.Node) error { + if pn, ok := n.(*dag.ProtoNode); ok { + fsn, err := ft.FSNodeFromBytes(pn.Data()) + if err != nil { + return err + } + fsn.SetModTime(db.fileModTime) + fsn.SetMode(db.fileMode) + + d, err := fsn.GetBytes() + if err != nil { + return err + } + + pn.SetData(d) + } + + return nil +} + // FSNodeOverDag encapsulates an `unixfs.FSNode` that will be stored in a // `dag.ProtoNode`. Instead of just having a single `ipld.Node` that // would need to be constantly (un)packed to access and modify its @@ -288,7 +329,7 @@ type FSNodeOverDag struct { } // NewFSNodeOverDag creates a new `dag.ProtoNode` and `ft.FSNode` -// decoupled from one onther (and will continue in that way until +// decoupled from one anonther (and will continue in that way until // `Commit` is called), with `fsNodeType` specifying the type of // the UnixFS layer node (either `File` or `Raw`). func (db *DagBuilderHelper) NewFSNodeOverDag(fsNodeType pb.Data_DataType) *FSNodeOverDag { @@ -374,6 +415,26 @@ func (n *FSNodeOverDag) SetFileData(fileData []byte) { n.file.SetData(fileData) } +// SetMode sets the file mode of the associated `ft.FSNode`. +func (n *FSNodeOverDag) SetMode(mode os.FileMode) { + n.file.SetMode(mode) +} + +// SetModTime sets the file modification time of the associated `ft.FSNode`. +func (n *FSNodeOverDag) SetModTime(ts time.Time) { + n.file.SetModTime(ts) +} + +// Mode returns the file mode of the associated `ft.FSNode` +func (n *FSNodeOverDag) Mode() os.FileMode { + return n.file.Mode() +} + +// ModTime returns the last modification time of the associated `ft.FSNode` +func (n *FSNodeOverDag) ModTime() time.Time { + return n.file.ModTime() +} + // GetDagNode fills out the proper formatting for the FSNodeOverDag node // inside of a DAG node and returns the dag node. // TODO: Check if we have committed (passed the UnixFS information diff --git a/ipld/unixfs/importer/trickle/trickle_test.go b/ipld/unixfs/importer/trickle/trickle_test.go index 9078fdc02..61899c9ff 100644 --- a/ipld/unixfs/importer/trickle/trickle_test.go +++ b/ipld/unixfs/importer/trickle/trickle_test.go @@ -7,6 +7,7 @@ import ( "io" mrand "math/rand" "testing" + "time" ft "github.com/ipfs/boxo/ipld/unixfs" h "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" @@ -40,6 +41,10 @@ func buildTestDag(ds ipld.DAGService, spl chunker.Splitter, rawLeaves UseRawLeav RawLeaves: bool(rawLeaves), } + return buildTestDagWithParams(ds, spl, dbp) +} + +func buildTestDagWithParams(ds ipld.DAGService, spl chunker.Splitter, dbp h.DagBuilderParams) (*merkledag.ProtoNode, error) { db, err := dbp.New(spl) if err != nil { return nil, err @@ -59,7 +64,7 @@ func buildTestDag(ds ipld.DAGService, spl chunker.Splitter, rawLeaves UseRawLeav Getter: ds, Direct: dbp.Maxlinks, LayerRepeat: depthRepeat, - RawLeaves: bool(rawLeaves), + RawLeaves: dbp.RawLeaves, }) } @@ -668,3 +673,105 @@ func TestAppendSingleBytesToEmpty(t *testing.T) { t.Fatal(err) } } + +func TestAppendWithModTime(t *testing.T) { + const nbytes = 128 * 1024 + + timestamp := time.Now() + ds := mdtest.Mock() + buf := random.Bytes(nbytes) + + nd := new(merkledag.ProtoNode) + nd.SetData(ft.FilePBDataWithStat(buf[:nbytes/2], nbytes/2, 0, timestamp)) + + dbp := &h.DagBuilderParams{ + Dagserv: ds, + Maxlinks: h.DefaultLinksPerBlock, + } + + r := bytes.NewReader(buf[nbytes/2:]) + db, err := dbp.New(chunker.NewSizeSplitter(r, 500)) + if err != nil { + t.Fatal(err) + } + + nd2, err := Append(context.Background(), nd, db) + if err != nil { + t.Fatal(err) + } + + fsn, _ := ft.ExtractFSNode(nd2) + + if !fsn.ModTime().After(timestamp) { + t.Errorf("expected modification time to be updated") + } + +} + +func TestAppendToEmptyWithModTime(t *testing.T) { + timestamp := time.Now() + ds := mdtest.Mock() + nd := new(merkledag.ProtoNode) + nd.SetData(ft.FilePBDataWithStat(nil, 0, 0, timestamp)) + + dbp := &h.DagBuilderParams{ + Dagserv: ds, + Maxlinks: h.DefaultLinksPerBlock, + } + + db, err := dbp.New(chunker.DefaultSplitter(bytes.NewReader([]byte("test")))) + if err != nil { + t.Fatal(err) + } + + nd2, err := Append(context.Background(), nd, db) + if err != nil { + t.Fatal(err) + } + + fsn, _ := ft.ExtractFSNode(nd2) + + if !fsn.ModTime().After(timestamp) { + t.Errorf("expected modification time to be updated") + } +} + +func TestMetadata(t *testing.T) { + runBothSubtests(t, testMetadata) +} + +func testMetadata(t *testing.T, rawLeaves UseRawLeaves) { + const nbytes = 3 * chunker.DefaultBlockSize + buf := new(bytes.Buffer) + _, err := io.CopyN(buf, random.NewRand(), nbytes) + if err != nil { + t.Fatal(err) + } + + dagserv := mdtest.Mock() + dbp := h.DagBuilderParams{ + Dagserv: dagserv, + Maxlinks: h.DefaultLinksPerBlock, + RawLeaves: bool(rawLeaves), + FileMode: 0522, + FileModTime: time.Unix(1638111600, 76552), + } + + nd, err := buildTestDagWithParams(dagserv, chunker.DefaultSplitter(buf), dbp) + if err != nil { + t.Fatal(err) + } + + dr, err := uio.NewDagReader(context.Background(), nd, dagserv) + if err != nil { + t.Fatal(err) + } + + if !dr.ModTime().Equal(dbp.FileModTime) { + t.Errorf("got modtime %v, wanted %v", dr.ModTime(), dbp.FileModTime) + } + + if dr.Mode() != dbp.FileMode { + t.Errorf("got filemode %o, wanted %o", dr.Mode(), dbp.FileMode) + } +} diff --git a/ipld/unixfs/importer/trickle/trickledag.go b/ipld/unixfs/importer/trickle/trickledag.go index 09a8b8672..2b9d31dfa 100644 --- a/ipld/unixfs/importer/trickle/trickledag.go +++ b/ipld/unixfs/importer/trickle/trickledag.go @@ -19,6 +19,7 @@ import ( "context" "errors" "fmt" + "time" ft "github.com/ipfs/boxo/ipld/unixfs" h "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" @@ -43,6 +44,13 @@ func Layout(db *h.DagBuilderHelper) (ipld.Node, error) { return nil, err } + if db.HasFileAttributes() { + err = db.SetFileAttributes(root) + } + + if err != nil { + return nil, err + } return root, db.Add(root) } @@ -94,7 +102,6 @@ func Append(ctx context.Context, basen ipld.Node, db *h.DagBuilderHelper) (out i } // Convert to unixfs node for working with easily - fsn, err := h.NewFSNFromDag(base) if err != nil { return nil, err @@ -109,9 +116,10 @@ func Append(ctx context.Context, basen ipld.Node, db *h.DagBuilderHelper) (out i } if db.Done() { - // TODO: If `FillNodeLayer` stop `Commit`ing this should be - // the place (besides the function end) to call it. - return fsn.GetDagNode() + if !fsn.ModTime().IsZero() { + fsn.SetModTime(time.Now()) + } + return fsn.Commit() } // If continuing, our depth has increased by one @@ -142,11 +150,7 @@ func Append(ctx context.Context, basen ipld.Node, db *h.DagBuilderHelper) (out i } } } - _, err = fsn.Commit() - if err != nil { - return nil, err - } - return fsn.GetDagNode() + return fsn.Commit() } func appendFillLastChild(ctx context.Context, fsn *h.FSNodeOverDag, depth int, repeatNumber int, db *h.DagBuilderHelper) error { diff --git a/ipld/unixfs/io/dagreader.go b/ipld/unixfs/io/dagreader.go index 77dc8d921..bb1c83800 100644 --- a/ipld/unixfs/io/dagreader.go +++ b/ipld/unixfs/io/dagreader.go @@ -5,6 +5,8 @@ import ( "context" "errors" "io" + "os" + "time" mdag "github.com/ipfs/boxo/ipld/merkledag" unixfs "github.com/ipfs/boxo/ipld/unixfs" @@ -29,6 +31,8 @@ var ( type DagReader interface { ReadSeekCloser Size() uint64 + Mode() os.FileMode + ModTime() time.Time CtxReadFull(context.Context, []byte) (int, error) } @@ -44,6 +48,8 @@ type ReadSeekCloser interface { // the given node, using the passed in DAGService for data retrieval. func NewDagReader(ctx context.Context, n ipld.Node, serv ipld.NodeGetter) (DagReader, error) { var size uint64 + var mode os.FileMode + var modTime time.Time switch n := n.(type) { case *mdag.RawNode: @@ -55,6 +61,9 @@ func NewDagReader(ctx context.Context, n ipld.Node, serv ipld.NodeGetter) (DagRe return nil, err } + mode = fsNode.Mode() + modTime = fsNode.ModTime() + switch fsNode.Type() { case unixfs.TFile, unixfs.TRaw: size = fsNode.FileSize() @@ -93,6 +102,8 @@ func NewDagReader(ctx context.Context, n ipld.Node, serv ipld.NodeGetter) (DagRe cancel: cancel, serv: serv, size: size, + mode: mode, + modTime: modTime, rootNode: n, dagWalker: ipld.NewWalker(ctxWithCancel, ipld.NewNavigableIPLDNode(n, serv)), }, nil @@ -129,7 +140,19 @@ type dagReader struct { // Passed to the `dagWalker` that will use it to request nodes. // TODO: Revisit name. - serv ipld.NodeGetter + serv ipld.NodeGetter + mode os.FileMode + modTime time.Time +} + +// Mode returns the UnixFS file mode or 0 if not set. +func (dr *dagReader) Mode() os.FileMode { + return dr.mode +} + +// ModTime returns the UnixFS file last modification time if set. +func (dr *dagReader) ModTime() time.Time { + return dr.modTime } // Size returns the total size of the data from the DAG structured file. diff --git a/ipld/unixfs/mod/dagmodifier.go b/ipld/unixfs/mod/dagmodifier.go index f662a0a71..c075523f8 100644 --- a/ipld/unixfs/mod/dagmodifier.go +++ b/ipld/unixfs/mod/dagmodifier.go @@ -7,6 +7,7 @@ import ( "context" "errors" "io" + "time" ft "github.com/ipfs/boxo/ipld/unixfs" help "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" @@ -258,6 +259,9 @@ func (dm *DagModifier) modifyDag(n ipld.Node, offset uint64) (cid.Cid, error) { } // Update newly written node.. + if !fsn.ModTime().IsZero() { + fsn.SetModTime(time.Now()) + } b, err := fsn.GetBytes() if err != nil { return cid.Cid{}, err @@ -527,8 +531,17 @@ func (dm *DagModifier) dagTruncate(ctx context.Context, n ipld.Node, size uint64 if err != nil { return nil, err } - nd.SetData(ft.WrapData(fsn.Data()[:size])) - return nd, nil + + fsn.SetData(fsn.Data()[:size]) + if !fsn.ModTime().IsZero() { + fsn.SetModTime(time.Now()) + } + data, err := fsn.GetBytes() + if err != nil { + return nil, err + } + + return mdag.NodeWithData(data), nil case *mdag.RawNode: return mdag.NewRawNodeWPrefix(nd.RawData()[:size], nd.Cid().Prefix()) } diff --git a/ipld/unixfs/pb/unixfs.pb.go b/ipld/unixfs/pb/unixfs.pb.go index 805c11289..d02e110f2 100644 --- a/ipld/unixfs/pb/unixfs.pb.go +++ b/ipld/unixfs/pb/unixfs.pb.go @@ -82,6 +82,8 @@ type Data struct { Blocksizes []uint64 `protobuf:"varint,4,rep,name=blocksizes" json:"blocksizes,omitempty"` HashType *uint64 `protobuf:"varint,5,opt,name=hashType" json:"hashType,omitempty"` Fanout *uint64 `protobuf:"varint,6,opt,name=fanout" json:"fanout,omitempty"` + Mode *uint32 `protobuf:"varint,7,opt,name=mode" json:"mode,omitempty"` + Mtime *IPFSTimestamp `protobuf:"bytes,8,opt,name=mtime" json:"mtime,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -158,6 +160,20 @@ func (m *Data) GetFanout() uint64 { return 0 } +func (m *Data) GetMode() uint32 { + if m != nil && m.Mode != nil { + return *m.Mode + } + return 0 +} + +func (m *Data) GetMtime() *IPFSTimestamp { + if m != nil { + return m.Mtime + } + return nil +} + type Metadata struct { MimeType *string `protobuf:"bytes,1,opt,name=MimeType" json:"MimeType,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -201,31 +217,91 @@ func (m *Metadata) GetMimeType() string { return "" } +// mostly copied from proto 3 - with int32 nanos changed to fixed32 for js-ipfs compatibility +// https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/timestamp.proto +type IPFSTimestamp struct { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + Seconds *int64 `protobuf:"varint,1,req,name=seconds" json:"seconds,omitempty"` + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + Nanos *uint32 `protobuf:"fixed32,2,opt,name=nanos" json:"nanos,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *IPFSTimestamp) Reset() { *m = IPFSTimestamp{} } +func (m *IPFSTimestamp) String() string { return proto.CompactTextString(m) } +func (*IPFSTimestamp) ProtoMessage() {} +func (*IPFSTimestamp) Descriptor() ([]byte, []int) { + return fileDescriptor_e2fd76cc44dfc7c3, []int{2} +} +func (m *IPFSTimestamp) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_IPFSTimestamp.Unmarshal(m, b) +} +func (m *IPFSTimestamp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_IPFSTimestamp.Marshal(b, m, deterministic) +} +func (m *IPFSTimestamp) XXX_Merge(src proto.Message) { + xxx_messageInfo_IPFSTimestamp.Merge(m, src) +} +func (m *IPFSTimestamp) XXX_Size() int { + return xxx_messageInfo_IPFSTimestamp.Size(m) +} +func (m *IPFSTimestamp) XXX_DiscardUnknown() { + xxx_messageInfo_IPFSTimestamp.DiscardUnknown(m) +} + +var xxx_messageInfo_IPFSTimestamp proto.InternalMessageInfo + +func (m *IPFSTimestamp) GetSeconds() int64 { + if m != nil && m.Seconds != nil { + return *m.Seconds + } + return 0 +} + +func (m *IPFSTimestamp) GetNanos() uint32 { + if m != nil && m.Nanos != nil { + return *m.Nanos + } + return 0 +} + func init() { proto.RegisterEnum("unixfs.v1.pb.Data_DataType", Data_DataType_name, Data_DataType_value) proto.RegisterType((*Data)(nil), "unixfs.v1.pb.Data") proto.RegisterType((*Metadata)(nil), "unixfs.v1.pb.Metadata") + proto.RegisterType((*IPFSTimestamp)(nil), "unixfs.pb.IPFSTimestamp") } func init() { proto.RegisterFile("unixfs.proto", fileDescriptor_e2fd76cc44dfc7c3) } var fileDescriptor_e2fd76cc44dfc7c3 = []byte{ - // 267 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x4c, 0x90, 0x41, 0x4f, 0x83, 0x30, - 0x18, 0x86, 0x05, 0xba, 0x0d, 0xbe, 0xa1, 0x69, 0xbe, 0x83, 0x21, 0x9a, 0x18, 0xc2, 0xc1, 0x70, - 0xc2, 0xe8, 0x3f, 0xd0, 0x2c, 0xc6, 0x0b, 0x97, 0x6e, 0xf1, 0xe0, 0xc5, 0x94, 0xad, 0x84, 0x66, - 0x8c, 0x12, 0xe8, 0x54, 0xfc, 0x1b, 0xfe, 0x61, 0x53, 0x18, 0xdb, 0x2e, 0x4d, 0x9e, 0xf6, 0x79, - 0x9b, 0x37, 0x2f, 0xf8, 0xfb, 0x4a, 0xfe, 0xe4, 0x6d, 0x52, 0x37, 0x4a, 0x2b, 0x1c, 0xe9, 0xeb, - 0x31, 0xa9, 0xb3, 0xe8, 0xcf, 0x06, 0xb2, 0xe0, 0x9a, 0xe3, 0x03, 0x90, 0x55, 0x57, 0x8b, 0xc0, - 0x0a, 0xed, 0xf8, 0xea, 0xe9, 0x36, 0x39, 0xb7, 0x12, 0x63, 0xf4, 0x87, 0x51, 0x58, 0x2f, 0x22, - 0x0e, 0xc1, 0xc0, 0x0e, 0xad, 0xd8, 0x67, 0xc3, 0x27, 0x37, 0xe0, 0xe6, 0xb2, 0x14, 0xad, 0xfc, - 0x15, 0x81, 0x13, 0x5a, 0x31, 0x61, 0x47, 0xc6, 0x3b, 0x80, 0xac, 0x54, 0xeb, 0xad, 0x81, 0x36, - 0x20, 0xa1, 0x13, 0x13, 0x76, 0x76, 0x63, 0xb2, 0x05, 0x6f, 0x8b, 0xbe, 0xc4, 0x64, 0xc8, 0x8e, - 0x8c, 0xd7, 0x30, 0xcd, 0x79, 0xa5, 0xf6, 0x3a, 0x98, 0xf6, 0x2f, 0x07, 0x8a, 0xde, 0xc1, 0x1d, - 0x5b, 0xe1, 0x0c, 0x1c, 0xc6, 0xbf, 0xe9, 0x05, 0x5e, 0x82, 0xb7, 0x90, 0x8d, 0x58, 0x6b, 0xd5, - 0x74, 0xd4, 0x42, 0x17, 0xc8, 0xab, 0x2c, 0x05, 0xb5, 0xd1, 0x07, 0x37, 0x15, 0x9a, 0x6f, 0xb8, - 0xe6, 0xd4, 0xc1, 0x39, 0xcc, 0x96, 0xdd, 0xae, 0x94, 0xd5, 0x96, 0x12, 0x93, 0x79, 0x7b, 0x4e, - 0x57, 0xcb, 0x82, 0x37, 0x1b, 0x3a, 0x89, 0xee, 0x4f, 0xa6, 0xe9, 0x95, 0xca, 0x9d, 0x38, 0x8c, - 0x63, 0xc5, 0x1e, 0x3b, 0xf2, 0xcb, 0xfc, 0xc3, 0x1b, 0x76, 0xfa, 0xac, 0xb3, 0xff, 0x00, 0x00, - 0x00, 0xff, 0xff, 0xbd, 0x16, 0xf8, 0x45, 0x67, 0x01, 0x00, 0x00, + // 322 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x91, 0x5f, 0x4b, 0xc3, 0x30, + 0x14, 0xc5, 0xed, 0xbf, 0xb5, 0xbb, 0xdb, 0xa4, 0x5c, 0x44, 0x82, 0x0f, 0x52, 0xfa, 0x20, 0x7d, + 0x90, 0x3e, 0xf8, 0x05, 0x44, 0x18, 0x43, 0x1f, 0x06, 0x92, 0x0d, 0xdf, 0xb3, 0x35, 0x63, 0x61, + 0x4d, 0x33, 0x9a, 0x0c, 0x9d, 0x9f, 0xd3, 0x0f, 0x24, 0x49, 0xd7, 0xe9, 0x5e, 0x4a, 0x7f, 0xb9, + 0xe7, 0x84, 0x73, 0x6e, 0x60, 0x7c, 0x68, 0xc4, 0xd7, 0x46, 0x97, 0xfb, 0x56, 0x19, 0x85, 0xc3, + 0x9e, 0x56, 0xf9, 0x8f, 0x0f, 0xe1, 0x94, 0x19, 0x86, 0x8f, 0x10, 0x2e, 0x8f, 0x7b, 0x4e, 0xbc, + 0xcc, 0x2f, 0xae, 0x9f, 0x48, 0x79, 0x96, 0x94, 0x76, 0xec, 0x3e, 0x76, 0x4e, 0x9d, 0x0a, 0xb1, + 0x73, 0x11, 0x3f, 0xf3, 0x8a, 0x31, 0xed, 0x6e, 0xb8, 0x83, 0x64, 0x23, 0x6a, 0xae, 0xc5, 0x37, + 0x27, 0x41, 0xe6, 0x15, 0x21, 0x3d, 0x33, 0xde, 0x03, 0xac, 0x6a, 0xb5, 0xde, 0x59, 0xd0, 0x24, + 0xcc, 0x82, 0x22, 0xa4, 0xff, 0x4e, 0xac, 0x77, 0xcb, 0xf4, 0xd6, 0x25, 0x88, 0x3a, 0x6f, 0xcf, + 0x78, 0x0b, 0x83, 0x0d, 0x6b, 0xd4, 0xc1, 0x90, 0x81, 0x9b, 0x9c, 0xc8, 0x66, 0x90, 0xaa, 0xe2, + 0x24, 0xce, 0xbc, 0x62, 0x42, 0xdd, 0x3f, 0x96, 0x10, 0x49, 0x23, 0x24, 0x27, 0x49, 0xe6, 0x15, + 0xa3, 0x8b, 0x1a, 0x6f, 0xef, 0xb3, 0xc5, 0x52, 0x48, 0xae, 0x0d, 0x93, 0x7b, 0xda, 0xc9, 0xf2, + 0x0f, 0x48, 0xfa, 0x66, 0x18, 0x43, 0x40, 0xd9, 0x67, 0x7a, 0x85, 0x13, 0x18, 0x4e, 0x45, 0xcb, + 0xd7, 0x46, 0xb5, 0xc7, 0xd4, 0xc3, 0x04, 0xc2, 0x99, 0xa8, 0x79, 0xea, 0xe3, 0x18, 0x92, 0x39, + 0x37, 0xac, 0x62, 0x86, 0xa5, 0x01, 0x8e, 0x20, 0x5e, 0x1c, 0x65, 0x2d, 0x9a, 0x5d, 0x1a, 0x5a, + 0xcf, 0xeb, 0xcb, 0x7c, 0xb9, 0xd8, 0xb2, 0xb6, 0x4a, 0xa3, 0xfc, 0xe1, 0x4f, 0x69, 0xbb, 0xcd, + 0x85, 0xe4, 0xa7, 0xed, 0x7a, 0xc5, 0x90, 0x9e, 0x39, 0x7f, 0x86, 0xc9, 0x45, 0x2e, 0x24, 0x10, + 0x6b, 0xbe, 0x56, 0x4d, 0xa5, 0xdd, 0x4b, 0x04, 0xb4, 0x47, 0xbc, 0x81, 0xa8, 0x61, 0x8d, 0xd2, + 0x6e, 0xe7, 0x31, 0xed, 0xe0, 0x37, 0x00, 0x00, 0xff, 0xff, 0x36, 0xaf, 0xfa, 0x7c, 0xd9, 0x01, + 0x00, 0x00, } diff --git a/ipld/unixfs/pb/unixfs.proto b/ipld/unixfs/pb/unixfs.proto index f65673f54..bd02aa410 100644 --- a/ipld/unixfs/pb/unixfs.proto +++ b/ipld/unixfs/pb/unixfs.proto @@ -21,8 +21,25 @@ message Data { optional uint64 hashType = 5; optional uint64 fanout = 6; + optional uint32 mode = 7; + optional IPFSTimestamp mtime = 8; } message Metadata { optional string MimeType = 1; } + +// mostly copied from proto 3 - with int32 nanos changed to fixed32 for js-ipfs compatibility +// https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/timestamp.proto +message IPFSTimestamp { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + required int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + optional fixed32 nanos = 2; +} diff --git a/ipld/unixfs/unixfs.go b/ipld/unixfs/unixfs.go index 4131df837..bdc54518f 100644 --- a/ipld/unixfs/unixfs.go +++ b/ipld/unixfs/unixfs.go @@ -6,10 +6,12 @@ package unixfs import ( "errors" "fmt" + "os" + "time" proto "github.com/gogo/protobuf/proto" + files "github.com/ipfs/boxo/files" dag "github.com/ipfs/boxo/ipld/merkledag" - pb "github.com/ipfs/boxo/ipld/unixfs/pb" ipld "github.com/ipfs/go-ipld-format" ) @@ -69,6 +71,22 @@ func FilePBData(data []byte, totalsize uint64) []byte { return data } +func FilePBDataWithStat(data []byte, totalsize uint64, mode os.FileMode, mtime time.Time) []byte { + pbfile := new(pb.Data) + typ := pb.Data_File + pbfile.Type = &typ + pbfile.Data = data + pbfile.Filesize = proto.Uint64(totalsize) + + pbDataAddStat(pbfile, mode, mtime) + + data, err := proto.Marshal(pbfile) + if err != nil { + panic(err) + } + return data +} + // FolderPBData returns Bytes that represent a Directory. func FolderPBData() []byte { pbfile := new(pb.Data) @@ -83,6 +101,36 @@ func FolderPBData() []byte { return data } +func FolderPBDataWithStat(mode os.FileMode, mtime time.Time) []byte { + pbfile := new(pb.Data) + typ := pb.Data_Directory + pbfile.Type = &typ + + pbDataAddStat(pbfile, mode, mtime) + + data, err := proto.Marshal(pbfile) + if err != nil { + //this really shouldnt happen, i promise + panic(err) + } + return data +} + +func pbDataAddStat(data *pb.Data, mode os.FileMode, mtime time.Time) { + if mode != 0 { + data.Mode = proto.Uint32(files.ModePermsToUnixPerms(mode)) + } + if !mtime.IsZero() { + data.Mtime = &pb.IPFSTimestamp{ + Seconds: proto.Int64(mtime.Unix()), + } + + if nanos := uint32(mtime.Nanosecond()); nanos > 0 { + data.Mtime.Nanos = &nanos + } + } +} + // WrapData marshals raw bytes into a `Data_Raw` type protobuf message. func WrapData(b []byte) []byte { pbdata := new(pb.Data) @@ -303,6 +351,93 @@ func (n *FSNode) IsDir() bool { } } +// Mode returns the optionally stored file permissions +func (n *FSNode) Mode() (m os.FileMode) { + perms := n.format.GetMode() & 0xFFF + if perms != 0 { + m = files.UnixPermsToModePerms(perms) + switch n.Type() { + case pb.Data_Directory, pb.Data_HAMTShard: + m |= os.ModeDir + case pb.Data_Symlink: + m |= os.ModeSymlink + } + } + return m +} + +// SetMode stores the given mode permissions, or nullifies stored permissions +// if none were provided and there are no extended bits set. +func (n *FSNode) SetMode(m os.FileMode) { + n.SetModeFromUnixPermissions(files.ModePermsToUnixPerms(m)) +} + +// SetModeFromUnixPermissions stores the given unix permissions, or nullifies stored permissions +// if none were provided and there are no extended bits set. +func (n *FSNode) SetModeFromUnixPermissions(unixPerms uint32) { + // preserve existing most significant 20 bits + newMode := (n.format.GetMode() & 0xFFFFF000) | (unixPerms & 0xFFF) + + if unixPerms == 0 { + if newMode&0xFFFFF000 == 0 { + n.format.Mode = nil + return + } + } + n.format.Mode = &newMode +} + +// ExtendedMode returns the 20 bits of extended file mode +func (n *FSNode) ExtendedMode() uint32 { + return (n.format.GetMode() & 0xFFFFF000) >> 12 +} + +// SetExtendedMode stores the 20 bits of extended file mode, only the first +// 20 bits of the `mode` argument are used, the remaining 12 bits are ignored. +func (n *FSNode) SetExtendedMode(mode uint32) { + newMode := (mode << 12) | (0xFFF & n.format.GetMode()) + if newMode == 0 { + n.format.Mode = nil + } else { + n.format.Mode = &newMode + } +} + +// ModTime returns the stored last modified timestamp if available. +func (n *FSNode) ModTime() time.Time { + ts := n.format.GetMtime() + if ts == nil || ts.Seconds == nil { + return time.Time{} + } + if ts.Nanos == nil { + return time.Unix(*ts.Seconds, 0) + } + if *ts.Nanos < 1 || *ts.Nanos > 999999999 { + return time.Time{} + } + + return time.Unix(*ts.Seconds, int64(*ts.Nanos)) +} + +// SetModTime stores the given last modified timestamp, otherwise nullifies stored timestamp. +func (n *FSNode) SetModTime(ts time.Time) { + if ts.IsZero() { + n.format.Mtime = nil + return + } + + if n.format.Mtime == nil { + n.format.Mtime = &pb.IPFSTimestamp{} + } + + n.format.Mtime.Seconds = proto.Int64(ts.Unix()) + if ts.Nanosecond() > 0 { + n.format.Mtime.Nanos = proto.Uint32(uint32(ts.Nanosecond())) + } else { + n.format.Mtime.Nanos = nil + } +} + // Metadata is used to store additional FSNode information. type Metadata struct { MimeType string @@ -360,6 +495,10 @@ func EmptyDirNode() *dag.ProtoNode { return dag.NodeWithData(FolderPBData()) } +func EmptyDirNodeWithStat(mode os.FileMode, mtime time.Time) *dag.ProtoNode { + return dag.NodeWithData(FolderPBDataWithStat(mode, mtime)) +} + // EmptyFileNode creates an empty file Protonode. func EmptyFileNode() *dag.ProtoNode { return dag.NodeWithData(FilePBData(nil, 0)) diff --git a/ipld/unixfs/unixfs_test.go b/ipld/unixfs/unixfs_test.go index b785be8ad..4cbc22ca8 100644 --- a/ipld/unixfs/unixfs_test.go +++ b/ipld/unixfs/unixfs_test.go @@ -2,7 +2,9 @@ package unixfs import ( "bytes" + "os" "testing" + "time" proto "github.com/gogo/protobuf/proto" @@ -183,3 +185,251 @@ func TestIsDir(t *testing.T) { } } } + +func (n *FSNode) getPbData(t *testing.T) *pb.Data { + b, err := n.GetBytes() + if err != nil { + t.Fatal(err) + } + + pbn := new(pb.Data) + err = proto.Unmarshal(b, pbn) + if err != nil { + t.Fatal(err) + } + return pbn +} + +func TestMode(t *testing.T) { + fsn := NewFSNode(TDirectory) + fsn.SetMode(1) + if !fsn.Mode().IsDir() { + t.Fatal("expected mode for directory") + } + + fsn = NewFSNode(TSymlink) + fsn.SetMode(1) + if fsn.Mode()&os.ModeSymlink != os.ModeSymlink { + t.Fatal("expected mode for symlink") + } + + fsn = NewFSNode(TFile) + + // not stored + if fsn.Mode() != 0 { + t.Fatal("expected mode not to be set") + } + + fileMode := os.FileMode(0640) + fsn.SetMode(fileMode) + if !fsn.Mode().IsRegular() { + t.Fatal("expected a regular file mode") + } + mode := fsn.Mode() + + if mode&os.ModePerm != fileMode { + t.Fatalf("expected permissions to be %O but got %O", fileMode, mode&os.ModePerm) + } + if mode&0xFFFFF000 != 0 { + t.Fatalf("expected high-order 20 bits of mode to be clear but got %b", (mode&0xFFFFF000)>>12) + } + + fsn.SetMode(fileMode | os.ModeSticky) + mode = fsn.Mode() + if mode&os.ModePerm != fileMode&os.ModePerm { + t.Fatalf("expected permissions to be %O but got %O", fileMode, mode&os.ModePerm) + } + if mode&os.ModeSticky == 0 { + t.Fatal("expected permissions to have sticky bit set") + } + if mode&os.ModeSetuid != 0 { + t.Fatal("expected permissions to have setuid bit unset") + } + if mode&os.ModeSetgid != 0 { + t.Fatal("expected permissions to have setgid bit unset") + } + + fsn.SetMode(fileMode | os.ModeSticky | os.ModeSetuid) + mode = fsn.Mode() + if mode&os.ModePerm != fileMode&os.ModePerm { + t.Fatalf("expected permissions to be %O but got %O", fileMode, mode&os.ModePerm) + } + if mode&os.ModeSticky == 0 { + t.Fatal("expected permissions to have sticky bit set") + } + if mode&os.ModeSetuid == 0 { + t.Fatal("expected permissions to have setuid bit set") + } + if mode&os.ModeSetgid != 0 { + t.Fatal("expected permissions to have setgid bit unset") + } + + fsn.SetMode(fileMode | os.ModeSetuid | os.ModeSetgid) + mode = fsn.Mode() + if mode&os.ModePerm != fileMode&os.ModePerm { + t.Fatalf("expected permissions to be %O but got %O", fileMode, mode&os.ModePerm) + } + if mode&os.ModeSticky != 0 { + t.Fatal("expected permissions to have sticky bit unset") + } + if mode&os.ModeSetuid == 0 { + t.Fatal("expected permissions to have setuid bit set") + } + if mode&os.ModeSetgid == 0 { + t.Fatal("expected permissions to have setgid bit set") + } + + // check the internal format (unix permissions) + fsn.SetMode(fileMode | os.ModeSetuid | os.ModeSticky) + pbn := fsn.getPbData(t) + // unix perms setuid and sticky bits should also be set + expected := uint32(05000 | (fileMode & os.ModePerm)) + if *pbn.Mode != expected { + t.Fatalf("expected stored permissions to be %O but got %O", expected, *pbn.Mode) + } + + fsn.SetMode(0) + pbn = fsn.getPbData(t) + if pbn.Mode != nil { + t.Fatal("expected file mode to be unset") + } + + fsn.SetExtendedMode(1) + fsn.SetMode(0) + pbn = fsn.getPbData(t) + if pbn.Mode == nil { + t.Fatal("expected extended mode to be preserved") + } +} + +func TestExtendedMode(t *testing.T) { + fsn := NewFSNode(TFile) + fsn.SetMode(os.ModePerm | os.ModeSetuid | os.ModeSticky) + const expectedUnixMode = uint32(05777) + + expectedExtMode := uint32(0xAAAAA) + fsn.SetExtendedMode(expectedExtMode) + extMode := fsn.ExtendedMode() + if extMode != expectedExtMode { + t.Fatalf("expected extended mode to be %X but got %X", expectedExtMode, extMode) + } + pbn := fsn.getPbData(t) + expectedPbMode := (expectedExtMode << 12) | (expectedUnixMode & 0xFFF) + if *pbn.Mode != expectedPbMode { + t.Fatalf("expected stored mode to be %b but got %b", expectedPbMode, *pbn.Mode) + } + + expectedExtMode = uint32(0x55555) + fsn.SetExtendedMode(expectedExtMode) + extMode = fsn.ExtendedMode() + if extMode != expectedExtMode { + t.Fatalf("expected extended mode to be %X but got %X", expectedExtMode, extMode) + } + pbn = fsn.getPbData(t) + expectedPbMode = (expectedExtMode << 12) | (expectedUnixMode & 0xFFF) + if *pbn.Mode != expectedPbMode { + t.Fatalf("expected stored mode to be %b but got %b", expectedPbMode, *pbn.Mode) + } + + // ignore bits 21..32 + expectedExtMode = uint32(0xFFFFF) + fsn.SetExtendedMode(0xAAAFFFFF) + extMode = fsn.ExtendedMode() + if extMode != expectedExtMode { + t.Fatalf("expected extended mode to be %X but got %X", expectedExtMode, extMode) + } + pbn = fsn.getPbData(t) + expectedPbMode = (expectedExtMode << 12) | (expectedUnixMode & 0xFFF) + if *pbn.Mode != expectedPbMode { + t.Fatalf("expected raw mode to be %b but got %b", expectedPbMode, *pbn.Mode) + } + + fsn.SetMode(0) + fsn.SetExtendedMode(0) + pbn = fsn.getPbData(t) + if pbn.Mode != nil { + t.Fatal("expected file mode to be unset") + } +} + +func (n *FSNode) setPbModTime(seconds *int64, nanos *uint32) { + if n.format.Mtime == nil { + n.format.Mtime = &pb.IPFSTimestamp{} + } + + n.format.Mtime.Seconds = seconds + n.format.Mtime.Nanos = nanos +} + +func TestModTime(t *testing.T) { + tm := time.Now() + expectedUnix := tm.Unix() + n := NewFSNode(TFile) + + // not stored + mt := n.ModTime() + if !mt.IsZero() { + t.Fatal("expected modification time not to be set") + } + + // valid timestamps + n.SetModTime(tm) + mt = n.ModTime() + if !mt.Equal(tm) { + t.Fatalf("expected modification time to be %v but got %v", tm, mt) + } + + tm = time.Unix(expectedUnix, 0) + n.SetModTime(tm) + pbn := n.getPbData(t) + if pbn.Mtime.Nanos != nil { + t.Fatal("expected nanoseconds to be nil") + } + mt = n.ModTime() + if !mt.Equal(tm) { + t.Fatalf("expected modification time to be %v but got %v", tm, mt) + } + + tm = time.Unix(expectedUnix, 3489753) + n.SetModTime(tm) + mt = n.ModTime() + if !mt.Equal(tm) { + t.Fatalf("expected modification time to be %v but got %v", tm, mt) + } + + tm = time.Time{} + n.SetModTime(tm) + pbn = n.getPbData(t) + if pbn.Mtime != nil { + t.Fatal("expected modification time to be unset") + } + mt = n.ModTime() + if !mt.Equal(tm) { + t.Fatalf("expected modification time to be %v but got %v", tm, mt) + } + + n.setPbModTime(&expectedUnix, nil) + mt = n.ModTime() + if !mt.Equal(time.Unix(expectedUnix, 0)) { + t.Fatalf("expected modification time to be %v but got %v", time.Unix(expectedUnix, 0), mt) + } + + // invalid timestamps + n.setPbModTime(nil, proto.Uint32(1000)) + mt = n.ModTime() + if !mt.IsZero() { + t.Fatal("expected modification time not to be set") + } + + n.setPbModTime(&expectedUnix, proto.Uint32(0)) + mt = n.ModTime() + if !mt.IsZero() { + t.Fatal("expected modification time not to be set") + } + + n.setPbModTime(&expectedUnix, proto.Uint32(1000000000)) + mt = n.ModTime() + if !mt.IsZero() { + t.Fatal("expected modification time not to be set") + } +} diff --git a/mfs/dir.go b/mfs/dir.go index 86c85d1c5..38302ac39 100644 --- a/mfs/dir.go +++ b/mfs/dir.go @@ -41,8 +41,6 @@ type Directory struct { // UnixFS directory implementation used for creating, // reading and editing directories. unixfsDir uio.Directory - - modTime time.Time } // NewDirectory constructs a new MFS directory. @@ -64,7 +62,6 @@ func NewDirectory(ctx context.Context, name string, node ipld.Node, parent paren ctx: ctx, unixfsDir: db, entriesCache: make(map[string]FSNode), - modTime: time.Now(), }, nil } @@ -135,8 +132,6 @@ func (d *Directory) updateChild(c child) error { return err } - d.modTime = time.Now() - return nil } @@ -292,6 +287,10 @@ func (d *Directory) ForEachEntry(ctx context.Context, f func(NodeListing) error) } func (d *Directory) Mkdir(name string) (*Directory, error) { + return d.MkdirWithOpts(name, MkdirOpts{}) +} + +func (d *Directory) MkdirWithOpts(name string, opts MkdirOpts) (*Directory, error) { d.lock.Lock() defer d.lock.Unlock() @@ -307,7 +306,7 @@ func (d *Directory) Mkdir(name string) (*Directory, error) { } } - ndir := ft.EmptyDirNode() + ndir := ft.EmptyDirNodeWithStat(opts.Mode, opts.ModTime) ndir.SetCidBuilder(d.GetCidBuilder()) err = d.dagService.Add(d.ctx, ndir) @@ -367,7 +366,6 @@ func (d *Directory) AddChild(name string, nd ipld.Node) error { return err } - d.modTime = time.Now() return nil } @@ -427,3 +425,68 @@ func (d *Directory) GetNode() (ipld.Node, error) { return nd.Copy(), err } + +func (d *Directory) SetMode(mode os.FileMode) error { + nd, err := d.GetNode() + if err != nil { + return err + } + + fsn, err := ft.ExtractFSNode(nd) + if err != nil { + return err + } + + fsn.SetMode(mode) + data, err := fsn.GetBytes() + if err != nil { + return err + } + + return d.setNodeData(data, nd.Links()) +} + +func (d *Directory) SetModTime(ts time.Time) error { + nd, err := d.GetNode() + if err != nil { + return err + } + + fsn, err := ft.ExtractFSNode(nd) + if err != nil { + return err + } + + fsn.SetModTime(ts) + data, err := fsn.GetBytes() + if err != nil { + return err + } + + return d.setNodeData(data, nd.Links()) +} + +func (d *Directory) setNodeData(data []byte, links []*ipld.Link) error { + nd := dag.NodeWithData(data) + nd.SetLinks(links) + + err := d.dagService.Add(d.ctx, nd) + if err != nil { + return err + } + + err = d.parent.updateChildEntry(child{d.name, nd}) + if err != nil { + return err + } + + d.lock.Lock() + defer d.lock.Unlock() + db, err := uio.NewDirectoryFromNode(d.dagService, nd) + if err != nil { + return err + } + d.unixfsDir = db + + return nil +} diff --git a/mfs/file.go b/mfs/file.go index 56c2b0046..ef8ce5dfa 100644 --- a/mfs/file.go +++ b/mfs/file.go @@ -3,7 +3,9 @@ package mfs import ( "context" "errors" + "os" "sync" + "time" dag "github.com/ipfs/boxo/ipld/merkledag" ft "github.com/ipfs/boxo/ipld/unixfs" @@ -177,3 +179,91 @@ func (fi *File) Sync() error { func (fi *File) Type() NodeType { return TFile } + +func (fi *File) Mode() (os.FileMode, error) { + fi.nodeLock.RLock() + defer fi.nodeLock.RUnlock() + + nd, err := fi.GetNode() + if err == nil { + fsn, err := ft.ExtractFSNode(nd) + if err == nil { + return fsn.Mode() & 0xFFF, nil + } + } + + return 0, err +} + +func (fi *File) SetMode(mode os.FileMode) error { + nd, err := fi.GetNode() + if err != nil { + return err + } + + fsn, err := ft.ExtractFSNode(nd) + if err != nil { + return err + } + + fsn.SetMode(mode) + data, err := fsn.GetBytes() + if err != nil { + return err + } + + return fi.setNodeData(data) +} + +// ModTime returns the files' last modification time +func (fi *File) ModTime() (time.Time, error) { + fi.nodeLock.RLock() + defer fi.nodeLock.RUnlock() + + nd, err := fi.GetNode() + if err == nil { + fsn, err := ft.ExtractFSNode(nd) + if err == nil { + return fsn.ModTime(), nil + } + } + + return time.Time{}, err +} + +// SetModTime sets the files' last modification time +func (fi *File) SetModTime(ts time.Time) error { + nd, err := fi.GetNode() + if err != nil { + return err + } + + fsn, err := ft.ExtractFSNode(nd) + if err != nil { + return err + } + + fsn.SetModTime(ts) + data, err := fsn.GetBytes() + if err != nil { + return err + } + + return fi.setNodeData(data) +} + +func (fi *File) setNodeData(data []byte) error { + nd := dag.NodeWithData(data) + err := fi.inode.dagService.Add(context.TODO(), nd) + if err != nil { + return err + } + + fi.nodeLock.Lock() + defer fi.nodeLock.Unlock() + fi.node = nd + parent := fi.inode.parent + name := fi.inode.name + + return parent.updateChildEntry(child{name, fi.node}) +} diff --git a/mfs/mfs_test.go b/mfs/mfs_test.go index eb5585a64..8497dbb0e 100644 --- a/mfs/mfs_test.go +++ b/mfs/mfs_test.go @@ -511,7 +511,19 @@ func TestMfsFile(t *testing.T) { fi := fsn.(*File) if fi.Type() != TFile { - t.Fatal("some is seriously wrong here") + t.Fatal("something is seriously wrong here") + } + + if m, err := fi.Mode(); err != nil { + t.Fatal("failed to get file mode: ", err) + } else if m != 0 { + t.Fatal("mode should not be set on a new file") + } + + if ts, err := fi.ModTime(); err != nil { + t.Fatal("failed to get file mtime: ", err) + } else if !ts.IsZero() { + t.Fatal("modification time should not be set on a new file") } wfd, err := fi.Open(Flags{Read: true, Write: true, Sync: true}) @@ -615,6 +627,12 @@ func TestMfsFile(t *testing.T) { t.Fatal(err) } + if ts, err := fi.ModTime(); err != nil { + t.Fatal("failed to get file mtime: ", err) + } else if !ts.IsZero() { + t.Fatal("file with unset modification time should not update modification time") + } + // make sure we can get node. TODO: verify it later _, err = fi.GetNode() if err != nil { @@ -622,6 +640,141 @@ func TestMfsFile(t *testing.T) { } } +func TestMfsModeAndModTime(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ds, rt := setupRoot(ctx, t) + rootdir := rt.GetDirectory() + nd := getRandFile(t, ds, 1000) + + err := rootdir.AddChild("file", nd) + if err != nil { + t.Fatal(err) + } + + fsn, err := rootdir.Child("file") + if err != nil { + t.Fatal(err) + } + + fi := fsn.(*File) + + if fi.Type() != TFile { + t.Fatal("something is seriously wrong here") + } + + var mode os.FileMode + ts, ts2 := time.Now(), time.Time{} + + // can set mode + if err = fi.SetMode(0644); err == nil { + if mode, err = fi.Mode(); mode != 0644 { + t.Fatal("failed to get correct mode of file") + } + } + if err != nil { + t.Fatal("failed to check file mode: ", err) + } + + // can set last modification time + if err = fi.SetModTime(ts); err == nil { + if ts2, err = fi.ModTime(); !ts2.Equal(ts) { + t.Fatal("failed to get correct modification time of file") + } + } + if err != nil { + t.Fatal("failed to check file modification time: ", err) + } + + // test modification time update after write (on closing file) + wfd, err := fi.Open(Flags{Read: false, Write: true, Sync: true}) + if err != nil { + t.Fatal(err) + } + _, err = wfd.Write([]byte("test")) + if err != nil { + t.Fatal(err) + } + err = wfd.Close() + if err != nil { + t.Fatal(err) + } + ts2, err = fi.ModTime() + if err != nil { + t.Fatal(err) + } + if !ts2.After(ts) { + t.Fatal("modification time should be updated after file write") + } + + // writeAt + ts = ts2 + wfd, err = fi.Open(Flags{Read: false, Write: true, Sync: true}) + if err != nil { + t.Fatal(err) + } + _, err = wfd.WriteAt([]byte("test"), 42) + if err != nil { + t.Fatal(err) + } + err = wfd.Close() + if err != nil { + t.Fatal(err) + } + ts2, err = fi.ModTime() + if err != nil { + t.Fatal(err) + } + if !ts2.After(ts) { + t.Fatal("modification time should be updated after file writeAt") + } + + // truncate (shrink) + ts = ts2 + wfd, err = fi.Open(Flags{Read: false, Write: true, Sync: true}) + if err != nil { + t.Fatal(err) + } + err = wfd.Truncate(100) + if err != nil { + t.Fatal(err) + } + err = wfd.Close() + if err != nil { + t.Fatal(err) + } + ts2, err = fi.ModTime() + if err != nil { + t.Fatal(err) + } + if !ts2.After(ts) { + t.Fatal("modification time should be updated after file truncate (shrink)") + } + + // truncate (expand) + ts = ts2 + wfd, err = fi.Open(Flags{Read: false, Write: true, Sync: true}) + if err != nil { + t.Fatal(err) + } + err = wfd.Truncate(1500) + if err != nil { + t.Fatal(err) + } + err = wfd.Close() + if err != nil { + t.Fatal(err) + } + ts2, err = fi.ModTime() + if err != nil { + t.Fatal(err) + } + if !ts2.After(ts) { + t.Fatal("modification time should be updated after file truncate (expand)") + } +} + func TestMfsDirListNames(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/mfs/ops.go b/mfs/ops.go index 693264704..09dbab00f 100644 --- a/mfs/ops.go +++ b/mfs/ops.go @@ -7,6 +7,7 @@ import ( "os" gopath "path" "strings" + "time" cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" @@ -122,6 +123,8 @@ type MkdirOpts struct { Mkparents bool Flush bool CidBuilder cid.Builder + Mode os.FileMode + ModTime time.Time } // Mkdir creates a directory at 'path' under the directory 'd', creating @@ -171,7 +174,7 @@ func Mkdir(r *Root, pth string, opts MkdirOpts) error { cur = next } - final, err := cur.Mkdir(parts[len(parts)-1]) + final, err := cur.MkdirWithOpts(parts[len(parts)-1], opts) if err != nil { if !opts.Mkparents || err != os.ErrExist || final == nil { return err @@ -243,3 +246,21 @@ func FlushPath(ctx context.Context, rt *Root, pth string) (ipld.Node, error) { rt.repub.WaitPub(ctx) return nd.GetNode() } + +func Chmod(rt *Root, pth string, mode os.FileMode) error { + nd, err := Lookup(rt, pth) + if err != nil { + return err + } + + return nd.SetMode(mode) +} + +func Touch(rt *Root, pth string, ts time.Time) error { + nd, err := Lookup(rt, pth) + if err != nil { + return err + } + + return nd.SetModTime(ts) +} diff --git a/mfs/root.go b/mfs/root.go index c08d2d053..5a7cb7ed1 100644 --- a/mfs/root.go +++ b/mfs/root.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "os" "time" dag "github.com/ipfs/boxo/ipld/merkledag" @@ -73,6 +74,8 @@ type FSNode interface { Flush() error Type() NodeType + SetModTime(ts time.Time) error + SetMode(mode os.FileMode) error } // IsDir checks whether the FSNode is dir type From 0686a87d44a55a52ef82d4b04408916f7e05a7a7 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Mon, 12 Aug 2024 20:07:44 -0700 Subject: [PATCH 2/5] lint fix --- mfs/mfs_test.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/mfs/mfs_test.go b/mfs/mfs_test.go index 8497dbb0e..4793ae141 100644 --- a/mfs/mfs_test.go +++ b/mfs/mfs_test.go @@ -665,7 +665,7 @@ func TestMfsModeAndModTime(t *testing.T) { } var mode os.FileMode - ts, ts2 := time.Now(), time.Time{} + ts, _ := time.Now(), time.Time{} // can set mode if err = fi.SetMode(0644); err == nil { @@ -679,7 +679,11 @@ func TestMfsModeAndModTime(t *testing.T) { // can set last modification time if err = fi.SetModTime(ts); err == nil { - if ts2, err = fi.ModTime(); !ts2.Equal(ts) { + ts2, err := fi.ModTime() + if err != nil { + t.Fatal(err) + } + if !ts2.Equal(ts) { t.Fatal("failed to get correct modification time of file") } } @@ -700,7 +704,7 @@ func TestMfsModeAndModTime(t *testing.T) { if err != nil { t.Fatal(err) } - ts2, err = fi.ModTime() + ts2, err := fi.ModTime() if err != nil { t.Fatal(err) } From bfa8674d96906cabba270d4f64e45f0dccce5eee Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Mon, 12 Aug 2024 20:14:48 -0700 Subject: [PATCH 3/5] fix windows test --- mfs/mfs_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mfs/mfs_test.go b/mfs/mfs_test.go index 4793ae141..223830d18 100644 --- a/mfs/mfs_test.go +++ b/mfs/mfs_test.go @@ -11,6 +11,7 @@ import ( "math/rand" "os" gopath "path" + "runtime" "sort" "strings" "sync" @@ -692,6 +693,9 @@ func TestMfsModeAndModTime(t *testing.T) { } // test modification time update after write (on closing file) + if runtime.GOOS == "windows" { + time.Sleep(3 * time.Second) // for os with low-res mod time. + } wfd, err := fi.Open(Flags{Read: false, Write: true, Sync: true}) if err != nil { t.Fatal(err) From 738001b382b13a51cf37501d475094c04f3f8086 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Mon, 12 Aug 2024 20:33:47 -0700 Subject: [PATCH 4/5] Fix windows tests for modtime --- mfs/mfs_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mfs/mfs_test.go b/mfs/mfs_test.go index 223830d18..80d0026a1 100644 --- a/mfs/mfs_test.go +++ b/mfs/mfs_test.go @@ -718,6 +718,9 @@ func TestMfsModeAndModTime(t *testing.T) { // writeAt ts = ts2 + if runtime.GOOS == "windows" { + time.Sleep(3 * time.Second) // for os with low-res mod time. + } wfd, err = fi.Open(Flags{Read: false, Write: true, Sync: true}) if err != nil { t.Fatal(err) @@ -740,6 +743,9 @@ func TestMfsModeAndModTime(t *testing.T) { // truncate (shrink) ts = ts2 + if runtime.GOOS == "windows" { + time.Sleep(3 * time.Second) // for os with low-res mod time. + } wfd, err = fi.Open(Flags{Read: false, Write: true, Sync: true}) if err != nil { t.Fatal(err) @@ -762,6 +768,9 @@ func TestMfsModeAndModTime(t *testing.T) { // truncate (expand) ts = ts2 + if runtime.GOOS == "windows" { + time.Sleep(3 * time.Second) // for os with low-res mod time. + } wfd, err = fi.Open(Flags{Read: false, Write: true, Sync: true}) if err != nil { t.Fatal(err) From 1b5bb03a0b963cbc2109bae66f56ccd57f7314a9 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Mon, 12 Aug 2024 20:48:11 -0700 Subject: [PATCH 5/5] fix more windows tests --- ipld/unixfs/importer/trickle/trickle_test.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/ipld/unixfs/importer/trickle/trickle_test.go b/ipld/unixfs/importer/trickle/trickle_test.go index 61899c9ff..d495fd208 100644 --- a/ipld/unixfs/importer/trickle/trickle_test.go +++ b/ipld/unixfs/importer/trickle/trickle_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io" mrand "math/rand" + "runtime" "testing" "time" @@ -678,14 +679,17 @@ func TestAppendWithModTime(t *testing.T) { const nbytes = 128 * 1024 timestamp := time.Now() - ds := mdtest.Mock() buf := random.Bytes(nbytes) nd := new(merkledag.ProtoNode) nd.SetData(ft.FilePBDataWithStat(buf[:nbytes/2], nbytes/2, 0, timestamp)) + if runtime.GOOS == "windows" { + time.Sleep(3 * time.Second) // for os with low-res mod time. + } + dbp := &h.DagBuilderParams{ - Dagserv: ds, + Dagserv: mdtest.Mock(), Maxlinks: h.DefaultLinksPerBlock, } @@ -710,12 +714,15 @@ func TestAppendWithModTime(t *testing.T) { func TestAppendToEmptyWithModTime(t *testing.T) { timestamp := time.Now() - ds := mdtest.Mock() nd := new(merkledag.ProtoNode) nd.SetData(ft.FilePBDataWithStat(nil, 0, 0, timestamp)) + if runtime.GOOS == "windows" { + time.Sleep(3 * time.Second) // for os with low-res mod time. + } + dbp := &h.DagBuilderParams{ - Dagserv: ds, + Dagserv: mdtest.Mock(), Maxlinks: h.DefaultLinksPerBlock, }