Merge pull request ipfs/go-unixfsnode#4 from ipfs/feat/support-hamt

Support HAMT

This commit was moved from ipfs/go-unixfsnode@5a89cd4

unixfs/node/data/builder/builder.go

@@ -0,0 +1,143 @@
+package builder
+
+import (
+	"errors"
+	"strconv"
+	"time"
+
+	"github.com/ipfs/go-unixfsnode/data"
+	"github.com/ipld/go-ipld-prime"
+	"github.com/ipld/go-ipld-prime/fluent/qp"
+)
+
+// BuildUnixFS provides a clean, validated interface to building data structures
+// that match the UnixFS protobuf encoded in the Data member of a ProtoNode
+// with sensible defaults
+//
+//   smallFileData, err := BuildUnixFS(func(b *Builder) {
+//      Data(b, []byte{"hello world"})
+//      Mtime(b, func(tb TimeBuilder) {
+//				Time(tb, time.Now())
+//			})
+//   })
+//
+func BuildUnixFS(fn func(*Builder)) (data.UnixFSData, error) {
+	nd, err := qp.BuildMap(data.Type.UnixFSData, -1, func(ma ipld.MapAssembler) {
+		b := &Builder{MapAssembler: ma}
+		fn(b)
+		if !b.hasBlockSizes {
+			qp.MapEntry(ma, data.Field__BlockSizes, qp.List(0, func(ipld.ListAssembler) {}))
+		}
+		if !b.hasDataType {
+			qp.MapEntry(ma, data.Field__DataType, qp.Int(data.Data_File))
+		}
+	})
+	if err != nil {
+		return nil, err
+	}
+	return nd.(data.UnixFSData), nil
+}
+
+// Builder is an interface for making UnixFS data nodes
+type Builder struct {
+	ipld.MapAssembler
+	hasDataType   bool
+	hasBlockSizes bool
+}
+
+// DataType sets the default on a builder for a UnixFS node - default is File
+func DataType(b *Builder, dataType int64) {
+	_, ok := data.DataTypeNames[dataType]
+	if !ok {
+		panic(data.ErrInvalidDataType{dataType})
+	}
+	qp.MapEntry(b.MapAssembler, data.Field__DataType, qp.Int(dataType))
+	b.hasDataType = true
+}
+
+// Data sets the data member inside the UnixFS data
+func Data(b *Builder, dataBytes []byte) {
+	qp.MapEntry(b.MapAssembler, data.Field__Data, qp.Bytes(dataBytes))
+}
+
+// FileSize sets the file size which should be the size of actual bytes underneath
+// this node for large files, w/o additional bytes to encode intermediate nodes
+func FileSize(b *Builder, fileSize uint64) {
+	qp.MapEntry(b.MapAssembler, data.Field__FileSize, qp.Int(int64(fileSize)))
+}
+
+// BlockSizes encodes block sizes for each child node
+func BlockSizes(b *Builder, blockSizes []uint64) {
+	qp.MapEntry(b.MapAssembler, data.Field__BlockSizes, qp.List(int64(len(blockSizes)), func(la ipld.ListAssembler) {
+		for _, bs := range blockSizes {
+			qp.ListEntry(la, qp.Int(int64(bs)))
+		}
+	}))
+	b.hasBlockSizes = true
+}
+
+// HashType sets the hash function for this node -- only applicable to HAMT
+func HashType(b *Builder, hashType uint64) {
+	qp.MapEntry(b.MapAssembler, data.Field__HashType, qp.Int(int64(hashType)))
+}
+
+// Fanout sets the fanout in a HAMT tree
+func Fanout(b *Builder, fanout uint64) {
+	qp.MapEntry(b.MapAssembler, data.Field__Fanout, qp.Int(int64(fanout)))
+}
+
+// Permissions sets file permissions for the Mode member of the UnixFS node
+func Permissions(b *Builder, mode int) {
+	mode = mode & 0xFFF
+	qp.MapEntry(b.MapAssembler, data.Field__Mode, qp.Int(int64(mode)))
+}
+
+func parseModeString(modeString string) (uint64, error) {
+	if len(modeString) > 0 && modeString[0] == '0' {
+		return strconv.ParseUint(modeString, 8, 32)
+	}
+	return strconv.ParseUint(modeString, 10, 32)
+}
+
+// PermissionsString sets file permissions for the Mode member of the UnixFS node,
+// parsed from a typical octect encoded permission string (eg '0755')
+func PermissionsString(b *Builder, modeString string) {
+	mode64, err := parseModeString(modeString)
+	if err != nil {
+		panic(err)
+	}
+	mode64 = mode64 & 0xFFF
+	qp.MapEntry(b.MapAssembler, data.Field__Mode, qp.Int(int64(mode64)))
+}
+
+// Mtime sets the modification time for this node using the time builder interface
+// and associated methods
+func Mtime(b *Builder, fn func(tb TimeBuilder)) {
+	qp.MapEntry(b.MapAssembler, data.Field__Mtime, qp.Map(-1, func(ma ipld.MapAssembler) {
+		fn(ma)
+	}))
+}
+
+// TimeBuilder is a simple interface for constructing the time member of UnixFS data
+type TimeBuilder ipld.MapAssembler
+
+// Time sets the modification time from a golang time value
+func Time(ma TimeBuilder, t time.Time) {
+	Seconds(ma, t.Unix())
+	FractionalNanoseconds(ma, int32(t.Nanosecond()))
+}
+
+// Seconds sets the seconds for a modification time
+func Seconds(ma TimeBuilder, seconds int64) {
+	qp.MapEntry(ma, data.Field__Seconds, qp.Int(seconds))
+
+}
+
+// FractionalNanoseconds sets the nanoseconds for a modification time (must
+// be between 0 & a billion)
+func FractionalNanoseconds(ma TimeBuilder, nanoseconds int32) {
+	if nanoseconds < 0 || nanoseconds > 999999999 {
+		panic(errors.New("mtime-nsecs must be within the range [0,999999999]"))
+	}
+	qp.MapEntry(ma, data.Field__Nanoseconds, qp.Int(int64(nanoseconds)))
+}

unixfs/node/data/datatypes.go

@@ -0,0 +1,40 @@
+package data
+
+const (
+	Data_Raw       int64 = 0
+	Data_Directory int64 = 1
+	Data_File      int64 = 2
+	Data_Metadata  int64 = 3
+	Data_Symlink   int64 = 4
+	Data_HAMTShard int64 = 5
+)
+
+var DataTypeNames = map[int64]string{
+	Data_Raw:       "Raw",
+	Data_Directory: "Directory",
+	Data_File:      "File",
+	Data_Metadata:  "Metadata",
+	Data_Symlink:   "Symlink",
+	Data_HAMTShard: "HAMTShard",
+}
+
+var DataTypeValues = map[string]int64{
+	"Raw":       Data_Raw,
+	"Directory": Data_Directory,
+	"File":      Data_File,
+	"Metadata":  Data_Metadata,
+	"Symlink":   Data_Symlink,
+	"HAMTShard": Data_HAMTShard,
+}
+
+const Field__DataType = "DataType"
+const Field__Data = "Data"
+const Field__FileSize = "FileSize"
+const Field__BlockSizes = "BlockSizes"
+const Field__HashType = "HashType"
+const Field__Fanout = "Fanout"
+const Field__Mode = "Mode"
+const Field__Mtime = "Mtime"
+const Field__Seconds = "Seconds"
+const Field__Nanoseconds = "FractionalNanoseconds"
+const Field__MimeType = "MimeType"

unixfs/node/data/doc.go

@@ -0,0 +1,14 @@
+/*
+Package data provides tools for working with the UnixFS data structure that
+is encoded in the "Data" field of the larger a DagPB encoded IPLD node.
+
+See https://github.com/ipfs/specs/blob/master/UNIXFS.md for more information
+about this data structure.
+
+This package provides an IPLD Prime compatible node interface for this data
+structure, as well as methods for serializing and deserializing the data
+structure to protobuf
+*/
+package data
+
+//go:generate go run ./gen

unixfs/node/data/errors.go

@@ -0,0 +1,43 @@
+package data
+
+import (
+	"fmt"
+
+	"google.golang.org/protobuf/encoding/protowire"
+)
+
+type ErrWrongNodeType struct {
+	Expected int64
+	Actual   int64
+}
+
+func (e ErrWrongNodeType) Error() string {
+	expectedName, ok := DataTypeNames[e.Expected]
+	if !ok {
+		expectedName = "Unknown Type"
+	}
+	actualName, ok := DataTypeNames[e.Actual]
+	if !ok {
+		actualName = "Unknown Type"
+	}
+	return fmt.Sprintf("incorrect Node Type: (UnixFSData) expected type: %s, actual type: %s", expectedName, actualName)
+}
+
+type ErrWrongWireType struct {
+	Module   string
+	Field    string
+	Expected protowire.Type
+	Actual   protowire.Type
+}
+
+func (e ErrWrongWireType) Error() string {
+	return fmt.Sprintf("protobuf: (%s) invalid wireType, field: %s, expected %d, got %d", e.Module, e.Field, e.Expected, e.Actual)
+}
+
+type ErrInvalidDataType struct {
+	DataType int64
+}
+
+func (e ErrInvalidDataType) Error() string {
+	return fmt.Sprintf("type: %d is not valid", e.DataType)
+}

unixfs/node/data/fixtures/directory.unixfs

@@ -0,0 +1 @@
+

unixfs/node/data/fixtures/directory/file.txt

@@ -0,0 +1 @@
+Hello UnixFS

unixfs/node/data/fixtures/file.txt

@@ -0,0 +1 @@
+Hello UnixFS

unixfs/node/data/fixtures/file.txt.unixfs

@@ -0,0 +1,2 @@
+Hello UnixFS
+

unixfs/node/data/fixtures/raw.unixfs

@@ -0,0 +1,2 @@
+Hello UnixFS
+

unixfs/node/data/fixtures/symlink.txt

@@ -0,0 +1 @@
+Hello UnixFS

unixfs/node/data/fixtures/symlink.txt.unixfs

@@ -0,0 +1 @@
+file.txt