From 219eae76143d7171721239a57e2a3e03be6911d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 5 Jan 2018 01:50:14 +0100 Subject: [PATCH 1/5] coreapi: draft block API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/interface/interface.go | 18 ++++++++++++++++++ core/coreapi/interface/options/block.go | 14 ++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 core/coreapi/interface/options/block.go diff --git a/core/coreapi/interface/interface.go b/core/coreapi/interface/interface.go index ddcdc8db67f..147a85412b3 100644 --- a/core/coreapi/interface/interface.go +++ b/core/coreapi/interface/interface.go @@ -53,6 +53,11 @@ type Key interface { Path() Path } +type BlockStat interface { + Size() int + Path() Path +} + // CoreAPI defines an unified interface to IPFS for Go programs. type CoreAPI interface { // Unixfs returns an implementation of Unixfs API. @@ -87,6 +92,19 @@ type UnixfsAPI interface { Ls(context.Context, Path) ([]*Link, error) } +type BlockAPI interface { + Put(context.Context, io.Reader) (Path, error) + WithCodec(codec uint64) options.BlockPutOption + WithHash(mhType uint64, mhLen int) options.BlockPutOption + + Get(context.Context) (io.Reader, error) + + Rm(context.Context) error + WithForce(force bool) options.BlockRmOption + + Stat(context.Context) (BlockStat, error) +} + // DagAPI specifies the interface to IPLD type DagAPI interface { // Put inserts data using specified format and input encoding. diff --git a/core/coreapi/interface/options/block.go b/core/coreapi/interface/options/block.go new file mode 100644 index 00000000000..e2473e3f737 --- /dev/null +++ b/core/coreapi/interface/options/block.go @@ -0,0 +1,14 @@ +package options + +type BlockPutSettings struct { + Codec uint64 + MhType uint64 + MhLength int +} + +type BlockRmSettings struct { + Force bool +} + +type BlockPutOption func(*BlockPutSettings) error +type BlockRmOption func(*BlockRmSettings) error From bdc9f6a96adfdac63dd607b1301613a183400eea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 6 Jan 2018 16:57:41 +0100 Subject: [PATCH 2/5] coreapi: implement block API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/block.go | 131 ++++++++++++++++++++++++ core/coreapi/coreapi.go | 4 + core/coreapi/interface/interface.go | 12 ++- core/coreapi/interface/options/block.go | 61 ++++++++++- 4 files changed, 202 insertions(+), 6 deletions(-) create mode 100644 core/coreapi/block.go diff --git a/core/coreapi/block.go b/core/coreapi/block.go new file mode 100644 index 00000000000..53d14dcc63a --- /dev/null +++ b/core/coreapi/block.go @@ -0,0 +1,131 @@ +package coreapi + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "io/ioutil" + + util "github.com/ipfs/go-ipfs/blocks/blockstore/util" + coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" + caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + + blocks "gx/ipfs/QmYsEQydGrsxNZfAiskvQ76N2xE9hDQtSAkRSynwMiUK3c/go-block-format" + cid "gx/ipfs/QmeSrf6pzut73u6zLQkRFQ3ygt3k6XFT2kjdYP8Tnkwwyg/go-cid" +) + +type BlockAPI struct { + *CoreAPI + *caopts.BlockOptions +} + +type BlockStat struct { + path coreiface.Path + size int +} + +func (api *BlockAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.BlockPutOption) (coreiface.Path, error) { + settings, err := caopts.BlockPutOptions(opts...) + if err != nil { + return nil, err + } + + data, err := ioutil.ReadAll(src) + if err != nil { + return nil, err + } + + var pref cid.Prefix + pref.Version = 1 + + formatval, ok := cid.Codecs[settings.Codec] + if !ok { + return nil, fmt.Errorf("unrecognized format: %s", settings.Codec) + } + if settings.Codec == "v0" { + pref.Version = 0 + } + pref.Codec = formatval + + pref.MhType = settings.MhType + pref.MhLength = settings.MhLength + + bcid, err := pref.Sum(data) + if err != nil { + return nil, err + } + + b, err := blocks.NewBlockWithCid(data, bcid) + if err != nil { + return nil, err + } + + k, err := api.node.Blocks.AddBlock(b) + if err != nil { + return nil, err + } + + return ParseCid(k), nil +} + +func (api *BlockAPI) Get(ctx context.Context, p coreiface.Path) (io.Reader, error) { + b, err := api.node.Blocks.GetBlock(ctx, p.Cid()) + if err != nil { + return nil, err + } + + return bytes.NewReader(b.RawData()), nil +} + +func (api *BlockAPI) Rm(ctx context.Context, p coreiface.Path, opts ...caopts.BlockRmOption) error { + settings, err := caopts.BlockRmOptions(opts...) + if err != nil { + return err + } + cids := []*cid.Cid{p.Cid()} + o := util.RmBlocksOpts{Force: settings.Force} + + out, err := util.RmBlocks(api.node.Blockstore, api.node.Pinning, cids, o) + if err != nil { + return err + } + + select { + case res := <-out: + remBlock, ok := res.(*util.RemovedBlock) + if !ok { + return errors.New("got unexpected output from util.RmBlocks") + } + + if remBlock.Error != "" { + return errors.New(remBlock.Error) + } + return nil + case <-ctx.Done(): + return ctx.Err() + } + + return nil +} + +func (api *BlockAPI) Stat(ctx context.Context, p coreiface.Path) (coreiface.BlockStat, error) { + b, err := api.node.Blocks.GetBlock(ctx, p.Cid()) + if err != nil { + return nil, err + } + + return &BlockStat{ + path: ParseCid(b.Cid()), + size: len(b.RawData()), + }, nil +} + +func (bs *BlockStat) Size() int { + return bs.size +} + +func (bs *BlockStat) Path() coreiface.Path { + return bs.path +} diff --git a/core/coreapi/coreapi.go b/core/coreapi/coreapi.go index b289bb38253..ed8e3f097f8 100644 --- a/core/coreapi/coreapi.go +++ b/core/coreapi/coreapi.go @@ -26,6 +26,10 @@ func (api *CoreAPI) Unixfs() coreiface.UnixfsAPI { return (*UnixfsAPI)(api) } +func (api *CoreAPI) Block() coreiface.BlockAPI { + return &BlockAPI{api, nil} +} + // Dag returns the DagAPI interface backed by the go-ipfs node func (api *CoreAPI) Dag() coreiface.DagAPI { return &DagAPI{api, nil} diff --git a/core/coreapi/interface/interface.go b/core/coreapi/interface/interface.go index 147a85412b3..bbe544344bd 100644 --- a/core/coreapi/interface/interface.go +++ b/core/coreapi/interface/interface.go @@ -62,6 +62,8 @@ type BlockStat interface { type CoreAPI interface { // Unixfs returns an implementation of Unixfs API. Unixfs() UnixfsAPI + // Block returns an implementation of Block API. + Block() BlockAPI // Dag returns an implementation of Dag API. Dag() DagAPI // Name returns an implementation of Name API. @@ -93,16 +95,16 @@ type UnixfsAPI interface { } type BlockAPI interface { - Put(context.Context, io.Reader) (Path, error) - WithCodec(codec uint64) options.BlockPutOption + Put(context.Context, io.Reader, ...options.BlockPutOption) (Path, error) + WithFormat(codec string) options.BlockPutOption WithHash(mhType uint64, mhLen int) options.BlockPutOption - Get(context.Context) (io.Reader, error) + Get(context.Context, Path) (io.Reader, error) - Rm(context.Context) error + Rm(context.Context, Path, ...options.BlockRmOption) error WithForce(force bool) options.BlockRmOption - Stat(context.Context) (BlockStat, error) + Stat(context.Context, Path) (BlockStat, error) } // DagAPI specifies the interface to IPLD diff --git a/core/coreapi/interface/options/block.go b/core/coreapi/interface/options/block.go index e2473e3f737..7e6ad3230bb 100644 --- a/core/coreapi/interface/options/block.go +++ b/core/coreapi/interface/options/block.go @@ -1,7 +1,12 @@ package options +import ( + //cid "gx/ipfs/QmeSrf6pzut73u6zLQkRFQ3ygt3k6XFT2kjdYP8Tnkwwyg/go-cid" + "gx/ipfs/QmYeKnKpubCMRiq3PGZcTREErthbb5Q9cXsCoSkD9bjEBd/go-multihash" +) + type BlockPutSettings struct { - Codec uint64 + Codec string MhType uint64 MhLength int } @@ -12,3 +17,57 @@ type BlockRmSettings struct { type BlockPutOption func(*BlockPutSettings) error type BlockRmOption func(*BlockRmSettings) error + +func BlockPutOptions(opts ...BlockPutOption) (*BlockPutSettings, error) { + options := &BlockPutSettings{ + Codec: "v0", + MhType: multihash.SHA2_256, + MhLength: -1, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +func BlockRmOptions(opts ...BlockRmOption) (*BlockRmSettings, error) { + options := &BlockRmSettings{ + Force: false, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + return options, nil +} + +type BlockOptions struct{} + +func (api *BlockOptions) WithFormat(codec string) BlockPutOption { + return func(settings *BlockPutSettings) error { + settings.Codec = codec + return nil + } +} + +func (api *BlockOptions) WithHash(mhType uint64, mhLen int) BlockPutOption { + return func(settings *BlockPutSettings) error { + settings.MhType = mhType + settings.MhLength = mhLen + return nil + } +} + +func (api *BlockOptions) WithForce(force bool) BlockRmOption { + return func(settings *BlockRmSettings) error { + settings.Force = force + return nil + } +} From b125c89ac78de94ab244d411979aa2f43e2be285 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 6 Jan 2018 17:13:33 +0100 Subject: [PATCH 3/5] corapi: block docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/interface/interface.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/core/coreapi/interface/interface.go b/core/coreapi/interface/interface.go index bbe544344bd..2402ecf8138 100644 --- a/core/coreapi/interface/interface.go +++ b/core/coreapi/interface/interface.go @@ -94,16 +94,35 @@ type UnixfsAPI interface { Ls(context.Context, Path) ([]*Link, error) } +// BlockAPI specifies the interface to the block layer type BlockAPI interface { + // Put imports raw block data, hashing it using specified settings. Put(context.Context, io.Reader, ...options.BlockPutOption) (Path, error) + + // WithFormat is an option for Put which specifies the multicodec to use to + // serialize the object. Default is "v0" WithFormat(codec string) options.BlockPutOption + + // WithHash is an option for Put which specifies the multihash settings to use + // when hashing the object. Default is mh.SHA2_256 (0x12). + // If mhLen is set to -1, default length for the hash will be used WithHash(mhType uint64, mhLen int) options.BlockPutOption + // Get attempts to resolve the path and return a reader for data in the block Get(context.Context, Path) (io.Reader, error) + // Rm removes the block specified by the path from local blockstore. + // By default an error will be returned if the block can't be found locally. + // + // NOTE: If the specified block is pinned it won't be removed and no error + // will be returned Rm(context.Context, Path, ...options.BlockRmOption) error + + // WithForce is an option for Rm which, when set to true, will ignore + // non-existing blocks WithForce(force bool) options.BlockRmOption + // Stat returns information on Stat(context.Context, Path) (BlockStat, error) } From bb6bf4ce53c6a6a06d27997514807b379cade1c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 8 Jan 2018 14:01:18 +0100 Subject: [PATCH 4/5] coreapi: block tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/block.go | 9 +- core/coreapi/block_test.go | 167 +++++++++++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+), 2 deletions(-) create mode 100644 core/coreapi/block_test.go diff --git a/core/coreapi/block.go b/core/coreapi/block.go index 53d14dcc63a..561389f074f 100644 --- a/core/coreapi/block.go +++ b/core/coreapi/block.go @@ -12,6 +12,7 @@ import ( coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" + mh "gx/ipfs/QmYeKnKpubCMRiq3PGZcTREErthbb5Q9cXsCoSkD9bjEBd/go-multihash" blocks "gx/ipfs/QmYsEQydGrsxNZfAiskvQ76N2xE9hDQtSAkRSynwMiUK3c/go-block-format" cid "gx/ipfs/QmeSrf6pzut73u6zLQkRFQ3ygt3k6XFT2kjdYP8Tnkwwyg/go-cid" ) @@ -44,7 +45,7 @@ func (api *BlockAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.Bloc if !ok { return nil, fmt.Errorf("unrecognized format: %s", settings.Codec) } - if settings.Codec == "v0" { + if settings.Codec == "v0" && settings.MhType == mh.SHA2_256 { pref.Version = 0 } pref.Codec = formatval @@ -93,7 +94,11 @@ func (api *BlockAPI) Rm(ctx context.Context, p coreiface.Path, opts ...caopts.Bl } select { - case res := <-out: + case res, ok := <-out: + if !ok { + return nil + } + remBlock, ok := res.(*util.RemovedBlock) if !ok { return errors.New("got unexpected output from util.RmBlocks") diff --git a/core/coreapi/block_test.go b/core/coreapi/block_test.go new file mode 100644 index 00000000000..bef532fdc4a --- /dev/null +++ b/core/coreapi/block_test.go @@ -0,0 +1,167 @@ +package coreapi_test + +import ( + "context" + "io/ioutil" + "strings" + "testing" + + mh "gx/ipfs/QmYeKnKpubCMRiq3PGZcTREErthbb5Q9cXsCoSkD9bjEBd/go-multihash" +) + +func TestBlockPut(t *testing.T) { + ctx := context.Background() + _, api, err := makeAPI(ctx) + if err != nil { + t.Error(err) + } + + res, err := api.Block().Put(ctx, strings.NewReader(`Hello`)) + if err != nil { + t.Error(err) + } + + if res.Cid().String() != "QmPyo15ynbVrSTVdJL9th7JysHaAbXt9dM9tXk1bMHbRtk" { + t.Errorf("got wrong cid: %s", res.Cid().String()) + } +} + +func TestBlockPutFormat(t *testing.T) { + ctx := context.Background() + _, api, err := makeAPI(ctx) + if err != nil { + t.Error(err) + } + + res, err := api.Block().Put(ctx, strings.NewReader(`Hello`), api.Block().WithFormat("cbor")) + if err != nil { + t.Error(err) + } + + if res.Cid().String() != "zdpuAn4amuLWo8Widi5v6VQpuo2dnpnwbVE3oB6qqs7mDSeoa" { + t.Errorf("got wrong cid: %s", res.Cid().String()) + } +} + +func TestBlockPutHash(t *testing.T) { + ctx := context.Background() + _, api, err := makeAPI(ctx) + if err != nil { + t.Error(err) + } + + res, err := api.Block().Put(ctx, strings.NewReader(`Hello`), api.Block().WithHash(mh.KECCAK_512, -1)) + if err != nil { + t.Error(err) + } + + if res.Cid().String() != "zBurKB9YZkcDf6xa53WBE8CFX4ydVqAyf9KPXBFZt5stJzEstaS8Hukkhu4gwpMtc1xHNDbzP7sPtQKyWsP3C8fbhkmrZ" { + t.Errorf("got wrong cid: %s", res.Cid().String()) + } +} + +func TestBlockGet(t *testing.T) { + ctx := context.Background() + _, api, err := makeAPI(ctx) + if err != nil { + t.Error(err) + } + + res, err := api.Block().Put(ctx, strings.NewReader(`Hello`), api.Block().WithHash(mh.KECCAK_512, -1)) + if err != nil { + t.Error(err) + } + + r, err := api.Block().Get(ctx, res) + if err != nil { + t.Error(err) + } + + d, err := ioutil.ReadAll(r) + if err != nil { + t.Error(err) + } + + if string(d) != "Hello" { + t.Error("didn't get correct data back") + } +} + +func TestBlockRm(t *testing.T) { + ctx := context.Background() + _, api, err := makeAPI(ctx) + if err != nil { + t.Error(err) + } + + res, err := api.Block().Put(ctx, strings.NewReader(`Hello`)) + if err != nil { + t.Error(err) + } + + r, err := api.Block().Get(ctx, res) + if err != nil { + t.Error(err) + } + + d, err := ioutil.ReadAll(r) + if err != nil { + t.Error(err) + } + + if string(d) != "Hello" { + t.Error("didn't get correct data back") + } + + err = api.Block().Rm(ctx, res) + if err != nil { + t.Error(err) + } + + _, err = api.Block().Get(ctx, res) + if err == nil { + t.Error("expected err to exist") + } + if err.Error() != "blockservice: key not found" { + t.Errorf("unexpected error; %s", err.Error()) + } + + err = api.Block().Rm(ctx, res) + if err == nil { + t.Error("expected err to exist") + } + if err.Error() != "blockstore: block not found" { + t.Errorf("unexpected error; %s", err.Error()) + } + + err = api.Block().Rm(ctx, res, api.Block().WithForce(true)) + if err != nil { + t.Error(err) + } +} + +func TestBlockStat(t *testing.T) { + ctx := context.Background() + _, api, err := makeAPI(ctx) + if err != nil { + t.Error(err) + } + + res, err := api.Block().Put(ctx, strings.NewReader(`Hello`)) + if err != nil { + t.Error(err) + } + + stat, err := api.Block().Stat(ctx, res) + if err != nil { + t.Error(err) + } + + if stat.Path().String() != res.String() { + t.Error("paths don't match") + } + + if stat.Size() != len("Hello") { + t.Error("length doesn't match") + } +} From e77b93816104605adf0bf4830fa6922cac261e10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 31 Jan 2018 00:34:51 +0100 Subject: [PATCH 5/5] coreapi: update block after update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/block.go | 10 +++++----- core/coreapi/block_test.go | 2 +- core/coreapi/interface/interface.go | 4 ++++ core/coreapi/interface/options/block.go | 3 +-- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/core/coreapi/block.go b/core/coreapi/block.go index 561389f074f..ffb619597d3 100644 --- a/core/coreapi/block.go +++ b/core/coreapi/block.go @@ -12,9 +12,9 @@ import ( coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - mh "gx/ipfs/QmYeKnKpubCMRiq3PGZcTREErthbb5Q9cXsCoSkD9bjEBd/go-multihash" - blocks "gx/ipfs/QmYsEQydGrsxNZfAiskvQ76N2xE9hDQtSAkRSynwMiUK3c/go-block-format" - cid "gx/ipfs/QmeSrf6pzut73u6zLQkRFQ3ygt3k6XFT2kjdYP8Tnkwwyg/go-cid" + mh "gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash" + cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" + blocks "gx/ipfs/Qmej7nf81hi2x2tvjRBF3mcp74sQyuDH4VMYDGd1YtXjb2/go-block-format" ) type BlockAPI struct { @@ -63,12 +63,12 @@ func (api *BlockAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.Bloc return nil, err } - k, err := api.node.Blocks.AddBlock(b) + err = api.node.Blocks.AddBlock(b) if err != nil { return nil, err } - return ParseCid(k), nil + return ParseCid(b.Cid()), nil } func (api *BlockAPI) Get(ctx context.Context, p coreiface.Path) (io.Reader, error) { diff --git a/core/coreapi/block_test.go b/core/coreapi/block_test.go index bef532fdc4a..7fcdbca7ea8 100644 --- a/core/coreapi/block_test.go +++ b/core/coreapi/block_test.go @@ -6,7 +6,7 @@ import ( "strings" "testing" - mh "gx/ipfs/QmYeKnKpubCMRiq3PGZcTREErthbb5Q9cXsCoSkD9bjEBd/go-multihash" + mh "gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash" ) func TestBlockPut(t *testing.T) { diff --git a/core/coreapi/interface/interface.go b/core/coreapi/interface/interface.go index 2402ecf8138..95351e7d045 100644 --- a/core/coreapi/interface/interface.go +++ b/core/coreapi/interface/interface.go @@ -62,12 +62,16 @@ type BlockStat interface { type CoreAPI interface { // Unixfs returns an implementation of Unixfs API. Unixfs() UnixfsAPI + // Block returns an implementation of Block API. Block() BlockAPI + // Dag returns an implementation of Dag API. Dag() DagAPI + // Name returns an implementation of Name API. Name() NameAPI + // Key returns an implementation of Key API. Key() KeyAPI diff --git a/core/coreapi/interface/options/block.go b/core/coreapi/interface/options/block.go index 7e6ad3230bb..bbb14612f44 100644 --- a/core/coreapi/interface/options/block.go +++ b/core/coreapi/interface/options/block.go @@ -1,8 +1,7 @@ package options import ( - //cid "gx/ipfs/QmeSrf6pzut73u6zLQkRFQ3ygt3k6XFT2kjdYP8Tnkwwyg/go-cid" - "gx/ipfs/QmYeKnKpubCMRiq3PGZcTREErthbb5Q9cXsCoSkD9bjEBd/go-multihash" + "gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash" ) type BlockPutSettings struct {