From bef3f26f019443e517a4ee82a29aa3e8f5e4ddc9 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 30 Jul 2020 14:50:38 +0100 Subject: [PATCH] fix: replace node Buffers with Uint8Arrays Relaxes checks on `.Data` to allow for `Uint8Array`s in place of node `Buffer`s. BREAKING CHANGES: - `dagNode.Data` can now be a `Uint8Array` --- README.md | 9 ++-- package.json | 14 ++++--- src/dag-link/dagLink.js | 8 ++-- src/dag-node/dagNode.js | 14 ++++--- src/dag-node/rmLink.js | 6 +-- src/dag-node/sortLinks.js | 7 +++- src/resolver.js | 4 +- src/serialize.js | 4 +- src/util.js | 9 ++-- test/dag-link-test.js | 7 ++-- test/dag-node-test.js | 88 +++++++++++++++++++-------------------- test/resolver.spec.js | 21 ++++------ test/util.spec.js | 5 +-- 13 files changed, 99 insertions(+), 97 deletions(-) diff --git a/README.md b/README.md index 2d571ee..4c1ec77 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ dagPB.util #### Create a DAGNode ```JavaScript -const node1 = new DAGNode(Buffer.from('some data')) +const node1 = new DAGNode(new TextEncoder('utf8').encode('some data')) // node2 will have the same data as node1 const node2 = new DAGNode('some data') @@ -114,7 +114,7 @@ const DAGNode = dagPB.DAGNode #### DAGNode constructor -- `data` - type: Buffer +- `data` - type: Uint8Array or String - `links`- (optional) type: Array of DAGLink instances or Array of DAGLink instances in its json format (link.toJSON) - `serializedSize`- (optional) type: Number of bytes the serialized node has. If none is given, it will automatically be calculated. @@ -198,7 +198,7 @@ node.rmLink('Link1') #### `node.serialize()` -Serialize the DAGNode instance to its portable binary format. Yields the same result as `dagPB.util.serialize(node)`. Returns a `Buffer`. +Serialize the DAGNode instance to its portable binary format. Yields the same result as `dagPB.util.serialize(node)`. Returns a `Uint8Array`. ### DAGLink functions @@ -238,7 +238,6 @@ const link = new DAGLink( > See: https://github.com/ipld/interface-ipld-format#local-resolver-methods - #### `dagPB.resolver.resolve` #### `dagPB.resolver.tree` @@ -251,7 +250,7 @@ const link = new DAGLink( ### `dagPB.util.serialize` -Serialize the DAGNode instance to its portable binary format. Yields the same result as `node.serialize()`. Returns a `Buffer`. +Serialize the DAGNode instance to its portable binary format. Yields the same result as `node.serialize()`. Returns a `Uint8Array`. ### `dagPB.util.deserialize` diff --git a/package.json b/package.json index 1fc55f3..d8e2760 100644 --- a/package.json +++ b/package.json @@ -65,22 +65,24 @@ "npm": ">=3.0.0" }, "dependencies": { - "buffer": "^5.6.0", - "cids": "~0.8.3", + "cids": "multiformats/js-cid#fix/support-uint8arrays", "class-is": "^1.1.0", + "ipfs-utils": "ipfs/js-ipfs-utils#feat/add-utility-uint8array-functions", "multicodec": "^1.0.3", "multihashing-async": "^1.0.0", + "npm": "^6.14.7", "protons": "^1.2.1", + "reset": "^0.1.0", + "run": "^1.4.0", "stable": "^0.1.8" }, "devDependencies": { "aegir": "^23.0.0", "fs-extra": "^9.0.1", - "ipfs-block-service": "~0.17.1", + "ipfs-block-service": "^0.17.1", "ipfs-repo": "^4.0.0", - "ipfs-utils": "^2.3.1", - "ipld-block": "~0.9.2", - "multibase": "^1.0.1", + "ipld-block": "^0.9.2", + "multibase": "^2.0.0", "multihashes": "^1.0.1" } } diff --git a/src/dag-link/dagLink.js b/src/dag-link/dagLink.js index b661845..facd3ab 100644 --- a/src/dag-link/dagLink.js +++ b/src/dag-link/dagLink.js @@ -2,7 +2,7 @@ const CID = require('cids') const withIs = require('class-is') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('ipfs-utils/src/uint8arrays/from-string') // Link represents an IPFS Merkle DAG Link between Nodes. class DAGLink { @@ -39,15 +39,15 @@ class DAGLink { return Object.assign({}, this._json) } - // Memoize the Buffer representation of name + // Memoize the Uint8Array representation of name // We need this to sort the links, otherwise - // we will reallocate new buffers every time + // we will reallocate new Uint8Arrays every time get nameAsBuffer () { if (this._nameBuf !== null) { return this._nameBuf } - this._nameBuf = Buffer.from(this.Name) + this._nameBuf = uint8ArrayFromString(this.Name) return this._nameBuf } } diff --git a/src/dag-node/dagNode.js b/src/dag-node/dagNode.js index 878fb2b..fac5275 100644 --- a/src/dag-node/dagNode.js +++ b/src/dag-node/dagNode.js @@ -1,24 +1,26 @@ 'use strict' const withIs = require('class-is') -const { Buffer } = require('buffer') const sortLinks = require('./sortLinks') const DAGLink = require('../dag-link/dagLink') const { serializeDAGNode } = require('../serialize.js') const toDAGLink = require('./toDagLink') const addLink = require('./addLink') const rmLink = require('./rmLink') +const uint8ArrayFromString = require('ipfs-utils/src/uint8arrays/from-string') +const uint8ArrayToString = require('ipfs-utils/src/uint8arrays/to-string') class DAGNode { constructor (data, links = [], serializedSize = null) { if (!data) { - data = Buffer.alloc(0) + data = new Uint8Array(0) } if (typeof data === 'string') { - data = Buffer.from(data) + data = uint8ArrayFromString(data) } - if (!Buffer.isBuffer(data)) { - throw new Error('Passed \'data\' is not a buffer or a string!') + + if (!(data instanceof Uint8Array)) { + throw new Error('Passed \'data\' is not a Uint8Array or a String!') } if (serializedSize !== null && typeof serializedSize !== 'number') { @@ -53,7 +55,7 @@ class DAGNode { } toString () { - return `DAGNode ` + return `DAGNode ` } _invalidateCached () { diff --git a/src/dag-node/rmLink.js b/src/dag-node/rmLink.js index 7822a2f..6f3534f 100644 --- a/src/dag-node/rmLink.js +++ b/src/dag-node/rmLink.js @@ -1,7 +1,7 @@ 'use strict' const CID = require('cids') -const { Buffer } = require('buffer') +const uint8ArrayEquals = require('ipfs-utils/src/uint8arrays/equals') const rmLink = (dagNode, nameOrCid) => { let predicate = null @@ -9,8 +9,8 @@ const rmLink = (dagNode, nameOrCid) => { // It's a name if (typeof nameOrCid === 'string') { predicate = (link) => link.Name === nameOrCid - } else if (Buffer.isBuffer(nameOrCid) || CID.isCID(nameOrCid)) { - predicate = (link) => link.Hash.equals(nameOrCid) + } else if (nameOrCid instanceof Uint8Array || CID.isCID(nameOrCid)) { + predicate = (link) => uint8ArrayEquals(link.Hash, nameOrCid) } if (predicate) { diff --git a/src/dag-node/sortLinks.js b/src/dag-node/sortLinks.js index 6f39161..e0ba440 100644 --- a/src/dag-node/sortLinks.js +++ b/src/dag-node/sortLinks.js @@ -1,10 +1,13 @@ 'use strict' -const { Buffer } = require('buffer') const sort = require('stable') +const uint8ArraySort = require('ipfs-utils/src/uint8arrays/sort') const linkSort = (a, b) => { - return Buffer.compare(a.nameAsBuffer, b.nameAsBuffer) + const buf1 = a.nameAsBuffer + const buf2 = b.nameAsBuffer + + return uint8ArraySort(buf1, buf2) } /** diff --git a/src/resolver.js b/src/resolver.js index d5ebed3..a65c568 100644 --- a/src/resolver.js +++ b/src/resolver.js @@ -10,7 +10,7 @@ const util = require('./util') * Returns the value or a link and the partial mising path. This way the * IPLD Resolver can fetch the link and continue to resolve. * - * @param {Buffer} binaryBlob - Binary representation of a PB block + * @param {Uint8Array} binaryBlob - Binary representation of a PB block * @param {string} [path='/'] - Path that should be resolved * @returns {Object} result - Result of the path it it was resolved successfully * @returns {*} result.value - Value the path resolves to @@ -58,7 +58,7 @@ exports.resolve = (binaryBlob, path) => { * Return all available paths of a block. * * @generator - * @param {Buffer} binaryBlob - Binary representation of a PB block + * @param {Uint8Array} binaryBlob - Binary representation of a PB block * @yields {string} - A single path */ exports.tree = function * (binaryBlob) { diff --git a/src/serialize.js b/src/serialize.js index fb4aec1..0d3bfc5 100644 --- a/src/serialize.js +++ b/src/serialize.js @@ -13,7 +13,7 @@ const toProtoBuf = (node) => { pbn.Data = node.Data } else { // NOTE: this has to be null in order to match go-ipfs serialization - // `null !== new Buffer(0)` + // `null !== new Uint8Array(0)` pbn.Data = null } @@ -35,7 +35,7 @@ const toProtoBuf = (node) => { * Serialize internal representation into a binary PB block. * * @param {Object} node - Internal representation of a PB block - * @returns {Buffer} - The encoded binary representation + * @returns {Uint8Array} - The encoded binary representation */ const serializeDAGNode = (node) => { const data = node.Data diff --git a/src/util.js b/src/util.js index 76e4267..53fcc9d 100644 --- a/src/util.js +++ b/src/util.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const protons = require('protons') const proto = protons(require('./dag.proto')) const DAGLink = require('./dag-link/dagLink') @@ -30,7 +29,7 @@ const cid = (binaryBlob, userOptions) => { * Serialize internal representation into a binary PB block. * * @param {Object} node - Internal representation of a CBOR block - * @returns {Buffer} - The encoded binary representation + * @returns {Uint8Array} - The encoded binary representation */ const serialize = (node) => { if (DAGNode.isDAGNode(node)) { @@ -43,7 +42,7 @@ const serialize = (node) => { /** * Deserialize PB block into the internal representation. * - * @param {Buffer} buffer - Binary representation of a PB block + * @param {Uint8Array} buffer - Binary representation of a PB block * @returns {Object} - An object that conforms to the IPLD Data Model */ const deserialize = (buffer) => { @@ -53,9 +52,9 @@ const deserialize = (buffer) => { return new DAGLink(link.Name, link.Tsize, link.Hash) }) - const data = pbn.Data == null ? Buffer.alloc(0) : pbn.Data + const data = pbn.Data == null ? new Uint8Array(0) : pbn.Data - return new DAGNode(data, links, buffer.length) + return new DAGNode(data, links, buffer.byteLength) } exports.serialize = serialize diff --git a/test/dag-link-test.js b/test/dag-link-test.js index 232fec3..76f1552 100644 --- a/test/dag-link-test.js +++ b/test/dag-link-test.js @@ -2,10 +2,11 @@ 'use strict' const chai = require('aegir/utils/chai') -const { Buffer } = require('buffer') const expect = chai.expect const CID = require('cids') const DAGLink = require('../src').DAGLink +const uint8ArrayFromString = require('ipfs-utils/src/uint8arrays/from-string') +const uint8ArrayToString = require('ipfs-utils/src/uint8arrays/to-string') module.exports = (repo) => { describe('DAGLink', () => { @@ -13,7 +14,7 @@ module.exports = (repo) => { it('string', () => { const link = new DAGLink('hello', 3, 'QmXg9Pp2ytZ14xgmQjYEiHjVjMFXzCVVEcRTWJBmLgR39U') - expect(link.Hash.buffer.toString('hex')) + expect(uint8ArrayToString(link.Hash.buffer, 'base16')) .to.equal('12208ab7a6c5e74737878ac73863cb76739d15d4666de44e5756bf55a2f9e9ab5f43') }) @@ -23,7 +24,7 @@ module.exports = (repo) => { }) it('create with multihash as a multihash Buffer', () => { - const link = new DAGLink('hello', 3, Buffer.from('12208ab7a6c5e74737878ac73863cb76739d15d4666de44e5756bf55a2f9e9ab5f43', 'hex')) + const link = new DAGLink('hello', 3, uint8ArrayFromString('12208ab7a6c5e74737878ac73863cb76739d15d4666de44e5756bf55a2f9e9ab5f43', 'base16')) expect(new CID(link.Hash).toBaseEncodedString()) .to.equal('QmXg9Pp2ytZ14xgmQjYEiHjVjMFXzCVVEcRTWJBmLgR39U') diff --git a/test/dag-node-test.js b/test/dag-node-test.js index 686654d..464af92 100644 --- a/test/dag-node-test.js +++ b/test/dag-node-test.js @@ -2,7 +2,6 @@ 'use strict' const chai = require('aegir/utils/chai') -const { Buffer } = require('buffer') const expect = chai.expect const dagPB = require('../src') @@ -18,6 +17,7 @@ const Block = require('ipld-block') const CID = require('cids') const multibase = require('multibase') const loadFixture = require('aegir/fixtures') +const uint8ArrayFromString = require('ipfs-utils/src/uint8arrays/from-string') const testBlockNamedLinks = loadFixture('test/fixtures/test-block-named-links') const testBlockUnnamedLinks = loadFixture('test/fixtures/test-block-unnamed-links') @@ -27,11 +27,11 @@ module.exports = (repo) => { describe('DAGNode', () => { it('create a node', () => { - const data = Buffer.from('some data') + const data = uint8ArrayFromString('some data') const node = new DAGNode(data) expect(node.Data.length).to.be.above(0) - expect(Buffer.isBuffer(node.Data)).to.be.true() + expect(node.Data).to.be.an.instanceOf(Uint8Array) expect(node.size).to.be.above(0) const serialized = dagPB.util.serialize(node) @@ -40,7 +40,7 @@ module.exports = (repo) => { }) it('dagPB.util.serialize same as node.serialize()', () => { - const node = new DAGNode(Buffer.from('some data')) + const node = new DAGNode(uint8ArrayFromString('some data')) const serialized = dagPB.util.serialize(node) expect(serialized).to.eql(node.serialize()) }) @@ -50,7 +50,7 @@ module.exports = (repo) => { const node = new DAGNode(data) expect(node.Data.length).to.be.above(0) - expect(Buffer.isBuffer(node.Data)).to.be.true() + expect(node.Data).to.be.an.instanceOf(Uint8Array) expect(node.size).to.be.above(0) const serialized = dagPB.util.serialize(node) @@ -70,7 +70,7 @@ module.exports = (repo) => { Tsize: 10 }] - const someData = Buffer.from('some data') + const someData = uint8ArrayFromString('some data') const node1 = new DAGNode(someData, l1) const l2 = l1.map((l) => { @@ -135,7 +135,7 @@ module.exports = (repo) => { Tsize: 262158 }] - const node = new DAGNode(Buffer.from('some data'), links) + const node = new DAGNode(uint8ArrayFromString('some data'), links) const serialized = node.serialize() const deserialized = dagPB.util.deserialize(serialized) @@ -144,14 +144,14 @@ module.exports = (repo) => { }) it('create with empty link name', () => { - const node = new DAGNode(Buffer.from('hello'), [ + const node = new DAGNode(uint8ArrayFromString('hello'), [ new DAGLink('', 10, 'QmXg9Pp2ytZ14xgmQjYEiHjVjMFXzCVVEcRTWJBmLgR39U') ]) expect(node.Links[0].Name).to.be.eql('') }) it('create with undefined link name', () => { - const node = new DAGNode(Buffer.from('hello'), [ + const node = new DAGNode(uint8ArrayFromString('hello'), [ new DAGLink(undefined, 10, 'QmXg9Pp2ytZ14xgmQjYEiHjVjMFXzCVVEcRTWJBmLgR39U') ]) expect(node.Links[0].Name).to.be.eql('') @@ -166,9 +166,9 @@ module.exports = (repo) => { it('create an empty node', () => { // this node is not in the repo as we don't copy node data to the browser - const node = new DAGNode(Buffer.alloc(0)) + const node = new DAGNode(new Uint8Array(0)) expect(node.Data.length).to.be.equal(0) - expect(Buffer.isBuffer(node.Data)).to.be.true() + expect(node.Data).to.be.an.instanceOf(Uint8Array) expect(node.size).to.be.equal(0) const serialized = dagPB.util.serialize(node) @@ -178,16 +178,16 @@ module.exports = (repo) => { it('fail to create a node with other data types', () => { expect(() => new DAGNode({})).to.throw( - 'Passed \'data\' is not a buffer or a string!' + 'Passed \'data\' is not a Uint8Array or a String!' ) expect(() => new DAGNode([])).to.throw( - 'Passed \'data\' is not a buffer or a string!' + 'Passed \'data\' is not a Uint8Array or a String!' ) }) it('addLink by DAGNode', async () => { - const node1 = new DAGNode(Buffer.from('1')) - const node2 = new DAGNode(Buffer.from('2')) + const node1 = new DAGNode(uint8ArrayFromString('1')) + const node2 = new DAGNode(uint8ArrayFromString('2')) node1.addLink(await node2.toDAGLink()) expect(node1.Links.length).to.equal(1) expect(node1.Links[0].Tsize).to.eql(node2.size) @@ -195,8 +195,8 @@ module.exports = (repo) => { }) it('addLink by DAGLink', async () => { - const node1 = new DAGNode(Buffer.from('1')) - const node2 = new DAGNode(Buffer.from('2')) + const node1 = new DAGNode(uint8ArrayFromString('1')) + const node2 = new DAGNode(uint8ArrayFromString('2')) const link = await node2.toDAGLink() node1.addLink(link) expect(node1.Links.length).to.equal(1) @@ -205,8 +205,8 @@ module.exports = (repo) => { }) it('addLink by object', async () => { - const node1 = new DAGNode(Buffer.from('1')) - const node2 = new DAGNode(Buffer.from('2')) + const node1 = new DAGNode(uint8ArrayFromString('1')) + const node2 = new DAGNode(uint8ArrayFromString('2')) const link = await node2.toDAGLink() const linkObject = link.toJSON() node1.addLink(linkObject) @@ -216,8 +216,8 @@ module.exports = (repo) => { }) it('addLink by name', async () => { - const node1 = new DAGNode(Buffer.from('1')) - const node2 = new DAGNode(Buffer.from('2')) + const node1 = new DAGNode(uint8ArrayFromString('1')) + const node2 = new DAGNode(uint8ArrayFromString('2')) const link = await node2.toDAGLink({ name: 'banana' }) expect(node1.Links.length).to.equal(0) node1.addLink(link) @@ -227,22 +227,22 @@ module.exports = (repo) => { }) it('addLink - add several links', async () => { - const node1 = new DAGNode(Buffer.from('1')) + const node1 = new DAGNode(uint8ArrayFromString('1')) expect(node1.Links.length).to.equal(0) - const node2 = new DAGNode(Buffer.from('2')) + const node2 = new DAGNode(uint8ArrayFromString('2')) node1.addLink(await node2.toDAGLink()) expect(node1.Links.length).to.equal(1) - const node3 = new DAGNode(Buffer.from('3')) + const node3 = new DAGNode(uint8ArrayFromString('3')) node1.addLink(await node3.toDAGLink()) expect(node1.Links.length).to.equal(2) }) it('addLink by DAGNode.Links', async () => { const linkName = 'link-name' - const remote = new DAGNode(Buffer.from('2')) - const source = new DAGNode(Buffer.from('1')) + const remote = new DAGNode(uint8ArrayFromString('2')) + const source = new DAGNode(uint8ArrayFromString('1')) source.addLink(await remote.toDAGLink({ name: linkName })) expect(source.Links.length).to.equal(1) @@ -256,11 +256,11 @@ module.exports = (repo) => { }) it('rmLink by name', async () => { - const node1 = new DAGNode(Buffer.from('1')) + const node1 = new DAGNode(uint8ArrayFromString('1')) expect(node1.Links.length).to.eql(0) const withoutLink = node1.toJSON() - const node2 = new DAGNode(Buffer.from('2')) + const node2 = new DAGNode(uint8ArrayFromString('2')) const link = await node2.toDAGLink({ name: 'banana' }) node1.addLink(link) @@ -271,11 +271,11 @@ module.exports = (repo) => { }) it('rmLink by hash', async () => { - const node1 = new DAGNode(Buffer.from('1')) + const node1 = new DAGNode(uint8ArrayFromString('1')) expect(node1.Links.length).to.eql(0) const withoutLink = node1.toJSON() - const node2 = new DAGNode(Buffer.from('2')) + const node2 = new DAGNode(uint8ArrayFromString('2')) const link = await node2.toDAGLink({ name: 'banana' }) node1.addLink(link) @@ -286,7 +286,7 @@ module.exports = (repo) => { }) it('get node CID', async () => { - const node = new DAGNode(Buffer.from('some data')) + const node = new DAGNode(uint8ArrayFromString('some data')) const serialized = dagPB.util.serialize(node) const cid = await dagPB.util.cid(serialized) expect(cid.multihash).to.exist() @@ -297,7 +297,7 @@ module.exports = (repo) => { }) it('get node CID with version', async () => { - const node = new DAGNode(Buffer.from('some data')) + const node = new DAGNode(uint8ArrayFromString('some data')) const serialized = dagPB.util.serialize(node) const cid = await dagPB.util.cid(serialized, { cidVersion: 0 }) expect(cid.multihash).to.exist() @@ -308,7 +308,7 @@ module.exports = (repo) => { }) it('get node CID with hashAlg', async () => { - const node = new DAGNode(Buffer.from('some data')) + const node = new DAGNode(uint8ArrayFromString('some data')) const serialized = dagPB.util.serialize(node) const cid = await dagPB.util.cid(serialized, { hashAlg: multicodec.SHA2_512 }) expect(cid.multihash).to.exist() @@ -328,7 +328,7 @@ module.exports = (repo) => { } async function rawBlockCid (str) { - const raw = Buffer.from(str) + const raw = uint8ArrayFromString(str) const rawHash = await multihashing(raw, 'sha2-256') return new CID(1, 'raw', rawHash) } @@ -388,10 +388,10 @@ module.exports = (repo) => { }) it('marshal a node and store it with block-service', async () => { - const node = new DAGNode(Buffer.from('some data')) + const node = new DAGNode(uint8ArrayFromString('some data')) const serialized = dagPB.util.serialize(node) const cid = await dagPB.util.cid(serialized) - const block = new Block(Buffer.from(serialized), cid) + const block = new Block(serialized, cid) await bs.put(block) const retrievedBlock = await bs.get(block.cid) @@ -498,14 +498,14 @@ module.exports = (repo) => { }) it('dagNode.toJSON with empty Node', () => { - const node = new DAGNode(Buffer.alloc(0)) - expect(node.toJSON().data).to.eql(Buffer.alloc(0)) + const node = new DAGNode(new Uint8Array(0)) + expect(node.toJSON().data).to.eql(new Uint8Array(0)) expect(node.toJSON().links).to.eql([]) expect(node.toJSON().size).to.exist() }) it('dagNode.toJSON with data no links', () => { - const data = Buffer.from('La cucaracha') + const data = uint8ArrayFromString('La cucaracha') const node = new DAGNode(data) expect(node.toJSON().data).to.eql(data) expect(node.toJSON().links).to.eql([]) @@ -535,19 +535,19 @@ module.exports = (repo) => { multibase.decode('z' + l2.Hash) ) - const node = new DAGNode(Buffer.from('hiya'), [link1, link2]) + const node = new DAGNode(uint8ArrayFromString('hiya'), [link1, link2]) expect(node.Links).to.have.lengthOf(2) }) it('toString', () => { - const node = new DAGNode(Buffer.from('hello world')) + const node = new DAGNode(uint8ArrayFromString('hello world')) const expected = 'DAGNode