From 107b04013860161d257e45115dbb4cb5de1a8ba7 Mon Sep 17 00:00:00 2001 From: Kelvin Fichter Date: Thu, 13 Jun 2019 21:13:34 -0400 Subject: [PATCH 1/6] Started rewriting Merkle Interval Tree for production --- packages/utils/src/index.ts | 2 +- packages/utils/src/merkle-interval-tree.ts | 290 ++++++++++++++++++ packages/utils/src/sum-tree.ts | 261 ---------------- packages/utils/src/utils.ts | 13 + ...e.spec.ts => merkle-interval-tree.spec.ts} | 20 +- 5 files changed, 314 insertions(+), 272 deletions(-) create mode 100644 packages/utils/src/merkle-interval-tree.ts delete mode 100644 packages/utils/src/sum-tree.ts rename packages/utils/test/{sum-tree.spec.ts => merkle-interval-tree.spec.ts} (89%) diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index 3ee072fd..6c42cc01 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -1,6 +1,6 @@ export * from './utils' export * from './constants' export * from './eth' -export * from './sum-tree' +export * from './merkle-interval-tree' export * from './interfaces' export * from './data-types' diff --git a/packages/utils/src/merkle-interval-tree.ts b/packages/utils/src/merkle-interval-tree.ts new file mode 100644 index 00000000..b62f0ce8 --- /dev/null +++ b/packages/utils/src/merkle-interval-tree.ts @@ -0,0 +1,290 @@ +/* External Imports */ +import BigNumber = require('bn.js') + +/* Internal Imports */ +import { bnMin, bnMax, except } from './utils' + +/** + * Computes the index of the sibling of a node. + * @param index Index of a node. + * @returns the index of the sibling of that node. + */ +const getSiblingIndex = (index: number): number => { + return index + (index % 2 === 0 ? 1 : -1) +} + +/** + * Computes the index of the parent of a node + * @param index Index of a node. + * @returns the index of the parent of that node. + */ +const getParentIndex = (index: number): number => { + return index === 0 ? 0 : Math.floor(index / 2) +} + +/** + * Checks if two ranges overlap. + * @param rangeA First range to check. + * @param rangeB Second range to check. + * @returns `true` if the ranges overlap, `false` otherwise. +const intersects = (rangeA: Range, rangeB: Range): boolean => { + const maxStart = bnMax(rangeA.start, rangeB.start) + const minEnd = bnMin(rangeA.end, rangeB.end) + return maxStart.lt(minEnd) +} + +/** + * Checks if a given index is out of bounds for an array. + * @param list Array to check against. + * @param index Index to check. + * @returns `true` if the index is out of bounds, `false` otherwise. + */ +const outOfBounds = (list: any[], index: number): boolean => { + return index < 0 || index >= list.length +} + +export interface Range { + start: BigNumber + end: BigNumber +} + +export interface MerkleIntervalTreeLeafNode { + start: BigNumber + end: BigNumber + data: Buffer +} + +export interface MerkleIntervalTreeInternalNode { + index: BigNumber + hash: Buffer +} + +export type MerkleIntervalTreeInclusionProof = MerkleIntervalTreeInternalNode[] + +export class MerkleIntervalTree { + private levels: MerkleIntervalTreeInternalNode[][] + + constructor( + private leaves: MerkleIntervalTreeLeafNode[], + private hashfn: (value: Buffer) => Buffer + ) { + this.validateLeaves(this.leaves) + + this.leaves.sort((a, b) => { + return a.start.sub(b.start) + }) + + const bottom = this.leaves.map((leaf) => { + return this.parseLeaf(leaf) + }) + + this.levels = [bottom] + this.generateTree() + } + + /** + * Generates an inclusion proof for a given leaf node. + * @param leafPosition Index of the leaf node in the list of leaves. + * @returns an inclusion proof for the given leaf. + */ + public getInclusionProof( + leafPosition: number + ): MerkleIntervalTreeInclusionProof { + if (outOfBounds(this.leaves, leafPosition)) { + throw new Error('Leaf position is out of bounds.') + } + + const inclusionProof: MerkleIntervalTreeInclusionProof = [] + let childIndex = leafPosition + let siblingIndex = getSiblingIndex(childIndex) + + for (let i = 0; i < this.levels.length - 1; i++) { + const currentLevel = this.levels[i] + const childNode = currentLevel[childIndex] + const siblingNode = outOfBounds(currentLevel, siblingIndex) + ? currentLevel[siblingIndex] + : this.createEmptyNode(childNode.index) + + inclusionProof.push(siblingNode) + + childIndex = getParentIndex(childIndex) + siblingIndex = getSiblingIndex(childIndex) + } + + return inclusionProof + } + + /** + * Checks an inclusion proof. Throws if the proof is invalid at any point. + * @param leafNode Leaf node to check inclusion of. + * @param leafPosition Index of the leaf in the list of leaves. + * @param inclusionProof Inclusion proof for the leaf node. + * @param rootHash Hash of the root of the tree. + * @returns the "implicit range" covered by leaf node if the proof is valid. + */ + public checkInclusionProof( + leafNode: MerkleIntervalTreeLeafNode, + leafPosition: number, + inclusionProof: MerkleIntervalTreeInternalNode, + rootHash: Buffer + ): Range { + if (leafPosition < 0) { + throw new Error('Invalid leaf index.') + } + + const path = reverse( + new BigNum(leafPosition).toString(2, inclusionProof.length) + ) + + const firstRightSiblingIndex = path.indexOf('0') + const firstRightSibling = + firstRightSiblingIndex >= 0 + ? inclusionProof[firstRightSiblingIndex] + : null + + let computedNode = this.parseLeaf(leafNode) + let leftChild: MerkleIntervalTreeInternalNode + let rightChild: MerkleIntervalTreeInternalNode + + for (let i = 0; i < inclusionProof.length; i++) { + const siblingNode = inclusionProof[i] + + if (path[i] === '1') { + leftChild = siblingNode + rightChild = computedNode + } else { + leftChild = computedNode + rightChild = siblingNode + + if ( + firstRightSibling !== null && + rightChild.index.lt(firstRightSibling.index) + ) { + throw new Error( + 'Invalid Merkle Interval Tree proof -- potential intersection detected.' + ) + } + } + + computedNode = this.computeParent(leftChild, rightChild) + } + + if (Buffer.compare(computedNode.hash, rootHash) !== 0) { + throw new Error( + 'Invalid Merkle Interval Tree proof -- invalid root hash.' + ) + } + + const implicitStart = leafPosition === 0 ? new BigNumber(0) : leafNode.index + const implicitEnd = + firstRightSibling !== null ? firstRightSibling.index : null + + return { + start: implicitStart, + end: implicitEnd, + } + } + + /** + * Validates that a set of leaf nodes are valid by checking that there are no + * overlapping leaves. Throws if any two leaves are overlapping. + * @param leaves Set of leaf nodes to check. + */ + private validateLeaves(leaves: MerkleIntervalTreeLeafNode[]): void { + const valid = leaves.every((leaf) => { + const others = except(leaves, leaf) + return others.every((other) => { + return !intersects(leaf, other) + }) + }) + + if (!valid) { + throw new Error('Merkle Interval Tree leaves must not overlap.') + } + } + + /** + * Parses a leaf node into an internal node. + * @param leaf Leaf to parse. + * @returns the parsed internal node. + */ + private parseLeaf( + leaf: MerkleIntervalTreeLeafNode + ): MerkleIntervalTreeInternalNode { + return { + index: leaf.start, + hash: this.hashfn( + Buffer.concat([ + leaf.start.toBuffer('BE', 16), + leaf.end.toBuffer('BE', 16), + leaf.data, + ]) + ), + } + } + + /** + * Creates an empty node for when there are an odd number of elements in a + * specific layer in the tree. + * @param siblingIndex Index of the node's sibling. + * @returns the empty node. + */ + private createEmptyNode( + siblingIndex: BigNumber + ): MerkleIntervalTreeInternalNode { + return { + index: siblingIndex, + hash: this.hashfn(Buffer.from('0')), + } + } + + /** + * Computes the parent of two internal nodes. + * @param leftChild Left child of the parent. + * @param rightChild Right child of the parent. + * @returns the parent of the two children. + */ + private computeParent( + leftChild: MerkleIntervalTreeInternalNode, + rightChild: MerkleIntervalTreeInternalNode + ): MerkleIntervalTreeInternalNode { + const data = Buffer.concat([ + leftChild.index, + leftChild.hash, + rightChild.index, + rightChild.hash, + ]) + const hash = this.hashfn(data) + const index = leftChild.index + + return { + index, + hash, + } + } + + /** + * Generates the tree recursively. + */ + private generateTree(): void { + const children = this.levels[this.levels.length - 1] + + if (children.length <= 1) { + return + } + + const parents: MerkleIntervalTreeInternalNode[] = [] + + for (let i = 0; i < children.length; i += 2) { + const leftChild = children[i] + const rightChild = outOfBounds(children, i + 1) + ? this.createEmptyNode(leftChild.index) + : children[i + 1] + const parent = this.computeParent(leftChild, rightChild) + parents.push(parent) + } + + this.levels.push(parents) + this.generateTree() + } +} diff --git a/packages/utils/src/sum-tree.ts b/packages/utils/src/sum-tree.ts deleted file mode 100644 index 95208732..00000000 --- a/packages/utils/src/sum-tree.ts +++ /dev/null @@ -1,261 +0,0 @@ -/* External Imports */ -import BigNum = require('bn.js') - -/* Internal Imports */ -import { keccak256 } from './eth' -import { reverse } from './utils' -import { NULL_HASH } from './constants' - -export interface ImplicitBounds { - implicitStart: BigNum - implicitEnd: BigNum -} - -export interface MerkleTreeNode { - end: BigNum - data: string -} - -export interface MerkleSumTreeOptions { - leaves?: MerkleTreeNode[] - hash?: (value: string) => string - maxTreeSize?: BigNum -} - -/** - * Computes the index of the parent of a node - * @param index Index of a node. - * @returns the index of the parent of that node. - */ -const getParentIndex = (index: number): number => { - return index === 0 ? 0 : Math.floor(index / 2) -} - -/** - * Computes the index of the sibling of a node. - * @param index Index of a node. - * @returns the index of the sibling of that node. - */ -const getSiblingIndex = (index: number): number => { - return index + (index % 2 === 0 ? 1 : -1) -} - -/** - * Basic MerkleSumTree implementation. - */ -export class MerkleSumTree { - private tree: MerkleTreeNode[][] = [] - private hash: (value: string) => string - private maxTreeSize: BigNum - - constructor({ - hash = keccak256, - leaves = [], - maxTreeSize = new BigNum('ffffffffffffffffffffffffffffffff', 16), - }: MerkleSumTreeOptions = {}) { - this.hash = hash - this.maxTreeSize = new BigNum(maxTreeSize, 'hex') - this.generateTree(this.parseLeaves(leaves)) - } - - /** - * @returns the root of the tree. - */ - get root(): string { - return this.tree[0].length > 0 - ? this.computeHash(this.tree[this.tree.length - 1][0]) - : null - } - - /** - * @returns the leaf nodes in the tree. - */ - get leaves(): MerkleTreeNode[] { - return this.tree[0] - } - - /** - * @returns all levels in the tree. - */ - get levels(): MerkleTreeNode[][] { - return this.tree - } - - /** - * Checks a Merkle proof. - * @param leaf Leaf node to check. - * @param leafIndex Position of the leaf in the tree. - * @param inclusionProof Inclusion proof for that transaction. - * @param root The root node of the tree to check. - * @returns the implicit bounds covered by the leaf if the proof is valid. - */ - public verify( - leaf: MerkleTreeNode, - leafIndex: number, - inclusionProof: MerkleTreeNode[], - root: string - ): ImplicitBounds { - if (leafIndex < 0) { - throw new Error('Invalid leaf index.') - } - - // Leaf data is unhashed, so hash it. - leaf.data = this.hash(leaf.data) - - // Compute the path based on the leaf index. - const path = reverse( - new BigNum(leafIndex).toString(2, inclusionProof.length) - ) - - // Need the first left sibling to ensure - // that the tree is monotonically increasing. - const firstLeftSiblingIndex = path.indexOf('1') - const firstLeftSibling = - firstLeftSiblingIndex >= 0 - ? inclusionProof[firstLeftSiblingIndex] - : undefined - - let computed = leaf - let left: MerkleTreeNode - let right: MerkleTreeNode - for (let i = 0; i < inclusionProof.length; i++) { - const sibling = inclusionProof[i] - - if (path[i] === '0') { - left = computed - right = sibling - } else { - left = sibling - right = computed - - // Some left node further up the tree - // is greater than the first left node - // so tree construction must be invalid. - if (left.end.gt(firstLeftSibling.end)) { - throw new Error('Invalid Merkle Sum Tree proof.') - } - } - - // Values at left nodes must always be - // less than values at right nodes. - if (left.end.gt(right.end)) { - throw new Error('Invalid Merkle Sum Tree proof.') - } - - computed = this.computeParent(left, right) - } - - // Check that the roots match. - if (this.computeHash(computed) !== root) { - throw new Error('Invalid Merkle Sum Tree proof.') - } - - const isLastLeaf = new BigNum(2) - .pow(new BigNum(inclusionProof.length)) - .subn(1) - .eqn(leafIndex) - - return { - implicitStart: firstLeftSibling ? firstLeftSibling.end : new BigNum(0), - implicitEnd: isLastLeaf ? this.maxTreeSize : leaf.end, - } - } - - /** - * Returns an inclusion proof for the leaf at a given index. - * @param leafIndex Index of the leaf to generate a proof for. - * @returns an inclusion proof for that leaf. - */ - public getInclusionProof(leafIndex: number): MerkleTreeNode[] { - if (leafIndex >= this.leaves.length || leafIndex < 0) { - throw new Error('Invalid leaf index.') - } - - const inclusionProof: MerkleTreeNode[] = [] - let parentIndex: number - let siblingIndex = getSiblingIndex(leafIndex) - for (let i = 0; i < this.tree.length - 1; i++) { - const node = this.tree[i][siblingIndex] || this.createEmptyNode() - inclusionProof.push(node) - - // Figure out the parent and then figure out the parent's sibling. - parentIndex = getParentIndex(siblingIndex) - siblingIndex = getSiblingIndex(parentIndex) - } - - return inclusionProof - } - - /** - * @returns an empty Merkle tree node. - */ - private createEmptyNode(): MerkleTreeNode { - return { - ...{ - end: this.maxTreeSize, - data: NULL_HASH, - }, - } - } - - /** - * Parses leaf nodes by hashing their data. - * @param leaves Leaf nodes to parse. - * @returns the parsed leaves. - */ - private parseLeaves(leaves: MerkleTreeNode[]): MerkleTreeNode[] { - return leaves.map((leaf) => { - return { - end: leaf.end, - data: this.hash(leaf.data), - } - }) - } - - /** - * Computes the unique hash for a node. - * @param node Node to hash. - * @returns the hash of the node. - */ - private computeHash(node: MerkleTreeNode): string { - const data = node.end.toString('hex') + node.data - return this.hash(data) - } - - /** - * Computes the parent of two nodes. - * @param left Left child node. - * @param right Right child node. - * @returns the parent of the two nodes. - */ - private computeParent( - left: MerkleTreeNode, - right: MerkleTreeNode - ): MerkleTreeNode { - return { - end: right.end, - data: this.hash(this.computeHash(left) + this.computeHash(right)), - } - } - - /** - * Recursively generates the Merkle tree. - * @param children Nodes in the last generated level. - */ - private generateTree(children: MerkleTreeNode[]): void { - this.tree.push(children) - if (children.length <= 1) { - return - } - - const parents: MerkleTreeNode[] = [] - for (let i = 0; i < children.length; i += 2) { - const left = children[i] - const right = - i + 1 === children.length ? this.createEmptyNode() : children[i + 1] - parents.push(this.computeParent(left, right)) - } - - this.generateTree(parents) - } -} diff --git a/packages/utils/src/utils.ts b/packages/utils/src/utils.ts index 574fe931..0faec69d 100644 --- a/packages/utils/src/utils.ts +++ b/packages/utils/src/utils.ts @@ -118,3 +118,16 @@ export const hexStringify = (value: BigNum | Buffer): string => { export const hexStrToBuf = (hexString: string): Buffer => { return Buffer.from(hexString.slice(2), 'hex') } + +/** + * Creates a new version of a list with all instances of a specific element + * removed. + * @param list List to remove elements from. + * @param element Element to remove from the list. + * @returns the list without the given element. + */ +export const except = (list: T[], element: T): T[] => { + return list.filter((item) => { + return item !== element + }) +} diff --git a/packages/utils/test/sum-tree.spec.ts b/packages/utils/test/merkle-interval-tree.spec.ts similarity index 89% rename from packages/utils/test/sum-tree.spec.ts rename to packages/utils/test/merkle-interval-tree.spec.ts index 416a1cb8..9b4ad066 100644 --- a/packages/utils/test/sum-tree.spec.ts +++ b/packages/utils/test/merkle-interval-tree.spec.ts @@ -4,12 +4,12 @@ import { should } from './setup' import BigNum = require('bn.js') /* Internal Imports */ -import { MerkleSumTree, MerkleTreeNode } from '../src/sum-tree' +import { MerkleIntervalTree, MerkleTreeNode } from '../src/merkle-interval-tree' -describe('MerkleSumTree', () => { +describe('MerkleIntervalTree', () => { describe('construction', () => { it('should be created correctly with no leaves', () => { - const tree = new MerkleSumTree() + const tree = new MerkleIntervalTree() should.not.exist(tree.root) tree.leaves.should.deep.equal([]) @@ -23,7 +23,7 @@ describe('MerkleSumTree', () => { data: '0x123', }, ] - const tree = new MerkleSumTree({ + const tree = new MerkleIntervalTree({ leaves, }) @@ -43,7 +43,7 @@ describe('MerkleSumTree', () => { data: '0x456', }, ] - const tree = new MerkleSumTree({ + const tree = new MerkleIntervalTree({ leaves, }) @@ -55,7 +55,7 @@ describe('MerkleSumTree', () => { describe('verify', () => { it('should correctly verify a valid proof', () => { - const tree = new MerkleSumTree() + const tree = new MerkleIntervalTree() const leaf: MerkleTreeNode = { end: new BigNum(100), data: '0x123', @@ -76,7 +76,7 @@ describe('MerkleSumTree', () => { }) it('should correctly reject a proof with the wrong root', () => { - const tree = new MerkleSumTree() + const tree = new MerkleIntervalTree() const leaf: MerkleTreeNode = { end: new BigNum(100), data: '0x123', @@ -96,7 +96,7 @@ describe('MerkleSumTree', () => { }) it('should correctly reject a proof with the wrong siblings', () => { - const tree = new MerkleSumTree() + const tree = new MerkleIntervalTree() const leaf: MerkleTreeNode = { end: new BigNum(100), data: '0x123', @@ -116,7 +116,7 @@ describe('MerkleSumTree', () => { }) it('should correctly reject a proof with an invalid sibling', () => { - const tree = new MerkleSumTree() + const tree = new MerkleIntervalTree() const leaf: MerkleTreeNode = { end: new BigNum(100), data: '0x123', @@ -149,7 +149,7 @@ describe('MerkleSumTree', () => { data: '0x456', }, ] - const tree = new MerkleSumTree({ + const tree = new MerkleIntervalTree({ leaves, }) const expected: MerkleTreeNode[] = [ From 1f81a8faad828b50100093c35b6cc510d0cfeaa7 Mon Sep 17 00:00:00 2001 From: Kelvin Fichter Date: Sun, 16 Jun 2019 13:56:14 -0400 Subject: [PATCH 2/6] Started porting new Merkle Interval Tree --- packages/utils/src/eth/utils.ts | 8 +- packages/utils/src/merkle-interval-tree.ts | 46 +++-- packages/utils/test/eth/utils.spec.ts | 13 +- .../utils/test/merkle-interval-tree.spec.ts | 166 ++++++++++-------- 4 files changed, 135 insertions(+), 98 deletions(-) diff --git a/packages/utils/src/eth/utils.ts b/packages/utils/src/eth/utils.ts index 96f27a51..5797cb4b 100644 --- a/packages/utils/src/eth/utils.ts +++ b/packages/utils/src/eth/utils.ts @@ -2,7 +2,7 @@ import { ethers } from 'ethers' /* Internal Imports */ -import { add0x } from '../utils' +import { add0x, remove0x } from '../utils' export const abi = new ethers.utils.AbiCoder() @@ -11,7 +11,7 @@ export const abi = new ethers.utils.AbiCoder() * @param value Value to hash * @returns the hash of the value. */ -export const keccak256 = (value: string): string => { - const preimage = add0x(value.replace(/0x/g, '')) - return ethers.utils.keccak256(preimage) +export const keccak256 = (value: Buffer): Buffer => { + const preimage = add0x(value.toString('hex')) + return Buffer.from(remove0x(ethers.utils.keccak256(preimage)), 'hex') } diff --git a/packages/utils/src/merkle-interval-tree.ts b/packages/utils/src/merkle-interval-tree.ts index b62f0ce8..236d3160 100644 --- a/packages/utils/src/merkle-interval-tree.ts +++ b/packages/utils/src/merkle-interval-tree.ts @@ -2,7 +2,8 @@ import BigNumber = require('bn.js') /* Internal Imports */ -import { bnMin, bnMax, except } from './utils' +import { keccak256 } from './eth/utils' +import { bnMin, bnMax, except, reverse } from './utils' /** * Computes the index of the sibling of a node. @@ -27,6 +28,7 @@ const getParentIndex = (index: number): number => { * @param rangeA First range to check. * @param rangeB Second range to check. * @returns `true` if the ranges overlap, `false` otherwise. + */ const intersects = (rangeA: Range, rangeB: Range): boolean => { const maxStart = bnMax(rangeA.start, rangeB.start) const minEnd = bnMin(rangeA.end, rangeB.end) @@ -65,13 +67,19 @@ export class MerkleIntervalTree { private levels: MerkleIntervalTreeInternalNode[][] constructor( - private leaves: MerkleIntervalTreeLeafNode[], - private hashfn: (value: Buffer) => Buffer + private leaves: MerkleIntervalTreeLeafNode[] = [], + private hashfn: (value: Buffer) => Buffer = keccak256 ) { this.validateLeaves(this.leaves) this.leaves.sort((a, b) => { - return a.start.sub(b.start) + if (a.start.lt(b.start)) { + return -1 + } else if (a.start.gt(b.start)) { + return 1 + } else { + return 0 + } }) const bottom = this.leaves.map((leaf) => { @@ -82,6 +90,22 @@ export class MerkleIntervalTree { this.generateTree() } + /** + * @returns the root of the tree. + */ + public getRoot(): MerkleIntervalTreeInternalNode { + return this.levels[0].length > 0 + ? this.levels[this.levels.length - 1][0] + : null + } + + /** + * @returns the levels of the tree. + */ + public getLevels(): MerkleIntervalTreeInternalNode[][] { + return this.levels + } + /** * Generates an inclusion proof for a given leaf node. * @param leafPosition Index of the leaf node in the list of leaves. @@ -125,7 +149,7 @@ export class MerkleIntervalTree { public checkInclusionProof( leafNode: MerkleIntervalTreeLeafNode, leafPosition: number, - inclusionProof: MerkleIntervalTreeInternalNode, + inclusionProof: MerkleIntervalTreeInclusionProof, rootHash: Buffer ): Range { if (leafPosition < 0) { @@ -133,7 +157,7 @@ export class MerkleIntervalTree { } const path = reverse( - new BigNum(leafPosition).toString(2, inclusionProof.length) + new BigNumber(leafPosition).toString(2, inclusionProof.length) ) const firstRightSiblingIndex = path.indexOf('0') @@ -175,7 +199,7 @@ export class MerkleIntervalTree { ) } - const implicitStart = leafPosition === 0 ? new BigNumber(0) : leafNode.index + const implicitStart = leafPosition === 0 ? new BigNumber(0) : leafNode.start const implicitEnd = firstRightSibling !== null ? firstRightSibling.index : null @@ -215,8 +239,8 @@ export class MerkleIntervalTree { index: leaf.start, hash: this.hashfn( Buffer.concat([ - leaf.start.toBuffer('BE', 16), - leaf.end.toBuffer('BE', 16), + leaf.start.toBuffer('be', 16), + leaf.end.toBuffer('be', 16), leaf.data, ]) ), @@ -249,9 +273,9 @@ export class MerkleIntervalTree { rightChild: MerkleIntervalTreeInternalNode ): MerkleIntervalTreeInternalNode { const data = Buffer.concat([ - leftChild.index, + leftChild.index.toBuffer('be', 16), leftChild.hash, - rightChild.index, + rightChild.index.toBuffer('be', 16), rightChild.hash, ]) const hash = this.hashfn(data) diff --git a/packages/utils/test/eth/utils.spec.ts b/packages/utils/test/eth/utils.spec.ts index 90f40838..895c36bd 100644 --- a/packages/utils/test/eth/utils.spec.ts +++ b/packages/utils/test/eth/utils.spec.ts @@ -6,21 +6,20 @@ import { keccak256 } from '../../src/eth/utils' describe('Ethereum Utils', () => { describe('keccak256', () => { it('should return the keccak256 hash of a value', () => { - const value = '0x123' + const value = Buffer.from('1234', 'hex') const hash = keccak256(value) - hash.should.equal( - '0x667d3611273365cfb6e64399d5af0bf332ec3e5d6986f76bc7d10839b680eb58' - ) + const expected = Buffer.from('387a8233c96e1fc0ad5e284353276177af2186e7afa85296f106336e376669f7', 'hex') + hash.should.deep.equal(expected) }) it('should automatically add 0x if it does not exist', () => { - const valueA = '123' - const valueB = '0x123' + const valueA = Buffer.from('123') + const valueB = Buffer.from('0x123') const hashA = keccak256(valueA) const hashB = keccak256(valueB) - hashA.should.equal(hashB) + hashA.should.deep.equal(hashB) }) }) }) diff --git a/packages/utils/test/merkle-interval-tree.spec.ts b/packages/utils/test/merkle-interval-tree.spec.ts index 9b4ad066..2e726a9f 100644 --- a/packages/utils/test/merkle-interval-tree.spec.ts +++ b/packages/utils/test/merkle-interval-tree.spec.ts @@ -1,162 +1,176 @@ import { should } from './setup' /* External Imports */ -import BigNum = require('bn.js') +import BigNumber = require('bn.js') /* Internal Imports */ -import { MerkleIntervalTree, MerkleTreeNode } from '../src/merkle-interval-tree' +import { + MerkleIntervalTree, + MerkleIntervalTreeLeafNode, + MerkleIntervalTreeInternalNode, +} from '../src/merkle-interval-tree' describe('MerkleIntervalTree', () => { describe('construction', () => { it('should be created correctly with no leaves', () => { const tree = new MerkleIntervalTree() - should.not.exist(tree.root) - tree.leaves.should.deep.equal([]) - tree.levels.should.deep.equal([[]]) + should.not.exist(tree.getRoot()) + tree.getLevels().should.deep.equal([[]]) }) it('should be created correctly with one leaf', () => { - const leaves: MerkleTreeNode[] = [ + const leaves: MerkleIntervalTreeLeafNode[] = [ { - end: new BigNum(100), - data: '0x123', + start: new BigNumber(0), + end: new BigNumber(100), + data: Buffer.from('0x123'), }, ] - const tree = new MerkleIntervalTree({ - leaves, - }) + const tree = new MerkleIntervalTree(leaves) - tree.root.should.equal( - '0x9b64c516a177f40236cfa25c63deb358d93081decbcac3a99dbdcfe856f1d20b' - ) + tree + .getRoot() + .should.equal( + '0x9b64c516a177f40236cfa25c63deb358d93081decbcac3a99dbdcfe856f1d20b' + ) }) it('should be created correctly with two leaves', () => { - const leaves: MerkleTreeNode[] = [ + const leaves: MerkleIntervalTreeLeafNode[] = [ { - end: new BigNum(100), - data: '0x123', + start: new BigNumber(0), + end: new BigNumber(100), + data: Buffer.from('0x123'), }, { - end: new BigNum(200), - data: '0x456', + start: new BigNumber(100), + end: new BigNumber(200), + data: Buffer.from('0x456'), }, ] - const tree = new MerkleIntervalTree({ - leaves, - }) + const tree = new MerkleIntervalTree(leaves) - tree.root.should.equal( - '0xec76338e61e80c68c487626bf8c793d88f189f553af34ca9d5683c2a1e81a9f5' - ) + tree + .getRoot() + .should.equal( + '0xec76338e61e80c68c487626bf8c793d88f189f553af34ca9d5683c2a1e81a9f5' + ) }) }) - describe('verify', () => { + describe('checkInclusionProof', () => { it('should correctly verify a valid proof', () => { const tree = new MerkleIntervalTree() - const leaf: MerkleTreeNode = { - end: new BigNum(100), - data: '0x123', + const leaf: MerkleIntervalTreeLeafNode = { + start: new BigNumber(0), + end: new BigNumber(100), + data: Buffer.from('0x123'), } - const inclusionProof: MerkleTreeNode[] = [ + const inclusionProof: MerkleIntervalTreeInternalNode[] = [ { - end: new BigNum(200), - data: - '0x2d8f2d36584051e513680eb7387c21fab7f2511d711694ada0674d669d89022d', + index: new BigNumber(200), + hash: Buffer.from( + '0x2d8f2d36584051e513680eb7387c21fab7f2511d711694ada0674d669d89022d' + ), }, ] - const root = - '0xec76338e61e80c68c487626bf8c793d88f189f553af34ca9d5683c2a1e81a9f5' + const rootHash = + Buffer.from('0xec76338e61e80c68c487626bf8c793d88f189f553af34ca9d5683c2a1e81a9f5') should.not.Throw(() => { - tree.verify(leaf, 0, inclusionProof, root) + tree.checkInclusionProof(leaf, 0, inclusionProof, rootHash) }) }) it('should correctly reject a proof with the wrong root', () => { const tree = new MerkleIntervalTree() - const leaf: MerkleTreeNode = { - end: new BigNum(100), - data: '0x123', + const leaf: MerkleIntervalTreeLeafNode = { + start: new BigNumber(0), + end: new BigNumber(100), + data: Buffer.from('0x123'), } - const inclusionProof: MerkleTreeNode[] = [ + const inclusionProof: MerkleIntervalTreeInternalNode[] = [ { - end: new BigNum(200), - data: - '0x2d8f2d36584051e513680eb7387c21fab7f2511d711694ada0674d669d89022d', + index: new BigNumber(200), + hash: Buffer.from( + '0x2d8f2d36584051e513680eb7387c21fab7f2511d711694ada0674d669d89022d' + ), }, ] - const root = '0x000000' + const rootHash = Buffer.from('0x000000') should.Throw(() => { - tree.verify(leaf, 0, inclusionProof, root) + tree.checkInclusionProof(leaf, 0, inclusionProof, rootHash) }, 'Invalid Merkle Sum Tree proof.') }) it('should correctly reject a proof with the wrong siblings', () => { const tree = new MerkleIntervalTree() - const leaf: MerkleTreeNode = { - end: new BigNum(100), - data: '0x123', + const leaf: MerkleIntervalTreeLeafNode = { + start: new BigNumber(0), + end: new BigNumber(100), + data: Buffer.from('0x123'), } - const inclusionProof: MerkleTreeNode[] = [ + const inclusionProof: MerkleIntervalTreeInternalNode[] = [ { - end: new BigNum(200), - data: '0x00000000', + index: new BigNumber(200), + hash: Buffer.from('0x00000000'), }, ] - const root = - '0xec76338e61e80c68c487626bf8c793d88f189f553af34ca9d5683c2a1e81a9f5' + const rootHash = + Buffer.from('0xec76338e61e80c68c487626bf8c793d88f189f553af34ca9d5683c2a1e81a9f5') should.Throw(() => { - tree.verify(leaf, 0, inclusionProof, root) + tree.checkInclusionProof(leaf, 0, inclusionProof, rootHash) }, 'Invalid Merkle Sum Tree proof.') }) it('should correctly reject a proof with an invalid sibling', () => { const tree = new MerkleIntervalTree() - const leaf: MerkleTreeNode = { - end: new BigNum(100), - data: '0x123', + const leaf: MerkleIntervalTreeLeafNode = { + start: new BigNumber(0), + end: new BigNumber(100), + data: Buffer.from('0x123'), } - const inclusionProof: MerkleTreeNode[] = [ + const inclusionProof: MerkleIntervalTreeInternalNode[] = [ { - end: new BigNum(50), - data: - '0x2d8f2d36584051e513680eb7387c21fab7f2511d711694ada0674d669d89022d', + index: new BigNumber(50), + hash: Buffer.from( + '0x2d8f2d36584051e513680eb7387c21fab7f2511d711694ada0674d669d89022d' + ), }, ] - const root = - '0xec76338e61e80c68c487626bf8c793d88f189f553af34ca9d5683c2a1e81a9f5' + const rootHash = + Buffer.from('0xec76338e61e80c68c487626bf8c793d88f189f553af34ca9d5683c2a1e81a9f5') should.Throw(() => { - tree.verify(leaf, 0, inclusionProof, root) + tree.checkInclusionProof(leaf, 0, inclusionProof, rootHash) }, 'Invalid Merkle Sum Tree proof.') }) }) describe('getInclusionProof', () => { it('should return a valid proof for a node', () => { - const leaves: MerkleTreeNode[] = [ + const leaves: MerkleIntervalTreeLeafNode[] = [ { - end: new BigNum(100), - data: '0x123', + start: new BigNumber(0), + end: new BigNumber(100), + data: Buffer.from('0x123'), }, { - end: new BigNum(200), - data: '0x456', + start: new BigNumber(0), + end: new BigNumber(200), + data: Buffer.from('0x456'), }, ] - const tree = new MerkleIntervalTree({ - leaves, - }) - const expected: MerkleTreeNode[] = [ + const tree = new MerkleIntervalTree(leaves) + const expected: MerkleIntervalTreeInternalNode[] = [ { - end: new BigNum(200), - data: - '0x2d8f2d36584051e513680eb7387c21fab7f2511d711694ada0674d669d89022d', + index: new BigNumber(200), + hash: Buffer.from( + '0x2d8f2d36584051e513680eb7387c21fab7f2511d711694ada0674d669d89022d' + ), }, ] From 1d94fffdc9db54f9596b299df55cace68164e3fa Mon Sep 17 00:00:00 2001 From: Kelvin Fichter Date: Sun, 16 Jun 2019 14:06:06 -0400 Subject: [PATCH 3/6] Fixed keccak256 tests --- packages/utils/test/eth/utils.spec.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/utils/test/eth/utils.spec.ts b/packages/utils/test/eth/utils.spec.ts index 895c36bd..eabe9a36 100644 --- a/packages/utils/test/eth/utils.spec.ts +++ b/packages/utils/test/eth/utils.spec.ts @@ -9,17 +9,16 @@ describe('Ethereum Utils', () => { const value = Buffer.from('1234', 'hex') const hash = keccak256(value) - const expected = Buffer.from('387a8233c96e1fc0ad5e284353276177af2186e7afa85296f106336e376669f7', 'hex') + const expected = Buffer.from('56570de287d73cd1cb6092bb8fdee6173974955fdef345ae579ee9f475ea7432', 'hex') hash.should.deep.equal(expected) }) - it('should automatically add 0x if it does not exist', () => { - const valueA = Buffer.from('123') - const valueB = Buffer.from('0x123') - const hashA = keccak256(valueA) - const hashB = keccak256(valueB) + it('should return the keccak256 of the empty string', () => { + const value = Buffer.from('', 'hex') + const hash = keccak256(value) - hashA.should.deep.equal(hashB) + const expected = Buffer.from('c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470', 'hex') + hash.should.deep.equal(expected) }) }) }) From bb8d6edd82aa1603b71ebc4ce72a20105a9ce1ae Mon Sep 17 00:00:00 2001 From: Kelvin Fichter Date: Sun, 16 Jun 2019 15:47:10 -0400 Subject: [PATCH 4/6] Fixed tests and linted everything --- packages/utils/src/merkle-interval-tree.ts | 30 +-- packages/utils/test/eth/utils.spec.ts | 10 +- .../utils/test/merkle-interval-tree.spec.ts | 240 ++++++++++++------ 3 files changed, 190 insertions(+), 90 deletions(-) diff --git a/packages/utils/src/merkle-interval-tree.ts b/packages/utils/src/merkle-interval-tree.ts index 236d3160..ac7d5449 100644 --- a/packages/utils/src/merkle-interval-tree.ts +++ b/packages/utils/src/merkle-interval-tree.ts @@ -2,6 +2,7 @@ import BigNumber = require('bn.js') /* Internal Imports */ +import { Range } from './interfaces' import { keccak256 } from './eth/utils' import { bnMin, bnMax, except, reverse } from './utils' @@ -45,11 +46,6 @@ const outOfBounds = (list: any[], index: number): boolean => { return index < 0 || index >= list.length } -export interface Range { - start: BigNumber - end: BigNumber -} - export interface MerkleIntervalTreeLeafNode { start: BigNumber end: BigNumber @@ -126,8 +122,8 @@ export class MerkleIntervalTree { const currentLevel = this.levels[i] const childNode = currentLevel[childIndex] const siblingNode = outOfBounds(currentLevel, siblingIndex) - ? currentLevel[siblingIndex] - : this.createEmptyNode(childNode.index) + ? this.createEmptyNode(childNode.index) + : currentLevel[siblingIndex] inclusionProof.push(siblingNode) @@ -160,15 +156,10 @@ export class MerkleIntervalTree { new BigNumber(leafPosition).toString(2, inclusionProof.length) ) - const firstRightSiblingIndex = path.indexOf('0') - const firstRightSibling = - firstRightSiblingIndex >= 0 - ? inclusionProof[firstRightSiblingIndex] - : null - let computedNode = this.parseLeaf(leafNode) let leftChild: MerkleIntervalTreeInternalNode let rightChild: MerkleIntervalTreeInternalNode + let prevRightSibling: MerkleIntervalTreeInternalNode = null for (let i = 0; i < inclusionProof.length; i++) { const siblingNode = inclusionProof[i] @@ -181,13 +172,16 @@ export class MerkleIntervalTree { rightChild = siblingNode if ( - firstRightSibling !== null && - rightChild.index.lt(firstRightSibling.index) + (prevRightSibling !== null && + rightChild.index.lt(prevRightSibling.index)) || + rightChild.index.lt(leafNode.end) ) { throw new Error( 'Invalid Merkle Interval Tree proof -- potential intersection detected.' ) } + + prevRightSibling = rightChild } computedNode = this.computeParent(leftChild, rightChild) @@ -199,6 +193,12 @@ export class MerkleIntervalTree { ) } + const firstRightSiblingIndex = path.indexOf('0') + const firstRightSibling = + firstRightSiblingIndex >= 0 + ? inclusionProof[firstRightSiblingIndex] + : null + const implicitStart = leafPosition === 0 ? new BigNumber(0) : leafNode.start const implicitEnd = firstRightSibling !== null ? firstRightSibling.index : null diff --git a/packages/utils/test/eth/utils.spec.ts b/packages/utils/test/eth/utils.spec.ts index eabe9a36..32536f78 100644 --- a/packages/utils/test/eth/utils.spec.ts +++ b/packages/utils/test/eth/utils.spec.ts @@ -9,7 +9,10 @@ describe('Ethereum Utils', () => { const value = Buffer.from('1234', 'hex') const hash = keccak256(value) - const expected = Buffer.from('56570de287d73cd1cb6092bb8fdee6173974955fdef345ae579ee9f475ea7432', 'hex') + const expected = Buffer.from( + '56570de287d73cd1cb6092bb8fdee6173974955fdef345ae579ee9f475ea7432', + 'hex' + ) hash.should.deep.equal(expected) }) @@ -17,7 +20,10 @@ describe('Ethereum Utils', () => { const value = Buffer.from('', 'hex') const hash = keccak256(value) - const expected = Buffer.from('c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470', 'hex') + const expected = Buffer.from( + 'c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470', + 'hex' + ) hash.should.deep.equal(expected) }) }) diff --git a/packages/utils/test/merkle-interval-tree.spec.ts b/packages/utils/test/merkle-interval-tree.spec.ts index 2e726a9f..c19a89a0 100644 --- a/packages/utils/test/merkle-interval-tree.spec.ts +++ b/packages/utils/test/merkle-interval-tree.spec.ts @@ -8,175 +8,269 @@ import { MerkleIntervalTree, MerkleIntervalTreeLeafNode, MerkleIntervalTreeInternalNode, + MerkleIntervalTreeInclusionProof, } from '../src/merkle-interval-tree' +/** + * Converts a string to a hex buffer. + * @param value String value to convert. + * @returns the string as a hex buffer. + */ +const hexify = (value: string): Buffer => { + return Buffer.from(value, 'hex') +} + describe('MerkleIntervalTree', () => { describe('construction', () => { it('should be created correctly with no leaves', () => { const tree = new MerkleIntervalTree() - should.not.exist(tree.getRoot()) - tree.getLevels().should.deep.equal([[]]) + const root = tree.getRoot() + const levels = tree.getLevels() + + should.not.exist(root) + levels.should.deep.equal([[]]) }) it('should be created correctly with one leaf', () => { - const leaves: MerkleIntervalTreeLeafNode[] = [ + const tree = new MerkleIntervalTree([ { start: new BigNumber(0), end: new BigNumber(100), - data: Buffer.from('0x123'), + data: hexify('1234'), }, - ] - const tree = new MerkleIntervalTree(leaves) + ]) - tree - .getRoot() - .should.equal( - '0x9b64c516a177f40236cfa25c63deb358d93081decbcac3a99dbdcfe856f1d20b' - ) + const root = tree.getRoot() + + root.should.deep.equal({ + index: new BigNumber(0), + hash: hexify( + '4d7654430bd809384e15ef3e842aad2449b6310b25c652a220af74716b37e0ae' + ), + }) }) it('should be created correctly with two leaves', () => { - const leaves: MerkleIntervalTreeLeafNode[] = [ + const tree = new MerkleIntervalTree([ { start: new BigNumber(0), end: new BigNumber(100), - data: Buffer.from('0x123'), + data: hexify('1234'), }, { start: new BigNumber(100), end: new BigNumber(200), - data: Buffer.from('0x456'), + data: hexify('5678'), }, - ] - const tree = new MerkleIntervalTree(leaves) + ]) - tree - .getRoot() - .should.equal( - '0xec76338e61e80c68c487626bf8c793d88f189f553af34ca9d5683c2a1e81a9f5' - ) + const root = tree.getRoot() + + root.should.deep.equal({ + index: new BigNumber(0), + hash: hexify( + 'e1b53cab461af771ad8d060145d2e27a04ee7c2e671efe4feac748de8cef1fc5' + ), + }) }) }) describe('checkInclusionProof', () => { it('should correctly verify a valid proof', () => { - const tree = new MerkleIntervalTree() const leaf: MerkleIntervalTreeLeafNode = { start: new BigNumber(0), end: new BigNumber(100), - data: Buffer.from('0x123'), + data: hexify('1234'), } - const inclusionProof: MerkleIntervalTreeInternalNode[] = [ + const inclusionProof: MerkleIntervalTreeInclusionProof = [ { - index: new BigNumber(200), - hash: Buffer.from( - '0x2d8f2d36584051e513680eb7387c21fab7f2511d711694ada0674d669d89022d' + index: new BigNumber(100), + hash: hexify( + '05cc573cfe77fad641c92f62241633a64f5656275753ae9b8bf67b44f29a777b' ), }, ] - const rootHash = - Buffer.from('0xec76338e61e80c68c487626bf8c793d88f189f553af34ca9d5683c2a1e81a9f5') + const rootHash = hexify( + 'e1b53cab461af771ad8d060145d2e27a04ee7c2e671efe4feac748de8cef1fc5' + ) - should.not.Throw(() => { - tree.checkInclusionProof(leaf, 0, inclusionProof, rootHash) + const bounds = new MerkleIntervalTree().checkInclusionProof( + leaf, + 0, + inclusionProof, + rootHash + ) + + bounds.should.deep.equal({ + start: new BigNumber(0), + end: new BigNumber(100), }) }) it('should correctly reject a proof with the wrong root', () => { - const tree = new MerkleIntervalTree() const leaf: MerkleIntervalTreeLeafNode = { start: new BigNumber(0), end: new BigNumber(100), - data: Buffer.from('0x123'), + data: hexify('1234'), } - const inclusionProof: MerkleIntervalTreeInternalNode[] = [ + const inclusionProof: MerkleIntervalTreeInclusionProof = [ { - index: new BigNumber(200), - hash: Buffer.from( - '0x2d8f2d36584051e513680eb7387c21fab7f2511d711694ada0674d669d89022d' + index: new BigNumber(100), + hash: hexify( + '05cc573cfe77fad641c92f62241633a64f5656275753ae9b8bf67b44f29a777b' ), }, ] - const rootHash = Buffer.from('0x000000') + const rootHash = hexify( + '0000000000000000000000000000000000000000000000000000000000000000' + ) should.Throw(() => { - tree.checkInclusionProof(leaf, 0, inclusionProof, rootHash) - }, 'Invalid Merkle Sum Tree proof.') + new MerkleIntervalTree().checkInclusionProof( + leaf, + 0, + inclusionProof, + rootHash + ) + }, 'Invalid Merkle Interval Tree proof -- invalid root hash.') }) - it('should correctly reject a proof with the wrong siblings', () => { - const tree = new MerkleIntervalTree() + it('should correctly reject a proof with an invalid sibling hash', () => { const leaf: MerkleIntervalTreeLeafNode = { start: new BigNumber(0), end: new BigNumber(100), - data: Buffer.from('0x123'), + data: hexify('1234'), } - const inclusionProof: MerkleIntervalTreeInternalNode[] = [ + const inclusionProof: MerkleIntervalTreeInclusionProof = [ { - index: new BigNumber(200), - hash: Buffer.from('0x00000000'), + index: new BigNumber(100), + hash: hexify( + '0000000000000000000000000000000000000000000000000000000000000000' + ), }, ] - const rootHash = - Buffer.from('0xec76338e61e80c68c487626bf8c793d88f189f553af34ca9d5683c2a1e81a9f5') + const rootHash = hexify( + 'e1b53cab461af771ad8d060145d2e27a04ee7c2e671efe4feac748de8cef1fc5' + ) should.Throw(() => { - tree.checkInclusionProof(leaf, 0, inclusionProof, rootHash) - }, 'Invalid Merkle Sum Tree proof.') + new MerkleIntervalTree().checkInclusionProof( + leaf, + 0, + inclusionProof, + rootHash + ) + }, 'Invalid Merkle Interval Tree proof -- invalid root hash.') }) - it('should correctly reject a proof with an invalid sibling', () => { - const tree = new MerkleIntervalTree() + it('should correctly reject a proof with an overlapping sibling', () => { const leaf: MerkleIntervalTreeLeafNode = { start: new BigNumber(0), end: new BigNumber(100), - data: Buffer.from('0x123'), + data: hexify('1234'), } - const inclusionProof: MerkleIntervalTreeInternalNode[] = [ + const inclusionProof: MerkleIntervalTreeInclusionProof = [ { index: new BigNumber(50), - hash: Buffer.from( - '0x2d8f2d36584051e513680eb7387c21fab7f2511d711694ada0674d669d89022d' + hash: hexify( + '85c599e3cc2588f5c561128ab27347805ad71c33ea8db75d18823e7117bb9d4b' ), }, ] - const rootHash = - Buffer.from('0xec76338e61e80c68c487626bf8c793d88f189f553af34ca9d5683c2a1e81a9f5') + const rootHash = hexify( + '224678c7f9b59e07eb036c1914798220b3d1b8d56beb18518f6698d9a0146b84' + ) should.Throw(() => { - tree.checkInclusionProof(leaf, 0, inclusionProof, rootHash) - }, 'Invalid Merkle Sum Tree proof.') + new MerkleIntervalTree().checkInclusionProof( + leaf, + 0, + inclusionProof, + rootHash + ) + }, 'Invalid Merkle Interval Tree proof -- potential intersection detected.') + }) + + it('should correctly reject a proof with a non-monotonic right sibling', () => { + const leaf: MerkleIntervalTreeLeafNode = { + start: new BigNumber(0), + end: new BigNumber(100), + data: hexify('1234'), + } + const inclusionProof: MerkleIntervalTreeInclusionProof = [ + { + index: new BigNumber(300), + hash: hexify( + '12c3f0bbe76afc5f6aefd5b3584fa90bc9e49f945ebd6fe5f4b86205b44e2b71' + ), + }, + { + index: new BigNumber(100), + hash: hexify( + '89e643ad387e04a149ba8c0d7b62ac42ff32c212183035b1a7a720c0ee24699e' + ), + }, + ] + const rootHash = hexify( + 'd40fc6083f76dd701db66dfbc465a945d8148067181c4c6e007e8f02c90853e3' + ) + + should.Throw(() => { + new MerkleIntervalTree().checkInclusionProof( + leaf, + 0, + inclusionProof, + rootHash + ) + }, 'Invalid Merkle Interval Tree proof -- potential intersection detected.') }) }) describe('getInclusionProof', () => { it('should return a valid proof for a node', () => { - const leaves: MerkleIntervalTreeLeafNode[] = [ + const tree = new MerkleIntervalTree([ { start: new BigNumber(0), end: new BigNumber(100), - data: Buffer.from('0x123'), + data: hexify('1234'), }, { - start: new BigNumber(0), + start: new BigNumber(100), end: new BigNumber(200), - data: Buffer.from('0x456'), + data: hexify('5678'), }, - ] - const tree = new MerkleIntervalTree(leaves) - const expected: MerkleIntervalTreeInternalNode[] = [ + ]) + + const inclusionProof = tree.getInclusionProof(0) + + inclusionProof.should.deep.equal([ { - index: new BigNumber(200), - hash: Buffer.from( - '0x2d8f2d36584051e513680eb7387c21fab7f2511d711694ada0674d669d89022d' + index: new BigNumber(100), + hash: hexify( + '05cc573cfe77fad641c92f62241633a64f5656275753ae9b8bf67b44f29a777b' ), }, - ] + ]) + }) - const inclusionProof = tree.getInclusionProof(0) + it('should throw an error when getting a proof for a non-existent node', () => { + const tree = new MerkleIntervalTree([ + { + start: new BigNumber(0), + end: new BigNumber(100), + data: hexify('1234'), + }, + { + start: new BigNumber(100), + end: new BigNumber(200), + data: hexify('5678'), + }, + ]) - inclusionProof.should.deep.equal(expected) + should.Throw(() => { + tree.getInclusionProof(2) + }, 'Leaf position is out of bounds.') }) }) }) From c7f1efe4bb3708227a3ee8c03afe1cf2490b9cac Mon Sep 17 00:00:00 2001 From: Kelvin Fichter Date: Sun, 16 Jun 2019 16:13:18 -0400 Subject: [PATCH 5/6] Fixed small bug --- .travis.yml | 1 + packages/core/src/app/common/utils/merkle-interval-tree.ts | 7 ++++++- packages/core/src/app/common/utils/misc.ts | 1 - .../common/utils/merkle-interval-tree.interface.ts | 3 +++ .../test/app/common/utils/merkle-interval-tree.spec.ts | 6 ++---- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index a307e054..9f2aebdd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,5 +7,6 @@ cache: - npm script: + - yarn lint - yarn build - yarn test diff --git a/packages/core/src/app/common/utils/merkle-interval-tree.ts b/packages/core/src/app/common/utils/merkle-interval-tree.ts index 414ee541..07a27233 100644 --- a/packages/core/src/app/common/utils/merkle-interval-tree.ts +++ b/packages/core/src/app/common/utils/merkle-interval-tree.ts @@ -2,7 +2,12 @@ import BigNumber = require('bn.js') /* Internal Imports */ -import { Range, MerkleIntervalTreeLeafNode, MerkleIntervalTreeInternalNode, MerkleIntervalTreeInclusionProof } from '../../../interfaces' +import { + Range, + MerkleIntervalTreeLeafNode, + MerkleIntervalTreeInternalNode, + MerkleIntervalTreeInclusionProof, +} from '../../../interfaces' import { keccak256 } from '../eth/utils' import { bnMin, bnMax, except, reverse } from './misc' diff --git a/packages/core/src/app/common/utils/misc.ts b/packages/core/src/app/common/utils/misc.ts index 1b455edf..8c1890a7 100644 --- a/packages/core/src/app/common/utils/misc.ts +++ b/packages/core/src/app/common/utils/misc.ts @@ -196,7 +196,6 @@ export const hexStrToBuf = (hexString: string): Buffer => { return Buffer.from(hexString.slice(2), 'hex') } - /** * Creates a new version of a list with all instances of a specific element * removed. diff --git a/packages/core/src/interfaces/common/utils/merkle-interval-tree.interface.ts b/packages/core/src/interfaces/common/utils/merkle-interval-tree.interface.ts index 0bcdeb99..748905e8 100644 --- a/packages/core/src/interfaces/common/utils/merkle-interval-tree.interface.ts +++ b/packages/core/src/interfaces/common/utils/merkle-interval-tree.interface.ts @@ -1,3 +1,6 @@ +/* External Imports */ +import BigNumber = require('bn.js') + export interface MerkleIntervalTreeLeafNode { start: BigNumber end: BigNumber diff --git a/packages/core/test/app/common/utils/merkle-interval-tree.spec.ts b/packages/core/test/app/common/utils/merkle-interval-tree.spec.ts index 040b0758..ed2cbe04 100644 --- a/packages/core/test/app/common/utils/merkle-interval-tree.spec.ts +++ b/packages/core/test/app/common/utils/merkle-interval-tree.spec.ts @@ -4,11 +4,9 @@ import { should } from '../../../setup' import BigNumber = require('bn.js') /* Internal Imports */ -import { - MerkleIntervalTree, -} from '../../../../src/app/common/utils/merkle-interval-tree' +import { MerkleIntervalTree } from '../../../../src/app/common/utils/merkle-interval-tree' -import { +import { MerkleIntervalTreeLeafNode, MerkleIntervalTreeInternalNode, MerkleIntervalTreeInclusionProof, From 50df20b9fb62f07270125a3cf7844d1836c8872e Mon Sep 17 00:00:00 2001 From: Kelvin Fichter Date: Thu, 13 Jun 2019 17:40:24 -0400 Subject: [PATCH 6/6] Added initial BlockManager and BlockDB implementations --- packages/core/package.json | 1 + packages/core/src/app/index.ts | 1 + packages/core/src/app/operator/block-db.ts | 146 ++++++++++++++++++ .../core/src/app/operator/block-manager.ts | 57 +++++++ packages/core/src/app/operator/index.ts | 2 + .../client/commitment-contract.interface.ts | 3 + packages/core/src/interfaces/client/index.ts | 1 + packages/core/src/interfaces/index.ts | 2 + .../interfaces/operator/block-db.interface.ts | 10 ++ .../operator/block-manager.interface.ts | 9 ++ .../core/src/interfaces/operator/index.ts | 2 + 11 files changed, 234 insertions(+) create mode 100644 packages/core/src/app/operator/block-db.ts create mode 100644 packages/core/src/app/operator/block-manager.ts create mode 100644 packages/core/src/interfaces/client/commitment-contract.interface.ts create mode 100644 packages/core/src/interfaces/operator/block-db.interface.ts create mode 100644 packages/core/src/interfaces/operator/block-manager.interface.ts diff --git a/packages/core/package.json b/packages/core/package.json index 7d0b88c8..997195be 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -37,6 +37,7 @@ "test": "mocha --require ts-node/register 'test/**/*.spec.ts'" }, "dependencies": { + "@pigi/utils": "^0.0.1-alpha.10", "abstract-leveldown": "^6.0.3", "axios": "^0.19.0", "bn.js": "^4.11.8", diff --git a/packages/core/src/app/index.ts b/packages/core/src/app/index.ts index 40b88d6f..a2bcfefb 100644 --- a/packages/core/src/app/index.ts +++ b/packages/core/src/app/index.ts @@ -1,3 +1,4 @@ export * from './client' export * from './common' export * from './core' +export * from './operator' diff --git a/packages/core/src/app/operator/block-db.ts b/packages/core/src/app/operator/block-db.ts new file mode 100644 index 00000000..0ee41527 --- /dev/null +++ b/packages/core/src/app/operator/block-db.ts @@ -0,0 +1,146 @@ +/* External Imports */ +import BigNumber = require('bn.js') +import { MerkleIntervalTree } from '@pigi/utils' + +/* Internal Imports */ +import { BaseKey } from '../common' +import { KeyValueStore, BlockDB, StateUpdate } from '../../interfaces' + +const PREFIXES = { + VARS: new BaseKey('v').encode(), + BLOCKS: new BaseKey('b').encode(), +} + +const KEYS = { + NEXT_BLOCK: Buffer.from('nextblock'), + BLOCK: new BaseKey('b', ['uint32']), +} + +/** + * Simple BlockDB implementation. + */ +export class DefaultBlockDB implements BlockDB { + private vars: KeyValueStore + private blocks: KeyValueStore + + /** + * Initializes the database wrapper. + * @param db Database to store values in. + */ + constructor(private db: KeyValueStore) { + this.vars = this.db.bucket(PREFIXES.VARS) + this.blocks = this.db.bucket(PREFIXES.BLOCKS) + } + + /** + * @returns the next plasma block number. + */ + public async getNextBlockNumber(): Promise { + // NOTE: You could theoretically cache the next block number as to avoid + // having to read from the database repeatedly. However, this introduces + // the need to ensure that the cached value and stored value never fall out + // of sync. It's probably better to hold off on implementing caching until + // this becomes a performance bottleneck. + + const buf = await this.vars.get(KEYS.NEXT_BLOCK) + return buf.readUInt32BE(0) + } + + /** + * Adds a state update to the list of updates to be published in the next + * plasma block. + * @param stateUpdate State update to publish in the next block. + * @returns a promise that resolves once the update has been added. + */ + public async addPendingStateUpdate(stateUpdate: StateUpdate): Promise { + const block = await this.getNextBlockStore() + const start = stateUpdate.id.start + const end = stateUpdate.id.end + + // TODO: Figure out how to implement locking here so two state updates + // can't be added at the same time. + + if (await block.has(start, end)) { + throw new Error('Block already contains a state update over that range.') + } + + const value = Buffer.from(JSON.stringify(stateUpdate)) + await block.put(start, end, value) + } + + /** + * @returns the list of state updates waiting to be published in the next + * plasma block. + */ + public async getPendingStateUpdates(): Promise { + const blockNumber = await this.getNextBlockNumber() + return this.getStateUpdates(blockNumber) + } + + /** + * Computes the Merkle Interval Tree root of a given block. + * @param blockNumber Block to compute a root for. + * @returns the root of the block. + */ + public async getMerkleRoot(blockNumber: number): Promise { + const stateUpdates = await this.getStateUpdates(blockNumber) + + const leaves = stateUpdates.map((stateUpdate) => { + const encodedStateUpdate = JSON.stringify(stateUpdate) // TODO: Actually encode this. + return { + end: stateUpdate.id.end, + data: encodedStateUpdate, + } + }) + const tree = new MerkleIntervalTree({ + leaves, + }) + return tree.root + } + + /** + * Finalizes the next plasma block so that it can be published. + * @returns a promise that resolves once the block has been finalized. + */ + public async finalizeNextBlock(): Promise { + const prevBlockNumber = await this.getNextBlockNumber() + const nextBlockNumber = Buffer.allocUnsafe(4) + nextBlockNumber.writeUInt32BE(prevBlockNumber + 1, 0) + await this.vars.put(KEYS.NEXT_BLOCK, nextBlockNumber) + } + + /** + * Opens the RangeDB for a specific block. + * @param blockNumber Block to open the RangeDB for. + * @returns the RangeDB instance for the given block. + */ + private async getBlockStore(blockNumber: number): Promise { + const key = KEYS.BLOCK.encode([blockNumber]) + const bucket = this.blocks.bucket(key) + return new RangeDB(bucket) + } + + /** + * @returns the RangeDB instance for the next block to be published. + */ + private async getNextBlockStore(): Promise { + const blockNumber = await this.getNextBlockNumber() + return this.getBlockStore(blockNumber) + } + + /** + * Queries all of the state updates within a given block. + * @param blockNumber Block to query state updates for. + * @returns the list of state updates for that block. + */ + private async getStateUpdates(blockNumber: number): Promise { + const block = await this.getBlockStore(blockNumber) + const values = await block.get( + new BigNumber(0), + new BigNumber('0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF') + ) + return values.map((value) => { + return JSON.parse(value.toString()) + }) + } +} diff --git a/packages/core/src/app/operator/block-manager.ts b/packages/core/src/app/operator/block-manager.ts new file mode 100644 index 00000000..8a3fb426 --- /dev/null +++ b/packages/core/src/app/operator/block-manager.ts @@ -0,0 +1,57 @@ +/* Internal Imports */ +import { + BlockManager, + BlockDB, + CommitmentContract, + StateUpdate, +} from '../../interfaces' + +/** + * Simple BlockManager implementation. + */ +export class DefaultBlockManager implements BlockManager { + /** + * Initializes the manager. + * @param blockdb BlockDB instance to store/query data from. + * @param commitmentContract Contract wrapper used to publish block roots. + */ + constructor( + private blockdb: BlockDB, + private commitmentContract: CommitmentContract + ) {} + + /** + * @returns the next plasma block number. + */ + public async getNextBlockNumber(): Promise { + return this.blockdb.getNextBlockNumber() + } + + /** + * Adds a state update to the list of updates to be published in the next + * plasma block. + * @param stateUpdate State update to add to the next block. + * @returns a promise that resolves once the update has been added. + */ + public async addPendingStateUpdate(stateUpdate: StateUpdate): Promise { + await this.blockdb.addPendingStateUpdate(stateUpdate) + } + + /** + * @returns the state updates to be published in the next block. + */ + public async getPendingStateUpdates(): Promise { + return this.blockdb.getPendingStateUpdates() + } + + /** + * Finalizes the next block and submits the block root to Ethereum. + * @returns a promise that resolves once the block has been published. + */ + public async submitNextBlock(): Promise { + const blockNumber = await this.getNextBlockNumber() + await this.blockdb.finalizeNextBlock() + const root = await this.blockdb.getMerkleRoot(blockNumber) + await this.commitmentContract.submitBlock(root) + } +} diff --git a/packages/core/src/app/operator/index.ts b/packages/core/src/app/operator/index.ts index e69de29b..49052407 100644 --- a/packages/core/src/app/operator/index.ts +++ b/packages/core/src/app/operator/index.ts @@ -0,0 +1,2 @@ +import './block-db' +import './block-manager' diff --git a/packages/core/src/interfaces/client/commitment-contract.interface.ts b/packages/core/src/interfaces/client/commitment-contract.interface.ts new file mode 100644 index 00000000..a157a78e --- /dev/null +++ b/packages/core/src/interfaces/client/commitment-contract.interface.ts @@ -0,0 +1,3 @@ +export interface CommitmentContract { + submitBlock(root: string): Promise +} diff --git a/packages/core/src/interfaces/client/index.ts b/packages/core/src/interfaces/client/index.ts index bb713d02..cf450eeb 100644 --- a/packages/core/src/interfaces/client/index.ts +++ b/packages/core/src/interfaces/client/index.ts @@ -2,3 +2,4 @@ export * from './state-manager.interface' export * from './state-db.interface' export * from './predicate-plugin.interface' export * from './plugin-manager.interface' +export * from './commitment-contract.interface' diff --git a/packages/core/src/interfaces/index.ts b/packages/core/src/interfaces/index.ts index 40b88d6f..b66fb95b 100644 --- a/packages/core/src/interfaces/index.ts +++ b/packages/core/src/interfaces/index.ts @@ -1,3 +1,5 @@ export * from './client' export * from './common' export * from './core' +export * from './client' +export * from './operator' diff --git a/packages/core/src/interfaces/operator/block-db.interface.ts b/packages/core/src/interfaces/operator/block-db.interface.ts new file mode 100644 index 00000000..622900e1 --- /dev/null +++ b/packages/core/src/interfaces/operator/block-db.interface.ts @@ -0,0 +1,10 @@ +/* Internal Imports */ +import { StateUpdate } from '../common/utils/state.interface' + +export interface BlockDB { + getNextBlockNumber(): Promise + addPendingStateUpdate(stateUpdate: StateUpdate): Promise + getPendingStateUpdates(): Promise + getMerkleRoot(blockNumber: number): Promise + finalizeNextBlock(): Promise +} diff --git a/packages/core/src/interfaces/operator/block-manager.interface.ts b/packages/core/src/interfaces/operator/block-manager.interface.ts new file mode 100644 index 00000000..ffc980d2 --- /dev/null +++ b/packages/core/src/interfaces/operator/block-manager.interface.ts @@ -0,0 +1,9 @@ +/* Internal Imports */ +import { StateUpdate } from '../common/utils/state.interface' + +export interface BlockManager { + getNextBlockNumber(): Promise + addPendingStateUpdate(stateUpdate: StateUpdate): Promise + getPendingStateUpdates(): Promise + submitNextBlock(): Promise +} diff --git a/packages/core/src/interfaces/operator/index.ts b/packages/core/src/interfaces/operator/index.ts index e69de29b..5b24a254 100644 --- a/packages/core/src/interfaces/operator/index.ts +++ b/packages/core/src/interfaces/operator/index.ts @@ -0,0 +1,2 @@ +export * from './block-db.interface' +export * from './block-manager.interface'