diff --git a/hamt/hamt.go b/hamt/hamt.go index 241d6d829..e49134e8f 100644 --- a/hamt/hamt.go +++ b/hamt/hamt.go @@ -69,6 +69,9 @@ type Shard struct { dserv ipld.DAGService + // FIXME: Remove. We don't actually store "value nodes". This confusing + // abstraction just removes the maxpadlen from the link names to extract + // the actual value link the trie is storing. // leaf node key string val *ipld.Link diff --git a/io/directory_test.go b/io/directory_test.go index 5a1bbe273..70e0f201f 100644 --- a/io/directory_test.go +++ b/io/directory_test.go @@ -266,8 +266,12 @@ func TestHAMTEnumerationWhenComputingSize(t *testing.T) { //DefaultShardWidth = 8 treeHeight := 2 - //HAMTShardingSize = int(math.Pow(float64(DefaultShardWidth), float64(treeHeight))) - HAMTShardingSize = DefaultShardWidth + thresholdToWidthRatio := 4 + HAMTShardingSize = DefaultShardWidth * thresholdToWidthRatio + // We will fetch enough nodes to get to the leaf shards (`treeHeight - 1`) + // and then enough shards (each with DefaultShardWidth value links) to + // reach the HAMTShardingSize. + nodesToFetch := treeHeight - 1 + thresholdToWidthRatio ds := mdtest.Mock() node, err := hamt.CreateFullShard(ds, treeHeight) @@ -282,19 +286,11 @@ func TestHAMTEnumerationWhenComputingSize(t *testing.T) { sequentialDagService.ResetCounter() // FIXME: Revert to const timeout after debugging. - _, timeoutExceeded := hamtDir.sizeBelowThreshold(1, func(ctx context.Context, _ *ipld.Link) { + below, timeoutExceeded := hamtDir.sizeBelowThreshold(1, func(ctx context.Context, _ *ipld.Link) { }) + assert.False(t, below) assert.False(t, timeoutExceeded) - // Fetched nodes: One child (internal) shard. First child of the root - // (key of 0). - // THen: how many children from async? 256 internal children? - // FIXME: This is currently fetching *all* children. Not working. - nodesFetched := 2 // Main and one child with all the values. - // FIXME: Values are encoded in a different shard or just as links? - /// Seems like the second one but get confirmation on this. - assert.Equal(t, nodesFetched, sequentialDagService.UniqueCidsRequested()) - // FIXME: how much? This needs to be deduced - // from the DefaultShardWidth + assert.Equal(t, nodesToFetch, sequentialDagService.UniqueCidsFetched()) } // Compare entries in the leftDir against the rightDir and possibly @@ -441,9 +437,9 @@ func TestDirBuilder(t *testing.T) { type serialFetchDag struct { ipld.DAGService - nextNode <-chan struct{} - cidsRequested map[cid.Cid]struct{} - mapLock sync.Mutex + nextNode <-chan struct{} + cidsFetched map[cid.Cid]struct{} + mapLock sync.Mutex } var _ ipld.DAGService = (*serialFetchDag)(nil) @@ -460,26 +456,33 @@ func newSerialFetchDag(ds ipld.DAGService) (*serialFetchDag, chan<- struct{}) { } func (d *serialFetchDag) ResetCounter() { - d.cidsRequested = make(map[cid.Cid]struct{}) + d.cidsFetched = make(map[cid.Cid]struct{}) } -func (d *serialFetchDag) UniqueCidsRequested() int { +func (d *serialFetchDag) UniqueCidsFetched() int { d.mapLock.Lock() defer d.mapLock.Unlock() - return len(d.cidsRequested) + return len(d.cidsFetched) } // Retrieve only one node at a time. func (d *serialFetchDag) Get(ctx context.Context, c cid.Cid) (ipld.Node, error) { - d.mapLock.Lock() - d.cidsRequested[c] = struct{}{} - d.mapLock.Unlock() + // Simulate a context cancel during a fetch operation (over the network). + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } node, err := d.DAGService.Get(ctx, c) if err != nil { return node, err } + d.mapLock.Lock() + d.cidsFetched[c] = struct{}{} + d.mapLock.Unlock() + // Only block for leaf shard nodes. shard, shardErr := hamt.NewHamtFromDag(nil, node) // (We can pass a nil, we won't use the dag service.) @@ -487,13 +490,13 @@ func (d *serialFetchDag) Get(ctx context.Context, c cid.Cid) (ipld.Node, error) return node, err } - fmt.Printf("looking at channel\n") - select { - case <-d.nextNode: - case <-ctx.Done(): - return nil, ctx.Err() - } - fmt.Printf("green light\n") + //fmt.Printf("looking at channel\n") + //select { + //case <-d.nextNode: + //case <-ctx.Done(): + // return nil, ctx.Err() + //} + //fmt.Printf("green light\n") return node, err }