fix and document TestHAMTEnumerationWhenComputingSize

hamt/hamt.go

@@ -69,6 +69,9 @@ type Shard struct {
 
 	dserv ipld.DAGService
 
+	// FIXME: Remove. We don't actually store "value nodes". This confusing
+	//  abstraction just removes the maxpadlen from the link names to extract
+	//  the actual value link the trie is storing.
 	// leaf node
 	key string
 	val *ipld.Link

io/directory_test.go

@@ -266,8 +266,12 @@ func TestHAMTEnumerationWhenComputingSize(t *testing.T) {
 	//DefaultShardWidth = 8
 
 	treeHeight := 2
-	//HAMTShardingSize = int(math.Pow(float64(DefaultShardWidth), float64(treeHeight)))
-	HAMTShardingSize = DefaultShardWidth
+	thresholdToWidthRatio := 4
+	HAMTShardingSize = DefaultShardWidth * thresholdToWidthRatio
+	// We will fetch enough nodes to get to the leaf shards (`treeHeight - 1`)
+	// and then enough shards (each with DefaultShardWidth value links) to
+	// reach the HAMTShardingSize.
+	nodesToFetch := treeHeight - 1 + thresholdToWidthRatio
 
 	ds := mdtest.Mock()
 	node, err := hamt.CreateFullShard(ds, treeHeight)
@@ -282,19 +286,11 @@ func TestHAMTEnumerationWhenComputingSize(t *testing.T) {
 
 	sequentialDagService.ResetCounter()
 	// FIXME: Revert to const timeout after debugging.
-	_, timeoutExceeded := hamtDir.sizeBelowThreshold(1, func(ctx context.Context, _ *ipld.Link) {
+	below, timeoutExceeded := hamtDir.sizeBelowThreshold(1, func(ctx context.Context, _ *ipld.Link) {
 	})
+	assert.False(t, below)
 	assert.False(t, timeoutExceeded)
-	// Fetched nodes: One child (internal) shard. First child of the root
-	// (key of 0).
-	// THen: how many children from async? 256 internal children?
-	// FIXME: This is currently fetching *all* children. Not working.
-	nodesFetched := 2 // Main and one child with all the values.
-	// FIXME: Values are encoded in a different shard or just as links?
-	///  Seems like the second one but get confirmation on this.
-	assert.Equal(t, nodesFetched, sequentialDagService.UniqueCidsRequested())
-	// FIXME: how much? This needs to be deduced
-	//                         from the DefaultShardWidth
+	assert.Equal(t, nodesToFetch, sequentialDagService.UniqueCidsFetched())
 }
 
 // Compare entries in the leftDir against the rightDir and possibly
@@ -441,9 +437,9 @@ func TestDirBuilder(t *testing.T) {
 type serialFetchDag struct {
 	ipld.DAGService
 
-	nextNode      <-chan struct{}
-	cidsRequested map[cid.Cid]struct{}
-	mapLock       sync.Mutex
+	nextNode    <-chan struct{}
+	cidsFetched map[cid.Cid]struct{}
+	mapLock     sync.Mutex
 }
 
 var _ ipld.DAGService = (*serialFetchDag)(nil)
@@ -460,40 +456,47 @@ func newSerialFetchDag(ds ipld.DAGService) (*serialFetchDag, chan<- struct{}) {
 }
 
 func (d *serialFetchDag) ResetCounter() {
-	d.cidsRequested = make(map[cid.Cid]struct{})
+	d.cidsFetched = make(map[cid.Cid]struct{})
 }
 
-func (d *serialFetchDag) UniqueCidsRequested() int {
+func (d *serialFetchDag) UniqueCidsFetched() int {
 	d.mapLock.Lock()
 	defer d.mapLock.Unlock()
-	return len(d.cidsRequested)
+	return len(d.cidsFetched)
 }
 
 // Retrieve only one node at a time.
 func (d *serialFetchDag) Get(ctx context.Context, c cid.Cid) (ipld.Node, error) {
-	d.mapLock.Lock()
-	d.cidsRequested[c] = struct{}{}
-	d.mapLock.Unlock()
+	// Simulate a context cancel during a fetch operation (over the network).
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	default:
+	}
 
 	node, err := d.DAGService.Get(ctx, c)
 	if err != nil {
 		return node, err
 	}
 
+	d.mapLock.Lock()
+	d.cidsFetched[c] = struct{}{}
+	d.mapLock.Unlock()
+
 	// Only block for leaf shard nodes.
 	shard, shardErr := hamt.NewHamtFromDag(nil, node)
 	// (We can pass a nil, we won't use the dag service.)
 	if shardErr != nil || !shard.IsValueNode() {
 		return node, err
 	}
 
-	fmt.Printf("looking at channel\n")
-	select {
-	case <-d.nextNode:
-	case <-ctx.Done():
-		return nil, ctx.Err()
-	}
-	fmt.Printf("green light\n")
+	//fmt.Printf("looking at channel\n")
+	//select {
+	//case <-d.nextNode:
+	//case <-ctx.Done():
+	//	return nil, ctx.Err()
+	//}
+	//fmt.Printf("green light\n")
 
 	return node, err
 }