Skip to content

Commit

Permalink
feat(traversal): support lists and low cardinality
Browse files Browse the repository at this point in the history 
Improve the traverse function so that it only explores relevant fields for low cardinality selectors
and traverses high cardinality selectors for lists correctly
  • Loading branch information
@hannahhoward
hannahhoward committed Aug 6, 2019
1 parent fcc1cc5 commit 31c9bb7
Show file tree
Hide file tree
Showing 3 changed files with 175 additions and 34 deletions.
45 changes: 45 additions & 0 deletions traversal/selector/selector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,3 +107,48 @@ func (ps PathSegmentInt) String() string {
func (ps PathSegmentInt) Index() (int, error) {
return ps.I, nil
}

// SegmentIterator iterates either a list or a map, generating PathSegments
// instead of indexes or keys
type SegmentIterator interface {
Next() (pathSegment PathSegment, value ipld.Node, err error)
Done() bool
}

// NewSegmentIterator generates a new iterator based on the node type
func NewSegmentIterator(n ipld.Node) SegmentIterator {
if n.ReprKind() == ipld.ReprKind_List {
return listSegmentIterator{n.ListIterator()}
}
return mapSegmentIterator{n.MapIterator()}
}

type listSegmentIterator struct {
ipld.ListIterator
}

func (lsi listSegmentIterator) Next() (pathSegment PathSegment, value ipld.Node, err error) {
i, v, err := lsi.ListIterator.Next()
return PathSegmentInt{i}, v, err
}

func (lsi listSegmentIterator) Done() bool {
return lsi.ListIterator.Done()
}

type mapSegmentIterator struct {
ipld.MapIterator
}

func (msi mapSegmentIterator) Next() (pathSegment PathSegment, value ipld.Node, err error) {
k, v, err := msi.MapIterator.Next()
if err != nil {
return nil, v, err
}
kstr, _ := k.AsString()
return PathSegmentString{kstr}, v, err
}

func (msi mapSegmentIterator) Done() bool {
return msi.MapIterator.Done()
}
87 changes: 54 additions & 33 deletions traversal/traverse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,46 +50,67 @@ func (tp TraversalProgress) traverseInformatively(n ipld.Node, s selector.Select
default:
return nil
}
// TODO: should only do this full loop if high-cardinality indicated.
// attn := s.Interests()
// if attn == nil {
// FIXME need another kind switch here, and list support!
for itr := n.MapIterator(); !itr.Done(); {
k, v, err := itr.Next()
if err != nil {
return err
}
kstr, _ := k.AsString()
sNext := s.Explore(n, selector.PathSegmentString{kstr})
if sNext != nil {
tpNext := tp
tpNext.Path = tp.Path.AppendSegment(kstr)
if v.ReprKind() == ipld.ReprKind_Link {
lnk, _ := v.AsLink()
// Assemble the LinkContext in case the Loader or NBChooser want it.
lnkCtx := ipld.LinkContext{
LinkPath: tpNext.Path,
LinkNode: v,
ParentNode: n,
attn := s.Interests()
if attn == nil {
for itr := selector.NewSegmentIterator(n); !itr.Done(); {
ps, v, err := itr.Next()
if err != nil {
return err
}
sNext := s.Explore(n, ps)
if sNext != nil {
err = tp.traverseChild(n, v, ps, sNext, fn)
if err != nil {
return err
}
// Load link!
v, err = lnk.Load(
tpNext.Cfg.Ctx,
lnkCtx,
tpNext.Cfg.LinkNodeBuilderChooser(lnk, lnkCtx),
tpNext.Cfg.LinkLoader,
)
}
}
} else {
for _, ps := range attn {
// TODO: Probably not the most efficient way to be doing this...
v, err := n.TraverseField(ps.String())
if err != nil {
continue
}
sNext := s.Explore(n, ps)
if sNext != nil {
err = tp.traverseChild(n, v, ps, sNext, fn)
if err != nil {
return fmt.Errorf("error traversing node at %q: could not load link %q: %s", tpNext.Path, lnk, err)
return err
}
}
// TODO when link load is implemented, it should go roughly here.
}
}
return nil
}

if err := tpNext.traverseInformatively(v, sNext, fn); err != nil {
return err
}
func (tp TraversalProgress) traverseChild(n ipld.Node, v ipld.Node, ps selector.PathSegment, sNext selector.Selector, fn AdvVisitFn) error {
var err error
tpNext := tp
tpNext.Path = tp.Path.AppendSegment(ps.String())
if v.ReprKind() == ipld.ReprKind_Link {
lnk, _ := v.AsLink()
// Assemble the LinkContext in case the Loader or NBChooser want it.
lnkCtx := ipld.LinkContext{
LinkPath: tpNext.Path,
LinkNode: v,
ParentNode: n,
}
// Load link!
v, err = lnk.Load(
tpNext.Cfg.Ctx,
lnkCtx,
tpNext.Cfg.LinkNodeBuilderChooser(lnk, lnkCtx),
tpNext.Cfg.LinkLoader,
)
if err != nil {
return fmt.Errorf("error traversing node at %q: could not load link %q: %s", tpNext.Path, lnk, err)
}
}

if err := tpNext.traverseInformatively(v, sNext, fn); err != nil {
return err
}
return nil
}

Expand Down
77 changes: 76 additions & 1 deletion traversal/traverse_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import (
"github.com/ipld/go-ipld-prime/traversal"
"github.com/ipld/go-ipld-prime/traversal/selector"
"github.com/ipld/go-ipld-prime/traversal/selector/builder"

)

/* Remember, we've got the following fixtures in scope:
Expand Down Expand Up @@ -156,4 +155,80 @@ func TestTraverse(t *testing.T) {
Wish(t, err, ShouldEqual, nil)
Wish(t, order, ShouldEqual, 6)
})
t.Run("traversing lists should work", func(t *testing.T) {
ss := ssb.ExploreRange(0, 3, ssb.Matcher())
s, err := ss.Selector()
var order int
err = traversal.TraversalProgress{
Cfg: &traversal.TraversalConfig{
LinkLoader: func(lnk ipld.Link, _ ipld.LinkContext) (io.Reader, error) {
return bytes.NewBuffer(storage[lnk]), nil
},
},
}.Traverse(middleListNode, s, func(tp traversal.TraversalProgress, n ipld.Node) error {
switch order {
case 0:
Wish(t, n, ShouldEqual, fnb.CreateString("alpha"))
Wish(t, tp.Path.String(), ShouldEqual, "0")
case 1:
Wish(t, n, ShouldEqual, fnb.CreateString("alpha"))
Wish(t, tp.Path.String(), ShouldEqual, "1")
case 2:
Wish(t, n, ShouldEqual, fnb.CreateString("beta"))
Wish(t, tp.Path.String(), ShouldEqual, "2")
}
order++
return nil
})
Wish(t, err, ShouldEqual, nil)
Wish(t, order, ShouldEqual, 3)
})
t.Run("multiple layers of link traversal should work", func(t *testing.T) {
ss := ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) {
efsb.Insert("linkedList", ssb.ExploreAll(ssb.Matcher()))
efsb.Insert("linkedMap", ssb.ExploreRecursive(3, ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) {
efsb.Insert("foo", ssb.Matcher())
efsb.Insert("nonlink", ssb.Matcher())
efsb.Insert("alink", ssb.Matcher())
efsb.Insert("nested", ssb.ExploreRecursiveEdge())
})))
})
s, err := ss.Selector()
var order int
err = traversal.TraversalProgress{
Cfg: &traversal.TraversalConfig{
LinkLoader: func(lnk ipld.Link, _ ipld.LinkContext) (io.Reader, error) {
return bytes.NewBuffer(storage[lnk]), nil
},
},
}.Traverse(rootNode, s, func(tp traversal.TraversalProgress, n ipld.Node) error {
switch order {
case 0:
Wish(t, n, ShouldEqual, fnb.CreateString("alpha"))
Wish(t, tp.Path.String(), ShouldEqual, "linkedList/0")
case 1:
Wish(t, n, ShouldEqual, fnb.CreateString("alpha"))
Wish(t, tp.Path.String(), ShouldEqual, "linkedList/1")
case 2:
Wish(t, n, ShouldEqual, fnb.CreateString("beta"))
Wish(t, tp.Path.String(), ShouldEqual, "linkedList/2")
case 3:
Wish(t, n, ShouldEqual, fnb.CreateString("alpha"))
Wish(t, tp.Path.String(), ShouldEqual, "linkedList/3")
case 4:
Wish(t, n, ShouldEqual, fnb.CreateBool(true))
Wish(t, tp.Path.String(), ShouldEqual, "linkedMap/foo")
case 5:
Wish(t, n, ShouldEqual, fnb.CreateString("zoo"))
Wish(t, tp.Path.String(), ShouldEqual, "linkedMap/nested/nonlink")
case 6:
Wish(t, n, ShouldEqual, fnb.CreateString("alpha"))
Wish(t, tp.Path.String(), ShouldEqual, "linkedMap/nested/alink")
}
order++
return nil
})
Wish(t, err, ShouldEqual, nil)
Wish(t, order, ShouldEqual, 7)
})
}

0 comments on commit 31c9bb7

Please sign in to comment.