Skip to content

Commit

Permalink
Use CID to prevent incorrect usage in netmap (#396)
Browse files Browse the repository at this point in the history
close #216
  • Loading branch information
roman-khimov authored Apr 21, 2023
2 parents 16daed8 + 83ca99a commit dbbb22c
Show file tree
Hide file tree
Showing 5 changed files with 61 additions and 20 deletions.
45 changes: 38 additions & 7 deletions netmap/json_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"path/filepath"
"testing"

cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/stretchr/testify/require"
)

Expand Down Expand Up @@ -37,6 +39,20 @@ func compareNodes(t testing.TB, expected [][]int, nodes nodes, actual [][]NodeIn
}
}

func compareNodesIgnoreOrder(t testing.TB, expected [][]int, nodes nodes, actual [][]NodeInfo) {
require.Equal(t, len(expected), len(actual))
for i := range expected {
require.Equal(t, len(expected[i]), len(actual[i]))

var expectedNodes []NodeInfo
for _, index := range expected[i] {
expectedNodes = append(expectedNodes, nodes[index])
}

require.ElementsMatch(t, expectedNodes, actual[i])
}
}

func TestPlacementPolicy_Interopability(t *testing.T) {
const testsDir = "./json_tests"

Expand All @@ -62,18 +78,24 @@ func TestPlacementPolicy_Interopability(t *testing.T) {

for name, tt := range tc.Tests {
t.Run(name, func(t *testing.T) {
v, err := nm.ContainerNodes(tt.Policy, tt.Pivot)
var pivot cid.ID
copy(pivot[:], tt.Pivot)

v, err := nm.ContainerNodes(tt.Policy, pivot)
if tt.Result == nil {
require.Error(t, err)
require.Contains(t, err.Error(), tt.Error)
} else {
require.NoError(t, err)
require.Equal(t, srcNodes, tc.Nodes)

compareNodes(t, tt.Result, tc.Nodes, v)
compareNodesIgnoreOrder(t, tt.Result, tc.Nodes, v)

if tt.Placement.Result != nil {
res, err := nm.PlacementVectors(v, tt.Placement.Pivot)
var placementPivot oid.ID
copy(placementPivot[:], tt.Placement.Pivot)

res, err := nm.PlacementVectors(v, placementPivot)
require.NoError(t, err)
compareNodes(t, tt.Placement.Result, tc.Nodes, res)
require.Equal(t, srcNodes, tc.Nodes)
Expand Down Expand Up @@ -108,23 +130,29 @@ func BenchmarkPlacementPolicyInteropability(b *testing.B) {

for name, tt := range tc.Tests {
b.Run(name, func(b *testing.B) {
var pivot cid.ID
copy(pivot[:], tt.Pivot)

b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StartTimer()
v, err := nm.ContainerNodes(tt.Policy, tt.Pivot)
v, err := nm.ContainerNodes(tt.Policy, pivot)
b.StopTimer()
if tt.Result == nil {
require.Error(b, err)
require.Contains(b, err.Error(), tt.Error)
} else {
require.NoError(b, err)

compareNodes(b, tt.Result, tc.Nodes, v)
compareNodesIgnoreOrder(b, tt.Result, tc.Nodes, v)

if tt.Placement.Result != nil {
var placementPivot oid.ID
copy(placementPivot[:], tt.Placement.Pivot)

b.StartTimer()
res, err := nm.PlacementVectors(v, tt.Placement.Pivot)
res, err := nm.PlacementVectors(v, placementPivot)
b.StopTimer()
require.NoError(b, err)
compareNodes(b, tt.Placement.Result, tc.Nodes, res)
Expand All @@ -150,11 +178,14 @@ func BenchmarkManySelects(b *testing.B) {
var nm NetMap
nm.SetNodes(tc.Nodes)

var pivot cid.ID
copy(pivot[:], tt.Pivot)

b.ResetTimer()
b.ReportAllocs()

for i := 0; i < b.N; i++ {
_, err = nm.ContainerNodes(tt.Policy, tt.Pivot)
_, err = nm.ContainerNodes(tt.Policy, pivot)
if err != nil {
b.FailNow()
}
Expand Down
4 changes: 2 additions & 2 deletions netmap/json_tests/many_selects.json
Original file line number Diff line number Diff line change
Expand Up @@ -183,9 +183,9 @@
"policy": {"replicas":[{"count":1,"selector":"SameRU"},{"count":1,"selector":"DistinctRU"},{"count":1,"selector":"Good"},{"count":1,"selector":"Main"}],"containerBackupFactor":2,"selectors":[{"name":"SameRU","count":2,"clause":"SAME","attribute":"City","filter":"FromRU"},{"name":"DistinctRU","count":2,"clause":"DISTINCT","attribute":"City","filter":"FromRU"},{"name":"Good","count":2,"clause":"DISTINCT","attribute":"Country","filter":"Good"},{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[{"name":"FromRU","key":"Country","op":"EQ","value":"Russia"},{"name":"Good","key":"Rating","op":"GE","value":"4"}]},
"result": [
[0, 5, 9, 10],
[2, 6, 0, 5],
[0, 5, 2, 6],
[1, 8, 2, 5],
[3, 4, 1, 7, 0, 2]
[0, 2, 1, 7, 3, 4]
]
}
}
Expand Down
6 changes: 3 additions & 3 deletions netmap/json_tests/multiple_rep_asymmetric.json
Original file line number Diff line number Diff line change
Expand Up @@ -321,10 +321,10 @@
4
],
[
8,
12,
5,
10
10,
8,
12
]
]
}
Expand Down
21 changes: 15 additions & 6 deletions netmap/netmap.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
package netmap

import (
"crypto/sha256"
"fmt"

"github.com/nspcc-dev/hrw"
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
)

// NetMap represents NeoFS network map. It includes information about all
Expand Down Expand Up @@ -140,11 +143,14 @@ func flattenNodes(ns []nodes) nodes {
}

// PlacementVectors sorts container nodes returned by ContainerNodes method
// and returns placement vectors for the entity identified by the given pivot.
// and returns placement vectors for the entity identified by the given object id.
// For example, in order to build node list to store the object, binary-encoded
// object identifier can be used as pivot. Result is deterministic for
// the fixed NetMap and parameters.
func (m NetMap) PlacementVectors(vectors [][]NodeInfo, pivot []byte) ([][]NodeInfo, error) {
func (m NetMap) PlacementVectors(vectors [][]NodeInfo, objectID oid.ID) ([][]NodeInfo, error) {
pivot := make([]byte, sha256.Size)
objectID.Encode(pivot)

h := hrw.Hash(pivot)
wf := defaultWeightFunc(m.nodes)
result := make([][]NodeInfo, len(vectors))
Expand All @@ -162,15 +168,18 @@ func (m NetMap) PlacementVectors(vectors [][]NodeInfo, pivot []byte) ([][]NodeIn
// given PlacementPolicy to the NetMap. Each line of the list corresponds to a
// replica descriptor. Line order corresponds to order of ReplicaDescriptor list
// in the policy. Nodes are pre-filtered according to the Filter list from
// the policy, and then selected by Selector list. Result is deterministic for
// the fixed NetMap and parameters.
// the policy, and then selected by Selector list. Result is not deterministic and
// node order in each vector may vary for call.
//
// Result can be used in PlacementVectors.
func (m NetMap) ContainerNodes(p PlacementPolicy, pivot []byte) ([][]NodeInfo, error) {
func (m NetMap) ContainerNodes(p PlacementPolicy, containerID cid.ID) ([][]NodeInfo, error) {
c := newContext(m)
c.setPivot(pivot)
c.setCBF(p.backupFactor)

pivot := make([]byte, sha256.Size)
containerID.Encode(pivot)
c.setPivot(pivot)

if err := c.processFilters(p); err != nil {
return nil, err
}
Expand Down
5 changes: 3 additions & 2 deletions netmap/selector_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (

"github.com/nspcc-dev/hrw"
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/stretchr/testify/require"
)

Expand Down Expand Up @@ -129,7 +130,7 @@ func BenchmarkPolicyHRWType(b *testing.B) {

b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := nm.ContainerNodes(p, []byte{1})
_, err := nm.ContainerNodes(p, cid.ID{1})
if err != nil {
b.Fatal()
}
Expand Down Expand Up @@ -173,7 +174,7 @@ func TestPlacementPolicy_DeterministicOrder(t *testing.T) {
nm.SetNodes(nodeList)

getIndices := func(t *testing.T) (uint64, uint64) {
v, err := nm.ContainerNodes(p, []byte{1})
v, err := nm.ContainerNodes(p, cid.ID{1})
require.NoError(t, err)

nss := make([]nodes, len(v))
Expand Down

0 comments on commit dbbb22c

Please sign in to comment.