diff --git a/pkg/local_object_storage/metabase/metadata_test.go b/pkg/local_object_storage/metabase/metadata_test.go index b2cbb587df..19a521cd8e 100644 --- a/pkg/local_object_storage/metabase/metadata_test.go +++ b/pkg/local_object_storage/metabase/metadata_test.go @@ -3,6 +3,7 @@ package meta import ( "bytes" "encoding/base64" + "fmt" "math" "math/big" "math/rand" @@ -10,6 +11,7 @@ import ( "strconv" "testing" + objectcore "github.com/nspcc-dev/neofs-node/pkg/core/object" "github.com/nspcc-dev/neofs-sdk-go/checksum" checksumtest "github.com/nspcc-dev/neofs-sdk-go/checksum/test" "github.com/nspcc-dev/neofs-sdk-go/client" @@ -30,6 +32,14 @@ func sortObjectIDs(ids []oid.ID) []oid.ID { return s } +func searchResultForIDs(ids []oid.ID) []client.SearchResultItem { + res := make([]client.SearchResultItem, len(ids)) + for i := range ids { + res[i].ID = ids[i] + } + return res +} + func appendAttribute(obj *object.Object, k, v string) { obj.SetAttributes(append(obj.Attributes(), *object.NewAttribute(k, v))...) } @@ -413,6 +423,105 @@ func TestNewSearchCursorFromString(t *testing.T) { require.Equal(t, 1+len(attr)+attributeDelimiterLen, res.ValIDOff) } +func cloneIntFilterMap(src map[int]ParsedIntFilter) map[int]ParsedIntFilter { + if src == nil { + return nil + } + dst := make(map[int]ParsedIntFilter) + for k, f := range src { + var n *big.Int + if f.n != nil { + n = new(big.Int).Set(f.n) + } + dst[k] = ParsedIntFilter{ + auto: f.auto, + n: n, + b: slices.Clone(f.b), + } + } + return dst +} + +func _assertSearchResultWithLimit(t testing.TB, db *DB, cnr cid.ID, fs object.SearchFilters, attrs []string, all []client.SearchResultItem, lim uint16) { + fInt, ok := PreprocessIntFilters(fs) + if !ok { + require.Empty(t, all) + return + } + fIntClone := cloneIntFilterMap(fInt) + + var primAttr string + var primInt bool + if len(attrs) > 0 { + require.NotEmpty(t, fs) + require.Equal(t, attrs[0], fs[0].Header()) + if fs[0].Operation() != object.MatchNotPresent { + primAttr, primInt = attrs[0], objectcore.IsIntegerSearchOp(fs[0].Operation()) + } + } + + var strCursor string + var cursor *SearchCursor + nAttr := len(attrs) + for { + res, c, err := db.Search(cnr, fs, fInt, attrs, cursor, lim) + if len(fInt) > 0 { + require.Equal(t, fIntClone, fInt, "int filter map mutation detected", "cursor: %q", strCursor) + } + require.NoError(t, err, "cursor: %q", strCursor) + + n := min(len(all), int(lim)) + require.Len(t, res, n) + for i := range n { // all[:n] == res assert can lead to huge output when failed + require.Equalf(t, all[i].ID, res[i].ID, "cursor: %q, i: %d", strCursor, i) + require.Len(t, res[i].Attributes, nAttr) + if nAttr > 0 { + require.Equal(t, all[i].Attributes[:nAttr], res[i].Attributes) + } + } + + if all = all[n:]; len(all) == 0 { + require.Nilf(t, c, "cursor: %q", strCursor) + break + } + require.NotNilf(t, c, "cursor: %q", strCursor) + + cc, err := CalculateCursor(fs, res[n-1]) + require.NoError(t, err, "cursor: %q", strCursor) + require.Equal(t, c.Key, cc.Key, "cursor: %q", strCursor) + + strCursor = base64.StdEncoding.EncodeToString(c.Key) + cursor, err = NewSearchCursorFromString(strCursor, primAttr, primInt) + require.NoErrorf(t, err, "cursor: %q", strCursor) + } +} + +func assertSearchResultWithLimit(t testing.TB, db *DB, cnr cid.ID, fs object.SearchFilters, attrs []string, expRes []client.SearchResultItem, lim uint16) { + _assertSearchResultWithLimit(t, db, cnr, fs, attrs, expRes, lim) + if len(attrs) > 0 { + expRes = slices.Clone(expRes) + slices.SortFunc(expRes, func(a, b client.SearchResultItem) int { return bytes.Compare(a.ID[:], b.ID[:]) }) + _assertSearchResultWithLimit(t, db, cnr, fs, nil, expRes, lim) + } +} + +func assertSearchResult(t *testing.T, db *DB, cnr cid.ID, fs object.SearchFilters, attrs []string, expRes []client.SearchResultItem) { + for _, lim := range []uint16{1000, 1, 2} { + t.Run(fmt.Sprintf("limit=%d", lim), func(t *testing.T) { + assertSearchResultWithLimit(t, db, cnr, fs, attrs, expRes, lim) + }) + } +} + +func assertSearchResultIndexes(t *testing.T, db *DB, cnr cid.ID, fs object.SearchFilters, attrs []string, all []client.SearchResultItem, inds []uint) { + require.True(t, slices.IsSorted(inds)) + expRes := make([]client.SearchResultItem, len(inds)) + for i := range inds { + expRes[i] = all[inds[i]] + } + assertSearchResult(t, db, cnr, fs, attrs, expRes) +} + func TestDB_SearchObjects(t *testing.T) { db := newDB(t) t.Run("no filters", func(t *testing.T) { @@ -427,10 +536,7 @@ func TestDB_SearchObjects(t *testing.T) { cnr := cidtest.ID() t.Run("no objects", func(t *testing.T) { - res, cursor, err := db.Search(cnr, nil, nil, nil, nil, 1) - require.NoError(t, err) - require.Empty(t, cursor) - require.Empty(t, res) + assertSearchResult(t, db, cnr, nil, nil, nil) }) const n = 10 @@ -446,53 +552,10 @@ func TestDB_SearchObjects(t *testing.T) { } idsSorted := sortObjectIDs(ids) + all := searchResultForIDs(idsSorted) - t.Run("all at once", func(t *testing.T) { - for _, tc := range []struct { - name string - count uint16 - }{ - {name: "exact", count: n}, - {name: "more", count: n + 1}, - } { - res, cursor, err := db.Search(cnr, nil, nil, nil, nil, tc.count) - require.NoError(t, err) - require.Empty(t, cursor) - require.Len(t, res, n) - for i := range idsSorted { - require.Equal(t, idsSorted[i], res[i].ID, i) - require.Empty(t, res[i].Attributes) - } - } - }) t.Run("paginated", func(t *testing.T) { - // request 3 first - res, cursor, err := db.Search(cnr, nil, nil, nil, nil, 3) - require.NoError(t, err) - require.Len(t, res, 3) - require.NotNil(t, cursor) - for i := range 3 { - require.Equal(t, idsSorted[i], res[i].ID, i) - require.Empty(t, res[i].Attributes) - } - // then 6 more - cursor.Key = slices.Concat([]byte{0}, cursor.Key) - res, cursor, err = db.Search(cnr, nil, nil, nil, cursor, 6) - require.NoError(t, err) - require.Len(t, res, 6) - require.NotNil(t, cursor) - for i := range 6 { - require.Equal(t, idsSorted[3+i], res[i].ID, i) - require.Empty(t, res[i].Attributes) - } - // and up to 2 more - cursor.Key = slices.Concat([]byte{0}, cursor.Key) - res, cursor, err = db.Search(cnr, nil, nil, nil, cursor, 2) - require.NoError(t, err) - require.Nil(t, cursor) - require.Len(t, res, 1) - require.Equal(t, idsSorted[n-1], res[0].ID) - require.Empty(t, res[0].Attributes) + assertSearchResult(t, db, cnr, nil, nil, all) }) t.Run("corrupted element", func(t *testing.T) { err := db.boltDB.Update(func(tx *bbolt.Tx) error { @@ -636,27 +699,16 @@ func TestDB_SearchObjects(t *testing.T) { require.NoError(t, db.Put(&phys[i], nil, nil)) } - check := func(k string, m object.SearchMatchType, v string, matchInds []uint) { - var fs object.SearchFilters - fs.AddFilter(k, v, m) - fInt, ok := PreprocessIntFilters(fs) - if !ok { - require.Empty(t, matchInds) - return - } - require.True(t, ok) - res, cursor, err := db.Search(cnr, fs, fInt, nil, nil, nAll) - require.NoError(t, err) - require.Empty(t, cursor) - require.Len(t, res, len(matchInds)) - for i, ind := range matchInds { - require.Equal(t, ids[ind], res[i].ID) - require.Empty(t, res[i].Attributes) - } + check := func(t *testing.T, k string, m object.SearchMatchType, v string, matchInds []uint) { + t.Run(fmt.Sprintf("%s %s", m, v), func(t *testing.T) { + var fs object.SearchFilters + fs.AddFilter(k, v, m) + assertSearchResultIndexes(t, db, cnr, fs, nil, searchResultForIDs(ids[:]), matchInds) + }) } t.Run("all", func(t *testing.T) { - check("attr_common", object.MatchStringEqual, "val_common", all) + check(t, "attr_common", object.MatchStringEqual, "val_common", all) }) t.Run("user attributes", func(t *testing.T) { // unique @@ -664,15 +716,15 @@ func TestDB_SearchObjects(t *testing.T) { si := strconv.Itoa(i) key := "unique_attr_" + si val := "unique_val_" + si - check(key, object.MatchStringEqual, val, []uint{uint(i)}) - check(key, object.MatchStringNotEqual, "other_val", []uint{uint(i)}) + check(t, key, object.MatchStringEqual, val, []uint{uint(i)}) + check(t, key, object.MatchStringNotEqual, "other_val", []uint{uint(i)}) for j := range val { - check(key, object.MatchCommonPrefix, val[:j], []uint{uint(i)}) + check(t, key, object.MatchCommonPrefix, val[:j], []uint{uint(i)}) } for _, matcher := range []object.SearchMatchType{ object.MatchNumGT, object.MatchNumGE, object.MatchNumLT, object.MatchNumLE, } { - check(key, matcher, val, nil) + check(t, key, matcher, val, nil) } var others []uint for j := range all { @@ -680,28 +732,28 @@ func TestDB_SearchObjects(t *testing.T) { others = append(others, uint(j)) } } - check(key, object.MatchNotPresent, "", others) + check(t, key, object.MatchNotPresent, "", others) } // group const val1 = "group_val_1" - check("group_attr_1", object.MatchStringEqual, val1, group1) - check("group_attr_1", object.MatchStringNotEqual, val1, nil) - check("group_attr_1", object.MatchNotPresent, val1, group2) + check(t, "group_attr_1", object.MatchStringEqual, val1, group1) + check(t, "group_attr_1", object.MatchStringNotEqual, val1, nil) + check(t, "group_attr_1", object.MatchNotPresent, val1, group2) for i := range val1 { - check("group_attr_1", object.MatchCommonPrefix, val1[:i], group1) + check(t, "group_attr_1", object.MatchCommonPrefix, val1[:i], group1) } const val2 = "group_val_2" - check("group_attr_2", object.MatchStringEqual, val2, group2) - check("group_attr_2", object.MatchStringNotEqual, val2, nil) - check("group_attr_2", object.MatchNotPresent, val2, group1) + check(t, "group_attr_2", object.MatchStringEqual, val2, group2) + check(t, "group_attr_2", object.MatchStringNotEqual, val2, nil) + check(t, "group_attr_2", object.MatchNotPresent, val2, group1) for i := range val1 { - check("group_attr_2", object.MatchCommonPrefix, val2[:i], group2) + check(t, "group_attr_2", object.MatchCommonPrefix, val2[:i], group2) } for _, matcher := range []object.SearchMatchType{ object.MatchNumGT, object.MatchNumGE, object.MatchNumLT, object.MatchNumLE, } { - check("group_attr_1", matcher, val1, nil) - check("group_attr_2", matcher, val2, nil) + check(t, "group_attr_1", matcher, val1, nil) + check(t, "group_attr_2", matcher, val2, nil) } }) t.Run("ROOT", func(t *testing.T) { @@ -709,22 +761,22 @@ func TestDB_SearchObjects(t *testing.T) { object.MatchUnspecified, object.MatchStringEqual, object.MatchStringNotEqual, object.MatchNumGT, object.MatchNumGE, object.MatchNumLT, object.MatchNumLE, } { - check("$Object:ROOT", matcher, "", []uint{0}) + check(t, "$Object:ROOT", matcher, "", []uint{0}) } - check("$Object:ROOT", object.MatchNotPresent, "", nil) + check(t, "$Object:ROOT", object.MatchNotPresent, "", nil) }) t.Run("PHY", func(t *testing.T) { for _, matcher := range []object.SearchMatchType{ object.MatchUnspecified, object.MatchStringEqual, object.MatchStringNotEqual, object.MatchNumGT, object.MatchNumGE, object.MatchNumLT, object.MatchNumLE, } { - check("$Object:PHY", matcher, "", []uint{2, 3, 4, 5}) + check(t, "$Object:PHY", matcher, "", []uint{2, 3, 4, 5}) } - check("$Object:PHY", object.MatchNotPresent, "", nil) + check(t, "$Object:PHY", object.MatchNotPresent, "", nil) }) t.Run("version", func(t *testing.T) { check := func(m object.SearchMatchType, v string, matchInds []uint) { - check("$Object:version", m, v, matchInds) + check(t, "$Object:version", m, v, matchInds) } check(object.MatchStringEqual, "v100.200", group1) check(object.MatchStringNotEqual, "v100.200", group2) @@ -750,7 +802,7 @@ func TestDB_SearchObjects(t *testing.T) { }) t.Run("owner", func(t *testing.T) { check := func(m object.SearchMatchType, v string, matchInds []uint) { - check("$Object:ownerID", m, v, matchInds) + check(t, "$Object:ownerID", m, v, matchInds) } check(object.MatchStringEqual, "NfzJyPrn1hRGuVJNvMYLTfWZGW2ZVR9Qmj", group1) check(object.MatchStringNotEqual, "NfzJyPrn1hRGuVJNvMYLTfWZGW2ZVR9Qmj", group2) @@ -770,7 +822,7 @@ func TestDB_SearchObjects(t *testing.T) { }) t.Run("type", func(t *testing.T) { check := func(m object.SearchMatchType, v string, matchInds []uint) { - check("$Object:objectType", m, v, matchInds) + check(t, "$Object:objectType", m, v, matchInds) } check(object.MatchStringEqual, "REGULAR", group1) check(object.MatchStringNotEqual, "REGULAR", group2) @@ -797,7 +849,7 @@ func TestDB_SearchObjects(t *testing.T) { }) t.Run("payload checksum", func(t *testing.T) { check := func(m object.SearchMatchType, v string, matchInds []uint) { - check("$Object:payloadHash", m, v, matchInds) + check(t, "$Object:payloadHash", m, v, matchInds) } check(object.MatchStringEqual, "8a61b9ff3de0983ed7ad7aa21db22ff91e5a2a07128cd45e3646282f90e4efd7", []uint{0}) check(object.MatchStringEqual, "d501baff2dec96b7dec7d634e5ec13ed8be33048bfa4e8285a37dabc0537e677", []uint{1}) @@ -825,7 +877,7 @@ func TestDB_SearchObjects(t *testing.T) { }) t.Run("payload homomorphic checksum", func(t *testing.T) { check := func(m object.SearchMatchType, v string, matchInds []uint) { - check("$Object:homomorphicHash", m, v, matchInds) + check(t, "$Object:homomorphicHash", m, v, matchInds) } check(object.MatchStringEqual, "a73a37d54475df580b324d70f3d1ac922200af91f196dd9cb0f8f1cca5fefdf0cb3dbc4aaac639416e3fdd4c540e616e6b44ac6b56a3b194e8011925192a8be2", []uint{0}) check(object.MatchStringEqual, "f72b6eb562c6dd5e69930ab51ca8a98b13bfa18013cd89df3254dbc615f86b8f8c042649fe76e01f54bea7216957fe6716ec0a33d6b6de25ec15a53f295196d1", []uint{1}) @@ -854,7 +906,7 @@ func TestDB_SearchObjects(t *testing.T) { }) t.Run("split ID", func(t *testing.T) { check := func(m object.SearchMatchType, v string, matchInds []uint) { - check("$Object:split.splitID", m, v, matchInds) + check(t, "$Object:split.splitID", m, v, matchInds) } group1, group2, all := []uint{2, 4}, []uint{3, 5}, []uint{2, 3, 4, 5} check(object.MatchStringEqual, "8b69e76d-5e95-4639-8213-46786c41ab73", group1) @@ -873,7 +925,7 @@ func TestDB_SearchObjects(t *testing.T) { }) t.Run("first ID", func(t *testing.T) { check := func(m object.SearchMatchType, v string, matchInds []uint) { - check("$Object:split.first", m, v, matchInds) + check(t, "$Object:split.first", m, v, matchInds) } group1, group2, all := []uint{2, 4}, []uint{3, 5}, []uint{2, 3, 4, 5} check(object.MatchStringEqual, "61hnJaKip8c1QxvC2iT4Txfpxf37QBNRaw1XCeq72DbC", group1) @@ -894,7 +946,7 @@ func TestDB_SearchObjects(t *testing.T) { }) t.Run("parent ID", func(t *testing.T) { check := func(m object.SearchMatchType, v string, matchInds []uint) { - check("$Object:split.parent", m, v, matchInds) + check(t, "$Object:split.parent", m, v, matchInds) } group1, group2, all := []uint{2, 4}, []uint{3, 5}, []uint{2, 3, 4, 5} check(object.MatchStringEqual, "RSYscGLzKw1nkeVRGpowYTGgtgodXJrMyyiHTGGJW3S", group1) @@ -916,39 +968,39 @@ func TestDB_SearchObjects(t *testing.T) { for _, matcher := range []object.SearchMatchType{ object.MatchNumGT, object.MatchNumGE, object.MatchNumLT, object.MatchNumLE, } { - check("global_non_integer", matcher, "123", nil) + check(t, "global_non_integer", matcher, "123", nil) // TODO: also check that BoltDB is untouched in following cases - check("attr_int", matcher, "text", nil) - check("attr_int", matcher, "1.5", nil) + check(t, "attr_int", matcher, "text", nil) + check(t, "attr_int", matcher, "1.5", nil) } - check("attr_int", object.MatchNumLT, "-115792089237316195423570985008687907853269984665640564039457584007913129639936", nil) - check("attr_int", object.MatchNumLE, "-115792089237316195423570985008687907853269984665640564039457584007913129639936", nil) - check("attr_int", object.MatchNumGT, "-115792089237316195423570985008687907853269984665640564039457584007913129639936", nil) - check("attr_int", object.MatchNumGE, "-115792089237316195423570985008687907853269984665640564039457584007913129639936", nil) - check("attr_int", object.MatchNumLT, "115792089237316195423570985008687907853269984665640564039457584007913129639936", nil) - check("attr_int", object.MatchNumLE, "115792089237316195423570985008687907853269984665640564039457584007913129639936", nil) - check("attr_int", object.MatchNumGT, "115792089237316195423570985008687907853269984665640564039457584007913129639936", nil) - check("attr_int", object.MatchNumGE, "115792089237316195423570985008687907853269984665640564039457584007913129639936", nil) - check("attr_int", object.MatchNumLT, "-115792089237316195423570985008687907853269984665640564039457584007913129639935", nil) - check("attr_int", object.MatchNumLT, "-18446744073709551615", []uint{0}) - check("attr_int", object.MatchNumLT, "0", []uint{0, 2}) - check("attr_int", object.MatchNumLT, "18446744073709551615", []uint{0, 2, 3}) - check("attr_int", object.MatchNumLT, "115792089237316195423570985008687907853269984665640564039457584007913129639935", []uint{0, 2, 3, 4}) - check("attr_int", object.MatchNumLE, "-115792089237316195423570985008687907853269984665640564039457584007913129639935", []uint{0}) - check("attr_int", object.MatchNumLE, "-18446744073709551615", []uint{0, 2}) - check("attr_int", object.MatchNumLE, "0", []uint{0, 2, 3}) - check("attr_int", object.MatchNumLE, "18446744073709551615", []uint{0, 2, 3, 4}) - check("attr_int", object.MatchNumLE, "115792089237316195423570985008687907853269984665640564039457584007913129639935", allInt) - check("attr_int", object.MatchNumGT, "-115792089237316195423570985008687907853269984665640564039457584007913129639935", []uint{1, 2, 3, 4}) - check("attr_int", object.MatchNumGT, "-18446744073709551615", []uint{1, 3, 4}) - check("attr_int", object.MatchNumGT, "0", []uint{1, 4}) - check("attr_int", object.MatchNumGT, "18446744073709551615", []uint{1}) - check("attr_int", object.MatchNumGT, "115792089237316195423570985008687907853269984665640564039457584007913129639935", nil) - check("attr_int", object.MatchNumGE, "-115792089237316195423570985008687907853269984665640564039457584007913129639935", allInt) - check("attr_int", object.MatchNumGE, "-18446744073709551615", []uint{1, 2, 3, 4}) - check("attr_int", object.MatchNumGE, "0", []uint{1, 3, 4}) - check("attr_int", object.MatchNumGE, "18446744073709551615", []uint{1, 4}) - check("attr_int", object.MatchNumGE, "115792089237316195423570985008687907853269984665640564039457584007913129639935", []uint{1}) + check(t, "attr_int", object.MatchNumLT, "-115792089237316195423570985008687907853269984665640564039457584007913129639936", nil) + check(t, "attr_int", object.MatchNumLE, "-115792089237316195423570985008687907853269984665640564039457584007913129639936", nil) + check(t, "attr_int", object.MatchNumGT, "-115792089237316195423570985008687907853269984665640564039457584007913129639936", nil) + check(t, "attr_int", object.MatchNumGE, "-115792089237316195423570985008687907853269984665640564039457584007913129639936", nil) + check(t, "attr_int", object.MatchNumLT, "115792089237316195423570985008687907853269984665640564039457584007913129639936", nil) + check(t, "attr_int", object.MatchNumLE, "115792089237316195423570985008687907853269984665640564039457584007913129639936", nil) + check(t, "attr_int", object.MatchNumGT, "115792089237316195423570985008687907853269984665640564039457584007913129639936", nil) + check(t, "attr_int", object.MatchNumGE, "115792089237316195423570985008687907853269984665640564039457584007913129639936", nil) + check(t, "attr_int", object.MatchNumLT, "-115792089237316195423570985008687907853269984665640564039457584007913129639935", nil) + check(t, "attr_int", object.MatchNumLT, "-18446744073709551615", []uint{0}) + check(t, "attr_int", object.MatchNumLT, "0", []uint{0, 2}) + check(t, "attr_int", object.MatchNumLT, "18446744073709551615", []uint{0, 2, 3}) + check(t, "attr_int", object.MatchNumLT, "115792089237316195423570985008687907853269984665640564039457584007913129639935", []uint{0, 2, 3, 4}) + check(t, "attr_int", object.MatchNumLE, "-115792089237316195423570985008687907853269984665640564039457584007913129639935", []uint{0}) + check(t, "attr_int", object.MatchNumLE, "-18446744073709551615", []uint{0, 2}) + check(t, "attr_int", object.MatchNumLE, "0", []uint{0, 2, 3}) + check(t, "attr_int", object.MatchNumLE, "18446744073709551615", []uint{0, 2, 3, 4}) + check(t, "attr_int", object.MatchNumLE, "115792089237316195423570985008687907853269984665640564039457584007913129639935", allInt) + check(t, "attr_int", object.MatchNumGT, "-115792089237316195423570985008687907853269984665640564039457584007913129639935", []uint{1, 2, 3, 4}) + check(t, "attr_int", object.MatchNumGT, "-18446744073709551615", []uint{1, 3, 4}) + check(t, "attr_int", object.MatchNumGT, "0", []uint{1, 4}) + check(t, "attr_int", object.MatchNumGT, "18446744073709551615", []uint{1}) + check(t, "attr_int", object.MatchNumGT, "115792089237316195423570985008687907853269984665640564039457584007913129639935", nil) + check(t, "attr_int", object.MatchNumGE, "-115792089237316195423570985008687907853269984665640564039457584007913129639935", allInt) + check(t, "attr_int", object.MatchNumGE, "-18446744073709551615", []uint{1, 2, 3, 4}) + check(t, "attr_int", object.MatchNumGE, "0", []uint{1, 3, 4}) + check(t, "attr_int", object.MatchNumGE, "18446744073709551615", []uint{1, 4}) + check(t, "attr_int", object.MatchNumGE, "115792089237316195423570985008687907853269984665640564039457584007913129639935", []uint{1}) for _, tc := range []struct { name, key string val1, val2 string @@ -957,30 +1009,30 @@ func TestDB_SearchObjects(t *testing.T) { {name: "payload length", key: "$Object:payloadLength", val1: "20", val2: "21"}, } { t.Run(tc.name, func(t *testing.T) { - check(tc.key, object.MatchNumLT, "-115792089237316195423570985008687907853269984665640564039457584007913129639935", nil) - check(tc.key, object.MatchNumLT, "0", nil) - check(tc.key, object.MatchNumLT, tc.val1, nil) - check(tc.key, object.MatchNumLT, tc.val2, group1) - check(tc.key, object.MatchNumLT, "18446744073709551615", all) - check(tc.key, object.MatchNumLT, "115792089237316195423570985008687907853269984665640564039457584007913129639935", all) - check(tc.key, object.MatchNumLE, "-115792089237316195423570985008687907853269984665640564039457584007913129639935", nil) - check(tc.key, object.MatchNumLE, "0", nil) - check(tc.key, object.MatchNumLE, tc.val1, group1) - check(tc.key, object.MatchNumLE, tc.val2, all) - check(tc.key, object.MatchNumLE, "18446744073709551615", all) - check(tc.key, object.MatchNumLE, "115792089237316195423570985008687907853269984665640564039457584007913129639935", all) - check(tc.key, object.MatchNumGT, "-115792089237316195423570985008687907853269984665640564039457584007913129639935", all) - check(tc.key, object.MatchNumGT, "0", all) - check(tc.key, object.MatchNumGT, tc.val1, group2) - check(tc.key, object.MatchNumGT, tc.val2, nil) - check(tc.key, object.MatchNumGT, "18446744073709551615", nil) - check(tc.key, object.MatchNumGT, "115792089237316195423570985008687907853269984665640564039457584007913129639935", nil) - check(tc.key, object.MatchNumGE, "-115792089237316195423570985008687907853269984665640564039457584007913129639935", all) - check(tc.key, object.MatchNumGE, "0", all) - check(tc.key, object.MatchNumGE, tc.val1, all) - check(tc.key, object.MatchNumGE, tc.val2, group2) - check(tc.key, object.MatchNumGE, "18446744073709551615", nil) - check(tc.key, object.MatchNumGE, "115792089237316195423570985008687907853269984665640564039457584007913129639935", nil) + check(t, tc.key, object.MatchNumLT, "-115792089237316195423570985008687907853269984665640564039457584007913129639935", nil) + check(t, tc.key, object.MatchNumLT, "0", nil) + check(t, tc.key, object.MatchNumLT, tc.val1, nil) + check(t, tc.key, object.MatchNumLT, tc.val2, group1) + check(t, tc.key, object.MatchNumLT, "18446744073709551615", all) + check(t, tc.key, object.MatchNumLT, "115792089237316195423570985008687907853269984665640564039457584007913129639935", all) + check(t, tc.key, object.MatchNumLE, "-115792089237316195423570985008687907853269984665640564039457584007913129639935", nil) + check(t, tc.key, object.MatchNumLE, "0", nil) + check(t, tc.key, object.MatchNumLE, tc.val1, group1) + check(t, tc.key, object.MatchNumLE, tc.val2, all) + check(t, tc.key, object.MatchNumLE, "18446744073709551615", all) + check(t, tc.key, object.MatchNumLE, "115792089237316195423570985008687907853269984665640564039457584007913129639935", all) + check(t, tc.key, object.MatchNumGT, "-115792089237316195423570985008687907853269984665640564039457584007913129639935", all) + check(t, tc.key, object.MatchNumGT, "0", all) + check(t, tc.key, object.MatchNumGT, tc.val1, group2) + check(t, tc.key, object.MatchNumGT, tc.val2, nil) + check(t, tc.key, object.MatchNumGT, "18446744073709551615", nil) + check(t, tc.key, object.MatchNumGT, "115792089237316195423570985008687907853269984665640564039457584007913129639935", nil) + check(t, tc.key, object.MatchNumGE, "-115792089237316195423570985008687907853269984665640564039457584007913129639935", all) + check(t, tc.key, object.MatchNumGE, "0", all) + check(t, tc.key, object.MatchNumGE, tc.val1, all) + check(t, tc.key, object.MatchNumGE, tc.val2, group2) + check(t, tc.key, object.MatchNumGE, "18446744073709551615", nil) + check(t, tc.key, object.MatchNumGE, "115792089237316195423570985008687907853269984665640564039457584007913129639935", nil) }) } t.Run("mixed", func(t *testing.T) { @@ -1001,59 +1053,41 @@ func TestDB_SearchObjects(t *testing.T) { objs[i].SetPayloadChecksum(checksumtest.Checksum()) // Put requires require.NoError(t, db.Put(&objs[i], nil, nil)) } - check := func(tb testing.TB, m object.SearchMatchType, val string, inds ...int) { + fullRes := searchResultForIDs(ids) + for i := range vals { + fullRes[i].Attributes = []string{vals[i]} + } + check := func(t *testing.T, m object.SearchMatchType, val string, inds ...uint) { var fs object.SearchFilters fs.AddFilter(attr, val, m) - fInt, ok := PreprocessIntFilters(fs) - require.True(t, ok) - res, _, err := db.Search(cnr, fs, fInt, []string{attr}, nil, 1000) - require.NoError(t, err) - require.Len(t, res, len(inds)) - for i, ind := range inds { - require.Equal(t, ids[ind], res[i].ID, vals[i]) - } + assertSearchResultIndexes(t, db, cnr, fs, []string{attr}, fullRes, inds) } - all := make([]int, len(vals)) + all := make([]uint, len(vals)) for i := range vals { - all[i] = i + all[i] = uint(i) } t.Run("EQ", func(t *testing.T) { for i := range vals { - check(t, object.MatchStringEqual, vals[i], i) + check(t, object.MatchStringEqual, vals[i], uint(i)) } }) t.Run("NE", func(t *testing.T) { for i := range vals { - others := make([]int, 0, len(vals)-1) + others := make([]uint, 0, len(vals)-1) for j := range vals { if j != i { - others = append(others, j) + others = append(others, uint(j)) } } check(t, object.MatchStringNotEqual, vals[i], others...) } - t.Run("jump while not yet limit", func(t *testing.T) { - // iterator starts from int attribute key space, and once it is finished - jumps - // to the plain one. It must not forget to check plain attributes since there - // can be matching elements that should be returned according to the sorting. - var fs object.SearchFilters - fs.AddFilter(attr, "missing", object.MatchStringNotEqual) - fInt, ok := PreprocessIntFilters(fs) - require.True(t, ok) - res, cursor, err := db.Search(cnr, fs, fInt, []string{attr}, nil, 2) - require.NoError(t, err) - require.Len(t, res, 2) - require.Equal(t, ids[0], res[0].ID) - require.Equal(t, ids[1], res[1].ID) - require.NotEmpty(t, cursor) - }) t.Run("empty", func(t *testing.T) { check(t, object.MatchStringNotEqual, "", all...) }) }) t.Run("PREFIX", func(t *testing.T) { t.Run("negative", func(t *testing.T) { - check := func(t testing.TB, val string) { + check := func(t *testing.T, val string) { check(t, object.MatchCommonPrefix, val, 0, 1) } t.Run("no digits", func(t *testing.T) { check(t, "-") }) @@ -1168,19 +1202,7 @@ func TestDB_SearchObjects(t *testing.T) { for _, f := range tc.fs { fs.AddFilter(f.k, f.v, f.m) } - fInt, ok := PreprocessIntFilters(fs) - if !ok { - require.Empty(t, tc.is) - return - } - res, cursor, err := db.Search(cnr, fs, fInt, nil, nil, nAll) - require.NoError(t, err) - require.Empty(t, cursor) - require.Len(t, res, len(tc.is)) - for i, ind := range tc.is { - require.Equal(t, ids[ind], res[i].ID) - require.Empty(t, res[i].Attributes) - } + assertSearchResultIndexes(t, db, cnr, fs, nil, searchResultForIDs(ids[:]), tc.is) }) } }) @@ -1273,99 +1295,23 @@ func TestDB_SearchObjects(t *testing.T) { } { var fs object.SearchFilters set(&fs) - fInt, ok := PreprocessIntFilters(fs) - require.True(t, ok) - res, cursor, err := db.Search(cnr, fs, fInt, nil, nil, 1000) - require.NoError(t, err) - require.Empty(t, cursor) - require.Empty(t, res) + assertSearchResult(t, db, cnr, fs, nil, nil) } }) t.Run("all", func(t *testing.T) { t.Run("unfiltered", func(t *testing.T) { - res, cursor, err := db.Search(cnr, nil, nil, nil, nil, 1000) - require.NoError(t, err) - require.Empty(t, cursor) - require.Len(t, res, len(objs)) - for i := range res { - require.Equal(t, ids[i], res[i].ID) - require.Empty(t, res[i].Attributes) - } + assertSearchResult(t, db, cnr, nil, nil, searchResultForIDs(ids)) }) var fs object.SearchFilters fs.AddFilter(heightAttr, "0", object.MatchNumGE) t.Run("w/o attributes", func(t *testing.T) { - res, cursor, err := db.Search(cnr, fs, nil, nil, nil, 1000) - require.NoError(t, err) - require.Empty(t, cursor) - require.Len(t, res, len(heightSorted)) - for i := range res { - // expected order by IDs - require.Equal(t, ids[i], res[i].ID) - require.Empty(t, res[i].Attributes) - } - t.Run("paging", func(t *testing.T) { - res, cursor, err := db.Search(cnr, fs, nil, nil, nil, 2) - require.NoError(t, err) - require.Len(t, res, 2) - for i := range 2 { - require.Equal(t, ids[i], res[i].ID) - require.Empty(t, res[i].Attributes) - } - require.NotEmpty(t, cursor) - cursor.Key = slices.Concat([]byte{0}, cursor.Key) - //nolint:staticcheck // drop with t.Skip - res, cursor, err = db.Search(cnr, fs, nil, nil, cursor, 6) - require.NoError(t, err) - t.Skip("paging is broken when prim attribute is not requested, see also https://github.com/nspcc-dev/neofs-node/issues/3058#issuecomment-2553193094") - require.Len(t, res, 6) - for i := range 6 { - require.Equal(t, ids[2+i], res[i].ID, i) - require.Empty(t, res[i].Attributes) - } - require.NotEmpty(t, cursor) - cursor.Key = slices.Concat([]byte{0}, cursor.Key) - res, cursor, err = db.Search(cnr, fs, nil, nil, cursor, 3) - require.NoError(t, err) - require.Len(t, res, 2) - for i := range 2 { - require.Equal(t, ids[8+i], res[i].ID) - require.Empty(t, res[i].Attributes) - } - require.Empty(t, cursor) - }) + assertSearchResult(t, db, cnr, fs, nil, searchResultForIDs(ids)) }) t.Run("single attribute", func(t *testing.T) { - res, cursor, err := db.Search(cnr, fs, nil, []string{heightAttr}, nil, 1000) - require.NoError(t, err) - require.Empty(t, cursor) - require.Len(t, res, len(heightSorted)) - for i, r := range heightSorted { - require.Equal(t, r.ID, res[i].ID) - require.Equal(t, []string{r.Attributes[0]}, res[i].Attributes) - } + assertSearchResult(t, db, cnr, fs, []string{heightAttr}, heightSorted) }) t.Run("two attributes", func(t *testing.T) { - res, cursor, err := db.Search(cnr, fs, nil, []string{heightAttr, otherAttr}, nil, 1000) - require.NoError(t, err) - require.Empty(t, cursor) - require.Equal(t, heightSorted, res) - t.Run("paging", func(t *testing.T) { - res, cursor, err := db.Search(cnr, fs, nil, []string{heightAttr, otherAttr}, nil, 2) - require.NoError(t, err) - require.Equal(t, heightSorted[:2], res) - require.NotEmpty(t, cursor) - cursor.Key = slices.Concat([]byte{0}, cursor.Key) - res, cursor, err = db.Search(cnr, fs, nil, []string{heightAttr, otherAttr}, cursor, 6) - require.NoError(t, err) - require.Equal(t, heightSorted[2:8], res) - require.NotEmpty(t, cursor) - cursor.Key = slices.Concat([]byte{0}, cursor.Key) - res, cursor, err = db.Search(cnr, fs, nil, []string{heightAttr, otherAttr}, cursor, 3) - require.NoError(t, err) - require.Equal(t, heightSorted[8:], res) - require.Empty(t, cursor) - }) + assertSearchResult(t, db, cnr, fs, []string{heightAttr, otherAttr}, heightSorted) }) }) t.Run("partial", func(t *testing.T) { @@ -1375,86 +1321,13 @@ func TestDB_SearchObjects(t *testing.T) { heightSorted := heightSorted[2:9] ids := ids[1:8] t.Run("w/o attributes", func(t *testing.T) { - fInt, ok := PreprocessIntFilters(fs) - require.True(t, ok) - res, cursor, err := db.Search(cnr, fs, fInt, nil, nil, 1000) - require.NoError(t, err) - require.Empty(t, cursor) - require.Len(t, res, len(ids)) - for i := range res { - // expected order by IDs - require.Equal(t, ids[i], res[i].ID) - require.Empty(t, res[i].Attributes) - } - t.Run("paging", func(t *testing.T) { - fInt, ok := PreprocessIntFilters(fs) - require.True(t, ok) - res, cursor, err := db.Search(cnr, fs, fInt, nil, nil, 2) - require.NoError(t, err) - require.Len(t, res, 2) - for i := range 2 { - require.Equal(t, ids[i], res[i].ID) - require.Empty(t, res[i].Attributes) - } - require.NotEmpty(t, cursor) - cursor.Key = slices.Concat([]byte{0}, cursor.Key) - //nolint:staticcheck // drop with t.Skip - res, cursor, err = db.Search(cnr, fs, nil, nil, cursor, 6) - require.NoError(t, err) - t.Skip("paging is broken when prim attribute is not requested, see also https://github.com/nspcc-dev/neofs-node/issues/3058#issuecomment-2553193094") - require.Len(t, res, 6) - for i := range 6 { - require.Equal(t, ids[2+i], res[i].ID, i) - require.Empty(t, res[i].Attributes) - } - require.NotEmpty(t, cursor) - res, cursor, err = db.Search(cnr, fs, nil, nil, cursor, 3) - require.NoError(t, err) - require.Len(t, res, 2) - for i := range 2 { - require.Equal(t, ids[8+i], res[i].ID) - require.Empty(t, res[i].Attributes) - } - require.Empty(t, cursor) - }) + assertSearchResult(t, db, cnr, fs, nil, searchResultForIDs(ids)) }) t.Run("single attribute", func(t *testing.T) { - fInt, ok := PreprocessIntFilters(fs) - require.True(t, ok) - res, cursor, err := db.Search(cnr, fs, fInt, []string{heightAttr}, nil, 1000) - require.NoError(t, err) - require.Empty(t, cursor) - require.Len(t, res, len(heightSorted)) - for i, r := range heightSorted { - require.Equal(t, r.ID, res[i].ID) - require.Equal(t, []string{r.Attributes[0]}, res[i].Attributes) - } + assertSearchResult(t, db, cnr, fs, []string{heightAttr}, heightSorted) }) t.Run("two attributes", func(t *testing.T) { - fInt, ok := PreprocessIntFilters(fs) - require.True(t, ok) - res, cursor, err := db.Search(cnr, fs, fInt, []string{heightAttr, otherAttr}, nil, 1000) - require.NoError(t, err) - require.Empty(t, cursor) - require.Equal(t, heightSorted, res) - t.Run("paging", func(t *testing.T) { - fInt, ok := PreprocessIntFilters(fs) - require.True(t, ok) - res, cursor, err := db.Search(cnr, fs, fInt, []string{heightAttr, otherAttr}, nil, 2) - require.NoError(t, err) - require.Equal(t, heightSorted[:2], res) - require.NotEmpty(t, cursor) - cursor.Key = slices.Concat([]byte{0}, cursor.Key) - res, cursor, err = db.Search(cnr, fs, fInt, []string{heightAttr, otherAttr}, cursor, 3) - require.NoError(t, err) - require.Equal(t, heightSorted[2:5], res) - require.NotEmpty(t, cursor) - cursor.Key = slices.Concat([]byte{0}, cursor.Key) - res, cursor, err = db.Search(cnr, fs, fInt, []string{heightAttr, otherAttr}, cursor, 3) - require.NoError(t, err) - require.Equal(t, heightSorted[5:], res) - require.Empty(t, cursor) - }) + assertSearchResult(t, db, cnr, fs, []string{heightAttr, otherAttr}, heightSorted) }) }) }) @@ -1486,20 +1359,14 @@ func TestDB_SearchObjects(t *testing.T) { t.Run("none", func(t *testing.T) { var fs object.SearchFilters fs.AddFilter(object.AttributeFilePath, "cat4.jpg", object.MatchStringEqual) - res, cursor, err := db.Search(cnr, fs, nil, []string{object.AttributeFilePath, object.AttributeTimestamp}, nil, 1000) - require.NoError(t, err) - require.Empty(t, cursor) - require.Empty(t, res) + assertSearchResult(t, db, cnr, fs, []string{object.AttributeFilePath, object.AttributeTimestamp}, nil) }) t.Run("single", func(t *testing.T) { var fs object.SearchFilters fs.AddFilter(object.AttributeFilePath, "cat1.jpg", object.MatchStringEqual) - res, cursor, err := db.Search(cnr, fs, nil, []string{object.AttributeFilePath, object.AttributeTimestamp}, nil, 1000) - require.NoError(t, err) - require.Empty(t, cursor) - require.Equal(t, []client.SearchResultItem{ + assertSearchResult(t, db, cnr, fs, []string{object.AttributeFilePath, object.AttributeTimestamp}, []client.SearchResultItem{ {ID: ids[0], Attributes: []string{"cat1.jpg", "1738760790"}}, - }, res) + }) }) t.Run("multiple", func(t *testing.T) { t.Run("both attributes", func(t *testing.T) { @@ -1510,21 +1377,7 @@ func TestDB_SearchObjects(t *testing.T) { } var fs object.SearchFilters fs.AddFilter(object.AttributeFilePath, "cat2.jpg", object.MatchStringEqual) - res, cursor, err := db.Search(cnr, fs, nil, []string{object.AttributeFilePath, object.AttributeTimestamp}, nil, 1000) - require.NoError(t, err) - require.Equal(t, fullRes, res) - require.Empty(t, cursor) - t.Run("paging", func(t *testing.T) { - res, cursor, err := db.Search(cnr, fs, nil, []string{object.AttributeFilePath, object.AttributeTimestamp}, nil, 2) - require.NoError(t, err) - require.Equal(t, fullRes[:2], res) - require.NotEmpty(t, cursor) - cursor.Key = slices.Concat([]byte{0}, cursor.Key) - res, cursor, err = db.Search(cnr, fs, nil, []string{object.AttributeFilePath, object.AttributeTimestamp}, cursor, 1000) - require.NoError(t, err) - require.Equal(t, fullRes[2:], res) - require.Empty(t, cursor) - }) + assertSearchResult(t, db, cnr, fs, []string{object.AttributeFilePath, object.AttributeTimestamp}, fullRes) }) }) }) @@ -1567,53 +1420,33 @@ func TestDB_SearchObjects(t *testing.T) { var fs object.SearchFilters fs.AddFilter(object.AttributeFilePath, "/home/Downloads/", object.MatchCommonPrefix) fs.AddFilter("Type", "JPEG", object.MatchStringEqual) - res, cursor, err := db.Search(cnr, fs, nil, attrs, nil, 1000) - require.NoError(t, err) - require.Empty(t, cursor) - require.Equal(t, []client.SearchResultItem{ + assertSearchResult(t, db, cnr, fs, attrs, []client.SearchResultItem{ {ID: ids[1], Attributes: []string{"/home/Downloads/cat.jpg", "val1_3", "val2_3"}}, {ID: ids[3], Attributes: []string{"/home/Downloads/dog.jpg", "val1_1", "val2_1"}}, - }, res) + }) fs = fs[:0] fs.AddFilter(object.AttributeFilePath, "/usr", object.MatchCommonPrefix) fs.AddFilter("Type", "BIN", object.MatchStringEqual) - res, cursor, err = db.Search(cnr, fs, nil, attrs, nil, 1000) - require.NoError(t, err) - require.Empty(t, cursor) - require.Equal(t, []client.SearchResultItem{ + assertSearchResult(t, db, cnr, fs, attrs, []client.SearchResultItem{ {ID: ids[2], Attributes: []string{"/usr/local/bin/go", "val1_2", "val2_2"}}, - }, res) + }) fs = fs[:0] fs.AddFilter(object.AttributeFilePath, "/", object.MatchCommonPrefix) fs.AddFilter("Type", "BIN", object.MatchStringNotEqual) - res, cursor, err = db.Search(cnr, fs, nil, attrs, nil, 1000) - require.NoError(t, err) - require.Empty(t, cursor) - require.Equal(t, []client.SearchResultItem{ + assertSearchResult(t, db, cnr, fs, attrs, []client.SearchResultItem{ {ID: ids[1], Attributes: []string{"/home/Downloads/cat.jpg", "val1_3", "val2_3"}}, {ID: ids[3], Attributes: []string{"/home/Downloads/dog.jpg", "val1_1", "val2_1"}}, {ID: ids[0], Attributes: []string{"/var/log/neofs/node", "val1_4", "val2_4"}}, - }, res) + }) - t.Run("paging", func(t *testing.T) { - fs = fs[:0] - fs.AddFilter(object.AttributeFilePath, "/home/", object.MatchCommonPrefix) - fs.AddFilter("Type", "TEXT", object.MatchStringNotEqual) - res, cursor, err := db.Search(cnr, fs, nil, attrs, nil, 1) - require.NoError(t, err) - require.Equal(t, []client.SearchResultItem{ - {ID: ids[1], Attributes: []string{"/home/Downloads/cat.jpg", "val1_3", "val2_3"}}, - }, res) - require.NotEmpty(t, cursor) - cursor.Key = slices.Concat([]byte{0}, cursor.Key) - res, cursor, err = db.Search(cnr, fs, nil, attrs, cursor, 1) - require.NoError(t, err) - require.Equal(t, []client.SearchResultItem{ - {ID: ids[3], Attributes: []string{"/home/Downloads/dog.jpg", "val1_1", "val2_1"}}, - }, res) - require.Empty(t, cursor) + fs = fs[:0] + fs.AddFilter(object.AttributeFilePath, "/home/", object.MatchCommonPrefix) + fs.AddFilter("Type", "TEXT", object.MatchStringNotEqual) + assertSearchResult(t, db, cnr, fs, attrs, []client.SearchResultItem{ + {ID: ids[1], Attributes: []string{"/home/Downloads/cat.jpg", "val1_3", "val2_3"}}, + {ID: ids[3], Attributes: []string{"/home/Downloads/dog.jpg", "val1_1", "val2_1"}}, }) }) }) @@ -1640,24 +1473,10 @@ func TestDB_SearchObjects(t *testing.T) { require.NoError(t, db.Put(&objs[i], nil, nil)) } - check := func(t testing.TB, exp []oid.ID) { - check := func(res []client.SearchResultItem, err error) { - require.NoError(t, err) - require.Len(t, res, len(exp)) - for i := range exp { - require.Equal(t, exp[i], res[i].ID) - } - } - res, _, err := db.Search(cnr, nil, nil, nil, nil, 1000) - check(res, err) + check := func(t *testing.T, exp []oid.ID) { var fs object.SearchFilters fs.AddFilter(attrPlain, valPlain, object.MatchStringEqual) - res, _, err = db.Search(cnr, fs, nil, []string{attrPlain}, nil, 1000) - check(res, err) - fs = fs[:0] - fs.AddFilter(attrInt, valInt, object.MatchStringEqual) - res, _, err = db.Search(cnr, fs, nil, []string{attrInt}, nil, 1000) - check(res, err) + assertSearchResult(t, db, cnr, fs, nil, searchResultForIDs(exp)) } // all available check(t, ids)