diff --git a/pkg/local_object_storage/metabase/metadata.go b/pkg/local_object_storage/metabase/metadata.go index 61e865116a..49ecca614f 100644 --- a/pkg/local_object_storage/metabase/metadata.go +++ b/pkg/local_object_storage/metabase/metadata.go @@ -196,66 +196,178 @@ func deleteMetadata(tx *bbolt.Tx, cnr cid.ID, id oid.ID) error { return nil } +// ErrUnreachableQuery is returned when no object ever matches a particular +// search query. +var ErrUnreachableQuery = errors.New("unreachable query") + +var ( + errInvalidCursor = errors.New("invalid cursor") + errInvalidPrimaryFilter = errors.New("invalid primary filter") + errWrongPrimaryAttribute = errors.New("wrong primary attribute") + errWrongKeyValDelim = errors.New("wrong key-value delimiter") + errWrongValOIDDelim = errors.New("wrong value-OID delimiter") +) + // SearchCursor is a cursor used for continuous search in the DB. -type SearchCursor struct { - Key []byte // PREFIX_ATTR_DELIM_VAL_ID, prefix is unset - ValIDOff int -} +type SearchCursor struct{ primKeysPrefix, primSeekKey []byte } -// splits VAL_DELIM_OID. -func splitValOID(b []byte) ([]byte, []byte, error) { - if len(b) < attributeDelimiterLen+oid.Size+1 { // +1 because VAL cannot be empty - return nil, nil, fmt.Errorf("too short len %d", len(b)) - } - idOff := len(b) - oid.Size - valLn := idOff - attributeDelimiterLen - if !bytes.Equal(b[valLn:idOff], attributeDelimiter) { - return nil, nil, errors.New("no delimiter before OID") +var metaOIDPrefix = []byte{metaPrefixID} + +func decodeOIDFromCursor(cursor string) ([]byte, error) { + key := make([]byte, 1+base64.StdEncoding.DecodedLen(len(cursor))) + n, err := base64.StdEncoding.Decode(key[1:], []byte(cursor)) + if err != nil { + return nil, fmt.Errorf("decode Base64: %w", err) + } else if n != oid.Size { + return nil, fmt.Errorf("wrong len %d for listing query", n) } - return b[:valLn], b[idOff:], nil + key[0] = metaPrefixID + return key[:1+n], nil } -// NewSearchCursorFromString decodes cursor from the string according to the -// primary attribute. -func NewSearchCursorFromString(s, primAttr string, isInt bool) (*SearchCursor, error) { - if s == "" { - return nil, nil - } - b := make([]byte, 1+base64.StdEncoding.DecodedLen(len(s))) - n, err := base64.StdEncoding.Decode(b[1:], []byte(s)) - if err != nil { - return nil, fmt.Errorf("decode cursor from Base64: %w", err) +// PreprocessSearchQuery accepts verified search filters, requested attributes +// along and continuation cursor, verifies the cursor and returns additional +// arguments to pass into [DB.Search]. If the query is valid but unreachable, +// [ErrUnreachableQuery] is returned. +func PreprocessSearchQuery(fs object.SearchFilters, attrs []string, cursor string) (*SearchCursor, map[int]ParsedIntFilter, error) { + if len(fs) == 0 { + if cursor != "" { + primSeekKey, err := decodeOIDFromCursor(cursor) + if err != nil { + return nil, nil, fmt.Errorf("%w: %w", errInvalidCursor, err) + } + return &SearchCursor{primKeysPrefix: metaOIDPrefix, primSeekKey: primSeekKey}, nil, nil + } + return &SearchCursor{primKeysPrefix: metaOIDPrefix, primSeekKey: metaOIDPrefix}, nil, nil } - b = b[:1+n] - if primAttr == "" { - if n != oid.Size { - return nil, fmt.Errorf("wrong OID cursor len %d", n) + + primMatcher, primVal := convertFilterValue(fs[0]) + oidSorted := len(attrs) == 0 || primMatcher == object.MatchNotPresent + var primValDB []byte + if !oidSorted && cursor == "" && primMatcher != object.MatchStringNotEqual && !objectcore.IsIntegerSearchOp(primMatcher) { + switch attr := fs[0].Header(); attr { + default: + primValDB = []byte(primVal) + case object.FilterOwnerID, object.FilterFirstSplitObject, object.FilterParentID: + var err error + if primValDB, err = base58.Decode(primVal); err != nil { + return nil, nil, fmt.Errorf("%w: decode %q attribute value from Base58: %w", errInvalidPrimaryFilter, attr, err) + } + case object.FilterPayloadChecksum, object.FilterPayloadHomomorphicHash: + var err error + if primValDB, err = hex.DecodeString(primVal); err != nil { + return nil, nil, fmt.Errorf("%w: decode %q attribute value from HEX: %w", errInvalidPrimaryFilter, attr, err) + } + case object.FilterSplitID: + uid, err := uuid.Parse(primVal) + if err != nil { + return nil, nil, fmt.Errorf("%w: code %q UUID attribute: %w", errInvalidPrimaryFilter, attr, err) + } + primValDB = uid[:] } - return &SearchCursor{Key: b}, nil } - if n > object.MaxHeaderLen { - return nil, fmt.Errorf("cursor len %d exceeds the limit %d", n, object.MaxHeaderLen) + + var primKeysPrefix, primSeekKey []byte + if oidSorted { + if cursor != "" { + var err error + if primSeekKey, err = decodeOIDFromCursor(cursor); err != nil { + return nil, nil, fmt.Errorf("%w: %w", errInvalidCursor, err) + } + } + } else if cursor != "" { + // TODO: wrap into "invalid cursor" error + primSeekKey = make([]byte, 1+base64.StdEncoding.DecodedLen(len(cursor))) + n, err := base64.StdEncoding.Decode(primSeekKey[1:], []byte(cursor)) + if err != nil { + return nil, nil, fmt.Errorf("%w: decode Base64: %w", errInvalidCursor, err) + } + if n > object.MaxHeaderLen { + return nil, nil, fmt.Errorf("%w: len %d exceeds the limit %d", errInvalidCursor, n, object.MaxHeaderLen) + } + primSeekKey = primSeekKey[:1+n] + if objectcore.IsIntegerSearchOp(primMatcher) { + if n != len(attrs[0])+attributeDelimiterLen+intValLen+oid.Size { + return nil, nil, fmt.Errorf("%w: wrong len %d for int query", errInvalidCursor, n) + } + primKeysPrefix = primSeekKey[:1+len(attrs[0])+attributeDelimiterLen] + if !bytes.Equal(primKeysPrefix[1:1+len(attrs[0])], []byte(attrs[0])) { + return nil, nil, fmt.Errorf("%w: %w", errInvalidCursor, errWrongPrimaryAttribute) + } + if !bytes.Equal(primKeysPrefix[1+len(attrs[0]):], attributeDelimiter) { + return nil, nil, fmt.Errorf("%w: %w", errInvalidCursor, errWrongKeyValDelim) + } + if primSeekKey[len(primKeysPrefix)] > 1 { + return nil, nil, fmt.Errorf("%w: invalid sign byte 0x%02X", errInvalidCursor, primSeekKey[len(primKeysPrefix)]) + } + } else { + if n < len(attrs[0])+attributeDelimiterLen+1+attributeDelimiterLen+oid.Size { // +1 because VAL cannot be empty + return nil, nil, fmt.Errorf("%w: too short len %d", errInvalidCursor, n) + } + primKeysPrefix = primSeekKey[:1+len(attrs[0])+attributeDelimiterLen] + if !bytes.Equal(primKeysPrefix[1:1+len(attrs[0])], []byte(attrs[0])) { + return nil, nil, fmt.Errorf("%w: %w", errInvalidCursor, errWrongPrimaryAttribute) + } + if !bytes.Equal(primKeysPrefix[1+len(attrs[0]):], attributeDelimiter) { + return nil, nil, fmt.Errorf("%w: %w", errInvalidCursor, errWrongKeyValDelim) + } + if !bytes.Equal(primSeekKey[len(primSeekKey)-oid.Size-attributeDelimiterLen:][:attributeDelimiterLen], attributeDelimiter) { + return nil, nil, fmt.Errorf("%w: %w", errInvalidCursor, errWrongValOIDDelim) + } + } } - ind := bytes.Index(b[1:], attributeDelimiter) // 1st is prefix - if ind < 0 { - return nil, errors.New("missing delimiter") + + if blindlyProcess(fs) { + return nil, nil, ErrUnreachableQuery } - if !bytes.Equal(b[1:1+ind], []byte(primAttr)) { - return nil, errors.New("wrong attribute") + fInt, ok := parseIntFilters(fs) + if !ok { + return nil, nil, ErrUnreachableQuery } - var res SearchCursor - res.ValIDOff = 1 + len(primAttr) + len(attributeDelimiter) - if isInt { - if len(b[res.ValIDOff:]) <= oid.Size { - return nil, errors.New("missing value") + + if oidSorted { + if cursor == "" { + primSeekKey = metaOIDPrefix } + primKeysPrefix = metaOIDPrefix } else { - if _, _, err = splitValOID(b[res.ValIDOff:]); err != nil { - return nil, fmt.Errorf("invalid VAL_OID: %w", err) + if cursor != "" { + if objectcore.IsIntegerSearchOp(primMatcher) { + primSeekKey[0] = metaPrefixAttrIDInt // fins primKeysPrefix also + } else { + primSeekKey[0] = metaPrefixAttrIDPlain // fins primKeysPrefix also + } + } else { + if objectcore.IsIntegerSearchOp(primMatcher) { + f := fInt[0] + if !f.auto && (primMatcher == object.MatchNumGE || primMatcher == object.MatchNumGT) { + primSeekKey = slices.Concat([]byte{metaPrefixAttrIDInt}, []byte(attrs[0]), attributeDelimiter, f.b) + primKeysPrefix = primSeekKey[:1+len(attrs[0])+attributeDelimiterLen] + } else { + primSeekKey = slices.Concat([]byte{metaPrefixAttrIDInt}, []byte(attrs[0]), attributeDelimiter) + primKeysPrefix = primSeekKey + } + } else { + // according to the condition above, primValDB is empty for '!=' matcher as it should be + primSeekKey = slices.Concat([]byte{metaPrefixAttrIDPlain}, []byte(attrs[0]), attributeDelimiter, primValDB) + primKeysPrefix = primSeekKey[:1+len(attrs[0])+attributeDelimiterLen] + } } } - res.Key = b - return &res, nil + return &SearchCursor{primKeysPrefix: primKeysPrefix, primSeekKey: primSeekKey}, fInt, nil +} + +// splits VAL_DELIM_OID. +func splitValOID(b []byte) ([]byte, []byte, error) { + if len(b) < attributeDelimiterLen+oid.Size+1 { // +1 because VAL cannot be empty + return nil, nil, fmt.Errorf("too short len %d", len(b)) + } + idOff := len(b) - oid.Size + valLn := idOff - attributeDelimiterLen + if !bytes.Equal(b[valLn:idOff], attributeDelimiter) { + return nil, nil, errWrongValOIDDelim + } + return b[:valLn], b[idOff:], nil } // ParsedIntFilter is returned by [PreprocessIntFilters] to pass into the @@ -269,9 +381,6 @@ type ParsedIntFilter struct { // Search selects up to count container's objects from the given container // matching the specified filters. func (db *DB) Search(cnr cid.ID, fs object.SearchFilters, fInt map[int]ParsedIntFilter, attrs []string, cursor *SearchCursor, count uint16) ([]client.SearchResultItem, []byte, error) { - if blindlyProcess(fs) { - return nil, nil, nil - } var res []client.SearchResultItem var newCursor []byte var err error @@ -305,69 +414,19 @@ func (db *DB) searchTx(tx *bbolt.Tx, cnr cid.ID, fs object.SearchFilters, fInt m if metaBkt == nil { return nil, nil, nil } - // TODO: make as much as possible outside the Bolt tx - primMatcher, primVal := convertFilterValue(fs[0]) - intPrimMatcher := objectcore.IsIntegerSearchOp(primMatcher) - notPresentPrimMatcher := primMatcher == object.MatchNotPresent - idIter := len(attrs) == 0 || notPresentPrimMatcher - primAttr := fs[0].Header() // attribute emptiness already prevented - var primSeekKey, primSeekPrefix []byte - if idIter { - if cursor != nil { - primSeekKey = cursor.Key - } else { - primSeekKey = make([]byte, 1) - } - primSeekKey[0] = metaPrefixID - primSeekPrefix = primSeekKey[:1] - } else if cursor != nil { - primSeekKey = cursor.Key - if intPrimMatcher { - primSeekKey[0] = metaPrefixAttrIDInt - } else { - primSeekKey[0] = metaPrefixAttrIDPlain - } - primSeekPrefix = primSeekKey[:cursor.ValIDOff] - } else { - if intPrimMatcher { - // we seek 0x01_ATTR_DELIM_VAL either w/ or w/o VAL. We ignore VAL when we need - // to start from the lowest int, i.e. filter is auto-matched (e.g. <=MaxUint256) or <(=). - f := fInt[0] - withVal := !f.auto && (primMatcher == object.MatchNumGT || primMatcher == object.MatchNumGE) - kln := 1 + len(primAttr) + attributeDelimiterLen // prefix 1st - if withVal { - kln += intValLen - } - primSeekKey = make([]byte, kln) - primSeekKey[0] = metaPrefixAttrIDInt - off := 1 + copy(primSeekKey[1:], primAttr) - off += copy(primSeekKey[off:], attributeDelimiter) - primSeekPrefix = primSeekKey[:off] - if withVal { - copy(primSeekKey[off:], f.b) - } - } else { - var err error - if primMatcher != object.MatchStringNotEqual { - primSeekKey, primSeekPrefix, err = seekKeyForAttribute(primAttr, primVal) - } else { - primSeekKey, primSeekPrefix, err = seekKeyForAttribute(primAttr, "") - } - if err != nil { - return nil, nil, fmt.Errorf("invalid primary filter value: %w", err) - } - } - } primCursor := metaBkt.Cursor() - primKey, _ := primCursor.Seek(primSeekKey) - if bytes.Equal(primKey, primSeekKey) { // points to the last response element, so go next + primKey, _ := primCursor.Seek(cursor.primSeekKey) + if bytes.Equal(primKey, cursor.primSeekKey) { // points to the last response element, so go next primKey, _ = primCursor.Next() } if primKey == nil { return nil, nil, nil } + primMatcher, _ := convertFilterValue(fs[0]) + intPrimMatcher := objectcore.IsIntegerSearchOp(primMatcher) + idIter := len(attrs) == 0 || primMatcher == object.MatchNotPresent res := make([]client.SearchResultItem, count) var lastMatchedPrimKey []byte var n uint16 @@ -378,13 +437,13 @@ func (db *DB) searchTx(tx *bbolt.Tx, cnr cid.ID, fs object.SearchFilters, fInt m curEpoch := db.epochState.CurrentEpoch() dbValInt := new(big.Int) nextPrimKey: - for ; bytes.HasPrefix(primKey, primSeekPrefix); primKey, _ = primCursor.Next() { + for ; bytes.HasPrefix(primKey, cursor.primKeysPrefix); primKey, _ = primCursor.Next() { if idIter { if id = primKey[1:]; len(id) != oid.Size { return nil, nil, invalidMetaBucketKeyErr(primKey, fmt.Errorf("invalid OID len %d", len(id))) } } else { // apply primary filter - valID := primKey[len(primSeekPrefix):] // VAL_OID + valID := primKey[len(cursor.primKeysPrefix):] // VAL_OID if intPrimMatcher { if len(valID) <= oid.Size { return nil, nil, invalidMetaBucketKeyErr(primKey, fmt.Errorf("too small VAL_OID len %d", len(valID))) @@ -400,7 +459,7 @@ nextPrimKey: // there may be several filters by primary key, e.g. N >= 10 && N <= 20. We // check them immediately before moving through the DB. attr := fs[i].Header() - if i > 0 && attr != primAttr { + if i > 0 && attr != fs[0].Header() { continue } mch, val := convertFilterValue(fs[i]) @@ -519,13 +578,6 @@ nextPrimKey: // TODO: can be merged with filtered code? func (db *DB) searchUnfiltered(cnr cid.ID, cursor *SearchCursor, count uint16) ([]client.SearchResultItem, []byte, error) { - var seekKey []byte - if cursor != nil { - seekKey = cursor.Key - seekKey[0] = metaPrefixID - } else { - seekKey = []byte{metaPrefixID} - } res := make([]client.SearchResultItem, count) var n uint16 var newCursor []byte @@ -537,8 +589,8 @@ func (db *DB) searchUnfiltered(cnr cid.ID, cursor *SearchCursor, count uint16) ( } mbc := mb.Cursor() - k, _ := mbc.Seek(seekKey) - if cursor != nil && bytes.Equal(k, seekKey) { // cursor is the last response element, so go next + k, _ := mbc.Seek(cursor.primSeekKey) + if cursor != nil && bytes.Equal(k, cursor.primSeekKey) { // cursor is the last response element, so go next k, _ = mbc.Next() } for ; k[0] == metaPrefixID; k, _ = mbc.Next() { @@ -563,41 +615,6 @@ func (db *DB) searchUnfiltered(cnr cid.ID, cursor *SearchCursor, count uint16) ( return res[:n], newCursor, nil } -func seekKeyForAttribute(attr, fltVal string) ([]byte, []byte, error) { - key := make([]byte, 1+len(attr)+attributeDelimiterLen+len(fltVal)) // prefix 1st - key[0] = metaPrefixAttrIDPlain - off := 1 + copy(key[1:], attr) - off += copy(key[off:], attributeDelimiter) - prefix := key[:off] - if fltVal == "" { - return key, prefix, nil - } - var dbVal []byte - switch attr { - default: - dbVal = []byte(fltVal) - case object.FilterOwnerID, object.FilterFirstSplitObject, object.FilterParentID: - var err error - if dbVal, err = base58.Decode(fltVal); err != nil { - return nil, nil, fmt.Errorf("decode %q attribute value from Base58: %w", attr, err) - } - case object.FilterPayloadChecksum, object.FilterPayloadHomomorphicHash: - var err error - if dbVal, err = hex.DecodeString(fltVal); err != nil { - return nil, nil, fmt.Errorf("decode %q attribute value from HEX: %w", attr, err) - } - case object.FilterSplitID: - uid, err := uuid.Parse(fltVal) - if err != nil { - return nil, nil, fmt.Errorf("decode %q UUID attribute: %w", attr, err) - } - dbVal = uid[:] - case object.FilterVersion, object.FilterType, object.FilterRoot, object.FilterPhysical: - } - copy(key[off:], dbVal) - return key, prefix, nil -} - // combines attribute's DB and NeoFS API SearchV2 values to the matchable // format. Returns DB errors only. func combineValues(attr string, dbVal []byte, fltVal string) ([]byte, []byte, error) { @@ -927,10 +944,7 @@ func CalculateCursor(fs object.SearchFilters, lastItem client.SearchResultItem) return res, nil } -// PreprocessIntFilters checks whether any object can match numeric filters from -// the given set, and returns false if not. Otherwise, it returns map of decoded -// values. -func PreprocessIntFilters(fs object.SearchFilters) (map[int]ParsedIntFilter, bool) { +func parseIntFilters(fs object.SearchFilters) (map[int]ParsedIntFilter, bool) { fInt := make(map[int]ParsedIntFilter, len(fs)) // number of filters is limited by pretty small value, so we can afford it for i := range fs { m, val := convertFilterValue(fs[i]) diff --git a/pkg/local_object_storage/metabase/metadata_test.go b/pkg/local_object_storage/metabase/metadata_test.go index 41178478ef..ab2b0d8e0b 100644 --- a/pkg/local_object_storage/metabase/metadata_test.go +++ b/pkg/local_object_storage/metabase/metadata_test.go @@ -11,7 +11,6 @@ import ( "strconv" "testing" - objectcore "github.com/nspcc-dev/neofs-node/pkg/core/object" "github.com/nspcc-dev/neofs-sdk-go/checksum" checksumtest "github.com/nspcc-dev/neofs-sdk-go/checksum/test" "github.com/nspcc-dev/neofs-sdk-go/client" @@ -366,61 +365,170 @@ func TestIntBucketOrder(t *testing.T) { }, collected) } -func TestNewSearchCursorFromString(t *testing.T) { - t.Run("empty", func(t *testing.T) { - res, err := NewSearchCursorFromString("", "any", false) - require.NoError(t, err) - require.Nil(t, res) +func assertInvalidCursorErr(t *testing.T, fs object.SearchFilters, attrs []string, cursor, msg string) { + _, _, err := PreprocessSearchQuery(fs, attrs, cursor) + require.ErrorIs(t, err, errInvalidCursor) + require.EqualError(t, err, errInvalidCursor.Error()+": "+msg) +} + +func assertCursor(t *testing.T, fs object.SearchFilters, attrs []string, cursor string, expPrefix, expKey []byte) { + c, _, err := PreprocessSearchQuery(fs, attrs, cursor) + require.NoError(t, err) + require.NotNil(t, c) + require.Equal(t, expPrefix, c.primKeysPrefix) + require.Equal(t, expKey, c.primSeekKey) +} + +var invalidListingCursorTestcases = []struct{ name, err, cursor string }{ + {name: "not a Base64", err: "decode Base64: illegal base64 data at input byte 0", cursor: "???"}, + {name: "undersize", err: "wrong len 31 for listing query", cursor: "q/WZCxCa19Y5lnEkCl/eL3TuEQdRmEtItzOe8TdsJA=="}, + {name: "oversize", err: "wrong len 33 for listing query", cursor: "ebTksjW7LcatKlCnNIiqQXyhZKdD2iMvcDsYSokVYyYB"}, +} + +// for 'attr: 123' last result. +var invalidIntCursorTestcases = []struct{ name, err, cursor string }{ + {name: "not a Base64", err: "decode Base64: illegal base64 data at input byte 0", cursor: "???"}, + {name: "undersize", err: "wrong len 69 for int query", + cursor: "YXR0cgABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHtK9i0PSRtkhlwPKwf9Zq0nzwbbzlJYFufLmRJyRPPI"}, + {name: "oversize", err: "wrong len 71 for int query", + cursor: "YXR0cgABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHt8duMfXsAHeMC6T9hwwUvq/LP7HQ0ovGK8PSK2cddFGAA="}, + {name: "other primary attribute", err: "wrong primary attribute", + cursor: "YnR0cgABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHvAZL3wws1klEuoU+mT625g8fNHuSjTyDL/leSvB2hNOA=="}, + {name: "wrong delimiter", err: "wrong key-value delimiter", + cursor: "YXR0cgEBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHvgYmA9Vxu0yP68nUexGmMRce5YyV/7EQ3g5jjj7ELcRg=="}, + {name: "invalid sign", err: "invalid sign byte 0xFF", + cursor: "YXR0cgD/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHv3+vwsBFrKukaYtBo1r7SCNPeLBr1d+4RDR9viyyRiZw=="}, +} + +// for 'attr: hello' last result. +var invalidNonIntCursorTestcases = []struct{ name, err, cursor string }{ + {name: "not a Base64", err: "decode Base64: illegal base64 data at input byte 0", cursor: "???"}, + {name: "no value", err: "too short len 38", + cursor: "YnR0cgAAR7tSRzMOSbFjFs5YvSPr3V6Ps8hmv+GdwAt3PMmVnYs="}, + {name: "other primary attribute", err: "wrong primary attribute", + cursor: "YnR0cgBoZWxsbwC5XU/eTk5N+i+RuLa4XQ4lcFd3wqN0LFye13unXZ2SBA=="}, + {name: "wrong key-value delimiter", err: "wrong key-value delimiter", + cursor: "YXR0cv9oZWxsbwD3kqge4Gmjjus4zLTKQs4gxxbRD4pK1N5Lu6NQuJ43UQ=="}, + {name: "wrong value-OID delimiter", err: "wrong value-OID delimiter", + cursor: "YXR0cgBoZWxsb/+IlGDk7Bu+PC410JNSmNyajZ0lphLjqtgWDLyNn5Gh4w=="}, +} + +func TestPreprocessSearchQuery_Cursors(t *testing.T) { + t.Run("listing", func(t *testing.T) { + test := func(t *testing.T, fs object.SearchFilters, attrs []string) { + t.Run("initial", func(t *testing.T) { + assertCursor(t, fs, attrs, "", []byte{0x00}, []byte{0x00}) + }) + t.Run("invalid cursor", func(t *testing.T) { + for _, tc := range invalidListingCursorTestcases { + t.Run(tc.name, func(t *testing.T) { assertInvalidCursorErr(t, fs, attrs, tc.cursor, tc.err) }) + } + }) + id := oidtest.ID() + assertCursor(t, fs, attrs, base64.StdEncoding.EncodeToString(id[:]), []byte{0x00}, slices.Concat([]byte{0x00}, id[:])) + } + t.Run("unfiltered", func(t *testing.T) { test(t, nil, nil) }) + t.Run("w/o attributes", func(t *testing.T) { + var fs object.SearchFilters + fs.AddFilter("attr", "val", object.MatchStringNotEqual) + test(t, fs, nil) + }) + t.Run("filter no attribute", func(t *testing.T) { + var fs object.SearchFilters + fs.AddFilter("attr", "", object.MatchNotPresent) + test(t, fs, []string{"attr"}) + }) }) - t.Run("not a Base64", func(t *testing.T) { - _, err := NewSearchCursorFromString("???", "any", false) - require.ErrorContains(t, err, "decode cursor from Base64") + t.Run("int", func(t *testing.T) { + t.Run("initial", func(t *testing.T) { + for _, op := range []object.SearchMatchType{object.MatchNumGT, object.MatchNumGE, object.MatchNumLT, object.MatchNumLE} { + t.Run(op.String(), func(t *testing.T) { + var fs object.SearchFilters + fs.AddFilter("attr", "123", op) + pref := slices.Concat([]byte{0x01}, []byte("attr"), []byte{0x00}) + if op == object.MatchNumGT || op == object.MatchNumGE { + assertCursor(t, fs, []string{"attr"}, "", pref, slices.Concat(pref, intBytes(big.NewInt(123)))) + } else { + assertCursor(t, fs, []string{"attr"}, "", pref, pref) + } + }) + } + }) + t.Run("invalid cursor", func(t *testing.T) { + var fs object.SearchFilters + fs.AddFilter("attr", "123", object.MatchNumGT) + for _, tc := range invalidIntCursorTestcases { + t.Run(tc.name, func(t *testing.T) { assertInvalidCursorErr(t, fs, []string{"attr"}, tc.cursor, tc.err) }) + } + t.Run("header overflow", func(t *testing.T) { + b := make([]byte, object.MaxHeaderLen+1) + rand.Read(b) //nolint:staticcheck + assertInvalidCursorErr(t, fs, []string{"attr"}, base64.StdEncoding.EncodeToString(b), "len 16385 exceeds the limit 16384") + }) + }) + id := oidtest.ID() + for _, n := range []*big.Int{ + maxUint256Neg, big.NewInt(math.MinInt64), big.NewInt(-1), big.NewInt(0), + big.NewInt(1), big.NewInt(math.MaxInt64), maxUint256, + } { + ib := intBytes(n) + b := slices.Concat([]byte("attr"), []byte{0x00}, ib, id[:]) + var fs object.SearchFilters + fs.AddFilter("attr", n.String(), object.MatchNumGE) + + c, fInt, err := PreprocessSearchQuery(fs, []string{"attr"}, base64.StdEncoding.EncodeToString(b)) + require.NoError(t, err) + require.NotNil(t, c) + + pref := slices.Concat([]byte{0x01}, []byte("attr"), []byte{0x00}) + require.Equal(t, pref, c.primKeysPrefix) + + require.Len(t, fInt, 1) + f, ok := fInt[0] + require.True(t, ok) + if n.Cmp(maxUint256Neg) == 0 { + require.Equal(t, ParsedIntFilter{auto: true}, f) + } else { + require.Equal(t, ParsedIntFilter{n: n, b: ib}, f) + } + } }) - t.Run("no attribute", func(t *testing.T) { - t.Run("undersize", func(t *testing.T) { - _, err := NewSearchCursorFromString("q/WZCxCa19Y5lnEkCl/eL3TuEQdRmEtItzOe8TdsJA==", "", false) - require.EqualError(t, err, "wrong OID cursor len 31") + t.Run("non-int", func(t *testing.T) { + t.Run("initial", func(t *testing.T) { + for _, op := range []object.SearchMatchType{object.MatchStringEqual, object.MatchStringNotEqual, object.MatchCommonPrefix} { + t.Run(op.String(), func(t *testing.T) { + var fs object.SearchFilters + fs.AddFilter("attr", "hello", op) + pref := slices.Concat([]byte{0x02}, []byte("attr"), []byte{0x00}) + key := pref + if op != object.MatchStringNotEqual { + key = slices.Concat(pref, []byte("hello")) + } + assertCursor(t, fs, []string{"attr"}, "", pref, key) + }) + } }) - t.Run("oversize", func(t *testing.T) { - _, err := NewSearchCursorFromString("ebTksjW7LcatKlCnNIiqQXyhZKdD2iMvcDsYSokVYyYB", "", false) - require.EqualError(t, err, "wrong OID cursor len 33") + var fs object.SearchFilters + fs.AddFilter("attr", "hello", object.MatchStringEqual) + t.Run("invalid cursor", func(t *testing.T) { + for _, tc := range invalidNonIntCursorTestcases { + t.Run(tc.name, func(t *testing.T) { assertInvalidCursorErr(t, fs, []string{"attr"}, tc.cursor, tc.err) }) + } + t.Run("header overflow", func(t *testing.T) { + b := make([]byte, object.MaxHeaderLen+1) + rand.Read(b) //nolint:staticcheck + assertInvalidCursorErr(t, fs, []string{"attr"}, base64.StdEncoding.EncodeToString(b), "len 16385 exceeds the limit 16384") + }) }) id := oidtest.ID() - res, err := NewSearchCursorFromString(base64.StdEncoding.EncodeToString(id[:]), "", false) + pref := slices.Concat([]byte("attr"), []byte{0x00}) + b := slices.Concat(pref, []byte("hello"), []byte{0x00}, id[:]) + c, fInt, err := PreprocessSearchQuery(fs, []string{"attr"}, base64.StdEncoding.EncodeToString(b)) require.NoError(t, err) - require.NotEmpty(t, res) - require.Equal(t, id[:], res.Key[1:]) - require.Zero(t, res.ValIDOff) - }) - const attr = "any_attr" - t.Run("header overflow", func(t *testing.T) { - b := make([]byte, object.MaxHeaderLen+1) - rand.Read(b) //nolint:staticcheck - _, err := NewSearchCursorFromString(base64.StdEncoding.EncodeToString(b), attr, false) - require.EqualError(t, err, "cursor len 16385 exceeds the limit 16384") - }) - t.Run("no delimiter", func(t *testing.T) { - _, err := NewSearchCursorFromString(base64.StdEncoding.EncodeToString([]byte(attr)), attr, false) - require.EqualError(t, err, "missing delimiter") + require.Empty(t, fInt) + require.Equal(t, slices.Concat([]byte{0x02}, pref), c.primKeysPrefix) + require.Equal(t, slices.Concat([]byte{0x02}, b), c.primSeekKey) }) - t.Run("wrong attribute", func(t *testing.T) { - b := slices.Concat([]byte(attr+"other"), attributeDelimiter) - _, err := NewSearchCursorFromString(base64.StdEncoding.EncodeToString(b), attr, false) - require.EqualError(t, err, "wrong attribute") - }) - t.Run("no value and OID", func(t *testing.T) { - b := slices.Concat([]byte(attr), attributeDelimiter) - _, err := NewSearchCursorFromString(base64.StdEncoding.EncodeToString(b), attr, false) - require.EqualError(t, err, "invalid VAL_OID: too short len 0") - }) - const val = "any_val" - id := oidtest.ID() - b := slices.Concat([]byte(attr), attributeDelimiter, []byte(val), attributeDelimiter, id[:]) - res, err := NewSearchCursorFromString(base64.StdEncoding.EncodeToString(b), attr, false) - require.NoError(t, err) - require.NotEmpty(t, res.Key) - require.Equal(t, b, res.Key[1:]) - require.Equal(t, 1+len(attr)+attributeDelimiterLen, res.ValIDOff) } func cloneIntFilterMap(src map[int]ParsedIntFilter) map[int]ParsedIntFilter { @@ -442,32 +550,33 @@ func cloneIntFilterMap(src map[int]ParsedIntFilter) map[int]ParsedIntFilter { return dst } -func _assertSearchResultWithLimit(t testing.TB, db *DB, cnr cid.ID, fs object.SearchFilters, attrs []string, all []client.SearchResultItem, lim uint16) { - fInt, ok := PreprocessIntFilters(fs) - if !ok { - require.Empty(t, all) - return - } - fIntClone := cloneIntFilterMap(fInt) - - var primAttr string - var primInt bool - if len(attrs) > 0 { - require.NotEmpty(t, fs) - require.Equal(t, attrs[0], fs[0].Header()) - if fs[0].Operation() != object.MatchNotPresent { - primAttr, primInt = attrs[0], objectcore.IsIntegerSearchOp(fs[0].Operation()) - } +func cloneSearchCursor(c *SearchCursor) *SearchCursor { + if c == nil { + return nil } + return &SearchCursor{primKeysPrefix: slices.Clone(c.primKeysPrefix), primSeekKey: slices.Clone(c.primSeekKey)} +} +func _assertSearchResultWithLimit(t testing.TB, db *DB, cnr cid.ID, fs object.SearchFilters, attrs []string, all []client.SearchResultItem, lim uint16) { var strCursor string - var cursor *SearchCursor nAttr := len(attrs) for { - res, c, err := db.Search(cnr, fs, fInt, attrs, cursor, lim) - if len(fInt) > 0 { - require.Equal(t, fIntClone, fInt, "int filter map mutation detected", "cursor: %q", strCursor) + cursor, fInt, err := PreprocessSearchQuery(fs, attrs, strCursor) + if err != nil { + if len(all) == 0 { + require.ErrorIs(t, err, ErrUnreachableQuery) + } else { + require.NoError(t, err) + } + return } + + cursorClone := cloneSearchCursor(cursor) + fIntClone := cloneIntFilterMap(fInt) + + res, c, err := db.Search(cnr, fs, fInt, attrs, cursor, lim) + require.Equal(t, cursorClone, cursor, "cursor mutation detected", "cursor: %q", strCursor) + require.Equal(t, fIntClone, fInt, "int filter map mutation detected", "cursor: %q", strCursor) require.NoError(t, err, "cursor: %q", strCursor) n := min(len(all), int(lim)) @@ -491,8 +600,6 @@ func _assertSearchResultWithLimit(t testing.TB, db *DB, cnr cid.ID, fs object.Se require.Equal(t, c, cc, "cursor: %q", strCursor) strCursor = base64.StdEncoding.EncodeToString(c) - cursor, err = NewSearchCursorFromString(strCursor, primAttr, primInt) - require.NoErrorf(t, err, "cursor: %q", strCursor) } } @@ -573,7 +680,10 @@ func TestDB_SearchObjects(t *testing.T) { }) require.NoError(t, err) - _, _, err = db.Search(cnr, nil, nil, nil, nil, n) + cursor, fInt, err := PreprocessSearchQuery(nil, nil, "") + require.NoError(t, err) + + _, _, err = db.Search(cnr, nil, fInt, nil, cursor, n) require.EqualError(t, err, "view BoltDB: invalid meta bucket key (prefix 0x0): unexpected object key len 32") }) }) diff --git a/pkg/local_object_storage/metabase/version_test.go b/pkg/local_object_storage/metabase/version_test.go index d733a81177..896bd9e2f3 100644 --- a/pkg/local_object_storage/metabase/version_test.go +++ b/pkg/local_object_storage/metabase/version_test.go @@ -18,7 +18,6 @@ import ( "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode" "github.com/nspcc-dev/neofs-sdk-go/checksum" checksumtest "github.com/nspcc-dev/neofs-sdk-go/checksum/test" - "github.com/nspcc-dev/neofs-sdk-go/client" apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test" @@ -459,17 +458,12 @@ func TestMigrate3to4(t *testing.T) { }) require.NoError(t, err) - res, _, err := db.Search(objs[0].GetContainerID(), nil, nil, nil, nil, 1000) - require.NoError(t, err) - require.Len(t, res, 2) - require.True(t, slices.ContainsFunc(res, func(r client.SearchResultItem) bool { return r.ID == objs[0].GetID() })) - require.True(t, slices.ContainsFunc(res, func(r client.SearchResultItem) bool { return r.ID == par.GetID() })) + exp := searchResultForIDs(sortObjectIDs([]oid.ID{objs[0].GetID(), par.GetID()})) + assertSearchResult(t, db, objs[0].GetContainerID(), nil, nil, exp) for i := range objs[1:] { - res, _, err := db.Search(objs[1+i].GetContainerID(), nil, nil, nil, nil, 1000) - require.NoError(t, err, i) - require.Len(t, res, 1, i) - require.Equal(t, objs[1+i].GetID(), res[0].ID, i) + exp := searchResultForIDs([]oid.ID{objs[1+i].GetID()}) + assertSearchResult(t, db, objs[1+i].GetContainerID(), nil, nil, exp) } for _, tc := range []struct { @@ -547,41 +541,31 @@ func TestMigrate3to4(t *testing.T) { } { var fs object.SearchFilters fs.AddFilter(tc.attr, tc.val, object.MatchStringEqual) - res, _, err := db.Search(tc.cnr, fs, nil, nil, nil, 1000) - require.NoError(t, err, tc) if !tc.par { - require.Len(t, res, 1, tc) - require.Equal(t, tc.exp, res[0].ID, tc) + exp = searchResultForIDs([]oid.ID{tc.exp}) } else { - require.Len(t, res, 2, tc) - require.True(t, slices.ContainsFunc(res, func(r client.SearchResultItem) bool { return r.ID == objs[0].GetID() })) - require.True(t, slices.ContainsFunc(res, func(r client.SearchResultItem) bool { return r.ID == par.GetID() })) + exp = searchResultForIDs(sortObjectIDs([]oid.ID{objs[0].GetID(), par.GetID()})) } + assertSearchResult(t, db, tc.cnr, fs, nil, exp) } for i := range objs { var fs object.SearchFilters fs.AddRootFilter() - res, _, err = db.Search(objs[i].GetContainerID(), fs, nil, nil, nil, 1000) - require.NoError(t, err, i) if i == 0 { - require.Len(t, res, 1) - require.Equal(t, par.GetID(), res[0].ID) + exp = searchResultForIDs([]oid.ID{par.GetID()}) } else { - require.Empty(t, res, i) + exp = nil } + assertSearchResult(t, db, objs[i].GetContainerID(), fs, nil, exp) fs = fs[:0] fs.AddPhyFilter() - res, _, err = db.Search(objs[i].GetContainerID(), fs, nil, nil, nil, 1000) - require.NoError(t, err, i) if i == 0 { - require.Len(t, res, 2) - require.True(t, slices.ContainsFunc(res, func(r client.SearchResultItem) bool { return r.ID == objs[0].GetID() })) - require.True(t, slices.ContainsFunc(res, func(r client.SearchResultItem) bool { return r.ID == par.GetID() })) + exp = searchResultForIDs(sortObjectIDs([]oid.ID{objs[0].GetID(), par.GetID()})) } else { - require.Len(t, res, 1) - require.Equal(t, objs[i].GetID(), res[0].ID, i) + exp = searchResultForIDs([]oid.ID{objs[i].GetID()}) } + assertSearchResult(t, db, objs[i].GetContainerID(), fs, nil, exp) } t.Run("failure", func(t *testing.T) { t.Run("zero by in attribute", func(t *testing.T) { diff --git a/pkg/services/object/server.go b/pkg/services/object/server.go index b50f6bc923..7612defd84 100644 --- a/pkg/services/object/server.go +++ b/pkg/services/object/server.go @@ -1989,19 +1989,12 @@ func (s *server) processSearchRequest(ctx context.Context, req *protoobject.Sear if err := fs.FromProtoMessage(body.Filters); err != nil { return nil, fmt.Errorf("invalid filters: %w", err) } - fInt, ok := meta.PreprocessIntFilters(fs) - if !ok { - return nil, nil - } - var primAttr string - var primInt bool - if len(fs) > 0 { - primAttr = fs[0].Header() - primInt = objectcore.IsIntegerSearchOp(fs[0].Operation()) - } - cursor, err := meta.NewSearchCursorFromString(body.Cursor, primAttr, primInt) + cursor, fInt, err := meta.PreprocessSearchQuery(fs, body.Attributes, body.Cursor) if err != nil { - return nil, fmt.Errorf("invalid cursor: %w", err) + if errors.Is(err, meta.ErrUnreachableQuery) { + return nil, nil + } + return nil, err } var res []sdkclient.SearchResultItem