Skip to content

Commit

Permalink
node/metabase: Shorten Search method return
Browse files Browse the repository at this point in the history 
Single field of the returned struct is used, so this refactors the
method and cursor re-calculation func to return this field only.

Refs #3058.

Signed-off-by: Leonard Lyubich <leonard@morphbits.io>
  • Loading branch information
@cthulhu-rider
cthulhu-rider committed Feb 24, 2025
1 parent 613d4d4 commit 65f8f68
Show file tree
Hide file tree
Showing 7 changed files with 49 additions and 58 deletions.
2 changes: 1 addition & 1 deletion cmd/neofs-node/object.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -629,7 +629,7 @@ type storageForObjectService struct {
}

// SearchObjects implements [objectService.Storage] interface.
func (x storageForObjectService) SearchObjects(cnr cid.ID, fs objectSDK.SearchFilters, fInt map[int]meta.ParsedIntFilter, attrs []string, cursor *meta.SearchCursor, count uint16) ([]client.SearchResultItem, *meta.SearchCursor, error) {
func (x storageForObjectService) SearchObjects(cnr cid.ID, fs objectSDK.SearchFilters, fInt map[int]meta.ParsedIntFilter, attrs []string, cursor *meta.SearchCursor, count uint16) ([]client.SearchResultItem, []byte, error) {
return x.local.Search(cnr, fs, fInt, attrs, cursor, count)
}

Expand Down
4 changes: 2 additions & 2 deletions pkg/local_object_storage/engine/select.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ func (e *StorageEngine) Select(cnr cid.ID, filters object.SearchFilters) ([]oid.
// Search performs Search op on all underlying shards and returns merged result.
//
// Fails instantly if executions are blocked (see [StorageEngine.BlockExecution]).
func (e *StorageEngine) Search(cnr cid.ID, fs object.SearchFilters, fInt map[int]meta.ParsedIntFilter, attrs []string, cursor *meta.SearchCursor, count uint16) ([]client.SearchResultItem, *meta.SearchCursor, error) {
func (e *StorageEngine) Search(cnr cid.ID, fs object.SearchFilters, fInt map[int]meta.ParsedIntFilter, attrs []string, cursor *meta.SearchCursor, count uint16) ([]client.SearchResultItem, []byte, error) {
if e.metrics != nil {
defer elapsed(e.metrics.AddSearchDuration)()
}
Expand Down Expand Up @@ -102,5 +102,5 @@ func (e *StorageEngine) Search(cnr cid.ID, fs object.SearchFilters, fInt map[int
// note: if the last res element is the last element of some shard, we could
// skip cursor calculation and win a bit. At the same time, this would require
// merging logic complication, so for now we just always do it.
return res[:count], &c, nil
return res[:count], c, nil
}
77 changes: 35 additions & 42 deletions pkg/local_object_storage/metabase/metadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -268,12 +268,12 @@ type ParsedIntFilter struct {

// Search selects up to count container's objects from the given container
// matching the specified filters.
func (db *DB) Search(cnr cid.ID, fs object.SearchFilters, fInt map[int]ParsedIntFilter, attrs []string, cursor *SearchCursor, count uint16) ([]client.SearchResultItem, *SearchCursor, error) {
func (db *DB) Search(cnr cid.ID, fs object.SearchFilters, fInt map[int]ParsedIntFilter, attrs []string, cursor *SearchCursor, count uint16) ([]client.SearchResultItem, []byte, error) {
if blindlyProcess(fs) {
return nil, nil, nil
}
var res []client.SearchResultItem
var newCursor *SearchCursor
var newCursor []byte
var err error
if len(fs) == 0 {
res, newCursor, err = db.searchUnfiltered(cnr, cursor, count)
Expand All @@ -286,9 +286,9 @@ func (db *DB) Search(cnr cid.ID, fs object.SearchFilters, fInt map[int]ParsedInt
return res, newCursor, nil
}

func (db *DB) search(cnr cid.ID, fs object.SearchFilters, fInt map[int]ParsedIntFilter, attrs []string, cursor *SearchCursor, count uint16) ([]client.SearchResultItem, *SearchCursor, error) {
func (db *DB) search(cnr cid.ID, fs object.SearchFilters, fInt map[int]ParsedIntFilter, attrs []string, cursor *SearchCursor, count uint16) ([]client.SearchResultItem, []byte, error) {
var res []client.SearchResultItem
var newCursor *SearchCursor
var newCursor []byte
err := db.boltDB.View(func(tx *bbolt.Tx) error {
var err error
res, newCursor, err = db.searchTx(tx, cnr, fs, fInt, attrs, cursor, count)
Expand All @@ -300,7 +300,7 @@ func (db *DB) search(cnr cid.ID, fs object.SearchFilters, fInt map[int]ParsedInt
return res, newCursor, nil
}

func (db *DB) searchTx(tx *bbolt.Tx, cnr cid.ID, fs object.SearchFilters, fInt map[int]ParsedIntFilter, attrs []string, cursor *SearchCursor, count uint16) ([]client.SearchResultItem, *SearchCursor, error) {
func (db *DB) searchTx(tx *bbolt.Tx, cnr cid.ID, fs object.SearchFilters, fInt map[int]ParsedIntFilter, attrs []string, cursor *SearchCursor, count uint16) ([]client.SearchResultItem, []byte, error) {
metaBkt := tx.Bucket(metaBucketKey(cnr))
if metaBkt == nil {
return nil, nil, nil
Expand Down Expand Up @@ -557,18 +557,15 @@ nextPrimKey:
n++
}
}
var newCursor *SearchCursor
var newCursor []byte
if more {
newCursor = &SearchCursor{
Key: slices.Clone(collectedPrimKeys[n-1][1:]),
ValIDOff: len(primSeekPrefix),
}
newCursor = slices.Clone(collectedPrimKeys[n-1][1:])
}
return res[:n], newCursor, nil
}

// TODO: can be merged with filtered code?
func (db *DB) searchUnfiltered(cnr cid.ID, cursor *SearchCursor, count uint16) ([]client.SearchResultItem, *SearchCursor, error) {
func (db *DB) searchUnfiltered(cnr cid.ID, cursor *SearchCursor, count uint16) ([]client.SearchResultItem, []byte, error) {
var seekKey []byte
if cursor != nil {
seekKey = cursor.Key
Expand All @@ -578,7 +575,7 @@ func (db *DB) searchUnfiltered(cnr cid.ID, cursor *SearchCursor, count uint16) (
}
res := make([]client.SearchResultItem, count)
var n uint16
var newCursor *SearchCursor
var newCursor []byte
curEpoch := db.epochState.CurrentEpoch()
err := db.boltDB.View(func(tx *bbolt.Tx) error {
mb := tx.Bucket(metaBucketKey(cnr))
Expand All @@ -593,7 +590,7 @@ func (db *DB) searchUnfiltered(cnr cid.ID, cursor *SearchCursor, count uint16) (
}
for ; k[0] == metaPrefixID; k, _ = mbc.Next() {
if n == count { // there are still elements
newCursor = &SearchCursor{Key: res[n-1].ID[:]}
newCursor = res[n-1].ID[:]
return nil
}
if len(k) != oid.Size+1 {
Expand Down Expand Up @@ -907,15 +904,15 @@ func convertFilterValue(f object.SearchFilter) (object.SearchMatchType, string)
}

// CalculateCursor calculates cursor for the given last search result item.
func CalculateCursor(fs object.SearchFilters, lastItem client.SearchResultItem) (SearchCursor, error) {
func CalculateCursor(fs object.SearchFilters, lastItem client.SearchResultItem) ([]byte, error) {
if len(lastItem.Attributes) == 0 || len(fs) == 0 || fs[0].Operation() == object.MatchNotPresent {
return SearchCursor{Key: lastItem.ID[:]}, nil
return lastItem.ID[:], nil
}
attr := fs[0].Header()
var lastItemVal string
if len(lastItem.Attributes) == 0 {
if attr != object.FilterRoot && attr != object.FilterPhysical {
return SearchCursor{Key: lastItem.ID[:]}, nil
return lastItem.ID[:], nil
}
lastItemVal = binPropMarker
} else {
Expand All @@ -927,57 +924,53 @@ func CalculateCursor(fs object.SearchFilters, lastItem client.SearchResultItem)
if objectcore.IsIntegerSearchOp(fs[0].Operation()) {
n, ok := new(big.Int).SetString(lastItemVal, 10)
if !ok {
return SearchCursor{}, fmt.Errorf("non-int attribute value %q with int matcher", lastItemVal)
return nil, fmt.Errorf("non-int attribute value %q with int matcher", lastItemVal)
}
var res SearchCursor
res.Key = make([]byte, len(attr)+attributeDelimiterLen+intValLen+oid.Size)
off := copy(res.Key, attr)
res.ValIDOff = off + copy(res.Key[off:], attributeDelimiter)
putInt(res.Key[res.ValIDOff:res.ValIDOff+intValLen], n)
copy(res.Key[res.ValIDOff+intValLen:], lastItem.ID[:])
res := make([]byte, len(attr)+attributeDelimiterLen+intValLen+oid.Size)
off := copy(res, attr)
off += copy(res[off:], attributeDelimiter)
putInt(res[off:off+intValLen], n)
copy(res[off+intValLen:], lastItem.ID[:])
return res, nil
}
case object.FilterOwnerID, object.FilterFirstSplitObject, object.FilterParentID:
var err error
if val, err = base58.Decode(lastItemVal); err != nil {
return SearchCursor{}, fmt.Errorf("decode %q attribute value from Base58: %w", attr, err)
return nil, fmt.Errorf("decode %q attribute value from Base58: %w", attr, err)
}
case object.FilterPayloadChecksum, object.FilterPayloadHomomorphicHash:
ln := hex.DecodedLen(len(lastItemVal))
if attr == object.FilterPayloadChecksum && ln != sha256.Size || attr == object.FilterPayloadHomomorphicHash && ln != tz.Size {
return SearchCursor{}, fmt.Errorf("wrong %q attribute decoded len %d", attr, ln)
return nil, fmt.Errorf("wrong %q attribute decoded len %d", attr, ln)
}
var res SearchCursor
res.Key = make([]byte, len(attr)+attributeDelimiterLen+ln+attributeDelimiterLen+oid.Size)
off := copy(res.Key, attr)
res.ValIDOff = off + copy(res.Key[off:], attributeDelimiter)
res := make([]byte, len(attr)+attributeDelimiterLen+ln+attributeDelimiterLen+oid.Size)
off := copy(res, attr)
off += copy(res[off:], attributeDelimiter)
var err error
if _, err = hex.Decode(res.Key[res.ValIDOff:], []byte(lastItemVal)); err != nil {
return SearchCursor{}, fmt.Errorf("decode %q attribute from HEX: %w", attr, err)
if _, err = hex.Decode(res[off:], []byte(lastItemVal)); err != nil {
return nil, fmt.Errorf("decode %q attribute from HEX: %w", attr, err)
}
off = res.ValIDOff + ln
off += copy(res.Key[off:], attributeDelimiter)
copy(res.Key[off:], lastItem.ID[:])
off += copy(res[off+ln:], attributeDelimiter)
copy(res[off:], lastItem.ID[:])
return res, nil
case object.FilterSplitID:
uid, err := uuid.Parse(lastItemVal)
if err != nil {
return SearchCursor{}, fmt.Errorf("decode %q attribute from HEX: %w", attr, err)
return nil, fmt.Errorf("decode %q attribute from HEX: %w", attr, err)
}
val = uid[:]
case object.FilterVersion, object.FilterType:
}
if val == nil {
val = []byte(lastItemVal)
}
var res SearchCursor
kln := len(attr) + attributeDelimiterLen + len(val) + attributeDelimiterLen + oid.Size
res.Key = make([]byte, kln)
off := copy(res.Key, attr)
res.ValIDOff = off + copy(res.Key[off:], attributeDelimiter)
off = res.ValIDOff + copy(res.Key[res.ValIDOff:], val)
off += copy(res.Key[off:], attributeDelimiter)
copy(res.Key[off:], lastItem.ID[:])
res := make([]byte, kln)
off := copy(res, attr)
off += copy(res[off:], attributeDelimiter)
off += copy(res[off:], val)
off += copy(res[off:], attributeDelimiter)
copy(res[off:], lastItem.ID[:])
return res, nil
}

Expand Down
4 changes: 2 additions & 2 deletions pkg/local_object_storage/metabase/metadata_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -488,9 +488,9 @@ func _assertSearchResultWithLimit(t testing.TB, db *DB, cnr cid.ID, fs object.Se

cc, err := CalculateCursor(fs, res[n-1])
require.NoError(t, err, "cursor: %q", strCursor)
require.Equal(t, c.Key, cc.Key, "cursor: %q", strCursor)
require.Equal(t, c, cc, "cursor: %q", strCursor)

strCursor = base64.StdEncoding.EncodeToString(c.Key)
strCursor = base64.StdEncoding.EncodeToString(c)
cursor, err = NewSearchCursorFromString(strCursor, primAttr, primInt)
require.NoErrorf(t, err, "cursor: %q", strCursor)
}
Expand Down
6 changes: 3 additions & 3 deletions pkg/local_object_storage/shard/select.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,15 +33,15 @@ func (s *Shard) Select(cnr cid.ID, filters object.SearchFilters) ([]oid.Address,
}

// Search performs Search op on the underlying metabase if it is not disabled.
func (s *Shard) Search(cnr cid.ID, fs object.SearchFilters, fInt map[int]meta.ParsedIntFilter, attrs []string, cursor *meta.SearchCursor, count uint16) ([]client.SearchResultItem, *meta.SearchCursor, error) {
func (s *Shard) Search(cnr cid.ID, fs object.SearchFilters, fInt map[int]meta.ParsedIntFilter, attrs []string, cursor *meta.SearchCursor, count uint16) ([]client.SearchResultItem, []byte, error) {
s.m.RLock()
defer s.m.RUnlock()
if s.info.Mode.NoMetabase() {
return nil, nil, ErrDegradedMode
}
res, cursor, err := s.metaBase.Search(cnr, fs, fInt, attrs, cursor, count)
res, newCursor, err := s.metaBase.Search(cnr, fs, fInt, attrs, cursor, count)
if err != nil {
return nil, nil, fmt.Errorf("call metabase: %w", err)
}
return res, cursor, nil
return res, newCursor, nil
}
10 changes: 4 additions & 6 deletions pkg/services/object/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ type Storage interface {

// SearchObjects selects up to count container's objects from the given
// container matching the specified filters.
SearchObjects(_ cid.ID, _ object.SearchFilters, _ map[int]meta.ParsedIntFilter, attrs []string, cursor *meta.SearchCursor, count uint16) ([]sdkclient.SearchResultItem, *meta.SearchCursor, error)
SearchObjects(_ cid.ID, _ object.SearchFilters, _ map[int]meta.ParsedIntFilter, attrs []string, cursor *meta.SearchCursor, count uint16) ([]sdkclient.SearchResultItem, []byte, error)
}

// ACLInfoExtractor is the interface that allows to fetch data required for ACL
Expand Down Expand Up @@ -2005,7 +2005,7 @@ func (s *server) processSearchRequest(ctx context.Context, req *protoobject.Sear
}

var res []sdkclient.SearchResultItem
var newCursor *meta.SearchCursor
var newCursor []byte
count := uint16(body.Count) // legit according to the limit
if ttl == 1 {
if res, newCursor, err = s.storage.SearchObjects(cnr, fs, fInt, body.Attributes, cursor, count); err != nil {
Expand Down Expand Up @@ -2077,11 +2077,9 @@ func (s *server) processSearchRequest(ctx context.Context, req *protoobject.Sear
return nil, fmt.Errorf("merge results from container nodes: %w", err)
}
if more {
c, err := meta.CalculateCursor(fs, res[len(res)-1])
if err != nil {
if newCursor, err = meta.CalculateCursor(fs, res[len(res)-1]); err != nil {
return nil, fmt.Errorf("recalculate cursor: %w", err)
}
newCursor = &c
}
}

Expand All @@ -2095,7 +2093,7 @@ func (s *server) processSearchRequest(ctx context.Context, req *protoobject.Sear
}
}
if newCursor != nil {
resBody.Cursor = base64.StdEncoding.EncodeToString(newCursor.Key)
resBody.Cursor = base64.StdEncoding.EncodeToString(newCursor)
}
return resBody, nil
}
Expand Down
4 changes: 2 additions & 2 deletions pkg/services/object/server_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ func (*noCallTestFSChain) LocalNodeUnderMaintenance() bool { panic("must not be

type noCallTestStorage struct{}

func (noCallTestStorage) SearchObjects(cid.ID, object.SearchFilters, map[int]meta.ParsedIntFilter, []string, *meta.SearchCursor, uint16) ([]client.SearchResultItem, *meta.SearchCursor, error) {
func (noCallTestStorage) SearchObjects(cid.ID, object.SearchFilters, map[int]meta.ParsedIntFilter, []string, *meta.SearchCursor, uint16) ([]client.SearchResultItem, []byte, error) {
panic("must not be called")
}
func (noCallTestStorage) VerifyAndStoreObjectLocally(object.Object) error {
Expand Down Expand Up @@ -587,7 +587,7 @@ func (nopStorage) VerifyAndStoreObjectLocally(object.Object) error { return nil
func (nopStorage) GetSessionPrivateKey(user.ID, uuid.UUID) (ecdsa.PrivateKey, error) {
return ecdsa.PrivateKey{}, apistatus.ErrSessionTokenNotFound
}
func (nopStorage) SearchObjects(cid.ID, object.SearchFilters, map[int]meta.ParsedIntFilter, []string, *meta.SearchCursor, uint16) ([]client.SearchResultItem, *meta.SearchCursor, error) {
func (nopStorage) SearchObjects(cid.ID, object.SearchFilters, map[int]meta.ParsedIntFilter, []string, *meta.SearchCursor, uint16) ([]client.SearchResultItem, []byte, error) {
return nil, nil, nil
}

Expand Down

0 comments on commit 65f8f68

Please sign in to comment.