Skip to content

Commit

Permalink
node/metabase: Shorten Search method return
Browse files Browse the repository at this point in the history 
Single field of the returned struct is used, so this refactors the
method and cursor re-calculation func to return this field only.

Refs #3058.

Signed-off-by: Leonard Lyubich <leonard@morphbits.io>
  • Loading branch information
@cthulhu-rider
cthulhu-rider committed Feb 21, 2025
1 parent 402e64d commit fdac166
Show file tree
Hide file tree
Showing 7 changed files with 49 additions and 58 deletions.
2 changes: 1 addition & 1 deletion cmd/neofs-node/object.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -629,7 +629,7 @@ type storageForObjectService struct {
}

// SearchObjects implements [objectService.Storage] interface.
func (x storageForObjectService) SearchObjects(cnr cid.ID, fs objectSDK.SearchFilters, fInt map[int]meta.ParsedIntFilter, attrs []string, cursor *meta.SearchCursor, count uint16) ([]client.SearchResultItem, *meta.SearchCursor, error) {
func (x storageForObjectService) SearchObjects(cnr cid.ID, fs objectSDK.SearchFilters, fInt map[int]meta.ParsedIntFilter, attrs []string, cursor *meta.SearchCursor, count uint16) ([]client.SearchResultItem, []byte, error) {

Check warning on line 632 in cmd/neofs-node/object.go

View check run for this annotation

Codecov / codecov/patch

cmd/neofs-node/object.go#L632 

Added line #L632 was not covered by tests
return x.local.Search(cnr, fs, fInt, attrs, cursor, count)
}

Expand Down
4 changes: 2 additions & 2 deletions pkg/local_object_storage/engine/select.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ func (e *StorageEngine) Select(cnr cid.ID, filters object.SearchFilters) ([]oid.
// Search performs Search op on all underlying shards and returns merged result.
//
// Fails instantly if executions are blocked (see [StorageEngine.BlockExecution]).
func (e *StorageEngine) Search(cnr cid.ID, fs object.SearchFilters, fInt map[int]meta.ParsedIntFilter, attrs []string, cursor *meta.SearchCursor, count uint16) ([]client.SearchResultItem, *meta.SearchCursor, error) {
func (e *StorageEngine) Search(cnr cid.ID, fs object.SearchFilters, fInt map[int]meta.ParsedIntFilter, attrs []string, cursor *meta.SearchCursor, count uint16) ([]client.SearchResultItem, []byte, error) {

Check warning on line 59 in pkg/local_object_storage/engine/select.go

View check run for this annotation

Codecov / codecov/patch

pkg/local_object_storage/engine/select.go#L59 

Added line #L59 was not covered by tests
if e.metrics != nil {
defer elapsed(e.metrics.AddSearchDuration)()
}
Expand Down Expand Up @@ -99,5 +99,5 @@ func (e *StorageEngine) Search(cnr cid.ID, fs object.SearchFilters, fInt map[int
// note: if the last res element is the last element of some shard, we could
// skip cursor calculation and win a bit. At the same time, this would require
// merging logic complication, so for now we just always do it.
return res[:count], &c, nil
return res[:count], c, nil

Check warning on line 102 in pkg/local_object_storage/engine/select.go

View check run for this annotation

Codecov / codecov/patch

pkg/local_object_storage/engine/select.go#L102 

Added line #L102 was not covered by tests
}
77 changes: 35 additions & 42 deletions pkg/local_object_storage/metabase/metadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,12 +271,12 @@ type ParsedIntFilter struct {

// Search selects up to count container's objects from the given container
// matching the specified filters.
func (db *DB) Search(cnr cid.ID, fs object.SearchFilters, fInt map[int]ParsedIntFilter, attrs []string, cursor *SearchCursor, count uint16) ([]client.SearchResultItem, *SearchCursor, error) {
func (db *DB) Search(cnr cid.ID, fs object.SearchFilters, fInt map[int]ParsedIntFilter, attrs []string, cursor *SearchCursor, count uint16) ([]client.SearchResultItem, []byte, error) {
if blindlyProcess(fs) {
return nil, nil, nil
}
var res []client.SearchResultItem
var newCursor *SearchCursor
var newCursor []byte
var err error
if len(fs) == 0 {
res, newCursor, err = db.searchUnfiltered(cnr, cursor, count)
Expand All @@ -289,9 +289,9 @@ func (db *DB) Search(cnr cid.ID, fs object.SearchFilters, fInt map[int]ParsedInt
return res, newCursor, nil
}

func (db *DB) search(cnr cid.ID, fs object.SearchFilters, fInt map[int]ParsedIntFilter, attrs []string, cursor *SearchCursor, count uint16) ([]client.SearchResultItem, *SearchCursor, error) {
func (db *DB) search(cnr cid.ID, fs object.SearchFilters, fInt map[int]ParsedIntFilter, attrs []string, cursor *SearchCursor, count uint16) ([]client.SearchResultItem, []byte, error) {
var res []client.SearchResultItem
var newCursor *SearchCursor
var newCursor []byte
err := db.boltDB.View(func(tx *bbolt.Tx) error {
var err error
res, newCursor, err = db.searchTx(tx, cnr, fs, fInt, attrs, cursor, count)
Expand All @@ -303,7 +303,7 @@ func (db *DB) search(cnr cid.ID, fs object.SearchFilters, fInt map[int]ParsedInt
return res, newCursor, nil
}

func (db *DB) searchTx(tx *bbolt.Tx, cnr cid.ID, fs object.SearchFilters, fInt map[int]ParsedIntFilter, attrs []string, cursor *SearchCursor, count uint16) ([]client.SearchResultItem, *SearchCursor, error) {
func (db *DB) searchTx(tx *bbolt.Tx, cnr cid.ID, fs object.SearchFilters, fInt map[int]ParsedIntFilter, attrs []string, cursor *SearchCursor, count uint16) ([]client.SearchResultItem, []byte, error) {
metaBkt := tx.Bucket(metaBucketKey(cnr))
if metaBkt == nil {
return nil, nil, nil
Expand Down Expand Up @@ -561,18 +561,15 @@ nextPrimKey:
n++
}
}
var newCursor *SearchCursor
var newCursor []byte
if more {
newCursor = &SearchCursor{
Key: slices.Clone(collectedPrimKeys[n-1][1:]),
ValIDOff: len(primSeekPrefix),
}
newCursor = slices.Clone(collectedPrimKeys[n-1][1:])
}
return res[:n], newCursor, nil
}

// TODO: can be merged with filtered code?
func (db *DB) searchUnfiltered(cnr cid.ID, cursor *SearchCursor, count uint16) ([]client.SearchResultItem, *SearchCursor, error) {
func (db *DB) searchUnfiltered(cnr cid.ID, cursor *SearchCursor, count uint16) ([]client.SearchResultItem, []byte, error) {
var seekKey []byte
if cursor != nil {
seekKey = cursor.Key
Expand All @@ -582,7 +579,7 @@ func (db *DB) searchUnfiltered(cnr cid.ID, cursor *SearchCursor, count uint16) (
}
res := make([]client.SearchResultItem, count)
var n uint16
var newCursor *SearchCursor
var newCursor []byte
curEpoch := db.epochState.CurrentEpoch()
err := db.boltDB.View(func(tx *bbolt.Tx) error {
mb := tx.Bucket(metaBucketKey(cnr))
Expand All @@ -597,7 +594,7 @@ func (db *DB) searchUnfiltered(cnr cid.ID, cursor *SearchCursor, count uint16) (
}
for ; k[0] == metaPrefixID; k, _ = mbc.Next() {
if n == count { // there are still elements
newCursor = &SearchCursor{Key: res[n-1].ID[:]}
newCursor = res[n-1].ID[:]
return nil
}
if len(k) != oid.Size+1 {
Expand Down Expand Up @@ -911,15 +908,15 @@ func convertFilterValue(f object.SearchFilter) (object.SearchMatchType, string)
}

// CalculateCursor calculates cursor for the given last search result item.
func CalculateCursor(fs object.SearchFilters, lastItem client.SearchResultItem) (SearchCursor, error) {
func CalculateCursor(fs object.SearchFilters, lastItem client.SearchResultItem) ([]byte, error) {
if len(lastItem.Attributes) == 0 || len(fs) == 0 || fs[0].Operation() == object.MatchNotPresent {
return SearchCursor{Key: lastItem.ID[:]}, nil
return lastItem.ID[:], nil
}
attr := fs[0].Header()
var lastItemVal string
if len(lastItem.Attributes) == 0 {
if attr != object.FilterRoot && attr != object.FilterPhysical {
return SearchCursor{Key: lastItem.ID[:]}, nil
return lastItem.ID[:], nil

Check warning on line 919 in pkg/local_object_storage/metabase/metadata.go

View check run for this annotation

Codecov / codecov/patch

pkg/local_object_storage/metabase/metadata.go#L919 

Added line #L919 was not covered by tests
}
lastItemVal = binPropMarker
} else {
Expand All @@ -931,57 +928,53 @@ func CalculateCursor(fs object.SearchFilters, lastItem client.SearchResultItem)
if objectcore.IsIntegerSearchOp(fs[0].Operation()) {
n, ok := new(big.Int).SetString(lastItemVal, 10)
if !ok {
return SearchCursor{}, fmt.Errorf("non-int attribute value %q with int matcher", lastItemVal)
return nil, fmt.Errorf("non-int attribute value %q with int matcher", lastItemVal)
}

Check warning on line 932 in pkg/local_object_storage/metabase/metadata.go

View check run for this annotation

Codecov / codecov/patch

pkg/local_object_storage/metabase/metadata.go#L931-L932 

Added lines #L931 - L932 were not covered by tests
var res SearchCursor
res.Key = make([]byte, len(attr)+attributeDelimiterLen+intValLen+oid.Size)
off := copy(res.Key, attr)
res.ValIDOff = off + copy(res.Key[off:], attributeDelimiter)
putInt(res.Key[res.ValIDOff:res.ValIDOff+intValLen], n)
copy(res.Key[res.ValIDOff+intValLen:], lastItem.ID[:])
res := make([]byte, len(attr)+attributeDelimiterLen+intValLen+oid.Size)
off := copy(res, attr)
off += copy(res[off:], attributeDelimiter)
putInt(res[off:off+intValLen], n)
copy(res[off+intValLen:], lastItem.ID[:])
return res, nil
}
case object.FilterOwnerID, object.FilterFirstSplitObject, object.FilterParentID:
var err error
if val, err = base58.Decode(lastItemVal); err != nil {
return SearchCursor{}, fmt.Errorf("decode %q attribute value from Base58: %w", attr, err)
return nil, fmt.Errorf("decode %q attribute value from Base58: %w", attr, err)

Check warning on line 943 in pkg/local_object_storage/metabase/metadata.go

View check run for this annotation

Codecov / codecov/patch

pkg/local_object_storage/metabase/metadata.go#L943 

Added line #L943 was not covered by tests
}
case object.FilterPayloadChecksum, object.FilterPayloadHomomorphicHash:
ln := hex.DecodedLen(len(lastItemVal))
if attr == object.FilterPayloadChecksum && ln != sha256.Size || attr == object.FilterPayloadHomomorphicHash && ln != tz.Size {
return SearchCursor{}, fmt.Errorf("wrong %q attribute decoded len %d", attr, ln)
return nil, fmt.Errorf("wrong %q attribute decoded len %d", attr, ln)

Check warning on line 948 in pkg/local_object_storage/metabase/metadata.go

View check run for this annotation

Codecov / codecov/patch

pkg/local_object_storage/metabase/metadata.go#L948 

Added line #L948 was not covered by tests
}
var res SearchCursor
res.Key = make([]byte, len(attr)+attributeDelimiterLen+ln+attributeDelimiterLen+oid.Size)
off := copy(res.Key, attr)
res.ValIDOff = off + copy(res.Key[off:], attributeDelimiter)
res := make([]byte, len(attr)+attributeDelimiterLen+ln+attributeDelimiterLen+oid.Size)
off := copy(res, attr)
off += copy(res[off:], attributeDelimiter)

Check warning on line 952 in pkg/local_object_storage/metabase/metadata.go

View check run for this annotation

Codecov / codecov/patch

pkg/local_object_storage/metabase/metadata.go#L950-L952 

Added lines #L950 - L952 were not covered by tests
var err error
if _, err = hex.Decode(res.Key[res.ValIDOff:], []byte(lastItemVal)); err != nil {
return SearchCursor{}, fmt.Errorf("decode %q attribute from HEX: %w", attr, err)
if _, err = hex.Decode(res[off:], []byte(lastItemVal)); err != nil {
return nil, fmt.Errorf("decode %q attribute from HEX: %w", attr, err)

Check warning on line 955 in pkg/local_object_storage/metabase/metadata.go

View check run for this annotation

Codecov / codecov/patch

pkg/local_object_storage/metabase/metadata.go#L954-L955 

Added lines #L954 - L955 were not covered by tests
}
off = res.ValIDOff + ln
off += copy(res.Key[off:], attributeDelimiter)
copy(res.Key[off:], lastItem.ID[:])
off += copy(res[off+ln:], attributeDelimiter)
copy(res[off:], lastItem.ID[:])

Check warning on line 958 in pkg/local_object_storage/metabase/metadata.go

View check run for this annotation

Codecov / codecov/patch

pkg/local_object_storage/metabase/metadata.go#L957-L958 

Added lines #L957 - L958 were not covered by tests
return res, nil
case object.FilterSplitID:
uid, err := uuid.Parse(lastItemVal)
if err != nil {
return SearchCursor{}, fmt.Errorf("decode %q attribute from HEX: %w", attr, err)
return nil, fmt.Errorf("decode %q attribute from HEX: %w", attr, err)

Check warning on line 963 in pkg/local_object_storage/metabase/metadata.go

View check run for this annotation

Codecov / codecov/patch

pkg/local_object_storage/metabase/metadata.go#L963 

Added line #L963 was not covered by tests
}
val = uid[:]
case object.FilterVersion, object.FilterType:
}
if val == nil {
val = []byte(lastItemVal)
}
var res SearchCursor
kln := len(attr) + attributeDelimiterLen + len(val) + attributeDelimiterLen + oid.Size
res.Key = make([]byte, kln)
off := copy(res.Key, attr)
res.ValIDOff = off + copy(res.Key[off:], attributeDelimiter)
off = res.ValIDOff + copy(res.Key[res.ValIDOff:], val)
off += copy(res.Key[off:], attributeDelimiter)
copy(res.Key[off:], lastItem.ID[:])
res := make([]byte, kln)
off := copy(res, attr)
off += copy(res[off:], attributeDelimiter)
off += copy(res[off:], val)
off += copy(res[off:], attributeDelimiter)
copy(res[off:], lastItem.ID[:])
return res, nil
}

Expand Down
4 changes: 2 additions & 2 deletions pkg/local_object_storage/metabase/metadata_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -488,9 +488,9 @@ func _assertSearchResultWithLimit(t testing.TB, db *DB, cnr cid.ID, fs object.Se

cc, err := CalculateCursor(fs, res[n-1])
require.NoError(t, err, "cursor: %q", strCursor)
require.Equal(t, c.Key, cc.Key, "cursor: %q", strCursor)
require.Equal(t, c, cc, "cursor: %q", strCursor)

strCursor = base64.StdEncoding.EncodeToString(c.Key)
strCursor = base64.StdEncoding.EncodeToString(c)
cursor, err = NewSearchCursorFromString(strCursor, primAttr, primInt)
require.NoErrorf(t, err, "cursor: %q", strCursor)
}
Expand Down
6 changes: 3 additions & 3 deletions pkg/local_object_storage/shard/select.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,15 +33,15 @@ func (s *Shard) Select(cnr cid.ID, filters object.SearchFilters) ([]oid.Address,
}

// Search performs Search op on the underlying metabase if it is not disabled.
func (s *Shard) Search(cnr cid.ID, fs object.SearchFilters, fInt map[int]meta.ParsedIntFilter, attrs []string, cursor *meta.SearchCursor, count uint16) ([]client.SearchResultItem, *meta.SearchCursor, error) {
func (s *Shard) Search(cnr cid.ID, fs object.SearchFilters, fInt map[int]meta.ParsedIntFilter, attrs []string, cursor *meta.SearchCursor, count uint16) ([]client.SearchResultItem, []byte, error) {

Check warning on line 36 in pkg/local_object_storage/shard/select.go

View check run for this annotation

Codecov / codecov/patch

pkg/local_object_storage/shard/select.go#L36 

Added line #L36 was not covered by tests
s.m.RLock()
defer s.m.RUnlock()
if s.info.Mode.NoMetabase() {
return nil, nil, ErrDegradedMode
}
res, cursor, err := s.metaBase.Search(cnr, fs, fInt, attrs, cursor, count)
res, newCursor, err := s.metaBase.Search(cnr, fs, fInt, attrs, cursor, count)

Check warning on line 42 in pkg/local_object_storage/shard/select.go

View check run for this annotation

Codecov / codecov/patch

pkg/local_object_storage/shard/select.go#L42 

Added line #L42 was not covered by tests
if err != nil {
return nil, nil, fmt.Errorf("call metabase: %w", err)
}
return res, cursor, nil
return res, newCursor, nil

Check warning on line 46 in pkg/local_object_storage/shard/select.go

View check run for this annotation

Codecov / codecov/patch

pkg/local_object_storage/shard/select.go#L46 

Added line #L46 was not covered by tests
}
10 changes: 4 additions & 6 deletions pkg/services/object/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ type Storage interface {

// SearchObjects selects up to count container's objects from the given
// container matching the specified filters.
SearchObjects(_ cid.ID, _ object.SearchFilters, _ map[int]meta.ParsedIntFilter, attrs []string, cursor *meta.SearchCursor, count uint16) ([]sdkclient.SearchResultItem, *meta.SearchCursor, error)
SearchObjects(_ cid.ID, _ object.SearchFilters, _ map[int]meta.ParsedIntFilter, attrs []string, cursor *meta.SearchCursor, count uint16) ([]sdkclient.SearchResultItem, []byte, error)
}

// ACLInfoExtractor is the interface that allows to fetch data required for ACL
Expand Down Expand Up @@ -2005,7 +2005,7 @@ func (s *server) processSearchRequest(ctx context.Context, req *protoobject.Sear
}

var res []sdkclient.SearchResultItem
var newCursor *meta.SearchCursor
var newCursor []byte

Check warning on line 2008 in pkg/services/object/server.go

View check run for this annotation

Codecov / codecov/patch

pkg/services/object/server.go#L2008 

Added line #L2008 was not covered by tests
count := uint16(body.Count) // legit according to the limit
if ttl == 1 {
if res, newCursor, err = s.storage.SearchObjects(cnr, fs, fInt, body.Attributes, cursor, count); err != nil {
Expand Down Expand Up @@ -2074,11 +2074,9 @@ func (s *server) processSearchRequest(ctx context.Context, req *protoobject.Sear
return nil, fmt.Errorf("merge results from container nodes: %w", err)
}
if more {
c, err := meta.CalculateCursor(fs, res[len(res)-1])
if err != nil {
if newCursor, err = meta.CalculateCursor(fs, res[len(res)-1]); err != nil {

Check warning on line 2077 in pkg/services/object/server.go

View check run for this annotation

Codecov / codecov/patch

pkg/services/object/server.go#L2077 

Added line #L2077 was not covered by tests
return nil, fmt.Errorf("recalculate cursor: %w", err)
}
newCursor = &c
}
}

Expand All @@ -2092,7 +2090,7 @@ func (s *server) processSearchRequest(ctx context.Context, req *protoobject.Sear
}
}
if newCursor != nil {
resBody.Cursor = base64.StdEncoding.EncodeToString(newCursor.Key)
resBody.Cursor = base64.StdEncoding.EncodeToString(newCursor)

Check warning on line 2093 in pkg/services/object/server.go

View check run for this annotation

Codecov / codecov/patch

pkg/services/object/server.go#L2093 

Added line #L2093 was not covered by tests
}
return resBody, nil
}
Expand Down
4 changes: 2 additions & 2 deletions pkg/services/object/server_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ func (*noCallTestFSChain) LocalNodeUnderMaintenance() bool { panic("must not be

type noCallTestStorage struct{}

func (noCallTestStorage) SearchObjects(cid.ID, object.SearchFilters, map[int]meta.ParsedIntFilter, []string, *meta.SearchCursor, uint16) ([]client.SearchResultItem, *meta.SearchCursor, error) {
func (noCallTestStorage) SearchObjects(cid.ID, object.SearchFilters, map[int]meta.ParsedIntFilter, []string, *meta.SearchCursor, uint16) ([]client.SearchResultItem, []byte, error) {
panic("must not be called")
}
func (noCallTestStorage) VerifyAndStoreObjectLocally(object.Object) error {
Expand Down Expand Up @@ -587,7 +587,7 @@ func (nopStorage) VerifyAndStoreObjectLocally(object.Object) error { return nil
func (nopStorage) GetSessionPrivateKey(user.ID, uuid.UUID) (ecdsa.PrivateKey, error) {
return ecdsa.PrivateKey{}, apistatus.ErrSessionTokenNotFound
}
func (nopStorage) SearchObjects(cid.ID, object.SearchFilters, map[int]meta.ParsedIntFilter, []string, *meta.SearchCursor, uint16) ([]client.SearchResultItem, *meta.SearchCursor, error) {
func (nopStorage) SearchObjects(cid.ID, object.SearchFilters, map[int]meta.ParsedIntFilter, []string, *meta.SearchCursor, uint16) ([]client.SearchResultItem, []byte, error) {
return nil, nil, nil
}

Expand Down

0 comments on commit fdac166

Please sign in to comment.