Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Integrate search v2 #1082

Merged
merged 13 commits into from
Feb 21, 2025
Merged

Integrate search v2 #1082

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
6ee2469
multipart: Drop set-only struct fields
cthulhu-rider Feb 11, 2025
82d05da
multipart: Inline a once called method
cthulhu-rider Feb 11, 2025
30b92db
api/layer: Drop unused comment
cthulhu-rider Feb 11, 2025
0cb4ff2
go.mod: Update SDK
smallhive Feb 14, 2025
9ce5aac
layer: Use search-v2 for multipart
smallhive Feb 19, 2025
04ab5c9
layer: Use search-v2 for tags removal
smallhive Feb 19, 2025
c14a3c0
layer: Use search-v2 for file listing
smallhive Feb 12, 2025
95b3304
handler: Remove obsolete tests
smallhive Feb 21, 2025
bf6f214
layer: Use search-v2 for tags retrieving
smallhive Feb 19, 2025
bfb2204
layer: Use search-v2 for locks retrieving
smallhive Feb 14, 2025
acfb3ce
*: Remove GetExtendObjectInfo usage where possible
smallhive Feb 17, 2025
c05b659
layer: Remove last search-v1 usage
smallhive Feb 18, 2025
17d951f
layer: Remove unused
smallhive Feb 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions api/data/info.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,17 @@ type (
Headers map[string]string
}

// ObjectListResponseContent holds response data for object listing.
ObjectListResponseContent struct {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What are the uses for ObjectInfo? Can this be embedded into ObjectInfo? Do we need both or just one is sufficient?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ObjectInfo in general appears with GetExtendedObjectInfo method to get full info about object. Moreover, it contains all object attributes, which are used for responses and other data manipulations

ObjectListResponseContent is a shorter version, which is enough to show a bucket listing

ID oid.ID
IsDir bool
Size int64
Owner user.ID
HashSum string
Created time.Time
Name string
}

// NotificationInfo store info to send s3 notification.
NotificationInfo struct {
Name string
Expand Down
26 changes: 19 additions & 7 deletions api/handler/copy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package handler

import (
"errors"
"net/http"
"net/url"
"regexp"
Expand Down Expand Up @@ -85,18 +86,29 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
}

if settingsSrc.VersioningEnabled() && srcObjPrm.VersionID == "" {
headObjectPrm := &layer.HeadObjectParams{
BktInfo: srcObjPrm.BktInfo,
Object: srcObject,
shortInfoParams := &layer.ShortInfoParams{
Owner: srcObjPrm.BktInfo.Owner,
CID: srcObjPrm.BktInfo.CID,
Object: srcObject,
}

ei, err := h.obj.GetExtendedObjectInfo(r.Context(), headObjectPrm)
ei, err := h.obj.GetIDForVersioningContainer(r.Context(), shortInfoParams)
if err != nil {
h.logAndSendError(w, "could not find object", reqInfo, err)
return
if !errors.Is(err, layer.ErrNodeNotFound) {
h.logAndSendError(w, "could not find object", reqInfo, err)
return
}

// CopyObject can copy object from versioned container, but it can contain only "null" versions.
// In this case we should find actual one.
shortInfoParams.FindNullVersion = true
if ei, err = h.obj.GetIDForVersioningContainer(r.Context(), shortInfoParams); err != nil {
h.logAndSendError(w, "could not find object", reqInfo, err)
return
}
}

srcObjPrm.VersionID = ei.ObjectInfo.VersionID()
srcObjPrm.VersionID = ei.EncodeToString()
}

dstBktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
Expand Down
33 changes: 18 additions & 15 deletions api/handler/locking.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,18 +159,19 @@ func (h *handler) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Reque
}

if settings.VersioningEnabled() && p.ObjVersion.VersionID == "" {
headObjectPrm := &layer.HeadObjectParams{
BktInfo: bktInfo,
Object: reqInfo.ObjectName,
shortInfoParams := &layer.ShortInfoParams{
Owner: bktInfo.Owner,
CID: bktInfo.CID,
Object: reqInfo.ObjectName,
}

ei, err := h.obj.GetExtendedObjectInfo(r.Context(), headObjectPrm)
ei, err := h.obj.GetIDForVersioningContainer(r.Context(), shortInfoParams)
if err != nil {
h.logAndSendError(w, "could not find object", reqInfo, err)
return
}

p.ObjVersion.VersionID = ei.ObjectInfo.VersionID()
p.ObjVersion.VersionID = ei.EncodeToString()
}

if err = h.obj.PutLockInfo(r.Context(), p); err != nil {
Expand Down Expand Up @@ -259,18 +260,19 @@ func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
}

if settings.VersioningEnabled() && p.ObjVersion.VersionID == "" {
headObjectPrm := &layer.HeadObjectParams{
BktInfo: bktInfo,
Object: reqInfo.ObjectName,
shortInfoParams := &layer.ShortInfoParams{
Owner: bktInfo.Owner,
CID: bktInfo.CID,
Object: reqInfo.ObjectName,
}

ei, err := h.obj.GetExtendedObjectInfo(r.Context(), headObjectPrm)
ei, err := h.obj.GetIDForVersioningContainer(r.Context(), shortInfoParams)
if err != nil {
h.logAndSendError(w, "could not find object", reqInfo, err)
return
}

p.ObjVersion.VersionID = ei.ObjectInfo.VersionID()
p.ObjVersion.VersionID = ei.EncodeToString()
}

if err = h.obj.PutLockInfo(r.Context(), p); err != nil {
Expand Down Expand Up @@ -307,18 +309,19 @@ func (h *handler) GetObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
}

if settings.VersioningEnabled() && p.VersionID == "" {
headObjectPrm := &layer.HeadObjectParams{
BktInfo: bktInfo,
Object: reqInfo.ObjectName,
shortInfoParams := &layer.ShortInfoParams{
Owner: bktInfo.Owner,
CID: bktInfo.CID,
Object: reqInfo.ObjectName,
}

ei, err := h.obj.GetExtendedObjectInfo(r.Context(), headObjectPrm)
ei, err := h.obj.GetIDForVersioningContainer(r.Context(), shortInfoParams)
if err != nil {
h.logAndSendError(w, "could not find object", reqInfo, err)
return
}

p.VersionID = ei.ObjectInfo.VersionID()
p.VersionID = ei.EncodeToString()
}

lockInfo, err := h.obj.GetLockInfo(r.Context(), p)
Expand Down
18 changes: 4 additions & 14 deletions api/handler/object_list.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/nspcc-dev/neofs-s3-gw/api/s3errors"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
)

// ListObjectsV1Handler handles objects listing requests for API version 1.
Expand Down Expand Up @@ -187,9 +186,9 @@ func parseListObjectArgs(reqInfo *api.ReqInfo) (*layer.ListObjectsParamsCommon,
}

func parseContinuationToken(queryValues url.Values) (string, error) {
// There is a tricky situation. If a continuation-token has been passed, it must not be empty.
if val, ok := queryValues["continuation-token"]; ok {
var objID oid.ID
if err := objID.DecodeString(val[0]); err != nil {
if len(val) == 0 || val[0] == "" {
return "", s3errors.GetAPIError(s3errors.ErrIncorrectContinuationToken)
}
return val[0], nil
Expand All @@ -207,11 +206,11 @@ func fillPrefixes(src []string, encode string) []CommonPrefix {
return dst
}

func fillContentsWithOwner(src []*data.ObjectInfo, encode string) ([]Object, error) {
func fillContentsWithOwner(src []data.ObjectListResponseContent, encode string) ([]Object, error) {
return fillContents(src, encode, true)
}

func fillContents(src []*data.ObjectInfo, encode string, fetchOwner bool) ([]Object, error) {
func fillContents(src []data.ObjectListResponseContent, encode string, fetchOwner bool) ([]Object, error) {
var dst []Object
for _, obj := range src {
res := Object{
Expand All @@ -221,15 +220,6 @@ func fillContents(src []*data.ObjectInfo, encode string, fetchOwner bool) ([]Obj
ETag: obj.HashSum,
}

if size, ok := obj.Headers[layer.AttributeDecryptedSize]; ok {
sz, err := strconv.ParseInt(size, 10, 64)
if err != nil {
return nil, fmt.Errorf("parse decrypted size %s: %w", size, err)
}

res.Size = sz
}

if fetchOwner {
res.Owner = &Owner{
ID: obj.Owner.String(),
Expand Down
19 changes: 1 addition & 18 deletions api/handler/object_list_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,24 +20,6 @@ func TestParseContinuationToken(t *testing.T) {
_, err = parseContinuationToken(queryValues)
require.Error(t, err)
})

t.Run("invalid not empty token", func(t *testing.T) {
var queryValues = map[string][]string{
"continuation-token": {"asd"},
}
_, err = parseContinuationToken(queryValues)
require.Error(t, err)
})

t.Run("valid token", func(t *testing.T) {
tokenStr := "75BTT5Z9o79XuKdUeGqvQbqDnxu6qWcR5EhxW8BXFf8t"
var queryValues = map[string][]string{
"continuation-token": {tokenStr},
}
token, err := parseContinuationToken(queryValues)
require.NoError(t, err)
require.Equal(t, tokenStr, token)
})
}

func TestListObjectNullVersions(t *testing.T) {
Expand Down Expand Up @@ -101,6 +83,7 @@ func TestS3BucketListDelimiterBasic(t *testing.T) {
}

func TestS3BucketListV2DelimiterPrefix(t *testing.T) {
t.Skip("mocked storage implementation requires improvement")
tc := prepareHandlerContext(t)

bktName := "bucket-for-listingv2"
Expand Down
33 changes: 18 additions & 15 deletions api/handler/tagging.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,18 +55,19 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request
}

if settings.VersioningEnabled() && tagPrm.ObjectVersion.VersionID == "" {
headObjectPrm := &layer.HeadObjectParams{
BktInfo: bktInfo,
Object: reqInfo.ObjectName,
shortInfoParams := &layer.ShortInfoParams{
Owner: bktInfo.Owner,
CID: bktInfo.CID,
Object: reqInfo.ObjectName,
}

ei, err := h.obj.GetExtendedObjectInfo(r.Context(), headObjectPrm)
ei, err := h.obj.GetIDForVersioningContainer(r.Context(), shortInfoParams)
if err != nil {
h.logAndSendError(w, "could not find object", reqInfo, err)
return
}

tagPrm.ObjectVersion.VersionID = ei.ObjectInfo.VersionID()
tagPrm.ObjectVersion.VersionID = ei.EncodeToString()
}

if err = h.obj.PutObjectTagging(r.Context(), tagPrm); err != nil {
Expand Down Expand Up @@ -113,18 +114,19 @@ func (h *handler) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request
}

if settings.VersioningEnabled() && tagPrm.ObjectVersion.VersionID == "" {
headObjectPrm := &layer.HeadObjectParams{
BktInfo: bktInfo,
Object: reqInfo.ObjectName,
shortInfoParams := &layer.ShortInfoParams{
Owner: bktInfo.Owner,
CID: bktInfo.CID,
Object: reqInfo.ObjectName,
}

ei, err := h.obj.GetExtendedObjectInfo(r.Context(), headObjectPrm)
ei, err := h.obj.GetIDForVersioningContainer(r.Context(), shortInfoParams)
if err != nil {
h.logAndSendError(w, "could not find object", reqInfo, err)
return
}

tagPrm.ObjectVersion.VersionID = ei.ObjectInfo.VersionID()
tagPrm.ObjectVersion.VersionID = ei.EncodeToString()
}

versionID, tagSet, err := h.obj.GetObjectTagging(r.Context(), tagPrm)
Expand Down Expand Up @@ -163,18 +165,19 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ
}

if settings.VersioningEnabled() && p.VersionID == "" {
headObjectPrm := &layer.HeadObjectParams{
BktInfo: bktInfo,
Object: reqInfo.ObjectName,
shortInfoParams := &layer.ShortInfoParams{
Owner: bktInfo.Owner,
CID: bktInfo.CID,
Object: reqInfo.ObjectName,
}

ei, err := h.obj.GetExtendedObjectInfo(r.Context(), headObjectPrm)
ei, err := h.obj.GetIDForVersioningContainer(r.Context(), shortInfoParams)
if err != nil {
h.logAndSendError(w, "could not find object", reqInfo, err)
return
}

p.VersionID = ei.ObjectInfo.VersionID()
p.VersionID = ei.EncodeToString()
}

if err = h.obj.DeleteObjectTagging(r.Context(), p); err != nil {
Expand Down
5 changes: 4 additions & 1 deletion api/layer/compound.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,10 @@ func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *ObjectV

lockInfo, err = n.getLockDataFromObjects(ctx, objVersion.BktInfo, objVersion.ObjectName, objVersion.VersionID)
if err != nil {
return nil, nil, err
// lock info can be missed - OK. Despite it some tags above may appear, and we should return them.
if !errorsStd.Is(err, ErrNodeNotFound) {
return nil, nil, err
}
}

n.cache.PutTagging(owner, objectTaggingCacheKey(objVersion), tags)
Expand Down
Loading
Loading