Skip to content
This repository has been archived by the owner on Mar 19, 2024. It is now read-only.

[#76] Added HEAD methods #79

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,8 @@ $ neofs-cli -r 192.168.130.72:8080 -k 6PYLKJhiSub5imt6WCVy6Quxtd9xu176omev1vWYov

#### Requests

The following requests support GET/HEAD methods.

##### By IDs

Basic downloading involves container ID and object ID and is done via GET
Expand Down Expand Up @@ -311,7 +313,7 @@ $ wget http://localhost:8082/get/Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ/2m

#### Replies

You get object contents in the reply body, but at the same time you also get a
You get object contents in the reply body (if GET method was used), but at the same time you also get a
set of reply headers generated using the following rules:
* `Content-Length` is set to the length of the object
* `Content-Type` is autodetected dynamically by gateway
Expand Down
2 changes: 2 additions & 0 deletions app.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,8 +200,10 @@ func (a *app) Serve(ctx context.Context) {
r.POST("/upload/{cid}", a.logger(uploader.Upload))
a.log.Info("added path /upload/{cid}")
r.GET("/get/{cid}/{oid}", a.logger(downloader.DownloadByAddress))
r.HEAD("/get/{cid}/{oid}", a.logger(downloader.HeadByAddress))
a.log.Info("added path /get/{cid}/{oid}")
r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloader.DownloadByAttribute))
r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.logger(downloader.HeadByAttribute))
a.log.Info("added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}")
// enable metrics
if a.cfg.GetBool(cmdMetrics) {
Expand Down
124 changes: 77 additions & 47 deletions downloader/download.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ type (
}
)

var errObjectNotFound = errors.New("object not found")

func newReader(data []byte, err error) *errReader {
return &errReader{data: data, err: err}
}
Expand Down Expand Up @@ -112,7 +114,7 @@ func isValidValue(s string) bool {
return true
}

func (r *request) receiveFile(clnt client.Object, objectAddress *object.Address) {
func (r request) receiveFile(clnt client.Object, objectAddress *object.Address) {
var (
err error
dis = "inline"
Expand All @@ -133,31 +135,9 @@ func (r *request) receiveFile(clnt client.Object, objectAddress *object.Address)
readDetector.Detect()
})

obj, err = clnt.GetObject(
r.RequestCtx,
options,
)
obj, err = clnt.GetObject(r.RequestCtx, options, bearerOpts(r.RequestCtx))
if err != nil {
r.log.Error(
"could not receive object",
zap.Stringer("elapsed", time.Since(start)),
zap.Error(err),
)
var (
msg = fmt.Sprintf("could not receive object: %v", err)
code = fasthttp.StatusBadRequest
cause = err
)
for unwrap := errors.Unwrap(err); unwrap != nil; unwrap = errors.Unwrap(cause) {
cause = unwrap
}
if st, ok := status.FromError(cause); ok && st != nil {
if st.Code() == codes.NotFound {
code = fasthttp.StatusNotFound
}
msg = st.Message()
}
r.Error(msg, code)
r.handleNeoFSErr(err, start)
return
}
if r.Request.URI().QueryArgs().GetBool("download") {
Expand Down Expand Up @@ -209,6 +189,36 @@ func (r *request) receiveFile(clnt client.Object, objectAddress *object.Address)
r.Response.Header.Set("Content-Disposition", dis+"; filename="+path.Base(filename))
}

func bearerOpts(ctx context.Context) client.CallOption {
if tkn, err := tokens.LoadBearerToken(ctx); err == nil {
return client.WithBearer(tkn)
}
return client.WithBearer(nil)
}

func (r *request) handleNeoFSErr(err error, start time.Time) {
r.log.Error(
"could not receive object",
zap.Stringer("elapsed", time.Since(start)),
zap.Error(err),
)
var (
msg = fmt.Sprintf("could not receive object: %v", err)
code = fasthttp.StatusBadRequest
cause = err
)
for unwrap := errors.Unwrap(err); unwrap != nil; unwrap = errors.Unwrap(cause) {
cause = unwrap
}
if st, ok := status.FromError(cause); ok && st != nil {
if st.Code() == codes.NotFound {
code = fasthttp.StatusNotFound
}
msg = st.Message()
}
r.Error(msg, code)
}

func (o objectIDs) Slice() []string {
res := make([]string, 0, len(o))
for _, oid := range o {
Expand Down Expand Up @@ -242,53 +252,74 @@ func (d *Downloader) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *requ

// DownloadByAddress handles download requests using simple cid/oid format.
func (d *Downloader) DownloadByAddress(c *fasthttp.RequestCtx) {
d.byAddress(c, request.receiveFile)
}

// byAddress is wrapper for function (e.g. request.headObject, request.receiveFile) that
// prepares request and object address to it.
func (d *Downloader) byAddress(c *fasthttp.RequestCtx, f func(request, client.Object, *object.Address)) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It will be appreciated to see some comments in code about f here and reference it in byAttribute.
Alternatively you can write comments about byAddress function that it prepares input data for f() to process. It will help for other developers to get into it.

var (
err error
address = object.NewAddress()
cid, _ = c.UserValue("cid").(string)
oid, _ = c.UserValue("oid").(string)
val = strings.Join([]string{cid, oid}, "/")
log = d.log.With(zap.String("cid", cid), zap.String("oid", oid))
)
if err = address.Parse(val); err != nil {
if err := address.Parse(val); err != nil {
log.Error("wrong object address", zap.Error(err))
c.Error("wrong object address", fasthttp.StatusBadRequest)
return
}

d.newRequest(c, log).receiveFile(d.pool, address)
f(*d.newRequest(c, log), d.pool, address)
}

// DownloadByAttribute handles attribute-based download requests.
func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) {
d.byAttribute(c, request.receiveFile)
}

// byAttribute is wrapper similar to byAddress.
func (d *Downloader) byAttribute(c *fasthttp.RequestCtx, f func(request, client.Object, *object.Address)) {
var (
err error
scid, _ = c.UserValue("cid").(string)
key, _ = c.UserValue("attr_key").(string)
val, _ = c.UserValue("attr_val").(string)
log = d.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val))
ids []*object.ID
httpStatus = fasthttp.StatusBadRequest
scid, _ = c.UserValue("cid").(string)
key, _ = c.UserValue("attr_key").(string)
val, _ = c.UserValue("attr_val").(string)
log = d.log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val))
)
cid := cid.New()
if err = cid.Parse(scid); err != nil {
containerID := cid.New()
if err := containerID.Parse(scid); err != nil {
log.Error("wrong container id", zap.Error(err))
c.Error("wrong container id", fasthttp.StatusBadRequest)
c.Error("wrong container id", httpStatus)
return
}

address, err := d.searchObject(c, log, containerID, key, val)
if err != nil {
log.Error("couldn't search object", zap.Error(err))
if errors.Is(err, errObjectNotFound) {
httpStatus = fasthttp.StatusNotFound
}
c.Error("couldn't search object", httpStatus)
return
}

f(*d.newRequest(c, log), d.pool, address)
}

func (d *Downloader) searchObject(c *fasthttp.RequestCtx, log *zap.Logger, cid *cid.ID, key, val string) (*object.Address, error) {
options := object.NewSearchFilters()
options.AddRootFilter()
options.AddFilter(key, val, object.MatchStringEqual)

sops := new(client.SearchObjectParams).WithContainerID(cid).WithSearchFilters(options)
if ids, err = d.pool.SearchObject(c, sops); err != nil {
log.Error("something went wrong", zap.Error(err))
c.Error("something went wrong", fasthttp.StatusBadRequest)
return
} else if len(ids) == 0 {
log.Debug("object not found")
c.Error("object not found", fasthttp.StatusNotFound)
return
ids, err := d.pool.SearchObject(c, sops)
if err != nil {
return nil, err
}
if len(ids) == 0 {
return nil, errObjectNotFound
}
if len(ids) > 1 {
log.Debug("found multiple objects",
Expand All @@ -298,6 +329,5 @@ func (d *Downloader) DownloadByAttribute(c *fasthttp.RequestCtx) {
address := object.NewAddress()
address.SetContainerID(cid)
address.SetObjectID(ids[0])

d.newRequest(c, log).receiveFile(d.pool, address)
return address, nil
}
88 changes: 88 additions & 0 deletions downloader/head.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
package downloader

import (
"net/http"
"strconv"
"time"

"github.com/nspcc-dev/neofs-api-go/pkg/client"
"github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-http-gw/tokens"
"github.com/valyala/fasthttp"
"go.uber.org/zap"
)

const sizeToDetectType = 512

func (r request) headObject(clnt client.Object, objectAddress *object.Address) {
var start = time.Now()
if err := tokens.StoreBearerToken(r.RequestCtx); err != nil {
r.log.Error("could not fetch and store bearer token", zap.Error(err))
r.Error("could not fetch and store bearer token", fasthttp.StatusBadRequest)
return
}

options := new(client.ObjectHeaderParams).WithAddress(objectAddress)
bearerOpt := bearerOpts(r.RequestCtx)
obj, err := clnt.GetObjectHeader(r.RequestCtx, options, bearerOpt)
if err != nil {
r.handleNeoFSErr(err, start)
return
}

r.Response.Header.Set("Content-Length", strconv.FormatUint(obj.PayloadSize(), 10))
var contentType string
for _, attr := range obj.Attributes() {
key := attr.Key()
val := attr.Value()
if !isValidToken(key) || !isValidValue(val) {
continue
}
r.Response.Header.Set("X-Attribute-"+key, val)
switch key {
case object.AttributeTimestamp:
value, err := strconv.ParseInt(val, 10, 64)
if err != nil {
r.log.Info("couldn't parse creation date",
zap.String("key", key),
zap.String("val", val),
zap.Error(err))
continue
}
r.Response.Header.Set("Last-Modified", time.Unix(value, 0).UTC().Format(http.TimeFormat))
case object.AttributeContentType:
contentType = val
}
}
r.Response.Header.Set("x-object-id", obj.ID().String())
r.Response.Header.Set("x-owner-id", obj.OwnerID().String())
r.Response.Header.Set("x-container-id", obj.ContainerID().String())

if len(contentType) == 0 {
objRange := object.NewRange()
objRange.SetOffset(0)
if sizeToDetectType < obj.PayloadSize() {
objRange.SetLength(sizeToDetectType)
} else {
objRange.SetLength(obj.PayloadSize())
}
ops := new(client.RangeDataParams).WithAddress(objectAddress).WithRange(objRange)
data, err := clnt.ObjectPayloadRangeData(r.RequestCtx, ops, bearerOpt)
if err != nil {
r.handleNeoFSErr(err, start)
return
}
contentType = http.DetectContentType(data)
}
r.SetContentType(contentType)
}

// HeadByAddress handles head requests using simple cid/oid format.
func (d *Downloader) HeadByAddress(c *fasthttp.RequestCtx) {
d.byAddress(c, request.headObject)
}

// HeadByAttribute handles attribute-based head requests.
func (d *Downloader) HeadByAttribute(c *fasthttp.RequestCtx) {
d.byAttribute(c, request.headObject)
}