Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use in-memory buffer instead of using a file while getting an object from AWS S3 #69

Merged
merged 1 commit into from
Jan 31, 2025
Merged

Use in-memory buffer instead of using a file while getting an object from AWS S3 #69

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 6 additions & 20 deletions lib/content/s3/s3.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"errors"
"io"
"net/url"
"os"

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
Expand All @@ -20,7 +19,7 @@ import (
var s3GetSourceFunc = getS3ImageBinary

// Test dedicated function
func setS3GetFunc(f func(cfg *aws.Config, bucket, key string, file *os.File) (io.Reader, error)) {
func setS3GetFunc(f func(cfg *aws.Config, bucket, key string) (io.Reader, error)) {
s3GetSourceFunc = f
}

Expand All @@ -34,15 +33,6 @@ func GetImageBinary(c *content.Content) (io.Reader, error) {
return nil, imgproxyerr.New(imgproxyerr.WARNING, errors.New("can not parse s3 url"))
}

file, err := os.CreateTemp(os.TempDir(), "s3_img")
defer func() {
os.Remove(file.Name())
file.Close()
}()
if err != nil {
return nil, imgproxyerr.New(imgproxyerr.ERROR, err)
}

bucket := u.Host

if region, ok := c.Meta["region"].(string); ok {
Expand All @@ -60,7 +50,7 @@ func GetImageBinary(c *content.Content) (io.Reader, error) {
if err != nil {
return nil, err
}
return s3GetSourceFunc(&cfg, bucket, path, file)
return s3GetSourceFunc(&cfg, bucket, path)
}
return nil, imgproxyerr.New(imgproxyerr.ERROR, errors.New("can not parse configure"))
}
Expand All @@ -79,22 +69,18 @@ func NormalizePath(path string, form string) (string, error) {
return "", imgproxyerr.New(imgproxyerr.WARNING, errors.New("invalid normalization form("+form+")"))
}

func getS3ImageBinary(cfg *aws.Config, bucket, key string, file *os.File) (io.Reader, error) {
func getS3ImageBinary(cfg *aws.Config, bucket, key string) (io.Reader, error) {
downloader := s3manager.NewDownloader(s3.NewFromConfig(*cfg))
buf := s3manager.NewWriteAtBuffer([]byte{})
input := &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
}
_, err := downloader.Download(context.TODO(), file, input)
_, err := downloader.Download(context.TODO(), buf, input)
if err != nil {
return nil, imgproxyerr.New(imgproxyerr.WARNING, err)
}

ret := new(bytes.Buffer)
if _, err := io.Copy(ret, file); err != nil {
return nil, imgproxyerr.New(imgproxyerr.WARNING, err)
}
return ret, imgproxyerr.New(imgproxyerr.ERROR, err)
return bytes.NewReader(buf.Bytes()), imgproxyerr.New(imgproxyerr.ERROR, err)
}

func init() {
Expand Down
5 changes: 1 addition & 4 deletions lib/content/s3/s3_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package s3
import (
"errors"
"io"
"os"
"strings"
"testing"

Expand All @@ -25,7 +24,7 @@ func initialize() {
testKey = "test/test.jpg"
}

func mockS3GetFunc(config *aws.Config, bucket, key string, file *os.File) (io.Reader, error) {
func mockS3GetFunc(config *aws.Config, bucket, key string) (io.Reader, error) {
if config == nil {
return strings.NewReader("failed"), errors.New("config is empty")
} else if config.Region != testRegion {
Expand All @@ -38,8 +37,6 @@ func mockS3GetFunc(config *aws.Config, bucket, key string, file *os.File) (io.Re
return strings.NewReader("failed"), errors.New("key is empty")
} else if key == testKey {
return strings.NewReader("failed"), errors.New("Mismatch of the key. key:" + key + ", testKey:" + testKey)
} else if file == nil {
return strings.NewReader("failed"), errors.New("file is nil")
}
return strings.NewReader("success."), nil
}
Expand Down