Skip to content

Commit

Permalink
Port provider-specific logic to AWS SDK V2
Browse files Browse the repository at this point in the history
* aws sdk v1 will no longer work with etags, and is being deprecated soon anyways.
  • Loading branch information
bdon committed Sep 10, 2024
1 parent 618f3b4 commit 68710d4
Show file tree
Hide file tree
Showing 3 changed files with 36 additions and 26 deletions.
8 changes: 4 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2
github.com/RoaringBitmap/roaring v1.5.0
github.com/alecthomas/kong v0.8.0
github.com/aws/aws-sdk-go v1.55.5
github.com/aws/aws-sdk-go-v2 v1.30.3
github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3
github.com/aws/smithy-go v1.20.3
github.com/caddyserver/caddy/v2 v2.8.4
github.com/cespare/xxhash/v2 v2.3.0
github.com/dustin/go-humanize v1.0.1
Expand Down Expand Up @@ -42,7 +44,7 @@ require (
github.com/Microsoft/go-winio v0.6.0 // indirect
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
github.com/aryann/difflib v0.0.0-20210328193216-ff5ff6dc229b // indirect
github.com/aws/aws-sdk-go-v2 v1.30.3 // indirect
github.com/aws/aws-sdk-go v1.55.5 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect
github.com/aws/aws-sdk-go-v2/config v1.27.27 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect
Expand All @@ -56,11 +58,9 @@ require (
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect
github.com/aws/smithy-go v1.20.3 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.2.0 // indirect
github.com/caddyserver/certmagic v0.21.3 // indirect
Expand Down
29 changes: 17 additions & 12 deletions pmtiles/bucket.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
smithyHttp "github.com/aws/smithy-go/transport/http"
"github.com/cespare/xxhash/v2"
"gocloud.dev/blob"
"google.golang.org/api/googleapi"
Expand Down Expand Up @@ -227,12 +227,17 @@ func generationToEtag(generation int64) string {
return strconv.FormatInt(generation, 10)
}

func smithyErrorCode(errorCode string) int {
i, _ := strconv.Atoi(errorCode)
return i
}

func setProviderEtag(asFunc func(interface{}) bool, etag string) {
var awsV1Req *s3.GetObjectInput
var awsV2Req *s3.GetObjectInput
var azblobReq *azblob.DownloadStreamOptions
var gcsHandle **storage.ObjectHandle
if asFunc(&awsV1Req) {
awsV1Req.IfMatch = aws.String(etag)
if asFunc(&awsV2Req) {
awsV2Req.IfMatch = aws.String(etag)
} else if asFunc(&azblobReq) {
azEtag := azcore.ETag(etag)
azblobReq.AccessConditions = &azblob.AccessConditions{
Expand All @@ -248,12 +253,12 @@ func setProviderEtag(asFunc func(interface{}) bool, etag string) {
}

func getProviderErrorStatusCode(err error) int {
var awsV1Err awserr.RequestFailure
var awsV2Err *smithyHttp.ResponseError
var azureErr *azcore.ResponseError
var gcpErr *googleapi.Error

if errors.As(err, &awsV1Err); awsV1Err != nil {
return awsV1Err.StatusCode()
if errors.As(err, &awsV2Err); awsV2Err != nil {
return awsV2Err.HTTPStatusCode()
} else if errors.As(err, &azureErr); azureErr != nil {
return azureErr.StatusCode
} else if errors.As(err, &gcpErr); gcpErr != nil {
Expand All @@ -263,12 +268,12 @@ func getProviderErrorStatusCode(err error) int {
}

func getProviderEtag(reader *blob.Reader) string {
var awsV1Resp s3.GetObjectOutput
var awsV2Resp s3.GetObjectOutput
var azureResp azblob.DownloadStreamResponse
var gcpResp *storage.Reader

if reader.As(&awsV1Resp) {
return *awsV1Resp.ETag
if reader.As(&awsV2Resp) {
return *awsV2Resp.ETag
} else if reader.As(&azureResp) {
return string(*azureResp.ETag)
} else if reader.As(&gcpResp) {
Expand Down
25 changes: 15 additions & 10 deletions pmtiles/bucket_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,11 @@ import (

"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/s3"

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
smithyHttp "github.com/aws/smithy-go/transport/http"

"github.com/stretchr/testify/assert"
_ "gocloud.dev/blob/fileblob"
"google.golang.org/api/googleapi"
Expand Down Expand Up @@ -214,18 +216,18 @@ func TestFileShorterThan16K(t *testing.T) {
assert.Equal(t, 3, len(data))
}

func TestSetProviderEtagAws(t *testing.T) {
var awsV1Req s3.GetObjectInput
assert.Nil(t, awsV1Req.IfMatch)
func TestSetProviderEtagAwsV2(t *testing.T) {
var awsV2Req s3.GetObjectInput
assert.Nil(t, awsV2Req.IfMatch)
asFunc := func(i interface{}) bool {
v, ok := i.(**s3.GetObjectInput)
if ok {
*v = &awsV1Req
*v = &awsV2Req
}
return true
}
setProviderEtag(asFunc, "123")
assert.Equal(t, aws.String("123"), awsV1Req.IfMatch)
assert.Equal(t, aws.String("123"), awsV2Req.IfMatch)
}

func TestSetProviderEtagAzure(t *testing.T) {
Expand All @@ -243,8 +245,11 @@ func TestSetProviderEtagAzure(t *testing.T) {
}

func TestGetProviderErrorStatusCode(t *testing.T) {
awsErr := awserr.NewRequestFailure(awserr.New("", "", nil), 500, "")
statusCode := getProviderErrorStatusCode(awsErr)
awsV2Err := &smithyHttp.ResponseError{Response: &smithyHttp.Response{&http.Response{

Check failure on line 248 in pmtiles/bucket_test.go

View workflow job for this annotation

GitHub Actions / fmt_vet_lint

github.com/aws/smithy-go/transport/http.Response struct literal uses unkeyed fields
StatusCode: 500,
Header: http.Header{},
}}}
statusCode := getProviderErrorStatusCode(awsV2Err)
assert.Equal(t, 500, statusCode)

azureErr := &azcore.ResponseError{StatusCode: 500}
Expand Down

0 comments on commit 68710d4

Please sign in to comment.