diff --git a/handlers/hash_utils.go b/handlers/hash_utils.go index b21dfbe..7e95d3b 100644 --- a/handlers/hash_utils.go +++ b/handlers/hash_utils.go @@ -6,7 +6,6 @@ import ( "crypto/sha256" "crypto/sha512" "encoding/hex" - "fmt" "hash" "hash/crc32" "io" @@ -63,24 +62,24 @@ func CalculateBasicHashes(client *AwsClient, bucket string, key string) (*HashIn } log.Printf("Size %d", *objectSize) - start := int64(0) - step := int64(ChunkSize) + result, _ := GetS3ObjectOutput(client, bucket, key) + p := make([]byte, ChunkSize) + for { - chunkRange := fmt.Sprintf("bytes: %d-%d", start, minOf(start+step, *objectSize-1)) + n, err := result.Body.Read(p) - buff, err := GetChunkDataFromS3(client, bucket, key, chunkRange) - if err != nil { - log.Printf("Can not stream chunk data of %s. Detail %s\n\n", key, err) - return nil, -1, err + if err != nil && err != io.EOF { + return nil, int64(-1), err } - hashCollection, err = UpdateBasicHashes(hashCollection, buff) - - if err != nil { - log.Printf("Can not compute hashes. Detail %s\n\n", err) + var err2 error + hashCollection, err2 = UpdateBasicHashes(hashCollection, p[:n]) + if err2 != nil { + log.Printf("Can not update hashes. Detail %s\n\n", err2) + return nil, int64(-1), err2 } - start = minOf(start+step, *objectSize-1) + 1 - if start >= *objectSize { + + if err == io.EOF { break } } @@ -98,7 +97,6 @@ func CalculateBasicHashes(client *AwsClient, bucket string, key string) (*HashIn // UpdateBasicHashes updates a hashes collection func UpdateBasicHashes(hashCollection *HashCollection, rd []byte) (*HashCollection, error) { - hashCollection.Reset() multiWriter := io.MultiWriter(hashCollection.Crc32c, hashCollection.Md5, hashCollection.Sha1, hashCollection.Sha256, hashCollection.Sha512) _, err := multiWriter.Write(rd) diff --git a/handlers/s3client.go b/handlers/s3client.go index 00dc4d8..894d7dd 100644 --- a/handlers/s3client.go +++ b/handlers/s3client.go @@ -2,7 +2,6 @@ package handlers import ( "fmt" - "io/ioutil" "os" "github.com/aws/aws-sdk-go/aws" @@ -35,14 +34,13 @@ func CreateNewAwsClient() (*AwsClient, error) { return client, nil } -// GetChunkDataFromS3 downloads chunk data from s3 -func GetChunkDataFromS3(client *AwsClient, bucket string, key string, byteRange string) ([]byte, error) { +// GetS3ObjectOutput gets object output from s3 +func GetS3ObjectOutput(client *AwsClient, bucket string, key string) (*s3.GetObjectOutput, error) { svc := s3.New(client.session) input := &s3.GetObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), - Range: aws.String(byteRange), } result, err := svc.GetObject(input) @@ -61,11 +59,8 @@ func GetChunkDataFromS3(client *AwsClient, bucket string, key string, byteRange } return nil, err } - body, err := ioutil.ReadAll(result.Body) - if err != nil { - fmt.Println(err.Error()) - } - return body, nil + + return result, nil } diff --git a/pull_request_template.md b/pull_request_template.md new file mode 100644 index 0000000..4105e39 --- /dev/null +++ b/pull_request_template.md @@ -0,0 +1,21 @@ +Description about what this pull request does. + +Please make sure to follow the [DEV guidelines](https://gen3.org/resources/developer/dev-introduction/) before asking for review. + +### New Features +- Implemented XXX + +### Breaking Changes + + +### Bug Fixes + + +### Improvements + + +### Dependency updates + + +### Deployment changes +