Merge pull request #12 from uc-cdis/fix/oom_error

Fix/oom error

handlers/hash_utils.go

@@ -6,7 +6,6 @@ import (
 	"crypto/sha256"
 	"crypto/sha512"
 	"encoding/hex"
-	"fmt"
 	"hash"
 	"hash/crc32"
 	"io"
@@ -63,24 +62,24 @@ func CalculateBasicHashes(client *AwsClient, bucket string, key string) (*HashIn
 	}
 	log.Printf("Size %d", *objectSize)
 
-	start := int64(0)
-	step := int64(ChunkSize)
+	result, _ := GetS3ObjectOutput(client, bucket, key)
+	p := make([]byte, ChunkSize)
+
 	for {
-		chunkRange := fmt.Sprintf("bytes: %d-%d", start, minOf(start+step, *objectSize-1))
+		n, err := result.Body.Read(p)
 
-		buff, err := GetChunkDataFromS3(client, bucket, key, chunkRange)
-		if err != nil {
-			log.Printf("Can not stream chunk data of %s. Detail %s\n\n", key, err)
-			return nil, -1, err
+		if err != nil && err != io.EOF {
+			return nil, int64(-1), err
 		}
 
-		hashCollection, err = UpdateBasicHashes(hashCollection, buff)
-
-		if err != nil {
-			log.Printf("Can not compute hashes. Detail %s\n\n", err)
+		var err2 error
+		hashCollection, err2 = UpdateBasicHashes(hashCollection, p[:n])
+		if err2 != nil {
+			log.Printf("Can not update hashes. Detail %s\n\n", err2)
+			return nil, int64(-1), err2
 		}
-		start = minOf(start+step, *objectSize-1) + 1
-		if start >= *objectSize {
+
+		if err == io.EOF {
 			break
 		}
 	}
@@ -98,7 +97,6 @@ func CalculateBasicHashes(client *AwsClient, bucket string, key string) (*HashIn
 // UpdateBasicHashes updates a hashes collection
 func UpdateBasicHashes(hashCollection *HashCollection, rd []byte) (*HashCollection, error) {
 
-	hashCollection.Reset()
 	multiWriter := io.MultiWriter(hashCollection.Crc32c, hashCollection.Md5, hashCollection.Sha1, hashCollection.Sha256, hashCollection.Sha512)
 	_, err := multiWriter.Write(rd)
 

handlers/s3client.go

@@ -2,7 +2,6 @@ package handlers
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -35,14 +34,13 @@ func CreateNewAwsClient() (*AwsClient, error) {
 	return client, nil
 }
 
-// GetChunkDataFromS3 downloads chunk data from s3
-func GetChunkDataFromS3(client *AwsClient, bucket string, key string, byteRange string) ([]byte, error) {
+// GetS3ObjectOutput gets object output from s3
+func GetS3ObjectOutput(client *AwsClient, bucket string, key string) (*s3.GetObjectOutput, error) {
 
 	svc := s3.New(client.session)
 	input := &s3.GetObjectInput{
 		Bucket: aws.String(bucket),
 		Key:    aws.String(key),
-		Range:  aws.String(byteRange),
 	}
 
 	result, err := svc.GetObject(input)
@@ -61,11 +59,8 @@ func GetChunkDataFromS3(client *AwsClient, bucket string, key string, byteRange
 		}
 		return nil, err
 	}
-	body, err := ioutil.ReadAll(result.Body)
-	if err != nil {
-		fmt.Println(err.Error())
-	}
-	return body, nil
+
+	return result, nil
 
 }
 

pull_request_template.md

@@ -0,0 +1,21 @@
+Description about what this pull request does.
+
+Please make sure to follow the [DEV guidelines](https://gen3.org/resources/developer/dev-introduction/) before asking for review.
+
+### New Features
+- Implemented XXX
+
+### Breaking Changes
+
+
+### Bug Fixes
+
+
+### Improvements
+
+
+### Dependency updates
+
+
+### Deployment changes
+