Added a new route for supporting a secondary bucket

.gitignore

@@ -5,3 +5,4 @@ s3-helper/s3-helper
 *coverage.out
 vendor/
 s3-helper
+evs-s3helper

Makefile

@@ -6,6 +6,9 @@ setup:
 build:
 	GOSUMDB=off GOPROXY=direct go build -o $(APPNAME)
 
+test:
+	go test ./... -v
+
 setup-linters:
 	go get -u github.com/alecthomas/gometalinter
 	go get -u github.com/client9/misspell/cmd/misspell

README.md

@@ -6,10 +6,6 @@ bucket).
 
 ## Building
 
-The glide vendoring is used by our build system which relies on https URLs and API keys to access private
-repos.  Sometime after making this public we will drop the use of glide for this, at which point we will
-also remove the vendoring from this repo.
-
 Clone this repo, then:
 
 ```
@@ -22,15 +18,29 @@ make build
 Create a config in `s3-helper.yml`.  Start the service with:
 
 ```
-$ ./s3-helper -config s3-helper.yml
+$ ./s3-helper -config=s3-helper.yml
 ```
 
 Run "s3-helper -h" which list possible flags.
 
+## Usage
+
+```
+\\ for  basic get request (goes to the default media bucket)
+127.0.0.1/{bucket-name}/{object-name}
+
+\\ for get request with byte range
+curl -H "range: bytes=0-199" 127.0.0.1/{bucket-name}/{object-name}
 
+\\ basic get request (goes to the ad media bucket)
+127.0.0.1/avod/{bucket-name}/{object-name}
+
+\\ for get request with byte range
+curl -H "range: bytes=0-199" 127.0.0.1/avod/{bucket-name}/{object-name}
+```
 ## Configuration
 
-s3helper reads its configuration from a file in yml format.  The default location is /mob/etc/s3-helper.yml,
+s3helper reads its configuration from a file in yml format.  The default location is /etc/s3-helper.yml,
 but this can be changed with the -config option, e.g. "-config=./test.yml"
 
 ```yml
@@ -45,13 +55,14 @@ but this can be changed with the -config option, e.g. "-config=./test.yml"
         name:    <newrelic name, default is "">
         license: <newrelic license, default is "">
 
-    s3_bucket:  <name of S3 bucket to forward object requests to>
-    s3_region:  <region of S3 bucket>
-    s3_path:    <optional prefix to prepend to object requests>
-    s3_retries; <maximum number of S3 retries>
-    s3_timeout: <timeout for S3 requests>
+    s3_bucket:    <name of S3 bucket to forward object requests to>
+    s3_ad_bucket: <name of S3 bucket to forward ad object requests to>
+    s3_region:    <region of S3 bucket>
+    s3_path:      <optional prefix to prepend to object requests>
+    s3_retries:   <maximum number of S3 retries>
+    s3_timeout:   <timeout for S3 requests>
 ```
-    
+
 ## Behavior
 
 Assume the configuration consists of:

app.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/pprof"
+	"os"
+
+	"github.com/crunchyroll/evs-s3helper/awsclient"
+	"github.com/rs/zerolog/log"
+)
+
+// PORT - The default port no, used no config doesn't have a port no defined.
+const PORT = 3300
+
+// App - a struct to hold the entire application context
+type App struct {
+	router   *http.ServeMux
+	s3Client *awsclient.S3Client
+}
+
+// Initialize - start the app with a path to config yaml
+func (a *App) Initialize(pprofFlag *bool, s3Region string) {
+	s3Clinet, err := awsclient.NewS3Client(s3Region)
+	if err != nil {
+		fmt.Printf("App failed to initiate due to invalid S3 client. error: %+v", err)
+		os.Exit(1) // kill the app
+	}
+
+	a.s3Client = s3Clinet
+	a.router = http.NewServeMux()
+
+	initRuntime()
+
+	a.router.Handle("/avod/", http.HandlerFunc(a.forwardToS3ForAd))
+	a.router.Handle("/", http.HandlerFunc(forwardToS3ForMedia))
+
+	if *pprofFlag {
+		a.router.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
+		a.router.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
+		a.router.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
+		a.router.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
+		log.Info().Msg("pprof is enabled")
+	}
+
+	log.Info().Msg(fmt.Sprintf("Accepting connections on %v", conf.Listen))
+	return
+}
+
+// Run - run the application with loaded App struct
+func (a *App) Run(port string) {
+	fmt.Printf("App start up initiated.")
+	errLNS := http.ListenAndServe(port, a.router)
+	defer fmt.Print("App shutting down")
+
+	if errLNS != nil {
+		fmt.Printf("App failed to start up. Error: %+v", errLNS)
+		os.Exit(1)
+	}
+}
+
+func (a *App) initializeRoutes() {
+}

awsclient/mock.go

@@ -0,0 +1,32 @@
+package awsclient
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"path"
+
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+)
+
+type mockS3Client struct {
+	s3iface.S3API
+	files map[string]interface{}
+	// files map[string][]byte this is for later
+}
+
+func (m *mockS3Client) GetObject(in *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+	key := path.Join(*in.Bucket, *in.Key)
+	if _, ok := m.files[key]; !ok {
+		return &s3.GetObjectOutput{}, errors.New("Key does not exist")
+	}
+	cLength := int64(1)
+	etag := "this-is-a-dummy-etag"
+	return &s3.GetObjectOutput{
+		Body:          ioutil.NopCloser(bytes.NewReader([]byte(fmt.Sprintf("%v", m.files[key])))),
+		ContentLength: &cLength,
+		ETag:          &etag,
+	}, nil
+}

awsclient/s3.go

@@ -0,0 +1,77 @@
+package awsclient
+
+import (
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+)
+
+// S3Client - manages a persistent connection with downstream S3 bucket
+type S3Client struct {
+	s3Manager s3iface.S3API
+}
+
+// NewS3Client -  creates a new instance for S3Client with a aws session manager
+// embedded inside.
+// The objective of S3Client will allow callers to manager a persistent connection
+// for a given bucket through it's life-time.
+func NewS3Client(region string) (*S3Client, error) {
+	sess, err := session.NewSessionWithOptions(session.Options{
+		Config:            aws.Config{Region: aws.String(region)},
+		SharedConfigState: session.SharedConfigEnable,
+	})
+
+	if err != nil {
+		return &S3Client{}, fmt.Errorf("Failed to initiate an S3Client. Error: %+v", err)
+	}
+
+	return &S3Client{
+		s3Manager: s3.New(sess),
+	}, nil
+}
+
+// GetObjectOutput- constructs the output result from S3 GetObject call
+type GetObjectOutput struct {
+	Body            io.ReadCloser `type:"blob"`
+	CacheControl    *string       `location:"header" locationName:"Cache-Control" type:"string"`
+	ContentEncoding *string       `location:"header" locationName:"Content-Encoding" type:"string"`
+	ContentLength   *int64        `location:"header" locationName:"Content-Length" type:"long"`
+	ContentRange    *string       `location:"header" locationName:"Content-Range" type:"string"`
+	ContentType     *string       `location:"header" locationName:"Content-Type" type:"string"`
+	ETag            *string       `location:"header" locationName:"ETag" type:"string"`
+	Expiration      *string       `location:"header" locationName:"x-amz-expiration" type:"string"`
+	LastModified    *time.Time    `location:"header" locationName:"Last-Modified" type:"timestamp"`
+	StorageClass    *string       `location:"header" locationName:"x-amz-storage-class" type:"string" enum:"StorageClass"`
+	TagCount        *int64        `location:"header" locationName:"x-amz-tagging-count" type:"integer"`
+	VersionId       *string       `location:"header" locationName:"x-amz-version-id" type:"string"`
+}
+
+// GetObject - talks to S3 to get content/byte-range from the bucket
+func (client *S3Client) GetObject(bucket, s3Path, byterange string) (*GetObjectOutput, error) {
+	getInput := &s3.GetObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(s3Path),
+		Range:  aws.String(byterange),
+	}
+
+	result, err := client.s3Manager.GetObject(getInput)
+	if err != nil {
+		return &GetObjectOutput{}, err
+	}
+
+	return &GetObjectOutput{
+		Body:            result.Body,
+		CacheControl:    result.CacheControl,
+		ContentEncoding: result.ContentEncoding,
+		ContentLength:   result.ContentLength,
+		ContentRange:    result.ContentRange,
+		ContentType:     result.ContentType,
+		ETag:            result.ETag,
+		VersionId:       result.VersionId,
+	}, nil
+}

awsclient/s3_test.go

@@ -0,0 +1,117 @@
+package awsclient
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+type media struct {
+	cmsId     string
+	pHash     string
+	createdAt string
+}
+
+func TestS3Client_Success(t *testing.T) {
+	dummyFiles := map[string]interface{}{
+		"avod/mediaId-0": struct{ cmsId string }{"A"},
+		"avod/mediaId-1": media{cmsId: "A", pHash: "abcd-12345", createdAt: "01/01/2021"},
+		"avod/mediaId-2": media{cmsId: "B", pHash: "abcd-1234", createdAt: "02/02/2021"},
+		"svod/mediaId-0": struct{ cmsId string }{"A"},
+		"svod/mediaId-1": media{cmsId: "A", pHash: "abcd-12345", createdAt: "01/01/2021"},
+		"svod/mediaId-2": media{cmsId: "B", pHash: "abcd-1234", createdAt: "02/02/2021"},
+	}
+
+	mS3 := mockS3Client{files: dummyFiles}
+	var getInput *s3.GetObjectInput
+	var getOutput *s3.GetObjectOutput
+	var err error
+
+	getInput = &s3.GetObjectInput{Bucket: aws.String("avod"), Key: aws.String("mediaId-1")}
+	getOutput, err = mS3.GetObject(getInput)
+	if err != nil {
+		t.Fatalf("S3:GetObject response has error: %v", err)
+	}
+
+	if len(*getOutput.ETag) == 0 {
+		t.Fatalf("S3:GetObject response has empty etag: %v", *getOutput)
+	}
+
+	if *getOutput.ContentLength == 0 {
+		t.FailNow()
+	}
+
+	getInput = &s3.GetObjectInput{Bucket: aws.String("avod"), Key: aws.String("mediaId-0")}
+	getOutput, err = mS3.GetObject(getInput)
+	if err != nil {
+		t.Fatalf("S3:GetObject response has error: %v", err)
+	}
+
+	if len(*getOutput.ETag) == 0 {
+		t.Fatalf("S3:GetObject response has empty etag: %v", *getOutput)
+	}
+
+	if *getOutput.ContentLength == 0 {
+		t.FailNow()
+	}
+
+	getInput = &s3.GetObjectInput{Bucket: aws.String("svod"), Key: aws.String("mediaId-1")}
+	getOutput, err = mS3.GetObject(getInput)
+	if err != nil {
+		t.Fatalf("S3:GetObject response has error: %v", err)
+	}
+
+	if len(*getOutput.ETag) == 0 {
+		t.Fatalf("S3:GetObject response has empty etag: %v", *getOutput)
+	}
+
+	if *getOutput.ContentLength == 0 {
+		t.FailNow()
+	}
+
+	getInput = &s3.GetObjectInput{Bucket: aws.String("svod"), Key: aws.String("mediaId-0")}
+	getOutput, err = mS3.GetObject(getInput)
+	if err != nil {
+		t.Fatalf("S3:GetObject response has error: %v", err)
+	}
+
+	if len(*getOutput.ETag) == 0 {
+		t.Fatalf("S3:GetObject response has empty etag: %v", *getOutput)
+	}
+
+	if *getOutput.ContentLength == 0 {
+		t.FailNow()
+	}
+}
+
+func TestS3Client_Failure(t *testing.T) {
+	dummyFiles := map[string]interface{}{
+		"avod/mediaId-0": struct{ cmsId string }{"A"},
+		"avod/mediaId-1": media{cmsId: "A", pHash: "abcd-12345", createdAt: "01/01/2021"},
+		"avod/mediaId-2": media{cmsId: "B", pHash: "abcd-1234", createdAt: "02/02/2021"},
+		"svod/mediaId-0": struct{ cmsId string }{"A"},
+		"svod/mediaId-1": media{cmsId: "A", pHash: "abcd-12345", createdAt: "01/01/2021"},
+		"svod/mediaId-2": media{cmsId: "B", pHash: "abcd-1234", createdAt: "02/02/2021"},
+	}
+
+	mS3 := mockS3Client{files: dummyFiles}
+	var getInput *s3.GetObjectInput
+	var err error
+
+	getInput = &s3.GetObjectInput{Bucket: aws.String("avod"), Key: aws.String("mediaId-3")}
+	_, err = mS3.GetObject(getInput)
+
+	// this should fail, as the key is not present
+	if err == nil {
+		t.FailNow()
+	}
+
+	getInput = &s3.GetObjectInput{Bucket: aws.String("avod"), Key: aws.String("mediaId-3"), Range: aws.String("0-10")}
+	_, err = mS3.GetObject(getInput)
+
+	// this should fail, as the key is not present
+	if err == nil {
+		t.FailNow()
+	}
+}

config.go

@@ -6,11 +6,12 @@ import "time"
 type Config struct {
 	Listen string `yaml:"listen"`
 
-	S3Bucket  string        `yaml:"s3_bucket"`
-	S3Path    string        `yaml:"s3_prefix" optional:"true"`
-	S3Region  string        `yaml:"s3_region"`
-	S3Retries int           `yaml:"s3_retries"`
-	S3Timeout time.Duration `yaml:"s3_timeout"`
+	S3AdBucket string        `yaml:"s3_ad_bucket"`
+	S3Bucket   string        `yaml:"s3_bucket"`
+	S3Path     string        `yaml:"s3_prefix" optional:"true"`
+	S3Region   string        `yaml:"s3_region"`
+	S3Retries  int           `yaml:"s3_retries"`
+	S3Timeout  time.Duration `yaml:"s3_timeout"`
 
 	LogLevel    string `optional:"true"`
 	Concurrency int    `optional:"true"`

go.mod

@@ -3,6 +3,7 @@ module github.com/crunchyroll/evs-s3helper
 go 1.16
 
 require (
+	github.com/aws/aws-sdk-go v1.40.6
 	github.com/crunchyroll/evs-common v0.0.0-20170228001437-6a7a36a07b65
 	github.com/crunchyroll/go-aws-auth v0.0.0-20161116002905-6d1794bd97a6
 	github.com/rs/zerolog v1.13.0