From 69757db396ced27b508a404d111f57b0669a3c1c Mon Sep 17 00:00:00 2001 From: Som Poddar Date: Thu, 22 Jul 2021 00:11:28 -0700 Subject: [PATCH] Added a new route for supporting a secondary bucket - crunchyroll would like to extend this utility to support avod streaming in the platform. - This change adds this capability with the minimal amount of change to the existing functionality. - Updated to use aws sdk for s3 get call --- .gitignore | 1 + README.md | 24 +++++++++---- config.go | 11 +++--- go.mod | 1 + go.sum | 91 +++++++++++------------------------------------ main.go | 4 ++- s3.go | 57 +++++++++++++++++++++++++---- s3client/get.go | 41 +++++++++++++++++++++ s3client/types.go | 22 ++++++++++++ 9 files changed, 162 insertions(+), 90 deletions(-) create mode 100644 s3client/get.go create mode 100644 s3client/types.go diff --git a/.gitignore b/.gitignore index ae8de62..2a81fce 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ s3-helper/s3-helper *coverage.out vendor/ s3-helper +evs-s3helper diff --git a/README.md b/README.md index 1349a00..250c116 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,6 @@ bucket). ## Building -The glide vendoring is used by our build system which relies on https URLs and API keys to access private -repos. Sometime after making this public we will drop the use of glide for this, at which point we will -also remove the vendoring from this repo. - Clone this repo, then: ``` @@ -22,15 +18,29 @@ make build Create a config in `s3-helper.yml`. Start the service with: ``` -$ ./s3-helper -config s3-helper.yml +$ ./s3-helper -config=s3-helper.yml ``` Run "s3-helper -h" which list possible flags. +## Usage + +``` +\\ for basic get request (goes to the default media bucket) +127.0.0.1/{bucket-name}/{object-name} + +\\ for get request with byte range +curl -H "range: bytes=0-199" 127.0.0.1/{bucket-name}/{object-name} +\\ basic get request (goes to the ad media bucket) +127.0.0.1/avod/{bucket-name}/{object-name} + +\\ for get request with byte range +curl -H "range: bytes=0-199" 127.0.0.1/avod/{bucket-name}/{object-name} +``` ## Configuration -s3helper reads its configuration from a file in yml format. The default location is /mob/etc/s3-helper.yml, +s3helper reads its configuration from a file in yml format. The default location is /etc/s3-helper.yml, but this can be changed with the -config option, e.g. "-config=./test.yml" ```yml @@ -51,7 +61,7 @@ but this can be changed with the -config option, e.g. "-config=./test.yml" s3_retries; s3_timeout: ``` - + ## Behavior Assume the configuration consists of: diff --git a/config.go b/config.go index bd06cc3..8d092b1 100644 --- a/config.go +++ b/config.go @@ -6,11 +6,12 @@ import "time" type Config struct { Listen string `yaml:"listen"` - S3Bucket string `yaml:"s3_bucket"` - S3Path string `yaml:"s3_prefix" optional:"true"` - S3Region string `yaml:"s3_region"` - S3Retries int `yaml:"s3_retries"` - S3Timeout time.Duration `yaml:"s3_timeout"` + S3AdBucket string `yaml:"s3_ad_bucket"` + S3Bucket string `yaml:"s3_bucket"` + S3Path string `yaml:"s3_prefix" optional:"true"` + S3Region string `yaml:"s3_region"` + S3Retries int `yaml:"s3_retries"` + S3Timeout time.Duration `yaml:"s3_timeout"` LogLevel string `optional:"true"` Concurrency int `optional:"true"` diff --git a/go.mod b/go.mod index b75ac57..8cd7fdd 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/crunchyroll/evs-s3helper go 1.16 require ( + github.com/aws/aws-sdk-go v1.40.6 github.com/crunchyroll/evs-common v0.0.0-20170228001437-6a7a36a07b65 github.com/crunchyroll/go-aws-auth v0.0.0-20161116002905-6d1794bd97a6 github.com/rs/zerolog v1.13.0 diff --git a/go.sum b/go.sum index 5c61544..ea4c4b0 100644 --- a/go.sum +++ b/go.sum @@ -1,101 +1,50 @@ -cloud.google.com/go v0.16.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/aws/aws-sdk-go v1.40.6 h1:JCQfi5MD8cW0PCAzr88hj9tj4BdEJkAy8EyAJ6c8I/k= +github.com/aws/aws-sdk-go v1.40.6/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/crunchyroll/evs-common v0.0.0-20170228001437-6a7a36a07b65 h1:ItAoFc7R9t5fCJTxPE9dIJ8xD8ZEPsdH5cs/URSWGfw= github.com/crunchyroll/evs-common v0.0.0-20170228001437-6a7a36a07b65/go.mod h1:btgFZfUbbCA3kbQpTcqarXOHlow9yR4eLthCGIsnvjc= -github.com/crunchyroll/evs-common v1.3.3 h1:PyKz1rj7Jf78sTgtRrSPfwt2nmkYXBauJhLCzipg5EY= -github.com/crunchyroll/evs-common v1.3.3/go.mod h1:btgFZfUbbCA3kbQpTcqarXOHlow9yR4eLthCGIsnvjc= github.com/crunchyroll/go-aws-auth v0.0.0-20161116002905-6d1794bd97a6 h1:/X6ZeGc3CciMpUqBBRBpzty4GUhOxf25Lijo1K56f3k= github.com/crunchyroll/go-aws-auth v0.0.0-20161116002905-6d1794bd97a6/go.mod h1:Umr4qTpP68u5nM9bAOk2NGXdVHYlNk8ayhDUVW+P2dM= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fsnotify/fsnotify v1.4.3-0.20170329110642-4da3e2cfbabc/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= -github.com/go-stack/stack v1.6.0 h1:MmJCxYVKTJ0SplGKqFVX3SBnmaUhODHZrrFF6jMbpZk= -github.com/go-stack/stack v1.6.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f h1:16RtHeWGkJMc80Etb8RPCcKevXGldr57+LOyZt8zOlg= -github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f/go.mod h1:ijRvpgDJDI262hYq/IQVYgf8hd8IHUs93Ol0kvMBAx4= -github.com/golang/lint v0.0.0-20170918230701-e5d664eb928e/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.1.1-0.20171103154506-982329095285/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gregjones/httpcache v0.0.0-20170920190843-316c5e0ff04e/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/hashicorp/hcl v0.0.0-20170914154624-68e816d1c783/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= -github.com/inconshreveable/log15 v0.0.0-20170622235902-74a0988b5f80 h1:g/SJtZVYc1cxSB8lgrgqeOlIdi4MhqNNHYRAC8y+g4c= -github.com/inconshreveable/log15 v0.0.0-20170622235902-74a0988b5f80/go.mod h1:cOaXtrgN4ScfRrD9Bre7U1thNq5RtJ8ZoP4iXVGRj6o= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/magiconair/properties v1.7.4-0.20170902060319-8d7837e64d3c/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mattn/go-colorable v0.0.10-0.20170816031813-ad5389df28cd h1:Y4ZRx+RIPFlPL4gnD/I7bdqSNXHlNop1Q6NjQuHds00= -github.com/mattn/go-colorable v0.0.10-0.20170816031813-ad5389df28cd/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.2 h1:F+DnWktyadxnOrohKLNUC9/GjFii5RJgY4GFG6ilggw= -github.com/mattn/go-isatty v0.0.2/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mitchellh/mapstructure v0.0.0-20170523030023-d0303fe80992/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/pelletier/go-toml v1.0.1-0.20170904195809-1d6b12b7cb29/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0 h1:hSNcYHyxDWycfePW7pUI8swuFkcSMPKh3E63Pokg1Hk= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= -github.com/rs/zerolog v1.23.0 h1:UskrK+saS9P9Y789yNNulYKdARjPZuS35B8gJF2x60g= -github.com/rs/zerolog v1.23.0/go.mod h1:6c7hFfxPOy7TacJc4Fcdi24/J0NKYGzjG8FWRI916Qo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/spf13/afero v0.0.0-20170901052352-ee1bd8ee15a1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.1.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= -github.com/spf13/jwalterweatherman v0.0.0-20170901151539-12bd96e66386/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.1-0.20170901120850-7aff26db30c1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.0.0/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/sync v0.0.0-20170517211232-f52d1811a629/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20170424234030-8be79e1e0910/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.0.0-20170921000349-586095a6e407/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20170918111702-1e559d0a00ee/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/grpc v1.2.1-0.20170921194603-d4b75ebd4f9f/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/main.go b/main.go index 8e5a3ea..0ba96dc 100644 --- a/main.go +++ b/main.go @@ -63,7 +63,9 @@ func main() { initRuntime() mux := http.NewServeMux() - mux.Handle("/", http.HandlerFunc(forwardToS3)) + + mux.Handle("/avod/", http.HandlerFunc(forwardToS3ForAd)) + mux.Handle("/", http.HandlerFunc(forwardToS3ForMedia)) if *pprofFlag { mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index)) diff --git a/s3.go b/s3.go index 41c7f3d..80e226e 100644 --- a/s3.go +++ b/s3.go @@ -1,4 +1,3 @@ -// s3-helper is used to assist nginx with various AWS related tasks package main import ( @@ -10,7 +9,8 @@ import ( "strings" "time" - "github.com/crunchyroll/go-aws-auth" + "github.com/crunchyroll/evs-s3helper/s3client" + awsauth "github.com/crunchyroll/go-aws-auth" "github.com/rs/zerolog/log" ) @@ -18,12 +18,12 @@ var statRate float32 = 1 // List of headers to forward in response var headerForward = map[string]bool{ - "Date": true, "Content-Length": true, "Content-Range": true, "Content-Type": true, - "Last-Modified": true, + "Date": true, "ETag": true, + "Last-Modified": true, } const serverName = "VOD S3 Helper" @@ -41,7 +41,52 @@ func initRuntime() { runtime.GOMAXPROCS(conc) } -func forwardToS3(w http.ResponseWriter, r *http.Request) { +func forwardToS3ForAd(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Server", serverName) + + if r.Method != "GET" && r.Method != "HEAD" { + w.WriteHeader(405) + return + } + + // Make sure that Remote Address is 127.0.0.1 so it comes off a local proxy + a := strings.SplitN(r.RemoteAddr, ":", 2) + if len(a) != 2 || a[0] != "127.0.0.1" { + w.WriteHeader(403) + return + } + + p := []rune(r.URL.Path) + s3Path := string(p[6:]) + byterange := r.Header.Get("Range") + logger := log.With(). + Str("object", s3Path).Str("range", byterange).Str("method", r.Method).Logger() + + getObject, getErr := s3client.GetObject(conf.S3AdBucket, s3Path, byterange) + if getErr != nil { + logger.Error(). + Str("error", getErr.Error()). + Msg(fmt.Sprintf("s3:Get:Err - path:%s ", s3Path)) + w.WriteHeader(500) + return + } + + w.WriteHeader(200) + w.Header().Set("Content-Length", fmt.Sprintf("%d", *getObject.ContentLength)) + w.Header().Set("Content-Range", *getObject.ContentRange) + w.Header().Set("Content-Type", *getObject.ContentType) + w.Header().Set("ETag", *getObject.ETag) + w.Header().Set("Last-Modified", getObject.LastModified.Format(time.RFC3339)) + + io.Copy(w, getObject.Body) + logger.Info().Str("path", s3Path).Int64("content-length", *getObject.ContentLength).Msg("s3:get - success") +} + +func forwardToS3ForMedia(w http.ResponseWriter, r *http.Request) { + forwardToS3(w, r, conf.S3Bucket) +} + +func forwardToS3(w http.ResponseWriter, r *http.Request, bucket string) { w.Header().Set("Server", serverName) if r.Method != "GET" && r.Method != "HEAD" { @@ -63,7 +108,7 @@ func forwardToS3(w http.ResponseWriter, r *http.Request) { Str("range", byterange). Str("method", r.Method). Logger() - s3url := fmt.Sprintf("http://s3-%s.amazonaws.com/%s%s%s", conf.S3Region, conf.S3Bucket, conf.S3Path, upath) + s3url := fmt.Sprintf("http://s3-%s.amazonaws.com/%s%s%s", conf.S3Region, bucket, conf.S3Path, upath) r2, err := http.NewRequest(r.Method, s3url, nil) if err != nil { w.WriteHeader(403) diff --git a/s3client/get.go b/s3client/get.go new file mode 100644 index 0000000..da781d5 --- /dev/null +++ b/s3client/get.go @@ -0,0 +1,41 @@ +package s3client + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" +) + +// GetObject - talks to S3 to get content/byte-range from the bucket +func GetObject(bucket, s3Path, byterange string) (*GetObjectOutput, error) { + getInput := &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(s3Path), + Range: aws.String(byterange), + } + + sess, err := session.NewSessionWithOptions(session.Options{ + Config: aws.Config{Region: aws.String("us-west-2")}, + SharedConfigState: session.SharedConfigEnable, + }) + + s3Manager := s3.New(sess) + result, err := s3Manager.GetObject(getInput) + if err != nil { + return &GetObjectOutput{}, err + } + + return &GetObjectOutput{ + Body: result.Body, + CacheControl: result.CacheControl, + ContentEncoding: result.ContentEncoding, + ContentLength: result.ContentLength, + ContentRange: result.ContentRange, + ContentType: result.ContentType, + ETag: result.ETag, + Expiration: result.Expiration, + LastModified: result.LastModified, + StorageClass: result.StorageClass, + TagCount: result.TagCount, + VersionId: result.VersionId}, nil +} diff --git a/s3client/types.go b/s3client/types.go new file mode 100644 index 0000000..68d9570 --- /dev/null +++ b/s3client/types.go @@ -0,0 +1,22 @@ +package s3client + +import ( + "io" + "time" +) + +// GetObjectOutput- constructs the output result from S3 GetObject call +type GetObjectOutput struct { + Body io.ReadCloser `type:"blob"` + CacheControl *string `location:"header" locationName:"Cache-Control" type:"string"` + ContentEncoding *string `location:"header" locationName:"Content-Encoding" type:"string"` + ContentLength *int64 `location:"header" locationName:"Content-Length" type:"long"` + ContentRange *string `location:"header" locationName:"Content-Range" type:"string"` + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + ETag *string `location:"header" locationName:"ETag" type:"string"` + Expiration *string `location:"header" locationName:"x-amz-expiration" type:"string"` + LastModified *time.Time `location:"header" locationName:"Last-Modified" type:"timestamp"` + StorageClass *string `location:"header" locationName:"x-amz-storage-class" type:"string" enum:"StorageClass"` + TagCount *int64 `location:"header" locationName:"x-amz-tagging-count" type:"integer"` + VersionId *string `location:"header" locationName:"x-amz-version-id" type:"string"` +}