diff --git a/.circleci/config.yml b/.circleci/config.yml index 80d78d075..c25fc3ced 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -174,6 +174,30 @@ jobs: - store_artifacts: path: *WIN_TEST_RESULTS + go-smb-test: + docker: + - image: circleci/golang:<< parameters.go-version >> + parameters: + go-version: + type: string + steps: + - checkout + - setup_remote_docker + - run: + name: build and start smb server and gogetter containers + command: | + docker-compose build + docker-compose up -d + + - run: + name: wait for containers to start + command: sleep 60 + + - run: + name: run smb getter tests + command: | + docker exec -it gogetter bash -c "env ACC_SMB_TEST=1 go test -v ./... -run=TestSmb_" + workflows: go-getter: jobs: @@ -190,3 +214,7 @@ workflows: go-version: ["1.14.1"] gotestsum-version: ["0.4.1"] name: win-test-go-<< matrix.go-version >> + - go-smb-test: + matrix: + parameters: + go-version: ["1.14.1"] diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..7c3dd846e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +# Dockerfile to create a go-getter container with smbclient dependency that is used by the get_smb.go tests +FROM golang:latest + +COPY . /go-getter +WORKDIR /go-getter + +RUN go mod download +RUN apt-get update +RUN apt-get -y install smbclient diff --git a/Dockerfile-smbserver b/Dockerfile-smbserver new file mode 100644 index 000000000..48f68f450 --- /dev/null +++ b/Dockerfile-smbserver @@ -0,0 +1,9 @@ +# Dockerfile to create a smb server that is used by the get_smb.go tests +FROM dperson/samba + +# Create shared folders +RUN mkdir -p /public && mkdir -p /private + +# Create shared files and directories under the shared folders (data and mnt) +RUN echo 'Hello' > /public/file.txt && mkdir -p /public/subdir && echo 'Hello' > /public/subdir/file.txt +RUN echo 'Hello' > /private/file.txt && mkdir -p /private/subdir && echo 'Hello' > /private/subdir/file.txt diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..f920d84f1 --- /dev/null +++ b/Makefile @@ -0,0 +1,12 @@ +start-smb: + @docker-compose build + docker-compose up -d samba + +smbtests-prepare: + @docker-compose build + @docker-compose up -d + @sleep 60 + +smbtests: + @docker cp ./ gogetter:/go-getter/ + @docker exec -it gogetter bash -c "env ACC_SMB_TEST=1 go test -v ./... -run=TestSmb_" diff --git a/README.md b/README.md index eb81b5543..151b56bb4 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,7 @@ can be augmented at runtime by implementing the `Getter` interface. * HTTP * Amazon S3 * Google GCP + * SMB In addition to the above protocols, go-getter has what are called "detectors." These take a URL and attempt to automatically choose the best protocol for @@ -360,3 +361,45 @@ In order to access to GCS, authentication credentials should be provided. More i #### GCS Testing The tests for `get_gcs.go` require you to have GCP credentials set in your environment. These credentials can have any level of permissions to any project, they just need to exist. This means setting `GOOGLE_APPLICATION_CREDENTIALS="~/path/to/credentials.json"` or `GOOGLE_CREDENTIALS="{stringified-credentials-json}"`. Due to this configuration, `get_gcs_test.go` will fail for external contributors in CircleCI. + +### SMB (smb) + +There are two options that go-getter will use to download a file in a smb shared folder. The first option uses +[`smbclient`](https://www.samba.org/samba/docs/current/man-html/smbclient.1.html) and the second one uses the file system +to look for a file in a local mount of the shared folder in the OS specific volume folder. go-getter will try to download +files from a smb shared folder whenever the url is prefixed with `smb://`. + +⚠️ The [`smbclient`](https://www.samba.org/samba/docs/current/man-html/smbclient.1.html) command is available only for Linux. +This is the ONLY option for a Linux user and therefore the client must be installed. + +The `smbclient` cli is not available for Windows and MacOS. The go-getter +will try to get files using the file system, when this happens the getter uses the FileGetter implementation. + +When connecting to a smb server, the OS creates a local mount in a system specific volume folder, and go-getter will +try to access the following folders when looking for local mounts. + +- MacOS: /Volumes/ +- Windows: \\\\\\\\ + +The following examples work for all the OSes: +- smb://host/shared/dir (downloads directory content) +- smb://host/shared/dir/file (downloads file) + +The following examples work for Linux: +- smb://username:password@host/shared/dir (downloads directory content) +- smb://username@host/shared/dir +- smb://username:password@host/shared/dir/file (downloads file) +- smb://username@host/shared/dir/file + +⚠️ The above examples also work on the other OSes but the authentication is not used to access the file system. + + + +#### SMB Testing +The test for `get_smb.go` requires a smb server running which can be started inside a docker container by +running `make start-smb`. Once the container is up the shared folder can be accessed via `smb:///public/` or +`smb://user:password@/private/` by another container or machine in the same network. + +To run the tests inside `get_smb_test.go` and `client_test.go`, prepare the environment with `make smbtests-prepare`. On prepare some +mock files and directories will be added to the shared folder and a go-getter container will start together with the samba server. +Once the environment for testing is prepared, run `make smbtests` to run the tests. \ No newline at end of file diff --git a/client.go b/client.go index c5bba4743..2679fd6ff 100644 --- a/client.go +++ b/client.go @@ -10,6 +10,7 @@ import ( "strings" urlhelper "github.com/hashicorp/go-getter/v2/helper/url" + "github.com/hashicorp/go-multierror" safetemp "github.com/hashicorp/go-safetemp" ) @@ -19,18 +20,13 @@ import ( // Using a client directly allows more fine-grained control over how downloading // is done, as well as customizing the protocols supported. type Client struct { - - // Detectors is the list of detectors that are tried on the source. - // If this is nil, then the default Detectors will be used. - Detectors []Detector - // Decompressors is the map of decompressors supported by this client. // If this is nil, then the default value is the Decompressors global. Decompressors map[string]Decompressor - // Getters is the map of protocols supported by this client. If this + // Getters is the list of protocols supported by this client. If this // is nil, then the default Getters variable will be used. - Getters map[string]Getter + Getters []Getter } // GetResult is the result of a Client.Get @@ -50,43 +46,75 @@ func (c *Client) Get(ctx context.Context, req *Request) (*GetResult, error) { req.Mode = ModeAny } - var err error - req.Src, err = Detect(req.Src, req.Pwd, c.Detectors) - if err != nil { - return nil, err - } - - var force string - // Determine if we have a forced protocol, i.e. "git::http://..." - force, req.Src = getForcedGetter(req.Src) - // If there is a subdir component, then we download the root separately // and then copy over the proper subdir. - var realDst, subDir string - req.Src, subDir = SourceDirSubdir(req.Src) - if subDir != "" { + req.Src, req.subDir = SourceDirSubdir(req.Src) + if req.subDir != "" { td, tdcloser, err := safetemp.Dir("", "getter") if err != nil { return nil, err } defer tdcloser.Close() - realDst = req.Dst + req.realDst = req.Dst req.Dst = td } - req.u, err = urlhelper.Parse(req.Src) - if err != nil { - return nil, err + var multierr []error + for _, g := range c.Getters { + shouldDownload, err := Detect(req, g) + if err != nil { + return nil, err + } + if !shouldDownload { + // the request should not be processed by that getter + continue + } + + result, getErr := c.get(ctx, req, g) + if getErr != nil { + if getErr.Fatal { + return nil, getErr.Err + } + multierr = append(multierr, getErr.Err) + continue + } + + return result, nil } - if force == "" { - force = req.u.Scheme + + if len(multierr) == 1 { + // This is for keeping the error original format + return nil, multierr[0] + } + + if multierr != nil { + var result *multierror.Error + result = multierror.Append(result, multierr...) + return nil, fmt.Errorf("error downloading '%s': %s", req.Src, result.Error()) } - g, ok := c.Getters[force] - if !ok { - return nil, fmt.Errorf( - "download not supported for scheme '%s'", force) + return nil, fmt.Errorf("error downloading '%s'", req.Src) +} + +// getError is the Error response object returned by get(context.Context, *Request, Getter) +// to tell the client whether to halt (Fatal) Get or to keep trying to get an artifact. +type getError struct { + // When Fatal is true something went wrong with get(context.Context, *Request, Getter) + // and the client should halt and return the Err. + Fatal bool + Err error +} + +func (ge *getError) Error() string { + return ge.Err.Error() +} + +func (c *Client) get(ctx context.Context, req *Request, g Getter) (*GetResult, *getError) { + u, err := urlhelper.Parse(req.Src) + req.u = u + if err != nil { + return nil, &getError{true, err} } // We have magic query parameters that we use to signal different features @@ -105,8 +133,7 @@ func (c *Client) Get(ctx context.Context, req *Request) (*GetResult, error) { if b, err := strconv.ParseBool(archiveV); err == nil && !b { archiveV = "-" } - } - if archiveV == "" { + } else { // We don't appear to... but is it part of the filename? matchingLen := 0 for k := range c.Decompressors { @@ -128,8 +155,8 @@ func (c *Client) Get(ctx context.Context, req *Request) (*GetResult, error) { // this at the end of everything. td, err := ioutil.TempDir("", "getter") if err != nil { - return nil, fmt.Errorf( - "Error creating temporary directory for archive: %s", err) + return nil, &getError{true, fmt.Errorf( + "Error creating temporary directory for archive: %s", err)} } defer os.RemoveAll(td) @@ -144,7 +171,7 @@ func (c *Client) Get(ctx context.Context, req *Request) (*GetResult, error) { // Determine checksum if we have one checksum, err := c.GetChecksum(ctx, req) if err != nil { - return nil, fmt.Errorf("invalid checksum: %s", err) + return nil, &getError{true, fmt.Errorf("invalid checksum: %s", err)} } // Delete the query parameter if we have it. @@ -155,7 +182,7 @@ func (c *Client) Get(ctx context.Context, req *Request) (*GetResult, error) { // Ask the getter which client mode to use req.Mode, err = g.Mode(ctx, req.u) if err != nil { - return nil, err + return nil, &getError{false, err} } // Destination is the base name of the URL path in "any" mode when @@ -187,14 +214,13 @@ func (c *Client) Get(ctx context.Context, req *Request) (*GetResult, error) { } } if getFile { - err := g.GetFile(ctx, req) - if err != nil { - return nil, err + if err := g.GetFile(ctx, req); err != nil { + return nil, &getError{false, err} } if checksum != nil { if err := checksum.Checksum(req.Dst); err != nil { - return nil, err + return nil, &getError{true, err} } } } @@ -204,7 +230,7 @@ func (c *Client) Get(ctx context.Context, req *Request) (*GetResult, error) { // into the final destination with the proper mode. err := decompressor.Decompress(decompressDst, req.Dst, decompressDir) if err != nil { - return nil, err + return nil, &getError{true, err} } // Swap the information back @@ -232,36 +258,67 @@ func (c *Client) Get(ctx context.Context, req *Request) (*GetResult, error) { // If we're getting a directory, then this is an error. You cannot // checksum a directory. TODO: test if checksum != nil { - return nil, fmt.Errorf( - "checksum cannot be specified for directory download") + return nil, &getError{true, fmt.Errorf( + "checksum cannot be specified for directory download")} } // We're downloading a directory, which might require a bit more work // if we're specifying a subdir. - err := g.Get(ctx, req) - if err != nil { - err = fmt.Errorf("error downloading '%s': %s", req.Src, err) - return nil, err + if err := g.Get(ctx, req); err != nil { + return nil, &getError{false, err} } } // If we have a subdir, copy that over - if subDir != "" { - if err := os.RemoveAll(realDst); err != nil { - return nil, err + if req.subDir != "" { + if err := os.RemoveAll(req.realDst); err != nil { + return nil, &getError{true, err} } - if err := os.MkdirAll(realDst, 0755); err != nil { - return nil, err + if err := os.MkdirAll(req.realDst, 0755); err != nil { + return nil, &getError{true, err} } // Process any globs - subDir, err := SubdirGlob(req.Dst, subDir) + subDir, err := SubdirGlob(req.Dst, req.subDir) if err != nil { - return nil, err + return nil, &getError{true, err} } - return &GetResult{realDst}, copyDir(ctx, realDst, subDir, false) + err = copyDir(ctx, req.realDst, subDir, false) + if err != nil { + return nil, &getError{false, err} + } + return &GetResult{req.realDst}, nil } return &GetResult{req.Dst}, nil + +} + +func (c *Client) checkArchive(req *Request) string { + q := req.u.Query() + archiveV := q.Get("archive") + if archiveV != "" { + // Delete the paramter since it is a magic parameter we don't + // want to pass on to the Getter + q.Del("archive") + req.u.RawQuery = q.Encode() + + // If we can parse the value as a bool and it is false, then + // set the archive to "-" which should never map to a decompressor + if b, err := strconv.ParseBool(archiveV); err == nil && !b { + archiveV = "-" + } + } + if archiveV == "" { + // We don't appear to... but is it part of the filename? + matchingLen := 0 + for k := range c.Decompressors { + if strings.HasSuffix(req.u.Path, "."+k) && len(k) > matchingLen { + archiveV = k + matchingLen = len(k) + } + } + } + return archiveV } diff --git a/client_option.go b/client_option.go index a76e48b40..c17f73fe9 100644 --- a/client_option.go +++ b/client_option.go @@ -6,14 +6,9 @@ func (c *Client) configure() error { if c.Decompressors == nil { c.Decompressors = Decompressors } - // Default detector values - if c.Detectors == nil { - c.Detectors = Detectors - } // Default getter values if c.Getters == nil { c.Getters = Getters } - return nil } diff --git a/client_test.go b/client_test.go new file mode 100644 index 000000000..6581134e1 --- /dev/null +++ b/client_test.go @@ -0,0 +1,85 @@ +package getter + +import ( + "context" + testing_helper "github.com/hashicorp/go-getter/v2/helper/testing" + "os" + "path/filepath" + "testing" +) + +func TestSmb_ClientGet(t *testing.T) { + smbTestsPreCheck(t) + + tests := []struct { + name string + rawURL string + mode Mode + file string + fail bool + }{ + { + "smb scheme subdir with registered authentication in private share", + "smb://user:password@samba/private/subdir", + ModeDir, + "file.txt", + false, + }, + { + "smb scheme file with registered authentication with file in private share", + "smb://user:password@samba/private/subdir/file.txt", + ModeFile, + "file.txt", + false, + }, + { + "smb scheme file without authentication in public share", + "smb://samba/public/subdir/file.txt", + ModeFile, + "file.txt", + false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dst := testing_helper.TempDir(t) + defer os.RemoveAll(dst) + + if tt.mode == ModeFile { + dst = filepath.Join(dst, tt.file) + } + + req := &Request{ + Dst: dst, + Src: tt.rawURL, + Mode: tt.mode, + } + + result, err := DefaultClient.Get(context.Background(), req) + + fail := err != nil + if tt.fail != fail { + if fail { + t.Fatalf("err: unexpected error %s", err.Error()) + } + t.Fatalf("err: expecting to fail but it did not") + } + + if !tt.fail { + if result == nil { + t.Fatalf("err: get result should not be nil") + } + if result.Dst != dst { + t.Fatalf("err: expected destination: %s \n actual destination: %s", dst, result.Dst) + } + if tt.mode == ModeDir { + dst = filepath.Join(dst, tt.file) + } + // Verify if the file was successfully downloaded + // and exists at the destination folder + testing_helper.AssertContents(t, dst, "Hello\n") + } + }) + } +} diff --git a/cmd/go-getter/go.mod b/cmd/go-getter/go.mod index 19f90517f..d291e885a 100644 --- a/cmd/go-getter/go.mod +++ b/cmd/go-getter/go.mod @@ -13,7 +13,7 @@ require ( github.com/fatih/color v1.9.0 // indirect github.com/hashicorp/go-getter/gcs/v2 v2.0.0-00010101000000-000000000000 github.com/hashicorp/go-getter/s3/v2 v2.0.0-00010101000000-000000000000 - github.com/hashicorp/go-getter/v2 v2.0.0-00010101000000-000000000000 + github.com/hashicorp/go-getter/v2 v2.0.0-20200511090339-3107ec4af37a github.com/mattn/go-runewidth v0.0.8 // indirect gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect ) diff --git a/cmd/go-getter/go.sum b/cmd/go-getter/go.sum index 87fbf6b2d..5b3ce4875 100644 --- a/cmd/go-getter/go.sum +++ b/cmd/go-getter/go.sum @@ -29,11 +29,13 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= github.com/aws/aws-sdk-go v1.30.8 h1:4BHbh8K3qKmcnAgToZ2LShldRF9inoqIBccpCLNCy3I= github.com/aws/aws-sdk-go v1.30.8/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/cheggaaa/pb v1.0.28 h1:kWGpdAcSp3MxMU9CCHOwz/8V0kCHN4+9yQm2MzWuI98= github.com/cheggaaa/pb v1.0.28/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -42,8 +44,10 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -84,8 +88,11 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-getter v1.4.1/go.mod h1:7qxyCd8rBfcShwsvxgIguu4KbS3l8bUCwg2Umn7RjeY= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0= @@ -94,6 +101,7 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -105,11 +113,14 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.8 h1:3tS41NlGYSmhhe/8fhGRzc+z3AYCw1Fe1WAyLuujKs0= github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0= @@ -122,6 +133,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -313,6 +325,7 @@ google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= diff --git a/cmd/go-getter/main.go b/cmd/go-getter/main.go index 7ac583188..0f4569d5a 100644 --- a/cmd/go-getter/main.go +++ b/cmd/go-getter/main.go @@ -61,8 +61,8 @@ func main() { client := getter.DefaultClient getters := getter.Getters - getters["gcs"] = new(gcs.Getter) - getters["s3"] = new(s3.Getter) + getters = append(getters, new(gcs.Getter)) + getters = append(getters, new(s3.Getter)) client.Getters = getters errChan := make(chan error, 2) diff --git a/detect.go b/detect.go index fc6da4a2b..062b849f1 100644 --- a/detect.go +++ b/detect.go @@ -2,9 +2,7 @@ package getter import ( "fmt" - "path/filepath" - - "github.com/hashicorp/go-getter/v2/helper/url" + "github.com/hashicorp/go-getter/helper/url" ) // Detector defines the interface that an invalid URL or a URL with a blank @@ -16,90 +14,45 @@ type Detector interface { Detect(string, string) (string, bool, error) } -// Detectors is the list of detectors that are tried on an invalid URL. -// This is also the order they're tried (index 0 is first). -var Detectors []Detector +// Detect is a method used to detect if a Getter is a candidate for downloading an artifact +// by calling the Getter.Detect(*Request) method +func Detect(req *Request, getter Getter) (bool, error) { + originalSrc := req.Src -func init() { - Detectors = []Detector{ - new(GitHubDetector), - new(GitDetector), - new(BitBucketDetector), - new(S3Detector), - new(GCSDetector), - new(FileDetector), + getForce, getSrc := getForcedGetter(req.Src) + if getForce != "" { + req.Forced = getForce } -} - -// Detect turns a source string into another source string if it is -// detected to be of a known pattern. -// -// The third parameter should be the list of detectors to use in the -// order to try them. If you don't want to configure this, just use -// the global Detectors variable. -// -// This is safe to be called with an already valid source string: Detect -// will just return it. -func Detect(src string, pwd string, ds []Detector) (string, error) { - getForce, getSrc := getForcedGetter(src) - - // Separate out the subdir if there is one, we don't pass that to detect - getSrc, subDir := SourceDirSubdir(getSrc) - u, err := url.Parse(getSrc) - if err == nil && u.Scheme != "" { - // Valid URL - return src, nil + req.Src = getSrc + ok, err := getter.Detect(req) + if err != nil { + return true, err + } + if !ok { + // Write back the original source + req.Src = originalSrc + return ok, nil } - for _, d := range ds { - result, ok, err := d.Detect(getSrc, pwd) - if err != nil { - return "", err - } - if !ok { - continue - } - - var detectForce string - detectForce, result = getForcedGetter(result) - result, detectSubdir := SourceDirSubdir(result) - - // If we have a subdir from the detection, then prepend it to our - // requested subdir. - if detectSubdir != "" { - if subDir != "" { - subDir = filepath.Join(detectSubdir, subDir) - } else { - subDir = detectSubdir - } - } - - if subDir != "" { - u, err := url.Parse(result) - if err != nil { - return "", fmt.Errorf("Error parsing URL: %s", err) - } - u.Path += "//" + subDir - - // a subdir may contain wildcards, but in order to support them we - // have to ensure the path isn't escaped. - u.RawPath = u.Path + result, detectSubdir := SourceDirSubdir(req.Src) - result = u.String() + // If we have a subdir from the detection, then prepend it to our + // requested subdir. + if detectSubdir != "" { + u, err := url.Parse(result) + if err != nil { + return true, fmt.Errorf("Error parsing URL: %s", err) } + u.Path += "//" + detectSubdir - // Preserve the forced getter if it exists. We try to use the - // original set force first, followed by any force set by the - // detector. - if getForce != "" { - result = fmt.Sprintf("%s::%s", getForce, result) - } else if detectForce != "" { - result = fmt.Sprintf("%s::%s", detectForce, result) - } + // a subdir may contain wildcards, but in order to support them we + // have to ensure the path isn't escaped. + u.RawPath = u.Path - return result, nil + result = u.String() } - return "", fmt.Errorf("invalid source string: %s", src) + req.Src = result + return true, nil } diff --git a/detect_file.go b/detect_file.go index 4ef41ea73..8a0256e77 100644 --- a/detect_file.go +++ b/detect_file.go @@ -56,12 +56,6 @@ func fmtFileURL(path string) string { if runtime.GOOS == "windows" { // Make sure we're using "/" on Windows. URLs are "/"-based. path = filepath.ToSlash(path) - return fmt.Sprintf("file://%s", path) } - - // Make sure that we don't start with "/" since we add that below. - if path[0] == '/' { - path = path[1:] - } - return fmt.Sprintf("file:///%s", path) + return path } diff --git a/detect_file_test.go b/detect_file_test.go index 0f7a055c9..4b74b3200 100644 --- a/detect_file_test.go +++ b/detect_file_test.go @@ -5,33 +5,32 @@ import ( "os" "path/filepath" "runtime" - "strings" "testing" ) type fileTest struct { in, pwd, out string - err bool + symlink, err bool } var fileTests = []fileTest{ - {"./foo", "/pwd", "file:///pwd/foo", false}, - {"./foo?foo=bar", "/pwd", "file:///pwd/foo?foo=bar", false}, - {"foo", "/pwd", "file:///pwd/foo", false}, + {"./foo", "/pwd", "/pwd/foo", false, false}, + {"./foo?foo=bar", "/pwd", "/pwd/foo?foo=bar", false, false}, + {"foo", "/pwd", "/pwd/foo", false, false}, } var unixFileTests = []fileTest{ {"./foo", "testdata/detect-file-symlink-pwd/syml/pwd", - "testdata/detect-file-symlink-pwd/real/foo", false}, + "testdata/detect-file-symlink-pwd/real/foo", true, false}, - {"/foo", "/pwd", "file:///foo", false}, - {"/foo?bar=baz", "/pwd", "file:///foo?bar=baz", false}, + {"/foo", "/pwd", "/foo", false, false}, + {"/foo?bar=baz", "/pwd", "/foo?bar=baz", false, false}, } var winFileTests = []fileTest{ - {"/foo", "/pwd", "file:///pwd/foo", false}, - {`C:\`, `/pwd`, `file://C:/`, false}, - {`C:\?bar=baz`, `/pwd`, `file://C:/?bar=baz`, false}, + {"/foo", "/pwd", "/pwd/foo", false, false}, + {`C:\`, `/pwd`, `C:/`, false, false}, + {`C:\?bar=baz`, `/pwd`, `C:/?bar=baz`, false, false}, } func TestFileDetector(t *testing.T) { @@ -65,8 +64,8 @@ func TestFileDetector(t *testing.T) { } expected := tc.out - if !strings.HasPrefix(expected, "file://") { - expected = "file://" + filepath.Join(pwdRoot, expected) + if tc.symlink { + expected = filepath.Join(pwdRoot, expected) } if out != expected { @@ -83,12 +82,12 @@ var noPwdFileTests = []fileTest{ } var noPwdUnixFileTests = []fileTest{ - {in: "/foo", pwd: "", out: "file:///foo", err: false}, + {in: "/foo", pwd: "", out: "/foo", err: false}, } var noPwdWinFileTests = []fileTest{ {in: "/foo", pwd: "", out: "", err: true}, - {in: `C:\`, pwd: ``, out: `file://C:/`, err: false}, + {in: `C:\`, pwd: ``, out: `C:/`, err: false}, } func TestFileDetector_noPwd(t *testing.T) { diff --git a/detect_git_test.go b/detect_git_test.go index a71dde2c3..60463e33e 100644 --- a/detect_git_test.go +++ b/detect_git_test.go @@ -41,28 +41,22 @@ func TestGitDetector(t *testing.T) { "git@github.xyz.com:org/project.git//module/a?ref=test-branch", "git::ssh://git@github.xyz.com/org/project.git//module/a?ref=test-branch", }, - { - // Already in the canonical form, so no rewriting required - // When the ssh: protocol is used explicitly, we recognize it as - // URL form rather than SCP-like form, so the part after the colon - // is a port number, not part of the path. - "git::ssh://git@git.example.com:2222/hashicorp/foo.git", - "git::ssh://git@git.example.com:2222/hashicorp/foo.git", - }, } pwd := "/pwd" f := new(GitDetector) - ds := []Detector{f} - for _, tc := range cases { + for i, tc := range cases { t.Run(tc.Input, func(t *testing.T) { - output, err := Detect(tc.Input, pwd, ds) + out, ok, err := f.Detect(tc.Input, pwd) if err != nil { - t.Fatalf("unexpected error: %s", err) + t.Fatalf("%d: err: %s", i, err) + } + if !ok { + t.Fatal("not ok") } - if output != tc.Output { - t.Errorf("wrong result\ninput: %s\ngot: %s\nwant: %s", tc.Input, output, tc.Output) + if out != tc.Output { + t.Fatalf("%d: bad: %#v", i, out) } }) } diff --git a/detect_test.go b/detect_test.go index 9bef662a7..bc7933a89 100644 --- a/detect_test.go +++ b/detect_test.go @@ -6,86 +6,98 @@ import ( ) func TestDetect(t *testing.T) { + gitGetter := &GitGetter{[]Detector{ + new(GitDetector), + new(BitBucketDetector), + new(GitHubDetector), + }, + } cases := []struct { Input string Pwd string Output string Err bool + getter Getter }{ - {"./foo", "/foo", "file:///foo/foo", false}, - {"git::./foo", "/foo", "git::file:///foo/foo", false}, + {"./foo", "/foo", "/foo/foo", false, new(FileGetter)}, + {"git::./foo", "/foo", "/foo/foo", false, gitGetter}, { "git::github.com/hashicorp/foo", "", - "git::https://github.com/hashicorp/foo.git", + "https://github.com/hashicorp/foo.git", false, + gitGetter, }, { - "./foo//bar", + "./foo", "/foo", - "file:///foo/foo//bar", - false, - }, - { - "git::github.com/hashicorp/foo//bar", - "", - "git::https://github.com/hashicorp/foo.git//bar", + "/foo/foo", false, + new(FileGetter), }, { "git::https://github.com/hashicorp/consul.git", "", - "git::https://github.com/hashicorp/consul.git", + "https://github.com/hashicorp/consul.git", false, + gitGetter, }, { "git::https://person@someothergit.com/foo/bar", "", - "git::https://person@someothergit.com/foo/bar", + "https://person@someothergit.com/foo/bar", false, + gitGetter, }, { "git::https://person@someothergit.com/foo/bar", "/bar", - "git::https://person@someothergit.com/foo/bar", - false, - }, - { - "./foo/archive//*", - "/bar", - "file:///bar/foo/archive//*", + "https://person@someothergit.com/foo/bar", false, + gitGetter, }, // https://github.com/hashicorp/go-getter/pull/124 { "git::ssh://git@my.custom.git/dir1/dir2", "", - "git::ssh://git@my.custom.git/dir1/dir2", + "ssh://git@my.custom.git/dir1/dir2", false, + gitGetter, }, { "git::git@my.custom.git:dir1/dir2", "/foo", - "git::ssh://git@my.custom.git/dir1/dir2", + "ssh://git@my.custom.git/dir1/dir2", false, + gitGetter, }, { "git::git@my.custom.git:dir1/dir2", "", - "git::ssh://git@my.custom.git/dir1/dir2", + "ssh://git@my.custom.git/dir1/dir2", false, + gitGetter, }, } for i, tc := range cases { t.Run(fmt.Sprintf("%d %s", i, tc.Input), func(t *testing.T) { - output, err := Detect(tc.Input, tc.Pwd, Detectors) + req := &Request{ + Src: tc.Input, + Pwd: tc.Pwd, + } + ok, err := Detect(req, tc.getter) if err != nil != tc.Err { t.Fatalf("%d: bad err: %s", i, err) } - if output != tc.Output { - t.Fatalf("%d: bad output: %s\nexpected: %s", i, output, tc.Output) + + if !tc.Err && !ok { + t.Fatalf("%d: should be ok", i) + } + + if req.Src != tc.Output { + t.Fatalf("%d: bad output: %s\nexpected: %s", i, req.Src, tc.Output) } }) } diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..008c413ac --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,34 @@ +version: '3.2' + +services: + samba: + build: + dockerfile: Dockerfile-smbserver + context: . + container_name: samba + environment: + USERID: "0" + GROUPID: "0" + networks: + - default + ports: + - "139:139/tcp" # default smb port + - "445:445/tcp" # default smb port + volumes: + - /mnt:/mnt:z + command: '-s "public;/public" -s "private;/private;yes;no;no;user" -u "user;password" -p' + + gogetter: + build: + context: . + container_name: gogetter + depends_on: + - samba + networks: + - default + volumes: + - /mnt:/mnt + command: tail -f /dev/null + +networks: + default: diff --git a/detect_gcs.go b/gcs/detect_gcs.go similarity index 69% rename from detect_gcs.go rename to gcs/detect_gcs.go index 11363737c..d5846ec4b 100644 --- a/detect_gcs.go +++ b/gcs/detect_gcs.go @@ -1,4 +1,4 @@ -package getter +package gcs import ( "fmt" @@ -6,11 +6,11 @@ import ( "strings" ) -// GCSDetector implements Detector to detect GCS URLs and turn +// Detector implements Detector to detect GCS URLs and turn // them into URLs that the GCSGetter can understand. -type GCSDetector struct{} +type Detector struct{} -func (d *GCSDetector) Detect(src, _ string) (string, bool, error) { +func (d *Detector) Detect(src, _ string) (string, bool, error) { if len(src) == 0 { return "", false, nil } @@ -22,7 +22,7 @@ func (d *GCSDetector) Detect(src, _ string) (string, bool, error) { return "", false, nil } -func (d *GCSDetector) detectHTTP(src string) (string, bool, error) { +func (d *Detector) detectHTTP(src string) (string, bool, error) { parts := strings.Split(src, "/") if len(parts) < 5 { @@ -39,5 +39,5 @@ func (d *GCSDetector) detectHTTP(src string) (string, bool, error) { return "", false, fmt.Errorf("error parsing GCS URL: %s", err) } - return "gcs::" + url.String(), true, nil + return url.String(), true, nil } diff --git a/detect_gcs_test.go b/gcs/detect_gcs_test.go similarity index 70% rename from detect_gcs_test.go rename to gcs/detect_gcs_test.go index 94e1ec42b..9a9a3e280 100644 --- a/detect_gcs_test.go +++ b/gcs/detect_gcs_test.go @@ -1,4 +1,4 @@ -package getter +package gcs import ( "testing" @@ -11,20 +11,20 @@ func TestGCSDetector(t *testing.T) { }{ { "www.googleapis.com/storage/v1/bucket/foo", - "gcs::https://www.googleapis.com/storage/v1/bucket/foo", + "https://www.googleapis.com/storage/v1/bucket/foo", }, { "www.googleapis.com/storage/v1/bucket/foo/bar", - "gcs::https://www.googleapis.com/storage/v1/bucket/foo/bar", + "https://www.googleapis.com/storage/v1/bucket/foo/bar", }, { "www.googleapis.com/storage/v1/foo/bar.baz", - "gcs::https://www.googleapis.com/storage/v1/foo/bar.baz", + "https://www.googleapis.com/storage/v1/foo/bar.baz", }, } pwd := "/pwd" - f := new(GCSDetector) + f := new(Detector) for i, tc := range cases { output, ok, err := f.Detect(tc.Input, pwd) if err != nil { diff --git a/gcs/get_gcs.go b/gcs/get_gcs.go index e8d4f13f7..dd9b6f256 100644 --- a/gcs/get_gcs.go +++ b/gcs/get_gcs.go @@ -15,8 +15,7 @@ import ( // Getter is a Getter implementation that will download a module from // a GCS bucket. -type Getter struct { -} +type Getter struct {} func (g *Getter) Mode(ctx context.Context, u *url.URL) (getter.Mode, error) { @@ -165,3 +164,48 @@ func (g *Getter) parseURL(u *url.URL) (bucket, path string, err error) { } return } + +func (g *Getter) Detect(req *getter.Request) (bool, error) { + src := req.Src + if len(src) == 0 { + return false, nil + } + + if req.Forced != "" { + // There's a getter being forced + if !g.validScheme(req.Forced) { + // Current getter is not the forced one + // Don't use it to try to download the artifact + return false, nil + } + } + isForcedGetter := req.Forced != "" && g.validScheme(req.Forced) + + u, err := url.Parse(src) + if err == nil && u.Scheme != "" { + if isForcedGetter { + // Is the forced getter and source is a valid url + return true, nil + } + if g.validScheme(u.Scheme) { + return true, nil + } + // Valid url with a scheme that is not valid for current getter + return false, nil + } + + src, ok, err := new(Detector).Detect(src, req.Pwd) + if err != nil { + return ok, err + } + if ok { + req.Src = src + return ok, nil + } + + return false, nil +} + +func (g *Getter) validScheme(scheme string) bool { + return scheme == "gcs" +} diff --git a/gcs/get_gcs_test.go b/gcs/get_gcs_test.go index 2af171628..d504b4458 100644 --- a/gcs/get_gcs_test.go +++ b/gcs/get_gcs_test.go @@ -46,7 +46,7 @@ func TestGetter(t *testing.T) { } c := getter.Client{ - Getters: map[string]getter.Getter{"gcs": g}, + Getters: []getter.Getter{g}, } // With a dir that doesn't exist @@ -75,7 +75,7 @@ func TestGetter_subdir(t *testing.T) { } c := getter.Client{ - Getters: map[string]getter.Getter{"gcs": g}, + Getters: []getter.Getter{g}, } // With a dir that doesn't exist @@ -106,7 +106,7 @@ func TestGetter_GetFile(t *testing.T) { } c := getter.Client{ - Getters: map[string]getter.Getter{"gcs": g}, + Getters: []getter.Getter{g}, } // Download @@ -134,7 +134,7 @@ func TestGetter_GetFile_notfound(t *testing.T) { } c := getter.Client{ - Getters: map[string]getter.Getter{"gcs": g}, + Getters: []getter.Getter{g}, } // Download @@ -237,4 +237,4 @@ func TestGetter_Url(t *testing.T) { } }) } -} +} \ No newline at end of file diff --git a/gcs/go.sum b/gcs/go.sum index b1e4457bd..3b56d9766 100644 --- a/gcs/go.sum +++ b/gcs/go.sum @@ -28,16 +28,20 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -73,8 +77,14 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-getter v1.4.1 h1:3A2Mh8smGFcf5M+gmcv898mZdrxpseik45IpcyISLsA= +github.com/hashicorp/go-getter v1.4.1/go.mod h1:7qxyCd8rBfcShwsvxgIguu4KbS3l8bUCwg2Umn7RjeY= +github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0= @@ -82,6 +92,7 @@ github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -89,6 +100,9 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= @@ -97,6 +111,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok= github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= @@ -274,6 +289,7 @@ google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/get.go b/get.go index 3810c9fcb..c599a6f6a 100644 --- a/get.go +++ b/get.go @@ -42,13 +42,21 @@ type Getter interface { // Mode returns the mode based on the given URL. This is used to // allow clients to let the getters decide which mode to use. Mode(context.Context, *url.URL) (Mode, error) + + // Detect detects whether the Request.Src matches a known pattern to + // turn it into a proper URL, and also transforms and update Request.Src + // when necessary. + // The Getter must validate if the Request.Src is a valid URL + // with a valid scheme for the Getter, and also check if the + // current Getter is the forced one and return true if that's the case. + Detect(*Request) (bool, error) } // Getters is the mapping of scheme to the Getter implementation that will // be used to get a dependency. -var Getters map[string]Getter +var Getters []Getter -// forcedRegexp is the regular expression that finds forced getters. This +// forcedRegexp is the regular expression that finds Forced getters. This // syntax is schema::url, example: git::https://foo.com var forcedRegexp = regexp.MustCompile(`^([A-Za-z0-9]+)::(.+)$`) @@ -57,7 +65,6 @@ var httpClient = cleanhttp.DefaultClient() var DefaultClient = &Client{ Getters: Getters, - Detectors: Detectors, Decompressors: Decompressors, } @@ -66,12 +73,20 @@ func init() { Netrc: true, } - Getters = map[string]Getter{ - "file": new(FileGetter), - "git": new(GitGetter), - "hg": new(HgGetter), - "http": httpGetter, - "https": httpGetter, + // The order of the Getters in the list may affect the result + // depending if the Request.Src is detected as valid by multiple getters + Getters = []Getter{ + &GitGetter{[]Detector{ + new(GitHubDetector), + new(GitDetector), + new(BitBucketDetector), + }, + }, + new(HgGetter), + new(SmbClientGetter), + new(SmbMountGetter), + httpGetter, + new(FileGetter), } } diff --git a/get_file.go b/get_file.go index 76515b427..1e7262315 100644 --- a/get_file.go +++ b/get_file.go @@ -6,12 +6,12 @@ import ( "net/url" "os" "path/filepath" + "runtime" ) // FileGetter is a Getter implementation that will download a module from // a file scheme. -type FileGetter struct { -} +type FileGetter struct{} func (g *FileGetter) Mode(ctx context.Context, u *url.URL) (Mode, error) { path := u.Path @@ -148,3 +148,54 @@ func (g *FileGetter) GetFile(ctx context.Context, req *Request) error { _, err = Copy(ctx, dstF, srcF) return err } + +func (g *FileGetter) Detect(req *Request) (bool, error) { + var src, pwd string + src = req.Src + pwd = req.Pwd + if len(src) == 0 { + return false, nil + } + + if req.Forced != "" { + // There's a getter being Forced + if !g.validScheme(req.Forced) { + // Current getter is not the Forced one + // Don't use it to try to download the artifact + return false, nil + } + } + isForcedGetter := req.Forced != "" && g.validScheme(req.Forced) + + u, err := url.Parse(src) + if err == nil && u.Scheme != "" { + if isForcedGetter { + // Is the Forced getter and source is a valid url + return true, nil + } + if g.validScheme(u.Scheme) { + return true, nil + } + if !(runtime.GOOS == "windows" && len(u.Scheme) == 1) { + return false, nil + } + // For windows, we try to get the artifact + // if it has a non valid scheme with 1 char + // e.g. C:/foo/bar for other cases a prefix file:// is necessary + } + + src, ok, err := new(FileDetector).Detect(src, pwd) + if err != nil { + return ok, err + } + if ok { + req.Src = src + return ok, nil + } + + return true, nil +} + +func (g *FileGetter) validScheme(scheme string) bool { + return scheme == "file" +} diff --git a/detect_file_unix_test.go b/get_file_unix_test.go similarity index 83% rename from detect_file_unix_test.go rename to get_file_unix_test.go index 657f1a41c..f99e199bf 100644 --- a/detect_file_unix_test.go +++ b/get_file_unix_test.go @@ -55,16 +55,20 @@ func TestFileDetector_relativeSymlink(t *testing.T) { // if detech doesn't fully resolve the pwd symlink, the output will be the // invalid path: "file:///../modules/foo" - f := new(FileDetector) - out, ok, err := f.Detect("../modules/foo", "./linkedPWD") + f := new(FileGetter) + req := &Request{ + Src: "../modules/foo", + Pwd: "./linkedPWD", + } + ok, err := f.Detect(req) if err != nil { t.Fatalf("err: %v", err) } if !ok { t.Fatal("not ok") } - if out != "file://"+filepath.Join(tmpDir, "modules/foo") { - t.Logf("expected: %v", "file://"+filepath.Join(tmpDir, "modules/foo")) + if req.Src != filepath.Join(tmpDir, "modules/foo") { + t.Logf("expected: %v", filepath.Join(tmpDir, "modules/foo")) t.Fatalf("bad: %v", out) } } diff --git a/get_git.go b/get_git.go index d4fe94015..314339129 100644 --- a/get_git.go +++ b/get_git.go @@ -23,6 +23,7 @@ import ( // GitGetter is a Getter implementation that will download a module from // a git repository. type GitGetter struct { + Detectors []Detector } var defaultBranchRegexp = regexp.MustCompile(`\s->\sorigin/(.*)`) @@ -313,3 +314,66 @@ func checkGitVersion(min string) error { return nil } + +func (g *GitGetter) Detect(req *Request) (bool, error) { + src := req.Src + if len(src) == 0 { + return false, nil + } + + if req.Forced != "" { + // There's a getter being Forced + if !g.validScheme(req.Forced) { + // Current getter is not the Forced one + // Don't use it to try to download the artifact + return false, nil + } + } + isForcedGetter := req.Forced != "" && g.validScheme(req.Forced) + + u, err := url.Parse(src) + if err == nil && u.Scheme != "" { + if isForcedGetter { + // Is the Forced getter and source is a valid url + return true, nil + } + if g.validScheme(u.Scheme) { + return true, nil + } + // Valid url with a scheme that is not valid for current getter + return false, nil + } + + for _, d := range g.Detectors { + src, ok, err := d.Detect(src, req.Pwd) + if err != nil { + return ok, err + } + forced, src := getForcedGetter(src) + if ok && g.validScheme(forced) { + req.Src = src + return ok, nil + } + } + + if _, err = url.Parse(req.Src); err != nil { + return true, nil + } + + if isForcedGetter { + // Is the Forced getter and should be used to download the artifact + if req.Pwd != "" && !filepath.IsAbs(src) { + // Make sure to add pwd to relative paths + src = filepath.Join(req.Pwd, src) + } + // Make sure we're using "/" on Windows. URLs are "/"-based. + req.Src = filepath.ToSlash(src) + return true, nil + } + + return false, nil +} + +func (g *GitGetter) validScheme(scheme string) bool { + return scheme == "git" || scheme == "ssh" +} diff --git a/get_git_test.go b/get_git_test.go index c5061ac73..7b9e7c8ee 100644 --- a/get_git_test.go +++ b/get_git_test.go @@ -394,7 +394,6 @@ func TestGitGetter_sshSCPStyle(t *testing.T) { } ctx := context.Background() - g := new(GitGetter) dst := testing_helper.TempDir(t) encodedKey := base64.StdEncoding.EncodeToString([]byte(testGitToken)) @@ -412,13 +411,14 @@ func TestGitGetter_sshSCPStyle(t *testing.T) { Mode: ModeDir, } + getter := &GitGetter{[]Detector{ + new(GitDetector), + new(BitBucketDetector), + new(GitHubDetector), + }, + } client := &Client{ - Detectors: []Detector{ - new(GitDetector), - }, - Getters: map[string]Getter{ - "git": g, - }, + Getters: []Getter{getter}, } if _, err := client.Get(ctx, req); err != nil { @@ -437,7 +437,6 @@ func TestGitGetter_sshExplicitPort(t *testing.T) { } ctx := context.Background() - g := new(GitGetter) dst := testing_helper.TempDir(t) encodedKey := base64.StdEncoding.EncodeToString([]byte(testGitToken)) @@ -456,13 +455,7 @@ func TestGitGetter_sshExplicitPort(t *testing.T) { Mode: ModeDir, } client := &Client{ - - Detectors: []Detector{ - new(GitDetector), - }, - Getters: map[string]Getter{ - "git": g, - }, + Getters: []Getter{new(GitGetter)}, } if _, err := client.Get(ctx, req); err != nil { @@ -481,7 +474,6 @@ func TestGitGetter_sshSCPStyleInvalidScheme(t *testing.T) { } ctx := context.Background() - g := new(GitGetter) dst := testing_helper.TempDir(t) encodedKey := base64.StdEncoding.EncodeToString([]byte(testGitToken)) @@ -501,24 +493,13 @@ func TestGitGetter_sshSCPStyleInvalidScheme(t *testing.T) { } client := &Client{ - Detectors: []Detector{ - new(GitDetector), - }, - Getters: map[string]Getter{ - "git": g, - }, + Getters: []Getter{new(GitGetter)}, } _, err := client.Get(ctx, req) if err == nil { t.Fatalf("get succeeded; want error") } - - got := err.Error() - want1, want2 := `invalid source string`, `invalid port number "hashicorp"` - if !(strings.Contains(got, want1) || strings.Contains(got, want2)) { - t.Fatalf("wrong error\ngot: %s\nwant: %q or %q", got, want1, want2) - } } func TestGitGetter_submodule(t *testing.T) { @@ -619,6 +600,136 @@ func TestGitGetter_setupGitEnvWithExisting_sshKey(t *testing.T) { } } +func TestGitGetter_GitHubDetector(t *testing.T) { + cases := []struct { + Input string + Output string + }{ + // HTTP + {"github.com/hashicorp/foo", "https://github.com/hashicorp/foo.git"}, + {"github.com/hashicorp/foo.git", "https://github.com/hashicorp/foo.git"}, + { + "github.com/hashicorp/foo/bar", + "https://github.com/hashicorp/foo.git//bar", + }, + { + "github.com/hashicorp/foo?foo=bar", + "https://github.com/hashicorp/foo.git?foo=bar", + }, + { + "github.com/hashicorp/foo.git?foo=bar", + "https://github.com/hashicorp/foo.git?foo=bar", + }, + } + + pwd := "/pwd" + f := &GitGetter{[]Detector{ + new(GitDetector), + new(BitBucketDetector), + new(GitHubDetector), + }, + } + for i, tc := range cases { + req := &Request{ + Src: tc.Input, + Pwd: pwd, + } + ok, err := Detect(req, f) + if err != nil { + t.Fatalf("err: %s", err) + } + if !ok { + t.Fatal("not ok") + } + + if req.Src != tc.Output { + t.Fatalf("%d: bad: %#v", i, req.Src) + } + } +} + +func TestGitGetter_Detector(t *testing.T) { + cases := []struct { + Input string + Output string + }{ + { + "git@github.com:hashicorp/foo.git", + "ssh://git@github.com/hashicorp/foo.git", + }, + { + "git@github.com:org/project.git?ref=test-branch", + "ssh://git@github.com/org/project.git?ref=test-branch", + }, + { + "git@github.com:hashicorp/foo.git//bar", + "ssh://git@github.com/hashicorp/foo.git//bar", + }, + { + "git@github.com:hashicorp/foo.git?foo=bar", + "ssh://git@github.com/hashicorp/foo.git?foo=bar", + }, + { + "git@github.xyz.com:org/project.git", + "ssh://git@github.xyz.com/org/project.git", + }, + { + "git@github.xyz.com:org/project.git?ref=test-branch", + "ssh://git@github.xyz.com/org/project.git?ref=test-branch", + }, + { + "git@github.xyz.com:org/project.git//module/a", + "ssh://git@github.xyz.com/org/project.git//module/a", + }, + { + "git@github.xyz.com:org/project.git//module/a?ref=test-branch", + "ssh://git@github.xyz.com/org/project.git//module/a?ref=test-branch", + }, + { + "git::ssh://git@git.example.com:2222/hashicorp/foo.git", + "ssh://git@git.example.com:2222/hashicorp/foo.git", + }, + { + "bitbucket.org/hashicorp/tf-test-git", + "https://bitbucket.org/hashicorp/tf-test-git.git", + }, + { + "bitbucket.org/hashicorp/tf-test-git.git", + "https://bitbucket.org/hashicorp/tf-test-git.git", + }, + { + "git::ssh://git@git.example.com:2222/hashicorp/foo.git", + "ssh://git@git.example.com:2222/hashicorp/foo.git", + }, + } + + pwd := "/pwd" + getter := &GitGetter{[]Detector{ + new(GitDetector), + new(BitBucketDetector), + new(GitHubDetector), + }, + } + for _, tc := range cases { + t.Run(tc.Input, func(t *testing.T) { + req := &Request{ + Src: tc.Input, + Pwd: pwd, + } + ok, err := Detect(req, getter) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + if !ok { + t.Fatalf("bad: should be ok") + } + if req.Src != tc.Output { + t.Errorf("wrong result\ninput: %s\ngot: %s\nwant: %s", tc.Input, req.Src, tc.Output) + } + }) + } +} + // gitRepo is a helper struct which controls a single temp git repo. type gitRepo struct { t *testing.T diff --git a/get_hg.go b/get_hg.go index c60141573..8cc4fe4c6 100644 --- a/get_hg.go +++ b/get_hg.go @@ -15,8 +15,7 @@ import ( // HgGetter is a Getter implementation that will download a module from // a Mercurial repository. -type HgGetter struct { -} +type HgGetter struct{} func (g *HgGetter) Mode(ctx context.Context, _ *url.URL) (Mode, error) { return ModeDir, nil @@ -125,6 +124,63 @@ func (g *HgGetter) update(ctx context.Context, dst string, u *url.URL, rev strin return getRunCommand(cmd) } +func (g *HgGetter) Detect(req *Request) (bool, error) { + src := req.Src + if len(src) == 0 { + return false, nil + } + + if req.Forced != "" { + // There's a getter being Forced + if !g.validScheme(req.Forced) { + // Current getter is not the Forced one + // Don't use it to try to download the artifact + return false, nil + } + } + isForcedGetter := req.Forced != "" && g.validScheme(req.Forced) + + u, err := url.Parse(src) + if err == nil && u.Scheme != "" { + if isForcedGetter { + // Is the Forced getter and source is a valid url + return true, nil + } + if g.validScheme(u.Scheme) { + return true, nil + } + // Valid url with a scheme that is not valid for current getter + return false, nil + } + + src, ok, err := new(BitBucketDetector).Detect(src, req.Pwd) + if err != nil { + return ok, err + } + forced, src := getForcedGetter(src) + if ok && g.validScheme(forced) { + req.Src = src + return ok, nil + } + + if isForcedGetter { + // Is the Forced getter and should be used to download the artifact + if req.Pwd != "" && !filepath.IsAbs(src) { + // Make sure to add pwd to relative paths + src = filepath.Join(req.Pwd, src) + } + // Make sure we're using "/" on Windows. URLs are "/"-based. + req.Src = filepath.ToSlash(src) + return true, nil + } + + return false, nil +} + +func (g *HgGetter) validScheme(scheme string) bool { + return scheme == "hg" +} + func fixWindowsDrivePath(u *url.URL) bool { // hg assumes a file:/// prefix for Windows drive letter file paths. // (e.g. file:///c:/foo/bar) diff --git a/get_hg_test.go b/get_hg_test.go index 41486bc30..a9b9f9bc7 100644 --- a/get_hg_test.go +++ b/get_hg_test.go @@ -2,9 +2,11 @@ package getter import ( "context" + "net/http" "os" "os/exec" "path/filepath" + "strings" "testing" testing_helper "github.com/hashicorp/go-getter/v2/helper/testing" @@ -118,3 +120,54 @@ func TestHgGetter_GetFile(t *testing.T) { } testing_helper.AssertContents(t, dst, "Hello\n") } + +func TestHgGetter_DetectBitBucketDetector(t *testing.T) { + t.Parallel() + + if _, err := http.Get(testBBUrl); err != nil { + t.Log("internet may not be working, skipping BB tests") + t.Skip() + } + + cases := []struct { + Input string + Output string + }{ + { + "bitbucket.org/hashicorp/tf-test-hg", + "https://bitbucket.org/hashicorp/tf-test-hg", + }, + } + + pwd := "/pwd" + for i, tc := range cases { + var err error + for i := 0; i < 3; i++ { + var ok bool + req := &Request{ + Src: tc.Input, + Pwd: pwd, + } + ok, err = Detect(req, new(HgGetter)) + if err != nil { + if strings.Contains(err.Error(), "invalid character") { + continue + } + + t.Fatalf("err: %s", err) + } + if !ok { + t.Fatal("not ok") + } + + if req.Src != tc.Output { + t.Fatalf("%d: bad: %#v", i, req.Src) + } + + break + } + if i >= 3 { + t.Fatalf("failure from bitbucket: %s", err) + } + } +} diff --git a/get_http.go b/get_http.go index 1c5f4ad34..f4dbfd346 100644 --- a/get_http.go +++ b/get_http.go @@ -319,3 +319,38 @@ func charsetReader(charset string, input io.Reader) (io.Reader, error) { return nil, fmt.Errorf("can't decode XML document using charset %q", charset) } } + +func (g *HttpGetter) Detect(req *Request) (bool, error) { + if len(req.Src) == 0 { + return false, nil + } + + if req.Forced != "" { + // There's a getter being Forced + if !g.validScheme(req.Forced) { + // Current getter is not the Forced one + // Don't use it to try to download the artifact + return false, nil + } + } + isForcedGetter := req.Forced != "" && g.validScheme(req.Forced) + + u, err := url.Parse(req.Src) + if err == nil && u.Scheme != "" { + if isForcedGetter { + // Is the Forced getter and source is a valid url + return true, nil + } + if g.validScheme(u.Scheme) { + return true, nil + } + // Valid url with a scheme that is not valid for current getter + return false, nil + } + + return false, nil +} + +func (g *HttpGetter) validScheme(scheme string) bool { + return scheme == "http" || scheme == "https" +} diff --git a/get_mock.go b/get_mock.go index 6690f80ef..a0aa1050d 100644 --- a/get_mock.go +++ b/get_mock.go @@ -52,3 +52,10 @@ func (g *MockGetter) Mode(ctx context.Context, u *url.URL) (Mode, error) { } return ModeFile, nil } + +func (g *MockGetter) Detect(req *Request) (bool, error) { + if g.Proxy != nil { + return g.Proxy.Detect(req) + } + return true, nil +} diff --git a/get_smb_mount.go b/get_smb_mount.go new file mode 100644 index 000000000..3e55cfd8c --- /dev/null +++ b/get_smb_mount.go @@ -0,0 +1,102 @@ +package getter + +import ( + "context" + "net/url" + "os" + "path/filepath" + "runtime" +) + +// SmbMountGetter is a Getter implementation that will download an artifact from +// a shared folder using the file system using FileGetter implementation. +// For Unix and MacOS users, the Getter will look for usual system specific mount paths such as: +// /Volumes/ for MacOS +// /run/user/1000/gvfs/smb-share:server=,share= for Unix +type SmbMountGetter struct{} + +func (g *SmbMountGetter) Mode(ctx context.Context, u *url.URL) (Mode, error) { + if u.Host == "" || u.Path == "" { + return 0, new(smbPathError) + } + + prefix, path := g.findPrefixAndPath(u) + u.Path = prefix + path + + return new(FileGetter).Mode(ctx, u) +} + +func (g *SmbMountGetter) Get(ctx context.Context, req *Request) error { + if req.u.Host == "" || req.u.Path == "" { + return new(smbPathError) + } + + prefix, path := g.findPrefixAndPath(req.u) + req.u.Path = prefix + path + + return new(FileGetter).Get(ctx, req) +} + +func (g *SmbMountGetter) GetFile(ctx context.Context, req *Request) error { + if req.u.Host == "" || req.u.Path == "" { + return new(smbPathError) + } + + prefix, path := g.findPrefixAndPath(req.u) + req.u.Path = prefix + path + + return new(FileGetter).GetFile(ctx, req) +} + +func (g *SmbMountGetter) findPrefixAndPath(u *url.URL) (string, string) { + var prefix, path string + switch runtime.GOOS { + case "windows": + prefix = string(os.PathSeparator) + string(os.PathSeparator) + path = filepath.Join(u.Host, u.Path) + case "darwin": + prefix = string(os.PathSeparator) + path = filepath.Join("Volumes", u.Path) + } + return prefix, path +} + +func (g *SmbMountGetter) Detect(req *Request) (bool, error) { + if runtime.GOOS == "linux" { + // Linux has the smbclient command which is a safer approach to retrieve an artifact from a samba shared folder. + // Therefore, this should be used instead of looking in the file system. + return false, nil + } + if len(req.Src) == 0 { + return false, nil + } + + if req.Forced != "" { + // There's a getter being Forced + if !g.validScheme(req.Forced) { + // Current getter is not the Forced one + // Don't use it to try to download the artifact + return false, nil + } + } + isForcedGetter := req.Forced != "" && g.validScheme(req.Forced) + + u, err := url.Parse(req.Src) + if err == nil && u.Scheme != "" { + if isForcedGetter { + // Is the Forced getter and source is a valid url + return true, nil + } + if g.validScheme(u.Scheme) { + return true, nil + } + // Valid url with a scheme that is not valid for current getter + return false, nil + } + + return false, nil +} + +func (g *SmbMountGetter) validScheme(scheme string) bool { + return scheme == "smb" +} diff --git a/get_smbclient.go b/get_smbclient.go new file mode 100644 index 000000000..802cb9f2d --- /dev/null +++ b/get_smbclient.go @@ -0,0 +1,339 @@ +package getter + +import ( + "bytes" + "context" + "fmt" + "net/url" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "syscall" +) + +// SmbClientGetter is a Getter implementation that will download a module from +// a shared folder using smbclient cli. +type SmbClientGetter struct{} + +func (g *SmbClientGetter) Mode(ctx context.Context, u *url.URL) (Mode, error) { + if u.Host == "" || u.Path == "" { + return 0, new(smbPathError) + } + + // Use smbclient cli to verify mode + mode, err := g.smbClientMode(u) + if err == nil { + return mode, nil + } + return 0, &smbGeneralError{err} +} + +func (g *SmbClientGetter) smbClientMode(u *url.URL) (Mode, error) { + hostPath, filePath, err := g.findHostAndFilePath(u) + if err != nil { + return 0, err + } + file := "" + // Get file and subdirectory name when existent + if strings.Contains(filePath, "/") { + i := strings.LastIndex(filePath, "/") + file = filePath[i+1:] + filePath = filePath[:i] + } else { + file = filePath + filePath = "." + } + + cmdArgs := g.smbclientCmdArgs(u.User, hostPath, filePath) + // check if file exists in the smb shared folder and check the mode + isDir, err := g.isDirectory(cmdArgs, file) + if err != nil { + return 0, err + } + if isDir { + return ModeDir, nil + } + return ModeFile, nil +} + +func (g *SmbClientGetter) Get(ctx context.Context, req *Request) error { + if req.u.Host == "" || req.u.Path == "" { + return new(smbPathError) + } + + // If dst folder doesn't exists, we need to remove the created on later in case of failures + dstExisted := false + if req.Dst != "" { + if _, err := os.Lstat(req.Dst); err == nil { + dstExisted = true + } + } + + // Download the directory content using smbclient cli + err := g.smbclientGet(req) + if err == nil { + return nil + } + + if !dstExisted { + // Remove the destination created for smbclient + os.Remove(req.Dst) + } + + return &smbGeneralError{err} +} + +func (g *SmbClientGetter) smbclientGet(req *Request) error { + hostPath, directory, err := g.findHostAndFilePath(req.u) + if err != nil { + return err + } + + cmdArgs := g.smbclientCmdArgs(req.u.User, hostPath, ".") + // check directory exists in the smb shared folder and is a directory + isDir, err := g.isDirectory(cmdArgs, directory) + if err != nil { + return err + } + if !isDir { + return fmt.Errorf("%s source path must be a directory", directory) + } + + // download everything that's inside the directory (files and subdirectories) + cmdArgs = append(cmdArgs, "-c") + cmdArgs = append(cmdArgs, "prompt OFF;recurse ON; mget *") + + if req.Dst != "" { + _, err := os.Lstat(req.Dst) + if err != nil { + if os.IsNotExist(err) { + // Create destination folder if it doesn't exist + if err := os.MkdirAll(req.Dst, 0755); err != nil { + return fmt.Errorf("failed to create destination path: %s", err.Error()) + } + } else { + return err + } + } + } + + _, err = g.runSmbClientCommand(req.Dst, cmdArgs) + return err +} + +func (g *SmbClientGetter) GetFile(ctx context.Context, req *Request) error { + if req.u.Host == "" || req.u.Path == "" { + return new(smbPathError) + } + + // If dst folder doesn't exist, we need to remove the created one later in case of failures + dstExisted := false + if req.Dst != "" { + if _, err := os.Lstat(req.Dst); err == nil { + dstExisted = true + } + } + + // If not mounted, try downloading the file using smbclient cli + err := g.smbclientGetFile(req) + if err == nil { + return nil + } + + if !dstExisted { + // Remove the destination created for smbclient + os.Remove(req.Dst) + } + + return &smbGeneralError{err} +} + +func (g *SmbClientGetter) smbclientGetFile(req *Request) error { + hostPath, filePath, err := g.findHostAndFilePath(req.u) + if err != nil { + return err + } + + // Get file and subdirectory name when existent + file := "" + if strings.Contains(filePath, "/") { + i := strings.LastIndex(filePath, "/") + file = filePath[i+1:] + filePath = filePath[:i] + } else { + file = filePath + filePath = "." + } + + cmdArgs := g.smbclientCmdArgs(req.u.User, hostPath, filePath) + // check file exists in the smb shared folder and is not a directory + isDir, err := g.isDirectory(cmdArgs, file) + if err != nil { + return err + } + if isDir { + return fmt.Errorf("%s source path must be a file", file) + } + + // download file + cmdArgs = append(cmdArgs, "-c") + if req.Dst != "" { + _, err := os.Lstat(req.Dst) + if err != nil { + if os.IsNotExist(err) { + // Create destination folder if it doesn't exist + if err := os.MkdirAll(filepath.Dir(req.Dst), 0755); err != nil { + return fmt.Errorf("failed to creat destination path: %s", err.Error()) + } + } else { + return err + } + } + cmdArgs = append(cmdArgs, fmt.Sprintf("get %s %s", file, req.Dst)) + } else { + cmdArgs = append(cmdArgs, fmt.Sprintf("get %s", file)) + } + + _, err = g.runSmbClientCommand("", cmdArgs) + return err +} + +func (g *SmbClientGetter) smbclientCmdArgs(used *url.Userinfo, hostPath string, fileDir string) (baseCmd []string) { + baseCmd = append(baseCmd, "-N") + + // Append auth user and password to baseCmd + auth := used.Username() + if auth != "" { + if password, ok := used.Password(); ok { + auth = auth + "%" + password + } + baseCmd = append(baseCmd, "-U") + baseCmd = append(baseCmd, auth) + } + + baseCmd = append(baseCmd, hostPath) + baseCmd = append(baseCmd, "--directory") + baseCmd = append(baseCmd, fileDir) + return baseCmd +} + +func (g *SmbClientGetter) findHostAndFilePath(u *url.URL) (string, string, error) { + // Host path + hostPath := "//" + u.Host + + // Get shared directory + path := strings.TrimPrefix(u.Path, "/") + splt := regexp.MustCompile(`/`) + directories := splt.Split(path, 2) + + if len(directories) > 0 { + hostPath = hostPath + "/" + directories[0] + } + + // Check file path + if len(directories) <= 1 || directories[1] == "" { + return "", "", fmt.Errorf("can not find file path and/or name in the smb url") + } + + return hostPath, directories[1], nil +} + +func (g *SmbClientGetter) isDirectory(args []string, object string) (bool, error) { + args = append(args, "-c") + args = append(args, fmt.Sprintf("allinfo %s", object)) + output, err := g.runSmbClientCommand("", args) + if err != nil { + return false, err + } + if strings.Contains(output, "OBJECT_NAME_NOT_FOUND") { + return false, fmt.Errorf("source path not found: %s", output) + } + return strings.Contains(output, "attributes: D"), nil +} + +func (g *SmbClientGetter) runSmbClientCommand(dst string, args []string) (string, error) { + cmd := exec.Command("smbclient", args...) + + if dst != "" { + cmd.Dir = dst + } + + var buf bytes.Buffer + cmd.Stdout = &buf + cmd.Stderr = &buf + + err := cmd.Run() + if err == nil { + return buf.String(), nil + } + if exiterr, ok := err.(*exec.ExitError); ok { + // The program has exited with an exit code != 0 + if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { + return buf.String(), fmt.Errorf( + "%s exited with %d: %s", + cmd.Path, + status.ExitStatus(), + buf.String()) + } + } + return buf.String(), fmt.Errorf("error running %s: %s", cmd.Path, buf.String()) +} + +func (g *SmbClientGetter) Detect(req *Request) (bool, error) { + if len(req.Src) == 0 { + return false, nil + } + + if req.Forced != "" { + // There's a getter being Forced + if !g.validScheme(req.Forced) { + // Current getter is not the Forced one + // Don't use it to try to download the artifact + return false, nil + } + } + isForcedGetter := req.Forced != "" && g.validScheme(req.Forced) + + u, err := url.Parse(req.Src) + if err == nil && u.Scheme != "" { + if isForcedGetter { + // Is the Forced getter and source is a valid url + return true, nil + } + if g.validScheme(u.Scheme) { + return true, nil + } + // Valid url with a scheme that is not valid for current getter + return false, nil + } + + return false, nil +} + +func (g *SmbClientGetter) validScheme(scheme string) bool { + return scheme == "smb" +} + +type smbPathError struct { + Path string +} + +func (e *smbPathError) Error() string { + if e.Path == "" { + return "samba path should contain valid host, filepath, and authentication if necessary (smb://:@/)" + } + return fmt.Sprintf("samba path should contain valid host, filepath, and authentication if necessary (%s)", e.Path) +} + +type smbGeneralError struct { + err error +} + +func (e *smbGeneralError) Error() string { + if e != nil { + return fmt.Sprintf("smbclient cli needs to be installed and credentials provided when necessary. \n err: %s", e.err.Error()) + } + return "smbclient cli needs to be installed and credentials provided when necessary." +} diff --git a/get_smbclient_test.go b/get_smbclient_test.go new file mode 100644 index 000000000..11c517be8 --- /dev/null +++ b/get_smbclient_test.go @@ -0,0 +1,329 @@ +package getter + +import ( + "context" + urlhelper "github.com/hashicorp/go-getter/helper/url" + testing_helper "github.com/hashicorp/go-getter/v2/helper/testing" + "os" + "path/filepath" + "testing" +) + +// smbTestsPreCheck checks whether ACC_SMB_TEST is set before running any SMB tests. +// SMB tests depends on a SMB server and should not run without the intention of it. +func smbTestsPreCheck(t *testing.T) { + r := os.Getenv("ACC_SMB_TEST") + if r != "1" { + t.Skip("smb getter tests won't run. ACC_SMB_TEST not set") + } +} + +func TestSmb_GetterImpl(t *testing.T) { + var _ Getter = new(SmbClientGetter) +} + +func TestSmb_GetterGet(t *testing.T) { + smbTestsPreCheck(t) + + tests := []struct { + name string + rawURL string + file string + fail bool + }{ + { + "smbclient with registered authentication in private share", + "smb://user:password@samba/private/subdir", + "file.txt", + false, + }, + { + "smbclient with registered authentication with file in private share", + "smb://user:password@samba/private/subdir/file.txt", + "file.txt", + true, + }, + { + "smbclient with only registered username authentication in private share", + "smb://user@samba/private/subdir", + "file.txt", + true, + }, + { + "smbclient with non registered username authentication in public share", + "smb://username@samba/public/subdir", + "file.txt", + false, + }, + { + "smbclient without authentication in private share", + "smb://samba/private/subdir", + "file.txt", + true, + }, + { + "smbclient without authentication in public share", + "smb://samba/public/subdir", + "file.txt", + false, + }, + { + "non existent directory in private share", + "smb://user:password@samba/private/invalid", + "", + true, + }, + { + "non existent directory in public share", + "smb://samba/public/invalid", + "", + true, + }, + { + "no hostname provided", + "smb://", + "", + true, + }, + { + "no filepath provided", + "smb://samba", + "", + true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dst := testing_helper.TempDir(t) + defer os.RemoveAll(dst) + + url, err := urlhelper.Parse(tt.rawURL) + if err != nil { + t.Fatalf("err: %s", err.Error()) + } + req := &Request{ + Dst: dst, + u: url, + } + + g := new(SmbClientGetter) + err = g.Get(context.Background(), req) + + fail := err != nil + if tt.fail != fail { + if fail { + t.Fatalf("err: unexpected error %s", err.Error()) + } + t.Fatalf("err: expecting to fail but it did not") + } + + if !tt.fail { + // Verify if the file was successfully downloaded + // and exists at the destination folder + testing_helper.AssertContents(t, filepath.Join(dst, tt.file), "Hello\n") + } + }) + } +} + +func TestSmb_GetterGetFile(t *testing.T) { + smbTestsPreCheck(t) + + tests := []struct { + name string + rawURL string + file string + fail bool + }{ + { + "smbclient with registered authentication in private share", + "smb://user:password@samba/private/file.txt", + "file.txt", + false, + }, + { + "smbclient with registered authentication and subdirectory in private share", + "smb://user:password@samba/private/subdir/file.txt", + "file.txt", + false, + }, + { + "smbclient with only registered username authentication in private share", + "smb://user@samba/private/file.txt", + "file.txt", + true, + }, + { + "smbclient with non registered username authentication in public share", + "smb://username@samba/public/file.txt", + "file.txt", + false, + }, + { + "smbclient without authentication in public share", + "smb://samba/public/file.txt", + "file.txt", + false, + }, + { + "smbclient without authentication in private share", + "smb://samba/private/file.txt", + "file.txt", + true, + }, + { + "smbclient get directory in private share", + "smb://user:password@samba/private/subdir", + "", + true, + }, + { + "smbclient get directory in public share", + "smb://samba/public/subdir", + "", + true, + }, + { + "non existent file in private share", + "smb://user:password@samba/private/invalidfile.txt", + "", + true, + }, + { + "non existent file in public share", + "smb://samba/public/invalidfile.txt", + "", + true, + }, + { + "no hostname provided", + "smb://", + "", + true, + }, + { + "no filepath provided", + "smb://samba", + "", + true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dst := testing_helper.TempDir(t) + defer os.RemoveAll(dst) + + url, err := urlhelper.Parse(tt.rawURL) + if err != nil { + t.Fatalf("err: %s", err.Error()) + } + req := &Request{ + Dst: filepath.Join(dst, tt.file), + u: url, + } + + g := new(SmbClientGetter) + err = g.GetFile(context.Background(), req) + + fail := err != nil + if tt.fail != fail { + if fail { + t.Fatalf("err: unexpected error %s", err.Error()) + } + t.Fatalf("err: expecting to fail but it did not") + } + + if !tt.fail { + // Verify if the file was successfully downloaded + // and exists at the destination folder + testing_helper.AssertContents(t, filepath.Join(dst, tt.file), "Hello\n") + } + }) + } +} + +func TestSmb_GetterMode(t *testing.T) { + smbTestsPreCheck(t) + + tests := []struct { + name string + rawURL string + expectedMode Mode + fail bool + }{ + { + "smbclient modefile for existing file in authenticated private share", + "smb://user:password@samba/private/file.txt", + ModeFile, + false, + }, + { + "smbclient modedir for existing directory in authenticated private share", + "smb://user:password@samba/private/subdir", + ModeDir, + false, + }, + { + "mode fail for non existent directory in authenticated private share", + "smb://user:password@samba/private/invaliddir", + 0, + true, + }, + { + "mode fail for non existent file in authenticated private share", + "smb://user:password@samba/private/invalidfile.txt", + 0, + true, + }, + { + "smbclient modefile for existing file in public share", + "smb://samba/public/file.txt", + ModeFile, + false, + }, + { + "smbclient modedir for existing directory in public share", + "smb://samba/public/subdir", + ModeDir, + false, + }, + { + "mode fail for non existent directory in public share", + "smb://samba/public/invaliddir", + 0, + true, + }, + { + "mode fail for non existent file in public share", + "smb://samba/public/invalidfile.txt", + 0, + true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + url, err := urlhelper.Parse(tt.rawURL) + if err != nil { + t.Fatalf("err: %s", err.Error()) + } + + g := new(SmbClientGetter) + mode, err := g.Mode(context.Background(), url) + + fail := err != nil + if tt.fail != fail { + if fail { + t.Fatalf("err: unexpected error %s", err.Error()) + } + t.Fatalf("err: expecting to fail but it did not") + } + + if mode != tt.expectedMode { + t.Fatalf("err: expeting mode %d, actual mode %d", tt.expectedMode, mode) + } + }) + } +} diff --git a/get_test.go b/get_test.go index dd4346b58..3e803d2d5 100644 --- a/get_test.go +++ b/get_test.go @@ -16,7 +16,7 @@ func TestGet_badSchema(t *testing.T) { dst := testing_helper.TempDir(t) u := testModule("basic") - u = strings.Replace(u, "file", "nope", -1) + u = "nope::" + u op, err := Get(ctx, dst, u) if err == nil { @@ -589,9 +589,7 @@ func TestGetFile_checksumURL(t *testing.T) { Mode: ModeFile, } client := &Client{ - Getters: map[string]Getter{ - "file": getter, - }, + Getters: []Getter{getter}, } op, err := client.Get(ctx, req) @@ -644,9 +642,7 @@ func TestGetFile_checksumSkip(t *testing.T) { Mode: ModeFile, } client := &Client{ - Getters: map[string]Getter{ - "file": getter, - }, + Getters: []Getter{getter}, } // get the file @@ -694,9 +690,7 @@ func TestGetFile_inplace(t *testing.T) { Inplace: true, } client := &Client{ - Getters: map[string]Getter{ - "file": getter, - }, + Getters: []Getter{getter}, } // get the file @@ -744,9 +738,7 @@ func TestGetFile_inplace_badChecksum(t *testing.T) { Inplace: true, } client := &Client{ - Getters: map[string]Getter{ - "file": getter, - }, + Getters: []Getter{getter}, } // get the file diff --git a/go.mod b/go.mod index 2657a23c2..b472fcc26 100644 --- a/go.mod +++ b/go.mod @@ -5,11 +5,13 @@ require ( github.com/google/go-cmp v0.3.0 github.com/hashicorp/go-cleanhttp v0.5.0 github.com/hashicorp/go-getter v1.4.1 + github.com/hashicorp/go-multierror v1.1.0 github.com/hashicorp/go-safetemp v1.0.0 github.com/hashicorp/go-version v1.1.0 github.com/mitchellh/go-homedir v1.0.0 github.com/mitchellh/go-testing-interface v1.0.0 github.com/ulikunitz/xz v0.5.5 + google.golang.org/api v0.9.0 ) go 1.13 diff --git a/go.sum b/go.sum index be76515c7..b2641bfc0 100644 --- a/go.sum +++ b/go.sum @@ -1,17 +1,138 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= +github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-getter v1.4.1 h1:3A2Mh8smGFcf5M+gmcv898mZdrxpseik45IpcyISLsA= +github.com/hashicorp/go-getter v1.4.1/go.mod h1:7qxyCd8rBfcShwsvxgIguu4KbS3l8bUCwg2Umn7RjeY= +github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0= github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok= github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/request.go b/request.go index 241adc9e7..5b4362696 100644 --- a/request.go +++ b/request.go @@ -18,6 +18,21 @@ type Request struct { Dst string Pwd string + // Forced is the forced getter detected in the Src string during the + // Getter detection. Forcing a getter means that go-getter will try + // to download the artifact only with the Getter that is being forced. + // + // For example: + // + // Request.Src Forced + // git::ssh://git@git.example.com:2222/foo/bar.git git + // + // This field is used by the Getters to validate when they are forced to download + // the artifact. + // If both Request.Src and Forced contains a forced getter, the one in the Request.Src will + // be considered and will override the value of this field. + Forced string + // Mode is the method of download the client will use. See Mode // for documentation. Mode Mode @@ -37,7 +52,8 @@ type Request struct { // By default a no op progress listener is used. ProgressListener ProgressTracker - u *url.URL + u *url.URL + subDir, realDst string } func (req *Request) URL() *url.URL { diff --git a/detect_s3.go b/s3/detect_s3.go similarity index 68% rename from detect_s3.go rename to s3/detect_s3.go index 8e0f4a03b..5749500ca 100644 --- a/detect_s3.go +++ b/s3/detect_s3.go @@ -1,4 +1,4 @@ -package getter +package s3 import ( "fmt" @@ -6,11 +6,11 @@ import ( "strings" ) -// S3Detector implements Detector to detect S3 URLs and turn +// Detector implements Detector to detect S3 URLs and turn // them into URLs that the S3 getter can understand. -type S3Detector struct{} +type Detector struct{} -func (d *S3Detector) Detect(src, _ string) (string, bool, error) { +func (d *Detector) Detect(src, _ string) (string, bool, error) { if len(src) == 0 { return "", false, nil } @@ -22,7 +22,7 @@ func (d *S3Detector) Detect(src, _ string) (string, bool, error) { return "", false, nil } -func (d *S3Detector) detectHTTP(src string) (string, bool, error) { +func (d *Detector) detectHTTP(src string) (string, bool, error) { parts := strings.Split(src, "/") if len(parts) < 2 { return "", false, fmt.Errorf( @@ -40,22 +40,22 @@ func (d *S3Detector) detectHTTP(src string) (string, bool, error) { } } -func (d *S3Detector) detectPathStyle(region string, parts []string) (string, bool, error) { +func (d *Detector) detectPathStyle(region string, parts []string) (string, bool, error) { urlStr := fmt.Sprintf("https://%s.amazonaws.com/%s", region, strings.Join(parts, "/")) url, err := url.Parse(urlStr) if err != nil { return "", false, fmt.Errorf("error parsing S3 URL: %s", err) } - return "s3::" + url.String(), true, nil + return url.String(), true, nil } -func (d *S3Detector) detectVhostStyle(region, bucket string, parts []string) (string, bool, error) { +func (d *Detector) detectVhostStyle(region, bucket string, parts []string) (string, bool, error) { urlStr := fmt.Sprintf("https://%s.amazonaws.com/%s/%s", region, bucket, strings.Join(parts, "/")) url, err := url.Parse(urlStr) if err != nil { return "", false, fmt.Errorf("error parsing S3 URL: %s", err) } - return "s3::" + url.String(), true, nil + return url.String(), true, nil } diff --git a/detect_s3_test.go b/s3/detect_s3_test.go similarity index 59% rename from detect_s3_test.go rename to s3/detect_s3_test.go index f410c785c..4b9cde48e 100644 --- a/detect_s3_test.go +++ b/s3/detect_s3_test.go @@ -1,4 +1,4 @@ -package getter +package s3 import ( "testing" @@ -12,62 +12,62 @@ func TestS3Detector(t *testing.T) { // Virtual hosted style { "bucket.s3.amazonaws.com/foo", - "s3::https://s3.amazonaws.com/bucket/foo", + "https://s3.amazonaws.com/bucket/foo", }, { "bucket.s3.amazonaws.com/foo/bar", - "s3::https://s3.amazonaws.com/bucket/foo/bar", + "https://s3.amazonaws.com/bucket/foo/bar", }, { "bucket.s3.amazonaws.com/foo/bar.baz", - "s3::https://s3.amazonaws.com/bucket/foo/bar.baz", + "https://s3.amazonaws.com/bucket/foo/bar.baz", }, { "bucket.s3-eu-west-1.amazonaws.com/foo", - "s3::https://s3-eu-west-1.amazonaws.com/bucket/foo", + "https://s3-eu-west-1.amazonaws.com/bucket/foo", }, { "bucket.s3-eu-west-1.amazonaws.com/foo/bar", - "s3::https://s3-eu-west-1.amazonaws.com/bucket/foo/bar", + "https://s3-eu-west-1.amazonaws.com/bucket/foo/bar", }, { "bucket.s3-eu-west-1.amazonaws.com/foo/bar.baz", - "s3::https://s3-eu-west-1.amazonaws.com/bucket/foo/bar.baz", + "https://s3-eu-west-1.amazonaws.com/bucket/foo/bar.baz", }, // Path style { "s3.amazonaws.com/bucket/foo", - "s3::https://s3.amazonaws.com/bucket/foo", + "https://s3.amazonaws.com/bucket/foo", }, { "s3.amazonaws.com/bucket/foo/bar", - "s3::https://s3.amazonaws.com/bucket/foo/bar", + "https://s3.amazonaws.com/bucket/foo/bar", }, { "s3.amazonaws.com/bucket/foo/bar.baz", - "s3::https://s3.amazonaws.com/bucket/foo/bar.baz", + "https://s3.amazonaws.com/bucket/foo/bar.baz", }, { "s3-eu-west-1.amazonaws.com/bucket/foo", - "s3::https://s3-eu-west-1.amazonaws.com/bucket/foo", + "https://s3-eu-west-1.amazonaws.com/bucket/foo", }, { "s3-eu-west-1.amazonaws.com/bucket/foo/bar", - "s3::https://s3-eu-west-1.amazonaws.com/bucket/foo/bar", + "https://s3-eu-west-1.amazonaws.com/bucket/foo/bar", }, { "s3-eu-west-1.amazonaws.com/bucket/foo/bar.baz", - "s3::https://s3-eu-west-1.amazonaws.com/bucket/foo/bar.baz", + "https://s3-eu-west-1.amazonaws.com/bucket/foo/bar.baz", }, // Misc tests { "s3-eu-west-1.amazonaws.com/bucket/foo/bar.baz?version=1234", - "s3::https://s3-eu-west-1.amazonaws.com/bucket/foo/bar.baz?version=1234", + "https://s3-eu-west-1.amazonaws.com/bucket/foo/bar.baz?version=1234", }, } pwd := "/pwd" - f := new(S3Detector) + f := new(Detector) for i, tc := range cases { output, ok, err := f.Detect(tc.Input, pwd) if err != nil { diff --git a/s3/get_s3.go b/s3/get_s3.go index c96b0a009..6696c9b00 100644 --- a/s3/get_s3.go +++ b/s3/get_s3.go @@ -20,8 +20,7 @@ import ( // Getter is a Getter implementation that will download a module from // a S3 bucket. -type Getter struct { -} +type Getter struct{} func (g *Getter) Mode(ctx context.Context, u *url.URL) (getter.Mode, error) { // Parse URL @@ -272,3 +271,47 @@ func (g *Getter) parseUrl(u *url.URL) (region, bucket, path, version string, cre return } + +func (g *Getter) Detect(req *getter.Request) (bool, error) { + src := req.Src + if len(src) == 0 { + return false, nil + } + + if req.Forced != "" { + // There's a getter being forced + if !g.validScheme(req.Forced) { + // Current getter is not the forced one + // Don't use it to try to download the artifact + return false, nil + } + } + isForcedGetter := req.Forced != "" && g.validScheme(req.Forced) + + u, err := url.Parse(src) + if err == nil && u.Scheme != "" { + if isForcedGetter { + // Is the forced getter and source is a valid url + return true, nil + } + if g.validScheme(u.Scheme) { + return true, nil + } + // Valid url with a scheme that is not valid for current getter + return false, nil + } + + src, ok, err := new(Detector).Detect(src, req.Pwd) + if err != nil { + return ok, err + } + if ok { + req.Src = src + } + + return ok, nil +} + +func (g *Getter) validScheme(scheme string) bool { + return scheme == "s3" +} diff --git a/s3/get_s3_test.go b/s3/get_s3_test.go index 6f71614bc..4de2a19af 100644 --- a/s3/get_s3_test.go +++ b/s3/get_s3_test.go @@ -42,7 +42,7 @@ func TestGetter(t *testing.T) { } c := getter.Client{ - Getters: map[string]getter.Getter{"s3": g}, + Getters: []getter.Getter{g}, } // With a dir that doesn't exist @@ -69,7 +69,7 @@ func TestGetter_subdir(t *testing.T) { } c := getter.Client{ - Getters: map[string]getter.Getter{"s3": g}, + Getters: []getter.Getter{g}, } // With a dir that doesn't exist @@ -99,7 +99,7 @@ func TestGetter_GetFile(t *testing.T) { } c := getter.Client{ - Getters: map[string]getter.Getter{"s3": g}, + Getters: []getter.Getter{g}, } // Download @@ -129,7 +129,7 @@ func TestGetter_GetFile_badParams(t *testing.T) { } c := getter.Client{ - Getters: map[string]getter.Getter{"s3": g}, + Getters: []getter.Getter{g}, } // Download @@ -157,7 +157,7 @@ func TestGetter_GetFile_notfound(t *testing.T) { } c := getter.Client{ - Getters: map[string]getter.Getter{"s3": g}, + Getters: []getter.Getter{g}, } // Download _, err := c.Get(ctx, req) @@ -311,4 +311,4 @@ func TestGetter_Url(t *testing.T) { } }) } -} +} \ No newline at end of file diff --git a/s3/go.sum b/s3/go.sum index e9766de23..9326a6fb7 100644 --- a/s3/go.sum +++ b/s3/go.sum @@ -1,20 +1,63 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= github.com/aws/aws-sdk-go v1.30.8 h1:4BHbh8K3qKmcnAgToZ2LShldRF9inoqIBccpCLNCy3I= github.com/aws/aws-sdk-go v1.30.8/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= +github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-getter v1.4.1 h1:3A2Mh8smGFcf5M+gmcv898mZdrxpseik45IpcyISLsA= +github.com/hashicorp/go-getter v1.4.1/go.mod h1:7qxyCd8rBfcShwsvxgIguu4KbS3l8bUCwg2Umn7RjeY= +github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0= github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= @@ -23,17 +66,92 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok= github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=