Support git shallow clones and additional ref fetches (#1521)

Implemented a `depth` field for git artifact configuration that, when
specified, will result in a shallow clone (and fetch) of the given
number of commits from the branch tip.

Implemented a `fetch` field for git artifact configuration that fetches
the given refspecs prior to checkout. This is necessary when one wants
to retrieve git revisions that exist in non-branch/-tag refs.

The motivation for these features is to support retrieval of patchset
refs from Gerrit code review (`refs/changes/[n]/[change]/[patch]`) but
these new fields should provide more flexibility to anyone integrating
with other git-based systems.

api/openapi-spec/swagger.json

@@ -258,6 +258,18 @@
         "repo"
       ],
       "properties": {
+        "depth": {
+          "description": "Depth specifies clones/fetches should be shallow and include the given number of commits from the branch tip",
+          "type": "integer",
+          "format": "int32"
+        },
+        "fetch": {
+          "description": "Fetch specifies a number of refs that should be fetched before checkout",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
         "insecureIgnoreHostKey": {
           "description": "InsecureIgnoreHostKey disables SSH strict host key checking during git clone",
           "type": "boolean"

examples/input-artifact-git.yaml

@@ -36,6 +36,17 @@ spec:
           # providers (github, bitbucket, gitlab, azure) as these keys are already baked into
           # the executor image which performs the clone.
           # insecureIgnoreHostKey: true
+          #
+          # Shallow clones/fetches can be performed by providing a `depth`.
+          # depth: 1
+          #
+          # Additional ref specs to fetch down prior to checkout can be
+          # provided with `fetch`. This may be necessary if `revision` is a
+          # non-branch/-tag ref and thus not covered by git's default fetch.
+          # See https://git-scm.com/book/en/v2/Git-Internals-The-Refspec for
+          # the refspec format.
+          # fetch: refs/meta/*
+          # fetch: refs/changes/*
     container:
       image: golang:1.10
       command: [sh, -c]

pkg/apis/workflow/v1alpha1/types.go

@@ -696,6 +696,13 @@ type GitArtifact struct {
 	// Revision is the git commit, tag, branch to checkout
 	Revision string `json:"revision,omitempty"`
 
+	// Depth specifies clones/fetches should be shallow and include the given
+	// number of commits from the branch tip
+	Depth *uint `json:"depth,omitempty"`
+
+	// Fetch specifies a number of refs that should be fetched before checkout
+	Fetch []string `json:"fetch,omitempty"`
+
 	// UsernameSecret is the secret selector to the repository username
 	UsernameSecret *apiv1.SecretKeySelector `json:"usernameSecret,omitempty"`
 

workflow/artifacts/git/git.go

@@ -11,6 +11,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/crypto/ssh"
 	git "gopkg.in/src-d/go-git.v4"
+	"gopkg.in/src-d/go-git.v4/config"
 	"gopkg.in/src-d/go-git.v4/plumbing/transport"
 	"gopkg.in/src-d/go-git.v4/plumbing/transport/http"
 	ssh2 "gopkg.in/src-d/go-git.v4/plumbing/transport/ssh"
@@ -86,10 +87,39 @@ func gitClone(path string, inputArtifact *wfv1.Artifact, auth transport.AuthMeth
 		RecurseSubmodules: git.DefaultSubmoduleRecursionDepth,
 		Auth:              auth,
 	}
-	_, err := git.PlainClone(path, false, &cloneOptions)
+	if inputArtifact.Git.Depth != nil {
+		cloneOptions.Depth = int(*inputArtifact.Git.Depth)
+	}
+
+	repo, err := git.PlainClone(path, false, &cloneOptions)
 	if err != nil {
 		return errors.InternalWrapError(err)
 	}
+
+	if inputArtifact.Git.Fetch != nil {
+		refSpecs := make([]config.RefSpec, len(inputArtifact.Git.Fetch))
+		for i, spec := range inputArtifact.Git.Fetch {
+			refSpecs[i] = config.RefSpec(spec)
+		}
+
+		fetchOptions := git.FetchOptions{
+			RefSpecs: refSpecs,
+		}
+		if inputArtifact.Git.Depth != nil {
+			fetchOptions.Depth = int(*inputArtifact.Git.Depth)
+		}
+
+		err = fetchOptions.Validate()
+		if err != nil {
+			return errors.InternalWrapError(err)
+		}
+
+		err = repo.Fetch(&fetchOptions)
+		if err != nil {
+			return errors.InternalWrapError(err)
+		}
+	}
+
 	if inputArtifact.Git.Revision != "" {
 		// We still rely on forking git for checkout, since go-git does not have a reliable
 		// way of resolving revisions (e.g. mybranch, HEAD^, v1.2.3)