From 08b10411ab8d37bf48614b9797889fba933de4fe Mon Sep 17 00:00:00 2001 From: Dan Duvall Date: Fri, 2 Aug 2019 10:04:48 -0700 Subject: [PATCH] Support git shallow clones and additional ref fetches Implemented a `depth` field for git artifact configuration that, when specified, will result in a shallow clone (and fetch) of the given number of commits from the branch tip. Implemented a `fetch` field for git artifact configuration that fetches the given refspecs prior to checkout. This is necessary when one wants to retrieve git revisions that exist in non-branch/-tag refs. The motivation for these features is to support retrieval of patchset refs from Gerrit code review (`refs/changes/[n]/[change]/[patch]`) but these new fields should provide more flexibility to anyone integrating with other git-based systems. --- api/openapi-spec/swagger.json | 12 +++++++ examples/input-artifact-git.yaml | 11 +++++++ .../workflow/v1alpha1/openapi_generated.go | 21 ++++++++++++ pkg/apis/workflow/v1alpha1/types.go | 7 ++++ .../v1alpha1/zz_generated.deepcopy.go | 10 ++++++ workflow/artifacts/git/git.go | 32 ++++++++++++++++++- 6 files changed, 92 insertions(+), 1 deletion(-) diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index df852a66e895..02ad885662be 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -258,6 +258,18 @@ "repo" ], "properties": { + "depth": { + "description": "Depth specifies clones/fetches should be shallow and include the given number of commits from the branch tip", + "type": "integer", + "format": "int32" + }, + "fetch": { + "description": "Fetch specifies a number of refs that should be fetched before checkout", + "type": "array", + "items": { + "type": "string" + } + }, "insecureIgnoreHostKey": { "description": "InsecureIgnoreHostKey disables SSH strict host key checking during git clone", "type": "boolean" diff --git a/examples/input-artifact-git.yaml b/examples/input-artifact-git.yaml index de237dfe2182..42c6cc4e334e 100644 --- a/examples/input-artifact-git.yaml +++ b/examples/input-artifact-git.yaml @@ -36,6 +36,17 @@ spec: # providers (github, bitbucket, gitlab, azure) as these keys are already baked into # the executor image which performs the clone. # insecureIgnoreHostKey: true + # + # Shallow clones/fetches can be performed by providing a `depth`. + # depth: 1 + # + # Additional ref specs to fetch down prior to checkout can be + # provided with `fetch`. This may be necessary if `revision` is a + # non-branch/-tag ref and thus not covered by git's default fetch. + # See https://git-scm.com/book/en/v2/Git-Internals-The-Refspec for + # the refspec format. + # fetch: refs/meta/* + # fetch: refs/changes/* container: image: golang:1.10 command: [sh, -c] diff --git a/pkg/apis/workflow/v1alpha1/openapi_generated.go b/pkg/apis/workflow/v1alpha1/openapi_generated.go index bc3bd4d5f5b4..6946e3d0e717 100644 --- a/pkg/apis/workflow/v1alpha1/openapi_generated.go +++ b/pkg/apis/workflow/v1alpha1/openapi_generated.go @@ -514,6 +514,27 @@ func schema_pkg_apis_workflow_v1alpha1_GitArtifact(ref common.ReferenceCallback) Format: "", }, }, + "depth": { + SchemaProps: spec.SchemaProps{ + Description: "Depth specifies clones/fetches should be shallow and include the given number of commits from the branch tip", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "fetch": { + SchemaProps: spec.SchemaProps{ + Description: "Fetch specifies a number of refs that should be fetched before checkout", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, "usernameSecret": { SchemaProps: spec.SchemaProps{ Description: "UsernameSecret is the secret selector to the repository username", diff --git a/pkg/apis/workflow/v1alpha1/types.go b/pkg/apis/workflow/v1alpha1/types.go index ea8d64b9ba29..7894f69beff3 100644 --- a/pkg/apis/workflow/v1alpha1/types.go +++ b/pkg/apis/workflow/v1alpha1/types.go @@ -696,6 +696,13 @@ type GitArtifact struct { // Revision is the git commit, tag, branch to checkout Revision string `json:"revision,omitempty"` + // Depth specifies clones/fetches should be shallow and include the given + // number of commits from the branch tip + Depth *uint `json:"depth,omitempty"` + + // Fetch specifies a number of refs that should be fetched before checkout + Fetch []string `json:"fetch,omitempty"` + // UsernameSecret is the secret selector to the repository username UsernameSecret *apiv1.SecretKeySelector `json:"usernameSecret,omitempty"` diff --git a/pkg/apis/workflow/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/workflow/v1alpha1/zz_generated.deepcopy.go index c59bbc6621cb..b52bc6fb4857 100644 --- a/pkg/apis/workflow/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/workflow/v1alpha1/zz_generated.deepcopy.go @@ -272,6 +272,16 @@ func (in *DAGTemplate) DeepCopy() *DAGTemplate { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GitArtifact) DeepCopyInto(out *GitArtifact) { *out = *in + if in.Depth != nil { + in, out := &in.Depth, &out.Depth + *out = new(uint) + **out = **in + } + if in.Fetch != nil { + in, out := &in.Fetch, &out.Fetch + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.UsernameSecret != nil { in, out := &in.UsernameSecret, &out.UsernameSecret *out = new(v1.SecretKeySelector) diff --git a/workflow/artifacts/git/git.go b/workflow/artifacts/git/git.go index eb2e98104abd..ae7554ebddc2 100644 --- a/workflow/artifacts/git/git.go +++ b/workflow/artifacts/git/git.go @@ -11,6 +11,7 @@ import ( log "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" git "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/plumbing/transport/http" ssh2 "gopkg.in/src-d/go-git.v4/plumbing/transport/ssh" @@ -86,10 +87,39 @@ func gitClone(path string, inputArtifact *wfv1.Artifact, auth transport.AuthMeth RecurseSubmodules: git.DefaultSubmoduleRecursionDepth, Auth: auth, } - _, err := git.PlainClone(path, false, &cloneOptions) + if inputArtifact.Git.Depth != nil { + cloneOptions.Depth = int(*inputArtifact.Git.Depth) + } + + repo, err := git.PlainClone(path, false, &cloneOptions) if err != nil { return errors.InternalWrapError(err) } + + if inputArtifact.Git.Fetch != nil { + refSpecs := make([]config.RefSpec, len(inputArtifact.Git.Fetch)) + for i, spec := range inputArtifact.Git.Fetch { + refSpecs[i] = config.RefSpec(spec) + } + + fetchOptions := git.FetchOptions{ + RefSpecs: refSpecs, + } + if inputArtifact.Git.Depth != nil { + fetchOptions.Depth = int(*inputArtifact.Git.Depth) + } + + err = fetchOptions.Validate() + if err != nil { + return errors.InternalWrapError(err) + } + + err = repo.Fetch(&fetchOptions) + if err != nil { + return errors.InternalWrapError(err) + } + } + if inputArtifact.Git.Revision != "" { // We still rely on forking git for checkout, since go-git does not have a reliable // way of resolving revisions (e.g. mybranch, HEAD^, v1.2.3)