campaigns: add and use volume mounts by default on Intel macOS

.github/workflows/docker.yml

@@ -0,0 +1,42 @@
+# For more information, refer to the "Dependent Docker images" section of
+# DEVELOPMENT.md.
+name: Publish Docker image dependencies
+
+# We only want to build on releases; this condition is 100% stolen from the
+# goreleaser action.
+on:
+  push:
+    tags:
+      - "*"
+      - "!latest"
+
+jobs:
+  publish:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      # We need buildx to be able to build a multi-architecture image.
+      - name: Set up Docker buildx
+        uses: docker/setup-buildx-action@v1
+
+      # We also need QEMU, since this is running on an AMD64 host and we want to
+      # build ARM64 images.
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: arm64
+
+      - run: ./docker/campaign-volume-workspace/push.py -d ./docker/campaign-volume-workspace/Dockerfile -i sourcegraph/src-campaign-volume-workspace -p linux/amd64,linux/arm64,linux/386
+        env:
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+          DOCKER_USERNAME: sourcegraphci
+
+      - name: Update Docker Hub description
+        uses: peter-evans/dockerhub-description@v2
+        with:
+          username: sourcegraphci
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          repository: sourcegraph/src-campaign-volume-workspace
+          readme-filepath: ./docker/campaign-volume-workspace/README.md

CHANGELOG.md

@@ -13,6 +13,8 @@ All notable changes to `src-cli` are documented in this file.
 
 ### Added
 
+- `src campaign [apply|preview]` can now make use of Docker volumes, rather than bind-mounting the host filesystem. This is now the default on macOS, as volume mounts have generally better performance there. The optional `-workspace` flag can be used to override the default. [#412](https://github.com/sourcegraph/src-cli/pull/412)
+
 ### Changed
 
 - `src login` now defaults to validating against `SRC_ENDPOINT` if configured.

DEVELOPMENT.md

@@ -38,3 +38,19 @@ For example, suppose we have the the recommended versions.
 If a new feature is added to a new `3.91.6` release of src-cli and this change requires only features available in Sourcegraph `3.99`, then this feature should also be present in a new `3.85.8` release of src-cli. Because a Sourcegraph instance will automatically select the highest patch version, all non-breaking changes should increment only the patch version. 
 
 Note that if instead the recommended src-cli version for Sourcegraph `3.99` was `3.90.4` in the example above, there is no additional step required, and the new patch version of src-cli will be available to both Sourcegraph versions.
+
+## Dependent Docker images
+
+`src campaign apply` and `src campaign preview` use a Docker image published as `sourcegraph/src-campaign-volume-workspace` for utility purposes when the volume workspace is selected, which is the default on macOS. This [Docker image](./docker/campaign-volume-workspace/Dockerfile) is built by [a Python script](./docker/campaign-volume-workspace/push.py) invoked by the GitHub Action workflow described in [`docker.yml`](.github/workflows/docker.yml).
+
+To build and develop this locally, you can build and tag the image with:
+
+```sh
+docker build -t sourcegraph/src-campaign-volume-workspace - < docker/campaign-volume-workspace/Dockerfile
+```
+
+To remove it and then force the upstream image on Docker Hub to be used again:
+
+```sh
+docker rmi sourcegraph/src-campaign-volume-workspace
+```

cmd/src/campaigns_apply.go

@@ -70,6 +70,7 @@ Examples:
 		svc := campaigns.NewService(&campaigns.ServiceOpts{
 			AllowUnsupported: flags.allowUnsupported,
 			Client:           cfg.apiClient(flags.api, flagSet.Output()),
+			Workspace:        flags.workspace,
 		})
 
 		if err := svc.DetermineFeatureFlags(ctx); err != nil {

cmd/src/campaigns_common.go

@@ -40,6 +40,7 @@ type campaignsApplyFlags struct {
 	namespace        string
 	parallelism      int
 	timeout          time.Duration
+	workspace        string
 	cleanArchives    bool
 	skipErrors       bool
 }
@@ -100,6 +101,20 @@ func newCampaignsApplyFlags(flagSet *flag.FlagSet, cacheDir, tempDir string) *ca
 		"If true, errors encountered while executing steps in a repository won't stop the execution of the campaign spec but only cause that repository to be skipped.",
 	)
 
+	// We default to bind workspaces on everything except ARM64 macOS at
+	// present. In the future, we'll likely want to switch the default for ARM64
+	// macOS as well, but this requires access to the hardware for testing.
+	var defaultWorkspace string
+	if runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" {
+		defaultWorkspace = "volume"
+	} else {
+		defaultWorkspace = "bind"
+	}
+	flagSet.StringVar(
+		&caf.workspace, "workspace", defaultWorkspace,
+		`Workspace mode to use ("bind" or "volume")`,
+	)
+
 	flagSet.BoolVar(verbose, "v", false, "print verbose output")
 
 	return caf
@@ -176,18 +191,20 @@ func campaignsExecute(ctx context.Context, out *output.Output, svc *campaigns.Se
 	}
 
 	opts := campaigns.ExecutorOpts{
-		Cache:      svc.NewExecutionCache(flags.cacheDir),
-		Creator:    svc.NewWorkspaceCreator(flags.cacheDir, flags.cleanArchives),
-		ClearCache: flags.clearCache,
-		KeepLogs:   flags.keepLogs,
-		Timeout:    flags.timeout,
-		TempDir:    flags.tempDir,
+		Cache:       svc.NewExecutionCache(flags.cacheDir),
+		Creator:     svc.NewWorkspaceCreator(flags.cacheDir),
+		RepoFetcher: svc.NewRepoFetcher(flags.cacheDir, flags.cleanArchives),
+		ClearCache:  flags.clearCache,
+		KeepLogs:    flags.keepLogs,
+		Timeout:     flags.timeout,
+		TempDir:     flags.tempDir,
 	}
 	if flags.parallelism <= 0 {
 		opts.Parallelism = runtime.GOMAXPROCS(0)
 	} else {
 		opts.Parallelism = flags.parallelism
 	}
+	out.VerboseLine(output.Linef("🚧", output.StyleSuccess, "Workspace creator: %T", opts.Creator))
 	executor := svc.NewExecutor(opts)
 
 	if errs != nil {
@@ -210,10 +227,10 @@ func campaignsExecute(ctx context.Context, out *output.Output, svc *campaigns.Se
 
 	imageProgress := out.Progress([]output.ProgressBar{{
 		Label: "Preparing container images",
-		Max:   float64(len(campaignSpec.Steps)),
+		Max:   1.0,
 	}}, nil)
-	err = svc.SetDockerImages(ctx, campaignSpec, func(step int) {
-		imageProgress.SetValue(0, float64(step))
+	err = svc.SetDockerImages(ctx, opts.Creator, campaignSpec, func(perc float64) {
+		imageProgress.SetValue(0, perc)
 	})
 	if err != nil {
 		return "", "", err

cmd/src/campaigns_preview.go

@@ -45,6 +45,7 @@ Examples:
 		svc := campaigns.NewService(&campaigns.ServiceOpts{
 			AllowUnsupported: flags.allowUnsupported,
 			Client:           cfg.apiClient(flags.api, flagSet.Output()),
+			Workspace:        flags.workspace,
 		})
 
 		if err := svc.DetermineFeatureFlags(ctx); err != nil {

docker/campaign-volume-workspace/Dockerfile

@@ -0,0 +1,13 @@
+# This Dockerfile builds the sourcegraph/src-campaign-volume-workspace image
+# that we use to run curl, git, and unzip against a Docker volume when using
+# the volume workspace.
+
+FROM alpine:3.12.3
+
+# Note that we have to configure git's user.email and user.name settings to
+# avoid issues when committing changes. These values are not used when creating
+# changesets, since we only extract the diff from the container and not a full
+# patch, but need to be set to avoid git erroring out.
+RUN apk add --update curl git unzip && \
+    git config --global user.email campaigns@sourcegraph.com && \
+    git config --global user.name 'Sourcegraph Campaigns'

docker/campaign-volume-workspace/README.md

@@ -0,0 +1,7 @@
+# `src` volume workspace base image
+
+Sourcegraph `src` executes campaigns using either a bind or volume workspace. In the latter case (which is the default on macOS), this utility image is used to initialise the volume workspace within Docker, and then to extract the diff used when creating the changeset.
+
+This image is based on Alpine, and adds the tools we need: curl, git, and unzip.
+
+For more information, please refer to the [`src-cli` repository](https://github.com/sourcegraph/src-cli/tree/main/docker/campaign-volume-workspace).

docker/campaign-volume-workspace/push.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+
+# This is a very simple script to build and push the Docker image used by the
+# Docker volume workspace driver. It's normally run from the "Publish Docker
+# image dependencies" GitHub Action, but can be run locally if necessary.
+#
+# This script requires Python 3.8 or later, and Docker 19.03 (for buildx). You
+# are strongly encouraged to use Black to format this file after modifying it.
+#
+# To run it locally, you'll need to be logged into Docker Hub, and create an
+# image in a namespace that you have access to. For example, if your username is
+# "alice", you could build and push the image as follows:
+#
+# $ ./push.py -d Dockerfile -i alice/src-campaign-volume-workspace
+#
+# By default, only the "latest" tag will be built and pushed. The script refers
+# to the HEAD ref given to it, either via $GITHUB_REF or the -r argument. If
+# this is in the form refs/tags/X.Y.Z, then we'll also push X, X.Y, and X.Y.Z
+# tags.
+#
+# Finally, if you have your environment configured to allow multi-architecture
+# builds with docker buildx, you can provide a --platform argument that will be
+# passed through verbatim to docker buildx build. (This is how we build ARM64
+# builds in our CI.) For example, you could build ARM64 and AMD64 images with:
+#
+# $ ./push.py --platform linux/arm64,linux/amd64 ...
+#
+# For this to work, you will need a builder with the relevant platforms enabled.
+# More instructions on this can be found at
+# https://docs.docker.com/buildx/working-with-buildx/#build-multi-platform-images.
+
+import argparse
+import itertools
+import os
+import subprocess
+
+from typing import BinaryIO, Optional, Sequence
+
+
+def calculate_tags(ref: str) -> Sequence[str]:
+    # The tags always include latest.
+    tags = ["latest"]
+
+    # If the ref is a tag ref, then we should parse the version out and add each
+    # component to our tag list: for example, for X.Y.Z, we want tags X, X.Y,
+    # and X.Y.Z.
+    if ref.startswith("refs/tags/"):
+        tags.extend(
+            [
+                # Join the version components back into a string.
+                ".".join(vc)
+                for vc in itertools.accumulate(
+                    # Split the version string into its components.
+                    ref.split("/", 2)[2].split("."),
+                    # Accumulate each component we've seen into a new list
+                    # entry.
+                    lambda vlist, v: vlist + [v],
+                    initial=[],
+                )
+                # We also get the initial value, so we need to skip that.
+                if len(vc) > 0
+            ]
+        )
+
+    return tags
+
+
+def docker_build(
+    dockerfile: BinaryIO, platform: Optional[str], image: str, tags: Sequence[str]
+):
+    args = ["docker", "buildx", "build", "--push"]
+
+    for tag in tags:
+        args.extend(["-t", f"{image}:{tag}"])
+
+    if platform is not None:
+        args.extend(["--platform", platform])
+
+    # Since we provide the Dockerfile via stdin, we don't need to provide it
+    # here. (Doing so means that we don't carry an unncessary context into the
+    # builder.)
+    args.append("-")
+
+    run(args, stdin=dockerfile)
+
+
+def docker_login(username: str, password: str):
+    run(
+        ["docker", "login", f"-u={username}", "--password-stdin"],
+        input=password,
+        text=True,
+    )
+
+
+def run(args: Sequence[str], /, **kwargs) -> subprocess.CompletedProcess:
+    print(f"+ {' '.join(args)}")
+    return subprocess.run(args, check=True, **kwargs)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-d", "--dockerfile", required=True, help="the Dockerfile to build"
+    )
+    parser.add_argument("-i", "--image", required=True, help="Docker image to push")
+    parser.add_argument(
+        "-p",
+        "--platform",
+        help="platforms to build using docker buildx (if omitted, the default will be used)",
+    )
+    parser.add_argument(
+        "-r",
+        "--ref",
+        default=os.environ.get("GITHUB_REF"),
+        help="current ref in refs/heads/... or refs/tags/... form",
+    )
+    args = parser.parse_args()
+
+    tags = calculate_tags(args.ref)
+    print(f"will push tags: {', '.join(tags)}")
+
+    print("logging into Docker Hub")
+    try:
+        docker_login(os.environ["DOCKER_USERNAME"], os.environ["DOCKER_PASSWORD"])
+    except KeyError as e:
+        print(f"error retrieving environment variables: {e}")
+        raise
+
+    print("building and pushing image")
+    docker_build(open(args.dockerfile, "rb"), args.platform, args.image, tags)
+
+    print("success!")
+
+
+if __name__ == "__main__":
+    main()

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/Masterminds/semver v1.5.0
 	github.com/dustin/go-humanize v1.0.0
 	github.com/efritz/pentimento v0.0.0-20190429011147-ade47d831101
+	github.com/gobwas/glob v0.2.3
 	github.com/google/go-cmp v0.5.2
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.0