WIP: Integration test build improvements

.circleci/config.yml

@@ -1,32 +1,28 @@
 version: 2
+
+# https://circleci.com/blog/circleci-hacks-reuse-yaml-in-your-circleci-config-with-yaml/
 defaults: &defaults
-  machine:
-    docker_layer_caching: true
-  working_directory: ~/weaveworks/service
+  docker:
+    - image: weaveworks/service-build:master-8245ec7d
+  working_directory: /go/src/github.com/weaveworks/service
+
 jobs:
   lint:
     <<: *defaults
     steps:
       - checkout
-      - run: make RM= lint
+      - run: make BUILD_IN_CONTAINER=false lint
 
   build:
     <<: *defaults
     steps:
       - checkout
-      - run: sudo docker info >/dev/null 2>&1 || sudo service docker start;
-      # FIXME: we probably don't need to copy the code and change the working dir
-      - run: echo -e "export GOPATH=$HOME\nexport SRCDIR=$HOME/src/github.com/weaveworks/service\nexport PATH=/usr/local/go/bin:$HOME/bin:$PATH" >> $BASH_ENV
-      - run: mkdir -p $(dirname $SRCDIR) && cp -r $(pwd)/ $SRCDIR
-      - run: |
-          cd $SRCDIR/build
-          ../tools/rebuild-image quay.io/weaveworks/build . build.sh Dockerfile
-          touch .uptodate
-      - run: cd $SRCDIR; make RM=
+      - setup_remote_docker
+      - run: make BUILD_IN_CONTAINER=false SUDO=
       - run: |
           set -e
           set -o pipefail
-          cd $SRCDIR; if [ $(./tools/image-tag | grep -e '-WIP$') ]; then
+          if [ $(./tools/image-tag | grep -e '-WIP$') ]; then
             echo "WIP build; exiting with error to mark as a failure"
             git --no-pager diff
             exit 1
@@ -46,15 +42,22 @@ jobs:
               docker tag ${tagged_image} ${repo_tmp_image}
               docker push $repo_tmp_image
             done
+      - persist_to_workspace:
+          root: .
+          paths:
+          - users/client/mock_client.go
+          - gcp-service/grpc/gcp-service.pb.go
+          - users/users.pb.go
+          - common/billing/grpc/billing.pb.go
+          - kubectl-service/grpc/kubectl-service.pb.go
+
   test:
     <<: *defaults
     steps:
       - checkout
-      - run: echo -e "export GOPATH=$HOME\nexport PATH=/usr/local/go/bin:$HOME/bin:$PATH" >> $BASH_ENV
-      # Link the working directory so that it appears in GOPATH. This is required for cover to map coverage statistics to sources.
-      - run: mkdir -p ${GOPATH}/src/github.com/weaveworks && ln -s $(pwd) ${GOPATH}/src/github.com/weaveworks/service
+      - setup_remote_docker
       - run: go get github.com/mattn/goveralls
-      - run: COVERDIR=./coverage make RM= test
+      - run: COVERDIR=./coverage make BUILD_IN_CONTAINER=false test
       - run: ./tools/cover/gather_coverage.sh ./coverage $CIRCLE_WORKING_DIRECTORY/coverage
       - run: goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$CIRCLE_WORKING_DIRECTORY/profile.cov -service=circleci || true
       - run: mkdir /tmp/coverage && cp coverage.* /tmp/coverage
@@ -65,6 +68,7 @@ jobs:
     <<: *defaults
     steps:
       - checkout
+      - setup_remote_docker
       - &pull-tmp-images
         run:
           name: Download docker images build earlier in workflow
@@ -84,28 +88,29 @@ jobs:
               docker tag ${repo_tmp_image} ${image}:latest
               docker image rm ${repo_tmp_image}
             done
-      # FIXME: we probably don't need to copy the code and change the working dir
-      - run: echo -e "export GOPATH=$HOME\nexport SRCDIR=$HOME/src/github.com/weaveworks/service\nexport PATH=/usr/local/go/bin:$HOME/bin:$PATH" >> $BASH_ENV
-      - run: mkdir -p $(dirname $SRCDIR) && cp -r $(pwd)/ $SRCDIR
+      - attach_workspace:
+          at: .
       - run: go get github.com/nats-io/gnatsd
       - run:
           command: gnatsd
           background: true
       - run: |
-          cd $SRCDIR
-          make RM= notebooks-integration-test
-          make RM= users-integration-test
-          make RM= billing-integration-test
-          make RM= pubsub-integration-test
-          make RM= flux-integration-test
-          make RM= kubectl-service-integration-test
-          make RM= gcp-service-integration-test
-          make RM= notification-integration-test
+          touch users/client/mock_client.go gcp-service/grpc/gcp-service.pb.go users/users.pb.go common/billing/grpc/billing.pb.go kubectl-service/grpc/kubectl-service.pb.go
+          make touch-uptodate
+          make BUILD_IN_CONTAINER=false notification-integration-test
+          make BUILD_IN_CONTAINER=false notebooks-integration-test
+          make BUILD_IN_CONTAINER=false users-integration-test
+          make BUILD_IN_CONTAINER=false billing-integration-test
+          make BUILD_IN_CONTAINER=false pubsub-integration-test
+          make BUILD_IN_CONTAINER=false flux-integration-test
+          make BUILD_IN_CONTAINER=false kubectl-service-integration-test
+          make BUILD_IN_CONTAINER=false gcp-service-integration-test
 
   upload:
     <<: *defaults
     steps:
       - checkout
+      - setup_remote_docker
       - <<: *pull-tmp-images
       - run:
           name: Publish docker images to final repository
@@ -119,6 +124,7 @@ jobs:
     <<: *defaults
     steps:
       - checkout
+      - setup_remote_docker
       - <<: *pull-tmp-images
       - run:
           name: Dry-run of publishing docker images

Makefile

@@ -29,6 +29,9 @@ images:
 	$(info $(IMAGE_NAMES))
 	@echo > /dev/null
 
+touch-uptodate:
+	touch $(UPTODATE_FILES)
+
 all: $(UPTODATE_FILES)
 
 # Generating proto code is automated.
@@ -176,105 +179,103 @@ $(EXES) test: build/$(UPTODATE) $(PROTO_GOS)
 		-e TESTDIRS=${TESTDIRS} \
 		$(IMAGE_PREFIX)/build $@
 
-billing-integration-test: build/$(UPTODATE)
-	@mkdir -p $(shell pwd)/.pkg
-	DB_CONTAINER="$$(docker run -d -e 'POSTGRES_DB=billing_test' postgres:9.5)"; \
-	$(SUDO) docker run $(RM) -ti \
-		-v $(shell pwd)/.pkg:/go/pkg \
-		-v $(shell pwd):/go/src/github.com/weaveworks/service \
-		-v $(shell pwd)/billing-api/db/migrations:/migrations \
-		--workdir /go/src/github.com/weaveworks/service \
-		--link "$$DB_CONTAINER":billing-db.weave.local \
-		$(IMAGE_PREFIX)/build $@; \
-	status=$$?; \
-	test -n "$(CIRCLECI)" || docker rm -f "$$DB_CONTAINER"; \
-	exit $$status
-
-flux-integration-test: build/$(UPTODATE)
-	@mkdir -p $(shell pwd)/.pkg
-	NATS_CONTAINER="$$(docker run -d nats)"; \
-	POSTGRES_CONTAINER="$$(docker run -d postgres)"; \
-	$(SUDO) docker run $(RM) -ti \
-		-v $(shell pwd)/.pkg:/go/pkg \
-		-v $(shell pwd):/go/src/github.com/weaveworks/service \
-		-v $(shell pwd)/billing-api/db/migrations:/migrations \
-		--workdir /go/src/github.com/weaveworks/service \
-		--link "$$NATS_CONTAINER":nats \
-		--link "$$POSTGRES_CONTAINER":postgres \
-		$(IMAGE_PREFIX)/build $@; \
-	status=$$?; \
-	test -n "$(CIRCLECI)" || docker rm -f "$$NATS_CONTAINER" "$$POSTGRES_CONTAINER"; \
-	exit $$status
-
 else
 
-$(EXES): build/$(UPTODATE) $(PROTO_GOS)
+$(EXES): $(PROTO_GOS)
 	go build $(GO_FLAGS) -o $@ ./$(@D)
 	$(NETGO_CHECK)
 
-%.pb.go: build/$(UPTODATE)
+%.pb.go:
 	protoc -I ./vendor:./$(@D) --gogoslick_out=plugins=grpc:./$(@D) ./$(patsubst %.pb.go,%.proto,$@)
 
-lint: build/$(UPTODATE)
+lint:
 	./tools/lint .
 
-test: build/$(UPTODATE) $(PROTO_GOS) $(MOCK_GOS)
+test: $(PROTO_GOS) $(MOCK_GOS)
 	TESTDIRS=${TESTDIRS} ./tools/test -netgo -no-race
 
-$(MOCK_USERS): build/$(UPTODATE)
+$(MOCK_USERS):
 	mockgen -destination=$@ github.com/weaveworks/service/users UsersClient \
 		&& sed -i'' s,github.com/weaveworks/service/vendor/,, $@
 
-$(MOCK_BILLING_DB): build/$(UPTODATE) $(BILLING_DB)/db.go
+$(MOCK_BILLING_DB): $(BILLING_DB)/db.go
 	mockgen -destination=$@ github.com/weaveworks/service/$(BILLING_DB) DB
 
-$(MOCK_BILLING_GRPC): build/$(UPTODATE)
+$(MOCK_BILLING_GRPC):
 	mockgen -source=$(BILLING_GRPC) -destination=$@ -package=grpc
 
-$(MOCK_COMMON_GCP_PARTNER_CLIENT): build/$(UPTODATE)
+$(MOCK_COMMON_GCP_PARTNER_CLIENT):
 	mockgen -destination=$@ github.com/weaveworks/service/common/gcp/partner API \
 		&& sed -i'' s,github.com/weaveworks/service/vendor/,, $@
 
-$(MOCK_COMMON_GCP_PARTNER_ACCESS): build/$(UPTODATE)
+$(MOCK_COMMON_GCP_PARTNER_ACCESS):
 	mockgen -destination=$@ github.com/weaveworks/service/common/gcp/partner Accessor \
 		&& sed -i'' s,github.com/weaveworks/service/vendor/,, $@
 
-billing-integration-test: build/$(UPTODATE) $(MOCK_GOS)
-	/bin/bash -c "go test -tags 'netgo integration' -timeout 30s $(BILLING_TEST_DIRS)"
+endif
+
+
+# Test and misc stuff
+INTEGRATION_TEST_IMAGE=golang:1.9.2-stretch
+
+billing-integration-test: $(MOCK_GOS)
+	@mkdir -p $(shell pwd)/.pkg
+	DB_CONTAINER="$$(docker run -d -e 'POSTGRES_DB=billing_test' postgres:9.5)"; \
+	$(SUDO) docker run $(RM) -ti \
+		-v $(shell pwd)/.pkg:/go/pkg \
+		-v $(shell pwd):/go/src/github.com/weaveworks/service \
+		-v $(shell pwd)/billing-api/db/migrations:/migrations \
+		--workdir /go/src/github.com/weaveworks/service \
+		--link "$$DB_CONTAINER":billing-db.weave.local \
+		$(INTEGRATION_TEST_IMAGE) \
+		/bin/bash -c "go test -tags 'netgo integration' -timeout 30s $(BILLING_TEST_DIRS)"; \
+	status=$$?; \
+	test -n "$(CIRCLECI)" || docker rm -f "$$DB_CONTAINER"; \
+	exit $$status
 
 flux-integration-test:
+	@mkdir -p $(shell pwd)/.pkg
 # These packages must currently be tested in series because
 # otherwise they will all race to run migrations.
-	/bin/bash -c "go test -tags integration -timeout 30s ./flux-api"
-	/bin/bash -c "go test -tags integration -timeout 30s ./flux-api/bus/nats"
-	/bin/bash -c "go test -tags integration -timeout 30s ./flux-api/history/sql"
-	/bin/bash -c "go test -tags integration -timeout 30s ./flux-api/instance/sql"
-
-endif
-
+	NATS_CONTAINER="$$(docker run -d nats)"; \
+	POSTGRES_CONTAINER="$$(docker run -d postgres)"; \
+	$(SUDO) docker run $(RM) -ti \
+		-v $(shell pwd)/.pkg:/go/pkg \
+		-v $(shell pwd):/go/src/github.com/weaveworks/service \
+		-v $(shell pwd)/billing-api/db/migrations:/migrations \
+		--workdir /go/src/github.com/weaveworks/service \
+		--link "$$NATS_CONTAINER":nats \
+		--link "$$POSTGRES_CONTAINER":postgres \
+		$(INTEGRATION_TEST_IMAGE) \
+		/bin/bash -c "go test -tags integration -timeout 30s ./flux-api && \
+		              go test -tags integration -timeout 30s ./flux-api/bus/nats &&\
+		              go test -tags integration -timeout 30s ./flux-api/history/sql &&\
+		              go test -tags integration -timeout 30s ./flux-api/instance/sql"; \
+	status=$$?; \
+	test -n "$(CIRCLECI)" || docker rm -f "$$NATS_CONTAINER" "$$POSTGRES_CONTAINER"; \
+	exit $$status
 
-# Test and misc stuff
-notebooks-integration-test: $(NOTEBOOKS_UPTODATE)
+notebooks-integration-test:
 	DB_CONTAINER="$$(docker run -d -e 'POSTGRES_DB=notebooks_test' postgres:9.5)"; \
 	docker run $(RM) \
 		-v $(shell pwd):/go/src/github.com/weaveworks/service \
 		-v $(shell pwd)/notebooks/db/migrations:/migrations \
 		--workdir /go/src/github.com/weaveworks/service/notebooks \
 		--link "$$DB_CONTAINER":configs-db.weave.local \
-		golang:1.9.2-stretch \
+		$(INTEGRATION_TEST_IMAGE) \
 		/bin/bash -c "go test -tags integration -timeout 30s ./..."; \
 	status=$$?; \
 	test -n "$(CIRCLECI)" || docker rm -f "$$DB_CONTAINER"; \
 	exit $$status
 
-users-integration-test: $(USERS_UPTODATE) $(PROTO_GOS) $(MOCK_GOS)
+users-integration-test: $(PROTO_GOS) $(MOCK_GOS)
 	DB_CONTAINER="$$(docker run -d -e 'POSTGRES_DB=users_test' postgres:9.5)"; \
 	docker run $(RM) \
 		-v $(shell pwd):/go/src/github.com/weaveworks/service \
 		-v $(shell pwd)/users/db/migrations:/migrations \
 		--workdir /go/src/github.com/weaveworks/service/users \
 		--link "$$DB_CONTAINER":users-db.weave.local \
-		golang:1.9.2-stretch \
+		$(INTEGRATION_TEST_IMAGE) \
 		/bin/bash -c "go test -tags integration -timeout 30s ./..."; \
 	status=$$?; \
 	test -n "$(CIRCLECI)" || docker rm -f "$$DB_CONTAINER"; \
@@ -286,31 +287,31 @@ pubsub-integration-test:
 		-v $(shell pwd):/go/src/github.com/weaveworks/service \
 		--net=host -p 127.0.0.1:1337:1337 \
 		--workdir /go/src/github.com/weaveworks/service/common/gcp/pubsub \
-		golang:1.9.2-stretch \
+		$(INTEGRATION_TEST_IMAGE) \
 		/bin/bash -c "RUN_MANUAL_TEST=1 go test -tags integration -timeout 30s ./..."; \
 	status=$$?; \
 	test -n "$(CIRCLECI)" || docker rm -f "$$PUBSUB_EMU_CONTAINER"; \
 	exit $$status
 
-kubectl-service-integration-test: kubectl-service/$(UPTODATE) kubectl-service/grpc/kubectl-service.pb.go
+kubectl-service-integration-test: kubectl-service/grpc/kubectl-service.pb.go
 	SVC_CONTAINER="$$(docker run -d -p 4887:4772 -p 8887:80 $(IMAGE_PREFIX)/kubectl-service -dry-run=true)"; \
 	docker run $(RM) \
 		-v $(shell pwd):/go/src/github.com/weaveworks/service \
 		--workdir /go/src/github.com/weaveworks/service/kubectl-service \
 		--link "$$SVC_CONTAINER":kubectl-service.weave.local \
-		golang:1.9.2-stretch \
+		$(INTEGRATION_TEST_IMAGE) \
 		/bin/bash -c "go test -tags integration -timeout 30s ./..."; \
 	status=$$?; \
 	test -n "$(CIRCLECI)" || docker rm -f "$$SVC_CONTAINER"; \
 	exit $$status
 
-gcp-service-integration-test: gcp-service/$(UPTODATE) gcp-service/grpc/gcp-service.pb.go
+gcp-service-integration-test: gcp-service/grpc/gcp-service.pb.go
 	SVC_CONTAINER="$$(docker run -d -p 4888:4772 -p 8888:80 $(IMAGE_PREFIX)/gcp-service -dry-run=true)"; \
 	docker run $(RM) \
 		-v $(shell pwd):/go/src/github.com/weaveworks/service \
 		--workdir /go/src/github.com/weaveworks/service/gcp-service \
 		--link "$$SVC_CONTAINER":gcp-service.weave.local \
-		golang:1.9.2-stretch \
+		$(INTEGRATION_TEST_IMAGE) \
 		/bin/bash -c "go test -tags integration -timeout 30s ./..."; \
 	status=$$?; \
 	test -n "$(CIRCLECI)" || docker rm -f "$$SVC_CONTAINER"; \