Merge pull request #8041 from hashicorp/x-gomod

Switch to Go modules for dependency management

.circleci/config/commands/run-tests.yml

@@ -16,4 +16,9 @@ steps:
 
         mkdir -p /tmp/test-reports
         sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs
-        sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad
+
+        if [ ! -z $GOTEST_MOD ]; then
+          sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad-module
+        else
+          sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad
+        fi

.circleci/config/jobs/build-darwin-binaries.yml

@@ -10,7 +10,7 @@ steps:
       target_directory: ~/goinstall
   - run: source ${BASH_ENV} && make deps
   - run: brew install protobuf
-  - run: sudo -E PATH="$GOPATH/bin:${HOME}/goinstall/go/bin:$PATH" make generate-structs
+  - run: PATH="$GOPATH/bin:${HOME}/goinstall/go/bin:$PATH" make generate-structs
   - run: source ${BASH_ENV} && make pkg/darwin_amd64.zip
   - store_artifacts:
       path: pkg/darwin_amd64.zip

.circleci/config/jobs/test-machine.yml

@@ -3,6 +3,9 @@ parameters:
   test_packages:
     type: string
     default: ""
+  test_module:
+    type: string
+    default: ""
   exclude_packages:
     type: string
     default: ""
@@ -15,6 +18,7 @@ parameters:
 environment:
   GOTEST_PKGS_EXCLUDE: "<< parameters.exclude_packages >>"
   GOTEST_PKGS: "<< parameters.test_packages >>"
+  GOTEST_MOD: "<< parameters.test_module >>"
   GOTESTARCH: "<< parameters.goarch >>"
 steps:
   - checkout

.circleci/config/workflows/build-test.yml

@@ -32,7 +32,7 @@ jobs:
       # API Tests run in a VM rather than container due to the FS tests
       # requiring `mount` priviliges.
       name: "test-api"
-      test_packages: "./api/..."
+      test_module: "api"
       filters: *backend_branches_filter
   - test-container:
       name: "test-devices"

.gitignore

@@ -60,6 +60,7 @@ nomad_linux_amd64
 nomad_darwin_amd64
 TODO.md
 codecgen-*.generated.go
+**/structs.generated.go
 GNUMakefile.local
 
 .terraform
@@ -104,3 +105,6 @@ azure-hashistack.pem
 
 # generated keys for e2e tests
 e2e/terraform/keys/
+
+# darwin CGO files copied for build
+vendor/github.com/shirou/gopsutil/host/include/

GNUmakefile

@@ -44,6 +44,9 @@ endif
 # On MacOS, we only build for MacOS
 ifeq (Darwin,$(THIS_OS))
 ALL_TARGETS += darwin_amd64
+# Copy CGO files for darwin into place
+#   TODO(shoenig) remove pending github.com/shirou/gopsutil/pull/885
+$(shell cp -R lib/darwin/include vendor/github.com/shirou/gopsutil/host)
 endif
 
 # On FreeBSD, we only build for FreeBSD
@@ -166,22 +169,22 @@ bootstrap: deps lint-deps git-hooks # Install all dependencies
 
 .PHONY: deps
 deps:  ## Install build and development dependencies
+## Keep versions in sync with tools/go.mod for now (see https://github.com/golang/go/issues/30515)
 	@echo "==> Updating build dependencies..."
-	GO111MODULE=on go get -u github.com/kardianos/govendor
-	GO111MODULE=on go get -u github.com/hashicorp/go-bindata/go-bindata@master
-	GO111MODULE=on go get -u github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs@master
-	GO111MODULE=on go get -u github.com/a8m/tree/cmd/tree
-	GO111MODULE=on go get -u github.com/magiconair/vendorfmt/cmd/vendorfmt
-	GO111MODULE=on go get -u gotest.tools/gotestsum
-	GO111MODULE=on go get -u github.com/hashicorp/hcl/v2/cmd/hclfmt@v2.5.1
-	GO111MODULE=on go get -u github.com/golang/protobuf/protoc-gen-go@v1.3.4
-	GO111MODULE=on go get -u github.com/hashicorp/go-msgpack/codec/codecgen@v1.1.5
+	GO111MODULE=on cd tools && go get github.com/hashicorp/go-bindata/go-bindata@v3.0.7+incompatible
+	GO111MODULE=on cd tools && go get github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs@v1.0.0
+	GO111MODULE=on cd tools && go get github.com/a8m/tree/cmd/tree
+	GO111MODULE=on cd tools && go get gotest.tools/gotestsum@v0.4.2
+	GO111MODULE=on cd tools && go get github.com/hashicorp/hcl/v2/cmd/hclfmt@v2.5.1
+	GO111MODULE=on cd tools && go get github.com/golang/protobuf/protoc-gen-go@v1.3.4
+	GO111MODULE=on cd tools && go get github.com/hashicorp/go-msgpack/codec/codecgen@v1.1.5
 
 .PHONY: lint-deps
 lint-deps: ## Install linter dependencies
+## Keep versions in sync with tools/go.mod (see https://github.com/golang/go/issues/30515)
 	@echo "==> Updating linter dependencies..."
-	GO111MODULE=on go get -u github.com/golangci/golangci-lint/cmd/golangci-lint@v1.24.0
-	GO111MODULE=on go get -u github.com/client9/misspell/cmd/misspell
+	GO111MODULE=on cd tools && go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.24.0
+	GO111MODULE=on cd tools && go get github.com/client9/misspell/cmd/misspell@v0.3.4
 
 .PHONY: git-hooks
 git-dir = $(shell git rev-parse --git-dir)
@@ -207,10 +210,7 @@ check: ## Lint the source code
 	@if (git status -s | grep -q -e '\.hcl$$' -e '\.nomad$$'); then echo the following HCL files are out of sync; git status -s | grep -e '\.hcl$$' -e '\.nomad$$'; exit 1; fi
 
 	@echo "==> Check API package is isolated from rest"
-	@if go list --test -f '{{ join .Deps "\n" }}' ./api | grep github.com/hashicorp/nomad/ | grep -v -e /vendor/ -e /nomad/api/ -e nomad/api.test; then echo "  /api package depends the ^^ above internal nomad packages.  Remove such dependency"; exit 1; fi
-
-	@echo "==> Check non-vendored packages"
-	@if go list --test -tags "$(GO_TAGS)" -f '{{join .Deps "\n"}}' . | grep -v github.com/hashicorp/nomad.test | xargs go list -tags "$(GO_TAGS)" -f '{{if not .Standard}}{{.ImportPath}}{{end}}' | grep -v -e github.com/hashicorp/nomad; then echo "  found referenced packages ^^ that are not vendored"; exit 1; fi
+	@cd ./api && if go list --test -f '{{ join .Deps "\n" }}' . | grep github.com/hashicorp/nomad/ | grep -v -e /vendor/ -e /nomad/api/ -e nomad/api.test; then echo "  /api package depends the ^^ above internal nomad packages.  Remove such dependency"; exit 1; fi
 
 .PHONY: checkscripts
 checkscripts: ## Lint shell scripts
@@ -239,11 +239,6 @@ generate-examples: command/job_init.bindata_assetfs.go
 command/job_init.bindata_assetfs.go: command/assets/*
 	go-bindata-assetfs -pkg command -o command/job_init.bindata_assetfs.go ./command/assets/...
 
-.PHONY: vendorfmt
-vendorfmt:
-	@echo "--> Formatting vendor/vendor.json"
-	vendorfmt
-
 .PHONY: changelogfmt
 changelogfmt:
 	@echo "--> Making [GH-xxxx] references clickable..."
@@ -257,12 +252,25 @@ hclfmt:
 	@find . -path ./terraform -prune -o -name 'upstart.nomad' -prune -o \( -name '*.nomad' -o -name '*.hcl' \) -exec \
 sh -c 'hclfmt -w {} || echo in path {}' ';'
 
+.PHONY: tidy
+tidy:
+	@echo "--> Tidy up submodules"
+	@cd tools && go mod tidy
+	@cd api && go mod tidy
+	@echo "--> Tidy nomad module"
+	@go mod tidy
+
+.PHONY: sync
+sync: tidy
+	@echo "--> Sync vendor directory"
+	@go mod vendor
+
 .PHONY: dev
 dev: GOOS=$(shell go env GOOS)
 dev: GOARCH=$(shell go env GOARCH)
 dev: GOPATH=$(shell go env GOPATH)
 dev: DEV_TARGET=pkg/$(GOOS)_$(GOARCH)/nomad
-dev: vendorfmt changelogfmt hclfmt ## Build for the current development platform
+dev: tidy changelogfmt hclfmt ## Build for the current development platform
 	@echo "==> Removing old development build..."
 	@rm -f $(PROJECT_ROOT)/$(DEV_TARGET)
 	@rm -f $(PROJECT_ROOT)/bin/nomad
@@ -313,6 +321,19 @@ test-nomad: dev ## Run Nomad test suites
 		bash -C "$(PROJECT_ROOT)/scripts/test_check.sh" ; \
 	fi
 
+.PHONY: test-nomad-module
+test-nomad-module: dev ## Run Nomad test suites on a sub-module
+	@echo "==> Running Nomad test suites on sub-module:"
+	@cd $(GOTEST_MOD) && $(if $(ENABLE_RACE),GORACE="strip_path_prefix=$(GOPATH)/src") $(GO_TEST_CMD) \
+		$(if $(ENABLE_RACE),-race) $(if $(VERBOSE),-v) \
+		-cover \
+		-timeout=15m \
+		-tags "$(GO_TAGS)" \
+		./... $(if $(VERBOSE), >test.log ; echo $$? > exit-code)
+	@if [ $(VERBOSE) ] ; then \
+		bash -C "$(PROJECT_ROOT)/scripts/test_check.sh" ; \
+	fi
+
 .PHONY: e2e-test
 e2e-test: dev ## Run the Nomad e2e test suite
 	@echo "==> Running Nomad E2E test suites:"
@@ -383,7 +404,7 @@ help: ## Display this usage information
 .PHONY: ui-screenshots
 ui-screenshots:
 	@echo "==> Collecting UI screenshots..."
-	# Build the screenshots image if it doesn't exist yet
+        # Build the screenshots image if it doesn't exist yet
 	@if [[ "$$(docker images -q nomad-ui-screenshots 2> /dev/null)" == "" ]]; then \
 		docker build --tag="nomad-ui-screenshots" ./scripts/screenshots; \
 	fi

README.md

@@ -229,3 +229,8 @@ $ make release
 $ ls ./pkg
 ...
 ```
+
+API Compatibility
+--------------------
+
+Only the `api/` and `plugins/` packages are intended to be imported by other projects. The root Nomad module does not follow semver and is not intended to be imported directly by other projects.