Fix noupstream in makefile (#1362)

Removes the NoUpstream option hack from #1278 This replaces the last uses of the option within the makefile. This adds one, more targeted option for the `GenName` which can be used to change the `tfgen` binary name to `gen` for "generic" providers. We can replace the conditional around the upstream target with a check to see if upstream.sh exists before executing it. The `LDFLAGS_UPSTREAM_VERSION` conditional had no effect and so can just be removed. > Somewhat confusingly, `ProviderVersion` is only set [for 3 providers](https://github.com/search?q=org%3Apulumi+path%3A.ci-mgmt.yaml+%22providerVersion%3A%22&type=code) (aws, gcp & azure) in order to set the version within the upstream codebase. Therefore, generic providers would never set this option and we don't need to add the extra guard.
pulumi · Feb 6, 2025 · 8eda7e3 · 8eda7e3
1 parent 030556b
commit 8eda7e3
Show file tree

Hide file tree

Showing 16 changed files with 866 additions and 452 deletions.
diff --git a/provider-ci/internal/pkg/config.go b/provider-ci/internal/pkg/config.go
@@ -268,9 +268,9 @@ type Config struct {
 	// https://github.com/search?q=org%3Apulumi+path%3A.ci-mgmt.yaml+%22makeTemplate%3A%22&type=code
 	MakeTemplate string `yaml:"makeTemplate"`
 
-	// NoUpstream is a temporary hack to disable bridge-specific workflow steps
-	// as part of the work to consolidate these with native providers.
-	NoUpstream bool
+	// Customizes the name of the "gen" program.
+	// Defaults to "tfgen" for bridged providers and "gen" for generic providers.
+	GenName string `yaml:"genName"`
 
 	// Customizes the Make function build_provider_cmd.
 	//

diff --git a/provider-ci/internal/pkg/generate.go b/provider-ci/internal/pkg/generate.go
@@ -45,9 +45,15 @@ func GeneratePackage(opts GenerateOpts) error {
 	if err != nil {
 		return fmt.Errorf("error getting template directories: %w", err)
 	}
-	if opts.Config.Template == "generic" {
-		opts.Config.NoUpstream = true
-		opts.Config.CheckUpstreamUpgrade = false
+
+	// GenName defaults to "tfgen" for bridged providers and "gen" for others
+	if opts.Config.GenName == "" {
+		switch opts.TemplateName {
+		case "generic":
+			opts.Config.GenName = "gen"
+		default:
+			opts.Config.GenName = "tfgen"
+		}
 	}
 
 	if opts.Config.ToolVersions.PulumiCTL == "" {
@@ -107,15 +113,15 @@ func getDeletedFiles(templateName string) []string {
 			".github/actions/download-codegen/action.yml",
 			".github/workflows/check-upstream-upgrade.yml",
 			".github/workflows/resync-build.yml",
-			"scripts/upstream.sh",
+			"upstream.sh",
 			".goreleaser.yml",
 			".goreleaser.prerelease.yml",
 		}
 	case "external-bridged-provider":
 		return []string{
 			".github/actions/download-bin/action.yml",
 			".github/actions/download-codegen/action.yml",
-			"scripts/upstream.sh",
+			"upstream.sh",
 			".goreleaser.yml",
 			".goreleaser.prerelease.yml",
 		}

diff --git a/provider-ci/internal/pkg/templates/base/Makefile b/provider-ci/internal/pkg/templates/base/Makefile
@@ -9,11 +9,7 @@ PROVIDER_PATH := provider/v#{{ .Config.MajorVersion }}#
 PROVIDER_PATH := provider
 #{{- end }}#
 VERSION_PATH := $(PROVIDER_PATH)/pkg/version.Version
-#{{- if .Config.NoUpstream }}#
-CODEGEN := pulumi-gen-$(PACK)
-#{{- else }}#
-CODEGEN := pulumi-tfgen-$(PACK)
-#{{- end }}#
+CODEGEN := pulumi-#{{ .Config.GenName }}#-$(PACK)
 PROVIDER := pulumi-resource-$(PACK)
 JAVA_GEN := pulumi-java-gen
 TESTPARALLELISM := 10
@@ -43,7 +39,7 @@ endif
 # Strips debug information from the provider binary to reduce its size and speed up builds
 LDFLAGS_STRIP_SYMBOLS=-s -w
 LDFLAGS_PROJ_VERSION=-X $(PROJECT)/$(VERSION_PATH)=$(PROVIDER_VERSION)#{{if .Config.ProviderVersion}}# -X #{{ .Config.ProviderVersion }}#=$(PROVIDER_VERSION)#{{end}}#
-#{{- if and (not (eq .Config.ProviderVersion "")) (not .Config.NoUpstream) }}#
+#{{- if not (eq .Config.ProviderVersion "") }}#
 LDFLAGS_UPSTREAM_VERSION=-X #{{ .Config.ProviderVersion }}#=v$(PROVIDER_VERSION)
 #{{- else }}#
 LDFLAGS_UPSTREAM_VERSION=
@@ -312,26 +308,19 @@ tfgen_build_only: bin/$(CODEGEN)
 bin/$(CODEGEN): provider/*.go provider/go.* .make/upstream
 	(cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(CODEGEN) -ldflags "$(LDFLAGS_PROJ_VERSION) $(LDFLAGS_EXTRAS)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(CODEGEN))
 .PHONY: tfgen schema tfgen_no_deps tfgen_build_only
-#{{ if .Config.NoUpstream }}#
-upstream: .make/upstream
-.make/upstream:
-	@touch $@
-#{{- else }}#
+
 # Apply patches to the upstream submodule, if it exists
 upstream: .make/upstream
-# Re-run if the upstream commit or the patches change
-.make/upstream: $(wildcard patches/*) $(shell ./upstream.sh file_target)
-ifneq ("$(wildcard upstream)","")
-	./upstream.sh init
-endif
+# Re-run if the upstream commit or the patches change.
+.make/upstream: $(wildcard patches/*) $(shell ./scripts/upstream.sh file_target)
+	./scripts/upstream.sh init
 #{{- if .Config.XrunUpstreamTools }}#
 	# Ensure tool is installed
 	cd upstream-tools && yarn install --frozen-lockfile
 	# Apply all automated changes
 	cd upstream-tools && yarn --silent run apply
 #{{- end }}#
 	@touch $@
-#{{- end }}#
 .PHONY: upstream
 
 bin/pulumi-java-gen: PULUMI_JAVA_VERSION := $(shell cat .pulumi-java-gen.version)

diff --git a/...nternal/pkg/templates/bridged/upstream.sh → ...al/pkg/templates/base/scripts/upstream.sh b/...nternal/pkg/templates/bridged/upstream.sh → ...al/pkg/templates/base/scripts/upstream.sh
@@ -182,7 +182,10 @@ init() {
     esac
   done
 
-  assert_upstream_exists
+  if [[ ! -d upstream ]]; then
+    echo "No 'upstream' directory detected. Skipping init."
+    exit 0
+  fi
 
   if [[ "${force}" != "true" ]]; then
     assert_not_checked_out

diff --git a/provider-ci/internal/pkg/templates/defaults.config.yaml b/provider-ci/internal/pkg/templates/defaults.config.yaml
@@ -36,6 +36,10 @@ major-version: 2
 # Only set for 5 providers: https://github.com/search?q=org%3Apulumi+path%3A.ci-mgmt.yaml+%22upstream-provider-repo%22&type=code
 #upstream-provider-repo: terraform-provider-xyz
 
+# Customizes the name of the "gen" program.
+# Defaults to "tfgen" for bridged providers and "gen" for generic providers.
+#genName: tfgen
+
 # lint includes an extra lint job in workflows if enabled (default). Can be set explicit to false.
 # This is false in around 8 provider repos: https://github.com/search?q=org%3Apulumi+path%3A.ci-mgmt.yaml+%22lint%3A+false%22&type=code
 lint: true

diff --git a/provider-ci/test-providers/acme/Makefile b/provider-ci/test-providers/acme/Makefile
@@ -259,11 +259,9 @@ bin/$(CODEGEN): provider/*.go provider/go.* .make/upstream
 
 # Apply patches to the upstream submodule, if it exists
 upstream: .make/upstream
-# Re-run if the upstream commit or the patches change
-.make/upstream: $(wildcard patches/*) $(shell ./upstream.sh file_target)
-ifneq ("$(wildcard upstream)","")
-	./upstream.sh init
-endif
+# Re-run if the upstream commit or the patches change.
+.make/upstream: $(wildcard patches/*) $(shell ./scripts/upstream.sh file_target)
+	./scripts/upstream.sh init
 	@touch $@
 .PHONY: upstream
 

diff --git a/provider-ci/test-providers/acme/upstream.sh → ...i/test-providers/acme/scripts/upstream.sh b/provider-ci/test-providers/acme/upstream.sh → ...i/test-providers/acme/scripts/upstream.sh
@@ -182,7 +182,10 @@ init() {
     esac
   done
 
-  assert_upstream_exists
+  if [[ ! -d upstream ]]; then
+    echo "No 'upstream' directory detected. Skipping init."
+    exit 0
+  fi
 
   if [[ "${force}" != "true" ]]; then
     assert_not_checked_out

diff --git a/provider-ci/test-providers/aws/Makefile b/provider-ci/test-providers/aws/Makefile
@@ -269,11 +269,9 @@ bin/$(CODEGEN): provider/*.go provider/go.* .make/upstream
 
 # Apply patches to the upstream submodule, if it exists
 upstream: .make/upstream
-# Re-run if the upstream commit or the patches change
-.make/upstream: $(wildcard patches/*) $(shell ./upstream.sh file_target)
-ifneq ("$(wildcard upstream)","")
-	./upstream.sh init
-endif
+# Re-run if the upstream commit or the patches change.
+.make/upstream: $(wildcard patches/*) $(shell ./scripts/upstream.sh file_target)
+	./scripts/upstream.sh init
 	# Ensure tool is installed
 	cd upstream-tools && yarn install --frozen-lockfile
 	# Apply all automated changes

diff --git a/provider-ci/test-providers/aws/upstream.sh → ...ci/test-providers/aws/scripts/upstream.sh b/provider-ci/test-providers/aws/upstream.sh → ...ci/test-providers/aws/scripts/upstream.sh
@@ -182,7 +182,10 @@ init() {
     esac
   done
 
-  assert_upstream_exists
+  if [[ ! -d upstream ]]; then
+    echo "No 'upstream' directory detected. Skipping init."
+    exit 0
+  fi
 
   if [[ "${force}" != "true" ]]; then
     assert_not_checked_out

diff --git a/provider-ci/test-providers/cloudflare/Makefile b/provider-ci/test-providers/cloudflare/Makefile
@@ -269,11 +269,9 @@ bin/$(CODEGEN): provider/*.go provider/go.* .make/upstream
 
 # Apply patches to the upstream submodule, if it exists
 upstream: .make/upstream
-# Re-run if the upstream commit or the patches change
-.make/upstream: $(wildcard patches/*) $(shell ./upstream.sh file_target)
-ifneq ("$(wildcard upstream)","")
-	./upstream.sh init
-endif
+# Re-run if the upstream commit or the patches change.
+.make/upstream: $(wildcard patches/*) $(shell ./scripts/upstream.sh file_target)
+	./scripts/upstream.sh init
 	@touch $@
 .PHONY: upstream
 

diff --git a/...-ci/test-providers/cloudflare/upstream.sh → ...-providers/cloudflare/scripts/upstream.sh b/...-ci/test-providers/cloudflare/upstream.sh → ...-providers/cloudflare/scripts/upstream.sh
@@ -182,7 +182,10 @@ init() {
     esac
   done
 
-  assert_upstream_exists
+  if [[ ! -d upstream ]]; then
+    echo "No 'upstream' directory detected. Skipping init."
+    exit 0
+  fi
 
   if [[ "${force}" != "true" ]]; then
     assert_not_checked_out

diff --git a/provider-ci/test-providers/docker/Makefile b/provider-ci/test-providers/docker/Makefile
@@ -272,11 +272,9 @@ bin/$(CODEGEN): provider/*.go provider/go.* .make/upstream
 
 # Apply patches to the upstream submodule, if it exists
 upstream: .make/upstream
-# Re-run if the upstream commit or the patches change
-.make/upstream: $(wildcard patches/*) $(shell ./upstream.sh file_target)
-ifneq ("$(wildcard upstream)","")
-	./upstream.sh init
-endif
+# Re-run if the upstream commit or the patches change.
+.make/upstream: $(wildcard patches/*) $(shell ./scripts/upstream.sh file_target)
+	./scripts/upstream.sh init
 	@touch $@
 .PHONY: upstream