Merge pull request #1114 from justaugustus/krel-gcbmgr

[krel] gcbmgr: Dynamically retrieve build version from CI markers

cmd/krel/cmd/gcbmgr.go

@@ -109,7 +109,7 @@ func init() {
 	gcbmgrCmd.PersistentFlags().StringVar(
 		&gcbmgrOpts.buildVersion,
 		"build-version",
-		"FAKE+BUILD+POINT",
+		"",
 		"Build version",
 	)
 
@@ -260,18 +260,14 @@ func runGcbmgr() error {
 func setGCBSubstitutions(o *gcbmgrOptions) (map[string]string, error) {
 	gcbSubs := map[string]string{}
 
-	releaseToolRepo := os.Getenv("RELEASE_TOOL_REPO")
-	if releaseToolRepo == "" {
-		releaseToolRepo = release.DefaultReleaseToolRepo
-	}
+	toolOrg := release.GetToolOrg()
+	gcbSubs["TOOL_ORG"] = toolOrg
 
-	releaseToolBranch := os.Getenv("RELEASE_TOOL_BRANCH")
-	if releaseToolBranch == "" {
-		releaseToolBranch = release.DefaultReleaseToolBranch
-	}
+	toolRepo := release.GetToolRepo()
+	gcbSubs["TOOL_REPO"] = toolRepo
 
-	gcbSubs["RELEASE_TOOL_REPO"] = releaseToolRepo
-	gcbSubs["RELEASE_TOOL_BRANCH"] = releaseToolBranch
+	toolBranch := release.GetToolBranch()
+	gcbSubs["TOOL_BRANCH"] = toolBranch
 
 	gcpUser := o.gcpUser
 	if gcpUser == "" {
@@ -312,25 +308,33 @@ func setGCBSubstitutions(o *gcbmgrOptions) (map[string]string, error) {
 		gcbSubs["RC"] = ""
 	}
 
+	branch := o.branch
+	if branch == "" {
+		return gcbSubs, errors.New("Release branch must be set to continue")
+	}
+
+	gcbSubs["RELEASE_BRANCH"] = branch
+
 	// TODO: Remove once we remove support for --built-at-head.
 	gcbSubs["BUILD_AT_HEAD"] = ""
 
-	buildpoint := o.buildVersion
+	buildVersion := o.buildVersion
+	if buildVersion == "" {
+		var versionErr error
+		buildVersion, versionErr = release.GetCIKubeVersion(o.branch, false)
+		if versionErr != nil {
+			return gcbSubs, versionErr
+		}
+	}
+
+	buildpoint := buildVersion
 	buildpoint = strings.ReplaceAll(buildpoint, "+", "-")
 	gcbSubs["BUILD_POINT"] = buildpoint
 
 	// TODO: Add conditionals for find_green_build
-	buildVersion := o.buildVersion
 	buildVersion = fmt.Sprintf("--buildversion=%s", buildVersion)
 	gcbSubs["BUILDVERSION"] = buildVersion
 
-	branch := o.branch
-	if branch == "" {
-		return gcbSubs, errors.New("Release branch must be set to continue")
-	}
-
-	gcbSubs["RELEASE_BRANCH"] = branch
-
 	kubecrossBranches := []string{
 		branch,
 		"master",

cmd/krel/cmd/gcbmgr_test.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"fmt"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -96,6 +97,9 @@ func TestSetGCBSubstitutionsSuccess(t *testing.T) {
 	testcases := []struct {
 		name       string
 		gcbmgrOpts gcbmgrOptions
+		toolOrg    string
+		toolRepo   string
+		toolBranch string
 		expected   map[string]string
 	}{
 		{
@@ -106,16 +110,15 @@ func TestSetGCBSubstitutionsSuccess(t *testing.T) {
 				gcpUser:     "test-user",
 			},
 			expected: map[string]string{
-				"BUILDVERSION":        "--buildversion=",
-				"BUILD_AT_HEAD":       "",
-				"BUILD_POINT":         "",
-				"OFFICIAL":            "",
-				"OFFICIAL_TAG":        "",
-				"RC":                  "",
-				"RC_TAG":              "",
-				"RELEASE_BRANCH":      "master",
-				"RELEASE_TOOL_BRANCH": "master",
-				"RELEASE_TOOL_REPO":   "https://github.com/kubernetes/release",
+				"BUILD_AT_HEAD":  "",
+				"OFFICIAL":       "",
+				"OFFICIAL_TAG":   "",
+				"RC":             "",
+				"RC_TAG":         "",
+				"RELEASE_BRANCH": "master",
+				"TOOL_ORG":       "kubernetes",
+				"TOOL_REPO":      "release",
+				"TOOL_BRANCH":    "master",
 			},
 		},
 		{
@@ -126,16 +129,15 @@ func TestSetGCBSubstitutionsSuccess(t *testing.T) {
 				gcpUser:     "test-user",
 			},
 			expected: map[string]string{
-				"BUILDVERSION":        "--buildversion=",
-				"BUILD_AT_HEAD":       "",
-				"BUILD_POINT":         "",
-				"OFFICIAL":            "",
-				"OFFICIAL_TAG":        "",
-				"RC":                  "--rc",
-				"RC_TAG":              "rc",
-				"RELEASE_BRANCH":      "release-1.14",
-				"RELEASE_TOOL_BRANCH": "master",
-				"RELEASE_TOOL_REPO":   "https://github.com/kubernetes/release",
+				"BUILD_AT_HEAD":  "",
+				"OFFICIAL":       "",
+				"OFFICIAL_TAG":   "",
+				"RC":             "--rc",
+				"RC_TAG":         "rc",
+				"RELEASE_BRANCH": "release-1.14",
+				"TOOL_ORG":       "kubernetes",
+				"TOOL_REPO":      "release",
+				"TOOL_BRANCH":    "master",
 			},
 		},
 		{
@@ -146,16 +148,37 @@ func TestSetGCBSubstitutionsSuccess(t *testing.T) {
 				gcpUser:     "test-user",
 			},
 			expected: map[string]string{
-				"BUILDVERSION":        "--buildversion=",
-				"BUILD_AT_HEAD":       "",
-				"BUILD_POINT":         "",
-				"OFFICIAL":            "--official",
-				"OFFICIAL_TAG":        "official",
-				"RC":                  "",
-				"RC_TAG":              "",
-				"RELEASE_BRANCH":      "release-1.15",
-				"RELEASE_TOOL_BRANCH": "master",
-				"RELEASE_TOOL_REPO":   "https://github.com/kubernetes/release",
+				"BUILD_AT_HEAD":  "",
+				"OFFICIAL":       "--official",
+				"OFFICIAL_TAG":   "official",
+				"RC":             "",
+				"RC_TAG":         "",
+				"RELEASE_BRANCH": "release-1.15",
+				"TOOL_ORG":       "kubernetes",
+				"TOOL_REPO":      "release",
+				"TOOL_BRANCH":    "master",
+			},
+		},
+		{
+			name: "release-1.16 official with custom tool org, repo, and branch",
+			gcbmgrOpts: gcbmgrOptions{
+				branch:      "release-1.16",
+				releaseType: "official",
+				gcpUser:     "test-user",
+			},
+			toolOrg:    "honk",
+			toolRepo:   "best-tools",
+			toolBranch: "tool-branch",
+			expected: map[string]string{
+				"BUILD_AT_HEAD":  "",
+				"OFFICIAL":       "--official",
+				"OFFICIAL_TAG":   "official",
+				"RC":             "",
+				"RC_TAG":         "",
+				"RELEASE_BRANCH": "release-1.16",
+				"TOOL_ORG":       "honk",
+				"TOOL_REPO":      "best-tools",
+				"TOOL_BRANCH":    "tool-branch",
 			},
 		},
 	}
@@ -164,6 +187,9 @@ func TestSetGCBSubstitutionsSuccess(t *testing.T) {
 		t.Logf("Test case: %s", tc.name)
 
 		opts := tc.gcbmgrOpts
+		os.Setenv("TOOL_ORG", tc.toolOrg)
+		os.Setenv("TOOL_REPO", tc.toolRepo)
+		os.Setenv("TOOL_BRANCH", tc.toolBranch)
 
 		subs, err := setGCBSubstitutions(&opts)
 		actual := dropDynamicSubstitutions(subs)
@@ -205,7 +231,7 @@ func dropDynamicSubstitutions(orig map[string]string) (result map[string]string)
 	result = orig
 
 	for k := range result {
-		if k == "GCP_USER_TAG" || k == "KUBE_CROSS_VERSION" {
+		if k == "BUILDVERSION" || k == "BUILD_POINT" || k == "GCP_USER_TAG" || k == "KUBE_CROSS_VERSION" {
 			delete(result, k)
 		}
 	}

pkg/kubepkg/BUILD.bazel

@@ -10,6 +10,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/command:go_default_library",
+        "//pkg/release:go_default_library",
         "//pkg/util:go_default_library",
         "@com_github_blang_semver//:go_default_library",
         "@com_github_google_go_github_v29//github:go_default_library",
@@ -36,8 +37,5 @@ go_test(
     name = "go_default_test",
     srcs = ["kubepkg_test.go"],
     embed = [":go_default_library"],
-    deps = [
-        "@com_github_stretchr_testify//assert:go_default_library",
-        "@com_github_stretchr_testify//require:go_default_library",
-    ],
+    deps = ["@com_github_stretchr_testify//assert:go_default_library"],
 )

pkg/kubepkg/kubepkg.go

@@ -20,7 +20,6 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
-	"net/http"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -34,6 +33,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"k8s.io/release/pkg/command"
+	"k8s.io/release/pkg/release"
 	"k8s.io/release/pkg/util"
 )
 
@@ -60,6 +60,8 @@ const (
 	templateRootDir = "templates"
 
 	kubeadmConf = "10-kubeadm.conf"
+
+	useSemver = true
 )
 
 var (
@@ -391,46 +393,12 @@ func getKubernetesVersion(packageDef *PackageDefinition) (string, error) {
 	}
 	switch packageDef.Channel {
 	case ChannelTesting:
-		return getTestingKubeVersion()
+		return release.GetStablePrereleaseKubeVersion(useSemver)
 	case ChannelNightly:
-		return getNightlyKubeVersion()
-	}
-
-	return getReleaseKubeVersion()
-}
-
-func getReleaseKubeVersion() (string, error) {
-	logrus.Info("Retrieving Kubernetes release version...")
-	return fetchVersion("https://dl.k8s.io/release/stable.txt")
-}
-
-func getTestingKubeVersion() (string, error) {
-	logrus.Info("Retrieving Kubernetes testing version...")
-	return fetchVersion("https://dl.k8s.io/release/latest.txt")
-}
-
-func getNightlyKubeVersion() (string, error) {
-	logrus.Info("Retrieving Kubernetes nightly version...")
-	return fetchVersion("https://dl.k8s.io/ci/k8s-master.txt")
-}
-
-func fetchVersion(url string) (string, error) {
-	res, err := http.Get(url)
-	if err != nil {
-		return "", err
+		return release.GetLatestCIKubeVersion(useSemver)
 	}
 
-	versionBytes, err := ioutil.ReadAll(res.Body)
-	res.Body.Close()
-	if err != nil {
-		return "", err
-	}
-
-	// Remove a newline and the v prefix from the string
-	version := strings.Replace(strings.Replace(string(versionBytes), "v", "", 1), "\n", "", 1)
-
-	logrus.Infof("Retrieved Kubernetes version: %s", version)
-	return version, nil
+	return release.GetStableReleaseKubeVersion(useSemver)
 }
 
 func getCNIVersion(packageDef *PackageDefinition) (string, error) {
@@ -584,7 +552,7 @@ func getCIBuildsDownloadLinkBase(packageDef *PackageDefinition) (string, error)
 	ciVersion := packageDef.KubernetesVersion
 	if ciVersion == "" {
 		var err error
-		ciVersion, err = getNightlyKubeVersion()
+		ciVersion, err = release.GetLatestCIKubeVersion(useSemver)
 		if err != nil {
 			return "", err
 		}

pkg/kubepkg/kubepkg_test.go

@@ -20,7 +20,6 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 func TestGetPackageVersionSuccess(t *testing.T) {
@@ -120,57 +119,6 @@ func TestGetKubernetesVersionFailure(t *testing.T) {
 	a.Error(err)
 }
 
-func TestFetchVersionSuccess(t *testing.T) {
-	testcases := []struct {
-		name     string
-		url      string
-		expected string
-	}{
-		{
-			name:     "Release URL",
-			url:      "https://dl.k8s.io/release/stable-1.13.txt",
-			expected: "1.13.12",
-		},
-		{
-			name:     "CI URL",
-			url:      "https://dl.k8s.io/ci/latest-1.14.txt",
-			expected: "1.14.11-beta.1.2+c8b135d0b49c44",
-		},
-	}
-
-	for _, tc := range testcases {
-		actual, err := fetchVersion(tc.url)
-
-		if err != nil {
-			t.Fatalf("did not expect an error: %v", err)
-		}
-
-		assert.Equal(t, tc.expected, actual)
-	}
-}
-
-func TestFetchVersionFailure(t *testing.T) {
-	testcases := []struct {
-		name string
-		url  string
-	}{
-		{
-			name: "Empty URL string",
-			url:  "",
-		},
-		{
-			name: "Bad URL",
-			url:  "https://fake.url",
-		},
-	}
-
-	for _, tc := range testcases {
-		_, err := fetchVersion(tc.url)
-
-		require.Error(t, err)
-	}
-}
-
 func TestGetCNIVersionSuccess(t *testing.T) {
 	testcases := []struct {
 		name        string