Use a version file instead of artifact API in integration tests (#4331)

Now there is a new mage target `mage integrations:updateVersions` that requests a list of version from the artifact API according to the set requirements and creates a `.agent-versions.json` file in the root of the repository. This file is later used for running integration tests. Also, there is a Github action that runs the target and opens a PR when the file changes.
elastic · Mar 6, 2024 · baee6e3 · baee6e3
1 parent 4aeba5b
commit baee6e3
Show file tree

Hide file tree

Showing 15 changed files with 363 additions and 283 deletions.
diff --git a/.agent-versions.json b/.agent-versions.json
@@ -0,0 +1,8 @@
+{
+  "testVersions": [
+    "8.14.0-SNAPSHOT",
+    "8.13.0-SNAPSHOT",
+    "8.12.2",
+    "7.17.18"
+  ]
+}
diff --git a/.github/workflows/bump-agent-versions.yml b/.github/workflows/bump-agent-versions.yml
@@ -0,0 +1,57 @@
+---
+name: update-agent-versions
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+  update_versions:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.21
+
+      - name: Set up branch
+        run: git checkout -b update-agent-versions-$GITHUB_RUN_ID
+
+      - name: Update the agent version file
+        uses: magefile/mage-action@v3
+        with:
+          version: v1.13.0
+          args: integration:updateVersions
+
+      - name: Check for file changes
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          changes=$(git status -s -uno .agent-versions.json)
+          if [ -z "$changes" ]
+          then
+            echo "The versions file didn't change, skipping..."
+          else
+            echo "The versions file changed"
+            open=$(gh pr list --repo "$GITHUB_REPOSITORY" --label="update-versions" --limit 1 --state open --base "$GITHUB_REF_NAME")
+            if [ -n "$open" ]
+            then
+              echo "Another PR for $GITHUB_REF_NAME is in review, skipping..."
+              exit 0
+            fi
+            git diff -p
+            git add ".agent-versions.json"
+            git commit -m "[$GITHUB_REF_NAME](automation) Update .agent-versions.json" -m "This file is used for picking agent versions in integration tests. It's content is based on the reponse from https://artifacts-api.elastic.co/v1/versions/"
+            git push --set-upstream origin "update-agent-versions-$GITHUB_RUN_ID"
+            gh pr create \
+              --base "$GITHUB_REF_NAME" \
+              --fill-first \
+              --head "update-agent-versions-$GITHUB_RUN_ID" \
+              --label 'Team:Elastic-Agent' \
+              --label 'update-versions' \
+              --reviewer 'elastic/elastic-agent-control-plane' \
+              --repo $GITHUB_REPOSITORY
+          fi
diff --git a/magefile.go b/magefile.go
@@ -36,7 +36,9 @@ import (
 	"github.com/elastic/elastic-agent/pkg/testing/multipass"
 	"github.com/elastic/elastic-agent/pkg/testing/ogc"
 	"github.com/elastic/elastic-agent/pkg/testing/runner"
+	"github.com/elastic/elastic-agent/pkg/testing/tools"
 	"github.com/elastic/elastic-agent/pkg/version"
+	"github.com/elastic/elastic-agent/testing/upgradetest"
 	bversion "github.com/elastic/elastic-agent/version"
 
 	// mage:import
@@ -1571,6 +1573,41 @@ func (Integration) Single(ctx context.Context, testName string) error {
 	return integRunner(ctx, false, testName)
 }
 
+// UpdateVersions runs an update on the `.agent-versions.json` fetching
+// the latest version list from the artifact API.
+func (Integration) UpdateVersions(ctx context.Context) error {
+	// test 2 current 8.x version, 1 previous 7.x version and 1 recent snapshot
+	reqs := upgradetest.VersionRequirements{
+		UpgradeToVersion: bversion.Agent,
+		CurrentMajors:    2,
+		PreviousMinors:   1,
+		PreviousMajors:   1,
+		RecentSnapshots:  1,
+	}
+
+	aac := tools.NewArtifactAPIClient(tools.WithLogFunc(log.Default().Printf))
+	versions, err := upgradetest.FetchUpgradableVersions(ctx, aac, reqs)
+	if err != nil {
+		return fmt.Errorf("failed to fetch upgradable versions: %w", err)
+	}
+	versionFileData := upgradetest.AgentVersions{
+		TestVersions: versions,
+	}
+	file, err := os.OpenFile(upgradetest.AgentVersionsFilename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		return fmt.Errorf("failed to open %s for write: %w", upgradetest.AgentVersionsFilename, err)
+	}
+	defer file.Close()
+
+	encoder := json.NewEncoder(file)
+	encoder.SetIndent("", "  ")
+	err = encoder.Encode(versionFileData)
+	if err != nil {
+		return fmt.Errorf("failed to encode JSON to file %s: %w", upgradetest.AgentVersionsFilename, err)
+	}
+	return nil
+}
+
 var stateDir = ".integration-cache"
 var stateFile = "state.yml"
 

diff --git a/pkg/testing/tools/artifacts_api.go b/pkg/testing/tools/artifacts_api.go
@@ -12,11 +12,9 @@ import (
 	"io"
 	"net/http"
 	"net/url"
-	"runtime"
 	"sort"
 	"time"
 
-	"github.com/elastic/elastic-agent/pkg/testing"
 	"github.com/elastic/elastic-agent/pkg/version"
 )
 
@@ -28,9 +26,7 @@ const (
 	artifactAPIV1BuildDetailsEndpoint   = "v1/versions/%s/builds/%s"
 	// artifactAPIV1SearchVersionPackage = "v1/search/%s/%s"
 
-	artifactElasticAgentProject = "elastic-agent-package"
-	artifactReleaseCDN          = "https://artifacts.elastic.co/downloads/beats/elastic-agent"
-
+	artifactElasticAgentProject      = "elastic-agent-package"
 	maxAttemptsForArtifactsAPICall   = 6
 	retryIntervalForArtifactsAPICall = 5 * time.Second
 )
@@ -126,8 +122,10 @@ func WithUrl(url string) ArtifactAPIClientOpt {
 	return func(aac *ArtifactAPIClient) { aac.url = url }
 }
 
-func WithCDNUrl(url string) ArtifactAPIClientOpt {
-	return func(aac *ArtifactAPIClient) { aac.cdnURL = url }
+type logFunc func(format string, args ...interface{})
+
+func WithLogFunc(logf logFunc) ArtifactAPIClientOpt {
+	return func(aac *ArtifactAPIClient) { aac.logFunc = logf }
 }
 
 func WithHttpClient(client httpDoer) ArtifactAPIClientOpt {
@@ -138,20 +136,17 @@ func WithHttpClient(client httpDoer) ArtifactAPIClientOpt {
 // More information about the API can be found at https://artifacts-api.elastic.co/v1
 // which will print a list of available operations
 type ArtifactAPIClient struct {
-	c      httpDoer
-	url    string
-	cdnURL string
-
-	logger logger
+	c       httpDoer
+	url     string
+	logFunc logFunc
 }
 
 // NewArtifactAPIClient creates a new Artifact API client
-func NewArtifactAPIClient(logger logger, opts ...ArtifactAPIClientOpt) *ArtifactAPIClient {
+func NewArtifactAPIClient(opts ...ArtifactAPIClientOpt) *ArtifactAPIClient {
 	c := &ArtifactAPIClient{
-		url:    defaultArtifactAPIURL,
-		cdnURL: artifactReleaseCDN,
-		c:      new(http.Client),
-		logger: logger,
+		url:     defaultArtifactAPIURL,
+		c:       new(http.Client),
+		logFunc: func(string, ...interface{}) {},
 	}
 
 	for _, opt := range opts {
@@ -177,68 +172,6 @@ func (aac ArtifactAPIClient) GetVersions(ctx context.Context) (list *VersionList
 	return checkResponseAndUnmarshal[VersionList](resp)
 }
 
-// RemoveUnreleasedVersions from the list
-// There is a period of time when a version is already marked as released
-// but not published on the CDN. This happens when we already have build candidates.
-// This function checks if a version marked as released actually has published artifacts.
-// If there are no published artifacts, the version is removed from the list.
-func (aac ArtifactAPIClient) RemoveUnreleasedVersions(ctx context.Context, vList *VersionList) error {
-	suffix, err := testing.GetPackageSuffix(runtime.GOOS, runtime.GOARCH)
-	if err != nil {
-		return fmt.Errorf("failed to generate the artifact suffix: %w", err)
-	}
-
-	results := make([]string, 0, len(vList.Versions))
-
-	for _, versionItem := range vList.Versions {
-		parsedVersion, err := version.ParseVersion(versionItem)
-		if err != nil {
-			return fmt.Errorf("failed to parse version %s: %w", versionItem, err)
-		}
-		// we check only release versions without `-SNAPSHOT`
-		if parsedVersion.Prerelease() != "" {
-			results = append(results, versionItem)
-			continue
-		}
-		url := fmt.Sprintf("%s/elastic-agent-%s-%s", aac.cdnURL, versionItem, suffix)
-		// using method `HEAD` to avoid downloading the file
-		req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
-		if err != nil {
-			return fmt.Errorf("failed to create an HTTP request to %q: %w", url, err)
-		}
-
-		resp, err := aac.c.Do(req)
-		if err != nil {
-			return fmt.Errorf("failed to request %q: %w", url, err)
-		}
-
-		// we don't read the response. However, we must drain when it's present,
-		// so the connection can be re-used later, see:
-		// https://cs.opensource.google/go/go/+/refs/tags/go1.22.0:src/net/http/response.go;l=62-64
-		_, _ = io.Copy(io.Discard, resp.Body)
-		_ = resp.Body.Close()
-
-		switch resp.StatusCode {
-		case http.StatusNotFound:
-			continue
-		case http.StatusOK:
-			results = append(results, versionItem)
-			continue
-		default:
-			return fmt.Errorf("unexpected status code from %s - %d", url, resp.StatusCode)
-		}
-	}
-
-	// nothing changed
-	if len(vList.Versions) == len(results) {
-		return nil
-	}
-
-	vList.Versions = results
-
-	return nil
-}
-
 // GetBuildsForVersion returns a list of builds for a specific version.
 // version should be one of the version strings returned by the GetVersions (expected format is semver
 // with optional prerelease but no build metadata, for example 8.9.0-SNAPSHOT)
@@ -336,7 +269,7 @@ func (aac ArtifactAPIClient) createAndPerformRequest(ctx context.Context, URL st
 			)
 		}
 
-		aac.logger.Logf(
+		aac.logFunc(
 			"failed attempt %d of %d executing http request %s %s: %s; retrying after %v...",
 			numAttempts+1, maxAttemptsForArtifactsAPICall, req.Method, req.URL, err.Error(), retryIntervalForArtifactsAPICall,
 		)
@@ -372,10 +305,6 @@ func checkResponseAndUnmarshal[T any](resp *http.Response) (*T, error) {
 	return result, nil
 }
 
-type logger interface {
-	Logf(format string, args ...any)
-}
-
 func (aac ArtifactAPIClient) GetLatestSnapshotVersion(ctx context.Context) (*version.ParsedSemVer, error) {
 	vList, err := aac.GetVersions(ctx)
 	if err != nil {
@@ -390,7 +319,7 @@ func (aac ArtifactAPIClient) GetLatestSnapshotVersion(ctx context.Context) (*ver
 	for _, v := range vList.Versions {
 		pv, err := version.ParseVersion(v)
 		if err != nil {
-			aac.logger.Logf("invalid version retrieved from artifact API: %q", v)
+			aac.logFunc("invalid version retrieved from artifact API: %q", v)
 			return nil, ErrInvalidVersionRetrieved
 		}
 		sortedParsedVersions = append(sortedParsedVersions, pv)

diff --git a/pkg/testing/tools/artifacts_api_test.go b/pkg/testing/tools/artifacts_api_test.go
@@ -6,15 +6,12 @@ package tools
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
-	"regexp"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 const (
@@ -171,7 +168,7 @@ func TestDefaultArtifactAPIClientErrorHttpStatus(t *testing.T) {
 			testSrv := httptest.NewServer(errorHandler)
 			defer testSrv.Close()
 
-			aac := NewArtifactAPIClient(t, WithUrl(testSrv.URL))
+			aac := NewArtifactAPIClient(WithUrl(testSrv.URL), WithLogFunc(t.Logf))
 			_, err := aac.GetVersions(context.Background())
 			assert.ErrorIs(t, err, ErrBadHTTPStatusCode, "Expected ErrBadHTTPStatusCode for status code %d", httpErrorCode)
 			_, err = aac.GetBuildsForVersion(context.Background(), "1.2.3-SNAPSHOT")
@@ -201,7 +198,7 @@ func TestDefaultArtifactAPIClient(t *testing.T) {
 	testSrv := httptest.NewServer(cannedRespHandler)
 	defer testSrv.Close()
 
-	aac := NewArtifactAPIClient(t, WithUrl(testSrv.URL))
+	aac := NewArtifactAPIClient(WithUrl(testSrv.URL), WithLogFunc(t.Logf))
 	versions, err := aac.GetVersions(context.Background())
 	assert.NoError(t, err)
 	assert.NotNil(t, versions)
@@ -219,74 +216,3 @@ func TestDefaultArtifactAPIClient(t *testing.T) {
 	assert.NotEmpty(t, buildDetails.Build.Projects)
 	assert.Contains(t, buildDetails.Build.Projects, "elastic-agent")
 }
-
-func createVersionList(t *testing.T) *VersionList {
-	var list VersionList
-	err := json.Unmarshal([]byte(cannedVersions), &list)
-	require.NoError(t, err)
-	return &list
-}
-
-func TestRemoveUnreleasedVersions(t *testing.T) {
-	versionRegExp := regexp.MustCompile(`/elastic-agent-(\d+\.\d+\.\d+)-.+$`)
-	var unreleasedVersions map[string]struct{}
-	server := httptest.NewServer(http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-		matches := versionRegExp.FindAllStringSubmatch(req.URL.Path, 2)
-		if len(matches) == 0 || len(matches[0]) < 2 {
-			resp.WriteHeader(http.StatusBadRequest)
-			return
-		}
-		version := matches[0][1]
-		if _, unreleased := unreleasedVersions[version]; unreleased {
-			resp.WriteHeader(http.StatusNotFound)
-			return
-		}
-
-		resp.WriteHeader(http.StatusOK)
-	}))
-
-	client := NewArtifactAPIClient(t, WithCDNUrl(server.URL))
-
-	t.Run("removes unreleased versions", func(t *testing.T) {
-		unreleasedVersions = map[string]struct{}{
-			"8.6.1": {},
-			"8.8.0": {},
-		}
-
-		versionList := createVersionList(t)
-		err := client.RemoveUnreleasedVersions(context.Background(), versionList)
-		require.NoError(t, err)
-		exp := []string{
-			"7.17.9",
-			"7.17.10",
-			"8.6.0",
-			"8.6.2",
-			"8.7.0",
-			"8.7.1",
-			"8.8.1",
-			"8.9.0-SNAPSHOT",
-		}
-		require.Equal(t, exp, versionList.Versions)
-	})
-
-	t.Run("does not change the list if all released", func(t *testing.T) {
-		unreleasedVersions = map[string]struct{}{} // everything is released
-
-		versionList := createVersionList(t)
-		err := client.RemoveUnreleasedVersions(context.Background(), versionList)
-		require.NoError(t, err)
-		exp := []string{
-			"7.17.9",
-			"7.17.10",
-			"8.6.0",
-			"8.6.1",
-			"8.6.2",
-			"8.7.0",
-			"8.7.1",
-			"8.8.0",
-			"8.8.1",
-			"8.9.0-SNAPSHOT",
-		}
-		require.Equal(t, exp, versionList.Versions)
-	})
-}
diff --git a/testing/integration/upgrade_broken_package_test.go b/testing/integration/upgrade_broken_package_test.go
@@ -41,7 +41,7 @@ func TestUpgradeBrokenPackageVersion(t *testing.T) {
 	require.NoError(t, err)
 
 	// Upgrade to an old build.
-	upgradeToVersion, err := upgradetest.PreviousMinor(ctx, define.Version(), t)
+	upgradeToVersion, err := upgradetest.PreviousMinor()
 	require.NoError(t, err)
 	endFixture, err := atesting.NewFixture(
 		t,