From 20617a489ffef52000cf062a48b742f133d03c05 Mon Sep 17 00:00:00 2001 From: Shwetha Gururaj Date: Fri, 6 Sep 2024 12:45:07 -0400 Subject: [PATCH 1/3] Use single env --- .github/ops-files/diego-cell-instances.yml | 2 +- .../workflows/tests-integration-reusable.yml | 142 +++------------- .github/workflows/tests-integration.yml | 153 +++++++++++++++++- integration/helpers/name_generator.go | 5 + integration/helpers/route.go | 43 +++++ .../helpers/servicebrokerstub/app_deploy.go | 22 ++- .../global/update_buildpack_command_test.go | 2 +- .../v7/isolated/unmap_route_command_test.go | 2 +- 8 files changed, 243 insertions(+), 128 deletions(-) diff --git a/.github/ops-files/diego-cell-instances.yml b/.github/ops-files/diego-cell-instances.yml index 87d715d696c..19ea43d8077 100644 --- a/.github/ops-files/diego-cell-instances.yml +++ b/.github/ops-files/diego-cell-instances.yml @@ -1,4 +1,4 @@ --- - type: replace path: /instance_groups/name=diego-cell/instances - value: 3 + value: 4 diff --git a/.github/workflows/tests-integration-reusable.yml b/.github/workflows/tests-integration-reusable.yml index f812f95f273..fd15bead800 100644 --- a/.github/workflows/tests-integration-reusable.yml +++ b/.github/workflows/tests-integration-reusable.yml @@ -19,16 +19,13 @@ on: name: required: true type: string - pool-name: - type: string - default: ${{ vars.SHEPHERD_POOL_NAME }} - pool-namespace: + lease-id: + required: true type: string - default: 'official' gitRef: type: string default: ${{github.event.workflow_run.head_sha}} - + jobs: run-integration-tests: defaults: @@ -42,92 +39,41 @@ jobs: with: ref: ${{inputs.gitRef}} - - name: Checkout cli-ci - uses: actions/checkout@v4 - with: - repository: cloudfoundry/cli-ci - path: cli-ci - - - id: read-min-capi - name: Read MIN CAPI - run: | - wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq &&\ - chmod +x /usr/bin/yq - version=$(yq '.capi-version-min' build_data.yml) - echo "version=$version" >> $GITHUB_OUTPUT - - - name: Checkout cf-deployment Min CAPI - if: ${{ inputs.capi-version != 'edge' }} - uses: actions/checkout@v4 - with: - repository: cloudfoundry/cf-deployment - path: cf-deployment - ref: ${{ steps.read-min-capi.outputs.version }} - - - name: Checkout cf-deployment - uses: actions/checkout@v4 - with: - repository: cloudfoundry/cf-deployment - path: cf-deployment - - name: Checkout CF deployment tasks uses: actions/checkout@v4 with: repository: cloudfoundry/cf-deployment-concourse-tasks path: cf-deployment-concourse-tasks - - id: claim-env - name: Claim Environment + - name: Set Up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + check-latest: true + + - name: Install Tools env: account_token: ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }} - pool_name: ${{ inputs.pool-name }} - pool_namespace: ${{ inputs.pool-namespace }} run: | - shepherd login service-account ${account_token} - - echo "shepherd create lease --duration 8h --pool ${pool_name} --pool-namespace ${pool_namespace} --namespace tas-devex --description 'CLI GHA'" - lease_id=$(shepherd create lease --duration 8h --pool ${pool_name} --pool-namespace ${pool_namespace} --namespace tas-devex --json | jq -r .id) - # Give sometime for the lease to complete. Shepherd may take upto an 3 hours to create an env - # if the pool is empty. - count=0 - while [ $count -lt 360 ] ; do - sleep 30 - status=$(shepherd get lease ${lease_id} --namespace tas-devex --json | jq -r .status) - if [ $status == "LEASED" ] ; then - shepherd get lease ${lease_id} --namespace tas-devex --json | jq .output > metadata.json - break - elif [ $status == "FAILED" -o $status == "EXPIRED" ] ; then - echo "There was an error obtaining the lease. Lease status is ${status}." - exit 1 - else - echo "Waiting for environment to be ready. Lease status is ${status}." - fi - count=$(($count+1)) - done - - env_name=$(jq -r .name metadata.json) - cat metadata.json | jq -r '.name' - echo "lease-id=$lease_id" >> "${GITHUB_OUTPUT}" + go version - - name: Install Tools - run: | if [[ ${{ inputs.os }} =~ "windows" ]] then install_location=/usr/bin bbl_artifact=bbl-v8.4.110_windows.exe - bosh_cli_artifact=bosh-cli-7.0.1-windows-amd64.exe + bosh_cli_artifact=bosh-cli-7.7.2-windows-amd64.exe credhub_artifact=credhub-windows-2.9.4.tgz else install_location=/usr/local/bin bbl_artifact=bbl-v8.4.110_linux_x86-64 - bosh_cli_artifact=bosh-cli-7.0.1-linux-amd64 + bosh_cli_artifact=bosh-cli-7.7.2-linux-amd64 credhub_artifact=credhub-linux-2.9.4.tgz fi curl https://github.com/cloudfoundry/bosh-bootloader/releases/download/v8.4.110/${bbl_artifact} --silent --location --output $install_location/bbl chmod +x $install_location/bbl bbl --version - curl https://s3.amazonaws.com/bosh-cli-artifacts/$bosh_cli_artifact --silent --output $install_location/bosh --location + curl https://github.com/cloudfoundry/bosh-cli/releases/download/v7.7.2/$bosh_cli_artifact --silent --output $install_location/bosh --location chmod +x $install_location/bosh bosh --version @@ -138,51 +84,10 @@ jobs: credhub --version apt-get update - apt-get install -y build-essential - - - name: Upload latest CAPI release - env: - capi_release_version: ${{ vars.CAPI_RELEASE_VERSION }} - run: | - if [ -z "$capi_release_version" ] - then - capi_release_version=$(curl -s https://api.github.com/repos/cloudfoundry/capi-release/releases/latest | jq -r .tag_name) - fi - - echo "Latest CAPI release is $capi_release_version" - - eval "$(bbl print-env --metadata-file metadata.json)" - env_name=$(jq -r .name metadata.json) - jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${env_name}.priv - bosh upload-release "https://bosh.io/d/github.com/cloudfoundry/capi-release?v=$capi_release_version" + apt-get install -y build-essential unzip - - name: Deploy Isolation Segment and OIDC Provider - run: | - env_name=$(jq -r .name metadata.json) - jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${env_name}.priv - eval "$(bbl print-env --metadata-file metadata.json)" - - # deploy - bosh -d cf manifest > /tmp/manifest.yml - bosh interpolate /tmp/manifest.yml \ - -o cf-deployment/operations/use-internal-lookup-for-route-services.yml \ - -o cf-deployment/operations/add-persistent-isolation-segment-diego-cell.yml \ - -o .github/ops-files/use-latest-capi.yml \ - -o .github/ops-files/add-oidc-provider.yml \ - -o .github/ops-files/add-uaa-client-credentials.yml \ - -o .github/ops-files/diego-cell-instances.yml \ - -v client-secret="${{ secrets.CLIENT_SECRET }}" \ - > ./director.yml - - bosh -d cf deploy director.yml -n - echo "Deployed CAPI version:" - bosh -d cf releases | grep capi - - - name: Set Up Go - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - check-latest: true + shepherd login service-account ${account_token} + shepherd get lease ${{ inputs.lease-id }} --namespace tas-devex --json | jq .output > metadata.json - name: Run Integration Tests if: ${{ !inputs.run-with-client-creds }} @@ -202,6 +107,7 @@ jobs: export GOPATH=$PWD/go export PATH="$GOPATH/bin:$PATH" export PATH="$PWD/out:$PATH" + export CF_INT_TEST_NAME="int" make build @@ -232,6 +138,7 @@ jobs: export GOPATH=$PWD/go export PATH="$GOPATH/bin:$PATH" export PATH="$PWD/out:$PATH" + export CF_INT_TEST_NAME="cc" make build @@ -239,13 +146,4 @@ jobs: cf api ${CF_INT_API} --skip-ssl-validation cf auth - make integration-tests-full-ci - - - name: Unclaim environment - if: always() - env: - account_token: ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }} - run: | - shepherd login service-account ${account_token} - set -x - shepherd delete lease ${{ steps.claim-env.outputs.lease-id }} --namespace tas-devex + make integration-tests-full-ci \ No newline at end of file diff --git a/.github/workflows/tests-integration.yml b/.github/workflows/tests-integration.yml index 09909916d9e..42b59e6cee4 100644 --- a/.github/workflows/tests-integration.yml +++ b/.github/workflows/tests-integration.yml @@ -15,7 +15,6 @@ on: - run-integration-tests-cf-env-with-client-creds push: tags: - - "v9.*" - "v8.*" - "v7.*" pull_request_target: @@ -62,11 +61,140 @@ jobs: check-latest: true - name: Run Units run: make units + + claim-env: + name: Claim and Prep Environment + outputs: + leaseid: ${{steps.claim.outputs.leaseid}} + defaults: + run: + shell: bash + runs-on: ubuntu-latest + container: us-west2-docker.pkg.dev/shepherd-268822/shepherd2/concourse-resource:latest + needs: + - get-sha + - units + steps: + - name: Checkout cli + uses: actions/checkout@v4 + with: + ref: ${{needs.get-sha.outputs.gitRef}} + + - name: claim + id: claim + env: + account_token: ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }} + pool_name: ${{ vars.SHEPHERD_POOL_NAME }} + pool_namespace: official + run: | + shepherd login service-account ${account_token} + + echo "shepherd create lease --duration 8h --pool ${pool_name} --pool-namespace ${pool_namespace} --namespace tas-devex --description 'CLI GHA'" + lease_id=$(shepherd create lease --duration 8h --pool ${pool_name} --pool-namespace ${pool_namespace} --namespace tas-devex --json | jq -r .id) + + # Give sometime for the lease to complete. Shepherd may take upto an 3 hours to create an env + # if the pool is empty. + count=0 + while [ $count -lt 360 ] ; do + sleep 30 + status=$(shepherd get lease ${lease_id} --namespace tas-devex --json | jq -r .status) + if [ $status == "LEASED" ] ; then + shepherd get lease ${lease_id} --namespace tas-devex --json | jq .output > metadata.json + break + elif [ $status == "FAILED" -o $status == "EXPIRED" ] ; then + echo "There was an error obtaining the lease. Lease status is ${status}." + exit 1 + else + echo "Waiting for environment to be ready. Lease status is ${status}." + fi + count=$(($count+1)) + done + + env_name=$(jq -r .name metadata.json) + echo "env name is ${env_name}" + echo "leaseid=${lease_id}" >> "${GITHUB_OUTPUT}" + + cf_deployment_version=$(jq -r '."cf-deployment_version"' metadata.json) + echo "cf_deployment_version is ${cf_deployment_version}" + echo "cf_deployment_version=${cf_deployment_version}" >> "${GITHUB_OUTPUT}" + + - name: Set Up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + check-latest: true + + - name: Install Tools + run: | + go version + + install_location=/usr/local/bin + bbl_artifact=bbl-v8.4.110_linux_x86-64 + bosh_cli_artifact=bosh-cli-7.7.2-linux-amd64 + + curl https://github.com/cloudfoundry/bosh-bootloader/releases/download/v8.4.110/${bbl_artifact} --silent --location --output $install_location/bbl + chmod +x $install_location/bbl + bbl --version + + curl https://github.com/cloudfoundry/bosh-cli/releases/download/v7.7.2/$bosh_cli_artifact --silent --output $install_location/bosh --location + chmod +x $install_location/bosh + bosh --version + + apt-get update + apt-get install -y build-essential unzip + + - name: Upload latest CAPI release + env: + capi_release_version: ${{ vars.CAPI_RELEASE_VERSION }} + run: | + if [ -z "$capi_release_version" ] + then + capi_release_version=$(curl -s https://api.github.com/repos/cloudfoundry/capi-release/releases/latest | jq -r .tag_name) + fi + + echo "Latest CAPI release is $capi_release_version" + + eval "$(bbl print-env --metadata-file metadata.json)" + env_name=$(jq -r .name metadata.json) + jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${env_name}.priv + + bosh upload-release "https://bosh.io/d/github.com/cloudfoundry/capi-release?v=$capi_release_version" + + - name: Checkout cf-deployment + uses: actions/checkout@v4 + with: + repository: cloudfoundry/cf-deployment + path: cf-deployment + ref: ${{steps.claim.outputs.cf_deployment_version}} + + - name: Deploy Isolation Segment and OIDC Provider + run: | + env_name=$(jq -r .name metadata.json) + jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${env_name}.priv + eval "$(bbl print-env --metadata-file metadata.json)" + + # deploy + bosh -d cf manifest > /tmp/manifest.yml + bosh interpolate /tmp/manifest.yml \ + -o cf-deployment/operations/use-internal-lookup-for-route-services.yml \ + -o cf-deployment/operations/add-persistent-isolation-segment-diego-cell.yml \ + -o .github/ops-files/use-latest-capi.yml \ + -o .github/ops-files/add-oidc-provider.yml \ + -o .github/ops-files/add-uaa-client-credentials.yml \ + -o .github/ops-files/diego-cell-instances.yml \ + -v client-secret="${{ secrets.CLIENT_SECRET }}" \ + > ./director.yml + + bosh -d cf deploy director.yml -n + echo "Deployed CAPI version:" + bosh -d cf releases | grep capi + run-integration-tests-cf-env: name: Integration tests needs: - get-sha - units + - claim-env if: ${{ github.event_name != 'workflow_dispatch' || inputs.workflow == 'all' || inputs.workflow == 'run-integration-tests-cf-env' }} uses: ./.github/workflows/tests-integration-reusable.yml with: @@ -74,6 +202,7 @@ jobs: os: ubuntu-latest name: Integration gitRef: ${{needs.get-sha.outputs.gitRef}} + lease-id: ${{ needs.claim-env.outputs.leaseid }} secrets: inherit run-integration-tests-cf-env-with-client-creds: @@ -81,6 +210,7 @@ jobs: needs: - get-sha - units + - claim-env if: ${{ github.event_name != 'workflow_dispatch' || inputs.workflow == 'all' || inputs.workflow == 'run-integration-tests-cf-env-with-client-creds' }} uses: ./.github/workflows/tests-integration-reusable.yml with: @@ -88,4 +218,23 @@ jobs: os: ubuntu-latest name: Integration client creds gitRef: ${{needs.get-sha.outputs.gitRef}} - secrets: inherit \ No newline at end of file + lease-id: ${{ needs.claim-env.outputs.leaseid }} + secrets: inherit + + unclaim-env: + name: Unclaim environment + runs-on: ubuntu-latest + container: us-west2-docker.pkg.dev/shepherd-268822/shepherd2/concourse-resource:latest + needs: + - claim-env + - run-integration-tests-cf-env + - run-integration-tests-cf-env-with-client-creds + if: always() + steps: + - name: unclaim + env: + account_token: ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }} + run: | + shepherd login service-account ${account_token} + set -x + shepherd delete lease ${{ needs.claim-env.outputs.leaseid }} --namespace tas-devex diff --git a/integration/helpers/name_generator.go b/integration/helpers/name_generator.go index 1dc75bf99f4..2f22730cc26 100644 --- a/integration/helpers/name_generator.go +++ b/integration/helpers/name_generator.go @@ -2,6 +2,7 @@ package helpers import ( "fmt" + "os" "sort" "strings" @@ -152,6 +153,10 @@ func NewPath() string { // PrefixedRandomName provides a random name with structure "namePrefix-randomstring" func PrefixedRandomName(namePrefix string) string { + testName := os.Getenv("CF_INT_TEST_NAME") + if len(testName) > 0 { + namePrefix += "-" + testName + } return namePrefix + "-" + RandomName() } diff --git a/integration/helpers/route.go b/integration/helpers/route.go index 6123d10b17a..d9ad43c9b12 100644 --- a/integration/helpers/route.go +++ b/integration/helpers/route.go @@ -2,9 +2,12 @@ package helpers import ( "fmt" + "os" "path" "regexp" + "strings" + "code.cloudfoundry.org/cli/resources" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gbytes" . "github.com/onsi/gomega/gexec" @@ -109,3 +112,43 @@ func (r Route) String() string { return routeString } + +func (r Route) GUID() string { + var domainReceiver struct { + Domains []resources.Domain `json:"resources"` + } + Curl(&domainReceiver, "/v3/domains?names=%s", r.Domain) + Expect(domainReceiver.Domains).To(HaveLen(1)) + + query := []string{fmt.Sprintf("domain_guids=%s", domainReceiver.Domains[0].GUID)} + if r.Host != "" { + query = append(query, fmt.Sprintf("hosts=%s", r.Host)) + } + if r.Path != "" { + path := r.Path + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + query = append(query, fmt.Sprintf("paths=%s", path)) + } + if r.Port != 0 { + query = append(query, fmt.Sprintf("ports=%d", r.Port)) + } + + var routeReceiver struct { + Routes []resources.Route `json:"resources"` + } + Curl(&routeReceiver, "/v3/routes?%s", strings.Join(query, "&")) + Expect(routeReceiver.Routes).To(HaveLen(1)) + + return routeReceiver.Routes[0].GUID +} + +func GetPort() int { + port := 1024 + testName := os.Getenv("CF_INT_TEST_NAME") + if testName == "cc" { + port = 1025 + } + return port +} diff --git a/integration/helpers/servicebrokerstub/app_deploy.go b/integration/helpers/servicebrokerstub/app_deploy.go index d5c21c389e8..1612868ec5c 100644 --- a/integration/helpers/servicebrokerstub/app_deploy.go +++ b/integration/helpers/servicebrokerstub/app_deploy.go @@ -6,6 +6,7 @@ import ( "os" "regexp" "strings" + "sync" "time" "github.com/onsi/ginkgo/v2" @@ -14,14 +15,25 @@ import ( . "github.com/onsi/gomega" ) +var appOrg = "fakeservicebroker" + const ( appNamePrefix = "hydrabroker" - appOrg = "fakeservicebroker" appSpace = "integration" defaultMemoryLimit = "32M" pathToApp = "../../assets/hydrabroker" ) +var once sync.Once +var mutex sync.Mutex + +func initialize() { + testName := os.Getenv("CF_INT_TEST_NAME") + if len(testName) > 0 { + appOrg += "-" + testName + } +} + func ensureAppIsDeployed() { if !appResponds() { ensureAppIsPushed() @@ -71,6 +83,14 @@ func ensureAppIsPushed() { } } + once.Do(func() { + initialize() + }) + + // mutex protects from goroutines, and we retry later to protect from other test processes + mutex.Lock() + defer mutex.Unlock() + helpers.CreateOrgAndSpaceUnlessExists(appOrg, appSpace) helpers.WithRandomHomeDir(func() { helpers.SetAPI() diff --git a/integration/v7/global/update_buildpack_command_test.go b/integration/v7/global/update_buildpack_command_test.go index 789473d2043..8409fd6c447 100644 --- a/integration/v7/global/update_buildpack_command_test.go +++ b/integration/v7/global/update_buildpack_command_test.go @@ -105,7 +105,7 @@ var _ = Describe("update-buildpack command", func() { Eventually(createSession).Should(Exit(0)) }) - listSession := helpers.CF("buildpacks") + listSession := helpers.CF("buildpacks").Wait() Eventually(listSession).Should(Say(helpers.BuildpacksOutputRegex(helpers.BuildpackFields{ Name: buildpackName, Stack: stacks[0]}))) Eventually(listSession).Should(Say(helpers.BuildpacksOutputRegex(helpers.BuildpackFields{Name: buildpackName}))) diff --git a/integration/v7/isolated/unmap_route_command_test.go b/integration/v7/isolated/unmap_route_command_test.go index 13bd47c4b7e..7143031a98b 100644 --- a/integration/v7/isolated/unmap_route_command_test.go +++ b/integration/v7/isolated/unmap_route_command_test.go @@ -117,7 +117,7 @@ var _ = Describe("unmap-route command", func() { }) When("it's a TCP route", func() { BeforeEach(func() { - port = 1024 + port = helpers.GetPort() tcpRoute = helpers.NewTCPRoute(spaceName, tcpDomain.Name, port) session := helpers.CF("map-route", appName, tcpDomain.Name, "--port", fmt.Sprintf("%d", tcpRoute.Port)) Eventually(session).Should(Exit(0)) From d4a14de270849eba0e8480fe27ebe22e0a68f6cd Mon Sep 17 00:00:00 2001 From: Shwetha Gururaj Date: Thu, 3 Oct 2024 16:46:27 -0400 Subject: [PATCH 2/3] Update .github/workflows/tests-integration-reusable.yml Co-authored-by: AL Berez --- .github/workflows/tests-integration-reusable.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/tests-integration-reusable.yml b/.github/workflows/tests-integration-reusable.yml index fd15bead800..eba6db5be37 100644 --- a/.github/workflows/tests-integration-reusable.yml +++ b/.github/workflows/tests-integration-reusable.yml @@ -25,7 +25,6 @@ on: gitRef: type: string default: ${{github.event.workflow_run.head_sha}} - jobs: run-integration-tests: defaults: From 3a2926d1a96324ecadc306b830ef296cf7b0e20d Mon Sep 17 00:00:00 2001 From: Shwetha Gururaj Date: Thu, 3 Oct 2024 16:46:43 -0400 Subject: [PATCH 3/3] Update .github/workflows/tests-integration-reusable.yml Co-authored-by: AL Berez --- .github/workflows/tests-integration-reusable.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-integration-reusable.yml b/.github/workflows/tests-integration-reusable.yml index eba6db5be37..6ddcb98e5f7 100644 --- a/.github/workflows/tests-integration-reusable.yml +++ b/.github/workflows/tests-integration-reusable.yml @@ -145,4 +145,4 @@ jobs: cf api ${CF_INT_API} --skip-ssl-validation cf auth - make integration-tests-full-ci \ No newline at end of file + make integration-tests-full-ci