From 0b06e564ff77fe0ab7830bab7b35184fff588c17 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Thu, 30 Jan 2025 11:27:56 +0100 Subject: [PATCH] feat: add CLI tests workflow --- .gitattributes | 1 + .github/workflows/build-and-integ.yml | 123 +++++++++++ .gitignore | 1 + .projen/files.json | 1 + .projenrc.ts | 22 ++ projenrc/cdk-cli-integ-tests.ts | 290 ++++++++++++++++++++++++++ 6 files changed, 438 insertions(+) create mode 100644 .github/workflows/build-and-integ.yml create mode 100644 projenrc/cdk-cli-integ-tests.ts diff --git a/.gitattributes b/.gitattributes index 1cde247d..74128203 100644 --- a/.gitattributes +++ b/.gitattributes @@ -7,6 +7,7 @@ /.github/pull_request_template.md linguist-generated /.github/workflows/auto-approve.yml linguist-generated /.github/workflows/auto-queue.yml linguist-generated +/.github/workflows/build-and-integ.yml linguist-generated /.github/workflows/build.yml linguist-generated /.github/workflows/pull-request-lint.yml linguist-generated /.github/workflows/release.yml linguist-generated diff --git a/.github/workflows/build-and-integ.yml b/.github/workflows/build-and-integ.yml new file mode 100644 index 00000000..bca1fe0b --- /dev/null +++ b/.github/workflows/build-and-integ.yml @@ -0,0 +1,123 @@ +# ~~ Generated by projen. To modify, edit .projenrc.ts and run "npx projen". + +name: build-and-integ +on: + pull_request_target: + branches: + - main +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + environment: integ-approval + env: + CI: "true" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: lts/* + - name: Install dependencies + run: yarn install --check-files + - name: build + run: npx projen build + - name: Upload artifact + uses: actions/upload-artifact@v4.4.0 + with: + name: build-artifact + path: packages/**/dist/js/*.tgz + overwrite: "true" + integ: + needs: build + runs-on: aws-cdk_ubuntu-latest_4-core + permissions: + contents: read + id-token: write + environment: run-tests + env: + MAVEN_ARGS: --no-transfer-progress + IS_CANARY: "true" + CI: "true" + steps: + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: build-artifact + path: packages + - name: Set up JDK 18 + if: matrix.suite == 'init-java' || matrix.suite == 'cli-integ-tests' + uses: actions/setup-java@v4 + with: + java-version: "18" + distribution: corretto + - name: Authenticate Via OIDC Role + id: creds + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-east-1 + role-duration-seconds: 14400 + role-to-assume: ${{ vars.AWS_ROLE_TO_ASSUME_FOR_TESTING }} + role-session-name: run-tests@aws-cdk-cli-integ + output-credentials: true + - name: Set git identity + run: |- + git config --global user.name "aws-cdk-cli-integ" + git config --global user.email "noreply@example.com" + - name: Install Verdaccio + run: npm install -g verdaccio pm2 + - name: Create Verdaccio config + run: |- + mkdir -p $HOME/.config/verdaccio + echo '{"storage":"./storage","auth":{"htpasswd":{"file":"./htpasswd"}},"uplinks":{"npmjs":{"url":"https://registry.npmjs.org/"}},"packages":{"@aws-cdk/cloud-assembly-schema":{"access":"$all","publish":"$all","proxy":"none"},"@aws-cdk/cloudformation-diff":{"access":"$all","publish":"$all","proxy":"none"},"cdk-assets":{"access":"$all","publish":"$all","proxy":"none"},"aws-cdk":{"access":"$all","publish":"$all","proxy":"none"},"@aws-cdk/cli-lib-alpha":{"access":"$all","publish":"$all","proxy":"none"},"cdk":{"access":"$all","publish":"$all","proxy":"none"},"**":{"access":"$all","proxy":"npmjs"}}}' > $HOME/.config/verdaccio/config.yaml + - name: Start Verdaccio + run: |- + pm2 start verdaccio -- --config $HOME/.config/verdaccio/config.yaml + sleep 5 # Wait for Verdaccio to start + - name: Configure npm to use local registry + run: |- + npm config set registry http://localhost:4873/ + echo '//localhost:4873/:_authToken="MWRjNDU3OTE1NTljYWUyOTFkMWJkOGUyYTIwZWMwNTI6YTgwZjkyNDE0NzgwYWQzNQ=="' > ~/.npmrc + - name: Find an locally publish all tarballs + run: find packages -name \*.tgz -print0 | xargs -0 -n1 npm publish --registry http://localhost:4873/ + - name: Download and install the test artifact + run: |- + npm install @aws-cdk-testing/cli-integ + mv ./node_modules/@aws-cdk-testing/cli-integ/* . + - name: Determine latest CLI version + id: cli_version + run: |- + CLI_VERSION=$(cd ${TMPDIR:-/tmp} && npm view aws-cdk version) + echo "CLI version: ${CLI_VERSION}" + echo "cli_version=${CLI_VERSION}" >> $GITHUB_OUTPUT + - name: "Run the test suite: ${{ matrix.suite }}" + env: + JEST_TEST_CONCURRENT: ${{ matrix.suite == 'cli-integ-tests' && 'true' || 'false' }} + JSII_SILENCE_WARNING_DEPRECATED_NODE_VERSION: "true" + JSII_SILENCE_WARNING_UNTESTED_NODE_VERSION: "true" + JSII_SILENCE_WARNING_KNOWN_BROKEN_NODE_VERSION: "true" + DOCKERHUB_DISABLED: "true" + AWS_REGIONS: us-east-2,eu-west-1,eu-north-1,ap-northeast-1,ap-south-1 + CDK_MAJOR_VERSION: "2" + RELEASE_TAG: latest + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bin/run-suite --use-cli-release=${{ steps.cli_version.outputs.cli_version }} ${{ matrix.suite }} + strategy: + fail-fast: false + matrix: + suite: + - cli-integ-tests + - init-csharp + - init-fsharp + - init-go + - init-java + - init-javascript + - init-python + - init-typescript-app + - init-typescript-lib + - tool-integrations diff --git a/.gitignore b/.gitignore index 45f77379..44d98820 100644 --- a/.gitignore +++ b/.gitignore @@ -44,5 +44,6 @@ jspm_packages/ /dist/ !/aws-cdk-cli.code-workspace !/.github/dependabot.yml +!/.github/workflows/build-and-integ.yml !/.projenrc.ts !/.github/workflows/release.yml diff --git a/.projen/files.json b/.projen/files.json index 015d41d6..dbea2670 100644 --- a/.projen/files.json +++ b/.projen/files.json @@ -5,6 +5,7 @@ ".github/pull_request_template.md", ".github/workflows/auto-approve.yml", ".github/workflows/auto-queue.yml", + ".github/workflows/build-and-integ.yml", ".github/workflows/build.yml", ".github/workflows/pull-request-lint.yml", ".github/workflows/release.yml", diff --git a/.projenrc.ts b/.projenrc.ts index 4f738f83..a51e51d6 100644 --- a/.projenrc.ts +++ b/.projenrc.ts @@ -4,11 +4,17 @@ import { ESLINT_RULES } from './projenrc/eslint'; import { JsiiBuild } from './projenrc/jsii'; import { BundleCli } from './projenrc/bundle'; import { Stability } from 'projen/lib/cdk'; +import { CdkCliIntegTestsWorkflow } from './projenrc/cdk-cli-integ-tests'; // 5.7 sometimes gives a weird error in `ts-jest` in `@aws-cdk/cli-lib-alpha` // https://github.com/microsoft/TypeScript/issues/60159 const TYPESCRIPT_VERSION = "5.6"; +const APPROVAL_ENVIRONMENT = 'integ-approval'; +const TEST_ENVIRONMENT = 'run-tests'; +const TEST_RUNNER = 'aws-cdk_ubuntu-latest_4-core'; + + /** * Projen depends on TypeScript-eslint 7 by default. * @@ -827,4 +833,20 @@ new pj.YamlFile(repo, ".github/dependabot.yml", { committed: true, }); +new CdkCliIntegTestsWorkflow(repo, { + approvalEnvironment: APPROVAL_ENVIRONMENT, + buildRunsOn: workflowRunsOn[0], + testEnvironment: TEST_ENVIRONMENT, + testRunsOn: TEST_RUNNER, + localPackages: [ + cloudAssemblySchema.name, + cloudFormationDiff.name, + cdkAssets.name, + cli.name, + cliLib.name, + cdkAliasPackage.name, + ], +}); + + repo.synth(); diff --git a/projenrc/cdk-cli-integ-tests.ts b/projenrc/cdk-cli-integ-tests.ts new file mode 100644 index 00000000..05e73adb --- /dev/null +++ b/projenrc/cdk-cli-integ-tests.ts @@ -0,0 +1,290 @@ +import { Component, github, javascript } from "projen"; + +export interface CdkCliIntegTestsWorkflowProps { + /** + * Runners for the workflow + */ + buildRunsOn: string; + + /** + * Runners for the workflow + */ + testRunsOn: string; + + /** + * GitHub environment name for approvals + * + * MUST be configured to require manual approval. + */ + approvalEnvironment: string; + + /** + * GitHub environment name for running the tests + * + * MUST be configured without approvals, and with the following vars and secrets: + * + * - vars: AWS_ROLE_TO_ASSUME_FOR_TESTING + * + * And the role needs to be configured to allow the AssumeRole operation. + */ + testEnvironment: string; + + /** + * Packages that are locally transfered (we will never use the upstream versions) + */ + localPackages: string[]; +} + +/** + * Add a workflow for running the tests + * + * This MUST be a separate workflow that runs in privileged context. We have a couple + * of options: + * + * - `workflow_run`: we can trigger a privileged workflow run after the unprivileged + * `pull_request` workflow finishes and reuse its output artifacts. The + * problem is that the second run is disconnected from the PR so we would need + * to script in visibility for approvals and success (by posting comments, for + * example) + * - Use only a `pull_request_target` workflow on the PR: this either would run + * a privileged workflow on any user code submission (might be fine given the + * workflow's `permissions`, but I'm sure this will make our security team uneasy + * anyway), OR this would mean any build needs human confirmation which means slow + * feedback. + * - Use a `pull_request` for a regular fast-feedback build, and a separate + * `pull_request_target` for the integ tests. This means we're building twice. + * + * Ultimately, our build isn't heavy enough to put in a lot of effort deduping + * it, so we'll go with the simplest solution which is the last one: 2 + * independent workflows. + * + * projen doesn't make it easy to copy the relevant parts of the 'build' workflow, + * so they're unfortunately duplicated here. + */ +export class CdkCliIntegTestsWorkflow extends Component { + constructor(repo: javascript.NodeProject, props: CdkCliIntegTestsWorkflowProps) { + super(repo); + + const buildWorkflow = repo.buildWorkflow; + const runTestsWorkflow = repo.github?.addWorkflow('build-and-integ'); + if (!buildWorkflow || !runTestsWorkflow) { + throw new Error('Expected build and run tests workflow'); + } + ((buildWorkflow as any).workflow as github.GithubWorkflow) + + runTestsWorkflow.on({ + pullRequestTarget: { + branches: ['main'], + }, + }); + // The 'build' part runs on the 'integ-approval' environment, which requires + // approval. The actual runs access the real environment, not requiring approval + // anymore. + runTestsWorkflow.addJob('build', { + environment: props.approvalEnvironment, + runsOn: [props.buildRunsOn], + permissions: { + contents: github.workflows.JobPermission.READ, + }, + env: { + CI: 'true', + }, + steps: [ + { + name: 'Checkout', + uses: 'actions/checkout@v4', + with: { + ref: '${{ github.event.pull_request.head.ref }}', + repository: '${{ github.event.pull_request.head.repo.full_name }}', + } + }, + { + name: 'Setup Node.js', + uses: 'actions/setup-node@v4', + with: { + 'node-version': 'lts/*', + } + }, + { + name: 'Install dependencies', + run: 'yarn install --check-files', + }, + { + name: 'build', + run: 'npx projen build', + }, + { + name: 'Upload artifact', + uses: 'actions/upload-artifact@v4.4.0', + with: { + name: 'build-artifact', + path: 'packages/**/dist/js/*.tgz', + overwrite: 'true', + } + }, + ], + }); + + const verdaccioConfig = { + storage: './storage', + auth: { htpasswd: { file: './htpasswd' } }, + uplinks: { npmjs: { url: 'https://registry.npmjs.org/' } }, + packages: {} as Record, + }; + + for (const pack of props.localPackages) { + verdaccioConfig.packages[pack] = { + access: '$all', + publish: '$all', + proxy: 'none', + }; + }; + verdaccioConfig.packages['**'] = { + access: '$all', + proxy: 'npmjs', + }; + + runTestsWorkflow.addJob('integ', { + environment: props.testEnvironment, + runsOn: [props.testRunsOn], + needs: ['build'], + permissions: { + contents: github.workflows.JobPermission.READ, + idToken: github.workflows.JobPermission.WRITE, + }, + env: { + // Otherwise Maven is too noisy + MAVEN_ARGS: '--no-transfer-progress', + // This is not actually a canary, but this prevents the tests from making + // assumptions about the availability of source packages. + IS_CANARY: 'true', + CI: 'true', + }, + strategy: { + failFast: false, + matrix: { + domain: { + suite: [ + 'cli-integ-tests', + 'init-csharp', + 'init-fsharp', + 'init-go', + 'init-java', + 'init-javascript', + 'init-python', + 'init-typescript-app', + 'init-typescript-lib', + 'tool-integrations', + ], + }, + }, + }, + steps: [ + { + name: 'Download build artifacts', + uses: 'actions/download-artifact@v4', + with: { + name: 'build-artifact', + path: 'packages', + } + }, + { + name: 'Set up JDK 18', + if: 'matrix.suite == \'init-java\' || matrix.suite == \'cli-integ-tests\'', + uses: 'actions/setup-java@v4', + with: { + 'java-version': '18', + 'distribution': 'corretto', + }, + }, + { + name: "Authenticate Via OIDC Role", + id: "creds", + uses: "aws-actions/configure-aws-credentials@v4", + with: { + "aws-region": "us-east-1", + "role-duration-seconds": 4 * 60 * 60, + // Expect this in Environment Variables + "role-to-assume": "${{ vars.AWS_ROLE_TO_ASSUME_FOR_TESTING }}", + "role-session-name": "run-tests@aws-cdk-cli-integ", + "output-credentials": true, + }, + }, + // This is necessary for the init tests to succeed, they set up a git repo. + { + name: 'Set git identity', + run: [ + 'git config --global user.name "aws-cdk-cli-integ"', + 'git config --global user.email "noreply@example.com"', + ].join('\n'), + }, + { + name: 'Install Verdaccio', + run: 'npm install -g verdaccio pm2', + }, + { + name: 'Create Verdaccio config', + run: [ + 'mkdir -p $HOME/.config/verdaccio', + `echo '${JSON.stringify(verdaccioConfig)}' > $HOME/.config/verdaccio/config.yaml`, + ].join('\n'), + }, + { + name: 'Start Verdaccio', + run: [ + 'pm2 start verdaccio -- --config $HOME/.config/verdaccio/config.yaml', + 'sleep 5 # Wait for Verdaccio to start', + ].join('\n'), + }, + { + name: 'Configure npm to use local registry', + run: [ + 'npm config set registry http://localhost:4873/', + 'echo \'//localhost:4873/:_authToken="MWRjNDU3OTE1NTljYWUyOTFkMWJkOGUyYTIwZWMwNTI6YTgwZjkyNDE0NzgwYWQzNQ=="\' > ~/.npmrc', + ].join('\n'), + }, + { + name: 'Find an locally publish all tarballs', + run: `find packages -name \\*.tgz -print0 | xargs -0 -n1 npm publish --registry http://localhost:4873/`, + }, + { + name: "Download and install the test artifact", + run: [ + `npm install @aws-cdk-testing/cli-integ`, + // Move the installed files to the current directory, because as + // currently configured the tests won't run from an installed + // node_modules directory. + "mv ./node_modules/@aws-cdk-testing/cli-integ/* .", + ].join("\n"), + }, + { + name: "Determine latest CLI version", + id: 'cli_version', + run: [ + "CLI_VERSION=$(cd ${TMPDIR:-/tmp} && npm view aws-cdk version)", + 'echo "CLI version: ${CLI_VERSION}"', + 'echo "cli_version=${CLI_VERSION}" >> $GITHUB_OUTPUT', + ].join('\n'), + }, + { + name: "Run the test suite: ${{ matrix.suite }}", + run: [ + "bin/run-suite --use-cli-release=${{ steps.cli_version.outputs.cli_version }} ${{ matrix.suite }}", + ].join('\n'), + env: { + // Concurrency only for long-running cli-integ-tests + JEST_TEST_CONCURRENT: "${{ matrix.suite == 'cli-integ-tests' && 'true' || 'false' }}", + JSII_SILENCE_WARNING_DEPRECATED_NODE_VERSION: 'true', + JSII_SILENCE_WARNING_UNTESTED_NODE_VERSION: 'true', + JSII_SILENCE_WARNING_KNOWN_BROKEN_NODE_VERSION: 'true', + DOCKERHUB_DISABLED: 'true', + AWS_REGIONS: ['us-east-2', 'eu-west-1', 'eu-north-1', 'ap-northeast-1', 'ap-south-1'].join(','), + CDK_MAJOR_VERSION: '2', + RELEASE_TAG: 'latest', + GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}', + }, + }, + ], + }); + } +}