From a5143eb51a76dcb2cc01b53917ddae0fecb1466f Mon Sep 17 00:00:00 2001 From: Sunny Date: Thu, 30 May 2024 10:21:22 +0000 Subject: [PATCH 1/2] Add workflow for AWS tests & document the setup - Add a new workflow integration-aws for running the AWS integration tests. - Add setup docs for configuring the AWS account and GitHub repository for running the tests. Signed-off-by: Sunny --- .github/workflows/integration-aws.yaml | 77 +++++++ tests/integration/README.md | 272 +++++++++++++++++++++++++ 2 files changed, 349 insertions(+) create mode 100644 .github/workflows/integration-aws.yaml diff --git a/.github/workflows/integration-aws.yaml b/.github/workflows/integration-aws.yaml new file mode 100644 index 00000000..051583b9 --- /dev/null +++ b/.github/workflows/integration-aws.yaml @@ -0,0 +1,77 @@ +name: integration-aws + +on: + workflow_dispatch: + schedule: + - cron: "0 6 * * *" + # push: + # branches: + # - main + +permissions: + contents: read + id-token: write # Required for obtaining AWS OIDC federated credential. + +jobs: + test: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./tests/integration + steps: + - name: Checkout + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + - name: Set up Go + uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 + with: + go-version-file: 'tests/integration/go.mod' + cache-dependency-path: tests/integration/go.sum + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 + with: + role-to-assume: arn:aws:iam::${{ secrets.IRC_E2E_AWS_ACCOUNT_ID }}:role/${{ secrets.IRC_E2E_AWS_ASSUME_ROLE_NAME }} + role-session-name: IRC_GH_Actions + aws-region: ${{ vars.IRC_E2E_AWS_REGION }} + - name: Set up QEMU + uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 + - name: Cache Docker layers + uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + id: cache + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-ghcache-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-ghcache- + - name: Set dynamic variables in .env + run: | + cat > .env <:role/flux-test-*", + "arn:aws:iam:::role/blue-eks-node-group-*", + "arn:aws:iam:::role/green-eks-node-group-*" + ] + } + ] +} +``` + +#### IAM and CI Setup + +To create all the necessary IAM role and policy with all the permissions, set up +CI secrets and variables using +[aws-gh-actions](https://github.com/fluxcd/test-infra/tree/main/tf-modules/aws/github-actions) +with the terraform configuration below. Please make sure all the requirements of +aws-gh-actions are followed before running it, especially registering GitHub +OIDC as an identity provider in the AWS account. + +**NOTE:** When running the following for a repo under an organization, set the +environment variable `GITHUB_ORGANIZATION` if setting the `owner` in the +`github` provider doesn't work. + +```hcl +module "aws_gh_actions" { + source = "git::https://github.com/fluxcd/test-infra.git//tf-modules/aws/github-actions" + + aws_region = "us-east-2" + aws_policy_name = "irc-e2e" + aws_policy_description = "policy for image-reflector-controller e2e tests" + aws_provision_perms = [ + "ec2:AllocateAddress", + "ec2:AssociateRouteTable", + "ec2:AttachInternetGateway", + "ec2:AuthorizeSecurityGroupEgress", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateInternetGateway", + "ec2:CreateLaunchTemplate", + "ec2:CreateLaunchTemplateVersion", + "ec2:CreateNatGateway", + "ec2:CreateNetworkAcl", + "ec2:CreateNetworkAclEntry", + "ec2:CreateRoute", + "ec2:CreateRouteTable", + "ec2:CreateSecurityGroup", + "ec2:CreateSubnet", + "ec2:CreateTags", + "ec2:CreateVpc", + "ec2:DeleteInternetGateway", + "ec2:DeleteLaunchTemplate", + "ec2:DeleteNatGateway", + "ec2:DeleteNetworkAcl", + "ec2:DeleteNetworkAclEntry", + "ec2:DeleteRoute", + "ec2:DeleteRouteTable", + "ec2:DeleteSecurityGroup", + "ec2:DeleteSubnet", + "ec2:DeleteTags", + "ec2:DeleteVolume", + "ec2:DeleteVpc", + "ec2:DescribeAddresses", + "ec2:DescribeAddressesAttribute", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeInternetGateways", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeNatGateways", + "ec2:DescribeNetworkAcls", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroupRules", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeTags", + "ec2:DescribeVpcAttribute", + "ec2:DescribeVpcs", + "ec2:DetachInternetGateway", + "ec2:DisassociateAddress", + "ec2:DisassociateRouteTable", + "ec2:ModifyVpcAttribute", + "ec2:ReleaseAddress", + "ec2:RevokeSecurityGroupEgress", + "ec2:RevokeSecurityGroupIngress", + "ec2:RunInstances", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability", + "ecr:CreateRepository", + "ecr:CompleteLayerUpload", + "ecr:DeleteRepository", + "ecr:DescribeRepositories", + "ecr:GetAuthorizationToken", + "ecr:InitiateLayerUpload", + "ecr:ListTagsForResource", + "ecr:PutImage", + "ecr:TagResource", + "ecr:UploadLayerPart", + "eks:AssociateAccessPolicy", + "eks:CreateAccessEntry", + "eks:CreateAddon", + "eks:CreateCluster", + "eks:CreateNodegroup", + "eks:DeleteAccessEntry", + "eks:DeleteAddon", + "eks:DeleteCluster", + "eks:DeleteNodegroup", + "eks:DescribeAccessEntry", + "eks:DescribeAddon", + "eks:DescribeAddonVersions", + "eks:DescribeCluster", + "eks:DescribeNodegroup", + "eks:DisassociateAccessPolicy", + "eks:ListAssociatedAccessPolicies", + "eks:ListNodegroups", + "eks:TagResource", + "eks:UpdateNodegroupConfig", + "eks:UpdateNodegroupVersion", + "iam:AttachRolePolicy", + "iam:CreateOpenIDConnectProvider", + "iam:CreateRole", + "iam:DeleteOpenIDConnectProvider", + "iam:DeleteRole", + "iam:DetachRolePolicy", + "iam:GetOpenIDConnectProvider", + "iam:GetRole", + "iam:ListAttachedRolePolicies", + "iam:ListInstanceProfilesForRole", + "iam:ListRolePolicies", + "iam:TagOpenIDConnectProvider", + "iam:TagRole", + "ssm:GetParameters" + ] + aws_cluster_role_prefix = [ + "flux-test-", + "blue-eks-node-group-", + "green-eks-node-group-" + ] + aws_role_name = "irc-e2e" + aws_role_description = "role to assume in image-reflector-controller e2e test" + github_repo_owner = "fluxcd" + github_project = "image-reflector-controller" + github_repo_branch_ref = "*" + + github_secret_accound_id_name = "IRC_E2E_AWS_ACCOUNT_ID" + github_secret_assume_role_name = "IRC_E2E_AWS_ASSUME_ROLE_NAME" + github_variable_region_name = "IRC_E2E_AWS_REGION" +} +``` + +**NOTE:** Change the various names and environment variables above as necessary. ### Microsoft Azure From 610998fcab719656a89ce4eef04a630a8a529696 Mon Sep 17 00:00:00 2001 From: Sunny Date: Fri, 2 Aug 2024 18:22:41 +0000 Subject: [PATCH 2/2] aws-e2e: rename variables and secrets Rename the variables and secrets that are common, which can be used with other aws related workflows in the same repository. Signed-off-by: Sunny --- .github/workflows/integration-aws.yaml | 8 ++++---- tests/integration/README.md | 2 -- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration-aws.yaml b/.github/workflows/integration-aws.yaml index 051583b9..d911dbee 100644 --- a/.github/workflows/integration-aws.yaml +++ b/.github/workflows/integration-aws.yaml @@ -29,9 +29,9 @@ jobs: - name: configure aws credentials uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 with: - role-to-assume: arn:aws:iam::${{ secrets.IRC_E2E_AWS_ACCOUNT_ID }}:role/${{ secrets.IRC_E2E_AWS_ASSUME_ROLE_NAME }} + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.IRC_E2E_AWS_ASSUME_ROLE_NAME }} role-session-name: IRC_GH_Actions - aws-region: ${{ vars.IRC_E2E_AWS_REGION }} + aws-region: ${{ vars.AWS_REGION }} - name: Set up QEMU uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 - name: Set up Docker Buildx @@ -69,9 +69,9 @@ jobs: - name: Run tests run: . .env && make test-aws TEST_IMG=fluxcd/image-reflector-controller:dev env: - AWS_REGION: ${{ vars.IRC_E2E_AWS_REGION }} + AWS_REGION: ${{ vars.AWS_REGION }} - name: Ensure resource cleanup if: ${{ always() }} run: . .env && make destroy-aws env: - AWS_REGION: ${{ vars.IRC_E2E_AWS_REGION }} + AWS_REGION: ${{ vars.AWS_REGION }} diff --git a/tests/integration/README.md b/tests/integration/README.md index de73feb4..462c474e 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -274,9 +274,7 @@ module "aws_gh_actions" { github_project = "image-reflector-controller" github_repo_branch_ref = "*" - github_secret_accound_id_name = "IRC_E2E_AWS_ACCOUNT_ID" github_secret_assume_role_name = "IRC_E2E_AWS_ASSUME_ROLE_NAME" - github_variable_region_name = "IRC_E2E_AWS_REGION" } ```