diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml new file mode 100644 index 000000000..0b3ededd1 --- /dev/null +++ b/.github/workflows/perf.yml @@ -0,0 +1,122 @@ +name: libp2p perf test + +# How to configure a repository for running this workflow: +# 1. Run 'make ssh-keygen' in 'perf' to generate a new SSH key pair named 'user' in 'perf/terraform/region/files' +# 2. Export AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY for the account of your choice +# 3. Run 'terraform apply' in 'perf/terraform' to create the AWS resources +# 4. Run 'terraform output' in 'perf/terraform' to get the bucket name +# 5. Go to https://console.aws.amazon.com/iamv2/home?#/users/details/perf?section=security_credentials +# 6. Click 'Create access key' to get the access key ID and secret access key +# 7. Go to https://github.com/libp2p/test-plans/settings/secrets/actions +# 8. Click 'New repository secret', set the name to 'PERF_AWS_SECRET_ACCESS_KEY', and paste the secret access key from step 6 +# 9. Click 'New repository secret', set the name to 'PERF_SSH_PRIVATE_KEY', and paste the private key from step 1 +# 10. Go to https://github.com/libp2p/test-plans/settings/variables/actions +# 11. Click 'New repository variable', set the name to 'PERF_AWS_ACCESS_KEY_ID', and paste the access key ID from step 6 +# 12. Click 'New repository variable', set the name to 'PERF_AWS_BUCKET', and paste the bucket name from step 4 + +on: + workflow_dispatch: + workflow_call: + # Example: + # uses: libp2p/test-plans/.github/workflows/perf.yml@master + # with: + # aws-access-key-id: ${{ vars.PERF_AWS_ACCESS_KEY_ID }} + # aws-bucket: ${{ vars.PERF_AWS_BUCKET }} + # ref: master + # secrets: + # PERF_AWS_SECRET_ACCESS_KEY: ${{ secrets.PERF_AWS_SECRET_ACCESS_KEY }} + # PERF_SSH_PRIVATE_KEY: ${{ secrets.PERF_SSH_PRIVATE_KEY }} + inputs: + aws-access-key-id: + type: string + required: true + description: The AWS access key ID of 'aws_iam_user.perf' + aws-bucket: + type: string + required: true + description: The AWS bucket as output by 'output.bucket_name' + ref: + type: string + required: false + description: The ref of the test-plans repo to use (defaults to 'master') + default: master + secrets: + PERF_AWS_SECRET_ACCESS_KEY: # The AWS secret access key of 'aws_iam_user.perf' + required: true + PERF_SSH_PRIVATE_KEY: # The SSH private key for 'aws_key_pair.perf' + required: true + +jobs: + perf: + name: Perf + runs-on: ubuntu-latest + timeout-minutes: 40 + defaults: + run: + shell: bash + working-directory: perf + env: + AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id || vars.PERF_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.PERF_AWS_SECRET_ACCESS_KEY }} + steps: + - name: Configure SSH + uses: webfactory/ssh-agent@d4b9b8ff72958532804b70bbe600ad43b36d5f2e # v0.8.0 + with: + ssh-private-key: ${{ secrets.PERF_SSH_PRIVATE_KEY }} + - name: Checkout test-plans + uses: actions/checkout@v3 + with: + repository: libp2p/test-plans + ref: ${{ inputs.ref || github.ref }} + - id: server + name: Provision server + run: echo "id=$(make provision-server | tail -n 1)" >> $GITHUB_OUTPUT + - id: client + name: Provision client + run: echo "id=$(make provision-client | tail -n 1)" >> $GITHUB_OUTPUT + - id: ip + name: Wait for client/server IP + env: + SERVER_ID: ${{ steps.server.outputs.id }} + CLIENT_ID: ${{ steps.client.outputs.id }} + run: | + read SERVER_IP CLIENT_IP <<< $(make wait SERVER_ID=$SERVER_ID CLIENT_ID=$CLIENT_ID | tail -n 1) + echo "server=$SERVER_IP" >> $GITHUB_OUTPUT + echo "client=$CLIENT_IP" >> $GITHUB_OUTPUT + - name: Download dependencies + run: npm ci + working-directory: perf/runner + - name: Run tests + env: + SERVER_IP: ${{ steps.ip.outputs.server }} + CLIENT_IP: ${{ steps.ip.outputs.client }} + run: npm run start -- --client-public-ip $CLIENT_IP --server-public-ip $SERVER_IP + working-directory: perf/runner + - name: Archive results + uses: actions/upload-artifact@v2 + with: + name: results + path: perf/runner/benchmark-results.json + - id: s3 + name: Upload results + env: + AWS_BUCKET: ${{ inputs.aws-bucket || vars.PERF_AWS_BUCKET }} + AWS_BUCKET_PATH: ${{ github.repository }}/${{ github.run_id }}/${{ github.run_attempt }}/benchmark-results.json + run: | + aws s3 cp benchmark-results.json s3://$AWS_BUCKET/$AWS_BUCKET_PATH --acl public-read --region us-west-2 + echo "url=https://$AWS_BUCKET.s3.amazonaws.com/$AWS_BUCKET_PATH" >> $GITHUB_OUTPUT + working-directory: perf/runner + - name: Set summary + env: + URL: ${{ steps.s3.outputs.url }} + run: echo "$URL" >> $GITHUB_STEP_SUMMARY + - name: Deprovision client + if: always() && steps.client.outputs.id != '' + env: + CLIENT_ID: ${{ steps.client.outputs.id }} + run: make deprovision-client CLIENT_ID=$CLIENT_ID + - name: Deprovision server + if: always() && steps.server.outputs.id != '' + env: + SERVER_ID: ${{ steps.server.outputs.id }} + run: make deprovision-server SERVER_ID=$SERVER_ID diff --git a/perf/Makefile b/perf/Makefile new file mode 100644 index 000000000..999510748 --- /dev/null +++ b/perf/Makefile @@ -0,0 +1,26 @@ +ssh-keygen: + ssh-keygen -t ed25519 -f ./terraform/region/files/user -N '' + +ssh-add: + ssh-add ./terraform/region/files/user + +provision-server: + aws ec2 run-instances --region=us-west-2 --launch-template LaunchTemplateName=perf-node --count 1 --query 'Instances[0].InstanceId' --output text + +provision-client: + aws ec2 run-instances --region=us-east-1 --launch-template LaunchTemplateName=perf-node --count 1 --query 'Instances[0].InstanceId' --output text + +wait: + aws ec2 wait instance-running --region us-west-2 --instance-ids "$(SERVER_ID)" + aws ec2 wait instance-running --region us-east-1 --instance-ids "$(CLIENT_ID)" + echo "$$(aws ec2 describe-instances --region us-west-2 --instance-ids "$(SERVER_ID)" --query 'Reservations[0].Instances[0].PublicIpAddress' --output text)" "$$(aws ec2 describe-instances --region us-east-1 --instance-ids "$(CLIENT_ID)" --query 'Reservations[0].Instances[0].PublicIpAddress' --output text)" + +deprovision-server: + aws ec2 terminate-instances --region=us-west-2 --instance-ids "$(SERVER_ID)" + +deprovision-client: + aws ec2 terminate-instances --region=us-east-1 --instance-ids "$(CLIENT_ID)" + +# https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install.html +scale-down: + sam local invoke ScaleDown --template terraform/common/files/scale_down.yml --event terraform/common/files/scale_down.json diff --git a/perf/README.md b/perf/README.md index 5463f410b..234a828be 100644 --- a/perf/README.md +++ b/perf/README.md @@ -10,21 +10,51 @@ Benchmark results can be visualized with https://observablehq.com/@mxinden-works ## Provision infrastructure -1. `cd terraform` -2. Save your public SSH key as the file `./user.pub`. +### Bootstrap + +1. Save your public SSH key as the file `./regions/files/user.pub`; or generate a new key pair with `make ssh-keygen` and add it to your SSH agent with `make ssh-add`. +2. `cd terraform` 3. `terraform init` 4. `terraform apply` +#### [OPTIONAL] Limited AWS credentials + +If you want to limit the AWS credentials used by subsequent steps, you can create Access Keys for the `perf` user that terraform created. + +1. Go to https://console.aws.amazon.com/iamv2/home?#/users/details/perf?section=security_credentials +2. Create access key +3. Download `perf_accessKeys.csv` +4. Configure AWS CLI to use the credentials. For example: +```bash +export AWS_ACCESS_KEY_ID=$(cat perf_accessKeys.csv | tail -n 1 | cut -d, -f1) +export AWS_SECRET_ACCESS_KEY=$(cat perf_accessKeys.csv | tail -n 1 | cut -d, -f2) +``` + +### Nodes + +1. `SERVER_ID=$(make provision-server | tail -n 1)` +2. `CLIENT_ID=$(make provision-client | tail -n 1)` +3. `read SERVER_IP CLIENT_IP <<< $(make wait SERVER_ID=$SERVER_ID CLIENT_ID=$CLIENT_ID | tail -n 1)` + ## Build and run implementations +_WARNING_: Running the perf tests might take a while. + 1. `cd runner` 2. `npm ci` -3. `npm run start -- --client-public-ip $(terraform output -raw -state ../terraform/terraform.tfstate client_public_ip) --server-public-ip $(terraform output -raw -state ../terraform/terraform.tfstate server_public_ip)` +3. `npm run start -- --client-public-ip $CLIENT_IP --server-public-ip $SERVER_IP` ## Deprovision infrastructure +### Nodes + +1. `make deprovision-client CLIENT_ID=$CLIENT_ID` +2. `make deprovision-server SERVER_ID=$SERVER_ID ` + +### Bootstrap + 1. `cd terraform` -3. `terraform destroy` +2. `terraform destroy` ## Adding a new implementation diff --git a/perf/terraform/.gitignore b/perf/terraform/.gitignore index bb09c5300..6304eb3c1 100644 --- a/perf/terraform/.gitignore +++ b/perf/terraform/.gitignore @@ -32,5 +32,3 @@ override.tf.json # Ignore CLI configuration files .terraformrc terraform.rc - -*.pub diff --git a/perf/terraform/.terraform.lock.hcl b/perf/terraform/.terraform.lock.hcl index e54d18d34..05146c6ea 100644 --- a/perf/terraform/.terraform.lock.hcl +++ b/perf/terraform/.terraform.lock.hcl @@ -1,25 +1,63 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. +provider "registry.terraform.io/hashicorp/archive" { + version = "2.3.0" + hashes = [ + "h1:NaDbOqAcA9d8DiAS5/6+5smXwN3/+twJGb3QRiz6pNw=", + "zh:0869128d13abe12b297b0cd13b8767f10d6bf047f5afc4215615aabc39c2eb4f", + "zh:481ed837d63ba3aa45dd8736da83e911e3509dee0e7961bf5c00ed2644f807b3", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9f08fe2977e2166849be24fb9f394e4d2697414d463f7996fd0d7beb4e19a29c", + "zh:9fe566deeafd460d27999ca0bbfd85426a5fcfcb40007b23884deb76da127b6f", + "zh:a1bd9a60925d9769e0da322e4523330ee86af9dc2e770cba1d0247a999ef29cb", + "zh:bb4094c8149f74308b22a87e1ac19bcccca76e8ef021b571074d9bccf1c0c6f0", + "zh:c8984c9def239041ce41ec8e19bbd76a49e74ed2024ff736dad60429dee89bcc", + "zh:ea4bb5ae73db1de3a586e62f39106f5e56770804a55aa5e6b4f642df973e0e75", + "zh:f44a9d596ecc3a8c5653f56ba0cd202ad93b49f76767f4608daf7260b813289e", + "zh:f5c5e6cc9f7f070020ab7d95fcc9ed8e20d5cf219978295a71236e22cbb6d508", + "zh:fd2273f51dcc8f43403bf1e425ba9db08a57c3ddcba5ad7a51742ccde21ca611", + ] +} + provider "registry.terraform.io/hashicorp/aws" { - version = "4.61.0" - constraints = "~> 4.0" + version = "4.67.0" + constraints = "4.67.0" hashes = [ - "h1:mJSchOA6VkYwEsi+tuspadRmyyE+FGZGYJFUt5kHV+M=", - "zh:051e2588410b7448a5c4c30d668948dd6fdfa8037700bfc00fb228986ccbf3a5", - "zh:082fbcf9706b48d0880ba552a11c29527e228dadd6d83668d0789abda24e5922", - "zh:0e0e72f214fb24f4f9c601cab088a2d8e00ec3327c451bc753911951d773214a", - "zh:3af6d38ca733ca66cce15c6a5735ded7c18348ad26040ebd9a59778b2cd9cf6c", - "zh:404898bc2258bbb9527fa06c72cb927ca011fd9bc3f4b90931c0912652c3f9e9", - "zh:4f617653b0f17a7708bc896f029c4ab0b677a1a1c987bd77166acad1d82db469", - "zh:5dbe393355ac137aa3fd329e3d24871f27012d3ba93d714485b55820df240349", - "zh:6067c2127eb5c879227aca671f101de6dcba909d0d8d15d5711480351962a248", + "h1:5Zfo3GfRSWBaXs4TGQNOflr1XaYj6pRnVJLX5VAjFX4=", + "zh:0843017ecc24385f2b45f2c5fce79dc25b258e50d516877b3affee3bef34f060", + "zh:19876066cfa60de91834ec569a6448dab8c2518b8a71b5ca870b2444febddac6", + "zh:24995686b2ad88c1ffaa242e36eee791fc6070e6144f418048c4ce24d0ba5183", + "zh:4a002990b9f4d6d225d82cb2fb8805789ffef791999ee5d9cb1fef579aeff8f1", + "zh:559a2b5ace06b878c6de3ecf19b94fbae3512562f7a51e930674b16c2f606e29", + "zh:6a07da13b86b9753b95d4d8218f6dae874cf34699bca1470d6effbb4dee7f4b7", + "zh:768b3bfd126c3b77dc975c7c0e5db3207e4f9997cf41aa3385c63206242ba043", + "zh:7be5177e698d4b547083cc738b977742d70ed68487ce6f49ecd0c94dbf9d1362", + "zh:8b562a818915fb0d85959257095251a05c76f3467caa3ba95c583ba5fe043f9b", "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:a939f94461f91aa3b7ec7096271e2714309bd917fe9a03e02f68afb556d65e0f", - "zh:b21227b9082e5fafe8b7c415dc6a99c0d82da05492457377a5fe7d4acaed80e2", - "zh:b8d9f09ed5fc8c654b768b7bee1237eaf1e2287c898249e740695055fb0fe072", - "zh:d360e1e185b148ff6b1d0ed4f7d574e08f2391697ab43df62085b04a1a5b1284", - "zh:da962da17ddda744911cb1e92b983fa3874d73a28f3ee72faa9ddb6680a63774", - "zh:e2f1c4f5ebeb4fd7ef690178168a4c529025b54a91bb7a087dcea48e0b82737a", + "zh:9c385d03a958b54e2afd5279cd8c7cbdd2d6ca5c7d6a333e61092331f38af7cf", + "zh:b3ca45f2821a89af417787df8289cb4314b273d29555ad3b2a5ab98bb4816b3b", + "zh:da3c317f1db2469615ab40aa6baba63b5643bae7110ff855277a1fb9d8eb4f2c", + "zh:dc6430622a8dc5cdab359a8704aec81d3825ea1d305bbb3bbd032b1c6adfae0c", + "zh:fac0d2ddeadf9ec53da87922f666e1e73a603a611c57bcbc4b86ac2821619b1d", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.5.1" + hashes = [ + "h1:IL9mSatmwov+e0+++YX2V6uel+dV6bn+fC/cnGDK3Ck=", + "zh:04e3fbd610cb52c1017d282531364b9c53ef72b6bc533acb2a90671957324a64", + "zh:119197103301ebaf7efb91df8f0b6e0dd31e6ff943d231af35ee1831c599188d", + "zh:4d2b219d09abf3b1bb4df93d399ed156cadd61f44ad3baf5cf2954df2fba0831", + "zh:6130bdde527587bbe2dcaa7150363e96dbc5250ea20154176d82bc69df5d4ce3", + "zh:6cc326cd4000f724d3086ee05587e7710f032f94fc9af35e96a386a1c6f2214f", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:b6d88e1d28cf2dfa24e9fdcc3efc77adcdc1c3c3b5c7ce503a423efbdd6de57b", + "zh:ba74c592622ecbcef9dc2a4d81ed321c4e44cddf7da799faa324da9bf52a22b2", + "zh:c7c5cde98fe4ef1143bd1b3ec5dc04baf0d4cc3ca2c5c7d40d17c0e9b2076865", + "zh:dac4bad52c940cd0dfc27893507c1e92393846b024c5a9db159a93c534a3da03", + "zh:de8febe2a2acd9ac454b844a4106ed295ae9520ef54dc8ed2faf29f12716b602", + "zh:eab0d0495e7e711cca367f7d4df6e322e6c562fc52151ec931176115b83ed014", ] } diff --git a/perf/terraform/common/files/.gitignore b/perf/terraform/common/files/.gitignore new file mode 100644 index 000000000..3fc577864 --- /dev/null +++ b/perf/terraform/common/files/.gitignore @@ -0,0 +1,2 @@ +# generated ZIP for AWS Lambda +scale_down.zip diff --git a/perf/terraform/common/files/scale_down.json b/perf/terraform/common/files/scale_down.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/perf/terraform/common/files/scale_down.json @@ -0,0 +1 @@ +{} diff --git a/perf/terraform/common/files/scale_down.py b/perf/terraform/common/files/scale_down.py new file mode 100644 index 000000000..e8d0f530c --- /dev/null +++ b/perf/terraform/common/files/scale_down.py @@ -0,0 +1,42 @@ +import boto3 +import os +import json +import datetime + +regions = json.loads(os.environ['REGIONS']) # Assuming this is a JSON array +tags = json.loads(os.environ['TAGS']) # Assuming this is a JSON object +max_age_minutes = int(os.environ['MAX_AGE_MINUTES']) # Assuming this is an integer + +def lambda_handler(event, context): + # iterate over all regions + for region in regions: + ec2 = boto3.client('ec2', region_name=region) + + now = datetime.datetime.now(datetime.timezone.utc) + + filters = [{'Name': 'instance-state-name', 'Values': ['running']}] + filters = filters + [{ + 'Name': 'tag:' + k, + 'Values': [v] + } for k, v in tags.items()] + + response = ec2.describe_instances(Filters=filters) + + instances = [] + + for reservation in response['Reservations']: + for instance in reservation['Instances']: + launch_time = instance['LaunchTime'] + instance_id = instance['InstanceId'] + + print( + f'Instance ID: {instance_id} has been running since {launch_time}.') + + if launch_time < now - datetime.timedelta(minutes=max_age_minutes): + print( + f'Instance ID: {instance_id} has been running for more than {max_age_minutes} minutes.') + instances.append(instance_id) + + if instances: + ec2.terminate_instances(InstanceIds=instances) + print(f'Terminating instances: {instances}') diff --git a/perf/terraform/common/files/scale_down.yml b/perf/terraform/common/files/scale_down.yml new file mode 100644 index 000000000..afafca1c7 --- /dev/null +++ b/perf/terraform/common/files/scale_down.yml @@ -0,0 +1,19 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: An AWS Lambda application. + +Resources: + ScaleDown: + Type: AWS::Serverless::Function + Properties: + Handler: scale_down.lambda_handler + Runtime: python3.9 + CodeUri: . + Environment: + Variables: + REGIONS: '["us-west-2", "us-east-1"]' + TAGS: '{"Project":"perf", "Name":"node"}' + MAX_AGE_MINUTES: '30' + Policies: + - AmazonEC2FullAccess + Timeout: 30 diff --git a/perf/terraform/common/main.tf b/perf/terraform/common/main.tf new file mode 100644 index 000000000..ceddadc1f --- /dev/null +++ b/perf/terraform/common/main.tf @@ -0,0 +1,126 @@ +variable "region" { + type = string + description = "The AWS region to create resources in" +} + +variable "common_tags" { + type = map(string) + description = "Common tags to apply to all resources" +} + +variable "bucket_name" { + type = string + description = "The name of the S3 bucket to create" +} + +provider "aws" { + region = var.region + + default_tags { + tags = var.common_tags + } +} + +resource "aws_iam_user" "perf" { + name = "perf" +} + +data "aws_iam_policy_document" "perf" { + statement { + actions = ["ec2:*"] + resources = ["*"] + effect = "Allow" + } + statement { + actions = ["iam:PassRole"] + resources = [aws_iam_role.perf_role.arn] + effect = "Allow" + } +} + +resource "aws_iam_user_policy" "perf" { + name = "perf" + user = aws_iam_user.perf.name + + policy = data.aws_iam_policy_document.perf.json +} + +resource "aws_s3_bucket" "perf" { + bucket = var.bucket_name +} + +resource "aws_s3_bucket_ownership_controls" "perf" { + bucket = aws_s3_bucket.perf.id + rule { + object_ownership = "ObjectWriter" + } +} + +resource "aws_s3_bucket_acl" "perf" { + depends_on = [aws_s3_bucket_ownership_controls.perf] + + bucket = aws_s3_bucket.perf.id + acl = "private" +} + +resource "aws_s3_bucket_public_access_block" "perf" { + bucket = aws_s3_bucket.perf.id + + block_public_acls = false + block_public_policy = false + ignore_public_acls = false + restrict_public_buckets = false +} + + +data "aws_iam_policy_document" "perf_assume_role" { + statement { + sid = "" + effect = "Allow" + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + actions = ["sts:AssumeRole"] + } +} + +resource "aws_iam_role" "perf_role" { + name = "perf-node-role" + assume_role_policy = data.aws_iam_policy_document.perf_assume_role.json +} + +resource "aws_iam_instance_profile" "perf_profile" { + name = "perf-node-profile" + role = aws_iam_role.perf_role.name +} + +data "aws_iam_policy_document" "perf_bucket" { + statement { + actions = ["s3:GetObject", "s3:GetObjectAcl", "s3:PutObject", "s3:PutObjectAcl"] + resources = ["${aws_s3_bucket.perf.arn}/*"] + effect = "Allow" + } + statement { + actions = ["s3:ListBucket"] + resources = ["${aws_s3_bucket.perf.arn}"] + effect = "Allow" + } +} + +resource "aws_iam_role_policy" "perf_bucket" { + name = "perf-bucket-policy" + role = aws_iam_role.perf_role.name + policy = data.aws_iam_policy_document.perf_bucket.json +} + +output "bucket_name" { + value = aws_s3_bucket.perf.bucket +} + +resource "aws_iam_user_policy" "perf_bucket" { + name = "perf-bucket" + user = aws_iam_user.perf.name + + policy = data.aws_iam_policy_document.perf_bucket.json +} diff --git a/perf/terraform/common/scale-down.tf b/perf/terraform/common/scale-down.tf new file mode 100644 index 000000000..6044876d0 --- /dev/null +++ b/perf/terraform/common/scale-down.tf @@ -0,0 +1,112 @@ +locals { + regions = ["us-east-1", "us-west-2"] + tags = merge(var.common_tags, { "Name" = "node" }) +} + +data "archive_file" "scale_down" { + type = "zip" + source_file = "${path.module}/files/scale_down.py" + output_path = "${path.module}/files/scale_down.zip" +} + +resource "aws_lambda_function" "scale_down" { + filename = data.archive_file.scale_down.output_path + source_code_hash = data.archive_file.scale_down.output_base64sha256 + function_name = "perf-scale-down" + role = aws_iam_role.scale_down.arn + handler = "scale_down.lambda_handler" + runtime = "python3.9" + memory_size = 128 + timeout = 30 + + environment { + variables = { + REGIONS = jsonencode(local.regions) + TAGS = jsonencode(local.tags) + MAX_AGE_MINUTES = 50 + } + } +} + +resource "aws_cloudwatch_log_group" "scale_down" { + name = "/aws/lambda/${aws_lambda_function.scale_down.function_name}" + retention_in_days = 7 +} + +resource "aws_cloudwatch_event_rule" "scale_down" { + name = "perf-scale-down-rule" + schedule_expression = "cron(37 * * * ? *)" # 00:37, 01:37, 02:37, ..., 23:37 +} + +resource "aws_cloudwatch_event_target" "scale_down" { + rule = aws_cloudwatch_event_rule.scale_down.name + arn = aws_lambda_function.scale_down.arn +} + +resource "aws_lambda_permission" "scale_down" { + statement_id = "AllowExecutionFromCloudWatch" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.scale_down.function_name + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.scale_down.arn +} + +data "aws_iam_policy_document" "scale_down_assume_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "scale_down" { + name = "perf-scale-down-lambda-role" + assume_role_policy = data.aws_iam_policy_document.scale_down_assume_role.json +} + +data "aws_iam_policy_document" "scale_down" { + statement { + actions = ["ec2:DescribeInstances", "ec2:DescribeTags"] + resources = ["*"] + effect = "Allow" + } + + statement { + actions = ["ec2:TerminateInstances"] + resources = ["*"] + effect = "Allow" + + dynamic "condition" { + for_each = local.tags + + content { + test = "StringEquals" + variable = "ec2:ResourceTag/${condition.key}" + values = [condition.value] + } + } + } +} + +resource "aws_iam_role_policy" "scale_down" { + name = "perf-scale-down-lamda-policy" + role = aws_iam_role.scale_down.name + policy = data.aws_iam_policy_document.scale_down.json +} + +data "aws_iam_policy_document" "scale_down_logging" { + statement { + actions = ["logs:CreateLogStream", "logs:PutLogEvents"] + resources = ["${aws_cloudwatch_log_group.scale_down.arn}*"] + effect = "Allow" + } +} + +resource "aws_iam_role_policy" "scale_down_logging" { + name = "perf-lambda-logging" + role = aws_iam_role.scale_down.name + policy = data.aws_iam_policy_document.scale_down_logging.json +} diff --git a/perf/terraform/region/files/.gitignore b/perf/terraform/region/files/.gitignore new file mode 100644 index 000000000..d1ddebdff --- /dev/null +++ b/perf/terraform/region/files/.gitignore @@ -0,0 +1,3 @@ +# generated SSH key +user +user.pub diff --git a/perf/terraform/user-data.sh b/perf/terraform/region/files/user-data.sh similarity index 100% rename from perf/terraform/user-data.sh rename to perf/terraform/region/files/user-data.sh diff --git a/perf/terraform/region/main.tf b/perf/terraform/region/main.tf index 81a3db2bc..14ebf8663 100644 --- a/perf/terraform/region/main.tf +++ b/perf/terraform/region/main.tf @@ -113,31 +113,38 @@ resource "aws_security_group" "restricted_inbound" { resource "aws_key_pair" "perf" { key_name = "user-public-key" - public_key = file("./user.pub") + public_key = file("${path.module}/files/user.pub") } -resource "aws_instance" "node" { - ami = var.ami +resource "aws_launch_template" "perf" { + name = "perf-node" + image_id = var.ami instance_type = "m5n.8xlarge" - subnet_id = aws_subnet.perf.id - key_name = aws_key_pair.perf.key_name - vpc_security_group_ids = [aws_security_group.restricted_inbound.id] - # Debug via: # - /var/log/cloud-init.log and # - /var/log/cloud-init-output.log - user_data = file("./user-data.sh") - user_data_replace_on_change = true + user_data = filebase64("${path.module}/files/user-data.sh") - tags = merge(var.common_tags, { - Name = "node" - }) -} + tag_specifications { + resource_type = "instance" -output "node_public_ip" { - value = aws_instance.node.public_ip - description = "Public IP address of the node instance" + tags = merge(var.common_tags, { + Name = "node" + }) + } + + instance_initiated_shutdown_behavior = "terminate" + + iam_instance_profile { + name = "perf-node-profile" + } + + network_interfaces { + subnet_id = aws_subnet.perf.id + security_groups = [aws_security_group.restricted_inbound.id] + delete_on_termination = true + } } diff --git a/perf/terraform/terraform.tf b/perf/terraform/terraform.tf index 79ad18050..c3b20e500 100644 --- a/perf/terraform/terraform.tf +++ b/perf/terraform/terraform.tf @@ -2,37 +2,45 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.0" + version = "4.67.0" } } } +locals { + tags = { + Project = "perf" + } +} + +resource "random_id" "bucket_id" { + byte_length = 4 +} + +module "common" { + source = "./common" + region = "us-west-2" + + common_tags = local.tags + bucket_name = "perf-test-bucket-${random_id.bucket_id.hex}" +} + module "server_region" { source = "./region" region = "us-west-2" - ami = "ami-0747e613a2a1ff483" + ami = "ami-0747e613a2a1ff483" - common_tags = { - Project = "perf" - } + common_tags = local.tags } module "client_region" { source = "./region" region = "us-east-1" - ami = "ami-06e46074ae430fba6" - - common_tags = { - Project = "perf" - } -} + ami = "ami-06e46074ae430fba6" -output "server_public_ip" { - value = module.server_region.node_public_ip - description = "Public IP address of the server instance" + common_tags = local.tags } -output "client_public_ip" { - value = module.client_region.node_public_ip - description = "Public IP address of the client instance" +output "bucket_name" { + value = module.common.bucket_name }