From 5e5f66a2e5d6056df981429111c45d8fd6c804c6 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Mon, 27 Jan 2025 12:16:41 -0500 Subject: [PATCH 01/16] Use direct PE_API_URL instead of CD proxy --- backend/worker/generate_config.sh | 2 +- infrastructure/integration.tfvars | 1 + infrastructure/pe_worker.tf | 4 ++++ infrastructure/prod.tfvars | 1 + infrastructure/stage-cd.tfvars | 1 + infrastructure/stage.tfvars | 1 + infrastructure/vars.tf | 6 ++++++ infrastructure/worker.tf | 7 +++++++ 8 files changed, 22 insertions(+), 1 deletion(-) diff --git a/backend/worker/generate_config.sh b/backend/worker/generate_config.sh index dbec44545..89c9e7367 100755 --- a/backend/worker/generate_config.sh +++ b/backend/worker/generate_config.sh @@ -14,7 +14,7 @@ key1=${PE_SHODAN_API_KEYS} [pe_api] pe_api_key=${PE_API_KEY} -pe_api_url=https://api.staging-cd.crossfeed.cyber.dhs.gov/pe/apiv1/ +pe_api_url=${PE_API_URL} cf_api_key=${CF_API_KEY} [staging] diff --git a/infrastructure/integration.tfvars b/infrastructure/integration.tfvars index 3d181e967..1e64694a4 100644 --- a/infrastructure/integration.tfvars +++ b/infrastructure/integration.tfvars @@ -57,6 +57,7 @@ ssm_sixgill_client_secret = "/crossfeed/integration/SIXGILL_CLIENT_SE ssm_lg_api_key = "/crossfeed/integration/LG_API_KEY" ssm_lg_workspace_name = "/crossfeed/integration/LG_WORKSPACE_NAME" ssm_pe_api_key = "/crossfeed/integration/PE_API_KEY" +ssm_pe_api_url = "/crossfeed/integration/PE_API_URL" ssm_cf_api_key = "/crossfeed/integration/CF_API_KEY" db_group_name = "crossfeed-integration-db-group" worker_ecs_repository_name = "crossfeed-integration-worker" diff --git a/infrastructure/pe_worker.tf b/infrastructure/pe_worker.tf index 80e8d50ec..443839d68 100644 --- a/infrastructure/pe_worker.tf +++ b/infrastructure/pe_worker.tf @@ -142,6 +142,10 @@ resource "aws_ecs_task_definition" "pe_worker" { "name": "PE_API_KEY", "valueFrom": "${data.aws_ssm_parameter.pe_api_key.arn}" }, + { + "name": "PE_API_URL", + "valueFrom": "${data.aws_ssm_parameter.pe_api_url.arn}" + }, { "name": "CF_API_KEY", "valueFrom": "${data.aws_ssm_parameter.cf_api_key.arn}" diff --git a/infrastructure/prod.tfvars b/infrastructure/prod.tfvars index aa7d9b9dd..b730ae5c9 100644 --- a/infrastructure/prod.tfvars +++ b/infrastructure/prod.tfvars @@ -117,6 +117,7 @@ ssm_redshift_database = "/crossfeed/prod/REDSHIFT_DATABASE" ssm_redshift_user = "/crossfeed/prod/REDSHIFT_USER" ssm_redshift_password = "/crossfeed/prod/REDSHIFT_PASSWORD" ssm_pe_api_key = "/crossfeed/prod/PE_API_KEY" +ssm_pe_api_url = "/crossfeed/prod/PE_API_URL" ssm_cf_api_key = "/crossfeed/prod/CF_API_KEY" ssm_intelx_api_key = "/crossfeed/prod/INTELX_API_KEY" ssm_xpanse_api_key = "/crossfeed/prod/XPANSE_API_KEY" diff --git a/infrastructure/stage-cd.tfvars b/infrastructure/stage-cd.tfvars index 041c4b7f9..52003de50 100644 --- a/infrastructure/stage-cd.tfvars +++ b/infrastructure/stage-cd.tfvars @@ -59,6 +59,7 @@ ssm_xpanse_auth_id = "/crossfeed/staging/XPANSE_AUTH_ID" ssm_lg_api_key = "/crossfeed/staging/LG_API_KEY" ssm_lg_workspace_name = "/crossfeed/staging/LG_WORKSPACE_NAME" ssm_pe_api_key = "/crossfeed/staging/PE_API_KEY" +ssm_pe_api_url = "/crossfeed/staging/PE_API_URL" ssm_cf_api_key = "/crossfeed/staging/CF_API_KEY" db_group_name = "crossfeed-staging-db-group" worker_ecs_repository_name = "crossfeed-staging-worker" diff --git a/infrastructure/stage.tfvars b/infrastructure/stage.tfvars index d7222e6e0..c035b6e05 100644 --- a/infrastructure/stage.tfvars +++ b/infrastructure/stage.tfvars @@ -117,5 +117,6 @@ ssm_redshift_database = "/crossfeed/staging/REDSHIFT_DATABASE" ssm_redshift_user = "/crossfeed/staging/REDSHIFT_USER" ssm_redshift_password = "/crossfeed/staging/REDSHIFT_PASSWORD" ssm_pe_api_key = "/crossfeed/staging/PE_API_KEY" +ssm_pe_api_url = "/crossfeed/staging/PE_API_URL" ssm_cf_api_key = "/crossfeed/staging/CF_API_KEY" create_elasticache_cluster = true diff --git a/infrastructure/vars.tf b/infrastructure/vars.tf index 1d7130b3a..e0481b499 100644 --- a/infrastructure/vars.tf +++ b/infrastructure/vars.tf @@ -711,6 +711,12 @@ variable "ssm_pe_api_key" { default = "/crossfeed/staging/PE_API_KEY" } +variable "ssm_pe_api_url" { + description = "ssm_pe_api_url" + type = string + default = "/crossfeed/staging/PE_API_URL" +} + variable "ssm_cf_api_key" { description = "ssm_cf_api_key" type = string diff --git a/infrastructure/worker.tf b/infrastructure/worker.tf index b7907d278..6a6d55bd7 100644 --- a/infrastructure/worker.tf +++ b/infrastructure/worker.tf @@ -100,6 +100,7 @@ resource "aws_iam_role_policy" "worker_task_execution_role_policy" { "${data.aws_ssm_parameter.https_proxy.arn}", "${aws_ssm_parameter.es_endpoint.arn}", "${data.aws_ssm_parameter.pe_api_key.arn}", + "${data.aws_ssm_parameter.pe_api_url.arn}", "${data.aws_ssm_parameter.cf_api_key.arn}", "${data.aws_ssm_parameter.ssm_mdl_name.arn}", "${data.aws_ssm_parameter.ssm_mdl_username.arn}", @@ -332,6 +333,10 @@ resource "aws_ecs_task_definition" "worker" { "name": "PE_API_KEY", "valueFrom": "${data.aws_ssm_parameter.pe_api_key.arn}" }, + { + "name": "PE_API_URL", + "valueFrom": "${data.aws_ssm_parameter.pe_api_url.arn}" + }, { "name": "CF_API_KEY", "valueFrom": "${data.aws_ssm_parameter.cf_api_key.arn}" @@ -438,6 +443,8 @@ data "aws_ssm_parameter" "https_proxy" { name = var.ssm_https_proxy } data "aws_ssm_parameter" "pe_api_key" { name = var.ssm_pe_api_key } +data "aws_ssm_parameter" "pe_api_url" { name = var.ssm_pe_api_url } + data "aws_ssm_parameter" "cf_api_key" { name = var.ssm_cf_api_key } data "aws_ssm_parameter" "ssm_mdl_name" { name = var.ssm_mdl_name } From 2bc6d7efabdc1e9e978d5be42da037f8d0382874 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Tue, 28 Jan 2025 14:37:28 -0500 Subject: [PATCH 02/16] Allow API Keys to be passed through Authorization header --- backend/src/xfd_django/xfd_api/auth.py | 51 ++++++++++++++------------ 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index 3dc53dd5b..4d92d7b45 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -5,6 +5,7 @@ import hashlib from hashlib import sha256 import os +import re from typing import List, Optional from urllib.parse import urlencode import uuid @@ -272,34 +273,38 @@ def get_current_active_user( if api_key: user = get_user_by_api_key(api_key) elif token: - try: - # Decode token in Authorization header to get user - payload = jwt.decode(token, JWT_SECRET, algorithms=[JWT_ALGORITHM]) - user_id = payload.get("id") - - if user_id is None: - print("No user ID found in token") + # Check if token is an API key + if re.match(r"^[A-Fa-f0-9]{32}$", token): + user = get_user_by_api_key(token) + else: + try: + # Decode token in Authorization header to get user + payload = jwt.decode(token, JWT_SECRET, algorithms=[JWT_ALGORITHM]) + user_id = payload.get("id") + + if user_id is None: + print("No user ID found in token") + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid token", + headers={"WWW-Authenticate": "Bearer"}, + ) + # Fetch the user by ID from the database + user = User.objects.get(id=user_id) + except jwt.ExpiredSignatureError: + print("Token has expired") + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Token has expired", + headers={"WWW-Authenticate": "Bearer"}, + ) + except jwt.InvalidTokenError: + print("Invalid token") raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid token", headers={"WWW-Authenticate": "Bearer"}, ) - # Fetch the user by ID from the database - user = User.objects.get(id=user_id) - except jwt.ExpiredSignatureError: - print("Token has expired") - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Token has expired", - headers={"WWW-Authenticate": "Bearer"}, - ) - except jwt.InvalidTokenError: - print("Invalid token") - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Invalid token", - headers={"WWW-Authenticate": "Bearer"}, - ) else: raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, From 3f5675665ab7fcd184c709cf76e973cfad479850 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Tue, 28 Jan 2025 16:24:16 -0500 Subject: [PATCH 03/16] Add -y flag ensure that apt installs the packages without prompting for user confirmation --- backend/Dockerfile.pe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/Dockerfile.pe b/backend/Dockerfile.pe index d40a383fd..100921e99 100644 --- a/backend/Dockerfile.pe +++ b/backend/Dockerfile.pe @@ -7,7 +7,7 @@ COPY ./package* ./ COPY src ./src -RUN apt update && apt install git zlib1g-dev +RUN apt update && apt install -y git zlib1g-dev RUN apt-get update && apt-get install -y jq From 02c82dbe3b6e5095e3192b58eeac1e9b22bde4ee Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 30 Jan 2025 12:24:10 -0500 Subject: [PATCH 04/16] Create EC2 instance for sending emails --- infrastructure/connectToEmailSender.sh | 66 +++++++++++++++++++++ infrastructure/email-sender-install.sh | 25 ++++++++ infrastructure/email-sender.tf | 82 ++++++++++++++++++++++++++ infrastructure/integration.tfvars | 2 + infrastructure/prod.tfvars | 2 + infrastructure/stage-cd.tfvars | 2 + infrastructure/stage.tfvars | 2 + infrastructure/vars.tf | 12 ++++ 8 files changed, 193 insertions(+) create mode 100644 infrastructure/connectToEmailSender.sh create mode 100644 infrastructure/email-sender-install.sh create mode 100644 infrastructure/email-sender.tf diff --git a/infrastructure/connectToEmailSender.sh b/infrastructure/connectToEmailSender.sh new file mode 100644 index 000000000..fca339e43 --- /dev/null +++ b/infrastructure/connectToEmailSender.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Configuration +AWS_PROFILE=${AWS_PROFILE:-"default"} +INSTANCE_ID=${INSTANCE_ID:-"your-instance-id"} +REGION="us-east-1" + +function log_info() { + echo "$(date '+%Y-%m-%d %H:%M:%S') INFO: $1" +} + +function log_error() { + echo "$(date '+%Y-%m-%d %H:%M:%S') ERROR: $1" >&2 +} + +# Check if the instance is running +function get_instance_status() { + log_info "Checking instance status..." + STATUS=$(aws ec2 describe-instance-status \ + --instance-ids "$INSTANCE_ID" \ + --profile "$AWS_PROFILE" \ + --query 'InstanceStatuses[0].InstanceState.Name' \ + --output text 2>/dev/null) + + echo "$STATUS" +} + +# Start the instance if it's not running +function start_instance() { + log_info "Starting instance $INSTANCE_ID..." + aws ec2 start-instances \ + --instance-ids "$INSTANCE_ID" \ + --profile "$AWS_PROFILE" \ + > /dev/null + + log_info "Instance started. Waiting for initialization (2 minutes)..." + sleep 120 +} + +# Connect to the instance using SSM +function connect_to_instance() { + log_info "Connecting to instance $INSTANCE_ID via SSM..." + aws ssm start-session \ + --target "$INSTANCE_ID" \ + --profile "$AWS_PROFILE" +} + +# Main script logic +log_info "Starting EC2 connection process..." +if [ -z "$INSTANCE_ID" ]; then + log_error "INSTANCE_ID is not set. Please set it as an environment variable or update the script." + exit 1 +fi + +STATUS=$(get_instance_status) + +if [[ "$STATUS" == "running" ]]; then + log_info "Instance is already running." +elif [[ "$STATUS" == "stopped" || "$STATUS" == "stopping" ]]; then + start_instance +else + log_error "Unexpected instance status: $STATUS" + exit 1 +fi + +connect_to_instance diff --git a/infrastructure/email-sender-install.sh b/infrastructure/email-sender-install.sh new file mode 100644 index 000000000..efc695cea --- /dev/null +++ b/infrastructure/email-sender-install.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Create temporary directory for SSM Agent installation +sudo mkdir -p /tmp/ssm +cd /tmp/ssm || return + +# Download and install the SSM Agent +wget https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/debian_amd64/amazon-ssm-agent.deb +sudo dpkg -i amazon-ssm-agent.deb +sudo systemctl enable amazon-ssm-agent +sudo systemctl start amazon-ssm-agent +rm amazon-ssm-agent.deb + +# Update packages +sudo apt-get update -y + +# Install Python3 and pip +sudo apt-get install -y python3 python3-pip + +# Install necessary Python libraries +pip3 install boto3 pandas + +# Create working directory for email script +sudo mkdir -p /var/www/email_sender +sudo chmod -R 755 /var/www/email_sender diff --git a/infrastructure/email-sender.tf b/infrastructure/email-sender.tf new file mode 100644 index 000000000..f6b2aac2a --- /dev/null +++ b/infrastructure/email-sender.tf @@ -0,0 +1,82 @@ + +resource "aws_iam_role" "email_sender" { + count = var.create_email_sender_instance ? 1 : 0 + name = "crossfeed-email-sender-${var.stage}" + assume_role_policy = < Date: Thu, 30 Jan 2025 12:32:23 -0500 Subject: [PATCH 05/16] run pre-commit --- infrastructure/connectToEmailSender.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/connectToEmailSender.sh b/infrastructure/connectToEmailSender.sh index fca339e43..c40399c68 100644 --- a/infrastructure/connectToEmailSender.sh +++ b/infrastructure/connectToEmailSender.sh @@ -20,7 +20,7 @@ function get_instance_status() { --instance-ids "$INSTANCE_ID" \ --profile "$AWS_PROFILE" \ --query 'InstanceStatuses[0].InstanceState.Name' \ - --output text 2>/dev/null) + --output text 2> /dev/null) echo "$STATUS" } From 72b88ab026f85ab2c2c523eaeb80162611c43532 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 30 Jan 2025 12:50:04 -0500 Subject: [PATCH 06/16] update emailSenderConnect with ssh --- ...ToEmailSender.sh => emailSenderConnect.sh} | 48 ++++++++++++++----- 1 file changed, 35 insertions(+), 13 deletions(-) rename infrastructure/{connectToEmailSender.sh => emailSenderConnect.sh} (51%) mode change 100644 => 100755 diff --git a/infrastructure/connectToEmailSender.sh b/infrastructure/emailSenderConnect.sh old mode 100644 new mode 100755 similarity index 51% rename from infrastructure/connectToEmailSender.sh rename to infrastructure/emailSenderConnect.sh index c40399c68..067b2f00f --- a/infrastructure/connectToEmailSender.sh +++ b/infrastructure/emailSenderConnect.sh @@ -1,9 +1,13 @@ #!/bin/bash # Configuration -AWS_PROFILE=${AWS_PROFILE:-"default"} -INSTANCE_ID=${INSTANCE_ID:-"your-instance-id"} -REGION="us-east-1" +AWS_PROFILE=${EMAIL_AWS_PROFILE:-"default"} +INSTANCE_ID=${EMAIL_SENDER_INSTANCE_ID:-"your-instance-id"} +AVAILABILITY_ZONE="us-east-1b" +LOCAL_PORT=9995 +REMOTE_PORT=22 +SSH_USER="ubuntu" +SSH_KEY_PATH=${EMAIL_SSH_KEY_PATH:-""} function log_info() { echo "$(date '+%Y-%m-%d %H:%M:%S') INFO: $1" @@ -15,14 +19,11 @@ function log_error() { # Check if the instance is running function get_instance_status() { - log_info "Checking instance status..." - STATUS=$(aws ec2 describe-instance-status \ + aws ec2 describe-instance-status \ --instance-ids "$INSTANCE_ID" \ --profile "$AWS_PROFILE" \ --query 'InstanceStatuses[0].InstanceState.Name' \ - --output text 2> /dev/null) - - echo "$STATUS" + --output text 2> /dev/null } # Start the instance if it's not running @@ -37,11 +38,29 @@ function start_instance() { sleep 120 } -# Connect to the instance using SSM -function connect_to_instance() { - log_info "Connecting to instance $INSTANCE_ID via SSM..." +# Inject SSH Public Key using EC2 Instance Connect +function send_ssh_public_key() { + log_info "Sending SSH public key..." + aws ec2-instance-connect send-ssh-public-key \ + --instance-id "$INSTANCE_ID" \ + --availability-zone "$AVAILABILITY_ZONE" \ + --instance-os-user "$SSH_USER" \ + --ssh-public-key "file://$SSH_KEY_PATH" \ + --profile "$AWS_PROFILE" + + if [[ $? -ne 0 ]]; then + log_error "Failed to send SSH public key." + exit 1 + fi +} + +# Start port forwarding with AWS SSM +function start_port_forwarding() { + log_info "Starting port forwarding via SSM..." aws ssm start-session \ --target "$INSTANCE_ID" \ + --document-name AWS-StartPortForwardingSession \ + --parameters "{\"portNumber\":[\"$REMOTE_PORT\"], \"localPortNumber\":[\"$LOCAL_PORT\"]}" \ --profile "$AWS_PROFILE" } @@ -52,7 +71,9 @@ if [ -z "$INSTANCE_ID" ]; then exit 1 fi -STATUS=$(get_instance_status) +STATUS=$(get_instance_status | tr -d '\r') + +log_info "Current instance status: $STATUS" if [[ "$STATUS" == "running" ]]; then log_info "Instance is already running." @@ -63,4 +84,5 @@ else exit 1 fi -connect_to_instance +send_ssh_public_key +start_port_forwarding From 7355816fd740991ca9deca301750066e4024ee42 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 30 Jan 2025 13:05:20 -0500 Subject: [PATCH 07/16] Fix email-sener subnet_id --- infrastructure/email-sender.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/email-sender.tf b/infrastructure/email-sender.tf index f6b2aac2a..525cdd561 100644 --- a/infrastructure/email-sender.tf +++ b/infrastructure/email-sender.tf @@ -70,7 +70,7 @@ resource "aws_instance" "email_sender" { } vpc_security_group_ids = [var.is_dmz ? aws_security_group.allow_internal[0].id : aws_security_group.allow_internal_lz[0].id] - subnet_id = var.is_dmz ? aws_subnet.backend[0].id : data.aws_ssm_parameter.subnet_db_1_id.value + subnet_id = var.is_dmz ? aws_subnet.backend[0].id : data.aws_ssm_parameter.subnet_db_1_id[0].value iam_instance_profile = aws_iam_instance_profile.ses_sender.name user_data = file("./email-sender-install.sh") From b687cca3d1f24c8a1e47335b2a691f721495d86b Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 30 Jan 2025 13:37:49 -0500 Subject: [PATCH 08/16] Fix undeclared reference in terraform --- infrastructure/email-sender.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/email-sender.tf b/infrastructure/email-sender.tf index 525cdd561..f96685e20 100644 --- a/infrastructure/email-sender.tf +++ b/infrastructure/email-sender.tf @@ -72,7 +72,7 @@ resource "aws_instance" "email_sender" { vpc_security_group_ids = [var.is_dmz ? aws_security_group.allow_internal[0].id : aws_security_group.allow_internal_lz[0].id] subnet_id = var.is_dmz ? aws_subnet.backend[0].id : data.aws_ssm_parameter.subnet_db_1_id[0].value - iam_instance_profile = aws_iam_instance_profile.ses_sender.name + iam_instance_profile = aws_iam_instance_profile.email_sender.name user_data = file("./email-sender-install.sh") lifecycle { From ef2868a7ff4dfbf1afc1c11a86189ef386f7d0f0 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 30 Jan 2025 14:04:31 -0500 Subject: [PATCH 09/16] fix terraform count --- infrastructure/email-sender.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/email-sender.tf b/infrastructure/email-sender.tf index f96685e20..756aae9be 100644 --- a/infrastructure/email-sender.tf +++ b/infrastructure/email-sender.tf @@ -54,7 +54,7 @@ resource "aws_instance" "email_sender" { associate_public_ip_address = false depends_on = [ - aws_iam_instance_profile.email_sender, + aws_iam_instance_profile.email_sender[0], aws_security_group.allow_internal, aws_subnet.backend ] @@ -72,7 +72,7 @@ resource "aws_instance" "email_sender" { vpc_security_group_ids = [var.is_dmz ? aws_security_group.allow_internal[0].id : aws_security_group.allow_internal_lz[0].id] subnet_id = var.is_dmz ? aws_subnet.backend[0].id : data.aws_ssm_parameter.subnet_db_1_id[0].value - iam_instance_profile = aws_iam_instance_profile.email_sender.name + iam_instance_profile = aws_iam_instance_profile.email_sender[0].name user_data = file("./email-sender-install.sh") lifecycle { From 8ce2f89a2da938216a0f1276b28a9b9c119124bd Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 30 Jan 2025 14:12:12 -0500 Subject: [PATCH 10/16] fix variable type for instance type --- infrastructure/vars.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/vars.tf b/infrastructure/vars.tf index 5463570bd..7c4ef47ef 100644 --- a/infrastructure/vars.tf +++ b/infrastructure/vars.tf @@ -553,7 +553,7 @@ variable "create_email_sender_instance" { variable "email_sender_instance_type" { description = "Instance type of the email sender instance." - type = bool + type = string default = false } From c76bc5c81c629881fd781096fd7c8527d1aae8d6 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 30 Jan 2025 14:37:48 -0500 Subject: [PATCH 11/16] Fix linter --- infrastructure/emailSenderConnect.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/infrastructure/emailSenderConnect.sh b/infrastructure/emailSenderConnect.sh index 067b2f00f..ede27ee03 100755 --- a/infrastructure/emailSenderConnect.sh +++ b/infrastructure/emailSenderConnect.sh @@ -41,14 +41,12 @@ function start_instance() { # Inject SSH Public Key using EC2 Instance Connect function send_ssh_public_key() { log_info "Sending SSH public key..." - aws ec2-instance-connect send-ssh-public-key \ + if ! aws ec2-instance-connect send-ssh-public-key \ --instance-id "$INSTANCE_ID" \ --availability-zone "$AVAILABILITY_ZONE" \ --instance-os-user "$SSH_USER" \ --ssh-public-key "file://$SSH_KEY_PATH" \ - --profile "$AWS_PROFILE" - - if [[ $? -ne 0 ]]; then + --profile "$AWS_PROFILE"; then log_error "Failed to send SSH public key." exit 1 fi From 7789222c8a15a40d85e87b31cbe69d5ef3146091 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 30 Jan 2025 14:40:37 -0500 Subject: [PATCH 12/16] Fix terraform typo --- infrastructure/email-sender.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/email-sender.tf b/infrastructure/email-sender.tf index 756aae9be..51645376d 100644 --- a/infrastructure/email-sender.tf +++ b/infrastructure/email-sender.tf @@ -40,7 +40,7 @@ resource "aws_iam_policy_attachment" "email_sender_ec2_policy_1" { } resource "aws_iam_policy_attachment" "email_sender_ec2_policy_2" { - count = var.create_db_accessor_instance ? 1 : 0 + count = var.create_email_sender_instance ? 1 : 0 name = "crossfeed-email-sender-${var.stage}" roles = [aws_iam_role.email_sender[0].id] policy_arn = "arn:${var.aws_partition}:iam::aws:policy/service-role/AmazonEC2RoleforSSM" From 70d0ecc4692221f266cc8086ac3d93cb567af9de Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 30 Jan 2025 15:36:13 -0500 Subject: [PATCH 13/16] Fix aws_instance definition --- infrastructure/email-sender.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/email-sender.tf b/infrastructure/email-sender.tf index 51645376d..7b189a2b3 100644 --- a/infrastructure/email-sender.tf +++ b/infrastructure/email-sender.tf @@ -54,7 +54,7 @@ resource "aws_instance" "email_sender" { associate_public_ip_address = false depends_on = [ - aws_iam_instance_profile.email_sender[0], + aws_iam_instance_profile.email_sender, aws_security_group.allow_internal, aws_subnet.backend ] @@ -72,7 +72,7 @@ resource "aws_instance" "email_sender" { vpc_security_group_ids = [var.is_dmz ? aws_security_group.allow_internal[0].id : aws_security_group.allow_internal_lz[0].id] subnet_id = var.is_dmz ? aws_subnet.backend[0].id : data.aws_ssm_parameter.subnet_db_1_id[0].value - iam_instance_profile = aws_iam_instance_profile.email_sender[0].name + iam_instance_profile = aws_iam_instance_profile.email_sender[0].id user_data = file("./email-sender-install.sh") lifecycle { From ae78d8e5aeefb6b570697d476a2fa268516ca40c Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 30 Jan 2025 15:48:47 -0500 Subject: [PATCH 14/16] Ensure profile exists when creating EC2 --- infrastructure/email-sender.tf | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/infrastructure/email-sender.tf b/infrastructure/email-sender.tf index 7b189a2b3..78f49871a 100644 --- a/infrastructure/email-sender.tf +++ b/infrastructure/email-sender.tf @@ -54,7 +54,10 @@ resource "aws_instance" "email_sender" { associate_public_ip_address = false depends_on = [ + aws_iam_role.email_sender, aws_iam_instance_profile.email_sender, + aws_iam_policy_attachment.email_sender_ec2_policy_1, + aws_iam_policy_attachment.email_sender_ec2_policy_2, aws_security_group.allow_internal, aws_subnet.backend ] @@ -65,6 +68,7 @@ resource "aws_instance" "email_sender" { Name = "email_sender" Owner = "Crossfeed managed resource" } + root_block_device { volume_size = 50 } @@ -72,11 +76,10 @@ resource "aws_instance" "email_sender" { vpc_security_group_ids = [var.is_dmz ? aws_security_group.allow_internal[0].id : aws_security_group.allow_internal_lz[0].id] subnet_id = var.is_dmz ? aws_subnet.backend[0].id : data.aws_ssm_parameter.subnet_db_1_id[0].value - iam_instance_profile = aws_iam_instance_profile.email_sender[0].id + iam_instance_profile = var.create_email_sender_instance ? aws_iam_instance_profile.email_sender[0].id : null user_data = file("./email-sender-install.sh") lifecycle { ignore_changes = [ami] } - } From 20e1bce9f64d55033bda1fc6376119845157f679 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 30 Jan 2025 16:20:40 -0500 Subject: [PATCH 15/16] Update the ami to differentiate DMZ/LZ --- infrastructure/database.tf | 2 +- infrastructure/email-sender.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/database.tf b/infrastructure/database.tf index 26f6615f8..06f8cd85e 100644 --- a/infrastructure/database.tf +++ b/infrastructure/database.tf @@ -192,7 +192,7 @@ resource "aws_iam_role_policy" "sqs_send_message_policy" { resource "aws_instance" "db_accessor" { count = var.create_db_accessor_instance ? 1 : 0 - ami = var.ami_id + ami = var.is_dmz ? data.aws_ami.ubuntu.id : var.ami_id instance_type = var.db_accessor_instance_class associate_public_ip_address = false diff --git a/infrastructure/email-sender.tf b/infrastructure/email-sender.tf index 78f49871a..3e7c8248e 100644 --- a/infrastructure/email-sender.tf +++ b/infrastructure/email-sender.tf @@ -49,7 +49,7 @@ resource "aws_iam_policy_attachment" "email_sender_ec2_policy_2" { # EC2 Instance for SES resource "aws_instance" "email_sender" { count = var.create_email_sender_instance ? 1 : 0 - ami = var.ami_id + ami = var.is_dmz ? data.aws_ami.ubuntu.id : var.ami_id instance_type = var.email_sender_instance_type associate_public_ip_address = false From 371366b9a023adf4e8519e2b6b5bdcca8c48e42f Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 30 Jan 2025 16:23:31 -0500 Subject: [PATCH 16/16] Add count to ubuntu ami call --- infrastructure/database.tf | 2 +- infrastructure/email-sender.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/database.tf b/infrastructure/database.tf index 06f8cd85e..80f7a3e3b 100644 --- a/infrastructure/database.tf +++ b/infrastructure/database.tf @@ -192,7 +192,7 @@ resource "aws_iam_role_policy" "sqs_send_message_policy" { resource "aws_instance" "db_accessor" { count = var.create_db_accessor_instance ? 1 : 0 - ami = var.is_dmz ? data.aws_ami.ubuntu.id : var.ami_id + ami = var.is_dmz ? data.aws_ami.ubuntu[0].id : var.ami_id instance_type = var.db_accessor_instance_class associate_public_ip_address = false diff --git a/infrastructure/email-sender.tf b/infrastructure/email-sender.tf index 3e7c8248e..0fea391af 100644 --- a/infrastructure/email-sender.tf +++ b/infrastructure/email-sender.tf @@ -49,7 +49,7 @@ resource "aws_iam_policy_attachment" "email_sender_ec2_policy_2" { # EC2 Instance for SES resource "aws_instance" "email_sender" { count = var.create_email_sender_instance ? 1 : 0 - ami = var.is_dmz ? data.aws_ami.ubuntu.id : var.ami_id + ami = var.is_dmz ? data.aws_ami.ubuntu[0].id : var.ami_id instance_type = var.email_sender_instance_type associate_public_ip_address = false