diff --git a/backend/Dockerfile.pe b/backend/Dockerfile.pe index d40a383fd..100921e99 100644 --- a/backend/Dockerfile.pe +++ b/backend/Dockerfile.pe @@ -7,7 +7,7 @@ COPY ./package* ./ COPY src ./src -RUN apt update && apt install git zlib1g-dev +RUN apt update && apt install -y git zlib1g-dev RUN apt-get update && apt-get install -y jq diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index 3dc53dd5b..4d92d7b45 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -5,6 +5,7 @@ import hashlib from hashlib import sha256 import os +import re from typing import List, Optional from urllib.parse import urlencode import uuid @@ -272,34 +273,38 @@ def get_current_active_user( if api_key: user = get_user_by_api_key(api_key) elif token: - try: - # Decode token in Authorization header to get user - payload = jwt.decode(token, JWT_SECRET, algorithms=[JWT_ALGORITHM]) - user_id = payload.get("id") - - if user_id is None: - print("No user ID found in token") + # Check if token is an API key + if re.match(r"^[A-Fa-f0-9]{32}$", token): + user = get_user_by_api_key(token) + else: + try: + # Decode token in Authorization header to get user + payload = jwt.decode(token, JWT_SECRET, algorithms=[JWT_ALGORITHM]) + user_id = payload.get("id") + + if user_id is None: + print("No user ID found in token") + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid token", + headers={"WWW-Authenticate": "Bearer"}, + ) + # Fetch the user by ID from the database + user = User.objects.get(id=user_id) + except jwt.ExpiredSignatureError: + print("Token has expired") + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Token has expired", + headers={"WWW-Authenticate": "Bearer"}, + ) + except jwt.InvalidTokenError: + print("Invalid token") raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid token", headers={"WWW-Authenticate": "Bearer"}, ) - # Fetch the user by ID from the database - user = User.objects.get(id=user_id) - except jwt.ExpiredSignatureError: - print("Token has expired") - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Token has expired", - headers={"WWW-Authenticate": "Bearer"}, - ) - except jwt.InvalidTokenError: - print("Invalid token") - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Invalid token", - headers={"WWW-Authenticate": "Bearer"}, - ) else: raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, diff --git a/backend/worker/generate_config.sh b/backend/worker/generate_config.sh index dbec44545..89c9e7367 100755 --- a/backend/worker/generate_config.sh +++ b/backend/worker/generate_config.sh @@ -14,7 +14,7 @@ key1=${PE_SHODAN_API_KEYS} [pe_api] pe_api_key=${PE_API_KEY} -pe_api_url=https://api.staging-cd.crossfeed.cyber.dhs.gov/pe/apiv1/ +pe_api_url=${PE_API_URL} cf_api_key=${CF_API_KEY} [staging] diff --git a/infrastructure/database.tf b/infrastructure/database.tf index 26f6615f8..80f7a3e3b 100644 --- a/infrastructure/database.tf +++ b/infrastructure/database.tf @@ -192,7 +192,7 @@ resource "aws_iam_role_policy" "sqs_send_message_policy" { resource "aws_instance" "db_accessor" { count = var.create_db_accessor_instance ? 1 : 0 - ami = var.ami_id + ami = var.is_dmz ? data.aws_ami.ubuntu[0].id : var.ami_id instance_type = var.db_accessor_instance_class associate_public_ip_address = false diff --git a/infrastructure/email-sender-install.sh b/infrastructure/email-sender-install.sh new file mode 100644 index 000000000..efc695cea --- /dev/null +++ b/infrastructure/email-sender-install.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Create temporary directory for SSM Agent installation +sudo mkdir -p /tmp/ssm +cd /tmp/ssm || return + +# Download and install the SSM Agent +wget https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/debian_amd64/amazon-ssm-agent.deb +sudo dpkg -i amazon-ssm-agent.deb +sudo systemctl enable amazon-ssm-agent +sudo systemctl start amazon-ssm-agent +rm amazon-ssm-agent.deb + +# Update packages +sudo apt-get update -y + +# Install Python3 and pip +sudo apt-get install -y python3 python3-pip + +# Install necessary Python libraries +pip3 install boto3 pandas + +# Create working directory for email script +sudo mkdir -p /var/www/email_sender +sudo chmod -R 755 /var/www/email_sender diff --git a/infrastructure/email-sender.tf b/infrastructure/email-sender.tf new file mode 100644 index 000000000..0fea391af --- /dev/null +++ b/infrastructure/email-sender.tf @@ -0,0 +1,85 @@ + +resource "aws_iam_role" "email_sender" { + count = var.create_email_sender_instance ? 1 : 0 + name = "crossfeed-email-sender-${var.stage}" + assume_role_policy = <&2 +} + +# Check if the instance is running +function get_instance_status() { + aws ec2 describe-instance-status \ + --instance-ids "$INSTANCE_ID" \ + --profile "$AWS_PROFILE" \ + --query 'InstanceStatuses[0].InstanceState.Name' \ + --output text 2> /dev/null +} + +# Start the instance if it's not running +function start_instance() { + log_info "Starting instance $INSTANCE_ID..." + aws ec2 start-instances \ + --instance-ids "$INSTANCE_ID" \ + --profile "$AWS_PROFILE" \ + > /dev/null + + log_info "Instance started. Waiting for initialization (2 minutes)..." + sleep 120 +} + +# Inject SSH Public Key using EC2 Instance Connect +function send_ssh_public_key() { + log_info "Sending SSH public key..." + if ! aws ec2-instance-connect send-ssh-public-key \ + --instance-id "$INSTANCE_ID" \ + --availability-zone "$AVAILABILITY_ZONE" \ + --instance-os-user "$SSH_USER" \ + --ssh-public-key "file://$SSH_KEY_PATH" \ + --profile "$AWS_PROFILE"; then + log_error "Failed to send SSH public key." + exit 1 + fi +} + +# Start port forwarding with AWS SSM +function start_port_forwarding() { + log_info "Starting port forwarding via SSM..." + aws ssm start-session \ + --target "$INSTANCE_ID" \ + --document-name AWS-StartPortForwardingSession \ + --parameters "{\"portNumber\":[\"$REMOTE_PORT\"], \"localPortNumber\":[\"$LOCAL_PORT\"]}" \ + --profile "$AWS_PROFILE" +} + +# Main script logic +log_info "Starting EC2 connection process..." +if [ -z "$INSTANCE_ID" ]; then + log_error "INSTANCE_ID is not set. Please set it as an environment variable or update the script." + exit 1 +fi + +STATUS=$(get_instance_status | tr -d '\r') + +log_info "Current instance status: $STATUS" + +if [[ "$STATUS" == "running" ]]; then + log_info "Instance is already running." +elif [[ "$STATUS" == "stopped" || "$STATUS" == "stopping" ]]; then + start_instance +else + log_error "Unexpected instance status: $STATUS" + exit 1 +fi + +send_ssh_public_key +start_port_forwarding diff --git a/infrastructure/integration.tfvars b/infrastructure/integration.tfvars index 3d181e967..d244cd0ea 100644 --- a/infrastructure/integration.tfvars +++ b/infrastructure/integration.tfvars @@ -57,6 +57,7 @@ ssm_sixgill_client_secret = "/crossfeed/integration/SIXGILL_CLIENT_SE ssm_lg_api_key = "/crossfeed/integration/LG_API_KEY" ssm_lg_workspace_name = "/crossfeed/integration/LG_WORKSPACE_NAME" ssm_pe_api_key = "/crossfeed/integration/PE_API_KEY" +ssm_pe_api_url = "/crossfeed/integration/PE_API_URL" ssm_cf_api_key = "/crossfeed/integration/CF_API_KEY" db_group_name = "crossfeed-integration-db-group" worker_ecs_repository_name = "crossfeed-integration-worker" @@ -107,3 +108,5 @@ ssm_redshift_user = "/crossfeed/integration/REDSHIFT_USER" ssm_redshift_password = "/crossfeed/integration/REDSHIFT_PASSWORD" create_elasticache_cluster = true matomo_availability_zone = "us-east-1a" +create_email_sender_instance = false +email_sender_instance_type = "t3.small" diff --git a/infrastructure/pe_worker.tf b/infrastructure/pe_worker.tf index 80e8d50ec..443839d68 100644 --- a/infrastructure/pe_worker.tf +++ b/infrastructure/pe_worker.tf @@ -142,6 +142,10 @@ resource "aws_ecs_task_definition" "pe_worker" { "name": "PE_API_KEY", "valueFrom": "${data.aws_ssm_parameter.pe_api_key.arn}" }, + { + "name": "PE_API_URL", + "valueFrom": "${data.aws_ssm_parameter.pe_api_url.arn}" + }, { "name": "CF_API_KEY", "valueFrom": "${data.aws_ssm_parameter.cf_api_key.arn}" diff --git a/infrastructure/prod.tfvars b/infrastructure/prod.tfvars index aa7d9b9dd..7843debca 100644 --- a/infrastructure/prod.tfvars +++ b/infrastructure/prod.tfvars @@ -117,7 +117,10 @@ ssm_redshift_database = "/crossfeed/prod/REDSHIFT_DATABASE" ssm_redshift_user = "/crossfeed/prod/REDSHIFT_USER" ssm_redshift_password = "/crossfeed/prod/REDSHIFT_PASSWORD" ssm_pe_api_key = "/crossfeed/prod/PE_API_KEY" +ssm_pe_api_url = "/crossfeed/prod/PE_API_URL" ssm_cf_api_key = "/crossfeed/prod/CF_API_KEY" ssm_intelx_api_key = "/crossfeed/prod/INTELX_API_KEY" ssm_xpanse_api_key = "/crossfeed/prod/XPANSE_API_KEY" ssm_xpanse_auth_id = "/crossfeed/prod/XPANSE_AUTH_ID" +create_email_sender_instance = false +email_sender_instance_type = "t3.small" diff --git a/infrastructure/stage-cd.tfvars b/infrastructure/stage-cd.tfvars index 041c4b7f9..e87b5f25e 100644 --- a/infrastructure/stage-cd.tfvars +++ b/infrastructure/stage-cd.tfvars @@ -59,6 +59,7 @@ ssm_xpanse_auth_id = "/crossfeed/staging/XPANSE_AUTH_ID" ssm_lg_api_key = "/crossfeed/staging/LG_API_KEY" ssm_lg_workspace_name = "/crossfeed/staging/LG_WORKSPACE_NAME" ssm_pe_api_key = "/crossfeed/staging/PE_API_KEY" +ssm_pe_api_url = "/crossfeed/staging/PE_API_URL" ssm_cf_api_key = "/crossfeed/staging/CF_API_KEY" db_group_name = "crossfeed-staging-db-group" worker_ecs_repository_name = "crossfeed-staging-worker" @@ -110,3 +111,5 @@ ssm_redshift_database = "/crossfeed/staging/REDSHIFT_DATABASE" ssm_redshift_user = "/crossfeed/staging/REDSHIFT_USER" ssm_redshift_password = "/crossfeed/staging/REDSHIFT_PASSWORD" create_elasticache_cluster = true +create_email_sender_instance = true +email_sender_instance_type = "t3.small" diff --git a/infrastructure/stage.tfvars b/infrastructure/stage.tfvars index d7222e6e0..0e0604525 100644 --- a/infrastructure/stage.tfvars +++ b/infrastructure/stage.tfvars @@ -117,5 +117,8 @@ ssm_redshift_database = "/crossfeed/staging/REDSHIFT_DATABASE" ssm_redshift_user = "/crossfeed/staging/REDSHIFT_USER" ssm_redshift_password = "/crossfeed/staging/REDSHIFT_PASSWORD" ssm_pe_api_key = "/crossfeed/staging/PE_API_KEY" +ssm_pe_api_url = "/crossfeed/staging/PE_API_URL" ssm_cf_api_key = "/crossfeed/staging/CF_API_KEY" create_elasticache_cluster = true +create_email_sender_instance = false +email_sender_instance_type = "t3.small" diff --git a/infrastructure/vars.tf b/infrastructure/vars.tf index 1d7130b3a..f36312c37 100644 --- a/infrastructure/vars.tf +++ b/infrastructure/vars.tf @@ -545,6 +545,18 @@ variable "create_db_accessor_instance" { default = false } +variable "create_email_sender_instance" { + description = "Whether to create a email sending EC2 instance. This instance can be used to access AWS SES and is spun up in a private subnet. It can be accessed using AWS Systems Manager Session Manager." + type = bool + default = false +} + +variable "email_sender_instance_type" { + description = "Instance type of the email sender instance." + type = string + default = false +} + variable "db_accessor_instance_class" { description = "db_accessor_instance_class" type = string @@ -711,6 +723,12 @@ variable "ssm_pe_api_key" { default = "/crossfeed/staging/PE_API_KEY" } +variable "ssm_pe_api_url" { + description = "ssm_pe_api_url" + type = string + default = "/crossfeed/staging/PE_API_URL" +} + variable "ssm_cf_api_key" { description = "ssm_cf_api_key" type = string diff --git a/infrastructure/worker.tf b/infrastructure/worker.tf index b7907d278..6a6d55bd7 100644 --- a/infrastructure/worker.tf +++ b/infrastructure/worker.tf @@ -100,6 +100,7 @@ resource "aws_iam_role_policy" "worker_task_execution_role_policy" { "${data.aws_ssm_parameter.https_proxy.arn}", "${aws_ssm_parameter.es_endpoint.arn}", "${data.aws_ssm_parameter.pe_api_key.arn}", + "${data.aws_ssm_parameter.pe_api_url.arn}", "${data.aws_ssm_parameter.cf_api_key.arn}", "${data.aws_ssm_parameter.ssm_mdl_name.arn}", "${data.aws_ssm_parameter.ssm_mdl_username.arn}", @@ -332,6 +333,10 @@ resource "aws_ecs_task_definition" "worker" { "name": "PE_API_KEY", "valueFrom": "${data.aws_ssm_parameter.pe_api_key.arn}" }, + { + "name": "PE_API_URL", + "valueFrom": "${data.aws_ssm_parameter.pe_api_url.arn}" + }, { "name": "CF_API_KEY", "valueFrom": "${data.aws_ssm_parameter.cf_api_key.arn}" @@ -438,6 +443,8 @@ data "aws_ssm_parameter" "https_proxy" { name = var.ssm_https_proxy } data "aws_ssm_parameter" "pe_api_key" { name = var.ssm_pe_api_key } +data "aws_ssm_parameter" "pe_api_url" { name = var.ssm_pe_api_url } + data "aws_ssm_parameter" "cf_api_key" { name = var.ssm_cf_api_key } data "aws_ssm_parameter" "ssm_mdl_name" { name = var.ssm_mdl_name }