Merge remote-tracking branch 'origin/develop' into matomo

backend/Dockerfile.pe

@@ -7,7 +7,7 @@ COPY ./package* ./
 
 COPY src ./src
 
-RUN apt update && apt install git zlib1g-dev
+RUN apt update && apt install -y git zlib1g-dev
 
 RUN apt-get update && apt-get install -y jq
 

backend/src/xfd_django/xfd_api/auth.py

@@ -5,6 +5,7 @@
 import hashlib
 from hashlib import sha256
 import os
+import re
 from typing import List, Optional
 from urllib.parse import urlencode
 import uuid
@@ -272,34 +273,38 @@ def get_current_active_user(
     if api_key:
         user = get_user_by_api_key(api_key)
     elif token:
-        try:
-            # Decode token in Authorization header to get user
-            payload = jwt.decode(token, JWT_SECRET, algorithms=[JWT_ALGORITHM])
-            user_id = payload.get("id")
-
-            if user_id is None:
-                print("No user ID found in token")
+        # Check if token is an API key
+        if re.match(r"^[A-Fa-f0-9]{32}$", token):
+            user = get_user_by_api_key(token)
+        else:
+            try:
+                # Decode token in Authorization header to get user
+                payload = jwt.decode(token, JWT_SECRET, algorithms=[JWT_ALGORITHM])
+                user_id = payload.get("id")
+
+                if user_id is None:
+                    print("No user ID found in token")
+                    raise HTTPException(
+                        status_code=status.HTTP_401_UNAUTHORIZED,
+                        detail="Invalid token",
+                        headers={"WWW-Authenticate": "Bearer"},
+                    )
+                # Fetch the user by ID from the database
+                user = User.objects.get(id=user_id)
+            except jwt.ExpiredSignatureError:
+                print("Token has expired")
+                raise HTTPException(
+                    status_code=status.HTTP_401_UNAUTHORIZED,
+                    detail="Token has expired",
+                    headers={"WWW-Authenticate": "Bearer"},
+                )
+            except jwt.InvalidTokenError:
+                print("Invalid token")
                 raise HTTPException(
                     status_code=status.HTTP_401_UNAUTHORIZED,
                     detail="Invalid token",
                     headers={"WWW-Authenticate": "Bearer"},
                 )
-            # Fetch the user by ID from the database
-            user = User.objects.get(id=user_id)
-        except jwt.ExpiredSignatureError:
-            print("Token has expired")
-            raise HTTPException(
-                status_code=status.HTTP_401_UNAUTHORIZED,
-                detail="Token has expired",
-                headers={"WWW-Authenticate": "Bearer"},
-            )
-        except jwt.InvalidTokenError:
-            print("Invalid token")
-            raise HTTPException(
-                status_code=status.HTTP_401_UNAUTHORIZED,
-                detail="Invalid token",
-                headers={"WWW-Authenticate": "Bearer"},
-            )
     else:
         raise HTTPException(
             status_code=status.HTTP_401_UNAUTHORIZED,

backend/worker/generate_config.sh

@@ -14,7 +14,7 @@ key1=${PE_SHODAN_API_KEYS}
 
 [pe_api]
 pe_api_key=${PE_API_KEY}
-pe_api_url=https://api.staging-cd.crossfeed.cyber.dhs.gov/pe/apiv1/
+pe_api_url=${PE_API_URL}
 cf_api_key=${CF_API_KEY}
 
 [staging]

infrastructure/database.tf

@@ -192,7 +192,7 @@ resource "aws_iam_role_policy" "sqs_send_message_policy" {
 
 resource "aws_instance" "db_accessor" {
   count                       = var.create_db_accessor_instance ? 1 : 0
-  ami                         = var.ami_id
+  ami                         = var.is_dmz ? data.aws_ami.ubuntu[0].id : var.ami_id
   instance_type               = var.db_accessor_instance_class
   associate_public_ip_address = false
 

infrastructure/email-sender-install.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Create temporary directory for SSM Agent installation
+sudo mkdir -p /tmp/ssm
+cd /tmp/ssm || return
+
+# Download and install the SSM Agent
+wget https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/debian_amd64/amazon-ssm-agent.deb
+sudo dpkg -i amazon-ssm-agent.deb
+sudo systemctl enable amazon-ssm-agent
+sudo systemctl start amazon-ssm-agent
+rm amazon-ssm-agent.deb
+
+# Update packages
+sudo apt-get update -y
+
+# Install Python3 and pip
+sudo apt-get install -y python3 python3-pip
+
+# Install necessary Python libraries
+pip3 install boto3 pandas
+
+# Create working directory for email script
+sudo mkdir -p /var/www/email_sender
+sudo chmod -R 755 /var/www/email_sender

infrastructure/email-sender.tf

@@ -0,0 +1,85 @@
+
+resource "aws_iam_role" "email_sender" {
+  count              = var.create_email_sender_instance ? 1 : 0
+  name               = "crossfeed-email-sender-${var.stage}"
+  assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": "sts:AssumeRole",
+      "Principal": {
+        "Service": "ec2.amazonaws.com"
+      },
+      "Effect": "Allow",
+      "Sid": ""
+    }
+  ]
+}
+EOF
+
+  tags = {
+    Project = var.project
+    Stage   = var.stage
+    Owner   = "Crossfeed managed resource"
+  }
+}
+
+resource "aws_iam_instance_profile" "email_sender" {
+  count = var.create_email_sender_instance ? 1 : 0
+  name  = "crossfeed-email-sender-${var.stage}"
+  role  = aws_iam_role.email_sender[0].id
+}
+
+# Attach Policies to the Email EC2 Role
+resource "aws_iam_policy_attachment" "email_sender_ec2_policy_1" {
+  count      = var.create_email_sender_instance ? 1 : 0
+  name       = "crossfeed-email-sender-${var.stage}"
+  roles      = [aws_iam_role.email_sender[0].id, "AmazonSSMRoleForInstancesQuickSetup"]
+  policy_arn = "arn:${var.aws_partition}:iam::aws:policy/AmazonSSMManagedInstanceCore"
+}
+
+resource "aws_iam_policy_attachment" "email_sender_ec2_policy_2" {
+  count      = var.create_email_sender_instance ? 1 : 0
+  name       = "crossfeed-email-sender-${var.stage}"
+  roles      = [aws_iam_role.email_sender[0].id]
+  policy_arn = "arn:${var.aws_partition}:iam::aws:policy/service-role/AmazonEC2RoleforSSM"
+}
+
+# EC2 Instance for SES
+resource "aws_instance" "email_sender" {
+  count                       = var.create_email_sender_instance ? 1 : 0
+  ami                         = var.is_dmz ? data.aws_ami.ubuntu[0].id : var.ami_id
+  instance_type               = var.email_sender_instance_type
+  associate_public_ip_address = false
+
+  depends_on = [
+    aws_iam_role.email_sender,
+    aws_iam_instance_profile.email_sender,
+    aws_iam_policy_attachment.email_sender_ec2_policy_1,
+    aws_iam_policy_attachment.email_sender_ec2_policy_2,
+    aws_security_group.allow_internal,
+    aws_subnet.backend
+  ]
+
+  tags = {
+    Project = var.project
+    Stage   = var.stage
+    Name    = "email_sender"
+    Owner   = "Crossfeed managed resource"
+  }
+
+  root_block_device {
+    volume_size = 50
+  }
+
+  vpc_security_group_ids = [var.is_dmz ? aws_security_group.allow_internal[0].id : aws_security_group.allow_internal_lz[0].id]
+  subnet_id              = var.is_dmz ? aws_subnet.backend[0].id : data.aws_ssm_parameter.subnet_db_1_id[0].value
+
+  iam_instance_profile = var.create_email_sender_instance ? aws_iam_instance_profile.email_sender[0].id : null
+  user_data            = file("./email-sender-install.sh")
+
+  lifecycle {
+    ignore_changes = [ami]
+  }
+}

infrastructure/emailSenderConnect.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+# Configuration
+AWS_PROFILE=${EMAIL_AWS_PROFILE:-"default"}
+INSTANCE_ID=${EMAIL_SENDER_INSTANCE_ID:-"your-instance-id"}
+AVAILABILITY_ZONE="us-east-1b"
+LOCAL_PORT=9995
+REMOTE_PORT=22
+SSH_USER="ubuntu"
+SSH_KEY_PATH=${EMAIL_SSH_KEY_PATH:-""}
+
+function log_info() {
+  echo "$(date '+%Y-%m-%d %H:%M:%S') INFO: $1"
+}
+
+function log_error() {
+  echo "$(date '+%Y-%m-%d %H:%M:%S') ERROR: $1" >&2
+}
+
+# Check if the instance is running
+function get_instance_status() {
+  aws ec2 describe-instance-status \
+    --instance-ids "$INSTANCE_ID" \
+    --profile "$AWS_PROFILE" \
+    --query 'InstanceStatuses[0].InstanceState.Name' \
+    --output text 2> /dev/null
+}
+
+# Start the instance if it's not running
+function start_instance() {
+  log_info "Starting instance $INSTANCE_ID..."
+  aws ec2 start-instances \
+    --instance-ids "$INSTANCE_ID" \
+    --profile "$AWS_PROFILE" \
+    > /dev/null
+
+  log_info "Instance started. Waiting for initialization (2 minutes)..."
+  sleep 120
+}
+
+# Inject SSH Public Key using EC2 Instance Connect
+function send_ssh_public_key() {
+  log_info "Sending SSH public key..."
+  if ! aws ec2-instance-connect send-ssh-public-key \
+    --instance-id "$INSTANCE_ID" \
+    --availability-zone "$AVAILABILITY_ZONE" \
+    --instance-os-user "$SSH_USER" \
+    --ssh-public-key "file://$SSH_KEY_PATH" \
+    --profile "$AWS_PROFILE"; then
+    log_error "Failed to send SSH public key."
+    exit 1
+  fi
+}
+
+# Start port forwarding with AWS SSM
+function start_port_forwarding() {
+  log_info "Starting port forwarding via SSM..."
+  aws ssm start-session \
+    --target "$INSTANCE_ID" \
+    --document-name AWS-StartPortForwardingSession \
+    --parameters "{\"portNumber\":[\"$REMOTE_PORT\"], \"localPortNumber\":[\"$LOCAL_PORT\"]}" \
+    --profile "$AWS_PROFILE"
+}
+
+# Main script logic
+log_info "Starting EC2 connection process..."
+if [ -z "$INSTANCE_ID" ]; then
+  log_error "INSTANCE_ID is not set. Please set it as an environment variable or update the script."
+  exit 1
+fi
+
+STATUS=$(get_instance_status | tr -d '\r')
+
+log_info "Current instance status: $STATUS"
+
+if [[ "$STATUS" == "running" ]]; then
+  log_info "Instance is already running."
+elif [[ "$STATUS" == "stopped" || "$STATUS" == "stopping" ]]; then
+  start_instance
+else
+  log_error "Unexpected instance status: $STATUS"
+  exit 1
+fi
+
+send_ssh_public_key
+start_port_forwarding

infrastructure/integration.tfvars

@@ -57,6 +57,7 @@ ssm_sixgill_client_secret            = "/crossfeed/integration/SIXGILL_CLIENT_SE
 ssm_lg_api_key                       = "/crossfeed/integration/LG_API_KEY"
 ssm_lg_workspace_name                = "/crossfeed/integration/LG_WORKSPACE_NAME"
 ssm_pe_api_key                       = "/crossfeed/integration/PE_API_KEY"
+ssm_pe_api_url                       = "/crossfeed/integration/PE_API_URL"
 ssm_cf_api_key                       = "/crossfeed/integration/CF_API_KEY"
 db_group_name                        = "crossfeed-integration-db-group"
 worker_ecs_repository_name           = "crossfeed-integration-worker"
@@ -107,3 +108,5 @@ ssm_redshift_user                    = "/crossfeed/integration/REDSHIFT_USER"
 ssm_redshift_password                = "/crossfeed/integration/REDSHIFT_PASSWORD"
 create_elasticache_cluster           = true
 matomo_availability_zone             = "us-east-1a"
+create_email_sender_instance         = false
+email_sender_instance_type           = "t3.small"

infrastructure/pe_worker.tf

@@ -142,6 +142,10 @@ resource "aws_ecs_task_definition" "pe_worker" {
         "name": "PE_API_KEY",
         "valueFrom": "${data.aws_ssm_parameter.pe_api_key.arn}"
       },
+      {
+        "name": "PE_API_URL",
+        "valueFrom": "${data.aws_ssm_parameter.pe_api_url.arn}"
+      },
       {
         "name": "CF_API_KEY",
         "valueFrom": "${data.aws_ssm_parameter.cf_api_key.arn}"

infrastructure/prod.tfvars

@@ -117,7 +117,10 @@ ssm_redshift_database             = "/crossfeed/prod/REDSHIFT_DATABASE"
 ssm_redshift_user                 = "/crossfeed/prod/REDSHIFT_USER"
 ssm_redshift_password             = "/crossfeed/prod/REDSHIFT_PASSWORD"
 ssm_pe_api_key                    = "/crossfeed/prod/PE_API_KEY"
+ssm_pe_api_url                    = "/crossfeed/prod/PE_API_URL"
 ssm_cf_api_key                    = "/crossfeed/prod/CF_API_KEY"
 ssm_intelx_api_key                = "/crossfeed/prod/INTELX_API_KEY"
 ssm_xpanse_api_key                = "/crossfeed/prod/XPANSE_API_KEY"
 ssm_xpanse_auth_id                = "/crossfeed/prod/XPANSE_AUTH_ID"
+create_email_sender_instance      = false
+email_sender_instance_type        = "t3.small"

infrastructure/stage-cd.tfvars

@@ -59,6 +59,7 @@ ssm_xpanse_auth_id                   = "/crossfeed/staging/XPANSE_AUTH_ID"
 ssm_lg_api_key                       = "/crossfeed/staging/LG_API_KEY"
 ssm_lg_workspace_name                = "/crossfeed/staging/LG_WORKSPACE_NAME"
 ssm_pe_api_key                       = "/crossfeed/staging/PE_API_KEY"
+ssm_pe_api_url                       = "/crossfeed/staging/PE_API_URL"
 ssm_cf_api_key                       = "/crossfeed/staging/CF_API_KEY"
 db_group_name                        = "crossfeed-staging-db-group"
 worker_ecs_repository_name           = "crossfeed-staging-worker"
@@ -110,3 +111,5 @@ ssm_redshift_database                = "/crossfeed/staging/REDSHIFT_DATABASE"
 ssm_redshift_user                    = "/crossfeed/staging/REDSHIFT_USER"
 ssm_redshift_password                = "/crossfeed/staging/REDSHIFT_PASSWORD"
 create_elasticache_cluster           = true
+create_email_sender_instance         = true
+email_sender_instance_type           = "t3.small"

infrastructure/stage.tfvars

@@ -117,5 +117,8 @@ ssm_redshift_database             = "/crossfeed/staging/REDSHIFT_DATABASE"
 ssm_redshift_user                 = "/crossfeed/staging/REDSHIFT_USER"
 ssm_redshift_password             = "/crossfeed/staging/REDSHIFT_PASSWORD"
 ssm_pe_api_key                    = "/crossfeed/staging/PE_API_KEY"
+ssm_pe_api_url                    = "/crossfeed/staging/PE_API_URL"
 ssm_cf_api_key                    = "/crossfeed/staging/CF_API_KEY"
 create_elasticache_cluster        = true
+create_email_sender_instance      = false
+email_sender_instance_type        = "t3.small"

infrastructure/vars.tf

@@ -545,6 +545,18 @@ variable "create_db_accessor_instance" {
   default     = false
 }
 
+variable "create_email_sender_instance" {
+  description = "Whether to create a email sending EC2 instance. This instance can be used to access AWS SES and is spun up in a private subnet. It can be accessed using AWS Systems Manager Session Manager."
+  type        = bool
+  default     = false
+}
+
+variable "email_sender_instance_type" {
+  description = "Instance type of the email sender instance."
+  type        = string
+  default     = false
+}
+
 variable "db_accessor_instance_class" {
   description = "db_accessor_instance_class"
   type        = string
@@ -711,6 +723,12 @@ variable "ssm_pe_api_key" {
   default     = "/crossfeed/staging/PE_API_KEY"
 }
 
+variable "ssm_pe_api_url" {
+  description = "ssm_pe_api_url"
+  type        = string
+  default     = "/crossfeed/staging/PE_API_URL"
+}
+
 variable "ssm_cf_api_key" {
   description = "ssm_cf_api_key"
   type        = string