Terraform module for deploying ECS-based Remote Ingestion Executor

remote-ingestion-executor/data.tf

@@ -0,0 +1 @@
+data "aws_region" "current" {}

remote-ingestion-executor/example/main.tf

@@ -0,0 +1,78 @@
+locals {
+  datahub = {
+    url       = "https://<your-company>.acryl.io/gms"
+    queue_url = "https://sqs.us-east-1.amazonaws.com/111111111111/xxx"
+    queue_arn = "arn:aws:sqs:us-east-1:11111111111:xxx"
+  }
+}
+
+module "example" {
+  source = "../"
+
+  cluster_name = "remote-ingestion-executor-example"
+
+  create_tasks_iam_role = true
+  tasks_iam_role_policies = {
+    SQS_Policy = aws_iam_policy.sqs-policy.arn
+  }
+
+  create_task_exec_iam_role = true
+  task_exec_secret_arns = [
+    aws_secretsmanager_secret.datahub_access_token.arn,
+  ]
+
+  datahub = local.datahub
+
+  secrets = [
+    {
+      name      = "DATAHUB_ACCESS_TOKEN"
+      valueFrom = aws_secretsmanager_secret.datahub_access_token.arn
+    },
+  ]
+
+  subnet_ids = ["subnet-XXX"]
+
+  assign_public_ip = true
+
+  security_group_rules = {
+    egress_all = {
+      type        = "egress"
+      from_port   = 0
+      to_port     = 0
+      protocol    = "-1"
+      cidr_blocks = ["0.0.0.0/0"]
+    },
+  }
+}
+
+resource "aws_secretsmanager_secret" "datahub_access_token" {
+  name = "datahub_access_token"
+}
+
+resource "aws_secretsmanager_secret_version" "service_user" {
+  secret_id     = aws_secretsmanager_secret.datahub_access_token.id
+  secret_string = "XXX"
+}
+
+resource "aws_iam_policy" "sqs-policy" {
+  name   = "remote-ingestion-executor-example-sqs"
+  path   = "/"
+  policy = data.aws_iam_policy_document.sqs-policy-document.json
+}
+
+data "aws_iam_policy_document" "sqs-policy-document" {
+  statement {
+    sid = "Allow"
+
+    actions = [
+      "sqs:ChangeMessageVisibility",
+      "sqs:DeleteMessage",
+      "sqs:ReceiveMessage",
+      "sqs:GetQueueUrl",
+    ]
+
+    resources = [
+      local.datahub.queue_arn,
+    ]
+  }
+}

remote-ingestion-executor/main.tf

@@ -0,0 +1,86 @@
+module "ecs_cluster" {
+  source  = "terraform-aws-modules/ecs/aws//modules/cluster"
+  version = "5.4.0"
+
+  cluster_name          = var.cluster_name
+  cluster_configuration = var.cluster_configuration
+
+  tags = var.tags
+}
+
+module "ecs_service" {
+  source  = "terraform-aws-modules/ecs/aws//modules/service"
+  version = "5.4.0"
+
+  cluster_arn = module.ecs_cluster.arn
+  name        = var.service_name
+
+  create_tasks_iam_role   = var.create_tasks_iam_role
+  tasks_iam_role_arn      = var.tasks_iam_role_arn
+  tasks_iam_role_name     = var.tasks_iam_role_name
+  tasks_iam_role_policies = var.tasks_iam_role_policies
+
+  create_task_exec_iam_role = var.create_task_exec_iam_role
+  task_exec_iam_role_name   = var.task_exec_iam_role_name
+
+  create_task_exec_policy     = var.create_task_exec_policy
+  task_exec_iam_role_policies = var.task_exec_iam_role_policies
+  task_exec_ssm_param_arns    = var.task_exec_ssm_param_arns
+  task_exec_secret_arns       = var.task_exec_secret_arns
+
+  cpu           = var.cpu
+  memory        = var.memory
+  desired_count = var.desired_count
+  launch_type   = "FARGATE"
+
+  enable_execute_command   = var.enable_execute_command
+  requires_compatibilities = var.requires_compatibilities
+
+  subnet_ids           = var.subnet_ids
+  security_group_ids   = var.security_group_ids
+  security_group_rules = var.security_group_rules
+  assign_public_ip     = var.assign_public_ip
+
+  container_definitions = {
+    dh-sqs-remote-executor = {
+      cpu    = var.cpu
+      memory = var.memory
+      image  = format("%s:%s", var.datahub.image, var.datahub.image_tag)
+
+      network_mode = var.network_mode
+
+      port_mappings = [
+        {
+          containerPort = 80
+        }
+      ]
+
+      enable_cloudwatch_logging   = var.enable_cloudwatch_logging
+      create_cloudwatch_log_group = var.create_cloudwatch_log_group
+      log_configuration           = var.log_configuration
+
+      secrets = var.secrets
+
+      environment = concat(var.environment, [
+        {
+          name  = "DATAHUB_BASE_URL"
+          value = var.datahub.url
+        },
+        {
+          name  = "AWS_COMMAND_QUEUE_URL"
+          value = var.datahub.queue_url
+        },
+        {
+          name  = "EXECUTOR_ID"
+          value = var.datahub.executor_id
+        },
+        {
+          name  = "AWS_REGION"
+          value = data.aws_region.current.name
+        },
+      ])
+    }
+  }
+
+  tags = var.tags
+}

remote-ingestion-executor/variables.tf

@@ -0,0 +1,194 @@
+variable "datahub" {
+  description = "Acryl Executor configuration"
+  type = object({
+    # The container image
+    image     = optional(string, "795586375822.dkr.ecr.us-west-2.amazonaws.com/acryl-sqs-remote-executor")
+    image_tag = optional(string, "v0.0.3.9")
+    # Acryl DataHub URL: The URL for your DataHub instance, e.g. <your-company>.acryl.io/gms
+    url = string
+    # Unique Executor Id. Warning - do not change this without consulting with your Acryl rep
+    executor_id = optional(string, "remote")
+    # SQS Queue ARN
+    queue_url = string
+  })
+}
+
+variable "cluster_name" {
+  description = "Name of the cluster (up to 255 letters, numbers, hyphens, and underscores)"
+  type        = string
+  default     = ""
+}
+
+variable "cluster_configuration" {
+  description = "The execute command configuration for the cluster"
+  type        = any
+  default     = {}
+}
+
+variable "create_tasks_iam_role" {
+  description = "Determines whether the ECS tasks IAM role should be created"
+  type        = bool
+  default     = true
+}
+
+variable "tasks_iam_role_arn" {
+  description = "Existing IAM role ARN"
+  type        = string
+  default     = null
+}
+
+variable "tasks_iam_role_name" {
+  description = "Name to use on IAM role created"
+  type        = string
+  default     = null
+}
+
+variable "tasks_iam_role_policies" {
+  description = "Map of IAM role policy ARNs to attach to the IAM role"
+  type        = map(string)
+  default     = {}
+}
+
+variable "create_task_exec_iam_role" {
+  description = "Determines whether the ECS task definition IAM role should be created"
+  type        = bool
+  default     = false
+}
+
+variable "task_exec_iam_role_name" {
+  description = "Name to use on IAM role created"
+  type        = string
+  default     = null
+}
+
+variable "create_task_exec_policy" {
+  description = "Determines whether the ECS task definition IAM policy should be created. This includes permissions included in AmazonECSTaskExecutionRolePolicy as well as access to secrets and SSM parameters"
+  type        = bool
+  default     = true
+}
+
+variable "task_exec_iam_role_policies" {
+  description = "Map of IAM role policy ARNs to attach to the IAM role"
+  type        = map(string)
+  default     = {}
+}
+
+variable "task_exec_ssm_param_arns" {
+  description = "List of SSM parameter ARNs the task execution role will be permitted to get/read"
+  type        = list(string)
+  default     = []
+}
+
+variable "task_exec_secret_arns" {
+  description = "List of SecretsManager secret ARNs the task execution role will be permitted to get/read"
+  type        = list(string)
+  default     = []
+}
+
+variable "service_name" {
+  description = "Name of the service (up to 255 letters, numbers, hyphens, and underscores)"
+  type        = string
+  default     = "dh-sqs-remote-executor"
+}
+
+variable "cpu" {
+  description = "Number of cpu units used by the task"
+  type        = number
+  default     = 1024
+}
+
+variable "memory" {
+  description = "Amount (in MiB) of memory used by the task"
+  type        = number
+  default     = 2048
+}
+
+variable "network_mode" {
+  description = "Docker networking mode to use for the containers in the task"
+  type        = string
+  default     = "awsvpc"
+}
+
+variable "security_group_ids" {
+  description = "List of security groups to associate with the task"
+  type        = list(string)
+  default     = []
+}
+
+variable "security_group_rules" {
+  description = "Security group rules to add to the security group created"
+  type        = any
+  default     = {}
+}
+
+variable "subnet_ids" {
+  description = "List of subnets to associate with the task"
+  type        = list(string)
+  default     = []
+}
+
+variable "assign_public_ip" {
+  description = "Assign a public IP address to the ENI"
+  type        = bool
+  default     = true
+}
+
+variable "enable_cloudwatch_logging" {
+  description = "Determines whether CloudWatch logging is configured for the container definition"
+  type        = bool
+  default     = true
+}
+
+variable "create_cloudwatch_log_group" {
+  description = "Determines whether a log group is created by this module. If not, AWS will automatically create one if logging is enabled"
+  type        = bool
+  default     = true
+}
+
+variable "log_configuration" {
+  description = "The log configuration for the container"
+  type        = any
+  default     = {}
+}
+
+variable "secrets" {
+  description = "The secrets to pass to the container. For more information, see [Specifying Sensitive Data](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/specifying-sensitive-data.html) in the Amazon Elastic Container Service Developer Guide"
+  type = list(object({
+    name      = string
+    valueFrom = string
+  }))
+  default = []
+}
+
+variable "environment" {
+  description = "The environment variables to pass to the container"
+  type = list(object({
+    name  = string
+    value = string
+  }))
+  default = []
+}
+
+variable "desired_count" {
+  description = "Number of instances of the task definition to place and keep running"
+  type        = number
+  default     = 1
+}
+
+variable "enable_execute_command" {
+  description = "Specifies whether to enable Amazon ECS Exec for the tasks within the service"
+  type        = bool
+  default     = true
+}
+
+variable "requires_compatibilities" {
+  description = "Set of launch types required by the task"
+  type        = list(string)
+  default     = ["EC2", "FARGATE"]
+}
+
+variable "tags" {
+  description = "A map of tags to add to all resources"
+  type        = map(string)
+  default     = {}
+}