From 0e91a1cce65c2a112047e989b2600d40e61ddec3 Mon Sep 17 00:00:00 2001 From: Davi Arnaut Date: Tue, 31 Oct 2023 15:20:11 -0700 Subject: [PATCH] Terraform module for deploying ECS-based Remote Ingestion Executor --- remote-ingestion-executor/data.tf | 1 + remote-ingestion-executor/example/main.tf | 78 +++++++++ remote-ingestion-executor/main.tf | 86 ++++++++++ remote-ingestion-executor/variables.tf | 194 ++++++++++++++++++++++ 4 files changed, 359 insertions(+) create mode 100644 remote-ingestion-executor/data.tf create mode 100644 remote-ingestion-executor/example/main.tf create mode 100644 remote-ingestion-executor/main.tf create mode 100644 remote-ingestion-executor/variables.tf diff --git a/remote-ingestion-executor/data.tf b/remote-ingestion-executor/data.tf new file mode 100644 index 0000000..2502393 --- /dev/null +++ b/remote-ingestion-executor/data.tf @@ -0,0 +1 @@ +data "aws_region" "current" {} diff --git a/remote-ingestion-executor/example/main.tf b/remote-ingestion-executor/example/main.tf new file mode 100644 index 0000000..16b757e --- /dev/null +++ b/remote-ingestion-executor/example/main.tf @@ -0,0 +1,78 @@ +locals { + datahub = { + url = "https://.acryl.io/gms" + queue_url = "https://sqs.us-east-1.amazonaws.com/111111111111/xxx" + queue_arn = "arn:aws:sqs:us-east-1:11111111111:xxx" + } +} + +module "example" { + source = "../" + + cluster_name = "remote-ingestion-executor-example" + + create_tasks_iam_role = true + tasks_iam_role_policies = { + SQS_Policy = aws_iam_policy.sqs-policy.arn + } + + create_task_exec_iam_role = true + task_exec_secret_arns = [ + aws_secretsmanager_secret.datahub_access_token.arn, + ] + + datahub = local.datahub + + secrets = [ + { + name = "DATAHUB_ACCESS_TOKEN" + valueFrom = aws_secretsmanager_secret.datahub_access_token.arn + }, + ] + + subnet_ids = ["subnet-XXX"] + + assign_public_ip = true + + security_group_rules = { + egress_all = { + type = "egress" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + }, + } +} + +resource "aws_secretsmanager_secret" "datahub_access_token" { + name = "datahub_access_token" +} + +resource "aws_secretsmanager_secret_version" "service_user" { + secret_id = aws_secretsmanager_secret.datahub_access_token.id + secret_string = "XXX" +} + +resource "aws_iam_policy" "sqs-policy" { + name = "remote-ingestion-executor-example-sqs" + path = "/" + policy = data.aws_iam_policy_document.sqs-policy-document.json +} + +data "aws_iam_policy_document" "sqs-policy-document" { + statement { + sid = "Allow" + + actions = [ + "sqs:ChangeMessageVisibility", + "sqs:DeleteMessage", + "sqs:ReceiveMessage", + "sqs:GetQueueUrl", + ] + + resources = [ + local.datahub.queue_arn, + ] + } +} diff --git a/remote-ingestion-executor/main.tf b/remote-ingestion-executor/main.tf new file mode 100644 index 0000000..8144f5b --- /dev/null +++ b/remote-ingestion-executor/main.tf @@ -0,0 +1,86 @@ +module "ecs_cluster" { + source = "terraform-aws-modules/ecs/aws//modules/cluster" + version = "5.4.0" + + cluster_name = var.cluster_name + cluster_configuration = var.cluster_configuration + + tags = var.tags +} + +module "ecs_service" { + source = "terraform-aws-modules/ecs/aws//modules/service" + version = "5.4.0" + + cluster_arn = module.ecs_cluster.arn + name = var.service_name + + create_tasks_iam_role = var.create_tasks_iam_role + tasks_iam_role_arn = var.tasks_iam_role_arn + tasks_iam_role_name = var.tasks_iam_role_name + tasks_iam_role_policies = var.tasks_iam_role_policies + + create_task_exec_iam_role = var.create_task_exec_iam_role + task_exec_iam_role_name = var.task_exec_iam_role_name + + create_task_exec_policy = var.create_task_exec_policy + task_exec_iam_role_policies = var.task_exec_iam_role_policies + task_exec_ssm_param_arns = var.task_exec_ssm_param_arns + task_exec_secret_arns = var.task_exec_secret_arns + + cpu = var.cpu + memory = var.memory + desired_count = var.desired_count + launch_type = "FARGATE" + + enable_execute_command = var.enable_execute_command + requires_compatibilities = var.requires_compatibilities + + subnet_ids = var.subnet_ids + security_group_ids = var.security_group_ids + security_group_rules = var.security_group_rules + assign_public_ip = var.assign_public_ip + + container_definitions = { + dh-sqs-remote-executor = { + cpu = var.cpu + memory = var.memory + image = format("%s:%s", var.datahub.image, var.datahub.image_tag) + + network_mode = var.network_mode + + port_mappings = [ + { + containerPort = 80 + } + ] + + enable_cloudwatch_logging = var.enable_cloudwatch_logging + create_cloudwatch_log_group = var.create_cloudwatch_log_group + log_configuration = var.log_configuration + + secrets = var.secrets + + environment = concat(var.environment, [ + { + name = "DATAHUB_BASE_URL" + value = var.datahub.url + }, + { + name = "AWS_COMMAND_QUEUE_URL" + value = var.datahub.queue_url + }, + { + name = "EXECUTOR_ID" + value = var.datahub.executor_id + }, + { + name = "AWS_REGION" + value = data.aws_region.current.name + }, + ]) + } + } + + tags = var.tags +} diff --git a/remote-ingestion-executor/variables.tf b/remote-ingestion-executor/variables.tf new file mode 100644 index 0000000..86c5ff1 --- /dev/null +++ b/remote-ingestion-executor/variables.tf @@ -0,0 +1,194 @@ +variable "datahub" { + description = "Acryl Executor configuration" + type = object({ + # The container image + image = optional(string, "795586375822.dkr.ecr.us-west-2.amazonaws.com/acryl-sqs-remote-executor") + image_tag = optional(string, "v0.0.3.9") + # Acryl DataHub URL: The URL for your DataHub instance, e.g. .acryl.io/gms + url = string + # Unique Executor Id. Warning - do not change this without consulting with your Acryl rep + executor_id = optional(string, "remote") + # SQS Queue ARN + queue_url = string + }) +} + +variable "cluster_name" { + description = "Name of the cluster (up to 255 letters, numbers, hyphens, and underscores)" + type = string + default = "" +} + +variable "cluster_configuration" { + description = "The execute command configuration for the cluster" + type = any + default = {} +} + +variable "create_tasks_iam_role" { + description = "Determines whether the ECS tasks IAM role should be created" + type = bool + default = true +} + +variable "tasks_iam_role_arn" { + description = "Existing IAM role ARN" + type = string + default = null +} + +variable "tasks_iam_role_name" { + description = "Name to use on IAM role created" + type = string + default = null +} + +variable "tasks_iam_role_policies" { + description = "Map of IAM role policy ARNs to attach to the IAM role" + type = map(string) + default = {} +} + +variable "create_task_exec_iam_role" { + description = "Determines whether the ECS task definition IAM role should be created" + type = bool + default = false +} + +variable "task_exec_iam_role_name" { + description = "Name to use on IAM role created" + type = string + default = null +} + +variable "create_task_exec_policy" { + description = "Determines whether the ECS task definition IAM policy should be created. This includes permissions included in AmazonECSTaskExecutionRolePolicy as well as access to secrets and SSM parameters" + type = bool + default = true +} + +variable "task_exec_iam_role_policies" { + description = "Map of IAM role policy ARNs to attach to the IAM role" + type = map(string) + default = {} +} + +variable "task_exec_ssm_param_arns" { + description = "List of SSM parameter ARNs the task execution role will be permitted to get/read" + type = list(string) + default = [] +} + +variable "task_exec_secret_arns" { + description = "List of SecretsManager secret ARNs the task execution role will be permitted to get/read" + type = list(string) + default = [] +} + +variable "service_name" { + description = "Name of the service (up to 255 letters, numbers, hyphens, and underscores)" + type = string + default = "dh-sqs-remote-executor" +} + +variable "cpu" { + description = "Number of cpu units used by the task" + type = number + default = 1024 +} + +variable "memory" { + description = "Amount (in MiB) of memory used by the task" + type = number + default = 2048 +} + +variable "network_mode" { + description = "Docker networking mode to use for the containers in the task" + type = string + default = "awsvpc" +} + +variable "security_group_ids" { + description = "List of security groups to associate with the task" + type = list(string) + default = [] +} + +variable "security_group_rules" { + description = "Security group rules to add to the security group created" + type = any + default = {} +} + +variable "subnet_ids" { + description = "List of subnets to associate with the task" + type = list(string) + default = [] +} + +variable "assign_public_ip" { + description = "Assign a public IP address to the ENI" + type = bool + default = true +} + +variable "enable_cloudwatch_logging" { + description = "Determines whether CloudWatch logging is configured for the container definition" + type = bool + default = true +} + +variable "create_cloudwatch_log_group" { + description = "Determines whether a log group is created by this module. If not, AWS will automatically create one if logging is enabled" + type = bool + default = true +} + +variable "log_configuration" { + description = "The log configuration for the container" + type = any + default = {} +} + +variable "secrets" { + description = "The secrets to pass to the container. For more information, see [Specifying Sensitive Data](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/specifying-sensitive-data.html) in the Amazon Elastic Container Service Developer Guide" + type = list(object({ + name = string + valueFrom = string + })) + default = [] +} + +variable "environment" { + description = "The environment variables to pass to the container" + type = list(object({ + name = string + value = string + })) + default = [] +} + +variable "desired_count" { + description = "Number of instances of the task definition to place and keep running" + type = number + default = 1 +} + +variable "enable_execute_command" { + description = "Specifies whether to enable Amazon ECS Exec for the tasks within the service" + type = bool + default = true +} + +variable "requires_compatibilities" { + description = "Set of launch types required by the task" + type = list(string) + default = ["EC2", "FARGATE"] +} + +variable "tags" { + description = "A map of tags to add to all resources" + type = map(string) + default = {} +}