diff --git a/README.md b/README.md index 043de92..9bfd273 100644 --- a/README.md +++ b/README.md @@ -3,12 +3,13 @@ Terraform module designed to easily backup EFS filesystems to S3 using DataPipeline. The workflow is simple: + * Periodically launch resource (EC2 instance) based on schedule * Execute the shell command defined in the activity on the instance -* Execute sync data from Production EFS to S3 Bucket by aws-cli -* The execution log of the activity is stored in S3 -* Publish to the SNS topic that defined the success or failure of the activity -* Automatic backup rotation using `S3 lifecycle rule` +* Sync data from Production EFS to S3 Bucket by using `aws-cli` +* The execution log of the activity is stored in `S3` +* Publish the success or failure of the activity to an `SNS` topic +* Automatically rotate the backups using `S3 lifecycle rule` ## Usage @@ -22,10 +23,9 @@ module "efs_backup" { name = "${var.name}" stage = "${var.stage}" namespace = "${var.namespace}" - region = "${var.region}" vpc_id = "${var.vpc_id}" efs_mount_target_id = "${var.efs_mount_target_id}" - use_ip_address = "true" + use_ip_address = "false" noncurrent_version_expiration_days = "${var.noncurrent_version_expiration_days}" ssh_key_pair = "${var.ssh_key_pair}" datapipeline_config = "${var.datapipeline_config}" @@ -40,19 +40,22 @@ output "efs_backup_security_group" { ## Variables -| Name | Default | Description | Required | -|:-----------------------------------|:--------------:|:------------------------------------------------------------------------------------|:--------:| -| namespace | `` | Namespace (e.g. `cp` or `cloudposse`) | Yes | -| stage | `` | Stage (e.g. `prod`, `dev`, `staging`) | Yes | -| name | `` | Name (e.g. `efs-backup`) | Yes | -| region | `us-east-1` | AWS Region where module should operate (e.g. `us-east-1`) | Yes | -| vpc_id | `` | AWS VPC ID where module should operate (e.g. `vpc-a22222ee`) | Yes | -| efs_mount_target_id | `` | Elastic File System Mount Target ID (e.g. `fsmt-279bfc62`) | Yes | -| use_ip_address | `false` | If set to `true` will be used IP address instead DNS name of Elastic File System | Yes | -| modify_security_group | `false` | Should the module modify EFS security groups (if set to `false` backups will fail) | Yes | -| noncurrent_version_expiration_days | `3` | S3 object versions expiration period (days) | Yes | -| ssh_key_pair | `` | A ssh key that will be deployed on DataPipeline's instance | Yes | -| datapipeline_config | `${map("instance_type", "t2.micro", "email", "", "period", "24 hours")}"`| Essential Datapipeline configuration options | Yes | +| Name | Default | Description | Required | +|:-----------------------------------|:--------------:|:----------------------------------------------------------------------------------------------|:--------:| +| namespace | `` | Namespace (e.g. `cp` or `cloudposse`) | Yes | +| stage | `` | Stage (e.g. `prod`, `dev`, `staging`) | Yes | +| name | `` | Name (e.g. `app` or `wordpress`) | Yes | +| region | `us-east-1` | (Optional) AWS Region. If not specified, will be derived from 'aws_region' data source | No | +| vpc_id | `` | AWS VPC ID where module should operate (_e.g._ `vpc-a22222ee`) | Yes | +| efs_mount_target_id | `` | Elastic File System Mount Target ID (_e.g._ `fsmt-279bfc62`) | Yes | +| use_ip_address | `false` | If set to `true`, will use IP address instead of DNS name to connect to the `EFS` | Yes | +| modify_security_group | `false` | Should the module modify the `EFS` security group | No | +| noncurrent_version_expiration_days | `35` | S3 object versions expiration period (days) | Yes | +| ssh_key_pair | `` | `SSH` key that will be deployed on DataPipeline's instance | No | +| datapipeline_config | `${map("instance_type", "t2.micro", "email", "", "period", "24 hours", "timeout", "60 Minutes")}"`| DataPipeline configuration options | Yes | +| attributes | `[]` | Additional attributes (_e.g._ `efs-backup`) | No | +| tags | `{}` | Additional tags (e.g. `map("BusinessUnit","XYZ")` | No | +| delimiter | `-` | Delimiter to be used between `name`, `namespace`, `stage` and `attributes` | No | ### `datapipeline_config` variables @@ -60,16 +63,70 @@ output "efs_backup_security_group" { | Name | Default | Description | Required | |:-----------------------------------|:--------------:|:------------------------------------------------------------|:--------:| | instance_type | `t2.micro` | Instance type to use | Yes | -| email | `` | Email to use in SNS | Yes | +| email | `` | Email to use in `SNS` | Yes | | period | `24 hours` | Frequency of pipeline execution (frequency of backups) | Yes | +| timeout | `60 Minutes` | Pipeline execution timeout | Yes | + + + +## Integration with `EFS` + +To enable connectivity between the `DataPipeline` instances and the `EFS`, use one of the following methods to configure Security Groups: + +1. Explicitly add the `DataPipeline` SG (the output of this module `security_group_id`) to the list of the `ingress` rules of the `EFS` SG. For example: +```hcl +module "elastic_beanstalk_environment" { + source = "git::https://github.com/cloudposse/terraform-aws-elastic-beanstalk-environment.git?ref=master" + namespace = "${var.namespace}" + name = "${var.name}" + stage = "${var.stage}" + delimiter = "${var.delimiter}" + attributes = ["${compact(concat(var.attributes, list("eb-env")))}"] + tags = "${var.tags}" + + # .............................. +} +module "efs" { + source = "git::https://github.com/cloudposse/terraform-aws-efs.git?ref=tmaster" + namespace = "${var.namespace}" + name = "${var.name}" + stage = "${var.stage}" + delimiter = "${var.delimiter}" + attributes = ["${compact(concat(var.attributes, list("efs")))}"] + tags = "${var.tags}" + + # Allow EB/EC2 instances and DataPipeline instances to connect to the EFS + security_groups = ["${module.elastic_beanstalk_environment.security_group_id}", "${module.efs_backup.security_group_id}"] +} -## Integration with EFS +module "efs_backup" { + source = "git::https://github.com/cloudposse/terraform-aws-efs-backup.git?ref=master" + name = "${var.name}" + stage = "${var.stage}" + namespace = "${var.namespace}" + delimiter = "${var.delimiter}" + attributes = ["${compact(concat(var.attributes, list("efs-backup")))}"] + tags = "${var.tags}" + + # Important to set it to `false` since we added the `DataPipeline` SG (output of the `efs_backup` module) to the `security_groups` of the `efs` module + # See NOTE below for more information + modify_security_group = "false" + + # .............................. +} +``` -It's necessary to configure your EFS filesystem security groups to permit backups from the DataPipeline instances. +2. Set `modify_security_group` attribute to `true` so the module will modify the `EFS` SG to allow the `DataPipeline` to connect to the `EFS` -Add the security group ID from the `efs_backup_security_group` output to a security group of EFS Filesystems. +**NOTE:** Do not mix these two methods together. +`Terraform` does not support using a Security Group with in-line rules in conjunction with any Security Group Rule resources. +https://www.terraform.io/docs/providers/aws/r/security_group_rule.html +> NOTE on Security Groups and Security Group Rules: Terraform currently provides both a standalone Security Group Rule resource +(a single ingress or egress rule), and a Security Group resource with ingress and egress rules defined in-line. +At this time you cannot use a Security Group with in-line rules in conjunction with any Security Group Rule resources. +Doing so will cause a conflict of rule settings and will overwrite rules. ## References diff --git a/cloudformation.tf b/cloudformation.tf index f16b1eb..3e4e097 100644 --- a/cloudformation.tf +++ b/cloudformation.tf @@ -3,7 +3,9 @@ module "sns_label" { namespace = "${var.namespace}" stage = "${var.stage}" name = "${var.name}" - attributes = ["sns"] + delimiter = "${var.delimiter}" + attributes = ["${compact(concat(var.attributes, list("sns")))}"] + tags = "${var.tags}" } resource "aws_cloudformation_stack" "sns" { @@ -13,6 +15,8 @@ resource "aws_cloudformation_stack" "sns" { parameters { Email = "${var.datapipeline_config["email"]}" } + + tags = "${module.sns_label.tags}" } module "datapipeline_label" { @@ -20,7 +24,9 @@ module "datapipeline_label" { namespace = "${var.namespace}" stage = "${var.stage}" name = "${var.name}" - attributes = ["datapipeline"] + delimiter = "${var.delimiter}" + attributes = ["${compact(concat(var.attributes, list("datapipeline")))}"] + tags = "${var.tags}" } resource "aws_cloudformation_stack" "datapipeline" { @@ -31,9 +37,9 @@ resource "aws_cloudformation_stack" "datapipeline" { myInstanceType = "${var.datapipeline_config["instance_type"]}" mySubnetId = "${data.aws_subnet_ids.default.ids[0]}" mySecurityGroupId = "${aws_security_group.datapipeline.id}" - myEFSHost = "${var.use_ip_address ? data.aws_efs_mount_target.default.ip_address : data.aws_efs_mount_target.default.dns_name }" + myEFSHost = "${var.use_ip_address ? data.aws_efs_mount_target.default.ip_address : format("%s.efs.%s.amazonaws.com", data.aws_efs_mount_target.default.file_system_id, (signum(length(var.region)) == 1 ? var.region : data.aws_region.default.name))}" myS3BackupsBucket = "${aws_s3_bucket.backups.id}" - myRegion = "${var.region}" + myRegion = "${signum(length(var.region)) == 1 ? var.region : data.aws_region.default.name}" myImageId = "${data.aws_ami.amazon_linux.id}" myTopicArn = "${aws_cloudformation_stack.sns.outputs["TopicArn"]}" myS3LogBucket = "${aws_s3_bucket.logs.id}" @@ -44,4 +50,6 @@ resource "aws_cloudformation_stack" "datapipeline" { Tag = "${module.label.id}" myExecutionTimeout = "${var.datapipeline_config["timeout"]}" } + + tags = "${module.datapipeline_label.tags}" } diff --git a/efs.tf b/efs.tf index f6740eb..424c874 100644 --- a/efs.tf +++ b/efs.tf @@ -1,4 +1,4 @@ -# Get Elastic File System Mount Target (EFS) +# Get Elastic File System Mount Target data "aws_efs_mount_target" "default" { mount_target_id = "${var.efs_mount_target_id}" } diff --git a/iam.tf b/iam.tf index 9cc93a6..ecc37fa 100644 --- a/iam.tf +++ b/iam.tf @@ -16,7 +16,9 @@ module "resource_role_label" { namespace = "${var.namespace}" stage = "${var.stage}" name = "${var.name}" - attributes = ["resource-role"] + delimiter = "${var.delimiter}" + attributes = ["${compact(concat(var.attributes, list("resource-role")))}"] + tags = "${var.tags}" } resource "aws_iam_role" "resource_role" { @@ -56,7 +58,9 @@ module "role_label" { namespace = "${var.namespace}" stage = "${var.stage}" name = "${var.name}" - attributes = ["role"] + delimiter = "${var.delimiter}" + attributes = ["${compact(concat(var.attributes, list("role")))}"] + tags = "${var.tags}" } resource "aws_iam_role" "role" { diff --git a/main.tf b/main.tf index 2ea7abf..c773b8c 100644 --- a/main.tf +++ b/main.tf @@ -3,14 +3,21 @@ terraform { } provider "aws" { - region = "${var.region}" + region = "${signum(length(var.region)) == 1 ? var.region : data.aws_region.default.name}" +} + +data "aws_region" "default" { + current = true } module "label" { - source = "git::https://github.com/cloudposse/terraform-null-label.git?ref=tags/0.2.1" - namespace = "${var.namespace}" - stage = "${var.stage}" - name = "${var.name}" + source = "git::https://github.com/cloudposse/terraform-null-label.git?ref=tags/0.2.1" + namespace = "${var.namespace}" + stage = "${var.stage}" + name = "${var.name}" + delimiter = "${var.delimiter}" + attributes = "${var.attributes}" + tags = "${var.tags}" } data "aws_ami" "amazon_linux" { diff --git a/network.tf b/network.tf index 8af5654..fabfdcb 100644 --- a/network.tf +++ b/network.tf @@ -3,7 +3,7 @@ data "aws_vpc" "default" { id = "${var.vpc_id}" } -# Get all subnets from the necessary vpc +# Get all subnets from the VPC data "aws_subnet_ids" "default" { vpc_id = "${data.aws_vpc.default.id}" } diff --git a/s3.tf b/s3.tf index 24035bf..6e2383e 100644 --- a/s3.tf +++ b/s3.tf @@ -3,7 +3,9 @@ module "logs_label" { namespace = "${var.namespace}" stage = "${var.stage}" name = "${var.name}" - attributes = ["logs"] + delimiter = "${var.delimiter}" + attributes = ["${compact(concat(var.attributes, list("logs")))}"] + tags = "${var.tags}" } resource "aws_s3_bucket" "logs" { @@ -17,7 +19,9 @@ module "backups_label" { namespace = "${var.namespace}" stage = "${var.stage}" name = "${var.name}" - attributes = ["backups"] + delimiter = "${var.delimiter}" + attributes = ["${compact(concat(var.attributes, list("backups")))}"] + tags = "${var.tags}" } resource "aws_s3_bucket" "backups" { diff --git a/variables.tf b/variables.tf index 3a8c629..afddb21 100644 --- a/variables.tf +++ b/variables.tf @@ -1,12 +1,24 @@ -variable "name" {} +variable "name" { + type = "string" +} -variable "namespace" {} +variable "namespace" { + type = "string" +} -variable "stage" {} +variable "stage" { + type = "string" +} -variable "region" {} +variable "region" { + type = "string" + default = "" + description = "(Optional) AWS Region. If not specified, will be derived from 'aws_region' data source" +} -variable "vpc_id" {} +variable "vpc_id" { + type = "string" +} # https://www.terraform.io/docs/configuration/variables.html # simply using string values rather than booleans for variables is recommended @@ -25,15 +37,38 @@ variable "datapipeline_config" { } } -variable "efs_mount_target_id" {} +variable "efs_mount_target_id" { + type = "string" + description = "EFS Mount Target ID (e.g. `fsmt-279bfc62`)" +} variable "modify_security_group" { - default = false + default = "false" } # Set a name of ssh key that will be deployed on DataPipeline's instance. The key should be present in AWS. -variable "ssh_key_pair" {} +variable "ssh_key_pair" { + type = "string" +} variable "noncurrent_version_expiration_days" { default = "35" } + +variable "delimiter" { + type = "string" + default = "-" + description = "Delimiter to be used between `name`, `namespace`, `stage`, etc." +} + +variable "attributes" { + type = "list" + default = [] + description = "Additional attributes (e.g. `efs-backup`)" +} + +variable "tags" { + type = "map" + default = {} + description = "Additional tags (e.g. `map('BusinessUnit`,`XYZ`)" +}