From 8e75969983ce57a9a2e7c636f514631fa049ab88 Mon Sep 17 00:00:00 2001 From: Vadim Hleif Date: Tue, 17 Jul 2018 11:49:41 +0300 Subject: [PATCH] Migrate readme yaml (#26) * Migrate readme yaml --- .gitignore | 3 + .travis.yml | 10 +- Makefile | 6 +- README.md | 242 +++++++++++++++++++++++++++++++++++++++------- README.yaml | 153 +++++++++++++++++++++++++++++ docs/targets.md | 9 ++ docs/terraform.md | 29 ++++++ outputs.tf | 12 ++- variables.tf | 24 +++-- 9 files changed, 432 insertions(+), 56 deletions(-) create mode 100644 README.yaml create mode 100644 docs/targets.md create mode 100644 docs/terraform.md diff --git a/.gitignore b/.gitignore index e673710..788d15a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ secrets.tfvars .terraform .idea *.iml + +.build-harness +build-harness \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 0bca29a..241026e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,8 +9,8 @@ install: - make init script: - - make terraform:install - - make terraform:get-plugins - - make terraform:get-modules - - make terraform:lint - - make terraform:validate + - make terraform/install + - make terraform/get-plugins + - make terraform/get-modules + - make terraform/lint + - make terraform/validate diff --git a/Makefile b/Makefile index b0f7470..655f630 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,10 @@ SHELL := /bin/bash +# List of targets the `readme` target should call before generating the readme +export README_DEPS ?= docs/targets.md docs/terraform.md + -include $(shell curl -sSL -o .build-harness "https://git.io/build-harness"; echo .build-harness) +## Lint terraform code lint: - $(SELF) terraform:install terraform:get-modules terraform:get-plugins terraform:lint terraform:validate + $(SELF) terraform/install terraform/get-modules terraform/get-plugins terraform/lint terraform/validate \ No newline at end of file diff --git a/README.md b/README.md index 68dbf69..3f522cc 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,9 @@ -# terraform-aws-efs-backup [![Build Status](https://travis-ci.org/cloudposse/terraform-aws-efs-backup.svg)](https://travis-ci.org/cloudposse/terraform-aws-efs-backup) + + +[![Cloud Posse](https://cloudposse.com/logo-300x69.svg)](https://cloudposse.com) + +# terraform-aws-efs-backup [![Build Status](https://travis-ci.org/cloudposse/terraform-aws-efs-backup.svg?branch=master)](https://travis-ci.org/cloudposse/terraform-aws-efs-backup) [![Latest Release](https://img.shields.io/github/release/cloudposse/terraform-aws-efs-backup.svg)](https://github.com/cloudposse/terraform-aws-efs-backup/releases/latest) [![Slack Community](https://slack.cloudposse.com/badge.svg)](https://slack.cloudposse.com) + Terraform module designed to easily backup EFS filesystems to S3 using DataPipeline. @@ -12,6 +17,21 @@ The workflow is simple: * Automatically rotate the backups using `S3 lifecycle rule` +--- + +This project is part of our comprehensive ["SweetOps"](https://docs.cloudposse.com) approach towards DevOps. + + +It's 100% Open Source and licensed under the [APACHE2](LICENSE). + + + + + + + + + ## Usage Include this module in your existing terraform code: @@ -36,39 +56,6 @@ output "efs_backup_security_group" { value = "${module.efs_backup.security_group_id}" } ``` - - -## Variables - -| Name | Default | Description | Required | -|:-----------------------------------|:--------------:|:----------------------------------------------------------------------------------------------|:--------:| -| namespace | `` | Namespace (e.g. `cp` or `cloudposse`) | Yes | -| stage | `` | Stage (e.g. `prod`, `dev`, `staging`) | Yes | -| name | `` | Name (e.g. `app` or `wordpress`) | Yes | -| region | `us-east-1` | (Optional) AWS Region. If not specified, will be derived from 'aws_region' data source | No | -| vpc_id | `` | AWS VPC ID where module should operate (_e.g._ `vpc-a22222ee`) | Yes | -| efs_mount_target_id | `` | Elastic File System Mount Target ID (_e.g._ `fsmt-279bfc62`) | Yes | -| use_ip_address | `false` | If set to `true`, will use IP address instead of DNS name to connect to the `EFS` | Yes | -| modify_security_group | `false` | Should the module modify the `EFS` security group | No | -| noncurrent_version_expiration_days | `35` | S3 object versions expiration period (days) | Yes | -| ssh_key_pair | `` | `SSH` key that will be deployed on DataPipeline's instance | No | -| datapipeline_config | `${map("instance_type", "t2.micro", "email", "", "period", "24 hours", "timeout", "60 Minutes")}"`| DataPipeline configuration options | Yes | -| attributes | `[]` | Additional attributes (_e.g._ `efs-backup`) | No | -| tags | `{}` | Additional tags (e.g. `map("BusinessUnit","XYZ")` | No | -| delimiter | `-` | Delimiter to be used between `name`, `namespace`, `stage` and `attributes` | No | - - -### `datapipeline_config` variables - -| Name | Default | Description | Required | -|:-----------------------------------|:--------------:|:-----------------------------------------------------------------------------|:--------:| -| instance_type | `t2.micro` | Instance type to use | Yes | -| email | `` | Email to use in `SNS`. Needs to be provided, otherwise the module will fail | Yes | -| period | `24 hours` | Frequency of pipeline execution (frequency of backups) | Yes | -| timeout | `60 Minutes` | Pipeline execution timeout | Yes | - - - ## Integration with `EFS` To enable connectivity between the `DataPipeline` instances and the `EFS`, use one of the following methods to configure Security Groups: @@ -129,11 +116,192 @@ At this time you cannot use a Security Group with in-line rules in conjunction w Doing so will cause a conflict of rule settings and will overwrite rules. + + + + +## Makefile Targets +``` +Available targets: + + help This help screen + help/all Display help for all targets + lint Lint terraform code + +``` + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| attributes | Additional attributes (e.g. `efs-backup`) | list | `` | no | +| datapipeline_config | DataPipeline configuration options | map | `` | no | +| delimiter | Delimiter to be used between `name`, `namespace`, `stage`, etc. | string | `-` | no | +| efs_mount_target_id | EFS Mount Target ID (e.g. `fsmt-279bfc62`) | string | - | yes | +| modify_security_group | Should the module modify the `EFS` security group | string | `false` | no | +| name | The Name of the application or solution (e.g. `bastion` or `portal`) | string | - | yes | +| namespace | Namespace (e.g. `cp` or `cloudposse`) | string | - | yes | +| noncurrent_version_expiration_days | S3 object versions expiration period (days) | string | `35` | no | +| region | (Optional) AWS Region. If not specified, will be derived from 'aws_region' data source | string | `` | no | +| ssh_key_pair | `SSH` key that will be deployed on DataPipeline's instance | string | - | yes | +| stage | Stage (e.g. `prod`, `dev`, `staging`) | string | - | yes | +| tags | Additional tags (e.g. `map('BusinessUnit`,`XYZ`) | map | `` | no | +| use_ip_address | If set to `true`, will use IP address instead of DNS name to connect to the `EFS` | string | `false` | no | +| vpc_id | VPC ID | string | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| backups_bucket_name | Backups bucket name | +| datapipeline_ids | Datapipeline ids | +| logs_bucket_name | Logs bucket name | +| security_group_id | Security group id | + + + + +## Related Projects + +Check out these related projects. + +- [terraform-aws-efs](https://github.com/cloudposse/terraform-aws-efs) - Terraform Module to define an EFS Filesystem (aka NFS) +- [terraform-aws-efs-cloudwatch-sns-alarms](https://github.com/cloudposse/terraform-aws-efs-cloudwatch-sns-alarms) - Terraform module that configures CloudWatch SNS alerts for EFS + + + + ## References -* Thanks https://github.com/knakayama/datapipeline-efs-backup-demo for inspiration +For additional context, refer to some of these links. + +- [datapipeline-efs-backup-demo](https://github.com/knakayama/datapipeline-efs-backup-demo) - Thanks for inspiration + + +## Help + +**Got a question?** + +File a GitHub [issue](https://github.com/cloudposse/terraform-aws-efs-backup/issues), send us an [email][email] or join our [Slack Community][slack]. + +## Commerical Support + +Work directly with our team of DevOps experts via email, slack, and video conferencing. + +We provide *commercial support* for all of our [Open Source][github] projects. As a *Dedicated Support* customer, you have access to our team of subject matter experts at a fraction of the cost of a fulltime engineer. + +[![E-Mail](https://img.shields.io/badge/email-hello@cloudposse.com-blue.svg)](mailto:hello@cloudposse.com) + +- **Questions.** We'll use a Shared Slack channel between your team and ours. +- **Troubleshooting.** We'll help you triage why things aren't working. +- **Code Reviews.** We'll review your Pull Requests and provide constructive feedback. +- **Bug Fixes.** We'll rapidly work to fix any bugs in our projects. +- **Build New Terraform Modules.** We'll develop original modules to provision infrastructure. +- **Cloud Architecture.** We'll assist with your cloud strategy and design. +- **Implementation.** We'll provide hands on support to implement our reference architectures. + + +## Community Forum + +Get access to our [Open Source Community Forum][slack] on Slack. It's **FREE** to join for everyone! Our "SweetOps" community is where you get to talk with others who share a similar vision for how to rollout and manage infrastructure. This is the best place to talk shop, ask questions, solicit feedback, and work together as a community to build *sweet* infrastructure. + +## Contributing + +### Bug Reports & Feature Requests + +Please use the [issue tracker](https://github.com/cloudposse/terraform-aws-efs-backup/issues) to report any bugs or file feature requests. + +### Developing + +If you are interested in being a contributor and want to get involved in developing this project or [help out](https://github.com/orgs/cloudposse/projects/3) with our other projects, we would love to hear from you! Shoot us an [email](mailto:hello@cloudposse.com). + +In general, PRs are welcome. We follow the typical "fork-and-pull" Git workflow. + + 1. **Fork** the repo on GitHub + 2. **Clone** the project to your own machine + 3. **Commit** changes to your own branch + 4. **Push** your work back up to your fork + 5. Submit a **Pull Request** so that we can review your changes + +**NOTE:** Be sure to merge the latest changes from "upstream" before making a pull request! + + +## Copyright + +Copyright © 2017-2018 [Cloud Posse, LLC](https://cloudposse.com) + + + +## License + +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +See [LICENSE](LICENSE) for full details. + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + + + + + + + +## Trademarks + +All other trademarks referenced herein are the property of their respective owners. + +## About + +This project is maintained and funded by [Cloud Posse, LLC][website]. Like it? Please let us know at + +[![Cloud Posse](https://cloudposse.com/logo-300x69.svg)](https://cloudposse.com) + +We're a [DevOps Professional Services][hire] company based in Los Angeles, CA. We love [Open Source Software](https://github.com/cloudposse/)! + +We offer paid support on all of our projects. + +Check out [our other projects][github], [apply for a job][jobs], or [hire us][hire] to help with your cloud strategy and implementation. + + [docs]: https://docs.cloudposse.com/ + [website]: https://cloudposse.com/ + [github]: https://github.com/cloudposse/ + [jobs]: https://cloudposse.com/jobs/ + [hire]: https://cloudposse.com/contact/ + [slack]: https://slack.cloudposse.com/ + [linkedin]: https://www.linkedin.com/company/cloudposse + [twitter]: https://twitter.com/cloudposse/ + [email]: mailto:hello@cloudposse.com + + +### Contributors + +| [![Erik Osterman][osterman_avatar]][osterman_homepage]
[Erik Osterman][osterman_homepage] | [![Andriy Knysh][aknysh_avatar]][aknysh_homepage]
[Andriy Knysh][aknysh_homepage] | [![Sergey Vasilyev][s2504s_avatar]][s2504s_homepage]
[Sergey Vasilyev][s2504s_homepage] | [![abferm][abferm_avatar]][abferm_homepage]
[abferm][abferm_homepage] | +|---|---|---|---| + [osterman_homepage]: https://github.com/osterman + [osterman_avatar]: https://github.com/osterman.png?size=150 + [aknysh_homepage]: https://github.com/aknysh + [aknysh_avatar]: https://github.com/aknysh.png?size=150 + [s2504s_homepage]: https://github.com/s2504s + [s2504s_avatar]: https://github.com/s2504s.png?size=150 + [abferm_homepage]: https://github.com/abferm + [abferm_avatar]: https://github.com/abferm.png?size=150 -## License -Apache 2 License. See [`LICENSE`](LICENSE) for full details. diff --git a/README.yaml b/README.yaml new file mode 100644 index 0000000..8951298 --- /dev/null +++ b/README.yaml @@ -0,0 +1,153 @@ +--- +# +# This is the canonical configuration for the `README.md` +# Run `make readme` to rebuild the `README.md` +# + +# Name of this project +name: terraform-aws-efs-backup + +# Logo for this project +#logo: docs/logo.png + +# License of this project +license: "APACHE2" + +# Canonical GitHub repo +github_repo: cloudposse/terraform-aws-efs-backup + +# Badges to display +badges: + - name: "Build Status" + image: "https://travis-ci.org/cloudposse/terraform-aws-efs-backup.svg?branch=master" + url: "https://travis-ci.org/cloudposse/terraform-aws-efs-backup" + - name: "Latest Release" + image: "https://img.shields.io/github/release/cloudposse/terraform-aws-efs-backup.svg" + url: "https://github.com/cloudposse/terraform-aws-efs-backup/releases/latest" + - name: "Slack Community" + image: "https://slack.cloudposse.com/badge.svg" + url: "https://slack.cloudposse.com" + +related: + - name: "terraform-aws-efs" + description: "Terraform Module to define an EFS Filesystem (aka NFS)" + url: "https://github.com/cloudposse/terraform-aws-efs" + - name: "terraform-aws-efs-cloudwatch-sns-alarms" + description: "Terraform module that configures CloudWatch SNS alerts for EFS" + url: "https://github.com/cloudposse/terraform-aws-efs-cloudwatch-sns-alarms" + +# Short description of this project +description: |- + Terraform module designed to easily backup EFS filesystems to S3 using DataPipeline. + + The workflow is simple: + + * Periodically launch resource (EC2 instance) based on schedule + * Execute the shell command defined in the activity on the instance + * Sync data from Production EFS to S3 Bucket by using `aws-cli` + * The execution log of the activity is stored in `S3` + * Publish the success or failure of the activity to an `SNS` topic + * Automatically rotate the backups using `S3 lifecycle rule` + +# How to use this project +usage: |- + Include this module in your existing terraform code: + + ```hcl + module "efs_backup" { + source = "git::https://github.com/cloudposse/terraform-aws-efs-backup.git?ref=master" + + name = "${var.name}" + stage = "${var.stage}" + namespace = "${var.namespace}" + vpc_id = "${var.vpc_id}" + efs_mount_target_id = "${var.efs_mount_target_id}" + use_ip_address = "false" + noncurrent_version_expiration_days = "${var.noncurrent_version_expiration_days}" + ssh_key_pair = "${var.ssh_key_pair}" + datapipeline_config = "${var.datapipeline_config}" + modify_security_group = "true" + } + + output "efs_backup_security_group" { + value = "${module.efs_backup.security_group_id}" + } + ``` + ## Integration with `EFS` + + To enable connectivity between the `DataPipeline` instances and the `EFS`, use one of the following methods to configure Security Groups: + + 1. Explicitly add the `DataPipeline` SG (the output of this module `security_group_id`) to the list of the `ingress` rules of the `EFS` SG. For example: + + ```hcl + module "elastic_beanstalk_environment" { + source = "git::https://github.com/cloudposse/terraform-aws-elastic-beanstalk-environment.git?ref=master" + namespace = "${var.namespace}" + name = "${var.name}" + stage = "${var.stage}" + delimiter = "${var.delimiter}" + attributes = ["${compact(concat(var.attributes, list("eb-env")))}"] + tags = "${var.tags}" + + # .............................. + } + + module "efs" { + source = "git::https://github.com/cloudposse/terraform-aws-efs.git?ref=tmaster" + namespace = "${var.namespace}" + name = "${var.name}" + stage = "${var.stage}" + delimiter = "${var.delimiter}" + attributes = ["${compact(concat(var.attributes, list("efs")))}"] + tags = "${var.tags}" + + # Allow EB/EC2 instances and DataPipeline instances to connect to the EFS + security_groups = ["${module.elastic_beanstalk_environment.security_group_id}", "${module.efs_backup.security_group_id}"] + } + + module "efs_backup" { + source = "git::https://github.com/cloudposse/terraform-aws-efs-backup.git?ref=master" + name = "${var.name}" + stage = "${var.stage}" + namespace = "${var.namespace}" + delimiter = "${var.delimiter}" + attributes = ["${compact(concat(var.attributes, list("efs-backup")))}"] + tags = "${var.tags}" + + # Important to set it to `false` since we added the `DataPipeline` SG (output of the `efs_backup` module) to the `security_groups` of the `efs` module + # See NOTE below for more information + modify_security_group = "false" + + # .............................. + } + ``` + + 2. Set `modify_security_group` attribute to `true` so the module will modify the `EFS` SG to allow the `DataPipeline` to connect to the `EFS` + + **NOTE:** Do not mix these two methods together. + `Terraform` does not support using a Security Group with in-line rules in conjunction with any Security Group Rule resources. + https://www.terraform.io/docs/providers/aws/r/security_group_rule.html + > NOTE on Security Groups and Security Group Rules: Terraform currently provides both a standalone Security Group Rule resource + (a single ingress or egress rule), and a Security Group resource with ingress and egress rules defined in-line. + At this time you cannot use a Security Group with in-line rules in conjunction with any Security Group Rule resources. + Doing so will cause a conflict of rule settings and will overwrite rules. + +references: + - name: "datapipeline-efs-backup-demo" + description: 'Thanks for inspiration' + url: "https://github.com/knakayama/datapipeline-efs-backup-demo" + +include: + - "docs/targets.md" + - "docs/terraform.md" + +# Contributors to this project +contributors: + - name: "Erik Osterman" + github: "osterman" + - name: "Andriy Knysh" + github: "aknysh" + - name: "Sergey Vasilyev" + github: "s2504s" + - name: "abferm" + github: "abferm" \ No newline at end of file diff --git a/docs/targets.md b/docs/targets.md new file mode 100644 index 0000000..09c39cd --- /dev/null +++ b/docs/targets.md @@ -0,0 +1,9 @@ +## Makefile Targets +``` +Available targets: + + help This help screen + help/all Display help for all targets + lint Lint terraform code + +``` diff --git a/docs/terraform.md b/docs/terraform.md new file mode 100644 index 0000000..8d7dfad --- /dev/null +++ b/docs/terraform.md @@ -0,0 +1,29 @@ + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| attributes | Additional attributes (e.g. `efs-backup`) | list | `` | no | +| datapipeline_config | DataPipeline configuration options | map | `` | no | +| delimiter | Delimiter to be used between `name`, `namespace`, `stage`, etc. | string | `-` | no | +| efs_mount_target_id | EFS Mount Target ID (e.g. `fsmt-279bfc62`) | string | - | yes | +| modify_security_group | Should the module modify the `EFS` security group | string | `false` | no | +| name | The Name of the application or solution (e.g. `bastion` or `portal`) | string | - | yes | +| namespace | Namespace (e.g. `cp` or `cloudposse`) | string | - | yes | +| noncurrent_version_expiration_days | S3 object versions expiration period (days) | string | `35` | no | +| region | (Optional) AWS Region. If not specified, will be derived from 'aws_region' data source | string | `` | no | +| ssh_key_pair | `SSH` key that will be deployed on DataPipeline's instance | string | - | yes | +| stage | Stage (e.g. `prod`, `dev`, `staging`) | string | - | yes | +| tags | Additional tags (e.g. `map('BusinessUnit`,`XYZ`) | map | `` | no | +| use_ip_address | If set to `true`, will use IP address instead of DNS name to connect to the `EFS` | string | `false` | no | +| vpc_id | VPC ID | string | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| backups_bucket_name | Backups bucket name | +| datapipeline_ids | Datapipeline ids | +| logs_bucket_name | Logs bucket name | +| security_group_id | Security group id | + diff --git a/outputs.tf b/outputs.tf index a5ec53b..cdbe518 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,15 +1,19 @@ output "logs_bucket_name" { - value = "${aws_s3_bucket.logs.bucket_domain_name}" + value = "${aws_s3_bucket.logs.bucket_domain_name}" + description = "Logs bucket name" } output "backups_bucket_name" { - value = "${aws_s3_bucket.backups.bucket_domain_name}" + value = "${aws_s3_bucket.backups.bucket_domain_name}" + description = "Backups bucket name" } output "datapipeline_ids" { - value = "${aws_cloudformation_stack.datapipeline.outputs["DataPipelineId"]}" + value = "${aws_cloudformation_stack.datapipeline.outputs["DataPipelineId"]}" + description = "Datapipeline ids" } output "security_group_id" { - value = "${aws_security_group.datapipeline.id}" + value = "${aws_security_group.datapipeline.id}" + description = "Security group id" } diff --git a/variables.tf b/variables.tf index afddb21..714c00c 100644 --- a/variables.tf +++ b/variables.tf @@ -1,13 +1,13 @@ variable "name" { - type = "string" + description = "The Name of the application or solution (e.g. `bastion` or `portal`)" } variable "namespace" { - type = "string" + description = "Namespace (e.g. `cp` or `cloudposse`)" } variable "stage" { - type = "string" + description = "Stage (e.g. `prod`, `dev`, `staging`)" } variable "region" { @@ -17,17 +17,20 @@ variable "region" { } variable "vpc_id" { - type = "string" + default = "" + description = "VPC ID" } # https://www.terraform.io/docs/configuration/variables.html # simply using string values rather than booleans for variables is recommended variable "use_ip_address" { - default = "false" + default = "false" + description = "If set to `true`, will use IP address instead of DNS name to connect to the `EFS`" } variable "datapipeline_config" { - type = "map" + description = "DataPipeline configuration options" + type = "map" default = { instance_type = "t2.micro" @@ -43,16 +46,19 @@ variable "efs_mount_target_id" { } variable "modify_security_group" { - default = "false" + default = "false" + description = " Should the module modify the `EFS` security group" } # Set a name of ssh key that will be deployed on DataPipeline's instance. The key should be present in AWS. variable "ssh_key_pair" { - type = "string" + type = "string" + description = "`SSH` key that will be deployed on DataPipeline's instance" } variable "noncurrent_version_expiration_days" { - default = "35" + default = "35" + description = "S3 object versions expiration period (days)" } variable "delimiter" {