Single AMI changes for Amazon Marketplace (#2486)

* Add single AMI build/setup * Add ACM support to Terraform and Letsencrypt support to single AMI * Finish Letsencrypt support for Single AMI, also add ACM to Single AMI and tidy up Terraform versioning * Fix Letsencrypt cert acquistion, reduce startup timers from 5 minutes to 3 minutes, tweaks for ACM/non-ACM in Terraform * Remove AWS-based license from Enterprise AMI to convert to BYOL * Tidy up - move Cloudformation into a separate subdirectory and remove old Terraform code * Updated TIG stack to latest versions and tested * Tidy up CloudFormation builds and improve instructions * Fix VPC variable name
gravitational · Jan 29, 2019 · 1265d4c · 1265d4c
1 parent 04c2b67
commit 1265d4c
Show file tree

Hide file tree

Showing 88 changed files with 3,054 additions and 1,368 deletions.
diff --git a/assets/marketplace/Jenkinsfile b/assets/marketplace/Jenkinsfile
@@ -7,83 +7,12 @@ pipeline {
         timestamps()
     }
     stages {
-        stage('Create files/build directory') {
-            steps {
-                dir('assets/marketplace') {
-                    sh 'mkdir -p files/build'
-                }
-            }
-        }
         stage('Run Packer') {
             steps {
                 dir('assets/marketplace') {
                     sh 'make oss'
                 }
             }
         }
-        stage('Update AMI IDs)') {
-            steps {
-                dir('assets/marketplace') {
-                    sh 'make update-ami-ids-oss'
-                }
-            }
-        }
-        stage('Run Cloudformation and wait for stack') {
-            environment {
-                JENKINS_CLOUDFORMATION_PARAMETERS_PATH = credentials('7d731ad8-ed03-4e42-a411-605c40c182f6')
-            }
-            steps {
-                dir('assets/marketplace') {
-                    sh 'make jenkins-create-stack-vpc'
-                    sh 'make jenkins-wait-for-stack-creation-vpc'
-                    // get VPC stack outputs as a JSON list of {ParameterKey: key, ParameterValue: value} pairs
-                    // write this to vpc.json
-                    sh 'make jenkins-describe-stack-outputs-vpc | jq -r "map ( { ParameterKey: ( .OutputKey | tostring ), ParameterValue: ( .OutputValue | tostring) } )" > vpc.json'
-                    // use jq to combine Jenkins JSON stack parameters and vpc.json into one list of pairs
-                    sh "jq -s add ${JENKINS_CLOUDFORMATION_PARAMETERS_PATH} vpc.json > full-parameters.json"
-                    // generate 16 random characters and update the domain name we're using (beware: there is a 64 character limit on the common name field)
-                    sh 'dd if=/dev/urandom bs=128 count=1 | tr -dc "a-z0-9" | fold -w 16 | head -n 1 > random-string'
-                    sh 'sed -i "s/jenkins-teleport-quickstart-test-stack.gravitational.io/jenkins-teleport-quickstart-$(cat random-string).gravitational.io/g" full-parameters.json'
-                    sh 'jq -r \'.[] | select(.ParameterKey == "DomainName") | .ParameterValue\' full-parameters.json > generated-domain-name'
-                    sh 'cat full-parameters.json'
-                    sh 'cat generated-domain-name'
-                    // update path to parameters file and create ASG stack
-                    sh 'JENKINS_CLOUDFORMATION_PARAMETERS_PATH=full-parameters.json make jenkins-create-stack'
-                    sh 'make jenkins-wait-for-stack-creation'
-                    sh 'make jenkins-get-s3-bucket-id > s3-bucket-id'
-                    sh 'cat s3-bucket-id'
-                }
-            }
-        }
-        stage('Wait for Teleport Web UI to come up') {
-            steps {
-                timeout(5) {
-                    waitUntil {
-                        dir('assets/marketplace') {
-                            script {
-                                sh 'cat generated-domain-name'
-                                def r = sh script: 'wget -q https://$(cat generated-domain-name)/web/login', returnStatus: true
-                                return (r == 0);
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-    post {
-        always {
-            dir('assets/marketplace') {
-                script {
-                    // we want to run all these shutdown steps in order regardless of success/failure (in case part of the ASG fails)
-                    // so we use '|| true' to suppress return codes > 0
-                    sh 'make jenkins-delete-stack || true'
-                    sh 'make jenkins-wait-for-stack-deletion || true'
-                    sh 'make jenkins-delete-stack-vpc || true'
-                    sh 'make jenkins-wait-for-stack-deletion-vpc || true'
-                    sh 'S3_BUCKET_ID=$(cat s3-bucket-id) make jenkins-delete-s3-bucket || true'
-                }
-            }            
-        }
     }
 }
diff --git a/assets/marketplace/Jenkinsfile-build-ent b/assets/marketplace/Jenkinsfile-build-ent
@@ -7,19 +7,13 @@ pipeline {
         timestamps()
     }
     parameters {
-        string(name: 'version', defaultValue: '2.6.1', description: 'Teleport version to build')
-        string(name: 'teleport_license_uri', defaultValue: 's3://', description: 'Path to load license file from')
+        string(name: 'version', defaultValue: '3.1.1', description: 'Teleport version to build')
     }
     stages {
         stage('Run Packer to build specified version') {
             steps {
                 dir('assets/marketplace') {
-                    sh """
-BUILD_AMI_NAME=cloudformation-gravitational-teleport-ent-${params.version} \
-TELEPORT_VERSION=${params.version} \
-TELEPORT_LICENSE_URI=${params.teleport_license_uri} \
-make ent-name-build
-"""
+                    sh "BUILD_AMI_NAME=gravitational-teleport-ami-ent-${params.version} TELEPORT_VERSION=${params.version} make ent-jenkins-build"
                 }
             }
         }

diff --git a/assets/marketplace/Jenkinsfile-build-oss b/assets/marketplace/Jenkinsfile-build-oss
@@ -7,13 +7,13 @@ pipeline {
         timestamps()
     }
     parameters {
-        string(name: 'version', defaultValue: '2.6.1', description: 'Teleport version to build')
+        string(name: 'version', defaultValue: '3.1.1', description: 'Teleport version to build')
     }
     stages {
         stage('Run Packer to build specified version') {
             steps {
                 dir('assets/marketplace') {
-                    sh "BUILD_AMI_NAME=cloudformation-gravitational-teleport-oss-${params.version} TELEPORT_VERSION=${params.version} make oss-name-build"
+                    sh "BUILD_AMI_NAME=gravitational-teleport-ami-oss-${params.version} TELEPORT_VERSION=${params.version} make oss-jenkins-build"
                 }
             }
         }

diff --git a/assets/marketplace/Makefile b/assets/marketplace/Makefile
@@ -1,5 +1,3 @@
-
-
 # VPC ID used for builds
 BUILD_VPC_ID ?=
 
@@ -13,7 +11,7 @@ BUILD_AMI_NAME ?=
 AWS_REGION ?= us-west-2
 
 # Teleport version
-TELEPORT_VERSION ?= 3.0.1
+TELEPORT_VERSION ?= 3.1.1
 
 # Teleport UID is the UID of a non-privileged 'teleport' user
 TELEPORT_UID ?= 1007
@@ -26,192 +24,56 @@ INSTANCE_TYPE ?= t2.micro
 # https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/EncryptionAtRest.html
 DESTINATION_REGIONS ?= us-west-2,us-east-1,us-east-2,eu-west-1
 
-# Cloudformation stack name to create, e.g. test1
-STACK ?=
-
-# Stack parameters, e.g ParameterKey=KeyName,ParameterValue=KeyName ParameterKey=DomainName,ParameterValue=teleport.example.com ParameterKey=DomainAdminEmail,ParameterValue=admin@example.com ParameterKey=HostedZoneID,ParameterValue=AWSZONEID
-STACK_PARAMS ?=
-
-# License S3 URI for enterprise builds, e.g. s3://example.com/path/to/license
-TELEPORT_LICENSE_URI ?=
-
 # Generate timestamp for builds
 BUILD_TIMESTAMP := $(shell TZ=UTC /bin/date "+%Y%m%d-%H%M%S%Z")
 
-# Jenkins test CloudFormation stack name
-JENKINS_TEST_STACK_NAME ?= jenkins-teleport-quickstart-test-stack
+# S3 Bucket ID
+S3_BUCKET_ID ?=
 
-# Jenkins Cloudformation parameters path
-JENKINS_CLOUDFORMATION_PARAMETERS_PATH ?= ./files/build/jenkins-parameters.json
+# Telegraf version
+TELEGRAF_VERSION ?= 1.9.3
 
-# YAML filename
-CF_YAML_FILENAME ?= ./oss.yaml
+# InfluxDB version
+INFLUXDB_VERSION ?= 1.7.1
 
-# S3 Bucket ID
-S3_BUCKET_ID ?=
+# Grafana version
+GRAFANA_VERSION ?= 5.4.3
 export
 
+
 .PHONY: oss
 oss: TELEPORT_TYPE=oss
+oss: check-vars
 oss:
 	@echo "Building image $(TELEPORT_VERSION) $(TELEPORT_TYPE)"
 	@echo "BUILD_TIMESTAMP=$(BUILD_TIMESTAMP)"
-	packer build -force -var build_timestamp=$(BUILD_TIMESTAMP) template.json
-	@echo "$(BUILD_TIMESTAMP)" > files/build/oss_build_timestamp.txt
+	packer build -force -var build_timestamp=$(BUILD_TIMESTAMP) single-ami.json
 
-.PHONY: oss-name-build
-oss-name-build: TELEPORT_TYPE=oss
-oss-name-build:
+.PHONY: oss-jenkins-build
+oss-jenkins-build: TELEPORT_TYPE=oss
+oss-jenkins-build: check-vars
+oss-jenkins-build:
 	@echo "Building image $(TELEPORT_VERSION) $(TELEPORT_TYPE) with name $(BUILD_AMI_NAME)"
-	packer build -force -var ami_name=$(BUILD_AMI_NAME) template.json
+	packer build -force -var ami_name=$(BUILD_AMI_NAME) single-ami.json
 
 .PHONY: ent
 ent: TELEPORT_TYPE=ent
-ent: check-ent-vars
-	aws s3 cp $(TELEPORT_LICENSE_URI) files/system/license.pem
+ent: check-vars
 	@echo "Building image $(TELEPORT_VERSION) $(TELEPORT_TYPE)"
 	@echo "BUILD_TIMESTAMP=$(BUILD_TIMESTAMP)"
-	packer build -force -var build_timestamp=$(BUILD_TIMESTAMP) template.json
-	@echo "$(BUILD_TIMESTAMP)" > files/build/ent_build_timestamp.txt
-
-.PHONY: ent-name-build
-ent-name-build: TELEPORT_TYPE=ent
-ent-name-build: check-ent-vars
-ent-name-build:
-	aws s3 cp $(TELEPORT_LICENSE_URI) files/system/license.pem
-	@echo "Building image $(TELEPORT_VERSION) $(TELEPORT_TYPE) with name $(BUILD_AMI_NAME)"
-	packer build -force -var ami_name=$(BUILD_AMI_NAME) template.json
-
-.PHONY: update-ami-ids-oss
-update-ami-ids-oss:
-	files/update-ami-ids.sh oss
-
-.PHONY: update-ami-ids-ent
-update-ami-ids-ent:
-	files/update-ami-ids.sh ent
-
-.PHONY: validate-template
-validate-template:
-	aws cloudformation validate-template --template-body file://$(CF_YAML_FILENAME)
+	packer build -force -var build_timestamp=$(BUILD_TIMESTAMP) single-ami.json
 
+.PHONY: ent-jenkins-build
+ent-jenkins-build: TELEPORT_TYPE=ent
+ent-jenkins-build: check-vars
+ent-jenkins-build:
+	@echo "Building image $(TELEPORT_VERSION) $(TELEPORT_TYPE) with name $(BUILD_AMI_NAME)"
+	packer build -force -var ami_name=$(BUILD_AMI_NAME) single-ami.json
 
-# Stack functionality
-# Create
-.PHONY: create-stack
-create-stack:
-	$(MAKE) validate-template
-	aws --region=$(AWS_REGION) cloudformation create-stack --capabilities CAPABILITY_IAM --stack-name $(STACK) --template-body file://$(CF_YAML_FILENAME) --parameters $(STACK_PARAMS)
-
-.PHONY: create-stack-vpc
-create-stack-vpc: CF_YAML_FILENAME=./oss-vpc.yaml
-create-stack-vpc:
-	$(MAKE) create-stack
-
-
-# Update
-.PHONY: update-stack
-update-stack:
-	$(MAKE) validate-template
-	aws --region=$(AWS_REGION) cloudformation update-stack --capabilities CAPABILITY_IAM --stack-name $(STACK) --template-body file://$(CF_YAML_FILENAME) --parameters $(STACK_PARAMS)
-
-.PHONY: update-stack-vpc
-update-stack-vpc: CF_YAML_FILENAME=./oss-vpc.yaml
-update-stack-vpc:
-	$(MAKE) update-stack
-
-
-# Describe
-.PHONY: describe-stack
-describe-stack:
-	@aws --region=$(AWS_REGION) cloudformation describe-stacks --stack-name $(STACK)
-
-.PHONY: describe-stack-outputs
-describe-stack-outputs:
-	@aws --region=$(AWS_REGION) cloudformation describe-stacks --stack-name $(STACK) --query 'Stacks[].Outputs[]'
-
-.PHONY: describe-stack-outputs-vpc
-describe-stack-outputs-vpc: JENKINS_TEST_STACK_NAME=jenkins-teleport-quickstart-test-stack-vpc
-describe-stack-outputs-vpc:
-	@aws --region=$(AWS_REGION) cloudformation describe-stacks --stack-name $(STACK) --query 'Stacks[].Outputs[]'
-
-
-# Delete
-.PHONY: delete-stack
-delete-stack:
-	aws --region=$(AWS_REGION) cloudformation delete-stack --stack-name $(STACK)
-
-
-# Jenkins helpers
-.PHONY: jenkins-create-stack
-jenkins-create-stack:
-	$(MAKE) validate-template
-	aws --region=$(AWS_REGION) cloudformation create-stack --capabilities CAPABILITY_IAM --stack-name $(JENKINS_TEST_STACK_NAME) --template-body file://$(CF_YAML_FILENAME) --parameters file://$(JENKINS_CLOUDFORMATION_PARAMETERS_PATH)
-
-.PHONY: jenkins-create-stack-vpc
-jenkins-create-stack-vpc: CF_YAML_FILENAME=./oss-vpc.yaml
-jenkins-create-stack-vpc: JENKINS_TEST_STACK_NAME=jenkins-teleport-quickstart-test-stack-vpc
-jenkins-create-stack-vpc:
-	$(MAKE) jenkins-create-stack
-
-.PHONY: jenkins-wait-for-stack-creation
-jenkins-wait-for-stack-creation:
-	aws --region=$(AWS_REGION) cloudformation wait stack-create-complete --stack-name $(JENKINS_TEST_STACK_NAME)
-
-.PHONY: jenkins-wait-for-stack-creation-vpc
-jenkins-wait-for-stack-creation-vpc: JENKINS_TEST_STACK_NAME=jenkins-teleport-quickstart-test-stack-vpc
-jenkins-wait-for-stack-creation-vpc:
-	$(MAKE) jenkins-wait-for-stack-creation
-
-.PHONY: jenkins-describe-stack
-jenkins-describe-stack:
-	@aws --region=$(AWS_REGION) cloudformation describe-stacks --stack-name $(JENKINS_TEST_STACK_NAME)
-
-.PHONY: jenkins-describe-stack-outputs
-jenkins-describe-stack-outputs:
-	@aws --region=$(AWS_REGION) cloudformation describe-stacks --stack-name $(JENKINS_TEST_STACK_NAME) --query 'Stacks[].Outputs[]'
-
-.PHONY: jenkins-describe-stack-vpc
-jenkins-describe-stack-vpc: JENKINS_TEST_STACK_NAME=jenkins-teleport-quickstart-test-stack-vpc
-jenkins-describe-stack-vpc:
-	$(MAKE) jenkins-describe-stack
-
-.PHONY: jenkins-describe-stack-outputs-vpc
-jenkins-describe-stack-outputs-vpc: JENKINS_TEST_STACK_NAME=jenkins-teleport-quickstart-test-stack-vpc
-jenkins-describe-stack-outputs-vpc:
-	@$(MAKE) -s jenkins-describe-stack-outputs
-
-.PHONY: jenkins-get-s3-bucket-id
-jenkins-get-s3-bucket-id:
-	@$(MAKE) -s jenkins-describe-stack-outputs | jq -r '.[] | select(.OutputKey == "S3BucketID") | .OutputValue'
-
-.PHONY: jenkins-delete-s3-bucket
-jenkins-delete-s3-bucket:
-	files/s3-delete-all-object-versions.sh $(S3_BUCKET_ID)
-	aws s3api delete-bucket --region=$(AWS_REGION) --bucket $(S3_BUCKET_ID)
-
-.PHONY: jenkins-delete-stack
-jenkins-delete-stack:
-	aws --region=$(AWS_REGION) cloudformation delete-stack --stack-name $(JENKINS_TEST_STACK_NAME)
-
-.PHONY: jenkins-delete-stack-vpc
-jenkins-delete-stack-vpc: JENKINS_TEST_STACK_NAME=jenkins-teleport-quickstart-test-stack-vpc
-jenkins-delete-stack-vpc:
-	$(MAKE) jenkins-delete-stack
-
-.PHONY: jenkins-wait-for-stack-deletion
-jenkins-wait-for-stack-deletion:
-	aws --region=$(AWS_REGION) cloudformation wait stack-delete-complete --stack-name $(JENKINS_TEST_STACK_NAME)
-
-.PHONY: jenkins-wait-for-stack-deletion-vpc
-jenkins-wait-for-stack-deletion-vpc: JENKINS_TEST_STACK_NAME=jenkins-test-stack-vpc
-jenkins-wait-for-stack-deletion-vpc:
-	$(MAKE) jenkins-wait-for-stack-deletion
 
-.PHONY: check-ent-vars
-check-ent-vars:
+# Other helpers
+.PHONY: check-vars
+check-vars:
 	@if [ -z "$(TELEPORT_VERSION)" ]; then \
 	  echo "TELEPORT_VERSION is not set"; exit 1; \
 	fi;
-	@if [ -z "$(TELEPORT_LICENSE_URI)" ]; then \
-	  echo "TELEPORT_LICENSE_URI is not set"; exit 1; \
-	fi;