From 6d6e2313568612453b97afc8b7e3c557cc6f5c8b Mon Sep 17 00:00:00 2001 From: Dave Kerr Date: Wed, 21 Feb 2018 10:25:54 +0700 Subject: [PATCH] wip: adds support for openshift 3.7 --- README.md | 21 +++++++++ install-from-bastion.sh | 12 +++-- inventory.template.cfg | 11 +++++ main.tf | 2 + modules/openshift/00-variables.tf | 8 ++++ modules/openshift/01-tags.tf | 27 ++++++++++++ modules/openshift/{01-amis.tf => 02-amis.tf} | 0 modules/openshift/{02-vpc.tf => 03-vpc.tf} | 44 ++++++++++++------- ...curity-groups.tf => 04-security-groups.tf} | 44 ++++++++++++------- .../openshift/{04-roles.tf => 05-roles.tf} | 0 .../openshift/{05-nodes.tf => 06-nodes.tf} | 40 +++++++++-------- modules/openshift/{06-dns.tf => 07-dns.tf} | 0 .../{07-bastion.tf => 08-bastion.tf} | 11 +++-- .../{08-inventory.tf => 09-inventory.tf} | 1 + variables.tf | 4 +- 15 files changed, 164 insertions(+), 61 deletions(-) create mode 100644 modules/openshift/01-tags.tf rename modules/openshift/{01-amis.tf => 02-amis.tf} (100%) rename modules/openshift/{02-vpc.tf => 03-vpc.tf} (65%) rename modules/openshift/{03-security-groups.tf => 04-security-groups.tf} (80%) rename modules/openshift/{04-roles.tf => 05-roles.tf} (100%) rename modules/openshift/{05-nodes.tf => 06-nodes.tf} (88%) rename modules/openshift/{06-dns.tf => 07-dns.tf} (100%) rename modules/openshift/{07-bastion.tf => 08-bastion.tf} (79%) rename modules/openshift/{08-inventory.tf => 09-inventory.tf} (95%) diff --git a/README.md b/README.md index 938778e..29b239b 100644 --- a/README.md +++ b/README.md @@ -219,6 +219,8 @@ When you run `make openshift`, all that happens is the `inventory.template.cfg` ## Choosing the OpenShift Version +Currently, OpenShift 3.7 is installed. + To change the version, just update the version identifier in this line of the [`./install-from-bastion.sh`](./install-from-bastion.sh) script: ```bash @@ -227,6 +229,13 @@ git clone -b release-3.6 https://github.com/openshift/openshift-ansible Available versions are listed [here](https://github.com/openshift/openshift-ansible#getting-the-correct-version). + +| Version | Status | +|---------|--------| +| 3.5 | Tested successfully. | +| 3.6 | Tested successfully. | +| 3.7 | Work in progress. | + OpenShift 3.5 is fully tested, and has a slightly different setup. You can build 3.5 by checking out the [`release/openshift-3.5`](https://github.com/dwmkerr/terraform-aws-openshift/tree/release/openshift-3.5) branch. ## Destroying the Cluster @@ -329,6 +338,18 @@ https://github.com/dwmkerr/terraform-aws-openshift/issues/40 At this stage if the AWS generated hostnames for OpenShift nodes are specified in the inventory, then this problem should disappear. If internal DNS names are used (e.g. node1.openshift.internal) then this issue will occur. +**Unable to restart service origin-master-api** + +``` +Failure summary: + + + 1. Hosts: ip-10-0-1-129.ec2.internal + Play: Configure masters + Task: restart master api + Message: Unable to restart service origin-master-api: Job for origin-master-api.service failed because the control process exited with error code. See "systemctl status origin-master-api.service" and "journalctl -xe" for details. +``` + ## Developer Guide This section is intended for those who want to update or modify the code. diff --git a/install-from-bastion.sh b/install-from-bastion.sh index 7deff71..a67065d 100644 --- a/install-from-bastion.sh +++ b/install-from-bastion.sh @@ -3,12 +3,16 @@ set -x # Elevate priviledges, retaining the environment. sudo -E su -# Install dev tools and Ansible 2.2 +# Install dev tools. yum install -y "@Development Tools" python2-pip openssl-devel python-devel gcc libffi-devel -pip install -Iv ansible==2.3.0.0 -# Clone the openshift-ansible repo, which contains the installer. -git clone -b release-3.6 https://github.com/openshift/openshift-ansible +# Get the OpenShift 3.7 installer. +pip install -Iv ansible==2.4.1.0 +git clone -b release-3.7 https://github.com/openshift/openshift-ansible + +# Get the OpenShift 3.6 installer. +# pip install -Iv ansible==2.3.0.0 +# git clone -b release-3.6 https://github.com/openshift/openshift-ansible # Run the playbook. ANSIBLE_HOST_KEY_CHECKING=False /usr/local/bin/ansible-playbook -i ./inventory.cfg ./openshift-ansible/playbooks/byo/config.yml # uncomment for verbose! -vvv diff --git a/inventory.template.cfg b/inventory.template.cfg index 03d8c64..9351d47 100644 --- a/inventory.template.cfg +++ b/inventory.template.cfg @@ -37,6 +37,17 @@ openshift_cloudprovider_kind=aws openshift_cloudprovider_aws_access_key=${access_key} openshift_cloudprovider_aws_secret_key=${secret_key} +# Set the cluster_id. +openshift_clusterid=${cluster_id} + +# - https://github.com/openshift/openshift-ansible/issues/5534 +# - https://github.com/openshift/openshift-ansible/issues/6087 +# - https://github.com/coreos/etcd/issues/8603#issuecomment-341508947 +etcd_version=3.1.9 +# etcd_version=3.2.7 +osm_cluster_network_cidr=11.0.0.0/16 + + # Create the masters host group. Note that due do: # https://github.com/dwmkerr/terraform-aws-openshift/issues/40 # We cannot use the internal DNS names (such as master.openshift.local) as there diff --git a/main.tf b/main.tf index 04bded9..126d399 100644 --- a/main.tf +++ b/main.tf @@ -13,6 +13,8 @@ module "openshift" { subnet_cidr = "10.0.1.0/24" key_name = "openshift" public_key_path = "${var.public_key_path}" + cluster_name = "openshift-cluster" + cluster_id = "openshift-cluster-${var.region}" } // Output some useful variables for quick SSH access etc. diff --git a/modules/openshift/00-variables.tf b/modules/openshift/00-variables.tf index 0da883e..a5bb2ea 100644 --- a/modules/openshift/00-variables.tf +++ b/modules/openshift/00-variables.tf @@ -26,3 +26,11 @@ variable "key_name" { variable "public_key_path" { description = "The local public key path, e.g. ~/.ssh/id_rsa.pub" } + +variable "cluster_name" { + description = "Name of the cluster, e.g: 'openshift-cluster'. Useful when running multiple clusters in the same AWS account." +} + +variable "cluster_id" { + description = "ID of the cluster, e.g: 'openshift-cluster-us-east-1'. Useful when running multiple clusters in the same AWS account." +} diff --git a/modules/openshift/01-tags.tf b/modules/openshift/01-tags.tf new file mode 100644 index 0000000..542dbd4 --- /dev/null +++ b/modules/openshift/01-tags.tf @@ -0,0 +1,27 @@ +// Wherever possible, we will use a common set of tags for resources. This +// makes it much easier to set up resource based billing, tag based access, +// resource groups and more. +// +// We are also required to set certain tags on resources to support Kubernetes +// and AWS integration, which is needed for dynamic volume provisioning. +// +// This is quite fiddly, the following resources should be useful: +// +// - Terraform: Local Values: https://www.terraform.io/docs/configuration/locals.html +// - Terraform: Default Tags for Resources in Terraform: https://github.com/hashicorp/terraform/issues/2283 +// - Terraform: Variable Interpolation for Tags: https://github.com/hashicorp/terraform/issues/14516 +// - OpenShift: Cluster Labelling Requirements: https://docs.openshift.org/latest/install_config/configuring_aws.html#aws-cluster-labeling + +// Define our common tags. +// - Project: Purely for my own organision, delete or change as you like! +// - KubernetesCluster: Set to , required for OpenShift < 3.7 +// - kubernetes.io/cluster/: Set to , required for OpenShift >= 3.7 +// The syntax below is ugly, but needed as we are using dynamic key names. +locals { + common_tags = "${map( + "Project", "openshift", + "KubernetesCluster", "${var.cluster_id}", + "kubernetes.io/cluster/${var.cluster_name}", "${var.cluster_id}" + )}" +} + diff --git a/modules/openshift/01-amis.tf b/modules/openshift/02-amis.tf similarity index 100% rename from modules/openshift/01-amis.tf rename to modules/openshift/02-amis.tf diff --git a/modules/openshift/02-vpc.tf b/modules/openshift/03-vpc.tf similarity index 65% rename from modules/openshift/02-vpc.tf rename to modules/openshift/03-vpc.tf index 08dbb35..d8d5425 100644 --- a/modules/openshift/02-vpc.tf +++ b/modules/openshift/03-vpc.tf @@ -3,20 +3,26 @@ resource "aws_vpc" "openshift" { cidr_block = "${var.vpc_cidr}" enable_dns_hostnames = true - tags { - Name = "OpenShift VPC" - Project = "openshift" - } + // Use our common tags and add a specific name. + tags = "${merge( + local.common_tags, + map( + "Name", "OpenShift VPC" + ) + )}" } // Create an Internet Gateway for the VPC. resource "aws_internet_gateway" "openshift" { vpc_id = "${aws_vpc.openshift.id}" - tags { - Name = "OpenShift IGW" - Project = "openshift" - } + // Use our common tags and add a specific name. + tags = "${merge( + local.common_tags, + map( + "Name", "OpenShift IGW" + ) + )}" } // Create a public subnet. @@ -27,10 +33,13 @@ resource "aws_subnet" "public-subnet" { map_public_ip_on_launch = true depends_on = ["aws_internet_gateway.openshift"] - tags { - Name = "OpenShift Public Subnet" - Project = "openshift" - } + // Use our common tags and add a specific name. + tags = "${merge( + local.common_tags, + map( + "Name", "OpenShift Public Subnet" + ) + )}" } // Create a route table allowing all addresses access to the IGW. @@ -42,10 +51,13 @@ resource "aws_route_table" "public" { gateway_id = "${aws_internet_gateway.openshift.id}" } - tags { - Name = "OpenShift Public Route Table" - Project = "openshift" - } + // Use our common tags and add a specific name. + tags = "${merge( + local.common_tags, + map( + "Name", "OpenShift Public Route Table" + ) + )}" } // Now associate the route table with the public subnet - giving diff --git a/modules/openshift/03-security-groups.tf b/modules/openshift/04-security-groups.tf similarity index 80% rename from modules/openshift/03-security-groups.tf rename to modules/openshift/04-security-groups.tf index 6eecbf1..8d20146 100644 --- a/modules/openshift/03-security-groups.tf +++ b/modules/openshift/04-security-groups.tf @@ -19,10 +19,13 @@ resource "aws_security_group" "openshift-vpc" { self = true } - tags { - Name = "OpenShift Internal VPC" - Project = "openshift" - } + // Use our common tags and add a specific name. + tags = "${merge( + local.common_tags, + map( + "Name", "OpenShift Internal VPC" + ) + )}" } // This security group allows public ingress to the instances for HTTP, HTTPS @@ -64,10 +67,13 @@ resource "aws_security_group" "openshift-public-ingress" { cidr_blocks = ["0.0.0.0/0"] } - tags { - Name = "OpenShift Public Access" - Project = "openshift" - } + // Use our common tags and add a specific name. + tags = "${merge( + local.common_tags, + map( + "Name", "OpenShift Public Ingress" + ) + )}" } // This security group allows public egress from the instances for HTTP and @@ -93,10 +99,13 @@ resource "aws_security_group" "openshift-public-egress" { cidr_blocks = ["0.0.0.0/0"] } - tags { - Name = "OpenShift Public Access" - Project = "openshift" - } + // Use our common tags and add a specific name. + tags = "${merge( + local.common_tags, + map( + "Name", "OpenShift Public Egress" + ) + )}" } // Security group which allows SSH access to a host. Used for the bastion. @@ -113,8 +122,11 @@ resource "aws_security_group" "openshift-ssh" { cidr_blocks = ["0.0.0.0/0"] } - tags { - Name = "OpenShift SSH Access" - Project = "openshift" - } + // Use our common tags and add a specific name. + tags = "${merge( + local.common_tags, + map( + "Name", "OpenShift SSH Access" + ) + )}" } diff --git a/modules/openshift/04-roles.tf b/modules/openshift/05-roles.tf similarity index 100% rename from modules/openshift/04-roles.tf rename to modules/openshift/05-roles.tf diff --git a/modules/openshift/05-nodes.tf b/modules/openshift/06-nodes.tf similarity index 88% rename from modules/openshift/05-nodes.tf rename to modules/openshift/06-nodes.tf index ba6d02c..ffa2251 100644 --- a/modules/openshift/05-nodes.tf +++ b/modules/openshift/06-nodes.tf @@ -42,14 +42,14 @@ resource "aws_instance" "master" { } key_name = "${aws_key_pair.keypair.key_name}" - - tags { - Name = "OpenShift Master" - Project = "openshift" - // this tag is required for dynamic EBS PVCs - // see https://github.com/kubernetes/kubernetes/issues/39178 - KubernetesCluster = "openshift-${var.region}" - } + + // Use our common tags and add a specific name. + tags = "${merge( + local.common_tags, + map( + "Name", "OpenShift Master" + ) + )}" } // Create the node userdata script. @@ -91,11 +91,13 @@ resource "aws_instance" "node1" { key_name = "${aws_key_pair.keypair.key_name}" - tags { - Name = "OpenShift Node 1" - Project = "openshift" - KubernetesCluster = "openshift-${var.region}" - } + // Use our common tags and add a specific name. + tags = "${merge( + local.common_tags, + map( + "Name", "OpenShift Node 1" + ) + )}" } resource "aws_instance" "node2" { ami = "${data.aws_ami.rhel7_2.id}" @@ -126,9 +128,11 @@ resource "aws_instance" "node2" { key_name = "${aws_key_pair.keypair.key_name}" - tags { - Name = "OpenShift Node 2" - Project = "openshift" - KubernetesCluster = "openshift-${var.region}" - } + // Use our common tags and add a specific name. + tags = "${merge( + local.common_tags, + map( + "Name", "OpenShift Node 2" + ) + )}" } diff --git a/modules/openshift/06-dns.tf b/modules/openshift/07-dns.tf similarity index 100% rename from modules/openshift/06-dns.tf rename to modules/openshift/07-dns.tf diff --git a/modules/openshift/07-bastion.tf b/modules/openshift/08-bastion.tf similarity index 79% rename from modules/openshift/07-bastion.tf rename to modules/openshift/08-bastion.tf index 2ab992f..dc6908f 100644 --- a/modules/openshift/07-bastion.tf +++ b/modules/openshift/08-bastion.tf @@ -13,8 +13,11 @@ resource "aws_instance" "bastion" { key_name = "${aws_key_pair.keypair.key_name}" - tags { - Name = "OpenShift Bastion" - Project = "openshift" - } + // Use our common tags and add a specific name. + tags = "${merge( + local.common_tags, + map( + "Name", "OpenShift Bastion" + ) + )}" } diff --git a/modules/openshift/08-inventory.tf b/modules/openshift/09-inventory.tf similarity index 95% rename from modules/openshift/08-inventory.tf rename to modules/openshift/09-inventory.tf index 0cb2ab8..3d7d9ff 100644 --- a/modules/openshift/08-inventory.tf +++ b/modules/openshift/09-inventory.tf @@ -10,6 +10,7 @@ data "template_file" "inventory" { master_hostname = "${aws_instance.master.private_dns}" node1_hostname = "${aws_instance.node1.private_dns}" node2_hostname = "${aws_instance.node1.private_dns}" + cluster_id = "${var.cluster_id}" } } diff --git a/variables.tf b/variables.tf index 0c5f945..639355b 100644 --- a/variables.tf +++ b/variables.tf @@ -1,9 +1,7 @@ // The region we will deploy our cluster into. variable "region" { description = "Region to deploy the cluster into" - // The default below will be fine for many, but to make it clear for first - // time users, there's no default, so you will be prompted for a region. - // default = "us-east-1" + default = "us-east-1" } // The public key to use for SSH access.