From 4b4e2d1da41266e9b3b5e1073c04b516afe1d413 Mon Sep 17 00:00:00 2001 From: joneszc Date: Mon, 16 Sep 2024 16:58:32 -0400 Subject: [PATCH 1/8] Add config option (amazon_web_services.eks_kms_arn) to specify KMS-key ARN to encrypt EKS cluster secrets --- src/_nebari/stages/infrastructure/__init__.py | 3 ++ .../infrastructure/template/aws/main.tf | 1 + .../template/aws/modules/kubernetes/main.tf | 12 ++++++++ .../template/aws/modules/kubernetes/policy.tf | 28 +++++++++++++++++++ .../aws/modules/kubernetes/variables.tf | 6 ++++ .../infrastructure/template/aws/variables.tf | 6 ++++ 6 files changed, 56 insertions(+) diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 0820940f20..a441782c19 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -149,6 +149,7 @@ class AWSInputVars(schema.Base): eks_endpoint_access: Optional[ Literal["private", "public", "public_and_private"] ] = "public" + eks_kms_arn: Optional[str] = None node_groups: List[AWSNodeGroupInputVars] availability_zones: List[str] vpc_cidr_block: str @@ -471,6 +472,7 @@ class AmazonWebServicesProvider(schema.Base): eks_endpoint_access: Optional[ Literal["private", "public", "public_and_private"] ] = "public" + eks_kms_arn: Optional[str] = None existing_subnet_ids: Optional[List[str]] = None existing_security_group_id: Optional[str] = None vpc_cidr_block: str = "10.10.0.0/16" @@ -815,6 +817,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]): name=self.config.escaped_project_name, environment=self.config.namespace, eks_endpoint_access=self.config.amazon_web_services.eks_endpoint_access, + eks_kms_arn=self.config.amazon_web_services.eks_kms_arn, existing_subnet_ids=self.config.amazon_web_services.existing_subnet_ids, existing_security_group_id=self.config.amazon_web_services.existing_security_group_id, region=self.config.amazon_web_services.region, diff --git a/src/_nebari/stages/infrastructure/template/aws/main.tf b/src/_nebari/stages/infrastructure/template/aws/main.tf index feffd35291..ec0cbb6606 100644 --- a/src/_nebari/stages/infrastructure/template/aws/main.tf +++ b/src/_nebari/stages/infrastructure/template/aws/main.tf @@ -99,6 +99,7 @@ module "kubernetes" { endpoint_public_access = var.eks_endpoint_access == "private" ? false : true endpoint_private_access = var.eks_endpoint_access == "public" ? false : true + eks_kms_arn = var.eks_kms_arn public_access_cidrs = var.eks_public_access_cidrs permissions_boundary = var.permissions_boundary } diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf index 6ca547ab32..20cbc5e73d 100644 --- a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf +++ b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf @@ -14,8 +14,20 @@ resource "aws_eks_cluster" "main" { public_access_cidrs = var.public_access_cidrs } + # Only set encryption_config if eks_kms_arn is not null + dynamic "encryption_config" { + for_each = var.eks_kms_arn != null ? [1] : [] + content { + provider { + key_arn = var.eks_kms_arn + } + resources = ["secrets"] + } + } + depends_on = [ aws_iam_role_policy_attachment.cluster-policy, + aws_iam_role_policy_attachment.cluster_encryption, ] tags = merge({ Name = var.name }, var.tags) diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/policy.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/policy.tf index 6916bc6532..f6ad38e6ca 100644 --- a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/policy.tf +++ b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/policy.tf @@ -32,6 +32,34 @@ resource "aws_iam_role_policy_attachment" "cluster-policy" { role = aws_iam_role.cluster.name } +data "aws_iam_policy_document" "cluster_encryption" { + count = var.eks_kms_arn != null ? 1 : 0 + statement { + actions = [ + "kms:Encrypt", + "kms:Decrypt", + "kms:ListGrants", + "kms:DescribeKey" + ] + resources = [var.eks_kms_arn] + } +} + +resource "aws_iam_policy" "cluster_encryption" { + count = var.eks_kms_arn != null ? 1 : 0 + name = "${var.name}-eks-encryption-policy" + description = "IAM policy for EKS cluster encryption" + policy = data.aws_iam_policy_document.cluster_encryption.json +} + +# Grant the EKS Cluster role KMS permissions if a key-arn is specified +resource "aws_iam_role_policy_attachment" "cluster_encryption" { + count = var.eks_kms_arn != null ? 1 : 0 + + policy_arn = aws_iam_policy.cluster_encryption.arn + role = aws_iam_role.cluster.name +} + # ======================================================= # Kubernetes Node Group Policies # ======================================================= diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/variables.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/variables.tf index 87f5e7c95a..318337fc84 100644 --- a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/variables.tf +++ b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/variables.tf @@ -70,6 +70,12 @@ variable "endpoint_private_access" { default = false } +variable "eks_kms_arn" { + description = "kms key arn for EKS cluster encryption_config" + type = string + default = null +} + variable "public_access_cidrs" { type = list(string) default = ["0.0.0.0/0"] diff --git a/src/_nebari/stages/infrastructure/template/aws/variables.tf b/src/_nebari/stages/infrastructure/template/aws/variables.tf index 372109babb..1f92f4c102 100644 --- a/src/_nebari/stages/infrastructure/template/aws/variables.tf +++ b/src/_nebari/stages/infrastructure/template/aws/variables.tf @@ -67,6 +67,12 @@ variable "eks_endpoint_private_access" { default = false } +variable "eks_kms_arn" { + description = "kms key arn for EKS cluster encryption_config" + type = string + default = null +} + variable "eks_public_access_cidrs" { type = list(string) default = ["0.0.0.0/0"] From 858d185d878e872e1fd987a5e2d16c55a1fca8cf Mon Sep 17 00:00:00 2001 From: joneszc Date: Tue, 17 Sep 2024 10:31:08 -0400 Subject: [PATCH 2/8] Add config option (amazon_web_services.eks_kms_arn) to specify KMS-key ARN to encrypt EKS cluster secrets --- .../infrastructure/template/aws/modules/kubernetes/policy.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/policy.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/policy.tf index f6ad38e6ca..bed7b12a96 100644 --- a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/policy.tf +++ b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/policy.tf @@ -49,14 +49,14 @@ resource "aws_iam_policy" "cluster_encryption" { count = var.eks_kms_arn != null ? 1 : 0 name = "${var.name}-eks-encryption-policy" description = "IAM policy for EKS cluster encryption" - policy = data.aws_iam_policy_document.cluster_encryption.json + policy = data.aws_iam_policy_document.cluster_encryption[count.index].json } # Grant the EKS Cluster role KMS permissions if a key-arn is specified resource "aws_iam_role_policy_attachment" "cluster_encryption" { count = var.eks_kms_arn != null ? 1 : 0 - policy_arn = aws_iam_policy.cluster_encryption.arn + policy_arn = aws_iam_policy.cluster_encryption[count.index].arn role = aws_iam_role.cluster.name } From 7f29ffcbaf709345b7d27efb3b0dab4c04bb2a1c Mon Sep 17 00:00:00 2001 From: joneszc Date: Tue, 17 Sep 2024 11:18:28 -0400 Subject: [PATCH 3/8] Add config option (amazon_web_services.eks_kms_arn) to specify KMS-key ARN to encrypt EKS cluster secrets --- .../template/aws/modules/kubernetes/variables.tf | 4 ++-- src/_nebari/stages/infrastructure/template/aws/variables.tf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/variables.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/variables.tf index 318337fc84..07b416bc38 100644 --- a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/variables.tf +++ b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/variables.tf @@ -72,8 +72,8 @@ variable "endpoint_private_access" { variable "eks_kms_arn" { description = "kms key arn for EKS cluster encryption_config" - type = string - default = null + type = string + default = null } variable "public_access_cidrs" { diff --git a/src/_nebari/stages/infrastructure/template/aws/variables.tf b/src/_nebari/stages/infrastructure/template/aws/variables.tf index 1f92f4c102..a0a45cabad 100644 --- a/src/_nebari/stages/infrastructure/template/aws/variables.tf +++ b/src/_nebari/stages/infrastructure/template/aws/variables.tf @@ -69,8 +69,8 @@ variable "eks_endpoint_private_access" { variable "eks_kms_arn" { description = "kms key arn for EKS cluster encryption_config" - type = string - default = null + type = string + default = null } variable "eks_public_access_cidrs" { From e13fdb311e0889601488ebe35d77951c8a49ba3c Mon Sep 17 00:00:00 2001 From: joneszc Date: Fri, 27 Sep 2024 17:30:22 -0400 Subject: [PATCH 4/8] Add validation checks for config option amazon_web_services.eks_kms_arn to ensure KMS-key ARN available --- .../provider/cloud/amazon_web_services.py | 24 +++++++++++++++++++ src/_nebari/stages/infrastructure/__init__.py | 19 +++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/src/_nebari/provider/cloud/amazon_web_services.py b/src/_nebari/provider/cloud/amazon_web_services.py index 1123c07fe0..f057b8ab19 100644 --- a/src/_nebari/provider/cloud/amazon_web_services.py +++ b/src/_nebari/provider/cloud/amazon_web_services.py @@ -121,6 +121,30 @@ def instances(region: str) -> Dict[str, str]: return {t: t for t in instance_types} +@functools.lru_cache() +def kms_key_arns(region: str) -> Dict[str, dict]: + """Return dict of available/enabled KMS key IDs and associated KeyMetadata for the AWS region.""" + session = aws_session(region=region) + client = session.client("kms") + paginator = client.get_paginator("list_keys") + schema = [ + "Arn", + "KeyUsage", + "KeyState", + "Origin", + "KeyManager", + "KeySpec", + "EncryptionAlgorithms", + "MultiRegion", + ] + kms_keys = [ + client.describe_key(KeyId=j["KeyId"]).get("KeyMetadata") + for i in paginator.paginate() + for j in i["Keys"] + ] + return {i["KeyId"]: {k: i[k] for k in schema} for i in kms_keys if i["Enabled"]} + + def aws_get_vpc_id(name: str, namespace: str, region: str) -> Optional[str]: """Return VPC ID for the EKS cluster namedd `{name}-{namespace}`.""" cluster_name = f"{name}-{namespace}" diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 49e4795612..37365829e6 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -562,6 +562,25 @@ def _check_input(cls, data: Any) -> Any: f"Amazon Web Services instance {node_group.instance} not one of available instance types={available_instances}" ) + # check if kms key is valid + available_kms_keys = amazon_web_services.kms_key_arns(data["region"]) + if "eks_kms_arn" in data: + key_id = [id for id in available_kms_keys.keys() if id in data["eks_kms_arn"]] + if len(key_id) == 1 and available_kms_keys[key_id[0]]["Arn"] == data["eks_kms_arn"]: + key_id = key_id[0] + if available_kms_keys[key_id]["KeyUsage"] != "ENCRYPT_DECRYPT": + raise ValueError( + f"Amazon Web Services KMS Key with ID {key_id} does not have KeyUsage configured to encrypt and decrypt data" + ) + if available_kms_keys[key_id]["KeySpec"] != "SYMMETRIC_DEFAULT": + raise ValueError( + f"Amazon Web Services KMS Key with ID {key_id} is not a Symmetric key" + ) + else: + raise ValueError( + f"Amazon Web Services KMS Key with ARN {data['eks_kms_arn']} not one of available/enabled keys={[v['Arn'] for v in available_kms_keys.values()]}" + ) + return data From a89989c7a2e71f4e356e4161f05b3f57fe2c4711 Mon Sep 17 00:00:00 2001 From: joneszc Date: Fri, 27 Sep 2024 17:48:30 -0400 Subject: [PATCH 5/8] Add validation checks for config option amazon_web_services.eks_kms_arn to ensure KMS-key ARN available --- src/_nebari/stages/infrastructure/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 37365829e6..0c1f57bd6f 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -565,8 +565,13 @@ def _check_input(cls, data: Any) -> Any: # check if kms key is valid available_kms_keys = amazon_web_services.kms_key_arns(data["region"]) if "eks_kms_arn" in data: - key_id = [id for id in available_kms_keys.keys() if id in data["eks_kms_arn"]] - if len(key_id) == 1 and available_kms_keys[key_id[0]]["Arn"] == data["eks_kms_arn"]: + key_id = [ + id for id in available_kms_keys.keys() if id in data["eks_kms_arn"] + ] + if ( + len(key_id) == 1 + and available_kms_keys[key_id[0]]["Arn"] == data["eks_kms_arn"] + ): key_id = key_id[0] if available_kms_keys[key_id]["KeyUsage"] != "ENCRYPT_DECRYPT": raise ValueError( From f31f209185841253e37ef33af04abe8a89a21d4a Mon Sep 17 00:00:00 2001 From: joneszc Date: Wed, 2 Oct 2024 14:20:31 -0400 Subject: [PATCH 6/8] Add validation checks for config option amazon_web_services.eks_kms_arn to ensure KMS-key ARN available --- .../provider/cloud/amazon_web_services.py | 14 +++++------ src/_nebari/stages/infrastructure/__init__.py | 25 ++++++++++++------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/src/_nebari/provider/cloud/amazon_web_services.py b/src/_nebari/provider/cloud/amazon_web_services.py index f057b8ab19..8fb9c0be39 100644 --- a/src/_nebari/provider/cloud/amazon_web_services.py +++ b/src/_nebari/provider/cloud/amazon_web_services.py @@ -127,22 +127,22 @@ def kms_key_arns(region: str) -> Dict[str, dict]: session = aws_session(region=region) client = session.client("kms") paginator = client.get_paginator("list_keys") - schema = [ + fields = [ "Arn", "KeyUsage", - "KeyState", - "Origin", - "KeyManager", "KeySpec", - "EncryptionAlgorithms", - "MultiRegion", + #"KeyState", + #"Origin", + #"KeyManager", + #"EncryptionAlgorithms", + #"MultiRegion", ] kms_keys = [ client.describe_key(KeyId=j["KeyId"]).get("KeyMetadata") for i in paginator.paginate() for j in i["Keys"] ] - return {i["KeyId"]: {k: i[k] for k in schema} for i in kms_keys if i["Enabled"]} + return {i["KeyId"]: {k: i[k] for k in fields} for i in kms_keys if i["Enabled"]} def aws_get_vpc_id(name: str, namespace: str, region: str) -> Optional[str]: diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index d2287c7dc2..4403ae302f 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -564,7 +564,7 @@ def _check_input(cls, data: Any) -> Any: # check if kms key is valid available_kms_keys = amazon_web_services.kms_key_arns(data["region"]) - if "eks_kms_arn" in data: + if "eks_kms_arn" in data and data["eks_kms_arn"] is not None: key_id = [ id for id in available_kms_keys.keys() if id in data["eks_kms_arn"] ] @@ -573,14 +573,21 @@ def _check_input(cls, data: Any) -> Any: and available_kms_keys[key_id[0]]["Arn"] == data["eks_kms_arn"] ): key_id = key_id[0] - if available_kms_keys[key_id]["KeyUsage"] != "ENCRYPT_DECRYPT": - raise ValueError( - f"Amazon Web Services KMS Key with ID {key_id} does not have KeyUsage configured to encrypt and decrypt data" - ) - if available_kms_keys[key_id]["KeySpec"] != "SYMMETRIC_DEFAULT": - raise ValueError( - f"Amazon Web Services KMS Key with ID {key_id} is not a Symmetric key" - ) + # Symmetric KMS keys with Encrypt and decrypt key-usage have the SYMMETRIC_DEFAULT key-spec + # EKS cluster encryption requires a Symmetric key that is set to encrypt and decrypt data + if available_kms_keys[key_id]["KeySpec"] is not "SYMMETRIC_DEFAULT": + if available_kms_keys[key_id]["KeyUsage"] is "GENERATE_VERIFY_MAC": + raise ValueError( + f"Amazon Web Services KMS Key with ID {key_id} does not have KeyUsage set to 'Encrypt and decrypt' data" + ) + elif available_kms_keys[key_id]["KeyUsage"] is not "ENCRYPT_DECRYPT": + raise ValueError( + f"Amazon Web Services KMS Key with ID {key_id} is not of type Symmetric, and KeyUsage not set to 'Encrypt and decrypt' data" + ) + else: + raise ValueError( + f"Amazon Web Services KMS Key with ID {key_id} is not of type Symmetric" + ) else: raise ValueError( f"Amazon Web Services KMS Key with ARN {data['eks_kms_arn']} not one of available/enabled keys={[v['Arn'] for v in available_kms_keys.values()]}" From cb391065eee7af4cd790ab39f34d5911b24052c6 Mon Sep 17 00:00:00 2001 From: joneszc Date: Wed, 2 Oct 2024 15:04:13 -0400 Subject: [PATCH 7/8] Add validation checks for config option amazon_web_services.eks_kms_arn to ensure KMS-key ARN available --- src/_nebari/provider/cloud/amazon_web_services.py | 10 +++++----- src/_nebari/stages/infrastructure/__init__.py | 8 +++++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/_nebari/provider/cloud/amazon_web_services.py b/src/_nebari/provider/cloud/amazon_web_services.py index 8fb9c0be39..3a491c8b26 100644 --- a/src/_nebari/provider/cloud/amazon_web_services.py +++ b/src/_nebari/provider/cloud/amazon_web_services.py @@ -131,11 +131,11 @@ def kms_key_arns(region: str) -> Dict[str, dict]: "Arn", "KeyUsage", "KeySpec", - #"KeyState", - #"Origin", - #"KeyManager", - #"EncryptionAlgorithms", - #"MultiRegion", + # "KeyState", + # "Origin", + # "KeyManager", + # "EncryptionAlgorithms", + # "MultiRegion", ] kms_keys = [ client.describe_key(KeyId=j["KeyId"]).get("KeyMetadata") diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 4403ae302f..463360b19f 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -575,12 +575,14 @@ def _check_input(cls, data: Any) -> Any: key_id = key_id[0] # Symmetric KMS keys with Encrypt and decrypt key-usage have the SYMMETRIC_DEFAULT key-spec # EKS cluster encryption requires a Symmetric key that is set to encrypt and decrypt data - if available_kms_keys[key_id]["KeySpec"] is not "SYMMETRIC_DEFAULT": - if available_kms_keys[key_id]["KeyUsage"] is "GENERATE_VERIFY_MAC": + if available_kms_keys[key_id]["KeySpec"] != "SYMMETRIC_DEFAULT": + if available_kms_keys[key_id]["KeyUsage"] == "GENERATE_VERIFY_MAC": raise ValueError( f"Amazon Web Services KMS Key with ID {key_id} does not have KeyUsage set to 'Encrypt and decrypt' data" ) - elif available_kms_keys[key_id]["KeyUsage"] is not "ENCRYPT_DECRYPT": + elif ( + available_kms_keys[key_id]["KeyUsage"] != "ENCRYPT_DECRYPT" + ): raise ValueError( f"Amazon Web Services KMS Key with ID {key_id} is not of type Symmetric, and KeyUsage not set to 'Encrypt and decrypt' data" ) From 385f7aa48fc7ebc28eb3fb016d6dd4c97a6d7f1b Mon Sep 17 00:00:00 2001 From: joneszc Date: Wed, 2 Oct 2024 15:21:01 -0400 Subject: [PATCH 8/8] Add validation checks for config option amazon_web_services.eks_kms_arn to ensure KMS-key ARN available --- src/_nebari/stages/infrastructure/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 463360b19f..b043f84bd4 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -580,9 +580,7 @@ def _check_input(cls, data: Any) -> Any: raise ValueError( f"Amazon Web Services KMS Key with ID {key_id} does not have KeyUsage set to 'Encrypt and decrypt' data" ) - elif ( - available_kms_keys[key_id]["KeyUsage"] != "ENCRYPT_DECRYPT" - ): + elif available_kms_keys[key_id]["KeyUsage"] != "ENCRYPT_DECRYPT": raise ValueError( f"Amazon Web Services KMS Key with ID {key_id} is not of type Symmetric, and KeyUsage not set to 'Encrypt and decrypt' data" )