Feature | data-science data-lake ref arch v1 + https://north.cloud ro…

…les added. (#646)

* Updating users, build.env and sso policy

* data-science data-lake reference architecture

* Updating data-science/aurora-mysql layer to integrate with secrets manager

* Adding data-science/secrets-manager layer to be used in the database-mysql layer and others

* Remove not necessary data-science/database-aurora-mysql export to s3 sub-layer

* Adding ref arch data-science/datalake-demo layer

* updating dms tf module source

* data-science/datalake-demo minor updates to dms module

* Adding data-lake readme.md

* renaming aurora mysql layer to reflect it's not provisioned

* data-science/datalake-demo updating README.md

* data-science/data-lake-demo layer sufix -- added since its not provisioned.

* https://north.cloud co tool roles integration

* data-science/glob/base-identities layer replaced by an empty placeholder.

* removing metal ec2 type in sso devops policy

* fixing conflict at devops policy

* Adding data-scinece mysql layer to infracost

apps-devstg/global/base-identities/policies.tf

@@ -532,4 +532,47 @@ data "aws_iam_policy_document" "lambda_costs_explorer_access" {
       "*"
     ]
   }
-}
+}
+
+resource "aws_iam_policy" "north_cloud_tool_access" {
+	name        = "NorthCostAndUsageReadOnlyPolicy"
+	description = "Read-only policy for North Inc. cost and usage"
+
+	policy = jsonencode ({
+		Version = "2012-10-17"
+		Statement = 	[
+		{
+			Sid    = "NorthCostAndUsageReadOnlyPolicyID"
+			Effect = "Allow"
+			Action = 	[
+			"ce:Get*",
+			"ce:Describe*",
+			"ce:List*",
+			"ce:Start*",
+			"account:GetAccountInformation",
+			"billing:Get*",
+			"payments:List*",
+			"payments:Get*",
+			"tax:List*",
+			"tax:Get*",
+			"consolidatedbilling:Get*",
+			"consolidatedbilling:List*",
+			"invoicing:List*",
+			"invoicing:Get*",
+			"cur:Get*",
+			"cur:Validate*",
+			"freetier:Get*",
+			"ec2:DescribeCapacity*",
+			"ec2:DescribeReservedInstances*",
+			"ec2:DescribeSpot*",
+			"rds:DescribeReserved*",
+			"rds:DescribeDBRecommendations",
+			"rds:DescribeAccountAttributes",
+			"ecs:DescribeCapacityProviders",
+			"es:DescribeReserved*"
+			]
+			Resource = "*"
+			}
+		]
+	})
+}

apps-devstg/global/base-identities/roles.tf

@@ -295,4 +295,34 @@ module "iam_assumable_role_lambda_costs_explorer_access" {
   ]
 
   tags = local.tags
-}
+}
+
+#
+# Assumable Role: north.cloud
+#
+module "iam_assumable_role_north_cloud_access" {
+  source = "github.com/binbashar/terraform-aws-iam.git//modules/iam-assumable-role?ref=v5.3.3"
+
+  trusted_role_arns = [
+    "arn:aws:iam::${var.external_accounts.north_cloud.aws_account_id}:root"
+  ]
+
+  create_role = true
+  role_name   = "NorthCostAndUsageRole"
+  role_path   = "/"
+
+  #
+  # MFA setup
+  #
+  role_requires_mfa    = false
+  mfa_age              = 86400 # Maximum CLI/API session duration in seconds between 3600 and 43200
+  max_session_duration = 10800 # Max age of the session (in seconds) when assuming roles
+  custom_role_policy_arns = [
+    aws_iam_policy.north_cloud_tool_access.arn,
+  ]
+
+  tags = local.tags
+}
+
+
+

apps-prd/global/base-identities/policies.tf

@@ -275,4 +275,47 @@ data "aws_iam_policy_document" "lambda_costs_explorer_access" {
       "*"
     ]
   }
-}
+}
+
+resource "aws_iam_policy" "north_cloud_tool_access" {
+	name        = "NorthCostAndUsageReadOnlyPolicy"
+	description = "Read-only policy for North Inc. cost and usage"
+
+	policy = jsonencode ({
+		Version = "2012-10-17"
+		Statement = 	[
+		{
+			Sid    = "NorthCostAndUsageReadOnlyPolicyID"
+			Effect = "Allow"
+			Action = 	[
+			"ce:Get*",
+			"ce:Describe*",
+			"ce:List*",
+			"ce:Start*",
+			"account:GetAccountInformation",
+			"billing:Get*",
+			"payments:List*",
+			"payments:Get*",
+			"tax:List*",
+			"tax:Get*",
+			"consolidatedbilling:Get*",
+			"consolidatedbilling:List*",
+			"invoicing:List*",
+			"invoicing:Get*",
+			"cur:Get*",
+			"cur:Validate*",
+			"freetier:Get*",
+			"ec2:DescribeCapacity*",
+			"ec2:DescribeReservedInstances*",
+			"ec2:DescribeSpot*",
+			"rds:DescribeReserved*",
+			"rds:DescribeDBRecommendations",
+			"rds:DescribeAccountAttributes",
+			"ecs:DescribeCapacityProviders",
+			"es:DescribeReserved*"
+			]
+			Resource = "*"
+			}
+		]
+	})
+}

apps-prd/global/base-identities/roles.tf

@@ -264,3 +264,30 @@ module "iam_assumable_role_drata_auditor" {
 
   tags = local.tags
 }
+
+#
+# Assumable Role: north.cloud
+#
+module "iam_assumable_role_north_cloud_access" {
+  source = "github.com/binbashar/terraform-aws-iam.git//modules/iam-assumable-role?ref=v5.3.3"
+
+  trusted_role_arns = [
+    "arn:aws:iam::${var.external_accounts.north_cloud.aws_account_id}:root"
+  ]
+
+  create_role = true
+  role_name   = "NorthCostAndUsageRole"
+  role_path   = "/"
+
+  #
+  # MFA setup
+  #
+  role_requires_mfa    = false
+  mfa_age              = 86400 # Maximum CLI/API session duration in seconds between 3600 and 43200
+  max_session_duration = 10800 # Max age of the session (in seconds) when assuming roles
+  custom_role_policy_arns = [
+    aws_iam_policy.north_cloud_tool_access.arn,
+  ]
+
+  tags = local.tags
+}

config/common.tfvars.example

@@ -52,6 +52,9 @@ external_accounts = {
     aws_account_id  = ""
     aws_external_id = ""
   }
+  north_cloud = {
+    aws_account_id  = ""
+  }
 }
 
 # AWS SSO

data-science/us-east-1/databases-aurora-mysql--/cluster_demoapps.tf

@@ -0,0 +1,77 @@
+#
+# NOTE: Before deploying make sure the required secret is created via apps-devstg/us-east-1/secrets-manager layer
+#
+
+#
+# DB Administrator secret
+#
+data "aws_secretsmanager_secret_version" "administrator" {
+  secret_id = data.terraform_remote_state.secrets.outputs.secret_ids["/aurora-mysql/administrator"]
+}
+
+module "demoapps" {
+  source = "github.com/binbashar/terraform-aws-rds-aurora.git?ref=v7.2.2"
+
+  # General settings
+  name           = "${var.project}-${var.environment}-binbash-aurora-mysql"
+  engine         = "aurora-mysql"
+  engine_mode    = "provisioned"
+  engine_version = "5.7"
+
+  # Initial database and credentials
+  database_name          = "demoapps"
+  master_username        = "admin"
+  master_password        = jsondecode(data.aws_secretsmanager_secret_version.administrator.secret_string)["password"]
+  create_random_password = false
+
+  # VPC and Subnets
+  vpc_id  = data.terraform_remote_state.vpc.outputs.vpc_id
+  subnets = data.terraform_remote_state.vpc.outputs.private_subnets
+
+  # Instance type and desired instances
+  instance_class = "db.t3.small"
+  instances = {
+    one = {}
+  }
+
+
+  # Autoscaling settings
+  autoscaling_enabled = false
+  # autoscaling_min_capacity        = 1
+  # autoscaling_max_capacity         = 3
+  # autoscaling_target_cpu         = 85
+  # autoscaling_target_connections = 200
+
+  # Storage encrypted as default
+  storage_encrypted = true
+
+  # Determines whether or not any DB modifications are applied immediately, or during the maintenance window
+  # Only 'true' in test environments
+  apply_immediately = true
+
+  # Automatic backup settings
+  backup_retention_period = 1
+  preferred_backup_window = "14:00-15:00"
+
+  # This avoid a snapshot before destroy the cluster
+  skip_final_snapshot = true
+
+  # Monitoring settings
+  # enabled_cloudwatch_logs_exports = ["audit", "error", "general", "slowquery"]
+
+  # Database parameters: you can specify your own if you must
+  # db_parameter_group_name         = aws_db_parameter_group.aurora_db_57_parameter_group.id
+  # db_cluster_parameter_group_name = aws_rds_cluster_parameter_group.aurora_57_cluster_parameter_group.id
+
+  # If true, must add policy to iam auth (user or role)
+  iam_database_authentication_enabled = false
+
+  # Security group settings
+  create_security_group = true
+  allowed_cidr_blocks = [
+    "0.0.0.0/0",
+    data.terraform_remote_state.shared_vpc.outputs.vpc_cidr_block
+  ]
+
+  tags = local.tags
+}