diff --git a/examples/public-dns-external/custom.tf b/examples/public-dns-external/custom.tf index 63b1f2219..e128d9d5a 100644 --- a/examples/public-dns-external/custom.tf +++ b/examples/public-dns-external/custom.tf @@ -1,47 +1,8 @@ -locals { - infra_outputs = data.terraform_remote_state.infra.outputs - gcp_credentials = local.infra_outputs.deployments_credentials - aws_deployment_role_arn = local.infra_outputs.deployments_aws_role_arn - region = "us-west-1" -} - - -data "terraform_remote_state" "infra" { - backend = "remote" - config = { +terraform { + cloud { organization = "weights-and-biases" - workspaces = { name = "deployer-global" } - } -} - -provider "aws" { - region = local.region - access_key = module.aws_credentials.access_key - secret_key = module.aws_credentials.secret_key - token = module.aws_credentials.token - - default_tags { - tags = { - Owner = "Deployer" - Namespace = var.namespace + workspaces { + name = "apple-replica-msk" } } -} - -# Login using the deployment service account. -provider "google" { - project = "wandb-production" - region = "us-central1" - zone = "us-central1-c" - credentials = local.gcp_credentials -} - -# Create AWS credentials from GCP account -module "aws_credentials" { - source = "wandb/assume-aws-role/google" - version = "1.1.0" - - duration_seconds = 43200 # 12 hours - role_arn = local.aws_deployment_role_arn - session_name = "TerraformDeployment" -} +} \ No newline at end of file diff --git a/examples/public-dns-external/main.tf b/examples/public-dns-external/main.tf index bc4dbc93f..ab506adae 100644 --- a/examples/public-dns-external/main.tf +++ b/examples/public-dns-external/main.tf @@ -1,15 +1,15 @@ -# provider "aws" { -# region = "us-west-2" - -# default_tags { -# tags = { -# GithubRepo = "terraform-aws-wandb" -# GithubOrg = "wandb" -# Enviroment = "Example" -# Example = "PublicDnsExternal" -# } -# } -# } +provider "aws" { + region = "us-west-2" + + default_tags { + tags = { + GithubRepo = "terraform-aws-wandb" + GithubOrg = "wandb" + Enviroment = "Example" + Example = "PublicDnsExternal" + } + } +} module "wandb_infra" { source = "../../" diff --git a/main.tf b/main.tf index 14bafe24e..271af9450 100644 --- a/main.tf +++ b/main.tf @@ -44,6 +44,7 @@ module "networking" { elasticache_subnet_cidrs = var.network_elasticache_subnet_cidrs } + locals { network_id = var.create_vpc ? module.networking.vpc_id : var.network_id network_public_subnets = var.create_vpc ? module.networking.public_subnets : var.network_public_subnets @@ -59,6 +60,14 @@ locals { network_elasticache_subnet_group_name = module.networking.elasticache_subnet_group_name } +module "msk" { + source = "./modules/msk" + namespace = var.namespace + + private_subnets = local.network_private_subnets + vpc_id = local.network_id +} + module "database" { source = "./modules/database" diff --git a/modules/msk/main.tf b/modules/msk/main.tf index 5610b4a20..a53a4848d 100644 --- a/modules/msk/main.tf +++ b/modules/msk/main.tf @@ -1,47 +1,56 @@ -resource "aws_security_group" "msk_brokers_sg" { - name = "msk-brokers-sg" - vpc_id = data.aws_vpc.existing_vpc.id - description = "Security group for MSK brokers" +# Security group for MSK (allows traffic within your VPC) +resource "aws_security_group" "msk" { + name = "${var.namespace}-msk-sg" + vpc_id = var.vpc_id + description = "Allow MSK traffic within the VPC" - # Restrict inbound traffic to only necessary ports from your VPC CIDR ingress { - from_port = 2181 # Zookeeper - to_port = 2181 - protocol = "tcp" - cidr_blocks = [data.aws_vpc.existing_vpc.cidr_block] + from_port = 9092 + to_port = 9092 + protocol = "tcp" + self = true } - # Add more ingress rules as needed for monitoring, etc. - egress { from_port = 0 to_port = 0 protocol = "-1" cidr_blocks = ["0.0.0.0/0"] } - - tags = { - Name = "msk-brokers-sg" - } } resource "aws_msk_cluster" "default" { - cluster_name = "${var.namespace}" - kafka_version = "3.4.0" # Choose your desired Kafka version - number_of_broker_nodes = 3 + cluster_name = var.namespace + kafka_version = "3.6.0" + number_of_broker_nodes = length(var.private_subnets) broker_node_group_info { - instance_type = "kafka.m5.large" # Adjust instance type as needed - client_subnets = data.aws_subnets.private_subnets.ids - security_groups = [aws_security_group.msk_brokers_sg.id] - # ebs_volume_size = 50 # In GB + instance_type = "kafka.m5.large" + + client_subnets = var.private_subnets + security_groups = [aws_security_group.msk.id] + + storage_info { + ebs_storage_info { + volume_size = 20 + } + } } encryption_info { encryption_in_transit { - client_broker = "TLS" + client_broker = "TLS" } } - depends_on = [aws_security_group.msk_brokers_sg] + depends_on = [aws_security_group.msk] +} + +output "zookeeper_connect_string" { + value = aws_msk_cluster.default.zookeeper_connect_string +} + +output "bootstrap_brokers_tls" { + description = "TLS connection host:port pairs" + value = aws_msk_cluster.default.bootstrap_brokers_tls } \ No newline at end of file diff --git a/modules/msk/variables.tf b/modules/msk/variables.tf index 6857747f6..a1ac111d5 100644 --- a/modules/msk/variables.tf +++ b/modules/msk/variables.tf @@ -1,3 +1,11 @@ variable "namespace" { type = string +} + +variable "vpc_id" { + type = string +} + +variable "private_subnets" { + type = list(string) } \ No newline at end of file