configure msk

examples/public-dns-external/custom.tf

@@ -1,47 +1,8 @@
-locals {
-  infra_outputs           = data.terraform_remote_state.infra.outputs
-  gcp_credentials         = local.infra_outputs.deployments_credentials
-  aws_deployment_role_arn = local.infra_outputs.deployments_aws_role_arn
-  region = "us-west-1"
-}
-
-
-data "terraform_remote_state" "infra" {
-  backend = "remote"
-  config = {
+terraform {
+  cloud {
     organization = "weights-and-biases"
-    workspaces   = { name = "deployer-global" }
-  }
-}
-
-provider "aws" {
-  region     = local.region
-  access_key = module.aws_credentials.access_key
-  secret_key = module.aws_credentials.secret_key
-  token      = module.aws_credentials.token
-
-  default_tags {
-    tags = {
-      Owner          = "Deployer"
-      Namespace      = var.namespace
+    workspaces {
+      name = "apple-replica-msk"
     }
   }
-}
-
-# Login using the deployment service account.
-provider "google" {
-  project     = "wandb-production"
-  region      = "us-central1"
-  zone        = "us-central1-c"
-  credentials = local.gcp_credentials
-}
-
-# Create AWS credentials from GCP account
-module "aws_credentials" {
-  source  = "wandb/assume-aws-role/google"
-  version = "1.1.0"
-
-  duration_seconds = 43200 # 12 hours
-  role_arn         = local.aws_deployment_role_arn
-  session_name     = "TerraformDeployment"
-}
+}

examples/public-dns-external/main.tf

@@ -1,15 +1,15 @@
-# provider "aws" {
-#   region = "us-west-2"
-
-#   default_tags {
-#     tags = {
-#       GithubRepo = "terraform-aws-wandb"
-#       GithubOrg  = "wandb"
-#       Enviroment = "Example"
-#       Example    = "PublicDnsExternal"
-#     }
-#   }
-# }
+provider "aws" {
+  region = "us-west-2"
+
+  default_tags {
+    tags = {
+      GithubRepo = "terraform-aws-wandb"
+      GithubOrg  = "wandb"
+      Enviroment = "Example"
+      Example    = "PublicDnsExternal"
+    }
+  }
+}
 
 module "wandb_infra" {
   source = "../../"

main.tf

@@ -44,6 +44,7 @@ module "networking" {
   elasticache_subnet_cidrs  = var.network_elasticache_subnet_cidrs
 }
 
+
 locals {
   network_id             = var.create_vpc ? module.networking.vpc_id : var.network_id
   network_public_subnets = var.create_vpc ? module.networking.public_subnets : var.network_public_subnets
@@ -59,6 +60,14 @@ locals {
   network_elasticache_subnet_group_name = module.networking.elasticache_subnet_group_name
 }
 
+module "msk" {
+  source    = "./modules/msk"
+  namespace = var.namespace
+
+  private_subnets = local.network_private_subnets
+  vpc_id          = local.network_id
+}
+
 module "database" {
   source = "./modules/database"
 

modules/msk/main.tf

@@ -1,47 +1,56 @@
-resource "aws_security_group" "msk_brokers_sg" {
-  name        = "msk-brokers-sg"
-  vpc_id      = data.aws_vpc.existing_vpc.id
-  description = "Security group for MSK brokers"
+# Security group for MSK (allows traffic within your VPC)
+resource "aws_security_group" "msk" {
+  name        = "${var.namespace}-msk-sg"
+  vpc_id      = var.vpc_id
+  description = "Allow MSK traffic within the VPC"
 
-  # Restrict inbound traffic to only necessary ports from your VPC CIDR
   ingress {
-    from_port   = 2181 # Zookeeper
-    to_port     = 2181
-    protocol    = "tcp"
-    cidr_blocks = [data.aws_vpc.existing_vpc.cidr_block] 
+    from_port = 9092
+    to_port   = 9092
+    protocol  = "tcp"
+    self      = true
   }
 
-  # Add more ingress rules as needed for monitoring, etc.
-
   egress {
     from_port   = 0
     to_port     = 0
     protocol    = "-1"
     cidr_blocks = ["0.0.0.0/0"]
   }
-
-  tags = {
-    Name = "msk-brokers-sg"
-  }
 }
 
 resource "aws_msk_cluster" "default" {
-  cluster_name                   = "${var.namespace}"
-  kafka_version          = "3.4.0" # Choose your desired Kafka version
-  number_of_broker_nodes = 3
+  cluster_name           = var.namespace
+  kafka_version          = "3.6.0"
+  number_of_broker_nodes = length(var.private_subnets)
 
   broker_node_group_info {
-    instance_type    = "kafka.m5.large"  # Adjust instance type as needed
-    client_subnets   = data.aws_subnets.private_subnets.ids
-    security_groups  = [aws_security_group.msk_brokers_sg.id]
-    # ebs_volume_size  = 50 # In GB
+    instance_type = "kafka.m5.large"
+
+    client_subnets  = var.private_subnets
+    security_groups = [aws_security_group.msk.id]
+
+    storage_info {
+      ebs_storage_info {
+        volume_size = 20
+      }
+    }
   }
 
   encryption_info {
     encryption_in_transit {
-      client_broker = "TLS" 
+      client_broker = "TLS"
     }
   }
 
-  depends_on = [aws_security_group.msk_brokers_sg] 
+  depends_on = [aws_security_group.msk]
+}
+
+output "zookeeper_connect_string" {
+  value = aws_msk_cluster.default.zookeeper_connect_string
+}
+
+output "bootstrap_brokers_tls" {
+  description = "TLS connection host:port pairs"
+  value       = aws_msk_cluster.default.bootstrap_brokers_tls
 }

modules/msk/variables.tf

@@ -1,3 +1,11 @@
 variable "namespace" {
   type = string
+}
+
+variable "vpc_id" {
+  type = string
+}
+
+variable "private_subnets" {
+  type = list(string)
 }