Implement Replace and Route as Step Function with Lambdas

Core/EC2/RDSBastion/main.tf

@@ -354,7 +354,7 @@ data "cloudinit_config" "startup" {
 
   part {
     content_type = "text/x-shellscript"
-    filename     = "4_viz_setup_foreign_tables.sh"
+    filename     = "4a_viz_setup_foreign_tables_external.sh"
     content      = templatefile("${path.module}/scripts/utils/setup_foreign_tables.tftpl", {
       db_name             = local.dbs["viz"]["db_name"]
       db_host             = local.dbs["viz"]["db_host"]
@@ -372,6 +372,26 @@ data "cloudinit_config" "startup" {
     })
   }
 
+  part {
+    content_type = "text/x-shellscript"
+    filename     = "4b_viz_setup_foreign_tables_rfcfcst.sh"
+    content      = templatefile("${path.module}/scripts/utils/setup_foreign_tables.tftpl", {
+      db_name             = local.dbs["viz"]["db_name"]
+      db_host             = local.dbs["viz"]["db_host"]
+      db_port             = local.dbs["viz"]["db_port"]
+      db_username         = local.dbs["viz"]["db_username"]
+      db_password         = local.dbs["viz"]["db_password"]
+      db_schema           = "wrds_rfcfcst"
+      foreign_db_name     = local.dbs["forecast"]["db_name"]
+      foreign_db_host     = local.dbs["forecast"]["db_host"]
+      foreign_db_port     = local.dbs["forecast"]["db_port"]
+      foreign_db_username = local.dbs["forecast"]["db_username"]
+      foreign_db_password = local.dbs["forecast"]["db_password"]
+      foreign_server      = "wrds_rfcfcst"
+      user_mappings       = [jsondecode(var.viz_proc_admin_rw_secret_string)["username"]]
+    })
+  }
+
   part {
     content_type = "text/x-shellscript"
     filename     = "5_egis_postgresql_setup.sh"

Core/EC2/rnr/templates/install.sh.tftpl

@@ -76,7 +76,9 @@ sudo tar -zxvf rnr_static.tgz
 sudo chown root static
 sudo chgrp root static
 sudo mkdir /rnr/owp-viz-replace-route/RESOURCES
+sudo mkdir /rnr/owp-viz-replace-route/RESOURCES_FROM_S3
 sudo cp -r static /rnr/owp-viz-replace-route/RESOURCES/
+sudo ln -s /rnr/owp-viz-replace-route/RESOURCES/static /rnr/owp-viz-replace-route/RESOURCES_FROM_S3/static
 sudo rm rnr_static.tgz
 
 echo "Updating Libnetcdf Link"
@@ -90,15 +92,15 @@ sudo mkdir /rnr/share
 sudo mkdir /rnr/share/log
 sudo mkdir /rnr/share/.archive
 sudo mkdir /rnr/share/Run
+sudo mkdir /rnr/share_from_s3
+sudo mkdir /rnr/share_from_s3/log
+sudo mkdir /rnr/share_from_s3/.archive
+sudo mkdir /rnr/share_from_s3/Run
+sudo ln -s /rn
 
 echo "Copying Rendered Template File to Replace and Route"
 sudo cp /deploy_files/conus.ini /rnr/owp-viz-replace-route/configs/conus.ini
 sudo cp /deploy_files/.env.devel /rnr/owp-viz-replace-route/.env.devel
 sudo dos2unix /rnr/owp-viz-replace-route/.env.devel
 
-echo "Setting up RNR Crontab"
-sudo crontab -l -u ec2-user > /tmp/mycrontab
-echo '10 * * * * cd /rnr/owp-viz-replace-route && sudo ./run.sh' >> /tmp/mycrontab
-sudo crontab -u ec2-user /tmp/mycrontab
-
 echo "Finished Setup"

Core/EventBridge/main.tf

@@ -1,58 +1,27 @@
-variable "scheduled_rules" {
-  type = map(map(string))
-}
-
-
 ###########################################
 ## All Scheduled Event EventBridge Rules ##
 ###########################################
 
-resource "aws_cloudwatch_event_rule" "scheduled_events" {
-  for_each = var.scheduled_rules
-
-  name                = each.key
-  description         = each.value.description
-  schedule_expression = each.value.schedule_expression
+resource "aws_cloudwatch_event_rule" "every_five_minutes" {
+  name                = "every_five_minutes"
+  description         = "Fires every 5 minutes"
+  schedule_expression = "cron(0/5 * * * ? *)"
 }
 
-#################################
-## Get Lambda Resource By Name ##
-#################################
-
-data "aws_lambda_function" "scheduled_rules_function" {
-  for_each = var.scheduled_rules
-
-  function_name = each.value.function_name
+resource "aws_cloudwatch_event_rule" "every_fifteen_minutes" {
+  name                = "every_fifteen_minutes"
+  description         = "Fires every 15 minutes"
+  schedule_expression = "cron(0/15 * * * ? *)"
 }
 
-###############################
-## Initialize Lambda Target  ##
-###############################
-
-resource "aws_cloudwatch_event_target" "eventbridge_targets" {
-  for_each = var.scheduled_rules
-
-  rule      = aws_cloudwatch_event_rule.scheduled_events[each.key].name
-  target_id = data.aws_lambda_function.scheduled_rules_function[each.key].function_name
-  arn       = data.aws_lambda_function.scheduled_rules_function[each.key].arn
-}
-
-resource "aws_lambda_permission" "allow_eventbridge_targets" {
-  for_each = var.scheduled_rules
-
-  statement_id_prefix  = "AllowExecutionFromCloudWatch"
-  action               = "lambda:InvokeFunction"
-  function_name        = data.aws_lambda_function.scheduled_rules_function[each.key].function_name
-  principal            = "events.amazonaws.com"
-  source_arn           = aws_cloudwatch_event_rule.scheduled_events[each.key].arn
-}
-
-
-
 #############
 ## Outputs ##
 #############
 
-output "scheduled_eventbridge_rules" {
-  value = { for k, v in resource.aws_cloudwatch_event_rule.scheduled_events : k => v }
+output "five_minute_eventbridge" {
+  value = aws_cloudwatch_event_rule.every_five_minutes
+}
+
+output "fifteen_minute_eventbridge" {
+  value = aws_cloudwatch_event_rule.every_fifteen_minutes
 }

Core/LAMBDA/rnr_functions/main.tf

@@ -0,0 +1,126 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      configuration_aliases = [ aws.sns ]
+    }
+  }
+}
+
+variable "environment" {
+  description = "Hydrovis environment"
+  type        = string
+}
+
+variable "region" {
+  description = "Hydrovis environment"
+  type        = string
+}
+
+variable "deployment_bucket" {
+  type = string
+}
+
+variable "db_lambda_security_groups" {
+  description = "Security group for the rnr lambdas."
+  type        = list(any)
+}
+
+variable "db_lambda_subnets" {
+  description = "Subnets to use for the rnr lambdas."
+  type        = list(any)
+}
+
+variable "viz_db_host" {
+  description = "Hostname of the viz processing RDS instance."
+  type        = string
+}
+
+variable "viz_db_name" {
+  description = "DB Name of the viz processing RDS instance."
+  type        = string
+}
+
+variable "viz_db_user_secret_string" {
+  description = "The secret string of the viz_processing data base user to write/read data as."
+  type        = string
+}
+
+variable "rnr_data_bucket" {
+  description = "S3 bucket where the rnr max flows will live."
+  type        = string
+}
+
+variable "lambda_role" {
+  description = "Role to use for the lambda functions."
+  type        = string
+}
+
+variable "xarray_layer" {
+  type = string
+}
+
+variable "psycopg2_sqlalchemy_layer" {
+  type = string
+}
+
+variable "viz_lambda_shared_funcs_layer" {
+  type = string
+}
+
+#############################
+##    REPLACE AND ROUTE    ##
+#############################
+data "archive_file" "rnr_domain_generator_zip" {
+  type = "zip"
+
+  source_dir = "${path.module}/rnr_domain_generator"
+
+  output_path = "${path.module}/temp/rnr_domain_generator_${var.environment}_${var.region}.zip"
+}
+
+resource "aws_s3_object" "rnr_domain_generator_zip_upload" {
+  bucket      = var.deployment_bucket
+  key         = "terraform_artifacts/${path.module}/rnr_domain_generator.zip"
+  source      = data.archive_file.rnr_domain_generator_zip.output_path
+  source_hash = data.archive_file.rnr_domain_generator_zip.output_md5
+}
+
+resource "aws_lambda_function" "rnr_domain_generator" {
+  function_name = "hv-vpp-${var.environment}-rnr-domain-generator"
+  description   = "Lambda function to run Replace and Route model."
+  memory_size   = 128
+  timeout       = 900
+  vpc_config {
+    security_group_ids = var.db_lambda_security_groups
+    subnet_ids         = var.db_lambda_subnets
+  }
+  environment {
+    variables = {
+      VIZ_DB_DATABASE = var.viz_db_name
+      VIZ_DB_HOST     = var.viz_db_host
+      VIZ_DB_USERNAME = jsondecode(var.viz_db_user_secret_string)["username"]
+      VIZ_DB_PASSWORD = jsondecode(var.viz_db_user_secret_string)["password"]
+      OUTPUT_BUCKET   = var.rnr_data_bucket
+      OUTPUT_PREFIX   = "rnr_runs"
+    }
+  }
+  s3_bucket        = aws_s3_object.rnr_domain_generator_zip_upload.bucket
+  s3_key           = aws_s3_object.rnr_domain_generator_zip_upload.key
+  source_code_hash = filebase64sha256(data.archive_file.rnr_domain_generator_zip.output_path)
+  runtime          = "python3.9"
+  handler          = "lambda_function.lambda_handler"
+  role             = var.lambda_role
+  layers = [
+    var.xarray_layer,
+    var.psycopg2_sqlalchemy_layer,
+    var.viz_lambda_shared_funcs_layer
+  ]
+  tags = {
+    "Name" = "rnr_domain_generator_${var.environment}"
+  }
+}
+
+output "rnr_domain_generator" {
+  value = aws_lambda_function.rnr_domain_generator
+}