Database Enhancements - Lock Up Fix (#520)

This PR includes a variety of minor Viz RDS Database / Pipeline tweaks, with the goal of preventing random memory crashes / lock-ups on the Viz database (more below). In short, it includes the following changes:

- Setting work_mem to 512MB on FIMPact spatial join SQL queries (this seems to solve the problem in most cases)
- Setting idle_in_transaction_session_timeout in the Viz RDS parameter group to 15 minutes (default was 1 day)
- Changed the drop table logic in ana past hour max flows query to truncate to prevent locks (this should hopefully also prevent the traffic jam, even if FIMpact does fail due to the memory crash)
- Removal of admin.ingest_status table logging (now redundant with step functions)
- It's not included in this PR, but we also requested the 24X7 team add a CloudWatch alarm to the viz database DB Load metric with a 15-minute average threshold of 7.

----------------

* Removal of admin ingest table logging (not using since step functions)

* Changing drop table on ana_past_hour to truncate to hopefully relieve risk of lock when srf_rate_of_change is taking a long time (I'm not really sure how this could happen, but I don't see any harm).

* Adding postgresql work_mem adjustment to fimpact spatial joins - this is something I was teting in the historic piplines, and it seems to improve the performance of these queries signifantly.

* Viz RDS Setting Change:  Idle transaction timeout changed from 1 day to 15 minutes (I'm not exactly sure if this is relevant, but don't think it hurts to try it).

Core/LAMBDA/viz_functions/viz_db_postprocess_sql/admin/ingest_finish.sql

@@ -1,11 +1,4 @@
 CREATE INDEX {index_name} ON {target_table} {index_columns};
 
 ALTER TABLE {target_table}
-ADD COLUMN reference_time TEXT DEFAULT '1900-01-01 00:00:00 UTC';
-
-UPDATE admin.ingest_status
-SET status = 'Import Complete',
-    update_time = now()::timestamp without time zone,
-    files_processed = {files_imported},
-    records_imported = {rows_imported}
-WHERE target = '{target_table}' AND reference_time = '1900-01-01 00:00:00';
+ADD COLUMN reference_time TEXT DEFAULT '1900-01-01 00:00:00 UTC';

Core/LAMBDA/viz_functions/viz_db_postprocess_sql/admin/ingest_prep.sql

@@ -1,7 +1,3 @@
---> Let the database know that ingest has started
-INSERT INTO admin.ingest_status (target, reference_time, status, update_time)
-VALUES ('{target_table}', '1900-01-01 00:00:00', 'Import Started', now()::timestamp without time zone);
-
 --> Drop target table index (if exists)
 DROP INDEX IF EXISTS {target_schema}.{index_name};
 

Core/LAMBDA/viz_functions/viz_db_postprocess_sql/max_flows/ana_max_flows.sql

@@ -1,8 +1,11 @@
 -- Create a past_hour ana table. This is an interim solution to Shawn's rate of change service.
 CREATE TABLE IF NOT EXISTS cache.max_flows_ana (feature_id bigint, reference_time timestamp without time zone, nwm_vers double precision, maxflow_1hour_cms double precision, maxflow_1hour_cfs double precision);
-DROP TABLE IF EXISTS cache.max_flows_ana_past_hour;
-SELECT * INTO cache.max_flows_ana_past_hour FROM cache.max_flows_ana;
+CREATE TABLE IF NOT EXISTS cache.max_flows_ana_past_hour (feature_id bigint, reference_time timestamp without time zone, nwm_vers double precision, maxflow_1hour_cms double precision, maxflow_1hour_cfs double precision);
+TRUNCATE TABLE cache.max_flows_ana_past_hour;
+INSERT INTO cache.max_flows_ana_past_hour
+SELECT * FROM cache.max_flows_ana;
 
+-- Regular ana max flows
 DROP TABLE IF EXISTS cache.max_flows_ana;
 
 SELECT forecasts.feature_id,

Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_inundation/building_footprints_fimpact.sql

@@ -1,3 +1,5 @@
+-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB)
+SET work_mem TO '512MB';
 --------------- Building Footprints ---------------
 DROP TABLE IF EXISTS publish.ana_inundation_building_footprints;
 SELECT

Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_inundation_hi/building_footprints_fimpact.sql

@@ -1,3 +1,5 @@
+-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB)
+SET work_mem TO '512MB';
 --------------- Building Footprints ---------------
 DROP TABLE IF EXISTS publish.ana_inundation_building_footprints_hi;
 SELECT

Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_inundation_prvi/building_footprints_fimpact.sql

@@ -1,3 +1,5 @@
+-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB)
+SET work_mem TO '512MB';
 --------------- Building Footprints ---------------
 DROP TABLE IF EXISTS publish.ana_inundation_building_footprints_prvi;
 SELECT

Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_past_14day_max_inundation/14day_building_footprints_fimpact.sql

@@ -1,3 +1,5 @@
+-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB)
+SET work_mem TO '512MB';
 --------------- Building Footprints ---------------
 DROP TABLE IF EXISTS publish.ana_past_14day_max_inundation_building_footprints;
 SELECT

Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_past_14day_max_inundation/7day_building_footprints_fimpact.sql

@@ -1,3 +1,5 @@
+-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB)
+SET work_mem TO '512MB';
 --------------- Building Footprints ---------------
 DROP TABLE IF EXISTS publish.ana_past_7day_max_inundation_building_footprints;
 SELECT

Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_gfs_10day_max_inundation/10day_building_footprints_fimpact.sql

@@ -1,3 +1,5 @@
+-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB)
+SET work_mem TO '512MB';
 --------------- Building Footprints ---------------
 DROP TABLE IF EXISTS publish.mrf_gfs_max_inundation_10day_building_footprints;
 SELECT

Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_gfs_10day_max_inundation/3day_building_footprints_fimpact.sql

@@ -1,3 +1,5 @@
+-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB)
+SET work_mem TO '512MB';
 --------------- Building Footprints ---------------
 DROP TABLE IF EXISTS publish.mrf_gfs_max_inundation_3day_building_footprints;
 SELECT

Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_gfs_10day_max_inundation/5day_building_footprints_fimpact.sql

@@ -1,3 +1,5 @@
+-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB)
+SET work_mem TO '512MB';
 --------------- Building Footprints ---------------
 DROP TABLE IF EXISTS publish.mrf_gfs_max_inundation_5day_building_footprints;
 SELECT

Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_nbm_10day_max_inundation/10day_building_footprints_fimpact.sql

@@ -1,3 +1,5 @@
+-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB)
+SET work_mem TO '512MB';
 --------------- Building Footprints ---------------
 DROP TABLE IF EXISTS publish.mrf_nbm_max_inundation_10day_building_footprints;
 SELECT

Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_nbm_10day_max_inundation/3day_building_footprints_fimpact.sql

@@ -1,3 +1,5 @@
+-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB)
+SET work_mem TO '512MB';
 --------------- Building Footprints ---------------
 DROP TABLE IF EXISTS publish.mrf_nbm_max_inundation_3day_building_footprints;
 SELECT

Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_nbm_10day_max_inundation/5day_building_footprints_fimpact.sql

@@ -1,3 +1,5 @@
+-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB)
+SET work_mem TO '512MB';
 --------------- Building Footprints ---------------
 DROP TABLE IF EXISTS publish.mrf_nbm_max_inundation_5day_building_footprints;
 SELECT

Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/rfc_based_5day_max_inundation/building_footprints_fimpact.sql

@@ -1,3 +1,5 @@
+-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB)
+SET work_mem TO '512MB';
 --------------- Building Footprints ---------------
 DROP TABLE IF EXISTS publish.rfc_based_5day_max_inundation_building_footprints;
 SELECT

Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/srf_18hr_max_inundation/building_footprints_fimpact.sql

@@ -1,3 +1,5 @@
+-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB)
+SET work_mem TO '512MB';
 --------------- Building Footprints ---------------
 DROP TABLE IF EXISTS publish.srf_18hr_max_inundation_building_footprints;
 SELECT

Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/srf_48hr_max_inundation_hi/building_footprints_fimpact.sql

@@ -1,3 +1,5 @@
+-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB)
+SET work_mem TO '512MB';
 --------------- Building Footprints ---------------
 DROP TABLE IF EXISTS publish.srf_48hr_max_inundation_building_footprints_hi;
 SELECT

Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/srf_48hr_max_inundation_prvi/building_footprints_fimpact.sql

@@ -1,3 +1,5 @@
+-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB)
+SET work_mem TO '512MB';
 --------------- Building Footprints ---------------
 DROP TABLE IF EXISTS publish.srf_48hr_max_inundation_building_footprints_prvi;
 SELECT

Core/RDS/viz/main.tf

@@ -55,6 +55,12 @@ resource "aws_db_parameter_group" "hydrovis" {
     value = "{DBInstanceClassMemory/10923}"
     apply_method = "pending-reboot"
   }
+
+  parameter {
+    name  = "idle_in_transaction_session_timeout"
+    value = "900000"
+    apply_method = "pending-reboot"
+  }
 
   parameter {
     name         = "rds.custom_dns_resolution"