diff --git a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/admin/ingest_finish.sql b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/admin/ingest_finish.sql index b6d1779b..6ecbdf47 100644 --- a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/admin/ingest_finish.sql +++ b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/admin/ingest_finish.sql @@ -1,11 +1,4 @@ CREATE INDEX {index_name} ON {target_table} {index_columns}; ALTER TABLE {target_table} -ADD COLUMN reference_time TEXT DEFAULT '1900-01-01 00:00:00 UTC'; - -UPDATE admin.ingest_status -SET status = 'Import Complete', - update_time = now()::timestamp without time zone, - files_processed = {files_imported}, - records_imported = {rows_imported} -WHERE target = '{target_table}' AND reference_time = '1900-01-01 00:00:00'; \ No newline at end of file +ADD COLUMN reference_time TEXT DEFAULT '1900-01-01 00:00:00 UTC'; \ No newline at end of file diff --git a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/admin/ingest_prep.sql b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/admin/ingest_prep.sql index f0edb895..73d5ceb9 100644 --- a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/admin/ingest_prep.sql +++ b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/admin/ingest_prep.sql @@ -1,7 +1,3 @@ ---> Let the database know that ingest has started -INSERT INTO admin.ingest_status (target, reference_time, status, update_time) -VALUES ('{target_table}', '1900-01-01 00:00:00', 'Import Started', now()::timestamp without time zone); - --> Drop target table index (if exists) DROP INDEX IF EXISTS {target_schema}.{index_name}; diff --git a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/max_flows/ana_max_flows.sql b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/max_flows/ana_max_flows.sql index 21717b53..97aeea5f 100644 --- a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/max_flows/ana_max_flows.sql +++ b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/max_flows/ana_max_flows.sql @@ -1,8 +1,11 @@ -- Create a past_hour ana table. This is an interim solution to Shawn's rate of change service. CREATE TABLE IF NOT EXISTS cache.max_flows_ana (feature_id bigint, reference_time timestamp without time zone, nwm_vers double precision, maxflow_1hour_cms double precision, maxflow_1hour_cfs double precision); -DROP TABLE IF EXISTS cache.max_flows_ana_past_hour; -SELECT * INTO cache.max_flows_ana_past_hour FROM cache.max_flows_ana; +CREATE TABLE IF NOT EXISTS cache.max_flows_ana_past_hour (feature_id bigint, reference_time timestamp without time zone, nwm_vers double precision, maxflow_1hour_cms double precision, maxflow_1hour_cfs double precision); +TRUNCATE TABLE cache.max_flows_ana_past_hour; +INSERT INTO cache.max_flows_ana_past_hour +SELECT * FROM cache.max_flows_ana; +-- Regular ana max flows DROP TABLE IF EXISTS cache.max_flows_ana; SELECT forecasts.feature_id, diff --git a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_inundation/building_footprints_fimpact.sql b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_inundation/building_footprints_fimpact.sql index 51d424e8..58785246 100644 --- a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_inundation/building_footprints_fimpact.sql +++ b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_inundation/building_footprints_fimpact.sql @@ -1,3 +1,5 @@ +-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB) +SET work_mem TO '512MB'; --------------- Building Footprints --------------- DROP TABLE IF EXISTS publish.ana_inundation_building_footprints; SELECT diff --git a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_inundation_hi/building_footprints_fimpact.sql b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_inundation_hi/building_footprints_fimpact.sql index 546d91a7..6bec9cfb 100644 --- a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_inundation_hi/building_footprints_fimpact.sql +++ b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_inundation_hi/building_footprints_fimpact.sql @@ -1,3 +1,5 @@ +-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB) +SET work_mem TO '512MB'; --------------- Building Footprints --------------- DROP TABLE IF EXISTS publish.ana_inundation_building_footprints_hi; SELECT diff --git a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_inundation_prvi/building_footprints_fimpact.sql b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_inundation_prvi/building_footprints_fimpact.sql index 3b3acacf..9f9843c0 100644 --- a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_inundation_prvi/building_footprints_fimpact.sql +++ b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_inundation_prvi/building_footprints_fimpact.sql @@ -1,3 +1,5 @@ +-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB) +SET work_mem TO '512MB'; --------------- Building Footprints --------------- DROP TABLE IF EXISTS publish.ana_inundation_building_footprints_prvi; SELECT diff --git a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_past_14day_max_inundation/14day_building_footprints_fimpact.sql b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_past_14day_max_inundation/14day_building_footprints_fimpact.sql index c2c2b61f..5844b1c0 100644 --- a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_past_14day_max_inundation/14day_building_footprints_fimpact.sql +++ b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_past_14day_max_inundation/14day_building_footprints_fimpact.sql @@ -1,3 +1,5 @@ +-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB) +SET work_mem TO '512MB'; --------------- Building Footprints --------------- DROP TABLE IF EXISTS publish.ana_past_14day_max_inundation_building_footprints; SELECT diff --git a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_past_14day_max_inundation/7day_building_footprints_fimpact.sql b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_past_14day_max_inundation/7day_building_footprints_fimpact.sql index 4599fa3d..7f166a46 100644 --- a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_past_14day_max_inundation/7day_building_footprints_fimpact.sql +++ b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/ana_past_14day_max_inundation/7day_building_footprints_fimpact.sql @@ -1,3 +1,5 @@ +-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB) +SET work_mem TO '512MB'; --------------- Building Footprints --------------- DROP TABLE IF EXISTS publish.ana_past_7day_max_inundation_building_footprints; SELECT diff --git a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_gfs_10day_max_inundation/10day_building_footprints_fimpact.sql b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_gfs_10day_max_inundation/10day_building_footprints_fimpact.sql index ae7c4a82..19d839b6 100644 --- a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_gfs_10day_max_inundation/10day_building_footprints_fimpact.sql +++ b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_gfs_10day_max_inundation/10day_building_footprints_fimpact.sql @@ -1,3 +1,5 @@ +-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB) +SET work_mem TO '512MB'; --------------- Building Footprints --------------- DROP TABLE IF EXISTS publish.mrf_gfs_max_inundation_10day_building_footprints; SELECT diff --git a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_gfs_10day_max_inundation/3day_building_footprints_fimpact.sql b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_gfs_10day_max_inundation/3day_building_footprints_fimpact.sql index 90d5acfe..7a163bdf 100644 --- a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_gfs_10day_max_inundation/3day_building_footprints_fimpact.sql +++ b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_gfs_10day_max_inundation/3day_building_footprints_fimpact.sql @@ -1,3 +1,5 @@ +-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB) +SET work_mem TO '512MB'; --------------- Building Footprints --------------- DROP TABLE IF EXISTS publish.mrf_gfs_max_inundation_3day_building_footprints; SELECT diff --git a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_gfs_10day_max_inundation/5day_building_footprints_fimpact.sql b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_gfs_10day_max_inundation/5day_building_footprints_fimpact.sql index 44c2ef57..dbbadca5 100644 --- a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_gfs_10day_max_inundation/5day_building_footprints_fimpact.sql +++ b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_gfs_10day_max_inundation/5day_building_footprints_fimpact.sql @@ -1,3 +1,5 @@ +-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB) +SET work_mem TO '512MB'; --------------- Building Footprints --------------- DROP TABLE IF EXISTS publish.mrf_gfs_max_inundation_5day_building_footprints; SELECT diff --git a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_nbm_10day_max_inundation/10day_building_footprints_fimpact.sql b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_nbm_10day_max_inundation/10day_building_footprints_fimpact.sql index c8c2ce30..03ff06d5 100644 --- a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_nbm_10day_max_inundation/10day_building_footprints_fimpact.sql +++ b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_nbm_10day_max_inundation/10day_building_footprints_fimpact.sql @@ -1,3 +1,5 @@ +-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB) +SET work_mem TO '512MB'; --------------- Building Footprints --------------- DROP TABLE IF EXISTS publish.mrf_nbm_max_inundation_10day_building_footprints; SELECT diff --git a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_nbm_10day_max_inundation/3day_building_footprints_fimpact.sql b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_nbm_10day_max_inundation/3day_building_footprints_fimpact.sql index e0df0f01..d7a51da0 100644 --- a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_nbm_10day_max_inundation/3day_building_footprints_fimpact.sql +++ b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_nbm_10day_max_inundation/3day_building_footprints_fimpact.sql @@ -1,3 +1,5 @@ +-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB) +SET work_mem TO '512MB'; --------------- Building Footprints --------------- DROP TABLE IF EXISTS publish.mrf_nbm_max_inundation_3day_building_footprints; SELECT diff --git a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_nbm_10day_max_inundation/5day_building_footprints_fimpact.sql b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_nbm_10day_max_inundation/5day_building_footprints_fimpact.sql index 188e8749..e3bb87c5 100644 --- a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_nbm_10day_max_inundation/5day_building_footprints_fimpact.sql +++ b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/mrf_nbm_10day_max_inundation/5day_building_footprints_fimpact.sql @@ -1,3 +1,5 @@ +-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB) +SET work_mem TO '512MB'; --------------- Building Footprints --------------- DROP TABLE IF EXISTS publish.mrf_nbm_max_inundation_5day_building_footprints; SELECT diff --git a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/rfc_based_5day_max_inundation/building_footprints_fimpact.sql b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/rfc_based_5day_max_inundation/building_footprints_fimpact.sql index b62cd0b1..8f0784d6 100644 --- a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/rfc_based_5day_max_inundation/building_footprints_fimpact.sql +++ b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/rfc_based_5day_max_inundation/building_footprints_fimpact.sql @@ -1,3 +1,5 @@ +-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB) +SET work_mem TO '512MB'; --------------- Building Footprints --------------- DROP TABLE IF EXISTS publish.rfc_based_5day_max_inundation_building_footprints; SELECT diff --git a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/srf_18hr_max_inundation/building_footprints_fimpact.sql b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/srf_18hr_max_inundation/building_footprints_fimpact.sql index efb45dc9..9d5d1a18 100644 --- a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/srf_18hr_max_inundation/building_footprints_fimpact.sql +++ b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/srf_18hr_max_inundation/building_footprints_fimpact.sql @@ -1,3 +1,5 @@ +-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB) +SET work_mem TO '512MB'; --------------- Building Footprints --------------- DROP TABLE IF EXISTS publish.srf_18hr_max_inundation_building_footprints; SELECT diff --git a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/srf_48hr_max_inundation_hi/building_footprints_fimpact.sql b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/srf_48hr_max_inundation_hi/building_footprints_fimpact.sql index 295c5de8..1303bb36 100644 --- a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/srf_48hr_max_inundation_hi/building_footprints_fimpact.sql +++ b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/srf_48hr_max_inundation_hi/building_footprints_fimpact.sql @@ -1,3 +1,5 @@ +-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB) +SET work_mem TO '512MB'; --------------- Building Footprints --------------- DROP TABLE IF EXISTS publish.srf_48hr_max_inundation_building_footprints_hi; SELECT diff --git a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/srf_48hr_max_inundation_prvi/building_footprints_fimpact.sql b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/srf_48hr_max_inundation_prvi/building_footprints_fimpact.sql index 27b5db02..d00b2137 100644 --- a/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/srf_48hr_max_inundation_prvi/building_footprints_fimpact.sql +++ b/Core/LAMBDA/viz_functions/viz_db_postprocess_sql/summaries/srf_48hr_max_inundation_prvi/building_footprints_fimpact.sql @@ -1,3 +1,5 @@ +-- We'll temporarily increase work_mem to 512MB, to help with performance on PostGIS spatial joins (default is 4MB) +SET work_mem TO '512MB'; --------------- Building Footprints --------------- DROP TABLE IF EXISTS publish.srf_48hr_max_inundation_building_footprints_prvi; SELECT diff --git a/Core/RDS/viz/main.tf b/Core/RDS/viz/main.tf index c128c306..0109db35 100644 --- a/Core/RDS/viz/main.tf +++ b/Core/RDS/viz/main.tf @@ -55,6 +55,12 @@ resource "aws_db_parameter_group" "hydrovis" { value = "{DBInstanceClassMemory/10923}" apply_method = "pending-reboot" } + + parameter { + name = "idle_in_transaction_session_timeout" + value = "900000" + apply_method = "pending-reboot" + } parameter { name = "rds.custom_dns_resolution"