Put data in a subdirectory (#192)

.gitignore

@@ -399,5 +399,5 @@ private_data/*
 .vscode/settings.json
 
 # Test data exceptions to the general data exclusion
-!pipelines/tests/covid-19_r_2024-01-29_f_2023-11-01_t_2024-01-29/model_runs/TD/data.csv
-!pipelines/tests/covid-19_r_2024-01-29_f_2023-11-01_t_2024-01-29/model_runs/TD/eval_data.tsv
+!pipelines/tests/covid-19_r_2024-01-29_f_2023-11-01_t_2024-01-29/model_runs/TD/data/data.tsv
+!pipelines/tests/covid-19_r_2024-01-29_f_2023-11-01_t_2024-01-29/model_runs/TD/data/eval_data.tsv

hewr/R/process_state_forecast.R

@@ -9,10 +9,10 @@
 process_state_forecast <- function(model_run_dir, save = TRUE) {
   disease_name_nssp <- parse_model_run_dir_path(model_run_dir)$disease
 
-  train_data_path <- fs::path(model_run_dir, "data", ext = "csv")
-  train_dat <- readr::read_csv(train_data_path, show_col_types = FALSE)
+  train_data_path <- fs::path(model_run_dir, "data", "data", ext = "tsv")
+  train_dat <- readr::read_tsv(train_data_path, show_col_types = FALSE)
 
-  eval_data_path <- fs::path(model_run_dir, "eval_data", ext = "tsv")
+  eval_data_path <- fs::path(model_run_dir, "data", "eval_data", ext = "tsv")
   eval_dat <- readr::read_tsv(eval_data_path, show_col_types = FALSE) |>
     dplyr::mutate(data_type = "eval")
 

pipelines/build_model.py

@@ -8,7 +8,7 @@
 
 
 def build_model_from_dir(model_dir):
-    data_path = model_dir / "data_for_model_fit.json"
+    data_path = model_dir / "data" / "data_for_model_fit.json"
     prior_path = model_dir / "priors.py"
 
     with open(

pipelines/forecast_state.py

@@ -253,7 +253,7 @@ def main(
         first_training_date=first_training_date,
         last_training_date=last_training_date,
         latest_comprehensive_path=eval_data_path,
-        output_data_dir=model_run_dir,
+        output_data_dir=Path(model_run_dir, "data"),
         last_eval_date=report_date + timedelta(days=n_forecast_days),
     )
 

pipelines/generate_epiweekly.R

@@ -31,7 +31,7 @@ purrr::walk(script_packages, \(pkg) {
 #' @return None. The function writes the epiweekly data to a CSV file in the
 #'  specified directory.
 convert_daily_to_epiweekly <- function(
-    model_run_dir, dataname = "data.csv",
+    model_run_dir, dataname = "data.tsv",
     strict = TRUE, day_of_week = 7) {
   ext <- path_ext(dataname)
   data_basename <- path_ext_remove(dataname)
@@ -42,7 +42,7 @@ convert_daily_to_epiweekly <- function(
   delim <- if (ext == "csv") "," else "\t"
   message(glue::glue("Generating epi-weekly data {model_run_dir}..."))
 
-  data_path <- path(model_run_dir, dataname)
+  data_path <- path(model_run_dir, "data", dataname)
 
   daily_data <- read_delim(
     data_path,
@@ -73,7 +73,7 @@ convert_daily_to_epiweekly <- function(
   # epiweek end date determines data_type classification
 
   output_file <- path(
-    model_run_dir,
+    model_run_dir, "data",
     glue::glue("epiweekly_{data_basename}"),
     ext = ext
   )
@@ -82,7 +82,7 @@ convert_daily_to_epiweekly <- function(
 }
 
 main <- function(model_run_dir) {
-  convert_daily_to_epiweekly(model_run_dir, dataname = "data.csv")
+  convert_daily_to_epiweekly(model_run_dir, dataname = "data.tsv")
   convert_daily_to_epiweekly(model_run_dir, dataname = "eval_data.tsv")
 }
 

pipelines/prep_data.py

@@ -332,16 +332,14 @@ def process_and_save_state(
         "state_pop": state_pop,
         "right_truncation_offset": right_truncation_offset,
     }
-
-    os.makedirs(model_run_dir, exist_ok=True)
+    data_dir = Path(model_run_dir, "data")
+    os.makedirs(data_dir, exist_ok=True)
 
     if logger is not None:
-        logger.info(f"Saving {state_abb} to {model_run_dir}")
-    data_to_save.write_csv(Path(model_run_dir, "data.csv"))
+        logger.info(f"Saving {state_abb} to {data_dir}")
+    data_to_save.write_csv(Path(data_dir, "data.tsv"), separator="\t")
 
-    with open(
-        Path(model_run_dir, "data_for_model_fit.json"), "w"
-    ) as json_file:
+    with open(Path(data_dir, "data_for_model_fit.json"), "w") as json_file:
         json.dump(data_for_model_fit, json_file)
 
     return None

pipelines/score_forecast.R

@@ -200,6 +200,7 @@ read_and_score_location <- function(model_run_dir,
   )
 
   truth_path <- fs::path(model_run_dir,
+    "data",
     eval_data_filename,
     ext = eval_data_file_ext
   )