Speed up output

src/troute-network/troute/AbstractNetwork.py

@@ -728,10 +728,10 @@ def build_forcing_sets(self,):
             raise AssertionError("Aborting simulation because the qlat_input_folder:", qlat_input_folder,"does not exist. Please check the the nexus_input_folder variable is correctly entered in the .yaml control file") from None
 
         forcing_glob_filter = forcing_parameters["qlat_file_pattern_filter"]
+        binary_folder = forcing_parameters.get('binary_nexus_file_folder', None)
 
-        if forcing_glob_filter=="nex-*":
+        if forcing_glob_filter=="nex-*" and binary_folder:
             print("Reformating qlat nexus files as hourly binary files...")
-            binary_folder = forcing_parameters.get('binary_nexus_file_folder', None)
             qlat_files = qlat_input_folder.glob(forcing_glob_filter)
 
             #Check that directory/files specified will work
@@ -748,9 +748,24 @@ def build_forcing_sets(self,):
             qlat_input_folder, forcing_glob_filter = nex_files_to_binary(qlat_files_list, binary_folder)
             forcing_parameters["qlat_input_folder"] = qlat_input_folder
             forcing_parameters["qlat_file_pattern_filter"] = forcing_glob_filter
+
+        if forcing_glob_filter=="nex-*":
+            all_files = sorted(qlat_input_folder.glob(forcing_glob_filter))
+            final_timestamp = pd.read_csv(all_files[0], header=None, index_col=[0]).tail(1).iloc[0,0]
+            final_timestamp = datetime.strptime(final_timestamp, "%Y-%m-%d %H:%M:%S")
+
+            all_files = [os.path.basename(f) for f in all_files]
+
+            run_sets = [
+                {
+                    'qlat_files': all_files,
+                    'nts': nts,
+                    'final_timestamp': final_timestamp
+                }
+            ]
 
         # TODO: Throw errors if insufficient input data are available
-        if run_sets:        
+        elif run_sets:        
             #FIXME: Change it for hyfeature
             '''
             # append final_timestamp variable to each set_list
@@ -893,7 +908,7 @@ def get_timesteps_from_nex(nexus_files):
     output_file_timestamps = []
     with open(nexus_files[0]) as f:
         for line in f:
-            output_file_timestamps.append(line.split(', ')[1])
+            output_file_timestamps.append(line.split(',')[1].strip())
     # Convert and reformat dates in the list
     output_file_timestamps = [pd.to_datetime(i).strftime("%Y%m%d%H%M") for i in output_file_timestamps]
 

src/troute-network/troute/HYFeaturesNetwork.py

@@ -10,6 +10,7 @@
 from joblib import delayed, Parallel
 from collections import defaultdict
 import xarray as xr
+from datetime import datetime
 import os
 
 import troute.nhd_io as nhd_io #FIXME
@@ -618,16 +619,35 @@ def build_qlateral_array(self, run,):
                     "qlat_file_pattern_filter", "*CHRT_OUT*"
                 )
                 qlat_files = sorted(qlat_input_folder.glob(qlat_file_pattern_filter))
-
+            
             dfs=[]
-            for f in qlat_files:
-                df = read_file(f)
-                df['feature_id'] = df['feature_id'].map(lambda x: int(str(x).removeprefix('nex-')) if str(x).startswith('nex') else int(x))
-                df = df.set_index('feature_id')
-                dfs.append(df)
 
-            # lateral flows [m^3/s] are stored at NEXUS points with NEXUS ids
-            nexuses_lateralflows_df = pd.concat(dfs, axis=1) 
+            #FIXME Temporary solution to allow t-route to use ngen nex-* output files as forcing files
+            # This capability should be here, but we need to think through how to handle all of this 
+            # data in memory for large domains and many timesteps... - shorvath, Feb 28, 2024
+            qlat_file_pattern_filter = self.forcing_parameters.get("qlat_file_pattern_filter", None)
+            if qlat_file_pattern_filter=="nex-*":
+                for f in qlat_files:
+                    df = pd.read_csv(f, names=['timestamp', 'qlat'], index_col=[0])
+                    df['timestamp'] = pd.to_datetime(df['timestamp']).dt.strftime('%Y%m%d%H%M')
+                    df = df.set_index('timestamp')
+                    df = df.T
+                    df.index = [int(os.path.basename(f).split('-')[1].split('_')[0])]
+                    df = df.rename_axis(None, axis=1)
+                    df.index.name = 'feature_id'
+                    dfs.append(df)
+
+                # lateral flows [m^3/s] are stored at NEXUS points with NEXUS ids
+                nexuses_lateralflows_df = pd.concat(dfs, axis=0) 
+            else:
+                for f in qlat_files:
+                    df = read_file(f)
+                    df['feature_id'] = df['feature_id'].map(lambda x: int(str(x).removeprefix('nex-')) if str(x).startswith('nex') else int(x))
+                    df = df.set_index('feature_id')
+                    dfs.append(df)
+
+                # lateral flows [m^3/s] are stored at NEXUS points with NEXUS ids
+                nexuses_lateralflows_df = pd.concat(dfs, axis=1) 
 
             # Take flowpath ids entering NEXUS and replace NEXUS ids by the upstream flowpath ids
             qlats_df = nexuses_lateralflows_df.rename(index=self.downstream_flowpath_dict)