From 8850801aeff65b9ffdc0cfea9a0c056e978c0b77 Mon Sep 17 00:00:00 2001 From: AminTorabi-NOAA Date: Thu, 23 May 2024 13:42:25 -0400 Subject: [PATCH 1/2] Update for usace issue --- src/troute-network/troute/nhd_io.py | 50 +++++++++++-------- .../test_AnA_V4_HYFeature.yaml | 2 +- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/src/troute-network/troute/nhd_io.py b/src/troute-network/troute/nhd_io.py index 7dd2a0279..5cd08d4e1 100644 --- a/src/troute-network/troute/nhd_io.py +++ b/src/troute-network/troute/nhd_io.py @@ -1059,27 +1059,30 @@ def _read_timeslice_file(f): stns = ds.variables['stationId'][:].filled(fill_value = np.nan) t = ds.variables['time'][:].filled(fill_value = np.nan) qual = ds.variables['discharge_quality'][:].filled(fill_value = np.nan) + + if discharge.size != 0 and stns.size != 0 and t.size != 0: + stationId = np.apply_along_axis(''.join, 1, stns.astype(str)) + time_str = np.apply_along_axis(''.join, 1, t.astype(str)) + stationId = np.char.strip(stationId) - stationId = np.apply_along_axis(''.join, 1, stns.astype(str)) - time_str = np.apply_along_axis(''.join, 1, t.astype(str)) - stationId = np.char.strip(stationId) - - timeslice_observations = (pd.DataFrame({ - 'stationId' : stationId, - 'datetime' : time_str, - 'discharge' : discharge - }). - set_index(['stationId', 'datetime']). - unstack(1, fill_value = np.nan)['discharge']) - - observation_quality = (pd.DataFrame({ - 'stationId' : stationId, - 'datetime' : time_str, - 'quality' : qual/100 - }). - set_index(['stationId', 'datetime']). - unstack(1, fill_value = np.nan)['quality']) - + timeslice_observations = (pd.DataFrame({ + 'stationId' : stationId, + 'datetime' : time_str, + 'discharge' : discharge + }). + set_index(['stationId', 'datetime']). + unstack(1, fill_value = np.nan)['discharge']) + + observation_quality = (pd.DataFrame({ + 'stationId' : stationId, + 'datetime' : time_str, + 'quality' : qual/100 + }). + set_index(['stationId', 'datetime']). + unstack(1, fill_value = np.nan)['quality']) + else: + timeslice_observations = pd.DataFrame() + observation_quality = pd.DataFrame() return timeslice_observations, observation_quality def _interpolate_one(df, interpolation_limit, frequency): @@ -1157,7 +1160,12 @@ def get_obs_from_timeslices( for f in timeslice_files: jobs.append(delayed(_read_timeslice_file)(f)) timeslice_dataframes = parallel(jobs) - + + all_empty = all(df.empty for tuple in timeslice_dataframes for df in tuple) + if all_empty: + LOG.debug(f'DataFrames in the list are empty.') + return pd.DataFrame() + # create lists of observations and obs quality dataframes returned # from _read_timeslice_file timeslice_obs_frames = [] diff --git a/test/LowerColorado_TX_v4/test_AnA_V4_HYFeature.yaml b/test/LowerColorado_TX_v4/test_AnA_V4_HYFeature.yaml index 5c1db9d3d..d17c64c3f 100644 --- a/test/LowerColorado_TX_v4/test_AnA_V4_HYFeature.yaml +++ b/test/LowerColorado_TX_v4/test_AnA_V4_HYFeature.yaml @@ -98,7 +98,7 @@ compute_parameters: data_assimilation_parameters: #---------- usgs_timeslices_folder : usgs_timeslices/ - usace_timeslices_folder : usace_timelices/ + usace_timeslices_folder : usace_timeslices/ streamflow_da: #---------- streamflow_nudging : True From 6f02d3da65db1f0dd8059894daaa1491a699d22f Mon Sep 17 00:00:00 2001 From: AminTorabi-NOAA Date: Tue, 28 May 2024 09:18:38 -0400 Subject: [PATCH 2/2] update on LOG file --- src/troute-network/troute/nhd_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/troute-network/troute/nhd_io.py b/src/troute-network/troute/nhd_io.py index 5cd08d4e1..52d235ab6 100644 --- a/src/troute-network/troute/nhd_io.py +++ b/src/troute-network/troute/nhd_io.py @@ -1163,7 +1163,7 @@ def get_obs_from_timeslices( all_empty = all(df.empty for tuple in timeslice_dataframes for df in tuple) if all_empty: - LOG.debug(f'DataFrames in the list are empty.') + LOG.debug(f'{crosswalk_gage_field} DataFrames is empty, check timeslice files.') return pd.DataFrame() # create lists of observations and obs quality dataframes returned