From 8a89114b8dd067e84c2036ee9ef6acb5b46a4896 Mon Sep 17 00:00:00 2001 From: Ben Koziol Date: Mon, 24 Feb 2025 09:36:57 -0700 Subject: [PATCH 1/2] test restart files on c6 [ci skip] --- tests/test_python/test_smoke_dust/conftest.py | 4 +- .../test_smoke_dust/test_core/test_cycle.py | 2 +- ush/smoke_dust/core/cycle.py | 44 ++++++++++--------- 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/tests/test_python/test_smoke_dust/conftest.py b/tests/test_python/test_smoke_dust/conftest.py index 11bb3108e..990a8e397 100644 --- a/tests/test_python/test_smoke_dust/conftest.py +++ b/tests/test_python/test_smoke_dust/conftest.py @@ -123,9 +123,9 @@ def create_fake_restart_files( cycle_dates: The series of dates to create the restart files for. shape: Output grid shape. """ - restart_dir = root_dir / "RESTART" - restart_dir.mkdir(exist_ok=True) for date in cycle_dates: + restart_dir = root_dir / date / "RESTART" + restart_dir.mkdir(exist_ok=True, parents=True) restart_file = restart_dir / f"{date[:8]}.{date[8:10]}0000.phy_data.nc" with Dataset(restart_file, "w") as nc_ds: nc_ds.createDimension("Time") diff --git a/tests/test_python/test_smoke_dust/test_core/test_cycle.py b/tests/test_python/test_smoke_dust/test_core/test_cycle.py index 00f2dffbc..eccde7441 100644 --- a/tests/test_python/test_smoke_dust/test_core/test_cycle.py +++ b/tests/test_python/test_smoke_dust/test_core/test_cycle.py @@ -58,7 +58,7 @@ def test_find_restart_files( create_fake_grid_out(tmp_path, fake_grid_out_shape) context = create_fake_context(tmp_path) cycle = SmokeDustCycleTwo(context) - create_fake_restart_files(context.nwges_dir, cycle.cycle_dates, fake_grid_out_shape) + create_fake_restart_files(context.hourly_hwpdir, cycle.cycle_dates, fake_grid_out_shape) create_fake_restart_files( context.nwges_dir, [ diff --git a/ush/smoke_dust/core/cycle.py b/ush/smoke_dust/core/cycle.py index bdb122efa..43db5f946 100644 --- a/ush/smoke_dust/core/cycle.py +++ b/ush/smoke_dust/core/cycle.py @@ -390,32 +390,36 @@ def _find_restart_files_( ) -> tuple[Path, ...]: root_dir = self._context.hourly_hwpdir self.log(f"_find_restart_files_: {root_dir=}") - filenames = glob.glob("**/*phy_data*nc", root_dir=root_dir, recursive=True) potential_restart_files = [ f"{cycle[:8]}.{cycle[8:10]}0000.phy_data.nc" for cycle in self.cycle_dates ] self.log(f"_find_restart_files_: {potential_restart_files=}") + potential_restart_dirs = [root_dir / cycle / "RESTART" for cycle in self.cycle_dates] + restart_dirs = [restart_dir for restart_dir in potential_restart_dirs if restart_dir.exists()] + self.log(f"_find_restart_files_: {restart_dirs=}") found_potentials = [] restart_files = [] - for filename in filenames: - self.log(f"_find_restart_files_: {filename=}", level=logging.DEBUG) - path = root_dir / filename - if path.name in potential_restart_files and path.name not in found_potentials: - try: - resolved = path.resolve(strict=True) - except FileNotFoundError: - self.log(f"restart file link not resolvable: {path=}", level=logging.WARN) - continue - with open_nc(resolved) as nc_ds: - variables = nc_ds.variables.keys() # pylint: disable=no-member - if all( - expected_var in variables for expected_var in self.expected_restart_varnames - ): - self.log( - f"_find_restart_files_: found restart path {path=}", level=logging.DEBUG - ) - restart_files.append(path) - found_potentials.append(path.name) + for restart_dir in restart_dirs: + filenames = glob.glob("**/*phy_data*nc", root_dir=restart_dir, recursive=True) + for filename in filenames: + self.log(f"_find_restart_files_: {filename=}", level=logging.DEBUG) + path = restart_dir / filename + if path.name in potential_restart_files and path.name not in found_potentials: + try: + resolved = path.resolve(strict=True) + except FileNotFoundError: + self.log(f"restart file link not resolvable: {path=}", level=logging.WARN) + continue + with open_nc(resolved, parallel=False) as nc_ds: + variables = nc_ds.variables.keys() # pylint: disable=no-member + if all( + expected_var in variables for expected_var in self.expected_restart_varnames + ): + self.log( + f"_find_restart_files_: found restart path {path=}", level=logging.DEBUG + ) + restart_files.append(path) + found_potentials.append(path.name) return tuple(restart_files) From 748c7b877816d735a5ba6222660473e5b7b13e17 Mon Sep 17 00:00:00 2001 From: Ben Koziol Date: Mon, 24 Feb 2025 10:17:59 -0700 Subject: [PATCH 2/2] code clean-up [ci skip] --- ush/smoke_dust/core/cycle.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ush/smoke_dust/core/cycle.py b/ush/smoke_dust/core/cycle.py index 43db5f946..b5379c1c0 100644 --- a/ush/smoke_dust/core/cycle.py +++ b/ush/smoke_dust/core/cycle.py @@ -395,7 +395,9 @@ def _find_restart_files_( ] self.log(f"_find_restart_files_: {potential_restart_files=}") potential_restart_dirs = [root_dir / cycle / "RESTART" for cycle in self.cycle_dates] - restart_dirs = [restart_dir for restart_dir in potential_restart_dirs if restart_dir.exists()] + restart_dirs = [ + restart_dir for restart_dir in potential_restart_dirs if restart_dir.exists() + ] self.log(f"_find_restart_files_: {restart_dirs=}") found_potentials = [] restart_files = [] @@ -413,10 +415,12 @@ def _find_restart_files_( with open_nc(resolved, parallel=False) as nc_ds: variables = nc_ds.variables.keys() # pylint: disable=no-member if all( - expected_var in variables for expected_var in self.expected_restart_varnames + expected_var in variables + for expected_var in self.expected_restart_varnames ): self.log( - f"_find_restart_files_: found restart path {path=}", level=logging.DEBUG + f"_find_restart_files_: found restart path {path=}", + level=logging.DEBUG, ) restart_files.append(path) found_potentials.append(path.name)