From 0432ce96a0a5eec4d20adb4d384505632a2db3dc Mon Sep 17 00:00:00 2001 From: scottrp <45947939+scottrp@users.noreply.github.com> Date: Thu, 9 Dec 2021 09:03:33 -0800 Subject: [PATCH] fix(paths): Path code made more robust so that non-standard model folder structures are supported (#1311) (#1316) * fix(paths): Path code made more robust so that non-standard model folder structures are supported. There are three basic path times, the path to the simulation, model relative paths to particular models in the simulation, and package/data relative paths to package or data files in a model or simulation. Model relative paths are relative to the simulation path. Package/data relative paths are relative to the simulation path for packages/data not attached to a model and relative to model relative path for packages/data attached to a model. (#1311) * test(paths): added more comprehensive testing of paths Co-authored-by: Scott Paulinski --- autotest/t505_test.py | 64 +++++++++++++++++++++++++------ flopy/mf6/data/mfdatastorage.py | 6 +++ flopy/mf6/mfbase.py | 24 +++++++----- flopy/mf6/mfmodel.py | 40 +++++++++++++++---- flopy/mf6/mfpackage.py | 37 +++++++++++++----- flopy/mf6/modflow/mfsimulation.py | 57 ++++++++++++++++++++++++++- 6 files changed, 187 insertions(+), 41 deletions(-) diff --git a/autotest/t505_test.py b/autotest/t505_test.py index 71953c946d..84d541c862 100644 --- a/autotest/t505_test.py +++ b/autotest/t505_test.py @@ -811,6 +811,7 @@ def test_np001(): } wel_package = ModflowGwfwel( model, + filename=f"well_folder\\{model_name}.wel", print_input=True, print_flows=True, save_flows=True, @@ -841,7 +842,7 @@ def test_np001(): riv_spd = { 0: { - "filename": "riv.txt", + "filename": os.path.join("riv_folder", "riv.txt"), "data": [((0, 0, 9), 110, 90.0, 100.0, 1.0, 2.0, 3.0)], } } @@ -914,6 +915,12 @@ def test_np001(): assert ( sim.simulation_data.max_columns_of_data == dis_package.ncol.get_data() ) + # test package file with relative path to simulation path + wel_path = os.path.join(run_folder, "well_folder", f"{model_name}.wel") + assert os.path.exists(wel_path) + # test data file with relative path to simulation path + riv_path = os.path.join(run_folder, "riv_folder", "riv.txt") + assert os.path.exists(riv_path) # run simulation if run: @@ -950,6 +957,17 @@ def test_np001(): sim.set_all_data_external(external_data_folder="data") sim.write_simulation() + # test file with relative path to model relative path + wel_path = os.path.join( + run_folder, md_folder, "well_folder", f"{model_name}.wel" + ) + assert os.path.exists(wel_path) + # test data file was recreated by set_all_data_external + riv_path = os.path.join( + run_folder, md_folder, "data", "np001_mod.riv_stress_period_data_1.txt" + ) + assert os.path.exists(riv_path) + assert ( sim.simulation_data.max_columns_of_data == dis_package.ncol.get_data() ) @@ -974,7 +992,6 @@ def test_np001(): outfile=outfile, ) - # budget_frf = sim.simulation_data.mfdata[(model_name, "CBC", "RIV")] budget_frf = model.output.budget().get_data(text="RIV", full3D=False) assert array_util.riv_array_comp(budget_frf_valid, budget_frf) @@ -1090,6 +1107,7 @@ def test_np001(): well_spd = {0: [(-1, -1, -1, -2000.0), (0, 0, 7, -2.0)], 1: []} wel_package = ModflowGwfwel( model, + pname="wel_1", filename="file_rename.wel", print_input=True, print_flows=True, @@ -1097,7 +1115,7 @@ def test_np001(): maxbound=2, stress_period_data=well_spd, ) - wel_package.write() + sim.write_simulation() found_begin = False found_end = False text_between_begin_and_end = False @@ -1121,7 +1139,7 @@ def test_np001(): spath, write_headers=False, ) - wel = test_sim.get_model().wel + wel = test_sim.get_model().get_package("wel_1") wel._filename = "np001_spd_test.wel" wel.write() found_begin = False @@ -1778,7 +1796,7 @@ def test005_advgw_tidal(): (31.0, 0.0, -600.0, -400.0), ] ts_dict = { - "filename": "well-rates.ts", + "filename": os.path.join("well-rates", "well-rates.ts"), "timeseries": timeseries, "time_series_namerecord": [ ("well_1_rate", "well_2_rate", "well_3_rate") @@ -2160,6 +2178,10 @@ def test005_advgw_tidal(): sim.set_all_data_external() sim.write_simulation() + # test time series data file with relative path to simulation path + ts_path = os.path.join(run_folder, "well-rates", "well-rates.ts") + assert os.path.exists(ts_path) + # run simulation sim.run_simulation() @@ -2180,22 +2202,24 @@ def test005_advgw_tidal(): package_type_dict = {} for package in model.packagelist: if not package.package_type in package_type_dict: - assert package.filename == f"new_name.{package.package_type}" + filename = os.path.split(package.filename)[1] + assert filename == f"new_name.{package.package_type}" package_type_dict[package.package_type] = 1 sim.write_simulation() name_file = os.path.join(run_folder, "new_name.nam") assert os.path.exists(name_file) dis_file = os.path.join(run_folder, "new_name.dis") assert os.path.exists(dis_file) + # test time series data file with relative path to simulation path + ts_path = os.path.join(run_folder, "well-rates", "new_name.ts") + assert os.path.exists(ts_path) sim.rename_all_packages("all_files_same_name") package_type_dict = {} for package in model.packagelist: if not package.package_type in package_type_dict: - assert ( - package.filename - == f"all_files_same_name.{package.package_type}" - ) + filename = os.path.split(package.filename)[1] + assert filename == f"all_files_same_name.{package.package_type}" package_type_dict[package.package_type] = 1 assert sim._tdis_file.filename == "all_files_same_name.tdis" for ims_file in sim._ims_files.values(): @@ -2207,6 +2231,9 @@ def test005_advgw_tidal(): assert os.path.exists(dis_file) tdis_file = os.path.join(run_folder, "all_files_same_name.tdis") assert os.path.exists(tdis_file) + # test time series data file with relative path to simulation path + ts_path = os.path.join(run_folder, "well-rates", "all_files_same_name.ts") + assert os.path.exists(ts_path) # load simulation sim_load = MFSimulation.load( @@ -3073,13 +3100,14 @@ def test006_2models_gnc(): # test exg delete newexgrecarray = exgrecarray[10:] + gnc_path = os.path.join("gnc", "test006_2models_gnc.gnc") exg_package = ModflowGwfgwf( sim, print_input=True, print_flows=True, save_flows=True, auxiliary="testaux", - gnc_filerecord="test006_2models_gnc.gnc", + gnc_filerecord=gnc_path, nexg=26, exchangedata=newexgrecarray, exgtype="gwf6-gwf6", @@ -3094,7 +3122,7 @@ def test006_2models_gnc(): print_flows=True, save_flows=True, auxiliary="testaux", - gnc_filerecord="test006_2models_gnc.gnc", + gnc_filerecord=gnc_path, nexg=36, exchangedata=exgrecarray, exgtype="gwf6-gwf6", @@ -3108,6 +3136,7 @@ def test006_2models_gnc(): new_gncrecarray = gncrecarray[10:] gnc_package = ModflowGwfgnc( sim, + filename=gnc_path, print_input=True, print_flows=True, numgnc=26, @@ -3118,6 +3147,7 @@ def test006_2models_gnc(): gnc_package = ModflowGwfgnc( sim, + filename=gnc_path, print_input=True, print_flows=True, numgnc=36, @@ -3131,6 +3161,10 @@ def test006_2models_gnc(): # write simulation to new location sim.write_simulation() + # test gnc file was created in correct location + gnc_full_path = os.path.join(run_folder, gnc_path) + assert os.path.exists(gnc_full_path) + # run simulation if run: sim.run_simulation() @@ -3172,6 +3206,9 @@ def test006_2models_gnc(): sim_path, "model2", "data", "model2.dis_botm.txt" ) assert os.path.exists(ext_file_path_2) + # test gnc file was created in correct location + gnc_full_path = os.path.join(sim_path, gnc_path) + assert os.path.exists(gnc_full_path) if run: sim.run_simulation() @@ -3183,6 +3220,9 @@ def test006_2models_gnc(): sim.rename_all_packages("file_rename") sim.set_sim_path(rename_folder) sim.write_simulation() + # test gnc file was created in correct location + gnc_full_path = os.path.join(rename_folder, "gnc", "file_rename.gnc") + assert os.path.exists(gnc_full_path) if run: sim.run_simulation() sim.delete_output_files() diff --git a/flopy/mf6/data/mfdatastorage.py b/flopy/mf6/data/mfdatastorage.py index 8153c96980..45ecbcb57d 100644 --- a/flopy/mf6/data/mfdatastorage.py +++ b/flopy/mf6/data/mfdatastorage.py @@ -1552,6 +1552,12 @@ def store_external( precision="double", ) else: + # make sure folder exists + file_path = os.path.split(fp)[0] + if not os.path.exists(file_path): + os.makedirs(file_path) + + # create file try: fd = open(fp, "w") except: diff --git a/flopy/mf6/mfbase.py b/flopy/mf6/mfbase.py index 9a5759e1f3..c3c50ab653 100644 --- a/flopy/mf6/mfbase.py +++ b/flopy/mf6/mfbase.py @@ -270,17 +270,23 @@ def _build_relative_path(self, model_name): def strip_model_relative_path(self, model_name, path): """Strip out the model relative path part of `path`. For internal FloPy use, not intended for end user.""" + new_path = path if model_name in self.model_relative_path: model_rel_path = self.model_relative_path[model_name] - new_path = None - while path: - path, leaf = os.path.split(path) - if leaf != model_rel_path: - if new_path: - new_path = os.path.join(leaf, new_path) - else: - new_path = leaf - return new_path + if ( + model_rel_path is not None + and len(model_rel_path) > 0 + and model_rel_path != "." + ): + model_rel_path_lst = model_rel_path.split(os.path.sep) + path_lst = path.split(os.path.sep) + new_path = "" + for i, mrp in enumerate(model_rel_path_lst): + if mrp != path_lst[i]: + new_path = os.path.join(new_path, path_lst[i]) + for rp in path_lst[len(model_rel_path_lst) :]: + new_path = os.path.join(new_path, rp) + return new_path @staticmethod def unique_file_name(file_name, lookup): diff --git a/flopy/mf6/mfmodel.py b/flopy/mf6/mfmodel.py index d12c3f570f..b99444eba0 100644 --- a/flopy/mf6/mfmodel.py +++ b/flopy/mf6/mfmodel.py @@ -1045,10 +1045,24 @@ def set_model_relative_path(self, model_ws): packages_data = packages.get_data() if packages_data is not None: for index, entry in enumerate(packages_data): - old_package_name = os.path.split(entry[1])[1] - packages_data[index][1] = os.path.join( - path, old_package_name - ) + # get package object associated with entry + package = None + if len(entry) >= 3: + package = self.get_package(entry[2]) + if package is None: + package = self.get_package(entry[0]) + if package is not None: + # combine model relative path with package path + packages_data[index][1] = os.path.join( + path, package.filename + ) + else: + # package not found, create path based on + # information in name file + old_package_name = os.path.split(entry[1])[-1] + packages_data[index][1] = os.path.join( + path, old_package_name + ) packages.set_data(packages_data) # update files referenced from within packages for package in self.packagelist: @@ -1186,12 +1200,15 @@ def update_package_filename(self, package, new_name): message=message, ) try: + file_mgr = self.simulation_data.mfpath + model_rel_path = file_mgr.model_relative_path[self.name] # update namefile package data with new name new_rec_array = None + old_leaf = os.path.split(package.filename)[1] for item in package_data: - base, leaf = os.path.split(item[1]) - if leaf == package.filename: - item[1] = os.path.join(base, new_name) + leaf = os.path.split(item[1])[1] + if leaf == old_leaf: + item[1] = os.path.join(model_rel_path, new_name) if new_rec_array is None: new_rec_array = np.rec.array( @@ -1246,7 +1263,14 @@ def rename_all_packages(self, name): package_type_count = {} for package in self.packagelist: if package.package_type not in package_type_count: - package.filename = f"{name}.{package.package_type}" + base_filename, leaf = os.path.split(package.filename) + new_fileleaf = f"{name}.{package.package_type}" + if base_filename != "": + package.filename = os.path.join( + base_filename, new_fileleaf + ) + else: + package.filename = new_fileleaf package_type_count[package.package_type] = 1 else: package_type_count[package.package_type] += 1 diff --git a/flopy/mf6/mfpackage.py b/flopy/mf6/mfpackage.py index 2b2bce5325..4b6c48186b 100644 --- a/flopy/mf6/mfpackage.py +++ b/flopy/mf6/mfpackage.py @@ -913,12 +913,19 @@ def load(self, block_header, fd, strict=True): print( f" loading child package {package_info[0]}..." ) + fname = package_info[1] + if package_info[2] is not None: + fname = os.path.join(package_info[2], fname) + filemgr = self._simulation_data.mfpath + fname = filemgr.strip_model_relative_path( + self._model_or_sim.name, fname + ) pkg = self._model_or_sim.load_package( package_info[0], - package_info[1], + fname, package_info[1], True, - package_info[2], + "", package_info[3], self._container_package, ) @@ -1038,12 +1045,19 @@ def _find_data_by_keyword(self, line, fd, initial_comment): print( f" loading child package {package_info[1]}..." ) + fname = package_info[1] + if package_info[2] is not None: + fname = os.path.join(package_info[2], fname) + filemgr = self._simulation_data.mfpath + fname = filemgr.strip_model_relative_path( + self._model_or_sim.name, fname + ) pkg = self._model_or_sim.load_package( package_info[0], - package_info[1], + fname, package_info[1], True, - package_info[2], + "", package_info[3], self._container_package, ) @@ -1094,12 +1108,19 @@ def _find_data_by_keyword(self, line, fd, initial_comment): print( f" loading child package {package_info[0]}..." ) + fname = package_info[1] + if package_info[2] is not None: + fname = os.path.join(package_info[2], fname) + filemgr = self._simulation_data.mfpath + fname = filemgr.strip_model_relative_path( + self._model_or_sim.name, fname + ) pkg = self._model_or_sim.load_package( package_info[0], - package_info[1], + fname, None, True, - package_info[2], + "", package_info[3], self._container_package, ) @@ -1626,10 +1647,6 @@ def __init__( message, model_or_sim.simulation_data.debug, ) - # only store the file name. model relative path handled - # internally - if model_or_sim.type.lower() == "model": - filename = os.path.split(filename)[-1] self._filename = MFFileMgmt.string_to_file_path(filename) self.path, self.structure = model_or_sim.register_package( self, not loading_package, pname is None, filename is None diff --git a/flopy/mf6/modflow/mfsimulation.py b/flopy/mf6/modflow/mfsimulation.py index 38eb72682b..ee64ebc09e 100644 --- a/flopy/mf6/modflow/mfsimulation.py +++ b/flopy/mf6/modflow/mfsimulation.py @@ -1166,12 +1166,17 @@ def _rename_package_group(group_dict, name): # update package file names and count for package in package_array: if package.package_type not in package_type_count: - package.filename = f"{name}.{package.package_type}" + file_name = f"{name}.{package.package_type}" package_type_count[package.package_type] = 1 else: package_type_count[package.package_type] += 1 ptc = package_type_count[package.package_type] - package.filename = f"{name}_{ptc}.{package.package_type}" + file_name = f"{name}_{ptc}.{package.package_type}" + base_filepath = os.path.split(package.filename)[0] + if base_filepath != "": + # include existing relative path in new file name + file_name = os.path.join(base_filepath, file_name) + package.filename = file_name def _rename_exchange_file(self, package, new_filename): self._exchange_files[package.filename] = package @@ -1264,10 +1269,12 @@ def update_package_filename(self, package, new_name): self._ims_files[new_name] = self._ims_files.pop(package.filename) self._update_ims_solution_group(package.filename, new_name) if package.filename in self._ghost_node_files: + self._update_exg_files_gnc(package.filename, new_name) self._ghost_node_files[new_name] = self._ghost_node_files.pop( package.filename ) if package.filename in self._mover_files: + self._update_exg_files_mvr(package.filename, new_name) self._mover_files[new_name] = self._mover_files.pop( package.filename ) @@ -2365,6 +2372,52 @@ def _is_in_solution_group(self, item, index, any_idx_after=False): return True return False + def _update_exg_files_gnc(self, gnc_filename, new_filename): + for exchange_file in self._exchange_files.values(): + if ( + hasattr(exchange_file, "gnc_filerecord") + and exchange_file.gnc_filerecord.has_data() + ): + try: + gnc_file = exchange_file.gnc_filerecord.get_data() + except MFDataException as mfde: + message = ( + "An error occurred while retrieving the ghost " + "node file record from exchange file " + '"{}".'.format(exchange_file.filename) + ) + raise MFDataException( + mfdata_except=mfde, + package=exchange_file._get_pname(), + message=message, + ) + if gnc_file[0][0] == gnc_filename: + gnc_file[0][0] = new_filename + exchange_file.gnc_filerecord.set_data(gnc_file) + + def _update_exg_file_mvr(self, mvr_filename, new_filename): + for exchange_file in self._exchange_files.values(): + if ( + hasattr(exchange_file, "mvr_filerecord") + and exchange_file.mvr_filerecord.has_data() + ): + try: + mvr_file = exchange_file.mvr_filerecord.get_data()[0][0] + except MFDataException as mfde: + message = ( + "An error occurred while retrieving the mover " + "file record from exchange file " + '"{}".'.format(exchange_file.filename) + ) + raise MFDataException( + mfdata_except=mfde, + package=exchange_file._get_pname(), + message=message, + ) + if mvr_file[0][0] == mvr_filename: + mvr_file[0][0] = new_filename + exchange_file.mvr_filerecord.set_data(mvr_file) + def plot(self, model_list=None, SelPackList=None, **kwargs): """ Plot simulation or models.