From 1a3cc05242574c0240c7a84e27766fd169a32a21 Mon Sep 17 00:00:00 2001 From: Scott Paulinski Date: Fri, 10 Dec 2021 10:32:00 -0800 Subject: [PATCH 1/2] fix(filenames): fixed how spaces in filenames are handled. filenames with spaces in them now are read correctly from the package files and are written to the package file in quotes (#1236) --- flopy/mf6/data/mfdata.py | 3 ++- flopy/mf6/data/mfdatascalar.py | 7 ++++++- flopy/mf6/data/mfdatastorage.py | 7 ++++--- flopy/mf6/data/mffileaccess.py | 7 +++++++ flopy/mf6/data/mfstructure.py | 19 ++++++++++++++++--- flopy/mf6/mfbase.py | 4 ++++ flopy/mf6/mfpackage.py | 21 ++++++++++++++++++--- flopy/mf6/modflow/mfsimulation.py | 2 +- flopy/utils/datautil.py | 19 +++++++++++++++++-- 9 files changed, 75 insertions(+), 14 deletions(-) diff --git a/flopy/mf6/data/mfdata.py b/flopy/mf6/data/mfdata.py index 9f5e2a0193..1fc4935b9b 100644 --- a/flopy/mf6/data/mfdata.py +++ b/flopy/mf6/data/mfdata.py @@ -10,6 +10,7 @@ from ..data.mfstructure import DatumType from ..coordinates.modeldimensions import DataDimensions, DiscretizationType from ...datbase import DataInterface, DataType +from ...utils import datautil from .mfdatastorage import DataStructureType from .mfdatautil import to_string from ...mbase import ModelInterface @@ -598,7 +599,7 @@ def _get_external_formatting_string(self, layer, ext_file_action): ext_file_path = file_mgmt.get_updated_path( layer_storage.fname, model_name, ext_file_action ) - layer_storage.fname = ext_file_path + layer_storage.fname = datautil.clean_filename(ext_file_path) ext_format = ["OPEN/CLOSE", f"'{ext_file_path}'"] if storage.data_structure_type != DataStructureType.recarray: if layer_storage.factor is not None: diff --git a/flopy/mf6/data/mfdatascalar.py b/flopy/mf6/data/mfdatascalar.py index 11e1327369..ee0fd8263d 100644 --- a/flopy/mf6/data/mfdatascalar.py +++ b/flopy/mf6/data/mfdatascalar.py @@ -4,6 +4,7 @@ from ..data import mfdata from ..mfbase import ExtFileAction, MFDataException from ...datbase import DataType +from ...utils.datautil import clean_filename from .mfdatautil import convert_data, to_string from .mffileaccess import MFFileAccessScalar from .mfdatastorage import DataStorage, DataStructureType, DataStorageType @@ -164,7 +165,11 @@ def set_data(self, data): data = [data] else: if isinstance(data, str): - data = data.strip().split()[-1] + if self.structure.file_data or self.structure.nam_file_data: + # clean up file name data + data = clean_filename(data) + else: + data = data.strip().split()[-1] else: while ( isinstance(data, list) diff --git a/flopy/mf6/data/mfdatastorage.py b/flopy/mf6/data/mfdatastorage.py index 45ecbcb57d..5881c52688 100644 --- a/flopy/mf6/data/mfdatastorage.py +++ b/flopy/mf6/data/mfdatastorage.py @@ -16,6 +16,7 @@ PyListUtil, ArrayIndexIter, MultiList, + clean_filename, ) from .mfdatautil import convert_data, MFComment from .mffileaccess import MFFileAccessArray, MFFileAccessList, MFFileAccess @@ -2065,9 +2066,9 @@ def process_open_close_line(self, arr_line, layer, store=True): layer, ) if arr_line[0].lower() == "open/close": - data_file = arr_line[1] + data_file = clean_filename(arr_line[1]) else: - data_file = arr_line[0] + data_file = clean_filename(arr_line[0]) elif isinstance(arr_line, dict): for key, value in arr_line.items(): if key.lower() == "factor": @@ -2104,7 +2105,7 @@ def process_open_close_line(self, arr_line, layer, store=True): if key.lower() == "data": data = value if "filename" in arr_line: - data_file = arr_line["filename"] + data_file = clean_filename(arr_line["filename"]) if data_file is None: message = ( diff --git a/flopy/mf6/data/mffileaccess.py b/flopy/mf6/data/mffileaccess.py index e78ecbf0fd..d7fb7787ce 100644 --- a/flopy/mf6/data/mffileaccess.py +++ b/flopy/mf6/data/mffileaccess.py @@ -2164,6 +2164,13 @@ def _append_data_list( data_item, sub_amt=sub_amt, ) + if ( + data_item.indicates_file_name() + or data_item.file_nam_in_nam_file() + ): + data_converted = datautil.clean_filename( + data_converted + ) if add_to_last_line: self._last_line_info[-1].append( [data_index, data_item.type, 0] diff --git a/flopy/mf6/data/mfstructure.py b/flopy/mf6/data/mfstructure.py index 8101c59197..648d2d4656 100644 --- a/flopy/mf6/data/mfstructure.py +++ b/flopy/mf6/data/mfstructure.py @@ -891,7 +891,8 @@ class MFDataItemStructure: def __init__(self): self.file_name_keywords = {"filein": False, "fileout": False} - self.contained_keywords = {"file_name": True} + self.file_name_key_seq = {"fname": True} + self.contained_keywords = {"fname": True, "file": True, "tdis6": True} self.block_name = None self.name = None self.display_name = None @@ -1151,11 +1152,16 @@ def get_keystring_desc(self, line_size, initial_indent, level_indent): ) return description + def file_nam_in_nam_file(self): + for key, item in self.contained_keywords.items(): + if self.name.lower().find(key) != -1: + return True + def indicates_file_name(self): if self.name.lower() in self.file_name_keywords: return True - for key, item in self.contained_keywords.items(): - if self.name.lower().find(key) != -1: + for key in self.file_name_key_seq.keys(): + if key in self.name.lower(): return True return False @@ -1415,6 +1421,7 @@ def __init__(self, data_item, model_data, package_type, dfn_list): self.num_data_items = len(data_item.data_items) self.record_within_record = False self.file_data = False + self.nam_file_data = False self.block_type = data_item.block_type self.block_variable = data_item.block_variable self.model_data = model_data @@ -1545,6 +1552,9 @@ def add_item(self, item, record=False, dfn_list=None): self.path, ) if isinstance(item, MFDataItemStructure): + self.nam_file_data = ( + self.nam_file_data or item.file_nam_in_nam_file() + ) self.file_data = ( self.file_data or item.indicates_file_name() ) @@ -1558,6 +1568,9 @@ def add_item(self, item, record=False, dfn_list=None): # insert placeholder in array self.data_item_structures.append(None) if isinstance(item, MFDataItemStructure): + self.nam_file_data = ( + self.nam_file_data or item.file_nam_in_nam_file() + ) self.file_data = ( self.file_data or item.indicates_file_name() ) diff --git a/flopy/mf6/mfbase.py b/flopy/mf6/mfbase.py index c3c50ab653..974667ca52 100644 --- a/flopy/mf6/mfbase.py +++ b/flopy/mf6/mfbase.py @@ -431,6 +431,10 @@ def resolve_path( else: file_path = path + # remove quote characters from file path + file_path = file_path.replace("'", "") + file_path = file_path.replace('"', "") + if os.path.isabs(file_path): # path is an absolute path if move_abs_paths: diff --git a/flopy/mf6/mfpackage.py b/flopy/mf6/mfpackage.py index 4b6c48186b..17c9f78021 100644 --- a/flopy/mf6/mfpackage.py +++ b/flopy/mf6/mfpackage.py @@ -852,7 +852,8 @@ def load(self, block_header, fd, strict=True): f' opening external file "{file_name}"...' ) external_file_info = arr_line - fd_block = open(os.path.join(root_path, arr_line[1]), "r") + file_name = datautil.clean_filename(arr_line[1]) + fd_block = open(os.path.join(root_path, file_name), "r") # read first line of external file line = fd_block.readline() arr_line = datautil.PyListUtil.split_data_line(line) @@ -1516,6 +1517,8 @@ class MFPackage(PackageContainer, PackageInterface): String defining the package type filename : str Filename of file where this package is stored + quoted_filename : str + Filename with quotes around it when there is a space in the name pname : str Package name loading_package : bool @@ -1647,7 +1650,9 @@ def __init__( message, model_or_sim.simulation_data.debug, ) - self._filename = MFFileMgmt.string_to_file_path(filename) + self._filename = MFFileMgmt.string_to_file_path( + datautil.clean_filename(filename) + ) self.path, self.structure = model_or_sim.register_package( self, not loading_package, pname is None, filename is None ) @@ -1714,6 +1719,13 @@ def filename(self): """Package's file name.""" return self._filename + @property + def quoted_filename(self): + """Package's file name with quotes if there is a space.""" + if " " in self._filename: + return f'"{self._filename}"' + return self._filename + @filename.setter def filename(self, fname): """Package's file name.""" @@ -1722,6 +1734,7 @@ def filename(self, fname): and self.structure.file_type in self.parent_file._child_package_groups ): + fname = datautil.clean_filename(fname) try: child_pkg_group = self.parent_file._child_package_groups[ self.structure.file_type @@ -2164,7 +2177,9 @@ def load(self, strict=True): """ # open file try: - fd_input_file = open(self.get_file_path(), "r") + fd_input_file = open( + datautil.clean_filename(self.get_file_path()), "r" + ) except OSError as e: if e.errno == errno.ENOENT: message = "File {} of type {} could not be opened.".format( diff --git a/flopy/mf6/modflow/mfsimulation.py b/flopy/mf6/modflow/mfsimulation.py index ee64ebc09e..9d78701531 100644 --- a/flopy/mf6/modflow/mfsimulation.py +++ b/flopy/mf6/modflow/mfsimulation.py @@ -1990,7 +1990,7 @@ def register_package( return path, self.structure.name_file_struct_obj elif package.package_type.lower() == "tdis": self._tdis_file = package - self._set_timing_block(package.filename) + self._set_timing_block(package.quoted_filename) return ( path, self.structure.package_struct_objs[ diff --git a/flopy/utils/datautil.py b/flopy/utils/datautil.py index 71bf50e4ca..c499f91daa 100644 --- a/flopy/utils/datautil.py +++ b/flopy/utils/datautil.py @@ -1,5 +1,18 @@ import os import numpy as np +import shlex + + +def clean_filename(file_name): + if ( + file_name[0] in PyListUtil.quote_list + and file_name[-1] in PyListUtil.quote_list + ): + # quoted string + # keep entire string and remove the quotes + f_name = file_name.strip('"') + return f_name.strip("'") + return file_name def clean_name(name): @@ -295,7 +308,8 @@ def split_data_line(line, external_file=False, delimiter_conf_length=15): else: # compare against the default split option without comments split comment_split = line.split("#", 1) - clean_line = comment_split[0].strip().split() + # first try standard split preserving quotes + clean_line = shlex.split(comment_split[0].strip(), posix=False) if len(comment_split) > 1: clean_line.append("#") clean_line.append(comment_split[1].strip()) @@ -342,7 +356,8 @@ def split_data_line(line, external_file=False, delimiter_conf_length=15): if item and item[0] in PyListUtil.quote_list: # starts with a quote, handle quoted text if item[-1] in PyListUtil.quote_list: - arr_fixed_line.append(item[1:-1]) + # if quoted on both ends, keep quotes + arr_fixed_line.append(item) else: arr_fixed_line.append(item[1:]) # loop until trailing quote found From abc06918723cb20f56d69821baa974903a4bd75e Mon Sep 17 00:00:00 2001 From: Scott Paulinski Date: Fri, 10 Dec 2021 10:34:40 -0800 Subject: [PATCH 2/2] fix(filenames): test cases now include filenames with spaces in them --- autotest/t505_test.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/autotest/t505_test.py b/autotest/t505_test.py index 84d541c862..fe7b1c3ad5 100644 --- a/autotest/t505_test.py +++ b/autotest/t505_test.py @@ -403,7 +403,7 @@ def test_array(): delc=5000.0, top=100.0, botm=[50.0, 0.0, -50.0, -100.0], - filename="{}.dis".format(model_name), + filename=f"{model_name} 1.dis", ) ic_package = mf6.ModflowGwfic( model, strt=90.0, filename=f"{model_name}.ic" @@ -526,9 +526,11 @@ def test_array(): write_headers=False, ) model = test_sim.get_model() + dis = model.get_package("dis") rcha = model.get_package("rcha") wel = model.get_package("wel") drn = model.get_package("drn") + assert os.path.split(dis.filename)[1] == f"{model_name} 1.dis" # do same tests as above val_irch = rcha.irch.array.sum(axis=(1, 2, 3)) assert val_irch[0] == 4 @@ -767,8 +769,8 @@ def test_np001(): oc_package = ModflowGwfoc( model, - budget_filerecord=[("np001_mod.cbc",)], - head_filerecord=[("np001_mod.hds",)], + budget_filerecord=[("np001_mod 1.cbc",)], + head_filerecord=[("np001_mod 1.hds",)], saverecord={ 0: [("HEAD", "ALL"), ("BUDGET", "ALL")], 1: [], @@ -803,7 +805,7 @@ def test_np001(): # test saving a binary file with list data well_spd = { 0: { - "filename": "wel0.bin", + "filename": "wel 0.bin", "binary": True, "data": [(0, 0, 4, -2000.0), (0, 0, 7, -2.0)], }, @@ -933,7 +935,7 @@ def test_np001(): ) # compare output to expected results - head_new = os.path.join(run_folder, "np001_mod.hds") + head_new = os.path.join(run_folder, "np001_mod 1.hds") outfile = os.path.join(run_folder, "head_compare.dat") assert pymake.compare_heads( None, @@ -982,7 +984,7 @@ def test_np001(): ) # compare output to expected results - head_new = os.path.join(run_folder_new, "np001_mod.hds") + head_new = os.path.join(run_folder_new, "np001_mod 1.hds") outfile = os.path.join(run_folder_new, "head_compare.dat") assert pymake.compare_heads( None, @@ -1217,13 +1219,13 @@ def test_np002(): sim.register_ims_package(ims_package, [model.name]) # get rid of top_data.txt so that a later test does not automatically pass - top_data_file = os.path.join(run_folder, "top_data.txt") + top_data_file = os.path.join(run_folder, "top data.txt") if os.path.isfile(top_data_file): os.remove(top_data_file) # test loading data to be stored in a file and loading data from a file # using the "dictionary" input format top = { - "filename": "top_data.txt", + "filename": "top data.txt", "factor": 1.0, "data": [ 100.0,