From a41a2356a36e278a5bd0295e79baa163094c7411 Mon Sep 17 00:00:00 2001 From: briochh Date: Tue, 19 Mar 2024 11:17:33 +1300 Subject: [PATCH 1/6] Dropping pandas pin + updating some actions in CI --- .github/workflows/ci.yml | 6 +++--- etc/environment.yml | 8 ++++---- pyproject.toml | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e33f4616..11c1a1a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Checkout repo - uses: actions/checkout@v3 # checksout this repo + uses: actions/checkout@v4 # checksout this repo - name: Set Windows ENV if: runner.os == 'Windows' @@ -85,7 +85,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Coveralls - uses: coverallsapp/github-action@master + uses: coverallsapp/github-action@v2 with: github-token: ${{ secrets.GITHUB_TOKEN }} flag-name: ${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.test-path }} @@ -97,7 +97,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Coveralls Finished - uses: coverallsapp/github-action@master + uses: coverallsapp/github-action@v2 with: github-token: ${{ secrets.GITHUB_TOKEN }} parallel-finished: true diff --git a/etc/environment.yml b/etc/environment.yml index a01ab3d8..efa493d4 100644 --- a/etc/environment.yml +++ b/etc/environment.yml @@ -4,10 +4,10 @@ channels: dependencies: # required - python>=3.8 - - numpy>=1.15.0 - - pandas<2.1.0 + - numpy + - pandas # optional - - matplotlib>=1.4.0 + - matplotlib - pyshp - jinja2 # tests @@ -20,4 +20,4 @@ dependencies: - shapely - pyproj - flopy - - modflow-devtools + - modflow-devtools \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 4cea4f50..9ac7cb84 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [build-system] requires = [ "setuptools >= 61", - 'tomli; python_version < "3.11"', + 'tomli; python_version <= "3.12"', ] build-backend = "setuptools.build_meta" From 41600ed206f225ea6250fa329567cd28e7e03e6f Mon Sep 17 00:00:00 2001 From: briochh Date: Tue, 19 Mar 2024 11:18:43 +1300 Subject: [PATCH 2/6] Adding 3.12 to tests... --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11c1a1a0..6270f548 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest] # , macos-latest] - python-version: [3.8, 3.9, "3.10", "3.11"] + python-version: [3.8, 3.9, "3.10", "3.11", "3.12"] run-type: [std] test-path: ["."] include: From 56b9894dd4be75b2b7eedcad4c5846dd081a1fc9 Mon Sep 17 00:00:00 2001 From: briochh Date: Tue, 19 Mar 2024 14:10:06 +1300 Subject: [PATCH 3/6] Fixing some more deprecations --- autotest/plot_tests.py | 6 +++--- autotest/pst_from_tests.py | 12 ++++++------ autotest/utils/freyberg.nam | 2 +- autotest/utils_tests.py | 2 +- pyemu/plot/plot_utils.py | 6 ++++-- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/autotest/plot_tests.py b/autotest/plot_tests.py index 08166d07..1c380bf0 100644 --- a/autotest/plot_tests.py +++ b/autotest/plot_tests.py @@ -369,11 +369,11 @@ def ensemble_change_test(tmp_path): os.chdir(bd) if __name__ == "__main__": - plot_summary_test('.') - # pst_plot_test() + # plot_summary_test('.') + pst_plot_test('.') #ensemble_summary_test('.') #ensemble_plot_test() - ensemble_1to1_test('.') + # ensemble_1to1_test('.') #ensemble_plot_test('.') #ensemble_change_test('.') diff --git a/autotest/pst_from_tests.py b/autotest/pst_from_tests.py index 1121175e..9337ce6c 100644 --- a/autotest/pst_from_tests.py +++ b/autotest/pst_from_tests.py @@ -2284,7 +2284,7 @@ def mf6_freyberg_direct_test(tmp_path): org_ghb = pd.read_csv(os.path.join(pf.new_d,"org","freyberg6.ghb_stress_period_data_1.txt"), header=None,names=["l","r","c","stage","cond"]) new_ghb = pd.read_csv(os.path.join(pf.new_d, "freyberg6.ghb_stress_period_data_1.txt"), - delim_whitespace=True, + sep=r"\s+", header=None, names=["l", "r", "c", "stage", "cond"]) d = org_ghb.stage - new_ghb.stage print(d) @@ -2299,7 +2299,7 @@ def mf6_freyberg_direct_test(tmp_path): org_ghb = pd.read_csv(os.path.join(pf.new_d, "org", "freyberg6.ghb_stress_period_data_1.txt"), header=None, names=["l", "r", "c", "stage", "cond"]) new_ghb = pd.read_csv(os.path.join(pf.new_d, "freyberg6.ghb_stress_period_data_1.txt"), - delim_whitespace=True, + sep=r"\s+", header=None, names=["l", "r", "c", "stage", "cond"]) d = (org_ghb.stage - new_ghb.stage).apply(np.abs) print(d) @@ -2311,9 +2311,9 @@ def mf6_freyberg_direct_test(tmp_path): pst.write(os.path.join(pf.new_d, "freyberg.pst")) pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) org_ghb = pd.read_csv(os.path.join(tmp_model_ws,"freyberg6.ghb_stress_period_data_1.txt"), - header=None, names=["l", "r", "c", "stage", "cond"],delim_whitespace=True) + header=None, names=["l", "r", "c", "stage", "cond"],sep=r"\s+") new_ghb = pd.read_csv(os.path.join(pf.new_d, "freyberg6.ghb_stress_period_data_1.txt"), - delim_whitespace=True, + sep=r"\s+", header=None, names=["l", "r", "c", "stage", "cond"]) d = org_ghb.stage - new_ghb.stage print(new_ghb.stage) @@ -2328,9 +2328,9 @@ def mf6_freyberg_direct_test(tmp_path): pst.write(os.path.join(pf.new_d, "freyberg.pst")) pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d) org_ghb = pd.read_csv(os.path.join(tmp_model_ws, "freyberg6.ghb_stress_period_data_1.txt"), - header=None, names=["l", "r", "c", "stage", "cond"], delim_whitespace=True) + header=None, names=["l", "r", "c", "stage", "cond"], sep=r"\s+") new_ghb = pd.read_csv(os.path.join(pf.new_d, "freyberg6.ghb_stress_period_data_1.txt"), - delim_whitespace=True, + sep=r"\s+", header=None, names=["l", "r", "c", "stage", "cond"]) d = (org_ghb.stage * 1.1) - new_ghb.stage print(new_ghb.stage) diff --git a/autotest/utils/freyberg.nam b/autotest/utils/freyberg.nam index 1e0cfcfe..7ef68814 100644 --- a/autotest/utils/freyberg.nam +++ b/autotest/utils/freyberg.nam @@ -1,5 +1,5 @@ # Name file for MODFLOW-NWT, generated by Flopy version 3.2.8. -#xul:0; yul:10000; rotation:0; proj4_str:+init=EPSG:3857; units:meters; lenuni:2; length_multiplier:1.0 ;start_datetime:1-1-1970 +#xul:0; yul:10000; rotation:0; proj4_str:EPSG:3857; units:meters; lenuni:2; length_multiplier:1.0 ;start_datetime:1-1-1970 LIST 2 freyberg.list DIS 11 freyberg.dis BAS6 13 freyberg.bas diff --git a/autotest/utils_tests.py b/autotest/utils_tests.py index 821c596b..f4a548c9 100644 --- a/autotest/utils_tests.py +++ b/autotest/utils_tests.py @@ -1511,7 +1511,7 @@ def sfr_reach_obs_test(tmp_path): import flopy import pandas as pd import numpy as np - [shutil.copy(os.path.join("utils",f"freyberg.{ext}"), tmp_path) + [shutil.copy(os.path.join("utils", f"freyberg.{ext}"), tmp_path) for ext in ["sfr.out", "nam", "dis", "bas"]] sfr_file = "freyberg.sfr.out" bd = os.getcwd() diff --git a/pyemu/plot/plot_utils.py b/pyemu/plot/plot_utils.py index b1324a5f..a0744ebc 100644 --- a/pyemu/plot/plot_utils.py +++ b/pyemu/plot/plot_utils.py @@ -404,7 +404,8 @@ def res_1to1( ) # plt.colorbar(ax=ax) else: - ax.scatter([obs_g.obsval], [obs_g.sim], marker=".", s=10, color="b") + ax.scatter(obs_g.obsval.values, obs_g.sim.values, + marker=".", s=10, color="b") ax.plot([mn, mx], [mn, mx], "k--", lw=1.0) xlim = (mn, mx) @@ -425,7 +426,8 @@ def res_1to1( if histogram == False: ax = axes[ax_count] - ax.scatter(obs_g.obsval, obs_g.res, marker=".", s=10, color="b") + ax.scatter(obs_g.obsval.values, obs_g.res.values, + marker=".", s=10, color="b") ylim = ax.get_ylim() mx = max(np.abs(ylim[0]), np.abs(ylim[1])) if obs_g.shape[0] == 1: From 61a901608ccaae510b46df436652f9141d29ac39 Mon Sep 17 00:00:00 2001 From: briochh Date: Tue, 19 Mar 2024 16:09:02 +1300 Subject: [PATCH 4/6] more proj-crs depr --- examples/freyberg_sfr_reaches/freyberg.nam | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/freyberg_sfr_reaches/freyberg.nam b/examples/freyberg_sfr_reaches/freyberg.nam index 1e0cfcfe..7ef68814 100644 --- a/examples/freyberg_sfr_reaches/freyberg.nam +++ b/examples/freyberg_sfr_reaches/freyberg.nam @@ -1,5 +1,5 @@ # Name file for MODFLOW-NWT, generated by Flopy version 3.2.8. -#xul:0; yul:10000; rotation:0; proj4_str:+init=EPSG:3857; units:meters; lenuni:2; length_multiplier:1.0 ;start_datetime:1-1-1970 +#xul:0; yul:10000; rotation:0; proj4_str:EPSG:3857; units:meters; lenuni:2; length_multiplier:1.0 ;start_datetime:1-1-1970 LIST 2 freyberg.list DIS 11 freyberg.dis BAS6 13 freyberg.bas From e3c4864926664f850336f090a04222e97c7a0f9a Mon Sep 17 00:00:00 2001 From: briochh Date: Wed, 20 Mar 2024 10:09:59 +1300 Subject: [PATCH 5/6] more delim_whitespace refac --- autotest/smoother/freyberg/template/run.py | 2 +- autotest/smoother_tests_ignore.py | 2 +- autotest/utils/to_pestpp.py | 3 ++- autotest/verf_test.py | 3 ++- examples/Freyberg/pest_freyberg.py | 2 +- examples/Freyberg_transient/forward_run.py | 4 ++-- pyemu/pst/pst_handler.py | 6 +++--- pyemu/pst/pst_utils.py | 2 +- pyemu/utils/helpers.py | 2 -- verification/Freyberg/pest_freyberg.py | 2 +- 10 files changed, 14 insertions(+), 14 deletions(-) diff --git a/autotest/smoother/freyberg/template/run.py b/autotest/smoother/freyberg/template/run.py index d9db9fd5..cf121302 100644 --- a/autotest/smoother/freyberg/template/run.py +++ b/autotest/smoother/freyberg/template/run.py @@ -193,7 +193,7 @@ def make_pst(): pdata.loc["sy","pargp"] = "storage" hds = pd.read_csv("freyberg.hds.truth.obf",header=None,names=["obsnme","obsval"],index_col=0, - delim_whitespace=True) + sep=r"\s+") pst.observation_data.loc[:,"weight"] = 0.0 pst.observation_data.loc[:,"obgnme"] = "forecast" groups = pst.observation_data.groupby(pst.observation_data.obsnme.apply(lambda x:x in obs_names)).groups diff --git a/autotest/smoother_tests_ignore.py b/autotest/smoother_tests_ignore.py index c105a002..65515dec 100644 --- a/autotest/smoother_tests_ignore.py +++ b/autotest/smoother_tests_ignore.py @@ -1820,7 +1820,7 @@ def setup_lorenz(): f.write("import os\nimport numpy as np\nimport pandas as pd\n") f.write("sigma,rho,beta = 10.0,28.0,2.66667\n") - f.write("df = pd.read_csv('{0}',delim_whitespace=True,index_col=0)\n".format(state_file)) + f.write("df = pd.read_csv('{0}',sep=r'\\s+',index_col=0)\n".format(state_file)) f.write("x,y,z,dt = df.loc[:,'prev'].values\n") f.write("df.loc['x','new'] = sigma * (y - x)\n") diff --git a/autotest/utils/to_pestpp.py b/autotest/utils/to_pestpp.py index 1bb81e89..11efd406 100644 --- a/autotest/utils/to_pestpp.py +++ b/autotest/utils/to_pestpp.py @@ -13,7 +13,8 @@ # get hedcon locations with open(hedcon_file,'r') as f: [f.readline() for _ in range(4)] - hed_df = pd.read_csv(f,usecols=[2,3],header=None,names=["row","col"],delim_whitespace=True) + hed_df = pd.read_csv(f,usecols=[2,3],header=None,names=["row","col"], + sep=r"\s+") hed_df.loc[:,'i'] = hed_df.pop("row") - 1 hed_df.loc[:,'j'] = hed_df.pop("col") - 1 diff --git a/autotest/verf_test.py b/autotest/verf_test.py index 4a111d88..b8ab0d2f 100644 --- a/autotest/verf_test.py +++ b/autotest/verf_test.py @@ -80,7 +80,8 @@ def predvar1b_test(): def ident_test(): - idf = pd.read_csv(os.path.join(verf_dir,"ident.out"),delim_whitespace=True,index_col="parameter") + idf = pd.read_csv(os.path.join(verf_dir, "ident.out"), + sep=r"\s+", index_col="parameter") la_ord_errvar = pyemu.ErrVar(jco=ord_base+".jco", predictions=predictions, diff --git a/examples/Freyberg/pest_freyberg.py b/examples/Freyberg/pest_freyberg.py index 4783aede..c3130903 100644 --- a/examples/Freyberg/pest_freyberg.py +++ b/examples/Freyberg/pest_freyberg.py @@ -20,7 +20,7 @@ ml.dis.sr.write_gridSpec(os.path.join("misc","freyberg.spc")) # write the bore coords file -obs_rowcol = pd.read_csv(os.path.join("misc","obs_rowcol.dat"),delim_whitespace=True) +obs_rowcol = pd.read_csv(os.path.join("misc","obs_rowcol.dat"),sep=r"\s+") obs_rowcol.loc[:,'x'] = ml.dis.sr.xcentergrid[obs_rowcol.row-1,obs_rowcol.col-1] obs_rowcol.loc[:,'y'] = ml.dis.sr.ycentergrid[obs_rowcol.row-1,obs_rowcol.col-1] obs_rowcol.loc[:,"top"] = ml.dis.top[obs_rowcol.row-1,obs_rowcol.col-1] diff --git a/examples/Freyberg_transient/forward_run.py b/examples/Freyberg_transient/forward_run.py index aad16ab4..21aaea4e 100644 --- a/examples/Freyberg_transient/forward_run.py +++ b/examples/Freyberg_transient/forward_run.py @@ -11,12 +11,12 @@ def prepare(): # apply drain conductance parameters - drn_df = pd.read_csv("drain_mlt.dat",delim_whitespace=True,header=None,names=["name","cond"]) + drn_df = pd.read_csv("drain_mlt.dat",sep=r"\s+",header=None,names=["name","cond"]) drn_df.index = drn_df.name.apply(lambda x: (int(x[-5:-3])+1,int(x[-2:])+1)) drn_files = [f for f in os.listdir(bak_dir) if "drn" in f.lower()] for drn_file in drn_files: df = pd.read_csv(os.path.join(bak_dir,drn_file),header=None, - names=["l","r","c","stage","cond"],delim_whitespace=True) + names=["l","r","c","stage","cond"],sep=r"\s+") df.index = df.apply(lambda x: (x.r,x.c),axis=1) df.loc[:,"cond"] = drn_df.cond diff --git a/pyemu/pst/pst_handler.py b/pyemu/pst/pst_handler.py index 99db84c0..c443356f 100644 --- a/pyemu/pst/pst_handler.py +++ b/pyemu/pst/pst_handler.py @@ -748,7 +748,7 @@ def _read_df(f, nrows, names, converters, defaults=None): # todo : drop method? header=None, names=names, nrows=nrows, - delim_whitespace=True, + sep=r"\s+", converters=converters, index_col=False, comment="#", @@ -942,7 +942,7 @@ def _cast_prior_df_from_lines(self, section, lines, pst_path="."): missing_vals = options.get("missing_values", None) if sep.lower() == "w": df = pd.read_csv( - filename, delim_whitespace=True, na_values=missing_vals,low_memory=False + filename, sep=r"\s+", na_values=missing_vals, low_memory=False ) else: df = pd.read_csv(filename, sep=sep, na_values=missing_vals,low_memory=False) @@ -2084,7 +2084,7 @@ def bounds_report(self, iterations=None): skiprows=1, index_col=0, usecols=[0, 1], - delim_whitespace=True, + sep=r"\s+", header=None, low_memory = False ) diff --git a/pyemu/pst/pst_utils.py b/pyemu/pst/pst_utils.py index e950e73b..0bc0365f 100644 --- a/pyemu/pst/pst_utils.py +++ b/pyemu/pst/pst_utils.py @@ -927,7 +927,7 @@ def _try_run_inschek(ins_file, out_file, cwd="."): pyemu.os_utils.run("inschek {0} {1}".format(ins_file, out_file), cwd=cwd) obf_file = os.path.join(cwd, ins_file.replace(".ins", ".obf")) df = pd.read_csv( - obf_file, delim_whitespace=True, skiprows=0, index_col=0, names=["obsval"] + obf_file, sep=r"\s+", skiprows=0, index_col=0, names=["obsval"] ) df.index = df.index.map(str.lower) return df diff --git a/pyemu/utils/helpers.py b/pyemu/utils/helpers.py index 99730326..96505223 100644 --- a/pyemu/utils/helpers.py +++ b/pyemu/utils/helpers.py @@ -2112,8 +2112,6 @@ def _process_list_file(model_file, df): else: sep = None datastrtrow = df_mf.head_rows.values[-1] - if fmt.lower() == "free" and sep == " ": - delim_whitespace = True if datastrtrow > 0: with open(org_file, "r") as fp: storehead = [next(fp) for _ in range(datastrtrow)] diff --git a/verification/Freyberg/pest_freyberg.py b/verification/Freyberg/pest_freyberg.py index 4783aede..12bc9efe 100644 --- a/verification/Freyberg/pest_freyberg.py +++ b/verification/Freyberg/pest_freyberg.py @@ -20,7 +20,7 @@ ml.dis.sr.write_gridSpec(os.path.join("misc","freyberg.spc")) # write the bore coords file -obs_rowcol = pd.read_csv(os.path.join("misc","obs_rowcol.dat"),delim_whitespace=True) +obs_rowcol = pd.read_csv(os.path.join("misc","obs_rowcol.dat"), sep=r"\s+") obs_rowcol.loc[:,'x'] = ml.dis.sr.xcentergrid[obs_rowcol.row-1,obs_rowcol.col-1] obs_rowcol.loc[:,'y'] = ml.dis.sr.ycentergrid[obs_rowcol.row-1,obs_rowcol.col-1] obs_rowcol.loc[:,"top"] = ml.dis.top[obs_rowcol.row-1,obs_rowcol.col-1] From bf53ee09de8f811b7ca115f6e54a3e41f972e96e Mon Sep 17 00:00:00 2001 From: briochh Date: Wed, 20 Mar 2024 15:08:31 +1300 Subject: [PATCH 6/6] reverting tomli python cond. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9ac7cb84..4cea4f50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [build-system] requires = [ "setuptools >= 61", - 'tomli; python_version <= "3.12"', + 'tomli; python_version < "3.11"', ] build-backend = "setuptools.build_meta"