Unpin Pandas and update CI

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest] # , macos-latest] 
-        python-version: [3.8, 3.9, "3.10", "3.11"]
+        python-version: [3.8, 3.9, "3.10", "3.11", "3.12"]
         run-type: [std]
         test-path: ["."]
         include:
@@ -24,7 +24,7 @@ jobs:
 
     steps:
     - name: Checkout repo
-      uses:  actions/checkout@v3 # checksout this repo
+      uses:  actions/checkout@v4 # checksout this repo
 
     - name: Set Windows ENV
       if: runner.os == 'Windows'
@@ -85,7 +85,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
     - name: Coveralls
-      uses: coverallsapp/github-action@master
+      uses: coverallsapp/github-action@v2
       with:
         github-token: ${{ secrets.GITHUB_TOKEN }}
         flag-name: ${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.test-path }}
@@ -97,7 +97,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Coveralls Finished
-      uses: coverallsapp/github-action@master
+      uses: coverallsapp/github-action@v2
       with:
         github-token: ${{ secrets.GITHUB_TOKEN }}
         parallel-finished: true

autotest/plot_tests.py

@@ -369,11 +369,11 @@ def ensemble_change_test(tmp_path):
     os.chdir(bd)
 
 if __name__ == "__main__":
-    plot_summary_test('.')
-    # pst_plot_test()
+    # plot_summary_test('.')
+    pst_plot_test('.')
     #ensemble_summary_test('.')
     #ensemble_plot_test()
-    ensemble_1to1_test('.')
+    # ensemble_1to1_test('.')
     #ensemble_plot_test('.')
     #ensemble_change_test('.')
 

autotest/pst_from_tests.py

@@ -2284,7 +2284,7 @@ def mf6_freyberg_direct_test(tmp_path):
         org_ghb = pd.read_csv(os.path.join(pf.new_d,"org","freyberg6.ghb_stress_period_data_1.txt"),
                               header=None,names=["l","r","c","stage","cond"])
         new_ghb = pd.read_csv(os.path.join(pf.new_d, "freyberg6.ghb_stress_period_data_1.txt"),
-                              delim_whitespace=True,
+                              sep=r"\s+",
                               header=None, names=["l", "r", "c", "stage", "cond"])
         d = org_ghb.stage - new_ghb.stage
         print(d)
@@ -2299,7 +2299,7 @@ def mf6_freyberg_direct_test(tmp_path):
         org_ghb = pd.read_csv(os.path.join(pf.new_d, "org", "freyberg6.ghb_stress_period_data_1.txt"),
                               header=None, names=["l", "r", "c", "stage", "cond"])
         new_ghb = pd.read_csv(os.path.join(pf.new_d, "freyberg6.ghb_stress_period_data_1.txt"),
-                              delim_whitespace=True,
+                              sep=r"\s+",
                               header=None, names=["l", "r", "c", "stage", "cond"])
         d = (org_ghb.stage - new_ghb.stage).apply(np.abs)
         print(d)
@@ -2311,9 +2311,9 @@ def mf6_freyberg_direct_test(tmp_path):
         pst.write(os.path.join(pf.new_d, "freyberg.pst"))
         pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d)
         org_ghb = pd.read_csv(os.path.join(tmp_model_ws,"freyberg6.ghb_stress_period_data_1.txt"),
-                              header=None, names=["l", "r", "c", "stage", "cond"],delim_whitespace=True)
+                              header=None, names=["l", "r", "c", "stage", "cond"],sep=r"\s+")
         new_ghb = pd.read_csv(os.path.join(pf.new_d, "freyberg6.ghb_stress_period_data_1.txt"),
-                              delim_whitespace=True,
+                              sep=r"\s+",
                               header=None, names=["l", "r", "c", "stage", "cond"])
         d = org_ghb.stage - new_ghb.stage
         print(new_ghb.stage)
@@ -2328,9 +2328,9 @@ def mf6_freyberg_direct_test(tmp_path):
         pst.write(os.path.join(pf.new_d, "freyberg.pst"))
         pyemu.os_utils.run("{0} freyberg.pst".format(ies_exe_path), cwd=pf.new_d)
         org_ghb = pd.read_csv(os.path.join(tmp_model_ws, "freyberg6.ghb_stress_period_data_1.txt"),
-                              header=None, names=["l", "r", "c", "stage", "cond"], delim_whitespace=True)
+                              header=None, names=["l", "r", "c", "stage", "cond"], sep=r"\s+")
         new_ghb = pd.read_csv(os.path.join(pf.new_d, "freyberg6.ghb_stress_period_data_1.txt"),
-                              delim_whitespace=True,
+                              sep=r"\s+",
                               header=None, names=["l", "r", "c", "stage", "cond"])
         d = (org_ghb.stage * 1.1) - new_ghb.stage
         print(new_ghb.stage)

autotest/smoother/freyberg/template/run.py

@@ -193,7 +193,7 @@ def make_pst():
     pdata.loc["sy","pargp"] = "storage"
 
     hds = pd.read_csv("freyberg.hds.truth.obf",header=None,names=["obsnme","obsval"],index_col=0,
-        delim_whitespace=True)
+                      sep=r"\s+")
     pst.observation_data.loc[:,"weight"] = 0.0
     pst.observation_data.loc[:,"obgnme"] = "forecast"
     groups = pst.observation_data.groupby(pst.observation_data.obsnme.apply(lambda x:x in obs_names)).groups

autotest/smoother_tests_ignore.py

@@ -1820,7 +1820,7 @@ def setup_lorenz():
         f.write("import os\nimport numpy as np\nimport pandas as pd\n")
         f.write("sigma,rho,beta = 10.0,28.0,2.66667\n")
 
-        f.write("df = pd.read_csv('{0}',delim_whitespace=True,index_col=0)\n".format(state_file))
+        f.write("df = pd.read_csv('{0}',sep=r'\\s+',index_col=0)\n".format(state_file))
         f.write("x,y,z,dt = df.loc[:,'prev'].values\n")
 
         f.write("df.loc['x','new'] = sigma * (y - x)\n")

autotest/utils/freyberg.nam

@@ -1,5 +1,5 @@
 # Name file for MODFLOW-NWT, generated by Flopy version 3.2.8.
-#xul:0; yul:10000; rotation:0; proj4_str:+init=EPSG:3857; units:meters; lenuni:2; length_multiplier:1.0 ;start_datetime:1-1-1970
+#xul:0; yul:10000; rotation:0; proj4_str:EPSG:3857; units:meters; lenuni:2; length_multiplier:1.0 ;start_datetime:1-1-1970
 LIST               2  freyberg.list
 DIS               11  freyberg.dis 
 BAS6              13  freyberg.bas 

autotest/utils/to_pestpp.py

@@ -13,7 +13,8 @@
 # get hedcon locations
 with open(hedcon_file,'r') as f:
     [f.readline() for _ in range(4)]
-    hed_df = pd.read_csv(f,usecols=[2,3],header=None,names=["row","col"],delim_whitespace=True)
+    hed_df = pd.read_csv(f,usecols=[2,3],header=None,names=["row","col"],
+                         sep=r"\s+")
 hed_df.loc[:,'i'] = hed_df.pop("row") - 1
 hed_df.loc[:,'j'] = hed_df.pop("col") - 1
 

autotest/utils_tests.py

@@ -1511,7 +1511,7 @@ def sfr_reach_obs_test(tmp_path):
     import flopy
     import pandas as pd
     import numpy as np
-    [shutil.copy(os.path.join("utils",f"freyberg.{ext}"), tmp_path)
+    [shutil.copy(os.path.join("utils", f"freyberg.{ext}"), tmp_path)
      for ext in ["sfr.out", "nam", "dis", "bas"]]
     sfr_file = "freyberg.sfr.out"
     bd = os.getcwd()

autotest/verf_test.py

@@ -80,7 +80,8 @@ def predvar1b_test():
 
 def ident_test():
 
-    idf = pd.read_csv(os.path.join(verf_dir,"ident.out"),delim_whitespace=True,index_col="parameter")
+    idf = pd.read_csv(os.path.join(verf_dir, "ident.out"),
+                      sep=r"\s+", index_col="parameter")
 
     la_ord_errvar = pyemu.ErrVar(jco=ord_base+".jco",
                                  predictions=predictions,

etc/environment.yml

@@ -4,10 +4,10 @@ channels:
 dependencies:
   # required
   - python>=3.8
-  - numpy>=1.15.0
-  - pandas<2.1.0
+  - numpy
+  - pandas
   # optional
-  - matplotlib>=1.4.0
+  - matplotlib
   - pyshp
   - jinja2
   # tests
@@ -20,4 +20,4 @@ dependencies:
   - shapely
   - pyproj
   - flopy
-  - modflow-devtools
+  - modflow-devtools

examples/Freyberg/pest_freyberg.py

@@ -20,7 +20,7 @@
 ml.dis.sr.write_gridSpec(os.path.join("misc","freyberg.spc"))
 
 # write the bore coords file
-obs_rowcol = pd.read_csv(os.path.join("misc","obs_rowcol.dat"),delim_whitespace=True)
+obs_rowcol = pd.read_csv(os.path.join("misc","obs_rowcol.dat"),sep=r"\s+")
 obs_rowcol.loc[:,'x'] = ml.dis.sr.xcentergrid[obs_rowcol.row-1,obs_rowcol.col-1]
 obs_rowcol.loc[:,'y'] = ml.dis.sr.ycentergrid[obs_rowcol.row-1,obs_rowcol.col-1]
 obs_rowcol.loc[:,"top"] = ml.dis.top[obs_rowcol.row-1,obs_rowcol.col-1]

examples/Freyberg_transient/forward_run.py

@@ -11,12 +11,12 @@
 def prepare():
 
 	# apply drain conductance parameters
-	drn_df = pd.read_csv("drain_mlt.dat",delim_whitespace=True,header=None,names=["name","cond"])
+	drn_df = pd.read_csv("drain_mlt.dat",sep=r"\s+",header=None,names=["name","cond"])
 	drn_df.index = drn_df.name.apply(lambda x: (int(x[-5:-3])+1,int(x[-2:])+1))
 	drn_files = [f for f in os.listdir(bak_dir) if "drn" in f.lower()]
 	for drn_file in drn_files:
 		df = pd.read_csv(os.path.join(bak_dir,drn_file),header=None,
-			names=["l","r","c","stage","cond"],delim_whitespace=True)
+			names=["l","r","c","stage","cond"],sep=r"\s+")
 		df.index = df.apply(lambda x: (x.r,x.c),axis=1)
 
 		df.loc[:,"cond"] = drn_df.cond

examples/freyberg_sfr_reaches/freyberg.nam

@@ -1,5 +1,5 @@
 # Name file for MODFLOW-NWT, generated by Flopy version 3.2.8.
-#xul:0; yul:10000; rotation:0; proj4_str:+init=EPSG:3857; units:meters; lenuni:2; length_multiplier:1.0 ;start_datetime:1-1-1970
+#xul:0; yul:10000; rotation:0; proj4_str:EPSG:3857; units:meters; lenuni:2; length_multiplier:1.0 ;start_datetime:1-1-1970
 LIST               2  freyberg.list
 DIS               11  freyberg.dis 
 BAS6              13  freyberg.bas 

pyemu/plot/plot_utils.py

@@ -404,7 +404,8 @@ def res_1to1(
             )
         #               plt.colorbar(ax=ax)
         else:
-            ax.scatter([obs_g.obsval], [obs_g.sim], marker=".", s=10, color="b")
+            ax.scatter(obs_g.obsval.values, obs_g.sim.values,
+                       marker=".", s=10, color="b")
 
         ax.plot([mn, mx], [mn, mx], "k--", lw=1.0)
         xlim = (mn, mx)
@@ -425,7 +426,8 @@ def res_1to1(
 
         if histogram == False:
             ax = axes[ax_count]
-            ax.scatter(obs_g.obsval, obs_g.res, marker=".", s=10, color="b")
+            ax.scatter(obs_g.obsval.values, obs_g.res.values,
+                       marker=".", s=10, color="b")
             ylim = ax.get_ylim()
             mx = max(np.abs(ylim[0]), np.abs(ylim[1]))
             if obs_g.shape[0] == 1:

pyemu/pst/pst_handler.py

@@ -748,7 +748,7 @@ def _read_df(f, nrows, names, converters, defaults=None):  # todo : drop method?
                 header=None,
                 names=names,
                 nrows=nrows,
-                delim_whitespace=True,
+                sep=r"\s+",
                 converters=converters,
                 index_col=False,
                 comment="#",
@@ -942,7 +942,7 @@ def _cast_prior_df_from_lines(self, section, lines, pst_path="."):
                 missing_vals = options.get("missing_values", None)
                 if sep.lower() == "w":
                     df = pd.read_csv(
-                        filename, delim_whitespace=True, na_values=missing_vals,low_memory=False
+                        filename, sep=r"\s+", na_values=missing_vals, low_memory=False
                     )
                 else:
                     df = pd.read_csv(filename, sep=sep, na_values=missing_vals,low_memory=False)
@@ -2084,7 +2084,7 @@ def bounds_report(self, iterations=None):
                     skiprows=1,
                     index_col=0,
                     usecols=[0, 1],
-                    delim_whitespace=True,
+                    sep=r"\s+",
                     header=None,
                     low_memory = False
                 )

pyemu/pst/pst_utils.py

@@ -927,7 +927,7 @@ def _try_run_inschek(ins_file, out_file, cwd="."):
         pyemu.os_utils.run("inschek {0} {1}".format(ins_file, out_file), cwd=cwd)
         obf_file = os.path.join(cwd, ins_file.replace(".ins", ".obf"))
         df = pd.read_csv(
-            obf_file, delim_whitespace=True, skiprows=0, index_col=0, names=["obsval"]
+            obf_file, sep=r"\s+", skiprows=0, index_col=0, names=["obsval"]
         )
         df.index = df.index.map(str.lower)
         return df

pyemu/utils/helpers.py

@@ -2112,8 +2112,6 @@ def _process_list_file(model_file, df):
     else:
         sep = None
     datastrtrow = df_mf.head_rows.values[-1]
-    if fmt.lower() == "free" and sep == " ":
-        delim_whitespace = True
     if datastrtrow > 0:
         with open(org_file, "r") as fp:
             storehead = [next(fp) for _ in range(datastrtrow)]

verification/Freyberg/pest_freyberg.py

@@ -20,7 +20,7 @@
 ml.dis.sr.write_gridSpec(os.path.join("misc","freyberg.spc"))
 
 # write the bore coords file
-obs_rowcol = pd.read_csv(os.path.join("misc","obs_rowcol.dat"),delim_whitespace=True)
+obs_rowcol = pd.read_csv(os.path.join("misc","obs_rowcol.dat"), sep=r"\s+")
 obs_rowcol.loc[:,'x'] = ml.dis.sr.xcentergrid[obs_rowcol.row-1,obs_rowcol.col-1]
 obs_rowcol.loc[:,'y'] = ml.dis.sr.ycentergrid[obs_rowcol.row-1,obs_rowcol.col-1]
 obs_rowcol.loc[:,"top"] = ml.dis.top[obs_rowcol.row-1,obs_rowcol.col-1]