refactor(generate_classes): deprecate branch for ref, introduce repo, test commit hashes

.github/workflows/commit.yml

@@ -209,18 +209,18 @@ jobs:
 
       - name: Update FloPy packages
         if: runner.os != 'Windows'
-        run: python -c 'import flopy; flopy.mf6.utils.generate_classes(branch="develop", backup=False)'
+        run: python -c 'import flopy; flopy.mf6.utils.generate_classes(ref="develop", backup=False)'
 
       - name: Update FloPy packages
         if: runner.os == 'Windows'
         shell: bash -l {0}
-        run: python -c 'import flopy; flopy.mf6.utils.generate_classes(branch="develop", backup=False)'
+        run: python -c 'import flopy; flopy.mf6.utils.generate_classes(ref="develop", backup=False)'
 
       - name: Run tests
         if: runner.os != 'Windows'
         working-directory: ./autotest
         run: |
-          pytest -v -m="not example and not regression" -n=auto --cov=flopy --cov-report=xml --durations=0 --keep-failed=.failed
+          pytest -v -m="not example and not regression" -n=auto --cov=flopy --cov-report=xml --durations=0 --keep-failed=.failed --dist loadfile
           coverage report
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -230,7 +230,7 @@ jobs:
         shell: bash -l {0}
         working-directory: ./autotest
         run: |
-          pytest -v -m="not example and not regression" -n=auto --cov=flopy --cov-report=xml --durations=0 --keep-failed=.failed
+          pytest -v -m="not example and not regression" -n=auto --cov=flopy --cov-report=xml --durations=0 --keep-failed=.failed --dist loadfile
           coverage report
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/examples.yml

@@ -86,12 +86,12 @@ jobs:
 
       - name: Update FloPy packages
         if: runner.os != 'Windows'
-        run: python -c 'import flopy; flopy.mf6.utils.generate_classes(branch="develop", backup=False)'
+        run: python -c 'import flopy; flopy.mf6.utils.generate_classes(ref="develop", backup=False)'
 
       - name: Update FloPy packages
         if: runner.os == 'Windows'
         shell: bash -l {0}
-        run: python -c 'import flopy; flopy.mf6.utils.generate_classes(branch="develop", backup=False)'
+        run: python -c 'import flopy; flopy.mf6.utils.generate_classes(ref="develop", backup=False)'
 
       - name: Run example tests
         if: runner.os != 'Windows'

.github/workflows/regression.yml

@@ -70,12 +70,12 @@ jobs:
 
       - name: Update FloPy packages
         if: runner.os != 'Windows'
-        run: python -c 'import flopy; flopy.mf6.utils.generate_classes(branch="develop", backup=False)'
+        run: python -c 'import flopy; flopy.mf6.utils.generate_classes(ref="develop", backup=False)'
 
       - name: Update FloPy packages
         if: runner.os == 'Windows'
         shell: bash -l {0}
-        run: python -c 'import flopy; flopy.mf6.utils.generate_classes(branch="develop", backup=False)'
+        run: python -c 'import flopy; flopy.mf6.utils.generate_classes(ref="develop", backup=False)'
 
       - name: Run regression tests
         if: runner.os != 'Windows'

.github/workflows/release.yml

@@ -64,14 +64,14 @@ jobs:
           echo "version=${ver#"v"}" >> $GITHUB_OUTPUT
 
       - name: Update FloPy packages
-        run: python -c 'import flopy; flopy.mf6.utils.generate_classes(branch="master", backup=False)'
+        run: python -c 'import flopy; flopy.mf6.utils.generate_classes(ref="master", backup=False)'
 
       - name: Lint Python files
         run: python scripts/pull_request_prepare.py
 
       - name: Run tests
         working-directory: autotest
-        run: pytest -v -m="not example and not regression" -n=auto --durations=0 --keep-failed=.failed
+        run: pytest -v -m="not example and not regression" -n=auto --durations=0 --keep-failed=.failed --dist loadfile
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

autotest/test_generate_classes.py

@@ -17,10 +17,14 @@ def nonempty(itr: Iterable):
 
 def pytest_generate_tests(metafunc):
     # defaults
+    owner = "MODFLOW-USGS"
+    repo = "modflow6"
     ref = [
-        "MODFLOW-USGS/modflow6/develop",
-        "MODFLOW-USGS/modflow6/master",
-        "MODFLOW-USGS/modflow6/6.4.1",
+        f"{owner}/{repo}/develop",
+        f"{owner}/{repo}/master",
+        f"{owner}/{repo}/6.4.1",
+        f"{owner}/{repo}/4458f9f",
+        f"{owner}/{repo}/4458f9f7a6244182e6acc2430a6996f9ca2df367",
     ]
 
     # refs provided as env vars override the defaults
@@ -51,7 +55,18 @@ def pytest_generate_tests(metafunc):
 
 @pytest.mark.mf6
 @pytest.mark.slow
-def test_generate_classes_from_dfn(virtualenv, project_root_path, ref):
+@pytest.mark.regression
+def test_generate_classes_from_github_refs(
+    request, virtualenv, project_root_path, ref, worker_id
+):
+    argv = (
+        request.config.workerinput["mainargv"]
+        if hasattr(request.config, "workerinput")
+        else []
+    )
+    if worker_id != "master" and "loadfile" not in argv:
+        pytest.skip("can't run in parallel")
+
     python = virtualenv.python
     venv = Path(python).parent
     print(
@@ -81,13 +96,16 @@ def test_generate_classes_from_dfn(virtualenv, project_root_path, ref):
     # generate classes
     spl = ref.split("/")
     owner = spl[0]
-    branch = spl[2]
+    repo = spl[1]
+    ref = spl[2]
     pprint(
         virtualenv.run(
             "python -c 'from flopy.mf6.utils import generate_classes; generate_classes(owner=\""
             + owner
-            + '", branch="'
-            + branch
+            + '", repo="'
+            + repo
+            + f'", ref="'
+            + ref
             + "\", backup=False)'"
         )
     )

docs/generate_classes.md

@@ -40,17 +40,26 @@ LIST OF FILES IN C:\Users\***\flopy\flopy\mf6\modflow
         ...
 ```
 
-The `generate_classes()` function has several optional arguments.
+The `generate_classes()` function has several optional parameters.
 
 ```python
 # use the develop branch instead
-generate_classes(branch="develop")
+generate_classes(ref="develop")
 
 # use a fork of modflow6
-generate_classes(owner="your-username", branch="your-branch")
+generate_classes(owner="your-username", ref="your-branch")
+
+# maybe your fork has a different name
+generate_classes(owner="your-username", repo="your-modflow6", ref="your-branch")
 
 # local copy of the repo
-generate_classes(dfnpath="../your/dfn/path"))
+generate_classes(dfnpath="../your/dfn/path")
 ```
 
-By default, a backup is made of FloPy's package classes before rewriting them. To disable backups, use `backup=False`.
+Branch names, commit hashes, or tags may be provided to `ref`.
+
+By default, a backup is made of FloPy's package classes before rewriting them. To disable backups, use `backup=False`.
+
+## Testing class generation
+
+Tests for the `generate_classes()` utility are located in `test_generate_classes.py`. The tests depend on [`pytest-virtualenv`](https://pypi.org/project/pytest-virtualenv/) and will be skipped if run in parallel without the `--dist loadfile` option for `pytest-xdist`.

docs/make_release.md

@@ -96,7 +96,7 @@ As described above, making a release manually involves the following steps:
 
 - Run `python scripts/update_version.py -v <semver>` to update the version number stored in `version.txt` and `flopy/version.py`. For an approved release use the `--approve` flag.
 
-- Update MODFLOW 6 dfn files in the repository and MODFLOW 6 package classes by running `python -c 'import flopy; flopy.mf6.utils.generate_classes(branch="master", backup=False)'`
+- Update MODFLOW 6 dfn files in the repository and MODFLOW 6 package classes by running `python -c 'import flopy; flopy.mf6.utils.generate_classes(ref="master", backup=False)'`
 
 - Run `isort` and `black` on the `flopy` module. This can be achieved by running `python scripts/pull_request_prepare.py` from the project root. The commands `isort .` and `black .` can also be run individually instead.
 

flopy/mf6/utils/generate_classes.py

@@ -2,6 +2,7 @@
 import shutil
 import tempfile
 import time
+from warnings import warn
 
 from .createpackages import create_packages
 
@@ -62,10 +63,15 @@ def download_dfn(owner, branch, new_dfn_pth):
     mf6url = mf6url.format(owner, branch)
     print(f"  Downloading MODFLOW 6 repository from {mf6url}")
     with tempfile.TemporaryDirectory() as tmpdirname:
-        download_and_unzip(mf6url, tmpdirname, verbose=True)
-        downloaded_dfn_pth = os.path.join(tmpdirname, f"modflow6-{branch}")
+        dl_path = download_and_unzip(mf6url, tmpdirname, verbose=True)
+        dl_contents = list(dl_path.glob("modflow6-*"))
+        proj_path = next(iter(dl_contents), None)
+        if not proj_path:
+            raise ValueError(
+                f"Could not find modflow6 project dir in {dl_path}: found {dl_contents}"
+            )
         downloaded_dfn_pth = os.path.join(
-            downloaded_dfn_pth, "doc", "mf6io", "mf6ivar", "dfn"
+            proj_path, "doc", "mf6io", "mf6ivar", "dfn"
         )
         shutil.copytree(downloaded_dfn_pth, new_dfn_pth)
 
@@ -104,7 +110,12 @@ def delete_mf6_classes():
 
 
 def generate_classes(
-    owner="MODFLOW-USGS", branch="master", dfnpath=None, backup=True
+    owner="MODFLOW-USGS",
+    repo="modflow6",
+    branch="master",
+    ref=None,
+    dfnpath=None,
+    backup=True,
 ):
     """
     Generate the MODFLOW 6 flopy classes using definition files from the
@@ -116,9 +127,16 @@ def generate_classes(
     owner : str
         Owner of the MODFLOW 6 repository to use to update the definition
         files and generate the MODFLOW 6 classes. Default is MODFLOW-USGS.
+    repo : str
+        Name of the MODFLOW 6 repository to use to update the definition.
     branch : str
         Branch name of the MODFLOW 6 repository to use to update the
         definition files and generate the MODFLOW 6 classes. Default is master.
+
+        .. deprecated:: 3.5.0
+            Use ref instead.
+    ref : str
+        Branch name, tag, or commit hash to use to update the definition.
     dfnpath : str
         Path to a definition file folder that will be used to generate the
         MODFLOW 6 classes.  Default is none, which means that the branch
@@ -140,11 +158,19 @@ def generate_classes(
     # user provided dfnpath
     if dfnpath is None:
         print(
-            f"  Updating the MODFLOW 6 classes using {owner}/modflow6/{branch}"
+            f"  Updating the MODFLOW 6 classes using {owner}/{repo}/{branch}"
         )
         timestr = time.strftime("%Y%m%d-%H%M%S")
         new_dfn_pth = os.path.join(flopypth, "mf6", "data", f"dfn_{timestr}")
-        download_dfn(owner, branch, new_dfn_pth)
+
+        # branch deprecated 3.5.0
+        if not ref:
+            if not branch:
+                raise ValueError("branch or ref must be provided")
+            warn("branch is deprecated, use ref instead", DeprecationWarning)
+            ref = branch
+
+        download_dfn(owner, ref, new_dfn_pth)
     else:
         print(f"  Updating the MODFLOW 6 classes using {dfnpath}")
         assert os.path.isdir(dfnpath)