From bd7d94b87592672922d09367d96e9dffdaa6af78 Mon Sep 17 00:00:00 2001 From: Wes Bonelli Date: Sat, 5 Aug 2023 01:07:28 -0400 Subject: [PATCH 1/2] refactor(generate_classes): deprecate branch for ref, test commit hashes --- .github/workflows/commit.yml | 8 +++--- .github/workflows/examples.yml | 4 +-- .github/workflows/regression.yml | 4 +-- .github/workflows/release.yml | 4 +-- autotest/test_generate_classes.py | 32 ++++++++++++++++++------ docs/generate_classes.md | 19 +++++++++++---- docs/make_release.md | 2 +- flopy/mf6/utils/generate_classes.py | 38 ++++++++++++++++++++++++----- 8 files changed, 82 insertions(+), 29 deletions(-) diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index 9864075694..daf46852a5 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -209,18 +209,18 @@ jobs: - name: Update FloPy packages if: runner.os != 'Windows' - run: python -c 'import flopy; flopy.mf6.utils.generate_classes(branch="develop", backup=False)' + run: python -c 'import flopy; flopy.mf6.utils.generate_classes(ref="develop", backup=False)' - name: Update FloPy packages if: runner.os == 'Windows' shell: bash -l {0} - run: python -c 'import flopy; flopy.mf6.utils.generate_classes(branch="develop", backup=False)' + run: python -c 'import flopy; flopy.mf6.utils.generate_classes(ref="develop", backup=False)' - name: Run tests if: runner.os != 'Windows' working-directory: ./autotest run: | - pytest -v -m="not example and not regression" -n=auto --cov=flopy --cov-report=xml --durations=0 --keep-failed=.failed + pytest -v -m="not example and not regression" -n=auto --cov=flopy --cov-report=xml --durations=0 --keep-failed=.failed --dist loadfile coverage report env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -230,7 +230,7 @@ jobs: shell: bash -l {0} working-directory: ./autotest run: | - pytest -v -m="not example and not regression" -n=auto --cov=flopy --cov-report=xml --durations=0 --keep-failed=.failed + pytest -v -m="not example and not regression" -n=auto --cov=flopy --cov-report=xml --durations=0 --keep-failed=.failed --dist loadfile coverage report env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 86eb6154e8..739c408e37 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -86,12 +86,12 @@ jobs: - name: Update FloPy packages if: runner.os != 'Windows' - run: python -c 'import flopy; flopy.mf6.utils.generate_classes(branch="develop", backup=False)' + run: python -c 'import flopy; flopy.mf6.utils.generate_classes(ref="develop", backup=False)' - name: Update FloPy packages if: runner.os == 'Windows' shell: bash -l {0} - run: python -c 'import flopy; flopy.mf6.utils.generate_classes(branch="develop", backup=False)' + run: python -c 'import flopy; flopy.mf6.utils.generate_classes(ref="develop", backup=False)' - name: Run example tests if: runner.os != 'Windows' diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 22981c46f8..daf9c8729a 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -70,12 +70,12 @@ jobs: - name: Update FloPy packages if: runner.os != 'Windows' - run: python -c 'import flopy; flopy.mf6.utils.generate_classes(branch="develop", backup=False)' + run: python -c 'import flopy; flopy.mf6.utils.generate_classes(ref="develop", backup=False)' - name: Update FloPy packages if: runner.os == 'Windows' shell: bash -l {0} - run: python -c 'import flopy; flopy.mf6.utils.generate_classes(branch="develop", backup=False)' + run: python -c 'import flopy; flopy.mf6.utils.generate_classes(ref="develop", backup=False)' - name: Run regression tests if: runner.os != 'Windows' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6ff6af9cf4..9fa496b5b3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -64,14 +64,14 @@ jobs: echo "version=${ver#"v"}" >> $GITHUB_OUTPUT - name: Update FloPy packages - run: python -c 'import flopy; flopy.mf6.utils.generate_classes(branch="master", backup=False)' + run: python -c 'import flopy; flopy.mf6.utils.generate_classes(ref="master", backup=False)' - name: Lint Python files run: python scripts/pull_request_prepare.py - name: Run tests working-directory: autotest - run: pytest -v -m="not example and not regression" -n=auto --durations=0 --keep-failed=.failed + run: pytest -v -m="not example and not regression" -n=auto --durations=0 --keep-failed=.failed --dist loadfile env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/autotest/test_generate_classes.py b/autotest/test_generate_classes.py index e48c73eb17..90ce076c6d 100644 --- a/autotest/test_generate_classes.py +++ b/autotest/test_generate_classes.py @@ -17,10 +17,14 @@ def nonempty(itr: Iterable): def pytest_generate_tests(metafunc): # defaults + owner = "MODFLOW-USGS" + repo = "modflow6" ref = [ - "MODFLOW-USGS/modflow6/develop", - "MODFLOW-USGS/modflow6/master", - "MODFLOW-USGS/modflow6/6.4.1", + f"{owner}/{repo}/develop", + f"{owner}/{repo}/master", + f"{owner}/{repo}/6.4.1", + f"{owner}/{repo}/4458f9f", + f"{owner}/{repo}/4458f9f7a6244182e6acc2430a6996f9ca2df367", ] # refs provided as env vars override the defaults @@ -51,7 +55,18 @@ def pytest_generate_tests(metafunc): @pytest.mark.mf6 @pytest.mark.slow -def test_generate_classes_from_dfn(virtualenv, project_root_path, ref): +@pytest.mark.regression +def test_generate_classes_from_github_refs( + request, virtualenv, project_root_path, ref, worker_id +): + argv = ( + request.config.workerinput["mainargv"] + if hasattr(request.config, "workerinput") + else [] + ) + if worker_id != "master" and "loadfile" not in argv: + pytest.skip("can't run in parallel") + python = virtualenv.python venv = Path(python).parent print( @@ -81,13 +96,16 @@ def test_generate_classes_from_dfn(virtualenv, project_root_path, ref): # generate classes spl = ref.split("/") owner = spl[0] - branch = spl[2] + repo = spl[1] + ref = spl[2] pprint( virtualenv.run( "python -c 'from flopy.mf6.utils import generate_classes; generate_classes(owner=\"" + owner - + '", branch="' - + branch + + '", repo="' + + repo + + f'", ref="' + + ref + "\", backup=False)'" ) ) diff --git a/docs/generate_classes.md b/docs/generate_classes.md index 79374d24fa..e01cb0c8cb 100644 --- a/docs/generate_classes.md +++ b/docs/generate_classes.md @@ -40,17 +40,26 @@ LIST OF FILES IN C:\Users\***\flopy\flopy\mf6\modflow ... ``` -The `generate_classes()` function has several optional arguments. +The `generate_classes()` function has several optional parameters. ```python # use the develop branch instead -generate_classes(branch="develop") +generate_classes(ref="develop") # use a fork of modflow6 -generate_classes(owner="your-username", branch="your-branch") +generate_classes(owner="your-username", ref="your-branch") + +# maybe your fork has a different name +generate_classes(owner="your-username", repo="your-modflow6", ref="your-branch") # local copy of the repo -generate_classes(dfnpath="../your/dfn/path")) +generate_classes(dfnpath="../your/dfn/path") ``` -By default, a backup is made of FloPy's package classes before rewriting them. To disable backups, use `backup=False`. \ No newline at end of file +Branch names, commit hashes, or tags may be provided to `ref`. + +By default, a backup is made of FloPy's package classes before rewriting them. To disable backups, use `backup=False`. + +## Testing class generation + +Tests for the `generate_classes()` utility are located in `test_generate_classes.py`. The tests depend on [`pytest-virtualenv`](https://pypi.org/project/pytest-virtualenv/) and will be skipped if run in parallel without the `--dist loadfile` option for `pytest-xdist`. \ No newline at end of file diff --git a/docs/make_release.md b/docs/make_release.md index 1825b3d7a3..4e0499916c 100644 --- a/docs/make_release.md +++ b/docs/make_release.md @@ -96,7 +96,7 @@ As described above, making a release manually involves the following steps: - Run `python scripts/update_version.py -v ` to update the version number stored in `version.txt` and `flopy/version.py`. For an approved release use the `--approve` flag. -- Update MODFLOW 6 dfn files in the repository and MODFLOW 6 package classes by running `python -c 'import flopy; flopy.mf6.utils.generate_classes(branch="master", backup=False)'` +- Update MODFLOW 6 dfn files in the repository and MODFLOW 6 package classes by running `python -c 'import flopy; flopy.mf6.utils.generate_classes(ref="master", backup=False)'` - Run `isort` and `black` on the `flopy` module. This can be achieved by running `python scripts/pull_request_prepare.py` from the project root. The commands `isort .` and `black .` can also be run individually instead. diff --git a/flopy/mf6/utils/generate_classes.py b/flopy/mf6/utils/generate_classes.py index ab078e33bb..bae168affc 100644 --- a/flopy/mf6/utils/generate_classes.py +++ b/flopy/mf6/utils/generate_classes.py @@ -2,6 +2,7 @@ import shutil import tempfile import time +from warnings import warn from .createpackages import create_packages @@ -62,10 +63,15 @@ def download_dfn(owner, branch, new_dfn_pth): mf6url = mf6url.format(owner, branch) print(f" Downloading MODFLOW 6 repository from {mf6url}") with tempfile.TemporaryDirectory() as tmpdirname: - download_and_unzip(mf6url, tmpdirname, verbose=True) - downloaded_dfn_pth = os.path.join(tmpdirname, f"modflow6-{branch}") + dl_path = download_and_unzip(mf6url, tmpdirname, verbose=True) + dl_contents = list(dl_path.glob("modflow6-*")) + proj_path = next(iter(dl_contents), None) + if not proj_path: + raise ValueError( + f"Could not find modflow6 project dir in {dl_path}: found {dl_contents}" + ) downloaded_dfn_pth = os.path.join( - downloaded_dfn_pth, "doc", "mf6io", "mf6ivar", "dfn" + proj_path, "doc", "mf6io", "mf6ivar", "dfn" ) shutil.copytree(downloaded_dfn_pth, new_dfn_pth) @@ -104,7 +110,12 @@ def delete_mf6_classes(): def generate_classes( - owner="MODFLOW-USGS", branch="master", dfnpath=None, backup=True + owner="MODFLOW-USGS", + repo="modflow6", + branch="master", + ref=None, + dfnpath=None, + backup=True, ): """ Generate the MODFLOW 6 flopy classes using definition files from the @@ -116,9 +127,16 @@ def generate_classes( owner : str Owner of the MODFLOW 6 repository to use to update the definition files and generate the MODFLOW 6 classes. Default is MODFLOW-USGS. + repo : str + Name of the MODFLOW 6 repository to use to update the definition. branch : str Branch name of the MODFLOW 6 repository to use to update the definition files and generate the MODFLOW 6 classes. Default is master. + + .. deprecated:: 3.5.0 + Use ref instead. + ref : str + Branch name, tag, or commit hash to use to update the definition. dfnpath : str Path to a definition file folder that will be used to generate the MODFLOW 6 classes. Default is none, which means that the branch @@ -140,11 +158,19 @@ def generate_classes( # user provided dfnpath if dfnpath is None: print( - f" Updating the MODFLOW 6 classes using {owner}/modflow6/{branch}" + f" Updating the MODFLOW 6 classes using {owner}/{repo}/{branch}" ) timestr = time.strftime("%Y%m%d-%H%M%S") new_dfn_pth = os.path.join(flopypth, "mf6", "data", f"dfn_{timestr}") - download_dfn(owner, branch, new_dfn_pth) + + # branch deprecated 3.5.0 + if not ref: + if not branch: + raise ValueError("branch or ref must be provided") + warn("branch is deprecated, use ref instead") + ref = branch + + download_dfn(owner, ref, new_dfn_pth) else: print(f" Updating the MODFLOW 6 classes using {dfnpath}") assert os.path.isdir(dfnpath) From ce681d6bad0b3bd62b6fe4e2bd6785d0fded261a Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Wed, 9 Aug 2023 18:15:10 -0400 Subject: [PATCH 2/2] specify DeprecationWarning Co-authored-by: Mike Taves --- flopy/mf6/utils/generate_classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flopy/mf6/utils/generate_classes.py b/flopy/mf6/utils/generate_classes.py index bae168affc..dd783a0909 100644 --- a/flopy/mf6/utils/generate_classes.py +++ b/flopy/mf6/utils/generate_classes.py @@ -167,7 +167,7 @@ def generate_classes( if not ref: if not branch: raise ValueError("branch or ref must be provided") - warn("branch is deprecated, use ref instead") + warn("branch is deprecated, use ref instead", DeprecationWarning) ref = branch download_dfn(owner, ref, new_dfn_pth)