From 24318d53f36edbde70e6b3c3bc911a2ac0cef47d Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 6 Mar 2023 11:27:53 +0100 Subject: [PATCH 1/9] Add jigsaw-python submodule --- .gitmodules | 3 +++ jigsaw-python | 1 + 2 files changed, 4 insertions(+) create mode 160000 jigsaw-python diff --git a/.gitmodules b/.gitmodules index f1a21370c0..e3f3b78a4e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "MALI-Dev"] path = MALI-Dev url = git@github.com:MALI-Dev/E3SM.git +[submodule "jigsaw-python"] + path = jigsaw-python + url = git@github.com:dengwirda/jigsaw-python.git diff --git a/jigsaw-python b/jigsaw-python new file mode 160000 index 0000000000..d9d70e60fa --- /dev/null +++ b/jigsaw-python @@ -0,0 +1 @@ +Subproject commit d9d70e60fae9b5686c85113d1d1d4b21ae341789 From c36ce9d2d4d2c503ff94e6fa0cd92f5367d6bf44 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 6 Mar 2023 14:08:24 +0100 Subject: [PATCH 2/9] Install jigsaw and jigsawpy from submodule --- conda/bootstrap.py | 59 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 13 deletions(-) diff --git a/conda/bootstrap.py b/conda/bootstrap.py index 41c0c205f3..d75cb3bb38 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -9,6 +9,7 @@ import socket import stat import subprocess +import time from configparser import ConfigParser import progressbar @@ -300,13 +301,6 @@ def build_conda_env(env_type, recreate, mpi, conda_mpi, version, f'conda create -y -n {env_name} {channels} ' \ f'--file {spec_filename} {packages}' check_call(commands, logger=logger) - - commands = \ - f'{activate_env} && ' \ - f'cd {source_path} && ' \ - f'python -m pip install -e .' - check_call(commands, logger=logger) - else: # conda packages don't like dashes version_conda = version.replace('-', '') @@ -323,16 +317,55 @@ def build_conda_env(env_type, recreate, mpi, conda_mpi, version, f'conda install -y -n {env_name} {channels} ' \ f'--file {spec_filename} {packages}' check_call(commands, logger=logger) - - commands = \ - f'{activate_env} && ' \ - f'cd {source_path} && ' \ - f'python -m pip install -e .' - check_call(commands, logger=logger) else: print(f'{env_name} already exists') if env_type == 'dev': + # remove conda jigsaw and jigsaw-python + t0 = time.time() + commands = \ + f'{activate_env} && ' \ + f'conda remove -y --force-remove jigsaw jigsawpy' + check_call(commands, logger=logger) + + commands = \ + f'{activate_env} && ' \ + f'cd {source_path} && ' \ + f'git submodule update --init jigsaw-python' + check_call(commands, logger=logger) + + print('Building JIGSAW\n') + commands = \ + f'{activate_env} && ' \ + f'conda install -y cxx-compiler && ' \ + f'cd {source_path}/jigsaw-python && ' \ + f'python setup.py build_external' + check_call(commands, logger=logger) + + print('Installing JIGSAW and JIGSAW-Python\n') + commands = \ + f'{activate_env} && ' \ + f'cd {source_path}/jigsaw-python && ' \ + f'python -m pip install --no-deps -e . && ' \ + f'cp jigsawpy/_bin/* ${{CONDA_PREFIX}}/bin' + check_call(commands, logger=logger) + + t1 = time.time() + total = t1 - t0 + message = f'JIGSAW install took {total} s.' + if logger is None: + print(message) + else: + logger.info(message) + + # install (or reinstall) compass in edit mode + print('Installing compass\n') + commands = \ + f'{activate_env} && ' \ + f'cd {source_path} && ' \ + f'python -m pip install -e .' + check_call(commands, logger=logger) + print('Installing pre-commit\n') commands = \ f'{activate_env} && ' \ From 9762b4aa55f426c62ebff43b9e27dc66a0d7a48c Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 6 Mar 2023 14:55:58 +0100 Subject: [PATCH 3/9] Use https instead of SSH to get git submodules ...in github actions --- .github/workflows/build_workflow.yml | 1 + .github/workflows/docs_workflow.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/build_workflow.yml b/.github/workflows/build_workflow.yml index 9de6cd42eb..28f1d470e8 100644 --- a/.github/workflows/build_workflow.yml +++ b/.github/workflows/build_workflow.yml @@ -94,6 +94,7 @@ jobs: - if: ${{ steps.skip_check.outputs.should_skip != 'true' }} name: Install compass run: | + git config --global url."https://github.com/".insteadOf "git@github.com:" ./conda/configure_compass_env.py \ --env_name compass_test \ --python=${{ matrix.python-version }} \ diff --git a/.github/workflows/docs_workflow.yml b/.github/workflows/docs_workflow.yml index fbe891766f..81913bdfa8 100644 --- a/.github/workflows/docs_workflow.yml +++ b/.github/workflows/docs_workflow.yml @@ -46,6 +46,7 @@ jobs: - if: ${{ steps.skip_check.outputs.should_skip != 'true' }} name: Install compass run: | + git config --global url."https://github.com/".insteadOf "git@github.com:" ./conda/configure_compass_env.py \ --env_only \ --env_name compass_test \ From dc143341ab05bd4523d24529996fe5142e53a13b Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 15 Mar 2023 13:23:25 +0100 Subject: [PATCH 4/9] Add --update_jigsaw flag Only install JIGSAW and JIGSAW-Python from the submodule if * creating a new environment * --recreate flag is used * --update_jigsaw flag is used --- conda/bootstrap.py | 53 +++++++++++++++++++++++++--------------------- conda/shared.py | 4 ++++ 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/conda/bootstrap.py b/conda/bootstrap.py index d75cb3bb38..d80751a574 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -241,7 +241,7 @@ def get_env_setup(args, config, machine, compiler, mpi, env_type, source_path, def build_conda_env(env_type, recreate, mpi, conda_mpi, version, python, source_path, conda_template_path, conda_base, env_name, env_path, activate_base, use_local, - local_conda_build, logger, local_mache): + local_conda_build, logger, local_mache, update_jigsaw): if env_type != 'dev': install_miniforge(conda_base, activate_base, logger) @@ -292,7 +292,10 @@ def build_conda_env(env_type, recreate, mpi, conda_mpi, version, else: spec_filename = None - if not os.path.exists(env_path) or recreate: + if not os.path.exists(env_path): + recreate = True + + if recreate: print(f'creating {env_name}') if env_type == 'dev': # install dev dependencies and compass itself @@ -334,29 +337,30 @@ def build_conda_env(env_type, recreate, mpi, conda_mpi, version, f'git submodule update --init jigsaw-python' check_call(commands, logger=logger) - print('Building JIGSAW\n') - commands = \ - f'{activate_env} && ' \ - f'conda install -y cxx-compiler && ' \ - f'cd {source_path}/jigsaw-python && ' \ - f'python setup.py build_external' - check_call(commands, logger=logger) + if recreate or update_jigsaw: + print('Building JIGSAW\n') + commands = \ + f'{activate_env} && ' \ + f'conda install -y cxx-compiler && ' \ + f'cd {source_path}/jigsaw-python && ' \ + f'python setup.py build_external' + check_call(commands, logger=logger) - print('Installing JIGSAW and JIGSAW-Python\n') - commands = \ - f'{activate_env} && ' \ - f'cd {source_path}/jigsaw-python && ' \ - f'python -m pip install --no-deps -e . && ' \ - f'cp jigsawpy/_bin/* ${{CONDA_PREFIX}}/bin' - check_call(commands, logger=logger) + print('Installing JIGSAW and JIGSAW-Python\n') + commands = \ + f'{activate_env} && ' \ + f'cd {source_path}/jigsaw-python && ' \ + f'python -m pip install --no-deps -e . && ' \ + f'cp jigsawpy/_bin/* ${{CONDA_PREFIX}}/bin' + check_call(commands, logger=logger) - t1 = time.time() - total = t1 - t0 - message = f'JIGSAW install took {total} s.' - if logger is None: - print(message) - else: - logger.info(message) + t1 = time.time() + total = t1 - t0 + message = f'JIGSAW install took {total} s.' + if logger is None: + print(message) + else: + logger.info(message) # install (or reinstall) compass in edit mode print('Installing compass\n') @@ -1020,7 +1024,8 @@ def main(): # noqa: C901 env_type, recreate, mpi, conda_mpi, compass_version, python, source_path, conda_template_path, conda_base, conda_env_name, conda_env_path, activate_base, args.use_local, - args.local_conda_build, logger, local_mache) + args.local_conda_build, logger, local_mache, + args.update_jigsaw) if local_mache: print('Install local mache\n') diff --git a/conda/shared.py b/conda/shared.py index 5dbe69fa39..b24248a56d 100644 --- a/conda/shared.py +++ b/conda/shared.py @@ -33,6 +33,10 @@ def parse_args(bootstrap): "install compilers or build SCORPIO") parser.add_argument("--recreate", dest="recreate", action='store_true', help="Recreate the environment if it exists") + parser.add_argument("--update_jigsaw", dest="update_jigsaw", + action='store_true', + help="Reinstall JIGSAW even if not recreating conda " + "environment.") parser.add_argument("-f", "--config_file", dest="config_file", help="Config file to override deployment config " "options") From 15b33c5d29691cb5197bdcea594a0389f90790da Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 10 Jan 2024 10:40:36 -0600 Subject: [PATCH 5/9] remove jigsaw and jigsawpy from spec-file Also remove specific version from setup.py --- conda/compass_env/spec-file.template | 2 -- setup.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/conda/compass_env/spec-file.template b/conda/compass_env/spec-file.template index 0156741d48..fdeddcf110 100644 --- a/conda/compass_env/spec-file.template +++ b/conda/compass_env/spec-file.template @@ -13,8 +13,6 @@ git gsw h5py ipython -jigsaw=0.9.14 -jigsawpy=0.3.3 jupyter lxml {% if include_mache %} diff --git a/setup.py b/setup.py index 5ac7e0a9cd..3d908a91cd 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ def package_files(directory, prefixes, extensions): 'gsw', 'h5py', 'ipython', - 'jigsawpy==0.3.3', + 'jigsawpy', 'jupyter', 'lxml', 'matplotlib', From 9703e6e70930eecca94540103e3c6695876fb869 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 10 Jan 2024 14:27:25 -0600 Subject: [PATCH 6/9] Update compass to 1.2.0-alpha.9 --- compass/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compass/version.py b/compass/version.py index 2dc4ea819f..7c2b0c7b44 100644 --- a/compass/version.py +++ b/compass/version.py @@ -1 +1 @@ -__version__ = '1.2.0-alpha.8' +__version__ = '1.2.0-alpha.9' From 2374cd73d7a3e76e890c32ba490dddd8a2623d4b Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 11 Jan 2024 04:37:35 -0600 Subject: [PATCH 7/9] Update docs with better Jigsaw acknowledgment Also include more about Jigsaw and Jigsaw-Python in the quick start. --- docs/developers_guide/quick_start.rst | 17 +++++++++++++---- docs/index.rst | 23 +++++++++++++++-------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/docs/developers_guide/quick_start.rst b/docs/developers_guide/quick_start.rst index ae78f72624..26d960143d 100644 --- a/docs/developers_guide/quick_start.rst +++ b/docs/developers_guide/quick_start.rst @@ -172,6 +172,11 @@ What the script does In addition to installing Miniforge3 and creating the conda environment for you, this script will also: +* install `Jigsaw `_ and + `Jigsaw-Python `_ from source + from the `jigsaw-python` submodule. These tools are used to create many of + the meshes used in Compass. + * install the ``compass`` package from the local branch in "development" mode so changes you make to the repo are immediately reflected in the conda environment. @@ -214,14 +219,18 @@ Optional flags ``--python`` Select a particular python version (the default is currently 3.8) -``--env-name`` +``--env_name`` Set the name of the environment (and the prefix for the activation script) to something other than the default (``dev_compass_`` or ``dev_compass__``). -``--with-albany`` - Install Albany for full MALI support (currently only with ``gnu`` - compilers) +``--update_jigsaw`` + Used to reinstall Jigsaw and Jigsaw-Python into the conda environment if + you have made changes to the Jigsaw (c++) code in the ``jigsaw-python`` + submodule. You should not need to reinstall Jigsaw-Python if you have made + changes only to the python code in ``jigsaw-python``, as the python package + is installed in + `edit mode `_. Activating the environment ~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/index.rst b/docs/index.rst index eeac3c13ad..93ff611954 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,10 +1,10 @@ -compass +Compass ======= -Configuration Of Model for Prediction Across Scales Setups (``compass``) is +Configuration Of Model for Prediction Across Scales Setups (Compass) is a python package that provides an automated system to set up test cases for Model for Prediction Across Scales (MPAS) components. The development version -of ``compass`` will be kept closely synchronized with the +of Compass will be kept closely synchronized with the `E3SM repo `_ and `MALI-Dev repo `_. Release versions will be compatible with specific tags of the MPAS components. @@ -16,26 +16,33 @@ MPAS code. Many compass test cases, such as those under the use data sets from observations to create create global and regional meshes, initial conditions, and boundary conditions. -``compass`` will be the tool used to create new land-ice and ocean meshes and +Compass will be the tool used to create new land-ice and ocean meshes and initial conditions for future versions of `E3SM `_, just as :ref:`legacy_compass` has been used to create meshes and initial conditions for `E3SM v1 `_ and `v2 `_. -We note that ``compass`` does *not* provide the tools for creating many of the +We note that Compass does *not* provide the tools for creating many of the files needed for full E3SM coupling, a process that requires expert help from the E3SM development team. The ``compass`` python package defines the test cases along with the commands to list and set up both test cases and test suites (groups of test cases). -``compass`` currently supports the ``landice`` and ``ocean`` dynamical cores +Compass currently supports the ``landice`` and ``ocean`` dynamical cores for MPAS. Nearly all test cases include calls that launch one of these dynamical cores. These runs are configured with namelists and streams files, -and one of the benefits of using ``compass`` over attempting to run one of the -MPAS components directly is that ``compass`` begins with default values for all +and one of the benefits of using Compass over attempting to run one of the +MPAS components directly is that Compass begins with default values for all namelists and streams, modifying only those options where the default is not appropriate. In this way, compass requires little alteration as the MPAS components themselves evolves and new functionality is added. +Compass makes extensive use of the +`Jigsaw `_ and +`Jigsaw-Python `_ tools to make all +but the simplest meshes for our test cases and E3SM initial conditions. These +tools, whithout which Compass would not be possible, are developed primarily by +`Darren Engwirda `_. + .. toctree:: :caption: User's guide :maxdepth: 2 From 96d4f7016d7fceb0624db40e697ae06834bbd104 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 18 Jan 2024 01:36:22 -0700 Subject: [PATCH 8/9] Update text about Jigsaw and fix typo Co-authored-by: Matt Hoffman --- docs/index.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 93ff611954..0bb5f9ea0f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -40,7 +40,8 @@ Compass makes extensive use of the `Jigsaw `_ and `Jigsaw-Python `_ tools to make all but the simplest meshes for our test cases and E3SM initial conditions. These -tools, whithout which Compass would not be possible, are developed primarily by +tools, without which Compass' mesh generation capabilities +would not be possible, are developed primarily by `Darren Engwirda `_. .. toctree:: From 345012d1e95c97f2ca2b5b0bd7e75a617a51beb5 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 19 Jan 2024 02:14:53 -0800 Subject: [PATCH 9/9] Leave alpha version in compass dev envs --- conda/bootstrap.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/conda/bootstrap.py b/conda/bootstrap.py index d80751a574..5ed9deecec 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -214,13 +214,16 @@ def get_env_setup(args, config, machine, compiler, mpi, env_type, source_path, ver = version.parse(compass_version) release_version = '.'.join(str(vr) for vr in ver.release) spack_env = f'dev_compass_{release_version}{env_suffix}' + compass_env = f'dev_compass_{compass_version}{env_suffix}' elif env_type == 'test_release': spack_env = f'test_compass_{compass_version}{env_suffix}' + compass_env = spack_env else: spack_env = f'compass_{compass_version}{env_suffix}' + compass_env = spack_env if env_name is None or env_type != 'dev': - env_name = spack_env + env_name = compass_env # add the compiler and MPI library to the spack env name spack_env = f'{spack_env}_{compiler}_{mpi}{lib_suffix}'