Merge pull request #755 from xylar/add-jigsaw-as-submodule

Add `jigsaw-python` as a submodule

.github/workflows/build_workflow.yml

@@ -94,6 +94,7 @@ jobs:
       - if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
         name: Install compass
         run: |
+          git config --global url."https://github.com/".insteadOf "git@github.com:"
           ./conda/configure_compass_env.py \
             --env_name compass_test \
             --python=${{ matrix.python-version }} \

.github/workflows/docs_workflow.yml

@@ -46,6 +46,7 @@ jobs:
       - if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
         name: Install compass
         run: |
+          git config --global url."https://github.com/".insteadOf "git@github.com:"
           ./conda/configure_compass_env.py \
             --env_only \
             --env_name compass_test \

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "MALI-Dev"]
 	path = MALI-Dev
 	url = git@github.com:MALI-Dev/E3SM.git
+[submodule "jigsaw-python"]
+	path = jigsaw-python
+	url = git@github.com:dengwirda/jigsaw-python.git

compass/version.py

@@ -1 +1 @@
-__version__ = '1.2.0-alpha.8'
+__version__ = '1.2.0-alpha.9'

conda/bootstrap.py

@@ -9,6 +9,7 @@
 import socket
 import stat
 import subprocess
+import time
 from configparser import ConfigParser
 
 import progressbar
@@ -213,13 +214,16 @@ def get_env_setup(args, config, machine, compiler, mpi, env_type, source_path,
         ver = version.parse(compass_version)
         release_version = '.'.join(str(vr) for vr in ver.release)
         spack_env = f'dev_compass_{release_version}{env_suffix}'
+        compass_env = f'dev_compass_{compass_version}{env_suffix}'
     elif env_type == 'test_release':
         spack_env = f'test_compass_{compass_version}{env_suffix}'
+        compass_env = spack_env
     else:
         spack_env = f'compass_{compass_version}{env_suffix}'
+        compass_env = spack_env
 
     if env_name is None or env_type != 'dev':
-        env_name = spack_env
+        env_name = compass_env
 
     # add the compiler and MPI library to the spack env name
     spack_env = f'{spack_env}_{compiler}_{mpi}{lib_suffix}'
@@ -240,7 +244,7 @@ def get_env_setup(args, config, machine, compiler, mpi, env_type, source_path,
 def build_conda_env(env_type, recreate, mpi, conda_mpi, version,
                     python, source_path, conda_template_path, conda_base,
                     env_name, env_path, activate_base, use_local,
-                    local_conda_build, logger, local_mache):
+                    local_conda_build, logger, local_mache, update_jigsaw):
 
     if env_type != 'dev':
         install_miniforge(conda_base, activate_base, logger)
@@ -291,7 +295,10 @@ def build_conda_env(env_type, recreate, mpi, conda_mpi, version,
     else:
         spec_filename = None
 
-    if not os.path.exists(env_path) or recreate:
+    if not os.path.exists(env_path):
+        recreate = True
+
+    if recreate:
         print(f'creating {env_name}')
         if env_type == 'dev':
             # install dev dependencies and compass itself
@@ -300,13 +307,6 @@ def build_conda_env(env_type, recreate, mpi, conda_mpi, version,
                 f'conda create -y -n {env_name} {channels} ' \
                 f'--file {spec_filename} {packages}'
             check_call(commands, logger=logger)
-
-            commands = \
-                f'{activate_env} && ' \
-                f'cd {source_path} && ' \
-                f'python -m pip install -e .'
-            check_call(commands, logger=logger)
-
         else:
             # conda packages don't like dashes
             version_conda = version.replace('-', '')
@@ -323,16 +323,56 @@ def build_conda_env(env_type, recreate, mpi, conda_mpi, version,
                 f'conda install -y -n {env_name} {channels} ' \
                 f'--file {spec_filename} {packages}'
             check_call(commands, logger=logger)
+        else:
+            print(f'{env_name} already exists')
+
+    if env_type == 'dev':
+        # remove conda jigsaw and jigsaw-python
+        t0 = time.time()
+        commands = \
+            f'{activate_env} && ' \
+            f'conda remove -y --force-remove jigsaw jigsawpy'
+        check_call(commands, logger=logger)
+
+        commands = \
+            f'{activate_env} && ' \
+            f'cd {source_path} && ' \
+            f'git submodule update --init jigsaw-python'
+        check_call(commands, logger=logger)
+
+        if recreate or update_jigsaw:
+            print('Building JIGSAW\n')
+            commands = \
+                f'{activate_env} && ' \
+                f'conda install -y cxx-compiler && ' \
+                f'cd {source_path}/jigsaw-python && ' \
+                f'python setup.py build_external'
+            check_call(commands, logger=logger)
 
+            print('Installing JIGSAW and JIGSAW-Python\n')
             commands = \
                 f'{activate_env} && ' \
-                f'cd {source_path} && ' \
-                f'python -m pip install -e .'
+                f'cd {source_path}/jigsaw-python && ' \
+                f'python -m pip install --no-deps -e . && ' \
+                f'cp jigsawpy/_bin/* ${{CONDA_PREFIX}}/bin'
             check_call(commands, logger=logger)
-        else:
-            print(f'{env_name} already exists')
 
-    if env_type == 'dev':
+            t1 = time.time()
+            total = t1 - t0
+            message = f'JIGSAW install took {total} s.'
+            if logger is None:
+                print(message)
+            else:
+                logger.info(message)
+
+        # install (or reinstall) compass in edit mode
+        print('Installing compass\n')
+        commands = \
+            f'{activate_env} && ' \
+            f'cd {source_path} && ' \
+            f'python -m pip install -e .'
+        check_call(commands, logger=logger)
+
         print('Installing pre-commit\n')
         commands = \
             f'{activate_env} && ' \
@@ -987,7 +1027,8 @@ def main():  # noqa: C901
                 env_type, recreate, mpi, conda_mpi, compass_version,
                 python, source_path, conda_template_path, conda_base,
                 conda_env_name, conda_env_path, activate_base, args.use_local,
-                args.local_conda_build, logger, local_mache)
+                args.local_conda_build, logger, local_mache,
+                args.update_jigsaw)
 
             if local_mache:
                 print('Install local mache\n')

conda/compass_env/spec-file.template

@@ -13,8 +13,6 @@ git
 gsw
 h5py
 ipython
-jigsaw=0.9.14
-jigsawpy=0.3.3
 jupyter
 lxml
 {% if include_mache %}

conda/shared.py

@@ -33,6 +33,10 @@ def parse_args(bootstrap):
                              "install compilers or build SCORPIO")
     parser.add_argument("--recreate", dest="recreate", action='store_true',
                         help="Recreate the environment if it exists")
+    parser.add_argument("--update_jigsaw", dest="update_jigsaw",
+                        action='store_true',
+                        help="Reinstall JIGSAW even if not recreating conda "
+                             "environment.")
     parser.add_argument("-f", "--config_file", dest="config_file",
                         help="Config file to override deployment config "
                              "options")

docs/developers_guide/quick_start.rst

@@ -172,6 +172,11 @@ What the script does
 In addition to installing Miniforge3 and creating the conda environment for
 you, this script will also:
 
+* install `Jigsaw <https://github.com/dengwirda/jigsaw>`_ and
+  `Jigsaw-Python <https://github.com/dengwirda/jigsaw-python>`_ from source
+  from the `jigsaw-python` submodule. These tools are used to create many of
+  the meshes used in Compass.
+
 * install the ``compass`` package from the local branch in "development" mode
   so changes you make to the repo are immediately reflected in the conda
   environment.
@@ -214,14 +219,18 @@ Optional flags
 ``--python``
     Select a particular python version (the default is currently 3.8)
 
-``--env-name``
+``--env_name``
     Set the name of the environment (and the prefix for the activation script)
     to something other than the default (``dev_compass_<version>`` or
     ``dev_compass_<version>_<mpi>``).
 
-``--with-albany``
-    Install Albany for full MALI support (currently only with ``gnu``
-    compilers)
+``--update_jigsaw``
+    Used to reinstall Jigsaw and Jigsaw-Python into the conda environment if
+    you have made changes to the Jigsaw (c++) code in the ``jigsaw-python``
+    submodule. You should not need to reinstall Jigsaw-Python if you have made
+    changes only to the python code in ``jigsaw-python``, as the python package
+    is installed in
+    `edit mode <https://setuptools.pypa.io/en/latest/userguide/development_mode.html>`_.
 
 Activating the environment
 ~~~~~~~~~~~~~~~~~~~~~~~~~~

docs/index.rst

@@ -1,10 +1,10 @@
-compass
+Compass
 =======
 
-Configuration Of Model for Prediction Across Scales Setups (``compass``) is
+Configuration Of Model for Prediction Across Scales Setups (Compass) is
 a python package that provides an automated system to set up test cases for
 Model for Prediction Across Scales (MPAS) components.  The development version
-of ``compass`` will be kept closely synchronized with the
+of Compass will be kept closely synchronized with the
 `E3SM repo <https://github.com/E3SM-Project/E3SM>`_ and
 `MALI-Dev repo <https://github.com/MALI-Dev/E3SM>`_. Release
 versions will be compatible with specific tags of the MPAS components.
@@ -16,26 +16,34 @@ MPAS code.  Many compass test cases, such as those under the
 use data sets from observations to create create global and regional meshes,
 initial conditions, and boundary conditions.
 
-``compass`` will be the tool used to create new land-ice and ocean meshes and
+Compass will be the tool used to create new land-ice and ocean meshes and
 initial conditions for future versions of `E3SM <https://e3sm.org/>`_, just as
 :ref:`legacy_compass` has been used to create meshes and initial conditions for
 `E3SM v1 <https://e3sm.org/model/e3sm-model-description/v1-description/>`_
 and `v2 <https://e3sm.org/research/science-campaigns/v2-planned-campaign/>`_.
-We note that ``compass`` does *not* provide the tools for creating many of the
+We note that Compass does *not* provide the tools for creating many of the
 files needed for full E3SM coupling, a process that requires expert help from
 the E3SM development team.
 
 The ``compass`` python package defines the test cases along with the commands
 to list and set up both test cases and test suites (groups of test cases).
-``compass`` currently supports the ``landice`` and ``ocean`` dynamical cores
+Compass currently supports the ``landice`` and ``ocean`` dynamical cores
 for MPAS.  Nearly all test cases include calls that launch one of these
 dynamical cores.  These runs are configured with namelists and streams files,
-and one of the benefits of using ``compass`` over attempting to run one of the
-MPAS components directly is that ``compass`` begins with default values for all
+and one of the benefits of using Compass over attempting to run one of the
+MPAS components directly is that Compass begins with default values for all
 namelists and streams, modifying only those options where the default is not
 appropriate. In this way, compass requires little alteration as the MPAS
 components themselves evolves and new functionality is added.
 
+Compass makes extensive use of the
+`Jigsaw <https://github.com/dengwirda/jigsaw>`_ and
+`Jigsaw-Python <https://github.com/dengwirda/jigsaw-python>`_ tools to make all
+but the simplest meshes for our test cases and E3SM initial conditions.  These
+tools, without which Compass' mesh generation capabilities
+would not be possible, are developed primarily by
+`Darren Engwirda <https://dengwirda.github.io/>`_.
+
 .. toctree::
    :caption: User's guide
    :maxdepth: 2

setup.py

@@ -24,7 +24,7 @@ def package_files(directory, prefixes, extensions):
      'gsw',
      'h5py',
      'ipython',
-     'jigsawpy==0.3.3',
+     'jigsawpy',
      'jupyter',
      'lxml',
      'matplotlib',