Add cylc-flow, cylc-rose, cylc-uiserver, bump Python to 3.10.13 (#894)

Necessary config and spack submodule pointer updates for adding cylc-flow, cylc-rose, cylc-uiserver.

Along with this, bump Python to version 3.10.13 (see JCSDA/spack#377 for some background information on why we can use newer Python versions than 3.9 with cylc in spack-stack. This Python version is also compatible with what is requested in #887.

I removed several legacy versions for Python packages as part of this process, since I ran into problems with duplicate packages.


Co-authored-by: Alex Richert <alexander.richert@noaa.gov>
Co-authored-by: Alex Richert <82525672+AlexanderRichert-NOAA@users.noreply.github.com>

.github/workflows/macos-ci-aarch64.yaml

@@ -137,7 +137,7 @@ jobs:
           module use ${ENVDIR}/install/modulefiles/Core
           module load stack-apple-clang/14.0.3
           module load stack-openmpi/4.1.6
-          module load stack-python/3.10.8
+          module load stack-python/3.10.13
           module available
 
           module load jedi-ufs-env/1.0.0

.github/workflows/macos-ci-x86_64.yaml

@@ -128,7 +128,7 @@ jobs:
           module use ${ENVDIR}/install/modulefiles/Core
           module load stack-apple-clang/14.0.0
           module load stack-openmpi/4.1.6
-          module load stack-python/3.10.8
+          module load stack-python/3.10.13
           module available
 
           module load jedi-ufs-env/1.0.0

.github/workflows/ubuntu-ci-x86_64.yaml

@@ -50,6 +50,7 @@ jobs:
 
           # Find external packages
           spack external find --scope system --exclude bison --exclude openssl
+          spack external find --scope system sed
           spack external find --scope system perl
           spack external find --scope system wget
           PATH="/usr/local/opt/curl/bin:$PATH" \
@@ -182,7 +183,7 @@ jobs:
           module use ${ENVDIR}/install/modulefiles/Core
           module load stack-intel/2022.1.0
           module load stack-intel-oneapi-mpi/2021.6.0
-          module load stack-python/3.10.8
+          module load stack-python/3.10.13
           module available
 
           module load jedi-ufs-env/1.0.0

.github/workflows/ubuntu-rnd-x86_64.yaml

@@ -162,7 +162,7 @@ jobs:
           module use /home/ubuntu/spack-stack/CI/unified-env/${TODAY}/modulefiles/Core
           module load stack-intel/2022.1.0
           module load stack-intel-oneapi-mpi/2021.6.0
-          module load stack-python/3.10.8
+          module load stack-python/3.10.13
           module available
 
           module load jedi-ufs-env/1.0.0

configs/common/packages.yaml

@@ -207,21 +207,20 @@
     proj:
       version: ['8.1.0']
       variants: ~tiff
-    # Check site configs for the following systems when making changes here:
-    # Derecho, Narwhal, Gaea-C5
     python:
-      version: ['3.10.8']
+      require: "@3.10.13"
     py-attrs:
       # https://github.com/JCSDA/spack-stack/issues/740
       version: ['21.4.0']
     py-cartopy:
       variants: +plotting
-    py-click:
-      version: ['8.0.3']
+      require: "@0.21.1"
     py-cryptography:
       variants: +rust_bootstrap
-    py-eccodes:
-      version: ['1.4.2']
+    # Introduced in https://github.com/JCSDA/spack-stack/pull/894, pin py-cython
+    # to avoid duplicate packages being built (cylc dependencies soft-want @3:)
+    py-cython:
+      require: "@0.29.36"
     py-h5py:
       version: ['3.7.0']
       variants: ~mpi
@@ -241,33 +240,13 @@
     # To avoid pip._vendor.pep517.wrappers.BackendInvalid errors with newer
     # versions of py-poetry-core when using external/homebrew Python as
     # we do at the moment in spack-stack.
-    py-poetry-core:
-      version: ['1.0.8']
-    py-pybind11:
-      version: ['2.8.1']
-    py-pygithub:
-      version: ['1.55']
-    py-pygrib:
-      version: ['2.1.4']
-    py-pyhdf:
-      version: ['0.10.4']
-    py-pyproj:
-      version: ['3.1.0']
-    py-python-dateutil:
-      version: ['2.8.2']
-    py-pythran:
-      # Versions earlier than 0.11.0 don't compile on macOS with llvm-clang/13.0.0 and Python/3.9,
-      # and 0.11.0 leads to downstream errors in py-scipy with the Intel compilers
-      version: ['0.12.2']
-    py-scipy:
-      version: ['1.9.3']
     # Pin the py-setuptools version to avoid duplicate Python packages
     py-setuptools:
       require: ['@63.4.3']
     py-setuptools-rust:
       variants: +rust_bootstrap
     py-shapely:
-      version: ['1.8.0']
+      require: ['@1.8.0']
     qt:
       version: ['5.15.3']
     scotch:

configs/sites/aws-pcluster/packages.yaml

@@ -115,6 +115,10 @@ packages:
     externals:
     - spec: qt@5.12.8
       prefix: /usr
+  sed:
+    externals:
+    - spec: sed@4.7
+      prefix: /usr
   tar:
     externals:
     - spec: tar@1.30

configs/templates/skylab-dev/spack.yaml

@@ -8,7 +8,7 @@ spack:
   definitions:
   - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel']
   - packages:
-      - ewok-env
+      - ewok-env +ecflow +cylc
       - jedi-fv3-env
       - jedi-mpas-env
       - jedi-neptune-env

configs/templates/unified-dev/spack.yaml

@@ -11,7 +11,7 @@ spack:
       - global-workflow-env
       - gmao-swell-env
       - gsi-env
-      - ewok-env
+      - ewok-env +ecflow +cylc
       - jedi-fv3-env
       - jedi-mpas-env
       - jedi-neptune-env