Merge develop into main for 2.22.0 release

.circleci/config.yml

@@ -1,24 +1,52 @@
 version: 2.1
 
+# Anchor to prevent forgetting to update a version
+baselibs_version: &baselibs_version v6.3.1
+
 orbs:
   ci: geos-esm/circleci-tools@1
 
 workflows:
   build-and-test:
     jobs:
 
-      # Builds MAPL in a "default" way
+      # Builds MAPL in a "default" way - Intel
       - ci/build:
           name: build-and-test-MAPL-on-<< matrix.compiler >>
           context:
             - docker-hub-creds
           matrix:
             parameters:
-              compiler: [gfortran, ifort]
+              compiler: [ifort]
+          baselibs_version: *baselibs_version
+          repo: MAPL
+          mepodevelop: false
+          run_unit_tests: true
+          ctest_options: "-LE 'PERFORMANCE|EXTDATA1G_BIG_TESTS|EXTDATA2G_BIG_TESTS' --output-on-failure"
+
+      # Builds MAPL in a "default" way - GNU
+      #
+      # NOTE: Currently Open MPI fails on the bundleio with:
+      #
+      #   The OSC pt2pt component does not support MPI_THREAD_MULTIPLE in this release.
+      #   Workarounds are to run on a single node, or to use a system with an RDMA
+      #   capable network such as Infiniband.
+      #
+      # For now, we run GNU/Open MPI without the bundleio tests. Some indications that
+      # Open MPI 5 will not have this limitation
+
+      - ci/build:
+          name: build-and-test-MAPL-on-<< matrix.compiler >>
+          context:
+            - docker-hub-creds
+          matrix:
+            parameters:
+              compiler: [gfortran]
+          baselibs_version: *baselibs_version
           repo: MAPL
           mepodevelop: false
           run_unit_tests: true
-          ctest_options: "-R MAPL -LE PERFORMANCE --output-on-failure"
+          ctest_options: "-E bundleio -LE 'PERFORMANCE|EXTDATA1G_BIG_TESTS|EXTDATA2G_BIG_TESTS' --output-on-failure"
 
       # Builds MAPL like UFS does (no FLAP and pFlogger, static)
       - ci/build:
@@ -28,11 +56,12 @@ workflows:
           matrix:
             parameters:
               compiler: [ifort]
+          baselibs_version: *baselibs_version
           repo: MAPL
           mepodevelop: false
           extra_cmake_options: "-DBUILD_WITH_FLAP=OFF -DBUILD_WITH_PFLOGGER=OFF -DBUILD_SHARED_MAPL=OFF"
           run_unit_tests: true
-          ctest_options: "-R MAPL -LE PERFORMANCE --output-on-failure"
+          ctest_options: "-LE 'PERFORMANCE|EXTDATA1G_BIG_TESTS|EXTDATA2G_BIG_TESTS' --output-on-failure"
 
       # Build GEOSgcm
       - ci/build:
@@ -42,11 +71,12 @@ workflows:
           matrix:
             parameters:
               compiler: [gfortran, ifort]
+          baselibs_version: *baselibs_version
           repo: GEOSgcm
           checkout_fixture: true
           mepodevelop: true
           checkout_mapl_branch: true
-          persist_workspace: false # Needs to be true to run fv3/gcm experiment, costs extra
+          persist_workspace: true # Needs to be true to run fv3/gcm experiment, costs extra, retained for one day
 
       # Build GEOSldas
       - ci/build:
@@ -56,6 +86,7 @@ workflows:
           matrix:
             parameters:
               compiler: [gfortran, ifort]
+          baselibs_version: *baselibs_version
           repo: GEOSldas
           mepodevelop: false
           checkout_fixture: true
@@ -71,6 +102,7 @@ workflows:
             parameters:
               compiler: [ifort]
           resource_class: xlarge
+          baselibs_version: *baselibs_version
           repo: GEOSadas
           checkout_fixture: true
           # This branch on GEOSadas will be used to track subrepos needed
@@ -80,15 +112,15 @@ workflows:
           mepodevelop: false
           rebuild_procs: 8
 
-      ##################################################
-      # - ci/run_fv3:                                  #
-      #     name: run-FV3-on-<< matrix.compiler >>     #
-      #     context:                                   #
-      #       - docker-hub-creds                       #
-      #     matrix:                                    #
-      #       parameters:                              #
-      #         compiler: [gfortran, ifort]            #
-      #     requires:                                  #
-      #       - build-GEOSgcm-on-<< matrix.compiler >> #
-      #     repo: GEOSgcm                              #
-      ##################################################
+      # Run GCM (1 hour, no ExtData)
+      - ci/run_gcm:
+          name: run-GCM-on-<< matrix.compiler >>
+          context:
+            - docker-hub-creds
+          matrix:
+            parameters:
+              compiler: [gfortran, ifort]
+          requires:
+            - build-GEOSgcm-on-<< matrix.compiler >>
+          repo: GEOSgcm
+          baselibs_version: *baselibs_version

.github/workflows/changelog-enforcer.yml

@@ -4,7 +4,7 @@ on:
       types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled]
 
 jobs:
-  # Enforces the update of a changelog file on every pull request 
+  # Enforces the update of a changelog file on every pull request
   changelog:
     runs-on: ubuntu-latest
     steps:

.github/workflows/validate_yaml_files.yml

@@ -0,0 +1,31 @@
+---
+
+# Based on code from https://github.com/marketplace/actions/yaml-lint
+
+name: Yaml Lint
+
+on:
+  pull_request:
+      types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled]
+
+# This validation is equivalent to running on the command line:
+#   yamllint -d relaxed --no-warnings
+# and is controlled by the .yamllint.yml file
+jobs:
+  validate-YAML:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - id: yaml-lint
+        name: yaml-lint
+        uses: ibiqlik/action-yamllint@v3
+        with:
+          no_warnings: true
+          format: colored
+          config_file: .yamllint.yml
+
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: yamllint-logfile
+          path: ${{ steps.yaml-lint.outputs.logfile }}

.github/workflows/workflow.yml

@@ -17,7 +17,7 @@ jobs:
     name: Build and Test MAPL
     runs-on: ubuntu-latest
     container:
-      image: gmao/ubuntu20-geos-env-mkl:v6.2.8-openmpi_4.0.6-gcc_11.2.0
+      image: gmao/ubuntu20-geos-env-mkl:v6.3.1-openmpi_4.1.2-gcc_11.2.0
       # Per https://github.com/actions/virtual-environments/issues/1445#issuecomment-713861495
       # It seems like we might not need secrets on GitHub Actions which is good for forked
       # pull requests
@@ -30,7 +30,7 @@ jobs:
       OMPI_MCA_btl_vader_single_copy_mechanism: none
     steps:
       - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.9.0
+        uses: styfle/cancel-workflow-action@0.9.1
         with:
           access_token: ${{ github.token }}
       - name: Checkout
@@ -61,85 +61,20 @@ jobs:
           cd build
           make -j4 build-tests
           # skip performance tests
-          ctest -R MAPL -LE PERFORMANCE --output-on-failure
-  ############################################################################################################################################################
-  # build_gcm:                                                                                                                                               #
-  #   name: Build GEOSgcm                                                                                                                                    #
-  #   if: "!contains(github.event.pull_request.labels.*.name, '0 diff trivial')"                                                                             #
-  #   runs-on: ubuntu-latest                                                                                                                                 #
-  #   container:                                                                                                                                             #
-  #     image: gmao/ubuntu20-geos-env-mkl:v6.2.8-openmpi_4.0.6-gcc_11.2.0                                                                                    #
-  #   env:                                                                                                                                                   #
-  #     OMPI_ALLOW_RUN_AS_ROOT: 1                                                                                                                            #
-  #     OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1                                                                                                                    #
-  #     OMPI_MCA_btl_vader_single_copy_mechanism: none                                                                                                       #
-  #   steps:                                                                                                                                                 #
-  #     - name: Cancel Previous Runs                                                                                                                         #
-  #       uses: styfle/cancel-workflow-action@0.9.0                                                                                                          #
-  #       with:                                                                                                                                              #
-  #         access_token: ${{ github.token }}                                                                                                                #
-  #     - name: Checkout GCM                                                                                                                                 #
-  #       uses: actions/checkout@v2                                                                                                                          #
-  #       with:                                                                                                                                              #
-  #         repository: GEOS-ESM/GEOSgcm                                                                                                                     #
-  #         fetch-depth: 1                                                                                                                                   #
-  #     - name: Versions etc.                                                                                                                                #
-  #       run: |                                                                                                                                             #
-  #         gfortran --version                                                                                                                               #
-  #         mpirun --version                                                                                                                                 #
-  #         echo $BASEDIR                                                                                                                                    #
-  #         echo ${GITHUB_HEAD_REF}                                                                                                                          #
-  #         echo ${{github.ref}}                                                                                                                             #
-  #         echo ${{github.head_ref}}                                                                                                                        #
-  #     - name: Mepo clone external repos                                                                                                                    #
-  #       run: |                                                                                                                                             #
-  #         mepo init                                                                                                                                        #
-  #         mepo clone                                                                                                                                       #
-  #         mepo status                                                                                                                                      #
-  #         mepo develop GEOSgcm_GridComp GEOSgcm_App                                                                                                        #
-  #         mepo status                                                                                                                                      #
-  #     # The next two actions use:                                                                                                                          #
-  #     #   https://github.uint.cloudmunity/t/is-there-a-way-to-tell-if-a-pr-is-from-a-forked-repository/134186/7?u=mathomp4                                        #
-  #     # to test if you are in a fork or not                                                                                                                #
-  #     - name: Git checkout MAPL branch (forks)                                                                                                             #
-  #       if: github.event.pull_request.head.repo.full_name != github.repository                                                                             #
-  #       working-directory: ./src/Shared/@MAPL                                                                                                              #
-  #       run: |                                                                                                                                             #
-  #         # GITHUB_REF will look like 'refs/pull/669/merge' and we want 'pull/669'                                                                         #
-  #         GITHUB_REF_STRIP_REFS=${GITHUB_REF##refs/}                                                                                                       #
-  #         GITHUB_REF_STRIP_MERGE=${GITHUB_REF_STRIP_REFS%%/merge}                                                                                          #
-  #         # This was figured out with looking at CircleCI output                                                                                           #
-  #         git fetch --force origin "${GITHUB_REF_STRIP_MERGE}/head:remotes/origin/${GITHUB_REF_STRIP_MERGE}"                                               #
-  #         mepo checkout ${GITHUB_REF_STRIP_MERGE} MAPL                                                                                                     #
-  #         mepo status                                                                                                                                      #
-  #     # If we are in the same org, this will work                                                                                                          #
-  #     - name: Mepo checkout MAPL branch (same org), MAPL on main/develop                                                                                   #
-  #       if: github.event.pull_request.head.repo.full_name == github.repository && (github.head_ref == 'main' || github.head_ref == 'develop')              #
-  #       run: |                                                                                                                                             #
-  #         mepo checkout ${GITHUB_HEAD_REF} MAPL                                                                                                            #
-  #         mepo status                                                                                                                                      #
-  #     # If we are in the same org, this will work                                                                                                          #
-  #     - name: Mepo checkout MAPL branch (same org), MAPL not on main/develop                                                                               #
-  #       if: github.event.pull_request.head.repo.full_name == github.repository && (github.head_ref != 'main' && github.head_ref != 'develop')              #
-  #       run: |                                                                                                                                             #
-  #         mepo checkout ${GITHUB_HEAD_REF} MAPL                                                                                                            #
-  #         mepo checkout-if-exists ${GITHUB_HEAD_REF}                                                                                                       #
-  #         mepo status                                                                                                                                      #
-  #     - name: CMake                                                                                                                                        #
-  #       run: |                                                                                                                                             #
-  #         mkdir build                                                                                                                                      #
-  #         cd build                                                                                                                                         #
-  #         cmake .. -DBASEDIR=$BASEDIR/Linux -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_BUILD_TYPE=Debug -DMPIEXEC_PREFLAGS='--oversubscribe' -DUSE_F2PY=OFF #
-  #     - name: Build                                                                                                                                        #
-  #       run: |                                                                                                                                             #
-  #         cd build                                                                                                                                         #
-  #         make -j4 install                                                                                                                                 #
-  ############################################################################################################################################################
+          # NOTE: Currently Open MPI fails on the bundleio with:
+          #
+          #   The OSC pt2pt component does not support MPI_THREAD_MULTIPLE in this release.
+          #   Workarounds are to run on a single node, or to use a system with an RDMA
+          #   capable network such as Infiniband.
+          #
+          # For now, we run GNU/Open MPI without the bundleio tests. Some indications that
+          # Open MPI 5 will not have this limitation
+          ctest -E bundleio -LE 'PERFORMANCE|EXTDATA1G_BIG_TESTS|EXTDATA2G_BIG_TESTS' --output-on-failure
   build_test_mapl_intel:
     name: Build and Test MAPL Intel
     runs-on: ubuntu-latest
     container:
-      image: gmao/ubuntu20-geos-env:v6.2.8-intelmpi_2021.3.0-intel_2021.3.0
+      image: gmao/ubuntu20-geos-env:v6.3.1-intelmpi_2021.3.0-intel_2021.3.0
       # Per https://github.com/actions/virtual-environments/issues/1445#issuecomment-713861495
       # It seems like we might not need secrets on GitHub Actions which is good for forked
       # pull requests
@@ -148,7 +83,7 @@ jobs:
         #password: ${{ secrets.DOCKERHUB_TOKEN }}
     steps:
       - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.9.0
+        uses: styfle/cancel-workflow-action@0.9.1
         with:
           access_token: ${{ github.token }}
       - name: Checkout
@@ -179,4 +114,4 @@ jobs:
           cd build
           make -j4 build-tests
           # skip performance tests
-          ctest -R MAPL -LE PERFORMANCE --output-on-failure
+          ctest -LE 'PERFORMANCE|EXTDATA1G_BIG_TESTS|EXTDATA2G_BIG_TESTS' --output-on-failure