Move spack-config into config/settings.json (#128)

* validate-repo-version: update README instructions * Remove reading/modification of `spack-config` from `[un]deploy-2-*.yml` workflows * Rework settings.json to include `spack-config`, updated schema * validate-deployment-settings: Added spack-config check, added comments * validate-deployment-settings: Update checks to work with new settings.json schema * settings-*.yml: Updated to deploy updates to `spack-config` * ci.yml: Added newline to comment to aid code formatting
ACCESS-NRI · Sep 23, 2024 · 0c65447 · 0c65447
1 parent 9bc6d7c
commit 0c65447
Show file tree

Hide file tree

Showing 9 changed files with 139 additions and 44 deletions.
diff --git a/.github/actions/validate-deployment-settings/action.yml b/.github/actions/validate-deployment-settings/action.yml
@@ -21,12 +21,15 @@ outputs:
 runs:
   using: composite
   steps:
+    ################
+    # SPACK CHECKS #
+    ################
     - name: Check All Spack Major Versions in Prerelease are in Release
       id: spack-maj-all-pre-in-rel
       shell: bash
       run: |
         majors_only_in_prerelease=$(jq --compact-output --raw-output \
-          '.deployment.${{ inputs.target }} | (.Prerelease.spack | keys) - (.Release.spack | keys) | @sh' \
+          '.deployment.${{ inputs.target }} | (.Prerelease | keys) - (.Release | keys) | @sh' \
           ${{ inputs.settings-path }}
         )
 
@@ -43,8 +46,21 @@ runs:
       # It then merges them together based on major version (Ex. [[{"0.20": "u298rer3", "0.20": "r98u349u"}], ...]).
       # And then finally compares the hashes for each major version, and returns the major versions where they differ.
       run: |
-        release_version_hashes=$(jq '.deployment.${{ inputs.target }}.Release.spack | to_entries' ${{ inputs.settings-path }})
-        prerelease_version_hashes=$(jq '.deployment.${{ inputs.target }}.Prerelease.spack | to_entries' ${{ inputs.settings-path }})
+        release_version_hashes=$(jq \
+          '.deployment.${{ inputs.target }}.Release
+          | [
+              keys[] as $major | .[$major].spack as $hash
+              | {($major): $hash} | to_entries[]
+            ]' ${{ inputs.settings-path }}
+        )
+
+        prerelease_version_hashes=$(jq \
+          '.deployment.${{ inputs.target }}.Prerelease
+          | [
+              keys[] as $major | .[$major].spack as $hash
+              | {($major): $hash} | to_entries[]
+            ]' ${{ inputs.settings-path }}
+        )
 
         unequal_hashes_for_version=$(jq --compact-output --raw-output --null-input \
           --argjson r "$release_version_hashes" \
@@ -76,23 +92,56 @@ runs:
       run: |
         gh repo clone access-nri/spack ${{ env.CHECKOUT_DIR }} -- --no-checkout --bare --filter=blob:none
 
-        # Essentially, pull out all the spack 'version: hash' sections and iterate
-        jq -cr '.deployment.${{ inputs.target }}[].spack | to_entries[] | "\(.key) \(.value)"' ${{ inputs.settings-path }} | while read -ra line; do
+        # Essentially, pull out all the spack 'major: hash' sections and iterate
+        version_hashes=$(jq --compact-output --raw-output \
+          '.deployment.${{ inputs.target }}[]
+            | keys[] as $major | .[$major].spack as $hash
+            | "\($major) \($hash)"' \
+          ${{ inputs.settings-path }}
+        )
+
+        # For each of the version hashes, check if $hash is in releases/v$major
+        while read -ra line; do
           version=${line[0]}
           hash=${line[1]}
           echo "Checking if $hash is in $version"
           if ! git -C ${{ env.CHECKOUT_DIR }} merge-base --is-ancestor $hash releases/v$version; then
             echo "::${{ inputs.error-level }}::Commit $hash does not exist on branch releases/v$version"
             failed=true
           fi
-        done
+        done <<< "$version_hashes"
 
         if [ -n "$failed" ]; then
           msg="Some commits referenced do not exist in access-nri/spack. Check the workflow logs."
           echo "::${{ inputs.error-level }}::$msg"
           echo "msg=$msg" >> $GITHUB_OUTPUT
         fi
 
+        rm -rf ${{ env.CHECKOUT_DIR }}
+
+    #######################
+    # SPACK-CONFIG CHECKS #
+    #######################
+    - name: Verify spack-config Tag Exists
+      id: spack-config-tag-exists
+      shell: bash
+      env:
+        GH_TOKEN: ${{ github.token }}
+        CHECKOUT_DIR: ./spack-config-metadata
+      run: |
+        gh repo clone access-nri/spack-config ${{ env.CHECKOUT_DIR }} -- --no-checkout --bare --filter=blob:none
+
+        if ! git -C ${{ env.CHECKOUT_DIR }} tag -l $spack_config_tag; then
+          msg="Tag $spack_config_tag does not exist in access-nri/spack-config"
+          echo "::${{ inputs.error-level }}::$msg"
+          echo "msg=$msg" >> $GITHUB_OUTPUT
+        fi
+
+        rm -rf ${{ env.CHECKOUT_DIR }}
+
+    #######################
+    # END OF ACTION STEPS #
+    #######################
     - name: Warning/Error Collector
       id: collector
       shell: bash

diff --git a/.github/actions/validate-repo-version/README.md b/.github/actions/validate-repo-version/README.md
@@ -6,7 +6,7 @@ This action checks that the tags specified in a models `config/versions.json` is
 
 | Name | Type | Description | Required | Default | Example |
 | ---- | ---- | ----------- | -------- | ------- | ------- |
-| `repo-to-check` | `string` | ACCESS-NRI repositories  to validate associated version in `config/versions.json` | `true` | N/A | `spack-packages spack-config` |
+| `repo-to-check` | `string` | ACCESS-NRI repository to validate associated version in `config/versions.json` | `true` | N/A | `spack-packages` |
 
 ## Outputs
 

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -241,6 +241,7 @@ jobs:
             where the binaries shall be on your `$PATH`.
             This Prerelease is also accessible on Gadi via `/g/data/vk83/prerelease/apps/spack/0.22/spack` in the `${{ needs.defaults.outputs.root-sbd }}-${{ needs.check-spack-yaml.outputs.prerelease }}` environment.
             </details>
+
             :hammer_and_wrench: Using: spack-packages `${{ needs.check-config.outputs.spack-packages-version}}`, spack-config `${{ needs.check-config.outputs.spack-config-version }}`
             <details>
             <summary>Details</summary>

diff --git a/.github/workflows/deploy-2-start.yml b/.github/workflows/deploy-2-start.yml
@@ -54,7 +54,6 @@ jobs:
         run: |
           echo "spack=$(jq --compact-output --raw-output '.spack' ./config/versions.json)" >> $GITHUB_OUTPUT
           echo "packages=$(jq --compact-output --raw-output '."spack-packages"' ./config/versions.json)" >> $GITHUB_OUTPUT
-          echo "config=$(jq --compact-output --raw-output '."spack-config"' ./config/versions.json)" >> $GITHUB_OUTPUT
 
       - name: Get ${{ inputs.deployment-environment }} Remote Paths
         id: path
@@ -100,11 +99,9 @@ jobs:
             exit 1
           fi
 
-          # Update spack-packages/config
+          # Update spack-packages
           git -C ${{ steps.path.outputs.spack-packages }} fetch
           git -C ${{ steps.path.outputs.spack-packages }} checkout --force ${{ steps.versions.outputs.packages }}
-          git -C ${{ steps.path.outputs.spack-config }} fetch
-          git -C ${{ steps.path.outputs.spack-config }} checkout --force ${{ steps.versions.outputs.config }}
 
           # Enable spack
           . ${{ steps.path.outputs.spack-config }}/spack-enable.bash

diff --git a/.github/workflows/settings-1-update.yml b/.github/workflows/settings-1-update.yml
@@ -69,7 +69,7 @@ jobs:
           update=$(jq --raw-output --compact-output \
             '[.deployment | to_entries[] | .key as $d | .value | keys[] | {
                 "deployment-environment": $d,
-                "type": .key
+                "type": .
               }]' ${{ env.CONFIG_SETTINGS_PATH }}
           )
           echo "updates=$update" >> $GITHUB_OUTPUT

diff --git a/.github/workflows/settings-2-deploy.yml b/.github/workflows/settings-2-deploy.yml
@@ -22,14 +22,27 @@ jobs:
 
       - name: Setup spack updates
         id: spack
-        # TODO: Since we can't format this json as an input for this job (see the earlier workflow) we need to do it here
+        # TODO: Since we can't format any of this later json as an input for this job (see the earlier workflow) we need to do it here
         # Create a newline-separated list of strings of the form "MAJOR_VERSION COMMIT_HASH" so we
         # update the remotes MAJOR_VERSION/spack to COMMIT_HASH. Ex: "0.20 y7834gtbf3jf3434rr34r34ru"
         run: |
           updates=$(jq --compact-output --raw-output \
             --arg env "${{ inputs.deployment-environment }}" \
             --arg type "${{ inputs.spack-type }}" \
-            '.deployment[$env][$type].spack | to_entries[] | "\(.key) \(.value)"' \
+            '.deployment[$env][$type] | to_entries[] | "\(.key) \(.value.spack)"' \
+            ${{ env.CONFIG_SETTINGS_PATH }}
+          )
+
+          echo "$updates"
+          echo "updates=$updates" >> $GITHUB_OUTPUT
+
+      - name: Setup spack-config updates
+        id: spack-config
+        run: |
+          updates=$(jq --compact-output --raw-output \
+            --arg env "${{ inputs.deployment-environment }}" \
+            --arg type "${{ inputs.spack-type }}" \
+            '.deployment[$env][$type] | to_entries[] | "\(.key) \(.value.spack-config"' \
             ${{ env.CONFIG_SETTINGS_PATH }}
           )
 
@@ -43,16 +56,16 @@ jobs:
           hosts: ${{ secrets.HOST }}
           private-key: ${{ secrets.SSH_KEY }}
 
-      - name: Update
+      - name: Update Spack
         continue-on-error: true
         run: |
           ssh ${{ secrets.USER}}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT'
           set +e
           while read -ra update; do
             version=${update[0]}
             new_commit=${update[1]}
-
             current_head_commit=$(git -C ${{ secrets.SPACK_INSTALLS_ROOT_LOCATION }}/$version/spack rev-parse HEAD)
+
             if [ $? -eq 128 ]; then
               # FIXME: Deploy spack instances in this job too.
               echo "::error::Error: ${{ inputs.deployment-environment }} ${{ inputs.spack-type }} $version spack does not exist. Deploy it via build-cds Create Deployment Spack workflow first."
@@ -73,3 +86,33 @@ jobs:
             fi
           done <<< "${{ steps.spack.outputs.updates }}"
           EOT
+
+      - name: Update spack-config
+        continue-on-error: true
+        run: |
+          ssh ${{ secrets.USER}}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT'
+          set +e
+          while read -ra update; do
+            version=${update[0]}
+            new_commit=${update[1]}
+            current_head_commit=$(git -C ${{ secrets.SPACK_INSTALLS_ROOT_LOCATION }}/$version/spack-config rev-parse HEAD)
+
+            if [ $? -eq 128 ]; then
+              echo "::error::Error: ${{ inputs.deployment-environment }} ${{ inputs.spack-type }} $version spack-config does not exist."
+              continue
+            fi
+
+            git -C ${{ secrets.SPACK_INSTALLS_ROOT_LOCATION }}/$version/spack-config fetch
+
+            if [[ "$current_head_commit" != "$new_commit" ]]; then
+              git -C ${{ secrets.SPACK_INSTALLS_ROOT_LOCATION }}/$version/spack-config checkout $new_commit
+              if [ $? -ne 0 ]; then
+                echo "::error::Error: ${{ inputs.deployment-environment }} ${{ inputs.spack-type }} $version spack-config failed checkout from $current_head_commit to $new_commit"
+              else
+                echo "::notice::Changed: ${{ inputs.deployment-environment }} ${{ inputs.spack-type }} $version spack-config changed from $current_head_commit to $new_commit"
+              fi
+            else
+              echo "::notice::Unchanged: ${{ inputs.deployment-environment }} ${{ inputs.spack-type }} $version spack-config left at $current_head_commit"
+            fi
+          done <<< "${{ steps.spack-config.outputs.updates }}"
+          EOT
diff --git a/.github/workflows/undeploy-2-start.yml b/.github/workflows/undeploy-2-start.yml
@@ -18,11 +18,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Get Versions From config/versions.json
+      - name: Get Spack Version From config/versions.json
         id: versions
-        run: |
-          echo "spack=$(jq --compact-output --raw-output '.spack' ./config/versions.json)" >> $GITHUB_OUTPUT
-          echo "config=$(jq --compact-output --raw-output '."spack-config"' ./config/versions.json)" >> $GITHUB_OUTPUT
+        run: echo "spack=$(jq --compact-output --raw-output '.spack' ./config/versions.json)" >> $GITHUB_OUTPUT
 
       - name: Get ${{ inputs.deployment-environment }} Remote Paths
         id: path

diff --git a/config/settings.json b/config/settings.json
@@ -3,16 +3,21 @@
   "deployment": {
     "Gadi": {
       "Release": {
-        "spack": {
-          "0.20": "6812713cf470b473a607f0de0e8e1cf53f804fb7",
-          "0.22": "21da7d7e2b5e2680cd9d2e0a2fb4a7d13d8baa9d"
+        "0.20": {
+          "spack": "6812713cf470b473a607f0de0e8e1cf53f804fb7",
+          "spack-config": "2024.03.22"
+        },
+        "0.22": {
+          "spack": "21da7d7e2b5e2680cd9d2e0a2fb4a7d13d8baa9d",
+          "spack-config": "2024.07.05"
         }
       },
       "Prerelease": {
-        "spack": {
-          "0.22": "21da7d7e2b5e2680cd9d2e0a2fb4a7d13d8baa9d"
+        "0.22": {
+          "spack": "21da7d7e2b5e2680cd9d2e0a2fb4a7d13d8baa9d",
+          "spack-config": "2024.07.05"
         }
       }
     }
   }
-}
+}
diff --git a/config/settings.schema.json b/config/settings.schema.json
@@ -15,39 +15,41 @@
           "properties": {
             "Release": {
               "type": "object",
-              "properties": {
-                "spack": {
+              "patternProperties": {
+                "^.+$": {
                   "type": "object",
-                  "patternProperties": {
-                    "^.+$": {
+                  "properties": {
+                    "spack": {
+                      "type": "string"
+                    },
+                    "spack-config": {
                       "type": "string"
                     }
                   },
-                  "additionalProperties": true
+                  "additionalProperties": true,
+                  "required": ["spack", "spack-config"]
                 }
               },
-              "additionalProperties": true,
-              "required": [
-                "spack"
-              ]
+              "additionalProperties": true
             },
             "Prerelease": {
               "type": "object",
-              "properties": {
-                "spack": {
+              "patternProperties": {
+                "^.+$": {
                   "type": "object",
-                  "patternProperties": {
-                    "": {
+                  "properties": {
+                    "spack": {
+                      "type": "string"
+                    },
+                    "spack-config": {
                       "type": "string"
                     }
                   },
-                  "additionalProperties": true
+                  "additionalProperties": true,
+                  "required": ["spack", "spack-config"]
                 }
               },
-              "additionalProperties": true,
-              "required": [
-                "spack"
-              ]
+              "additionalProperties": true
             }
           },
           "additionalProperties": true,