From 17a55492bb39f9611291a674eb7ac69c67b82bed Mon Sep 17 00:00:00 2001
From: miampf <miampf@proton.me>
Date: Tue, 29 Oct 2024 15:34:25 +0100
Subject: [PATCH] rework e2e test

---
 .../workflows/e2e_runtime-reproducibility.yml | 66 +++++++++----------
 1 file changed, 32 insertions(+), 34 deletions(-)

diff --git a/.github/workflows/e2e_runtime-reproducibility.yml b/.github/workflows/e2e_runtime-reproducibility.yml
index 65bc622fb9..a8323a3e76 100644
--- a/.github/workflows/e2e_runtime-reproducibility.yml
+++ b/.github/workflows/e2e_runtime-reproducibility.yml
@@ -8,7 +8,6 @@ on:
 jobs:
   os-matrix:
     strategy:
-      # If adjusting the matrix, remember to also adjust the targets/os_list found in the "Collect checksums" step
       matrix:
         os: [ubuntu-22.04, ubuntu-20.04]
         # As we do not use the Cachix cache containing the artifacts built by developers in this workflow,
@@ -17,7 +16,13 @@ jobs:
         # is reproducible across individual builds (as the --rebuild flag is used, causing Nix to rebuild the node-installer-image derivation)
         # and across independent builds on Ubuntu 20.04 and 22.04 (which also test the reproducibility of the transitive closure of our packages, as no shared
         # cache is present between the two machines)
-        build-target: ["microsoft.contrast-node-installer-image", "kata.contrast-node-installer-image"]
+        #
+        # If adjusting the build-target, remember to also adjust the matrix for the collect-checksums job
+        build-target:
+          [
+            "microsoft.contrast-node-installer-image",
+            "kata.contrast-node-installer-image",
+          ]
       fail-fast: false
     # Usually we would define the matrix outputs here, but as GitHub Actions don't seem to allow per-combination outputs,
     # we'll write the outputs without defining them here. See https://github.com/orgs/community/discussions/17245#discussioncomment-3814009.
@@ -51,16 +56,11 @@ jobs:
         with:
           name: ${{ matrix.build-target }}-${{ matrix.os }}-rebuild
           path: rebuild
-      - name: Upload reference checksum
+      - name: Upload checksums
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
         with:
-          name: ${{ matrix.build-target }}-${{ matrix.os }}-checksum-reference
-          path: ${{ matrix.build-target }}-${{ matrix.os }}-reference_checksum.txt
-      - name: Upload rebuild checksum
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
-        with:
-          name: ${{ matrix.build-target }}-${{ matrix.os }}-checksum-rebuild
-          path: ${{ matrix.build-target }}-${{ matrix.os }}-rebuild_checksum.txt
+          name: ${{ matrix.build-target }}-${{ matrix.os }}-checksums
+          path: ${{ matrix.build-target }}-${{ matrix.os }}-*_checksum.txt
       - name: Notify teams channel of failure
         if: ${{ failure() && github.ref == 'main' && github.run_attempt == 1 }}
         uses: ./.github/actions/post_to_teams
@@ -75,39 +75,36 @@ jobs:
     permissions:
       contents: read
     needs: os-matrix
+    strategy:
+      matrix:
+        build-target:
+          [
+            "microsoft.contrast-node-installer-image",
+            "kata.contrast-node-installer-image",
+          ]
     steps:
       - name: Download all checksum artifacts
         uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
         with:
-          pattern: "*-*-checksum-*"
+          pattern: "${{matrix.build-target}}-*-checksums"
+          path: "./checksums"
+          merge-multiple: true
       - name: Collect checksums
         id: collect
         shell: python
         run: |
-          import json, os
-          download_dir = os.getenv("GITHUB_WORKSPACE")
-          targets = ["microsoft.contrast-node-installer-image", "kata.contrast-node-installer-image"]
-          os_list = ["ubuntu-22.04", "ubuntu-20.04"]
+          import json, os, pathlib
           checksum_mismatch = False
-          for target in targets:
-              seen = {}
-              for system in os_list:
-                  reference_checksum_filename = f"{target}-{system}-reference_checksum.txt"
-                  rebuild_checksum_filename = f"{target}-{system}-rebuild_checksum.txt"
-                  with open(os.path.join(download_dir, f"{target}-{system}-checksum-reference", reference_checksum_filename)) as f_reference:
-                      with open(os.path.join(download_dir, f"{target}-{system}-checksum-rebuild", rebuild_checksum_filename)) as f_rebuild:
-                          reference_checksum = f_reference.readline()
-                          if not reference_checksum in seen:
-                              seen[reference_checksum] = []
-                          seen[reference_checksum].append(f"{target}-{system}-reference")
-                          rebuild_checksum = f_rebuild.readline()
-                          if not rebuild_checksum in seen:
-                              seen[rebuild_checksum] = []
-                          seen[rebuild_checksum].append(f"{target}-{system}-rebuild")
-              if len(seen) > 1:
-                  print(f"At least one checksum mismatched for {target}:")
-                  print(json.dumps(seen, indent=2))
-                  checksum_mismatch = True
+          seen = {}
+          for file in os.listdir("./checksums"):
+              checksum = pathlib.Path("./checksums", file).read_text()
+              if not checksum in seen:
+                  seen[checksum] = []
+          assert len(seen) > 0
+          if len(seen) > 1:
+              print(f"At least one checksum mismatched for {target}:")
+              print(json.dumps(seen, indent=2))
+              checksum_mismatch = True
           if checksum_mismatch:
               exit(1)
           print("All checksums were equal")
@@ -118,3 +115,4 @@ jobs:
           webhook: ${{ secrets.TEAMS_CI_WEBHOOK }}
           title: "Runtime reproducibility test failed"
           message: "failed to collect checksums"
+          additionalFields: '[{"title": "Build target", "value": "${{matrix.build-target}}"}]'