Github Actions CI Changes

.github/actions/setupenv/action.yml

@@ -0,0 +1,18 @@
+name: "Perform Repetitive Tasks"
+description: "Composite action that checks out repos, and adjusts directory permissions for runner"
+
+runs:
+  using: "composite"
+  steps:
+      - name: Checkout PandABlocks-rootfs
+        uses: actions/checkout@v4
+        with:
+          repository: PandABlocks/PandABlocks-rootfs
+          path: repos/PandABlocks-rootfs
+          fetch-depth: 0
+
+      - name: Give runner build perms, and adjust repos config path
+        shell: bash
+        run: |
+          sudo mkdir /build
+          sudo chmod -R 777 /build

.github/scripts/Find_Tests.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Modules path
+search_dir=$1
+work_dir=$2
+
+# Find files matching timing name pattern
+found_files=$(find "$search_dir" -type f -name "*.timing.ini")
+echo $found_files
+# Array of modules and their test count
+module_grps=()
+
+# Check if any files were found
+if [ -z "$found_files" ]; then
+    echo "No timing tests found"
+else
+    # Loop through each found file
+    for file in $found_files; do
+        # Count occurrences of [*] in the file
+        # Min 5 char to filter descriptions and index
+        count=$(grep -o '\[[^][]\{5,\}\]' "$file" | wc -l)
+        # Duplicate names allowed for multiple test files in same module subdirectory
+        module_name=$(basename "$(dirname "$file")")
+        module_grps+=("$module_name" "$count")
+    done
+    # Run python script to define job matrix based on found tests
+    python3 $work_dir/Group_Tests.py "${module_grps[@]}"
+fi

.github/scripts/Group_Tests.py

@@ -0,0 +1,43 @@
+import sys
+import json
+
+modules_grps = (sys.argv)[1:]
+print(modules_grps)
+modules = []
+num_jobs = 5
+
+
+# Allocate modules to jobs with lowest load
+def split_modules(modules, num_jobs):
+    modules.sort(key=lambda x: x[1], reverse=True)
+    jobs = [[] for _ in range(num_jobs)]
+    for module, count in modules:
+        min_sum_job_idx = min(
+            range(num_jobs), key=lambda i: sum(subset[1] for subset in jobs[i])
+        )
+        jobs[min_sum_job_idx].append([module, count])
+    return jobs
+
+
+# Produce matrix in format expected by 'make hdl_test' job step
+def generate_matrix(jobs):
+    matrix = {"modules": []}
+    for job in jobs:
+        job_include = " ".join(module[0] for module in job)
+        matrix["modules"].append(job_include)
+    return matrix
+
+
+# Convert bash array to sensible py array
+for i in range(0, len(modules_grps) - 1, 2):
+    # If duplicate module names are found, add their counts
+    if modules and modules_grps[i] == modules_grps[i - 2]:
+        modules[-1][1] += int(modules_grps[i + 1])
+    # If no duplicate present, append new element
+    else:
+        modules.append([modules_grps[i], int(modules_grps[i + 1])])
+print(modules)
+
+# Produce JSON file to pass to GH job
+with open("github_tests.json", "w") as matrix_file:
+    json.dump(generate_matrix(split_modules(modules, num_jobs)), matrix_file)

.github/workflows/_make_boot.yml

@@ -0,0 +1,49 @@
+on:
+  workflow_call:
+
+jobs:
+  make_boot:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: zynq
+            app: PandABox-no-fmc
+          - platform: zynqmp
+            app: xu5_st1-no-fmc
+          - platform: zynqmp
+            app: PandABrick
+    runs-on:
+      group: iris_runners
+    container:
+      image: ghcr.io/pandablocks/pandablocks-dev-container:latest
+      options: --privileged
+
+    steps:
+      # Necessary to find action.yml
+      - name: Checkout Source
+        uses: actions/checkout@v4
+        with:
+          path: repos/PandABlocks-fpga
+          fetch-depth: 0
+
+      - name: Checkout rootfs and Give Directory Perms
+        uses: ./repos/PandABlocks-fpga/.github/actions/setupenv
+
+      # Generate bootable image
+      - name: Make boot
+        id: make_boot
+        run: |
+          echo "boot_files=false" >> $GITHUB_OUTPUT
+          cd repos/PandABlocks-fpga
+          ln -s CONFIG.example CONFIG
+          make boot APP_NAME=${{ matrix.app }}
+          ps -Ao pid= -o comm= | awk '$2 ~ /Xvfb/ { print $1}' | xargs kill
+
+      # Upload artifacts if boot files present
+      - name: Upload boot
+        if: ${{ steps.make_boot.outputs.boot_files == 'true'}}
+        uses: actions/upload-artifact@v4
+        with:
+          name: boot-${{ matrix.app }}
+          path: /build/boot*.zip

.github/workflows/_make_zpkg.yml

@@ -0,0 +1,53 @@
+on:
+  workflow_call:
+
+env:
+  WORK_DIR: /__w/PandABlocks-FPGA/PandABlocks-FPGA
+
+jobs:
+  make_zpkg:
+    strategy:
+      fail-fast: false
+      matrix:
+        app: [
+          "PandABox-no-fmc",
+          "PandABox-fmc_24vio",
+          "PandABox-fmc_acq427",
+          "PandABox-fmc_acq430",
+          "PandABox-fmc_lback-sfp_lback",
+          "PandABrick",
+          "xu5_st1-no-fmc",
+          "xu5_st1-fmc_acq430"
+          ]
+    runs-on:
+      group: iris_runners
+    container:
+      image: ghcr.io/pandablocks/pandablocks-dev-container:latest
+      options: --privileged
+
+    steps:
+      # Necessary to find action.yml
+      - name: Checkout Source
+        uses: actions/checkout@v4
+        with:
+          path: repos/PandABlocks-fpga
+          fetch-depth: 0
+
+      - name: Checkout rootfs and Give Directory Perms
+        uses: ./repos/PandABlocks-fpga/.github/actions/setupenv
+
+      # Make zpkgs
+      - name: build carrier_ip and zpkg
+        id: make_zpkg
+        run: |
+          cd repos/PandABlocks-fpga
+          ln -s CONFIG.example CONFIG
+          make WORK_DIR=$WORK_DIR carrier_ip APP_NAME=${{ matrix.app }}
+          make WORK_DIR=$WORK_DIR zpkg APP_NAME=${{ matrix.app }}
+
+      # Artifacts
+      - name: Upload zpkg
+        uses: actions/upload-artifact@v4
+        with:
+          name: zpkgs-${{ matrix.app }}
+          path: /build/panda-fpga@*.zpg

.github/workflows/_release.yml

@@ -0,0 +1,25 @@
+on:
+  workflow_call:
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    # make a release on every tag
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          path: zpkgs
+          merge-multiple: true
+
+
+      - name: Github Release
+        # We pin to the SHA, not the tag, for security reasons.
+        # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions
+        uses: softprops/action-gh-release@2d72d869af3bf23602f9593a1e3fd739b80ac1eb  # v0.1.12
+        with:
+          prerelease: ${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc') }}
+          files: zpkgs/*
+          body: See [Changelog](CHANGELOG.rst) for more details
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/_test_hdl.yml

@@ -0,0 +1,36 @@
+on:
+  workflow_call:
+    inputs:
+      matrix:
+        required: true
+        type: string
+
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      # Using generated matrix from previous job
+      matrix: ${{fromJSON(inputs.matrix)}}
+    runs-on:
+      group: iris_runners
+    container:
+        image: ghcr.io/pandablocks/pandablocks-dev-container:latest
+        options: --privileged
+
+    steps:
+      # Necessary to find action.yml
+      - name: Checkout Source
+        uses: actions/checkout@v4
+        with:
+          path: repos/PandABlocks-fpga
+          fetch-depth: 0
+
+      - name: Checkout rootfs and Give Directory Perms
+        uses: ./repos/PandABlocks-fpga/.github/actions/setupenv
+
+      # Run tests
+      - name: Make hdl Tests
+        run: |
+          cd repos/PandABlocks-fpga
+          ln -s CONFIG.example CONFIG
+          make hdl_test MODULES="${{matrix.modules}}" 

.github/workflows/_test_matrix.yml

@@ -0,0 +1,32 @@
+on:
+  workflow_call:
+    outputs:
+      matrix: 
+          value: ${{ jobs.test_matrix.outputs.matrix }}
+
+env:
+  WORK_DIR: /__w/PandABlocks-FPGA/PandABlocks-FPGA
+
+jobs:
+    test_matrix:
+      runs-on:
+        group: iris_runners
+      container:
+          image: docker.io/shihabdls/pandablocks-container-extension:v2.5
+          options: --privileged
+      outputs:
+          matrix: ${{steps.make_matrix.outputs.matrix}} 
+      steps:
+        - name: Checkout Source
+          uses: actions/checkout@v4
+          with:
+            path: repos/PandABlocks-fpga
+            fetch-depth: 0
+
+        - name: Evaluate number of modules/tests & generate job matrix
+          id: make_matrix
+          run: |
+               bash $WORK_DIR/repos/PandABlocks-fpga/.github/scripts/Find_Tests.sh "$WORK_DIR/repos/PandABlocks-fpga/modules" "$WORK_DIR/repos/PandABlocks-fpga/.github/scripts"
+               json_content=$(cat github_tests.json)
+               echo "::set-output name=matrix::$json_content"
+               echo "$json_content"

.github/workflows/_test_python_autogen.yml

@@ -0,0 +1,30 @@
+on:
+    workflow_call:
+
+jobs:
+    test_python_autogen:
+      runs-on:
+        group: iris_runners
+      container:
+          image: ghcr.io/pandablocks/pandablocks-dev-container:latest
+          options: --privileged
+
+      steps:
+        # Necessary to find action.yml
+        - name: Checkout Source
+          uses: actions/checkout@v4
+          with:
+            path: repos/PandABlocks-fpga
+            fetch-depth: 0
+
+        - name: Checkout rootfs and Give Directory Perms
+          uses: ./repos/PandABlocks-fpga/.github/actions/setupenv
+
+        # Run tests
+        - name: Make Python Tests
+          run: |
+            cd repos/PandABlocks-fpga
+            ln -s CONFIG.example CONFIG
+            make python_tests
+            make python_timing
+            make autogen