Skip to content

Commit

Permalink
Filter the JVM matrix and include modules into it
Browse files Browse the repository at this point in the history
  • Loading branch information
gsmet committed Feb 21, 2025
1 parent 0d47320 commit b4a715c
Show file tree
Hide file tree
Showing 3 changed files with 76 additions and 33 deletions.
64 changes: 54 additions & 10 deletions .github/filter-jvm-tests-json.sh
Original file line number Diff line number Diff line change
@@ -1,20 +1,64 @@
#!/bin/bash

BASEDIR=$(dirname "$0")
# Purpose: Prints a filtered version of matrix-jvm-tests.json
# Note: This script is only for CI and does therefore not aim to be compatible with BSD/macOS.

# Filter out mac os from a matrix of build configurations.
# The reason we do this is that running mac on a fork won't work; the fork won't have a self-hosted runner
set -e -u -o pipefail
shopt -s failglob

# See https://stackoverflow.com/questions/65384420/how-to-make-a-github-action-matrix-element-conditional
export IFS=$'\n'

repoName=${GITHUB_REPOSITORY}
# path of this shell script
PRG_PATH=$( cd "$(dirname "$0")" ; pwd -P )

if [[ $repoName == "quarkusio/quarkus" ]]
JSON=$(cat ${PRG_PATH}/matrix-jvm-tests.json)
if [[ "${GITHUB_REPOSITORY:-DEFINED_ON_CI}" != "quarkusio/quarkus" ]]; then
# Filter out mac os from a matrix of build configurations.
# The reason we do this is that running mac on a fork won't work; the fork won't have a self-hosted runner
# See https://stackoverflow.com/questions/65384420/how-to-make-a-github-action-matrix-element-conditional
JSON=$(echo -n "$JSON" | jq 'map(. | select(.["os-name"]!="macos-arm64-latest"))')
fi

# Step 0: print unfiltered json and exit in case the parameter is '_all_' (full build) or print nothing if empty (no changes)
if [ "$1" == '_all_' ]
then
echo \{java: ${JSON}\}
exit 0
elif [ -z "$1" ]
then
matrix=$(cat $BASEDIR/matrix-jvm-tests.json)
echo -n ''
exit 0
fi

RUNTIME_MODULES=$(echo -n "$1" | grep -Pv '^(integration-tests|tcks)/.*' || echo '')
INTEGRATION_TESTS=$(echo -n "$1" | grep -Po '^integration-tests/.+' | grep -Pv '^integration-tests/(devtools|gradle|maven|devmode|kubernetes)/.*' || echo '')

if [ -z "$RUNTIME_MODULES" ] && [ -z "$INTEGRATION_TESTS" ]; then
echo -n ''
exit 0
fi

if [ -z "$RUNTIME_MODULES" ]; then
JSON=$(echo -n $JSON | jq --arg category Runtime 'del( .[] | select(.category == $category) )')
else
RUNTIME_MODULES_COMMAND="-pl '"
for RUNTIME_MODULE in $RUNTIME_MODULES; do
RUNTIME_MODULES_COMMAND+="${RUNTIME_MODULE},"
done
RUNTIME_MODULES_COMMAND+="'"
JSON=$(echo -n $JSON | jq --arg category Runtime --arg modules "${RUNTIME_MODULES_COMMAND}" '( .[] | select(.category == $category) ).modules = $modules')
fi

if [ -z "$INTEGRATION_TESTS" ]; then
JSON=$(echo -n $JSON | jq --arg category Integration 'del( .[] | select(.category == $category) )')
else
# Use jq to read in a json file that represents the matrix configuration.
matrix=$(jq 'map(. | select(.["os-name"]!="macos-arm64-latest"))' $BASEDIR/matrix-jvm-tests.json)
INTEGRATION_TESTS_COMMAND="-pl '"
for INTEGRATION_TEST in $INTEGRATION_TESTS; do
INTEGRATION_TESTS_COMMAND+="${INTEGRATION_TEST},"
done
INTEGRATION_TESTS_COMMAND+="'"
JSON=$(echo -n $JSON | jq --arg category Integration --arg modules "${INTEGRATION_TESTS_COMMAND}" '( .[] | select(.category == $category) ).modules = $modules')
fi

echo \{java: ${matrix}\}

echo \{java: ${JSON}\}
21 changes: 15 additions & 6 deletions .github/matrix-jvm-tests.json
Original file line number Diff line number Diff line change
@@ -1,33 +1,40 @@
[
{
"name": "JVM Tests - JDK 17",
"category": "Runtime",
"java-version": 17,
"maven_args": "$JVM_TEST_MAVEN_ARGS",
"maven_opts": "-Xmx2g -XX:MaxMetaspaceSize=1g",
"os-name": "ubuntu-latest"
"os-name": "ubuntu-latest",
"modules": "-pl !docs -Dno-test-modules"
},
{
"name": "JVM Tests - JDK 21",
"category": "Runtime",
"java-version": 21,
"java-version-gradle": 20,
"maven_args": "$JVM_TEST_MAVEN_ARGS",
"maven_opts": "-Xmx3g -XX:MaxMetaspaceSize=1g",
"os-name": "ubuntu-latest"
"os-name": "ubuntu-latest",
"modules": "-pl !docs -Dno-test-modules"
},
{
"name": "JVM Tests - JDK 17 Windows",
"category": "Runtime",
"java-version": 17,
"maven_args": "-DskipDocs -Dformat.skip",
"maven_opts": "-Xmx2g -XX:MaxMetaspaceSize=1g",
"os-name": "windows-latest"
"os-name": "windows-latest",
"modules": "-pl !docs -Dno-test-modules"
},
{
"name": "JVM Integration Tests - JDK 17",
"category": "Integration",
"java-version": 17,
"maven_args": "$JVM_TEST_MAVEN_ARGS",
"maven_opts": "-Xmx2g -XX:MaxMetaspaceSize=1g",
"os-name": "ubuntu-latest"
"os-name": "ubuntu-latest",
"modules": "-f integration-tests -pl !gradle -pl !maven -pl !devmode -pl !devtools"
},
{
"name": "JVM Integration Tests - JDK 21",
Expand All @@ -36,14 +43,16 @@
"java-version-gradle": 20,
"maven_args": "$JVM_TEST_MAVEN_ARGS",
"maven_opts": "-Xmx3g -XX:MaxMetaspaceSize=1g",
"os-name": "ubuntu-latest"
"os-name": "ubuntu-latest",
"modules": "-f integration-tests -pl !gradle -pl !maven -pl !devmode -pl !devtools"
},
{
"name": "JVM Integration Tests - JDK 17 Windows",
"category": "Integration",
"java-version": 17,
"maven_args": "-DskipDocs -Dformat.skip",
"maven_opts": "-Xmx2g -XX:MaxMetaspaceSize=1g",
"os-name": "windows-latest"
"os-name": "windows-latest",
"modules": "-f integration-tests -pl !gradle -pl !maven -pl !devmode -pl !devtools"
}
]
24 changes: 7 additions & 17 deletions .github/workflows/ci-actions-incremental.yml
Original file line number Diff line number Diff line change
Expand Up @@ -347,7 +347,6 @@ jobs:
native_matrix: ${{ steps.calc-native-matrix.outputs.matrix }}
jvm_matrix: ${{ steps.calc-jvm-matrix.outputs.matrix }}
virtual_threads_matrix: ${{ steps.calc-virtual-threads-matrix.outputs.matrix }}
run_jvm: ${{ steps.calc-run-flags.outputs.run_jvm }}
run_devtools: ${{ steps.calc-run-flags.outputs.run_devtools }}
run_gradle: ${{ steps.calc-run-flags.outputs.run_gradle }}
run_maven: ${{ steps.calc-run-flags.outputs.run_maven }}
Expand All @@ -366,7 +365,7 @@ jobs:
- name: Calculate matrix from matrix-jvm-tests.json
id: calc-jvm-matrix
run: |
json=$(.github/filter-jvm-tests-json.sh)
json=$(.github/filter-jvm-tests-json.sh "${GIB_IMPACTED_MODULES}")
echo "${json}"
echo "matrix=${json}" >> $GITHUB_OUTPUT
- name: Calculate matrix from virtual-threads-tests.json
Expand All @@ -379,24 +378,22 @@ jobs:
- name: Calculate run flags
id: calc-run-flags
run: |
run_jvm=true; run_devtools=true; run_gradle=true; run_maven=true; run_kubernetes=true; run_quickstarts=true; run_tcks=true
run_devtools=true; run_gradle=true; run_maven=true; run_kubernetes=true; run_quickstarts=true; run_tcks=true
if [ -z "${GIB_IMPACTED_MODULES}" ]
then
run_jvm=false; run_devtools=false; run_gradle=false; run_maven=false; run_kubernetes=false; run_quickstarts=false; run_tcks=false
run_devtools=false; run_gradle=false; run_maven=false; run_kubernetes=false; run_quickstarts=false; run_tcks=false
elif [ "${GIB_IMPACTED_MODULES}" != '_all_' ]
then
# Important: keep -pl ... in actual jobs in sync with the following grep commands!
# do not use grep -q as it can exit before echo has finished outputting and cause broken pipes
if ! (echo -n "${GIB_IMPACTED_MODULES}" | grep -Pv 'integration-tests/(devtools|gradle|maven|devmode|kubernetes/.*)|tcks/.*' > /dev/null); then run_jvm=false; fi
if ! (echo -n "${GIB_IMPACTED_MODULES}" | grep 'integration-tests/devtools' > /dev/null); then run_devtools=false; fi
if ! (echo -n "${GIB_IMPACTED_MODULES}" | grep 'integration-tests/gradle' > /dev/null); then run_gradle=false; fi
if ! (echo -n "${GIB_IMPACTED_MODULES}" | grep -P 'integration-tests/(maven|devmode)' > /dev/null); then run_maven=false; fi
if ! (echo -n "${GIB_IMPACTED_MODULES}" | grep -P 'integration-tests/kubernetes/.*' > /dev/null); then run_kubernetes=false; fi
if ! (echo -n "${GIB_IMPACTED_MODULES}" | grep -Pv '(docs|integration-tests|tcks)/.*' > /dev/null); then run_quickstarts=false; fi
if ! (echo -n "${GIB_IMPACTED_MODULES}" | grep 'tcks/.*' > /dev/null); then run_tcks=false; fi
fi
echo "run_jvm=${run_jvm}, run_devtools=${run_devtools}, run_gradle=${run_gradle}, run_maven=${run_maven}, run_kubernetes=${run_kubernetes}, run_quickstarts=${run_quickstarts}, run_tcks=${run_tcks}"
echo "run_jvm=${run_jvm}" >> $GITHUB_OUTPUT
echo "run_devtools=${run_devtools}, run_gradle=${run_gradle}, run_maven=${run_maven}, run_kubernetes=${run_kubernetes}, run_quickstarts=${run_quickstarts}, run_tcks=${run_tcks}"
echo "run_devtools=${run_devtools}" >> $GITHUB_OUTPUT
echo "run_gradle=${run_gradle}" >> $GITHUB_OUTPUT
echo "run_maven=${run_maven}" >> $GITHUB_OUTPUT
Expand All @@ -409,7 +406,7 @@ jobs:
runs-on: ${{ fromJson(needs.configure.outputs.config).runners[matrix.java.os-name].runsOn || matrix.java.os-name }}
needs: [configure, build-jdk17, calculate-test-jobs]
# Skip main in forks
if: ${{ !cancelled() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') && (needs.calculate-test-jobs.outputs.run_jvm == 'true' && (github.repository == 'quarkusio/quarkus' || !endsWith(github.ref, '/main'))) }}
if: ${{ !cancelled() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') && needs.calculate-test-jobs.outputs.jvm_matrix && (github.repository == 'quarkusio/quarkus' || !endsWith(github.ref, '/main')) }}
timeout-minutes: 400
env:
MAVEN_OPTS: ${{ matrix.java.maven_opts }}
Expand Down Expand Up @@ -507,14 +504,7 @@ jobs:
env:
CAPTURE_BUILD_SCAN: true
run: |
if [[ "${{ matrix.java.category }}" == *"Integration"* ]]; then
PL=$JVM_TEST_INTEGRATION_TESTS_SELECTOR
WORKAROUND=-Dgib.buildAll=true
else
PL=$JVM_TEST_NORMAL_TESTS_SELECTOR
fi
# Despite the pre-calculated run_jvm flag, GIB has to be re-run here to figure out the exact submodules to build.
./mvnw $COMMON_MAVEN_ARGS $COMMON_TEST_MAVEN_ARGS $PTS_MAVEN_ARGS $PL clean install -Dsurefire.timeout=1200 -Dno-test-kubernetes ${{ matrix.java.maven_args }} ${{ needs.build-jdk17.outputs.gib_args }} $WORKAROUND
./mvnw $COMMON_MAVEN_ARGS $COMMON_TEST_MAVEN_ARGS $PTS_MAVEN_ARGS $PL clean install -Dsurefire.timeout=1200 -Dno-test-kubernetes ${{ matrix.java.maven_args }}
- name: Clean Gradle temp directory
if: always()
run: devtools/gradle/gradlew --stop && rm -rf devtools/gradle/gradle-extension-plugin/build/tmp
Expand Down Expand Up @@ -638,7 +628,7 @@ jobs:
env:
CAPTURE_BUILD_SCAN: true
# Important: keep -pl ... in sync with "Calculate run flags"!
# Despite the pre-calculated run_jvm flag, GIB has to be re-run here to figure out the exact submodules to build.
# Despite the pre-calculated run_maven flag, GIB has to be re-run here to figure out the exact submodules to build.
run: ./mvnw $COMMON_MAVEN_ARGS $COMMON_TEST_MAVEN_ARGS $PTS_MAVEN_ARGS $JVM_TEST_MAVEN_ARGS clean install -pl 'integration-tests/maven' -pl 'integration-tests/devmode' ${{ needs.build-jdk17.outputs.gib_args }}
- name: Prepare failure archive (if maven failed)
if: failure()
Expand Down

0 comments on commit b4a715c

Please sign in to comment.