diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2f896849e2..28366fe239 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -31,9 +31,20 @@ defaults: shell: bash jobs: + platforms: + name: Run Platforms Workflow + uses: ./.github/workflows/platforms.yaml + permissions: + packages: write + contents: read + build-base: name: Run Build Workflow uses: ./.github/workflows/build_base.yaml + needs: [platforms] + with: + p: ${{needs.platforms.outputs.p}} + platform-matrix: ${{needs.platforms.outputs.platform-matrix}} permissions: packages: write contents: read @@ -68,14 +79,14 @@ jobs: dev-package: name: Build leap-dev package - needs: [build-base] - if: always() && needs.build-base.result == 'success' + needs: [platforms, build-base] + if: always() && needs.platforms.result == 'success' && needs.build-base.result == 'success' strategy: fail-fast: false matrix: platform: [ubuntu20, ubuntu22] runs-on: ubuntu-latest - container: ${{fromJSON(needs.build-base.outputs.p)[matrix.platform].image}} + container: ${{fromJSON(needs.platforms.outputs.p)[matrix.platform].image}} steps: - uses: actions/checkout@v3 with: @@ -104,15 +115,15 @@ jobs: tests: name: Tests - needs: [build-base] - if: always() && needs.build-base.result == 'success' + needs: [platforms, build-base] + if: always() && needs.platforms.result == 'success' && needs.build-base.result == 'success' strategy: fail-fast: false matrix: platform: [ubuntu20, ubuntu22] runs-on: ["self-hosted", "enf-x86-hightier"] container: - image: ${{fromJSON(needs.build-base.outputs.p)[matrix.platform].image}} + image: ${{fromJSON(needs.platforms.outputs.p)[matrix.platform].image}} options: --security-opt seccomp=unconfined steps: - uses: actions/checkout@v3 @@ -130,8 +141,8 @@ jobs: np-tests: name: NP Tests - needs: [build-base] - if: always() && needs.build-base.result == 'success' + needs: [platforms, build-base] + if: always() && needs.platforms.result == 'success' && needs.build-base.result == 'success' strategy: fail-fast: false matrix: @@ -146,7 +157,7 @@ jobs: - name: Run tests in parallel containers uses: ./.github/actions/parallel-ctest-containers with: - container: ${{fromJSON(needs.build-base.outputs.p)[matrix.platform].image}} + container: ${{fromJSON(needs.platforms.outputs.p)[matrix.platform].image}} error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd", "build/TestLogs"]' log-tarball-prefix: ${{matrix.platform}} tests-label: nonparallelizable_tests @@ -160,8 +171,8 @@ jobs: lr-tests: name: LR Tests - needs: [build-base] - if: always() && needs.build-base.result == 'success' + needs: [platforms, build-base] + if: always() && needs.platforms.result == 'success' && needs.build-base.result == 'success' strategy: fail-fast: false matrix: @@ -176,7 +187,7 @@ jobs: - name: Run tests in parallel containers uses: ./.github/actions/parallel-ctest-containers with: - container: ${{fromJSON(needs.build-base.outputs.p)[matrix.platform].image}} + container: ${{fromJSON(needs.platforms.outputs.p)[matrix.platform].image}} error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd", "build/TestLogs"]' log-tarball-prefix: ${{matrix.platform}} tests-label: long_running_tests @@ -190,15 +201,15 @@ jobs: libtester-tests: name: libtester tests - needs: [build-base, v, dev-package] - if: always() && needs.v.result == 'success' && needs.dev-package.result == 'success' + needs: [platforms, build-base, v, dev-package] + if: always() && needs.platforms.result == 'success' && needs.v.result == 'success' && needs.dev-package.result == 'success' strategy: fail-fast: false matrix: platform: [ubuntu20, ubuntu22] test: [build-tree, make-dev-install, deb-install] runs-on: ["self-hosted", "enf-x86-midtier"] - container: ${{ matrix.test != 'deb-install' && fromJSON(needs.build-base.outputs.p)[matrix.platform].image || matrix.platform == 'ubuntu20' && 'ubuntu:focal' || 'ubuntu:jammy' }} + container: ${{ matrix.test != 'deb-install' && fromJSON(needs.platforms.outputs.p)[matrix.platform].image || matrix.platform == 'ubuntu20' && 'ubuntu:focal' || 'ubuntu:jammy' }} env: DEBIAN_FRONTEND: noninteractive TZ: Etc/UTC diff --git a/.github/workflows/build_base.yaml b/.github/workflows/build_base.yaml index b1771fbfbf..51dd5d2167 100644 --- a/.github/workflows/build_base.yaml +++ b/.github/workflows/build_base.yaml @@ -1,12 +1,16 @@ name: "Build leap" on: - workflow_dispatch: workflow_call: - outputs: + inputs: p: description: "Discovered Build Platforms" - value: ${{ jobs.d.outputs.p }} + type: string + required: true + platform-matrix: + description: "Platform Matrix" + type: string + required: true permissions: packages: read @@ -17,57 +21,14 @@ defaults: shell: bash jobs: - d: - name: Discover Platforms - runs-on: ubuntu-latest - outputs: - missing-platforms: ${{steps.discover.outputs.missing-platforms}} - p: ${{steps.discover.outputs.platforms}} - steps: - - name: Discover Platforms - id: discover - uses: AntelopeIO/discover-platforms-action@v1 - with: - platform-file: .cicd/platforms.json - password: ${{secrets.GITHUB_TOKEN}} - package-name: builders - - build-platforms: - name: Build Platforms - needs: d - if: needs.d.outputs.missing-platforms != '[]' - strategy: - fail-fast: false - matrix: - platform: ${{fromJSON(needs.d.outputs.missing-platforms)}} - runs-on: ["self-hosted", "enf-x86-beefy"] - permissions: - packages: write - contents: read - steps: - - name: Login to Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{github.repository_owner}} - password: ${{secrets.GITHUB_TOKEN}} - - name: Build and push - uses: docker/build-push-action@v3 - with: - push: true - tags: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}} - file: ${{fromJSON(needs.d.outputs.p)[matrix.platform].dockerfile}} - Build: name: Build leap - needs: [d, build-platforms] - if: always() && needs.d.result == 'success' && (needs.build-platforms.result == 'success' || needs.build-platforms.result == 'skipped') strategy: fail-fast: false matrix: - platform: [ubuntu20, ubuntu22] + platform: ${{fromJSON(inputs.platform-matrix)}} runs-on: ["self-hosted", "enf-x86-beefy"] - container: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}} + container: ${{fromJSON(inputs.p)[matrix.platform].image}} steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/performance_harness_run.yaml b/.github/workflows/performance_harness_run.yaml index 90602d8345..1584edf80c 100644 --- a/.github/workflows/performance_harness_run.yaml +++ b/.github/workflows/performance_harness_run.yaml @@ -2,6 +2,16 @@ name: "Performance Harness Run" on: workflow_dispatch: + inputs: + platform-choice: + description: 'Select Platform' + type: choice + options: + - ubuntu20 + - ubuntu22 + override-test-params: + description: 'Override perf harness params' + type: string permissions: packages: read @@ -12,10 +22,99 @@ defaults: shell: bash jobs: - tmp: - name: Stub + + v: + name: Discover Inputs + runs-on: ubuntu-latest + outputs: + test-params: ${{steps.overrides.outputs.test-params}} + steps: + - name: Setup Input Params + id: overrides + run: | + echo test-params=testBpOpMode >> $GITHUB_OUTPUT + + if [[ "${{inputs.override-test-params}}" != "" ]]; then + echo test-params=${{inputs.override-test-params}} >> $GITHUB_OUTPUT + fi + + platforms: + name: Run Platforms Workflow + uses: ./.github/workflows/platforms.yaml + with: + override-build-matrix: ${{github.event.inputs.platform-choice}} + permissions: + packages: write + contents: read + + reuse-build: + name: Reuse leap build + needs: [v] + if: | + always() && + needs.v.result == 'success' runs-on: ubuntu-latest + outputs: + build-artifact: ${{steps.downloadBuild.outputs.downloaded-file}} steps: - - name: Workflow Stub + - name: Download builddir + id: downloadBuild + uses: AntelopeIO/asset-artifact-download-action@v3 + with: + owner: AntelopeIO + repo: leap + file: build.tar.zst + target: ${{github.sha}} + artifact-name: ${{github.event.inputs.platform-choice}}-build + fail-on-missing-target: false + + - name: Upload builddir + if: steps.downloadBuild.outputs.downloaded-file != '' + uses: AntelopeIO/upload-artifact-large-chunks-action@v1 + with: + name: ${{github.event.inputs.platform-choice}}-build + path: build.tar.zst + + build-base: + name: Run Build Workflow + needs: [platforms, reuse-build] + if: always() && needs.platforms.result == 'success' && needs.reuse-build.outputs.build-artifact == '' + uses: ./.github/workflows/build_base.yaml + with: + p: ${{needs.platforms.outputs.p}} + platform-matrix: ${{needs.platforms.outputs.platform-matrix}} + permissions: + packages: write + contents: read + + tests: + name: Tests + needs: [v, platforms, reuse-build, build-base] + if: always() && needs.platforms.result == 'success' && (needs.build-base.result == 'success' || needs.reuse-build.result == 'success') + runs-on: ubuntu-latest + container: + image: ${{fromJSON(needs.platforms.outputs.p)[github.event.inputs.platform-choice].image}} + steps: + - name: Download builddir + uses: actions/download-artifact@v3 + with: + name: ${{github.event.inputs.platform-choice}}-build + - name: Run Performance Test + run: | + zstdcat build.tar.zst | tar x + cd build + ./tests/performance_tests/performance_test.py ${{needs.v.outputs.test-params}} + - name: Prepare results + id: prep-results run: | - echo "Workflow Stub" + tar -pc build/performance_test | zstd --long -T0 -9 > performance_test_logs.tar.zst + - name: Upload results + uses: AntelopeIO/upload-artifact-large-chunks-action@v1 + with: + name: performance-test-results + path: performance_test_logs.tar.zst + - name: Upload report + uses: actions/upload-artifact@v3 + with: + name: performance-test-report + path: ./build/performance_test/**/report.json diff --git a/.github/workflows/ph_backward_compatibility.yaml b/.github/workflows/ph_backward_compatibility.yaml index 26037cbfee..0abe6b6400 100644 --- a/.github/workflows/ph_backward_compatibility.yaml +++ b/.github/workflows/ph_backward_compatibility.yaml @@ -12,25 +12,36 @@ defaults: shell: bash jobs: + platforms: + name: Run Platforms Workflow + uses: ./.github/workflows/platforms.yaml + permissions: + packages: write + contents: read + build-base: name: Run Build Workflow uses: ./.github/workflows/build_base.yaml + needs: [platforms] + with: + p: ${{needs.platforms.outputs.p}} + platform-matrix: ${{needs.platforms.outputs.platform-matrix}} permissions: packages: write contents: read tests: name: Tests - needs: [build-base] - if: always() && needs.build-base.result == 'success' + needs: [platforms, build-base] + if: always() && needs.platforms.result == 'success' && needs.build-base.result == 'success' strategy: fail-fast: false matrix: - platform: [ubuntu20, ubuntu22] + platform: ${{fromJSON(needs.platforms.outputs.platform-matrix)}} release: [3.1, 3.2, 4.0] runs-on: ["self-hosted", "enf-x86-lowtier"] container: - image: ${{fromJSON(needs.build-base.outputs.p)[matrix.platform].image}} + image: ${{fromJSON(needs.platforms.outputs.p)[matrix.platform].image}} options: --security-opt seccomp=unconfined steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/platforms.yaml b/.github/workflows/platforms.yaml new file mode 100644 index 0000000000..e44ce0ffda --- /dev/null +++ b/.github/workflows/platforms.yaml @@ -0,0 +1,88 @@ +name: "Platforms" + +on: + workflow_call: + inputs: + override-build-matrix: + description: 'Override build matrix' + type: string + required: false + outputs: + p: + description: "Discovered Build Platforms" + value: ${{ jobs.d.outputs.p }} + platform-matrix: + description: "Overridden Platform Matrix" + value: ${{ jobs.pm.outputs.platform-matrix }} + +permissions: + packages: read + contents: read + +defaults: + run: + shell: bash + +jobs: + d: + name: Discover Platforms + runs-on: ubuntu-latest + outputs: + missing-platforms: ${{steps.discover.outputs.missing-platforms}} + p: ${{steps.discover.outputs.platforms}} + steps: + - name: Discover Platforms + id: discover + uses: AntelopeIO/discover-platforms-action@v1 + with: + platform-file: .cicd/platforms.json + password: ${{secrets.GITHUB_TOKEN}} + package-name: builders + + build-platforms: + name: Build Platforms + needs: d + if: needs.d.outputs.missing-platforms != '[]' + strategy: + fail-fast: false + matrix: + platform: ${{fromJSON(needs.d.outputs.missing-platforms)}} + runs-on: ["self-hosted", "enf-x86-beefy"] + permissions: + packages: write + contents: read + steps: + - name: Login to Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{github.repository_owner}} + password: ${{secrets.GITHUB_TOKEN}} + - name: Build and push + uses: docker/build-push-action@v3 + with: + push: true + tags: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}} + file: ${{fromJSON(needs.d.outputs.p)[matrix.platform].dockerfile}} + + pm: + name: Platform Matrix + needs: [d] + if: always() && needs.d.result == 'success' + runs-on: ubuntu-latest + outputs: + platform-matrix: ${{steps.pm-results.outputs.platform-matrix}} + steps: + - name: Parse Platform Matrix + id: parse-pm + uses: actions/github-script@v6 + with: + script: return Object.keys(${{needs.d.outputs.p}}) + - name: Check | Override result + id: pm-results + run: | + echo 'platform-matrix=${{steps.parse-pm.outputs.result}}' >> $GITHUB_OUTPUT + + if [[ "${{inputs.override-build-matrix}}" != "" ]]; then + echo 'platform-matrix=["${{inputs.override-build-matrix}}"]' >> $GITHUB_OUTPUT + fi