Compile on AWS #368
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Compile on AWS | |
| run-name: Compile on AWS | |
| env: | |
| CTEST_OUTPUT_ON_FAILURE: 1 | |
| OMPI_ALLOW_RUN_AS_ROOT: 1 | |
| OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 | |
| OMPI_MCA_rmaps_base_oversubscribe: true | |
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | |
| DEBUG: ${{ github.event.inputs.debug }} | |
| on: | |
| pull_request: | |
| types: [opened, synchronize] | |
| # enable to manually trigger the tests | |
| workflow_dispatch: | |
| inputs: | |
| debug: | |
| description: 'Enable debug mode (true/false)' | |
| required: false | |
| default: 'false' | |
| jobs: | |
| start-runner: | |
| if: ${{contains(github.event.pull_request.labels.*.name, 'full-ci') || github.event_name == 'workflow_dispatch'}} | |
| name: Start self-hosted EC2 runner | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| instance: | |
| - TYPE: g4ad.xlarge | |
| AMI: ami-0f8c4824f9f2e449e | |
| index: 0 | |
| - TYPE: g4dn.xlarge | |
| AMI: ami-06615d18bfde0a1e7 | |
| index: 1 | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| #outputs: | |
| # label_${{ matrix.instance.index }}: ${{ steps.start-ec2-runner.outputs.label }} # Outputs JSON array of runner labels | |
| # ec2-instance-id_${{ matrix.instance.index }}: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| role-to-assume: arn:aws:iam::308634587211:role/Github-OIDC-Role-29bocUD8VBZr | |
| aws-region: us-east-1 | |
| - name: Start EC2 runner | |
| id: start-ec2-runner | |
| uses: machulav/ec2-github-runner@v2.3.8 | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| ec2-image-id: ${{matrix.instance.AMI}} | |
| ec2-instance-type: ${{matrix.instance.TYPE}} | |
| iam-role-name: Role4Github | |
| subnet-id: subnet-87eb68a6 | |
| security-group-id: sg-559f8967 | |
| aws-resource-tags: > # optional, requires additional permissions | |
| [ | |
| {"Key": "ucfd-project", "Value": "BMBF_2022_EXASIM"}, | |
| {"Key": "ucfd-client", "Value": "UCFD-RD"}, | |
| {"Key": "GitHubRepository", "Value": "${{ github.repository }}"} | |
| ] | |
| pre-runner-script: | | |
| #!/bin/bash | |
| . /root/spack/share/spack/setup-env.sh | |
| sudo rm /usr/local/cuda | |
| sudo rm /usr/local/cuda-12.1 | |
| sudo ln -s /usr/local/cuda-12.2 /usr/local/cuda | |
| - name: Store Runner Label | |
| run: | | |
| echo "${{ steps.start-ec2-runner.outputs.label }}" > label_${{ matrix.instance.index }}.txt | |
| echo "${{ steps.start-ec2-runner.outputs.ec2-instance-id }}" > ec2-instance-id_${{ matrix.instance.index }}.txt | |
| echo "${{ matrix.instance.TYPE }}" > type_${{ matrix.instance.index }}.txt | |
| - name: Upload Label Artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: labels_${{ matrix.instance.index }} | |
| path: | | |
| label_${{ matrix.instance.index }}.txt | |
| ec2-instance-id_${{ matrix.instance.index }}.txt | |
| type_${{ matrix.instance.index }}.txt | |
| retention-days: 1 | |
| aggregate-labels: | |
| needs: start-runner | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.aggregate.outputs.matrix }} | |
| steps: | |
| - name: Initialize Empty JSON Arrays | |
| run: | | |
| echo "MATRIX_JSON=[]" >> $GITHUB_ENV | |
| - name: Download Artifact 1 | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: labels_0 | |
| path: downloaded_data | |
| - name: Download Artifact 2 | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: labels_1 | |
| path: downloaded_data | |
| - name: Loop Through Matrix Indices and Download Artifacts | |
| id: aggregate | |
| run: | | |
| INSTANCES=() | |
| for index in {0..1}; do # Adjust max index if needed | |
| # Read values from downloaded files | |
| LABEL=$(cat downloaded_data/label_${index}.txt) | |
| EC2_ID=$(cat downloaded_data/ec2-instance-id_${index}.txt) | |
| EC2_TYPE=$(cat downloaded_data/type_${index}.txt) | |
| # Create JSON object for this instance | |
| INSTANCES+=("{\"runner\":\"$LABEL\", \"ec2-id\":\"$EC2_ID\", \"ec2-type\":\"$EC2_TYPE\"}") | |
| done | |
| # Convert arrays to JSON | |
| MATRIX_JSON=$(echo "${INSTANCES[@]}" | jq -s . | jq -c .) | |
| echo $MATRIX_JSON | |
| echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT | |
| - name: Show Aggregated Data | |
| run: | | |
| echo "Matrix: ${{ steps.aggregate.outputs.matrix }}" | |
| build-on-aws: | |
| name: Build on aws | |
| needs: aggregate-labels # required to start the main job when the runner is ready | |
| runs-on: ${{ matrix.instance.runner }} # run the job on the newly created runner | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| instance: ${{ fromJson(needs.aggregate-labels.outputs.matrix) }} | |
| preset: ["develop", "production"] | |
| steps: | |
| - name: Prepare environment | |
| shell: bash -i {0} | |
| run: | | |
| sudo rm /actions-runner/externals/node20/bin/node | |
| sudo ln -s /root/spack/opt/spack/linux-centos7-x86_64_v3/gcc-10.5.0/node-js*/bin/node /actions-runner/externals/node20/bin/node | |
| - name: Checkout NeoFOAM | |
| uses: actions/checkout@v2 | |
| - name: Set up cache | |
| uses: actions/cache@v3 | |
| if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip-cache')}} | |
| with: | |
| path: build | |
| key: aws_PR_${{ github.event.pull_request.number }}_${{matrix.preset}}_${{matrix.instance.ec2-type}} | |
| - name: Build NeoFOAM | |
| shell: bash -i {0} | |
| run: | | |
| . /root/spack/share/spack/setup-env.sh | |
| spack load cmake@3.30.5 | |
| spack load hip | |
| #spack load kokkos@4.3.00 | |
| module load clang/17 | |
| cmake --version | |
| #CC=clang \ | |
| #CXX=clang++ \ | |
| cmake --preset ${{matrix.preset}} \ | |
| -DNEOFOAM_BUILD_TESTS=ON \ | |
| -DNEOFOAM_DEVEL_TOOLS=OFF \ | |
| -DNEOFOAM_ENABLE_MPI_WITH_THREAD_SUPPORT=OFF | |
| cmake --build --preset ${{matrix.preset}} | |
| - name: Test NeoFOAM | |
| shell: bash -i {0} | |
| run: | | |
| . /root/spack/share/spack/setup-env.sh | |
| module load gnu/11 | |
| spack load cmake@3.30.5 | |
| ctest --preset ${{matrix.preset}} | |
| benchmark-on-aws: | |
| name: Benchmark on aws | |
| needs: [start-runner,aggregate-labels, build-on-aws] # required to start the main job when the runner is ready | |
| runs-on: ${{ matrix.instance.runner }} # run the job on the newly created runner | |
| if: github.event.inputs.debug != 'true' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| instance: ${{ fromJson(needs.aggregate-labels.outputs.matrix) }} | |
| steps: | |
| - name: Build NeoFOAM | |
| shell: bash -i {0} | |
| run: | | |
| . /root/spack/share/spack/setup-env.sh | |
| module load clang/17 | |
| spack load cmake@3.30.5 | |
| cmake --version | |
| python3 -m pip install xmltodict | |
| #CC=clang \ | |
| #CXX=clang++ \ | |
| cmake --preset profiling | |
| cmake --build --preset profiling | |
| module load gnu/11 | |
| ctest --preset profiling | |
| mkdir -p ${{github.event.number}}/main | |
| cd build/profiling/bin/benchmarks | |
| python3 ../../../../scripts/catch2json.py | |
| cd ../../../.. | |
| cp build/profiling/bin/benchmarks/*.json ${{github.event.number}}/ | |
| lscpu > ${{github.event.number}}/lscpu.log | |
| rm -rf build | |
| git fetch origin | |
| git checkout main | |
| #CC=clang \ | |
| #CXX=clang++ \ | |
| module load clang/17 | |
| cmake --preset profiling | |
| cmake --build --preset profiling | |
| module load gnu/11 | |
| ctest --preset profiling | |
| cd build/profiling/bin/benchmarks | |
| python3 ../../../../scripts/catch2json.py | |
| cd ../../../.. | |
| cp build/profiling/bin/benchmarks/*.json ${{github.event.number}}/main/ | |
| - name: Push Benchmark Data | |
| uses: cpina/github-action-push-to-another-repository@main | |
| env: | |
| API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }} | |
| with: | |
| source-directory: ${{github.event.number}} | |
| destination-github-username: 'exasim-project' | |
| destination-repository-name: 'NeoFOAM-BenchmarkData' | |
| target-directory: ${{github.event.number}}/${{matrix.instance.ec2-type}} | |
| user-email: github-actions@github.com | |
| target-branch: main | |
| stop-runner: | |
| name: Stop self-hosted EC2 runner | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| instance: ${{ fromJson(needs.aggregate-labels.outputs.matrix) }} | |
| needs: | |
| - start-runner # required to get output from the start-runner job | |
| - benchmark-on-aws # required to wait when the main job is done | |
| - aggregate-labels | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| # only try to run the stop job if the start runner hasn't been skipped | |
| if: always() && needs.start-runner.result != 'skipped' && github.event.inputs.debug != 'true' | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: arn:aws:iam::308634587211:role/Github-OIDC-Role-29bocUD8VBZr | |
| aws-region: us-east-1 | |
| - name: Stop EC2 runner | |
| uses: machulav/ec2-github-runner@v2.3.8 | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| label: ${{ matrix.instance.runner }} | |
| ec2-instance-id: ${{ matrix.instance.ec2-id }} |