Forward unittest workloads to AWS (#2887)

.github/workflows/pre_merge.yml

@@ -34,15 +34,11 @@ jobs:
       - name: Code quality checks
         run: tox -vv -e pre-commit-all-py310-pt1
   Unit-Test:
-    runs-on: [self-hosted, linux, x64, dev]
     needs: Code-Quality-Checks
-    timeout-minutes: 120
     strategy:
       fail-fast: false
       matrix:
         include:
-          - python-version: "3.8"
-            tox-env: "py38"
           - python-version: "3.9"
             tox-env: "py39"
           - python-version: "3.10"
@@ -52,19 +48,37 @@ jobs:
     concurrency:
       group: ${{ github.workflow }}-Unit-${{ github.event.pull_request.number || github.ref }}-${{ matrix.tox-env }}
       cancel-in-progress: true
+    uses: ./.github/workflows/run_tests_in_tox_custom.yml
+    with:
+      python-version: ${{ matrix.python-version }}
+      toxenv-pyver: ${{ matrix.tox-env }}
+      toxenv-task: all
+      tests-dir: tests/unit
+      timeout-minutes: 120
+      upload-artifact: true
+      artifact-prefix: "unit-test-results"
+      runs-on: "['otx-gpu-v100-1']"
+  Coverage-Test:
+    needs: Code-Quality-Checks
+    concurrency:
+      group: ${{ github.workflow }}-Coverage-${{ github.event.pull_request.number || github.ref }}}
+      cancel-in-progress: true
+    runs-on: ["otx-gpu-v100-1"]
+    container:
+      image: 219678651685.dkr.ecr.eu-central-1.amazonaws.com/ote-ci:pr-194-1334bef936df3b06d28bc3d1bd90e7248359b99e
+      options: --runtime=nvidia --env-file=/home/runner/.nvidia.env --shm-size=24g
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
       - name: Install Python
         uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: "3.8"
       - name: Install dependencies
         run: python -m pip install -r requirements/dev.txt
       - name: Run unit test
-        run: tox -vv -e unittest-all-${{ matrix.tox-env }}-pt1
+        run: tox -vv -e unittest-all-py38-pt1
       - name: Upload coverage artifact
-        if: ${{ matrix.python-version == '3.8' }}
         uses: actions/upload-artifact@v3
         with:
           name: coverage
@@ -88,7 +102,7 @@ jobs:
           chmod +x codecov
           ./codecov -t ${{ secrets.CODECOV_TOKEN }} --sha $COMMIT_ID -U $HTTP_PROXY -f .tox/coverage.xml -F ${{ matrix.tox-env }}
   Integration-Test:
-    needs: Unit-Test
+    needs: [Unit-Test, Coverage-Test]
     strategy:
       fail-fast: false
       matrix:

.github/workflows/run_tests_in_tox_custom.yml

@@ -0,0 +1,77 @@
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        type: string
+        default: "3.10"
+      toxenv-pyver:
+        description: "[py38, py39, py310]"
+        type: string
+        default: "py310"
+      toxenv-task:
+        description: "[all, act, ano, cls, det, seg, iseg]"
+        type: string
+        default: "all"
+      tests-dir:
+        type: string
+        default: ""
+      timeout-minutes:
+        type: number
+        default: 720
+      upload-artifact:
+        type: boolean
+        default: false
+      runs-on:
+        type: string
+        default: "['self-hosted', 'Linux', 'X64', 'dev']"
+      task:
+        type: string
+        default: "undefined"
+      artifact-prefix:
+        type: string
+        default: "test-results"
+      toxenv-ptver:
+        type: string
+        default: "pt1"
+      container-options:
+        type: string
+        default: "--runtime=nvidia --env-file=/home/runner/.nvidia.env --shm-size=24g"
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+  run_tests_on_custom:
+    # tricky workaround to pass list from the string input type
+    # https://github.com/orgs/community/discussions/11692
+    runs-on: ${{ fromJson(inputs.runs-on) }}
+    container:
+      image: 219678651685.dkr.ecr.eu-central-1.amazonaws.com/ote-ci:pr-194-1334bef936df3b06d28bc3d1bd90e7248359b99e
+      options: ${{ inputs.container-options }}
+    timeout-minutes: ${{ inputs.timeout-minutes }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ inputs.python-version }}
+      - name: Install dependencies
+        run: python -m pip install -r requirements/dev.txt
+      - name: Run Tests
+        env:
+          MLFLOW_TRACKING_SERVER_URI: ${{ vars.MLFLOW_TRACKING_SERVER_URI }}
+          BENCHMARK_RESULTS_CLEAR: ${{ vars.BENCHMARK_RESULTS_CLEAR }}
+          GH_CTX_REF_NAME: ${{ github.ref_name }}
+          GH_CTX_SHA: ${{ github.sha  }}
+        run: tox -vv -e tests-${{ inputs.toxenv-task }}-${{ inputs.toxenv-pyver }}-${{ inputs.toxenv-ptver }} -- ${{ inputs.tests-dir }}
+      - name: Upload test results
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ inputs.artifact-prefix }}-${{ inputs.toxenv-task }}-${{ inputs.toxenv-pyver }}-${{ inputs.toxenv-ptver }}
+          path: |
+            .tox/tests-${{ inputs.toxenv-task }}-${{ inputs.toxenv-pyver }}-${{ inputs.toxenv-ptver }}.csv
+            .tox/tests-reg_${{ inputs.task }}*.csv
+            .tox/perf-*.csv
+        # Use always() to always run this step to publish test results when there are test failures
+        if: ${{ inputs.upload-artifact && always() }}