use runner's HABANA_VISIBLE_DEVICES

huggingface · Feb 18, 2025 · b653727 · b653727
1 parent 356d898
commit b653727
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 77 deletions.
diff --git a/.github/workflows/test_accelerate_gaudi2.yml b/.github/workflows/test_accelerate_gaudi2.yml
@@ -1,10 +1,7 @@
-name: (Gaudi2) Accelerate integration tests
+name: Accelerate integration tests
 
 on:
   workflow_dispatch:
-  schedule:
-    - cron: "0 23 * * *" # every Wednesday and Saturday at 1am CET (midnight winter time)
-
   pull_request:
     branches: [main]
   push:
@@ -17,74 +14,27 @@ concurrency:
 jobs:
   accelerate:
     name: Test Accelerate integration
-
     runs-on: [self-hosted, linux, x64, gaudi2]
 
-    container:
-      image: vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest
-      options: --runtime=habana
-        --ipc=host
-        --cap-add=sys_nice
-        --env HABANA_VISIBLE_DEVICES=all
-        --env OMPI_MCA_btl_vader_single_copy_mechanism=none
-        --env HF_HOME=/data
-        --env GAUDI2_CI=1
-
     steps:
-      - name: Checkout Accelerate
+      - name: Checkout
         uses: actions/checkout@v4
-        with:
-          repository: huggingface/accelerate
-          branch: hpu-upstream
-          path: accelerate
-          fetch-depth: 0
-
-      - name: Install Accelerate
-        working-directory: ./accelerate
+      - name: Pull image
         run: |
-          pip install -e .[testing]
-          pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.19.0
-
-      - name: Run Accelerate Core tests
-        working-directory: ./accelerate
-        run: |
-          make test_core
-        env:
-          PT_ENABLE_INT64_SUPPORT: 1 # for tokenizers
-          PT_HPU_LAZY_MODE: 0 # for fsdp
-          RUN_SLOW: 1
-
-      - name: Run Accelerate CLI tests
-        working-directory: ./accelerate
-        run: |
-          make test_cli
-        env:
-          PT_ENABLE_INT64_SUPPORT: 1 # for tokenizers
-          RUN_SLOW: 1
-
-      - name: Run Accelerate FSDP tests
-        working-directory: ./accelerate
-        run: |
-          make test_fsdp
-        env:
-          PT_ENABLE_INT64_SUPPORT: 1 # for tokenizers
-          PT_HPU_LAZY_MODE: 0 # for fsdp
-          RUN_SLOW: 1
-
-      - name: Run Accelerate DeepSpeed tests
-        working-directory: ./accelerate
-        run: |
-          make test_deepspeed
-        env:
-          PT_ENABLE_INT64_SUPPORT: 1 # for tokenizers
-          PT_HPU_LAZY_MODE: 0 # for cpu offload
-          RUN_SLOW: 1
-
-      - name: Run Accelerate Big Modeling tests
-        working-directory: ./accelerate
+          docker pull vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest
+      - name: Run tests
         run: |
-          make test_big_modeling
-        env:
-          PT_ENABLE_INT64_SUPPORT: 1 # for tokenizers
-          PT_HPU_LAZY_MODE: 0 # for cpu offload
-          RUN_SLOW: 1
+          docker run \
+          --rm \
+          -v $PWD:/root/workspace \
+          -v /scratch-1:/data \
+          --workdir=/root/workspace \
+          --runtime=habana \
+          -e HABANA_VISIBLE_DEVICES=$DOCKER_HABANA_VISIBLE_DEVICES \
+          -e OMPI_MCA_btl_vader_single_copy_mechanism=none \
+          -e HF_HOME=/data \
+          --cap-add=sys_nice \
+          --net=host \
+          --ipc=host \
+          vault.habana.ai/gaudi-docker/1.19.0/ubuntu22.04/habanalabs/pytorch-installer-2.5.1:latest \
+          /bin/bash tests/ci/accelerate.sh
diff --git a/tests/ci/accelerate.sh b/tests/ci/accelerate.sh
@@ -1,10 +1,10 @@
-#!/bin/bash
+# Description: Accelerate CI script
 
-OPTIMUM_HABANA_PATH=${CI_OPTIMUM_HABANA_PATH:-/root/workspace/optimum-habana}
-ACCELERATE_PATH=${CI_SENTENCE_TRANSFORMER_PATH:-/root/workspace/accelerate}
+git clone https://github.com/huggingface/accelerate.git --branch hpu-support --depth 1 && cd accelerate
+pip install -e .[testing] git+https://github.com/HabanaAI/DeepSpeed.git@1.19.0
 
-python -m pip install --upgrade pip
-python -m pip install $OPTIMUM_HABANA_PATH[tests]
-cd $ACCELERATE_PATH
-python -m pip install .
-RUN_SLOW=1 pytest tests/test_accelerator.py
+RUN_SLOW=1 PT_ENABLE_INT64_SUPPORT=1 make test_cli
+RUN_SLOW=1 PT_ENABLE_INT64_SUPPORT=1 PT_HPU_LAZY_MODE=0 make test_core
+RUN_SLOW=1 PT_ENABLE_INT64_SUPPORT=1 PT_HPU_LAZY_MODE=0 make test_fsdp
+RUN_SLOW=1 PT_ENABLE_INT64_SUPPORT=1 PT_HPU_LAZY_MODE=0 make test_deepspeed
+RUN_SLOW=1 PT_ENABLE_INT64_SUPPORT=1 PT_HPU_LAZY_MODE=0 make test_big_modeling