add HF cache (vllm-project#65)

SUMMARY:
* update "set-env" action to set HF_HOME
* add action to mount HF cache
* pin some tests to devices "0,1" as this enables about 2k more test
points

TEST PLAN:
runs on remote push

---------

Co-authored-by: andy-neuma <andy@neuralmagic.com>

.github/actions/nm-hf-cache/action.yml

@@ -0,0 +1,15 @@
+name: HF cache
+description: 'mount HF cache'
+inputs:
+  fs_cache:
+    description: 'filesystem to use for HF cache'
+    required: true
+runs:
+  using: composite
+  steps:
+  - run: |
+      sudo apt install -y nfs-common
+      sudo mkdir -m 777 -p /EFS
+      sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport ${{ inputs.fs_cache }}:/ /EFS
+      sudo chown -R $(whoami):$(whoami) ${HF_HOME}
+    shell: bash

.github/actions/nm-set-env/action.yml

@@ -12,6 +12,7 @@ runs:
   steps:
   - run: |
       echo "HF_TOKEN=${HF_TOKEN_SECRET}" >> $GITHUB_ENV
+      echo "HF_HOME=/EFS/hf_home" >> $GITHUB_ENV
       NUM_THREADS=$(./.github/scripts/determine-threading -G ${{ inputs.Gi_per_thread }})
       echo "MAX_JOBS=${NUM_THREADS}" >> $GITHUB_ENV
       echo "VLLM_INSTALL_PUNICA_KERNELS=1" >> $GITHUB_ENV

.github/scripts/run-tests

@@ -59,9 +59,9 @@ do
     # this is a bit messy and brittle, but certain tests
     # need to be run with specific options
     if [[ "${TEST}" == *"kernels"* ]]; then
-        CUDA_VISIBLE_DEVICES=0 pytest --junitxml=${RESULT_XML} ${TEST} || LOCAL_SUCCESS=$?
+        CUDA_VISIBLE_DEVICES=0,1 pytest --junitxml=${RESULT_XML} ${TEST} || LOCAL_SUCCESS=$?
     elif [[ "${TEST}" == *"samplers"* ]]; then
-        CUDA_VISIBLE_DEVICES=0 pytest --junitxml=${RESULT_XML} ${TEST} || LOCAL_SUCCESS=$?
+        CUDA_VISIBLE_DEVICES=0,1 pytest --junitxml=${RESULT_XML} ${TEST} || LOCAL_SUCCESS=$?
     elif [[ "${TEST}" == *"distributed"* ]]; then
         pytest --forked --junitxml=${RESULT_XML} ${TEST} || LOCAL_SUCCESS=$?
     else

.github/workflows/build-test.yml

@@ -79,6 +79,12 @@ jobs:
                 python: ${{ inputs.python }}
                 venv: TEST
 
+            - name: hf cache
+              id: hf_cache
+              uses: ./.github/actions/nm-hf-cache/
+              with:
+                fs_cache: ${{ secrets.HF_FS_CACHE }}
+
             # TODO: testmo source is currently hardcoded.
             - name: create testmo run
               id: create_testmo_run