From e71f50558a61338af46e3fa200a0df18fee02d6b Mon Sep 17 00:00:00 2001 From: Thomas Parnell Date: Wed, 15 Jan 2025 19:23:24 +0000 Subject: [PATCH 1/4] Add github workflow to run Spyre tests Signed-off-by: Thomas Parnell --- .github/workflows/test-spyre.yml | 28 ++++++++++++++++++++++++++++ tests/spyre/spyre_util.py | 2 +- 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test-spyre.yml diff --git a/.github/workflows/test-spyre.yml b/.github/workflows/test-spyre.yml new file mode 100644 index 000000000..cd8b73c65 --- /dev/null +++ b/.github/workflows/test-spyre.yml @@ -0,0 +1,28 @@ +name: test-sypre + +on: + # Trigger the workflow on push or pull request, + # but only for the main branch + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + test-spyre: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Build docker images + run: docker build . -t vllm-spyre -f Dockerfile.spyre + - name: Run tests within docker image + run: docker run -it --entrypoint /bin/bash vllm-spyre -c ''' + python3.12 -c "from transformers import pipeline; pipeline(\"text-generation\", model=\"JackFram/llama-160m\")" && \ + export VARIANT=$(ls /root/.cache/huggingface/hub/models--JackFram--llama-160m/snapshots/) && \ + mkdir -p /models && \ + ln -s /root/.cache/huggingface/hub/models--JackFram--llama-160m/snapshots/${VARIANT} /models/llama-194m && \ + pip install pytest sentence-transformers && \ + python3.12 -m pytest tests/spyre + ''' diff --git a/tests/spyre/spyre_util.py b/tests/spyre/spyre_util.py index 7fc3bc10d..175ad0a58 100644 --- a/tests/spyre/spyre_util.py +++ b/tests/spyre/spyre_util.py @@ -134,7 +134,7 @@ def compare_results(model: str, prompts: List[str], print() assert DISABLE_ASSERTS or backend == 'sendnn_decoder' or\ - hf_result['text'] == vllm_result['text'] + hf_result['token_ids'] == vllm_result['token_ids'] if len(hf_result['tokens']) > 0: print(" token id. token logprob " From 476a29f89cf5ead120e812b2ab51cf18cb182c24 Mon Sep 17 00:00:00 2001 From: Thomas Parnell Date: Wed, 15 Jan 2025 19:26:54 +0000 Subject: [PATCH 2/4] Fix syntax error Signed-off-by: Thomas Parnell --- .github/workflows/test-spyre.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-spyre.yml b/.github/workflows/test-spyre.yml index cd8b73c65..7b42c42b8 100644 --- a/.github/workflows/test-spyre.yml +++ b/.github/workflows/test-spyre.yml @@ -18,11 +18,12 @@ jobs: - name: Build docker images run: docker build . -t vllm-spyre -f Dockerfile.spyre - name: Run tests within docker image - run: docker run -it --entrypoint /bin/bash vllm-spyre -c ''' - python3.12 -c "from transformers import pipeline; pipeline(\"text-generation\", model=\"JackFram/llama-160m\")" && \ - export VARIANT=$(ls /root/.cache/huggingface/hub/models--JackFram--llama-160m/snapshots/) && \ - mkdir -p /models && \ - ln -s /root/.cache/huggingface/hub/models--JackFram--llama-160m/snapshots/${VARIANT} /models/llama-194m && \ - pip install pytest sentence-transformers && \ - python3.12 -m pytest tests/spyre - ''' + run: | + docker run -it --entrypoint /bin/bash vllm-spyre -c ''' + python3.12 -c "from transformers import pipeline; pipeline(\"text-generation\", model=\"JackFram/llama-160m\")" && \ + export VARIANT=$(ls /root/.cache/huggingface/hub/models--JackFram--llama-160m/snapshots/) && \ + mkdir -p /models && \ + ln -s /root/.cache/huggingface/hub/models--JackFram--llama-160m/snapshots/${VARIANT} /models/llama-194m && \ + pip install pytest sentence-transformers && \ + python3.12 -m pytest tests/spyre + ''' From 7b8f6a7acc077027787045481f97fdb1bbd9af6e Mon Sep 17 00:00:00 2001 From: Thomas Parnell Date: Wed, 15 Jan 2025 19:33:11 +0000 Subject: [PATCH 3/4] Remove -t argument to docker run Signed-off-by: Thomas Parnell --- .github/workflows/test-spyre.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-spyre.yml b/.github/workflows/test-spyre.yml index 7b42c42b8..1d07a2aad 100644 --- a/.github/workflows/test-spyre.yml +++ b/.github/workflows/test-spyre.yml @@ -15,11 +15,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Build docker images + - name: Build docker image run: docker build . -t vllm-spyre -f Dockerfile.spyre - - name: Run tests within docker image + - name: Run Spyre tests within docker container run: | - docker run -it --entrypoint /bin/bash vllm-spyre -c ''' + docker run -i --rm --entrypoint /bin/bash vllm-spyre -c ''' python3.12 -c "from transformers import pipeline; pipeline(\"text-generation\", model=\"JackFram/llama-160m\")" && \ export VARIANT=$(ls /root/.cache/huggingface/hub/models--JackFram--llama-160m/snapshots/) && \ mkdir -p /models && \ From dd96ddf3b1225810b2b6814c486d6af9ffaa6ee6 Mon Sep 17 00:00:00 2001 From: Thomas Parnell Date: Wed, 15 Jan 2025 19:42:32 +0000 Subject: [PATCH 4/4] Make test verbose Signed-off-by: Thomas Parnell --- .github/workflows/test-spyre.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-spyre.yml b/.github/workflows/test-spyre.yml index 1d07a2aad..a15ef31f1 100644 --- a/.github/workflows/test-spyre.yml +++ b/.github/workflows/test-spyre.yml @@ -25,5 +25,5 @@ jobs: mkdir -p /models && \ ln -s /root/.cache/huggingface/hub/models--JackFram--llama-160m/snapshots/${VARIANT} /models/llama-194m && \ pip install pytest sentence-transformers && \ - python3.12 -m pytest tests/spyre + python3.12 -m pytest tests/spyre -v '''