diff --git a/.github/workflows/test-spyre.yml b/.github/workflows/test-spyre.yml
new file mode 100644
index 000000000..a15ef31f1
--- /dev/null
+++ b/.github/workflows/test-spyre.yml
@@ -0,0 +1,29 @@
+name: test-sypre
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  test-spyre:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+    - name: Build docker image
+      run: docker build . -t vllm-spyre -f Dockerfile.spyre
+    - name: Run Spyre tests within docker container
+      run: |
+        docker run -i --rm --entrypoint /bin/bash vllm-spyre -c '''
+          python3.12 -c "from transformers import pipeline; pipeline(\"text-generation\", model=\"JackFram/llama-160m\")" && \
+          export VARIANT=$(ls /root/.cache/huggingface/hub/models--JackFram--llama-160m/snapshots/) && \
+          mkdir -p /models && \
+          ln -s /root/.cache/huggingface/hub/models--JackFram--llama-160m/snapshots/${VARIANT} /models/llama-194m && \
+          pip install pytest sentence-transformers && \
+          python3.12 -m pytest tests/spyre -v
+        '''
diff --git a/tests/spyre/spyre_util.py b/tests/spyre/spyre_util.py
index 7fc3bc10d..175ad0a58 100644
--- a/tests/spyre/spyre_util.py
+++ b/tests/spyre/spyre_util.py
@@ -134,7 +134,7 @@ def compare_results(model: str, prompts: List[str],
         print()
 
         assert DISABLE_ASSERTS or backend == 'sendnn_decoder' or\
-            hf_result['text'] == vllm_result['text']
+            hf_result['token_ids'] == vllm_result['token_ids']
 
         if len(hf_result['tokens']) > 0:
             print("   token id. token               logprob      "