Merge branch 'main' of https://github.com/blinkbear/vllm

blinkbear · Sep 5, 2024 · 293c294 · 293c294
2 parents 856b8bf + 1f0ffd5
commit 293c294
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 1 deletion.
diff --git a/benchmarks/backend_request_func.py b/benchmarks/backend_request_func.py
@@ -344,6 +344,7 @@ async def async_request_openai_chat_completions(
                 },
             ],
             "temperature": 0.0,
+            "min_tokens": request_func_input.output_len,
             "max_tokens": request_func_input.output_len,
             "stream": True,
         }

diff --git a/examples/analysis/prefill_decode_time_compare.py b/examples/analysis/prefill_decode_time_compare.py
@@ -53,7 +53,7 @@ def create_init_prompts(
         selected_seqs = [seqs[1]] * init_prompt_nums
     elif prefill_mode == "horizonal":
         # create a batch whose size is 1 and each seq length is init_prompt_nums
-        selected_seqs = [seqs[init_prompt_nums]]
+        selected_seqs = [seqs[init_prompt_nums]] *3
     for i in range(len(selected_seqs)):
         prompts_queue.put(selected_seqs[i])