From db87eb6c67271eb61ba9fd8559ce811a1a398a4d Mon Sep 17 00:00:00 2001 From: youkaichao Date: Thu, 5 Dec 2024 20:30:41 -0800 Subject: [PATCH] [torch.compile] use size tuning for specific sizes (#10933) Signed-off-by: youkaichao --- vllm/compilation/backends.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vllm/compilation/backends.py b/vllm/compilation/backends.py index d49a83fe3981f..9773ba8cec779 100644 --- a/vllm/compilation/backends.py +++ b/vllm/compilation/backends.py @@ -43,6 +43,12 @@ def wrap_inductor(graph, if additional_inductor_config is not None: current_config.update(additional_inductor_config) + if isinstance(runtime_shape, int): + # for a specific batchsize, tuning triton kernel parameters + # can be beneficial + current_config["max_autotune"] = True + current_config["coordinate_descent_tuning"] = True + # inductor can inplace modify the graph, so we need to copy it # see https://github.com/pytorch/pytorch/issues/138980 graph = copy.deepcopy(graph)