Disable value splitting on G3 (vllm-project#58)

JHLEE17 · Jun 11, 2024 · 47c0c5b · 47c0c5b
1 parent b2d0e1e
commit 47c0c5b
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/vllm/hpu/ops.py b/vllm/hpu/ops.py
@@ -9,11 +9,14 @@
 import torch.nn as nn
 import torch.nn.functional as F
 import habana_frameworks.torch as htorch
+import habana_frameworks.torch.utils.experimental as htexp
 from typing import List, Optional, Tuple
 
 import vllm.hpu.utils as hpu_utils
 
-PA_SPLIT_VALUE = (os.environ.get('PA_SPLIT_VALUE', '1') == '1')
+# FIXME: For some reason splitting value causes DFAs on G3. This needs to be debugged
+PA_SPLIT_VALUE_DEFAULT = '0' if (htexp._get_device_type() == htexp.synDeviceType.synDeviceGaudi3) else '1'
+PA_SPLIT_VALUE = (os.environ.get('PA_SPLIT_VALUE', PA_SPLIT_VALUE_DEFAULT) == '1')
 
 
 def silu_and_mul(output, input):