Fix cache position update

huggingface · May 7, 2024 · e1b1c10 · e1b1c10
1 parent 043df8b
commit e1b1c10
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/src/transformers/generation/utils.py b/src/transformers/generation/utils.py
@@ -667,7 +667,11 @@ def _update_model_kwargs_for_generation(
                     dim=-1,
                 )
 
-        if "cache_position" in model_kwargs and model_kwargs["cache_position"] is not None:
+        if (
+            model_kwargs.get("use_cache", True)
+            and "cache_position" in model_kwargs
+            and model_kwargs["cache_position"] is not None
+        ):
             model_kwargs["cache_position"] = model_kwargs["cache_position"][-1:] + num_new_tokens
 
         return model_kwargs