diff --git a/src/transformers/models/whisper/generation_whisper.py b/src/transformers/models/whisper/generation_whisper.py
index b3865140f24e..42457ff8669c 100644
--- a/src/transformers/models/whisper/generation_whisper.py
+++ b/src/transformers/models/whisper/generation_whisper.py
@@ -1042,8 +1042,8 @@ def _set_return_timestamps(return_timestamps, is_shortform, generation_config):
                 "Make sure to initialize the generation config with the correct attributes that are needed such as `no_timestamps_token_id`. "
                 "For more details on how to generate the approtiate config, refer to https://github.com/huggingface/transformers/issues/21878#issuecomment-1451902363"
             )
-
-        generation_config.return_timestamps = return_timestamps
+        if return_timestamps is not None:
+            generation_config.return_timestamps = return_timestamps
 
     @staticmethod
     def _set_language_and_task(language, task, is_multilingual, generation_config):