diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py
index de7a736293f4..fa508a350770 100755
--- a/src/transformers/trainer.py
+++ b/src/transformers/trainer.py
@@ -1697,6 +1697,8 @@ def _inner_training_loop(
         use_accelerator_prepare = True if model is self.model else False
 
         if delay_optimizer_creation:
+            if use_accelerator_prepare:
+                self.model = self.accelerator.prepare(self.model)
             self.create_optimizer_and_scheduler(num_training_steps=max_steps)
 
         # prepare using `accelerator` prepare