diff --git a/examples/text-generation/utils.py b/examples/text-generation/utils.py
index 7cc09e9905..41ceede6c6 100644
--- a/examples/text-generation/utils.py
+++ b/examples/text-generation/utils.py
@@ -433,7 +433,7 @@ def setup_distributed_model(args, model_dtype, model_kwargs, logger):
         # Construct model with fake meta tensors, later will be replaced on devices during ds-inference ckpt load
         with deepspeed.OnDevice(dtype=model_dtype, device="meta"):
             if (
-                config.rope_scaling
+                hasattr(config, 'rope_scaling') and config.rope_scaling
                 and config.rope_scaling["rope_type"] == "llama3"
                 and config.max_position_embeddings > 8192
             ):