Check rope_scaling attr (huggingface#1609)

huijuanzh · Jan 7, 2025 · 7cc6221 · 7cc6221
1 parent dcce890
commit 7cc6221
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/examples/text-generation/utils.py b/examples/text-generation/utils.py
@@ -442,7 +442,7 @@ def setup_distributed_model(args, model_dtype, model_kwargs, logger):
         # Construct model with fake meta tensors, later will be replaced on devices during ds-inference ckpt load
         with deepspeed.OnDevice(dtype=model_dtype, device="meta"):
             if (
-                config.rope_scaling
+                hasattr(config, 'rope_scaling') and config.rope_scaling
                 and config.rope_scaling["rope_type"] == "llama3"
                 and config.max_position_embeddings > 8192
             ):