Merge pull request #1 from winglian/longchat-args

add rope scaling as a cli arg so openai server can load rope scaled models
togethercomputer · Aug 7, 2023 · b9012fb · b9012fb
2 parents 58e7121 + bb8e153
commit b9012fb
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py
@@ -1,5 +1,6 @@
 import argparse
 import dataclasses
+import json
 from dataclasses import dataclass
 from typing import Optional, Tuple
 
@@ -131,6 +132,10 @@ def add_cli_args(
         parser.add_argument('--disable-log-stats',
                             action='store_true',
                             help='disable logging statistics')
+        parser.add_argument('--rope-scaling',
+                            default=None,
+                            type=json.loads,
+                            help='RoPE scaling configuration')
         return parser
 
     @classmethod