From fe402a6fe34c7294baaa7d307eddc8449559d808 Mon Sep 17 00:00:00 2001
From: Wing Lian <wing.lian@gmail.com>
Date: Sat, 5 Aug 2023 07:55:03 -0400
Subject: [PATCH 1/2] add rope scaling as a cli arg so openai server can load
 rope scaled models

---
 vllm/engine/arg_utils.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py
index 0c8a5601e4ce5..4cb18f3a6e117 100644
--- a/vllm/engine/arg_utils.py
+++ b/vllm/engine/arg_utils.py
@@ -131,6 +131,9 @@ def add_cli_args(
         parser.add_argument('--disable-log-stats',
                             action='store_true',
                             help='disable logging statistics')
+        parser.add_argument('--rope-scaling',
+                            default=None,
+                            help='RoPE scaling configuration')
         return parser
 
     @classmethod

From bb8e1539f53f7a915eb856a7bed8b1c18adf822a Mon Sep 17 00:00:00 2001
From: Wing Lian <wing.lian@gmail.com>
Date: Sat, 5 Aug 2023 11:32:14 -0400
Subject: [PATCH 2/2] set rope-scaling arg as json.loads so it can load from
 cli

---
 vllm/engine/arg_utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py
index 4cb18f3a6e117..6895d550ba5d6 100644
--- a/vllm/engine/arg_utils.py
+++ b/vllm/engine/arg_utils.py
@@ -1,5 +1,6 @@
 import argparse
 import dataclasses
+import json
 from dataclasses import dataclass
 from typing import Optional, Tuple
 
@@ -133,6 +134,7 @@ def add_cli_args(
                             help='disable logging statistics')
         parser.add_argument('--rope-scaling',
                             default=None,
+                            type=json.loads,
                             help='RoPE scaling configuration')
         return parser