From e923b4f0198bbafc257b5f6031afe7fd9d200af0 Mon Sep 17 00:00:00 2001
From: takatost <takatost@gmail.com>
Date: Wed, 10 Apr 2024 11:05:35 +0800
Subject: [PATCH] feat: vision parameter support of OpenAI Compatible API

---
 .../openai_api_compatible/llm/llm.py          | 14 +++++++++-----
 .../openai_api_compatible.yaml                | 19 +++++++++++++++++++
 2 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py b/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py
index 04f5024207994f..58b5407c025bd9 100644
--- a/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py
+++ b/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py
@@ -167,23 +167,27 @@ def get_customizable_model_schema(self, model: str, credentials: dict) -> AIMode
         """
             generate custom model entities from credentials
         """
-        support_function_call = False
         features = []
+
         function_calling_type = credentials.get('function_calling_type', 'no_call')
         if function_calling_type == 'function_call':
-            features = [ModelFeature.TOOL_CALL]
-            support_function_call = True
+            features.append(ModelFeature.TOOL_CALL)
         endpoint_url = credentials["endpoint_url"]
         # if not endpoint_url.endswith('/'):
         #     endpoint_url += '/'
         # if 'https://api.openai.com/v1/' == endpoint_url:
-        #     features = [ModelFeature.STREAM_TOOL_CALL]
+        #     features.append(ModelFeature.STREAM_TOOL_CALL)
+
+        vision_support = credentials.get('vision_support', 'not_support')
+        if vision_support == 'support':
+            features.append(ModelFeature.VISION)
+
         entity = AIModelEntity(
             model=model,
             label=I18nObject(en_US=model),
             model_type=ModelType.LLM,
             fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
-            features=features if support_function_call else [],
+            features=features,
             model_properties={
                 ModelPropertyKey.CONTEXT_SIZE: int(credentials.get('context_size', "4096")),
                 ModelPropertyKey.MODE: credentials.get('mode'),
diff --git a/api/core/model_runtime/model_providers/openai_api_compatible/openai_api_compatible.yaml b/api/core/model_runtime/model_providers/openai_api_compatible/openai_api_compatible.yaml
index be99f7684ce3a9..cd53e149422fec 100644
--- a/api/core/model_runtime/model_providers/openai_api_compatible/openai_api_compatible.yaml
+++ b/api/core/model_runtime/model_providers/openai_api_compatible/openai_api_compatible.yaml
@@ -97,6 +97,25 @@ model_credential_schema:
           label:
             en_US: Not Support
             zh_Hans: 不支持
+    - variable: vision_support
+      show_on:
+        - variable: __model_type
+          value: llm
+      label:
+        zh_Hans: Vision 支持
+        en_US: Vision Support
+      type: select
+      required: false
+      default: no_support
+      options:
+        - value: support
+          label:
+            en_US: Support
+            zh_Hans: 支持
+        - value: no_support
+          label:
+            en_US: Not Support
+            zh_Hans: 不支持
     - variable: stream_mode_delimiter
       label:
         zh_Hans: 流模式返回结果的分隔符