From abed794dc3a122025ab4978fd7b4646aa8c6ae74 Mon Sep 17 00:00:00 2001
From: "Chang, Hui-Tang" <huitang.chang@instill.tech>
Date: Tue, 9 Jan 2024 14:18:04 +0800
Subject: [PATCH] feat(schema): add new instillFormat for chat history (#43)

Because

- We need to standardize the chat history format across all LLM tasks.

This commit

- add `structured/multi_modal_content` instillFormat
- add `structured/chat_messages` instillFormat
---
 schema.json | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 75 insertions(+)

diff --git a/schema.json b/schema.json
index ea82a15b..30d020ec 100644
--- a/schema.json
+++ b/schema.json
@@ -44,6 +44,43 @@
         "title": "Bounding Box",
         "type": "object"
       },
+      "chat_messages": {
+        "instillFormat": "structured/chat_messages",
+        "items": {
+          "instillFormat": "structured/chat_message",
+          "properties": {
+            "content": {
+              "$ref": "#/$defs/instill_types/multi_modal_content",
+              "description": "The message content",
+              "instillUIOrder": 1,
+              "title": "Content"
+            },
+            "metadata": {
+              "additionalProperties": true,
+              "description": "The message metadata",
+              "instillFormat": "semi-structured/object",
+              "instillUIOrder": 2,
+              "required": [],
+              "title": "Metadata",
+              "type": "object"
+            },
+            "role": {
+              "description": "The message role, i.e. 'system', 'user' or 'assistant'",
+              "instillFormat": "string",
+              "instillUIOrder": 0,
+              "title": "Role",
+              "type": "string"
+            }
+          },
+          "required": [
+            "role",
+            "content"
+          ],
+          "type": "object"
+        },
+        "title": "Chat Message",
+        "type": "array"
+      },
       "classification": {
         "additionalProperties": false,
         "properties": {
@@ -256,6 +293,44 @@
         ],
         "type": "object"
       },
+      "multi_modal_content": {
+        "instillFormat": "structured/multi_modal_content",
+        "items": {
+          "properties": {
+            "image_url": {
+              "properties": {
+                "url": {
+                  "description": "Either a URL of the image or the base64 encoded image data.",
+                  "type": "string"
+                }
+              },
+              "required": [
+                "url"
+              ],
+              "type": "object"
+            },
+            "text": {
+              "description": "The text content.",
+              "instillFormat": "string",
+              "type": "string"
+            },
+            "type": {
+              "description": "The type of the content part.",
+              "enum": [
+                "text",
+                "image_url"
+              ],
+              "instillFormat": "string",
+              "type": "string"
+            }
+          },
+          "required": [
+            "type"
+          ],
+          "type": "object"
+        },
+        "type": "array"
+      },
       "ocr": {
         "additionalProperties": false,
         "properties": {