feat(openai): support gpt-4o-2024-08-06 and structured output

ai/openai/v0/README.mdx

@@ -58,7 +58,7 @@ Provide text outputs in response to their inputs.
 | Temperature | `temperature` | number | What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.  We generally recommend altering this or `top-p` but not both.  |
 | N | `n` | integer | How many chat completion choices to generate for each input message. Note that you will be charged based on the number of generated tokens across all of the choices. Keep `n` as `1` to minimize costs. |
 | Max Tokens | `max-tokens` | integer | The maximum number of tokens that can be generated in the chat completion.  The total length of input tokens and generated tokens is limited by the model's context length. |
-| Response Format | `response-format` | object | An object specifying the format that the model must output. Used to enable JSON mode. |
+| Response Format (required) | `response-format` | object | Response format. |
 | Top P | `top-p` | number | An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.  We generally recommend altering this or `temperature` but not both.  |
 | Presence Penalty | `presence-penalty` | number | Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics. |
 | Frequency Penalty | `frequency-penalty` | number | Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. |

ai/openai/v0/config/tasks.json

@@ -309,6 +309,7 @@
             "gpt-4o-mini",
             "gpt-4o",
             "gpt-4o-2024-05-13",
+            "gpt-4o-2024-08-06",
             "gpt-4-turbo",
             "gpt-4-turbo-2024-04-09",
             "gpt-4-0125-preview",
@@ -329,7 +330,7 @@
             "gpt-3.5-turbo-0125",
             "gpt-3.5-turbo-16k-0613"
           ],
-          "example": "gpt-3.5-turbo",
+          "example": "gpt-4o",
           "type": "string",
           "instillAcceptFormats": [
             "string"
@@ -344,6 +345,7 @@
           "instillCredentialMap": {
             "values": [
               "gpt-4o",
+              "gpt-4o-2024-08-06",
               "gpt-4-turbo",
               "gpt-4-vision-preview",
               "gpt-4",
@@ -411,35 +413,68 @@
           "type": "string"
         },
         "response-format": {
-          "description": "An object specifying the format that the model must output. Used to enable JSON mode.",
+          "description": "Response format.",
           "instillUIOrder": 8,
-          "properties": {
-            "type": {
-              "default": "text",
-              "description": "Must be one of `text` or `json_object`.",
-              "enum": [
-                "text",
-                "json_object"
+          "additionalProperties": true,
+          "type": "object",
+          "required": [
+            "type"
+          ],
+          "oneOf": [
+            {
+              "properties": {
+                "type": {
+                  "type": "string",
+                  "const": "text"
+                }
+              },
+              "required": [
+                "type"
+              ],
+              "type": "object"
+            },
+            {
+              "properties": {
+                "type": {
+                  "type": "string",
+                  "const": "json_object"
+                }
+              },
+              "required": [
+                "type"
               ],
-              "type": "string",
-              "instillAcceptFormats": [
-                "string"
+              "type": "object"
+            },
+            {
+              "properties": {
+                "type": {
+                  "type": "string",
+                  "const": "json_schema"
+                },
+                "json-schema": {
+                  "description": "Set up the schema of the structured output.",
+                  "type": "string",
+                  "instillAcceptFormats": [
+                    "string"
+                  ],
+                  "title": "JSON Schema",
+                  "instillShortDescription": "Specify the schema of the structured output.",
+                  "instillUIOrder": 1,
+                  "instillUIMultiline": true,
+                  "instillUpstreamTypes": [
+                    "value",
+                    "reference"
+                  ]
+                }
+              },
+              "required": [
+                "type",
+                "json-schema"
               ],
-              "title": "Type",
-              "example": "text",
-              "instillShortDescription": "Setting to `json_object` enables JSON mode. ",
-              "instillUIOrder": 0,
-              "instillUpstreamTypes": [
-                "value",
-                "reference"
-              ]
+              "type": "object"
             }
-          },
-          "required": [
-            "type"
           ],
-          "title": "Response Format",
-          "type": "object"
+          "title": "Response Format"
         },
         "system-message": {
           "default": "You are a helpful assistant.",
@@ -501,7 +536,8 @@
       },
       "required": [
         "model",
-        "prompt"
+        "prompt",
+        "response-format"
       ],
       "title": "Input",
       "type": "object"

ai/openai/v0/main.go

@@ -174,7 +174,31 @@ func (e *execution) Execute(_ context.Context, inputs []*structpb.Struct) ([]*st
 
 			// workaround, the OpenAI service can not accept this param
 			if inputStruct.Model != "gpt-4-vision-preview" {
-				body.ResponseFormat = inputStruct.ResponseFormat
+				if inputStruct.ResponseFormat != nil {
+					body.ResponseFormat = &responseFormatReqStruct{
+						Type: inputStruct.ResponseFormat.Type,
+					}
+					if inputStruct.ResponseFormat.Type == "json_schema" {
+						if inputStruct.Model == "gpt-4o-mini" || inputStruct.Model == "gpt-4o-2024-08-06" {
+							sch := map[string]any{}
+							if inputStruct.ResponseFormat.JSONSchema != "" {
+								err = json.Unmarshal([]byte(inputStruct.ResponseFormat.JSONSchema), &sch)
+								if err != nil {
+									return nil, err
+								}
+								body.ResponseFormat = &responseFormatReqStruct{
+									Type:       inputStruct.ResponseFormat.Type,
+									JSONSchema: sch,
+								}
+							}
+
+						} else {
+							return nil, fmt.Errorf("this model doesn't support response format: json_schema")
+						}
+
+					}
+				}
+
 			}
 
 			resp := textCompletionResp{}

ai/openai/v0/text_generation.go

@@ -10,23 +10,24 @@ type textMessage struct {
 }
 
 type TextCompletionInput struct {
-	Prompt           string                `json:"prompt"`
-	Images           []string              `json:"images"`
-	ChatHistory      []*textMessage        `json:"chat-history,omitempty"`
-	Model            string                `json:"model"`
-	SystemMessage    *string               `json:"system-message,omitempty"`
-	Temperature      *float32              `json:"temperature,omitempty"`
-	TopP             *float32              `json:"top-p,omitempty"`
-	N                *int                  `json:"n,omitempty"`
-	Stop             *string               `json:"stop,omitempty"`
-	MaxTokens        *int                  `json:"max-tokens,omitempty"`
-	PresencePenalty  *float32              `json:"presence-penalty,omitempty"`
-	FrequencyPenalty *float32              `json:"frequency-penalty,omitempty"`
-	ResponseFormat   *responseFormatStruct `json:"response-format,omitempty"`
+	Prompt           string                     `json:"prompt"`
+	Images           []string                   `json:"images"`
+	ChatHistory      []*textMessage             `json:"chat-history,omitempty"`
+	Model            string                     `json:"model"`
+	SystemMessage    *string                    `json:"system-message,omitempty"`
+	Temperature      *float32                   `json:"temperature,omitempty"`
+	TopP             *float32                   `json:"top-p,omitempty"`
+	N                *int                       `json:"n,omitempty"`
+	Stop             *string                    `json:"stop,omitempty"`
+	MaxTokens        *int                       `json:"max-tokens,omitempty"`
+	PresencePenalty  *float32                   `json:"presence-penalty,omitempty"`
+	FrequencyPenalty *float32                   `json:"frequency-penalty,omitempty"`
+	ResponseFormat   *responseFormatInputStruct `json:"response-format,omitempty"`
 }
 
-type responseFormatStruct struct {
-	Type string `json:"type,omitempty"`
+type responseFormatInputStruct struct {
+	Type       string `json:"type,omitempty"`
+	JSONSchema string `json:"json-schema,omitempty"`
 }
 
 type TextCompletionOutput struct {
@@ -35,16 +36,21 @@ type TextCompletionOutput struct {
 }
 
 type textCompletionReq struct {
-	Model            string                `json:"model"`
-	Messages         []interface{}         `json:"messages"`
-	Temperature      *float32              `json:"temperature,omitempty"`
-	TopP             *float32              `json:"top_p,omitempty"`
-	N                *int                  `json:"n,omitempty"`
-	Stop             *string               `json:"stop,omitempty"`
-	MaxTokens        *int                  `json:"max_tokens,omitempty"`
-	PresencePenalty  *float32              `json:"presence_penalty,omitempty"`
-	FrequencyPenalty *float32              `json:"frequency_penalty,omitempty"`
-	ResponseFormat   *responseFormatStruct `json:"response_format,omitempty"`
+	Model            string                   `json:"model"`
+	Messages         []interface{}            `json:"messages"`
+	Temperature      *float32                 `json:"temperature,omitempty"`
+	TopP             *float32                 `json:"top_p,omitempty"`
+	N                *int                     `json:"n,omitempty"`
+	Stop             *string                  `json:"stop,omitempty"`
+	MaxTokens        *int                     `json:"max_tokens,omitempty"`
+	PresencePenalty  *float32                 `json:"presence_penalty,omitempty"`
+	FrequencyPenalty *float32                 `json:"frequency_penalty,omitempty"`
+	ResponseFormat   *responseFormatReqStruct `json:"response_format,omitempty"`
+}
+
+type responseFormatReqStruct struct {
+	Type       string         `json:"type,omitempty"`
+	JSONSchema map[string]any `json:"json_schema,omitempty"`
 }
 
 type multiModalMessage struct {

application/hubspot/v0/deal_test.go

@@ -89,7 +89,7 @@ func TestComponent_ExecuteGetDealTask(t *testing.T) {
 			client:             createMockClient(),
 		}
 		e.execute = e.GetDeal
-		
+
 
 		pbInput, err := structpb.NewStruct(map[string]any{
 			"deal-id": tc.input,
@@ -139,7 +139,7 @@ func TestComponent_ExecuteCreateDealTask(t *testing.T) {
 			client:             createMockClient(),
 		}
 		e.execute = e.CreateDeal
-		
+
 
 		pbInput, err := base.ConvertToStructpb(tc.inputDeal)
 

application/hubspot/v0/thread_test.go

@@ -112,7 +112,7 @@ func TestComponent_ExecuteGetThreadTask(t *testing.T) {
 			client:             createMockClient(),
 		}
 		e.execute = e.GetThread
-		
+
 		pbInput, err := structpb.NewStruct(map[string]any{
 			"thread-id": tc.input,
 		})
@@ -165,7 +165,7 @@ func TestComponent_ExecuteInsertMessageTask(t *testing.T) {
 			client:             createMockClient(),
 		}
 		e.execute = e.InsertMessage
-		
+
 
 		pbInput, err := base.ConvertToStructpb(tc.input)
 

application/hubspot/v0/ticket_test.go

@@ -80,7 +80,7 @@ func TestComponent_ExecuteGetTicketTask(t *testing.T) {
 			client:             createMockClient(),
 		}
 		e.execute = e.GetTicket
-		
+
 
 		pbInput, err := structpb.NewStruct(map[string]any{
 			"ticket-id": tc.input,
@@ -132,7 +132,7 @@ func TestComponent_ExecuteCreateTicketTask(t *testing.T) {
 			client:             createMockClient(),
 		}
 		e.execute = e.CreateTicket
-		
+
 
 		pbInput, err := base.ConvertToStructpb(tc.inputTicket)
 

application/slack/v0/component_test.go

@@ -146,7 +146,7 @@ func TestComponent_ExecuteWriteTask(t *testing.T) {
 				client:             &MockSlackClient{},
 			}
 			e.execute = e.sendMessage
-			
+
 
 			pbIn, err := base.ConvertToStructpb(tc.input)
 			c.Assert(err, qt.IsNil)
@@ -230,7 +230,7 @@ func TestComponent_ExecuteReadTask(t *testing.T) {
 				client:             &MockSlackClient{},
 			}
 			e.execute = e.readMessage
-			
+
 
 			pbIn, err := base.ConvertToStructpb(tc.input)
 			c.Assert(err, qt.IsNil)