From c725db0b07b41bee0f12981f956ed0f3cb3d73eb Mon Sep 17 00:00:00 2001 From: David Miguel Lozano Date: Fri, 26 Jan 2024 22:09:39 +0100 Subject: [PATCH] feat: Add support for shortening embeddings (#311) https://openai.com/blog/new-embedding-models-and-api-updates#native-support-for-shortening-embeddings --- .../schema/create_embedding_request.dart | 11 ++++++ .../src/generated/schema/schema.freezed.dart | 34 +++++++++++++++++-- .../lib/src/generated/schema/schema.g.dart | 2 ++ packages/openai_dart/oas/openapi_curated.yaml | 5 +++ .../openai_dart/oas/openapi_official.yaml | 5 +++ .../test/openai_client_embeddings_test.dart | 14 ++++++++ 6 files changed, 69 insertions(+), 2 deletions(-) diff --git a/packages/openai_dart/lib/src/generated/schema/create_embedding_request.dart b/packages/openai_dart/lib/src/generated/schema/create_embedding_request.dart index b5287cfa..fec9f621 100644 --- a/packages/openai_dart/lib/src/generated/schema/create_embedding_request.dart +++ b/packages/openai_dart/lib/src/generated/schema/create_embedding_request.dart @@ -26,6 +26,9 @@ class CreateEmbeddingRequest with _$CreateEmbeddingRequest { @Default(EmbeddingEncodingFormat.float) EmbeddingEncodingFormat encodingFormat, + /// The number of dimensions the resulting output embeddings should have. Only supported in `text-embedding-3` and later models. + @JsonKey(includeIfNull: false) int? dimensions, + /// A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse. [Learn more](https://platform.openai.com/docs/guides/safety-best-practices/end-user-ids). @JsonKey(includeIfNull: false) String? user, }) = _CreateEmbeddingRequest; @@ -39,11 +42,18 @@ class CreateEmbeddingRequest with _$CreateEmbeddingRequest { 'model', 'input', 'encoding_format', + 'dimensions', 'user' ]; + /// Validation constants + static const dimensionsMinValue = 1; + /// Perform validations on the schema property values String? validateSchema() { + if (dimensions != null && dimensions! < dimensionsMinValue) { + return "The value of 'dimensions' cannot be < $dimensionsMinValue"; + } return null; } @@ -53,6 +63,7 @@ class CreateEmbeddingRequest with _$CreateEmbeddingRequest { 'model': model, 'input': input, 'encoding_format': encodingFormat, + 'dimensions': dimensions, 'user': user, }; } diff --git a/packages/openai_dart/lib/src/generated/schema/schema.freezed.dart b/packages/openai_dart/lib/src/generated/schema/schema.freezed.dart index f2d39a05..ed7de3ef 100644 --- a/packages/openai_dart/lib/src/generated/schema/schema.freezed.dart +++ b/packages/openai_dart/lib/src/generated/schema/schema.freezed.dart @@ -10560,6 +10560,10 @@ mixin _$CreateEmbeddingRequest { EmbeddingEncodingFormat get encodingFormat => throw _privateConstructorUsedError; + /// The number of dimensions the resulting output embeddings should have. Only supported in `text-embedding-3` and later models. + @JsonKey(includeIfNull: false) + int? get dimensions => throw _privateConstructorUsedError; + /// A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse. [Learn more](https://platform.openai.com/docs/guides/safety-best-practices/end-user-ids). @JsonKey(includeIfNull: false) String? get user => throw _privateConstructorUsedError; @@ -10580,6 +10584,7 @@ abstract class $CreateEmbeddingRequestCopyWith<$Res> { {@_EmbeddingModelConverter() EmbeddingModel model, @_EmbeddingInputConverter() EmbeddingInput input, @JsonKey(name: 'encoding_format') EmbeddingEncodingFormat encodingFormat, + @JsonKey(includeIfNull: false) int? dimensions, @JsonKey(includeIfNull: false) String? user}); $EmbeddingModelCopyWith<$Res> get model; @@ -10603,6 +10608,7 @@ class _$CreateEmbeddingRequestCopyWithImpl<$Res, Object? model = null, Object? input = null, Object? encodingFormat = null, + Object? dimensions = freezed, Object? user = freezed, }) { return _then(_value.copyWith( @@ -10618,6 +10624,10 @@ class _$CreateEmbeddingRequestCopyWithImpl<$Res, ? _value.encodingFormat : encodingFormat // ignore: cast_nullable_to_non_nullable as EmbeddingEncodingFormat, + dimensions: freezed == dimensions + ? _value.dimensions + : dimensions // ignore: cast_nullable_to_non_nullable + as int?, user: freezed == user ? _value.user : user // ignore: cast_nullable_to_non_nullable @@ -10655,6 +10665,7 @@ abstract class _$$CreateEmbeddingRequestImplCopyWith<$Res> {@_EmbeddingModelConverter() EmbeddingModel model, @_EmbeddingInputConverter() EmbeddingInput input, @JsonKey(name: 'encoding_format') EmbeddingEncodingFormat encodingFormat, + @JsonKey(includeIfNull: false) int? dimensions, @JsonKey(includeIfNull: false) String? user}); @override @@ -10679,6 +10690,7 @@ class __$$CreateEmbeddingRequestImplCopyWithImpl<$Res> Object? model = null, Object? input = null, Object? encodingFormat = null, + Object? dimensions = freezed, Object? user = freezed, }) { return _then(_$CreateEmbeddingRequestImpl( @@ -10694,6 +10706,10 @@ class __$$CreateEmbeddingRequestImplCopyWithImpl<$Res> ? _value.encodingFormat : encodingFormat // ignore: cast_nullable_to_non_nullable as EmbeddingEncodingFormat, + dimensions: freezed == dimensions + ? _value.dimensions + : dimensions // ignore: cast_nullable_to_non_nullable + as int?, user: freezed == user ? _value.user : user // ignore: cast_nullable_to_non_nullable @@ -10710,6 +10726,7 @@ class _$CreateEmbeddingRequestImpl extends _CreateEmbeddingRequest { @_EmbeddingInputConverter() required this.input, @JsonKey(name: 'encoding_format') this.encodingFormat = EmbeddingEncodingFormat.float, + @JsonKey(includeIfNull: false) this.dimensions, @JsonKey(includeIfNull: false) this.user}) : super._(); @@ -10731,6 +10748,11 @@ class _$CreateEmbeddingRequestImpl extends _CreateEmbeddingRequest { @JsonKey(name: 'encoding_format') final EmbeddingEncodingFormat encodingFormat; + /// The number of dimensions the resulting output embeddings should have. Only supported in `text-embedding-3` and later models. + @override + @JsonKey(includeIfNull: false) + final int? dimensions; + /// A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse. [Learn more](https://platform.openai.com/docs/guides/safety-best-practices/end-user-ids). @override @JsonKey(includeIfNull: false) @@ -10738,7 +10760,7 @@ class _$CreateEmbeddingRequestImpl extends _CreateEmbeddingRequest { @override String toString() { - return 'CreateEmbeddingRequest(model: $model, input: $input, encodingFormat: $encodingFormat, user: $user)'; + return 'CreateEmbeddingRequest(model: $model, input: $input, encodingFormat: $encodingFormat, dimensions: $dimensions, user: $user)'; } @override @@ -10750,13 +10772,15 @@ class _$CreateEmbeddingRequestImpl extends _CreateEmbeddingRequest { (identical(other.input, input) || other.input == input) && (identical(other.encodingFormat, encodingFormat) || other.encodingFormat == encodingFormat) && + (identical(other.dimensions, dimensions) || + other.dimensions == dimensions) && (identical(other.user, user) || other.user == user)); } @JsonKey(ignore: true) @override int get hashCode => - Object.hash(runtimeType, model, input, encodingFormat, user); + Object.hash(runtimeType, model, input, encodingFormat, dimensions, user); @JsonKey(ignore: true) @override @@ -10779,6 +10803,7 @@ abstract class _CreateEmbeddingRequest extends CreateEmbeddingRequest { @_EmbeddingInputConverter() required final EmbeddingInput input, @JsonKey(name: 'encoding_format') final EmbeddingEncodingFormat encodingFormat, + @JsonKey(includeIfNull: false) final int? dimensions, @JsonKey(includeIfNull: false) final String? user}) = _$CreateEmbeddingRequestImpl; const _CreateEmbeddingRequest._() : super._(); @@ -10803,6 +10828,11 @@ abstract class _CreateEmbeddingRequest extends CreateEmbeddingRequest { EmbeddingEncodingFormat get encodingFormat; @override + /// The number of dimensions the resulting output embeddings should have. Only supported in `text-embedding-3` and later models. + @JsonKey(includeIfNull: false) + int? get dimensions; + @override + /// A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse. [Learn more](https://platform.openai.com/docs/guides/safety-best-practices/end-user-ids). @JsonKey(includeIfNull: false) String? get user; diff --git a/packages/openai_dart/lib/src/generated/schema/schema.g.dart b/packages/openai_dart/lib/src/generated/schema/schema.g.dart index 589e3a4a..efd7272b 100644 --- a/packages/openai_dart/lib/src/generated/schema/schema.g.dart +++ b/packages/openai_dart/lib/src/generated/schema/schema.g.dart @@ -1005,6 +1005,7 @@ _$CreateEmbeddingRequestImpl _$$CreateEmbeddingRequestImplFromJson( encodingFormat: $enumDecodeNullable( _$EmbeddingEncodingFormatEnumMap, json['encoding_format']) ?? EmbeddingEncodingFormat.float, + dimensions: json['dimensions'] as int?, user: json['user'] as String?, ); @@ -1023,6 +1024,7 @@ Map _$$CreateEmbeddingRequestImplToJson( } } + writeNotNull('dimensions', instance.dimensions); writeNotNull('user', instance.user); return val; } diff --git a/packages/openai_dart/oas/openapi_curated.yaml b/packages/openai_dart/oas/openapi_curated.yaml index 61a859ab..0baea463 100644 --- a/packages/openai_dart/oas/openapi_curated.yaml +++ b/packages/openai_dart/oas/openapi_curated.yaml @@ -2205,6 +2205,11 @@ components: default: "float" type: string enum: [ "float", "base64" ] + dimensions: + description: | + The number of dimensions the resulting output embeddings should have. Only supported in `text-embedding-3` and later models. + type: integer + minimum: 1 user: *end_user_param_configuration required: - model diff --git a/packages/openai_dart/oas/openapi_official.yaml b/packages/openai_dart/oas/openapi_official.yaml index b4be5a7b..61dbc555 100644 --- a/packages/openai_dart/oas/openapi_official.yaml +++ b/packages/openai_dart/oas/openapi_official.yaml @@ -6752,6 +6752,11 @@ components: default: "float" type: string enum: ["float", "base64"] + dimensions: + description: | + The number of dimensions the resulting output embeddings should have. Only supported in `text-embedding-3` and later models. + type: integer + minimum: 1 user: *end_user_param_configuration required: - model diff --git a/packages/openai_dart/test/openai_client_embeddings_test.dart b/packages/openai_dart/test/openai_client_embeddings_test.dart index fe98b9a4..6a200d54 100644 --- a/packages/openai_dart/test/openai_client_embeddings_test.dart +++ b/packages/openai_dart/test/openai_client_embeddings_test.dart @@ -59,5 +59,19 @@ void main() { expect(res.data.first.index, 0); expect(res.data.first.embeddingVectorBase64, isNotEmpty); }); + + test('Test shortening embeddings', () async { + const request = CreateEmbeddingRequest( + model: EmbeddingModel.model(EmbeddingModels.textEmbedding3Large), + input: EmbeddingInput.string( + 'The food was delicious and the waiter...', + ), + dimensions: 256, + ); + final res = await client.createEmbedding(request: request); + expect(res.data, hasLength(1)); + expect(res.data.first.index, 0); + expect(res.data.first.embeddingVector, hasLength(256)); + }); }); }