From cf41ed7db8cbfbdcf3146a3e05ac59de86bb4a86 Mon Sep 17 00:00:00 2001 From: Steve Lorello <42971704+slorello89@users.noreply.github.com> Date: Tue, 5 Dec 2023 08:27:32 -0500 Subject: [PATCH] Vector Search and Semantic Caching (#417) --- .gitattributes | 2 + .github/workflows/dotnet-core.yml | 2 + .gitignore | 2 + README.md | 104 +++++ Redis.OM.sln | 112 +++--- dockerfile | 5 +- src/Redis.OM.POC/RedisCommands.cs | 4 +- src/Redis.OM.POC/RedisConnection.cs | 12 +- src/Redis.OM.POC/RedisHash.cs | 2 +- src/Redis.OM.POC/XRangeResponse.cs | 15 +- .../AllMiniLML6V2Tokenizer.cs | 37 ++ .../LICENSE | 35 ++ .../Redis.OM.Vectorizers.AllMiniLML6V2.csproj | 49 +++ .../RedisConnectionProviderExtensions.cs | 32 ++ .../Resources/model.onnx | 3 + .../Resources/vocab.txt | 3 + .../SentenceVectorizer.cs | 188 +++++++++ .../SentenceVectorizerAttribute.cs | 22 ++ .../Tokenizers/CasedTokenizer.cs | 14 + .../Tokenizers/StringExtensions.cs | 25 ++ .../Tokenizers/TokenizerBase.cs | 117 ++++++ .../Tokenizers/Tokens.cs | 10 + .../Tokenizers/UncasedTokenizer.cs | 15 + .../DnnImageModelSelectorExtensions.cs | 50 +++ .../ImageModelObjects.cs | 26 ++ .../ImageVectorizer.cs | 108 +++++ .../ImageVectorizerAttribute.cs | 22 ++ .../Redis.OM.Vectorizers.Resnet18.csproj | 61 +++ .../Redis.OM.Vectorizers.Resnet18.props | 8 + .../Resources/ResNet18Onnx/ResNet18.onnx | 3 + .../ResNetPrepOnnx/ResNetPreprocess.onnx | 3 + .../AzureOpenAIVectorizer.cs | 77 ++++ .../AzureOpenAIVectorizerAttribute.cs | 48 +++ src/Redis.OM.Vectorizers/Configuration.cs | 95 +++++ .../HuggingFaceVectorizer.cs | 83 ++++ .../HuggingFaceVectorizerAttribute.cs | 81 ++++ src/Redis.OM.Vectorizers/OpenAIVectorizer.cs | 77 ++++ .../OpenAIVectorizerAttribute.cs | 47 +++ .../Redis.OM.Vectorizers.csproj | 44 +++ .../RedisConnectionProviderExtensions.cs | 87 ++++ src/Redis.OM.Vectorizers/RedisOMHttpUtil.cs | 9 + .../AggregationPredicates/QueryPredicate.cs | 4 +- .../Common/ExpressionParserUtilities.cs | 89 ++++- src/Redis.OM/Common/ExpressionTranslator.cs | 102 +++-- src/Redis.OM/Contracts/IRedisConnection.cs | 8 +- src/Redis.OM/Contracts/IRedisHydrateable.cs | 2 +- src/Redis.OM/Contracts/ISemanticCache.cs | 93 +++++ src/Redis.OM/Contracts/IVectorizer.cs | 28 ++ src/Redis.OM/Modeling/IndexedAttribute.cs | 72 ++++ .../Modeling/RedisCollectionStateManager.cs | 10 +- src/Redis.OM/Modeling/RedisSchemaField.cs | 96 ++++- src/Redis.OM/Modeling/SearchFieldType.cs | 5 + .../Modeling/Vectors/DistanceMetric.cs | 48 +++ .../Modeling/Vectors/DoubleVectorizer.cs | 29 ++ .../Vectors/DoubleVectorizerAttribute.cs | 40 ++ .../Modeling/Vectors/FloatVectorizer.cs | 30 ++ .../Vectors/FloatVectorizerAttribute.cs | 39 ++ .../Modeling/Vectors/JsonScoreConverter.cs | 27 ++ .../Modeling/Vectors/VectorAlgorithm.cs | 43 ++ .../Modeling/Vectors/VectorJsonConverter.cs | 184 +++++++++ src/Redis.OM/Modeling/Vectors/VectorResult.cs | 30 ++ .../Modeling/Vectors/VectorScoreField.cs | 18 + src/Redis.OM/Modeling/Vectors/VectorScores.cs | 48 +++ src/Redis.OM/Modeling/Vectors/VectorType.cs | 42 ++ src/Redis.OM/Modeling/Vectors/VectorUtils.cs | 224 +++++++++++ .../Modeling/Vectors/VectorizerAttribute.cs | 60 +++ src/Redis.OM/RediSearchCommands.cs | 50 +++ src/Redis.OM/Redis.OM.csproj | 8 +- src/Redis.OM/RedisCommands.cs | 48 +-- src/Redis.OM/RedisConnection.cs | 18 +- src/Redis.OM/RedisObjectHandler.cs | 253 +++++++++--- src/Redis.OM/RedisReply.cs | 39 +- src/Redis.OM/SearchExtensions.cs | 49 +++ .../Searching/Query/NearestNeighbors.cs | 36 ++ src/Redis.OM/Searching/Query/RedisQuery.cs | 44 ++- src/Redis.OM/Searching/RedisCollection.cs | 2 +- .../Searching/RedisCollectionEnumerator.cs | 2 +- src/Redis.OM/Searching/SearchResponse.cs | 6 +- src/Redis.OM/SemanticCache.cs | 248 ++++++++++++ src/Redis.OM/SemanticCacheResponse.cs | 50 +++ src/Redis.OM/Vector.cs | 96 +++++ src/Redis.OM/Vectors.cs | 60 +++ src/Redis.OM/stylecop.ruleset | 1 + test/Redis.OM.Unit.Tests/Address.cs | 1 - .../BasicTypeWithGeoLoc.cs | 1 + .../Redis.OM.Unit.Tests/ConfigurationTests.cs | 1 - test/Redis.OM.Unit.Tests/CoreTests.cs | 1 - test/Redis.OM.Unit.Tests/GeoLocTests.cs | 2 +- .../RediSearchTests/AggregationSetTests.cs | 76 ++-- .../RediSearchTests/Person.cs | 1 - .../RediSearchTests/SearchTests.cs | 320 +++++++-------- .../VectorTests/HuggingFaceVectors.cs | 24 ++ .../VectorTests/ObjectWithVector.cs | 53 +++ .../VectorTests/OpenAICompletionResponse.cs | 24 ++ .../VectorTests/OpenAIVectors.cs | 24 ++ .../VectorTests/SemanticCachingTests.cs | 65 +++ .../VectorTests/SimpleVectorizerAttribute.cs | 40 ++ .../VectorTests/VectorFunctionalTests.cs | 371 ++++++++++++++++++ .../VectorTests/VectorTests.cs | 215 ++++++++++ .../Redis.OM.Unit.Tests.csproj | 3 +- .../RedisSetupCollection.cs | 22 +- .../SearchJsonTests/RedisJsonIndexTests.cs | 1 - .../DocWithVectors.cs | 23 ++ .../Redis.OM.Vectorizer.Tests/GlobalUsings.cs | 1 + .../Redis.OM.Vectorizer.Tests.csproj | 38 ++ .../VectorizerFunctionalTests.cs | 45 +++ test/Redis.OM.Vectorizer.Tests/hal.jpg | Bin 0 -> 33140 bytes 107 files changed, 5008 insertions(+), 464 deletions(-) create mode 100644 .gitattributes create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/AllMiniLML6V2Tokenizer.cs create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/LICENSE create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/Redis.OM.Vectorizers.AllMiniLML6V2.csproj create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/RedisConnectionProviderExtensions.cs create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/Resources/model.onnx create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/Resources/vocab.txt create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizer.cs create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizerAttribute.cs create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/CasedTokenizer.cs create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/StringExtensions.cs create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/TokenizerBase.cs create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/Tokens.cs create mode 100644 src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/UncasedTokenizer.cs create mode 100644 src/Redis.OM.Vectorizers.Resnet18/DnnImageModelSelectorExtensions.cs create mode 100644 src/Redis.OM.Vectorizers.Resnet18/ImageModelObjects.cs create mode 100644 src/Redis.OM.Vectorizers.Resnet18/ImageVectorizer.cs create mode 100644 src/Redis.OM.Vectorizers.Resnet18/ImageVectorizerAttribute.cs create mode 100644 src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.csproj create mode 100644 src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.props create mode 100755 src/Redis.OM.Vectorizers.Resnet18/Resources/ResNet18Onnx/ResNet18.onnx create mode 100755 src/Redis.OM.Vectorizers.Resnet18/Resources/ResNetPrepOnnx/ResNetPreprocess.onnx create mode 100644 src/Redis.OM.Vectorizers/AzureOpenAIVectorizer.cs create mode 100644 src/Redis.OM.Vectorizers/AzureOpenAIVectorizerAttribute.cs create mode 100644 src/Redis.OM.Vectorizers/Configuration.cs create mode 100644 src/Redis.OM.Vectorizers/HuggingFaceVectorizer.cs create mode 100644 src/Redis.OM.Vectorizers/HuggingFaceVectorizerAttribute.cs create mode 100644 src/Redis.OM.Vectorizers/OpenAIVectorizer.cs create mode 100644 src/Redis.OM.Vectorizers/OpenAIVectorizerAttribute.cs create mode 100644 src/Redis.OM.Vectorizers/Redis.OM.Vectorizers.csproj create mode 100644 src/Redis.OM.Vectorizers/RedisConnectionProviderExtensions.cs create mode 100644 src/Redis.OM.Vectorizers/RedisOMHttpUtil.cs create mode 100644 src/Redis.OM/Contracts/ISemanticCache.cs create mode 100644 src/Redis.OM/Contracts/IVectorizer.cs create mode 100644 src/Redis.OM/Modeling/Vectors/DistanceMetric.cs create mode 100644 src/Redis.OM/Modeling/Vectors/DoubleVectorizer.cs create mode 100644 src/Redis.OM/Modeling/Vectors/DoubleVectorizerAttribute.cs create mode 100644 src/Redis.OM/Modeling/Vectors/FloatVectorizer.cs create mode 100644 src/Redis.OM/Modeling/Vectors/FloatVectorizerAttribute.cs create mode 100644 src/Redis.OM/Modeling/Vectors/JsonScoreConverter.cs create mode 100644 src/Redis.OM/Modeling/Vectors/VectorAlgorithm.cs create mode 100644 src/Redis.OM/Modeling/Vectors/VectorJsonConverter.cs create mode 100644 src/Redis.OM/Modeling/Vectors/VectorResult.cs create mode 100644 src/Redis.OM/Modeling/Vectors/VectorScoreField.cs create mode 100644 src/Redis.OM/Modeling/Vectors/VectorScores.cs create mode 100644 src/Redis.OM/Modeling/Vectors/VectorType.cs create mode 100644 src/Redis.OM/Modeling/Vectors/VectorUtils.cs create mode 100644 src/Redis.OM/Modeling/Vectors/VectorizerAttribute.cs create mode 100644 src/Redis.OM/Searching/Query/NearestNeighbors.cs create mode 100644 src/Redis.OM/SemanticCache.cs create mode 100644 src/Redis.OM/SemanticCacheResponse.cs create mode 100644 src/Redis.OM/Vector.cs create mode 100644 src/Redis.OM/Vectors.cs create mode 100644 test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/HuggingFaceVectors.cs create mode 100644 test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs create mode 100644 test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAICompletionResponse.cs create mode 100644 test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs create mode 100644 test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs create mode 100644 test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SimpleVectorizerAttribute.cs create mode 100644 test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs create mode 100644 test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs create mode 100644 test/Redis.OM.Vectorizer.Tests/DocWithVectors.cs create mode 100644 test/Redis.OM.Vectorizer.Tests/GlobalUsings.cs create mode 100644 test/Redis.OM.Vectorizer.Tests/Redis.OM.Vectorizer.Tests.csproj create mode 100644 test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs create mode 100644 test/Redis.OM.Vectorizer.Tests/hal.jpg diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..c3339f17 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +*.onnx filter=lfs diff=lfs merge=lfs -text +src/Redis.OM.Vectorizers.AllMiniLML6V2/Resources/vocab.txt filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/dotnet-core.yml b/.github/workflows/dotnet-core.yml index a1a8acca..731e3199 100644 --- a/.github/workflows/dotnet-core.yml +++ b/.github/workflows/dotnet-core.yml @@ -11,5 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + with: + lfs: true - name: execute run: docker-compose -f ./docker/docker-compose.yaml run dotnet \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8279cee0..c0196859 100644 --- a/.gitignore +++ b/.gitignore @@ -388,3 +388,5 @@ FodyWeavers.xsd # JetBrains Rider .idea/ *.sln.iml + +test/Redis.OM.Unit.Tests/appsettings.json.local \ No newline at end of file diff --git a/README.md b/README.md index 9a342df1..97ff9dc3 100644 --- a/README.md +++ b/README.md @@ -288,6 +288,110 @@ customers.Where(x => x.LastName == "Bond" && x.FirstName == "James"); customers.Where(x=>x.NickNames.Contains("Jim")); ``` +### Vectors + +Redis OM .NET also supports storing and querying Vectors stored in Redis. + +A `Vector` is a representation of an object that can be transformed into a vector by a Vectorizer. + +A `VectorizerAttribute` is the abstract class you use to decorate your Vector fields, it is responsible for defining the logic to convert the object's that `Vector` is a container for into actual vector embeddings needed. In the package `Redis.OM.Vectorizers` we provide vectorizers for HuggingFace, OpenAI, and AzureOpenAI to allow you to easily integrate them into your workflows. + +#### Define a Vector in your Model. + +To define a vector in your model, simply decorate a `Vector` field with an `Indexed` attribute which defines the algorithm and algorithmic parameters and a `Vectorizer` attribute which defines the shape of the vectors, (in this case we'll use OpenAI): + +```cs +[Document(StorageType = StorageType.Json)] +public class OpenAICompletionResponse +{ + [RedisIdField] + public string Id { get; set; } + + [Indexed(DistanceMetric = DistanceMetric.COSINE, Algorithm = VectorAlgorithm.HNSW, M = 16)] + [OpenAIVectorizer] + public Vector Prompt { get; set; } + + public string Response { get; set; } + + [Indexed] + public string Language { get; set; } + + [Indexed] + public DateTime TimeStamp { get; set; } +} +``` + +#### Insert Vectors into Redis + +With the vector defined in our model, all we need to do is create Vectors of the generic type, and insert them with our model. Using our `RedisCollection`, you can do this by simply using `Insert`: + +```cs +var collection = _provider.RedisCollection(); +var completionResult = new OpenAICompletionResponse +{ + Language = "en_us", + Prompt = Vector.Of("What is the Capital of France?"), + Response = "Paris", + TimeStamp = DateTime.Now - TimeSpan.FromHours(3) +}; +collection.Insert(completionResult); +``` + +The Vectorizer will manage the embedding generation for you without you having to intervene. + +#### Query Vectors in Redis + +To query vector fields in Redis, all you need to do is use the `VectorRange` method on a vector within our normal LINQ queries, and/or use the `NearestNeighbors` with whatever other filters you want to use, here's some examples: + +```cs +var prompt = "What really is the Capital of France?"; + +// simple vector range, find first within .15 +var result = collection.First(x => x.Prompt.VectorRange(prompt, .15)); + +// simple nearest neighbors query, finds first nearest neighbor +result = collection.NearestNeighbors(x => x.Prompt, 1, prompt).First(); + +// hybrid query, pre-filters result set for english responses, then runs a nearest neighbors search. +result = collection.Where(x=>x.Language == "en_us").NearestNeighbors(x => x.Prompt, 1, prompt).First(); + +// hybrid query, pre-filters responses newer than 4 hours, and finds first result within .15 +var ts = DateTimeOffset.Now - TimeSpan.FromHours(4); +result = collection.First(x=>x.TimeStamp > ts && x.Prompt.VectorRange(prompt, .15)); +``` + +#### What Happens to the Embeddings? + +With Redis OM, the embeddings can be completely transparent to you, they are generated and bound to the `Vector` when you query/insert your vectors. If however you needed your embedding after the insertion/Query, they are available at `Vector.Embedding`, and be queried either as the raw bytes, as an array of doubles or as an array of floats (depending on your vectorizer). + +#### Configuration + +The Vectorizers provided by the `Redis.OM.Vectorizers` package have some configuration parameters that it will pull in either from your `appsettings.json` file, or your environment variables (with your appsettings taking precedence). + +| Configuration Parameter | Description | +|-------------------------------- |-----------------------------------------------| +| REDIS_OM_HF_TOKEN | HuggingFace Authorization token. | +| REDIS_OM_OAI_TOKEN | OpenAI Authorization token | +| REDIS_OM_OAI_API_URL | OpenAI URL | +| REDIS_OM_AZURE_OAI_TOKEN | Azure OpenAI api key | +| REDIS_OM_AZURE_OAI_RESOURCE_NAME | Azure resource name | +| REDIS_OM_AZURE_OAI_DEPLOYMENT_NAME | Azure deployment | + +### Semantic Caching + +Redis OM also provides the ability to use Semantic Caching, as well as providers for OpenAI, HuggingFace, and Azure OpenAI to perform semantic caching. To use a Semantic Cache, simply pull one out of the RedisConnectionProvider and use `Store` to insert items, and `GetSimilar` to retrieve items. For example: + +```cs +var cache = _provider.OpenAISemanticCache(token, threshold: .15); +cache.Store("What is the capital of France?", "Paris"); +var res = cache.GetSimilar("What really is the capital of France?").First(); +``` + +### ML.NET Based Vectorizers + +We also provide the packages `Redis.OM.Vectorizers.ResNet18` and `Redis.OM.Vectorizers.AllMiniLML6V2` which have embedded models / ML Pipelines in them to +allow you to easily Vectorize Images and Sentences respectively without the need to depend on an external API. + ### 🖩 Aggregations We can also run aggregations on the customer object, again using expressions in LINQ: diff --git a/Redis.OM.sln b/Redis.OM.sln index 9846fb54..266ef612 100644 --- a/Redis.OM.sln +++ b/Redis.OM.sln @@ -10,13 +10,15 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Redis.OM.POC", "src\Redis.O EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Redis.OM.Unit.Tests", "test\Redis.OM.Unit.Tests\Redis.OM.Unit.Tests.csproj", "{570BF479-BCF4-4D1B-A702-2234CA0A3E7D}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Redis.OM.Test.ConsoleApp", "test\Redis.OM.Test.ConsoleApp\Redis.OM.Test.ConsoleApp.csproj", "{FC7E5ED3-51AC-45E6-A178-6287C9227975}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.Vectorizers", "src\Redis.OM.Vectorizers\Redis.OM.Vectorizers.csproj", "{4B9F4623-3126-48B7-B690-F28F702A4717}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.Analyzer", "src\Redis.OM.Analyzer\Redis.OM.Analyzer.csproj", "{44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Vectorizers", "Vectorizers", "{452DC80B-8195-44E8-A376-C246619492A8}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.AspNetCore", "src\Redis.OM.AspNetCore\Redis.OM.AspNetCore.csproj", "{230ED77D-D625-43BC-94D6-6BDBACEA3EAF}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.Vectorizers.AllMiniLML6V2", "src\Redis.OM.Vectorizers.AllMiniLML6V2\Redis.OM.Vectorizers.AllMiniLML6V2.csproj", "{081DEE32-9B26-44C6-B377-456E862D3813}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.Test.AspDotnetCore", "test\Redis.OM.Test.AspDotnetCore\Redis.OM.Test.AspDotnetCore.csproj", "{3F609AB2-1492-4EBE-9FF2-B47829307E9E}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.Vectorizer.Tests", "test\Redis.OM.Vectorizer.Tests\Redis.OM.Vectorizer.Tests.csproj", "{7C3E1D79-408C-45E9-931C-12195DFA268D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redis.OM.Vectorizers.Resnet18", "src\Redis.OM.Vectorizers.Resnet18\Redis.OM.Vectorizers.Resnet18.csproj", "{FE22706B-9A28-4045-9581-2058F32C4193}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -64,54 +66,54 @@ Global {570BF479-BCF4-4D1B-A702-2234CA0A3E7D}.Release|x64.Build.0 = Release|Any CPU {570BF479-BCF4-4D1B-A702-2234CA0A3E7D}.Release|x86.ActiveCfg = Release|Any CPU {570BF479-BCF4-4D1B-A702-2234CA0A3E7D}.Release|x86.Build.0 = Release|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Debug|Any CPU.Build.0 = Debug|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Debug|x64.ActiveCfg = Debug|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Debug|x64.Build.0 = Debug|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Debug|x86.ActiveCfg = Debug|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Debug|x86.Build.0 = Debug|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Release|Any CPU.ActiveCfg = Release|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Release|Any CPU.Build.0 = Release|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Release|x64.ActiveCfg = Release|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Release|x64.Build.0 = Release|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Release|x86.ActiveCfg = Release|Any CPU - {FC7E5ED3-51AC-45E6-A178-6287C9227975}.Release|x86.Build.0 = Release|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Debug|Any CPU.Build.0 = Debug|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Debug|x64.ActiveCfg = Debug|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Debug|x64.Build.0 = Debug|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Debug|x86.ActiveCfg = Debug|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Debug|x86.Build.0 = Debug|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Release|Any CPU.ActiveCfg = Release|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Release|Any CPU.Build.0 = Release|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Release|x64.ActiveCfg = Release|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Release|x64.Build.0 = Release|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Release|x86.ActiveCfg = Release|Any CPU - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1}.Release|x86.Build.0 = Release|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Debug|Any CPU.Build.0 = Debug|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Debug|x64.ActiveCfg = Debug|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Debug|x64.Build.0 = Debug|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Debug|x86.ActiveCfg = Debug|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Debug|x86.Build.0 = Debug|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Release|Any CPU.ActiveCfg = Release|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Release|Any CPU.Build.0 = Release|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Release|x64.ActiveCfg = Release|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Release|x64.Build.0 = Release|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Release|x86.ActiveCfg = Release|Any CPU - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF}.Release|x86.Build.0 = Release|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Debug|Any CPU.Build.0 = Debug|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Debug|x64.ActiveCfg = Debug|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Debug|x64.Build.0 = Debug|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Debug|x86.ActiveCfg = Debug|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Debug|x86.Build.0 = Debug|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Release|Any CPU.ActiveCfg = Release|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Release|Any CPU.Build.0 = Release|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Release|x64.ActiveCfg = Release|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Release|x64.Build.0 = Release|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Release|x86.ActiveCfg = Release|Any CPU - {3F609AB2-1492-4EBE-9FF2-B47829307E9E}.Release|x86.Build.0 = Release|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Debug|x64.ActiveCfg = Debug|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Debug|x64.Build.0 = Debug|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Debug|x86.ActiveCfg = Debug|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Debug|x86.Build.0 = Debug|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Release|Any CPU.Build.0 = Release|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Release|x64.ActiveCfg = Release|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Release|x64.Build.0 = Release|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Release|x86.ActiveCfg = Release|Any CPU + {4B9F4623-3126-48B7-B690-F28F702A4717}.Release|x86.Build.0 = Release|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Debug|Any CPU.Build.0 = Debug|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Debug|x64.ActiveCfg = Debug|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Debug|x64.Build.0 = Debug|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Debug|x86.ActiveCfg = Debug|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Debug|x86.Build.0 = Debug|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Release|Any CPU.ActiveCfg = Release|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Release|Any CPU.Build.0 = Release|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Release|x64.ActiveCfg = Release|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Release|x64.Build.0 = Release|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Release|x86.ActiveCfg = Release|Any CPU + {081DEE32-9B26-44C6-B377-456E862D3813}.Release|x86.Build.0 = Release|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Debug|x64.ActiveCfg = Debug|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Debug|x64.Build.0 = Debug|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Debug|x86.ActiveCfg = Debug|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Debug|x86.Build.0 = Debug|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Release|Any CPU.Build.0 = Release|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Release|x64.ActiveCfg = Release|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Release|x64.Build.0 = Release|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Release|x86.ActiveCfg = Release|Any CPU + {7C3E1D79-408C-45E9-931C-12195DFA268D}.Release|x86.Build.0 = Release|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Debug|x64.ActiveCfg = Debug|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Debug|x64.Build.0 = Debug|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Debug|x86.ActiveCfg = Debug|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Debug|x86.Build.0 = Debug|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Release|Any CPU.Build.0 = Release|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Release|x64.ActiveCfg = Release|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Release|x64.Build.0 = Release|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Release|x86.ActiveCfg = Release|Any CPU + {FE22706B-9A28-4045-9581-2058F32C4193}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -119,8 +121,10 @@ Global GlobalSection(NestedProjects) = preSolution {7994382C-28EF-4F55-9B6D-810D35247816} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} {E3A31119-E4F1-4793-B5C2-ED2D51502B01} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} - {44FAD9BB-C6DF-402C-BCE7-64E7C674F8D1} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} - {230ED77D-D625-43BC-94D6-6BDBACEA3EAF} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} + {452DC80B-8195-44E8-A376-C246619492A8} = {8D9ECCFF-E022-4B68-BB43-228CEA248DEC} + {4B9F4623-3126-48B7-B690-F28F702A4717} = {452DC80B-8195-44E8-A376-C246619492A8} + {081DEE32-9B26-44C6-B377-456E862D3813} = {452DC80B-8195-44E8-A376-C246619492A8} + {FE22706B-9A28-4045-9581-2058F32C4193} = {452DC80B-8195-44E8-A376-C246619492A8} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {E5752441-184B-4F17-BAD0-93823AC68607} diff --git a/dockerfile b/dockerfile index c2e45881..3cc5893c 100644 --- a/dockerfile +++ b/dockerfile @@ -1,5 +1,4 @@ -FROM mcr.microsoft.com/dotnet/sdk:6.0 - +FROM mcr.microsoft.com/dotnet/sdk:7.0 WORKDIR /app ADD . /app @@ -7,4 +6,4 @@ ADD . /app RUN ls /app RUN dotnet restore /app/Redis.OM.sln -ENTRYPOINT ["dotnet","test"] \ No newline at end of file +ENTRYPOINT ["dotnet", "test", "--framework", "net7.0" ] \ No newline at end of file diff --git a/src/Redis.OM.POC/RedisCommands.cs b/src/Redis.OM.POC/RedisCommands.cs index 10d6cc33..19a608e0 100644 --- a/src/Redis.OM.POC/RedisCommands.cs +++ b/src/Redis.OM.POC/RedisCommands.cs @@ -565,7 +565,7 @@ public static long XAck(this IRedisConnection connection, string streamId, strin public static async Task XAddAsync(this IRedisConnection connection, string streamId, object message, string messageId = "*", int maxLen = -1, string minId = "", bool trimApprox = true, bool makeStream = true) { var kvps = message.BuildHashSet(); - var args = new List { streamId }; + var args = new List { streamId }; if (!makeStream) { args.Add("NOMKSTREAM"); @@ -611,7 +611,7 @@ public static long XAck(this IRedisConnection connection, string streamId, strin public static string? XAdd(this IRedisConnection connection, string streamId, object message, string messageId = "*", int maxLen = -1, string minId = "", bool trimApprox = true, bool makeStream = true) { var kvps = message.BuildHashSet(); - var args = new List { streamId }; + var args = new List { streamId }; if (!makeStream) { args.Add("NOMKSTREAM"); diff --git a/src/Redis.OM.POC/RedisConnection.cs b/src/Redis.OM.POC/RedisConnection.cs index 0a1859bb..6ce3df34 100644 --- a/src/Redis.OM.POC/RedisConnection.cs +++ b/src/Redis.OM.POC/RedisConnection.cs @@ -42,19 +42,19 @@ public RedisList GetList(string listName, uint chunkSize = 100) return new RedisList(this, listName, chunkSize); } - public RedisReply Execute(string command, params string[] args) + public RedisReply Execute(string command, params object[] args) { - var commandBytes = RespHelper.BuildCommand(command, args); + var commandBytes = RespHelper.BuildCommand(command, args.Select(x=>x.ToString()).ToArray()); _socket.Send(commandBytes); return RespHelper.GetNextReplyFromSocket(_socket); } - public async Task ExecuteAsync(string command, params string[] args) + public async Task ExecuteAsync(string command, params object[] args) { await _semaphoreSlim.WaitAsync(); try { - var commandBytes = new ArraySegment(RespHelper.BuildCommand(command, args)); + var commandBytes = new ArraySegment(RespHelper.BuildCommand(command, args.Select(x=>x.ToString()).ToArray())); await _socket.SendAsync(commandBytes, SocketFlags.None); return await RespHelper.GetNextReplyFromSocketAsync(_socket); } @@ -66,7 +66,7 @@ public async Task ExecuteAsync(string command, params string[] args) } /// - public RedisReply[] ExecuteInTransaction(Tuple[] commandArgsTuples) + public RedisReply[] ExecuteInTransaction(Tuple[] commandArgsTuples) { var transaction = _db.CreateTransaction(); var tasks = new List>(); @@ -81,7 +81,7 @@ public RedisReply[] ExecuteInTransaction(Tuple[] commandArgsTu } /// - public async Task ExecuteInTransactionAsync(Tuple[] commandArgsTuples) + public async Task ExecuteInTransactionAsync(Tuple[] commandArgsTuples) { var transaction = _db.CreateTransaction(); var tasks = new List>(); diff --git a/src/Redis.OM.POC/RedisHash.cs b/src/Redis.OM.POC/RedisHash.cs index fed046e4..c4919a6d 100644 --- a/src/Redis.OM.POC/RedisHash.cs +++ b/src/Redis.OM.POC/RedisHash.cs @@ -23,7 +23,7 @@ public RedisHash(IRedisConnection connection, string keyName) public string this[string key] { get => _connection.HMGet(_keyName, key).FirstOrDefault() ?? ""; - set => _connection.HSet(_keyName, new KeyValuePair(key,value)); + set => _connection.HSet(_keyName, new KeyValuePair(key,value)); } public ICollection Keys => new RedisHashScanner(_keyName, this, _connection, false); diff --git a/src/Redis.OM.POC/XRangeResponse.cs b/src/Redis.OM.POC/XRangeResponse.cs index 09b6ed36..b651529b 100644 --- a/src/Redis.OM.POC/XRangeResponse.cs +++ b/src/Redis.OM.POC/XRangeResponse.cs @@ -13,15 +13,6 @@ public class XRangeResponse { public IDictionary Messages { get; set; } - public XRangeResponse(StreamEntry[] entries) - { - Messages = new Dictionary(); - foreach(var entry in entries) - { - var innerDict = entry.Values.ToDictionary(x => x.Name.ToString(), x => x.Value.ToString()); - Messages.Add(entry.Id, (T)RedisObjectHandler.FromHashSet(innerDict)); - } - } public XRangeResponse(RedisResult[] vals, string streamName) { Messages = new Dictionary(); @@ -43,10 +34,10 @@ public XRangeResponse(RedisResult[] vals, string streamName) { var id = (string)((RedisResult[])obj.ToArray()[0])[i]; var pairs = ((RedisResult[])((RedisResult[])obj.ToArray()[0])[i + 1]); - var messageDict = new Dictionary(); + var messageDict = new Dictionary(); for (var j = 0; j < pairs.Length; j += 2) { - messageDict.Add(((string)pairs[j]), ((string)pairs[j + 1])); + messageDict.Add(((string)pairs[j]), new RedisReply(pairs[j + 1])); } Messages.Add(id, (T)RedisObjectHandler.FromHashSet(messageDict)); } @@ -70,7 +61,7 @@ public XRangeResponse(RedisReply[] vals, string streamName) { var id = (string)obj.ToArray()[0].ToArray()[i]; var pairs = obj.ToArray()[0].ToArray()[i + 1].ToArray(); - var messageDict = new Dictionary(); + var messageDict = new Dictionary(); for (var j = 0; j < pairs.Length; j+=2) { messageDict.Add(pairs[j], pairs[j + 1]); diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/AllMiniLML6V2Tokenizer.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/AllMiniLML6V2Tokenizer.cs new file mode 100644 index 00000000..06b7f7a8 --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/AllMiniLML6V2Tokenizer.cs @@ -0,0 +1,37 @@ +using System.Reflection; +using Redis.OM.Vectorizers.AllMiniLML6V2.Tokenizers; + +namespace Redis.OM.Vectorizers.AllMiniLML6V2; + +internal class AllMiniLML6V2Tokenizer : UncasedTokenizer +{ + private AllMiniLML6V2Tokenizer(string[] vocabulary) : base(vocabulary) + { + } + + internal static AllMiniLML6V2Tokenizer Create() + { + var assembly = Assembly.GetExecutingAssembly(); + const string fileName = "Redis.OM.Vectorizers.AllMiniLML6V2.Resources.vocab.txt"; + using var stream = assembly.GetManifestResourceStream(fileName); + if (stream is null) + { + throw new FileNotFoundException("Could not find embedded resource file Resources.vocab.txt"); + } + using var reader = new StreamReader(stream); + + if (stream is null) + { + throw new Exception("Could not open stream reader."); + } + + var vocab = new List(); + string? line; + while ((line = reader.ReadLine()) is not null) + { + vocab.Add(line); + } + + return new AllMiniLML6V2Tokenizer(vocab.ToArray()); + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/LICENSE b/src/Redis.OM.Vectorizers.AllMiniLML6V2/LICENSE new file mode 100644 index 00000000..3939dec8 --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/LICENSE @@ -0,0 +1,35 @@ +Copyright (c) 2023 Redis Inc. + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following + conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS IN THE SOFTWARE. + + =============================================== + + Third Party Licenses: + + The BERT tokenization code heavily influenced by the BertTokenizers + Project: https://github.com/NMZivkovic/BertTokenizers which is + licensed under an MIT license: + https://github.com/NMZivkovic/BertTokenizers/blob/master/LICENSE.txt + + Some parts of the pre/post processing pipeline were adapted + from Curiosity AI's MiniLM project, which uses an MIT license: + https://github.com/curiosity-ai/MiniLM/blob/4a7c629c223b6244cb8a394f17920ea1de363dce/MiniLM/MiniLM.csproj#L13 \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Redis.OM.Vectorizers.AllMiniLML6V2.csproj b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Redis.OM.Vectorizers.AllMiniLML6V2.csproj new file mode 100644 index 00000000..30badfd0 --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Redis.OM.Vectorizers.AllMiniLML6V2.csproj @@ -0,0 +1,49 @@ + + + + net6.0;net7.0 + enable + enable + Redis.OM.Vectorizers.AllMiniLML6V2 + 0.6.0 + 0.6.0 + https://github.com/redis/redis-om-dotnet/releases/tag/v0.6.0 + Sentence Vectorizer for Redis OM .NET using all-MiniLM-L6-v2 + Redis OM all-MiniLM-L6-v2 Vectorizers + Steve Lorello + Redis Inc + https://github.com/redis/redis-om-dotnet + https://github.com/redis/redis-om-dotnet + Github + redis redisearch AI Vectors + icon-square.png + true + + + + + + + + + + + + + + + + + + + + + + + + + + bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml + + + diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/RedisConnectionProviderExtensions.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/RedisConnectionProviderExtensions.cs new file mode 100644 index 00000000..a0ea8395 --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/RedisConnectionProviderExtensions.cs @@ -0,0 +1,32 @@ +using Redis.OM.Contracts; + +namespace Redis.OM.Vectorizers.AllMiniLML6V2; + +/// +/// Static extensions for The RedisConnectionProvider. +/// +public static class RedisConnectionProviderExtensions +{ + /// + /// Creates a Semantic Cache using the All-MiniLM-L6-v2 Vectorizer + /// + /// The connection provider. + /// The Index that the cache will be stored in. + /// The threshold that will be considered a match + /// The Prefix. + /// The Time to Live for a record stored in Redis. + /// + public static ISemanticCache AllMiniLML6V2SemanticCache(this IRedisConnectionProvider provider, string indexName="AllMiniLML6V2SemanticCache", double threshold = .15, string? prefix = null, long? ttl = null) + { + var vectorizer = new SentenceVectorizer(); + var connection = provider.Connection; + var info = connection.GetIndexInfo(indexName); + var cache = new SemanticCache(indexName, prefix ?? indexName, threshold, ttl, vectorizer, connection); + if (info is null) + { + cache.CreateIndex(); + } + + return cache; + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Resources/model.onnx b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Resources/model.onnx new file mode 100644 index 00000000..49af6b9d --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Resources/model.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a64cee3d4134bbdc86eed96e1a660efec58975417204ecfcf134140edb6e0e2 +size 90893304 diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Resources/vocab.txt b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Resources/vocab.txt new file mode 100644 index 00000000..cada3e34 --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Resources/vocab.txt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07eced375cec144d27c900241f3e339478dec958f92fddbc551f295c992038a3 +size 231508 diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizer.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizer.cs new file mode 100644 index 00000000..92dd96e7 --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizer.cs @@ -0,0 +1,188 @@ +using System.Reflection; +using Microsoft.ML.OnnxRuntime; +using Microsoft.ML.OnnxRuntime.Tensors; +using Redis.OM.Contracts; +using Redis.OM.Modeling; +using Redis.OM.Vectorizers.AllMiniLML6V2.Tokenizers; + +namespace Redis.OM.Vectorizers.AllMiniLML6V2; + +/// +/// A vectorizer to Vectorize sentences using ALl Mini LM L6 V2 Model. +/// +public class SentenceVectorizer : IVectorizer +{ + /// + public VectorType VectorType => VectorType.FLOAT32; + + /// + public int Dim => 384; + private static Lazy Tokenizer => new Lazy(AllMiniLML6V2Tokenizer.Create); + private static Lazy InferenceSession => new Lazy(LoadInferenceSession); + + private static InferenceSession LoadInferenceSession() + { + var file = "Redis.OM.Vectorizers.AllMiniLML6V2.Resources.model.onnx"; + using var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream(file); + if (stream is null) + { + throw new InvalidOperationException("Could not find Model resource"); + } + + var resourceBytes = new byte[stream.Length]; + _ = stream.Read(resourceBytes, 0, resourceBytes.Length); + return new InferenceSession(resourceBytes); + } + + /// + public byte[] Vectorize(string obj) + { + return Vectorize(new[] { obj })[0].SelectMany(BitConverter.GetBytes).ToArray(); + } + + private static Lazy OutputNames => new (() => InferenceSession.Value.OutputMetadata.Keys.ToArray()); + + /// + /// Vectorizers an array of sentences (which are vectorized individually). + /// + /// The Sentences + /// + public static float[][] Vectorize(string[] sentences) + { + const int MaxTokens = 512; + var numSentences = sentences.Length; + + var tokenized = sentences.Select(x=>Tokenizer.Value.Tokenize(x)).ToArray(); + + var seqLen = tokenized.Max(t => Math.Min(MaxTokens, t.Count)); + + List<(long[] InputIds, long[] TokenTypeIds, long[] AttentionMask)> encoded = tokenized.Select(tokens => + { + var padding = Enumerable.Repeat(0L, seqLen - Math.Min(MaxTokens, tokens.Count)).ToList(); + + var tokenIndexes = tokens.Take(MaxTokens).Select(token => (long)token.VocabularyIndex).Concat(padding).ToArray(); + var segmentIndexes = tokens.Take(MaxTokens).Select(token => token.SegmentIndex).Concat(padding).ToArray(); + var inputMask = tokens.Take(MaxTokens).Select(_ => 1L).Concat(padding).ToArray(); + return (tokenIndexes, TokenTypeIds: segmentIndexes, inputMask); + }).ToList(); + var tokenCount = encoded.First().InputIds.Length; + + long[] flattenIDs = new long[encoded.Sum(s => s.InputIds.Length)]; + long[] flattenAttentionMask = new long[encoded.Sum(s => s.AttentionMask.Length)]; + long[] flattenTokenTypeIds = new long[encoded.Sum(s => s.TokenTypeIds.Length)]; + + var flattenIDsSpan = flattenIDs.AsSpan(); + var flattenAttentionMaskSpan = flattenAttentionMask.AsSpan(); + var flattenTokenTypeIdsSpan = flattenTokenTypeIds.AsSpan(); + + foreach (var (InputIds, TokenTypeIds, AttentionMask) in encoded) + { + InputIds.AsSpan().CopyTo(flattenIDsSpan); + flattenIDsSpan = flattenIDsSpan.Slice(InputIds.Length); + + AttentionMask.AsSpan().CopyTo(flattenAttentionMaskSpan); + flattenAttentionMaskSpan = flattenAttentionMaskSpan.Slice(AttentionMask.Length); + + TokenTypeIds.AsSpan().CopyTo(flattenTokenTypeIdsSpan); + flattenTokenTypeIdsSpan = flattenTokenTypeIdsSpan.Slice(TokenTypeIds.Length); + } + + var dimensions = new[] { numSentences, tokenCount }; + + var input = new [] + { + NamedOnnxValue.CreateFromTensor("input_ids", new DenseTensor(flattenIDs, dimensions)), + NamedOnnxValue.CreateFromTensor("attention_mask", new DenseTensor(flattenAttentionMask,dimensions)), + NamedOnnxValue.CreateFromTensor("token_type_ids", new DenseTensor(flattenTokenTypeIds, dimensions)) + }; + + using var runOptions = new RunOptions(); + + using var output = InferenceSession.Value.Run(input, OutputNames.Value, runOptions); + + var output_pooled = MeanPooling((DenseTensor)output.First().Value, encoded); + var output_pooled_normalized = Normalize(output_pooled); + + const int embDim = 384; + + var outputFlatten = new float[sentences.Length][]; + + for(int s = 0; s < sentences.Length; s++) + { + var emb = new float[embDim]; + outputFlatten[s] = emb; + + for (int i = 0; i < embDim; i++) + { + emb[i] = output_pooled_normalized[s, i]; + } + } + + return outputFlatten; + } + + internal static DenseTensor Normalize(DenseTensor input_dense, float eps = 1e-12f) + { + var sentencesCount = input_dense.Dimensions[0]; + var hiddenStates = input_dense.Dimensions[1]; + + var denom_dense = new float [sentencesCount]; + + for (int s = 0; s < sentencesCount; s++) + { + for (int i = 0; i < hiddenStates; i++) + { + denom_dense[s] += input_dense[s, i] * input_dense[s, i]; + } + + denom_dense[s] = MathF.Max(MathF.Sqrt(denom_dense[s]), eps); + } + + for (int s = 0; s < sentencesCount; s++) + { + var invNorm = 1 / denom_dense[s]; + + for (int i = 0; i < hiddenStates; i++) + { + input_dense[s, i] *= invNorm; + } + } + + return input_dense; + } + + + internal static DenseTensor MeanPooling(DenseTensor token_embeddings_dense, List<(long[] InputIds, long[] TokenTypeIds, long[] AttentionMask)> encodedSentences, float eps = 1e-9f) + { + var sentencesCount = token_embeddings_dense.Dimensions[0]; + var sentenceLength = token_embeddings_dense.Dimensions[1]; + var hiddenStates = token_embeddings_dense.Dimensions[2]; + + var result = new DenseTensor(new[] { sentencesCount, hiddenStates }); + + for (int s = 0; s < sentencesCount; s++) + { + var maskSum = 0f; + + var attentionMask = encodedSentences[s].AttentionMask; + + for (int t = 0; t < sentenceLength; t++) + { + maskSum += attentionMask[t]; + + for (int i = 0; i < hiddenStates; i++) + { + result[s, i] += token_embeddings_dense[s, t, i] * attentionMask[t]; + } + } + + var invSum = 1f / MathF.Max(maskSum, eps); + for (int i = 0; i < hiddenStates; i++) + { + result[s, i] *= invSum; + } + } + + return result; + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizerAttribute.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizerAttribute.cs new file mode 100644 index 00000000..39d89677 --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/SentenceVectorizerAttribute.cs @@ -0,0 +1,22 @@ +using Redis.OM.Contracts; +using Redis.OM.Modeling; + +namespace Redis.OM.Vectorizers.AllMiniLML6V2; + +/// +/// +/// +public class SentenceVectorizerAttribute : VectorizerAttribute +{ + /// + public override VectorType VectorType => Vectorizer.VectorType; + + /// + public override int Dim => Vectorizer.Dim; + + /// + public override byte[] Vectorize(object obj) => Vectorizer.Vectorize((string)obj); + + /// + public override IVectorizer Vectorizer => new SentenceVectorizer(); +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/CasedTokenizer.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/CasedTokenizer.cs new file mode 100644 index 00000000..3181eddc --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/CasedTokenizer.cs @@ -0,0 +1,14 @@ +namespace Redis.OM.Vectorizers.AllMiniLML6V2.Tokenizers; + +internal abstract class CasedTokenizer : TokenizerBase +{ + protected CasedTokenizer(string[] vocabulary) : base(vocabulary) + { + } + + protected override IEnumerable TokenizeSentence(string text) + { + return text.Split(new [] { " ", " ", "\r\n" }, StringSplitOptions.None) + .SelectMany(o => o.SplitAndKeep(".,;:\\/?!#$%()=+-*\"'–_`<>&^@{}[]|~'".ToArray())); + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/StringExtensions.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/StringExtensions.cs new file mode 100644 index 00000000..956caee4 --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/StringExtensions.cs @@ -0,0 +1,25 @@ +namespace Redis.OM.Vectorizers.AllMiniLML6V2.Tokenizers; + +internal static class StringExtension +{ + public static IEnumerable SplitAndKeep( + this string inputString, params char[] delimiters) + { + int start = 0, index; + + while ((index = inputString.IndexOfAny(delimiters, start)) != -1) + { + if (index - start > 0) + yield return inputString.Substring(start, index - start); + + yield return inputString.Substring(index, 1); + + start = index + 1; + } + + if (start < inputString.Length) + { + yield return inputString.Substring(start); + } + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/TokenizerBase.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/TokenizerBase.cs new file mode 100644 index 00000000..ef092d76 --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/TokenizerBase.cs @@ -0,0 +1,117 @@ +using System.Text.RegularExpressions; + +namespace Redis.OM.Vectorizers.AllMiniLML6V2.Tokenizers; + +internal abstract class TokenizerBase +{ + protected readonly string[] _vocabulary; + protected readonly Dictionary _vocabularyDict; + + public TokenizerBase(string[] vocabulary) + { + _vocabulary = vocabulary; + _vocabularyDict = new Dictionary(); + + for (int i = 0; i < _vocabulary.Length; i++) + { + _vocabularyDict[_vocabulary[i]] = i; + } + } + + public List<(string Token, int VocabularyIndex, long SegmentIndex)> Tokenize(params string[] texts) + { + IEnumerable tokens = new[] { Tokens.Classification }; + + foreach (var text in texts) + { + tokens = tokens.Concat(TokenizeSentence(text)); + tokens = tokens.Concat(new[] { Tokens.Separation }); + } + + var tokenAndIndex = tokens.SelectMany(TokenizeSubWords).ToArray(); + + var segmentIndexes = SegmentIndex(tokenAndIndex); + return tokenAndIndex.Zip(segmentIndexes, (tokenIndex, segmentIndex) => (tokenIndex.Token, tokenIndex.VocabularyIndex, segmentIndex)).ToList(); + } + + public List<(long InputIds, long TokenTypeIds, long AttentionMask)> Encode(int sequenceLength, params string[] texts) + { + var tokens = Tokenize(texts); + + var padding = Enumerable.Repeat(0L, sequenceLength - tokens.Count).ToArray(); + var tokenIndexes = tokens.Select(token => (long)token.VocabularyIndex).Concat(padding).ToArray(); + var segmentIndexes = tokens.Select(token => token.SegmentIndex).Concat(padding).ToArray(); + var inputMask = tokens.Select(o => 1L).Concat(padding).ToArray(); + + var output = tokenIndexes.Zip(segmentIndexes, Tuple.Create) + .Zip(inputMask, (t, z) => Tuple.Create(t.Item1, t.Item2, z)); + + return output.Select(x => (InputIds: x.Item1, TokenTypeIds: x.Item2, AttentionMask: x.Item3)).ToList(); + } + + private IEnumerable SegmentIndex(IEnumerable<(string token, int index)> tokens) + { + var segmentIndex = 0; + var segmentIndexes = new List(); + + foreach (var (token, index) in tokens) + { + segmentIndexes.Add(segmentIndex); + + if (token == Tokens.Separation) + { + segmentIndex++; + } + } + + return segmentIndexes; + } + + private IEnumerable<(string Token, int VocabularyIndex)> TokenizeSubWords(string word) + { + if (_vocabularyDict.ContainsKey(word)) + { + return new (string, int)[] { (word, _vocabularyDict[word]) }; + } + + var tokens = new List<(string, int)>(); + var remaining = word; + + while (!string.IsNullOrEmpty(remaining) && remaining.Length > 2) + { + string? prefix = null; + int subWordLength = remaining.Length; + while (subWordLength >= 1) + { + string subWord = remaining.Substring(0, subWordLength); + if (!_vocabularyDict.ContainsKey(subWord)) + { + subWordLength--; + continue; + } + + prefix = subWord; + break; + } + + if (prefix == null) + { + tokens.Add((Tokens.Unknown, _vocabularyDict[Tokens.Unknown])); + return tokens; + } + + var regex = new Regex(prefix); + remaining = regex.Replace(remaining, "##", 1); + + tokens.Add((prefix, _vocabularyDict[prefix])); + } + + if (!string.IsNullOrEmpty(word) && !tokens.Any()) + { + tokens.Add((Tokens.Unknown, _vocabularyDict[Tokens.Unknown])); + } + + return tokens; + } + protected abstract IEnumerable TokenizeSentence(string text); +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/Tokens.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/Tokens.cs new file mode 100644 index 00000000..5f7ea2ab --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/Tokens.cs @@ -0,0 +1,10 @@ +namespace Redis.OM.Vectorizers.AllMiniLML6V2.Tokenizers; + +internal class Tokens +{ + public const string Padding = ""; + public const string Unknown = "[UNK]"; + public const string Classification = "[CLS]"; + public const string Separation = "[SEP]"; + public const string Mask = "[MASK]"; +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/UncasedTokenizer.cs b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/UncasedTokenizer.cs new file mode 100644 index 00000000..d3581d2f --- /dev/null +++ b/src/Redis.OM.Vectorizers.AllMiniLML6V2/Tokenizers/UncasedTokenizer.cs @@ -0,0 +1,15 @@ +namespace Redis.OM.Vectorizers.AllMiniLML6V2.Tokenizers; + +internal abstract class UncasedTokenizer : TokenizerBase +{ + public UncasedTokenizer(string[] vocabulary) : base(vocabulary) + { + } + + protected override IEnumerable TokenizeSentence(string text) + { + return text.Split(new [] { " ", " ", "\r\n" }, StringSplitOptions.None) + .SelectMany(o => o.SplitAndKeep(".,;:\\/?!#$%()=+-*\"'–_`<>&^@{}[]|~'".ToArray())) + .Select(o => o.ToLower()); + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Resnet18/DnnImageModelSelectorExtensions.cs b/src/Redis.OM.Vectorizers.Resnet18/DnnImageModelSelectorExtensions.cs new file mode 100644 index 00000000..9b0fcb9f --- /dev/null +++ b/src/Redis.OM.Vectorizers.Resnet18/DnnImageModelSelectorExtensions.cs @@ -0,0 +1,50 @@ +using Microsoft.ML; +using Microsoft.ML.Data; +using Microsoft.ML.Runtime; +using Microsoft.ML.Transforms; +using Microsoft.ML.Transforms.Onnx; + +namespace Redis.OM.Vectorizers.Resnet18; + +/// +/// Extensions pulled and slightly modified from from ML.NET to service this package as the content files cannot be +/// reliably copied from transitive dependencies. +/// +internal static class DnnImageModelSelectorExtensions +{ + /// + /// Returns an estimator chain with the two corresponding models (a preprocessing one and a main one) required for the ResNet pipeline. + /// Also includes the renaming ColumnsCopyingTransforms required to be able to use arbitrary input and output column names. + /// This assumes both of the models are in the same location as the file containing this method, which they will be if used through the NuGet. + /// This should be the default way to use ResNet18 if importing the model from a NuGet. + /// + public static EstimatorChain ResNet18(this DnnImageModelSelector dnnModelContext, IHostEnvironment env, string outputColumnName, string inputColumnName, MLContext context) + { + return ResNet18(dnnModelContext, env, outputColumnName, inputColumnName, Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Resources"), context); + } + + /// + /// This allows a custom model location to be specified. This is useful is a custom model is specified, + /// or if the model is desired to be placed or shipped separately in a different folder from the main application. Note that because ONNX models + /// must be in a directory all by themselves for the OnnxTransformer to work, this method appends a ResNet18Onnx/ResNetPrepOnnx subdirectory + /// to the passed in directory to prevent having to make that directory manually each time. + /// + public static EstimatorChain ResNet18(this DnnImageModelSelector dnnModelContext, IHostEnvironment env, string outputColumnName, string inputColumnName, string modelDir, MLContext context) + { + var modelChain = new EstimatorChain(); + + var inputRename = context.Transforms.CopyColumns("OriginalInput", inputColumnName); + var midRename = context.Transforms.CopyColumns("Input247", "PreprocessedInput"); + var endRename = context.Transforms.CopyColumns(outputColumnName, "Pooling395_Output_0"); + + // There are two estimators created below. The first one is for image preprocessing and the second one is the actual DNN model. + var prepEstimator = context.Transforms.ApplyOnnxModel("PreprocessedInput", "OriginalInput", Path.Combine(modelDir, "ResNetPrepOnnx", "ResNetPreprocess.onnx")); + var mainEstimator = context.Transforms.ApplyOnnxModel("Pooling395_Output_0", "Input247", Path.Combine(modelDir, "ResNet18Onnx", "ResNet18.onnx")); + modelChain = modelChain.Append(inputRename); + var modelChain2 = modelChain.Append(prepEstimator); + modelChain = modelChain2.Append(midRename); + modelChain2 = modelChain.Append(mainEstimator); + modelChain = modelChain2.Append(endRename); + return modelChain; + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Resnet18/ImageModelObjects.cs b/src/Redis.OM.Vectorizers.Resnet18/ImageModelObjects.cs new file mode 100644 index 00000000..efe59d67 --- /dev/null +++ b/src/Redis.OM.Vectorizers.Resnet18/ImageModelObjects.cs @@ -0,0 +1,26 @@ +using Microsoft.ML.Data; +using Microsoft.ML.Transforms.Image; + +namespace Redis.OM.Vectorizers.Resnet18; + +internal class ImageInput +{ + [ColumnName(@"ImageSource")] + public string ImageSource { get; set; } + + public ImageInput(string imageSource) + { + ImageSource = imageSource; + } +} + +internal class InMemoryImageData +{ + [ImageType(224,224)] + public MLImage Image; + + public InMemoryImageData(MLImage image) + { + Image = image; + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizer.cs b/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizer.cs new file mode 100644 index 00000000..3ef48c14 --- /dev/null +++ b/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizer.cs @@ -0,0 +1,108 @@ +using Microsoft.ML; +using Microsoft.ML.Data; +using Microsoft.ML.Transforms; +using Redis.OM.Contracts; +using Redis.OM.Modeling; + +namespace Redis.OM.Vectorizers.Resnet18; + +/// +/// A Vectorizer that uses Resnet 18 to perform vectorization. It accepts either a file path or full URI to an image as +/// input and vectorizers the inputs returning a Float32 vector with a dimensionality of 512 +/// +public class ImageVectorizer : IVectorizer +{ + /// + public VectorType VectorType => VectorType.FLOAT32; + + /// + public int Dim => 512; + + /// + public byte[] Vectorize(string obj) + { + var isUri = Uri.TryCreate(obj, UriKind.Absolute, out var uri) && (uri.Scheme == Uri.UriSchemeHttp || uri.Scheme == Uri.UriSchemeHttps); + if (isUri) + { + var request = new HttpRequestMessage() + { + Method = HttpMethod.Get, + RequestUri = uri, + }; + var imageStream = Configuration.Instance.Client.Send(request).Content.ReadAsStream(); + var image = MLImage.CreateFromStream(imageStream); + var vector = VectorizeImages(new [] { image })[0].SelectMany(BitConverter.GetBytes).ToArray(); + return vector; + } + + if (!File.Exists(obj)) + { + throw new ArgumentException( + $"Input {obj} was not a well formed URI, and was not a file path that exists on this system.", nameof(obj)); + } + + return VectorizeFiles(new[] { obj })[0].SelectMany(BitConverter.GetBytes).ToArray(); + } + + private static readonly Lazy>> FilePipeline = new(CreateFilePipeline); + + private static readonly Lazy MlContext = new(()=>new MLContext()); + + private static EstimatorChain> CreateFilePipeline() + { + var mlContext = MlContext.Value; + var pipeline = mlContext.Transforms + .LoadImages("ImageObject", "", "ImageSource") + .Append(mlContext.Transforms.ResizeImages("ImageObject", 224, 224)) + .Append(mlContext.Transforms.ExtractPixels("Pixels", "ImageObject")) + .Append(mlContext.Transforms.DnnFeaturizeImage("Features", + m => m.ModelSelector.ResNet18(mlContext, m.OutputColumn, m.InputColumn, mlContext), "Pixels")); + + return pipeline; + } + + /// + /// Vectorizers a series of image file paths. + /// + /// + /// + public static float[][] VectorizeFiles(IEnumerable imagePaths) + { + var images = imagePaths.Select(x => new ImageInput(x)); + var mlContext = MlContext.Value; + var dataView = mlContext.Data.LoadFromEnumerable(images); + + var transformedData = FilePipeline.Value.Fit(dataView).Transform(dataView); + var vector = transformedData.GetColumn("Features").ToArray(); + return vector; + } + + private static readonly Lazy>> BitmapPipeline = new(CreateBitmapPipeline); + + private static EstimatorChain> CreateBitmapPipeline() + { + var mlContext = MlContext.Value; + var pipeline = mlContext.Transforms + .ResizeImages("Image", 224,224) + .Append(mlContext.Transforms.ExtractPixels("Pixels", "Image")) + .Append(mlContext.Transforms.DnnFeaturizeImage("Features", + m => m.ModelSelector.ResNet18(mlContext, m.OutputColumn, m.InputColumn, mlContext), "Pixels")); + + return pipeline; + } + + /// + /// Encodes a collection of images. + /// + /// + /// + public static float[][] VectorizeImages(IEnumerable mlImages) + { + var images = mlImages.Select(x => new InMemoryImageData(x)); + var mlContext = MlContext.Value; + var dataView = mlContext.Data.LoadFromEnumerable(images); + var transformedData = BitmapPipeline.Value.Fit(dataView).Transform(dataView); + var vector = transformedData.GetColumn("Features").ToArray(); + return vector; + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizerAttribute.cs b/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizerAttribute.cs new file mode 100644 index 00000000..29b58492 --- /dev/null +++ b/src/Redis.OM.Vectorizers.Resnet18/ImageVectorizerAttribute.cs @@ -0,0 +1,22 @@ +using Redis.OM.Contracts; +using Redis.OM.Modeling; + +namespace Redis.OM.Vectorizers.Resnet18; + +/// +/// A Vectorizer Attribute for encoding images +/// +public class ImageVectorizerAttribute : VectorizerAttribute +{ + /// + public override VectorType VectorType => Vectorizer.VectorType; + + /// + public override int Dim => Vectorizer.Dim; + + /// + public override byte[] Vectorize(object obj) => Vectorizer.Vectorize((string)obj); + + /// + public override IVectorizer Vectorizer { get; } = new ImageVectorizer(); +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.csproj b/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.csproj new file mode 100644 index 00000000..58d9d1e3 --- /dev/null +++ b/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.csproj @@ -0,0 +1,61 @@ + + + net6.0;net7.0 + enable + enable + Redis.OM.Vectorizers.Resnet18 + 0.6.0 + 0.6.0 + https://github.com/redis/redis-om-dotnet/releases/tag/v0.6.0 + Resnet 18 Vectorizers for Redis OM .NET. + Redis OM Resnet 18 Vectorizers + Steve Lorello + Redis Inc + https://github.com/redis/redis-om-dotnet + icon-square.png + MIT + https://github.com/redis/redis-om-dotnet + Github + redis redisearch indexing databases + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PreserveNewest + + + + + + + bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml + + + diff --git a/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.props b/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.props new file mode 100644 index 00000000..7f412f54 --- /dev/null +++ b/src/Redis.OM.Vectorizers.Resnet18/Redis.OM.Vectorizers.Resnet18.props @@ -0,0 +1,8 @@ + + + + Resources\%(RecursiveDir)%(Filename)%(Extension) + PreserveNewest + + + \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers.Resnet18/Resources/ResNet18Onnx/ResNet18.onnx b/src/Redis.OM.Vectorizers.Resnet18/Resources/ResNet18Onnx/ResNet18.onnx new file mode 100755 index 00000000..930f9590 --- /dev/null +++ b/src/Redis.OM.Vectorizers.Resnet18/Resources/ResNet18Onnx/ResNet18.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2552f19db2dfdb01b3669537b6a0f1b44da38d18279535492ed5dfc2ef1ff8f +size 63653533 diff --git a/src/Redis.OM.Vectorizers.Resnet18/Resources/ResNetPrepOnnx/ResNetPreprocess.onnx b/src/Redis.OM.Vectorizers.Resnet18/Resources/ResNetPrepOnnx/ResNetPreprocess.onnx new file mode 100755 index 00000000..88790121 --- /dev/null +++ b/src/Redis.OM.Vectorizers.Resnet18/Resources/ResNetPrepOnnx/ResNetPreprocess.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84bf8def78d0befaa3f6e73745d57d7016c6a192ead76f79686bd438c1047004 +size 602584 diff --git a/src/Redis.OM.Vectorizers/AzureOpenAIVectorizer.cs b/src/Redis.OM.Vectorizers/AzureOpenAIVectorizer.cs new file mode 100644 index 00000000..62b558e4 --- /dev/null +++ b/src/Redis.OM.Vectorizers/AzureOpenAIVectorizer.cs @@ -0,0 +1,77 @@ + +using System.Net.Http.Json; +using System.Text.Json; +using Redis.OM.Contracts; +using Redis.OM.Modeling; + +namespace Redis.OM.Vectorizers; + +/// +/// Vectorizer for Azure's OpenAI REST API +/// +public class AzureOpenAIVectorizer : IVectorizer +{ + private readonly string _apiKey; + private readonly string _resourceName; + private readonly string _deploymentName; + + /// + /// Initializes vectorizer + /// + /// The Vectorizers API Key + /// The Azure Resource Name. + /// The Azure Deployment Name. + /// The dimensions of the model addressed by this resource/deployment. + public AzureOpenAIVectorizer(string apiKey, string resourceName, string deploymentName, int dim) + { + _apiKey = apiKey; + _resourceName = resourceName; + _deploymentName = deploymentName; + Dim = dim; + } + + /// + public VectorType VectorType => VectorType.FLOAT32; + + /// + public int Dim { get; } + + /// + public byte[] Vectorize(string str) => GetFloats(str, _resourceName, _deploymentName, _apiKey).SelectMany(BitConverter.GetBytes).ToArray(); + + internal static float[] GetFloats(string s, string resourceName, string deploymentName, string apiKey) + { + var client = Configuration.Instance.Client; + var requestContent = JsonContent.Create(new { input = s }); + + var request = new HttpRequestMessage + { + Method = HttpMethod.Post, + RequestUri = new Uri( + $"https://{resourceName}.openai.azure.com/openai/deployments/{deploymentName}/embeddings?api-version=2023-05-15"), + Content = requestContent, + Headers = { { "api-key", apiKey } } + }; + + var res = client.Send(request); + if (!res.IsSuccessStatusCode) + { + throw new HttpRequestException( + $"Open AI did not respond with a positive error code: {res.StatusCode}, {res.ReasonPhrase}"); + } + var jsonObj = res.Content.ReadFromJsonAsync().Result; + + + if (!jsonObj.TryGetProperty("data", out var data)) + { + throw new Exception("Malformed Response"); + } + + if (data.GetArrayLength() < 1 || !data[0].TryGetProperty("embedding", out var embedding)) + { + throw new Exception("Malformed Response"); + } + + return embedding.Deserialize()!; + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers/AzureOpenAIVectorizerAttribute.cs b/src/Redis.OM.Vectorizers/AzureOpenAIVectorizerAttribute.cs new file mode 100644 index 00000000..43344ed2 --- /dev/null +++ b/src/Redis.OM.Vectorizers/AzureOpenAIVectorizerAttribute.cs @@ -0,0 +1,48 @@ +using Redis.OM.Contracts; +using Redis.OM.Modeling; + +namespace Redis.OM.Vectorizers; + +/// +public class AzureOpenAIVectorizerAttribute : VectorizerAttribute +{ + /// + public AzureOpenAIVectorizerAttribute(string deploymentName, string resourceName, int dim) + { + DeploymentName = deploymentName; + ResourceName = resourceName; + Dim = dim; + Vectorizer = new AzureOpenAIVectorizer(Configuration.Instance.AzureOpenAIApiKey, ResourceName, DeploymentName, Dim); + } + + /// + /// Gets the DeploymentName. + /// + public string DeploymentName { get; } + + /// + /// Gets the resource name. + /// + public string ResourceName { get; } + + /// + public override IVectorizer Vectorizer { get; } + + /// + public override VectorType VectorType => VectorType.FLOAT32; + + /// + public override int Dim { get; } + + /// + public override byte[] Vectorize(object obj) + { + if (obj is not string s) + { + throw new ArgumentException("Object must be a string to be embedded", nameof(obj)); + } + + var floats = AzureOpenAIVectorizer.GetFloats(s, ResourceName, DeploymentName, Configuration.Instance.AzureOpenAIApiKey); + return floats.SelectMany(BitConverter.GetBytes).ToArray(); + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers/Configuration.cs b/src/Redis.OM.Vectorizers/Configuration.cs new file mode 100644 index 00000000..7810fa3f --- /dev/null +++ b/src/Redis.OM.Vectorizers/Configuration.cs @@ -0,0 +1,95 @@ +using System.Net.Http.Headers; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.Configuration; +[assembly: InternalsVisibleTo("Redis.OM.Vectorizers.Resnet18")] +namespace Redis.OM; + +/// +/// Some Configuration Items. +/// +internal class Configuration +{ + /// + /// Gets the configuration item at the given key. + /// + /// + public string? this[string str] => _settings[str]; + + /// + /// The bearer authorization token for Hugging Face's model API. + /// + public string HuggingFaceAuthorizationToken => _settings["REDIS_OM_HF_TOKEN"] ?? string.Empty; + + /// + /// Bearer token for Open AI's API. + /// + public string OpenAiAuthorizationToken => _settings["REDIS_OM_OAI_TOKEN"] ?? string.Empty; + + /// + /// Azure OpenAI Api Key. + /// + public string AzureOpenAIApiKey => _settings["REDIS_OM_AZURE_OAI_TOKEN"] ?? string.Empty; + + /// + /// Hugging Face Model Id + /// + public string ModelId => _settings["REDIS_OM_HF_MODEL_ID"] ?? string.Empty; + + /// + /// Base Address for Hugging Face Feature Extraction API + /// + public string HuggingFaceBaseAddress => _settings["REDIS_OM_HF_FEATURE_EXTRACTION_URL"] ?? string.Empty; + + private const string DefaultHuggingFaceApiUrl = "https://api-inference.huggingface.co"; + + private const string DefaultOpenAiApiUrl = "https://api.openai.com"; + + /// + /// URL for OpenAI API. + /// + public string OpenAiApiUrl => _settings["REDIS_OM_OAI_API_URL"] ?? String.Empty; + + private readonly IConfiguration _settings; + + private static readonly object LockObject = new (); + private static Configuration? _instance; + + /// + /// Common HTTP Client. + /// + public readonly HttpClient Client; + + /// + /// Singleton Instance. + /// + public static Configuration Instance + { + get + { + if (_instance is not null) return _instance; + lock (LockObject) + { + _instance ??= new Configuration(); + } + + return _instance; + } + } + + internal Configuration() + { + var builder = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + {"REDIS_OM_HF_FEATURE_EXTRACTION_URL", DefaultHuggingFaceApiUrl}, + {"REDIS_OM_OAI_API_URL", DefaultOpenAiApiUrl}, + {"REDIS_OM_HF_TOKEN", Environment.GetEnvironmentVariable("REDIS_OM_HF_TOKEN")}, + {"REDIS_OM_OAI_TOKEN", Environment.GetEnvironmentVariable("REDIS_OM_OAI_TOKEN")}, + {"REDIS_OM_AZURE_OAI_TOKEN", Environment.GetEnvironmentVariable("REDIS_OM_AZURE_OAI_TOKEN")} + }) + .AddJsonFile("settings.json", true, true) + .AddJsonFile("appsettings.json", true, true); + _settings = builder.Build(); + Client = new HttpClient(); + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers/HuggingFaceVectorizer.cs b/src/Redis.OM.Vectorizers/HuggingFaceVectorizer.cs new file mode 100644 index 00000000..988fe7be --- /dev/null +++ b/src/Redis.OM.Vectorizers/HuggingFaceVectorizer.cs @@ -0,0 +1,83 @@ +using System.Net.Http.Json; +using System.Text.Json; +using Redis.OM.Contracts; +using Redis.OM.Modeling; + +namespace Redis.OM.Vectorizers; + +/// +/// Vectorizer for HuggingFace API. +/// +public class HuggingFaceVectorizer : IVectorizer +{ + /// + /// Initializes the Vectorizer. + /// + /// Auth token. + /// Model Id. + /// Dimensions for the output tensors of the model. + public HuggingFaceVectorizer(string authToken, string modelId, int dim) + { + _huggingFaceAuthToken = authToken; + ModelId = modelId; + Dim = dim; + } + + private readonly string _huggingFaceAuthToken; + + /// + /// The Model Id. + /// + public string ModelId { get; } + + /// + public VectorType VectorType => VectorType.FLOAT32; + + /// + public int Dim { get; } + + /// + public byte[] Vectorize(string str) + { + return GetFloats(str, ModelId, _huggingFaceAuthToken).SelectMany(BitConverter.GetBytes).ToArray(); + } + + /// + /// Gets the floats for the sentence. + /// + /// the string. + /// The Model Id. + /// The HF token. + /// + /// + internal static float[] GetFloats(string s, string modelId, string huggingFaceAuthToken) + { + var client = Configuration.Instance.Client; + var requestContent = JsonContent.Create(new + { + inputs = new [] { s }, + options = new { wait_for_model = true } + }); + + var request = new HttpRequestMessage() + { + Method = HttpMethod.Post, + Content = requestContent, + RequestUri = + new Uri($"{Configuration.Instance.HuggingFaceBaseAddress}/pipeline/feature-extraction/{modelId}"), + Headers = + { + { "Authorization", $"Bearer {huggingFaceAuthToken}" } + } + }; + + var res = client.Send(request); + var floats = JsonSerializer.Deserialize(RedisOMHttpUtil.ReadJsonSync(res)); + if (floats is null) + { + throw new Exception("Did not receive a response back from HuggingFace"); + } + + return floats.First(); + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers/HuggingFaceVectorizerAttribute.cs b/src/Redis.OM.Vectorizers/HuggingFaceVectorizerAttribute.cs new file mode 100644 index 00000000..ba1905c4 --- /dev/null +++ b/src/Redis.OM.Vectorizers/HuggingFaceVectorizerAttribute.cs @@ -0,0 +1,81 @@ +using Redis.OM.Contracts; +using Redis.OM.Modeling; + +namespace Redis.OM.Vectorizers; + +/// +/// An attribute that provides a Hugging Face API Sentence Vectorizer. +/// +public class HuggingFaceVectorizerAttribute : VectorizerAttribute +{ + /// + /// The Model Id. + /// + public string? ModelId { get; set; } + + private IVectorizer? _vectorizer; + + /// + public override IVectorizer Vectorizer + { + get + { + if (_vectorizer is null) + { + var modelId = ModelId ?? Configuration.Instance["REDIS_OM_HF_MODEL_ID"]; + if (modelId is null) + { + throw new InvalidOperationException("Need a Model ID in order to process vector"); + } + + _vectorizer = new HuggingFaceVectorizer(Configuration.Instance.HuggingFaceAuthorizationToken, modelId, Dim); + } + + return _vectorizer; + } + } + + + /// + public override VectorType VectorType => VectorType.FLOAT32; + private int? _dim; + + /// + public override int Dim + { + get + { + if (_dim is not null) return _dim.Value; + const string testString = "This is a vector dimensionality probing query"; + var floats = GetFloats(testString); + _dim = floats.Length; + + return _dim.Value; + } + } + + /// + public override byte[] Vectorize(object obj) + { + if (obj is not string s) + { + throw new ArgumentException("Object must be a string", nameof(obj)); + } + + var floats = GetFloats(s); + return floats.SelectMany(BitConverter.GetBytes).ToArray(); + } + + /// + /// Gets the embedded floats of the string from the HuggingFace API. + /// + /// the string. + /// the embedding's floats. + /// thrown if model id is not populated. + public float[] GetFloats(string s) + { + var modelId = ModelId ?? Configuration.Instance["REDIS_OM_HF_MODEL_ID"]; + if (modelId is null) throw new InvalidOperationException("Model Id Required to use Hugging Face API."); + return HuggingFaceVectorizer.GetFloats(s, modelId, Configuration.Instance.HuggingFaceAuthorizationToken); + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers/OpenAIVectorizer.cs b/src/Redis.OM.Vectorizers/OpenAIVectorizer.cs new file mode 100644 index 00000000..a6294a05 --- /dev/null +++ b/src/Redis.OM.Vectorizers/OpenAIVectorizer.cs @@ -0,0 +1,77 @@ +using System.Net.Http.Json; +using System.Text.Json; +using Redis.OM.Contracts; +using Redis.OM.Modeling; + +namespace Redis.OM.Vectorizers; + +/// +/// A Vectorizer leveraging Open AI's REST API +/// +public class OpenAIVectorizer : IVectorizer +{ + private readonly string _openAIAuthToken; + private readonly string _model; + + /// + /// Initializes the vectorizer. + /// + /// The Authorization Token. + /// The Model ID + /// The dimension of the model's output tensor. + public OpenAIVectorizer(string openAIAuthToken, string model = "text-embedding-ada-002", int dim = 1536) + { + _openAIAuthToken = openAIAuthToken; + _model = model; + Dim = dim; + } + + /// + public VectorType VectorType => VectorType.FLOAT32; + + /// + public int Dim { get; } + + /// + public byte[] Vectorize(string str) + { + var floats = GetFloats(str, _model, _openAIAuthToken); + return floats.SelectMany(BitConverter.GetBytes).ToArray(); + } + + internal static float[] GetFloats(string s, string model, string openAIAuthToken) + { + var client = Configuration.Instance.Client; + var requestContent = JsonContent.Create(new { input = s, model }); + + var request = new HttpRequestMessage + { + Method = HttpMethod.Post, + RequestUri = new Uri($"{Configuration.Instance.OpenAiApiUrl}/v1/embeddings"), + Content = requestContent, + Headers = { { "Authorization", $"Bearer {openAIAuthToken}" } } + }; + + var res = client.Send(request); + if (!res.IsSuccessStatusCode) + { + throw new HttpRequestException( + $"Open AI did not respond with a positive error code: {res.StatusCode}, {res.ReasonPhrase}"); + } + + var jsonObj = JsonSerializer.Deserialize(RedisOMHttpUtil.ReadJsonSync(res)); + + + if (!jsonObj.TryGetProperty("data", out var data)) + { + throw new Exception("Malformed Response"); + } + + if (data.GetArrayLength() < 1 || !data[0].TryGetProperty("embedding", out var embedding)) + { + throw new Exception("Malformed Response"); + } + + return embedding.Deserialize()!; + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers/OpenAIVectorizerAttribute.cs b/src/Redis.OM.Vectorizers/OpenAIVectorizerAttribute.cs new file mode 100644 index 00000000..026e5718 --- /dev/null +++ b/src/Redis.OM.Vectorizers/OpenAIVectorizerAttribute.cs @@ -0,0 +1,47 @@ +using Redis.OM.Contracts; +using Redis.OM.Modeling; + +namespace Redis.OM.Vectorizers; + +/// +/// An OpenAI Sentence Vectorizer. +/// +public class OpenAIVectorizerAttribute : VectorizerAttribute +{ + private const string DefaultModel = "text-embedding-ada-002"; + + /// + /// The ModelId. + /// + public string ModelId { get; } = DefaultModel; + + /// + public override VectorType VectorType => VectorType.FLOAT32; + + /// + public override int Dim => ModelId == DefaultModel ? 1536 : GetFloats("Probing model dimensions").Length; + + private IVectorizer? _vectorizer; + + /// + public override IVectorizer Vectorizer + { + get + { + return _vectorizer ??= new OpenAIVectorizer(Configuration.Instance.OpenAiAuthorizationToken, ModelId, Dim); + } + } + + /// + public override byte[] Vectorize(object obj) + { + var s = (string)obj; + var floats = GetFloats(s); + return floats.SelectMany(BitConverter.GetBytes).ToArray(); + } + + internal float[] GetFloats(string s) + { + return OpenAIVectorizer.GetFloats(s, ModelId, Configuration.Instance.OpenAiAuthorizationToken); + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers/Redis.OM.Vectorizers.csproj b/src/Redis.OM.Vectorizers/Redis.OM.Vectorizers.csproj new file mode 100644 index 00000000..830ac576 --- /dev/null +++ b/src/Redis.OM.Vectorizers/Redis.OM.Vectorizers.csproj @@ -0,0 +1,44 @@ + + + + net6.0;net7.0 + enable + enable + Redis.OM + 0.6.0 + 0.6.0 + https://github.com/redis/redis-om-dotnet/releases/tag/v0.6.0 + Core Vectorizers for Redis OM .NET. + Redis OM Vectorizers + Steve Lorello + Redis Inc + https://github.com/redis/redis-om-dotnet + icon-square.png + MIT + https://github.com/redis/redis-om-dotnet + Github + redis redisearch indexing databases + true + + + + + + + + + + + + + + + + + + + + bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml + + + diff --git a/src/Redis.OM.Vectorizers/RedisConnectionProviderExtensions.cs b/src/Redis.OM.Vectorizers/RedisConnectionProviderExtensions.cs new file mode 100644 index 00000000..4f2d017c --- /dev/null +++ b/src/Redis.OM.Vectorizers/RedisConnectionProviderExtensions.cs @@ -0,0 +1,87 @@ +using Redis.OM.Contracts; + +namespace Redis.OM.Vectorizers; + +/// +/// Static extensions for The RedisConnectionProvider. +/// +public static class RedisConnectionProviderExtensions +{ + + /// + /// Creates a Semantic Cache using the Hugging face model API + /// + /// The Connection Provider. + /// The API token for Hugging face. + /// The activation threshold. + /// The Model Id to use. + /// The dimensionality of the tensors. + /// The Index name. + /// The prefix. + /// The TTL + /// + public static ISemanticCache HuggingFaceSemanticCache(this IRedisConnectionProvider provider, string huggingFaceAuthToken, double threshold = .15, string modelId = "sentence-transformers/all-mpnet-base-v2", int dim = 768, string indexName = "HuggingFaceSemanticCache", string? prefix = null, long? ttl = null) + { + var vectorizer = new HuggingFaceVectorizer(huggingFaceAuthToken, modelId, dim); + var connection = provider.Connection; + var info = connection.GetIndexInfo(indexName); + var cache = new SemanticCache(indexName, prefix ?? indexName, threshold, ttl, vectorizer, connection); + if (info is null) + { + cache.CreateIndex(); + } + + return cache; + } + + /// + /// Creates a Semantic Cache that leverages OpenAI's REST API. + /// + /// The Provider. + /// The OpenAI bearer token. + /// The activation threshold for acceptable distance. + /// The index name to create. + /// The Prefix to use for the semantic cache. + /// The Time to Live for Items in the Semantic Cache. + /// + public static ISemanticCache OpenAISemanticCache(this IRedisConnectionProvider provider, string openAIAuthToken, double threshold = .15, string indexName = "OpenAISemanticCache", string? prefix = null, long? ttl = null) + { + var vectorizer = new OpenAIVectorizer(openAIAuthToken); + var connection = provider.Connection; + var info = connection.GetIndexInfo(indexName); + var cache = new SemanticCache(indexName, prefix ?? indexName, threshold, ttl, vectorizer, connection); + if (info is null) + { + cache.CreateIndex(); + } + + return cache; + } + + /// + /// Creates a Semantic Cache leveraging Azure's Open AI REST Api. + /// + /// The RedisConnectionProvider. + /// The API Key for Azure. + /// The Resource Name. + /// The Deployment ID + /// The dimension of the model at the given Resource/Deployment. + /// The Activation Threshold. + /// The Index name. + /// The Prefix. + /// The Time to Live for a record inserted into the cache. + /// + public static ISemanticCache AzureOpenAISemanticCache(this IRedisConnectionProvider provider, string apiKey, string resourceName, string deploymentId, int dim, double threshold = .15, string indexName = "AzureOpenAISemanticCache", string? prefix = null, long? ttl = null) + { + var vectorizer = new AzureOpenAIVectorizer(apiKey, resourceName, deploymentId, dim); + var connection = provider.Connection; + var cache = new SemanticCache(indexName, prefix ?? indexName, threshold, ttl, vectorizer, connection); + var info = connection.GetIndexInfo(indexName); + if (info is null) + { + cache.CreateIndex(); + } + + return cache; + } +} \ No newline at end of file diff --git a/src/Redis.OM.Vectorizers/RedisOMHttpUtil.cs b/src/Redis.OM.Vectorizers/RedisOMHttpUtil.cs new file mode 100644 index 00000000..cefbb826 --- /dev/null +++ b/src/Redis.OM.Vectorizers/RedisOMHttpUtil.cs @@ -0,0 +1,9 @@ +namespace Redis.OM; + +internal static class RedisOMHttpUtil +{ + public static string ReadJsonSync(HttpResponseMessage msg) + { + return new StreamReader(msg.Content.ReadAsStream()).ReadToEnd(); + } +} \ No newline at end of file diff --git a/src/Redis.OM/Aggregation/AggregationPredicates/QueryPredicate.cs b/src/Redis.OM/Aggregation/AggregationPredicates/QueryPredicate.cs index 5a6a27ef..87263bf8 100644 --- a/src/Redis.OM/Aggregation/AggregationPredicates/QueryPredicate.cs +++ b/src/Redis.OM/Aggregation/AggregationPredicates/QueryPredicate.cs @@ -81,7 +81,7 @@ protected override void ValidateAndPushOperand(Expression expression, Stack()); // hack - will need to revisit when integrating vectors into aggregations. stack.Push(BuildQueryPredicate(binaryExpression.NodeType, memberExpression, System.Linq.Expressions.Expression.Constant(val))); } } @@ -92,7 +92,7 @@ protected override void ValidateAndPushOperand(Expression expression, Stack())); } else if (expression is UnaryExpression uni) { diff --git a/src/Redis.OM/Common/ExpressionParserUtilities.cs b/src/Redis.OM/Common/ExpressionParserUtilities.cs index 5f7267f3..c2515611 100644 --- a/src/Redis.OM/Common/ExpressionParserUtilities.cs +++ b/src/Redis.OM/Common/ExpressionParserUtilities.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Collections.ObjectModel; using System.Globalization; using System.Linq; using System.Linq.Expressions; @@ -11,6 +12,7 @@ using Redis.OM.Aggregation; using Redis.OM.Aggregation.AggregationPredicates; using Redis.OM.Modeling; +using Redis.OM.Modeling.Vectors; using Redis.OM.Searching.Query; namespace Redis.OM.Common @@ -72,18 +74,19 @@ internal static string GetOperandString(MethodCallExpression exp) /// Gets the operand string from a search. /// /// expression. + /// The parameters. /// Treat enum as an integer. /// Whether or not to negate the result. /// the operand string. /// thrown if expression is un-parseable. - internal static string GetOperandStringForQueryArgs(Expression exp, bool treatEnumsAsInt = false, bool negate = false) + internal static string GetOperandStringForQueryArgs(Expression exp, List parameters, bool treatEnumsAsInt = false, bool negate = false) { var res = exp switch { ConstantExpression constExp => $"{constExp.Value}", MemberExpression member => GetOperandStringForMember(member, treatEnumsAsInt), - MethodCallExpression method => TranslateMethodStandardQuerySyntax(method), - UnaryExpression unary => GetOperandStringForQueryArgs(unary.Operand, treatEnumsAsInt, unary.NodeType == ExpressionType.Not), + MethodCallExpression method => TranslateMethodStandardQuerySyntax(method, parameters), + UnaryExpression unary => GetOperandStringForQueryArgs(unary.Operand, parameters, treatEnumsAsInt, unary.NodeType == ExpressionType.Not), _ => throw new ArgumentException("Unrecognized Expression type") }; @@ -171,20 +174,22 @@ internal static string ParseBinaryExpression(BinaryExpression rootBinaryExpressi /// Translates the method expression. /// /// the expression. + /// The parameters to be passed into the query. /// The expression translated. /// thrown if the method isn't recognized. - internal static string TranslateMethodExpressions(MethodCallExpression exp) + internal static string TranslateMethodExpressions(MethodCallExpression exp, List parameters) { return exp.Method.Name switch { - "Contains" => TranslateContainsStandardQuerySyntax(exp), + "Contains" => TranslateContainsStandardQuerySyntax(exp, parameters), nameof(StringExtension.FuzzyMatch) => TranslateFuzzyMatch(exp), nameof(StringExtension.MatchContains) => TranslateMatchContains(exp), nameof(StringExtension.MatchStartsWith) => TranslateMatchStartsWith(exp), nameof(StringExtension.MatchEndsWith) => TranslateMatchEndsWith(exp), + nameof(VectorExtensions.VectorRange) => TranslateVectorRange(exp, parameters), nameof(string.StartsWith) => TranslateStartsWith(exp), nameof(string.EndsWith) => TranslateEndsWith(exp), - "Any" => TranslateAnyForEmbeddedObjects(exp), + "Any" => TranslateAnyForEmbeddedObjects(exp, parameters), _ => throw new ArgumentException($"Unrecognized method for query translation:{exp.Method.Name}") }; } @@ -640,16 +645,17 @@ private static IEnumerable SplitBinaryExpression(BinaryExpress while (true); } - private static string TranslateMethodStandardQuerySyntax(MethodCallExpression exp) + private static string TranslateMethodStandardQuerySyntax(MethodCallExpression exp, List parameters) { return exp.Method.Name switch { nameof(StringExtension.FuzzyMatch) => TranslateFuzzyMatch(exp), nameof(string.Format) => TranslateFormatMethodStandardQuerySyntax(exp), - nameof(string.Contains) => TranslateContainsStandardQuerySyntax(exp), + nameof(string.Contains) => TranslateContainsStandardQuerySyntax(exp, parameters), nameof(string.StartsWith) => TranslateStartsWith(exp), nameof(string.EndsWith) => TranslateEndsWith(exp), - "Any" => TranslateAnyForEmbeddedObjects(exp), + nameof(VectorExtensions.VectorRange) => TranslateVectorRange(exp, parameters), + "Any" => TranslateAnyForEmbeddedObjects(exp, parameters), _ => throw new InvalidOperationException($"Unable to parse method {exp.Method.Name}") }; } @@ -786,7 +792,7 @@ private static string TranslateFuzzyMatch(MethodCallExpression exp) }; } - private static string TranslateContainsStandardQuerySyntax(MethodCallExpression exp) + private static string TranslateContainsStandardQuerySyntax(MethodCallExpression exp, List parameters) { MemberExpression? expression = null; Type type; @@ -797,7 +803,7 @@ private static string TranslateContainsStandardQuerySyntax(MethodCallExpression { var propertyExpression = (MemberExpression)exp.Arguments.Last(); var valuesExpression = (MemberExpression)exp.Arguments.First(); - literal = GetOperandStringForQueryArgs(propertyExpression); + literal = GetOperandStringForQueryArgs(propertyExpression, parameters); if (!literal.StartsWith("@")) { if (exp.Arguments.Count == 1 && exp.Object != null) @@ -836,7 +842,7 @@ private static string TranslateContainsStandardQuerySyntax(MethodCallExpression var valueType = Nullable.GetUnderlyingType(valuesExpression.Type) ?? valuesExpression.Type; memberName = GetOperandStringForMember(propertyExpression); var treatEnumsAsInts = type.IsEnum && !(propertyExpression.Member.GetCustomAttributes(typeof(JsonConverterAttribute)).FirstOrDefault() is JsonConverterAttribute converter && converter.ConverterType == typeof(JsonStringEnumConverter)); - literal = GetOperandStringForQueryArgs(valuesExpression, treatEnumsAsInts); + literal = GetOperandStringForQueryArgs(valuesExpression, parameters, treatEnumsAsInts); if ((valueType == typeof(List) || valueType == typeof(string[]) || type == typeof(string[]) || type == typeof(List) || type == typeof(Guid) || type == typeof(Guid[]) || type == typeof(List) || type == typeof(Guid[]) || type == typeof(List) || type == typeof(Ulid) || (type.IsEnum && !treatEnumsAsInts)) && attribute is IndexedAttribute) { @@ -887,7 +893,7 @@ private static string TranslateContainsStandardQuerySyntax(MethodCallExpression type = Nullable.GetUnderlyingType(expression.Type) ?? expression.Type; memberName = GetOperandStringForMember(expression); - literal = GetOperandStringForQueryArgs(exp.Arguments.Last()); + literal = GetOperandStringForQueryArgs(exp.Arguments.Last(), parameters); if (searchFieldAttribute is not null && searchFieldAttribute is SearchableAttribute) { @@ -936,12 +942,12 @@ private static string GetContainsStringForConstantExpression(string propertyName return sb.ToString(); } - private static string TranslateAnyForEmbeddedObjects(MethodCallExpression exp) + private static string TranslateAnyForEmbeddedObjects(MethodCallExpression exp, List parameters) { var type = exp.Arguments.Last().Type; var prefix = GetOperandString(exp.Arguments[0]); var lambda = (LambdaExpression)exp.Arguments.Last(); - var tempQuery = ExpressionTranslator.TranslateBinaryExpression((BinaryExpression)lambda.Body); + var tempQuery = ExpressionTranslator.TranslateBinaryExpression((BinaryExpression)lambda.Body, parameters); return tempQuery.Replace("@", $"{prefix}_"); } @@ -978,5 +984,58 @@ private static string GetConstantStringForArgs(ConstantExpression constExp) return $"{valueAsString}"; } + + private static object GetOperand(Expression expression) + { + return expression switch + { + MemberExpression me => GetValue(me.Member, ((ConstantExpression)me.Expression).Value), + ConstantExpression ce => ce.Value, + _ => throw new InvalidOperationException("Could not determine value.") + }; + } + + private static string TranslateVectorRange(MethodCallExpression exp, List parameters) + { + if (exp.Arguments[0] is not MemberExpression member) + { + throw new InvalidOperationException("Vector Range must be called on a member"); + } + + var field = GetOperandStringForMember(member); + var vectorizer = member.Member.GetCustomAttributes().FirstOrDefault(); + byte[] bytes; + + var operand = GetOperand(exp.Arguments[1]); + if (vectorizer is null) + { + throw new InvalidOperationException( + $"Attempting to run a vector range on a {member.Type} with no provided vectorizer"); + } + + if (operand is not Vector vector) + { + bytes = vectorizer.Vectorize(operand); + } + else + { + vector.Embed(vectorizer); + + bytes = vector.Embedding ?? throw new InvalidOperationException("Embedding was null"); + } + + var distance = (double)((ConstantExpression)exp.Arguments[2]).Value; + var distanceArgName = parameters.Count.ToString(); + parameters.Add(distance); + var vectorArgName = parameters.Count.ToString(); + parameters.Add(bytes); + if (exp.Arguments.Count > 3) + { + var scoreName = $"{GetOperand(exp.Arguments[3])}{VectorScores.RangeScoreSuffix}"; + return $"{field}:[VECTOR_RANGE ${distanceArgName} ${vectorArgName}]=>{{$YIELD_DISTANCE_AS: {scoreName}}}"; + } + + return $"{field}:[VECTOR_RANGE ${distanceArgName} ${vectorArgName}]"; + } } } \ No newline at end of file diff --git a/src/Redis.OM/Common/ExpressionTranslator.cs b/src/Redis.OM/Common/ExpressionTranslator.cs index 13215c5a..6de9866a 100644 --- a/src/Redis.OM/Common/ExpressionTranslator.cs +++ b/src/Redis.OM/Common/ExpressionTranslator.cs @@ -8,6 +8,7 @@ using Redis.OM.Aggregation; using Redis.OM.Aggregation.AggregationPredicates; using Redis.OM.Modeling; +using Redis.OM.Modeling.Vectors; using Redis.OM.Searching; using Redis.OM.Searching.Query; @@ -30,7 +31,7 @@ public static RedisAggregation BuildAggregationFromExpression(Expression express var attr = type.GetCustomAttribute(); if (attr == null) { - throw new InvalidOperationException("Aggregations can only be perfomred on objects decorated with a RedisObjectDefinitionAttribute that specifies a particular index"); + throw new InvalidOperationException("Aggregations can only be performed on objects decorated with a RedisObjectDefinitionAttribute that specifies a particular index"); } var indexName = string.IsNullOrEmpty(attr.IndexName) ? $"{type.Name.ToLower()}-idx" : attr.IndexName; @@ -183,6 +184,7 @@ internal static RedisQuery BuildQueryFromExpression(Expression expression, Type throw new InvalidOperationException("Searches can only be performed on objects decorated with a RedisObjectDefinitionAttribute that specifies a particular index"); } + var parameters = new List(); var indexName = string.IsNullOrEmpty(attr.IndexName) ? $"{type.Name.ToLower()}-idx" : attr.IndexName; var query = new RedisQuery(indexName!) { QueryText = "*" }; switch (expression) @@ -227,7 +229,10 @@ internal static RedisQuery BuildQueryFromExpression(Expression expression, Type query.GeoFilter = ExpressionParserUtilities.TranslateGeoFilter(exp); break; case "Where": - query.QueryText = TranslateWhereMethod(exp); + query.QueryText = TranslateWhereMethod(exp, parameters); + break; + case "NearestNeighbors": + query.NearestNeighbors = ParseNearestNeighborsFromExpression(exp); break; } } @@ -236,18 +241,67 @@ internal static RedisQuery BuildQueryFromExpression(Expression expression, Type } case LambdaExpression lambda: - query.QueryText = BuildQueryFromExpression(lambda.Body); + query.QueryText = BuildQueryFromExpression(lambda.Body, parameters); break; } if (mainBooleanExpression != null) { - query.QueryText = BuildQueryFromExpression(((LambdaExpression)mainBooleanExpression).Body); + parameters = new List(); + query.QueryText = BuildQueryFromExpression(((LambdaExpression)mainBooleanExpression).Body, parameters); } + query.Parameters = parameters; return query; } + /// + /// Builds a Nearest Neighbor query from provided expression. + /// + /// The expression. + /// The nearest neighbor query. + internal static NearestNeighbors ParseNearestNeighborsFromExpression(MethodCallExpression expression) + { + var memberExpression = (MemberExpression)((LambdaExpression)((UnaryExpression)expression.Arguments[1]).Operand).Body; + var attr = memberExpression.Member.GetCustomAttributes().FirstOrDefault() ?? throw new ArgumentException($"Could not find Vector attribute on {memberExpression.Member.Name}."); + var vectorizer = memberExpression.Member.GetCustomAttributes().FirstOrDefault(); + var propertyName = !string.IsNullOrEmpty(attr.PropertyName) ? attr.PropertyName : memberExpression.Member.Name; + var numNeighbors = (int)((ConstantExpression)expression.Arguments[2]).Value; + var value = ((ConstantExpression)expression.Arguments[3]).Value ?? throw new InvalidOperationException("Provided vector property was null"); + byte[] bytes; + + if (vectorizer is not null) + { + if (value is Vector vec) + { + if (vec.Embedding is null) + { + vec.Embed(vectorizer); + } + + bytes = vec.Embedding!; + } + else + { + bytes = vectorizer.Vectorize(value); + } + } + else if (memberExpression.Type == typeof(float[])) + { + bytes = ((float[])value).SelectMany(BitConverter.GetBytes).ToArray(); + } + else if (memberExpression.Type == typeof(double[])) + { + bytes = ((double[])value).SelectMany(BitConverter.GetBytes).ToArray(); + } + else + { + throw new ArgumentException($"{memberExpression.Type} was not valid without a Vectorizer"); + } + + return new NearestNeighbors(propertyName, numNeighbors, bytes); + } + /// /// Get's the index field type for the given member info. /// @@ -276,40 +330,41 @@ internal static SearchFieldType DetermineIndexFieldsType(MemberInfo member) /// Translates a binary expression. /// /// The Binary Expression. + /// The parameters of the query. /// The query string formatted from the binary expression. /// Thrown if expression is not parsable because of the arguments passed into it. - internal static string TranslateBinaryExpression(BinaryExpression binExpression) + internal static string TranslateBinaryExpression(BinaryExpression binExpression, List parameters) { var sb = new StringBuilder(); if (binExpression.Left is BinaryExpression leftBin && binExpression.Right is BinaryExpression rightBin) { sb.Append("("); - sb.Append(TranslateBinaryExpression(leftBin)); + sb.Append(TranslateBinaryExpression(leftBin, parameters)); sb.Append(SplitPredicateSeporators(binExpression.NodeType)); - sb.Append(TranslateBinaryExpression(rightBin)); + sb.Append(TranslateBinaryExpression(rightBin, parameters)); sb.Append(")"); } else if (binExpression.Left is BinaryExpression left) { sb.Append("("); - sb.Append(TranslateBinaryExpression(left)); + sb.Append(TranslateBinaryExpression(left, parameters)); sb.Append(SplitPredicateSeporators(binExpression.NodeType)); - sb.Append(ExpressionParserUtilities.GetOperandStringForQueryArgs(binExpression.Right)); + sb.Append(ExpressionParserUtilities.GetOperandStringForQueryArgs(binExpression.Right, parameters)); sb.Append(")"); } else if (binExpression.Right is BinaryExpression right) { sb.Append("("); - sb.Append(ExpressionParserUtilities.GetOperandStringForQueryArgs(binExpression.Left)); + sb.Append(ExpressionParserUtilities.GetOperandStringForQueryArgs(binExpression.Left, parameters)); sb.Append(SplitPredicateSeporators(binExpression.NodeType)); - sb.Append(TranslateBinaryExpression(right)); + sb.Append(TranslateBinaryExpression(right, parameters)); sb.Append(")"); } else { - var leftContent = ExpressionParserUtilities.GetOperandStringForQueryArgs(binExpression.Left); + var leftContent = ExpressionParserUtilities.GetOperandStringForQueryArgs(binExpression.Left, parameters); - var rightContent = ExpressionParserUtilities.GetOperandStringForQueryArgs(binExpression.Right); + var rightContent = ExpressionParserUtilities.GetOperandStringForQueryArgs(binExpression.Right, parameters); if (binExpression.Left is MemberExpression member) { @@ -677,26 +732,26 @@ private static RedisSortBy TranslateOrderByMethod(MethodCallExpression expressio var predicate = (UnaryExpression)expression.Arguments[1]; var lambda = (LambdaExpression)predicate.Operand; var memberExpression = (MemberExpression)lambda.Body; - sb.Field = ExpressionParserUtilities.GetSearchFieldNameFromMember(memberExpression); + sb.Field = memberExpression.Member.Name == nameof(VectorScores.NearestNeighborsScore) ? VectorScores.NearestNeighborScoreName : ExpressionParserUtilities.GetSearchFieldNameFromMember(memberExpression); sb.Direction = ascending ? SortDirection.Ascending : SortDirection.Descending; return sb; } - private static string BuildQueryFromExpression(Expression exp) + private static string BuildQueryFromExpression(Expression exp, List parameters) { if (exp is BinaryExpression binExp) { - return TranslateBinaryExpression(binExp); + return TranslateBinaryExpression(binExp, parameters); } if (exp is MethodCallExpression method) { - return ExpressionParserUtilities.TranslateMethodExpressions(method); + return ExpressionParserUtilities.TranslateMethodExpressions(method, parameters); } if (exp is UnaryExpression uni) { - var operandString = BuildQueryFromExpression(uni.Operand); + var operandString = BuildQueryFromExpression(uni.Operand, parameters); if (uni.NodeType == ExpressionType.Not) { operandString = $"-{operandString}"; @@ -714,18 +769,11 @@ private static string BuildQueryFromExpression(Expression exp) throw new ArgumentException("Unparseable Lambda Body detected"); } - private static string TranslateFirstMethod(MethodCallExpression expression) - { - var predicate = (UnaryExpression)expression.Arguments[1]; - var lambda = (LambdaExpression)predicate.Operand; - return BuildQueryFromExpression(lambda.Body); - } - - private static string TranslateWhereMethod(MethodCallExpression expression) + private static string TranslateWhereMethod(MethodCallExpression expression, List parameters) { var predicate = (UnaryExpression)expression.Arguments[1]; var lambda = (LambdaExpression)predicate.Operand; - return BuildQueryFromExpression(lambda.Body); + return BuildQueryFromExpression(lambda.Body, parameters); } private static string BuildQueryPredicate(ExpressionType expType, string left, string right, MemberExpression memberExpression) diff --git a/src/Redis.OM/Contracts/IRedisConnection.cs b/src/Redis.OM/Contracts/IRedisConnection.cs index fa275546..d21b8e39 100644 --- a/src/Redis.OM/Contracts/IRedisConnection.cs +++ b/src/Redis.OM/Contracts/IRedisConnection.cs @@ -14,7 +14,7 @@ public interface IRedisConnection : IDisposable /// The command name. /// The arguments. /// A redis Reply. - RedisReply Execute(string command, params string[] args); + RedisReply Execute(string command, params object[] args); /// /// Executes a command. @@ -22,7 +22,7 @@ public interface IRedisConnection : IDisposable /// The command name. /// The arguments. /// A redis Reply. - Task ExecuteAsync(string command, params string[] args); + Task ExecuteAsync(string command, params object[] args); /// /// Executes the contained commands within the context of a transaction. @@ -30,7 +30,7 @@ public interface IRedisConnection : IDisposable /// each tuple represents a command and /// it's arguments to execute inside a transaction. /// A redis Reply. - Task ExecuteInTransactionAsync(Tuple[] commandArgsTuples); + Task ExecuteInTransactionAsync(Tuple[] commandArgsTuples); /// /// Executes the contained commands within the context of a transaction. @@ -38,6 +38,6 @@ public interface IRedisConnection : IDisposable /// each tuple represents a command and /// it's arguments to execute inside a transaction. /// A redis Reply. - RedisReply[] ExecuteInTransaction(Tuple[] commandArgsTuples); + RedisReply[] ExecuteInTransaction(Tuple[] commandArgsTuples); } } diff --git a/src/Redis.OM/Contracts/IRedisHydrateable.cs b/src/Redis.OM/Contracts/IRedisHydrateable.cs index 7f1e6791..27e6499c 100644 --- a/src/Redis.OM/Contracts/IRedisHydrateable.cs +++ b/src/Redis.OM/Contracts/IRedisHydrateable.cs @@ -17,6 +17,6 @@ public interface IRedisHydrateable /// Converts object to dictionary for Redis. /// /// A dictionary for Redis. - IDictionary BuildHashSet(); + IDictionary BuildHashSet(); } } diff --git a/src/Redis.OM/Contracts/ISemanticCache.cs b/src/Redis.OM/Contracts/ISemanticCache.cs new file mode 100644 index 00000000..3dad9f69 --- /dev/null +++ b/src/Redis.OM/Contracts/ISemanticCache.cs @@ -0,0 +1,93 @@ +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace Redis.OM.Contracts +{ + /// + /// An interface for interacting with a Semantic Cache. + /// + public interface ISemanticCache + { + /// + /// Gets the index name of the cache. + /// + string IndexName { get; } + + /// + /// Gets the prefix to be used for the keys. + /// + string Prefix { get; } + + /// + /// Gets the threshold to be used for the keys. + /// + double Threshold { get; } + + /// + /// Gets the Time to live for the keys added to the cache. + /// + long? Ttl { get; } + + /// + /// Gets the vectorizer to use for the Semantic Cache. + /// + IVectorizer Vectorizer { get; } + + /// + /// Checks the cache to see if any close prompts have been added. + /// + /// The prompt. + /// How many results to pull back at most (defaults to 10). + /// The responses. + SemanticCacheResponse[] GetSimilar(string prompt, int maxNumResults = 10); + + /// + /// Checks the cache to see if any close prompts have been added. + /// + /// The prompt. + /// How many results to pull back at most (defaults to 10). + /// The responses. + Task GetSimilarAsync(string prompt, int maxNumResults = 10); + + /// + /// Stores the Prompt/response/metadata in Redis. + /// + /// The prompt. + /// The response. + /// The metadata. + void Store(string prompt, string response, object? metadata = null); + + /// + /// Stores the Prompt/response/metadata in Redis. + /// + /// The prompt. + /// The response. + /// The metadata. + /// A representing the asynchronous operation. + Task StoreAsync(string prompt, string response, object? metadata = null); + + /// + /// Deletes the cache from Redis. + /// + /// Whether or not to drop the records associated with the cache. Defaults to true. + void DeleteCache(bool dropRecords = true); + + /// + /// Deletes the cache from Redis. + /// + /// Whether or not to drop the records associated with the cache. Defaults to true. + /// A representing the asynchronous operation. + Task DeleteCacheAsync(bool dropRecords = true); + + /// + /// Creates the index for Semantic Cache. + /// + void CreateIndex(); + + /// + /// Creates the index for the Semantic Cache. + /// + /// A representing the asynchronous operation. + Task CreateIndexAsync(); + } +} \ No newline at end of file diff --git a/src/Redis.OM/Contracts/IVectorizer.cs b/src/Redis.OM/Contracts/IVectorizer.cs new file mode 100644 index 00000000..41b0c685 --- /dev/null +++ b/src/Redis.OM/Contracts/IVectorizer.cs @@ -0,0 +1,28 @@ +using Redis.OM.Modeling; + +namespace Redis.OM.Contracts +{ + /// + /// Converter of objects into vectors. + /// + /// The type to be vectorized. + public interface IVectorizer + { + /// + /// Gets the vector Type generated by the vectorizer. + /// + VectorType VectorType { get; } + + /// + /// Gets the vector dimension of the vectors generated by the vectorizer. + /// + int Dim { get; } + + /// + /// Converts the provided object to a vector. + /// + /// the object to convert. + /// A byte array containing the vectorized data. + byte[] Vectorize(T obj); + } +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/IndexedAttribute.cs b/src/Redis.OM/Modeling/IndexedAttribute.cs index cce2df7c..d844e765 100644 --- a/src/Redis.OM/Modeling/IndexedAttribute.cs +++ b/src/Redis.OM/Modeling/IndexedAttribute.cs @@ -17,7 +17,79 @@ public sealed class IndexedAttribute : SearchFieldAttribute /// public bool CaseSensitive { get; set; } = false; + /// + /// Gets or sets the vector storage algorithm to use. Defaults to Flat (which is brute force). + /// + public VectorAlgorithm Algorithm { get; set; } = VectorAlgorithm.FLAT; + + /// + /// Gets or sets the Supported distance metric. + /// + public DistanceMetric DistanceMetric { get; set; } = DistanceMetric.L2; + + /// + /// Gets or sets the Initial vector capacity in the index affecting memory allocation size of the index. + /// + public int InitialCapacity { get; set; } + + /// + /// Gets or sets Block size to hold BLOCK_SIZE amount of vectors in a contiguous array. This is useful when the + /// index is dynamic with respect to addition and deletion. Defaults to 1024. + /// + public int BlockSize { get; set; } + + /// + /// gets or sets the number of maximum allowed outgoing edges for each node in the graph in each layer. + /// On layer zero the maximal number of outgoing edges will be 2M. Default is 16. + /// + public int M { get; set; } + + /// + /// Gets or sets the number of maximum allowed potential outgoing edges candidates for each node in the graph, + /// during the graph building. Default is 200. + /// + public int EfConstructor { get; set; } + + /// + /// Gets or sets the number of maximum top candidates to hold during the KNN search. Higher values of + /// EfRuntime lead to more accurate results at the expense of a longer runtime. Default is 10. + /// + public int EfRuntime { get; set; } + + /// + /// Gets or sets Relative factor that sets the boundaries in which a range query may search for candidates. + /// That is, vector candidates whose distance from the query vector is radius*(1 + EPSILON) are potentially + /// scanned, allowing more extensive search and more accurate results (on the expense of runtime). Default is 0.01. + /// + public double Epsilon { get; set; } + /// internal override SearchFieldType SearchFieldType => SearchFieldType.INDEXED; + + /// + /// gets the number of arguments that will be produced by this attribute. + /// + internal int NumArgs + { + get + { + var numArgs = 6; + numArgs += InitialCapacity != default ? 2 : 0; + if (Algorithm == VectorAlgorithm.FLAT) + { + numArgs += BlockSize != default ? 2 : 0; + } + + if (Algorithm == VectorAlgorithm.HNSW) + { + numArgs += M != default ? 2 : 0; + numArgs += EfConstructor != default ? 2 : 0; + numArgs += EfRuntime != default ? 2 : 0; + numArgs += Epsilon != default ? 2 : 0; + } + + return numArgs; + } + } } } diff --git a/src/Redis.OM/Modeling/RedisCollectionStateManager.cs b/src/Redis.OM/Modeling/RedisCollectionStateManager.cs index f4ec792e..0a39a135 100644 --- a/src/Redis.OM/Modeling/RedisCollectionStateManager.cs +++ b/src/Redis.OM/Modeling/RedisCollectionStateManager.cs @@ -112,10 +112,11 @@ internal bool TryDetectDifferencesSingle(string key, object value, out IList)Snapshot[key]; + var snapshotHash = (IDictionary)Snapshot[key]; var deletedKeys = snapshotHash.Keys.Except(dataHash.Keys).Select(x => new KeyValuePair(x, string.Empty)); var modifiedKeys = dataHash.Where(x => - !snapshotHash.Keys.Contains(x.Key) || snapshotHash[x.Key] != x.Value); + !snapshotHash.Keys.Contains(x.Key) || snapshotHash[x.Key] != x.Value).Select(x => + new KeyValuePair(x.Key, x.Value.ToString())); differences = new List { new HashDiff(modifiedKeys, deletedKeys.Select(x => x.Key)), @@ -158,10 +159,11 @@ internal IDictionary> DetectDifferences() if (Data.ContainsKey(key)) { var dataHash = Data[key] !.BuildHashSet(); - var snapshotHash = (IDictionary)Snapshot[key]; + var snapshotHash = (IDictionary)Snapshot[key]; var deletedKeys = snapshotHash.Keys.Except(dataHash.Keys).Select(x => new KeyValuePair(x, string.Empty)); var modifiedKeys = dataHash.Where(x => - !snapshotHash.Keys.Contains(x.Key) || snapshotHash[x.Key] != x.Value); + !snapshotHash.Keys.Contains(x.Key) || snapshotHash[x.Key] != x.Value).Select(x => + new KeyValuePair(x.Key, x.Value.ToString())); var diff = new List { new HashDiff(modifiedKeys, deletedKeys.Select(x => x.Key)), diff --git a/src/Redis.OM/Modeling/RedisSchemaField.cs b/src/Redis.OM/Modeling/RedisSchemaField.cs index c5e16388..1a938eb5 100644 --- a/src/Redis.OM/Modeling/RedisSchemaField.cs +++ b/src/Redis.OM/Modeling/RedisSchemaField.cs @@ -3,7 +3,9 @@ using System.Globalization; using System.Linq; using System.Reflection; +using System.Runtime.InteropServices.ComTypes; using System.Text.Json.Serialization; +using Redis.OM.Modeling.Vectors; namespace Redis.OM.Modeling { @@ -61,7 +63,16 @@ internal static string[] SerializeArgsJson(this PropertyInfo info, int remaining { var innerType = Nullable.GetUnderlyingType(info.PropertyType) ?? info.PropertyType; - if (IsComplexType(innerType)) + if (attr is IndexedAttribute indexedAttribute && (innerType == typeof(Vector) || innerType.BaseType == typeof(Vector))) + { + var vectorizer = info.GetCustomAttributes().FirstOrDefault(x => x.GetType() != typeof(FloatVectorizerAttribute) && x.GetType() != typeof(DoubleVectorizerAttribute)); + var pathPostfix = vectorizer != null ? ".Vector" : string.Empty; + ret.Add(!string.IsNullOrEmpty(attr.PropertyName) ? $"{pathPrefix}{attr.PropertyName}{pathPostfix}" : $"{pathPrefix}{info.Name}{pathPostfix}"); + ret.Add("AS"); + ret.Add(!string.IsNullOrEmpty(attr.PropertyName) ? $"{aliasPrefix}{attr.PropertyName}" : $"{aliasPrefix}{info.Name}"); + ret.AddRange(CommonSerialization(attr, innerType, info)); + } + else if (IsComplexType(innerType)) { if (cascadeDepth > 0) { @@ -93,12 +104,22 @@ internal static string[] SerializeArgsJson(this PropertyInfo info, int remaining internal static string[] SerializeArgs(this PropertyInfo info) { var attr = Attribute.GetCustomAttribute(info, typeof(SearchFieldAttribute)) as SearchFieldAttribute; - if (attr == null) + if (attr is null) { return Array.Empty(); } - var ret = new List { !string.IsNullOrEmpty(attr.PropertyName) ? attr.PropertyName : info.Name }; + var suffix = string.Empty; + if (attr.SearchFieldType == SearchFieldType.INDEXED && (info.PropertyType == typeof(Vector) || info.PropertyType.BaseType == typeof(Vector))) + { + var vectorizer = info.GetCustomAttributes().FirstOrDefault(); + if (vectorizer is not null && vectorizer is not FloatVectorizerAttribute && vectorizer is not DoubleVectorizerAttribute) + { + suffix = ".Vector"; + } + } + + var ret = new List { !string.IsNullOrEmpty(attr.PropertyName) ? $"attr.PropertyName{suffix}" : $"{info.Name}{suffix}" }; var innerType = Nullable.GetUnderlyingType(info.PropertyType); ret.AddRange(CommonSerialization(attr, innerType ?? info.PropertyType, info)); return ret.ToArray(); @@ -177,6 +198,11 @@ private static string GetSearchFieldType(Type declaredType, SearchFieldAttribute return "GEO"; } + if (declaredType == typeof(Vector) || declaredType.BaseType == typeof(Vector)) + { + return "VECTOR"; + } + if (declaredType.IsEnum) { return propertyInfo.GetCustomAttributes(typeof(JsonConverterAttribute)).FirstOrDefault() is JsonConverterAttribute converter && converter.ConverterType == typeof(JsonStringEnumConverter) ? "TAG" : "NUMERIC"; @@ -187,6 +213,65 @@ private static string GetSearchFieldType(Type declaredType, SearchFieldAttribute private static bool IsEnumTypeFlags(Type type) => type.GetCustomAttributes(typeof(FlagsAttribute), false).Any(); + private static IEnumerable VectorSerialization(IndexedAttribute vectorAttribute, PropertyInfo propertyInfo) + { + var vectorizer = propertyInfo.GetCustomAttributes().FirstOrDefault(); + if (vectorizer is null) + { + throw new InvalidOperationException("Indexed vector fields must provide a vectorizer."); + } + + yield return vectorAttribute.Algorithm.AsRedisString(); + yield return vectorAttribute.NumArgs.ToString(); + yield return "TYPE"; + yield return vectorizer.VectorType.AsRedisString(); + + yield return "DIM"; + yield return vectorizer.Dim.ToString(); + yield return "DISTANCE_METRIC"; + yield return vectorAttribute.DistanceMetric.AsRedisString(); + if (vectorAttribute.InitialCapacity != default) + { + yield return "INITIAL_CAP"; + yield return vectorAttribute.InitialCapacity.ToString(); + } + + if (vectorAttribute.Algorithm == VectorAlgorithm.FLAT) + { + if (vectorAttribute.BlockSize != default) + { + yield return "BLOCK_SIZE"; + yield return vectorAttribute.BlockSize.ToString(); + } + } + else if (vectorAttribute.Algorithm == VectorAlgorithm.HNSW) + { + if (vectorAttribute.M != default) + { + yield return "M"; + yield return vectorAttribute.M.ToString(); + } + + if (vectorAttribute.EfConstructor != default) + { + yield return "EF_CONSTRUCTION"; + yield return vectorAttribute.EfConstructor.ToString(); + } + + if (vectorAttribute.EfRuntime != default) + { + yield return "EF_RUNTIME"; + yield return vectorAttribute.EfRuntime.ToString(); + } + + if (vectorAttribute.Epsilon != default) + { + yield return "EPSILON"; + yield return vectorAttribute.Epsilon.ToString(CultureInfo.InvariantCulture); + } + } + } + private static string[] CommonSerialization(SearchFieldAttribute attr, Type declaredType, PropertyInfo propertyInfo) { var searchFieldType = GetSearchFieldType(declaredType, attr, propertyInfo); @@ -230,6 +315,11 @@ private static string[] CommonSerialization(SearchFieldAttribute attr, Type decl } } + if (searchFieldType == "VECTOR" && attr is IndexedAttribute vector) + { + ret.AddRange(VectorSerialization(vector, propertyInfo)); + } + if (attr.Sortable || attr.Aggregatable) { ret.Add("SORTABLE"); diff --git a/src/Redis.OM/Modeling/SearchFieldType.cs b/src/Redis.OM/Modeling/SearchFieldType.cs index 8ab425ce..8c1b9853 100644 --- a/src/Redis.OM/Modeling/SearchFieldType.cs +++ b/src/Redis.OM/Modeling/SearchFieldType.cs @@ -29,5 +29,10 @@ internal enum SearchFieldType /// A generically indexed field - the library will figure out how to index. /// INDEXED = 4, + + /// + /// A vector index field. + /// + VECTOR = 5, } } diff --git a/src/Redis.OM/Modeling/Vectors/DistanceMetric.cs b/src/Redis.OM/Modeling/Vectors/DistanceMetric.cs new file mode 100644 index 00000000..af39609f --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/DistanceMetric.cs @@ -0,0 +1,48 @@ +using System; + +namespace Redis.OM.Modeling +{ + /// + /// The Vector distance metric to use. + /// + public enum DistanceMetric + { + /// + /// Euclidean distance. + /// + L2, + + /// + /// Inner Product. + /// + IP, + + /// + /// The Cosine distance. + /// + COSINE, + } + + /// + /// Quality of life extensions for distance metrics. + /// + internal static class DistanceMetricExtensions + { + /// + /// Gets the Distance metric as a Redis usable string. + /// + /// The distance Metric. + /// A Redis Usable string. + /// thrown if illegal ordinal encountered. + internal static string AsRedisString(this DistanceMetric distanceMetric) + { + return distanceMetric switch + { + DistanceMetric.L2 => "L2", + DistanceMetric.IP => "IP", + DistanceMetric.COSINE => "COSINE", + _ => throw new ArgumentOutOfRangeException(nameof(distanceMetric)) + }; + } + } +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/DoubleVectorizer.cs b/src/Redis.OM/Modeling/Vectors/DoubleVectorizer.cs new file mode 100644 index 00000000..1cc84080 --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/DoubleVectorizer.cs @@ -0,0 +1,29 @@ +using System; +using System.Linq; +using Redis.OM.Contracts; + +namespace Redis.OM.Modeling.Vectors; + +/// +/// A vectorizer for double arrays. +/// +public class DoubleVectorizer : IVectorizer +{ + /// + /// Initializes a new instance of the class. + /// + /// The dimensions. + public DoubleVectorizer(int dim) + { + Dim = dim; + } + + /// + public VectorType VectorType => VectorType.FLOAT64; + + /// + public int Dim { get; } + + /// + public byte[] Vectorize(double[] obj) => obj.SelectMany(BitConverter.GetBytes).ToArray(); +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/DoubleVectorizerAttribute.cs b/src/Redis.OM/Modeling/Vectors/DoubleVectorizerAttribute.cs new file mode 100644 index 00000000..e9148b52 --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/DoubleVectorizerAttribute.cs @@ -0,0 +1,40 @@ +using System; +using Redis.OM.Contracts; + +namespace Redis.OM.Modeling.Vectors; + +/// +/// A vectorizer attribute for doubles. +/// +public class DoubleVectorizerAttribute : VectorizerAttribute +{ + /// + /// Initializes a new instance of the class. + /// + /// the dimensions. + public DoubleVectorizerAttribute(int dim) + { + Dim = dim; + Vectorizer = new DoubleVectorizer(dim); + } + + /// + public override IVectorizer Vectorizer { get; } + + /// + public override VectorType VectorType => VectorType.FLOAT64; + + /// + public override int Dim { get; } + + /// + public override byte[] Vectorize(object obj) + { + if (obj is not double[] doubles) + { + throw new InvalidOperationException("Provided object for vectorization must be a double[]"); + } + + return Vectorizer.Vectorize(doubles); + } +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/FloatVectorizer.cs b/src/Redis.OM/Modeling/Vectors/FloatVectorizer.cs new file mode 100644 index 00000000..9e511b1a --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/FloatVectorizer.cs @@ -0,0 +1,30 @@ +using System; +using System.Linq; +using Redis.OM.Contracts; +using Redis.OM.Modeling; + +namespace Redis.OM; + +/// +/// A vectorizer for float arrays. +/// +public class FloatVectorizer : IVectorizer +{ + /// + /// Initializes a new instance of the class. + /// + /// The dimensions. + public FloatVectorizer(int dim) + { + Dim = dim; + } + + /// + public VectorType VectorType => VectorType.FLOAT32; + + /// + public int Dim { get; } + + /// + public byte[] Vectorize(float[] obj) => obj.SelectMany(BitConverter.GetBytes).ToArray(); +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/FloatVectorizerAttribute.cs b/src/Redis.OM/Modeling/Vectors/FloatVectorizerAttribute.cs new file mode 100644 index 00000000..bc9d952a --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/FloatVectorizerAttribute.cs @@ -0,0 +1,39 @@ +using System; +using System.Linq; +using Redis.OM.Contracts; + +namespace Redis.OM.Modeling.Vectors; + +/// +public class FloatVectorizerAttribute : VectorizerAttribute +{ + /// + /// Initializes a new instance of the class. + /// + /// The dimensions of the vector. + public FloatVectorizerAttribute(int dim) + { + Dim = dim; + Vectorizer = new FloatVectorizer(dim); + } + + /// + public override VectorType VectorType => VectorType.FLOAT32; + + /// + public override int Dim { get; } + + /// + public override IVectorizer Vectorizer { get; } + + /// + public override byte[] Vectorize(object obj) + { + if (obj is not float[] floats) + { + throw new InvalidOperationException("Must pass in an array of floats"); + } + + return Vectorizer.Vectorize(floats); + } +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/JsonScoreConverter.cs b/src/Redis.OM/Modeling/Vectors/JsonScoreConverter.cs new file mode 100644 index 00000000..e4c70284 --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/JsonScoreConverter.cs @@ -0,0 +1,27 @@ +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Redis.OM.Modeling +{ + /// + /// ignores the Json Score field. + /// + internal class JsonScoreConverter : JsonConverter + { + /// + public override bool CanConvert(Type typeToConvert) => typeToConvert == typeof(double) || typeToConvert == typeof(double?); + + /// + public override object? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + return -1.0; + } + + /// + public override void Write(Utf8JsonWriter writer, object value, JsonSerializerOptions options) + { + writer.WriteNumberValue(-1.0); + } + } +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/VectorAlgorithm.cs b/src/Redis.OM/Modeling/Vectors/VectorAlgorithm.cs new file mode 100644 index 00000000..f4d8c9db --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/VectorAlgorithm.cs @@ -0,0 +1,43 @@ +using System; + +namespace Redis.OM.Modeling +{ + /// + /// The Vector Algorithm. + /// + public enum VectorAlgorithm + { + /// + /// Uses a brute force algorithm to find nearest neighbors. + /// + FLAT = 0, + + /// + /// Uses the Hierarchical Small World Algorithm to build an efficient graph structure to + /// retrieve approximate nearest neighbors + /// + HNSW = 1, + } + + /// + /// Quality of life functions for VectorAlgorithm enum. + /// + internal static class VectorAlgorithmExtensions + { + /// + /// Returns the algorithm as a Redis Serialized String. + /// + /// The algorithm to use. + /// The algorithm's name. + /// Thrown if an invalid Algorithm is passed. + internal static string AsRedisString(this VectorAlgorithm algorithm) + { + return algorithm switch + { + VectorAlgorithm.FLAT => "FLAT", + VectorAlgorithm.HNSW => "HNSW", + _ => throw new ArgumentOutOfRangeException(nameof(algorithm)) + }; + } + } +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/VectorJsonConverter.cs b/src/Redis.OM/Modeling/Vectors/VectorJsonConverter.cs new file mode 100644 index 00000000..be74169c --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/VectorJsonConverter.cs @@ -0,0 +1,184 @@ +using System; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using Redis.OM.Modeling.Vectors; + +namespace Redis.OM.Modeling +{ + /// + /// Converts the provided object to a json vector. + /// + /// The type. + internal class VectorJsonConverter : JsonConverter> + where T : class + { + private readonly VectorizerAttribute _vectorizerAttribute; + + /// + /// Initializes a new instance of the class. + /// + /// the attribute that will be used for vectorization. + internal VectorJsonConverter(VectorizerAttribute attribute) + { + _vectorizerAttribute = attribute; + } + + /// + public override Vector? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + T res; + reader.Read(); + byte[] embedding; + if (_vectorizerAttribute is FloatVectorizerAttribute floatVectorizer) + { + float[] floats = new float[floatVectorizer.Dim]; + for (var i = 0; i < floatVectorizer.Dim; i++) + { + floats[i] = reader.GetSingle(); + reader.Read(); + } + + res = (floats as T) !; + embedding = floats.GetBytes(); + } + else if (_vectorizerAttribute is DoubleVectorizerAttribute doubleVectorizer) + { + double[] doubles = new double[doubleVectorizer.Dim]; + for (var i = 0; i < doubleVectorizer.Dim; i++) + { + doubles[i] = reader.GetDouble(); + reader.Read(); + } + + res = (doubles as T) !; + embedding = doubles.GetBytes(); + } + else + { + reader.Read(); + res = JsonSerializer.Deserialize(reader.GetString() !) !; + reader.Read(); + reader.Read(); // Vector + reader.Read(); // start array + if (_vectorizerAttribute.VectorType == VectorType.FLOAT32) + { + var floats = new float[_vectorizerAttribute.Dim]; + for (var i = 0; i < _vectorizerAttribute.Dim; i++) + { + floats[i] = reader.GetSingle(); + reader.Read(); // each item + } + + embedding = floats.GetBytes(); + } + else + { + var doubles = new double[_vectorizerAttribute.Dim]; + for (var i = 0; i < _vectorizerAttribute.Dim; i++) + { + doubles[i] = reader.GetDouble(); + reader.Read(); // each item + } + + embedding = doubles.GetBytes(); + } + + reader.Read(); // end array + } + + var vector = new Vector(res!) { Embedding = embedding }; + return vector; + } + + /// + public override void Write(Utf8JsonWriter writer, Vector value, JsonSerializerOptions options) + { + if (_vectorizerAttribute is DoubleVectorizerAttribute && value is Vector doubleVector) + { + writer.WriteStartArray(); + foreach (var d in doubleVector.Value) + { + writer.WriteNumberValue(d); + } + + writer.WriteEndArray(); + return; + } + + if (_vectorizerAttribute is FloatVectorizerAttribute && value is Vector floatVector) + { + writer.WriteStartArray(); + foreach (var d in floatVector.Value) + { + writer.WriteNumberValue(d); + } + + writer.WriteEndArray(); + return; + } + + writer.WriteStartObject(); + writer.WritePropertyName("Value"); + writer.WriteStringValue(JsonSerializer.Serialize(value.Obj)); + if (value.Embedding is null) + { + value.Embed(_vectorizerAttribute); + } + + var bytes = value.Embedding!; + var jagged = SplitIntoJaggedArray(bytes, _vectorizerAttribute.VectorType == VectorType.FLOAT32 ? 4 : 8); + writer.WritePropertyName("Vector"); + if (_vectorizerAttribute.VectorType == VectorType.FLOAT32) + { + var floats = jagged.Select(a => BitConverter.ToSingle(a, 0)).ToArray(); + writer.WriteStartArray(); + foreach (var f in floats) + { + writer.WriteNumberValue(f); + } + + writer.WriteEndArray(); + } + else + { + var doubles = jagged.Select(x => BitConverter.ToDouble(x, 0)).ToArray(); + writer.WriteStartArray(); + foreach (var d in doubles) + { + writer.WriteNumberValue(d); + } + + writer.WriteEndArray(); + } + + writer.WriteEndObject(); + } + + /// + public override bool CanConvert(Type typeToConvert) => true; + + /// + /// Converts input bytes to Jagged array. + /// + /// the bytes to parse. + /// Size of the jagged arrays. + /// A jagged array of bytes. + /// thrown if the vector is not correctly balanced. + internal static byte[][] SplitIntoJaggedArray(byte[] bytes, int numBytesPerArray) + { + if (bytes.Length % numBytesPerArray != 0) + { + throw new ArgumentException("Unbalanced vector."); + } + + var result = new byte[bytes.Length / numBytesPerArray][]; + for (var i = 0; i < bytes.Length; i += numBytesPerArray) + { + result[i / numBytesPerArray] = bytes.Skip(i).Take(numBytesPerArray).ToArray(); + } + + return result; + } + } +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/VectorResult.cs b/src/Redis.OM/Modeling/Vectors/VectorResult.cs new file mode 100644 index 00000000..29ea8c0d --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/VectorResult.cs @@ -0,0 +1,30 @@ +namespace Redis.OM +{ + /// + /// Represents a vector result with its score and the document associated with it. + /// + /// the document type. + public class VectorResult + { + /// + /// Initializes a new instance of the class. + /// + /// the document. + /// the score. + internal VectorResult(T document, double score) + { + Score = score; + Document = document; + } + + /// + /// Gets the document. + /// + public T Document { get; } + + /// + /// Gets the score. + /// + public double Score { get; } + } +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/VectorScoreField.cs b/src/Redis.OM/Modeling/Vectors/VectorScoreField.cs new file mode 100644 index 00000000..9584e7f0 --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/VectorScoreField.cs @@ -0,0 +1,18 @@ +using System; +using System.Text.Json.Serialization; + +namespace Redis.OM.Modeling +{ + /// + /// Attribute to decorate vector score field. A field decorated with this will have the sentinel value -1 when + /// the score is not present in the result. + /// + public class KnnVectorScore : JsonConverterAttribute + { + /// + public override JsonConverter? CreateConverter(Type typeToConvert) + { + return new JsonScoreConverter(); + } + } +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/VectorScores.cs b/src/Redis.OM/Modeling/Vectors/VectorScores.cs new file mode 100644 index 00000000..b6faefbb --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/VectorScores.cs @@ -0,0 +1,48 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Serialization; + +namespace Redis.OM.Modeling.Vectors +{ + /// + /// A collector for vector scores, binding this to your model causes Redis OM to bind all scores resulting from + /// a vector query to it. Otherwise it will be ignored when it is added to Redis. + /// + public class VectorScores + { + /// + /// The Range score suffix. + /// + internal const string RangeScoreSuffix = "_RangeScore"; + + /// + /// The Nearest neighbor score name. + /// + internal const string NearestNeighborScoreName = "KnnNeighborScore"; + + /// + /// Initializes a new instance of the class. + /// + internal VectorScores() + { + } + + /// + /// Gets the nearest neighbor score. + /// + [JsonIgnore] + public double? NearestNeighborsScore { get; internal set; } + + /// + /// Gets the first score from the vector ranges. + /// + [JsonIgnore] + public double? RangeScore => RangeScores.FirstOrDefault().Value; + + /// + /// Gets or sets the range score dictionary. + /// + [JsonIgnore] + internal Dictionary RangeScores { get; set; } = new (); + } +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/VectorType.cs b/src/Redis.OM/Modeling/Vectors/VectorType.cs new file mode 100644 index 00000000..dedf8e71 --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/VectorType.cs @@ -0,0 +1,42 @@ +using System; + +namespace Redis.OM.Modeling +{ + /// + /// Type of Vector. + /// + public enum VectorType + { + /// + /// Float 32s. + /// + FLOAT32, + + /// + /// Float 64s. + /// + FLOAT64, + } + + /// + /// Extensions for VectorType. + /// + internal static class VectorTypeExtensions + { + /// + /// Gets the Vector type as a Redis usable string. + /// + /// The Vector type. + /// A Redis Usable string. + /// Thrown if illegal value for Vector type is encountered. + internal static string AsRedisString(this VectorType vectorType) + { + return vectorType switch + { + VectorType.FLOAT32 => "FLOAT32", + VectorType.FLOAT64 => "FLOAT64", + _ => throw new ArgumentOutOfRangeException(nameof(vectorType)) + }; + } + } +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/VectorUtils.cs b/src/Redis.OM/Modeling/Vectors/VectorUtils.cs new file mode 100644 index 00000000..047f1bf1 --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/VectorUtils.cs @@ -0,0 +1,224 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; + +namespace Redis.OM.Modeling +{ + /// + /// Helper utilities for handling vectors in Redis. + /// + public static class VectorUtils + { + /// + /// Converts array of doubles to a vector string. + /// + /// the doubles. + /// the vector string. + public static string ToVecString(IEnumerable doubles) + { + var bytes = doubles.SelectMany(BitConverter.GetBytes).ToArray(); + return BytesToVecStr(bytes); + } + + /// + /// Converts array of floats to a vector string. + /// + /// the floats. + /// the vector string. + public static string ToVecString(IEnumerable floats) + { + var bytes = floats.SelectMany(BitConverter.GetBytes).ToArray(); + return BytesToVecStr(bytes); + } + + /// + /// Converts the double to a binary safe redis vector string. + /// + /// the double. + /// The binary safe redis vector string. + public static string DoubleToVecStr(double d) + { + return BytesToVecStr(BitConverter.GetBytes(d)); + } + + /// + /// Converts the bytes to a binary safe redis string. + /// + /// the bytes to convert. + /// the binary safe redis String. + public static string BytesToVecStr(byte[] bytes) + { + var sb = new StringBuilder(); + foreach (var b in bytes) + { + switch (b) + { + case 0x08: + sb.Append("\\b"); + break; + case 0x22: + sb.Append("\""); + break; + case >= 0x20 and <= 0x7f: + sb.Append((char)b); + break; + default: + sb.Append($"\\x{Convert.ToString(b, 16).PadLeft(2, '0')}"); + break; + } + } + + return sb.ToString(); + } + + /// + /// Converts Vector String to array of doubles. + /// + /// the reply. + /// the doubles. + /// Thrown if unbalanced. + public static double[] VecStrToDoubles(RedisReply reply) + { + var bytes = (byte[]?)reply ?? throw new InvalidCastException("Could not convert result to raw result."); + if (bytes.Length % 8 != 0) + { + throw new ArgumentException("Unbalanced Vector String"); + } + + var doubles = new double[bytes.Length / 8]; + for (var i = 0; i < bytes.Length; i += 8) + { + doubles[i / 8] = BitConverter.ToDouble(bytes, i); + } + + return doubles; + } + + /// + /// converts the vector bytes to an array of doubles. + /// + /// the bytes. + /// The doubles. + /// Thrown if unbalanced. + public static double[] VectorBytesToDoubles(byte[] bytes) + { + if (bytes.Length % 8 != 0) + { + throw new ArgumentException("Unbalanced Vector String"); + } + + var doubles = new double[bytes.Length / 8]; + for (var i = 0; i < bytes.Length; i += 8) + { + doubles[i / 8] = BitConverter.ToDouble(bytes, i); + } + + return doubles; + } + + /// + /// Converts Vector String to array of doubles. + /// + /// the reply. + /// the doubles. + /// Thrown if unbalanced. + public static double[] VecStrToDoubles(string reply) + { + var bytes = Encoding.ASCII.GetBytes(reply); + return VectorBytesToDoubles(bytes); + } + + /// + /// Parses the bytes into an array of floats. + /// + /// the bytes. + /// the floats. + /// Thrown if bytes are unbalanced. + public static float[] VectorBytesToFloats(byte[] bytes) + { + if (bytes.Length % 4 != 0) + { + throw new ArgumentException("Unbalanced Vector String"); + } + + var floats = new float[bytes.Length / 4]; + for (var i = 0; i < bytes.Length; i += 4) + { + floats[i / 4] = BitConverter.ToSingle(bytes, i); + } + + return floats; + } + + /// + /// Parses a vector string to an array of floats. + /// + /// the reply. + /// The floats. + /// thrown if unbalanced. + public static float[] VectorStrToFloats(RedisReply reply) + { + var bytes = (byte[]?)reply ?? throw new InvalidCastException("Could not convert result to raw result."); + return VectorBytesToFloats(bytes); + } + + /// + /// Converts binary safe Redis blob to double. + /// + /// the string. + /// the double the string represents. + public static double DoubleFromVecStr(string str) + { + var bytes = VecStrToBytes(str); + return BitConverter.ToDouble(bytes, 0); + } + + /// + /// Converts the binary safe vector string from Redis to an array of bytes. + /// + /// the string to convert back to bytes. + /// the bytes from the string. + public static byte[] VecStrToBytes(string str) + { + var bytes = new List(); + var i = 0; + while (i < str.Length) + { + if (str[i] == '\\' && i + 1 < str.Length && str[i + 1] == '\\') + { + bytes.Add((byte)'\\'); + i += 2; + } + else if (str[i] == '\\' && i + 3 < str.Length && str[i + 1] == 'x') + { + // byte literal, interpret from hex. + bytes.Add(byte.Parse(str.Substring(i + 2, 2), NumberStyles.HexNumber)); + i += 4; + } + else + { + bytes.Add((byte)str[i]); + i++; + } + } + + return bytes.ToArray(); + } + + /// + /// Converts doubles to array of bytes. + /// + /// the doubles. + /// the array of bytes. + internal static byte[] GetBytes(this double[] doubles) => doubles.SelectMany(BitConverter.GetBytes).ToArray(); + + /// + /// Converts floats to array of bytes. + /// + /// the floats. + /// the array of bytes. + internal static byte[] GetBytes(this float[] floats) => floats.SelectMany(BitConverter.GetBytes).ToArray(); + } +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/Vectors/VectorizerAttribute.cs b/src/Redis.OM/Modeling/Vectors/VectorizerAttribute.cs new file mode 100644 index 00000000..917f07ae --- /dev/null +++ b/src/Redis.OM/Modeling/Vectors/VectorizerAttribute.cs @@ -0,0 +1,60 @@ +using System; +using System.Text.Json.Serialization; +using Redis.OM.Contracts; + +namespace Redis.OM.Modeling +{ + /// + /// A vectorizer attribute. + /// + public abstract class VectorizerAttribute : JsonConverterAttribute + { + /// + /// Gets the number of bytes per vector. + /// + public int BytesPerVector => VectorType == VectorType.FLOAT32 ? 4 * Dim : 8 * Dim; + + /// + /// Gets the vector Type generated by the vectorizer. + /// + public abstract VectorType VectorType { get; } + + /// + /// Gets the vector dimension of the vectors generated by the vectorizer. + /// + public abstract int Dim { get; } + + /// + /// Converts the provided object to a vector. + /// + /// the object to convert. + /// A byte array containing the vectorized data. + public abstract byte[] Vectorize(object obj); + } + + /// + /// Method for converting a field into a vector. + /// + /// The type. +#pragma warning disable SA1402 + public abstract class VectorizerAttribute : VectorizerAttribute + where T : class +#pragma warning restore SA1402 + { + /// + /// Gets the vectorizer for this attribute. + /// + /// The vectorizer. + public abstract IVectorizer Vectorizer { get; } + + /// + /// Creates the json converter fulfilled by this attribute. + /// + /// The type to convert. + /// The Json Converter. + public override JsonConverter? CreateConverter(Type typeToConvert) + { + return new VectorJsonConverter(this); + } + } +} \ No newline at end of file diff --git a/src/Redis.OM/RediSearchCommands.cs b/src/Redis.OM/RediSearchCommands.cs index 1b75b776..fad794ff 100644 --- a/src/Redis.OM/RediSearchCommands.cs +++ b/src/Redis.OM/RediSearchCommands.cs @@ -91,6 +91,56 @@ public static async Task CreateIndexAsync(this IRedisConnection connection } } + /// + /// Get index information. + /// + /// the connection. + /// The index name. + /// Strong-typed result of FT.INFO idx. + public static RedisIndexInfo? GetIndexInfo(this IRedisConnection connection, string indexName) + { + try + { + var redisReply = connection.Execute("FT.INFO", indexName); + var redisIndexInfo = new RedisIndexInfo(redisReply); + return redisIndexInfo; + } + catch (Exception ex) + { + if (ex.Message.ToLower().Contains("unknown index name")) + { + return null; + } + + throw; + } + } + + /// + /// Get index information. + /// + /// the connection. + /// the index name. + /// Strong-typed result of FT.INFO idx. + public static async Task GetIndexInfoAsync(this IRedisConnection connection, string indexName) + { + try + { + var redisReply = await connection.ExecuteAsync("FT.INFO", indexName).ConfigureAwait(false); + var redisIndexInfo = new RedisIndexInfo(redisReply); + return redisIndexInfo; + } + catch (Exception ex) + { + if (ex.Message.ToLower().Contains("unknown index name")) + { + return null; + } + + throw; + } + } + /// /// Get index information. /// diff --git a/src/Redis.OM/Redis.OM.csproj b/src/Redis.OM/Redis.OM.csproj index b5b8e2f6..93e68ddc 100644 --- a/src/Redis.OM/Redis.OM.csproj +++ b/src/Redis.OM/Redis.OM.csproj @@ -2,13 +2,13 @@ netstandard2.0 - 9.0 + 11 Redis.OM enable true - 0.5.5 - 0.5.5 - https://github.com/redis/redis-om-dotnet/releases/tag/v0.5.5 + 0.6.0 + 0.6.0 + https://github.com/redis/redis-om-dotnet/releases/tag/v0.6.0 Object Mapping and More for Redis Redis OM Steve Lorello diff --git a/src/Redis.OM/RedisCommands.cs b/src/Redis.OM/RedisCommands.cs index 3d1700ee..2db3bce3 100644 --- a/src/Redis.OM/RedisCommands.cs +++ b/src/Redis.OM/RedisCommands.cs @@ -82,9 +82,9 @@ public static async Task SetAsync(this IRedisConnection connection, obje /// the key. /// the field value pairs to set. /// How many new fields were created. - public static async Task HSetAsync(this IRedisConnection connection, string key, params KeyValuePair[] fieldValues) + public static async Task HSetAsync(this IRedisConnection connection, string key, params KeyValuePair[] fieldValues) { - var args = new List { key }; + var args = new List { key }; foreach (var kvp in fieldValues) { args.Add(kvp.Key); @@ -102,9 +102,9 @@ public static async Task HSetAsync(this IRedisConnection connection, string /// the the timespan to set for your (TTL). /// the field value pairs to set. /// How many new fields were created. - public static async Task HSetAsync(this IRedisConnection connection, string key, TimeSpan timeSpan, params KeyValuePair[] fieldValues) + public static async Task HSetAsync(this IRedisConnection connection, string key, TimeSpan timeSpan, params KeyValuePair[] fieldValues) { - var args = new List { key }; + var args = new List { key }; foreach (var kvp in fieldValues) { args.Add(kvp.Key); @@ -224,9 +224,9 @@ public static async Task JsonSetAsync(this IRedisConnection connection, st /// the the timespan to set for your (TTL). /// the field value pairs to set. /// How many new fields were created. - public static int HSet(this IRedisConnection connection, string key, TimeSpan timeSpan, params KeyValuePair[] fieldValues) + public static int HSet(this IRedisConnection connection, string key, TimeSpan timeSpan, params KeyValuePair[] fieldValues) { - var args = new List(); + var args = new List(); args.Add(key); foreach (var kvp in fieldValues) { @@ -244,9 +244,9 @@ public static int HSet(this IRedisConnection connection, string key, TimeSpan ti /// the key. /// the field value pairs to set. /// How many new fields were created. - public static int HSet(this IRedisConnection connection, string key, params KeyValuePair[] fieldValues) + public static int HSet(this IRedisConnection connection, string key, params KeyValuePair[] fieldValues) { - var args = new List { key }; + var args = new List { key }; foreach (var kvp in fieldValues) { args.Add(kvp.Key); @@ -413,7 +413,7 @@ public static string Set(this IRedisConnection connection, object obj) } var kvps = obj.BuildHashSet(); - var argsList = new List(); + var argsList = new List(); int? res = null; argsList.Add(timespan != null ? ((long)timespan.Value.TotalMilliseconds).ToString() : "-1"); foreach (var kvp in kvps) @@ -464,7 +464,7 @@ public static string Set(this IRedisConnection connection, object obj) } var kvps = obj.BuildHashSet(); - var argsList = new List(); + var argsList = new List(); int? res = null; argsList.Add(timespan != null ? ((long)timespan.Value.TotalMilliseconds).ToString() : "-1"); foreach (var kvp in kvps) @@ -598,9 +598,9 @@ public static string Set(this IRedisConnection connection, object obj, TimeSpan /// the connection. /// the key name. /// the object serialized into a dictionary. - public static IDictionary HGetAll(this IRedisConnection connection, string keyName) + public static IDictionary HGetAll(this IRedisConnection connection, string keyName) { - var ret = new Dictionary(); + var ret = new Dictionary(); var res = connection.Execute("HGETALL", keyName).ToArray(); for (var i = 0; i < res.Length; i += 2) { @@ -616,9 +616,9 @@ public static IDictionary HGetAll(this IRedisConnection connecti /// the connection. /// the key name. /// the object serialized into a dictionary. - public static async Task> HGetAllAsync(this IRedisConnection connection, string keyName) + public static async Task> HGetAllAsync(this IRedisConnection connection, string keyName) { - var ret = new Dictionary(); + var ret = new Dictionary(); var res = (await connection.ExecuteAsync("HGETALL", keyName)).ToArray(); for (var i = 0; i < res.Length; i += 2) { @@ -638,7 +638,7 @@ public static async Task> HGetAllAsync(this IRedisCo /// the full script. /// the result. /// Thrown if the script cannot be resolved either the script is empty or the script name has not been encountered. - public static async Task CreateAndEvalAsync(this IRedisConnection connection, string scriptName, string[] keys, string[] argv, string fullScript = "") + public static async Task CreateAndEvalAsync(this IRedisConnection connection, string scriptName, string[] keys, object[] argv, string fullScript = "") { string sha; if (!Scripts.ShaCollection.TryGetValue(scriptName, out sha)) @@ -659,7 +659,7 @@ public static async Task> HGetAllAsync(this IRedisCo Scripts.ShaCollection[scriptName] = sha; } - var args = new List + var args = new List { sha, keys.Count().ToString(), @@ -691,7 +691,7 @@ public static async Task> HGetAllAsync(this IRedisCo /// the full script. /// the result. /// Thrown if the script cannot be resolved either the script is empty or the script name has not been encountered. - public static int? CreateAndEval(this IRedisConnection connection, string scriptName, string[] keys, string[] argv, string fullScript = "") + public static int? CreateAndEval(this IRedisConnection connection, string scriptName, string[] keys, object[] argv, string fullScript = "") { if (!Scripts.ShaCollection.ContainsKey(scriptName)) { @@ -712,7 +712,7 @@ public static async Task> HGetAllAsync(this IRedisCo Scripts.ShaCollection[scriptName] = sha; } - var args = new List + var args = new List { Scripts.ShaCollection[scriptName], keys.Count().ToString(), @@ -784,7 +784,7 @@ internal static void UnlinkAndSet(this IRedisConnection connection, string ke else { var hash = value.BuildHashSet(); - var args = new List((hash.Keys.Count * 2) + 1); + var args = new List((hash.Keys.Count * 2) + 1); args.Add(hash.Keys.Count.ToString()); foreach (var pair in hash) { @@ -815,7 +815,7 @@ internal static async Task UnlinkAndSetAsync(this IRedisConnection connection else { var hash = value.BuildHashSet(); - var args = new List((hash.Keys.Count * 2) + 1); + var args = new List((hash.Keys.Count * 2) + 1); args.Add(hash.Keys.Count.ToString()); foreach (var pair in hash) { @@ -830,24 +830,24 @@ internal static async Task UnlinkAndSetAsync(this IRedisConnection connection private static RedisReply[] SendCommandWithExpiry( this IRedisConnection connection, string command, - string[] args, + object[] args, string keyToExpire, TimeSpan ts) { var commandTuple = Tuple.Create(command, args); - var expireTuple = Tuple.Create("PEXPIRE", new[] { keyToExpire, ((long)ts.TotalMilliseconds).ToString(CultureInfo.InvariantCulture) }); + var expireTuple = Tuple.Create("PEXPIRE", new object[] { keyToExpire, ((long)ts.TotalMilliseconds).ToString(CultureInfo.InvariantCulture) }); return connection.ExecuteInTransaction(new[] { commandTuple, expireTuple }); } private static Task SendCommandWithExpiryAsync( this IRedisConnection connection, string command, - string[] args, + object[] args, string keyToExpire, TimeSpan ts) { var commandTuple = Tuple.Create(command, args); - var expireTuple = Tuple.Create("PEXPIRE", new[] { keyToExpire, ((long)ts.TotalMilliseconds).ToString(CultureInfo.InvariantCulture) }); + var expireTuple = Tuple.Create("PEXPIRE", new object[] { keyToExpire, ((long)ts.TotalMilliseconds).ToString(CultureInfo.InvariantCulture) }); return connection.ExecuteInTransactionAsync(new[] { commandTuple, expireTuple }); } } diff --git a/src/Redis.OM/RedisConnection.cs b/src/Redis.OM/RedisConnection.cs index 997f4ea9..5a36b7c3 100644 --- a/src/Redis.OM/RedisConnection.cs +++ b/src/Redis.OM/RedisConnection.cs @@ -24,7 +24,7 @@ internal RedisConnection(IDatabase db) } /// - public RedisReply Execute(string command, params string[] args) + public RedisReply Execute(string command, params object[] args) { try { @@ -38,11 +38,11 @@ public RedisReply Execute(string command, params string[] args) } /// - public async Task ExecuteAsync(string command, params string[] args) + public async Task ExecuteAsync(string command, params object[] args) { try { - var result = await _db.ExecuteAsync(command, args); + var result = await _db.ExecuteAsync(command, args).ConfigureAwait(false); return new RedisReply(result); } catch (Exception ex) @@ -52,7 +52,7 @@ public async Task ExecuteAsync(string command, params string[] args) } /// - public RedisReply[] ExecuteInTransaction(Tuple[] commandArgsTuples) + public async Task ExecuteInTransactionAsync(Tuple[] commandArgsTuples) { var transaction = _db.CreateTransaction(); var tasks = new List>(); @@ -61,13 +61,13 @@ public RedisReply[] ExecuteInTransaction(Tuple[] commandArgsTu tasks.Add(transaction.ExecuteAsync(tuple.Item1, tuple.Item2)); } - transaction.Execute(); - Task.WhenAll(tasks).Wait(); + await transaction.ExecuteAsync().ConfigureAwait(false); + await Task.WhenAll(tasks).ConfigureAwait(false); return tasks.Select(x => new RedisReply(x.Result)).ToArray(); } /// - public async Task ExecuteInTransactionAsync(Tuple[] commandArgsTuples) + public RedisReply[] ExecuteInTransaction(Tuple[] commandArgsTuples) { var transaction = _db.CreateTransaction(); var tasks = new List>(); @@ -76,8 +76,8 @@ public async Task ExecuteInTransactionAsync(Tuple new RedisReply(x.Result)).ToArray(); } diff --git a/src/Redis.OM/RedisObjectHandler.cs b/src/Redis.OM/RedisObjectHandler.cs index 55270a63..4f765670 100644 --- a/src/Redis.OM/RedisObjectHandler.cs +++ b/src/Redis.OM/RedisObjectHandler.cs @@ -6,8 +6,10 @@ using System.Runtime.CompilerServices; using System.Text; using System.Text.Json; +using System.Web; using Redis.OM.Contracts; using Redis.OM.Modeling; +using Redis.OM.Modeling.Vectors; [assembly: InternalsVisibleTo("Redis.OM.POC")] @@ -29,41 +31,6 @@ internal static class RedisObjectHandler { typeof(ulong), default(ulong) }, }; - /// - /// Builds object from provided hash set. - /// - /// Hash set to build item from. - /// The type to construct. - /// An instance of the requested object. - /// Throws an exception if Deserialization fails. - internal static T FromHashSet(IDictionary hash) - where T : notnull - { - if (typeof(IRedisHydrateable).IsAssignableFrom(typeof(T))) - { - var obj = Activator.CreateInstance(); - ((IRedisHydrateable)obj).Hydrate(hash); - return obj; - } - - var attr = Attribute.GetCustomAttribute(typeof(T), typeof(DocumentAttribute)) as DocumentAttribute; - string asJson; - if (attr != null && attr.StorageType == StorageType.Json && hash.ContainsKey("$")) - { - asJson = hash["$"]; - } - else if (hash.Keys.Count > 0 && hash.Keys.All(x => x.StartsWith("$"))) - { - asJson = hash.Values.First(); - } - else - { - asJson = SendToJson(hash, typeof(T)); - } - - return JsonSerializer.Deserialize(asJson, RedisSerializationSettings.JsonSerializerOptions) ?? throw new Exception("Deserialization fail"); - } - /// /// Tries to parse the hash set into a fully or partially hydrated object. /// @@ -87,16 +54,39 @@ internal static T FromHashSet(IDictionary hash) { asJson = hash["$"]; } - else if (attr != null) + else if (hash.Keys.Count > 0 && hash.Keys.All(x => x.StartsWith("$"))) { - asJson = SendToJson(stringDictionary, typeof(T)); + asJson = hash.Values.First(); } else { - throw new ArgumentException("Type must be decorated with a DocumentAttribute"); + asJson = SendToJson(hash, typeof(T)); } - return JsonSerializer.Deserialize(asJson, RedisSerializationSettings.JsonSerializerOptions) ?? throw new Exception("Deserialization fail"); + var res = JsonSerializer.Deserialize(asJson, RedisSerializationSettings.JsonSerializerOptions) ?? throw new Exception("Deserialization fail"); + if (hash.ContainsKey(VectorScores.NearestNeighborScoreName) || hash.Keys.Any(x => x.EndsWith(VectorScores.RangeScoreSuffix))) + { + var vectorScores = new VectorScores(); + if (hash.ContainsKey(VectorScores.NearestNeighborScoreName)) + { + vectorScores.NearestNeighborsScore = ParseScoreFromString(hash[VectorScores.NearestNeighborScoreName]); + } + + foreach (var key in hash.Keys.Where(x => x.EndsWith(VectorScores.RangeScoreSuffix))) + { + var strippedKey = key.Substring(0, key.Length - VectorScores.RangeScoreSuffix.Length); + var score = ParseScoreFromString(hash[key]); + vectorScores.RangeScores.Add(strippedKey, score); + } + + var scoreProperties = typeof(T).GetProperties().Where(x => x.PropertyType == typeof(VectorScores)); + foreach (var p in scoreProperties) + { + p.SetValue(res, vectorScores); + } + } + + return res; } /// @@ -105,7 +95,7 @@ internal static T FromHashSet(IDictionary hash) /// The hash. /// The type to deserialize to. /// the deserialized object. - internal static object? FromHashSet(IDictionary hash, Type type) + internal static object? FromHashSet(IDictionary hash, Type type) { var asJson = SendToJson(hash, type); return JsonSerializer.Deserialize(asJson, type); @@ -299,7 +289,7 @@ internal static void ExtractPropertyName(PropertyInfo property, ref string prope internal static T ToObject(this RedisReply val) where T : notnull { - var hash = new Dictionary(); + var hash = new Dictionary(); var vals = val.ToArray(); for (var i = 0; i < vals.Length; i += 2) { @@ -314,24 +304,47 @@ internal static T ToObject(this RedisReply val) /// /// object to be turned into a hash set. /// A hash set generated from the object. - internal static IDictionary BuildHashSet(this object obj) + internal static IDictionary BuildHashSet(this object obj) { if (obj is IRedisHydrateable hydrateable) { - return hydrateable.BuildHashSet(); + return hydrateable.BuildHashSet().ToDictionary(x => x.Key, x => (object)x.Value); } var properties = obj .GetType() .GetProperties() .Where(x => x.GetValue(obj) != null); - var hash = new Dictionary(); + var hash = new Dictionary(); foreach (var property in properties) { var type = Nullable.GetUnderlyingType(property.PropertyType) ?? property.PropertyType; var propertyName = property.Name; ExtractPropertyName(property, ref propertyName); + if (property.GetCustomAttributes().Any()) + { + var val = property.GetValue(obj); + var vectorizer = property.GetCustomAttributes().First(); + if (val is not Vector vector) + { + throw new InvalidOperationException("VectorizerAttribute must decorate vectors"); + } + + vector.Embed(vectorizer); + if (vectorizer is FloatVectorizerAttribute or DoubleVectorizerAttribute) + { + hash.Add(propertyName, vector.Embedding!); + } + else + { + hash.Add($"{propertyName}.Vector", vector.Embedding!); + hash.Add($"{propertyName}.Value", JsonSerializer.Serialize(vector.Obj)); + } + + continue; + } + if (type.IsPrimitive || type == typeof(decimal) || type == typeof(string) || type == typeof(GeoLoc) || type == typeof(Ulid) || type == typeof(Guid)) { var val = property.GetValue(obj); @@ -361,6 +374,16 @@ internal static IDictionary BuildHashSet(this object obj) hash.Add(propertyName, new DateTimeOffset(val).ToUnixTimeMilliseconds().ToString()); } } + else if (type == typeof(Vector)) + { + var val = (Vector)obj; + if (val.Embedding is null) + { + throw new InvalidOperationException("Could not use null embedding."); + } + + hash.Add(propertyName, val.Embedding); + } else if (type.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(IEnumerable<>))) { IEnumerable e; @@ -410,7 +433,7 @@ internal static IDictionary BuildHashSet(this object obj) return hash; } - private static string SendToJson(IDictionary hash, Type t) + private static string SendToJson(IDictionary hash, Type t) { var properties = t.GetProperties(); if ((!properties.Any() || t == typeof(Ulid) || t == typeof(Ulid?)) && hash.Count == 1) @@ -424,37 +447,97 @@ private static string SendToJson(IDictionary hash, Type t) var type = Nullable.GetUnderlyingType(property.PropertyType) ?? property.PropertyType; var propertyName = property.Name; ExtractPropertyName(property, ref propertyName); - if (!hash.Any(x => x.Key.StartsWith(propertyName))) + var vectorizer = property.GetCustomAttributes().FirstOrDefault(); + if (vectorizer is FloatVectorizerAttribute || vectorizer is DoubleVectorizerAttribute) + { + if (hash.ContainsKey(propertyName)) + { + string arrString; + if (vectorizer.VectorType == VectorType.FLOAT32) + { + var floats = VectorUtils.VectorStrToFloats(hash[propertyName]); + arrString = string.Join(",", floats); + } + else + { + var doubles = VectorUtils.VecStrToDoubles(hash[propertyName]); + arrString = string.Join(",", doubles); + } + + var valueStr = $"[{arrString}]"; + ret += $"\"{propertyName}\":{valueStr},"; + } + + continue; + } + + var isVectorized = vectorizer != null; + var lookupPropertyName = propertyName + (isVectorized ? ".Value" : string.Empty); + var vectorPropertyName = $"{propertyName}.Vector"; + if (isVectorized && !hash.ContainsKey($"{propertyName}.Value") && !hash.ContainsKey($"{propertyName}.Vector")) + { + continue; + } + + if (isVectorized) + { + ret += $"\"{propertyName}\":{{"; + propertyName = "Value"; + } + + if (!hash.Any(x => x.Key.StartsWith(lookupPropertyName))) { continue; } if (type == typeof(bool) || type == typeof(bool?)) { - if (!hash.ContainsKey(propertyName)) + if (!hash.ContainsKey(lookupPropertyName)) { continue; } - ret += $"\"{propertyName}\":{hash[propertyName].ToLower()},"; + ret += $"\"{propertyName}\":{((string)hash[lookupPropertyName]).ToLower()},"; } else if (type.IsPrimitive || type == typeof(decimal) || type.IsEnum) { - if (!hash.ContainsKey(propertyName)) + if (!hash.ContainsKey(lookupPropertyName)) { continue; } - ret += $"\"{propertyName}\":{hash[propertyName]},"; + ret += $"\"{propertyName}\":{hash[lookupPropertyName]},"; } else if (type == typeof(string) || type == typeof(GeoLoc) || type == typeof(DateTime) || type == typeof(DateTime?) || type == typeof(DateTimeOffset) || type == typeof(Guid) || type == typeof(Guid?) || type == typeof(Ulid) || type == typeof(Ulid?)) { - if (!hash.ContainsKey(propertyName)) + if (!hash.ContainsKey(lookupPropertyName)) { continue; } - ret += $"\"{propertyName}\":\"{hash[propertyName]}\","; + ret += $"\"{propertyName}\":\"{HttpUtility.JavaScriptStringEncode(hash[lookupPropertyName])}\","; + } + else if (type == typeof(Vector)) + { + if (!hash.ContainsKey(lookupPropertyName)) + { + continue; + } + + string arrString; + if (type == typeof(float[])) + { + var floats = VectorUtils.VectorStrToFloats(hash[lookupPropertyName]); + arrString = string.Join(",", floats); + } + else + { + var doubles = VectorUtils.VecStrToDoubles(hash[lookupPropertyName]); + arrString = string.Join(",", doubles); + } + + var valueStr = $"[{arrString}]"; + ret += $"\"{lookupPropertyName}\":{valueStr},"; } else if (type.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(IEnumerable<>))) { @@ -481,7 +564,7 @@ private static string SendToJson(IDictionary hash, Type t) if (innerType == typeof(bool) || innerType == typeof(bool?)) { var val = entries[$"{propertyName}[{i}]"]; - ret += $"{val.ToLower()},"; + ret += $"{((string)val).ToLower()},"; } else if (innerType.IsPrimitive || innerType == typeof(decimal)) { @@ -514,7 +597,7 @@ private static string SendToJson(IDictionary hash, Type t) else { var entries = hash.Where(x => x.Key.StartsWith($"{propertyName}.")) - .Select(x => new KeyValuePair(x.Key.Substring($"{propertyName}.".Length), x.Value)) + .Select(x => new KeyValuePair(x.Key.Substring($"{propertyName}.".Length), x.Value)) .ToDictionary(x => x.Key, x => x.Value); if (entries.Any()) { @@ -525,10 +608,39 @@ private static string SendToJson(IDictionary hash, Type t) else if (hash.ContainsKey(propertyName)) { ret += $"\"{propertyName}\":"; - ret += hash[propertyName]; + ret += hash[lookupPropertyName]; ret += ","; } } + + if (isVectorized) + { + if (vectorizer is null) + { + throw new InvalidOperationException( + "Vector field must be decorated with a vectorizer attribute"); + } + + if (hash.ContainsKey(lookupPropertyName)) + { + ret += $"\"Value\":\"{HttpUtility.JavaScriptStringEncode(hash[lookupPropertyName])}\","; + } + + string arrString; + if (vectorizer.VectorType == VectorType.FLOAT32) + { + var floats = VectorUtils.VectorStrToFloats(hash[vectorPropertyName]); + arrString = string.Join(",", floats); + } + else + { + var doubles = VectorUtils.VecStrToDoubles(hash[vectorPropertyName]); + arrString = string.Join(",", doubles); + } + + var valueStr = $"[{arrString}]"; + ret += $"\"Vector\":{valueStr}}}"; + } } ret = ret.TrimEnd(','); @@ -536,6 +648,22 @@ private static string SendToJson(IDictionary hash, Type t) return ret; } + private static double ParseScoreFromString(string scoreStr) + { + if (double.TryParse(scoreStr, NumberStyles.Any, CultureInfo.InvariantCulture, out var score)) + { + return score; + } + + return scoreStr switch + { + "inf" => double.PositiveInfinity, + "-inf" => double.NegativeInfinity, + "nan" => double.NaN, + _ => throw new ArgumentException($"Could not parse score from {scoreStr}", nameof(scoreStr)) + }; + } + private static Type GetEnumerableType(PropertyInfo pi) { var type = pi.PropertyType.GetElementType(); @@ -553,6 +681,21 @@ private static Type GetEnumerableType(PropertyInfo pi) return type; } + private static byte[] PrimitiveCollectionToVectorBytes(PropertyInfo pi, object obj, Type type) + { + if (type == typeof(double)) + { + return ((IEnumerable)pi.GetValue(obj)).SelectMany(BitConverter.GetBytes).ToArray(); + } + + if (type == typeof(float)) + { + return ((IEnumerable)pi.GetValue(obj)).SelectMany(BitConverter.GetBytes).ToArray(); + } + + throw new ArgumentException("Could not pull a usable type out from property info"); + } + private static IEnumerable PrimitiveCollectionToStrings(PropertyInfo pi, object obj, Type type) { if (type == typeof(bool)) diff --git a/src/Redis.OM/RedisReply.cs b/src/Redis.OM/RedisReply.cs index f29bd38a..7bb5b458 100644 --- a/src/Redis.OM/RedisReply.cs +++ b/src/Redis.OM/RedisReply.cs @@ -1,6 +1,7 @@ using System; using System.Globalization; using System.Linq; +using System.Text; using StackExchange.Redis; namespace Redis.OM @@ -15,7 +16,7 @@ public class RedisReply : IConvertible #pragma warning restore SA1018 private readonly double? _internalDouble; private readonly int? _internalInt; - private readonly string? _internalString; + private readonly byte[]? _raw; private readonly long? _internalLong; /// @@ -30,11 +31,11 @@ public RedisReply(RedisResult result) break; case ResultType.SimpleString: case ResultType.BulkString: - _internalString = (string)result; + _raw = (byte[])result; break; case ResultType.Error: Error = true; - _internalString = result.ToString(); + _raw = (byte[])result; break; case ResultType.Integer: _internalLong = (long)result; @@ -45,6 +46,15 @@ public RedisReply(RedisResult result) } } + /// + /// Initializes a new instance of the class. + /// + /// the raw bytes. + internal RedisReply(byte[] raw) + { + _raw = raw; + } + /// /// Initializes a new instance of the class. /// @@ -60,7 +70,7 @@ internal RedisReply(double val) /// the value. internal RedisReply(string val) { - _internalString = val; + _raw = Encoding.UTF8.GetBytes(val); } /// @@ -108,7 +118,7 @@ public static implicit operator double(RedisReply v) return (double)v._internalDouble; } - if (v._internalString != null && double.TryParse(v._internalString, NumberStyles.Number, CultureInfo.InvariantCulture, out var ret)) + if (v._raw != null && double.TryParse(Encoding.UTF8.GetString(v._raw), NumberStyles.Number, CultureInfo.InvariantCulture, out var ret)) { return ret; } @@ -126,6 +136,13 @@ public static implicit operator double(RedisReply v) throw new InvalidCastException("Could not cast to double"); } + /// + /// implicitly converts the reply to a byte array. + /// + /// the . + /// the byte array. + public static implicit operator byte[]?(RedisReply v) => v._raw; + /// /// implicitly converts the reply to a double. /// @@ -160,7 +177,7 @@ public static implicit operator double(RedisReply v) /// /// the reply. /// the string. - public static implicit operator string(RedisReply v) => v._internalString ?? string.Empty; + public static implicit operator string(RedisReply v) => v._raw is not null ? Encoding.UTF8.GetString(v._raw) : string.Empty; /// /// implicitly converts a string into a redis reply. @@ -182,7 +199,7 @@ public static implicit operator int(RedisReply v) return (int)v._internalInt; } - if (v._internalString != null && int.TryParse(v._internalString, out var ret)) + if (v._raw != null && int.TryParse(Encoding.UTF8.GetString(v._raw), out var ret)) { return ret; } @@ -227,7 +244,7 @@ public static implicit operator long(RedisReply v) return (long)v._internalLong; } - if (v._internalString != null && long.TryParse(v._internalString, out var ret)) + if (v._raw != null && long.TryParse(Encoding.UTF8.GetString(v._raw), out var ret)) { return ret; } @@ -293,9 +310,9 @@ public override string ToString() return _internalInt.ToString(); } - if (_internalString != null) + if (_raw != null) { - return _internalString; + return Encoding.UTF8.GetString(_raw); } return base.ToString(); @@ -325,7 +342,7 @@ public TypeCode GetTypeCode() return TypeCode.Int64; } - if (_internalString != null) + if (_raw != null) { return TypeCode.String; } diff --git a/src/Redis.OM/SearchExtensions.cs b/src/Redis.OM/SearchExtensions.cs index 8d08a37d..788d0f3d 100644 --- a/src/Redis.OM/SearchExtensions.cs +++ b/src/Redis.OM/SearchExtensions.cs @@ -100,6 +100,55 @@ public static IRedisCollection Where(this IRedisCollection source, Expr return new RedisCollection((RedisQueryProvider)source.Provider, exp, source.StateManager, combined, source.SaveState, source.ChunkSize); } + /// + /// Finds nearest neighbors to provided vector. + /// + /// The source. + /// The expression yielding the field to search on. + /// Number of neighbors to search for. + /// The vector or item to search on. + /// The indexed type. + /// The type of the vector. + /// A Redis Collection with a nearest neighbors expression attached to it. + public static IRedisCollection NearestNeighbors(this IRedisCollection source, Expression>> expression, int numNeighbors, Vector item) + where T : notnull + where TKnnType : class + { + var collection = (RedisCollection)source; + var booleanExpression = collection.BooleanExpression; + + var exp = Expression.Call( + null, + GetMethodInfo(NearestNeighbors, source, expression, numNeighbors, item), + new[] { source.Expression, Expression.Quote(expression), Expression.Constant(numNeighbors), Expression.Constant(item) }); + return new RedisCollection((RedisQueryProvider)source.Provider, exp, source.StateManager, booleanExpression, source.SaveState, source.ChunkSize); + } + + /// + /// Finds nearest neighbors to provided vector. + /// + /// The source. + /// The expression yielding the field to search on. + /// Number of neighbors to search for. + /// The vector or item to search on. + /// The indexed type. + /// The type of the vector. + /// A Redis Collection with a nearest neighbors expression attached to it. + public static IRedisCollection NearestNeighbors(this IRedisCollection source, Expression>> expression, int numNeighbors, TKnnType item) + where T : notnull + where TKnnType : class + { + var collection = (RedisCollection)source; + var booleanExpression = collection.BooleanExpression; + + var vector = Vector.Of(item); + var exp = Expression.Call( + null, + GetMethodInfo(NearestNeighbors, source, expression, numNeighbors, vector), + new[] { source.Expression, Expression.Quote(expression), Expression.Constant(numNeighbors), Expression.Constant(vector) }); + return new RedisCollection((RedisQueryProvider)source.Provider, exp, source.StateManager, booleanExpression, source.SaveState, source.ChunkSize); + } + /// /// Specifies which items to pull out of Redis. /// diff --git a/src/Redis.OM/Searching/Query/NearestNeighbors.cs b/src/Redis.OM/Searching/Query/NearestNeighbors.cs new file mode 100644 index 00000000..f2791e96 --- /dev/null +++ b/src/Redis.OM/Searching/Query/NearestNeighbors.cs @@ -0,0 +1,36 @@ +namespace Redis.OM.Searching.Query +{ + /// + /// Components of a KNN search. + /// + public class NearestNeighbors + { + /// + /// Initializes a new instance of the class. + /// + /// The property name to search on. + /// The number of nearest neighbors. + /// The vector blob. + public NearestNeighbors(string propertyName, int numNeighbors, byte[] vectorBlob) + { + PropertyName = propertyName; + NumNeighbors = numNeighbors; + VectorBlob = vectorBlob; + } + + /// + /// Gets the name of the property to perform the vector search on. + /// + public string PropertyName { get; } + + /// + /// Gets the number of neighbors to find. + /// + public int NumNeighbors { get; } + + /// + /// Gets the Vector blob to search on. + /// + public byte[] VectorBlob { get; } + } +} \ No newline at end of file diff --git a/src/Redis.OM/Searching/Query/RedisQuery.cs b/src/Redis.OM/Searching/Query/RedisQuery.cs index 2ee18626..f0e345bc 100644 --- a/src/Redis.OM/Searching/Query/RedisQuery.cs +++ b/src/Redis.OM/Searching/Query/RedisQuery.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.Linq; +using Redis.OM.Modeling.Vectors; namespace Redis.OM.Searching.Query { @@ -18,6 +20,16 @@ public RedisQuery(string index) this.Index = index; } + /// + /// Gets or sets the nearest neighbors query. + /// + public NearestNeighbors? NearestNeighbors { get; set; } + + /// + /// Gets or sets the parameters for the query. + /// + public List Parameters { get; set; } = new (); + /// /// Gets or sets the flags for the query options. /// @@ -63,16 +75,42 @@ public RedisQuery(string index) /// /// the serialized arguments. /// thrown if the index is null. - internal string[] SerializeQuery() + internal object[] SerializeQuery() { - var ret = new List(); + var parameters = new List(Parameters); + var ret = new List(); if (string.IsNullOrEmpty(Index)) { throw new ArgumentException("Index cannot be null"); } ret.Add(Index); - ret.Add(QueryText); + + if (NearestNeighbors is not null) + { + var queryText = $"({QueryText})=>[KNN {NearestNeighbors.NumNeighbors} @{NearestNeighbors.PropertyName} ${parameters.Count} AS {VectorScores.NearestNeighborScoreName}]"; + ret.Add(queryText); + parameters.Add(NearestNeighbors.VectorBlob); + } + else + { + ret.Add(QueryText); + } + + if (parameters.Any()) + { + ret.Add("PARAMS"); + ret.Add(parameters.Count * 2); + for (var i = 0; i < parameters.Count; i++) + { + ret.Add(i.ToString()); + ret.Add(parameters[i]); + } + + ret.Add("DIALECT"); + ret.Add(2); + } + foreach (var flag in (QueryFlags[])Enum.GetValues(typeof(QueryFlags))) { if ((Flags & (long)flag) == (long)flag) diff --git a/src/Redis.OM/Searching/RedisCollection.cs b/src/Redis.OM/Searching/RedisCollection.cs index 57b0d851..b73b9f6f 100644 --- a/src/Redis.OM/Searching/RedisCollection.cs +++ b/src/Redis.OM/Searching/RedisCollection.cs @@ -549,7 +549,7 @@ public T Single(Expression> expression) } /// - public IEnumerator GetEnumerator() + public virtual IEnumerator GetEnumerator() { StateManager.Clear(); return new RedisCollectionEnumerator(Expression, _connection, ChunkSize, StateManager, BooleanExpression, SaveState, RootType, typeof(T)); diff --git a/src/Redis.OM/Searching/RedisCollectionEnumerator.cs b/src/Redis.OM/Searching/RedisCollectionEnumerator.cs index bd6ecc5c..99adda4d 100644 --- a/src/Redis.OM/Searching/RedisCollectionEnumerator.cs +++ b/src/Redis.OM/Searching/RedisCollectionEnumerator.cs @@ -164,7 +164,7 @@ private async ValueTask GetNextChunkAsync() _query!.Limit!.Offset = _query.Limit.Offset + _query.Limit.Number; } - _records = await _connection.SearchAsync(_query); + _records = await _connection.SearchAsync(_query).ConfigureAwait(false); _index = 0; _started = true; ConcatenateRecords(); diff --git a/src/Redis.OM/Searching/SearchResponse.cs b/src/Redis.OM/Searching/SearchResponse.cs index e4eeba45..562e9e9a 100644 --- a/src/Redis.OM/Searching/SearchResponse.cs +++ b/src/Redis.OM/Searching/SearchResponse.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using Redis.OM; namespace Redis.OM.Searching { @@ -54,7 +53,8 @@ public IDictionary DocumentsAs() var dict = new Dictionary(); foreach (var kvp in Documents) { - var obj = RedisObjectHandler.FromHashSet(kvp.Value); + var rrDict = kvp.Value.ToDictionary(x => x.Key, x => (RedisReply)x.Value); + var obj = RedisObjectHandler.FromHashSet(rrDict); dict.Add(kvp.Key, obj); } @@ -112,7 +112,7 @@ public SearchResponse(RedisReply val) for (var i = 1; i < vals.Count(); i += 2) { var docId = (string)vals[i]; - var documentHash = new Dictionary(); + var documentHash = new Dictionary(); var docArray = vals[i + 1].ToArray(); if (docArray.Length > 1) { diff --git a/src/Redis.OM/SemanticCache.cs b/src/Redis.OM/SemanticCache.cs new file mode 100644 index 00000000..4aa3e6c6 --- /dev/null +++ b/src/Redis.OM/SemanticCache.cs @@ -0,0 +1,248 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; +using Redis.OM.Contracts; +using Redis.OM.Modeling; +using Redis.OM.Searching.Query; + +namespace Redis.OM +{ + /// + /// A semantic cache for Large Language Models. + /// + public class SemanticCache : ISemanticCache + { + private readonly IRedisConnection _connection; + + /// + /// Initializes a new instance of the class. + /// + /// The index name. + /// The prefix for indexed items. + /// The threshold to check against.. + /// The Time To Live for a record inserted. + /// The vectorizer to use. + /// The connection to redis. + public SemanticCache(string indexName, string prefix, double threshold, long? ttl, IVectorizer vectorizer, IRedisConnection connection) + { + IndexName = indexName; + Prefix = prefix; + Threshold = threshold; + Ttl = ttl; + Vectorizer = vectorizer; + _connection = connection; + } + + /// + public string IndexName { get; } + + /// + public string Prefix { get; } + + /// + public double Threshold { get; } + + /// + public long? Ttl { get; } + + /// + public IVectorizer Vectorizer { get; } + + /// + public SemanticCacheResponse[] GetSimilar(string prompt, int maxNumResults = 10) + { + var query = BuildCheckQuery(prompt, maxNumResults); + var res = (RedisReply[])_connection.Execute("FT.SEARCH", query.SerializeQuery()); + return BuildResponse(res); + } + + /// + public async Task GetSimilarAsync(string prompt, int maxNumResults = 10) + { + var query = BuildCheckQuery(prompt, maxNumResults); + var res = (RedisReply[])await _connection.ExecuteAsync("FT.SEARCH", query.SerializeQuery()).ConfigureAwait(false); + return BuildResponse(res); + } + + /// + public void Store(string prompt, string response, object? metadata = null) + { + var key = $"{Prefix}:{Sha256Hash(prompt)}"; + var hash = BuildDocumentHash(prompt, response, metadata); + if (Ttl is not null) + { + _connection.HSet(key, TimeSpan.FromMilliseconds((double)Ttl), hash.ToArray()); + } + else + { + _connection.HSet(key, hash.ToArray()); + } + } + + /// + public Task StoreAsync(string prompt, string response, object? metadata = null) + { + var key = $"{Prefix}:{Sha256Hash(prompt)}"; + var hash = BuildDocumentHash(prompt, response, metadata); + return Ttl is not null ? _connection.HSetAsync(key, TimeSpan.FromMilliseconds((double)Ttl), hash.ToArray()) : _connection.HSetAsync(key, hash.ToArray()); + } + + /// + public void DeleteCache(bool dropRecords = true) + { + try + { + if (dropRecords) + { + _connection.Execute("FT.DROPINDEX", IndexName, "DD"); + } + else + { + _connection.Execute("FT.DROPINDEX", IndexName); + } + } + catch (Exception ex) + { + if (!ex.Message.Contains("Unknown Index name")) + { + throw; + } + } + } + + /// + public Task DeleteCacheAsync(bool dropRecords = true) + { + try + { + return dropRecords ? _connection.ExecuteAsync("FT.DROPINDEX", IndexName, "DD") : _connection.ExecuteAsync("FT.DROPINDEX", IndexName); + } + catch (Exception ex) + { + if (!ex.Message.Contains("Unknown Index name")) + { + throw; + } + } + + return Task.CompletedTask; + } + + /// + public void CreateIndex() + { + try + { + var serializedParams = SerializedIndexArgs(); + _connection.Execute("FT.CREATE", serializedParams); + } + catch (Exception ex) + { + if (ex.Message.Contains("Index already exists")) + { + return; + } + + throw; + } + } + + /// + public Task CreateIndexAsync() + { + try + { + var serializedParams = SerializedIndexArgs(); + return _connection.ExecuteAsync("FT.CREATE", serializedParams); + } + catch (Exception ex) + { + if (ex.Message.Contains("Index already exists")) + { + return Task.CompletedTask; + } + + throw; + } + } + + private static string Sha256Hash(string value) + { + StringBuilder sb = new StringBuilder(); + + using (var hash = SHA256.Create()) + { + Encoding enc = Encoding.UTF8; + byte[] result = hash.ComputeHash(enc.GetBytes(value)); + + foreach (byte b in result) + { + sb.Append(b.ToString("x2")); + } + } + + return sb.ToString(); + } + + private RedisQuery BuildCheckQuery(string prompt, int maxNumResults) + { + var query = new RedisQuery(IndexName); + query.QueryText = "@embedding:[VECTOR_RANGE $0 $1]=>{$YIELD_DISTANCE_AS: semantic_score}"; + query.Parameters.Add(Threshold); + query.Parameters.Add(Vectorizer.Vectorize(prompt)); + if (maxNumResults != 10) + { + query.Limit = new SearchLimit { Number = maxNumResults, Offset = 0 }; + } + + return query; + } + + private SemanticCacheResponse[] BuildResponse(RedisReply[] res) + { + List results = new List(); + for (int i = 1; i < res.Length; i += 2) + { + var key = (string)res[i]; + var hashArr = (RedisReply[])res[i + 1]; + Dictionary hash = new Dictionary(); + for (var j = 0; j < hashArr.Length; j += 2) + { + hash.Add(hashArr[j], hashArr[j + 1]); + } + + var score = double.Parse(hash["semantic_score"], CultureInfo.InvariantCulture); + var response = hash["response"]; + hash.TryGetValue("metadata", out var metadata); + results.Add(new SemanticCacheResponse(key, response, score, metadata)); + } + + return results.ToArray(); + } + + private Dictionary BuildDocumentHash(string prompt, string response, object? metadata) + { + var bytes = Vectorizer.Vectorize(prompt); + Dictionary hash = new Dictionary(); + hash.Add("embedding", bytes); + hash.Add("response", response); + hash.Add("prompt", prompt); + if (metadata is not null) + { + hash.Add("metadata", metadata); + } + + return hash; + } + + private object[] SerializedIndexArgs() + { + return new object[] { IndexName, nameof(Prefix), 1, Prefix, "SCHEMA", "embedding", "VECTOR", "FLAT", 6, "DIM", Vectorizer.Dim, "TYPE", Vectorizer.VectorType.AsRedisString(), "DISTANCE_METRIC", "COSINE", }; + } + } +} \ No newline at end of file diff --git a/src/Redis.OM/SemanticCacheResponse.cs b/src/Redis.OM/SemanticCacheResponse.cs new file mode 100644 index 00000000..38e5cd37 --- /dev/null +++ b/src/Redis.OM/SemanticCacheResponse.cs @@ -0,0 +1,50 @@ +namespace Redis.OM +{ + /// + /// A response to a Semantic Cache Query. + /// + public class SemanticCacheResponse + { + /// + /// Initializes a new instance of the class. + /// + /// The key. + /// The string response. + /// The Score. + /// The metadata. + internal SemanticCacheResponse(string key, string response, double score, object? metaData) + { + Key = key; + Response = response; + Score = score; + MetaData = metaData; + } + + /// + /// Gets the key. + /// + public string Key { get; } + + /// + /// Gets the response. + /// + public string Response { get; } + + /// + /// Gets the score. + /// + public double Score { get; } + + /// + /// Gets the metadata. + /// + public object? MetaData { get; } + + /// + /// Converts response to string implicitly. + /// + /// The response. + /// The response string. + public static implicit operator string(SemanticCacheResponse response) => response.Response; + } +} \ No newline at end of file diff --git a/src/Redis.OM/Vector.cs b/src/Redis.OM/Vector.cs new file mode 100644 index 00000000..9e268a12 --- /dev/null +++ b/src/Redis.OM/Vector.cs @@ -0,0 +1,96 @@ +using System; +using Redis.OM.Modeling; + +namespace Redis.OM +{ + /// + /// Represents a vector created from an item. + /// + public abstract class Vector + { + /// + /// Gets or sets the Embedding. You may set the embedding yourself, if it's not set when Redis OM inserts the vector, it will generate it for you. + /// + public byte[]? Embedding { get; set; } + + /// + /// Gets the embedding represented as an array of floats. + /// + public float[]? Floats => Embedding is not null ? VectorUtils.VectorBytesToFloats(Embedding) : null; + + /// + /// Gets the embedding represented as an array of doubles. + /// + public double[]? Doubles => Embedding is not null ? VectorUtils.VectorBytesToDoubles(Embedding) : null; + + /// + /// Gets The object backed by this vector. + /// + internal abstract object? Obj { get; } + + /// + /// Gets a vector of the type. + /// + /// The value. + /// The type. + /// A vector of the given type. + public static Vector Of(T val) + where T : class + { + return new Vector(val); + } + + /// + /// Embeds the Vector using the provided vectorizer. + /// + /// The Vectorizer. + public abstract void Embed(VectorizerAttribute attr); + } + + /// + /// Represents a vector created from an item of type T. + /// + /// The type. +#pragma warning disable SA1402 + public sealed class Vector : Vector, IEquatable> + where T : class +#pragma warning restore SA1402 + { + /// + /// Initializes a new instance of the class. + /// + /// The item the vector will represent. + public Vector(T value) + { + Value = value; + } + + /// + /// Gets the item represented by the vector. + /// + public T Value { get; } + + /// + internal override object? Obj => Value; + + /// + /// Embeds the Vector using the provided vectorizer. + /// + /// The Vectorizer. + public override void Embed(VectorizerAttribute attr) + { + if (attr is not VectorizerAttribute vectorizerAttribute) + { + throw new InvalidOperationException($"VectorizerAttribute must be of the type {typeof(T).Name}"); + } + + Embedding = vectorizerAttribute.Vectorizer.Vectorize(Value); + } + + /// + public bool Equals(Vector other) + { + return Value == other.Value && Embedding == other.Embedding; + } + } +} \ No newline at end of file diff --git a/src/Redis.OM/Vectors.cs b/src/Redis.OM/Vectors.cs new file mode 100644 index 00000000..33199769 --- /dev/null +++ b/src/Redis.OM/Vectors.cs @@ -0,0 +1,60 @@ +using System; + +namespace Redis.OM +{ + /// + /// Container class for Vector extensions. + /// + public static class VectorExtensions + { + /// + /// Placeholder method to allow you to perform vector range operations. Only meant to be run + /// in context of a query. + /// + /// The vector field. + /// The comparison object. + /// The allowable distance from the provided object. + /// The type to compare. + /// Whether the queried vector is within the allowable distance. + public static bool VectorRange(this Vector obj, Vector comparisonObject, double range) + where T : class => throw new NotImplementedException("This method is only meant to be run within a query of Redis."); + + /// + /// Placeholder method to allow you to perform vector range operations. Only meant to be run + /// in context of a query. + /// + /// The vector field. + /// The comparison object. + /// The allowable distance from the provided object. + /// The name of the score in the output. + /// The type to compare. + /// Whether the queried vector is within the allowable distance. + public static bool VectorRange(this Vector obj, Vector comparisonObject, double range, string scoreName) + where T : class => throw new NotImplementedException("This method is only meant to be run within a query of Redis."); + + /// + /// Placeholder method to allow you to perform vector range operations. Only meant to be run + /// in context of a query. + /// + /// The vector field. + /// The comparison object. + /// The allowable distance from the provided object. + /// The type to compare. + /// Whether the queried vector is within the allowable distance. + public static bool VectorRange(this Vector obj, T comparisonObject, double range) + where T : class => throw new NotImplementedException("This method is only meant to be run within a query of Redis."); + + /// + /// Placeholder method to allow you to perform vector range operations. Only meant to be run + /// in context of a query. + /// + /// The vector field. + /// The comparison object. + /// The allowable distance from the provided object. + /// The name of the score in the output. + /// The type to compare. + /// Whether the queried vector is within the allowable distance. + public static bool VectorRange(this Vector obj, T comparisonObject, double range, string scoreName) + where T : class => throw new NotImplementedException("This method is only meant to be run within a query of Redis."); + } +} \ No newline at end of file diff --git a/src/Redis.OM/stylecop.ruleset b/src/Redis.OM/stylecop.ruleset index 59ef8844..90642d0c 100644 --- a/src/Redis.OM/stylecop.ruleset +++ b/src/Redis.OM/stylecop.ruleset @@ -18,6 +18,7 @@ + \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/Address.cs b/test/Redis.OM.Unit.Tests/Address.cs index 8deb6aa8..e6b0da15 100644 --- a/test/Redis.OM.Unit.Tests/Address.cs +++ b/test/Redis.OM.Unit.Tests/Address.cs @@ -5,7 +5,6 @@ using System.Text; using System.Text.Json.Serialization; using System.Threading.Tasks; -using Redis.OM; using Redis.OM.Modeling; namespace Redis.OM.Unit.Tests diff --git a/test/Redis.OM.Unit.Tests/BasicTypeWithGeoLoc.cs b/test/Redis.OM.Unit.Tests/BasicTypeWithGeoLoc.cs index 07649716..b518f8a9 100644 --- a/test/Redis.OM.Unit.Tests/BasicTypeWithGeoLoc.cs +++ b/test/Redis.OM.Unit.Tests/BasicTypeWithGeoLoc.cs @@ -2,6 +2,7 @@ namespace Redis.OM.Unit.Tests { + [Document] public class BasicTypeWithGeoLoc { public string Name { get; set; } diff --git a/test/Redis.OM.Unit.Tests/ConfigurationTests.cs b/test/Redis.OM.Unit.Tests/ConfigurationTests.cs index 3586362c..979fac7f 100644 --- a/test/Redis.OM.Unit.Tests/ConfigurationTests.cs +++ b/test/Redis.OM.Unit.Tests/ConfigurationTests.cs @@ -1,6 +1,5 @@ using System.Linq; using System.Net; -using Redis.OM; using Xunit; namespace Redis.OM.Unit.Tests diff --git a/test/Redis.OM.Unit.Tests/CoreTests.cs b/test/Redis.OM.Unit.Tests/CoreTests.cs index 5a59fd7e..c94aa0c7 100644 --- a/test/Redis.OM.Unit.Tests/CoreTests.cs +++ b/test/Redis.OM.Unit.Tests/CoreTests.cs @@ -4,7 +4,6 @@ using StackExchange.Redis; using System.Linq; using System.IO; -using Redis.OM; using Redis.OM.Modeling; using System.Threading; using System.Threading.Tasks; diff --git a/test/Redis.OM.Unit.Tests/GeoLocTests.cs b/test/Redis.OM.Unit.Tests/GeoLocTests.cs index d6b74d57..9a950a9f 100644 --- a/test/Redis.OM.Unit.Tests/GeoLocTests.cs +++ b/test/Redis.OM.Unit.Tests/GeoLocTests.cs @@ -20,7 +20,7 @@ public void TestParsingFromJson() [Fact] public void TestParsingFromFormattedHash() { - var hash = new Dictionary + var hash = new Dictionary { {"Name", "Foo"}, {"Location", "32.5,22.4"} diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/AggregationSetTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/AggregationSetTests.cs index 55e25b25..26fe2379 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/AggregationSetTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/AggregationSetTests.cs @@ -349,8 +349,8 @@ public void TestConfigurableChunkSize() public void TestLoad() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.Load(x => x.RecordShell.Name).ToList(); _substitute.Received().Execute("FT.AGGREGATE","person-idx","*","LOAD","1","Name","WITHCURSOR", "COUNT","10000"); } @@ -359,8 +359,8 @@ public void TestLoad() public void TestMultiVariant() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.Load(x => new {x.RecordShell.Name, x.RecordShell.Age}).ToList(); _substitute.Received().Execute("FT.AGGREGATE","person-idx","*","LOAD","2","Name", "Age","WITHCURSOR", "COUNT","10000"); } @@ -369,8 +369,8 @@ public void TestMultiVariant() public void TestLoadAll() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.LoadAll().ToList(); _substitute.Received().Execute("FT.AGGREGATE", "person-idx", "*", "LOAD", "*", "WITHCURSOR", "COUNT", "10000"); } @@ -379,8 +379,8 @@ public void TestLoadAll() public void TestMultipleOrderBys() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.OrderBy(x => x.RecordShell.Name).OrderByDescending(x => x.RecordShell.Age).ToList(); _substitute.Received().Execute("FT.AGGREGATE","person-idx", "*", "SORTBY", "4", "@Name", "ASC", "@Age", "DESC", "WITHCURSOR", "COUNT", "10000"); } @@ -389,8 +389,8 @@ public void TestMultipleOrderBys() public void TestRightSideStringTypeFilter() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.Apply(x => string.Format("{0} {1}", x.RecordShell.FirstName, x.RecordShell.LastName), "FullName").Filter(p => p.Aggregations["FullName"] == "Bruce Wayne").ToList(); _substitute.Received().Execute("FT.AGGREGATE", "person-idx", "*", "APPLY", "format(\"%s %s\",@FirstName,@LastName)", "AS", "FullName", "FILTER", "@FullName == 'Bruce Wayne'", "WITHCURSOR", "COUNT", "10000"); @@ -400,8 +400,8 @@ public void TestRightSideStringTypeFilter() public void TestNestedOrderBy() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.OrderBy(x => x.RecordShell.Address.State).ToList(); _substitute.Received().Execute("FT.AGGREGATE","person-idx", "*", "SORTBY", "2", "@Address_State", "ASC", "WITHCURSOR", "COUNT", "10000"); } @@ -410,8 +410,8 @@ public void TestNestedOrderBy() public void TestNestedGroup() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.GroupBy(x => x.RecordShell.Address.State).ToList(); _substitute.Received().Execute("FT.AGGREGATE","person-idx", "*", "GROUPBY", "1", "@Address_State", "WITHCURSOR", "COUNT", "10000"); } @@ -420,8 +420,8 @@ public void TestNestedGroup() public void TestNestedGroupMulti() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.GroupBy(x => new {x.RecordShell.Address.State, x.RecordShell.Address.ForwardingAddress.City}).ToList(); _substitute.Received().Execute("FT.AGGREGATE","person-idx", "*", "GROUPBY", "2", "@Address_State", "@Address_ForwardingAddress_City", "WITHCURSOR", "COUNT", "10000"); } @@ -430,8 +430,8 @@ public void TestNestedGroupMulti() public void TestNestedApply() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.Apply(x => x.RecordShell.Address.HouseNumber + 4, "house_num_modified").ToList(); _substitute.Received().Execute("FT.AGGREGATE","person-idx", "*", "APPLY", "@Address_HouseNumber + 4", "AS", "house_num_modified", "WITHCURSOR", "COUNT", "10000"); } @@ -440,8 +440,8 @@ public void TestNestedApply() public void TestMissedBinExpression() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.Apply(x => x.RecordShell.Address.HouseNumber + 4, "house_num_modified") .Apply(x=>x.RecordShell.Age + x["house_num_modified"] * 4 + x.RecordShell.Sales, "arbitrary_calculation").ToList(); _substitute.Received().Execute("FT.AGGREGATE","person-idx", "*", "APPLY", "@Address_HouseNumber + 4", "AS", "house_num_modified", "APPLY", "@Age + @house_num_modified * 4 + @Sales", "AS", "arbitrary_calculation", "WITHCURSOR", "COUNT", "10000"); @@ -456,8 +456,8 @@ public void TestWhereByComplexObjectOnTheRightSide() LastName = "Bond" }; var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.Where(x =>x.RecordShell.FirstName==customerFilter.FirstName) .ToList(); _substitute.Received().Execute("FT.AGGREGATE", "person-idx", "@FirstName:{James}", "WITHCURSOR", "COUNT", "10000"); } @@ -471,8 +471,8 @@ public void TestSequentialWhereClauseTranslation() LastName = "Bond" }; var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.Where(x => x.RecordShell.FirstName == customerFilter.FirstName).Where(p=>p.RecordShell.LastName==customerFilter.LastName).ToList(); _substitute.Received().Execute("FT.AGGREGATE", "person-idx", "@LastName:{Bond} @FirstName:{James}", "WITHCURSOR", "COUNT", "10000"); } @@ -480,8 +480,8 @@ public void TestSequentialWhereClauseTranslation() public void TestSkipTakeTranslatedLimit() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); _ = collection.OrderByDescending(p=>p.RecordShell.Age).Skip(0).Take(10).ToList(); _substitute.Received().Execute("FT.AGGREGATE", "person-idx","*","SORTBY","2","@Age","DESC","LIMIT","0","10", "WITHCURSOR", "COUNT", "10000"); } @@ -490,8 +490,8 @@ public void TestSkipTakeTranslatedLimit() public void RightBinExpressionOperator() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); Expression, bool>> query = a => a.RecordShell!.Age == 0 && (a.RecordShell!.Age == 2 || a.RecordShell!.Age == 50); _ = collection.Where(query).ToList(); @@ -502,8 +502,8 @@ public void RightBinExpressionOperator() public void RightBinExpressionWithUniaryOperator() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); Expression, bool>> query = a => a.RecordShell!.Name.Contains("Steve") && (a.RecordShell!.Age == 2 || a.RecordShell!.Age == 50); _ = collection.Where(query).ToList(); @@ -514,8 +514,8 @@ public void RightBinExpressionWithUniaryOperator() public void LeftBinExpressionWithUniaryOperator() { var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); Expression, bool>> query = a => (a.RecordShell!.Age == 2 || a.RecordShell!.Age == 50) && a.RecordShell!.Name.Contains("Steve"); _ = collection.Where(query).ToList(); @@ -531,8 +531,8 @@ public void PunctuationMarkInTagQuery() LastName = "White" }; var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); Expression, bool>> query = a => a.RecordShell.FirstName == customerFilter.FirstName; _ = collection.Where(query).ToList(); _substitute.Received().Execute("FT.AGGREGATE", "person-idx", "@FirstName:{Walter\\-Junior}", "WITHCURSOR", "COUNT", "10000"); @@ -542,8 +542,8 @@ public void PunctuationMarkInTagQuery() public void CustomPropertyNamesInQuery() { //Arrange - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); var collection = new RedisAggregationSet(_substitute, true, 10); @@ -568,8 +568,8 @@ public void DateTimeQuery() var dto = DateTimeOffset.Now.Subtract(TimeSpan.FromHours(3)); var dtoMs = dto.ToUnixTimeMilliseconds(); var collection = new RedisAggregationSet(_substitute, true, chunkSize: 10000); - _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); - _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); + _substitute.Execute("FT.AGGREGATE", Arg.Any()).Returns(MockedResult); + _substitute.Execute("FT.CURSOR", Arg.Any()).Returns(MockedResultCursorEnd); Expression, bool>> query = a => a.RecordShell.Timestamp > dt; _ = collection.Where(query).ToList(); diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/Person.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/Person.cs index c57ae65a..78ff0326 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/Person.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/Person.cs @@ -1,5 +1,4 @@ using System.Collections.Generic; -using Redis.OM; using Redis.OM.Modeling; namespace Redis.OM.Unit.Tests.RediSearchTests diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/SearchTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/SearchTests.cs index 3efc289b..d3d263f1 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/SearchTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/SearchTests.cs @@ -52,7 +52,7 @@ public class SearchTests public void TestBasicQuery() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Age < 33).ToList(); @@ -68,7 +68,7 @@ public void TestBasicQuery() public void TestBasicNegationQuery() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => !(x.Age < 33)).ToList(); _substitute.Received().Execute( @@ -84,7 +84,7 @@ public void TestBasicNegationQuery() public void TestBasicQueryWithVariable() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var y = 33; var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Age < y).ToList(); @@ -101,7 +101,7 @@ public void TestBasicQueryWithVariable() public void TestFirstOrDefaultWithMixedLocals() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var heightList = new List { 70.0, 68.0 }; var y = 33; foreach (var height in heightList) @@ -122,7 +122,7 @@ public void TestFirstOrDefaultWithMixedLocals() public void TestBasicQueryWithExactNumericMatch() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var y = 33; var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Age == y).ToList(); @@ -139,7 +139,7 @@ public void TestBasicQueryWithExactNumericMatch() public void TestBasicFirstOrDefaultQuery() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var y = 33; var collection = new RedisCollection(_substitute); _ = collection.FirstOrDefault(x => x.Age == y); @@ -156,7 +156,7 @@ public void TestBasicFirstOrDefaultQuery() public void TestBasicQueryNoNameIndex() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var y = 33; var collection = new RedisCollection(_substitute); _ = collection.FirstOrDefault(x => x.Age == y); @@ -173,7 +173,7 @@ public void TestBasicQueryNoNameIndex() public void TestBasicOrQuery() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Age < 33 || x.TagField == "Steve").ToList(); @@ -190,7 +190,7 @@ public void TestBasicOrQuery() public void TestBasicOrQueryTwoTags() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.TagField == "Bob" || x.TagField == "Steve").ToList(); @@ -207,7 +207,7 @@ public void TestBasicOrQueryTwoTags() public void TestBasicOrQueryWithNegation() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Age < 33 || x.TagField != "Steve" || x.Name != "Steve").ToList(); @@ -224,7 +224,7 @@ public void TestBasicOrQueryWithNegation() public void TestBasicAndQuery() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Age < 33 && x.TagField == "Steve").ToList(); @@ -241,7 +241,7 @@ public void TestBasicAndQuery() public void TestBasicTagQuery() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Age < 33 && x.TagField == "Steve").ToList(); @@ -258,7 +258,7 @@ public void TestBasicTagQuery() public void TestBasicThreeClauseQuery() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Age < 33 && x.TagField == "Steve" && x.Height >= 70).ToList(); @@ -275,7 +275,7 @@ public void TestBasicThreeClauseQuery() public void TestGroupedThreeClauseQuery() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Age < 33 && (x.TagField == "Steve" || x.Height >= 70)).ToList(); @@ -292,7 +292,7 @@ public void TestGroupedThreeClauseQuery() public void TestBasicQueryWithContains() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name.Contains("Ste")).ToList(); @@ -309,7 +309,7 @@ public void TestBasicQueryWithContains() public void TestBasicQueryWithStartsWith() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name.StartsWith("Ste")).ToList(); @@ -326,7 +326,7 @@ public void TestBasicQueryWithStartsWith() public void TestFuzzy() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name.FuzzyMatch("Ste", 2)).ToList(); @@ -343,7 +343,7 @@ public void TestFuzzy() public void TestMatchStartsWith() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name.MatchStartsWith("Ste")).ToList(); @@ -360,7 +360,7 @@ public void TestMatchStartsWith() public void TestMatchEndsWith() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name.MatchEndsWith("Ste")).ToList(); @@ -377,7 +377,7 @@ public void TestMatchEndsWith() public void TestMatchContains() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name.MatchContains("Ste")).ToList(); @@ -394,7 +394,7 @@ public void TestMatchContains() public void TestTagContains() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var ste = "Ste"; var person = new Person() { TagField = "ath" }; @@ -422,7 +422,7 @@ public void TestTagContains() public void TestTagStartsWith() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.TagField.StartsWith("Ste")).ToList(); @@ -439,7 +439,7 @@ public void TestTagStartsWith() public void TestTagEndsWith() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.TagField.EndsWith("Ste")).ToList(); @@ -456,7 +456,7 @@ public void TestTagEndsWith() public void TestTextEndsWith() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name.EndsWith("Ste")).ToList(); @@ -473,7 +473,7 @@ public void TestTextEndsWith() public void TestTextStartsWith() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name.StartsWith("Ste")).ToList(); @@ -490,7 +490,7 @@ public void TestTextStartsWith() public void TestBasicQueryWithEndsWith() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name.EndsWith("Ste")).ToList(); @@ -507,7 +507,7 @@ public void TestBasicQueryWithEndsWith() public void TestBasicQueryFromPropertyOfModel() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); var modelObject = new Person() { Name = "Steve" }; @@ -525,7 +525,7 @@ public void TestBasicQueryFromPropertyOfModel() public void TestBasicQueryFromPropertyOfModelWithStringInterpolation() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); var modelObject = new Person() { Name = "Steve" }; @@ -543,7 +543,7 @@ public void TestBasicQueryFromPropertyOfModelWithStringInterpolation() public void TestBasicQueryFromPropertyOfModelWithStringFormatFourArgs() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); var modelObject = new Person() { Name = "Steve" }; @@ -563,7 +563,7 @@ public void TestBasicQueryFromPropertyOfModelWithStringFormatFourArgs() public void TestBasicQueryWithContainsWithNegation() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => !x.Name.Contains("Ste")).ToList(); @@ -580,7 +580,7 @@ public void TestBasicQueryWithContainsWithNegation() public void TestTwoPredicateQueryWithContains() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name.Contains("Ste") || x.TagField == "John").ToList(); @@ -597,7 +597,7 @@ public void TestTwoPredicateQueryWithContains() public void TestTwoPredicateQueryWithPrefixMatching() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name.Contains("Ste*") || x.TagField == "John").ToList(); @@ -614,7 +614,7 @@ public void TestTwoPredicateQueryWithPrefixMatching() public void TestGeoFilter() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.GeoFilter(x => x.Home, 5, 6.7, 50, GeoLocDistanceUnit.Kilometers).ToList(); @@ -639,7 +639,7 @@ public void TestGeoFilter() public void TestGeoFilterWithWhereClause() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); var res = collection.Where(x => x.TagField == "Steve").GeoFilter(x => x.Home, 5, 6.7, 50, GeoLocDistanceUnit.Kilometers).ToList(); @@ -664,7 +664,7 @@ public void TestGeoFilterWithWhereClause() public void TestSelect() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Select(x => x.Name).ToList(); @@ -685,7 +685,7 @@ public void TestSelect() public void TestSelectComplexAnonType() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Select(x => new { x.Name }).ToList(); @@ -707,7 +707,7 @@ public void TestSelectComplexAnonType() public void TextEqualityExpression() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Name == "Steve").ToList(); @@ -724,7 +724,7 @@ public void TextEqualityExpression() public void TestPaginationChunkSizesSinglePredicate() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); _ = collection.Where(x => x.Name == "Steve").ToList(); @@ -741,7 +741,7 @@ public void TestPaginationChunkSizesSinglePredicate() public void TestPaginationChunkSizesMultiplePredicates() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); _ = collection.Where(x => x.TagField == "Steve").GeoFilter(x => x.Home, 5, 6.7, 50, GeoLocDistanceUnit.Kilometers).ToList(); @@ -765,7 +765,7 @@ public void TestPaginationChunkSizesMultiplePredicates() public void TestNestedObjectStringSearch() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); _ = collection.Where(x => x.Address.City == "Newark").ToList(); @@ -783,7 +783,7 @@ public void TestNestedObjectStringSearch() public void TestNestedObjectStringSearchNested2Levels() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); _ = collection.Where(x => x.Address.ForwardingAddress.City == "Newark").ToList(); @@ -801,7 +801,7 @@ public void TestNestedObjectStringSearchNested2Levels() public void TestNestedObjectNumericSearch() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); _ = collection.Where(x => x.Address.HouseNumber == 4).ToList(); @@ -819,7 +819,7 @@ public void TestNestedObjectNumericSearch() public void TestNestedObjectNumericSearch2Levels() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); _ = collection.Where(x => x.Address.ForwardingAddress.HouseNumber == 4).ToList(); @@ -837,7 +837,7 @@ public void TestNestedObjectNumericSearch2Levels() public void TestNestedQueryOfGeo() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); _ = collection.GeoFilter(x => x.Address.Location, 5, 6.7, 50, GeoLocDistanceUnit.Kilometers).ToList(); @@ -861,7 +861,7 @@ public void TestNestedQueryOfGeo() public void TestNestedQueryOfGeo2Levels() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); _ = collection.GeoFilter(x => x.Address.ForwardingAddress.Location, 5, 6.7, 50, GeoLocDistanceUnit.Kilometers).ToList(); @@ -885,7 +885,7 @@ public void TestNestedQueryOfGeo2Levels() public void TestArrayContains() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); _ = collection.Where(x => x.NickNames.Contains("Steve")).ToList(); @@ -903,7 +903,7 @@ public void TestArrayContains() public void TestArrayContainsSpecialChar() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); _ = collection.Where(x => x.NickNames.Contains("Steve@redis.com")).ToList(); @@ -921,7 +921,7 @@ public void TestArrayContainsSpecialChar() public void TestArrayContainsVar() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); var steve = "Steve"; @@ -940,7 +940,7 @@ public void TestArrayContainsVar() public void TestArrayContainsNested() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute, 1000); _ = collection.Where(x => x.Mother.NickNames.Contains("Di")).ToList(); @@ -957,10 +957,10 @@ public void TestArrayContainsNested() [Fact] public async Task TestUpdateJson() { - _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); - _substitute.ExecuteAsync("EVALSHA", Arg.Any()).Returns(Task.FromResult(new RedisReply("42"))); - _substitute.ExecuteAsync("SCRIPT", Arg.Any()) + _substitute.ExecuteAsync("EVALSHA", Arg.Any()).Returns(Task.FromResult(new RedisReply("42"))); + _substitute.ExecuteAsync("SCRIPT", Arg.Any()) .Returns(Task.FromResult(new RedisReply("cbbf1c4fab5064f419e469cc51c563f8bf51e6fb"))); var collection = new RedisCollection(_substitute); var steve = await collection.FirstAsync(x => x.Name == "Steve"); @@ -974,12 +974,12 @@ public async Task TestUpdateJson() public async Task TestUpdateJsonUnloadedScriptAsync() { - _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); - _substitute.ExecuteAsync("EVALSHA", Arg.Any()) + _substitute.ExecuteAsync("EVALSHA", Arg.Any()) .Throws(new RedisServerException("Failed on EVALSHA")); - _substitute.ExecuteAsync("EVAL", Arg.Any()).Returns(Task.FromResult(new RedisReply("42"))); - _substitute.ExecuteAsync("SCRIPT", Arg.Any()).Returns(Task.FromResult(new RedisReply("cbbf1c4fab5064f419e469cc51c563f8bf51e6fb"))); + _substitute.ExecuteAsync("EVAL", Arg.Any()).Returns(Task.FromResult(new RedisReply("42"))); + _substitute.ExecuteAsync("SCRIPT", Arg.Any()).Returns(Task.FromResult(new RedisReply("cbbf1c4fab5064f419e469cc51c563f8bf51e6fb"))); var collection = new RedisCollection(_substitute); var steve = await collection.FirstAsync(x => x.Name == "Steve"); steve.Age = 33; @@ -992,12 +992,12 @@ public async Task TestUpdateJsonUnloadedScriptAsync() [Fact] public void TestUpdateJsonUnloadedScript() { - _substitute.Execute("FT.SEARCH", Arg.Any()).Returns(_mockReply); + _substitute.Execute("FT.SEARCH", Arg.Any()).Returns(_mockReply); - _substitute.Execute("EVALSHA", Arg.Any()) + _substitute.Execute("EVALSHA", Arg.Any()) .Throws(new RedisServerException("Failed on EVALSHA")); - _substitute.Execute("EVAL", Arg.Any()).Returns(new RedisReply("42")); - _substitute.Execute("SCRIPT", Arg.Any()).Returns(new RedisReply("cbbf1c4fab5064f419e469cc51c563f8bf51e6fb")); + _substitute.Execute("EVAL", Arg.Any()).Returns(new RedisReply("42")); + _substitute.Execute("SCRIPT", Arg.Any()).Returns(new RedisReply("cbbf1c4fab5064f419e469cc51c563f8bf51e6fb")); var collection = new RedisCollection(_substitute); var steve = collection.First(x => x.Name == "Steve"); steve.Age = 33; @@ -1010,9 +1010,9 @@ public void TestUpdateJsonUnloadedScript() [Fact] public async Task TestUpdateJsonName() { - _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); - _substitute.ExecuteAsync("EVALSHA", Arg.Any()).Returns(new RedisReply("42")); - _substitute.ExecuteAsync("SCRIPT", Arg.Any()).Returns(new RedisReply("42")); + _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync("EVALSHA", Arg.Any()).Returns(new RedisReply("42")); + _substitute.ExecuteAsync("SCRIPT", Arg.Any()).Returns(new RedisReply("42")); var collection = new RedisCollection(_substitute); var steve = await collection.FirstAsync(x => x.Name == "Steve"); steve.Name = "Bob"; @@ -1024,9 +1024,9 @@ public async Task TestUpdateJsonName() [Fact] public async Task TestUpdateJsonNestedObject() { - _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); - _substitute.ExecuteAsync("EVALSHA", Arg.Any()).Returns(new RedisReply("42")); - _substitute.ExecuteAsync("SCRIPT", Arg.Any()).Returns(new RedisReply("cbbf1c4fab5064f419e469cc51c563f8bf51e6fb")); + _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync("EVALSHA", Arg.Any()).Returns(new RedisReply("42")); + _substitute.ExecuteAsync("SCRIPT", Arg.Any()).Returns(new RedisReply("cbbf1c4fab5064f419e469cc51c563f8bf51e6fb")); var collection = new RedisCollection(_substitute); var steve = await collection.FirstAsync(x => x.Name == "Steve"); steve.Address = new Address { State = "Florida" }; @@ -1045,9 +1045,9 @@ public async Task TestUpdateJsonNestedObject() [Fact] public async Task TestUpdateJsonWithDouble() { - _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); - _substitute.ExecuteAsync("EVALSHA", Arg.Any()).Returns(new RedisReply("42")); - _substitute.ExecuteAsync("SCRIPT", Arg.Any()).Returns(new RedisReply("42")); + _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync("EVALSHA", Arg.Any()).Returns(new RedisReply("42")); + _substitute.ExecuteAsync("SCRIPT", Arg.Any()).Returns(new RedisReply("42")); var collection = new RedisCollection(_substitute); var steve = await collection.FirstAsync(x => x.Name == "Steve"); steve.Age = 33; @@ -1061,8 +1061,8 @@ public async Task TestUpdateJsonWithDouble() public async Task TestDeleteAsync() { const string key = "Redis.OM.Unit.Tests.RediSearchTests.Person:01FVN836BNQGYMT80V7RCVY73N"; - _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); - _substitute.ExecuteAsync("UNLINK", Arg.Any()).Returns("1"); + _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync("UNLINK", Arg.Any()).Returns("1"); var collection = new RedisCollection(_substitute); var steve = await collection.FirstAsync(x => x.Name == "Steve"); Assert.True(collection.StateManager.Data.ContainsKey(key)); @@ -1077,8 +1077,8 @@ public async Task TestDeleteAsync() public void TestDelete() { const string key = "Redis.OM.Unit.Tests.RediSearchTests.Person:01FVN836BNQGYMT80V7RCVY73N"; - _substitute.Execute("FT.SEARCH", Arg.Any()).Returns(_mockReply); - _substitute.Execute("UNLINK", Arg.Any()).Returns(new RedisReply("1")); + _substitute.Execute("FT.SEARCH", Arg.Any()).Returns(_mockReply); + _substitute.Execute("UNLINK", Arg.Any()).Returns(new RedisReply("1")); var collection = new RedisCollection(_substitute); var steve = collection.First(x => x.Name == "Steve"); Assert.True(collection.StateManager.Data.ContainsKey(key)); @@ -1094,7 +1094,7 @@ public void TestDelete() [InlineData(false)] public async Task TestFirstAsync(bool useExpression) { - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var expectedPredicate = useExpression ? "(@TagField:{bob})" : "*"; @@ -1123,7 +1123,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestFirstAsyncNone(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReplyNone); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReplyNone); var expectedPredicate = useExpression ? "(@TagField:{bob})" : "*"; @@ -1151,7 +1151,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestFirstOrDefaultAsync(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var expectedPredicate = useExpression ? "(@TagField:{bob})" : "*"; @@ -1181,7 +1181,7 @@ await _substitute.Received().ExecuteAsync( [InlineData(false)] public async Task TestFirstOrDefaultAsyncNone(bool useExpression) { - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReplyNone); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReplyNone); var collection = new RedisCollection(_substitute); @@ -1213,7 +1213,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestSingleAsync(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var expectedPredicate = useExpression ? "(@TagField:{bob})" : "*"; var collection = new RedisCollection(_substitute); @@ -1242,7 +1242,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestSingleAsyncNone(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReplyNone); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReplyNone); var collection = new RedisCollection(_substitute); var expectedPredicate = useExpression ? "(@TagField:{bob})" : "*"; @@ -1270,7 +1270,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestSingleAsyncTwo(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); var collection = new RedisCollection(_substitute); var expectedPredicate = useExpression ? "(@TagField:{bob})" : "*"; @@ -1298,7 +1298,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestSingleOrDefaultAsync(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1330,7 +1330,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestSingleOrDefaultAsyncNone(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReplyNone); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReplyNone); var collection = new RedisCollection(_substitute); @@ -1362,7 +1362,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestSingleOrDefaultAsyncTwo(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); var collection = new RedisCollection(_substitute); @@ -1394,7 +1394,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestAnyAsync(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var expectedPredicate = useExpression ? "(@TagField:{bob})" : "*"; @@ -1418,7 +1418,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestAnyAsyncNone(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReplyNone); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReplyNone); var collection = new RedisCollection(_substitute); @@ -1441,7 +1441,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestCountAsync(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1463,7 +1463,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestCount2Async(bool useExpression) { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); var collection = new RedisCollection(_substitute); var expectedPredicate = useExpression ? "(@TagField:{bob})" : "*"; @@ -1482,7 +1482,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestOrderByWithAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); var collection = new RedisCollection(_substitute); var expectedPredicate = "*"; _ = await collection.OrderBy(x => x.Age).ToListAsync(); @@ -1502,7 +1502,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestOrderByDescendingWithAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); var collection = new RedisCollection(_substitute); var expectedPredicate = "*"; _ = await collection.OrderByDescending(x => x.Age).ToListAsync(); @@ -1522,7 +1522,7 @@ await _substitute.Received().ExecuteAsync( public async Task CombinedExpressionsWithFirstOrDefaultAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); var collection = new RedisCollection(_substitute); @@ -1542,7 +1542,7 @@ await _substitute.Received().ExecuteAsync( public async Task CombinedExpressionsWithFirstAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); var collection = new RedisCollection(_substitute); @@ -1562,7 +1562,7 @@ await _substitute.Received().ExecuteAsync( public async Task CombinedExpressionsWithAnyAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); var collection = new RedisCollection(_substitute); @@ -1582,7 +1582,7 @@ await _substitute.Received().ExecuteAsync( public async Task CombinedExpressionsSingleOrDefaultAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1602,7 +1602,7 @@ await _substitute.Received().ExecuteAsync( public async Task CombinedExpressionsSingleAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1622,7 +1622,7 @@ await _substitute.Received().ExecuteAsync( public async Task CombinedExpressionsCountAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1642,7 +1642,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestCombinedExpressionWithExpressionFirstOrDefaultAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply2Count); var collection = new RedisCollection(_substitute); @@ -1668,7 +1668,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestCombinedExpressionWithExpressionFirstAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1694,7 +1694,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestCombinedExpressionWithExpressionAnyAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1720,7 +1720,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestCombinedExpressionWithExpressionSingleAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1746,7 +1746,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestCombinedExpressionWithExpressionSingleOrDefaultAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1772,7 +1772,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestCombinedExpressionWithExpressionCountAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1798,7 +1798,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestCreateIndexWithNoStopwords() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); await _substitute.CreateIndexAsync(typeof(ObjectWithZeroStopwords)); @@ -1819,7 +1819,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestCreateIndexWithTwoStopwords() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); await _substitute.CreateIndexAsync(typeof(ObjectWithTwoStopwords)); @@ -1839,7 +1839,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestCreateIndexWithStringLikeValueTypes() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); await _substitute.CreateIndexAsync(typeof(ObjectWithStringLikeValueTypes)); @@ -1864,7 +1864,7 @@ await _substitute.Received().ExecuteAsync("FT.CREATE", public async Task TestCreateIndexWithStringLikeValueTypesHash() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); await _substitute.CreateIndexAsync(typeof(ObjectWithStringLikeValueTypesHash)); @@ -1887,7 +1887,7 @@ await _substitute.Received().ExecuteAsync("FT.CREATE", public async Task TestCreateIndexWithDatetimeValue() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); await _substitute.CreateIndexAsync(typeof(ObjectWithDateTime)); await _substitute.CreateIndexAsync(typeof(ObjectWithDateTimeHash)); @@ -1921,7 +1921,7 @@ await _substitute.Received().ExecuteAsync("FT.CREATE", public async Task TestQueryOfUlid() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1937,7 +1937,7 @@ public async Task TestQueryOfUlid() public async Task TestQueryOfGuid() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1954,7 +1954,7 @@ public async Task TestQueryOfGuid() public async Task TestQueryOfBoolean() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1971,7 +1971,7 @@ public async Task TestQueryOfBoolean() public async Task TestQueryOfEnum() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -1988,7 +1988,7 @@ public async Task TestQueryOfEnum() public async Task TestQueryOfEnumHash() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -2003,7 +2003,7 @@ public async Task TestQueryOfEnumHash() public async Task TestGreaterThanEnumQuery() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -2017,7 +2017,7 @@ public async Task TestGreaterThanEnumQuery() [Fact] public async Task TestIndexCreationWithEmbeddedListOfDocuments() { - _substitute.ExecuteAsync("FT.CREATE", Arg.Any()).Returns("OK"); + _substitute.ExecuteAsync("FT.CREATE", Arg.Any()).Returns("OK"); await _substitute.CreateIndexAsync(typeof(ObjectWithEmbeddedArrayOfObjects)); await _substitute.Received().ExecuteAsync("FT.CREATE", "objectwithembeddedarrayofobjects-idx", @@ -2042,7 +2042,7 @@ await _substitute.Received().ExecuteAsync("FT.CREATE", public async Task TestAnyQueryForArrayOfEmbeddedObjects() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -2062,7 +2062,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestAnyQueryForArrayOfEmbeddedObjectsEnum() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -2082,7 +2082,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestAnyQueryForArrayOfEmbeddedObjectsExtraPredicate() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -2102,7 +2102,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestAnyQueryForArrayOfEmbeddedObjectsMultipleAnyCalls() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -2122,7 +2122,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestAnyQueryForArrayOfEmbeddedObjectsMultiplePredicatesInsideAny() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -2142,7 +2142,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestAnyQueryForArrayOfEmbeddedObjectsOtherTypes() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var boolean = true; var ulid = Ulid.NewUlid(); @@ -2166,7 +2166,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestAnyQueryForListOfEmbeddedObjects() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -2186,7 +2186,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestAnyQueryForArrayOfEmbeddedObjectsMultiVariant() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -2206,7 +2206,7 @@ await _substitute.Received().ExecuteAsync( public async Task SearchWithMultipleWhereClauses() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -2228,7 +2228,7 @@ await _substitute.Received().ExecuteAsync( public async Task TestAsyncMaterializationMethodsWithCombinedQueries() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = await collection.Where(x => x.TagField == "CountAsync") .CountAsync(x => x.Age == 32); @@ -2292,7 +2292,7 @@ await _substitute.Received().ExecuteAsync( public void TestMaterializationMethodsWithCombinedQueries() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => x.Age == 32); _ = collection.Count(x => x.TagField == "Count"); _ = collection.Any(x => x.TagField == "Any"); @@ -2351,7 +2351,7 @@ public void SearchTagFieldContains() { var potentialTagFieldValues = new [] { "Steve", "Alice", "Bob" }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialTagFieldValues.Contains(x.TagField)); _ = collection.ToList(); _substitute.Received().Execute( @@ -2368,7 +2368,7 @@ public void SearchTextFieldContains() { var potentialTextFieldValues = new [] { "Steve", "Alice", "Bob" }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialTextFieldValues.Contains(x.Name)); _ = collection.ToList(); _substitute.Received().Execute( @@ -2385,7 +2385,7 @@ public void SearchNumericFieldContains() { var potentialTagFieldValues = new int?[] { 35, 50, 60 }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialTagFieldValues.Contains(x.Age)); _ = collection.ToList(); _substitute.Received().Execute( @@ -2401,7 +2401,7 @@ public void SearchNumericFieldContains() public void Issue201() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var p1 = new Person() { Name = "Steve" }; var collection = new RedisCollection(_substitute, 1000); @@ -2421,7 +2421,7 @@ public void Issue201() public void RangeOnDateTimeWithMultiplePredicates() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var fromDto = DateTimeOffset.UtcNow.Subtract(TimeSpan.FromHours(4)); var toDto = DateTimeOffset.UtcNow; @@ -2499,7 +2499,7 @@ public void RangeOnDateTimeWithMultiplePredicates() public void RangeOnDatetime() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var timestamp = DateTime.Now; var timeAnHourAgo = timestamp.Subtract(TimeSpan.FromHours(1)); @@ -2589,7 +2589,7 @@ public void RangeOnDatetime() public async Task RangeOnDatetimeAsync() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var timestamp = DateTime.Now; var timeAnHourAgo = timestamp.Subtract(TimeSpan.FromHours(1)); @@ -2678,7 +2678,7 @@ await _substitute.Received().ExecuteAsync( public async Task RangeOnDatetimeAsyncHash() { _substitute.ClearSubstitute(); - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns(_mockReply); var timestamp = DateTime.Now; var timeAnHourAgo = timestamp.Subtract(TimeSpan.FromHours(1)); @@ -2781,7 +2781,7 @@ public void SearchNumericFieldListContains() { var potentialTagFieldValues = new List { 35, 50, 60 }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialTagFieldValues.Contains(x.Age)); _ = collection.ToList(); _substitute.Received().Execute( @@ -2798,7 +2798,7 @@ public void SearchTagFieldAndTextListContains() { var potentialTagFieldValues = new List { "Steve", "Alice", "Bob" }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialTagFieldValues.Contains(x.TagField) || potentialTagFieldValues.Contains(x.Name)); _ = collection.ToList(); _substitute.Received().Execute( @@ -2819,7 +2819,7 @@ public void TestNullResponseDoc() var query = new RedisQuery("fake-idx"); _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()) + _substitute.Execute(Arg.Any(), Arg.Any()) .Returns((RedisReply)res); _ = _substitute.Search(query); @@ -2830,7 +2830,7 @@ public void SearchTagFieldAndTextListContainsWithEscapes() { var potentialTagFieldValues = new List { "steve@example.com", "alice@example.com", "bob@example.com" }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialTagFieldValues.Contains(x.TagField) || potentialTagFieldValues.Contains(x.Name)); _ = collection.ToList(); _substitute.Received().Execute( @@ -2846,7 +2846,7 @@ public void SearchTagFieldAndTextListContainsWithEscapes() public void SearchWithEmptyAny() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); var any = collection.Any(); _substitute.Received().Execute( @@ -2874,7 +2874,7 @@ public void SearchWithEmptyAny() public void TestContainsFromLocal() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); const string steve = "steve"; _ = collection.Where(x => x.NickNamesList.Contains(steve)).ToList(); @@ -2898,7 +2898,7 @@ public void SearchGuidFieldContains() var guid3Str = ExpressionParserUtilities.EscapeTagField(guid3.ToString()); var potentialFieldValues = new [] { guid1, guid2, guid3 }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialFieldValues.Contains(x.Guid)); _ = collection.ToList(); _substitute.Received().Execute( @@ -2914,7 +2914,7 @@ public void SearchGuidFieldContains() public void TestContainsFromProperty() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); var steve = new Person { @@ -2939,7 +2939,7 @@ public void SearchUlidFieldContains() var potentialFieldValues = new [] { ulid1, ulid2, ulid3 }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialFieldValues.Contains(x.Ulid)); _ = collection.ToList(); _substitute.Received().Execute( @@ -2960,7 +2960,7 @@ public void SearchEnumFieldContains() var potentialFieldValues = new [] { enum1, enum2, enum3 }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialFieldValues.Contains(x.AnEnum)); _ = collection.ToList(); _substitute.Received().Execute( @@ -2981,7 +2981,7 @@ public void SearchNumericEnumFieldContains() var potentialFieldValues = new [] { enum1, enum2, enum3 }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialFieldValues.Contains(x.AnEnumAsInt)); _ = collection.ToList(); _substitute.Received().Execute( @@ -3002,7 +3002,7 @@ public void SearchEnumFieldContainsList() var potentialFieldValues = new List { enum1, enum2, enum3 }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialFieldValues.Contains(x.AnEnum)); _ = collection.ToList(); _substitute.Received().Execute( @@ -3023,7 +3023,7 @@ public void SearchNumericEnumFieldContainsList() var potentialFieldValues = new List { enum1, enum2, enum3 }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialFieldValues.Contains(x.AnEnumAsInt)); _ = collection.ToList(); _substitute.Received().Execute( @@ -3044,7 +3044,7 @@ public void SearchEnumFieldContainsListAsProperty() var potentialFieldValues = new { list = new List { enum1, enum2, enum3 } }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialFieldValues.list.Contains(x.AnEnum)); _ = collection.ToList(); _substitute.Received().Execute( @@ -3065,7 +3065,7 @@ public void SearchNumericEnumFieldContainsListAsProperty() var potentialFieldValues = new { list = new List { enum1, enum2, enum3 } }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => potentialFieldValues.list.Contains(x.AnEnumAsInt)); _ = collection.ToList(); _substitute.Received().Execute( @@ -3081,7 +3081,7 @@ public void SearchNumericEnumFieldContainsListAsProperty() public void TestNestedOrderBy() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); _ = new RedisCollection(_substitute).OrderBy(x => x.Address.State).ToList(); _substitute.Received().Execute("FT.SEARCH", "person-idx", "*", "LIMIT", "0", "100", "SORTBY", "Address_State", "ASC"); } @@ -3090,7 +3090,7 @@ public void TestNestedOrderBy() public void TestGeoFilterNested() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.GeoFilter(x => x.Address.Location, 5, 6.7, 50, GeoLocDistanceUnit.Kilometers).ToList(); @@ -3114,7 +3114,7 @@ public void TestGeoFilterNested() public void TestSelectWithWhere() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.Age == 33).Select(x => x.Name).ToList(); _substitute.Received().Execute( @@ -3134,7 +3134,7 @@ public void TestNullableEnumQueries() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.AnEnum == AnEnum.one && x.NullableStringEnum == AnEnum.two).ToList(); @@ -3152,7 +3152,7 @@ public void TestNullableEnumQueries() public void TestEscapeForwardSlash() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Where(x => x.TagField == "a/test/string").ToList(); @@ -3170,7 +3170,7 @@ public void TestEscapeForwardSlash() public void TestMixedNestingIndexCreation() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply("OK")); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply("OK")); _substitute.CreateIndex(typeof(ComplexObjectWithCascadeAndJsonPath)); @@ -3196,7 +3196,7 @@ public void TestMixedNestingIndexCreation() public void TestMixedNestingQuerying() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); @@ -3234,7 +3234,7 @@ public void TestMixedNestingQuerying() [Fact] public async Task TestCreateIndexWithJsonPropertyName() { - _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); + _substitute.ExecuteAsync(Arg.Any(), Arg.Any()).Returns("OK"); await _substitute.CreateIndexAsync(typeof(ObjectWithPropertyNamesDefined)); @@ -3253,7 +3253,7 @@ await _substitute.Received().ExecuteAsync( public void QueryNamedPropertiesJson() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.FirstOrDefault(x => x.Key == "hello"); @@ -3272,7 +3272,7 @@ public void QueryNamedPropertiesJson() public void TestMultipleContains() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); Expression> whereExpressionFail = a => !a.FirstName.Contains("Andrey") && !a.LastName.Contains("Bred"); @@ -3302,7 +3302,7 @@ public void TestMultipleContains() public void TestSelectNestedObject() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); _ = collection.Select(x => x.Address).ToList(); @@ -3347,7 +3347,7 @@ public void NonNullableNumericFieldContains() var floats = new [] { 25.5F, 26, 27 }; var ushorts = new ushort[] { 28, 29, 30 }; _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute).Where(x => ints.Contains(x.Integer)); _ = collection.ToList(); var expected = $"@{nameof(ObjectWithNumerics.Integer)}:[1 1]|@{nameof(ObjectWithNumerics.Integer)}:[2 2]|@{nameof(ObjectWithNumerics.Integer)}:[3 3]"; @@ -3463,7 +3463,7 @@ public void NonNullableNumericFieldContains() public void TestConstantExpressionContains() { _substitute.ClearSubstitute(); - _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(_mockReply); var collection = new RedisCollection(_substitute); var parameter = Expression.Parameter(typeof(Person), "b"); var property = Expression.Property(parameter, "TagField"); diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/HuggingFaceVectors.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/HuggingFaceVectors.cs new file mode 100644 index 00000000..585de4fc --- /dev/null +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/HuggingFaceVectors.cs @@ -0,0 +1,24 @@ +using Redis.OM.Vectorizers; +using Redis.OM.Modeling; +using Redis.OM.Modeling.Vectors; + +namespace Redis.OM.Unit.Tests; + +[Document(StorageType = StorageType.Json)] +public class HuggingFaceVectors +{ + [RedisIdField] + public string Id { get; set; } + + [Indexed] + [HuggingFaceVectorizer(ModelId = "sentence-transformers/all-MiniLM-L6-v2")] + public Vector Sentence { get; set; } + + [Indexed] + public string Name { get; set; } + + [Indexed] + public int Age { get; set; } + + public VectorScores VectorScore { get; set; } +} \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs new file mode 100644 index 00000000..19fd97ab --- /dev/null +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/ObjectWithVector.cs @@ -0,0 +1,53 @@ +using Redis.OM.Modeling; +using Redis.OM.Modeling.Vectors; + +namespace Redis.OM.Unit.Tests; + +[Document(StorageType = StorageType.Json)] +public class ObjectWithVector +{ + [RedisIdField] + public string Id { get; set; } + + [Indexed] public string Name { get; set; } + + [Indexed] public int Num { get; set; } + + [Indexed(Algorithm = VectorAlgorithm.HNSW)] + [DoubleVectorizer(10)] + public Vector SimpleHnswVector { get; set; } + + [Indexed(Algorithm = VectorAlgorithm.FLAT)] + [SimpleVectorizer] + public Vector SimpleVectorizedVector { get; set; } + + public VectorScores VectorScores { get; set; } +} + +[Document(StorageType = StorageType.Hash)] +public class ObjectWithVectorHash +{ + [RedisIdField] + public string Id { get; set; } + + [Indexed] public string Name { get; set; } + + [Indexed] public int Num { get; set; } + + [Indexed(Algorithm = VectorAlgorithm.HNSW)] + [DoubleVectorizer(10)] + public Vector SimpleHnswVector { get; set; } + + [Indexed(Algorithm = VectorAlgorithm.FLAT)] + [SimpleVectorizer] + public Vector SimpleVectorizedVector { get; set; } + + public VectorScores VectorScores { get; set; } +} + +[Document(StorageType = StorageType.Json, Prefixes = new []{"Simple"})] +public class ToyVector +{ + [RedisIdField] public string Id { get; set; } + [Indexed][DoubleVectorizer(6)]public Vector SimpleVector { get; set; } +} \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAICompletionResponse.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAICompletionResponse.cs new file mode 100644 index 00000000..a8ef2dab --- /dev/null +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAICompletionResponse.cs @@ -0,0 +1,24 @@ +using System; +using Redis.OM.Modeling; +using Redis.OM.Vectorizers; + +namespace Redis.OM.Unit.Tests; + +[Document(StorageType = StorageType.Json)] +public class OpenAICompletionResponse +{ + [RedisIdField] + public string Id { get; set; } + + [Indexed(DistanceMetric = DistanceMetric.COSINE, Algorithm = VectorAlgorithm.HNSW, M = 16)] + [OpenAIVectorizer] + public Vector Prompt { get; set; } + + public string Response { get; set; } + + [Indexed] + public string Language { get; set; } + + [Indexed] + public DateTime TimeStamp { get; set; } +} \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs new file mode 100644 index 00000000..c99df107 --- /dev/null +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/OpenAIVectors.cs @@ -0,0 +1,24 @@ +using Redis.OM.Modeling; +using Redis.OM.Modeling.Vectors; +using Redis.OM.Vectorizers; + +namespace Redis.OM.Unit.Tests; + +[Document(StorageType = StorageType.Json)] +public class OpenAIVectors +{ + [RedisIdField] + public string Id { get; set; } + + [Indexed] + [OpenAIVectorizer] + public Vector Sentence { get; set; } + + [Indexed] + public string Name { get; set; } + + [Indexed] + public int Age { get; set; } + + public VectorScores VectorScore { get; set; } +} \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs new file mode 100644 index 00000000..8b61203d --- /dev/null +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SemanticCachingTests.cs @@ -0,0 +1,65 @@ +using System; +using System.Linq; +using Redis.OM.Contracts; +using Redis.OM.Vectorizers; +using Xunit; + +namespace Redis.OM.Unit.Tests; + +[Collection("Redis")] +public class SemanticCachingTests +{ + private readonly IRedisConnectionProvider _provider; + public SemanticCachingTests(RedisSetup setup) + { + _provider = setup.Provider; + } + + [SkipIfMissingEnvVar("REDIS_OM_OAI_TOKEN")] + public void OpenAISemanticCache() + { + var token = Environment.GetEnvironmentVariable("REDIS_OM_OAI_TOKEN"); + Assert.NotNull(token); + var cache = _provider.OpenAISemanticCache(token, threshold: .15); + cache.Store("What is the capital of France?", "Paris"); + var res = cache.GetSimilar("What really is the capital of France?").First(); + Assert.Equal("Paris",res.Response); + Assert.True(res.Score < .15); + } + + [SkipIfMissingEnvVar("REDIS_OM_HF_TOKEN")] + public void HuggingFaceSemanticCache() + { + var token = Environment.GetEnvironmentVariable("REDIS_OM_HF_TOKEN"); + Assert.NotNull(token); + var cache = _provider.HuggingFaceSemanticCache(token, threshold: .15); + cache.Store("What is the capital of France?", "Paris"); + var res = cache.GetSimilar("What really is the capital of France?").First(); + Assert.Equal("Paris",res.Response); + Assert.True(res.Score < .15); + } + + [SkipIfMissingEnvVar("REDIS_OM_AZURE_OAI_TOKEN")] + public void AzureOpenAISemanticCache() + { + var token = Environment.GetEnvironmentVariable("REDIS_OM_AZURE_OAI_TOKEN"); + var resource = Environment.GetEnvironmentVariable("REDIS_OM_AZURE_OAI_RESOURCE"); + var deployment = Environment.GetEnvironmentVariable("REDIS_OM_AZURE_OAI_DEPLOYMENT"); + var dimStr = Environment.GetEnvironmentVariable("REDIS_OM_AZURE_OAI_DIM"); + if (string.IsNullOrEmpty(dimStr) || !int.TryParse(dimStr, out var dim)) + { + throw new InvalidOperationException("REDIS_OM_AZURE_OAI_DIM must contain a valid integrer value."); + } + + + Assert.NotNull(token); + Assert.NotNull(resource); + Assert.NotNull(deployment); + var cache = _provider.AzureOpenAISemanticCache(token, resource, deployment, dim); + cache.Store("What is the capital of France?", "Paris"); + var res = cache.GetSimilar("What really is the capital of France?").First(); + Assert.Equal("Paris",res.Response); + Assert.True(res.Score < .15); + + } +} \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SimpleVectorizerAttribute.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SimpleVectorizerAttribute.cs new file mode 100644 index 00000000..1514e60f --- /dev/null +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/SimpleVectorizerAttribute.cs @@ -0,0 +1,40 @@ +using System; +using System.Linq; +using Redis.OM.Contracts; +using Redis.OM.Modeling; + +namespace Redis.OM.Unit.Tests; + +public class SimpleVectorizerAttribute : VectorizerAttribute +{ + public override VectorType VectorType => VectorType.FLOAT32; + public override int Dim => 30; + + public override IVectorizer Vectorizer => new SimpleVectorizer(); + + public override byte[] Vectorize(object obj) + { + if (obj is not string s) + { + throw new Exception("Could not vectorize non-string"); + } + + return Vectorizer.Vectorize(s); + } +} + +public class SimpleVectorizer : IVectorizer +{ + public VectorType VectorType => VectorType.FLOAT32; + public int Dim => 30; + public byte[] Vectorize(string obj) + { + var floats = new float[30]; + for (var i = 0; i < 30; i++) + { + floats[i] = i; + } + + return floats.SelectMany(BitConverter.GetBytes).ToArray(); + } +} \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs new file mode 100644 index 00000000..1a497587 --- /dev/null +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorFunctionalTests.cs @@ -0,0 +1,371 @@ +using System; +using System.Linq; +using System.Text; +using Redis.OM.Contracts; +using Redis.OM.Searching; +using Xunit; + +namespace Redis.OM.Unit.Tests; + +[Collection("Redis")] +public class VectorFunctionalTests +{ + private readonly IRedisConnection _connection; + + public VectorFunctionalTests(RedisSetup setup) + { + _connection = setup.Connection; + } + + [SkipIfMissingEnvVar("REDIS_OM_HF_TOKEN")] + public void TestHuggingFaceVectorizer() + { + _connection.DropIndexAndAssociatedRecords(typeof(HuggingFaceVectors)); + _connection.CreateIndex(typeof(HuggingFaceVectors)); + var collection = new RedisCollection(_connection); + var sentenceVector = Vector.Of("Hello World this is Hal."); + var obj = new HuggingFaceVectors + { + Age = 45, + Sentence = sentenceVector, + Name = "Hal" + }; + + collection.Insert(obj); + var queryVector = Vector.Of("Hello World this is Hal."); + var res = collection.NearestNeighbors(x => x.Sentence, 2, queryVector).First(); + Assert.Equal(obj.Id, res.Id); + Assert.Equal(0, res.VectorScore.NearestNeighborsScore); + Assert.Equal(obj.Sentence.Value, res.Sentence.Value); + } + + [SkipIfMissingEnvVar("REDIS_OM_HF_TOKEN")] + public void TestParis() + { + _connection.DropIndexAndAssociatedRecords(typeof(HuggingFaceVectors)); + _connection.CreateIndex(typeof(HuggingFaceVectors)); + var collection = new RedisCollection(_connection); + var sentenceVector = Vector.Of("What is the capital of France?"); + var obj = new HuggingFaceVectors + { + Age = 2259, + Sentence = sentenceVector, + Name = "Paris" + }; + + collection.Insert(obj); + var queryVector = Vector.Of("What really is the capital of France?"); + var res = collection + .First(x => x.Sentence.VectorRange(queryVector, .1, "range") && x.Age > 1000); + res = collection.NearestNeighbors(x => x.Sentence, 2, queryVector).First(x => x.Age > 1000); + Assert.Equal(obj.Id, res.Id); + Assert.True(res.VectorScore.RangeScore < .1); + Assert.Equal(sentenceVector.Value, res.Sentence.Value); + Assert.Equal(obj.Sentence.Embedding, res.Sentence.Embedding); + } + + [SkipIfMissingEnvVar("REDIS_OM_OAI_TOKEN")] + public void TestOpenAIVectorizer() + { + _connection.DropIndexAndAssociatedRecords(typeof(OpenAIVectors)); + _connection.CreateIndex(typeof(OpenAIVectors)); + var collection = new RedisCollection(_connection); + var sentenceVector = Vector.Of("Hello World this is Hal."); + var obj = new OpenAIVectors + { + Age = 45, + Sentence = sentenceVector, + Name = "Hal" + }; + + collection.Insert(obj); + var queryVector = Vector.Of("Hello World this is Hal."); + var res = collection.NearestNeighbors(x => x.Sentence, 2, queryVector).First(); + Assert.Equal(obj.Id, res.Id); + Assert.True(res.VectorScore.NearestNeighborsScore < .01); + Assert.Equal(obj.Sentence.Value, res.Sentence.Value); + Assert.Equal(obj.Sentence.Embedding, res.Sentence.Embedding); + } + + [SkipIfMissingEnvVar("REDIS_OM_OAI_TOKEN")] + public void TestOpenAIVectorRange() + { + _connection.DropIndexAndAssociatedRecords(typeof(OpenAIVectors)); + _connection.CreateIndex(typeof(OpenAIVectors)); + var collection = new RedisCollection(_connection); + var sentenceVector = Vector.Of("What is the capital of France?"); + var obj = new OpenAIVectors + { + Age = 2259, + Sentence = sentenceVector, + Name = "Paris" + }; + + collection.Insert(obj); + var queryVector = Vector.Of("What really is the capital of France?"); + var res = collection.First(x => x.Sentence.VectorRange(queryVector, 1, "range")); + Assert.Equal(obj.Id, res.Id); + Assert.True(res.VectorScore.RangeScore < .1); + Assert.Equal(obj.Sentence.Value, res.Sentence.Value); + Assert.Equal(obj.Sentence.Embedding, res.Sentence.Embedding); + } + + [Fact] + public void BasicRangeQuery() + { + _connection.CreateIndex(typeof(ObjectWithVector)); + var collection = new RedisCollection(_connection); + var simpleHnswVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); + var simpleVectorizedVector = Vector.Of("FooBarBaz"); + collection.Insert(new ObjectWithVector + { + Id = "helloWorld", + SimpleHnswVector = simpleHnswVector, + SimpleVectorizedVector = simpleVectorizedVector + }); + var queryVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); + var res = collection.First(x => x.SimpleHnswVector.VectorRange(queryVector, 5)); + Assert.Equal("helloWorld", res.Id); + } + + [Fact] + public void MultiRangeOnSameProperty() + { + _connection.CreateIndex(typeof(ObjectWithVector)); + var collection = new RedisCollection(_connection); + var simpleHnswVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); + var simpleVectorizedVector = Vector.Of("FooBarBaz"); + collection.Insert(new ObjectWithVector + { + Id = "helloWorld", + SimpleHnswVector = simpleHnswVector, + SimpleVectorizedVector = simpleVectorizedVector + }); + var queryVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); + var res = collection.First(x => x.SimpleHnswVector.VectorRange(queryVector, 5) && x.SimpleHnswVector.VectorRange(queryVector, 6)); + Assert.Equal("helloWorld", res.Id); + } + + [Fact] + public void RangeAndKnn() + { + _connection.CreateIndex(typeof(ObjectWithVector)); + var collection = new RedisCollection(_connection); + var simpleHnswVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); + var simpleVectorizedVector = Vector.Of("FooBarBaz"); + collection.Insert(new ObjectWithVector + { + Id = "helloWorld", + SimpleHnswVector = simpleHnswVector, + SimpleVectorizedVector = simpleVectorizedVector + }); + var queryVector =Enumerable.Range(0, 10).Select(x => (double)x).ToArray(); + queryVector[0] += 2; + var stringQueryVector = Vector.Of("FooBarBaz"); + var res = collection.NearestNeighbors(x=>x.SimpleVectorizedVector, 1, stringQueryVector) + .First(x => x.SimpleHnswVector.VectorRange(queryVector, 5, "range")); + Assert.Equal("helloWorld", res.Id); + Assert.Equal(4, res.VectorScores.RangeScore); + Assert.Equal(0, res.VectorScores.NearestNeighborsScore); + } + + [Fact] + public void RangeAndKnnWithVector() + { + _connection.CreateIndex(typeof(ObjectWithVector)); + var collection = new RedisCollection(_connection); + var simpleHnswVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); + var simpleVectorizedVector = Vector.Of("FooBarBaz"); + collection.Insert(new ObjectWithVector + { + Id = "helloWorld", + SimpleHnswVector = simpleHnswVector, + SimpleVectorizedVector = simpleVectorizedVector + }); + var queryVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); + queryVector.Value[0] += 2; + var stringQueryVector = Vector.Of("FooBarBaz"); + var res = collection.NearestNeighbors(x=>x.SimpleVectorizedVector, 1, stringQueryVector) + .First(x => x.SimpleHnswVector.VectorRange(queryVector, 5, "range")); + Assert.Equal("helloWorld", res.Id); + Assert.Equal(4, res.VectorScores.RangeScore); + Assert.Equal(0, res.VectorScores.NearestNeighborsScore); + } + + [Fact] + public void BasicQuery() + { + _connection.CreateIndex(typeof(ObjectWithVector)); + var collection = new RedisCollection(_connection); + var simpleHnswVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); + var simpleVectorizedVector = Vector.Of("FooBarBaz"); + collection.Insert(new ObjectWithVector + { + Id = "helloWorld", + SimpleHnswVector = simpleHnswVector, + SimpleVectorizedVector = simpleVectorizedVector + }); + var queryVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); + queryVector.Value[0] += 2; + + var stringQueryVector = Vector.Of("FooBarBaz"); + var res = collection.NearestNeighbors(x => x.SimpleVectorizedVector, 1, stringQueryVector).First(); + Assert.Equal("helloWorld", res.Id); + Assert.Equal(0, res.VectorScores.NearestNeighborsScore); + res = collection.NearestNeighbors(x => x.SimpleHnswVector, 1, queryVector).First(); + Assert.Equal(4, res.VectorScores.NearestNeighborsScore); + } + + [Fact] + public void ScoresOnHash() + { + _connection.DropIndexAndAssociatedRecords(typeof(ObjectWithVectorHash)); + _connection.CreateIndex(typeof(ObjectWithVectorHash)); + var simpleHnswVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); + var simpleVectorizedVector = Vector.Of("foo"); + var obj = new ObjectWithVectorHash + { + Id = "helloWorld", + SimpleHnswVector = simpleHnswVector, + SimpleVectorizedVector = simpleVectorizedVector, + }; + var collection = new RedisCollection(_connection); + collection.Insert(obj); + var res = collection.NearestNeighbors(x => x.SimpleHnswVector, 5, simpleHnswVector).First(); + + Assert.Equal(0, res.VectorScores.NearestNeighborsScore); + } + + [Fact] + public void HybridQueryTest() + { + _connection.DropIndexAndAssociatedRecords(typeof(ObjectWithVectorHash)); + _connection.CreateIndex(typeof(ObjectWithVectorHash)); + var simpleHnswVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); + var simpleVectorizedVector = Vector.Of("foo"); + var obj = new ObjectWithVectorHash + { + Id = "theOneWithStuff", + SimpleHnswVector = simpleHnswVector, + Name = "Steve", + Num = 6, + SimpleVectorizedVector = simpleVectorizedVector, + }; + var collection = new RedisCollection(_connection); + collection.Insert(obj); + var res = collection.Where(x=>x.Name == "Steve" && x.Num == 6).NearestNeighbors(x => x.SimpleHnswVector, 5, simpleHnswVector).First(); + + Assert.Equal(0, res.VectorScores.NearestNeighborsScore); + } + + [Fact] + public void TestIndex() + { + _connection.CreateIndex(typeof(ObjectWithVectorHash)); + + var simpleHnswVector = Vector.Of(Enumerable.Range(0, 10).Select(x => (double)x).ToArray()); + var simpleVectorizedVector = Vector.Of("foo"); + var obj = new ObjectWithVectorHash + { + Id = "helloWorld", + SimpleHnswVector = simpleHnswVector, + SimpleVectorizedVector = simpleVectorizedVector, + }; + + var key = _connection.Set(obj); + var res = _connection.Get(key); + Assert.Equal(simpleHnswVector.Value, res.SimpleHnswVector.Value); + Assert.Equal(simpleHnswVector.Embedding, res.SimpleHnswVector.Embedding); + + simpleVectorizedVector = Vector.Of("foobarbaz"); + key = _connection.Set(new ObjectWithVector() + { + Id = "helloWorld", + SimpleHnswVector = simpleHnswVector, + SimpleVectorizedVector = simpleVectorizedVector + }); + + var jsonRes = _connection.Get(key); + + Assert.Equal(simpleHnswVector.Value, jsonRes.SimpleHnswVector.Value); + Assert.Equal(simpleHnswVector.Embedding, jsonRes.SimpleHnswVector.Embedding); + Assert.Equal(simpleVectorizedVector.Value, jsonRes.SimpleVectorizedVector.Value); + Assert.Equal(simpleVectorizedVector.Embedding, jsonRes.SimpleVectorizedVector.Embedding); + } + + [Fact] + public void Insert() + { + var simpleHnswJsonStr = new StringBuilder(); + var vectorizedFlatVectorJsonStr = new StringBuilder(); + simpleHnswJsonStr.Append('['); + vectorizedFlatVectorJsonStr.Append('['); + var simpleHnswHash = new double[10]; + var vectorizedFlatHashVector = new float[30]; + for (var i = 0; i < 10; i++) + { + simpleHnswHash[i] = i; + } + + for (var i = 0; i < 30; i++) + { + vectorizedFlatHashVector[i] = i; + } + + simpleHnswJsonStr.Append(string.Join(',', simpleHnswHash)); + vectorizedFlatVectorJsonStr.Append(string.Join(',', vectorizedFlatHashVector)); + simpleHnswJsonStr.Append(']'); + vectorizedFlatVectorJsonStr.Append(']'); + + var simpleHnswVector = Vector.Of(simpleHnswHash); + var simpleVectorizedVector = Vector.Of("foobar"); + var hashObj = new ObjectWithVectorHash() + { + Id = "helloWorld", + SimpleHnswVector = simpleHnswVector, + SimpleVectorizedVector = simpleVectorizedVector + }; + + var key = _connection.Set(hashObj); + var res = _connection.Get(key); + Assert.Equal(simpleVectorizedVector.Value, res.SimpleVectorizedVector.Value); + Assert.Equal(simpleVectorizedVector.Embedding, res.SimpleVectorizedVector.Embedding); + } + + [SkipIfMissingEnvVar("REDIS_OM_OAI_TOKEN")] + public void OpenAIQueryTest() + { + var provider = new RedisConnectionProvider(""); + + provider.RedisCollection(); + _connection.DropIndexAndAssociatedRecords(typeof(OpenAICompletionResponse)); + _connection.CreateIndex(typeof(OpenAICompletionResponse)); + + var collection = new RedisCollection(_connection); + var query = new OpenAICompletionResponse + { + Language = "en_us", + Prompt = Vector.Of("What is the Capital of France?"), + Response = "Paris", + TimeStamp = DateTime.Now - TimeSpan.FromHours(3) + }; + collection.Insert(query); + var queryPrompt ="What really is the Capital of France?"; + var result = collection.First(x => x.Prompt.VectorRange(queryPrompt, .15)); + + Assert.Equal("Paris", result.Response); + + result = collection.NearestNeighbors(x => x.Prompt, 1, queryPrompt).First(); + Assert.Equal("Paris", result.Response); + + result = collection.Where(x=>x.Language == "en_us").NearestNeighbors(x => x.Prompt, 1, queryPrompt).First(); + Assert.Equal("Paris", result.Response); + + result = collection.First(x=>x.Language == "en_us" && x.Prompt.VectorRange(queryPrompt, .15)); + Assert.Equal("Paris", result.Response); + + var ts = DateTimeOffset.Now - TimeSpan.FromHours(4); + result = collection.First(x=>x.TimeStamp > ts && x.Prompt.VectorRange(queryPrompt, .15)); + Assert.Equal("Paris", result.Response); + } +} \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs new file mode 100644 index 00000000..0ab57f6d --- /dev/null +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/VectorTests/VectorTests.cs @@ -0,0 +1,215 @@ +using System; +using System.Linq; +using System.Text; +using System.Text.Json; +using NSubstitute; +using NSubstitute.ClearExtensions; +using Redis.OM.Contracts; +using Redis.OM.Modeling; +using Redis.OM.Modeling.Vectors; +using Redis.OM.Searching; +using Xunit; + +namespace Redis.OM.Unit.Tests; + +public class VectorIndexCreationTests +{ + private readonly IRedisConnection _substitute = Substitute.For(); + + [Fact] + public void CreateIndexWithVector() + { + _substitute.ClearSubstitute(); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply("OK")); + + _substitute.CreateIndex(typeof(ObjectWithVector)); + + _substitute.Received().Execute( + "FT.CREATE", + $"{nameof(ObjectWithVector).ToLower()}-idx", + "ON", + "Json", + "PREFIX", + "1", + $"Redis.OM.Unit.Tests.{nameof(ObjectWithVector)}:", + "SCHEMA", + "$.Name", "AS", "Name", "TAG", "SEPARATOR", "|", + "$.Num", "AS", "Num", "NUMERIC", + "$.SimpleHnswVector", "AS", "SimpleHnswVector", "VECTOR", "HNSW", "6", "TYPE", "FLOAT64", "DIM", "10", "DISTANCE_METRIC", "L2", + "$.SimpleVectorizedVector.Vector", "AS","SimpleVectorizedVector", "VECTOR", "FLAT", "6", "TYPE", "FLOAT32", "DIM", "30", "DISTANCE_METRIC", "L2" + ); + + _substitute.ClearSubstitute(); + _substitute.CreateIndex(typeof(ObjectWithVectorHash)); + _substitute.Received().Execute( + "FT.CREATE", + $"{nameof(ObjectWithVectorHash).ToLower()}-idx", + "ON", + "Hash", + "PREFIX", + "1", + $"Redis.OM.Unit.Tests.{nameof(ObjectWithVectorHash)}:", + "SCHEMA", + "Name", "TAG", "SEPARATOR", "|", + "Num", "NUMERIC", + "SimpleHnswVector", "VECTOR", "HNSW", "6", "TYPE", "FLOAT64", "DIM", "10", "DISTANCE_METRIC", "L2", + "SimpleVectorizedVector.Vector", "VECTOR", "FLAT", "6", "TYPE", "FLOAT32", "DIM", "30", "DISTANCE_METRIC", "L2" + ); + } + + [Fact] + public void SimpleKnnQuery() + { + _substitute.ClearSubstitute(); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply(0)); + var collection = new RedisCollection(_substitute); + var compVector = Vector.Of(new double[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + float[] floats = Enumerable.Range(0, 30).Select(x => (float)x).ToArray(); + var blob = compVector.Value.SelectMany(BitConverter.GetBytes).ToArray(); + var floatBlob = floats.SelectMany(BitConverter.GetBytes).ToArray(); + _ = collection.NearestNeighbors(x=>x.SimpleHnswVector, 5, compVector).ToList(); + + _substitute.Received().Execute("FT.SEARCH", + $"{nameof(ObjectWithVector).ToLower()}-idx", + $"(*)=>[KNN 5 @SimpleHnswVector $0 AS {VectorScores.NearestNeighborScoreName}]", + "PARAMS", 2, "0", Arg.Is(b=>b.SequenceEqual(blob)), "DIALECT", 2, "LIMIT", "0", "100"); + + _substitute.ClearSubstitute(); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply(0)); + var queryVector = Vector.Of("hello world"); + _ = collection.NearestNeighbors(x => x.SimpleVectorizedVector, 8, queryVector).ToArray(); + + _substitute.Received().Execute("FT.SEARCH", + $"{nameof(ObjectWithVector).ToLower()}-idx", + $"(*)=>[KNN 8 @SimpleVectorizedVector $0 AS {VectorScores.NearestNeighborScoreName}]", + "PARAMS", 2, "0", Arg.Is(b=>b.SequenceEqual(floatBlob)), "DIALECT", 2, "LIMIT", "0", "100"); + } + + [Fact] + public void SimpleRangeQuery() + { + _substitute.ClearSubstitute(); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply(0)); + var collection = new RedisCollection(_substitute); + var compVector = new double[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + float[] floats = Enumerable.Range(0, 30).Select(x => (float)x).ToArray(); + var floatBytes = floats.SelectMany(BitConverter.GetBytes).ToArray(); + var queryVector = Vector.Of("foobar"); + _ = collection.Where(x => x.SimpleVectorizedVector.VectorRange(queryVector, .3)).ToList(); + _substitute.Received().Execute("FT.SEARCH", $"{nameof(ObjectWithVector).ToLower()}-idx", + "@SimpleVectorizedVector:[VECTOR_RANGE $0 $1]", "PARAMS", 4, "0", .3, "1", Arg.Is(b => b.SequenceEqual(floatBytes)), + "DIALECT", 2, "LIMIT", "0", "100"); + + } + + [Fact] + public void SimpleKnnQueryWithSortBy() + { + _substitute.ClearSubstitute(); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply(0)); + var collection = new RedisCollection(_substitute); + var compVector = Vector.Of(new double[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + float[] floats = Enumerable.Range(0, 30).Select(x => (float)x).ToArray(); + var blob = compVector.Value.SelectMany(BitConverter.GetBytes).ToArray(); + var floatBlob = floats.SelectMany(BitConverter.GetBytes).ToArray(); + _ = collection.NearestNeighbors(x=>x.SimpleHnswVector, 5, compVector).OrderBy(x=>x.VectorScores.NearestNeighborsScore).ToList(); + + _substitute.Received().Execute("FT.SEARCH", + $"{nameof(ObjectWithVector).ToLower()}-idx", + $"(*)=>[KNN 5 @SimpleHnswVector $0 AS {VectorScores.NearestNeighborScoreName}]", + "PARAMS", 2, "0", Arg.Is(b=>b.SequenceEqual(blob)), "DIALECT", 2, "LIMIT", "0", "100", "SORTBY", VectorScores.NearestNeighborScoreName, "ASC"); + + _substitute.ClearSubstitute(); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply(0)); + var queryVector = Vector.Of("hello world"); + _ = collection.NearestNeighbors(x => x.SimpleVectorizedVector, 8, queryVector).OrderByDescending(x=>x.VectorScores.NearestNeighborsScore).ToArray(); + + _substitute.Received().Execute("FT.SEARCH", + $"{nameof(ObjectWithVector).ToLower()}-idx", + $"(*)=>[KNN 8 @SimpleVectorizedVector $0 AS {VectorScores.NearestNeighborScoreName}]", + "PARAMS", 2, "0", Arg.Is(b=>b.SequenceEqual(floatBlob)), "DIALECT", 2, "LIMIT", "0", "100", "SORTBY", VectorScores.NearestNeighborScoreName, "DESC"); + } + + + [Fact] + public void TestBinConversions() + { + var piStr = VectorUtils.DoubleToVecStr(Math.PI); + var pi = VectorUtils.DoubleFromVecStr(piStr); + Assert.Equal(Math.PI, pi); + } + + [Fact] + public void InsertVectors() + { + var simpleHnswJsonStr = new StringBuilder(); + var vectorizedFlatVectorJsonStr = new StringBuilder(); + simpleHnswJsonStr.Append('['); + vectorizedFlatVectorJsonStr.Append('['); + var simpleHnswHash = new double[10]; + var vectorizedFlatHashVector = new float[30]; + for (var i = 0; i < 10; i++) + { + simpleHnswHash[i] = i; + } + for (var i = 0; i < 30; i++) + { + vectorizedFlatHashVector[i] = i; + } + + simpleHnswJsonStr.Append(string.Join(',', simpleHnswHash)); + vectorizedFlatVectorJsonStr.Append(string.Join(',', vectorizedFlatHashVector)); + simpleHnswJsonStr.Append(']'); + vectorizedFlatVectorJsonStr.Append(']'); + + var simpleHnswBytes = simpleHnswHash.SelectMany(BitConverter.GetBytes).ToArray(); + var flatVectorizedBytes = vectorizedFlatHashVector.SelectMany(BitConverter.GetBytes).ToArray(); + + var simpleHnswVector = Vector.Of(simpleHnswHash); + var simpleVectorizedVector = Vector.Of("foobar"); + + var hashObj = new ObjectWithVectorHash() + { + Id = "foo", + SimpleHnswVector = simpleHnswVector, + SimpleVectorizedVector = simpleVectorizedVector + }; + + var jsonObj = new ObjectWithVector() + { + Id = "foo", + SimpleHnswVector = simpleHnswVector, + SimpleVectorizedVector = simpleVectorizedVector + }; + + var json = + $"{{\"Id\":\"foo\",\"Num\":0,\"SimpleHnswVector\":{simpleHnswJsonStr},\"SimpleVectorizedVector\":{{\"Value\":\"\\u0022foobar\\u0022\",\"Vector\":{vectorizedFlatVectorJsonStr}}}}}"; + + _substitute.Execute("HSET", Arg.Any()).Returns(new RedisReply("3")); + _substitute.Execute("JSON.SET", Arg.Any()).Returns(new RedisReply("OK")); + _substitute.Set(hashObj); + _substitute.Set(jsonObj); + _substitute.Received().Execute("HSET", "Redis.OM.Unit.Tests.ObjectWithVectorHash:foo", "Id", "foo", "Num", "0", "SimpleHnswVector", + Arg.Is(x=>x.SequenceEqual(simpleHnswBytes)), "SimpleVectorizedVector.Vector", Arg.Is(x=>x.SequenceEqual(flatVectorizedBytes)), "SimpleVectorizedVector.Value", "\"foobar\""); + _substitute.Received().Execute("JSON.SET", "Redis.OM.Unit.Tests.ObjectWithVector:foo", ".", json); + var deseralized = JsonSerializer.Deserialize(json); + Assert.Equal("foobar", deseralized.SimpleVectorizedVector.Value); + } + + [Fact] + public void HybridQuery() + { + _substitute.ClearSubstitute(); + _substitute.Execute(Arg.Any(), Arg.Any()).Returns(new RedisReply(0)); + var collection = new RedisCollection(_substitute); + var compVector = Vector.Of(new double[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + var blob = compVector.Value.SelectMany(BitConverter.GetBytes); + _ = collection.Where(x => x.Name == "Steve" && x.Num < 5) + .NearestNeighbors(x => x.SimpleHnswVector, 2, compVector).ToList(); + _substitute.Received().Execute("FT.SEARCH", + $"{nameof(ObjectWithVector).ToLower()}-idx", + $"(((@Name:{{Steve}}) (@Num:[-inf (5])))=>[KNN 2 @SimpleHnswVector $0 AS {VectorScores.NearestNeighborScoreName}]", + "PARAMS", 2, "0", Arg.Is(b=>b.SequenceEqual(blob)), "DIALECT", 2, "LIMIT", "0", "100"); + + } +} \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/Redis.OM.Unit.Tests.csproj b/test/Redis.OM.Unit.Tests/Redis.OM.Unit.Tests.csproj index 72fa0e34..f19b411a 100644 --- a/test/Redis.OM.Unit.Tests/Redis.OM.Unit.Tests.csproj +++ b/test/Redis.OM.Unit.Tests/Redis.OM.Unit.Tests.csproj @@ -1,6 +1,6 @@  - net6.0 + net6.0;net7.0 false @@ -15,6 +15,7 @@ + diff --git a/test/Redis.OM.Unit.Tests/RedisSetupCollection.cs b/test/Redis.OM.Unit.Tests/RedisSetupCollection.cs index 2e533f2a..374b1292 100644 --- a/test/Redis.OM.Unit.Tests/RedisSetupCollection.cs +++ b/test/Redis.OM.Unit.Tests/RedisSetupCollection.cs @@ -32,24 +32,18 @@ public RedisSetup() Connection.CreateIndex(typeof(ObjectWithDateTimeOffsetJson)); } - private IRedisConnection _connection = null; - public IRedisConnection Connection - { - get - { - if (_connection == null) - _connection = GetConnection(); - return _connection; - } - } + private IRedisConnectionProvider _provider; - private IRedisConnection GetConnection() + public IRedisConnectionProvider Provider => _provider ??= GetProvider(); + + public IRedisConnection Connection => Provider.Connection; + + private IRedisConnectionProvider GetProvider() { var host = Environment.GetEnvironmentVariable("STANDALONE_HOST_PORT") ?? "localhost:6379"; var connectionString = $"redis://{host}"; - var provider = new RedisConnectionProvider(connectionString); - return provider.Connection; - } + return new RedisConnectionProvider(connectionString); + } public void Dispose() { diff --git a/test/Redis.OM.Unit.Tests/SearchJsonTests/RedisJsonIndexTests.cs b/test/Redis.OM.Unit.Tests/SearchJsonTests/RedisJsonIndexTests.cs index c4453331..5d2a7dc2 100644 --- a/test/Redis.OM.Unit.Tests/SearchJsonTests/RedisJsonIndexTests.cs +++ b/test/Redis.OM.Unit.Tests/SearchJsonTests/RedisJsonIndexTests.cs @@ -1,6 +1,5 @@ using System; using System.Linq; -using Redis.OM; using Redis.OM.Modeling; using Xunit; diff --git a/test/Redis.OM.Vectorizer.Tests/DocWithVectors.cs b/test/Redis.OM.Vectorizer.Tests/DocWithVectors.cs new file mode 100644 index 00000000..417a2264 --- /dev/null +++ b/test/Redis.OM.Vectorizer.Tests/DocWithVectors.cs @@ -0,0 +1,23 @@ +using Redis.OM.Modeling; +using Redis.OM.Modeling.Vectors; +using Redis.OM.Vectorizers.AllMiniLML6V2; +using Redis.OM.Vectorizers.Resnet18; + +namespace Redis.OM.Vectorizer.Tests; + +[Document(StorageType = StorageType.Json)] +public class DocWithVectors +{ + [RedisIdField] + public string? Id { get; set; } + + [Indexed(Algorithm = VectorAlgorithm.HNSW)] + [SentenceVectorizer] + public Vector? Sentence { get; set; } + + [Indexed] + [ImageVectorizer] + public Vector? ImagePath { get; set; } + + public VectorScores? Scores { get; set; } +} \ No newline at end of file diff --git a/test/Redis.OM.Vectorizer.Tests/GlobalUsings.cs b/test/Redis.OM.Vectorizer.Tests/GlobalUsings.cs new file mode 100644 index 00000000..8c927eb7 --- /dev/null +++ b/test/Redis.OM.Vectorizer.Tests/GlobalUsings.cs @@ -0,0 +1 @@ +global using Xunit; \ No newline at end of file diff --git a/test/Redis.OM.Vectorizer.Tests/Redis.OM.Vectorizer.Tests.csproj b/test/Redis.OM.Vectorizer.Tests/Redis.OM.Vectorizer.Tests.csproj new file mode 100644 index 00000000..64e95962 --- /dev/null +++ b/test/Redis.OM.Vectorizer.Tests/Redis.OM.Vectorizer.Tests.csproj @@ -0,0 +1,38 @@ + + + net7.0 + enable + enable + latest + + false + true + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + + PreserveNewest + + + + diff --git a/test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs b/test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs new file mode 100644 index 00000000..97dd6aa4 --- /dev/null +++ b/test/Redis.OM.Vectorizer.Tests/VectorizerFunctionalTests.cs @@ -0,0 +1,45 @@ +using Redis.OM.Contracts; +using Redis.OM.Searching; +using Redis.OM.Vectorizers.AllMiniLML6V2; + +namespace Redis.OM.Vectorizer.Tests; + +public class VectorizerFunctionalTests +{ + private readonly IRedisConnectionProvider _provider; + public VectorizerFunctionalTests() + { + var host = Environment.GetEnvironmentVariable("STANDALONE_HOST_PORT") ?? "localhost"; + _provider = new RedisConnectionProvider($"redis://{host}"); + } + + [Fact] + public void Test() + { + var connection = _provider.Connection; + connection.CreateIndex(typeof(DocWithVectors)); + connection.Set(new DocWithVectors + { + Sentence = Vector.Of("Hello world this is Hal."), + ImagePath = Vector.Of("hal.jpg") + }); + + var collection = new RedisCollection(connection); + + // images + var res = collection.NearestNeighbors(x => x.ImagePath!, 5, "hal.jpg"); + Assert.Equal(0, res.First().Scores!.NearestNeighborsScore); + // sentences + collection.NearestNeighbors(x => x.Sentence!, 5, "Hello world this really is Hal."); + } + + [Fact] + public void SemanticCaching() + { + var cache = _provider.AllMiniLML6V2SemanticCache(); + cache.Store("What is the Capital of France?", "Paris"); + var res = cache.GetSimilar("What really is the capital of France?"); + Assert.NotEmpty(res); + Assert.Equal("Paris", res.First().Response); + } +} \ No newline at end of file diff --git a/test/Redis.OM.Vectorizer.Tests/hal.jpg b/test/Redis.OM.Vectorizer.Tests/hal.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d581779f572737118018ef9a8820f864e6ff0747 GIT binary patch literal 33140 zcmeFZbwE_#)-Zf#7`lcIiJ?Pk1_lO@ZlpoVAcvMtkr0uV7(znn6pIi$0SRdlL9qZ4 z5ET^=5aHc3Sik$+``qWd-}}D*yuS@+pS9OsYwfky+Gp3^NDTS&uH_!!ibq*^1%w8Kcm)KabgUInrY>&Y z{=_tdkwYt}$YE75I22l51%p<>$UzWUIqbJ?La^-*5CbU% zxyKVZx1T|Y5FHg26*UzdH8mX*Ee$Ob8zUVZBO50R3mXdyClehh{J8BT{xOEp)6&v2 z&?6ZbkgN<046MY6fpv$4>3!O4DRA|V(8Mg~W~$tfwQ$jRvBK_(L!LVy`fu4U=M zA{cZGL&2I;(WEUT%w~1ZRbC-DVOWP!gkZh&`jcA-mc6q1xTtQfjl16cs!&Cak$qG6 z&n=1VkHV(E=!@l5pJ>hR7@b+Jx!n2q&5Er@cv3-a+k>&$RV4#E&xqtxbyvEcy#2}q z!Qnt#1W6}K3UVw-1pzcO0%#B<$U=@emcvTa;GWg+lDx3%>)-@}jy0QGh{7ik3Zep( zqS*T*K#3|2cfEazq4=p^D*S5=#D1%={S=}jBbCVnX+WPJ-Pe-oLF9F_$tAJgH#TIw zr%i6cG9e;N*<-?=z$s}YaqVPgfG@etvzScZ($@zwoM9KS-#LZ@wvO6T?(N?bx|zET z9lE#;MOg&4NVgxh9pyf-@S^=DGwSK|dQI8Y*4-OV9>2Rd98fjyLys>h)>1t^!m|wx zSn0Z6v^>SO{IqZ;_|p+x6Wyh`I#4c3`@WzK1*Y(AB8~XA-4RhuJI%vMbPx-Q@M%l6jBydFRzdFTlSkh)ph#q8yEODtwDh+ z^?tsKvAs{<@r8}wJ@E01gN(t2+Gl3SO9i97(#>x(b#@)T5I(W_HBQNCQ)n#f*_nu% zln~$f7|Dh2mRU(%2N?q9#wUB{A#aN4#no4J%76q?eF>V zNsj3@GS|5GDYOFu`ix*qvZqankSg%l*&E z$G`cM?JkLa$3NzKXvRP1%&JM!0Uk$tEpJ2$DmnVV$P@Xw?<(7nVT0||nC)y~ATwiO zNDgh#;#cf80z`razndg$%9wK5?@!{G2T#qvhf_9{NF$pI^!Xf8A9rz-9kF%mO+Vgy;`{V1x zJhJyYc|XQ9_|H3uPn49$K#EC`M*I(n^rR%QtVj|i*RM1mN$M9;nwk`Y2u9KajQ7Mq zLmE4Y5S&E2LIgp%vK}5F#}C9{lm-Fz15;5{{0oC2;T=KzlLkw|NdLx=WUhfU1OEKi(UL8RHq4mUS3NoEeB9g7f?5OL-Mj|TufWC3|W0Z=HoY#=;z7}&%Q zasg>Pkqlh_82-Q!CH$I0{i6GW;1}+Ps>HN^n4cfG5vxt%8W8619%An5;|5Yc_^jMQ z(I7=?rZ!-z=NI7b3DTrsg7*mhGp!vOaCj%ZqlT5I*N=Qsz+}0L=_8N>ai|_Yc{Wp1}cOfqy|!1O$6~di&%3!wv%;Iz4NmbaN68r0HG4LIVu& z{`g>*P`o=pKsHf<_#ebHJH$i`F~{hzCkovDPuHI;EZEOR&)xkQFVx1x^Dj6CH$33OM}``O7}}VcxCR9H5d~8HME`}D+AARVh?bwX=TA#9?o{6J zCmL{4y5l`u!u&!3NF9z34*e(Owm;E-C8u@u)Cuql2>!#kOgr+m4Sr<+7RW5XpJ;39 z(11Ymu+R|vpACn`4>Y9zf~9i}2n7x5zu;+si@pAlob()u6AOaqzy;S>BEdV)9C1yv z(}XEVbVLe>A`e7Fvz3Dj^iZ%@5LAB`;3FYUFz5e!mGJi}VRrTJRl?t^guhn_f3Fe< z*4zJgRtco>NdSyU5VQ|mV9p6ajF1-O4km=IU=oP>V}ck$nkohY3YLFEfr4}(Ye<18 zaA!oM03aClAu3P+_}_lb+UOM;8mJ;G>mMTHLfqYxaSJ#s8|e}#ic`-ceql%R_baSQei4D}B1M-e5ufO&DK znjp}1rxJ$)e~JC4*7`~4?)Hl=5KR3@g}b}S;{EW4@&2J9z_Mtf0e%WKHT^~YPsJWS z{EIpyR4*K8^xwHGgg7IUwZex4gay0d^}<1kLO%-&3AMuiHqAdR?62&A{1Zu!=Y|8Yic%ucxbsR?@v<2}FMu`ODh_N!z6T~DMWh{V|Fhsl( zZ~};ds}#`6KqfIJ@+#wqyvjHtjWUi%qpVD#QAU$sqBLcaG-YLCiOM9I%0x}T)2%24 z+$jZ`fRv(?k`zu#Sqhj2jg~@Vq|owGXayfTTcY7wYzBCPR2d4&KD@i&5486`={x@Y^S>i9!V3?c+W)3E+#w4I@Cc1?3C3%Af=B(|b+)qq zO79St@PDJV_FsD~WvsrQmcF8#wvIkpUQb^6XN~{R8Z9S{LEC`$jc6q>7x>vN{|8$C zFs7TAi@zrx%xh!?|MPlaK?R@u<)M)E{HnzciRTJ{a`czZ&eH#r)C%^}n;gf7{aj z{n+|{+e5O%ndZ(zFZ+-E!A@Ft$BOzy$B<@lD*x=s|DUuyFr8Cz|Bq%3JLT?dy#9x? z24X8ARrZf*_D>X;yA#1*%LetIll#B#e_P;h3;b-fkZsaZ?bMmb1u_4ZMJ@@w6aBZDf6)DRAknuw`ClwaK7+CM{l@*~{EZ8! z1jjMJ!4kmDoz&ZgRj0fA2 zC|l6Vy!}0ayZzn0NgLzd{(srV_(Q^f$o7W>BDmA80l-qd1+hBGK*&K>i0pkIgkYhC z$PSc(9M~_vnNZt;_aYEv$NzSx-2)iplj6Tb@GOvohj^3Ddq6r?HYm5S;P4$11o1`= zjyo|ztPmGC6fFdaL(&iiQi4<=O-K(ihRngC90%wSB0 zDu&9SYN!Fa05wBbpqtP==mGQ)9G-p*J%e6BZ=iS360{0^0|$vHVe~K-7#EBmCIZ_H z!@zJbb(kK^1ZD+ufH}iFVg9f%*iqPVSUQXVD~46T8emPZYp{E;9@sE!0`?j<5Bm(; zfRn-L;H+?7xF}o}j)QB#_rh)92jQOZAovk@B0Ljb2(N^ngSWx&!5_jO!(YMQ!B@b! zYFaXOG66DaG8~x>nK_vwnFm=gSu9x^Spiuk*#)xeWL;#B$X<~xlC2{s5G)83LK>lp zFhuM_xFLcNv4{*rF`^#PhUh>%LcB&SA-2fr$$7|kldF&$k=v7dl1Gpykr$BHl3yn8 zB!5i)hJ2NRf`Xkwj6#XRkivn&ha#FHgW?RuMT)x=!xYmLE0mO!T$H;h)hNv=T_{5- zlPHTQ&r#l?9HD$exkg1x#ZQHy(x-By@~29mDxf+`b(?C0>MhkbY9zG?wFyk&>NnKiXqafkXw+zIXnbgn(G=2Lpy{NUr1?TiODjmLOlwK&O&d>JNPCI4 zoAx#BIvq2eB%Kc30lF}{Ou9O{yL1zDU+C%SMd>x^9qB{qGwB=X+v#7>uQ9MN$S@c& zxHH5tlrXe0j4*s)q-GRh)M7luc!aTlv4wGvaS=(46h-PFU68TJ667`H6XXgL3lo~j zoar!A2Gd!l9;P{FN@h`JJ!W_2MCK~ycIIgoG8Q2g9Tqp1<1E!Ioh)xyDOkl=jaYqH z(^=264zMnQRQRZH zqwrG^Y7v}>m&hrRE|IUIVxo4U$)Z<8=fpV0jKq$JHHuA&(~E0}2Z>jTk4jKT;3Rw{ z&PWVN!X>eiUXmq}gS+9o6?c2@KC^pRiX80W`b$+xJ&~r9){+jFZj^p4!!BbYlOS_N z=7X%TtfOp>Y?tho99GUxu3GLHni*}3PC#Erf5u2++%P4WQF(fKefc=~tMZ=|b}Qf& zN)^VjELc-)GWI_9yP}d}u;K;9c_m>bXQk6hPjIX_3tR@STbW#0TRBeort-RqqDrVr zv&u(RX;ojPdDWvJHS*!V0 zOG3*>>zvjHZ8_~=?aSIbnfbs=^E&!=?>^I>+RDk)_b8ZsPC!YsJ~=@HHbF2 zYe;Um*D%NMi4m`nn^A+&lChF;jB&?ay1mwWi}y~MNSXwjTsMWA8k^>tPMV3B`I}ua zhs=%4bIqSyh*<<#+_0puw6H9(oV7w*9kuGRX0dj%K5M;dqivIA^K75^zOa4wZ5eG3 z*f!XHwbQjD*uAoswU4oX=)mpZ?QqqR($Utj)^TOO-v0dkZw_D&Bpnz#D0VRNV9z1$ zLw<*DIWao9IJG#FJKH&*b>4I_cd2w)aW!&1?YiWq<95pJox7%cp8Gsr4Nt(o^-%L5 zc+7dKdFFb~duezTcrAMCco%zr@-g%&^ZDv)?px=(?PuqA@i5h4=fl_hnf-nJI|BFu zA_Im4cLyc~z6w$a$`4u!-WyyS0t-19ay66{Os4w6#KTU6y$)9oF9~0Zu#ISmWQz2U z?2D3&N{xDZME^)lG$Pt9`u$;ic5@p6K@b-e~jjs@3H;_ zxdcMO@^QQ4*AsaY;}fS(7@TNKVn_;3dXlV~T$X}J@lF{?l}{~7-AZ#$>rIzSFG%0a zaLedDi9T6)ay!!_b0AAGt2CP;+dq3OMJLUHjC=`^P zqB#|I>Q$jh;ngDHqU@s0Vz1)ICE6tyPxG8kJ^l5J`xoNChLw=Yxl3KUcY=p=0@F3ftw|_*luOrrn#MXd;89jJ1chs?=If+yf=N{>Hf2J`}WZe z%Z`E0y`9|;^dGc$X?5M{R_ngmqtbJ&7uS2`q2j}~K5XCRer*5c0mXr~L8ZZ~L&`(f zhxZKM9?=-N_el5AgHfZ=hmXx350C8|8-H@($*Xbq@wo}ViBC_%pMHB5KS@5B_8j@V z;04c%%9j!^FTPTIb?de6>%J-Lsb|w})9+?NW;Wj>&eG2oyybt}Fo&7DIj=uIw6K5S z&AY&N-xibJGrd3kLHtAON6n9YOZH1|J_Uc;{+#iJ>r36T!t(tUijFL@(jgupCjOIwe-Oa%N(_|b6yy}tRMgbq`y%kM4m~ZnXsD>^kaYA6j3Cg^F)=eC znHi8sMxqcRFZfV~5->8-P|*-6{=bFo``|Mom^VD03H||pO;-={J8aB4d zW92Kp#(ZA4?fy8DhOAHD(EWtCz;R+sM-t|(o0q5#3x%qz*nB1sDn*GPHQrg_yU zc-2T4TkD@Bql`GW961C|w`i(~zAft?)=D4~HOY>zHPq%3AaDuf3OB%Qr))AMhv`_$ zfx_$x)_Mv+3b=+%k#(>I|7gaz{^f1R?hVWP>AB89zLuF-ji;}>bdA4^d=HuodH)7K zK||$^OcbDUzn1`RxuPb7YRO&4c4w4#i%S?=6-e6;R~Acve^W9&gv9aD179`vPe*Iq zSgQCCD?9lDI~slT(C(?256@2d%JP;Sn~r&Ft`RkOhj1#p*3i9&hDuXM7~|eUAQLsR z644P>cJ{cMW}N|aHTD8^t&ynAV;yeM6NjeM4(2|Z{-Ac7la+Dzm)ob<6`|S8)zHt( zi}8u#2n=0zPp+sI6%d5Xr_v#?F$m^O6V=PBk*F^pt^_sq8a9O(gdR_fy547n$7@;- z`$x`4K>}1Txs5R9L9bU0xr8Yz(BO=WICG9B zL^kpvaw3W@uLtlT1gJm-bWn*H9W32(f`|@O$0-3CLmqI`C5f}6R=_aG8i@x-o~cNU z!oWP+u731W@Ej6;T+}3i$nBnFWF>FWL&cztoTkW>1Dve2C*cJWm5a-{N#%!}5|^_|NC4!w+y7P0V*4C^m&q~?C;K=wMLmgq1z;59bZD3z*WU>IPgPxO|93qxjz*bqH0IXG;Vx96; zG3wj|5($sDi00kHiJ;Je1gNbr)S&SKuJ}Yn&7g!radC3s8t}ZFum{CznX*o+Juh`) zrP73HX)T5>p&)G}k_#bXm{@O^4M5i8iPJ)@h^B|xXYYjQmS$GQYzw_6XnI%Y5JDw)m6r9>Z zLGe8_+7^JGE&)8V+$cFvCs1itz#%T4ZgZ)Tz1o^`&LF2Z0SQ@HL?6`HgRvm@fbs<` zxK3yS#p1gWC`WhFF3em6erBSC34w*8 zIB-!pJfT3|sj};2cR1yH?3el13Tq4xoSHqmr&csq9b@AT1R}}>G@(QUN&@hL3dPrI ziW8a1#mS*z8Y*kCx#8*NZ;dO5HN*y;(gLQtY?|Me9NmWWUosq*ugE$#7p|R%5C_%- zAEathLF6cuf;(7Tf=bc=?~o$|HWVT^Zwao>BpBcs5Z81k(iP)@g`JjCR~uTi=Mj5W5X2wNI=rZP@xw zEgyaMr2SLqr`0oiUQ~Yij9Gq`9S;YOHw1hOK`3zqMMgZ3077{+#L6wec=*)IhUse? z>anpG9%eoIu*a$V`>w&S*E7{N<1W4md#Up!Mk%1E>yB*Yld@?Q)4n{=S`r{Dl0uM$ zCO!C^h(;4U#l%H;Iqv}9BlE#e&mT@Kf8J0}o{eo8J34sd)^u<5ZM%|QgOV>i>VA8rbMW!|f6WQV@0^AEGs zt!^@uyj&kFxtN?48=(LC^BG1XLN;hAMDviukfTU9;*J0WeF%!OQX2Cd?jL-z9Q-xr zK+LMT*%l*UPL2(@b13fi=hvso=TRsLK&ol6qXw`sDXY876S9^npUfr;Ki*u) zPan)WAbKsDH|R^ocdKPpu!#hMHn1%*u*Q>aU;_c{HC^v~-F#+hS-!4EX}`XIl)u`E zw?p!u%lvLJ%xZ#s0j(WcP}LtmHSj6JJC0-Is@LE6o-n|k?pq&!Efv!^^uZm;iY$-+ zDIHk;SD*n5Z<`_OP&D!q+v*v1&?Cfy%<3J2U!oykQ--&Ur63Pk@s$^l{SpQ!|Bwf% z@$DzJ_VQhO1`0tI(Jni|tNSMekgD~sBsD=Xlw-Ge-GAW$z)k$*>u2~Vl++6$BxqnO zdxvZ_e`3KMR)zYVV)!D%s|i(~&`v0`xkdgLX@4ro@Bt@9rpYBiV-aJujLrVd|5yE0 zudXka17Rr^{O-5t|FyIQ^=K(^7!5=@!OM2-Phr1p27GuUtBB}J%E{2(|GLEb^>gqP z8>pP$dss&E4^c3~KN8F65~v(mEl!D^eS-P}ztiY{ASg#b8-lct&;Ettw-5g0`w5de zwV#RqdI$42CGe3q_`Vter$)fRht9+RV*(R6Bsm2$N&p;%WYwf(VPogOC}5q16t%Pk zg+)Xybiw3~bUc_G-WsFqc+yztw5gyFgAbE_OqP2PletZ$c9)Ljxs~Io9&J??AHo!h z9S3j&k`f(Zyn3moK0ZAAbgXE(xzAOl;cn`+^@hoHVWy9ID@L%k!;D*xZ#Z_8>mA={RvRwn7iy>*Na_Ul~hn8HRaV^S*YDf(1RKIu`>b-jN2eNfla^mf;yH|zM`yYm=c7~3my znY*WHsn5wZ=HnI7m}bvW(3AsPGJV`E>mP^BsDGUHT>0#Wos1UD^bzf9$HXKT5pGW? zEgTs2VOI^Bxw_eP#^#BlUs4}9r~=MF8+|kEq>R~ zH@9A%p7Cv0*m@q5saEE%|IKD)8}b?O!7`{12rUM!2i#H#c^N|Abgtu|yxsf$3%GB= zN>A%9e$xSdi`52E(0b3K?9I14F-@KA*w|;l9w9FUzn`sINFJZgiqV970^Yd*n&mh#=H)qzAYe&d$X~CyR^{b zB)Qq1b-C+C*5%D_i!mS6JFT&@-vh)gA{TvCrVdHokeUv39v=%@KDEEJB45Z-CO$Kj z5T-~t#awp6z*5=Mtk?cxOVFSZYXh%t7D2YCi*G>fGIw%IROtNdfoQpU4?0uHz>M78 z`urIKE90#^#*TB}(>1`Y*5X?_(BY5n8!`(4!sB8YF3>qdWm{yY-*4==)a^mpb+{&=D$Cn~~xYZ|Py7+zlI!34 zc$<%#+OXv{S=tC|>-(xXH>D#}FGz%@7RaUMY;?~>TdST5P(QP${X_kGWyBHo16H3F zj+#@KB=!e&@zm60UXNkmkQIN}fhNCLYOxB$-Z^7ddZez!%XO%gD+k}3MOEI>*^Pc6 zRyE(fw|6;Ic(PxXF;~6D$MrI!hiAw(M6D9FP}+GqNN6PTgieQe3%kIoMYso82do*r zU5MtXyG#klmbfDQ`t;3juu+o!#cr_e{QpA|AQ8Bukw;ln%hoK`*Sl|@$BL&ir{sVr zwLy2ZZk-<4fE=c0+R~$wH2DxqCf|AnsilHSIJmYAiCe7BMrgo_CbXGBpuf|j_pE4g zlOh1mszs%1Ny*%;Os(~lgmGmPuwv#+aTQ1#z)fA`)nnueQ7wxf*a)2y$<|V=NCsBW zfSp;MLnR=yKem6fCS|%H1dw}0JLI-T2j0Gt8}76-E0h|RGpcmX)xGCYPJ|umnnX)G zilZ647mezgJOF4^k(Ox#$ko+_x*kNXW$`mjPwhEs&?=}_!StRg%S%66Tj(hh-%yH) zzQR5`r@o3!RWMnR1=~`@AEW_G9vI6O;sJ!&>p1KpHQomJv}Ir?Q+<~FoIn|}WuS%{GO8PxTgJ|6V}c`;Ql z4q8JkVQ=RpC_`;!qQ2MB-{^aNC2IAw;z#vIiO=6YwUqusp?q&V;izY2>``wUElbw= zfy_I{B?9x=N3UQsxG=Nnw-khiW1ExLtQW=VlIu?JPi{joDI#wQZ$K;==AwtzXCA)X zQxyF+SN7;<{Kr(IVeO>(t=i%2QtH+7}g2_e}rwg=tspp+9ycp8C0T-$P&{V3SEHZGWnvrf{?bC$0q}D)F=F${bh|?^8}U{_GQ3$r ze2QHTW)|c9HGwIi=?IAlXg07SCP~!I}5bCNsF^RhL9W zP?1BkTj9_mK5dxK>YS~}tk5BXh+}6}n;U1>T|Z0K&dAGR%s~Q?{nUwX-O0#It!;L>1RIG_HZ)h5@dX`}s>h#K${_NQa()&=ebK5DJ1tDEfUp}tywif=T;pmjFOdyrw&WAD7TkAcfA#^c)% z>t?W9+5CXf_1Qz&tUi8(KS7m#F7WjJ*SowA zmIleR4~UFMb=+XYopO20yeLwD$<60kT2-E24Cxwwo7JTJ<;fzDuhKLgV0JEi*P}+> zq~(RKKM-fzc0KwR*LwZMB0r_i7I)l6S7aw^K1(NUVilx9{AQ(^4h>Vh)feh>p|9n8n&{_y26SDv&hXth>6ihBCCvRct>~buDkk|B@ zRoj(9NErNf6Q#nvPi*WWI@Tc-KNc4h8t~WC8Dj+zHwW3xrjYD?pTG|EXGX9qR5t0L zbScPLO29ejl~DX@si5HS2dR~!M{$R->lc{%qG<1Z!L?`llC}2jBp1r}LT(Ek%s0U#1vz7>zjSJ~3N+FQpUkc{8E#F;44~8CF`}@A%%)owAqV_GCMwTEO|DX z(dB*3O-%>==o3S)KS(n&@;$QWTBX=X3#WGE8e3CWyf(7Gy`>PHeZ|MUw$H4|V&Cqw zBQ-V?dI;f07n_j*Gu`tWe&qu&sfdGg>;pR2jr~s?b+wl`qhRH0^ulbFJ(aa8XxAl= zRKE9D@)dg(_R8yExlK-K-JZYdFLhhjayu>p8uu)|%gXb{&;q-D}b2xgwe|c4swC^H$Z2 z{G;)R7%`KwKnIBy{51*0qqRnfwIV&%O`g+>(p6h`XI{4R4!Vg69;b-XTP>`0lIRJO zu}b7)&AOV;y^5Ex==rc=F@fIe&ba*Xt`bhID6OPEP2zfG@MW_n;&He`!=Jh47gcZP zisYrvewmdCY|xmX!LO5^7V4(nKEzd zgC!>=kA%U`hK(y+E?%Ft6g$m_zj+f>3(Md)+NfZM4xww@zh7bM;g-Ue_nd3DMqF$C z%p@ZlR&(0)ir8c;ONX^@b5+i56u988j<~;3w>EVjk|vj!yj;_4iaja3WGwWpqeIeE zHvRSq@|Jx2YF-M4Xcega#vDuD^}$%zi)wZC0ynRf+82E!WL|}vEr=63#f;>2TD)@< z9I>3(hS|`>d(u*Jy%t^H@1H&Ie2-Yu>_UrO7evE!blz3^oI5)dXN1^>Qf+%4G`8Ec zX(?%+H>z%^Iy-zIC)WV$y0ty6RA}zKx=yZyGI@`Dol%k*#y9GSxAR zhfyI%$mqNQoXaSxrur;(0<1)*aD}MOBejI;tEr6|y%i=!mDRnos)n%)mSPTvE!4la zo)Kx2c&==DdBXvz80R^`!u)nUtJP{OWQ?)gsJ}Tn#N1x}nAiz5=M97tmCf*&3`fd| z`=5|k4+(P6c4)tjOgSLSM0;XNAn>N{U$asjEIh{;vj@#hbvUw^B0v2RWjI!K9NMEd z-EjyV`H8|;?r74HBhi!}8|ByWviZwL_7! z9H@A<4ZNHUI%YwWdY(Pqqg66H25#Wox?inw*tywP^b)7~F`-LHi8R+i3jnp80I?W~ z$3B@|pfe@N^}hm)Q-VO#Y(5b93zyc*9$C*tfZr(cq^K zbC!aRRiK*gz1AH5BQdIIHyJyaws%2@$5X+g4i*S=o zYWZmD$UBtQp(0$1s<(!U(>u_LlIM)rgo4hQEjqxii2JhaPi33GM)7e{>p;K#8r_V* z6Tpwj$1=~mdFqF&X*3{xDfQwoh_5OLjzaj&m!d$!4o23onD_?vzPO_K!ZYi zdS{LA)hWq>X_0M6*n?me#FBCH&c%v@A7v|AA4kO|U9DuHr!-IW4Is|~<8H$G1S3d9M@3DPtKsf1oDCgeG`o4BEWv?l^ z^CRbu6LbeEp9fK$BB(sgP40Wd>^ynQbJFLOlG|ywaHE4CMc889KP;jfoMu&IKZ~X7 zn7)=*;(NYjF*Hx1<)`?7(u#NJ{_XZ5HM$s`gJ$oOV?|eY=ZGwZi`1tqTj(Mssr08l zh!1vGZ78VSCzoH95q7(#*5kG-x5pprEa4=$sN3R?cJ8=xGliLO8}W|XoHwO`Cx+%) zSh#BPd~c0FQt>6B+Sp$97fAm;%>dCBqkV=CMaDRiJy?@tObpiWr-W#jQXaCJZ>~wr z&bgPrsTb70Wl|4HJbU zsJ+DCLwYobgol<9Kepi_4l9uh8){?vCLR$X9#ie>*u~$1r{VfLrA5R+D zF_C>ke{Ygp*W=HVxTmOUt`cL|H@hj((s`$w{Pg=99IDg$W{9#{w9hV|62g5xDodY?y4QI|!D7jZEDhqa( zp6ahZ#;Yq3!Jgm{or(^-RN*d-fCcKlyL51uy#*^;>csAAGY(IbWZK7*kN2u5G28JO z<38+aGZJ<08*Ex%ipOs2JTgDkU4cqR|ag1VW_;@K3 zIm6wjm*JPp&DuKW8Tc(LpI=hFO6H@ilP4QnZp#-Rwp=b|!`3G1^nIrHhr|e~AZpEXG;RON5w|C6TbN z%)XmRBR56%9J-H7&b?c(G-)m?eCAz`6x~HK8)OSNy|e)r=J>Hj-sG3V!RpiD-<%XH z?9bxd8O2IQH+`1ROKoCf<+<4iWx%lgv?1w%sfNBvC`8)1Sr)4^O# zl3kfIgON@BVVNBSBgOs(HKk^W)~$A&5t?V_LvrI&b*kdKXYU3pl^mjyvQwo;Uv%Zs zW}DVaSz(@1RDO{@E#;T;(lV9vft*VUAzaPcHIzxvwMh*zRP3r7MgyOlbR<8pC*Qey zL5y<^JJuOlp?vLp%45I9xv8rJznmMwCBg)~(vk_D?uId)^L<9I6d#UTyEMffqBKdx zA5z!9yg!FCbyTOI{qbU0an-0&!@~hdxn+jV3G)h7BiT8YB1HkJ97o#*WPMBHy%z_n z)53(%(nqPqHkmbE<%+aOBp|1`gH9fx>(Rn4sGu+*r4&k5Mhkdvimqc4oYf z;)PJvhMU9Wn`y7wQcAAMqKrU=OXD8|3zXcr6jc`u3v-_PEFIn0*Swo8?Ou>r_^Lo~ zLh{mV(jB)eF3KCs50pe6?3INDi5J9LtZb&m(22G;X`xTJIGHb|j>HWm7f+ll*w3K! zpo4d-Lw}4PF;lqAj8{JMCMqv~8#yE=E`9!bCH8u1?An?mLL3j83_w28azMf$W-u%j8?s9$XC~YzQ zlKZnpF&qyylM?%zdshgml>3z<$EOlo$74<=yM^;MjJvp>y!>SzxVB7B)hPXZs?}$q zjo{D0XF6WoNHudu7U-n#qNb+!HH}0^d|i#MaGaXps8ckV4UO&ONDFW~wR9(`)4^|V zhq7dlm7#YtKbNem2$#RyOVI~zQ7^ju#>?zdzq}*(X)iCQ*W`8!SqfbVk}1hEUo_LG ze)G{Kl(mH&?IV{4*o2<8`J zBs(WxbSp=e2``9R67m=GYr9xwneUaf=GwxY+=Z_We0(ah)VfwwejwuZL{rnP>U(tyXX8xo3!ZXUDF zc*keNMdheON;G$jOsDZ!H!FEqdX$#I^J2=TvD29jgXp z-=%ld`k7>xD!O^fsO({5sefL?GS(w@K2>b2XQi>s+!m3x4VjFalwkLRPb1!X3!bJ< z@5vD`biu1~L~+)I2Bf6Fs8-qYo{2|OVJ~$bK}A9>X^KIT&S$2hO;Es0J1sLznOE%i zWcqyN%#E%lJBzAw{bbE&%)CAPRO->pM!e_4%`I<;_H0AO8&2vCyPAP%#hVLsbKqE4 zJy}&_%1^s=HG)lV_oZ1b+g$Rt6=+kNI2!e?)k-8p%i-!1?b#~Wz`)e_ZEAT0HFe+J zuZHzW5xF@~hi2oss0=TgtHq_9n`J!D)42r&B&M_{@3ek6*{SQCYiW}2vUv4wG$7;~ z4C>BjwwX9+91jAAEDeQ*g^$Pz4H;Ao@^K-P)U>Wnpc11gr1g)#sJ@cR;M3HXt;04^ z_V~mM<`DV88O*nyWsH$}k=W7&^|E7%yJdt4G1S5(YNx+x+0crJAL(U`4d1*f+<)9xPC;$_a3VuvyL z;LIGUI%K}`;{}ZTDQri{uI|`Yf|y&ggnRB89fX)by{uO(Iagc5-f4G6?)tRk#J;$D zS0e7_+qq*KGr5eNS2tqyW^(;F_(pnaa_4hY>HG6;WDVWY}H-mll2*%HK<#kASX3&%|oGy1!|0}%Tzyc6P^^YOlH+|n1fWMUs_c6oZn zw@zc|tC{a~{x0_1>3rEI*d>?Kqn|!2o!Aqc5E8E2Ibo%)e&X1(vWkP5lixXoBqQI@ z@p*mW-{iayGO?`4$a_sEX|f_``ieN-JHp>$P3C>X&~tvhPjTaCzF+&ktLk*4hKAd% zFL5oY>)+U~J3ZZoTDlERZ$k-Qg+?r8{KE~>>dKZM-hb-NVp49Hp6Tm98eC;*<`cQj z*4CO31T(whu))L(5~a23#*!U0Vrn-3NWU`b7BLK;aJH9)ag`Xlf=gDhSE;2?TRAlk>Gvb^x-kwncE5I_sdMF&*luGW5}l`@?u}=tbHfX zI-a<-$LNl~-I{CibdHgPc(PdGHuNep19yTIyQ^j24eWkm^sHsyhkm;o*h7nB{{7*L zkE2@--Vmn#@W1-{5^$)yxBr8#?4TkI?Sy~x`v5&ED zk2ML|lA_3-wU9iNr;vmsQ6fw4x953(@7wSFU%&rcvz&82=f2Om?{lBI&UNnldrp7T z7vMO0d0-3S@2d{}zFq?X1g-IV6~SOU0{t%KugG#~UOI8GMg}wj4oNjGN8r&>E@%)K z4dG{}(W?6u(DARlSPoE{iv}6i1c9SbJY}h1WuY8PJHLk4*vRr%RF*LsY(*nb=rm(2 z3);|!N3(3a3TP_wqsY2EFcSm?LuqMXCJO{b5p}>|ENv3an}pQlwS^d?QDCiH)1ndM zbS56l0&C=XVrkvbfn2!?MOz3Ajevm|vQ1;42o@d-0!ce!vAK9?HV++1l4h}E2ZTXv zzBCjF%_4&!WKbLrlmW&j0K3K5MnDP%8R1CHG{%>KVCV=iG=U-~>V=NiY3kPn!ppEs zcoQ2{0A&n9z_AcxC^sykogExW0kL2eX)rn&#&HZ9!2*L=By0YN#ymese=o&gQpgy9wWA^VI z-e1uSh#_|bwZs}*Vy%!4$ZXO6^7{jd_loOFU7Z)X9<>i5D?iN`u(K7tRUrkEF>U@` z^DlUX<}GaF_Ob}I3GFX#UNCvHCcAkfR%`$|9W)Z!zIx&NPf!OorHJ*xup)b`%gE+E zVNRgAjBwiq`6_O-qq(gAtYjRoo;ZnaX@0jvq9^jKy47+1OXsG-wF|P+tbreiEY=N` zJ~NVOTcJnZpQnf)?-+5`l;{0vy}GWGhY#VXZGFUcymnUD<@}Y;kyW3afr2p6H;;MG zBTk3Wo;*{_XfVP(9WXj@l*%PxyNL>VCYn6@2{M}5|2%EuzAdeH7G7WP1Pwtg3(WZ*YZ~z(m?(LhC z$oJR{cE1V}cL47{IskQQe|Or2;{RHd&J)c@n7@Ru8$F}jT`9mJ>>$?E8w<6Q7PSz) zFAeiLoaE6zz!rvi?Pk`KK&g3Yb95>KL5rjhj8{ z^AuZ^W#5W2u~xE?wJ?oCVM!Sm?6y8fJ-{{g4U>Uvs`THb%uiKo^_>Zx)Wsw|LwkMs z{^cENsnOQx>uXQ$HEr+kCb0}|DLfB!N6D&Q=o=REQ+0d(u))vit^S1Cw%h1czYlNs;KDZxXIQ1*_u)ow*TOGs>>h-iWNFG77A)QO4O@_&=O^9? zYGQjK>K0vB<(=+|x`EQl+CwTA;E#zY^us)_hX1o}QCE^7+XELC&IbkL+xM zOT6BsK(7cgD(9zhLS>6>dCZWnj>?196Vq2Yjv#@>&TtUsh2cW**}RFK=3_z{UamfV zPE4Za;w4KP?mvP%f_8mGC+lJHXPU}N#)GU*m$Ybbl5|V|baGaXH)9ysdqHiENM82H zJMDJxpePwT2G<>h{-91MiW+3&9wsv_MamF|Pc7?~zr_9_GK!#fV41`VFWs!MX;l!I zTDWxPx8D2GbtfiSlN+}WM7F9`>!=1K&6q8;k*1_%?5JXOS}$Vlt(~&I?JA^snvmTB zOF{)qC!;mp$~(PBrL|NdI~nEfaqg@>GlD%jt%2Zrcbb{+Z`Sn$Kcjrl^ZuX#4YB~$ zQICW46w3<1gE`GePB(F&=BfuoqyUv=Io_nAs_XQ89*Qa=<|I}ux?xtV=s z2-#Ia8PZjXfX%}ofHeTa6hT1Kk48Dpy&v&0JpI3)_|GFgWkOf*H-Rvpe_U?ByINbsxQ`gYN9i#QLUx3N=o68N6_@~d|&a4dHlS8Eor5g$yQCJa=a@C;u zFDK^gZ1!~$|Duzjkz~gZ zJ6jp{DLmB<((5xF!2)m;_sNp(3Eg@wRftUU$)rjJ1U|afVi^omy z;RF}S$~1ljTRCM@(b>awTy`bNw=Y4Ct<{&T1cJ}E-$oKE4q^YYY})VLiw}(Jt904z zzBFEkPi!JTN!&|o1+DFk&%3CnC z#C!ZhT?&&K$wIi0zCqs5T3Zk}=`5FPJyeO_;W*3#JenyeW)w+7aeU!PsEdIZ?@Xx} zxx@WGOr~e^P1{^-#yXJ{t-ob!=L3J$vgK;sY0>5hpeUJdnSnb|-#J(Lno>rjii}Ke zOzx%ra7Eb+wt?^gd1&DQ^?*aDaY-7EINdR?0Pf=MwWm&NoV<?2Ht&Qn54Lyjyq=Im54kwGRC@H~<5Q2wmIdpwZ`1k79_tUtNCa3g zi}91CZs7Qa{SN0HnIsZ(zD5Rd7ZycV)i=Y`5REF)(N%Q;21{C##y!MikZ0(rtkDM# zDZV?r-5x7GlXF6Lv}`r5A3{SmuQ`=;s>)38!ZIV>i1CwiH_SQj?=4wTa>gbevt^iT z;?`Lq(r1}gJ@xPmoubbggOOs5w~dho&-h!$?p#DVAW{{hOiGp_Fc1TOmvhJ$@jUlC z;D_Wr&(XfnJ;g2bS}hGmF5O5LbX&)EdK9{>3b(0W+qaL)X7}_+7_mf`cvYThB>si) zca^Y9!{+g@VCQr)XE4}FD%1Co2>Y^rC$nKhzycCzlCUgm-xJueBjh_PWzrZjZ{bES zWZYua9?vbl{#B(ye?iMb!&c@e2xiqC8UD=Ec_a7WWd6YO^RHA@wM>Q2Ka>h8P7)t4 zF0k_De4n^0cmy74V`eYDdoE>ycThXgjCb_n~pBUw+hmKnCB=gheg)cbwwqK#FXm$jlu)y8^H%zt4B>V1694% zG-TF%Wxpe??r+3KC8!n_sX+P!yN8J?%2LG+#F1qhM%xq7)a8zfS4F?uj>hM{5t2KT z*w9!0LC$LwZ)Km@@3!R5v}Up#GjtT&D|H1U#uTU7d$JYrRk?+(J1V9)_{wI)as5@J z+qkEDG3gU~>KQQC9b|a=SNMTwZ||U+#maJRne1_;(K27V7NbEGhh+prqQwL)Ix!+S zKOl#d94?#NhUhNw4%@yAk$}a4P{3)v?mm+MB9`VI}h`gf@3~ zRsYGvK&j2X-q4{&`@sFIkhxla$;}eCp`+0j zkYja`MweJ_`{Elp1Ffiu)uj(cvWmoTmD`BwBZ&S7E7z)9+cT?dwWEV3Goq!6`UDu4 zK_$3AHnDSNA3jco-f!+Uzi>xfVu;R_$>G?icfF=o-hih!kLx=0dZgx?Y3)axJ7@ZB z8TO{mPjsyx-*gaq>#Li;BPiHiME~BJefY`bo(m)M9qxp=elg)C=UihtEvNrhzIa6C z^mCV@C4tz<@nN43A4*584*o^M!6$)Hyk_yO;=P+l0fLTl@62P;3RTv;Zb}hdGs|cC z#|N8lx+h#!7;UE)7B#=o8b+lsr4n>OU3ketnL2hdI~sa%!!ROmk=fMAdM+&3rZ1oM z+Ng}ua^Yyg<)sa9riQ7qq+LBm#<8;Vy#V65Yn&C0<=4QyF=De;bk7F%8ObL#A}D@n zoLhD1@g`;yT}D#Y))IGF>=01nvT9abH{jUOy&lD3yf7=Wf`5EG@__4w{S*! zc<*Zj>1zaWu8EZj)V#}*OKguV8hq$zS!b$!Vm+8o{OHJ+JqfRd-S}QtL@N-Y?asLe z4l9zNyzNuk75%Oak3y}UtmA9v3bbev(?rL9Q~ndSJpL@I+-EFkPKa(f+gaP!&V)l@ z9Qez+;>Zpr(Lrc!ZXfb`T(G*2dsA_wsG(mim&^;tgs%Xe-I_c<4anN#Iz<%Soky~q|b=|{m5Qrq6 zV{g%j-<`Gl(Xusgkbbz0b2Se?@E(SmTk%>|h>=?y`Wn zY&JJ!PlIK*wEuwemiU>wCnT}yKS2)h%7S5bdnMoQUcHGveOu|7%Am7;2yEuOD@mmz z`w{0mjaW`+U-^WRK-uY{>C1Dxb)sBioZ|xod-r{$uir&fg>mfH@rgm-_hK$;CSV>8 zGZBWzqztBuyKVj35TUbVxZ%6+Y3dgj1ogEQJk=NV6iG&oSQhM5-9tJzRKw=bGOWBd z%W}owa`#Xd@D(qO;IAVxP8UT#EDK5=d36VA)!uQ&J9~&XUvs5ISP9X}d_R3#(2v74 zvONmxAYl+v6QkqrZ&Dg0A-1i4yc~PwucHwGU*2OSWbk9&78p2B2PIq3US6e&AMz6< zCHWmr-@Er9zY!mH`J8y~`aJagF#lE0>X&3~X8uzO`oP1(2erhhDhV3&27F7zxwm|u zug6pFet}NB+fKUPH(eVPd9p%Ab|_2&>wuG@c-CEC=KMmua@goWX5m^~tz%a*+J77@;E&d%ty{_lbY#k^k_##b`hGh4R@#}e2r zBr5!NrZw5u1g|1lD<01C*o2ZhaA}^1B~*t<(NadNrCCdk!KBCewa<{&Q&o=H6(^rOTFP0+EqO9XrG(E*r zNa){9yX*T(N{x3l{ao)!CAJV3YL%lUr4Cbq#fV~f;;*th~p%#Dm zMstOh&dJ5(?L$XelBLZiq>n6LV-rsl0H@c$#m&=3j#`pTqX+2m?T5}jJ60OSLu4sh z-%RyBB82p0wv^U&=6WiX+L}f)TPgT8$L~(EM|NOV{n1uD-JXI1!#1qu^pRm#dN&o= z*}x?NH;Q&WE!`^C;K%r!I%S<5lg`U~*V{XWC=+Sk0>;Upb5&q5>Sv3Ny7lq1PsSAU zD>tgpo}2Z4I}gfaFyGK^DUuJRi(9b0&@`)BM#gbpz?-`Pzkb7IcL1aer_4J1}f z9~%>sj@|konX>$OE*;Q9$MlP4I6?;(H%Tv-F1gKPk$qKLWW&E(D#S%-;_S*6{Zz&7 ztrf=oaN>+@jYZWJ8~Gu`m$A}2^=bEG!#fyrQX_HV9LGe@Xzm&={{+$983@d|XEJW; zIlN|wJ6tRN&J_X&YUURWxN82o8Is;B@X+l4SN(|jk=!IZfOML;;P*<#quWm-}K&EeZuzGx{>gKWP4@_FDxCe z$gI9_D9v8$8bsPv!9*R4Z*PSG z*TV`O6GJW|Wwd0pwDmZI>`YMRB9*yHUIC*;n)t-CMGGLqY1#dft#s72^J>$9`Y{bTpFHxx@A zBTlhSOL3Xi7qhP(uReBcAL;!kd>);j`<0RJD#2+Qd;-pVMx;ofJiPyate$x_pXxo`s3pIaqweRsPW8Z{* z@*_9LpG;S(d7S8xSZSf4?xe+N9fj0Zu1|=2hH{x|tM8r-=ger{{}R3Ru+?(_Ohb>^F` z?W6kX1n!cp2gQv!_c|VHpM5xw{UfI5Qr8t*r;%#jD%~)pxmhB|g2$F!-`a@i$2w2n zz!!ic&308$;SG-tO7-PL7j>vox2G*lvF;-GbvYa7=WH$w?zU-TUDSRKneB(|Ma|Fu zQ0--3J3;VQQ&|4t`VK6loVx$)HFxFL=9jjgP<`j#e<^q~-}gUOGYmW{U3ctM;Dj7KTD8`MH-vbWAlZe&9>PPmsP8THatg?sCJH+EB|v zpvRXxF?oLo+fmm#u7A`HxyWaoNRKl2*u0ZQ@Tno2xFr|v^4E%JNZ~JCa|lKFuIfg8 z{CbgN#A8c0s@d&Im_fJ4)&mAIaWQn;sCcrpE$j(v-RHTA!GkAR>)m##61z__OS96X ziMKw#ub8uwkrLfueNX-%VYHPziWeRdN|uid_0A0fMRb9lG@yW|@u z8?tKa#3H6fE*wl@!I%uLL z*D{}eK>y|%xujgc(j>YOM^qVjB`%#!Sqw@IRuHcDXIf-9E-Ni594UD)Q0JDN;1tm+gZMc