Store embeddings in Azure Cognitive Search

microsoft · Jul 28, 2023 · ed114f0 · ed114f0
1 parent 6a8f4e9
commit ed114f0
Show file tree

Hide file tree

Showing 20 changed files with 817 additions and 135 deletions.
diff --git a/SemanticMemory.sln.DotSettings b/SemanticMemory.sln.DotSettings
@@ -1,4 +1,6 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:Boolean x:Key="/Default/CodeEditing/ContextActionTable/DisabledContextActions/=JetBrains_002EReSharper_002EIntentions_002ECSharp_002EContextActions_002EMisc_002ESurroundWithQuotesAction/@EntryIndexedValue">False</s:Boolean>
+	<s:Boolean x:Key="/Default/CodeEditing/ContextActionTable/DisabledContextActions/=JetBrains_002EReSharper_002EIntentions_002ECSharp_002EContextActions_002ENullPropagationToIfStatementAction/@EntryIndexedValue">False</s:Boolean>
 	<s:Boolean x:Key="/Default/CodeEditing/TypingAssist/Asp/FormatOnClosingTag/@EntryValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/CodeEditing/TypingAssist/Asp/FormatOnEnter/@EntryValue">True</s:Boolean>
 	<s:String x:Key="/Default/CodeEditing/TypingAssist/FormatOnPaste/@EntryValue">FullFormat</s:String>
@@ -136,6 +138,7 @@
 	<s:Int64 x:Key="/Default/Environment/Hierarchy/Build/BuildTool/MsBuildSolutionLoadingNodeCount/@EntryValue">2</s:Int64>
 	<s:Boolean x:Key="/Default/Environment/Hierarchy/Build/SolutionBuilderNext/LogToFile/@EntryValue">False</s:Boolean>
 	<s:Boolean x:Key="/Default/Environment/Hierarchy/Build/SolutionBuilderNext/ShouldRestoreNugetPackages/@EntryValue">True</s:Boolean>
+	<s:String x:Key="/Default/Environment/InlayHints/GeneralInlayHintsOptions/DefaultMode/@EntryValue">PushToShowHints</s:String>
 	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ECSharpAttributeForSingleLineMethodUpgrade/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ECSharpKeepExistingMigration/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ECSharpPlaceEmbeddedOnSameLineMigration/@EntryIndexedValue">True</s:Boolean>
@@ -150,6 +153,18 @@
 	<s:String x:Key="/Default/Housekeeping/UnitTestingMru/UnitTestSessionDefault/LogSeverity/@EntryValue">TRACE</s:String>
 	<s:Int64 x:Key="/Default/Housekeeping/UnitTestingMru/UnitTestSessionDefault/OutputLineNumberLimit/@EntryValue">8201</s:Int64>
 	<s:String x:Key="/Default/Housekeeping/UnitTestingMru/UnitTestSessionDefault/PlatformType/@EntryValue">x64</s:String>
+	<s:Boolean x:Key="/Default/PatternsAndTemplates/LiveTemplates/Template/=938AB0AF2F87A1479305AF74828124DA/@KeyIndexDefined">True</s:Boolean>
+	<s:Boolean x:Key="/Default/PatternsAndTemplates/LiveTemplates/Template/=938AB0AF2F87A1479305AF74828124DA/Applicability/=Live/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/PatternsAndTemplates/LiveTemplates/Template/=938AB0AF2F87A1479305AF74828124DA/IsBlessed/@EntryValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/PatternsAndTemplates/LiveTemplates/Template/=938AB0AF2F87A1479305AF74828124DA/Reformat/@EntryValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/PatternsAndTemplates/LiveTemplates/Template/=938AB0AF2F87A1479305AF74828124DA/Scope/=C3001E7C0DA78E4487072B7E050D86C5/@KeyIndexDefined">True</s:Boolean>
+	<s:String x:Key="/Default/PatternsAndTemplates/LiveTemplates/Template/=938AB0AF2F87A1479305AF74828124DA/Scope/=C3001E7C0DA78E4487072B7E050D86C5/CustomProperties/=minimumLanguageVersion/@EntryIndexedValue">2.0</s:String>
+	<s:String x:Key="/Default/PatternsAndTemplates/LiveTemplates/Template/=938AB0AF2F87A1479305AF74828124DA/Scope/=C3001E7C0DA78E4487072B7E050D86C5/Type/@EntryValue">InCSharpFile</s:String>
+	<s:String x:Key="/Default/PatternsAndTemplates/LiveTemplates/Template/=938AB0AF2F87A1479305AF74828124DA/Shortcut/@EntryValue">pragma</s:String>
+	<s:Boolean x:Key="/Default/PatternsAndTemplates/LiveTemplates/Template/=938AB0AF2F87A1479305AF74828124DA/ShortenQualifiedReferences/@EntryValue">True</s:Boolean>
+	<s:String x:Key="/Default/PatternsAndTemplates/LiveTemplates/Template/=938AB0AF2F87A1479305AF74828124DA/Text/@EntryValue">#pragma warning disable CA0000 // reason
+
+#pragma warning restore CA0000</s:String>
 	<s:Boolean x:Key="/Default/PatternsAndTemplates/LiveTemplates/Template/=CC90F25BDE5075498DCA20E411C14A16/@KeyIndexDefined">True</s:Boolean>
 	<s:Boolean x:Key="/Default/PatternsAndTemplates/LiveTemplates/Template/=CC90F25BDE5075498DCA20E411C14A16/Applicability/=Live/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/PatternsAndTemplates/LiveTemplates/Template/=CC90F25BDE5075498DCA20E411C14A16/Field/=METHOD/@KeyIndexDefined">False</s:Boolean>
@@ -200,6 +215,7 @@ public void It$SOMENAME$()
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=davinci/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Dotproduct/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=ENDPART/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=facetable/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=fareweller/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=fffffff/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=greaterthan/@EntryIndexedValue">True</s:Boolean>

diff --git a/lib/dotnet/Core.NetStandard20/SemanticMemoryException.cs b/lib/dotnet/Core.NetStandard20/SemanticMemoryException.cs
@@ -0,0 +1,35 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+
+namespace Microsoft.SemanticMemory.Core20;
+
+/// <summary>
+/// Provides the base exception from which all Semantic Kernel exceptions derive.
+/// </summary>
+public class SemanticMemoryException : Exception
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="SemanticMemoryException"/> class with a default message.
+    /// </summary>
+    public SemanticMemoryException()
+    {
+    }
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="SemanticMemoryException"/> class with its message set to <paramref name="message"/>.
+    /// </summary>
+    /// <param name="message">A string that describes the error.</param>
+    public SemanticMemoryException(string? message) : base(message)
+    {
+    }
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="SemanticMemoryException"/> class with its message set to <paramref name="message"/>.
+    /// </summary>
+    /// <param name="message">A string that describes the error.</param>
+    /// <param name="innerException">The exception that is the cause of the current exception.</param>
+    public SemanticMemoryException(string? message, Exception? innerException) : base(message, innerException)
+    {
+    }
+}
diff --git a/lib/dotnet/Core.NetStandard20/SemanticMemoryWebException.cs b/lib/dotnet/Core.NetStandard20/SemanticMemoryWebException.cs
@@ -4,28 +4,19 @@
 
 namespace Microsoft.SemanticMemory.Core20;
 
-public class SemanticMemoryWebException : Exception
+public class SemanticMemoryWebException : SemanticMemoryException
 {
-    /// <summary>
-    /// Initializes a new instance of the <see cref="SemanticMemoryWebException"/> class with a default message.
-    /// </summary>
+    /// <inheritdoc />
     public SemanticMemoryWebException()
     {
     }
 
-    /// <summary>
-    /// Initializes a new instance of the <see cref="SemanticMemoryWebException"/> class with its message set to <paramref name="message"/>.
-    /// </summary>
-    /// <param name="message">A string that describes the error.</param>
+    /// <inheritdoc />
     public SemanticMemoryWebException(string? message) : base(message)
     {
     }
 
-    /// <summary>
-    /// Initializes a new instance of the <see cref="SemanticMemoryWebException"/> class with its message set to <paramref name="message"/>.
-    /// </summary>
-    /// <param name="message">A string that describes the error.</param>
-    /// <param name="innerException">The exception that is the cause of the current exception.</param>
+    /// <inheritdoc />
     public SemanticMemoryWebException(string? message, Exception? innerException) : base(message, innerException)
     {
     }

diff --git a/lib/dotnet/Core/ContentStorage/EmbeddingFileContent.cs b/lib/dotnet/Core/ContentStorage/EmbeddingFileContent.cs
@@ -0,0 +1,34 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Text.Json.Serialization;
+using Microsoft.SemanticKernel.AI.Embeddings;
+
+namespace Microsoft.SemanticMemory.Core.ContentStorage;
+
+public class EmbeddingFileContent
+{
+    [JsonPropertyName("generator_name")]
+    [JsonPropertyOrder(1)]
+    public string GeneratorName { get; set; } = string.Empty;
+
+    [JsonPropertyName("generator_provider")]
+    [JsonPropertyOrder(2)]
+    public string GeneratorProvider { get; set; } = string.Empty;
+
+    [JsonPropertyName("vector_size")]
+    [JsonPropertyOrder(3)]
+    public int VectorSize { get; set; }
+
+    [JsonPropertyName("source_file_name")]
+    [JsonPropertyOrder(4)]
+    public string SourceFileName { get; set; } = string.Empty;
+
+    [JsonPropertyName("vector")]
+    [JsonPropertyOrder(100)]
+    public Embedding<float> Vector { get; set; }
+
+    [JsonPropertyName("timestamp")]
+    [JsonPropertyOrder(5)]
+    public DateTimeOffset TimeStamp { get; set; } = DateTimeOffset.UtcNow;
+}
diff --git a/lib/dotnet/Core/Core.csproj b/lib/dotnet/Core/Core.csproj
@@ -24,6 +24,7 @@
         <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.1" />
         <PackageReference Include="Microsoft.SemanticKernel" Version="0.17.230718.1-preview" />
         <PackageReference Include="RabbitMQ.Client" Version="6.5.0" />
+        <PackageReference Include="System.Linq.Async" Version="6.0.1" />
     </ItemGroup>
 
     <ItemGroup>

diff --git a/lib/dotnet/Core/Diagnostics/ContentStorageException.cs b/lib/dotnet/Core/Diagnostics/ContentStorageException.cs
@@ -1,10 +1,11 @@
 // Copyright (c) Microsoft. All rights reserved.
 
 using System;
+using Microsoft.SemanticMemory.Core20;
 
 namespace Microsoft.SemanticMemory.Core.Diagnostics;
 
-public class ContentStorageException : MemoryException
+public class ContentStorageException : SemanticMemoryException
 {
     /// <inheritdoc />
     public ContentStorageException() { }

diff --git a/lib/dotnet/Core/Diagnostics/MemoryException.cs b/lib/dotnet/Core/Diagnostics/MemoryException.cs
diff --git a/lib/dotnet/Core/Diagnostics/OrchestrationException.cs b/lib/dotnet/Core/Diagnostics/OrchestrationException.cs
@@ -1,10 +1,11 @@
 // Copyright (c) Microsoft. All rights reserved.
 
 using System;
+using Microsoft.SemanticMemory.Core20;
 
 namespace Microsoft.SemanticMemory.Core.Diagnostics;
 
-public class OrchestrationException : MemoryException
+public class OrchestrationException : SemanticMemoryException
 {
     /// <inheritdoc />
     public OrchestrationException() { }

diff --git a/lib/dotnet/Core/Diagnostics/PipelineCompletedException.cs b/lib/dotnet/Core/Diagnostics/PipelineCompletedException.cs
@@ -1,10 +1,11 @@
 // Copyright (c) Microsoft. All rights reserved.
 
 using System;
+using Microsoft.SemanticMemory.Core20;
 
 namespace Microsoft.SemanticMemory.Core.Diagnostics;
 
-public class PipelineCompletedException : MemoryException
+public class PipelineCompletedException : SemanticMemoryException
 {
     /// <summary>
     /// Initializes a new instance.