Allow to configure default index name for all storage engines (#354)

## Motivation and Context (Why the change? What's the scenario?) * Allow to configure default index name for all storage engines * Remove restriction blocking the deletion of the default index. See also #349 ## High level description (Approach, Design) * Added new configuration setting. * Moved the index name validation from internal storage classes out to the API layer of memory service, serverless memory, and orchestrator
microsoft · Mar 12, 2024 · b905d25 · b905d25
1 parent ebbd17c
commit b905d25
Show file tree

Hide file tree

Showing 43 changed files with 231 additions and 361 deletions.
diff --git a/KernelMemory.sln b/KernelMemory.sln
@@ -78,7 +78,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "root", "root", "{6EF76FD8-4
 		Directory.Packages.props = Directory.Packages.props
 		nuget-package.props = nuget-package.props
 		code-analysis.props = code-analysis.props
-		build.py = build.py
 		nuget.config = nuget.config
 		Dockerfile = Dockerfile
 		.dockerignore = .dockerignore

diff --git a/build.py b/build.py
diff --git a/clients/dotnet/WebClient/MemoryWebClient.cs b/clients/dotnet/WebClient/MemoryWebClient.cs
@@ -71,8 +71,7 @@ public Task<string> ImportDocumentAsync(
         DocumentUploadRequest uploadRequest,
         CancellationToken cancellationToken = default)
     {
-        var index = IndexExtensions.CleanName(uploadRequest.Index);
-        return this.ImportInternalAsync(index, uploadRequest, cancellationToken);
+        return this.ImportInternalAsync(uploadRequest.Index, uploadRequest, cancellationToken);
     }
 
     /// <inheritdoc />
@@ -152,7 +151,6 @@ public async Task<IEnumerable<IndexDetails>> ListIndexesAsync(CancellationToken
     /// <inheritdoc />
     public async Task DeleteIndexAsync(string? index = null, CancellationToken cancellationToken = default)
     {
-        index = IndexExtensions.CleanName(index);
         var url = Constants.HttpDeleteIndexEndpointWithParams
             .Replace(Constants.HttpIndexPlaceholder, index);
         HttpResponseMessage? response = await this._client.DeleteAsync(url, cancellationToken).ConfigureAwait(false);
@@ -181,7 +179,6 @@ public async Task DeleteDocumentAsync(string documentId, string? index = null, C
             throw new KernelMemoryException("The document ID is empty");
         }
 
-        index = IndexExtensions.CleanName(index);
         var url = Constants.HttpDeleteDocumentEndpointWithParams
             .Replace(Constants.HttpIndexPlaceholder, index)
             .Replace(Constants.HttpDocumentIdPlaceholder, documentId);
@@ -219,7 +216,6 @@ public async Task<bool> IsDocumentReadyAsync(
         string? index = null,
         CancellationToken cancellationToken = default)
     {
-        index = IndexExtensions.CleanName(index);
         var url = Constants.HttpUploadStatusEndpointWithParams
             .Replace(Constants.HttpIndexPlaceholder, index)
             .Replace(Constants.HttpDocumentIdPlaceholder, documentId);
@@ -254,7 +250,6 @@ public async Task<SearchResult> SearchAsync(
             filters.Add(filter);
         }
 
-        index = IndexExtensions.CleanName(index);
         SearchQuery request = new()
         {
             Index = index,
@@ -288,7 +283,6 @@ public async Task<MemoryAnswer> AskAsync(
             filters.Add(filter);
         }
 
-        index = IndexExtensions.CleanName(index);
         MemoryQuery request = new()
         {
             Index = index,

diff --git a/extensions/AzureAISearch/AzureAISearch.FunctionalTests/DefaultTests.cs b/extensions/AzureAISearch/AzureAISearch.FunctionalTests/DefaultTests.cs
@@ -18,6 +18,7 @@ public DefaultTests(IConfiguration cfg, ITestOutputHelper output) : base(cfg, ou
         Assert.False(string.IsNullOrEmpty(this.OpenAiConfig.APIKey));
 
         this._memory = new KernelMemoryBuilder()
+            .With(new KernelMemoryConfig { DefaultIndexName = "default4tests" })
             .WithSearchClientConfig(new SearchClientConfig { EmptyAnswer = NotFound })
             .WithOpenAI(this.OpenAiConfig)
             .WithAzureAISearchMemoryDb(this.AzureAiSearchConfig.Endpoint, this.AzureAiSearchConfig.APIKey)
@@ -66,6 +67,13 @@ public async Task ItNormalizesIndexNames()
         await IndexListTest.ItNormalizesIndexNames(this._memory, this.Log);
     }
 
+    [Fact]
+    [Trait("Category", "AzAISearch")]
+    public async Task ItUsesDefaultIndexName()
+    {
+        await IndexListTest.ItUsesDefaultIndexName(this._memory, this.Log, "default4tests");
+    }
+
     [Fact]
     [Trait("Category", "AzAISearch")]
     public async Task ItDeletesIndexes()

diff --git a/extensions/AzureAISearch/AzureAISearch/AzureAISearchMemory.cs b/extensions/AzureAISearch/AzureAISearch/AzureAISearchMemory.cs
@@ -119,12 +119,6 @@ public async Task<IEnumerable<string>> GetIndexesAsync(CancellationToken cancell
     public Task DeleteIndexAsync(string index, CancellationToken cancellationToken = default)
     {
         index = this.NormalizeIndexName(index);
-        if (string.Equals(index, Constants.DefaultIndex, StringComparison.OrdinalIgnoreCase))
-        {
-            this._log.LogWarning("The default index cannot be deleted");
-            return Task.CompletedTask;
-        }
-
         return this._adminClient.DeleteIndexAsync(index, cancellationToken);
     }
 
@@ -439,7 +433,7 @@ private string NormalizeIndexName(string index)
     {
         if (string.IsNullOrWhiteSpace(index))
         {
-            index = Constants.DefaultIndex;
+            throw new ArgumentNullException(nameof(index), "The index name is empty");
         }
 
         if (index.Length > 128)

diff --git a/extensions/Elasticsearch/Elasticsearch.FunctionalTests/DefaultTests.cs b/extensions/Elasticsearch/Elasticsearch.FunctionalTests/DefaultTests.cs
@@ -25,6 +25,7 @@ public DefaultTests(IConfiguration cfg, ITestOutputHelper output) : base(cfg, ou
         this._elasticsearchConfig = cfg.GetSection("KernelMemory:Services:Elasticsearch").Get<ElasticsearchConfig>()!;
 
         this._memory = new KernelMemoryBuilder()
+            .With(new KernelMemoryConfig { DefaultIndexName = "default4tests" })
             .WithSearchClientConfig(new SearchClientConfig { EmptyAnswer = NotFound })
             .WithOpenAI(this.OpenAiConfig)
             // .WithAzureOpenAITextGeneration(this.AzureOpenAITextConfiguration)
@@ -75,6 +76,13 @@ public async Task ItNormalizesIndexNames()
         await IndexListTest.ItNormalizesIndexNames(this._memory, this.Log);
     }
 
+    [Fact]
+    [Trait("Category", "Elasticsearch")]
+    public async Task ItUsesDefaultIndexName()
+    {
+        await IndexListTest.ItUsesDefaultIndexName(this._memory, this.Log, "default4tests");
+    }
+
     [Fact]
     [Trait("Category", "Elasticsearch")]
     public async Task ItDeletesIndexes()

diff --git a/extensions/LlamaSharp/LlamaSharp.FunctionalTests/appsettings.json b/extensions/LlamaSharp/LlamaSharp.FunctionalTests/appsettings.json
@@ -28,8 +28,7 @@
       },
       "Qdrant": {
         "Endpoint": "http://127.0.0.1:6333",
-        "APIKey": "",
-        "DefaultIndex": "default"
+        "APIKey": ""
       },
       "OpenAI": {
         // Name of the model used to generate text (text completion or chat completion)