diff --git a/KernelMemory.sln b/KernelMemory.sln index 44dc4729b..516d58e90 100644 --- a/KernelMemory.sln +++ b/KernelMemory.sln @@ -78,7 +78,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "root", "root", "{6EF76FD8-4 Directory.Packages.props = Directory.Packages.props nuget-package.props = nuget-package.props code-analysis.props = code-analysis.props - build.py = build.py nuget.config = nuget.config Dockerfile = Dockerfile .dockerignore = .dockerignore diff --git a/build.py b/build.py deleted file mode 100755 index f37334886..000000000 --- a/build.py +++ /dev/null @@ -1,244 +0,0 @@ -#!/usr/bin/env python -import glob -import os -import shutil -import subprocess -import sys - -cfg = { - "packages": [ - "Microsoft.KernelMemory.Abstractions", - "Microsoft.KernelMemory.Core", - "Microsoft.KernelMemory.WebClient", - "Microsoft.KernelMemory.SemanticKernelPlugin", - "Microsoft.KernelMemory.ContentStorage.AzureBlobs", - "Microsoft.KernelMemory.DataFormats.AzureAIDocIntel", - "Microsoft.KernelMemory.Orchestration.AzureQueues", - "Microsoft.KernelMemory.Orchestration.RabbitMQ", - "Microsoft.KernelMemory.AI.AzureOpenAI", - "Microsoft.KernelMemory.AI.OpenAI", - "Microsoft.KernelMemory.AI.LlamaSharp", - "Microsoft.KernelMemory.MemoryDb.AzureAISearch", - "Microsoft.KernelMemory.MemoryDb.Postgres", - "Microsoft.KernelMemory.MemoryDb.Qdrant", - ], - "clear_dotnet_cache": False, - # Enable this if using Bash and you want to see colors, though shell output won't stream in realtime - "bash_with_colors_no_stream": False, -} - - -def main(): - - # Move into project dir - change_working_dir_to_project() - print("Directory: ", os.getcwd()) - - # Delete and rebuild packages - print(bold("### Deleting previous builds")) - delete_files_by_extension(os.getcwd(), [".nupkg", ".snupkg"]) - - # Find SLN file - print(bold("### Build solution")) - solution = find_sln_file(os.getcwd()) - if not solution or solution.isspace(): - print(bold("# Error:"), red("No valid .sln file found.")) - exit(1) - - # Clear .NET cache - if cfg["clear_dotnet_cache"]: - print(bold("# Deleting .NET cache")) - run_shell_command(f"dotnet clean {solution} --nologo -c Debug --verbosity minimal") - run_shell_command(f"dotnet clean {solution} --nologo -c Release --verbosity minimal") - - # Build SLN - run_shell_command(f"dotnet build {solution} --nologo -c Release") - - print(bold("### Verify packages have been built")) - verify_packages_build(os.getcwd()) - - print(bold("### Clearing Nuget cache")) - delete_cached_packages() - - print(bold("### Copy Nuget packages to /packages")) - move_packages_build_to(os.getcwd(), os.path.join(os.getcwd(), "packages")) - - # print(bold("# Clean .NET build")) - # run_shell_command(f"dotnet restore {solution} --nologo --no-cache") - # run_shell_command(f"dotnet clean {solution} --nologo -c Debug --verbosity minimal") - # run_shell_command(f"dotnet clean {solution} --nologo -c Release --verbosity minimal") - - -def change_working_dir_to_project(): - os.chdir(os.path.dirname(os.path.abspath(__file__))) - - -def delete_files_by_extension(root_dir, extensions): - count = 0 - if isinstance(extensions, str): - extensions = [extensions] - - print("Deleting files from dir:", root_dir) - for dir_name, sub_dirs, filenames in os.walk(root_dir): - for filename in filenames: - if any(filename.endswith(ext) for ext in extensions): - file_path = os.path.join(dir_name, filename) - os.remove(file_path) - count += 1 - print(f"Deleted: {file_path[len(root_dir) + 1:]}") - print("Files deleted:", count) - - -def find_sln_file(root_folder): - for dir_name, sub_dirs, filenames in os.walk(root_folder): - for filename in filenames: - if filename.endswith(".sln") and "dev" not in filename.lower(): - return filename - - -def run_shell_command(command): - if cfg["bash_with_colors_no_stream"]: - run_shell_command_with_colors(command) - else: - run_shell_command_with_stream(command) - - -def bold(text): - return f"\033[1m{text}\033[0m" - - -def red(text): - return f"\033[91m{text}\033[0m" - - -def yellow(text): - return f"\033[93m{text}\033[0m" - - -def run_shell_command_with_stream(command): - try: - # Run the shell command printing output as it occurs. Note: this might strip colors - print(bold("# Command: "), command) - - process = subprocess.Popen( - command, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - bufsize=1, - universal_newlines=True, - env={**os.environ, "TERM": "xterm-256color"}, - ) - - # Print the command output in real-time for stdout and stderr - for line in process.stdout: - sys.stdout.write(line) - sys.stdout.flush() - - for line in process.stderr: - sys.stdout.write(f"# Error: {line}") - sys.stdout.flush() - - # Wait for the command to complete - process.wait() - - # Check the return code - if process.returncode != 0: - print(bold("# Return Code: "), process.returncode) - exit(1) - - except Exception as e: - print(bold("# Error: "), e) - exit(1) - - -def run_shell_command_with_colors(command): - try: - # Run the shell command and capture the output along with color codes - print(bold("# Command: "), command) - result = subprocess.run( - ["script", "-q", "/dev/null", "bash", "-c", command], - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - - if result.stdout and not result.stdout.isspace(): - print("# Command Output:") - print(result.stdout) - - except subprocess.CalledProcessError as e: - # Handle errors with original colors - print(bold("# Error Occurred: ")) - print(bold("# Return Code: "), e.returncode) - print(bold("# Standard Output: ")) - print(e.stdout) - if e.stderr and not e.stderr.isspace(): - print(bold("# Standard Error: ")) - print(e.stderr) - - # Exit the script with an error code - exit(1) - - -def verify_packages_build(root_dir): - for package in cfg["packages"]: - # Generate the expected file path - expected_file_path = os.path.join(root_dir, "**", "bin", "Release", f"{package}.*.nupkg") - - # Use glob to check if any file matches the pattern - matches = glob.glob(expected_file_path, recursive=True) - - if not matches: - print(f"# Error: {package}: package not found") - exit(1) - else: - for match in matches: - print(f"{match[len(root_dir) + 1:]}") - - -def move_packages_build_to(root_dir, destination_dir): - for package in cfg["packages"]: - # Generate the expected file path - expected_file_path = os.path.join(root_dir, "**", "bin", "Release", f"{package}.*.nupkg") - - # Use glob to check if any file matches the pattern - matches = glob.glob(expected_file_path, recursive=True) - - if not matches: - print(f"# Error: {package}: package not found") - exit(1) - else: - for match in matches: - shutil.move(match, destination_dir) - - -def delete_cached_packages(): - for package in cfg["packages"]: - nuget_packages_dir = os.path.join(os.environ["HOME"], ".nuget", "packages", package) - - if os.path.isdir(nuget_packages_dir): - print("Cache content before purge") - print(f"dir: {nuget_packages_dir}") - for entry in os.listdir(nuget_packages_dir): - print(entry) - - try: - shutil.rmtree(nuget_packages_dir) - except Exception as e: - print(f"ERROR: unable to clear cache at {nuget_packages_dir}") - print(e) - exit(1) - - if os.path.isdir(nuget_packages_dir): - print(f"ERROR: unable to clear cache at {nuget_packages_dir}") - exit(1) - - -def green(text): - return f"\033[92m{text}\033[0m" - - -main() diff --git a/clients/dotnet/WebClient/MemoryWebClient.cs b/clients/dotnet/WebClient/MemoryWebClient.cs index 56dc79245..730968374 100644 --- a/clients/dotnet/WebClient/MemoryWebClient.cs +++ b/clients/dotnet/WebClient/MemoryWebClient.cs @@ -71,8 +71,7 @@ public Task ImportDocumentAsync( DocumentUploadRequest uploadRequest, CancellationToken cancellationToken = default) { - var index = IndexExtensions.CleanName(uploadRequest.Index); - return this.ImportInternalAsync(index, uploadRequest, cancellationToken); + return this.ImportInternalAsync(uploadRequest.Index, uploadRequest, cancellationToken); } /// @@ -152,7 +151,6 @@ public async Task> ListIndexesAsync(CancellationToken /// public async Task DeleteIndexAsync(string? index = null, CancellationToken cancellationToken = default) { - index = IndexExtensions.CleanName(index); var url = Constants.HttpDeleteIndexEndpointWithParams .Replace(Constants.HttpIndexPlaceholder, index); HttpResponseMessage? response = await this._client.DeleteAsync(url, cancellationToken).ConfigureAwait(false); @@ -181,7 +179,6 @@ public async Task DeleteDocumentAsync(string documentId, string? index = null, C throw new KernelMemoryException("The document ID is empty"); } - index = IndexExtensions.CleanName(index); var url = Constants.HttpDeleteDocumentEndpointWithParams .Replace(Constants.HttpIndexPlaceholder, index) .Replace(Constants.HttpDocumentIdPlaceholder, documentId); @@ -219,7 +216,6 @@ public async Task IsDocumentReadyAsync( string? index = null, CancellationToken cancellationToken = default) { - index = IndexExtensions.CleanName(index); var url = Constants.HttpUploadStatusEndpointWithParams .Replace(Constants.HttpIndexPlaceholder, index) .Replace(Constants.HttpDocumentIdPlaceholder, documentId); @@ -254,7 +250,6 @@ public async Task SearchAsync( filters.Add(filter); } - index = IndexExtensions.CleanName(index); SearchQuery request = new() { Index = index, @@ -288,7 +283,6 @@ public async Task AskAsync( filters.Add(filter); } - index = IndexExtensions.CleanName(index); MemoryQuery request = new() { Index = index, diff --git a/extensions/AzureAISearch/AzureAISearch.FunctionalTests/DefaultTests.cs b/extensions/AzureAISearch/AzureAISearch.FunctionalTests/DefaultTests.cs index 1f35edc8e..314040830 100644 --- a/extensions/AzureAISearch/AzureAISearch.FunctionalTests/DefaultTests.cs +++ b/extensions/AzureAISearch/AzureAISearch.FunctionalTests/DefaultTests.cs @@ -18,6 +18,7 @@ public DefaultTests(IConfiguration cfg, ITestOutputHelper output) : base(cfg, ou Assert.False(string.IsNullOrEmpty(this.OpenAiConfig.APIKey)); this._memory = new KernelMemoryBuilder() + .With(new KernelMemoryConfig { DefaultIndexName = "default4tests" }) .WithSearchClientConfig(new SearchClientConfig { EmptyAnswer = NotFound }) .WithOpenAI(this.OpenAiConfig) .WithAzureAISearchMemoryDb(this.AzureAiSearchConfig.Endpoint, this.AzureAiSearchConfig.APIKey) @@ -66,6 +67,13 @@ public async Task ItNormalizesIndexNames() await IndexListTest.ItNormalizesIndexNames(this._memory, this.Log); } + [Fact] + [Trait("Category", "AzAISearch")] + public async Task ItUsesDefaultIndexName() + { + await IndexListTest.ItUsesDefaultIndexName(this._memory, this.Log, "default4tests"); + } + [Fact] [Trait("Category", "AzAISearch")] public async Task ItDeletesIndexes() diff --git a/extensions/AzureAISearch/AzureAISearch/AzureAISearchMemory.cs b/extensions/AzureAISearch/AzureAISearch/AzureAISearchMemory.cs index a6c2afe9a..26d8909ff 100644 --- a/extensions/AzureAISearch/AzureAISearch/AzureAISearchMemory.cs +++ b/extensions/AzureAISearch/AzureAISearch/AzureAISearchMemory.cs @@ -119,12 +119,6 @@ public async Task> GetIndexesAsync(CancellationToken cancell public Task DeleteIndexAsync(string index, CancellationToken cancellationToken = default) { index = this.NormalizeIndexName(index); - if (string.Equals(index, Constants.DefaultIndex, StringComparison.OrdinalIgnoreCase)) - { - this._log.LogWarning("The default index cannot be deleted"); - return Task.CompletedTask; - } - return this._adminClient.DeleteIndexAsync(index, cancellationToken); } @@ -439,7 +433,7 @@ private string NormalizeIndexName(string index) { if (string.IsNullOrWhiteSpace(index)) { - index = Constants.DefaultIndex; + throw new ArgumentNullException(nameof(index), "The index name is empty"); } if (index.Length > 128) diff --git a/extensions/Elasticsearch/Elasticsearch.FunctionalTests/DefaultTests.cs b/extensions/Elasticsearch/Elasticsearch.FunctionalTests/DefaultTests.cs index c0ba8d485..b0cd341f8 100644 --- a/extensions/Elasticsearch/Elasticsearch.FunctionalTests/DefaultTests.cs +++ b/extensions/Elasticsearch/Elasticsearch.FunctionalTests/DefaultTests.cs @@ -25,6 +25,7 @@ public DefaultTests(IConfiguration cfg, ITestOutputHelper output) : base(cfg, ou this._elasticsearchConfig = cfg.GetSection("KernelMemory:Services:Elasticsearch").Get()!; this._memory = new KernelMemoryBuilder() + .With(new KernelMemoryConfig { DefaultIndexName = "default4tests" }) .WithSearchClientConfig(new SearchClientConfig { EmptyAnswer = NotFound }) .WithOpenAI(this.OpenAiConfig) // .WithAzureOpenAITextGeneration(this.AzureOpenAITextConfiguration) @@ -75,6 +76,13 @@ public async Task ItNormalizesIndexNames() await IndexListTest.ItNormalizesIndexNames(this._memory, this.Log); } + [Fact] + [Trait("Category", "Elasticsearch")] + public async Task ItUsesDefaultIndexName() + { + await IndexListTest.ItUsesDefaultIndexName(this._memory, this.Log, "default4tests"); + } + [Fact] [Trait("Category", "Elasticsearch")] public async Task ItDeletesIndexes() diff --git a/extensions/LlamaSharp/LlamaSharp.FunctionalTests/appsettings.json b/extensions/LlamaSharp/LlamaSharp.FunctionalTests/appsettings.json index cc1f1e80a..06200cdf7 100644 --- a/extensions/LlamaSharp/LlamaSharp.FunctionalTests/appsettings.json +++ b/extensions/LlamaSharp/LlamaSharp.FunctionalTests/appsettings.json @@ -28,8 +28,7 @@ }, "Qdrant": { "Endpoint": "http://127.0.0.1:6333", - "APIKey": "", - "DefaultIndex": "default" + "APIKey": "" }, "OpenAI": { // Name of the model used to generate text (text completion or chat completion) diff --git a/extensions/Postgres/Postgres.FunctionalTests/DefaultTests.cs b/extensions/Postgres/Postgres.FunctionalTests/DefaultTests.cs index cc75e9779..544efea7f 100644 --- a/extensions/Postgres/Postgres.FunctionalTests/DefaultTests.cs +++ b/extensions/Postgres/Postgres.FunctionalTests/DefaultTests.cs @@ -15,6 +15,7 @@ public DefaultTests(IConfiguration cfg, ITestOutputHelper output) : base(cfg, ou Assert.False(string.IsNullOrEmpty(this.OpenAiConfig.APIKey)); this._memory = new KernelMemoryBuilder() + .With(new KernelMemoryConfig { DefaultIndexName = "default4tests" }) .WithSearchClientConfig(new SearchClientConfig { EmptyAnswer = NotFound }) .WithOpenAI(this.OpenAiConfig) // .WithAzureOpenAITextGeneration(this.AzureOpenAITextConfiguration) @@ -65,6 +66,13 @@ public async Task ItNormalizesIndexNames() await IndexListTest.ItNormalizesIndexNames(this._memory, this.Log); } + [Fact] + [Trait("Category", "Postgres")] + public async Task ItUsesDefaultIndexName() + { + await IndexListTest.ItUsesDefaultIndexName(this._memory, this.Log, "default4tests"); + } + [Fact] [Trait("Category", "Postgres")] public async Task ItDeletesIndexes() diff --git a/extensions/Postgres/Postgres/PostgresMemory.cs b/extensions/Postgres/Postgres/PostgresMemory.cs index d1149c129..992c69de5 100644 --- a/extensions/Postgres/Postgres/PostgresMemory.cs +++ b/extensions/Postgres/Postgres/PostgresMemory.cs @@ -103,11 +103,6 @@ public async Task DeleteIndexAsync( CancellationToken cancellationToken = default) { index = NormalizeIndexName(index); - if (string.Equals(index, Constants.DefaultIndex, StringComparison.OrdinalIgnoreCase)) - { - this._log.LogWarning("The default index cannot be deleted"); - return; - } try { @@ -241,7 +236,7 @@ private static string NormalizeIndexName(string index) { if (string.IsNullOrWhiteSpace(index)) { - index = Constants.DefaultIndex; + throw new ArgumentNullException(nameof(index), "The index name is empty"); } index = s_replaceIndexNameCharsRegex.Replace(index.Trim().ToLowerInvariant(), ValidSeparator); diff --git a/extensions/Qdrant/Qdrant.FunctionalTests/DefaultTests.cs b/extensions/Qdrant/Qdrant.FunctionalTests/DefaultTests.cs index b5dddda37..5cec4a8f1 100644 --- a/extensions/Qdrant/Qdrant.FunctionalTests/DefaultTests.cs +++ b/extensions/Qdrant/Qdrant.FunctionalTests/DefaultTests.cs @@ -16,6 +16,7 @@ public DefaultTests(IConfiguration cfg, ITestOutputHelper output) : base(cfg, ou Assert.False(string.IsNullOrEmpty(this.OpenAiConfig.APIKey)); this._memory = new KernelMemoryBuilder() + .With(new KernelMemoryConfig { DefaultIndexName = "default4tests" }) .WithSearchClientConfig(new SearchClientConfig { EmptyAnswer = NotFound }) .WithOpenAI(this.OpenAiConfig) .WithQdrantMemoryDb(this.QdrantConfig) @@ -64,6 +65,13 @@ public async Task ItNormalizesIndexNames() await IndexListTest.ItNormalizesIndexNames(this._memory, this.Log); } + [Fact] + [Trait("Category", "Qdrant")] + public async Task ItUsesDefaultIndexName() + { + await IndexListTest.ItUsesDefaultIndexName(this._memory, this.Log, "default4tests"); + } + [Fact] [Trait("Category", "Qdrant")] public async Task ItDeletesIndexes() diff --git a/extensions/Qdrant/Qdrant.FunctionalTests/appsettings.json b/extensions/Qdrant/Qdrant.FunctionalTests/appsettings.json index 939219e58..b5a34509e 100644 --- a/extensions/Qdrant/Qdrant.FunctionalTests/appsettings.json +++ b/extensions/Qdrant/Qdrant.FunctionalTests/appsettings.json @@ -7,8 +7,7 @@ "Services": { "Qdrant": { "Endpoint": "http://127.0.0.1:6333", - "APIKey": "", - "DefaultIndex": "default" + "APIKey": "" }, "OpenAI": { // Name of the model used to generate text (text completion or chat completion) diff --git a/extensions/Qdrant/Qdrant.TestApplication/appsettings.json b/extensions/Qdrant/Qdrant.TestApplication/appsettings.json index 117b1b2f5..19f7e200c 100644 --- a/extensions/Qdrant/Qdrant.TestApplication/appsettings.json +++ b/extensions/Qdrant/Qdrant.TestApplication/appsettings.json @@ -9,8 +9,7 @@ "Services": { "Qdrant": { "Endpoint": "http://127.0.0.1:6333", - "APIKey": "", - "DefaultIndex": "default" + "APIKey": "" } } } diff --git a/extensions/Qdrant/Qdrant/QdrantConfig.cs b/extensions/Qdrant/Qdrant/QdrantConfig.cs index 352317f4f..d635911a1 100644 --- a/extensions/Qdrant/Qdrant/QdrantConfig.cs +++ b/extensions/Qdrant/Qdrant/QdrantConfig.cs @@ -19,5 +19,4 @@ public class QdrantConfig public string Endpoint { get; set; } = string.Empty; public string APIKey { get; set; } = string.Empty; - public string DefaultIndex { get; set; } = Constants.DefaultIndex; } diff --git a/extensions/Qdrant/Qdrant/QdrantMemory.cs b/extensions/Qdrant/Qdrant/QdrantMemory.cs index e09f48fb8..2fcf83e8b 100644 --- a/extensions/Qdrant/Qdrant/QdrantMemory.cs +++ b/extensions/Qdrant/Qdrant/QdrantMemory.cs @@ -25,7 +25,6 @@ public class QdrantMemory : IMemoryDb private readonly ITextEmbeddingGenerator _embeddingGenerator; private readonly QdrantClient _qdrantClient; private readonly ILogger _log; - private readonly string _defaultIndex; /// /// Create new instance @@ -47,7 +46,6 @@ public QdrantMemory( this._log = log ?? DefaultLogger.Instance; this._qdrantClient = new QdrantClient(endpoint: config.Endpoint, apiKey: config.APIKey); - this._defaultIndex = !string.IsNullOrWhiteSpace(config.DefaultIndex) ? config.DefaultIndex : Constants.DefaultIndex; } /// @@ -55,7 +53,7 @@ public Task CreateIndexAsync( string index, int vectorSize, CancellationToken cancellationToken = default) { - index = this.NormalizeIndexName(index); + index = NormalizeIndexName(index); return this._qdrantClient.CreateCollectionAsync(index, vectorSize, cancellationToken); } @@ -73,13 +71,7 @@ public Task DeleteIndexAsync( string index, CancellationToken cancellationToken = default) { - index = this.NormalizeIndexName(index); - if (string.Equals(index, Constants.DefaultIndex, StringComparison.OrdinalIgnoreCase)) - { - this._log.LogWarning("The default index cannot be deleted"); - return Task.CompletedTask; - } - + index = NormalizeIndexName(index); return this._qdrantClient.DeleteCollectionAsync(index, cancellationToken); } @@ -89,7 +81,7 @@ public async Task UpsertAsync( MemoryRecord record, CancellationToken cancellationToken = default) { - index = this.NormalizeIndexName(index); + index = NormalizeIndexName(index); QdrantPoint qdrantPoint; @@ -134,7 +126,7 @@ public async Task UpsertAsync( bool withEmbeddings = false, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - index = this.NormalizeIndexName(index); + index = NormalizeIndexName(index); if (limit <= 0) { limit = int.MaxValue; } // Remove empty filters @@ -170,7 +162,7 @@ public async IAsyncEnumerable GetListAsync( bool withEmbeddings = false, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - index = this.NormalizeIndexName(index); + index = NormalizeIndexName(index); if (limit <= 0) { limit = int.MaxValue; } // Remove empty filters @@ -202,7 +194,7 @@ public async Task DeleteAsync( MemoryRecord record, CancellationToken cancellationToken = default) { - index = this.NormalizeIndexName(index); + index = NormalizeIndexName(index); QdrantPoint? existingPoint = await this._qdrantClient .GetVectorByPayloadIdAsync(index, record.Id, cancellationToken: cancellationToken) @@ -223,11 +215,11 @@ public async Task DeleteAsync( private static readonly Regex s_replaceIndexNameCharsRegex = new(@"[\s|\\|/|.|_|:]"); private const string ValidSeparator = "-"; - private string NormalizeIndexName(string index) + private static string NormalizeIndexName(string index) { if (string.IsNullOrWhiteSpace(index)) { - index = this._defaultIndex; + throw new ArgumentNullException(nameof(index), "The index name is empty"); } index = s_replaceIndexNameCharsRegex.Replace(index.Trim().ToLowerInvariant(), ValidSeparator); diff --git a/extensions/Redis/Redis.FunctionalTests/DefaultTests.cs b/extensions/Redis/Redis.FunctionalTests/DefaultTests.cs index 40aea5c00..d328ecb59 100644 --- a/extensions/Redis/Redis.FunctionalTests/DefaultTests.cs +++ b/extensions/Redis/Redis.FunctionalTests/DefaultTests.cs @@ -16,6 +16,7 @@ public DefaultTests(IConfiguration cfg, ITestOutputHelper output) : base(cfg, ou Assert.False(string.IsNullOrEmpty(this.OpenAiConfig.APIKey)); this._memory = new KernelMemoryBuilder() + .With(new KernelMemoryConfig { DefaultIndexName = "default4tests" }) .WithSearchClientConfig(new SearchClientConfig { EmptyAnswer = NotFound }) .WithOpenAI(this.OpenAiConfig) .WithRedisMemoryDb(this.RedisConfig) @@ -68,7 +69,7 @@ public async Task ItNormalizesIndexNames() [Trait("Category", "Redis")] public async Task ItUsesDefaultIndexName() { - await IndexListTest.ItUsesDefaultIndexName(this._memory, this.Log); + await IndexListTest.ItUsesDefaultIndexName(this._memory, this.Log, "default4tests"); } [Fact] diff --git a/extensions/Redis/Redis/RedisMemory.cs b/extensions/Redis/Redis/RedisMemory.cs index ed0b61d36..8adc14717 100644 --- a/extensions/Redis/Redis/RedisMemory.cs +++ b/extensions/Redis/Redis/RedisMemory.cs @@ -401,7 +401,7 @@ private static string NormalizeIndexName(string index, string? prefix = null) { if (string.IsNullOrWhiteSpace(index)) { - index = Constants.DefaultIndex; + throw new ArgumentNullException(nameof(index), "The index name is empty"); } var indexWithPrefix = !string.IsNullOrWhiteSpace(prefix) ? $"{prefix}{index}" : index; diff --git a/extensions/SQLServer/SQLServer.FunctionalTests/DefaultTests.cs b/extensions/SQLServer/SQLServer.FunctionalTests/DefaultTests.cs index 9dd9bfe5a..4c0c4651e 100644 --- a/extensions/SQLServer/SQLServer.FunctionalTests/DefaultTests.cs +++ b/extensions/SQLServer/SQLServer.FunctionalTests/DefaultTests.cs @@ -24,6 +24,7 @@ public DefaultTests(IConfiguration cfg, ITestOutputHelper output) : base(cfg, ou SqlServerConfig sqlServerConfig = cfg.GetSection("KernelMemory:Services:SqlServer").Get()!; this._memory = new KernelMemoryBuilder() + .With(new KernelMemoryConfig { DefaultIndexName = "default4tests" }) .WithSearchClientConfig(new SearchClientConfig { EmptyAnswer = NotFound }) .WithOpenAI(this.OpenAiConfig) // .WithAzureOpenAITextGeneration(this.AzureOpenAITextConfiguration) @@ -74,6 +75,13 @@ public async Task ItNormalizesIndexNames() await IndexListTest.ItNormalizesIndexNames(this._memory, this.Log); } + [Fact] + [Trait("Category", "SQLServer")] + public async Task ItUsesDefaultIndexName() + { + await IndexListTest.ItUsesDefaultIndexName(this._memory, this.Log, "default4tests"); + } + [Fact] [Trait("Category", "SQLServer")] public async Task ItDeletesIndexes() diff --git a/nuget-package.props b/nuget-package.props index 99558093a..b42dcce56 100644 --- a/nuget-package.props +++ b/nuget-package.props @@ -1,7 +1,7 @@  - 0.32.0 + 0.33.0 false diff --git a/service/Abstractions/Constants.cs b/service/Abstractions/Constants.cs index a15453a58..5dfb4ad2e 100644 --- a/service/Abstractions/Constants.cs +++ b/service/Abstractions/Constants.cs @@ -22,9 +22,6 @@ public static class Constants // Internal file used to track progress of asynchronous pipelines public const string PipelineStatusFilename = "__pipeline_status.json"; - // Index name used when none is specified - public const string DefaultIndex = "default"; - // Tags settings public const char ReservedEqualsChar = ':'; public const string ReservedTagsPrefix = "__"; diff --git a/service/Abstractions/Models/DocumentUploadRequest.cs b/service/Abstractions/Models/DocumentUploadRequest.cs index c52bcb9fb..cc534c752 100644 --- a/service/Abstractions/Models/DocumentUploadRequest.cs +++ b/service/Abstractions/Models/DocumentUploadRequest.cs @@ -75,7 +75,7 @@ public DocumentUploadRequest() /// How to process the files, e.g. how to extract/chunk etc. public DocumentUploadRequest(Document document, string? index = null, IEnumerable? steps = null) { - this.Index = IndexExtensions.CleanName(index); + this.Index = index ?? string.Empty; this.Steps = steps?.ToList() ?? new List(); this.DocumentId = document.Id; diff --git a/service/Abstractions/Models/IndexExtensions.cs b/service/Abstractions/Models/IndexExtensions.cs deleted file mode 100644 index 30a01d220..000000000 --- a/service/Abstractions/Models/IndexExtensions.cs +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.KernelMemory; - -public static class IndexExtensions -{ - public static string CleanName(string? name) - { - if (name == null) { return Constants.DefaultIndex; } - - name = name.Trim(); - return string.IsNullOrEmpty(name) ? Constants.DefaultIndex : name; - } -} diff --git a/service/Abstractions/Models/MemoryQuery.cs b/service/Abstractions/Models/MemoryQuery.cs index d2ced508e..079536f57 100644 --- a/service/Abstractions/Models/MemoryQuery.cs +++ b/service/Abstractions/Models/MemoryQuery.cs @@ -9,7 +9,7 @@ public class MemoryQuery { [JsonPropertyName("index")] [JsonPropertyOrder(0)] - public string Index { get; set; } = string.Empty; + public string? Index { get; set; } = string.Empty; [JsonPropertyName("question")] [JsonPropertyOrder(1)] diff --git a/service/Abstractions/Models/SearchQuery.cs b/service/Abstractions/Models/SearchQuery.cs index a597396ea..642a17bec 100644 --- a/service/Abstractions/Models/SearchQuery.cs +++ b/service/Abstractions/Models/SearchQuery.cs @@ -9,7 +9,7 @@ public class SearchQuery { [JsonPropertyName("index")] [JsonPropertyOrder(0)] - public string Index { get; set; } = string.Empty; + public string? Index { get; set; } = string.Empty; [JsonPropertyName("query")] [JsonPropertyOrder(1)] diff --git a/service/Abstractions/Pipeline/DataPipeline.cs b/service/Abstractions/Pipeline/DataPipeline.cs index cf1459b58..0bee2c748 100644 --- a/service/Abstractions/Pipeline/DataPipeline.cs +++ b/service/Abstractions/Pipeline/DataPipeline.cs @@ -15,8 +15,6 @@ namespace Microsoft.KernelMemory.Pipeline; /// public sealed class DataPipeline { - private string _index = string.Empty; - [JsonConverter(typeof(JsonStringEnumConverter))] public enum ArtifactTypes { @@ -210,11 +208,7 @@ public string GetHandlerOutputFileName(IPipelineStepHandler handler, int index = /// [JsonPropertyOrder(0)] [JsonPropertyName("index")] - public string Index - { - get { return this._index; } - set { this._index = IndexExtensions.CleanName(value); } - } + public string Index { get; set; } = string.Empty; /// /// Id of the document and the pipeline instance. diff --git a/service/Core/Configuration/KernelMemoryConfig.cs b/service/Core/Configuration/KernelMemoryConfig.cs index 663495d67..559bbf517 100644 --- a/service/Core/Configuration/KernelMemoryConfig.cs +++ b/service/Core/Configuration/KernelMemoryConfig.cs @@ -127,6 +127,11 @@ public class RetrievalConfig /// public string TextGeneratorType { get; set; } = string.Empty; + /// + /// Name of the index to use when none is specified. + /// + public string DefaultIndexName { get; set; } = "default"; + /// /// HTTP service authorization settings. /// diff --git a/service/Core/KernelMemoryBuilder.cs b/service/Core/KernelMemoryBuilder.cs index 190058756..8aeeeda1d 100644 --- a/service/Core/KernelMemoryBuilder.cs +++ b/service/Core/KernelMemoryBuilder.cs @@ -212,8 +212,7 @@ private MemoryServerless BuildServerlessClient() { try { - var serviceProvider = this._memoryServiceCollection.BuildServiceProvider(); - + ServiceProvider serviceProvider = this._memoryServiceCollection.BuildServiceProvider(); this.CompleteServerlessClient(serviceProvider); // In case the user didn't set the embedding generator and memory DB to use for ingestion, use the values set for retrieval @@ -261,16 +260,11 @@ private MemoryService BuildAsyncClient() // In case the user didn't set the embedding generator and memory DB to use for ingestion, use the values set for retrieval this.ReuseRetrievalEmbeddingGeneratorIfNecessary(serviceProvider); this.ReuseRetrievalMemoryDbIfNecessary(serviceProvider); + this.CheckForMissingDependencies(); // Recreate the service provider, in order to have the latest dependencies just configured serviceProvider = this._memoryServiceCollection.BuildServiceProvider(); - - var orchestrator = serviceProvider.GetService() ?? throw new ConfigurationException("Unable to build orchestrator"); - var searchClient = serviceProvider.GetService() ?? throw new ConfigurationException("Unable to build search client"); - - this.CheckForMissingDependencies(); - - return new MemoryService(orchestrator, searchClient); + return ActivatorUtilities.CreateInstance(serviceProvider); } private KernelMemoryBuilder CompleteServerlessClient(ServiceProvider serviceProvider) diff --git a/service/Core/MemoryServerless.cs b/service/Core/MemoryServerless.cs index 687674cf5..4d90aaab2 100644 --- a/service/Core/MemoryServerless.cs +++ b/service/Core/MemoryServerless.cs @@ -9,6 +9,7 @@ using System.Threading.Tasks; using Microsoft.KernelMemory.Configuration; using Microsoft.KernelMemory.Diagnostics; +using Microsoft.KernelMemory.Models; using Microsoft.KernelMemory.Pipeline; using Microsoft.KernelMemory.Search; @@ -24,8 +25,8 @@ namespace Microsoft.KernelMemory; public class MemoryServerless : IKernelMemory { private readonly ISearchClient _searchClient; - private readonly InProcessPipelineOrchestrator _orchestrator; + private readonly string? _defaultIndexName; /// /// Synchronous orchestrator used by the serverless memory. @@ -39,10 +40,15 @@ public class MemoryServerless : IKernelMemory public MemoryServerless( InProcessPipelineOrchestrator orchestrator, - ISearchClient searchClient) + ISearchClient searchClient, + KernelMemoryConfig? config = null) { this._orchestrator = orchestrator ?? throw new ConfigurationException("The orchestrator is NULL"); this._searchClient = searchClient ?? throw new ConfigurationException("The search client is NULL"); + + // A non-null config object is required in order to get a non-empty default index name + config ??= new KernelMemoryConfig(); + this._defaultIndexName = config.DefaultIndexName; } /// @@ -75,7 +81,7 @@ public Task ImportDocumentAsync( DocumentUploadRequest uploadRequest, CancellationToken cancellationToken = default) { - var index = IndexExtensions.CleanName(uploadRequest.Index); + var index = IndexName.CleanName(uploadRequest.Index, this._defaultIndexName); return this._orchestrator.ImportDocumentAsync(index, uploadRequest, cancellationToken); } @@ -147,12 +153,14 @@ public async Task> ListIndexesAsync(CancellationToken /// public Task DeleteIndexAsync(string? index = null, CancellationToken cancellationToken = default) { + index = IndexName.CleanName(index, this._defaultIndexName); return this._orchestrator.StartIndexDeletionAsync(index: index, cancellationToken); } /// public Task DeleteDocumentAsync(string documentId, string? index = null, CancellationToken cancellationToken = default) { + index = IndexName.CleanName(index, this._defaultIndexName); return this._orchestrator.StartDocumentDeletionAsync(documentId: documentId, index: index, cancellationToken); } @@ -162,7 +170,7 @@ public async Task IsDocumentReadyAsync( string? index = null, CancellationToken cancellationToken = default) { - index = IndexExtensions.CleanName(index); + index = IndexName.CleanName(index, this._defaultIndexName); return await this._orchestrator.IsDocumentReadyAsync(index: index, documentId, cancellationToken).ConfigureAwait(false); } @@ -172,7 +180,7 @@ public async Task IsDocumentReadyAsync( string? index = null, CancellationToken cancellationToken = default) { - index = IndexExtensions.CleanName(index); + index = IndexName.CleanName(index, this._defaultIndexName); try { DataPipeline? pipeline = await this._orchestrator.ReadPipelineStatusAsync(index: index, documentId, cancellationToken).ConfigureAwait(false); @@ -201,7 +209,7 @@ public Task SearchAsync( filters.Add(filter); } - index = IndexExtensions.CleanName(index); + index = IndexName.CleanName(index, this._defaultIndexName); return this._searchClient.SearchAsync( index: index, query: query, @@ -227,7 +235,7 @@ public Task AskAsync( filters.Add(filter); } - index = IndexExtensions.CleanName(index); + index = IndexName.CleanName(index, this._defaultIndexName); return this._searchClient.AskAsync( index: index, question: question, diff --git a/service/Core/MemoryService.cs b/service/Core/MemoryService.cs index 16250a935..b7267c060 100644 --- a/service/Core/MemoryService.cs +++ b/service/Core/MemoryService.cs @@ -9,6 +9,7 @@ using System.Threading.Tasks; using Microsoft.KernelMemory.Configuration; using Microsoft.KernelMemory.Diagnostics; +using Microsoft.KernelMemory.Models; using Microsoft.KernelMemory.Pipeline; using Microsoft.KernelMemory.Search; @@ -19,13 +20,19 @@ public class MemoryService : IKernelMemory { private readonly IPipelineOrchestrator _orchestrator; private readonly ISearchClient _searchClient; + private readonly string? _defaultIndexName; public MemoryService( IPipelineOrchestrator orchestrator, - ISearchClient searchClient) + ISearchClient searchClient, + KernelMemoryConfig? config = null) { this._orchestrator = orchestrator ?? throw new ConfigurationException("The orchestrator is NULL"); this._searchClient = searchClient ?? throw new ConfigurationException("The search client is NULL"); + + // A non-null config object is required in order to get a non-empty default index name + config ??= new KernelMemoryConfig(); + this._defaultIndexName = config.DefaultIndexName; } /// @@ -58,7 +65,7 @@ public Task ImportDocumentAsync( DocumentUploadRequest uploadRequest, CancellationToken cancellationToken = default) { - var index = IndexExtensions.CleanName(uploadRequest.Index); + var index = IndexName.CleanName(uploadRequest.Index, this._defaultIndexName); return this._orchestrator.ImportDocumentAsync(index, uploadRequest, cancellationToken); } @@ -131,12 +138,14 @@ public async Task> ListIndexesAsync(CancellationToken /// public Task DeleteIndexAsync(string? index = null, CancellationToken cancellationToken = default) { + index = IndexName.CleanName(index, this._defaultIndexName); return this._orchestrator.StartIndexDeletionAsync(index: index, cancellationToken); } /// public Task DeleteDocumentAsync(string documentId, string? index = null, CancellationToken cancellationToken = default) { + index = IndexName.CleanName(index, this._defaultIndexName); return this._orchestrator.StartDocumentDeletionAsync(documentId: documentId, index: index, cancellationToken); } @@ -146,7 +155,7 @@ public Task IsDocumentReadyAsync( string? index = null, CancellationToken cancellationToken = default) { - index = IndexExtensions.CleanName(index); + index = IndexName.CleanName(index, this._defaultIndexName); return this._orchestrator.IsDocumentReadyAsync(index: index, documentId, cancellationToken); } @@ -156,7 +165,7 @@ public Task IsDocumentReadyAsync( string? index = null, CancellationToken cancellationToken = default) { - index = IndexExtensions.CleanName(index); + index = IndexName.CleanName(index, this._defaultIndexName); return this._orchestrator.ReadPipelineSummaryAsync(index: index, documentId, cancellationToken); } @@ -177,7 +186,7 @@ public Task SearchAsync( filters.Add(filter); } - index = IndexExtensions.CleanName(index); + index = IndexName.CleanName(index, this._defaultIndexName); return this._searchClient.SearchAsync( index: index, query: query, @@ -203,7 +212,7 @@ public Task AskAsync( filters.Add(filter); } - index = IndexExtensions.CleanName(index); + index = IndexName.CleanName(index, this._defaultIndexName); return this._searchClient.AskAsync( index: index, question: question, diff --git a/service/Core/MemoryStorage/DevTools/SimpleTextDb.cs b/service/Core/MemoryStorage/DevTools/SimpleTextDb.cs index 11035af93..4ebd75924 100644 --- a/service/Core/MemoryStorage/DevTools/SimpleTextDb.cs +++ b/service/Core/MemoryStorage/DevTools/SimpleTextDb.cs @@ -188,7 +188,7 @@ private static string NormalizeIndexName(string index) { if (string.IsNullOrWhiteSpace(index)) { - index = Constants.DefaultIndex; + throw new ArgumentNullException(nameof(index), "The index name is empty"); } index = s_replaceIndexNameCharsRegex.Replace(index.Trim().ToLowerInvariant(), ValidSeparator); diff --git a/service/Core/MemoryStorage/DevTools/SimpleVectorDb.cs b/service/Core/MemoryStorage/DevTools/SimpleVectorDb.cs index 011ab1ad1..1ba4df624 100644 --- a/service/Core/MemoryStorage/DevTools/SimpleVectorDb.cs +++ b/service/Core/MemoryStorage/DevTools/SimpleVectorDb.cs @@ -195,7 +195,7 @@ private static string NormalizeIndexName(string index) { if (string.IsNullOrWhiteSpace(index)) { - index = Constants.DefaultIndex; + throw new ArgumentNullException(nameof(index), "The index name is empty"); } index = s_replaceIndexNameCharsRegex.Replace(index.Trim().ToLowerInvariant(), ValidSeparator); diff --git a/service/Core/Models/IndexName.cs b/service/Core/Models/IndexName.cs new file mode 100644 index 000000000..aa890a807 --- /dev/null +++ b/service/Core/Models/IndexName.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; + +namespace Microsoft.KernelMemory.Models; + +public static class IndexName +{ + /// + /// Clean the index name, returning a non empty value if possible + /// + /// Input index name + /// Default value to fall back when input is empty + /// Non empty index name + public static string CleanName(string? name, string? defaultName) + { + if (string.IsNullOrWhiteSpace(name) && string.IsNullOrWhiteSpace(defaultName)) + { + throw new ArgumentNullException(nameof(defaultName), + "Both index name and default fallback value are empty. Provide an index name or a default value to use when the index name is empty."); + } + + defaultName = defaultName?.Trim() ?? string.Empty; + if (name == null) { return defaultName; } + + name = name.Trim(); + return string.IsNullOrWhiteSpace(name) ? defaultName : name; + } +} diff --git a/service/Core/Pipeline/BaseOrchestrator.cs b/service/Core/Pipeline/BaseOrchestrator.cs index af8787145..79058954c 100644 --- a/service/Core/Pipeline/BaseOrchestrator.cs +++ b/service/Core/Pipeline/BaseOrchestrator.cs @@ -12,6 +12,7 @@ using Microsoft.KernelMemory.Diagnostics; using Microsoft.KernelMemory.FileSystem.DevTools; using Microsoft.KernelMemory.MemoryStorage; +using Microsoft.KernelMemory.Models; namespace Microsoft.KernelMemory.Pipeline; @@ -26,6 +27,7 @@ public abstract class BaseOrchestrator : IPipelineOrchestrator, IDisposable private readonly List _defaultIngestionSteps; private readonly IContentStorage _contentStorage; private readonly IMimeTypeDetection _mimeTypeDetection; + private readonly string? _defaultIndexName; protected ILogger Log { get; private set; } protected CancellationTokenSource CancellationTokenSource { get; private set; } @@ -48,6 +50,7 @@ protected BaseOrchestrator( this._embeddingGenerators = embeddingGenerators; this._memoryDbs = memoryDbs; this._textGenerator = textGenerator; + this._defaultIndexName = config?.DefaultIndexName; this._mimeTypeDetection = mimeTypeDetection ?? new MimeTypesDetection(); this.CancellationTokenSource = new CancellationTokenSource(); @@ -80,6 +83,8 @@ public async Task ImportDocumentAsync(string index, DocumentUploadReques { this.Log.LogInformation("Queueing upload of {0} files for further processing [request {1}]", uploadRequest.Files.Count, uploadRequest.DocumentId); + index = IndexName.CleanName(index, this._defaultIndexName); + var pipeline = this.PrepareNewDocumentUpload( index: index, documentId: uploadRequest.DocumentId, @@ -122,6 +127,8 @@ public DataPipeline PrepareNewDocumentUpload( TagCollection tags, IEnumerable? filesToUpload = null) { + index = IndexName.CleanName(index, this._defaultIndexName); + filesToUpload ??= new List(); var pipeline = new DataPipeline @@ -140,7 +147,7 @@ public DataPipeline PrepareNewDocumentUpload( /// public async Task ReadPipelineStatusAsync(string index, string documentId, CancellationToken cancellationToken = default) { - index = IndexExtensions.CleanName(index); + index = IndexName.CleanName(index, this._defaultIndexName); try { @@ -170,6 +177,8 @@ public DataPipeline PrepareNewDocumentUpload( /// public async Task ReadPipelineSummaryAsync(string index, string documentId, CancellationToken cancellationToken = default) { + index = IndexName.CleanName(index, this._defaultIndexName); + try { DataPipeline? pipeline = await this.ReadPipelineStatusAsync(index: index, documentId: documentId, cancellationToken).ConfigureAwait(false); @@ -184,6 +193,8 @@ public DataPipeline PrepareNewDocumentUpload( /// public async Task IsDocumentReadyAsync(string index, string documentId, CancellationToken cancellationToken = default) { + index = IndexName.CleanName(index, this._defaultIndexName); + try { DataPipeline? pipeline = await this.ReadPipelineStatusAsync(index: index, documentId, cancellationToken).ConfigureAwait(false); @@ -205,24 +216,28 @@ public Task StopAllPipelinesAsync() /// public async Task ReadTextFileAsync(DataPipeline pipeline, string fileName, CancellationToken cancellationToken = default) { + pipeline.Index = IndexName.CleanName(pipeline.Index, this._defaultIndexName); return (await this.ReadFileAsync(pipeline, fileName, cancellationToken).ConfigureAwait(false)).ToString(); } /// public Task ReadFileAsync(DataPipeline pipeline, string fileName, CancellationToken cancellationToken = default) { + pipeline.Index = IndexName.CleanName(pipeline.Index, this._defaultIndexName); return this._contentStorage.ReadFileAsync(pipeline.Index, pipeline.DocumentId, fileName, true, cancellationToken); } /// public Task WriteTextFileAsync(DataPipeline pipeline, string fileName, string fileContent, CancellationToken cancellationToken = default) { + pipeline.Index = IndexName.CleanName(pipeline.Index, this._defaultIndexName); return this.WriteFileAsync(pipeline, fileName, new BinaryData(fileContent), cancellationToken); } /// public Task WriteFileAsync(DataPipeline pipeline, string fileName, BinaryData fileContent, CancellationToken cancellationToken = default) { + pipeline.Index = IndexName.CleanName(pipeline.Index, this._defaultIndexName); return this._contentStorage.WriteFileAsync(pipeline.Index, pipeline.DocumentId, fileName, fileContent.ToStream(), cancellationToken); } @@ -250,6 +265,7 @@ public ITextGenerator GetTextGenerator() /// public Task StartIndexDeletionAsync(string? index = null, CancellationToken cancellationToken = default) { + index = IndexName.CleanName(index, this._defaultIndexName); DataPipeline pipeline = PrepareIndexDeletion(index: index); return this.RunPipelineAsync(pipeline, cancellationToken); } @@ -257,6 +273,7 @@ public Task StartIndexDeletionAsync(string? index = null, CancellationToken canc /// public Task StartDocumentDeletionAsync(string documentId, string? index = null, CancellationToken cancellationToken = default) { + index = IndexName.CleanName(index, this._defaultIndexName); DataPipeline pipeline = PrepareDocumentDeletion(index: index, documentId: documentId); return this.RunPipelineAsync(pipeline, cancellationToken); } diff --git a/service/Service/ServiceConfiguration.cs b/service/Service/ServiceConfiguration.cs index b98812da1..eace5268b 100644 --- a/service/Service/ServiceConfiguration.cs +++ b/service/Service/ServiceConfiguration.cs @@ -75,7 +75,7 @@ private IKernelMemoryBuilder BuildUsingConfiguration(IKernelMemoryBuilder builde } // Required by ctors expecting KernelMemoryConfig via DI - builder.AddSingleton(this._memoryConfiguration); + builder.AddSingleton(this._memoryConfiguration); this.ConfigureMimeTypeDetectionDependency(builder); diff --git a/service/Service/WebAPIEndpoints.cs b/service/Service/WebAPIEndpoints.cs index 658952ec0..2fc1231f5 100644 --- a/service/Service/WebAPIEndpoints.cs +++ b/service/Service/WebAPIEndpoints.cs @@ -260,8 +260,6 @@ async Task ( CancellationToken cancellationToken) => { log.LogTrace("New document status HTTP request"); - index = IndexExtensions.CleanName(index); - if (string.IsNullOrEmpty(documentId)) { return Results.Problem(detail: $"'{Constants.WebServiceDocumentIdField}' query parameter is missing or has no value", statusCode: 400); diff --git a/service/Service/appsettings.json b/service/Service/appsettings.json index 6483c2868..40ed6a544 100644 --- a/service/Service/appsettings.json +++ b/service/Service/appsettings.json @@ -96,6 +96,8 @@ "ContentStorageType": "SimpleFileStorage", // "AzureOpenAIText", "OpenAI" or "LlamaSharp" "TextGeneratorType": "", + // Name of the index to use when none is specified + "DefaultIndexName": "default", // Data ingestion pipelines configuration. "DataIngestion": { // - InProcess: in process .NET orchestrator, synchronous/no queues @@ -276,9 +278,7 @@ // Qdrant endpoint "Endpoint": "http://127.0.0.1:6333", // Qdrant API key, e.g. when using Qdrant cloud - "APIKey": "", - // Name of the default KM index/Qdrant collection, when none is specified - "DefaultIndex": "default" + "APIKey": "" }, "Redis": { // Redis connection string, e.g. "localhost:6379,password=..." diff --git a/service/tests/Core.FunctionalTests/DefaultTestCases/DocumentUploadTest.cs b/service/tests/Core.FunctionalTests/DefaultTestCases/DocumentUploadTest.cs index f8c1fa377..3aa88dd02 100644 --- a/service/tests/Core.FunctionalTests/DefaultTestCases/DocumentUploadTest.cs +++ b/service/tests/Core.FunctionalTests/DefaultTestCases/DocumentUploadTest.cs @@ -16,10 +16,12 @@ await memory.ImportDocumentAsync( documentId: Id, steps: Constants.PipelineWithoutSummary); + var count = 0; while (!await memory.IsDocumentReadyAsync(documentId: Id)) { + Assert.True(count++ <= 30, "Document import timed out"); log("Waiting for memory ingestion to complete..."); - await Task.Delay(TimeSpan.FromSeconds(2)); + await Task.Delay(TimeSpan.FromSeconds(1)); } // Act diff --git a/service/tests/Core.FunctionalTests/DefaultTestCases/IndexListTest.cs b/service/tests/Core.FunctionalTests/DefaultTestCases/IndexListTest.cs index 9275b1eef..128740424 100644 --- a/service/tests/Core.FunctionalTests/DefaultTestCases/IndexListTest.cs +++ b/service/tests/Core.FunctionalTests/DefaultTestCases/IndexListTest.cs @@ -21,20 +21,27 @@ public static async Task ItNormalizesIndexNames(IKernelMemory memory, Action log) + public static async Task ItUsesDefaultIndexName(IKernelMemory memory, Action log, string expectedDefault) { // Arrange string emptyIndexName = string.Empty; // Act - await memory.ImportTextAsync("something", index: emptyIndexName); + var id = await memory.ImportTextAsync("something", index: emptyIndexName); + var count = 0; + while (!await memory.IsDocumentReadyAsync(id)) + { + Assert.True(count++ <= 30, "Document import timed out"); + await Task.Delay(TimeSpan.FromSeconds(1)); + } + var list = (await memory.ListIndexesAsync()).ToList(); // Clean up before exceptions can occur await memory.DeleteIndexAsync(emptyIndexName); // Assert - Assert.True(list.Any(x => x.Name == "default")); + Assert.True(list.Any(x => x.Name == expectedDefault)); } public static async Task ItListsIndexes(IKernelMemory memory, Action log) diff --git a/service/tests/Core.FunctionalTests/Models/IndexNameTest.cs b/service/tests/Core.FunctionalTests/Models/IndexNameTest.cs new file mode 100644 index 000000000..a42c37021 --- /dev/null +++ b/service/tests/Core.FunctionalTests/Models/IndexNameTest.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.KernelMemory; +using Microsoft.KernelMemory.Models; + +namespace FunctionalTests.Models; + +public class IndexNameTest +{ + [Theory] + [Trait("Category", "UnitTest")] + [InlineData(null, "abc", "abc")] + [InlineData("", "bcd", "bcd")] + [InlineData(" ", "cde", "cde")] + [InlineData(" ", " def ", "def")] + [InlineData(" \n ", "cde", "cde")] + [InlineData(" \r ", " def ", "def")] + [InlineData(" \t ", " def ", "def")] + [InlineData("123", null, "123")] + [InlineData("123", "", "123")] + [InlineData("234", "xyz", "234")] + [InlineData(" 345 ", "xyz", "345")] + [InlineData(" 456 ", " xyz ", "456")] + public void ItReturnsExpectedIndexName(string? name, string defaultName, string expected) + { + Assert.Equal(expected, IndexName.CleanName(name, defaultName)); + } + + [Theory] + [Trait("Category", "UnitTest")] + [InlineData(null, null)] + [InlineData(null, "")] + [InlineData("", null)] + [InlineData("", "")] + [InlineData(" ", "")] + [InlineData("", " ")] + [InlineData(" ", " ")] + [InlineData(" \n ", " ")] + [InlineData(" ", " \n ")] + [InlineData(" \r ", " ")] + [InlineData(" ", " \r ")] + [InlineData(" \t ", " ")] + [InlineData(" ", " \t ")] + public void ItThrowsIfIndexNameCannotBeCalculated(string? name, string defaultName) + { + Assert.Throws(() => IndexName.CleanName(name, defaultName)); + } +} diff --git a/service/tests/Core.FunctionalTests/ServerLess/OpenAIDependencyInjectionTest.cs b/service/tests/Core.FunctionalTests/ServerLess/OpenAIDependencyInjectionTest.cs index 298fa6419..235f13166 100644 --- a/service/tests/Core.FunctionalTests/ServerLess/OpenAIDependencyInjectionTest.cs +++ b/service/tests/Core.FunctionalTests/ServerLess/OpenAIDependencyInjectionTest.cs @@ -28,6 +28,7 @@ public async Task TestExtensionMethod1() // Assert var answer = await memory.AskAsync("What year is it?"); + Console.WriteLine(answer.Result); Assert.Contains("2099", answer.Result); } diff --git a/service/tests/Core.FunctionalTests/VectorDbComparison/TestCosineSimilarity.cs b/service/tests/Core.FunctionalTests/VectorDbComparison/TestCosineSimilarity.cs index f01e58ce0..a4cfc149e 100644 --- a/service/tests/Core.FunctionalTests/VectorDbComparison/TestCosineSimilarity.cs +++ b/service/tests/Core.FunctionalTests/VectorDbComparison/TestCosineSimilarity.cs @@ -42,10 +42,13 @@ public async Task CompareCosineSimilarity() var postgres = new PostgresMemory(this.PostgresConfig, embeddingGenerator); var simpleVecDb = new SimpleVectorDb(this.SimpleVectorDbConfig, embeddingGenerator); - // TODO: revisit RedisMemory not to need this, e.g. not to connect in ctor - IConnectionMultiplexer redisMux; - if (RedisEnabled) { redisMux = await ConnectionMultiplexer.ConnectAsync(this.RedisConfig.ConnectionString); } - var redis = new RedisMemory(this.RedisConfig, redisMux, embeddingGenerator); + RedisMemory? redis = null; + if (RedisEnabled) + { + // TODO: revisit RedisMemory not to need this, e.g. not to connect in ctor + var redisMux = await ConnectionMultiplexer.ConnectAsync(this.RedisConfig.ConnectionString); + redis = new RedisMemory(this.RedisConfig, redisMux, embeddingGenerator); + } // == Delete indexes left over diff --git a/service/tests/Core.FunctionalTests/appsettings.json b/service/tests/Core.FunctionalTests/appsettings.json index 957a1ef1c..40099880b 100644 --- a/service/tests/Core.FunctionalTests/appsettings.json +++ b/service/tests/Core.FunctionalTests/appsettings.json @@ -28,8 +28,7 @@ }, "Qdrant": { "Endpoint": "http://127.0.0.1:6333", - "APIKey": "", - "DefaultIndex": "default" + "APIKey": "" }, "Redis": { // Redis connection string, e.g. "localhost:6379,password=..." diff --git a/service/tests/Service.FunctionalTests/DefaultTests.cs b/service/tests/Service.FunctionalTests/DefaultTests.cs index ca05df2a0..907c70abb 100644 --- a/service/tests/Service.FunctionalTests/DefaultTests.cs +++ b/service/tests/Service.FunctionalTests/DefaultTests.cs @@ -58,6 +58,13 @@ public async Task ItNormalizesIndexNames() await IndexListTest.ItNormalizesIndexNames(this._memory, this.Log); } + [Fact] + [Trait("Category", "WebService")] + public async Task ItUsesDefaultIndexName() + { + await IndexListTest.ItUsesDefaultIndexName(this._memory, this.Log, "default"); + } + [Fact] [Trait("Category", "WebService")] public async Task ItDeletesIndexes() diff --git a/service/tests/Service.FunctionalTests/appsettings.json b/service/tests/Service.FunctionalTests/appsettings.json index 5574dd021..701771a54 100644 --- a/service/tests/Service.FunctionalTests/appsettings.json +++ b/service/tests/Service.FunctionalTests/appsettings.json @@ -20,8 +20,7 @@ }, "Qdrant": { "Endpoint": "http://127.0.0.1:6333", - "APIKey": "", - "DefaultIndex": "default" + "APIKey": "" }, "AzureOpenAIText": { // "ApiKey" or "AzureIdentity"