* Add citations

* Complete Webclient Ask method
* Sync signatures, user ID first then Doc ID/Query/others
* Update docs and samples, show citations
* Add ExistsAsync method +  /upload-status stub

README.md

@@ -37,7 +37,7 @@ in your app.
 >
 > // Import a file specifying a User and Tags
 > await memory.ImportFileAsync("business-plan.docx",
->     new DocumentDetails("file1", "user@some.email")
+>     new DocumentDetails("user@some.email", "file1")
 >         .AddTag("collection", "business")
 >         .AddTag("collection", "plans")
 >         .AddTag("type", "doc"));
@@ -48,7 +48,7 @@ in your app.
 > ```csharp
 > string answer1 = await memory.AskAsync("How many people attended the meeting?");
 >
-> string answer2 = await memory.AskAsync("what's the project timeline?", "user@some.email");
+> string answer2 = await memory.AskAsync("user@some.email", "what's the project timeline?");
 > ```
 
 The code leverages the default documents ingestion pipeline:
@@ -167,7 +167,7 @@ await orchestrator.RunPipelineAsync(pipeline);
 
 1. [Using the web service](samples/dotnet-WebClient)
 2. [Importing files without the service (serverless ingestion)](samples/dotnet-Serverless)
-3. [How to upload files from command line with curl](samples/curl)
+3. [Upload files and get answers from command line with curl](samples/curl)
 4. [Writing a custom pipeline handler](samples/dotnet-CustomHandler)
 5. [Importing files with custom steps](samples/dotnet-ServerlessCustomPipeline)
 6. [Extracting text from documents](samples/dotnet-ExtractTextFromDocs)

dotnet/ClientLib/DocumentDetails.cs

@@ -22,8 +22,8 @@ public string UserId
     public TagCollection Tags { get; set; } = new();
 
     public DocumentDetails(
-        string documentId = "",
         string userId = Constants.DefaultDocumentOwnerUserId,
+        string documentId = "",
         TagCollection? tags = null)
     {
         this.DocumentId = string.IsNullOrEmpty(documentId) ? RandomId() : documentId;

dotnet/ClientLib/ISemanticMemoryClient.cs

@@ -7,9 +7,49 @@ namespace Microsoft.SemanticMemory.Client;
 
 public interface ISemanticMemoryClient
 {
+    /// <summary>
+    /// Import a file into memory. The file can have tags and other details.
+    /// </summary>
+    /// <param name="file">Details of the file to import</param>
+    /// <returns>Document ID</returns>
     public Task<string> ImportFileAsync(Document file);
+
+    /// <summary>
+    /// Import multiple files into memory. Each file can have tags and other details.
+    /// </summary>
+    /// <param name="files">Details of the files to import</param>
+    /// <returns>List of document IDs</returns>
     public Task<IList<string>> ImportFilesAsync(Document[] files);
+
+    /// <summary>
+    /// Import a file from disk into memory.
+    /// </summary>
+    /// <param name="fileName">Path and name of the file to import</param>
+    /// <returns>Document ID</returns>
     public Task<string> ImportFileAsync(string fileName);
+
+    /// <summary>
+    /// Import a files from disk into memory, with details such as tags and user ID.
+    /// </summary>
+    /// <param name="fileName">Path and name of the files to import</param>
+    /// <param name="details">File details such as tags and user ID</param>
+    /// <returns>Document ID</returns>
     public Task<string> ImportFileAsync(string fileName, DocumentDetails details);
-    public Task<string> AskAsync(string question, string userId);
+
+    /// <summary>
+    /// Search a user memory for an answer to the given query.
+    /// TODO: add support for tags.
+    /// </summary>
+    /// <param name="userId">ID of the user's memory to search</param>
+    /// <param name="query">Query/question to answer</param>
+    /// <returns>Answer to the query, if possible</returns>
+    public Task<MemoryAnswer> AskAsync(string userId, string query);
+
+    /// <summary>
+    /// Check if a document ID exists in a user memory.
+    /// </summary>
+    /// <param name="userId">ID of the user's memory to search</param>
+    /// <param name="documentId">Document ID</param>
+    /// <returns>True if the document has been successfully uploaded and imported</returns>
+    public Task<bool> ExistsAsync(string userId, string documentId);
 }

dotnet/ClientLib/MemoryAnswer.cs

@@ -0,0 +1,15 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Collections.Generic;
+using System.Text.Json.Serialization;
+
+namespace Microsoft.SemanticMemory.Client;
+
+public class MemoryAnswer
+{
+    [JsonPropertyName("Text")]
+    public string Text { get; set; } = string.Empty;
+
+    [JsonPropertyName("RelevantSources")]
+    public List<Dictionary<string, object>> RelevantSources { get; set; } = new();
+}

dotnet/ClientLib/MemoryWebClient.cs

@@ -4,6 +4,8 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Net.Http;
+using System.Text;
+using System.Text.Json;
 using System.Threading.Tasks;
 
 namespace Microsoft.SemanticMemory.Client;
@@ -46,19 +48,32 @@ public Task<string> ImportFileAsync(string fileName, DocumentDetails details)
         return this.ImportFileInternalAsync(new Document(fileName) { Details = details });
     }
 
-    public async Task<string> AskAsync(string question, string userId)
+    /// <inheritdoc />
+    public async Task<MemoryAnswer> AskAsync(string userId, string query)
     {
-        // Work in progress
+        var request = new { UserId = userId, Query = query, Tags = new TagCollection() };
+        using var content = new StringContent(JsonSerializer.Serialize(request), Encoding.UTF8, "application/json");
 
-        await Task.Delay(0).ConfigureAwait(false);
+        HttpResponseMessage? response = await this._client.PostAsync("/ask", content).ConfigureAwait(false);
+        response.EnsureSuccessStatusCode();
 
-        return "...work in progress...";
+        var json = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
+        return JsonSerializer.Deserialize<MemoryAnswer>(json, new JsonSerializerOptions { PropertyNameCaseInsensitive = true }) ?? new MemoryAnswer();
     }
 
-    public async Task<string> AskAsync(string question)
+    /// <inheritdoc />
+    public async Task<bool> ExistsAsync(string userId, string documentId)
     {
-        await Task.Delay(0).ConfigureAwait(false);
-        return "...work in progress...";
+        HttpResponseMessage? response = await this._client.GetAsync($"/upload-status?user={userId}&id={documentId}").ConfigureAwait(false);
+        response.EnsureSuccessStatusCode();
+
+        // WORK IN PROGRESS
+
+        var json = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
+
+        // WORK IN PROGRESS
+
+        return false;
     }
 
     #region private

dotnet/CoreLib/Configuration/SemanticMemoryConfig.cs

@@ -63,12 +63,7 @@ public class SemanticMemoryConfig
 
     public static SemanticMemoryConfig LoadFromAppSettings()
     {
-        if (s_host == null)
-        {
-            WebApplicationBuilder builder = WebApplication.CreateBuilder();
-            builder.Services.UseConfiguration(builder.Configuration);
-            s_host = builder.Build();
-        }
+        if (s_host == null) { s_host = PrepareHost(); }
 
         var config = s_host.Services.GetService<SemanticMemoryConfig>();
         if (config == null)
@@ -81,12 +76,7 @@ public static SemanticMemoryConfig LoadFromAppSettings()
 
     public static ILoggerFactory GetLogFactory()
     {
-        if (s_host == null)
-        {
-            WebApplicationBuilder builder = WebApplication.CreateBuilder();
-            builder.Services.UseConfiguration(builder.Configuration);
-            s_host = builder.Build();
-        }
+        if (s_host == null) { s_host = PrepareHost(); }
 
         var factory = s_host.Services.GetService<ILoggerFactory>();
         if (factory == null)
@@ -96,4 +86,12 @@ public static ILoggerFactory GetLogFactory()
 
         return factory;
     }
+
+    private static IHost PrepareHost()
+    {
+        WebApplicationBuilder builder = WebApplication.CreateBuilder();
+        builder.Logging.ConfigureLogger();
+        builder.Services.UseConfiguration(builder.Configuration);
+        return builder.Build();
+    }
 }

dotnet/CoreLib/Pipeline/MemoryServerlessClient.cs

@@ -67,9 +67,17 @@ public async Task<string> ImportFileAsync(string fileName, DocumentDetails detai
     }
 
     /// <inheritdoc />
-    public Task<string> AskAsync(string question, string userId)
+    public Task<MemoryAnswer> AskAsync(string userId, string query)
     {
-        return this._searchClient.SearchAsync(userId: userId, query: question);
+        return this._searchClient.SearchAsync(userId: userId, query: query);
+    }
+
+    public async Task<bool> ExistsAsync(string userId, string documentId)
+    {
+        // WORK IN PROGRESS
+        await Task.Delay(0).ConfigureAwait(false);
+
+        return false;
     }
 
     #region private

dotnet/CoreLib/Search/SearchClient.cs

@@ -44,13 +44,20 @@ public SearchClient(
         if (this._kernel == null) { throw new SemanticMemoryException("Semantic Kernel not configured"); }
     }
 
-    public Task<string> SearchAsync(SearchRequest request)
+    public Task<MemoryAnswer> SearchAsync(SearchRequest request)
     {
         return this.SearchAsync(request.UserId, request.Query);
     }
 
-    public async Task<string> SearchAsync(string userId, string query)
+    public async Task<MemoryAnswer> SearchAsync(string userId, string query)
     {
+        var noAnswer = new MemoryAnswer { Text = "INFO NOT FOUND" };
+
+        var answer = new MemoryAnswer
+        {
+            Text = "INFO NOT FOUND"
+        };
+
         this._log.LogTrace("Generating embedding for the query");
         IList<Embedding<float>> embeddings = await this._embeddingGenerator
             .GenerateEmbeddingsAsync(new List<string> { query }).ConfigureAwait(false);
@@ -67,7 +74,7 @@ public async Task<string> SearchAsync(string userId, string query)
                      "======\n" +
                      "Given only the facts above, provide a comprehensive/detailed answer.\n" +
                      "You don't know where the knowledge comes from, just answer.\n" +
-                     "If you don't have sufficient information, replay with 'INFO NOT FOUND'.\n" +
+                     "If you don't have sufficient information, reply with 'INFO NOT FOUND'.\n" +
                      "Question: {{$question}}\n" +
                      "Answer: ";
 
@@ -86,14 +93,24 @@ public async Task<string> SearchAsync(string userId, string query)
         await foreach ((MemoryRecord, double) memory in matches)
         {
             factsAvailableCount++;
-            var partition = memory.Item1.Metadata["text"].ToString()?.Trim() ?? "";
-            var fact = $"======\n{partition}\n";
+            var partitionText = memory.Item1.Metadata["text"].ToString()?.Trim() ?? "";
+            var fact = $"==== [Relevance: {memory.Item2:P1}]:\n{partitionText}\n";
             var size = GPT3Tokenizer.Encode(fact).Count;
             if (size < tokensAvailable)
             {
                 factsUsedCount++;
+                this._log.LogTrace("Adding text {0} with relevance {1}", factsUsedCount, memory.Item2);
                 facts += fact;
                 tokensAvailable -= size;
+
+                answer.RelevantSources.Add(new Dictionary<string, object>()
+                {
+                    { "File", memory.Item1.Metadata["file_name"] },
+                    { "LastUpdate", memory.Item1.Metadata["last_update"] },
+                    { "Relevance", memory.Item2 },
+                    { "PartitionSize", size },
+                });
+
                 continue;
             }
 
@@ -103,13 +120,13 @@ public async Task<string> SearchAsync(string userId, string query)
         if (factsAvailableCount == 0)
         {
             this._log.LogWarning("No memories available");
-            return "INFO NOT FOUND";
+            return noAnswer;
         }
 
         if (factsAvailableCount > 0 && factsUsedCount == 0)
         {
             this._log.LogError("Unable to inject memories in the prompt, not enough token available");
-            return "INFO NOT FOUND";
+            return noAnswer;
         }
 
         var context = this._kernel.CreateNewContext();
@@ -122,6 +139,8 @@ public async Task<string> SearchAsync(string userId, string query)
         var skFunction = this._kernel.CreateSemanticFunction(prompt.Trim(), maxTokens: AnswerTokens, temperature: 0);
         SKContext result = await skFunction.InvokeAsync(context).ConfigureAwait(false);
 
-        return result.Result;
+        answer.Text = result.Result;
+
+        return answer;
     }
 }

dotnet/Service/Program.cs

@@ -7,6 +7,7 @@
 using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using Microsoft.SemanticMemory.Client;
 using Microsoft.SemanticMemory.Core.AppBuilders;
 using Microsoft.SemanticMemory.Core.Configuration;
 using Microsoft.SemanticMemory.Core.Diagnostics;
@@ -53,7 +54,8 @@
     }
 });
 
-var config = app.Services.GetService<SemanticMemoryConfig>();
+var config = app.Services.GetService<SemanticMemoryConfig>()
+             ?? throw new ConfigurationException("Configuration is null");
 
 // ********************************************************
 // ************** WEB SERVICE ENDPOINTS *******************
@@ -98,7 +100,8 @@
         try
         {
             var id = await orchestrator.UploadFileAsync(input);
-            return Results.Accepted($"/upload-status?id={id}", new { Id = id, Message = "Upload completed, ingestion started" });
+            return Results.Accepted($"/upload-status?user={input.UserId}&id={id}",
+                new { Id = id, UserId = input.UserId, Message = "Upload completed, ingestion started" });
         }
         catch (Exception e)
         {
@@ -113,7 +116,17 @@
         ILogger<Program> log) =>
     {
         log.LogTrace("New search request");
-        return Results.Ok(await searchClient.SearchAsync(request));
+        MemoryAnswer answer = await searchClient.SearchAsync(request);
+        return Results.Ok(answer);
+    });
+
+    // Document status endpoint
+    app.MapGet("/upload-status", async Task<IResult> () =>
+    {
+        // WORK IN PROGRESS
+        await Task.Delay(0);
+
+        return Results.Ok("");
     });
 }
 #pragma warning restore CA1031