Refactoring + Examples (#2)

SemanticMemory.sln

@@ -1,21 +1,17 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
 
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Handlers", "handlers-dotnet\Handlers.csproj", "{9A23325F-FCF5-4BF7-9413-D325A779AE42}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PipelineService", "handlers-dotnet\PipelineService.csproj", "{9A23325F-FCF5-4BF7-9413-D325A779AE42}"
 EndProject
 
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "lib", "lib", "{8E36E21B-00CC-4B85-98FB-6CB2CA5F978C}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Configuration", "lib-dotnet\Configuration\Configuration.csproj", "{DB5CA047-1184-4878-B1FC-0C17C9FAE7AA}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Core", "lib-dotnet\Core\Core.csproj", "{DB5CA047-1184-4878-B1FC-0C17C9FAE7AA}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataFormats.Office", "lib-dotnet\DataFormats.Office\DataFormats.Office.csproj", "{174E9AA0-E379-4B5C-8125-54CD4EE57ADF}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataFormats.Pdf", "lib-dotnet\DataFormats.Pdf\DataFormats.Pdf.csproj", "{E383AB2F-CFCD-4EAF-A612-85E7FC85667F}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Diagnostics", "lib-dotnet\Diagnostics\Diagnostics.csproj", "{60130AA3-B980-4DF6-8BFF-6F0CF09AD629}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Storage", "lib-dotnet\Storage\Storage.csproj", "{E7906A61-09F5-44D4-ACDC-1486E72B4775}"
-EndProject
 
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "samples", "samples", "{6335B02C-9964-4B39-9795-C5F5F0392515}"
 	ProjectSection(SolutionItems) = preProject
@@ -54,8 +50,6 @@ Global
 		Release|Any CPU = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
-		{E7906A61-09F5-44D4-ACDC-1486E72B4775} = {8E36E21B-00CC-4B85-98FB-6CB2CA5F978C}
-		{60130AA3-B980-4DF6-8BFF-6F0CF09AD629} = {8E36E21B-00CC-4B85-98FB-6CB2CA5F978C}
 		{DB5CA047-1184-4878-B1FC-0C17C9FAE7AA} = {8E36E21B-00CC-4B85-98FB-6CB2CA5F978C}
 		{ACD9F6E9-0B7E-4B69-9468-28C42657AB10} = {6335B02C-9964-4B39-9795-C5F5F0392515}
 		{174E9AA0-E379-4B5C-8125-54CD4EE57ADF} = {8E36E21B-00CC-4B85-98FB-6CB2CA5F978C}
@@ -64,14 +58,6 @@ Global
 		{15CEB46F-AE80-4D0F-85E7-1F666078874D} = {6335B02C-9964-4B39-9795-C5F5F0392515}
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{E7906A61-09F5-44D4-ACDC-1486E72B4775}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{E7906A61-09F5-44D4-ACDC-1486E72B4775}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{E7906A61-09F5-44D4-ACDC-1486E72B4775}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{E7906A61-09F5-44D4-ACDC-1486E72B4775}.Release|Any CPU.Build.0 = Release|Any CPU
-		{60130AA3-B980-4DF6-8BFF-6F0CF09AD629}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{60130AA3-B980-4DF6-8BFF-6F0CF09AD629}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{60130AA3-B980-4DF6-8BFF-6F0CF09AD629}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{60130AA3-B980-4DF6-8BFF-6F0CF09AD629}.Release|Any CPU.Build.0 = Release|Any CPU
 		{DB5CA047-1184-4878-B1FC-0C17C9FAE7AA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{DB5CA047-1184-4878-B1FC-0C17C9FAE7AA}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{DB5CA047-1184-4878-B1FC-0C17C9FAE7AA}.Release|Any CPU.ActiveCfg = Release|Any CPU

handlers-dotnet/Handlers.csproj → handlers-dotnet/PipelineService.csproj

@@ -7,8 +7,8 @@
         <Nullable>enable</Nullable>
         <ImplicitUsings>disable</ImplicitUsings>
         <IsPackable>false</IsPackable>
-        <AssemblyName>Microsoft.SemanticKernel.Services.SemanticMemory.Handlers</AssemblyName>
-        <RootNamespace>Microsoft.SemanticKernel.Services.SemanticMemory.Handlers</RootNamespace>
+        <AssemblyName>Microsoft.SemanticKernel.Services.SemanticMemory.PipelineService</AssemblyName>
+        <RootNamespace>Microsoft.SemanticKernel.Services.SemanticMemory.PipelineService</RootNamespace>
     </PropertyGroup>
 
     <PropertyGroup>
@@ -20,10 +20,7 @@
     </ItemGroup>
 
     <ItemGroup>
-        <ProjectReference Include="..\lib-dotnet\Configuration\Configuration.csproj" />
-        <ProjectReference Include="..\lib-dotnet\DataFormats.Office\DataFormats.Office.csproj" />
-        <ProjectReference Include="..\lib-dotnet\DataFormats.Pdf\DataFormats.Pdf.csproj" />
-        <ProjectReference Include="..\lib-dotnet\Storage\Storage.csproj" />
+        <ProjectReference Include="..\lib-dotnet\Core\Core.csproj" />
     </ItemGroup>
 
     <ItemGroup>

handlers-dotnet/Program.cs

@@ -1,12 +1,14 @@
 // Copyright (c) Microsoft. All rights reserved.
 
 using Microsoft.Extensions.Hosting;
-using Microsoft.SemanticKernel.Services.Configuration;
-using Microsoft.SemanticKernel.Services.SemanticMemory.Handlers;
+using Microsoft.SemanticKernel.SemanticMemory.Core;
+using Microsoft.SemanticKernel.SemanticMemory.Core.Handlers;
+using Microsoft.SemanticKernel.Services.SemanticMemory.PipelineService;
 
 var builder = HostedHandlersBuilder.CreateApplicationBuilder();
+builder.Services.UseDefaultHandler<TextExtractionHandler>("extract");
 
-builder.AddHandler<TextExtractionHandler>("extract");
+builder.AddHandler<TextExtraction>("extract");
 // builder.AddHandler<TextPartitioningHandler>("partition"); // work in progress
 // builder.AddHandler<IndexingHandler>("index"); // work in progress
 

handlers-dotnet/TextExtraction.cs

@@ -0,0 +1,46 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Hosting;
+using Microsoft.SemanticKernel.SemanticMemory.Core.Handlers;
+using Microsoft.SemanticKernel.SemanticMemory.Core.Pipeline;
+
+namespace Microsoft.SemanticKernel.Services.SemanticMemory.PipelineService;
+
+public class TextExtraction : IHostedService, IPipelineStepHandler
+{
+    private readonly IPipelineOrchestrator _orchestrator;
+    private readonly TextExtractionHandler _handler;
+
+    public TextExtraction(
+        string stepName,
+        IPipelineOrchestrator orchestrator,
+        TextExtractionHandler handler)
+    {
+        this.StepName = stepName;
+        this._orchestrator = orchestrator;
+        this._handler = handler;
+    }
+
+    /// <inheritdoc />
+    public string StepName { get; }
+
+    /// <inheritdoc />
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        return this._orchestrator.AddHandlerAsync(this, cancellationToken);
+    }
+
+    /// <inheritdoc />
+    public Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync(DataPipeline pipeline, CancellationToken cancellationToken)
+    {
+        return this._handler.InvokeAsync(pipeline, cancellationToken);
+    }
+
+    /// <inheritdoc />
+    public Task StopAsync(CancellationToken cancellationToken)
+    {
+        return this._orchestrator.StopAllPipelinesAsync();
+    }
+}

handlers-dotnet/run.cmd

@@ -0,0 +1,5 @@
+@echo off
+
+dotnet restore
+dotnet build
+dotnet run

handlers-dotnet/run.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -e
+
+cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/"
+
+dotnet restore
+dotnet build
+dotnet run

lib-dotnet/Configuration/AppBuilder.cs → lib-dotnet/Core/Configuration/AppBuilder.cs

@@ -4,7 +4,7 @@
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Hosting;
 
-namespace Microsoft.SemanticKernel.Services.Configuration;
+namespace Microsoft.SemanticKernel.SemanticMemory.Core.Configuration;
 
 public static class AppBuilder
 {

lib-dotnet/Configuration/ContentStorageConfig.cs → lib-dotnet/Core/Configuration/ContentStorageConfig.cs

@@ -1,8 +1,8 @@
 // Copyright (c) Microsoft. All rights reserved.
 
-using Microsoft.SemanticKernel.Services.Storage.ContentStorage;
+using Microsoft.SemanticKernel.SemanticMemory.Core.ContentStorage;
 
-namespace Microsoft.SemanticKernel.Services.Configuration;
+namespace Microsoft.SemanticKernel.SemanticMemory.Core.Configuration;
 
 public class ContentStorageConfig
 {

lib-dotnet/Configuration/OrchestrationConfig.cs → lib-dotnet/Core/Configuration/OrchestrationConfig.cs

@@ -1,8 +1,8 @@
 // Copyright (c) Microsoft. All rights reserved.
 
-using Microsoft.SemanticKernel.Services.Storage.Pipeline;
+using Microsoft.SemanticKernel.SemanticMemory.Core.Pipeline;
 
-namespace Microsoft.SemanticKernel.Services.Configuration;
+namespace Microsoft.SemanticKernel.SemanticMemory.Core.Configuration;
 
 public class OrchestrationConfig
 {

lib-dotnet/Configuration/SKMemoryConfig.cs → lib-dotnet/Core/Configuration/SKMemoryConfig.cs

@@ -1,6 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 
-namespace Microsoft.SemanticKernel.Services.Configuration;
+namespace Microsoft.SemanticKernel.SemanticMemory.Core.Configuration;
 
 public class SKMemoryConfig
 {

lib-dotnet/Storage/ContentStorage/AzureBlob.cs → lib-dotnet/Core/ContentStorage/AzureBlob.cs

@@ -12,9 +12,9 @@
 using Azure.Storage.Blobs.Specialized;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
-using Microsoft.SemanticKernel.Services.Diagnostics;
+using Microsoft.SemanticKernel.SemanticMemory.Core.Diagnostics;
 
-namespace Microsoft.SemanticKernel.Services.Storage.ContentStorage;
+namespace Microsoft.SemanticKernel.SemanticMemory.Core.ContentStorage;
 
 public class AzureBlob : IContentStorage
 {

lib-dotnet/Storage/ContentStorage/AzureBlobConfig.cs → lib-dotnet/Core/ContentStorage/AzureBlobConfig.cs

@@ -1,6 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 
-namespace Microsoft.SemanticKernel.Services.Storage.ContentStorage;
+namespace Microsoft.SemanticKernel.SemanticMemory.Core.ContentStorage;
 
 public class AzureBlobConfig
 {

lib-dotnet/Storage/ContentStorage/FileSystem.cs → lib-dotnet/Core/ContentStorage/FileSystem.cs

@@ -6,9 +6,9 @@
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
-using Microsoft.SemanticKernel.Services.Diagnostics;
+using Microsoft.SemanticKernel.SemanticMemory.Core.Diagnostics;
 
-namespace Microsoft.SemanticKernel.Services.Storage.ContentStorage;
+namespace Microsoft.SemanticKernel.SemanticMemory.Core.ContentStorage;
 
 public class FileSystem : IContentStorage
 {

lib-dotnet/Storage/ContentStorage/FileSystemConfig.cs → lib-dotnet/Core/ContentStorage/FileSystemConfig.cs

@@ -1,6 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 
-namespace Microsoft.SemanticKernel.Services.Storage.ContentStorage;
+namespace Microsoft.SemanticKernel.SemanticMemory.Core.ContentStorage;
 
 public class FileSystemConfig
 {

lib-dotnet/Storage/ContentStorage/IContentStorage.cs → lib-dotnet/Core/ContentStorage/IContentStorage.cs

@@ -5,7 +5,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 
-namespace Microsoft.SemanticKernel.Services.Storage.ContentStorage;
+namespace Microsoft.SemanticKernel.SemanticMemory.Core.ContentStorage;
 
 public interface IContentStorage
 {

lib-dotnet/Storage/Storage.csproj → lib-dotnet/Core/Core.csproj

@@ -6,24 +6,24 @@
         <ImplicitUsings>disable</ImplicitUsings>
         <Nullable>enable</Nullable>
         <IsPackable>false</IsPackable>
-        <AssemblyName>Microsoft.SemanticKernel.Services.Storage</AssemblyName>
-        <RootNamespace>Microsoft.SemanticKernel.Services.Storage</RootNamespace>
+        <AssemblyName>Microsoft.SemanticKernel.SemanticMemory.Core</AssemblyName>
+        <RootNamespace>Microsoft.SemanticKernel.SemanticMemory.Core</RootNamespace>
     </PropertyGroup>
 
     <ItemGroup>
+        <PackageReference Include="Microsoft.AspNetCore.Http" Version="2.2.2" />
         <PackageReference Include="Azure.Identity" Version="1.9.0" />
         <PackageReference Include="Azure.Storage.Blobs" Version="12.16.0" />
         <PackageReference Include="Azure.Storage.Queues" Version="12.14.0" />
-        <PackageReference Include="Microsoft.AspNetCore.Http" Version="2.2.2" />
-        <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="7.0.4" />
-        <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="7.0.0" />
+        <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
         <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="7.0.0" />
         <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.1" />
         <PackageReference Include="RabbitMQ.Client" Version="6.5.0" />
     </ItemGroup>
 
     <ItemGroup>
-        <ProjectReference Include="..\Diagnostics\Diagnostics.csproj" />
+      <ProjectReference Include="..\DataFormats.Office\DataFormats.Office.csproj" />
+      <ProjectReference Include="..\DataFormats.Pdf\DataFormats.Pdf.csproj" />
     </ItemGroup>
 
 </Project>

lib-dotnet/Configuration/DependencyInjection.cs → lib-dotnet/Core/DependencyInjection.cs

@@ -4,11 +4,12 @@
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
-using Microsoft.SemanticKernel.Services.Storage.ContentStorage;
-using Microsoft.SemanticKernel.Services.Storage.Pipeline;
-using Microsoft.SemanticKernel.Services.Storage.Queue;
+using Microsoft.SemanticKernel.SemanticMemory.Core.Configuration;
+using Microsoft.SemanticKernel.SemanticMemory.Core.ContentStorage;
+using Microsoft.SemanticKernel.SemanticMemory.Core.Pipeline;
+using Microsoft.SemanticKernel.SemanticMemory.Core.Pipeline.Queue;
 
-namespace Microsoft.SemanticKernel.Services.Configuration;
+namespace Microsoft.SemanticKernel.SemanticMemory.Core;
 
 public static class DependencyInjection
 {
@@ -167,4 +168,9 @@ public static void UseDistributedPipeline(this IServiceCollection services, SKMe
                 throw new NotImplementedException($"Queue type '{config.Orchestration.DistributedPipeline.Type}' not available");
         }
     }
+
+    public static void UseDefaultHandler<THandler>(this IServiceCollection services, string stepName) where THandler : class
+    {
+        services.AddTransient<THandler>(serviceProvider => ActivatorUtilities.CreateInstance<THandler>(serviceProvider, stepName));
+    }
 }

lib-dotnet/Diagnostics/ContentStorageException.cs → lib-dotnet/Core/Diagnostics/ContentStorageException.cs

@@ -2,7 +2,7 @@
 
 using System;
 
-namespace Microsoft.SemanticKernel.Services.Diagnostics;
+namespace Microsoft.SemanticKernel.SemanticMemory.Core.Diagnostics;
 
 public class ContentStorageException : MemoryException
 {

lib-dotnet/Diagnostics/LoggerExtensions.cs → lib-dotnet/Core/Diagnostics/LoggerExtensions.cs

@@ -2,7 +2,7 @@
 
 using Microsoft.Extensions.Logging;
 
-namespace Microsoft.SemanticKernel.Services.Diagnostics;
+namespace Microsoft.SemanticKernel.SemanticMemory.Core.Diagnostics;
 
 public static class LoggerExtensions
 {

lib-dotnet/Diagnostics/MemoryException.cs → lib-dotnet/Core/Diagnostics/MemoryException.cs

@@ -2,7 +2,7 @@
 
 using System;
 
-namespace Microsoft.SemanticKernel.Services.Diagnostics;
+namespace Microsoft.SemanticKernel.SemanticMemory.Core.Diagnostics;
 
 /// <summary>
 /// Provides the base exception from which all Semantic Kernel exceptions derive.

lib-dotnet/Diagnostics/OrchestrationException.cs → lib-dotnet/Core/Diagnostics/OrchestrationException.cs

@@ -2,7 +2,7 @@
 
 using System;
 
-namespace Microsoft.SemanticKernel.Services.Diagnostics;
+namespace Microsoft.SemanticKernel.SemanticMemory.Core.Diagnostics;
 
 public class OrchestrationException : MemoryException
 {

lib-dotnet/Diagnostics/PipelineCompletedException.cs → lib-dotnet/Core/Diagnostics/PipelineCompletedException.cs

@@ -2,7 +2,7 @@
 
 using System;
 
-namespace Microsoft.SemanticKernel.Services.Diagnostics;
+namespace Microsoft.SemanticKernel.SemanticMemory.Core.Diagnostics;
 
 public class PipelineCompletedException : MemoryException
 {