diff --git a/docs/getting-started.md b/docs/getting-started.md index 74c20b90..efe28d4f 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -81,7 +81,8 @@ Python environments created by CSnakes are designed to be process-level singleto CSnakes comes with a host builder for the `Microsoft.Extensions.Hosting` library to make it easier to create a Python environment in your C# code. -CSnakes also needs to know where to find Python using one or many [Python Locators](reference.md#python-locators). This example uses the [NuGet locator](reference.md#nuget-locator), which is an easy way to get started on Windows. +CSnakes also needs to know where to find Python using one or many [Python Locators](reference.md#python-locators). +The simplest option is the `FromRedistributable` method, which will download a Python 3.12 redistributable and store it locally. This is compatible with Windows, macOS, and Linux. Here's an example of how you can create a Python environment in C#: @@ -101,7 +102,7 @@ var builder = Host.CreateDefaultBuilder(args) services .WithPython() .WithHome(home) - .FromNuGet("3.12.4"); // Add one or many Python Locators here + .FromRedistributable(); // Download Python 3.12 and store it locally }); var app = builder.Build(); diff --git a/docs/reference.md b/docs/reference.md index 113e7aa5..4471a2fc 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -102,6 +102,10 @@ CSnakes uses a `PythonLocator` to find the Python runtime on the host machine. T You can chain locators together to match use the first one that finds a Python runtime. This is a useful pattern for code that is designed to run on Windows, Linux, and MacOS. +### Redistributable Locator + +The `.FromRedistributable()` method automates the installation of a compatible version of Python. It will source Python 3.12 and cache it locally. This download is about 50-80MB, so the first time you run your application, it will download the redistributable and cache it locally. The next time you run your application, it will use the cached redistributable. This could take a minute or two depending on your bandwidth. + ### Environment Variable Locator The `.FromEnvironmentVariable()` method allows you to specify an environment variable that contains the path to the Python runtime. This is useful for scenarios where the Python runtime is installed in a non-standard location or where the path to the Python runtime is not known at compile time. @@ -289,4 +293,4 @@ The type of `.Send` is the `TSend` type parameter of the `Generator` type annota ```csharp var generator = env.ExampleGenerator(5); string nextValue= generator.Send(10); -``` \ No newline at end of file +``` diff --git a/src/CSnakes.Runtime/CSnakes.Runtime.csproj b/src/CSnakes.Runtime/CSnakes.Runtime.csproj index 6560b2c3..e37297bb 100644 --- a/src/CSnakes.Runtime/CSnakes.Runtime.csproj +++ b/src/CSnakes.Runtime/CSnakes.Runtime.csproj @@ -16,6 +16,7 @@ + diff --git a/src/CSnakes.Runtime/Locators/RedistributableLocator.cs b/src/CSnakes.Runtime/Locators/RedistributableLocator.cs new file mode 100644 index 00000000..3a4d25a4 --- /dev/null +++ b/src/CSnakes.Runtime/Locators/RedistributableLocator.cs @@ -0,0 +1,220 @@ +using System.Formats.Tar; +using System.Runtime.InteropServices; +using Microsoft.Extensions.Logging; +using ZstdSharp; + +namespace CSnakes.Runtime.Locators; + +internal class RedistributableLocator(ILogger logger, int installerTimeout = 360) : PythonLocator +{ + private const string standaloneRelease = "20250106"; + private static readonly Version defaultVersion = new(3, 12, 8, 0); + protected override Version Version { get; } = defaultVersion; + + protected override string GetPythonExecutablePath(string folder, bool freeThreaded = false) + { + string suffix = freeThreaded ? "t" : ""; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return Path.Combine(folder, $"python{suffix}.exe"); + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + return Path.Combine(folder, "bin", $"python{Version.Major}.{Version.Minor}{suffix}"); + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + return Path.Combine(folder, "bin", $"python{Version.Major}.{Version.Minor}{suffix}"); + } + + throw new PlatformNotSupportedException($"Unsupported platform: '{RuntimeInformation.OSDescription}'."); + } + + public override PythonLocationMetadata LocatePython() + { + var downloadPath = Path.Join(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "CSnakes", $"python{Version.Major}.{Version.Minor}.{Version.Build}"); + var installPath = Path.Join(downloadPath, "python", "install"); + var lockfile = Path.Join(downloadPath, "install.lock"); + + // Check if the install path already exists to save waiting + if (Directory.Exists(installPath) && !File.Exists(lockfile)) + { + return LocatePythonInternal(installPath); + } + + + if (File.Exists(Path.Join(downloadPath, "install.lock"))) // Someone else is installing, wait to finish + { + // Wait until it's finished + var loopCount = 0; + while (File.Exists(lockfile)) + { + Thread.Sleep(1000); + loopCount++; + if (loopCount > installerTimeout) + { + throw new TimeoutException("Python installation timed out."); + } + } + return LocatePythonInternal(installPath); + } + + // Create the folder and lock file, the install path is only created at the end. + Directory.CreateDirectory(downloadPath); + File.WriteAllText(lockfile, ""); + try + { + // Determine binary name, see https://gregoryszorc.com/docs/python-build-standalone/main/running.html#obtaining-distributions + string platform; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + platform = RuntimeInformation.ProcessArchitecture switch + { + Architecture.X86 => "i686-pc-windows-msvc-shared-pgo-full", + Architecture.X64 => "x86_64-pc-windows-msvc-shared-pgo-full", + _ => throw new PlatformNotSupportedException($"Unsupported architecture: '{RuntimeInformation.ProcessArchitecture}'.") + }; + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + platform = RuntimeInformation.ProcessArchitecture switch + { + // No such thing as i686 mac + Architecture.X64 => "x86_64-apple-darwin-pgo+lto-full", + Architecture.Arm64 => "aarch64-apple-darwin-pgo+lto-full", + _ => throw new PlatformNotSupportedException($"Unsupported architecture: '{RuntimeInformation.ProcessArchitecture}'.") + }; + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + platform = RuntimeInformation.ProcessArchitecture switch + { + Architecture.X86 => "i686-unknown-linux-gnu-pgo+lto-full", + Architecture.X64 => "x86_64-unknown-linux-gnu-pgo+lto-full", + Architecture.Arm64 => "aarch64-unknown-linux-gnu-pgo+lto-full", + // .NET doesn't run on armv7 anyway.. don't try that + _ => throw new PlatformNotSupportedException($"Unsupported architecture: '{RuntimeInformation.ProcessArchitecture}'.") + }; + } + else + { + throw new PlatformNotSupportedException($"Unsupported platform: '{RuntimeInformation.OSDescription}'."); + } + string downloadUrl = $"https://github.com/astral-sh/python-build-standalone/releases/download/{standaloneRelease}/cpython-{Version.Major}.{Version.Minor}.{Version.Build}+{standaloneRelease}-{platform}.tar.zst"; + + // Download and extract the Zstd tarball + logger.LogInformation("Downloading Python from {DownloadUrl}", downloadUrl); + string tempFilePath = DownloadFileToTempDirectoryAsync(downloadUrl).GetAwaiter().GetResult(); + string tarFilePath = DecompressZstFile(tempFilePath); + ExtractTar(tarFilePath, downloadPath, logger); + logger.LogInformation("Extracted Python to {downloadPath}", downloadPath); + + // Delete the tarball and temp file + File.Delete(tarFilePath); + File.Delete(tempFilePath); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to download and extract Python"); + // If the install failed somewhere, delete the folder incase it's half downloaded + if (Directory.Exists(installPath)) + { + Directory.Delete(installPath, true); + } + + throw; + } + finally + { + // Delete the lock file + File.Delete(lockfile); + } + return LocatePythonInternal(installPath); + } + + protected override string GetLibPythonPath(string folder, bool freeThreaded = false) + { + string suffix = freeThreaded ? "t" : ""; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return Path.Combine(folder, $"python{Version.Major}{Version.Minor}{suffix}.dll"); + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + return Path.Combine(folder, "lib", $"libpython{Version.Major}.{Version.Minor}{suffix}.dylib"); + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + return Path.Combine(folder, "lib", $"libpython{Version.Major}.so"); + } + + throw new PlatformNotSupportedException($"Unsupported platform: '{RuntimeInformation.OSDescription}'."); + } + + private static async Task DownloadFileToTempDirectoryAsync(string fileUrl) + { + using HttpClient client = new(); + using HttpResponseMessage response = await client.GetAsync(fileUrl); + response.EnsureSuccessStatusCode(); + + string tempFilePath = Path.GetTempFileName(); + using FileStream fileStream = new FileStream(tempFilePath, FileMode.Create, FileAccess.Write, FileShare.None); + await response.Content.CopyToAsync(fileStream); + + return tempFilePath; + } + + private static string DecompressZstFile(string zstFilePath) + { + string tarFilePath = Path.ChangeExtension(zstFilePath, ".tar"); + using var inputStream = new FileStream(zstFilePath, FileMode.Open, FileAccess.Read); + using var decompressor = new DecompressionStream(inputStream); + using var outputStream = new FileStream(tarFilePath, FileMode.Create, FileAccess.Write); + decompressor.CopyTo(outputStream); + return tarFilePath; + } + + private static void ExtractTar(string tarFilePath, string extractPath, ILogger logger) + { + using FileStream tarStream = File.OpenRead(tarFilePath); + using TarReader tarReader = new(tarStream); + TarEntry? entry; + List<(string, string)> symlinks = []; + while ((entry = tarReader.GetNextEntry()) is not null) + { + string entryPath = Path.Combine(extractPath, entry.Name); + if (entry.EntryType == TarEntryType.Directory) + { + Directory.CreateDirectory(entryPath); + logger.LogDebug("Creating directory: {EntryPath}", entryPath); + } + else if (entry.EntryType == TarEntryType.RegularFile) + { + Directory.CreateDirectory(Path.GetDirectoryName(entryPath)!); + entry.ExtractToFile(entryPath, true); + } else if (entry.EntryType == TarEntryType.SymbolicLink) { + // Delay the creation of symlinks until after all files have been extracted + symlinks.Add((entryPath, entry.LinkName)); + } else + { + logger.LogDebug("Skipping entry: {EntryPath} ({EntryType})", entryPath, entry.EntryType); + } + } + foreach (var (path, link) in symlinks) + { + logger.LogDebug("Creating symlink: {Path} -> {Link}", path, link); + try + { + File.CreateSymbolicLink(path, link); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to create symlink: {Path} -> {Link}", path, link); + } + } + } +} diff --git a/src/CSnakes.Runtime/ServiceCollectionExtensions.cs b/src/CSnakes.Runtime/ServiceCollectionExtensions.cs index 9858a8c1..2de13513 100644 --- a/src/CSnakes.Runtime/ServiceCollectionExtensions.cs +++ b/src/CSnakes.Runtime/ServiceCollectionExtensions.cs @@ -179,6 +179,18 @@ public static IPythonEnvironmentBuilder FromConda(this IPythonEnvironmentBuilder return builder; } + /// + /// Simplest option for getting started with CSnakes. + /// Downloads and installs the redistributable version of Python from GitHub and stores it in %APP_DATA%/csnakes. + /// + /// The to add the locator to. + /// + public static IPythonEnvironmentBuilder FromRedistributable(this IPythonEnvironmentBuilder builder) + { + builder.Services.AddSingleton(); + return builder; + } + /// /// Adds a pip package installer to the service collection. /// diff --git a/src/CSnakes.sln b/src/CSnakes.sln index db134f82..bce8510e 100644 --- a/src/CSnakes.sln +++ b/src/CSnakes.sln @@ -30,6 +30,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestUtilities", "TestUtilit EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Conda.Tests", "Conda.Tests\Conda.Tests.csproj", "{38604D9B-2C01-4B82-AFA1-A00E184BAE03}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RedistributablePython.Tests", "RedistributablePython.Tests\RedistributablePython.Tests.csproj", "{F43684F2-D7B3-403F-B6E8-1B4740513E2A}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -68,6 +70,10 @@ Global {38604D9B-2C01-4B82-AFA1-A00E184BAE03}.Debug|Any CPU.Build.0 = Debug|Any CPU {38604D9B-2C01-4B82-AFA1-A00E184BAE03}.Release|Any CPU.ActiveCfg = Release|Any CPU {38604D9B-2C01-4B82-AFA1-A00E184BAE03}.Release|Any CPU.Build.0 = Release|Any CPU + {F43684F2-D7B3-403F-B6E8-1B4740513E2A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F43684F2-D7B3-403F-B6E8-1B4740513E2A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F43684F2-D7B3-403F-B6E8-1B4740513E2A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F43684F2-D7B3-403F-B6E8-1B4740513E2A}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -81,6 +87,7 @@ Global {93264FC1-2880-4959-9576-50D260039BC2} = {E52DC71C-FB58-4E57-9CCA-8A78EAA49123} {641C9CD0-8529-4666-8F27-ECEB7F72043C} = {E52DC71C-FB58-4E57-9CCA-8A78EAA49123} {38604D9B-2C01-4B82-AFA1-A00E184BAE03} = {E52DC71C-FB58-4E57-9CCA-8A78EAA49123} + {F43684F2-D7B3-403F-B6E8-1B4740513E2A} = {E52DC71C-FB58-4E57-9CCA-8A78EAA49123} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {4ACC77F9-1BB8-42DE-B647-01C458922F49} diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index 17cae342..458f2469 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -2,13 +2,11 @@ true - 3.12.4 $(PYTHON_VERSION.Replace('alpha.','a').Replace('beta.','b').Replace('rc.','rc')) - @@ -17,21 +15,20 @@ + + - - - @@ -49,4 +46,4 @@ - + \ No newline at end of file diff --git a/src/RedistributablePython.Tests/BasicTests.cs b/src/RedistributablePython.Tests/BasicTests.cs new file mode 100644 index 00000000..906b0fe0 --- /dev/null +++ b/src/RedistributablePython.Tests/BasicTests.cs @@ -0,0 +1,12 @@ +namespace RedistributablePython.Tests; + +public class BasicTests : RedistributablePythonTestBase +{ + [Fact] + public void TestSimpleImport() + { + var testModule = Env.TestSimple(); + Assert.NotNull(testModule); + testModule.TestNothing(); + } +} diff --git a/src/RedistributablePython.Tests/RedistributablePython.Tests.csproj b/src/RedistributablePython.Tests/RedistributablePython.Tests.csproj new file mode 100644 index 00000000..b48e8744 --- /dev/null +++ b/src/RedistributablePython.Tests/RedistributablePython.Tests.csproj @@ -0,0 +1,39 @@ + + + + + false + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + Always + + + Always + + + \ No newline at end of file diff --git a/src/RedistributablePython.Tests/RedistributablePythonTestBase.cs b/src/RedistributablePython.Tests/RedistributablePythonTestBase.cs new file mode 100644 index 00000000..aa7bfa4c --- /dev/null +++ b/src/RedistributablePython.Tests/RedistributablePythonTestBase.cs @@ -0,0 +1,38 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace RedistributablePython.Tests; +public class RedistributablePythonTestBase : IDisposable +{ + private readonly IPythonEnvironment env; + private readonly IHost app; + + public RedistributablePythonTestBase() + { + string venvPath = Path.Join(Environment.CurrentDirectory, "python", ".venv"); + app = Host.CreateDefaultBuilder() + .ConfigureServices((context, services) => + { + var pb = services.WithPython(); + pb.WithHome(Path.Join(Environment.CurrentDirectory, "python")); + + pb.FromRedistributable() + .WithPipInstaller() + .WithVirtualEnvironment(venvPath); + + services.AddLogging(builder => builder.AddXUnit()); + }) + .Build(); + + env = app.Services.GetRequiredService(); + } + + public void Dispose() + { + GC.SuppressFinalize(this); + GC.Collect(); + } + + public IPythonEnvironment Env => env; +} diff --git a/src/RedistributablePython.Tests/python/requirements.txt b/src/RedistributablePython.Tests/python/requirements.txt new file mode 100644 index 00000000..cc634450 --- /dev/null +++ b/src/RedistributablePython.Tests/python/requirements.txt @@ -0,0 +1,2 @@ +httpx +numpy diff --git a/src/RedistributablePython.Tests/python/test_simple.py b/src/RedistributablePython.Tests/python/test_simple.py new file mode 100644 index 00000000..fa8548e3 --- /dev/null +++ b/src/RedistributablePython.Tests/python/test_simple.py @@ -0,0 +1,6 @@ +import httpx +import numpy as np + + +def test_nothing() -> None: + pass