From 8378d98ac033b6dcc5d9327d2ab490d1da85b1cb Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 14 Nov 2024 17:04:24 +1100 Subject: [PATCH 01/13] Add async enumerator --- src/CSnakes.Runtime/Python/GeneratorIterator.cs | 15 +++++++++++++++ src/CSnakes.Runtime/Python/IGeneratorIterator.cs | 2 +- src/CSnakes.Runtime/Python/PyObject.cs | 5 +++++ src/Integration.Tests/GeneratorTests.cs | 4 ++-- 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/CSnakes.Runtime/Python/GeneratorIterator.cs b/src/CSnakes.Runtime/Python/GeneratorIterator.cs index 5fe10f63..4f743ba2 100644 --- a/src/CSnakes.Runtime/Python/GeneratorIterator.cs +++ b/src/CSnakes.Runtime/Python/GeneratorIterator.cs @@ -68,4 +68,19 @@ private bool Send(PyObject value) } IEnumerator IEnumerable.GetEnumerator() => this; + + #region Async + public ValueTask DisposeAsync() + { + Dispose(); + return default; + } + + public async ValueTask MoveNextAsync() + { + using var result = await sendPyFunction.CallAsync(PyObject.None); + current = result.As(); + return true; + } + #endregion } diff --git a/src/CSnakes.Runtime/Python/IGeneratorIterator.cs b/src/CSnakes.Runtime/Python/IGeneratorIterator.cs index c474dcb7..46de916b 100644 --- a/src/CSnakes.Runtime/Python/IGeneratorIterator.cs +++ b/src/CSnakes.Runtime/Python/IGeneratorIterator.cs @@ -2,7 +2,7 @@ namespace CSnakes.Runtime.Python; -public interface IGeneratorIterator: IEnumerator, IEnumerable, IGeneratorIterator +public interface IGeneratorIterator: IEnumerator, IEnumerable, IGeneratorIterator, IAsyncEnumerator { /// /// The value is undefined until either or diff --git a/src/CSnakes.Runtime/Python/PyObject.cs b/src/CSnakes.Runtime/Python/PyObject.cs index d71581ca..58bde05b 100644 --- a/src/CSnakes.Runtime/Python/PyObject.cs +++ b/src/CSnakes.Runtime/Python/PyObject.cs @@ -317,6 +317,11 @@ public PyObject Call(params PyObject[] args) return CallWithArgs(args); } + internal async Task CallAsync(params PyObject[] args) + { + return await Task.Run(() => CallWithArgs(args)); + } + public PyObject CallWithArgs(PyObject[]? args = null) { RaiseOnPythonNotInitialized(); diff --git a/src/Integration.Tests/GeneratorTests.cs b/src/Integration.Tests/GeneratorTests.cs index 46ae268e..4953571a 100644 --- a/src/Integration.Tests/GeneratorTests.cs +++ b/src/Integration.Tests/GeneratorTests.cs @@ -8,9 +8,9 @@ public void TestGenerator() var generator = mod.ExampleGenerator(3); Assert.True(generator.MoveNext()); - Assert.Equal("Item 0", generator.Current); + Assert.Equal("Item 0", ((IEnumerator)generator).Current); Assert.True(generator.Send(10)); - Assert.Equal("Received 10", generator.Current); + Assert.Equal("Received 10", ((IEnumerator)generator).Current); Assert.Equal(["Item 1", "Item 2"], generator.ToArray()); Assert.True(generator.Return); } From 9cf18b409de67f2a61bfa33333bbb2b2391ca770 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 15 Nov 2024 08:06:27 +1100 Subject: [PATCH 02/13] Add a test, move the task up to the generator iterator --- src/CSnakes.Runtime/Python/GeneratorIterator.cs | 6 +++--- src/CSnakes.Runtime/Python/IGeneratorIterator.cs | 5 ++++- src/CSnakes.Runtime/Python/PyObject.cs | 5 ----- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/CSnakes.Runtime/Python/GeneratorIterator.cs b/src/CSnakes.Runtime/Python/GeneratorIterator.cs index 4f743ba2..35aa57bf 100644 --- a/src/CSnakes.Runtime/Python/GeneratorIterator.cs +++ b/src/CSnakes.Runtime/Python/GeneratorIterator.cs @@ -78,9 +78,9 @@ public ValueTask DisposeAsync() public async ValueTask MoveNextAsync() { - using var result = await sendPyFunction.CallAsync(PyObject.None); - current = result.As(); - return true; + return await Task.Run(() => Send(PyObject.None)); } + + public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) => this; #endregion } diff --git a/src/CSnakes.Runtime/Python/IGeneratorIterator.cs b/src/CSnakes.Runtime/Python/IGeneratorIterator.cs index 46de916b..3c570dcc 100644 --- a/src/CSnakes.Runtime/Python/IGeneratorIterator.cs +++ b/src/CSnakes.Runtime/Python/IGeneratorIterator.cs @@ -2,7 +2,10 @@ namespace CSnakes.Runtime.Python; -public interface IGeneratorIterator: IEnumerator, IEnumerable, IGeneratorIterator, IAsyncEnumerator +public interface IGeneratorIterator: + IEnumerator, IEnumerable, + IGeneratorIterator, + IAsyncEnumerator, IAsyncEnumerable { /// /// The value is undefined until either or diff --git a/src/CSnakes.Runtime/Python/PyObject.cs b/src/CSnakes.Runtime/Python/PyObject.cs index 58bde05b..d71581ca 100644 --- a/src/CSnakes.Runtime/Python/PyObject.cs +++ b/src/CSnakes.Runtime/Python/PyObject.cs @@ -317,11 +317,6 @@ public PyObject Call(params PyObject[] args) return CallWithArgs(args); } - internal async Task CallAsync(params PyObject[] args) - { - return await Task.Run(() => CallWithArgs(args)); - } - public PyObject CallWithArgs(PyObject[]? args = null) { RaiseOnPythonNotInitialized(); From a5af133b7d78a734f3ebf8f370ed3e9b78fbbed9 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 15 Nov 2024 08:06:42 +1100 Subject: [PATCH 03/13] Add integration test --- src/Integration.Tests/GeneratorTests.cs | 11 +++++++++++ src/Integration.Tests/python/test_generators.py | 7 +++++++ 2 files changed, 18 insertions(+) diff --git a/src/Integration.Tests/GeneratorTests.cs b/src/Integration.Tests/GeneratorTests.cs index 4953571a..1e9fd30a 100644 --- a/src/Integration.Tests/GeneratorTests.cs +++ b/src/Integration.Tests/GeneratorTests.cs @@ -23,4 +23,15 @@ public void TestNormalGenerator() var generator = mod.TestNormalGenerator(); Assert.Equal(["one", "two"], generator.ToArray()); } + + [Fact] + public async Task TestAsyncEnumerator() + { + var mod = Env.TestGenerators(); + List words = [ "foo", "bar", "baz"]; + await foreach (string word in mod.TestGeneratorSequence()) + { + Assert.Contains(word, words); + } + } } diff --git a/src/Integration.Tests/python/test_generators.py b/src/Integration.Tests/python/test_generators.py index dd94ed66..cbf1edd9 100644 --- a/src/Integration.Tests/python/test_generators.py +++ b/src/Integration.Tests/python/test_generators.py @@ -9,6 +9,13 @@ def example_generator(length: int) -> Generator[str, int, bool]: return True + def test_normal_generator() -> Generator[str, None, None]: yield "one" yield "two" + + +def test_generator_sequence() -> Generator[str, None, None]: + words = ["foo", "bar", "baz"] + for word in words: + yield word From 1ab2bb7a0173bbca05c245aac57fef66f7c92f48 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 15 Nov 2024 16:54:47 +1100 Subject: [PATCH 04/13] Basic coroutine bridge between the Python event loop and the C# Task AP --- src/CSnakes.Runtime/CPython/Coroutine.cs | 25 ++++++ .../PyObjectTypeConverter.Coroutine.cs | 31 ++++++++ src/CSnakes.Runtime/Python/Coroutine.cs | 46 +++++++++++ .../Python/GeneratorIterator.cs | 15 ---- src/CSnakes.Runtime/Python/ICoroutine.cs | 8 ++ .../Python/IGeneratorIterator.cs | 3 +- src/CSnakes.Runtime/Python/Import.cs | 12 ++- .../Python/PyObject.Imortals.cs | 1 - src/CSnakes.Runtime/Python/PyObject.cs | 1 + .../Parser/PythonParser.Function.cs | 5 +- .../Parser/Types/PythonFunctionDefinition.cs | 7 +- .../PythonStaticGenerator.cs | 2 +- .../Reflection/MethodReflection.cs | 77 ++++++++++++++++--- .../Reflection/TypeReflection.cs | 8 +- src/Integration.Tests/CoroutineTests.cs | 11 +++ src/Integration.Tests/GeneratorTests.cs | 11 --- .../python/test_coroutines.py | 7 ++ 17 files changed, 217 insertions(+), 53 deletions(-) create mode 100644 src/CSnakes.Runtime/CPython/Coroutine.cs create mode 100644 src/CSnakes.Runtime/PyObjectTypeConverter.Coroutine.cs create mode 100644 src/CSnakes.Runtime/Python/Coroutine.cs create mode 100644 src/CSnakes.Runtime/Python/ICoroutine.cs create mode 100644 src/Integration.Tests/CoroutineTests.cs create mode 100644 src/Integration.Tests/python/test_coroutines.py diff --git a/src/CSnakes.Runtime/CPython/Coroutine.cs b/src/CSnakes.Runtime/CPython/Coroutine.cs new file mode 100644 index 00000000..9d2f964a --- /dev/null +++ b/src/CSnakes.Runtime/CPython/Coroutine.cs @@ -0,0 +1,25 @@ +using CSnakes.Runtime.Python; + +namespace CSnakes.Runtime.CPython; +internal unsafe partial class CPythonAPI +{ + internal static bool IsPyCoroutine(PyObject p) + { + return HasAttr(p, "__await__"); + } + + private static PyObject? currentEventLoop = null; + private static PyObject? asyncioModule = null; + + internal static PyObject GetAsyncioModule() + { + asyncioModule ??= Import("asyncio"); + return asyncioModule; + } + + internal static PyObject GetCurrentEventLoop() + { + currentEventLoop ??= GetAsyncioModule().GetAttr("new_event_loop").Call(); + return currentEventLoop; + } +} diff --git a/src/CSnakes.Runtime/PyObjectTypeConverter.Coroutine.cs b/src/CSnakes.Runtime/PyObjectTypeConverter.Coroutine.cs new file mode 100644 index 00000000..bf43d17d --- /dev/null +++ b/src/CSnakes.Runtime/PyObjectTypeConverter.Coroutine.cs @@ -0,0 +1,31 @@ +using CSnakes.Runtime.CPython; +using CSnakes.Runtime.Python; +using System.Diagnostics; +using System.Reflection; + +namespace CSnakes.Runtime; +internal partial class PyObjectTypeConverter +{ + internal static object ConvertToCoroutine(PyObject pyObject, Type destinationType) + { + Debug.Assert(destinationType.IsGenericType); + Debug.Assert(!destinationType.IsGenericTypeDefinition); + Debug.Assert(destinationType.GetGenericTypeDefinition() == typeof(ICoroutine<,,>)); + + if (!CPythonAPI.IsPyCoroutine(pyObject)) + { + throw new InvalidCastException($"Cannot convert {pyObject.GetPythonType()} to a coroutine."); + } + + if (!knownDynamicTypes.TryGetValue(destinationType, out DynamicTypeInfo? typeInfo)) + { + var typeArgs = destinationType.GetGenericArguments(); + Type coroutineType = typeof(Coroutine<,,>).MakeGenericType(typeArgs); + ConstructorInfo ctor = coroutineType.GetConstructors().First(); + typeInfo = new(ctor); + knownDynamicTypes[destinationType] = typeInfo; + } + + return typeInfo.ReturnTypeConstructor.Invoke([pyObject.Clone()]); + } +} diff --git a/src/CSnakes.Runtime/Python/Coroutine.cs b/src/CSnakes.Runtime/Python/Coroutine.cs new file mode 100644 index 00000000..8137a55a --- /dev/null +++ b/src/CSnakes.Runtime/Python/Coroutine.cs @@ -0,0 +1,46 @@ +using CSnakes.Runtime.CPython; + +namespace CSnakes.Runtime.Python; + +public class Coroutine(PyObject coroutine) : ICoroutine +{ + private TYield current = default!; + private TReturn @return = default!; + + public TYield Current => current; + public TReturn Return => @return; + + public Task AsTask() + { + return Task.Run( + () => + { + try + { + using (GIL.Acquire()) + { + var loop = CPythonAPI.GetCurrentEventLoop(); + using PyObject task = loop.GetAttr("create_task").Call(coroutine); + using PyObject result = loop.GetAttr("run_until_complete").Call(task); + current = result.As(); + } + return current; + } + catch (PythonInvocationException ex) + { + if (ex.InnerException is PythonStopIterationException stopIteration) + { + using var @return = stopIteration.TakeValue(); + this.@return = @return.As(); + + // Coroutine has finished + // TODO: define behavior for this case + return default(TYield); + } + + throw; + } + } + ); + } +} diff --git a/src/CSnakes.Runtime/Python/GeneratorIterator.cs b/src/CSnakes.Runtime/Python/GeneratorIterator.cs index 35aa57bf..5fe10f63 100644 --- a/src/CSnakes.Runtime/Python/GeneratorIterator.cs +++ b/src/CSnakes.Runtime/Python/GeneratorIterator.cs @@ -68,19 +68,4 @@ private bool Send(PyObject value) } IEnumerator IEnumerable.GetEnumerator() => this; - - #region Async - public ValueTask DisposeAsync() - { - Dispose(); - return default; - } - - public async ValueTask MoveNextAsync() - { - return await Task.Run(() => Send(PyObject.None)); - } - - public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) => this; - #endregion } diff --git a/src/CSnakes.Runtime/Python/ICoroutine.cs b/src/CSnakes.Runtime/Python/ICoroutine.cs new file mode 100644 index 00000000..3e523051 --- /dev/null +++ b/src/CSnakes.Runtime/Python/ICoroutine.cs @@ -0,0 +1,8 @@ +namespace CSnakes.Runtime.Python; + +public interface ICoroutine : ICoroutine +{ + public Task AsTask(); +} + +public interface ICoroutine { } diff --git a/src/CSnakes.Runtime/Python/IGeneratorIterator.cs b/src/CSnakes.Runtime/Python/IGeneratorIterator.cs index 3c570dcc..624ac08e 100644 --- a/src/CSnakes.Runtime/Python/IGeneratorIterator.cs +++ b/src/CSnakes.Runtime/Python/IGeneratorIterator.cs @@ -4,8 +4,7 @@ namespace CSnakes.Runtime.Python; public interface IGeneratorIterator: IEnumerator, IEnumerable, - IGeneratorIterator, - IAsyncEnumerator, IAsyncEnumerable + IGeneratorIterator { /// /// The value is undefined until either or diff --git a/src/CSnakes.Runtime/Python/Import.cs b/src/CSnakes.Runtime/Python/Import.cs index 8731fb1f..ae2e9332 100644 --- a/src/CSnakes.Runtime/Python/Import.cs +++ b/src/CSnakes.Runtime/Python/Import.cs @@ -6,13 +6,17 @@ public static class Import { public static PyObject ImportModule(string module) { - return CPythonAPI.Import(module); + using (GIL.Acquire()) + return CPythonAPI.Import(module); } public static void ReloadModule(ref PyObject module) { - var newModule = CPythonAPI.ReloadModule(module); - module.Dispose(); - module = newModule; + using (GIL.Acquire()) + { + var newModule = CPythonAPI.ReloadModule(module); + module.Dispose(); + module = newModule; + } } } diff --git a/src/CSnakes.Runtime/Python/PyObject.Imortals.cs b/src/CSnakes.Runtime/Python/PyObject.Imortals.cs index 40c5de0d..88ac620a 100644 --- a/src/CSnakes.Runtime/Python/PyObject.Imortals.cs +++ b/src/CSnakes.Runtime/Python/PyObject.Imortals.cs @@ -9,5 +9,4 @@ public partial class PyObject public static PyObject One { get; } = new PyOneObject(); public static PyObject Zero { get; } = new PyZeroObject(); public static PyObject NegativeOne { get; } = new PyNegativeOneObject(); - } diff --git a/src/CSnakes.Runtime/Python/PyObject.cs b/src/CSnakes.Runtime/Python/PyObject.cs index d71581ca..2aaf21ab 100644 --- a/src/CSnakes.Runtime/Python/PyObject.cs +++ b/src/CSnakes.Runtime/Python/PyObject.cs @@ -470,6 +470,7 @@ internal object As(Type type) var t when t == typeof(byte[]) => CPythonAPI.PyBytes_AsByteArray(this), var t when t.IsAssignableTo(typeof(ITuple)) => PyObjectTypeConverter.ConvertToTuple(this, t), var t when t.IsAssignableTo(typeof(IGeneratorIterator)) => PyObjectTypeConverter.ConvertToGeneratorIterator(this, t), + var t when t.IsAssignableTo(typeof(ICoroutine)) => PyObjectTypeConverter.ConvertToCoroutine(this, t), var t => PyObjectTypeConverter.PyObjectToManagedType(this, t), }; } diff --git a/src/CSnakes.SourceGeneration/Parser/PythonParser.Function.cs b/src/CSnakes.SourceGeneration/Parser/PythonParser.Function.cs index 49e7a139..d8810c6c 100644 --- a/src/CSnakes.SourceGeneration/Parser/PythonParser.Function.cs +++ b/src/CSnakes.SourceGeneration/Parser/PythonParser.Function.cs @@ -11,12 +11,13 @@ namespace CSnakes.Parser; public static partial class PythonParser { public static TokenListParser PythonFunctionDefinitionParser { get; } = - (from def in Token.EqualTo(PythonToken.Def) + (from @async in Token.EqualTo(PythonToken.Async).OptionalOrDefault() + from def in Token.EqualTo(PythonToken.Def) from name in Token.EqualTo(PythonToken.Identifier) from parameters in PythonParameterListParser.AssumeNotNull() from arrow in Token.EqualTo(PythonToken.Arrow).Optional().Then(returnType => PythonTypeDefinitionParser.AssumeNotNull().OptionalOrDefault()) from colon in Token.EqualTo(PythonToken.Colon) - select new PythonFunctionDefinition(name.ToStringValue(), arrow, parameters)) + select new PythonFunctionDefinition(name.ToStringValue(), arrow, parameters, async.HasValue)) .Named("Function Definition"); /// diff --git a/src/CSnakes.SourceGeneration/Parser/Types/PythonFunctionDefinition.cs b/src/CSnakes.SourceGeneration/Parser/Types/PythonFunctionDefinition.cs index 5bbf5ee2..85785cc2 100644 --- a/src/CSnakes.SourceGeneration/Parser/Types/PythonFunctionDefinition.cs +++ b/src/CSnakes.SourceGeneration/Parser/Types/PythonFunctionDefinition.cs @@ -2,7 +2,7 @@ using System.Collections.Immutable; namespace CSnakes.Parser.Types; -public class PythonFunctionDefinition(string name, PythonTypeSpec? returnType, PythonFunctionParameter[] pythonFunctionParameter) +public class PythonFunctionDefinition(string name, PythonTypeSpec? returnType, PythonFunctionParameter[] pythonFunctionParameter, bool isAsync = false) { public string Name { get; private set; } = name; @@ -35,14 +35,13 @@ private static PythonFunctionParameter[] FixupArguments(PythonFunctionParameter[ public bool HasReturnTypeAnnotation() => returnType is not null; - public bool IsAsync { get; private set; } + public bool IsAsync => isAsync; public ImmutableArray SourceLines { get; private set; } = []; public PythonFunctionDefinition WithSourceLines(ImmutableArray value) => - value == SourceLines ? this : new(Name, ReturnType, Parameters) + value == SourceLines ? this : new(Name, ReturnType, Parameters, IsAsync) { - IsAsync = IsAsync, SourceLines = value }; } diff --git a/src/CSnakes.SourceGeneration/PythonStaticGenerator.cs b/src/CSnakes.SourceGeneration/PythonStaticGenerator.cs index 196ae73e..4a14db5c 100644 --- a/src/CSnakes.SourceGeneration/PythonStaticGenerator.cs +++ b/src/CSnakes.SourceGeneration/PythonStaticGenerator.cs @@ -187,7 +187,7 @@ select line.ToString() into line "/// " ], [ - $"{s.WithModifiers(s.Modifiers.RemoveAt(s.Modifiers.IndexOf(SyntaxKind.PublicKeyword))) + $"{s.WithModifiers(new SyntaxTokenList(s.Modifiers.Where(m => !m.IsKind(SyntaxKind.PublicKeyword) && !m.IsKind(SyntaxKind.AsyncKeyword)).ToList())) .WithBody(null) .NormalizeWhitespace()};" ] diff --git a/src/CSnakes.SourceGeneration/Reflection/MethodReflection.cs b/src/CSnakes.SourceGeneration/Reflection/MethodReflection.cs index 82e3a41a..e00afee2 100644 --- a/src/CSnakes.SourceGeneration/Reflection/MethodReflection.cs +++ b/src/CSnakes.SourceGeneration/Reflection/MethodReflection.cs @@ -16,19 +16,42 @@ public static class MethodReflection { public static MethodDefinition FromMethod(PythonFunctionDefinition function, string moduleName) { - // Step 1: Create a method declaration - - // Step 2: Determine the return type of the method + // Step 1: Determine the return type of the method PythonTypeSpec returnPythonType = function.ReturnType; TypeSyntax returnSyntax; - if (returnPythonType.Name == "None") + TypeSyntax? coroutineSyntax = null; + + if (!function.IsAsync) { - returnSyntax = PredefinedType(Token(SyntaxKind.VoidKeyword)); - } - else + if (returnPythonType.Name == "None") + { + returnSyntax = PredefinedType(Token(SyntaxKind.VoidKeyword)); + } + else + { + returnSyntax = TypeReflection.AsPredefinedType(returnPythonType, TypeReflection.ConversionDirection.FromPython); + } + } else { - returnSyntax = TypeReflection.AsPredefinedType(returnPythonType, TypeReflection.ConversionDirection.FromPython); + coroutineSyntax = TypeReflection.AsPredefinedType(returnPythonType, TypeReflection.ConversionDirection.FromPython); + if (returnPythonType.Name != "Coroutine" || !returnPythonType.HasArguments() || returnPythonType.Arguments.Length != 3) + { + throw new ArgumentException("Async function must return a Coroutine[T1, T2, T3]"); + } + var tYield = returnPythonType.Arguments[0]; + if (tYield.Name == "None") + { + returnSyntax = PredefinedType(Token(SyntaxKind.VoidKeyword)); + } + else + { + returnSyntax = TypeReflection.AsPredefinedType(tYield, TypeReflection.ConversionDirection.FromPython); + } + // return is a Task of + returnSyntax = GenericName(Identifier("Task")) + .WithTypeArgumentList(TypeArgumentList(SeparatedList([returnSyntax]))); + } // Step 3: Build arguments @@ -106,6 +129,7 @@ public static MethodDefinition FromMethod(PythonFunctionDefinition function, str ReturnStatementSyntax returnExpression = returnSyntax switch { + GenericNameSyntax g when g.Identifier.Text == "Task" && coroutineSyntax is not null => ProcessAsyncMethodWithReturnType(coroutineSyntax, parameterGenericArgs), PredefinedTypeSyntax s when s.Keyword.IsKind(SyntaxKind.VoidKeyword) => ReturnStatement(null), IdentifierNameSyntax { Identifier.ValueText: "PyObject" } => ReturnStatement(IdentifierName("__result_pyObject")), _ => ProcessMethodWithReturnType(returnSyntax, parameterGenericArgs) @@ -197,13 +221,16 @@ PredefinedTypeSyntax s when s.Keyword.IsKind(SyntaxKind.VoidKeyword) => true, .Select((a) => a.cSharpParameter) ); + var modifiers = function.IsAsync ? TokenList( + Token(SyntaxKind.PublicKeyword), + Token(SyntaxKind.AsyncKeyword) + ) + : TokenList(Token(SyntaxKind.PublicKeyword)); + var syntax = MethodDeclaration( returnSyntax, Identifier(function.Name.ToPascalCase())) - .WithModifiers( - TokenList( - Token(SyntaxKind.PublicKeyword)) - ) + .WithModifiers(modifiers) .WithBody(body) .WithParameterList(ParameterList(SeparatedList(methodParameters))); @@ -229,6 +256,32 @@ private static ReturnStatementSyntax ProcessMethodWithReturnType(TypeSyntax retu return returnExpression; } + private static ReturnStatementSyntax ProcessAsyncMethodWithReturnType(TypeSyntax returnSyntax, List parameterGenericArgs) + { + ReturnStatementSyntax returnExpression; + if (returnSyntax is GenericNameSyntax rg) + { + parameterGenericArgs.Add(rg); + } + var pyObjectAs = InvocationExpression( + MemberAccessExpression( + SyntaxKind.SimpleMemberAccessExpression, + IdentifierName("__result_pyObject"), + GenericName(Identifier("As")) + .WithTypeArgumentList(TypeArgumentList(SeparatedList([returnSyntax]))))); + + returnExpression = ReturnStatement( + AwaitExpression( + InvocationExpression( + MemberAccessExpression( + SyntaxKind.SimpleMemberAccessExpression, + pyObjectAs, + IdentifierName("AsTask"))) + )); + + return returnExpression; + } + private static InvocationExpressionSyntax GenerateParamsCall(IEnumerable<(PythonFunctionParameter pythonParameter, ParameterSyntax cSharpParameter)> parameterList) { IEnumerable pythonFunctionCallArguments = parameterList.Select((a) => Argument(IdentifierName($"{a.cSharpParameter.Identifier}_pyObject"))).ToList(); diff --git a/src/CSnakes.SourceGeneration/Reflection/TypeReflection.cs b/src/CSnakes.SourceGeneration/Reflection/TypeReflection.cs index ee9e2725..58161f12 100644 --- a/src/CSnakes.SourceGeneration/Reflection/TypeReflection.cs +++ b/src/CSnakes.SourceGeneration/Reflection/TypeReflection.cs @@ -2,7 +2,6 @@ using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.CSharp.Syntax; using CSnakes.Parser.Types; -using System.ComponentModel; namespace CSnakes.Reflection; @@ -32,6 +31,7 @@ public static TypeSyntax AsPredefinedType(PythonTypeSpec pythonType, ConversionD "typing.Sequence" or "Sequence" => CreateListType(pythonType.Arguments[0], direction), "typing.Optional" or "Optional" => AsPredefinedType(pythonType.Arguments[0], direction), "typing.Generator" or "Generator" => CreateGeneratorType(pythonType.Arguments[0], pythonType.Arguments[1], pythonType.Arguments[2], direction), + "typing.Coroutine" or "Coroutine" => CreateCoroutineType(pythonType.Arguments[0], pythonType.Arguments[1], pythonType.Arguments[2], direction), // Todo more types... see https://docs.python.org/3/library/stdtypes.html#standard-generic-classes _ => SyntaxFactory.ParseTypeName("PyObject"), }; @@ -59,6 +59,12 @@ private static TypeSyntax CreateGeneratorType(PythonTypeSpec yieldType, PythonTy AsPredefinedType(returnType, direction) ]); + private static TypeSyntax CreateCoroutineType(PythonTypeSpec yieldType, PythonTypeSpec sendType, PythonTypeSpec returnType, ConversionDirection direction) => CreateGenericType("ICoroutine", [ + AsPredefinedType(yieldType, direction), + AsPredefinedType(sendType, direction), + AsPredefinedType(returnType, direction) + ]); + private static TypeSyntax CreateTupleType(PythonTypeSpec[] tupleTypes, ConversionDirection direction) { if (tupleTypes.Length == 1) diff --git a/src/Integration.Tests/CoroutineTests.cs b/src/Integration.Tests/CoroutineTests.cs new file mode 100644 index 00000000..354a25fe --- /dev/null +++ b/src/Integration.Tests/CoroutineTests.cs @@ -0,0 +1,11 @@ +namespace Integration.Tests; +public class CoroutineTests(PythonEnvironmentFixture fixture) : IntegrationTestBase(fixture) +{ + [Fact] + public async Task BasicCoroutine() + { + var mod = Env.TestCoroutines(); + long result = await mod.TestCoroutine(); + Assert.Equal(5, result); + } +} diff --git a/src/Integration.Tests/GeneratorTests.cs b/src/Integration.Tests/GeneratorTests.cs index 1e9fd30a..4953571a 100644 --- a/src/Integration.Tests/GeneratorTests.cs +++ b/src/Integration.Tests/GeneratorTests.cs @@ -23,15 +23,4 @@ public void TestNormalGenerator() var generator = mod.TestNormalGenerator(); Assert.Equal(["one", "two"], generator.ToArray()); } - - [Fact] - public async Task TestAsyncEnumerator() - { - var mod = Env.TestGenerators(); - List words = [ "foo", "bar", "baz"]; - await foreach (string word in mod.TestGeneratorSequence()) - { - Assert.Contains(word, words); - } - } } diff --git a/src/Integration.Tests/python/test_coroutines.py b/src/Integration.Tests/python/test_coroutines.py new file mode 100644 index 00000000..bae5f3cc --- /dev/null +++ b/src/Integration.Tests/python/test_coroutines.py @@ -0,0 +1,7 @@ +from typing import Coroutine +import asyncio + + +async def test_coroutine() -> Coroutine[int, None, None]: + await asyncio.sleep(0.1) + return 5 From e3f3abd5ac0e8f82aab6c1b2d088e753ed4c62d2 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 15 Nov 2024 18:00:42 +1100 Subject: [PATCH 05/13] Create and call task outside of the GIL of the main thread to avoid deadlock --- .../Reflection/MethodReflection.cs | 89 +++++++++++++++---- 1 file changed, 70 insertions(+), 19 deletions(-) diff --git a/src/CSnakes.SourceGeneration/Reflection/MethodReflection.cs b/src/CSnakes.SourceGeneration/Reflection/MethodReflection.cs index e00afee2..75f4fe98 100644 --- a/src/CSnakes.SourceGeneration/Reflection/MethodReflection.cs +++ b/src/CSnakes.SourceGeneration/Reflection/MethodReflection.cs @@ -157,7 +157,11 @@ PredefinedTypeSyntax s when s.Keyword.IsKind(SyntaxKind.VoidKeyword) => true, IdentifierName($"__func_{function.Name}")))))) ); - var callStatement = LocalDeclarationStatement( + LocalDeclarationStatementSyntax callStatement; + + if (!function.IsAsync) + { + callStatement = LocalDeclarationStatement( VariableDeclaration( IdentifierName("PyObject")) .WithVariables( @@ -167,10 +171,23 @@ PredefinedTypeSyntax s when s.Keyword.IsKind(SyntaxKind.VoidKeyword) => true, .WithInitializer( EqualsValueClause( callExpression))))); - if (resultShouldBeDisposed) + + } else + { + callStatement = LocalDeclarationStatement( + VariableDeclaration( + IdentifierName("PyObject")) + .WithVariables( + SingletonSeparatedList( + VariableDeclarator( + Identifier("__result_pyObject")) + ))); + + } + if (resultShouldBeDisposed && !function.IsAsync) callStatement = callStatement.WithUsingKeyword(Token(SyntaxKind.UsingKeyword)); - StatementSyntax[] statements = [ - ExpressionStatement( + + var logStatement = ExpressionStatement( InvocationExpression( MemberAccessExpression( SyntaxKind.SimpleMemberAccessExpression, @@ -183,22 +200,56 @@ PredefinedTypeSyntax s when s.Keyword.IsKind(SyntaxKind.VoidKeyword) => true, Argument(LiteralExpression(SyntaxKind.StringLiteralExpression, Literal("Invoking Python function: {FunctionName}"))), Argument(LiteralExpression(SyntaxKind.StringLiteralExpression, Literal(function.Name))) ]))) + ); + + BlockSyntax body; + if (function.IsAsync) + { + ExpressionStatementSyntax localCallStatement = ExpressionStatement( + AssignmentExpression( + SyntaxKind.SimpleAssignmentExpression, + IdentifierName("__result_pyObject"), + callExpression)); + + StatementSyntax[] statements = [ + logStatement, + functionObject, + .. pythonConversionStatements, + localCallStatement + ]; + + body = Block( + callStatement, + UsingStatement( + null, + InvocationExpression( + MemberAccessExpression( + SyntaxKind.SimpleMemberAccessExpression, + IdentifierName("GIL"), + IdentifierName("Acquire"))), + Block(statements) ), - functionObject, - .. pythonConversionStatements, - callStatement, - returnExpression]; - var body = Block( - UsingStatement( - null, - InvocationExpression( - MemberAccessExpression( - SyntaxKind.SimpleMemberAccessExpression, - IdentifierName("GIL"), - IdentifierName("Acquire"))), - Block(statements) - )); - + returnExpression); + } else + { + StatementSyntax[] statements = [ + logStatement, + functionObject, + .. pythonConversionStatements, + callStatement, + returnExpression]; + body = Block( + UsingStatement( + null, + InvocationExpression( + MemberAccessExpression( + SyntaxKind.SimpleMemberAccessExpression, + IdentifierName("GIL"), + IdentifierName("Acquire"))), + Block(statements) + )); + } + // Sort the method parameters into this order // 1. All positional arguments // 2. All keyword-only arguments From 5bf155ae82c49955f8496dde94af652cb27284be Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 15 Nov 2024 18:18:54 +1100 Subject: [PATCH 06/13] Add a harder test --- src/Integration.Tests/CoroutineTests.cs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/Integration.Tests/CoroutineTests.cs b/src/Integration.Tests/CoroutineTests.cs index 354a25fe..c9787912 100644 --- a/src/Integration.Tests/CoroutineTests.cs +++ b/src/Integration.Tests/CoroutineTests.cs @@ -8,4 +8,17 @@ public async Task BasicCoroutine() long result = await mod.TestCoroutine(); Assert.Equal(5, result); } + + [Fact] + public async Task MultipleCoroutineCalls() + { + var mod = Env.TestCoroutines(); + var tasks = new List>(); + for (int i = 0; i < 10; i++) + { + tasks.Add(mod.TestCoroutine()); + } + var r = await Task.WhenAll(tasks); + Assert.All(r, x => Assert.Equal(5, x)); + } } From a27a7cc51e6a18ceddc56d0b5f97467b75c6584d Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 15 Nov 2024 18:25:48 +1100 Subject: [PATCH 07/13] Make the current event loop a thread static as they're per thread state --- src/CSnakes.Runtime/CPython/Coroutine.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSnakes.Runtime/CPython/Coroutine.cs b/src/CSnakes.Runtime/CPython/Coroutine.cs index 9d2f964a..0bccefa2 100644 --- a/src/CSnakes.Runtime/CPython/Coroutine.cs +++ b/src/CSnakes.Runtime/CPython/Coroutine.cs @@ -8,7 +8,7 @@ internal static bool IsPyCoroutine(PyObject p) return HasAttr(p, "__await__"); } - private static PyObject? currentEventLoop = null; + [ThreadStatic] private static PyObject? currentEventLoop = null; private static PyObject? asyncioModule = null; internal static PyObject GetAsyncioModule() From 7e3108973fcb29d57f6954249a74ed48557a4304 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 22 Nov 2024 15:47:51 -0600 Subject: [PATCH 08/13] Refactor event loops into a specific type and handle lifetimes. --- src/CSnakes.Runtime/CPython/Coroutine.cs | 73 +++++++++++++++++++----- src/CSnakes.Runtime/CPython/Init.cs | 8 ++- src/CSnakes.Runtime/Python/Coroutine.cs | 11 ++-- 3 files changed, 72 insertions(+), 20 deletions(-) diff --git a/src/CSnakes.Runtime/CPython/Coroutine.cs b/src/CSnakes.Runtime/CPython/Coroutine.cs index 0bccefa2..98e18fbf 100644 --- a/src/CSnakes.Runtime/CPython/Coroutine.cs +++ b/src/CSnakes.Runtime/CPython/Coroutine.cs @@ -1,4 +1,5 @@ using CSnakes.Runtime.Python; +using System.Threading.Tasks; namespace CSnakes.Runtime.CPython; internal unsafe partial class CPythonAPI @@ -7,19 +8,63 @@ internal static bool IsPyCoroutine(PyObject p) { return HasAttr(p, "__await__"); } + + internal class EventLoop : IDisposable + { + private readonly PyObject? loop = null; + + public EventLoop() + { + loop = NewEventLoopFactory!.Call(); + } + + public bool IsDisposed { get; private set; } + + private void Close() + { + if (loop is null) + { + throw new InvalidOperationException("Event loop not initialized"); + } + using var close = loop.GetAttr("close"); + close?.Call(); + } + + public void Dispose() + { + Close(); + loop?.Dispose(); + IsDisposed = true; + } + + public PyObject RunTaskUntilComplete(PyObject coroutine) + { + if (loop is null) + { + throw new InvalidOperationException("Event loop not initialized"); + } + using var taskFunc = loop.GetAttr("create_task"); + using var task = taskFunc?.Call(coroutine) ?? PyObject.None; + using var runUntilComplete = loop.GetAttr("run_until_complete"); + var result = runUntilComplete.Call(task); + return result; + } + } - [ThreadStatic] private static PyObject? currentEventLoop = null; - private static PyObject? asyncioModule = null; + [ThreadStatic] private static EventLoop? currentEventLoop = null; + private static PyObject? AsyncioModule = null; + private static PyObject? NewEventLoopFactory = null; - internal static PyObject GetAsyncioModule() - { - asyncioModule ??= Import("asyncio"); - return asyncioModule; - } - - internal static PyObject GetCurrentEventLoop() - { - currentEventLoop ??= GetAsyncioModule().GetAttr("new_event_loop").Call(); - return currentEventLoop; - } -} + internal static EventLoop WithEventLoop() + { + if (AsyncioModule is null) + { + throw new InvalidOperationException("Asyncio module not initialized"); + } + if (currentEventLoop is null || currentEventLoop.IsDisposed) + { + currentEventLoop = new EventLoop(); + } + return currentEventLoop!; + } +} diff --git a/src/CSnakes.Runtime/CPython/Init.cs b/src/CSnakes.Runtime/CPython/Init.cs index cf2e81a4..9f834138 100644 --- a/src/CSnakes.Runtime/CPython/Init.cs +++ b/src/CSnakes.Runtime/CPython/Init.cs @@ -1,5 +1,4 @@ using CSnakes.Runtime.Python; -using CSnakes.Runtime.Python.Interns; using System.Diagnostics; using System.Reflection; using System.Runtime.InteropServices; @@ -101,6 +100,8 @@ private void InitializeEmbeddedPython() PyBytesType = GetTypeRaw(PyBytes_FromByteSpan(new byte[] { })); ItemsStrIntern = AsPyUnicodeObject("items"); PyNone = GetBuiltin("None"); + AsyncioModule = Import("asyncio"); + NewEventLoopFactory = AsyncioModule.GetAttr("new_event_loop"); } PyEval_SaveThread(); } @@ -137,6 +138,11 @@ protected virtual void Dispose(bool disposing) if (!IsInitialized) return; + // Clean-up interns + NewEventLoopFactory?.Dispose(); + AsyncioModule?.Dispose(); + // TODO: Add more cleanup code here + Debug.WriteLine("Calling Py_Finalize()"); // Acquire the GIL only to dispose it immediately because `PyGILState_Release` diff --git a/src/CSnakes.Runtime/Python/Coroutine.cs b/src/CSnakes.Runtime/Python/Coroutine.cs index 8137a55a..28940269 100644 --- a/src/CSnakes.Runtime/Python/Coroutine.cs +++ b/src/CSnakes.Runtime/Python/Coroutine.cs @@ -18,11 +18,12 @@ public Task AsTask() try { using (GIL.Acquire()) - { - var loop = CPythonAPI.GetCurrentEventLoop(); - using PyObject task = loop.GetAttr("create_task").Call(coroutine); - using PyObject result = loop.GetAttr("run_until_complete").Call(task); - current = result.As(); + { + using (var loop = CPythonAPI.WithEventLoop()) + { + using PyObject result = loop.RunTaskUntilComplete(coroutine); + current = result.As(); + } } return current; } From 40ca27b39f920e2db4da54f870c1e0becc04a9e6 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Sun, 24 Nov 2024 14:21:21 -0800 Subject: [PATCH 09/13] Add remark --- src/CSnakes.Runtime/CPython/Coroutine.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/CSnakes.Runtime/CPython/Coroutine.cs b/src/CSnakes.Runtime/CPython/Coroutine.cs index 98e18fbf..4d5b39b6 100644 --- a/src/CSnakes.Runtime/CPython/Coroutine.cs +++ b/src/CSnakes.Runtime/CPython/Coroutine.cs @@ -63,6 +63,7 @@ internal static EventLoop WithEventLoop() } if (currentEventLoop is null || currentEventLoop.IsDisposed) { + // TODO: Reuse event loops currentEventLoop = new EventLoop(); } return currentEventLoop!; From c59473616171532797a0c3099a3207bb6c500b52 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Sun, 24 Nov 2024 16:50:06 -0800 Subject: [PATCH 10/13] Refactor finalize into a method --- src/CSnakes.Runtime/CPython/Init.cs | 47 ++++++++++++++++------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/src/CSnakes.Runtime/CPython/Init.cs b/src/CSnakes.Runtime/CPython/Init.cs index 9f834138..77833ed9 100644 --- a/src/CSnakes.Runtime/CPython/Init.cs +++ b/src/CSnakes.Runtime/CPython/Init.cs @@ -127,34 +127,39 @@ private void InitializeEmbeddedPython() [LibraryImport(PythonLibraryName, EntryPoint = "Py_SetPath", StringMarshallingCustomType = typeof(Utf32StringMarshaller), StringMarshalling = StringMarshalling.Custom)] internal static partial void Py_SetPath_UCS4_UTF32(string path); - protected virtual void Dispose(bool disposing) + protected void FinalizeEmbeddedPython() { - if (!disposedValue) + lock (initLock) { - if (disposing) - { - lock (initLock) - { - if (!IsInitialized) - return; + if (!IsInitialized) + return; + + // Clean-up interns + NewEventLoopFactory?.Dispose(); + AsyncioModule?.Dispose(); + // TODO: Add more cleanup code here - // Clean-up interns - NewEventLoopFactory?.Dispose(); - AsyncioModule?.Dispose(); - // TODO: Add more cleanup code here + Debug.WriteLine($"Calling Py_Finalize() on thread {GetNativeThreadId()}"); - Debug.WriteLine("Calling Py_Finalize()"); + // Acquire the GIL only to dispose it immediately because `PyGILState_Release` + // is not available after `Py_Finalize` is called. This is done primarily to + // trigger the disposal of handles that have been queued before the Python + // runtime is finalized. - // Acquire the GIL only to dispose it immediately because `PyGILState_Release` - // is not available after `Py_Finalize` is called. This is done primarily to - // trigger the disposal of handles that have been queued before the Python - // runtime is finalized. + GIL.Acquire().Dispose(); - GIL.Acquire().Dispose(); + PyGILState_Ensure(); + Py_Finalize(); + } + } - PyGILState_Ensure(); - Py_Finalize(); - } + protected virtual void Dispose(bool disposing) + { + if (!disposedValue) + { + if (disposing) + { + FinalizeEmbeddedPython(); } disposedValue = true; From d468d48ec4fcbd7c27f98f8e22ad7048fd3c49d0 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 28 Nov 2024 15:13:19 +1100 Subject: [PATCH 11/13] Run initialization and finalization in the same thread as a background task --- src/CSnakes.Runtime/CPython/Init.cs | 56 +++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/src/CSnakes.Runtime/CPython/Init.cs b/src/CSnakes.Runtime/CPython/Init.cs index 77833ed9..3cd4fcb3 100644 --- a/src/CSnakes.Runtime/CPython/Init.cs +++ b/src/CSnakes.Runtime/CPython/Init.cs @@ -14,6 +14,8 @@ internal unsafe partial class CPythonAPI : IDisposable private static readonly Lock initLock = new(); private static Version PythonVersion = new("0.0.0"); private bool disposedValue = false; + private Task? initializationTask = null; + private CancellationTokenSource? cts = null; public CPythonAPI(string pythonLibraryPath, Version version) { @@ -50,7 +52,8 @@ private static IntPtr DllImportResolver(string libraryName, Assembly assembly, D { // Override default dlopen flags on linux to allow global symbol resolution (required in extension modules) // See https://github.com/tonybaloney/CSnakes/issues/112#issuecomment-2290643468 - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)){ + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { return dlopen(pythonLibraryPath!, (int)(RTLD.LAZY | RTLD.GLOBAL)); } @@ -64,7 +67,47 @@ internal void Initialize() if (IsInitialized) return; - InitializeEmbeddedPython(); + /* Notes: + * + * The CPython initialization and finalization + * methods should be called from the same thread. + * + * Without doing so, CPython can hang on finalization. + * Especially if the code imports the threading module, or + * uses asyncio. + * + * Since we have no guarantee that the Dispose() of this + * class will be called from the same managed CLR thread + * as the one which called Init(), we create a Task with + * a cancellation token and use that as a mechanism to wait + * in the background for shutdown then call the finalization. + */ + + cts = new CancellationTokenSource(); + bool initialized = false; + initializationTask = Task.Run( + () => { + InitializeEmbeddedPython(); + initialized = true; + while (true) + { + if (cts.IsCancellationRequested) + { + FinalizeEmbeddedPython(); + break; + } + else + { + Thread.Sleep(500); + } + } + }, + cancellationToken: cts.Token); + + while (!initialized) // Wait for startup + { + continue; + } } private void InitializeEmbeddedPython() @@ -159,7 +202,14 @@ protected virtual void Dispose(bool disposing) { if (disposing) { - FinalizeEmbeddedPython(); + if (initializationTask != null) + { + if (cts == null) + throw new InvalidOperationException("Invalid runtime state"); + + cts.Cancel(); + initializationTask.Wait(); + } } disposedValue = true; From c21dc0f6f09f3daae7361ec3c9b07cc7673564ee Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 28 Nov 2024 15:21:40 +1100 Subject: [PATCH 12/13] Test that exceptions are handled gracefully --- src/Integration.Tests/CoroutineTests.cs | 11 +++++++++++ src/Integration.Tests/python/test_coroutines.py | 5 +++++ 2 files changed, 16 insertions(+) diff --git a/src/Integration.Tests/CoroutineTests.cs b/src/Integration.Tests/CoroutineTests.cs index c9787912..bed74d7f 100644 --- a/src/Integration.Tests/CoroutineTests.cs +++ b/src/Integration.Tests/CoroutineTests.cs @@ -20,5 +20,16 @@ public async Task MultipleCoroutineCalls() } var r = await Task.WhenAll(tasks); Assert.All(r, x => Assert.Equal(5, x)); + } + + [Fact] + public async Task CoroutineRaisesException() + { + var mod = Env.TestCoroutines(); + var task = mod.TestCoroutineRaisesException(); + var exception = await Assert.ThrowsAsync(async () => await task); + Assert.NotNull(exception.InnerException); + Assert.Equal("This is a Python exception", exception.InnerException.Message); + Assert.Equal("ValueError", exception.PythonExceptionType); } } diff --git a/src/Integration.Tests/python/test_coroutines.py b/src/Integration.Tests/python/test_coroutines.py index bae5f3cc..550daaa0 100644 --- a/src/Integration.Tests/python/test_coroutines.py +++ b/src/Integration.Tests/python/test_coroutines.py @@ -5,3 +5,8 @@ async def test_coroutine() -> Coroutine[int, None, None]: await asyncio.sleep(0.1) return 5 + + +async def test_coroutine_raises_exception() -> Coroutine[int, None, None]: + raise ValueError("This is a Python exception") + From 21ffe16336fbb2fca6befe19875887d8fd8735c0 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 28 Nov 2024 15:40:58 +1100 Subject: [PATCH 13/13] Reuse event loops and close them all at shutdown. --- src/CSnakes.Runtime/CPython/Coroutine.cs | 17 ++++- src/CSnakes.Runtime/CPython/Init.cs | 3 + src/CSnakes.Runtime/Python/Coroutine.cs | 89 ++++++++++++------------ src/CSnakes.Runtime/Python/ICoroutine.cs | 2 +- 4 files changed, 61 insertions(+), 50 deletions(-) diff --git a/src/CSnakes.Runtime/CPython/Coroutine.cs b/src/CSnakes.Runtime/CPython/Coroutine.cs index 4d5b39b6..f1da0302 100644 --- a/src/CSnakes.Runtime/CPython/Coroutine.cs +++ b/src/CSnakes.Runtime/CPython/Coroutine.cs @@ -1,4 +1,5 @@ using CSnakes.Runtime.Python; +using System.Collections.Concurrent; using System.Threading.Tasks; namespace CSnakes.Runtime.CPython; @@ -50,12 +51,13 @@ public PyObject RunTaskUntilComplete(PyObject coroutine) return result; } } - + + private static ConcurrentBag eventLoops = []; [ThreadStatic] private static EventLoop? currentEventLoop = null; private static PyObject? AsyncioModule = null; private static PyObject? NewEventLoopFactory = null; - internal static EventLoop WithEventLoop() + internal static EventLoop GetEventLoop() { if (AsyncioModule is null) { @@ -63,9 +65,18 @@ internal static EventLoop WithEventLoop() } if (currentEventLoop is null || currentEventLoop.IsDisposed) { - // TODO: Reuse event loops currentEventLoop = new EventLoop(); + eventLoops.Add(currentEventLoop); } return currentEventLoop!; } + + internal static void CloseEventLoops() + { + foreach (var eventLoop in eventLoops) + { + eventLoop.Dispose(); + } + eventLoops.Clear(); + } } diff --git a/src/CSnakes.Runtime/CPython/Init.cs b/src/CSnakes.Runtime/CPython/Init.cs index 3cd4fcb3..ecff15fd 100644 --- a/src/CSnakes.Runtime/CPython/Init.cs +++ b/src/CSnakes.Runtime/CPython/Init.cs @@ -177,6 +177,9 @@ protected void FinalizeEmbeddedPython() if (!IsInitialized) return; + // Shut down asyncio coroutines + CloseEventLoops(); + // Clean-up interns NewEventLoopFactory?.Dispose(); AsyncioModule?.Dispose(); diff --git a/src/CSnakes.Runtime/Python/Coroutine.cs b/src/CSnakes.Runtime/Python/Coroutine.cs index 28940269..97545fbd 100644 --- a/src/CSnakes.Runtime/Python/Coroutine.cs +++ b/src/CSnakes.Runtime/Python/Coroutine.cs @@ -1,47 +1,44 @@ -using CSnakes.Runtime.CPython; - -namespace CSnakes.Runtime.Python; - -public class Coroutine(PyObject coroutine) : ICoroutine -{ - private TYield current = default!; - private TReturn @return = default!; - - public TYield Current => current; - public TReturn Return => @return; - - public Task AsTask() - { - return Task.Run( - () => - { - try - { - using (GIL.Acquire()) +using CSnakes.Runtime.CPython; + +namespace CSnakes.Runtime.Python; + +public class Coroutine(PyObject coroutine) : ICoroutine +{ + private TYield current = default!; + private TReturn @return = default!; + + public TYield Current => current; + public TReturn Return => @return; + + public Task AsTask() + { + return Task.Run( + () => + { + try + { + using (GIL.Acquire()) { - using (var loop = CPythonAPI.WithEventLoop()) - { - using PyObject result = loop.RunTaskUntilComplete(coroutine); - current = result.As(); - } - } - return current; - } - catch (PythonInvocationException ex) - { - if (ex.InnerException is PythonStopIterationException stopIteration) - { - using var @return = stopIteration.TakeValue(); - this.@return = @return.As(); - - // Coroutine has finished - // TODO: define behavior for this case - return default(TYield); - } - - throw; - } - } - ); - } -} + using PyObject result = CPythonAPI.GetEventLoop().RunTaskUntilComplete(coroutine); + current = result.As(); + } + return current; + } + catch (PythonInvocationException ex) + { + if (ex.InnerException is PythonStopIterationException stopIteration) + { + using var @return = stopIteration.TakeValue(); + this.@return = @return.As(); + + // Coroutine has finished + // TODO: define behavior for this case + return default; + } + + throw; + } + } + ); + } +} diff --git a/src/CSnakes.Runtime/Python/ICoroutine.cs b/src/CSnakes.Runtime/Python/ICoroutine.cs index 3e523051..4dff8c8c 100644 --- a/src/CSnakes.Runtime/Python/ICoroutine.cs +++ b/src/CSnakes.Runtime/Python/ICoroutine.cs @@ -2,7 +2,7 @@ public interface ICoroutine : ICoroutine { - public Task AsTask(); + public Task AsTask(); } public interface ICoroutine { }