Add support for async functions

src/CSnakes.Runtime/CPython/Coroutine.cs

@@ -0,0 +1,82 @@
+using CSnakes.Runtime.Python;
+using System.Collections.Concurrent;
+using System.Threading.Tasks;
+
+namespace CSnakes.Runtime.CPython;
+internal unsafe partial class CPythonAPI
+{
+    internal static bool IsPyCoroutine(PyObject p)
+    {
+        return HasAttr(p, "__await__");
+    }
+
+    internal class EventLoop : IDisposable
+    {
+        private readonly PyObject? loop = null;
+
+        public EventLoop()
+        {
+            loop = NewEventLoopFactory!.Call();
+        }
+
+        public bool IsDisposed { get; private set; }
+
+        private void Close()
+        {
+            if (loop is null)
+            {
+                throw new InvalidOperationException("Event loop not initialized");
+            }
+            using var close = loop.GetAttr("close");
+            close?.Call();
+        }
+
+        public void Dispose()
+        {
+            Close();
+            loop?.Dispose();
+            IsDisposed = true;
+        }
+
+        public PyObject RunTaskUntilComplete(PyObject coroutine)
+        {
+            if (loop is null)
+            {
+                throw new InvalidOperationException("Event loop not initialized");
+            }
+            using var taskFunc = loop.GetAttr("create_task");
+            using var task = taskFunc?.Call(coroutine) ?? PyObject.None;
+            using var runUntilComplete = loop.GetAttr("run_until_complete");
+            var result = runUntilComplete.Call(task);
+            return result;
+        }
+    }
+
+    private static ConcurrentBag<EventLoop> eventLoops = [];
+    [ThreadStatic] private static EventLoop? currentEventLoop = null;
+    private static PyObject? AsyncioModule = null;
+    private static PyObject? NewEventLoopFactory = null;
+
+    internal static EventLoop GetEventLoop()
+    {
+        if (AsyncioModule is null)
+        {
+            throw new InvalidOperationException("Asyncio module not initialized");
+        }
+        if (currentEventLoop is null || currentEventLoop.IsDisposed)
+        {
+            currentEventLoop = new EventLoop();
+            eventLoops.Add(currentEventLoop);
+        }
+        return currentEventLoop!;
+    }
+
+    internal static void CloseEventLoops()
+    {
+        foreach (var eventLoop in eventLoops)
+        {
+            eventLoop.Dispose();
+        }
+        eventLoops.Clear();
+    }
+}

src/CSnakes.Runtime/CPython/Init.cs

@@ -1,5 +1,4 @@
 using CSnakes.Runtime.Python;
-using CSnakes.Runtime.Python.Interns;
 using System.Diagnostics;
 using System.Reflection;
 using System.Runtime.InteropServices;
@@ -15,6 +14,8 @@ internal unsafe partial class CPythonAPI : IDisposable
     private static readonly Lock initLock = new();
     private static Version PythonVersion = new("0.0.0");
     private bool disposedValue = false;
+    private Task? initializationTask = null;
+    private CancellationTokenSource? cts = null;
 
     public CPythonAPI(string pythonLibraryPath, Version version)
     {
@@ -51,7 +52,8 @@ private static IntPtr DllImportResolver(string libraryName, Assembly assembly, D
         {
             // Override default dlopen flags on linux to allow global symbol resolution (required in extension modules)
             // See https://github.com/tonybaloney/CSnakes/issues/112#issuecomment-2290643468
-            if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)){
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+            {
                 return dlopen(pythonLibraryPath!, (int)(RTLD.LAZY | RTLD.GLOBAL));
             }
 
@@ -65,7 +67,47 @@ internal void Initialize()
         if (IsInitialized)
             return;
 
-        InitializeEmbeddedPython();
+        /* Notes:
+         * 
+         * The CPython initialization and finalization
+         * methods should be called from the same thread.
+         * 
+         * Without doing so, CPython can hang on finalization.
+         * Especially if the code imports the threading module, or 
+         * uses asyncio.
+         * 
+         * Since we have no guarantee that the Dispose() of this 
+         * class will be called from the same managed CLR thread
+         * as the one which called Init(), we create a Task with 
+         * a cancellation token and use that as a mechanism to wait
+         * in the background for shutdown then call the finalization.
+         */
+
+        cts = new CancellationTokenSource();
+        bool initialized = false;
+        initializationTask = Task.Run(
+            () => {
+                InitializeEmbeddedPython();
+                initialized = true;
+                while (true)
+                {
+                    if (cts.IsCancellationRequested)
+                    {
+                        FinalizeEmbeddedPython();
+                        break;
+                    }
+                    else
+                    {
+                        Thread.Sleep(500);
+                    }
+                }
+            },
+            cancellationToken: cts.Token);
+
+        while (!initialized) // Wait for startup
+        {
+            continue;
+        }
     }
 
     private void InitializeEmbeddedPython()
@@ -101,6 +143,8 @@ private void InitializeEmbeddedPython()
                 PyBytesType = GetTypeRaw(PyBytes_FromByteSpan(new byte[] { }));
                 ItemsStrIntern = AsPyUnicodeObject("items");
                 PyNone = GetBuiltin("None");
+                AsyncioModule = Import("asyncio");
+                NewEventLoopFactory = AsyncioModule.GetAttr("new_event_loop");
             }
             PyEval_SaveThread();
         }
@@ -126,28 +170,48 @@ private void InitializeEmbeddedPython()
     [LibraryImport(PythonLibraryName, EntryPoint = "Py_SetPath", StringMarshallingCustomType = typeof(Utf32StringMarshaller), StringMarshalling = StringMarshalling.Custom)]
     internal static partial void Py_SetPath_UCS4_UTF32(string path);
 
+    protected void FinalizeEmbeddedPython()
+    {
+        lock (initLock)
+        {
+            if (!IsInitialized)
+                return;
+
+            // Shut down asyncio coroutines
+            CloseEventLoops();
+
+            // Clean-up interns
+            NewEventLoopFactory?.Dispose();
+            AsyncioModule?.Dispose();
+            // TODO: Add more cleanup code here
+
+            Debug.WriteLine($"Calling Py_Finalize() on thread {GetNativeThreadId()}");
+
+            // Acquire the GIL only to dispose it immediately because `PyGILState_Release`
+            // is not available after `Py_Finalize` is called. This is done primarily to
+            // trigger the disposal of handles that have been queued before the Python
+            // runtime is finalized.
+
+            GIL.Acquire().Dispose();
+
+            PyGILState_Ensure();
+            Py_Finalize();
+        }
+    }
+
     protected virtual void Dispose(bool disposing)
     {
         if (!disposedValue)
         {
             if (disposing)
             {
-                lock (initLock)
+                if (initializationTask != null)
                 {
-                    if (!IsInitialized)
-                        return;
-
-                    Debug.WriteLine("Calling Py_Finalize()");
-
-                    // Acquire the GIL only to dispose it immediately because `PyGILState_Release`
-                    // is not available after `Py_Finalize` is called. This is done primarily to
-                    // trigger the disposal of handles that have been queued before the Python
-                    // runtime is finalized.
-
-                    GIL.Acquire().Dispose();
+                    if (cts == null)
+                        throw new InvalidOperationException("Invalid runtime state");
 
-                    PyGILState_Ensure();
-                    Py_Finalize();
+                    cts.Cancel();
+                    initializationTask.Wait();
                 }
             }
 

src/CSnakes.Runtime/PyObjectTypeConverter.Coroutine.cs

@@ -0,0 +1,31 @@
+using CSnakes.Runtime.CPython;
+using CSnakes.Runtime.Python;
+using System.Diagnostics;
+using System.Reflection;
+
+namespace CSnakes.Runtime;
+internal partial class PyObjectTypeConverter
+{
+    internal static object ConvertToCoroutine(PyObject pyObject, Type destinationType)
+    {
+        Debug.Assert(destinationType.IsGenericType);
+        Debug.Assert(!destinationType.IsGenericTypeDefinition);
+        Debug.Assert(destinationType.GetGenericTypeDefinition() == typeof(ICoroutine<,,>));
+
+        if (!CPythonAPI.IsPyCoroutine(pyObject))
+        {
+            throw new InvalidCastException($"Cannot convert {pyObject.GetPythonType()} to a coroutine.");
+        }
+
+        if (!knownDynamicTypes.TryGetValue(destinationType, out DynamicTypeInfo? typeInfo))
+        {
+            var typeArgs = destinationType.GetGenericArguments();
+            Type coroutineType = typeof(Coroutine<,,>).MakeGenericType(typeArgs);
+            ConstructorInfo ctor = coroutineType.GetConstructors().First();
+            typeInfo = new(ctor);
+            knownDynamicTypes[destinationType] = typeInfo;
+        }
+
+        return typeInfo.ReturnTypeConstructor.Invoke([pyObject.Clone()]);
+    }
+}

src/CSnakes.Runtime/Python/Coroutine.cs

@@ -0,0 +1,44 @@
+using CSnakes.Runtime.CPython;
+
+namespace CSnakes.Runtime.Python;
+
+public class Coroutine<TYield, TSend, TReturn>(PyObject coroutine) : ICoroutine<TYield, TSend, TReturn>
+{
+    private TYield current = default!;
+    private TReturn @return = default!;
+
+    public TYield Current => current;
+    public TReturn Return => @return;
+
+    public Task<TYield?> AsTask()
+    {
+        return Task.Run(
+            () =>
+            {
+                try
+                {
+                    using (GIL.Acquire())
+                    {
+                        using PyObject result = CPythonAPI.GetEventLoop().RunTaskUntilComplete(coroutine);
+                        current = result.As<TYield>();
+                    }
+                    return current;
+                }
+                catch (PythonInvocationException ex)
+                {
+                    if (ex.InnerException is PythonStopIterationException stopIteration)
+                    {
+                        using var @return = stopIteration.TakeValue();
+                        this.@return = @return.As<TReturn>();
+
+                        // Coroutine has finished
+                        // TODO: define behavior for this case
+                        return default;
+                    }
+
+                    throw;
+                }
+            }
+        );
+    }
+}

src/CSnakes.Runtime/Python/ICoroutine.cs

@@ -0,0 +1,8 @@
+namespace CSnakes.Runtime.Python;
+
+public interface ICoroutine<TYield, TSend, TReturn> : ICoroutine
+{
+    public Task<TYield?> AsTask();
+}
+
+public interface ICoroutine { }

src/CSnakes.Runtime/Python/IGeneratorIterator.cs

@@ -2,7 +2,9 @@
 
 namespace CSnakes.Runtime.Python;
 
-public interface IGeneratorIterator<out TYield, in TSend, out TReturn>: IEnumerator<TYield>, IEnumerable<TYield>, IGeneratorIterator
+public interface IGeneratorIterator<out TYield, in TSend, out TReturn>:
+    IEnumerator<TYield>, IEnumerable<TYield>,
+    IGeneratorIterator
 {
     /// <remarks>
     /// The value is undefined until either <see cref="IEnumerator.MoveNext"/> or <see cref="Send"/>

src/CSnakes.Runtime/Python/Import.cs

@@ -6,13 +6,17 @@ public static class Import
 {
     public static PyObject ImportModule(string module)
     {
-        return CPythonAPI.Import(module);
+        using (GIL.Acquire())
+            return CPythonAPI.Import(module);
     }
 
     public static void ReloadModule(ref PyObject module)
     {
-        var newModule = CPythonAPI.ReloadModule(module);
-        module.Dispose();
-        module = newModule;
+        using (GIL.Acquire())
+        {
+            var newModule = CPythonAPI.ReloadModule(module);
+            module.Dispose();
+            module = newModule;
+        }
     }
 }

src/CSnakes.Runtime/Python/PyObject.Imortals.cs

@@ -9,5 +9,4 @@ public partial class PyObject
     public static PyObject One { get; } = new PyOneObject();
     public static PyObject Zero { get; } = new PyZeroObject();
     public static PyObject NegativeOne { get; } = new PyNegativeOneObject();
-
 }

src/CSnakes.Runtime/Python/PyObject.cs

@@ -16,7 +16,7 @@
     {
         if (pyObject == IntPtr.Zero)
         {
             throw ThrowPythonExceptionAsClrException();
         }
     }
 
@@ -470,6 +470,7 @@
                 var t when t == typeof(byte[]) => CPythonAPI.PyBytes_AsByteArray(this),
                 var t when t.IsAssignableTo(typeof(ITuple)) => PyObjectTypeConverter.ConvertToTuple(this, t),
                 var t when t.IsAssignableTo(typeof(IGeneratorIterator)) => PyObjectTypeConverter.ConvertToGeneratorIterator(this, t),
+                var t when t.IsAssignableTo(typeof(ICoroutine)) => PyObjectTypeConverter.ConvertToCoroutine(this, t),
                 var t => PyObjectTypeConverter.PyObjectToManagedType(this, t),
             };
         }

src/CSnakes.SourceGeneration/Parser/PythonParser.Function.cs

@@ -11,12 +11,13 @@ namespace CSnakes.Parser;
 public static partial class PythonParser
 {
     public static TokenListParser<PythonToken, PythonFunctionDefinition> PythonFunctionDefinitionParser { get; } =
-        (from def in Token.EqualTo(PythonToken.Def)
+        (from @async in Token.EqualTo(PythonToken.Async).OptionalOrDefault()
+         from def in Token.EqualTo(PythonToken.Def)
          from name in Token.EqualTo(PythonToken.Identifier)
          from parameters in PythonParameterListParser.AssumeNotNull()
          from arrow in Token.EqualTo(PythonToken.Arrow).Optional().Then(returnType => PythonTypeDefinitionParser.AssumeNotNull().OptionalOrDefault())
          from colon in Token.EqualTo(PythonToken.Colon)
-         select new PythonFunctionDefinition(name.ToStringValue(), arrow, parameters))
+         select new PythonFunctionDefinition(name.ToStringValue(), arrow, parameters, async.HasValue))
         .Named("Function Definition");
 
     /// <summary>