From a3c445a27643167d303a07ade7c437f1608404b6 Mon Sep 17 00:00:00 2001 From: Arkadiusz Kozdra Date: Tue, 27 Feb 2024 14:07:49 +0100 Subject: [PATCH] [#53417] PoC: Add portable dotnet package support --- .ci.yml | 16 +++- README.md | 2 + src/pyrenode3/__init__.py | 18 +++-- src/pyrenode3/env.py | 2 + src/pyrenode3/loader.py | 158 ++++++++++++++++++++++++++++++-------- 5 files changed, 156 insertions(+), 40 deletions(-) diff --git a/.ci.yml b/.ci.yml index 60fb230..c1d4392 100644 --- a/.ci.yml +++ b/.ci.yml @@ -46,7 +46,21 @@ test-dotnet-build: - pushd renode && ./build.sh --net && popd script: - export PYRENODE_RUNTIME=coreclr - - export PYRENODE_BUILD_DIR=$(pwd)/renode + - export PYRENODE_BUILD_DIR=$PWD/renode + - *run_tests + +test-dotnet-portable: + stage: build + tags: ['ace-x86_64'] + before_script: + - *install_dependencies + - *init_python + # Download Renode pkg + - wget --progress=dot:giga https://builds.renode.io/renode-latest.linux-portable-dotnet.tar.gz + script: + - tar xvf renode-latest.linux-portable-dotnet.tar.gz + - export PYRENODE_RUNTIME=coreclr + - export PYRENODE_BIN=$(realpath $PWD/renode_*-dotnet_portable/renode) - *run_tests test-mono-build: diff --git a/README.md b/README.md index 6440e35..61cdbe8 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ This will spawn a two-machine demo scenario and, when the Linux boots to shell, To modify the output directory used as a source of Renode binaries (location of `Renode.exe`), you must set the `PYRENODE_BUILD_OUTPUT` variable, with a path relative to `PYRENODE_BUILD_DIR`. - `PYRENODE_RUNTIME` -- Specifies runtime which is used to run Renode. Supported runtimes: `mono` (default), `dotnet` (.NET). +- `PYRENODE_BIN` -- Specifies the location of Renode portable binary that will be used by `pyrenode3`. `PYRENODE_PKG` and `PYRENODE_BUILD_DIR` are mutually exclusive. Exactly one of them must be specified to use `pyrenode3` successfully. @@ -47,3 +48,4 @@ Exactly one of them must be specified to use `pyrenode3` successfully. | :----------------- | :----------------: | :----------------: | | Package | :white_check_mark: | :x: | | Built from sources | :white_check_mark: | :white_check_mark: | +| Portable binary | :x: | :white_check_mark: | diff --git a/src/pyrenode3/__init__.py b/src/pyrenode3/__init__.py index 1ab9a2d..4de4b3c 100644 --- a/src/pyrenode3/__init__.py +++ b/src/pyrenode3/__init__.py @@ -9,12 +9,11 @@ runtime = env.pyrenode_runtime if runtime not in ["mono", "coreclr"]: - raise ImportError(f"Runtime '{runtime}' not supported") + raise ImportError(f"Runtime {runtime!r} not supported") - if env.pyrenode_pkg and env.pyrenode_build_dir: + if sum(map(bool, (env.pyrenode_pkg, env.pyrenode_build_dir, env.pyrenode_bin))) > 1: raise ImportError( - f"Both {env.PYRENODE_PKG} and {env.PYRENODE_BUILD_DIR} are set. " - "Please unset one of them." + f"Multiple of {env.PYRENODE_PKG}, {env.PYRENODE_BUILD_DIR}, {env.PYRENODE_BIN} are set. Please unset all but one of them." ) if env.pyrenode_pkg: @@ -30,10 +29,17 @@ elif runtime == "coreclr": RenodeLoader.from_net_build(env.pyrenode_build_dir) + elif env.pyrenode_bin: + if runtime == "mono": + raise ImportError("Using mono portable binary is not supported.") + elif runtime == "coreclr": + RenodeLoader.from_net_bin(env.pyrenode_bin) + if not RenodeLoader().is_initialized: msg = ( - f"Renode not found. Please set {env.PYRENODE_PKG} to the location of Renode package or " - f"{env.PYRENODE_BUILD_DIR} to the location of Renode build directory." + f"Renode not found. Please set {env.PYRENODE_PKG} to the location of Renode package " + f"or {env.PYRENODE_BUILD_DIR} to the location of Renode build directory, " + f"or {env.PYRENODE_BIN} to the location of Renode portable binary." ) raise ImportError(msg) diff --git a/src/pyrenode3/env.py b/src/pyrenode3/env.py index 38d9591..b88e922 100644 --- a/src/pyrenode3/env.py +++ b/src/pyrenode3/env.py @@ -1,6 +1,7 @@ import os # Env variable names +PYRENODE_BIN = "PYRENODE_BIN" PYRENODE_BUILD_DIR = "PYRENODE_BUILD_DIR" PYRENODE_BUILD_OUTPUT = "PYRENODE_BUILD_OUTPUT" PYRENODE_PKG = "PYRENODE_PKG" @@ -8,6 +9,7 @@ PYRENODE_SKIP_LOAD = "PYRENODE_SKIP_LOAD" # Values of env variables +pyrenode_bin = os.environ.get(PYRENODE_BIN) pyrenode_build_dir = os.environ.get(PYRENODE_BUILD_DIR) pyrenode_build_output = os.environ.get(PYRENODE_BUILD_OUTPUT) pyrenode_pkg = os.environ.get(PYRENODE_PKG) diff --git a/src/pyrenode3/loader.py b/src/pyrenode3/loader.py index 3f5eb46..8f8e7d9 100644 --- a/src/pyrenode3/loader.py +++ b/src/pyrenode3/loader.py @@ -1,14 +1,18 @@ import glob +import json import logging import os import pathlib +import re import sys import tarfile import tempfile from contextlib import contextmanager from typing import Union +from subprocess import check_output, STDOUT from pythonnet import load as pythonnet_load +from clr_loader.util.runtime_spec import DotnetCoreRuntimeSpec from pyrenode3 import env from pyrenode3.singleton import MetaSingleton @@ -18,6 +22,18 @@ class InitializationError(Exception): ... +def ensure_symlink(src, dst, relative=False, verbose=False): + if relative: + src = os.path.relpath(src, dst.parent) + try: + dst.symlink_to(src) + except FileExistsError: + return + if verbose: + logging.warning(f"{dst.name} is not in the expected location. Created symlink.") + logging.warning(f"{src} -> {dst}") + + class RenodeLoader(metaclass=MetaSingleton): """A class used for loading Renode DLLs, platforms and scripts from various sources.""" @@ -59,11 +75,12 @@ def from_mono_arch_pkg(cls, path: "Union[str, pathlib.Path]"): renode_dir = pathlib.Path(temp.name) / "opt/renode" + pythonnet_load("mono") + loader = cls() loader.__setup( renode_dir / "bin", renode_dir, - runtime="mono", temp=temp, add_dlls=["Renode.exe"] ) @@ -102,11 +119,13 @@ def discover_bin_dir(renode_dir, runtime): def from_mono_build(cls, path: "Union[str, pathlib.Path]"): """Load Renode from Mono build.""" renode_dir = pathlib.Path(path) + + pythonnet_load("mono") + loader = cls() loader.__setup( cls.discover_bin_dir(renode_dir, "mono"), renode_dir, - runtime="mono", add_dlls=["Renode.exe"] ) @@ -118,24 +137,113 @@ def from_net_build(cls, path: "Union[str, pathlib.Path]"): renode_bin_dir = cls.discover_bin_dir(renode_dir, "coreclr") # HACK: move libMonoPosixHelper.so to path where it is searched for - src = renode_bin_dir / "runtimes/linux-x64/native/libMonoPosixHelper.so" - dst = renode_bin_dir / "runtimes/linux-x64/lib/netstandard2.0/libMonoPosixHelper.so" - if not dst.exists(): - src_path = os.path.relpath(src, dst.parent) - logging.warning("libMonoPosixHelper.so is not in the expected location. Creating symlink.") - logging.warning(f"{src_path} -> {dst}") - os.symlink(src_path, dst) + bindll_dir = renode_bin_dir / "runtimes/linux-x64" + src = bindll_dir / "native/libMonoPosixHelper.so" + netstd_dir = bindll_dir / "lib/netstandard2.0" + ensure_symlink(src, netstd_dir / "libMonoPosixHelper.so", relative=True, verbose=True) + + pythonnet_load("coreclr", runtime_config=renode_bin_dir / "Renode.runtimeconfig.json") loader = cls() loader.__setup( renode_bin_dir, renode_dir, - runtime="coreclr", - add_dlls=["runtimes/linux-x64/lib/netstandard2.0/Mono.Posix.NETStandard.dll"], + add_dlls=[netstd_dir / "Mono.Posix.NETStandard.dll"], ) return loader + @classmethod + def from_net_bin(cls, path: "Union[str, pathlib.Path]"): + """Load Renode from binary.""" + renode_bin = pathlib.Path(path) + renode_dir = renode_bin.parent + + # As a side effect, executing the binary causes the embedded dlls to be extracted to: + # ~/.net/// + # The location gets printed to stderr (or selected file) if suitable environment variables are set. + out = check_output([renode_bin, "--version"], stderr=STDOUT, env=os.environ | {"COREHOST_TRACE": "1", "COREHOST_TRACEFILE": ""}, text=True) + + binaries = re.search(r"will be extracted to \[(.*)\] directory", out).group(1) + binaries = pathlib.Path(binaries) + + # There should be *some* way to specify a dll PATH, but it does not 'just work' e.g. in runtimeconfig.json. + # As a workaround, we create a directory hierarchy (can be anywhere, but we use ~/.net/...) like + # shared/Microsoft.NETCore.App/6.0.26/ + # libclrjit.so + # libcoreclr.so + # libSystem.Native.so + # libSystem.Security.Cryptography.Native.OpenSsl.so + # Microsoft.CSharp.dll + # ... + # Microsoft.NETCore.App.deps.json + # The DLLs are extracted, the .so libs are blended into the .text of the executable, so we ship them, + # and the deps.json can be pretty much any deps.json file, so we use the extracted Renode.deps.json. + + # We need to find *some* runtime version, although 6.0.0 is 'good enough' if we find nothing else. + # Luckily, deps.json has the runtime version info, and a list of system DLLs: + # { + # "runtimeTarget": { + # "name": ".NETCoreApp,Version=v6.0/linux-x64", + # "signature": "" + # }, + # "compilationOptions": {}, + # "targets": { + # ".NETCoreApp,Version=v6.0": {}, + # ".NETCoreApp,Version=v6.0/linux-x64": { + # "Renode/1.0.0": { + # "dependencies": { + # "AntShell": "1.0.0", + # ... + # "runtimepack.Microsoft.NETCore.App.Runtime.linux-x64": "6.0.26" + # }, + # "runtime": { + # "Renode.dll": {} + # } + # }, + # "runtimepack.Microsoft.NETCore.App.Runtime.linux-x64/6.0.26": { + # "runtime": { + # "Microsoft.CSharp.dll": { + # "assemblyVersion": "6.0.0.0", + # "fileVersion": "6.0.2623.60508" + # }, + # "Microsoft.VisualBasic.Core.dll": { ... }, + # }}}}} + SYSTEM_RUNTIME = "runtimepack.Microsoft.NETCore.App.Runtime.linux-x64" + + deps = json.load(open(binaries / "Renode.deps.json", "rb")) + target = deps["targets"][deps["runtimeTarget"]["name"]] + for lib, dlls in target.items(): + name, version = lib.split("/") + if name == SYSTEM_RUNTIME: + tfm_full = version + system_dlls = list(dlls["runtime"]) + break + else: + tfm_full = "6.0.0" + system_dlls = [dll.name for dll in binaries.glob("*.dll")] + logging.warning(f"Could not find {SYSTEM_RUNTIME} in deps.json. " + f"Assuming framework version {tfm_full}.") + + runtime = binaries / "shared/Microsoft.NETCore.App" / tfm_full + runtime.mkdir(parents=True, exist_ok=True) + for lib in renode_dir.glob("*.so"): + ensure_symlink(lib, runtime / lib.name) + + for lib in system_dlls: + ensure_symlink(binaries / lib, runtime / lib, relative=True) + + ensure_symlink(renode_dir / "libhostfxr.so", binaries / "libhostfxr.so") + ensure_symlink(binaries / "Renode.deps.json", runtime / "Microsoft.NETCore.App.deps.json", relative=True) + + loader = cls() + loader.__renode_dir = renode_dir + with loader.in_root(): + pythonnet_load("coreclr", dotnet_root=binaries, runtime_spec=DotnetCoreRuntimeSpec("Microsoft.NETCore.App", tfm_full, runtime)) + loader.__setup(binaries, renode_dir) + + return loader + @contextmanager def in_root(self): last_cwd = os.getcwd() @@ -145,38 +253,25 @@ def in_root(self): finally: os.chdir(last_cwd) - def __load_runtime(self): - params = {} - if self.__runtime == "coreclr": - # When using .NET Renode and the runtimeconfig.json file is present we should use that - # to specify exactly the runtime that is expected. - runtime_config = self.__bin_dir / "Renode.runtimeconfig.json" - if runtime_config.exists(): - params["runtime_config"] = runtime_config - else: - logging.warning( - "Can't find the Renode.runtimeconfig.json file. " - "Will use a default pythonnet runtime settings." - ) - - pythonnet_load(self.__runtime, **params) - def __load_asm(self): # Import clr here, because it must be done after the proper runtime is selected. # If the runtime isn't loaded, the clr module loads the default runtime (mono) automatically. # It is an issue when we use non-default runtime, e.g. coreclr. import clr - dlls = [*glob.glob(str(self.binaries / "*.dll"))] + dlls = [*self.binaries.glob("*.dll")] dlls.extend(self.__extra.get("add_dlls", [])) for dll in dlls: - clr.AddReference(str(self.binaries / dll)) + fullpath = self.binaries / dll + # We do not normally ship CoreLib (except portable), and it gets loaded by other dlls anyway, but loading it directly raises an error: + # System.IO.FileLoadException: Could not load file or assembly 'System.Private.CoreLib, Version=6.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e'. + if fullpath.exists() and fullpath.name != "System.Private.CoreLib.dll": + clr.AddReference(str(fullpath)) def __setup(self, bin_dir: "Union[str, pathlib.Path]", renode_dir: "Union[str, pathlib.Path]", - runtime: str, **kwargs, ): if self.__initialized: @@ -185,11 +280,8 @@ def __setup(self, self.__bin_dir = pathlib.Path(bin_dir).absolute() self.__renode_dir = pathlib.Path(renode_dir).absolute() - self.__runtime = runtime self.__extra = kwargs - self.__load_runtime() - self.__load_asm() self.__initialized = True