diff --git a/clr.py b/clr.py index 711333dd2..c993b517c 100644 --- a/clr.py +++ b/clr.py @@ -2,40 +2,8 @@ Legacy Python.NET loader for backwards compatibility """ -def _get_netfx_path(): - import os, sys - - if sys.maxsize > 2 ** 32: - arch = "amd64" - else: - arch = "x86" - - return os.path.join(os.path.dirname(__file__), "pythonnet", "netfx", arch, "clr.pyd") - - -def _get_mono_path(): - import os, glob - - paths = glob.glob(os.path.join(os.path.dirname(__file__), "pythonnet", "mono", "clr.*so")) - return paths[0] - - def _load_clr(): - import sys - from importlib import util - - if sys.platform == "win32": - path = _get_netfx_path() - else: - path = _get_mono_path() - - del sys.modules[__name__] - - spec = util.spec_from_file_location("clr", path) - clr = util.module_from_spec(spec) - spec.loader.exec_module(clr) - - sys.modules[__name__] = clr - + from pythonnet import load + load() _load_clr() diff --git a/pythonnet.sln b/pythonnet.sln index fcad97d5c..a803d0248 100644 --- a/pythonnet.sln +++ b/pythonnet.sln @@ -4,9 +4,9 @@ VisualStudioVersion = 16.0.30717.126 MinimumVisualStudioVersion = 15.0.26124.0 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Runtime", "src\runtime\Python.Runtime.csproj", "{4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Console", "src\console\Console.csproj", "{E6B01706-00BA-4144-9029-186AC42FBE9A}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Python.Loader", "src\loader\Python.Loader.csproj", "{B9F0A702-A977-47E3-88FF-6082E9614F40}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "clrmodule", "src\clrmodule\clrmodule.csproj", "{F9F5FA13-BC52-4C0B-BC1C-FE3C0B8FCCDD}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Console", "src\console\Console.csproj", "{E6B01706-00BA-4144-9029-186AC42FBE9A}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.EmbeddingTest", "src\embed_tests\Python.EmbeddingTest.csproj", "{819E089B-4770-400E-93C6-4F7A35F0EA12}" EndProject @@ -129,6 +129,30 @@ Global {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Release|x64.Build.0 = Release|x64 {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Release|x86.ActiveCfg = Release|x86 {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Release|x86.Build.0 = Release|x86 + {16288AA7-EB9F-45CC-90C0-3AFA7715BA8A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {16288AA7-EB9F-45CC-90C0-3AFA7715BA8A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {16288AA7-EB9F-45CC-90C0-3AFA7715BA8A}.Debug|x64.ActiveCfg = Debug|Any CPU + {16288AA7-EB9F-45CC-90C0-3AFA7715BA8A}.Debug|x64.Build.0 = Debug|Any CPU + {16288AA7-EB9F-45CC-90C0-3AFA7715BA8A}.Debug|x86.ActiveCfg = Debug|Any CPU + {16288AA7-EB9F-45CC-90C0-3AFA7715BA8A}.Debug|x86.Build.0 = Debug|Any CPU + {16288AA7-EB9F-45CC-90C0-3AFA7715BA8A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {16288AA7-EB9F-45CC-90C0-3AFA7715BA8A}.Release|Any CPU.Build.0 = Release|Any CPU + {16288AA7-EB9F-45CC-90C0-3AFA7715BA8A}.Release|x64.ActiveCfg = Release|Any CPU + {16288AA7-EB9F-45CC-90C0-3AFA7715BA8A}.Release|x64.Build.0 = Release|Any CPU + {16288AA7-EB9F-45CC-90C0-3AFA7715BA8A}.Release|x86.ActiveCfg = Release|Any CPU + {16288AA7-EB9F-45CC-90C0-3AFA7715BA8A}.Release|x86.Build.0 = Release|Any CPU + {B9F0A702-A977-47E3-88FF-6082E9614F40}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B9F0A702-A977-47E3-88FF-6082E9614F40}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B9F0A702-A977-47E3-88FF-6082E9614F40}.Debug|x64.ActiveCfg = Debug|Any CPU + {B9F0A702-A977-47E3-88FF-6082E9614F40}.Debug|x64.Build.0 = Debug|Any CPU + {B9F0A702-A977-47E3-88FF-6082E9614F40}.Debug|x86.ActiveCfg = Debug|Any CPU + {B9F0A702-A977-47E3-88FF-6082E9614F40}.Debug|x86.Build.0 = Debug|Any CPU + {B9F0A702-A977-47E3-88FF-6082E9614F40}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B9F0A702-A977-47E3-88FF-6082E9614F40}.Release|Any CPU.Build.0 = Release|Any CPU + {B9F0A702-A977-47E3-88FF-6082E9614F40}.Release|x64.ActiveCfg = Release|Any CPU + {B9F0A702-A977-47E3-88FF-6082E9614F40}.Release|x64.Build.0 = Release|Any CPU + {B9F0A702-A977-47E3-88FF-6082E9614F40}.Release|x86.ActiveCfg = Release|Any CPU + {B9F0A702-A977-47E3-88FF-6082E9614F40}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/pythonnet/__init__.py b/pythonnet/__init__.py index 5c197e146..422795a60 100644 --- a/pythonnet/__init__.py +++ b/pythonnet/__init__.py @@ -1,3 +1,64 @@ -def get_assembly_path(): - import os - return os.path.dirname(__file__) + "/runtime/Python.Runtime.dll" +import os +import sys +import clr_loader + +_RUNTIME = None +_LOADER_ASSEMBLY = None +_FFI = None +_LOADED = False + + +def set_runtime(runtime): + global _RUNTIME + _RUNTIME = runtime + + +def set_default_runtime(): + if sys.platform == 'win32': + set_runtime(clr_loader.get_netfx()) + else: + set_runtime(clr_loader.get_mono(gc="")) + + +def load(): + global _FFI, _LOADED, _LOADER_ASSEMBLY + + if _LOADED: + return + + from .util import find_libpython + from os.path import join, dirname, basename + + if _RUNTIME is None: + # TODO: Warn, in the future the runtime must be set explicitly, either as a + # config/env variable or via set_runtime + set_default_runtime() + + dll_path = join(dirname(__file__), "runtime", "Python.Loader.dll") + runtime_dll_path = join(dirname(dll_path), "Python.Runtime.dll") + libpython = basename(find_libpython()) + # TODO: Add dirname of libpython to (DY)LD_LIBRARY_PATH or PATH + + if _FFI is None and libpython != "__Internal" and sys.platform != "win32": + # Load and leak libpython handle s.t. the .NET runtime doesn't dlcloses it + import posix + + import cffi + _FFI = cffi.FFI() + _FFI.dlopen(libpython, posix.RTLD_NODELETE | posix.RTLD_LOCAL) + + _LOADER_ASSEMBLY = _RUNTIME.get_assembly(dll_path) + + func = _LOADER_ASSEMBLY["Python.Loader.Internal.Initialize"] + if func(f"{runtime_dll_path};{libpython}".encode("utf8")) != 0: + raise RuntimeError("Failed to initialize Python.Runtime.dll") + + import atexit + atexit.register(unload) + + +def unload(): + if _LOADER_ASSEMBLY is not None: + func = _LOADER_ASSEMBLY["Python.Loader.Internal.Shutdown"] + if func(b"") != 0: + raise RuntimeError("Failed to call Python.NET shutdown") diff --git a/pythonnet/util/__init__.py b/pythonnet/util/__init__.py new file mode 100644 index 000000000..75d4bad8c --- /dev/null +++ b/pythonnet/util/__init__.py @@ -0,0 +1 @@ +from .find_libpython import find_libpython diff --git a/pythonnet/util/find_libpython.py b/pythonnet/util/find_libpython.py new file mode 100644 index 000000000..422a0200a --- /dev/null +++ b/pythonnet/util/find_libpython.py @@ -0,0 +1,395 @@ +#!/usr/bin/env python + +""" +Locate libpython associated with this Python executable. +""" + +# License +# +# Copyright 2018, Takafumi Arakaki +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from __future__ import print_function, absolute_import + +from logging import getLogger +import ctypes.util +import functools +import os +import sys +import sysconfig + +logger = getLogger("find_libpython") + +is_windows = os.name == "nt" +is_apple = sys.platform == "darwin" + +SHLIB_SUFFIX = sysconfig.get_config_var("SHLIB_SUFFIX") +if SHLIB_SUFFIX is None: + if is_windows: + SHLIB_SUFFIX = ".dll" + else: + SHLIB_SUFFIX = ".so" +if is_apple: + # sysconfig.get_config_var("SHLIB_SUFFIX") can be ".so" in macOS. + # Let's not use the value from sysconfig. + SHLIB_SUFFIX = ".dylib" + + +def linked_libpython(): + """ + Find the linked libpython using dladdr (in *nix). + + Returns + ------- + path : str or None + A path to linked libpython. Return `None` if statically linked. + """ + if is_windows: + return _linked_libpython_windows() + return _linked_libpython_unix() + + +class Dl_info(ctypes.Structure): + _fields_ = [ + ("dli_fname", ctypes.c_char_p), + ("dli_fbase", ctypes.c_void_p), + ("dli_sname", ctypes.c_char_p), + ("dli_saddr", ctypes.c_void_p), + ] + + +def _linked_libpython_unix(): + libdl = ctypes.CDLL(ctypes.util.find_library("dl")) + libdl.dladdr.argtypes = [ctypes.c_void_p, ctypes.POINTER(Dl_info)] + libdl.dladdr.restype = ctypes.c_int + + dlinfo = Dl_info() + retcode = libdl.dladdr( + ctypes.cast(ctypes.pythonapi.Py_GetVersion, ctypes.c_void_p), + ctypes.pointer(dlinfo)) + if retcode == 0: # means error + return None + path = os.path.realpath(dlinfo.dli_fname.decode()) + if path == os.path.realpath(sys.executable): + return None + return path + + +def _linked_libpython_windows(): + """ + Based on: https://stackoverflow.com/a/16659821 + """ + from ctypes.wintypes import HANDLE, LPWSTR, DWORD + + GetModuleFileName = ctypes.windll.kernel32.GetModuleFileNameW + GetModuleFileName.argtypes = [HANDLE, LPWSTR, DWORD] + GetModuleFileName.restype = DWORD + + MAX_PATH = 260 + try: + buf = ctypes.create_unicode_buffer(MAX_PATH) + GetModuleFileName(ctypes.pythonapi._handle, buf, MAX_PATH) + return buf.value + except (ValueError, OSError): + return None + + + +def library_name(name, suffix=SHLIB_SUFFIX, is_windows=is_windows): + """ + Convert a file basename `name` to a library name (no "lib" and ".so" etc.) + + >>> library_name("libpython3.7m.so") # doctest: +SKIP + 'python3.7m' + >>> library_name("libpython3.7m.so", suffix=".so", is_windows=False) + 'python3.7m' + >>> library_name("libpython3.7m.dylib", suffix=".dylib", is_windows=False) + 'python3.7m' + >>> library_name("python37.dll", suffix=".dll", is_windows=True) + 'python37' + """ + if not is_windows and name.startswith("lib"): + name = name[len("lib"):] + if suffix and name.endswith(suffix): + name = name[:-len(suffix)] + return name + + +def append_truthy(list, item): + if item: + list.append(item) + + +def uniquifying(items): + """ + Yield items while excluding the duplicates and preserving the order. + + >>> list(uniquifying([1, 2, 1, 2, 3])) + [1, 2, 3] + """ + seen = set() + for x in items: + if x not in seen: + yield x + seen.add(x) + + +def uniquified(func): + """ Wrap iterator returned from `func` by `uniquifying`. """ + @functools.wraps(func) + def wrapper(*args, **kwds): + return uniquifying(func(*args, **kwds)) + return wrapper + + +@uniquified +def candidate_names(suffix=SHLIB_SUFFIX): + """ + Iterate over candidate file names of libpython. + + Yields + ------ + name : str + Candidate name libpython. + """ + LDLIBRARY = sysconfig.get_config_var("LDLIBRARY") + if LDLIBRARY: + yield LDLIBRARY + + LIBRARY = sysconfig.get_config_var("LIBRARY") + if LIBRARY: + yield os.path.splitext(LIBRARY)[0] + suffix + + dlprefix = "" if is_windows else "lib" + sysdata = dict( + v=sys.version_info, + # VERSION is X.Y in Linux/macOS and XY in Windows: + VERSION=(sysconfig.get_python_version() or + "{v.major}.{v.minor}".format(v=sys.version_info) or + sysconfig.get_config_var("VERSION")), + ABIFLAGS=(sysconfig.get_config_var("ABIFLAGS") or + sysconfig.get_config_var("abiflags") or ""), + ) + + for stem in [ + "python{VERSION}{ABIFLAGS}".format(**sysdata), + "python{VERSION}".format(**sysdata), + "python{v.major}".format(**sysdata), + "python", + ]: + yield dlprefix + stem + suffix + + + +@uniquified +def candidate_paths(suffix=SHLIB_SUFFIX): + """ + Iterate over candidate paths of libpython. + + Yields + ------ + path : str or None + Candidate path to libpython. The path may not be a fullpath + and may not exist. + """ + + yield linked_libpython() + + # List candidates for directories in which libpython may exist + lib_dirs = [] + append_truthy(lib_dirs, sysconfig.get_config_var('LIBPL')) + append_truthy(lib_dirs, sysconfig.get_config_var('srcdir')) + append_truthy(lib_dirs, sysconfig.get_config_var("LIBDIR")) + + # LIBPL seems to be the right config_var to use. It is the one + # used in python-config when shared library is not enabled: + # https://github.com/python/cpython/blob/v3.7.0/Misc/python-config.in#L55-L57 + # + # But we try other places just in case. + + if is_windows: + lib_dirs.append(os.path.join(os.path.dirname(sys.executable))) + else: + lib_dirs.append(os.path.join( + os.path.dirname(os.path.dirname(sys.executable)), + "lib")) + + # For macOS: + append_truthy(lib_dirs, sysconfig.get_config_var("PYTHONFRAMEWORKPREFIX")) + + lib_dirs.append(sys.exec_prefix) + lib_dirs.append(os.path.join(sys.exec_prefix, "lib")) + + lib_basenames = list(candidate_names(suffix=suffix)) + + for directory in lib_dirs: + for basename in lib_basenames: + yield os.path.join(directory, basename) + + # In macOS and Windows, ctypes.util.find_library returns a full path: + for basename in lib_basenames: + yield ctypes.util.find_library(library_name(basename)) + +# Possibly useful links: +# * https://packages.ubuntu.com/bionic/amd64/libpython3.6/filelist +# * https://github.com/Valloric/ycmd/issues/518 +# * https://github.com/Valloric/ycmd/pull/519 + + +def normalize_path(path, suffix=SHLIB_SUFFIX, is_apple=is_apple): + """ + Normalize shared library `path` to a real path. + + If `path` is not a full path, `None` is returned. If `path` does + not exists, append `SHLIB_SUFFIX` and check if it exists. + Finally, the path is canonicalized by following the symlinks. + + Parameters + ---------- + path : str ot None + A candidate path to a shared library. + """ + if not path: + return None + if not os.path.isabs(path): + return None + if os.path.exists(path): + return os.path.realpath(path) + if os.path.exists(path + suffix): + return os.path.realpath(path + suffix) + if is_apple: + return normalize_path(_remove_suffix_apple(path), + suffix=".so", is_apple=False) + return None + + +def _remove_suffix_apple(path): + """ + Strip off .so or .dylib. + + >>> _remove_suffix_apple("libpython.so") + 'libpython' + >>> _remove_suffix_apple("libpython.dylib") + 'libpython' + >>> _remove_suffix_apple("libpython3.7") + 'libpython3.7' + """ + if path.endswith(".dylib"): + return path[:-len(".dylib")] + if path.endswith(".so"): + return path[:-len(".so")] + return path + + +@uniquified +def finding_libpython(): + """ + Iterate over existing libpython paths. + + The first item is likely to be the best one. + + Yields + ------ + path : str + Existing path to a libpython. + """ + logger.debug("is_windows = %s", is_windows) + logger.debug("is_apple = %s", is_apple) + for path in candidate_paths(): + logger.debug("Candidate: %s", path) + normalized = normalize_path(path) + if normalized: + logger.debug("Found: %s", normalized) + yield normalized + else: + logger.debug("Not found.") + + +def find_libpython(): + """ + Return a path (`str`) to libpython or `None` if not found. + + Parameters + ---------- + path : str or None + Existing path to the (supposedly) correct libpython. + """ + for path in finding_libpython(): + return os.path.realpath(path) + + +def print_all(items): + for x in items: + print(x) + + +def cli_find_libpython(cli_op, verbose): + import logging + # Importing `logging` module here so that using `logging.debug` + # instead of `logger.debug` outside of this function becomes an + # error. + + if verbose: + logging.basicConfig( + format="%(levelname)s %(message)s", + level=logging.DEBUG) + + if cli_op == "list-all": + print_all(finding_libpython()) + elif cli_op == "candidate-names": + print_all(candidate_names()) + elif cli_op == "candidate-paths": + print_all(p for p in candidate_paths() if p and os.path.isabs(p)) + else: + path = find_libpython() + if path is None: + return 1 + print(path, end="") + + +def main(args=None): + import argparse + parser = argparse.ArgumentParser( + description=__doc__) + parser.add_argument( + "--verbose", "-v", action="store_true", + help="Print debugging information.") + + group = parser.add_mutually_exclusive_group() + group.add_argument( + "--list-all", + action="store_const", dest="cli_op", const="list-all", + help="Print list of all paths found.") + group.add_argument( + "--candidate-names", + action="store_const", dest="cli_op", const="candidate-names", + help="Print list of candidate names of libpython.") + group.add_argument( + "--candidate-paths", + action="store_const", dest="cli_op", const="candidate-paths", + help="Print list of candidate paths of libpython.") + + ns = parser.parse_args(args) + parser.exit(cli_find_libpython(**vars(ns))) + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 06a26ef95..ab6b8506e 100644 --- a/setup.py +++ b/setup.py @@ -140,8 +140,8 @@ def run(self): opts.extend(["--configuration", self.dotnet_config]) opts.extend(["--output", output]) - self.announce("Running dotnet build...", level=distutils.log.INFO) - self.spawn(["dotnet", "build", lib.path] + opts) + self.announce("Running dotnet publish...", level=distutils.log.INFO) + self.spawn(["dotnet", "publish", lib.path] + opts) for k, v in rename.items(): source = os.path.join(output, k) @@ -198,60 +198,12 @@ def install_for_development(self): dotnet_libs = [ DotnetLib( - "python-runtime", - "src/runtime/Python.Runtime.csproj", + "pythonnet-loader", + "src/loader/Python.Loader.csproj", output="pythonnet/runtime", ) ] -if BUILD_NETFX: - dotnet_libs.extend( - [ - DotnetLib( - "clrmodule-amd64", - "src/clrmodule/", - runtime="win-x64", - output="pythonnet/netfx/amd64", - rename={"clr.dll": "clr.pyd"}, - ), - DotnetLib( - "clrmodule-x86", - "src/clrmodule/", - runtime="win-x86", - output="pythonnet/netfx/x86", - rename={"clr.dll": "clr.pyd"}, - ), - ] - ) - -ext_modules = [] - -if BUILD_MONO: - try: - mono_libs = check_output( - "pkg-config --libs mono-2", shell=True, encoding="utf8" - ) - mono_cflags = check_output( - "pkg-config --cflags mono-2", shell=True, encoding="utf8" - ) - cflags = mono_cflags.strip() - libs = mono_libs.strip() - - # build the clr python module - clr_ext = Extension( - "pythonnet.mono.clr", - language="c++", - sources=["src/monoclr/clrmod.c"], - extra_compile_args=cflags.split(" "), - extra_link_args=libs.split(" "), - ) - ext_modules.append(clr_ext) - except Exception: - print( - "Failed to find mono libraries via pkg-config, skipping the Mono CLR loader" - ) - - setup( cmdclass=cmdclass, name="pythonnet", @@ -261,12 +213,11 @@ def install_for_development(self): license="MIT", author="The Contributors of the Python.NET Project", author_email="pythonnet@python.org", - packages=["pythonnet"], - install_requires=["pycparser"], + packages=["pythonnet", "pythonnet.util"], + install_requires=["pycparser", "clr_loader"], long_description=long_description, # data_files=[("{install_platlib}", ["{build_lib}/pythonnet"])], py_modules=["clr"], - ext_modules=ext_modules, dotnet_libs=dotnet_libs, classifiers=[ "Development Status :: 5 - Production/Stable", diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs deleted file mode 100644 index 7b0387d46..000000000 --- a/src/clrmodule/ClrModule.cs +++ /dev/null @@ -1,113 +0,0 @@ -//============================================================================ -// This file replaces the hand-maintained stub that used to implement clr.dll. -// This is a line-by-line port from IL back to C#. -// We now use RGiesecke.DllExport on the required static init method so it can be -// loaded by a standard CPython interpreter as an extension module. When it -// is loaded, it bootstraps the managed runtime integration layer and defers -// to it to do initialization and put the clr module into sys.modules, etc. - -// The "USE_PYTHON_RUNTIME_*" defines control what extra evidence is used -// to help the CLR find the appropriate Python.Runtime assembly. - -// If defined, the "pythonRuntimeVersionString" variable must be set to -// Python.Runtime's current version. -#define USE_PYTHON_RUNTIME_VERSION - -// If defined, the "PythonRuntimePublicKeyTokenData" data array must be -// set to Python.Runtime's public key token. (sn -T Python.Runtin.dll) -#define USE_PYTHON_RUNTIME_PUBLIC_KEY_TOKEN - -// If DEBUG is defined in the Build Properties, a few Console.WriteLine -// calls are made to indicate what's going on during the load... -//============================================================================ -using System; -using System.Diagnostics; -using System.Globalization; -using System.IO; -using System.Reflection; -using System.Runtime.InteropServices; -using NXPorts.Attributes; - -public class clrModule -{ - [DllExport("PyInit_clr", CallingConvention.StdCall)] - public static IntPtr PyInit_clr() - { - DebugPrint("Attempting to load 'Python.Runtime' using standard binding rules."); -#if USE_PYTHON_RUNTIME_PUBLIC_KEY_TOKEN - var pythonRuntimePublicKeyTokenData = new byte[] { 0x50, 0x00, 0xfe, 0xa6, 0xcb, 0xa7, 0x02, 0xdd }; -#endif - - // Attempt to find and load Python.Runtime using standard assembly binding rules. - // This roughly translates into looking in order: - // - GAC - // - ApplicationBase - // - A PrivateBinPath under ApplicationBase - // With an unsigned assembly, the GAC is skipped. - var pythonRuntimeName = new AssemblyName("Python.Runtime") - { -#if USE_PYTHON_RUNTIME_VERSION - // Has no effect until SNK works. Keep updated anyways. - Version = new Version("2.5.0"), -#endif - CultureInfo = CultureInfo.InvariantCulture - }; -#if USE_PYTHON_RUNTIME_PUBLIC_KEY_TOKEN - pythonRuntimeName.SetPublicKeyToken(pythonRuntimePublicKeyTokenData); -#endif - // We've got the AssemblyName with optional features; try to load it. - Assembly pythonRuntime; - try - { - pythonRuntime = Assembly.Load(pythonRuntimeName); - DebugPrint("Success loading 'Python.Runtime' using standard binding rules."); - } - catch (IOException) - { - DebugPrint("'Python.Runtime' not found using standard binding rules."); - try - { - // If the above fails for any reason, we fallback to attempting to load "Python.Runtime.dll" - // from the directory this assembly is running in. "This assembly" is probably "clr.pyd", - // sitting somewhere in PYTHONPATH. This is using Assembly.LoadFrom, and inherits all the - // caveats of that call. See MSDN docs for details. - // Suzanne Cook's blog is also an excellent source of info on this: - // http://blogs.msdn.com/suzcook/ - // http://blogs.msdn.com/suzcook/archive/2003/05/29/57143.aspx - // http://blogs.msdn.com/suzcook/archive/2003/06/13/57180.aspx - - Assembly executingAssembly = Assembly.GetExecutingAssembly(); - string assemblyDirectory = Path.GetDirectoryName(executingAssembly.Location); - if (assemblyDirectory == null) - { - throw new InvalidOperationException(executingAssembly.Location); - } - string pythonRuntimeDllPath = Path.Combine(assemblyDirectory, "Python.Runtime.dll"); - DebugPrint($"Attempting to load Python.Runtime from: '{pythonRuntimeDllPath}'."); - pythonRuntime = Assembly.LoadFrom(pythonRuntimeDllPath); - DebugPrint($"Success loading 'Python.Runtime' from: '{pythonRuntimeDllPath}'."); - } - catch (InvalidOperationException) - { - DebugPrint("Could not load 'Python.Runtime'."); - return IntPtr.Zero; - } - } - - // Once here, we've successfully loaded SOME version of Python.Runtime - // So now we get the PythonEngine and execute the InitExt method on it. - Type pythonEngineType = pythonRuntime.GetType("Python.Runtime.PythonEngine"); - - return (IntPtr)pythonEngineType.InvokeMember("InitExt", BindingFlags.InvokeMethod, null, null, null); - } - - /// - /// Substitute for Debug.Writeline(...). Ideally we would use Debug.Writeline - /// but haven't been able to configure the TRACE from within Python. - /// - [Conditional("DEBUG")] - private static void DebugPrint(string str) - { - Console.WriteLine(str); - } -} diff --git a/src/clrmodule/Properties/AssemblyInfo.cs b/src/clrmodule/Properties/AssemblyInfo.cs deleted file mode 100644 index 5e2e05ed4..000000000 --- a/src/clrmodule/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,5 +0,0 @@ -using System.Reflection; -using System.Runtime.InteropServices; - -// The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("ae10d6a4-55c2-482f-9716-9988e6c169e3")] diff --git a/src/clrmodule/clrmodule.csproj b/src/clrmodule/clrmodule.csproj deleted file mode 100644 index 8595fd0ba..000000000 --- a/src/clrmodule/clrmodule.csproj +++ /dev/null @@ -1,24 +0,0 @@ - - - net472 - win-x86;win-x64 - clr - - - - - - - 1.0.0 - all - runtime; build; native; contentfiles; analyzers - - - - - x86 - - - x64 - - diff --git a/src/loader/Loader.cs b/src/loader/Loader.cs new file mode 100644 index 000000000..6f69d0725 --- /dev/null +++ b/src/loader/Loader.cs @@ -0,0 +1,114 @@ +using System.Runtime.InteropServices; +using System.Text; +using System; +using System.IO; +using System.Reflection; +using Mono.Cecil; + +namespace Python.Loader +{ + public class RuntimeLoader + { + AssemblyDefinition assembly; + + static public RuntimeLoader FromFile(string filename) + { + return new RuntimeLoader(File.OpenRead(filename)); + } + + public RuntimeLoader(Stream stream) + { + var asmResolver = new NetStandardResolver(); + assembly = AssemblyDefinition.ReadAssembly(stream, new ReaderParameters + { + AssemblyResolver = asmResolver, + InMemory = true, + }); + } + + public void Remap(string pythonDll) + { + var moduleRef = new ModuleReference(pythonDll); + + var module = assembly.MainModule; + module.ModuleReferences.Add(moduleRef); + + foreach (var type in module.Types) + { + foreach (var func in type.Methods) + { + if (func.HasPInvokeInfo) + { + var info = func.PInvokeInfo; + if (info.Module.Name == "__Internal") + { + info.Module = moduleRef; + } + } + } + } + } + + public Assembly LoadAssembly() + { + using (var stream = new MemoryStream()) + { + assembly.Write(stream); + return Assembly.Load(stream.ToArray()); + } + } + } + + static class Internal + { + static Type PythonEngine = null; + + public static int Initialize(IntPtr data, int size) + { + try + { + var buf = new byte[size]; + Marshal.Copy(data, buf, 0, size); + var str = UTF8Encoding.Default.GetString(buf); + + var splitted = str.Split(';'); + + var dllPath = splitted[0]; + var pythonDll = splitted[1]; + + Console.WriteLine("Remapping __Internal in {0} to {1}", dllPath, pythonDll); + var loader = RuntimeLoader.FromFile(dllPath); + loader.Remap(pythonDll); + var assembly = loader.LoadAssembly(); + + PythonEngine = assembly.GetType("Python.Runtime.PythonEngine"); + var method = PythonEngine.GetMethod("InternalInitialize"); + var res = (int)method.Invoke(null, new object[] { data, size }); + Console.WriteLine("Done calling init: {0}", res); + return res; + } + catch (Exception exc) + { + Console.WriteLine($"{exc}\n{exc.StackTrace}"); + return -1; + } + } + + public static int Shutdown(IntPtr data, int size) + { + if (PythonEngine == null) + return -2; + + try + { + var method = PythonEngine.GetMethod("InternalShutdown"); + return (int)method.Invoke(null, new object[] { data, size }); + } + catch (Exception exc) + { + Console.WriteLine($"{exc}\n{exc.StackTrace}"); + return -1; + } + } + } +} diff --git a/src/loader/NetStandardResolver.cs b/src/loader/NetStandardResolver.cs new file mode 100644 index 000000000..d1c25efbc --- /dev/null +++ b/src/loader/NetStandardResolver.cs @@ -0,0 +1,23 @@ +using System.Runtime.CompilerServices; +using System.Reflection; +using System; +using System.Collections.Generic; + +using Mono.Cecil; + +namespace Python.Loader +{ + + class NetStandardResolver : DefaultAssemblyResolver + { + public override AssemblyDefinition Resolve(AssemblyNameReference name) + { + if (name.Name == "netstandard" || name.Name == "mscorlib") { + var asm = Assembly.Load(name.FullName); + // Inject facade directory + AddSearchDirectory(asm.Location + "/.."); + } + return base.Resolve(name); + } + } +} diff --git a/src/loader/Python.Loader.csproj b/src/loader/Python.Loader.csproj new file mode 100644 index 000000000..7ec4b7d20 --- /dev/null +++ b/src/loader/Python.Loader.csproj @@ -0,0 +1,21 @@ + + + + netstandard2.0 + true + + + + + false + Content + PreserveNewest + + + + + + + + + diff --git a/src/monoclr/clrmod.c b/src/monoclr/clrmod.c deleted file mode 100644 index cdfd89342..000000000 --- a/src/monoclr/clrmod.c +++ /dev/null @@ -1,215 +0,0 @@ -// #define Py_LIMITED_API 0x03050000 -#include - -#include "stdlib.h" - -#define MONO_VERSION "v4.0.30319.1" -#define MONO_DOMAIN "Python" - -#include -#include -#include -#include -#include - -#ifndef _WIN32 -#include "dirent.h" -#include "dlfcn.h" -#include "libgen.h" -#include "alloca.h" -#endif - -typedef struct -{ - MonoDomain *domain; - MonoAssembly *pr_assm; - MonoMethod *shutdown; - const char *pr_file; - char *error; - char *init_name; - char *shutdown_name; - PyObject *module; -} PyNet_Args; - -PyNet_Args *PyNet_Init(void); -static PyNet_Args *pn_args; - -PyMODINIT_FUNC -PyInit_clr(void) -{ - pn_args = PyNet_Init(); - if (pn_args->error) - { - return NULL; - } - - return pn_args->module; -} - -void PyNet_Finalize(PyNet_Args *); -void main_thread_handler(PyNet_Args *user_data); - -// initialize Mono and PythonNet -PyNet_Args *PyNet_Init() -{ - PyObject *pn_module; - PyObject *pn_path; - PyNet_Args *pn_args; - pn_args = (PyNet_Args *)malloc(sizeof(PyNet_Args)); - - pn_module = PyImport_ImportModule("pythonnet"); - if (pn_module == NULL) - { - pn_args->error = "Failed to import pythonnet"; - return pn_args; - } - - pn_path = PyObject_CallMethod(pn_module, "get_assembly_path", NULL); - if (pn_path == NULL) - { - Py_DecRef(pn_module); - pn_args->error = "Failed to get assembly path"; - return pn_args; - } - - pn_args->pr_file = PyUnicode_AsUTF8(pn_path); - pn_args->error = NULL; - pn_args->shutdown = NULL; - pn_args->module = NULL; - -#ifdef __linux__ - // Force preload libmono-2.0 as global. Without this, on some systems - // symbols from libmono are not found by libmononative (which implements - // some of the System.* namespaces). Since the only happened on Linux so - // far, we hardcode the library name, load the symbols into the global - // namespace and leak the handle. - dlopen("libmono-2.0.so", RTLD_LAZY | RTLD_GLOBAL); -#endif - - pn_args->init_name = "Python.Runtime:InitExt()"; - pn_args->shutdown_name = "Python.Runtime:Shutdown()"; - - pn_args->domain = mono_jit_init_version(MONO_DOMAIN, MONO_VERSION); - - // XXX: Skip setting config for now, should be derived from pr_file - // mono_domain_set_config(pn_args->domain, ".", "Python.Runtime.dll.config"); - - /* - * Load the default Mono configuration file, this is needed - * if you are planning on using the dllmaps defined on the - * system configuration - */ - mono_config_parse(NULL); - - main_thread_handler(pn_args); - - if (pn_args->error != NULL) - { - PyErr_SetString(PyExc_ImportError, pn_args->error); - } - return pn_args; -} - -char *PyNet_ExceptionToString(MonoObject *e); - -// Shuts down PythonNet and cleans up Mono -void PyNet_Finalize(PyNet_Args *pn_args) -{ - MonoObject *exception = NULL; - - if (pn_args->shutdown) - { - mono_runtime_invoke(pn_args->shutdown, NULL, NULL, &exception); - if (exception) - { - pn_args->error = PyNet_ExceptionToString(exception); - } - pn_args->shutdown = NULL; - } - - if (pn_args->domain) - { - mono_jit_cleanup(pn_args->domain); - pn_args->domain = NULL; - } - free(pn_args); -} - -MonoMethod *getMethodFromClass(MonoClass *cls, char *name) -{ - MonoMethodDesc *mdesc; - MonoMethod *method; - - mdesc = mono_method_desc_new(name, 1); - method = mono_method_desc_search_in_class(mdesc, cls); - mono_method_desc_free(mdesc); - - return method; -} - -void main_thread_handler(PyNet_Args *user_data) -{ - PyNet_Args *pn_args = user_data; - MonoMethod *init; - MonoImage *pr_image; - MonoClass *pythonengine; - MonoObject *exception = NULL; - MonoObject *init_result; - - pn_args->pr_assm = mono_domain_assembly_open(pn_args->domain, pn_args->pr_file); - if (!pn_args->pr_assm) - { - pn_args->error = "Unable to load assembly"; - return; - } - - pr_image = mono_assembly_get_image(pn_args->pr_assm); - if (!pr_image) - { - pn_args->error = "Unable to get image"; - return; - } - - pythonengine = mono_class_from_name(pr_image, "Python.Runtime", "PythonEngine"); - if (!pythonengine) - { - pn_args->error = "Unable to load class PythonEngine from Python.Runtime"; - return; - } - - init = getMethodFromClass(pythonengine, pn_args->init_name); - if (!init) - { - pn_args->error = "Unable to fetch Init method from PythonEngine"; - return; - } - - pn_args->shutdown = getMethodFromClass(pythonengine, pn_args->shutdown_name); - if (!pn_args->shutdown) - { - pn_args->error = "Unable to fetch shutdown method from PythonEngine"; - return; - } - - init_result = mono_runtime_invoke(init, NULL, NULL, &exception); - if (exception) - { - pn_args->error = PyNet_ExceptionToString(exception); - return; - } - - pn_args->module = *(PyObject**)mono_object_unbox(init_result); -} - -// Get string from a Mono exception -char *PyNet_ExceptionToString(MonoObject *e) -{ - MonoMethodDesc *mdesc = mono_method_desc_new(":ToString()", 0 /*FALSE*/); - MonoMethod *mmethod = mono_method_desc_search_in_class(mdesc, mono_get_object_class()); - mono_method_desc_free(mdesc); - - mmethod = mono_object_get_virtual_method(e, mmethod); - MonoString *monoString = (MonoString*) mono_runtime_invoke(mmethod, e, NULL, NULL); - mono_runtime_invoke(mmethod, e, NULL, NULL); - return mono_string_to_utf8(monoString); -} diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index df6cf7101..e3b1d7a9b 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -150,14 +150,39 @@ public static int RunSimpleString(string code) return Runtime.PyRun_SimpleString(code); } - public static void Initialize() + // Entrypoint for the cffi clr-module import + public static int InternalInitialize(IntPtr data, int size) { - Initialize(setSysArgv: true); + IntPtr gilState = Runtime.PyGILState_Ensure(); + try + { + Initialize(setSysArgv: false); + return 0; + } + finally + { + Runtime.PyGILState_Release(gilState); + } } - public static void Initialize(bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) + public static int InternalShutdown(IntPtr data, int size) { - Initialize(Enumerable.Empty(), setSysArgv: setSysArgv, initSigs: initSigs, mode); + IntPtr gilState = Runtime.PyGILState_Ensure(); + try + { + // Crashes right now + // Shutdown(); + return 0; + } + finally + { + Runtime.PyGILState_Release(gilState); + } + } + + public static void Initialize() + { + Initialize(setSysArgv: true); } /// @@ -170,8 +195,9 @@ public static void Initialize(bool setSysArgv = true, bool initSigs = false, Shu /// interpreter lock (GIL) to call this method. /// initSigs can be set to 1 to do default python signal configuration. This will override the way signals are handled by the application. /// - public static void Initialize(IEnumerable args, bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) + public static void Initialize(IEnumerable args = null, bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) { + args = args ?? new string[] { }; if (initialized) { return; @@ -184,6 +210,7 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, delegateManager = new DelegateManager(); Runtime.Initialize(initSigs, mode); initialized = true; + Exceptions.Clear(); // Make sure we clean up properly on app domain unload. @@ -235,16 +262,16 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, // and decorators into the main clr module. Runtime.PyDict_SetItemString(clr_dict, "_extras", module); using (var keys = locals.Keys()) - foreach (PyObject key in keys) - { - if (!key.ToString().StartsWith("_") || key.ToString().Equals("__version__")) + foreach (PyObject key in keys) { - PyObject value = locals[key]; - Runtime.PyDict_SetItem(clr_dict, key.Handle, value.Handle); - value.Dispose(); + if (!key.ToString().StartsWith("_") || key.ToString().Equals("__version__")) + { + PyObject value = locals[key]; + Runtime.PyDict_SetItem(clr_dict, key.Handle, value.Handle); + value.Dispose(); + } + key.Dispose(); } - key.Dispose(); - } } finally { @@ -405,7 +432,7 @@ public static void RemoveShutdownHandler(ShutdownHandler handler) /// static void ExecuteShutdownHandlers() { - while(ShutdownHandlers.Count > 0) + while (ShutdownHandlers.Count > 0) { var handler = ShutdownHandlers[ShutdownHandlers.Count - 1]; ShutdownHandlers.RemoveAt(ShutdownHandlers.Count - 1); diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index d9301acdc..24d743cfc 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -22,46 +22,13 @@ public class Runtime public static int UCS => _UCS; internal static readonly int _UCS = PyUnicode_GetMax() <= 0xFFFF ? 2 : 4; -#if PYTHON36 - const string _minor = "6"; -#elif PYTHON37 - const string _minor = "7"; -#elif PYTHON38 - const string _minor = "8"; -#elif PYTHON39 - const string _minor = "9"; -#else -#error You must define one of PYTHON36 to PYTHON39 -#endif - -#if WINDOWS - internal const string dllBase = "python3" + _minor; -#else - internal const string dllBase = "python3." + _minor; -#endif - -#if PYTHON_WITH_PYDEBUG - internal const string dllWithPyDebug = "d"; -#else - internal const string dllWithPyDebug = ""; -#endif -#if PYTHON_WITH_PYMALLOC - internal const string dllWithPyMalloc = "m"; -#else - internal const string dllWithPyMalloc = ""; -#endif - // C# compiler copies constants to the assemblies that references this library. // We needs to replace all public constants to static readonly fields to allow // binary substitution of different Python.Runtime.dll builds in a target application. public static readonly string PythonDLL = _PythonDll; -#if PYTHON_WITHOUT_ENABLE_SHARED && !NETSTANDARD internal const string _PythonDll = "__Internal"; -#else - internal const string _PythonDll = dllBase + dllWithPyDebug + dllWithPyMalloc; -#endif // set to true when python is finalizing internal static object IsFinalizingLock = new object(); @@ -121,6 +88,9 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd } ShutdownMode = mode; + bool mustReleaseGil = false; + IntPtr gilState = IntPtr.Zero; + if (Py_IsInitialized() == 0) { Py_InitializeEx(initSigs ? 1 : 0); @@ -140,7 +110,11 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd // If we're coming back from a domain reload or a soft shutdown, // we have previously released the thread state. Restore the main // thread state here. - PyGILState_Ensure(); + // + // When called from the cffi clr-module import, this must be released at + // the end. + mustReleaseGil = PyGILState_Check() == 0; + gilState = PyGILState_Ensure(); } MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; @@ -182,6 +156,11 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd } XDecref(item); AssemblyManager.UpdatePath(); + + if (mustReleaseGil) + { + PyGILState_Release(gilState); + } } private static void InitPyMembers() @@ -842,6 +821,9 @@ internal static unsafe long Refcount(IntPtr op) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyGILState_Release(IntPtr gs); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyGILState_Check(); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyGILState_GetThisThreadState();