From d1928dcfde6cca317a33abadde2da27fd09adc60 Mon Sep 17 00:00:00 2001 From: amos402 Date: Fri, 14 Jun 2019 02:35:50 +0800 Subject: [PATCH 1/7] Drop LoadLibrary dependency --- src/runtime/runtime.cs | 107 ++--------------------------------------- 1 file changed, 5 insertions(+), 102 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 7623200e0..0359dac1a 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -7,97 +7,6 @@ namespace Python.Runtime { - [SuppressUnmanagedCodeSecurity] - internal static class NativeMethods - { -#if MONO_LINUX || MONO_OSX -#if NETSTANDARD - private static int RTLD_NOW = 0x2; -#if MONO_LINUX - private static int RTLD_GLOBAL = 0x100; - private static IntPtr RTLD_DEFAULT = IntPtr.Zero; - private const string NativeDll = "libdl.so"; - public static IntPtr LoadLibrary(string fileName) - { - return dlopen($"lib{fileName}.so", RTLD_NOW | RTLD_GLOBAL); - } -#elif MONO_OSX - private static int RTLD_GLOBAL = 0x8; - private const string NativeDll = "/usr/lib/libSystem.dylib"; - private static IntPtr RTLD_DEFAULT = new IntPtr(-2); - - public static IntPtr LoadLibrary(string fileName) - { - return dlopen($"lib{fileName}.dylib", RTLD_NOW | RTLD_GLOBAL); - } -#endif -#else - private static int RTLD_NOW = 0x2; - private static int RTLD_SHARED = 0x20; -#if MONO_OSX - private static IntPtr RTLD_DEFAULT = new IntPtr(-2); - private const string NativeDll = "__Internal"; -#elif MONO_LINUX - private static IntPtr RTLD_DEFAULT = IntPtr.Zero; - private const string NativeDll = "libdl.so"; -#endif - - public static IntPtr LoadLibrary(string fileName) - { - return dlopen(fileName, RTLD_NOW | RTLD_SHARED); - } -#endif - - - public static void FreeLibrary(IntPtr handle) - { - dlclose(handle); - } - - public static IntPtr GetProcAddress(IntPtr dllHandle, string name) - { - // look in the exe if dllHandle is NULL - if (dllHandle == IntPtr.Zero) - { - dllHandle = RTLD_DEFAULT; - } - - // clear previous errors if any - dlerror(); - IntPtr res = dlsym(dllHandle, name); - IntPtr errPtr = dlerror(); - if (errPtr != IntPtr.Zero) - { - throw new Exception("dlsym: " + Marshal.PtrToStringAnsi(errPtr)); - } - return res; - } - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - public static extern IntPtr dlopen(String fileName, int flags); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - private static extern IntPtr dlsym(IntPtr handle, String symbol); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int dlclose(IntPtr handle); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr dlerror(); -#else // Windows - private const string NativeDll = "kernel32.dll"; - - [DllImport(NativeDll)] - public static extern IntPtr LoadLibrary(string dllToLoad); - - [DllImport(NativeDll)] - public static extern IntPtr GetProcAddress(IntPtr hModule, string procedureName); - - [DllImport(NativeDll)] - public static extern bool FreeLibrary(IntPtr hModule); -#endif - } - /// /// Encapsulates the low-level Python C API. Note that it is /// the responsibility of the caller to have acquired the GIL @@ -395,18 +304,12 @@ internal static void Initialize(bool initSigs = false) IntPtr dllLocal = IntPtr.Zero; - if (_PythonDll != "__Internal") - { - dllLocal = NativeMethods.LoadLibrary(_PythonDll); - } - _PyObject_NextNotImplemented = NativeMethods.GetProcAddress(dllLocal, "_PyObject_NextNotImplemented"); + var zipimport = PyImport_ImportModule("zipimport"); + var ZipImportError = PyObject_GetAttrString(zipimport, "ZipImportError"); + _PyObject_NextNotImplemented = Marshal.ReadIntPtr(ZipImportError, TypeOffset.tp_iternext); + XDecref(ZipImportError); + XDecref(zipimport); -#if !(MONO_LINUX || MONO_OSX) - if (dllLocal != IntPtr.Zero) - { - NativeMethods.FreeLibrary(dllLocal); - } -#endif // Initialize data about the platform we're running on. We need // this for the type manager and potentially other details. Must // happen after caching the python types, above. From f000e0846407a5f0a8c30b612b3c874981330feb Mon Sep 17 00:00:00 2001 From: amos402 Date: Fri, 14 Jun 2019 02:51:24 +0800 Subject: [PATCH 2/7] Explain for getting _PyObject_NextNotImplemented --- src/runtime/runtime.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 0359dac1a..7f0349da4 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -304,6 +304,10 @@ internal static void Initialize(bool initSigs = false) IntPtr dllLocal = IntPtr.Zero; + // Since `_PyObject_NextNotImplemented` would set to a heap class + // for tp_iternext which doesn't implement __next__. + // Thus we need a heap class to get it, the ZipImportError is a + // heap class and it's in builtins, so we can use it as a trick. var zipimport = PyImport_ImportModule("zipimport"); var ZipImportError = PyObject_GetAttrString(zipimport, "ZipImportError"); _PyObject_NextNotImplemented = Marshal.ReadIntPtr(ZipImportError, TypeOffset.tp_iternext); From f8824005d489f7d5f3d7bfb95f59af3a2dd03cfe Mon Sep 17 00:00:00 2001 From: amos402 Date: Fri, 14 Jun 2019 03:18:58 +0800 Subject: [PATCH 3/7] Update changelog and authors --- AUTHORS.md | 1 + CHANGELOG.md | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS.md b/AUTHORS.md index ba954b47d..4f4c9862e 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -52,6 +52,7 @@ - William Sardar ([@williamsardar])(https://github.com/williamsardar) - Xavier Dupré ([@sdpython](https://github.com/sdpython)) - Zane Purvis ([@zanedp](https://github.com/zanedp)) +- ([@amos402]https://github.com/amos402) - ([@bltribble](https://github.com/bltribble)) - ([@civilx64](https://github.com/civilx64)) - ([@GSPP](https://github.com/GSPP)) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5531bf47..d8430da0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - PythonEngine.Intialize will now call `Py_InitializeEx` with a default value of 0, so signals will not be configured by default on embedding. This is different from the previous behaviour, where `Py_Initialize` was called instead, which sets initSigs to 1. ([#449][i449]) - Refactored MethodBinder.Bind in preparation to make it extensible (#829) - Look for installed Windows 10 sdk's during installation instead of relying on specific versions. +- Remove `LoadLibrary` call. ([#880][p880]) ### Fixed @@ -57,7 +58,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Fixed conversion of 'float' and 'double' values ([#486][i486]) - Fixed 'clrmethod' for python 2 ([#492][i492]) - Fixed double calling of constructor when deriving from .NET class ([#495][i495]) -- Fixed `clr.GetClrType` when iterating over `System` members ([#607][p607]) +- Fixed `clr.GetClrType` when iterating over `System` members ([#607][p607]) - Fixed `LockRecursionException` when loading assemblies ([#627][i627]) - Fixed errors breaking .NET Remoting on method invoke ([#276][i276]) - Fixed PyObject.GetHashCode ([#676][i676]) From b3e889b9f6f320f7a40f67dfe6cbe315e03215bb Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 1 Dec 2019 21:35:05 +0800 Subject: [PATCH 4/7] * Load PyModuleType without LibraryLoader * Drop C module dependency when getting _PyObject_NextNotImplemented * Exception details for SetNoSiteFlag --- src/runtime/runtime.cs | 62 ++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 945c8f809..8b4ef6cbe 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -292,26 +292,18 @@ internal static void Initialize(bool initSigs = false) Error = new IntPtr(-1); + _PyObject_NextNotImplemented = Get_PyObject_NextNotImplemented(); + { + IntPtr sys = PyImport_ImportModule("sys"); + PyModuleType = PyObject_Type(sys); + XDecref(sys); + } + // Initialize data about the platform we're running on. We need // this for the type manager and potentially other details. Must // happen after caching the python types, above. InitializePlatformData(); - IntPtr dllLocal = IntPtr.Zero; - var loader = LibraryLoader.Get(OperatingSystem); - - // Since `_PyObject_NextNotImplemented` would set to a heap class - // for tp_iternext which doesn't implement __next__. - // Thus we need a heap class to get it, the ZipImportError is a - // heap class and it's in builtins, so we can use it as a trick. - var zipimport = PyImport_ImportModule("zipimport"); - var ZipImportError = PyObject_GetAttrString(zipimport, "ZipImportError"); - _PyObject_NextNotImplemented = Marshal.ReadIntPtr(ZipImportError, TypeOffset.tp_iternext); - XDecref(ZipImportError); - XDecref(zipimport); - PyModuleType = loader.GetFunction(dllLocal, "PyModule_Type"); - - // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); PyCLRMetaType = MetaType.Initialize(); @@ -328,6 +320,29 @@ internal static void Initialize(bool initSigs = false) AssemblyManager.UpdatePath(); } + private static IntPtr Get_PyObject_NextNotImplemented() + { + IntPtr globals = PyDict_New(); + if (PyDict_SetItemString(globals, "__builtins__", PyEval_GetBuiltins()) != 0) + { + XDecref(globals); + throw new PythonException(); + } + const string code = "class A(object): pass"; + IntPtr res = PyRun_String(code, (IntPtr)RunFlagType.File, globals, globals); + if (res == IntPtr.Zero) + { + XDecref(globals); + throw new PythonException(); + } + XDecref(res); + IntPtr A = PyDict_GetItemString(globals, "A"); + IntPtr iternext = Marshal.ReadIntPtr(A, TypeOffset.tp_iternext); + XDecref(globals); + XDecref(A); + return iternext; + } + /// /// Initializes the data about platforms. /// @@ -1893,14 +1908,16 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) internal static void SetNoSiteFlag() { + if (_PythonDll == "__Internal") + { + throw new NotSupportedException("SetNoSiteFlag didn't support on static compile"); + } var loader = LibraryLoader.Get(OperatingSystem); - - IntPtr dllLocal; - if (_PythonDll != "__Internal") + IntPtr dllLocal = loader.Load(_PythonDll); + if (dllLocal == IntPtr.Zero) { - dllLocal = loader.Load(_PythonDll); + throw new Exception($"Cannot load {_PythonDll}"); } - try { Py_NoSiteFlag = loader.GetFunction(dllLocal, "Py_NoSiteFlag"); @@ -1908,10 +1925,7 @@ internal static void SetNoSiteFlag() } finally { - if (dllLocal != IntPtr.Zero) - { - loader.Free(dllLocal); - } + loader.Free(dllLocal); } } } From 5150e618612a299af3a5c970f5ec299b3ca974f0 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 15 Dec 2019 02:34:28 +0800 Subject: [PATCH 5/7] Prevent exception override --- src/runtime/runtime.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 153738930..358a3946b 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -332,8 +332,14 @@ private static IntPtr Get_PyObject_NextNotImplemented() IntPtr res = PyRun_String(code, (IntPtr)RunFlagType.File, globals, globals); if (res == IntPtr.Zero) { - XDecref(globals); - throw new PythonException(); + try + { + throw new PythonException(); + } + finally + { + XDecref(globals); + } } XDecref(res); IntPtr A = PyDict_GetItemString(globals, "A"); From 65e209e00d76a39007b86b4a3c825ab1e38a1d2c Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 16 Dec 2019 01:33:42 +0800 Subject: [PATCH 6/7] Rollback symbol loading for __Internal --- src/runtime/runtime.cs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 358a3946b..495dd7bb7 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1946,15 +1946,15 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) internal static void SetNoSiteFlag() { - if (_PythonDll == "__Internal") - { - throw new NotSupportedException("SetNoSiteFlag didn't support on static compile"); - } var loader = LibraryLoader.Get(OperatingSystem); - IntPtr dllLocal = loader.Load(_PythonDll); - if (dllLocal == IntPtr.Zero) + IntPtr dllLocal; + if (_PythonDll != "__Internal") { - throw new Exception($"Cannot load {_PythonDll}"); + dllLocal = loader.Load(_PythonDll); + if (dllLocal == IntPtr.Zero) + { + throw new Exception($"Cannot load {_PythonDll}"); + } } try { @@ -1963,7 +1963,10 @@ internal static void SetNoSiteFlag() } finally { - loader.Free(dllLocal); + if (dllLocal != IntPtr.Zero) + { + loader.Free(dllLocal); + } } } } From 2b84394f00d0db083ea82b8ca6d1831d8e9e9b81 Mon Sep 17 00:00:00 2001 From: amos402 Date: Thu, 13 Feb 2020 01:16:20 +0800 Subject: [PATCH 7/7] * Fix refcnt error * Pick `SlotHelper` from #958 --- src/runtime/runtime.cs | 27 +++------------------------ src/runtime/typemanager.cs | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 24 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 5ead02378..7da70ddcd 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -340,30 +340,9 @@ internal static void Initialize(bool initSigs = false) private static IntPtr Get_PyObject_NextNotImplemented() { - IntPtr globals = PyDict_New(); - if (PyDict_SetItemString(globals, "__builtins__", PyEval_GetBuiltins()) != 0) - { - XDecref(globals); - throw new PythonException(); - } - const string code = "class A(object): pass"; - IntPtr res = PyRun_String(code, (IntPtr)RunFlagType.File, globals, globals); - if (res == IntPtr.Zero) - { - try - { - throw new PythonException(); - } - finally - { - XDecref(globals); - } - } - XDecref(res); - IntPtr A = PyDict_GetItemString(globals, "A"); - IntPtr iternext = Marshal.ReadIntPtr(A, TypeOffset.tp_iternext); - XDecref(globals); - XDecref(A); + IntPtr pyType = SlotHelper.CreateObjectType(); + IntPtr iternext = Marshal.ReadIntPtr(pyType, TypeOffset.tp_iternext); + Runtime.XDecref(pyType); return iternext; } diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index bb920b74f..c541f9dc2 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Reflection; using System.Runtime.InteropServices; @@ -820,4 +821,37 @@ internal static void CopySlot(IntPtr from, IntPtr to, int offset) Marshal.WriteIntPtr(to, offset, fp); } } + + + static class SlotHelper + { + public static IntPtr CreateObjectType() + { + IntPtr globals = Runtime.PyDict_New(); + if (Runtime.PyDict_SetItemString(globals, "__builtins__", Runtime.PyEval_GetBuiltins()) != 0) + { + Runtime.XDecref(globals); + throw new PythonException(); + } + const string code = "class A(object): pass"; + IntPtr res = Runtime.PyRun_String(code, (IntPtr)RunFlagType.File, globals, globals); + if (res == IntPtr.Zero) + { + try + { + throw new PythonException(); + } + finally + { + Runtime.XDecref(globals); + } + } + Runtime.XDecref(res); + IntPtr A = Runtime.PyDict_GetItemString(globals, "A"); + Debug.Assert(A != IntPtr.Zero); + Runtime.XIncref(A); + Runtime.XDecref(globals); + return A; + } + } }