diff --git a/.travis.yml b/.travis.yml index 1ffe7754b..e664a4696 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,8 @@ env: matrix: - BUILD_OPTS=--xplat NUNIT_PATH="~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe" RUN_TESTS=dotnet EMBED_TESTS_PATH=netcoreapp2.0_publish/ PERF_TESTS_PATH=net461/ - BUILD_OPTS="" NUNIT_PATH="./packages/NUnit.*/tools/nunit3-console.exe" RUN_TESTS="mono $NUNIT_PATH" EMBED_TESTS_PATH="" PERF_TESTS_PATH="" + - PYTHONNET_SHUTDOWN_MODE="Soft" BUILD_OPTS=--xplat NUNIT_PATH="~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe" RUN_TESTS=dotnet EMBED_TESTS_PATH=netcoreapp2.0_publish/ PERF_TESTS_PATH=net461/ + - PYTHONNET_SHUTDOWN_MODE="Soft" BUILD_OPTS="" NUNIT_PATH="./packages/NUnit.*/tools/nunit3-console.exe" RUN_TESTS="mono $NUNIT_PATH" EMBED_TESTS_PATH="" PERF_TESTS_PATH="" global: - LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so diff --git a/AUTHORS.md b/AUTHORS.md index ce6a79513..eeafd98e4 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -35,6 +35,7 @@ - David Lassonde ([@lassond](https://github.com/lassond)) - David Lechner ([@dlech](https://github.com/dlech)) - Dmitriy Se ([@dmitriyse](https://github.com/dmitriyse)) +- Félix Bourbonnais ([@BadSingleton](https://github.com/BadSingleton)) - Florian Treurniet ([@ftreurni](https://github.com/ftreurni)) - He-chien Tsai ([@t3476](https://github.com/t3476)) - Inna Wiesel ([@inna-w](https://github.com/inna-w)) @@ -66,6 +67,7 @@ - William Sardar ([@williamsardar])(https://github.com/williamsardar) - Xavier Dupré ([@sdpython](https://github.com/sdpython)) - Zane Purvis ([@zanedp](https://github.com/zanedp)) +- ([@amos402]https://github.com/amos402) - ([@bltribble](https://github.com/bltribble)) - ([@civilx64](https://github.com/civilx64)) - ([@GSPP](https://github.com/GSPP)) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4417fbaf5..5bdf5e32b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -113,6 +113,7 @@ This version improves performance on benchmarks significantly compared to 2.3. - PythonEngine.Intialize will now call `Py_InitializeEx` with a default value of 0, so signals will not be configured by default on embedding. This is different from the previous behaviour, where `Py_Initialize` was called instead, which sets initSigs to 1. ([#449][i449]) - Refactored MethodBinder.Bind in preparation to make it extensible (#829) - Look for installed Windows 10 sdk's during installation instead of relying on specific versions. +- Remove `LoadLibrary` call. ([#880][p880]) ### Fixed diff --git a/appveyor.yml b/appveyor.yml index f64fbf0f6..d45ab5b36 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -16,7 +16,6 @@ environment: matrix: - PYTHON_VERSION: 3.8 - BUILD_OPTS: --xplat - PYTHON_VERSION: 3.7 BUILD_OPTS: --xplat - PYTHON_VERSION: 3.6 @@ -24,7 +23,10 @@ environment: - PYTHON_VERSION: 3.8 - PYTHON_VERSION: 3.7 - PYTHON_VERSION: 3.6 - + - PYTHON_VERSION: 3.7 + PYTHONNET_SHUTDOWN_MODE: Soft + - PYTHON_VERSION: 3.8 + PYTHONNET_SHUTDOWN_MODE: Soft init: # Update Environment Variables based on matrix/platform - set PY_VER=%PYTHON_VERSION:.=% diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index b162d4eb0..3556df0f6 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -1,9 +1,12 @@ using System; -using System.CodeDom.Compiler; +using System.Collections.Generic; +using System.Diagnostics; using System.Reflection; +using System.Runtime.InteropServices; using NUnit.Framework; using Python.Runtime; +using PyRuntime = Python.Runtime.Runtime; // // This test case is disabled on .NET Standard because it doesn't have all the // APIs we use. We could work around that, but .NET Core doesn't implement @@ -16,6 +19,12 @@ namespace Python.EmbeddingTest { class TestDomainReload { + abstract class CrossCaller : MarshalByRefObject + { + public abstract ValueType Execute(ValueType arg); + } + + /// /// Test that the python runtime can survive a C# domain reload without crashing. /// @@ -50,173 +59,411 @@ class TestDomainReload [Test] public static void DomainReloadAndGC() { - // We're set up to run in the directory that includes the bin directory. - System.IO.Directory.SetCurrentDirectory(AppDomain.CurrentDomain.BaseDirectory); - - Assembly pythonRunner1 = BuildAssembly("test1"); - RunAssemblyAndUnload(pythonRunner1, "test1"); - - // Verify that python is not initialized even though we ran it. - Assert.That(Runtime.Runtime.Py_IsInitialized(), Is.Zero); - - // This caused a crash because objects allocated in pythonRunner1 - // still existed in memory, but the code to do python GC on those - // objects is gone. - Assembly pythonRunner2 = BuildAssembly("test2"); - RunAssemblyAndUnload(pythonRunner2, "test2"); - } - - // - // The code we'll test. All that really matters is - // using GIL { Python.Exec(pyScript); } - // but the rest is useful for debugging. - // - // What matters in the python code is gc.collect and clr.AddReference. - // - // Note that the language version is 2.0, so no $"foo{bar}" syntax. - // - const string TestCode = @" - using Python.Runtime; - using System; - class PythonRunner { - public static void RunPython() { - AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; - string name = AppDomain.CurrentDomain.FriendlyName; - Console.WriteLine(string.Format(""[{0} in .NET] In PythonRunner.RunPython"", name)); - using (Py.GIL()) { - try { - var pyScript = string.Format(""import clr\n"" - + ""print('[{0} in python] imported clr')\n"" - + ""clr.AddReference('System')\n"" - + ""print('[{0} in python] allocated a clr object')\n"" - + ""import gc\n"" - + ""gc.collect()\n"" - + ""print('[{0} in python] collected garbage')\n"", - name); - PythonEngine.Exec(pyScript); - } catch(Exception e) { - Console.WriteLine(string.Format(""[{0} in .NET] Caught exception: {1}"", name, e)); + Assert.IsFalse(PythonEngine.IsInitialized); + RunAssemblyAndUnload("test1"); + Assert.That(PyRuntime.Py_IsInitialized() != 0, + "On soft-shutdown mode, Python runtime should still running"); + + RunAssemblyAndUnload("test2"); + Assert.That(PyRuntime.Py_IsInitialized() != 0, + "On soft-shutdown mode, Python runtime should still running"); + + if (PythonEngine.DefaultShutdownMode == ShutdownMode.Normal) + { + // The default mode is a normal mode, + // it should shutdown the Python VM avoiding influence other tests. + PyRuntime.PyGILState_Ensure(); + PyRuntime.Py_Finalize(); + } + } + + #region CrossDomainObject + + class CrossDomainObjectStep1 : CrossCaller + { + public override ValueType Execute(ValueType arg) + { + try + { + // Create a C# user-defined object in Python. Asssing some values. + Type type = typeof(Python.EmbeddingTest.Domain.MyClass); + string code = string.Format(@" +import clr +clr.AddReference('{0}') + +from Python.EmbeddingTest.Domain import MyClass +obj = MyClass() +obj.Method() +obj.StaticMethod() +obj.Property = 1 +obj.Field = 10 +", Assembly.GetExecutingAssembly().FullName); + + using (Py.GIL()) + using (var scope = Py.CreateScope()) + { + scope.Exec(code); + using (PyObject obj = scope.Get("obj")) + { + Debug.Assert(obj.AsManagedObject(type).GetType() == type); + // We only needs its Python handle + PyRuntime.XIncref(obj.Handle); + return obj.Handle; } } } - static void OnDomainUnload(object sender, EventArgs e) { - System.Console.WriteLine(string.Format(""[{0} in .NET] unloading"", AppDomain.CurrentDomain.FriendlyName)); + catch (Exception e) + { + Debug.WriteLine(e); + throw; } - }"; + } + } + class CrossDomainObjectStep2 : CrossCaller + { + public override ValueType Execute(ValueType arg) + { + // handle refering a clr object created in previous domain, + // it should had been deserialized and became callable agian. + IntPtr handle = (IntPtr)arg; + try + { + using (Py.GIL()) + { + IntPtr tp = Runtime.Runtime.PyObject_TYPE(handle); + IntPtr tp_clear = Marshal.ReadIntPtr(tp, TypeOffset.tp_clear); + Assert.That(tp_clear, Is.Not.Null); + + using (PyObject obj = new PyObject(handle)) + { + obj.InvokeMethod("Method"); + obj.InvokeMethod("StaticMethod"); + + using (var scope = Py.CreateScope()) + { + scope.Set("obj", obj); + scope.Exec(@" +obj.Method() +obj.StaticMethod() +obj.Property += 1 +obj.Field += 10 +"); + } + var clrObj = obj.As(); + Assert.AreEqual(clrObj.Property, 2); + Assert.AreEqual(clrObj.Field, 20); + } + } + } + catch (Exception e) + { + Debug.WriteLine(e); + throw; + } + return 0; + } + } + /// - /// Build an assembly out of the source code above. - /// - /// This creates a file .dll in order - /// to support the statement "proxy.theAssembly = assembly" below. - /// That statement needs a file, can't run via memory. + /// Create a C# custom object in a domain, in python code. + /// Unload the domain, create a new domain. + /// Make sure the C# custom object created in the previous domain has been re-created /// - static Assembly BuildAssembly(string assemblyName) + [Test] + public static void CrossDomainObject() { - var provider = CodeDomProvider.CreateProvider("CSharp"); + RunDomainReloadSteps(); + } + + #endregion - var compilerparams = new CompilerParameters(); - compilerparams.ReferencedAssemblies.Add("Python.Runtime.dll"); - compilerparams.GenerateExecutable = false; - compilerparams.GenerateInMemory = false; - compilerparams.IncludeDebugInformation = false; - compilerparams.OutputAssembly = assemblyName; + #region TestClassReference - var results = provider.CompileAssemblyFromSource(compilerparams, TestCode); - if (results.Errors.HasErrors) + class ReloadClassRefStep1 : CrossCaller + { + public override ValueType Execute(ValueType arg) { - var errors = new System.Text.StringBuilder("Compiler Errors:\n"); - foreach (CompilerError error in results.Errors) + const string code = @" +from Python.EmbeddingTest.Domain import MyClass + +def test_obj_call(): + obj = MyClass() + obj.Method() + obj.StaticMethod() + obj.Property = 1 + obj.Field = 10 + +test_obj_call() +"; + const string name = "test_domain_reload_mod"; + using (Py.GIL()) { - errors.AppendFormat("Line {0},{1}\t: {2}\n", - error.Line, error.Column, error.ErrorText); + // Create a new module + IntPtr module = PyRuntime.PyModule_New(name); + Assert.That(module != IntPtr.Zero); + IntPtr globals = PyRuntime.PyObject_GetAttrString(module, "__dict__"); + Assert.That(globals != IntPtr.Zero); + try + { + // import builtins + // module.__dict__[__builtins__] = builtins + int res = PyRuntime.PyDict_SetItemString(globals, "__builtins__", + PyRuntime.PyEval_GetBuiltins()); + PythonException.ThrowIfIsNotZero(res); + + // Execute the code in the module's scope + PythonEngine.Exec(code, globals); + // import sys + // modules = sys.modules + IntPtr modules = PyRuntime.PyImport_GetModuleDict(); + // modules[name] = module + res = PyRuntime.PyDict_SetItemString(modules, name, module); + PythonException.ThrowIfIsNotZero(res); + } + catch + { + PyRuntime.XDecref(module); + throw; + } + finally + { + PyRuntime.XDecref(globals); + } + return module; } - throw new Exception(errors.ToString()); } - else + } + + class ReloadClassRefStep2 : CrossCaller + { + public override ValueType Execute(ValueType arg) { - return results.CompiledAssembly; + var module = (IntPtr)arg; + using (Py.GIL()) + { + var test_obj_call = PyRuntime.PyObject_GetAttrString(module, "test_obj_call"); + PythonException.ThrowIfIsNull(test_obj_call); + var args = PyRuntime.PyTuple_New(0); + var res = PyRuntime.PyObject_CallObject(test_obj_call, args); + PythonException.ThrowIfIsNull(res); + + PyRuntime.XDecref(args); + PyRuntime.XDecref(res); + } + return 0; } } + + [Test] /// - /// This is a magic incantation required to run code in an application - /// domain other than the current one. + /// Create a new Python module, define a function in it. + /// Unload the domain, load a new one. + /// Make sure the function (and module) still exists. /// - class Proxy : MarshalByRefObject + public void TestClassReference() { - Assembly theAssembly = null; + RunDomainReloadSteps(); + } + + #endregion + + #region Tempary tests - public void InitAssembly(string assemblyPath) + // https://github.com/pythonnet/pythonnet/pull/1074#issuecomment-596139665 + [Test] + public void CrossReleaseBuiltinType() + { + void ExecTest() { - theAssembly = Assembly.LoadFile(System.IO.Path.GetFullPath(assemblyPath)); + try + { + PythonEngine.Initialize(); + var numRef = CreateNumReference(); + Assert.True(numRef.IsAlive); + PythonEngine.Shutdown(); // <- "run" 1 ends + PythonEngine.Initialize(); // <- "run" 2 starts + + GC.Collect(); + GC.WaitForPendingFinalizers(); // <- this will put former `num` into Finalizer queue + Finalizer.Instance.Collect(forceDispose: true); + // ^- this will call PyObject.Dispose, which will call XDecref on `num.Handle`, + // but Python interpreter from "run" 1 is long gone, so it will corrupt memory instead. + Assert.False(numRef.IsAlive); + } + finally + { + PythonEngine.Shutdown(); + } } - public void RunPython() + var errorArgs = new List(); + void ErrorHandler(object sender, Finalizer.ErrorArgs e) { - Console.WriteLine("[Proxy] Entering RunPython"); + errorArgs.Add(e); + } + Finalizer.Instance.ErrorHandler += ErrorHandler; + try + { + for (int i = 0; i < 10; i++) + { + ExecTest(); + } + } + finally + { + Finalizer.Instance.ErrorHandler -= ErrorHandler; + } + Assert.AreEqual(errorArgs.Count, 0); + } - // Call into the new assembly. Will execute Python code - var pythonrunner = theAssembly.GetType("PythonRunner"); - var runPythonMethod = pythonrunner.GetMethod("RunPython"); - runPythonMethod.Invoke(null, new object[] { }); + [Test] + public void CrossReleaseCustomType() + { + void ExecTest() + { + try + { + PythonEngine.Initialize(); + var objRef = CreateConcreateObject(); + Assert.True(objRef.IsAlive); + PythonEngine.Shutdown(); // <- "run" 1 ends + PythonEngine.Initialize(); // <- "run" 2 starts + GC.Collect(); + GC.WaitForPendingFinalizers(); + Finalizer.Instance.Collect(forceDispose: true); + Assert.False(objRef.IsAlive); + } + finally + { + PythonEngine.Shutdown(); + } + } + + var errorArgs = new List(); + void ErrorHandler(object sender, Finalizer.ErrorArgs e) + { + errorArgs.Add(e); + } + Finalizer.Instance.ErrorHandler += ErrorHandler; + try + { + for (int i = 0; i < 10; i++) + { + ExecTest(); + } + } + finally + { + Finalizer.Instance.ErrorHandler -= ErrorHandler; + } + Assert.AreEqual(errorArgs.Count, 0); + } + + private static WeakReference CreateNumReference() + { + var num = 3216757418.ToPython(); + Assert.AreEqual(num.Refcount, 1); + WeakReference numRef = new WeakReference(num, false); + return numRef; + } + + private static WeakReference CreateConcreateObject() + { + var obj = new Domain.MyClass().ToPython(); + Assert.AreEqual(obj.Refcount, 1); + WeakReference numRef = new WeakReference(obj, false); + return numRef; + } + #endregion Tempary tests + + /// + /// This is a magic incantation required to run code in an application + /// domain other than the current one. + /// + class Proxy : MarshalByRefObject + { + public void RunPython() + { + Console.WriteLine("[Proxy] Entering RunPython"); + PythonRunner.RunPython(); Console.WriteLine("[Proxy] Leaving RunPython"); } + + public object Call(string methodName, params object[] args) + { + var pythonrunner = typeof(PythonRunner); + var method = pythonrunner.GetMethod(methodName); + return method.Invoke(null, args); + } + } + + static T CreateInstanceInstanceAndUnwrap(AppDomain domain) + { + Type type = typeof(T); + var theProxy = (T)domain.CreateInstanceAndUnwrap( + type.Assembly.FullName, + type.FullName); + return theProxy; } /// /// Create a domain, run the assembly in it (the RunPython function), /// and unload the domain. /// - static void RunAssemblyAndUnload(Assembly assembly, string assemblyName) + static void RunAssemblyAndUnload(string domainName) { - Console.WriteLine($"[Program.Main] === creating domain for assembly {assembly.FullName}"); - - // Create the domain. Make sure to set PrivateBinPath to a relative - // path from the CWD (namely, 'bin'). - // See https://stackoverflow.com/questions/24760543/createinstanceandunwrap-in-another-domain - var currentDomain = AppDomain.CurrentDomain; - var domainsetup = new AppDomainSetup() - { - ApplicationBase = currentDomain.SetupInformation.ApplicationBase, - ConfigurationFile = currentDomain.SetupInformation.ConfigurationFile, - LoaderOptimization = LoaderOptimization.SingleDomain, - PrivateBinPath = "." - }; - var domain = AppDomain.CreateDomain( - $"My Domain {assemblyName}", - currentDomain.Evidence, - domainsetup); + Console.WriteLine($"[Program.Main] === creating domain {domainName}"); + AppDomain domain = CreateDomain(domainName); // Create a Proxy object in the new domain, where we want the // assembly (and Python .NET) to reside - Type type = typeof(Proxy); - System.IO.Directory.SetCurrentDirectory(AppDomain.CurrentDomain.BaseDirectory); - var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( - type.Assembly.FullName, - type.FullName); + var theProxy = CreateInstanceInstanceAndUnwrap(domain); + theProxy.Call("InitPython", ShutdownMode.Soft); // From now on use the Proxy to call into the new assembly - theProxy.InitAssembly(assemblyName); theProxy.RunPython(); - Console.WriteLine($"[Program.Main] Before Domain Unload on {assembly.FullName}"); + theProxy.Call("ShutdownPython"); + Console.WriteLine($"[Program.Main] Before Domain Unload on {domainName}"); AppDomain.Unload(domain); - Console.WriteLine($"[Program.Main] After Domain Unload on {assembly.FullName}"); + Console.WriteLine($"[Program.Main] After Domain Unload on {domainName}"); // Validate that the assembly does not exist anymore try { Console.WriteLine($"[Program.Main] The Proxy object is valid ({theProxy}). Unexpected domain unload behavior"); + Assert.Fail($"{theProxy} should be invlaid now"); } - catch (Exception) + catch (AppDomainUnloadedException) { Console.WriteLine("[Program.Main] The Proxy object is not valid anymore, domain unload complete."); } } + private static AppDomain CreateDomain(string name) + { + // Create the domain. Make sure to set PrivateBinPath to a relative + // path from the CWD (namely, 'bin'). + // See https://stackoverflow.com/questions/24760543/createinstanceandunwrap-in-another-domain + var currentDomain = AppDomain.CurrentDomain; + var domainsetup = new AppDomainSetup() + { + ApplicationBase = currentDomain.SetupInformation.ApplicationBase, + ConfigurationFile = currentDomain.SetupInformation.ConfigurationFile, + LoaderOptimization = LoaderOptimization.SingleDomain, + PrivateBinPath = "." + }; + var domain = AppDomain.CreateDomain( + $"My Domain {name}", + currentDomain.Evidence, + domainsetup); + return domain; + } + /// /// Resolves the assembly. Why doesn't this just work normally? /// @@ -234,6 +481,141 @@ static Assembly ResolveAssembly(object sender, ResolveEventArgs args) return null; } + + static void RunDomainReloadSteps() where T1 : CrossCaller where T2 : CrossCaller + { + ValueType arg = null; + Type type = typeof(Proxy); + { + AppDomain domain = CreateDomain("test_domain_reload_1"); + try + { + var theProxy = CreateInstanceInstanceAndUnwrap(domain); + theProxy.Call("InitPython", ShutdownMode.Reload); + + var caller = CreateInstanceInstanceAndUnwrap(domain); + arg = caller.Execute(arg); + + theProxy.Call("ShutdownPython"); + } + finally + { + AppDomain.Unload(domain); + } + } + + { + AppDomain domain = CreateDomain("test_domain_reload_2"); + try + { + var theProxy = CreateInstanceInstanceAndUnwrap(domain); + theProxy.Call("InitPython", ShutdownMode.Reload); + + var caller = CreateInstanceInstanceAndUnwrap(domain); + caller.Execute(arg); + theProxy.Call("ShutdownPythonCompletely"); + } + finally + { + AppDomain.Unload(domain); + } + } + if (PythonEngine.DefaultShutdownMode == ShutdownMode.Normal) + { + Assert.IsTrue(PyRuntime.Py_IsInitialized() == 0); + } + } + } + + + // + // The code we'll test. All that really matters is + // using GIL { Python.Exec(pyScript); } + // but the rest is useful for debugging. + // + // What matters in the python code is gc.collect and clr.AddReference. + // + // Note that the language version is 2.0, so no $"foo{bar}" syntax. + // + static class PythonRunner + { + public static void RunPython() + { + AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; + string name = AppDomain.CurrentDomain.FriendlyName; + Console.WriteLine("[{0} in .NET] In PythonRunner.RunPython", name); + using (Py.GIL()) + { + try + { + var pyScript = string.Format("import clr\n" + + "print('[{0} in python] imported clr')\n" + + "clr.AddReference('System')\n" + + "print('[{0} in python] allocated a clr object')\n" + + "import gc\n" + + "gc.collect()\n" + + "print('[{0} in python] collected garbage')\n", + name); + PythonEngine.Exec(pyScript); + } + catch (Exception e) + { + Console.WriteLine(string.Format("[{0} in .NET] Caught exception: {1}", name, e)); + throw; + } + } + } + + + private static IntPtr _state; + + public static void InitPython(ShutdownMode mode) + { + PythonEngine.Initialize(mode: mode); + _state = PythonEngine.BeginAllowThreads(); + } + + public static void ShutdownPython() + { + PythonEngine.EndAllowThreads(_state); + PythonEngine.Shutdown(); + } + + public static void ShutdownPythonCompletely() + { + PythonEngine.EndAllowThreads(_state); + // XXX: Reload mode will reserve clr objects after `Runtime.Shutdown`, + // if it used a another mode(the default mode) in other tests, + // when other tests trying to access these reserved objects, it may cause Domain exception, + // thus it needs to reduct to Soft mode to make sure all clr objects remove from Python. + var defaultMode = PythonEngine.DefaultShutdownMode; + if (defaultMode != ShutdownMode.Reload) + { + PythonEngine.ShutdownMode = defaultMode; + } + PythonEngine.Shutdown(); + } + + static void OnDomainUnload(object sender, EventArgs e) + { + Console.WriteLine(string.Format("[{0} in .NET] unloading", AppDomain.CurrentDomain.FriendlyName)); + } } + } + + +namespace Python.EmbeddingTest.Domain +{ + [Serializable] + public class MyClass + { + public int Property { get; set; } + public int Field; + public void Method() { } + public static void StaticMethod() { } + } +} + + #endif diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 2d8c996bf..a54bc7a96 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -1,6 +1,9 @@ using NUnit.Framework; using Python.Runtime; using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics; using System.Linq; using System.Threading; @@ -25,10 +28,22 @@ public void TearDown() PythonEngine.Shutdown(); } - private static void FullGCCollect() + private static bool FullGCCollect() { GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced); - GC.WaitForPendingFinalizers(); + try + { + return GC.WaitForFullGCComplete() == GCNotificationStatus.Succeeded; + } + catch (NotImplementedException) + { + // Some clr runtime didn't implement GC.WaitForFullGCComplete yet. + return false; + } + finally + { + GC.WaitForPendingFinalizers(); + } } [Test] @@ -88,23 +103,33 @@ public void CollectBasicObject() } [Test] + [Ignore("Ignore temporarily")] public void CollectOnShutdown() { - MakeAGarbage(out var shortWeak, out var longWeak); - FullGCCollect(); - var garbage = Finalizer.Instance.GetCollectedObjects(); - Assert.IsNotEmpty(garbage); + IntPtr op = MakeAGarbage(out var shortWeak, out var longWeak); + int hash = shortWeak.Target.GetHashCode(); + List garbage; + if (!FullGCCollect()) + { + Assert.IsTrue(WaitForCollected(op, hash, 10000)); + } + Assert.IsFalse(shortWeak.IsAlive); + garbage = Finalizer.Instance.GetCollectedObjects(); + Assert.IsNotEmpty(garbage, "The garbage object should be collected"); + Assert.IsTrue(garbage.Any(r => ReferenceEquals(r.Target, longWeak.Target)), + "Garbage should contains the collected object"); + PythonEngine.Shutdown(); garbage = Finalizer.Instance.GetCollectedObjects(); Assert.IsEmpty(garbage); } - private static void MakeAGarbage(out WeakReference shortWeak, out WeakReference longWeak) + private static IntPtr MakeAGarbage(out WeakReference shortWeak, out WeakReference longWeak) { PyLong obj = new PyLong(1024); shortWeak = new WeakReference(obj); longWeak = new WeakReference(obj, true); - obj = null; + return obj.Handle; } private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale) @@ -226,7 +251,7 @@ public void ValidateRefCount() { if (!Finalizer.Instance.RefCountValidationEnabled) { - Assert.Pass("Only run with FINALIZER_CHECK"); + Assert.Ignore("Only run with FINALIZER_CHECK"); } IntPtr ptr = IntPtr.Zero; bool called = false; @@ -261,5 +286,28 @@ private static IntPtr CreateStringGarbage() return s1.Handle; } + private static bool WaitForCollected(IntPtr op, int hash, int milliseconds) + { + var stopwatch = Stopwatch.StartNew(); + do + { + var garbage = Finalizer.Instance.GetCollectedObjects(); + foreach (var item in garbage) + { + // The validation is not 100% precise, + // but it's rare that two conditions satisfied but they're still not the same object. + if (item.Target.GetHashCode() != hash) + { + continue; + } + var obj = (IPyDisposable)item.Target; + if (obj.GetTrackedHandles().Contains(op)) + { + return true; + } + } + } while (stopwatch.ElapsedMilliseconds < milliseconds); + return false; + } } } diff --git a/src/embed_tests/TestPyScope.cs b/src/embed_tests/TestPyScope.cs index 701e698ec..a94b8ce28 100644 --- a/src/embed_tests/TestPyScope.cs +++ b/src/embed_tests/TestPyScope.cs @@ -338,8 +338,8 @@ public void TestThread() //add function to the scope //can be call many times, more efficient than ast ps.Exec( - "import threading\n" + - "lock = threading.Lock()\n" + + "import threading\n"+ + "lock = threading.Lock()\n"+ "def update():\n" + " global res, th_cnt\n" + " with lock:\n" + diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index 4129d3df3..38878205c 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -28,16 +28,22 @@ public static void PlatformCache() { Runtime.Runtime.Initialize(); - Assert.That(Runtime.Runtime.Machine, Is.Not.EqualTo(MachineType.Other)); - Assert.That(Runtime.Runtime.OperatingSystem, Is.Not.EqualTo(OperatingSystemType.Other)); + Assert.That(NativeCodePageHelper.Machine, Is.Not.EqualTo(MachineType.Other)); + Assert.That(!string.IsNullOrEmpty(NativeCodePageHelper.MachineName)); - // Don't shut down the runtime: if the python engine was initialized - // but not shut down by another test, we'd end up in a bad state. - } + Assert.That(NativeCodePageHelper.OperatingSystem, Is.Not.EqualTo(OperatingSystemType.Other)); + Assert.That(!string.IsNullOrEmpty(NativeCodePageHelper.OperatingSystemName)); + + Runtime.Runtime.Shutdown(); + } [Test] public static void Py_IsInitializedValue() { + if (Runtime.Runtime.Py_IsInitialized() == 1) + { + Runtime.Runtime.PyGILState_Ensure(); + } Runtime.Runtime.Py_Finalize(); Assert.AreEqual(0, Runtime.Runtime.Py_IsInitialized()); Runtime.Runtime.Py_Initialize(); diff --git a/src/embed_tests/TestTypeManager.cs b/src/embed_tests/TestTypeManager.cs index 931c44236..43155e1bf 100644 --- a/src/embed_tests/TestTypeManager.cs +++ b/src/embed_tests/TestTypeManager.cs @@ -1,5 +1,6 @@ using NUnit.Framework; using Python.Runtime; +using Python.Runtime.Platform; using System.Runtime.InteropServices; namespace Python.EmbeddingTest @@ -15,22 +16,21 @@ public static void Init() [TearDown] public static void Fini() { - // Don't shut down the runtime: if the python engine was initialized - // but not shut down by another test, we'd end up in a bad state. + Runtime.Runtime.Shutdown(); } [Test] public static void TestNativeCode() { - Assert.That(() => { var _ = TypeManager.NativeCode.Active; }, Throws.Nothing); - Assert.That(TypeManager.NativeCode.Active.Code.Length, Is.GreaterThan(0)); + Assert.That(() => { var _ = NativeCodePageHelper.NativeCode.Active; }, Throws.Nothing); + Assert.That(NativeCodePageHelper.NativeCode.Active.Code.Length, Is.GreaterThan(0)); } [Test] public static void TestMemoryMapping() { - Assert.That(() => { var _ = TypeManager.CreateMemoryMapper(); }, Throws.Nothing); - var mapper = TypeManager.CreateMemoryMapper(); + Assert.That(() => { var _ = NativeCodePageHelper.CreateMemoryMapper(); }, Throws.Nothing); + var mapper = NativeCodePageHelper.CreateMemoryMapper(); // Allocate a read-write page. int len = 12; diff --git a/src/embed_tests/pyimport.cs b/src/embed_tests/pyimport.cs index d78b030ea..ebb4fabd0 100644 --- a/src/embed_tests/pyimport.cs +++ b/src/embed_tests/pyimport.cs @@ -40,6 +40,7 @@ public void SetUp() IntPtr str = Runtime.Runtime.PyString_FromString(testPath); BorrowedReference path = Runtime.Runtime.PySys_GetObject("path"); Runtime.Runtime.PyList_Append(path, str); + Runtime.Runtime.XDecref(str); } [TearDown] diff --git a/src/embed_tests/pyinitialize.cs b/src/embed_tests/pyinitialize.cs index ea1d8d023..c774680dd 100644 --- a/src/embed_tests/pyinitialize.cs +++ b/src/embed_tests/pyinitialize.cs @@ -24,9 +24,11 @@ public static void StartAndStopTwice() public static void LoadDefaultArgs() { using (new PythonEngine()) - using (var argv = new PyList(Runtime.Runtime.PySys_GetObject("argv"))) { - Assert.AreNotEqual(0, argv.Length()); + using(var argv = new PyList(Runtime.Runtime.PySys_GetObject("argv"))) + { + Assert.AreNotEqual(0, argv.Length()); + } } } @@ -35,10 +37,12 @@ public static void LoadSpecificArgs() { var args = new[] { "test1", "test2" }; using (new PythonEngine(args)) - using (var argv = new PyList(Runtime.Runtime.PySys_GetObject("argv"))) { - Assert.AreEqual(args[0], argv[0].ToString()); - Assert.AreEqual(args[1], argv[1].ToString()); + using (var argv = new PyList(Runtime.Runtime.PySys_GetObject("argv"))) + { + Assert.AreEqual(args[0], argv[0].ToString()); + Assert.AreEqual(args[1], argv[1].ToString()); + } } } @@ -134,5 +138,46 @@ public void ShutdownHandlers() // Wrong: (4 * 2) + 1 + 1 + 1 = 11 Assert.That(shutdown_count, Is.EqualTo(12)); } + + [Test] + public static void TestRunExitFuncs() + { + if (Runtime.Runtime.GetDefaultShutdownMode() == ShutdownMode.Normal) + { + // If the runtime using the normal mode, + // callback registered by atexit will be called after we release the clr information, + // thus there's no chance we can check it here. + Assert.Ignore("Skip on normal mode"); + } + Runtime.Runtime.Initialize(); + PyObject atexit; + try + { + atexit = Py.Import("atexit"); + } + catch (PythonException e) + { + string msg = e.ToString(); + Runtime.Runtime.Shutdown(); + + if (e.IsMatches(Exceptions.ImportError)) + { + Assert.Ignore("no atexit module"); + } + else + { + Assert.Fail(msg); + } + return; + } + bool called = false; + Action callback = () => + { + called = true; + }; + atexit.InvokeMethod("register", callback.ToPython()); + Runtime.Runtime.Shutdown(); + Assert.True(called); + } } } diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs index a3bf29056..8ae382e77 100644 --- a/src/runtime/BorrowedReference.cs +++ b/src/runtime/BorrowedReference.cs @@ -10,6 +10,7 @@ readonly ref struct BorrowedReference readonly IntPtr pointer; public bool IsNull => this.pointer == IntPtr.Zero; + /// Gets a raw pointer to the Python object public IntPtr DangerousGetAddress() => this.IsNull ? throw new NullReferenceException() : this.pointer; diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 8afa60f4f..08dc1d860 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,185 +1,188 @@ - - - - Debug - AnyCPU - {097B4AC0-74E9-4C58-BCF8-C69746EC8271} - Library - Python.Runtime - Python.Runtime - bin\Python.Runtime.xml - bin\ - v4.0 - - 1591 - ..\..\ - $(SolutionDir)\bin\ - Properties - 7.3 - true - false - ..\pythonnet.snk - - - - - - PYTHON2;PYTHON27;UCS4 - true - pdbonly - - - PYTHON3;PYTHON38;UCS4 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS4;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON38;UCS4;TRACE;DEBUG - false - full - - - PYTHON2;PYTHON27;UCS2 - true - pdbonly - - - PYTHON3;PYTHON38;UCS2 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS2;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON38;UCS2;TRACE;DEBUG - false - full - - - - - - - - - - - - - - Properties\SharedAssemblyInfo.cs - - - - + + + + Debug + AnyCPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271} + Library + Python.Runtime + Python.Runtime + bin\Python.Runtime.xml + bin\ + v4.0 + + 1591 + ..\..\ + $(SolutionDir)\bin\ + Properties + 7.3 + true + false + ..\pythonnet.snk + + + + + + PYTHON2;PYTHON27;UCS4 + true + pdbonly + + + PYTHON3;PYTHON38;UCS4 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS4;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON38;UCS4;TRACE;DEBUG + false + full + + + PYTHON2;PYTHON27;UCS2 + true + pdbonly + + + PYTHON3;PYTHON38;UCS2 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS2;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON38;UCS2;TRACE;DEBUG + false + full + + + + + + + + + + + + + + Properties\SharedAssemblyInfo.cs + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - clr.py - - - - - $(TargetPath) - $(TargetDir)$(TargetName).pdb - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + clr.py + + + + + $(TargetPath) + $(TargetDir)$(TargetName).pdb + + + + + diff --git a/src/runtime/arrayobject.cs b/src/runtime/arrayobject.cs index 364d91969..0db84dd90 100644 --- a/src/runtime/arrayobject.cs +++ b/src/runtime/arrayobject.cs @@ -8,6 +8,7 @@ namespace Python.Runtime /// the same as a ClassObject, except that it provides sequence semantics /// to support natural array usage (indexing) from Python. /// + [Serializable] internal class ArrayObject : ClassBase { internal ArrayObject(Type tp) : base(tp) diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 972380928..66153fbe1 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -1,6 +1,8 @@ using System; using System.Collections; +using System.Diagnostics; using System.Runtime.InteropServices; +using System.Runtime.Serialization; namespace Python.Runtime { @@ -12,6 +14,7 @@ namespace Python.Runtime /// concrete subclasses provide slot implementations appropriate for /// each variety of reflected type. /// + [Serializable] internal class ClassBase : ManagedType { internal Indexer indexer; @@ -28,13 +31,6 @@ internal virtual bool CanSubclass() return !type.IsEnum; } - /// - /// Implements __init__ for reflected classes and value types. - /// - public static int tp_init(IntPtr ob, IntPtr args, IntPtr kw) - { - return 0; - } /// /// Default implementation of [] semantics for reflected types. @@ -292,15 +288,44 @@ public static IntPtr tp_repr(IntPtr ob) public static void tp_dealloc(IntPtr ob) { ManagedType self = GetManagedObject(ob); - IntPtr dict = Marshal.ReadIntPtr(ob, ObjectOffset.TypeDictOffset(self.tpHandle)); - if (dict != IntPtr.Zero) - { - Runtime.XDecref(dict); - } + tp_clear(ob); Runtime.PyObject_GC_UnTrack(self.pyHandle); Runtime.PyObject_GC_Del(self.pyHandle); - Runtime.XDecref(self.tpHandle); - self.gcHandle.Free(); + self.FreeGCHandle(); + } + + public static int tp_clear(IntPtr ob) + { + ManagedType self = GetManagedObject(ob); + if (!self.IsTypeObject()) + { + ClearObjectDict(ob); + } + self.tpHandle = IntPtr.Zero; + return 0; + } + + protected override void OnSave(InterDomainContext context) + { + base.OnSave(context); + if (pyHandle != tpHandle) + { + IntPtr dict = GetObjectDict(pyHandle); + Runtime.XIncref(dict); + context.Storage.AddValue("dict", dict); + } + } + + protected override void OnLoad(InterDomainContext context) + { + base.OnLoad(context); + if (pyHandle != tpHandle) + { + IntPtr dict = context.Storage.GetValue("dict"); + SetObjectDict(pyHandle, dict); + } + gcHandle = AllocGCHandle(); + Marshal.WriteIntPtr(pyHandle, TypeOffset.magic(), (IntPtr)gcHandle); } diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index af16b1359..e55e89240 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -22,6 +22,7 @@ public interface IPythonDerivedType { } + [Serializable] internal class ClassDerivedObject : ClassObject { private static Dictionary assemblyBuilders; @@ -99,6 +100,10 @@ internal static IntPtr ToPython(IPythonDerivedType obj) // collected while Python still has a reference to it. if (Runtime.Refcount(self.pyHandle) == 1) { + +#if PYTHON_WITH_PYDEBUG + Runtime._Py_NewReference(self.pyHandle); +#endif GCHandle gc = GCHandle.Alloc(self, GCHandleType.Normal); Marshal.WriteIntPtr(self.pyHandle, ObjectOffset.magic(self.tpHandle), (IntPtr)gc); self.gcHandle.Free(); @@ -130,7 +135,7 @@ internal static Type CreateDerivedType(string name, if (null == assemblyName) { - assemblyName = Assembly.GetExecutingAssembly().FullName; + assemblyName = "Python.Runtime.Dynamic"; } ModuleBuilder moduleBuilder = GetModuleBuilder(assemblyName, moduleName); diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 08918adc1..15f3d821d 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -4,6 +4,7 @@ using System.Reflection; using System.Runtime.InteropServices; using System.Security; +using System.Linq; namespace Python.Runtime { @@ -18,7 +19,7 @@ namespace Python.Runtime internal class ClassManager { private static Dictionary cache; - private static Type dtype; + private static readonly Type dtype; private ClassManager() { @@ -26,7 +27,6 @@ private ClassManager() static ClassManager() { - cache = new Dictionary(128); // SEE: https://msdn.microsoft.com/en-us/library/96b1ayy4(v=vs.100).aspx // ""All delegates inherit from MulticastDelegate, which inherits from Delegate."" // Was Delegate, which caused a null MethodInfo returned from GetMethode("Invoke") @@ -39,6 +39,76 @@ public static void Reset() cache = new Dictionary(128); } + internal static void DisposePythonWrappersForClrTypes() + { + var visited = new HashSet(); + var visitedHandle = GCHandle.Alloc(visited); + var visitedPtr = (IntPtr)visitedHandle; + try + { + foreach (var cls in cache.Values) + { + // XXX: Force to release instance's managed resources + // but not dealloc itself immediately. + // These managed resources should preserve vacant shells + // since others may still referencing it. + cls.CallTypeTraverse(TraverseTypeClear, visitedPtr); + cls.CallTypeClear(); + cls.DecrRefCount(); + } + } + finally + { + visitedHandle.Free(); + } + cache.Clear(); + } + + private static int TraverseTypeClear(IntPtr ob, IntPtr arg) + { + var visited = (HashSet)GCHandle.FromIntPtr(arg).Target; + if (!visited.Add(ob)) + { + return 0; + } + var clrObj = ManagedType.GetManagedObject(ob); + if (clrObj != null) + { + clrObj.CallTypeTraverse(TraverseTypeClear, arg); + clrObj.CallTypeClear(); + } + return 0; + } + + internal static void SaveRuntimeData(RuntimeDataStorage storage) + { + var contexts = storage.AddValue("contexts", + new Dictionary()); + storage.AddValue("cache", cache); + foreach (var cls in cache.Values) + { + // This incref is for cache to hold the cls, + // thus no need for decreasing it at RestoreRuntimeData. + Runtime.XIncref(cls.pyHandle); + var context = contexts[cls.pyHandle] = new InterDomainContext(); + cls.Save(context); + } + } + + internal static Dictionary RestoreRuntimeData(RuntimeDataStorage storage) + { + cache = storage.GetValue>("cache"); + var contexts = storage.GetValue >("contexts"); + var loadedObjs = new Dictionary(); + foreach (var cls in cache.Values) + { + var context = contexts[cls.pyHandle]; + cls.Load(context); + loadedObjs.Add(cls, context); + } + return loadedObjs; + } + /// /// Return the ClassBase-derived instance that implements a particular /// reflected managed type, creating it if it doesn't yet exist. @@ -134,7 +204,6 @@ private static void InitClassBase(Type type, ClassBase impl) IntPtr tp = TypeManager.GetTypeHandle(impl, type); - impl.tpHandle = tp; // Finally, initialize the class __dict__ and return the object. IntPtr dict = Marshal.ReadIntPtr(tp, TypeOffset.tp_dict); @@ -146,6 +215,8 @@ private static void InitClassBase(Type type, ClassBase impl) var item = (ManagedType)iter.Value; var name = (string)iter.Key; Runtime.PyDict_SetItemString(dict, name, item.pyHandle); + // Decref the item now that it's been used. + item.DecrRefCount(); } // If class has constructors, generate an __doc__ attribute. @@ -180,6 +251,7 @@ private static void InitClassBase(Type type, ClassBase impl) // TODO: deprecate __overloads__ soon... Runtime.PyDict_SetItemString(dict, "__overloads__", ctors.pyHandle); Runtime.PyDict_SetItemString(dict, "Overloads", ctors.pyHandle); + ctors.DecrRefCount(); } // don't generate the docstring if one was already set from a DocStringAttribute. @@ -195,7 +267,7 @@ private static void InitClassBase(Type type, ClassBase impl) private static ClassInfo GetClassInfo(Type type) { - var ci = new ClassInfo(type); + var ci = new ClassInfo(); var methods = new Hashtable(); ArrayList list; MethodInfo meth; @@ -410,18 +482,22 @@ private static ClassInfo GetClassInfo(Type type) return ci; } - } - - - internal class ClassInfo - { - public Indexer indexer; - public Hashtable members; - - internal ClassInfo(Type t) + + /// + /// This class owns references to PyObjects in the `members` member. + /// The caller has responsibility to DECREF them. + /// + private class ClassInfo { - members = new Hashtable(); - indexer = null; + public Indexer indexer; + public Hashtable members; + + internal ClassInfo() + { + members = new Hashtable(); + indexer = null; + } } } + } diff --git a/src/runtime/classobject.cs b/src/runtime/classobject.cs index d7624ed6e..18816781f 100644 --- a/src/runtime/classobject.cs +++ b/src/runtime/classobject.cs @@ -9,6 +9,7 @@ namespace Python.Runtime /// Python type objects. Each of those type objects is associated with /// an instance of ClassObject, which provides its implementation. /// + [Serializable] internal class ClassObject : ClassBase { internal ConstructorBinder binder; diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 5c7ad7891..0b62fecba 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -3,12 +3,14 @@ namespace Python.Runtime { + [Serializable] internal class CLRObject : ManagedType { internal object inst; internal CLRObject(object ob, IntPtr tp) { + System.Diagnostics.Debug.Assert(tp != IntPtr.Zero); IntPtr py = Runtime.PyType_GenericAlloc(tp, 0); long flags = Util.ReadCLong(tp, TypeOffset.tp_flags); @@ -22,11 +24,10 @@ internal CLRObject(object ob, IntPtr tp) } } - GCHandle gc = GCHandle.Alloc(this); + GCHandle gc = AllocGCHandle(TrackTypes.Wrapper); Marshal.WriteIntPtr(py, ObjectOffset.magic(tp), (IntPtr)gc); tpHandle = tp; pyHandle = py; - gcHandle = gc; inst = ob; // Fix the BaseException args (and __cause__ in case of Python 3) @@ -34,6 +35,9 @@ internal CLRObject(object ob, IntPtr tp) Exceptions.SetArgsAndCause(py); } + protected CLRObject() + { + } static CLRObject GetInstance(object ob, IntPtr pyType) { @@ -68,5 +72,30 @@ internal static IntPtr GetInstHandle(object ob) CLRObject co = GetInstance(ob); return co.pyHandle; } + + internal static CLRObject Restore(object ob, IntPtr pyHandle, InterDomainContext context) + { + CLRObject co = new CLRObject() + { + inst = ob, + pyHandle = pyHandle, + tpHandle = Runtime.PyObject_TYPE(pyHandle) + }; + co.Load(context); + return co; + } + + protected override void OnSave(InterDomainContext context) + { + base.OnSave(context); + Runtime.XIncref(pyHandle); + } + + protected override void OnLoad(InterDomainContext context) + { + base.OnLoad(context); + GCHandle gc = AllocGCHandle(TrackTypes.Wrapper); + Marshal.WriteIntPtr(pyHandle, ObjectOffset.magic(tpHandle), (IntPtr)gc); + } } } diff --git a/src/runtime/constructorbinder.cs b/src/runtime/constructorbinder.cs index 973707f02..0cda3a3d9 100644 --- a/src/runtime/constructorbinder.cs +++ b/src/runtime/constructorbinder.cs @@ -11,6 +11,7 @@ namespace Python.Runtime /// standard MethodBinder because of a difference in invoking constructors /// using reflection (which is seems to be a CLR bug). /// + [Serializable] internal class ConstructorBinder : MethodBinder { private Type _containingType; diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index 3908628b9..0c81c0a93 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -19,18 +19,20 @@ namespace Python.Runtime /// and creating the BoundContructor object which contains ContructorInfo object. /// 3) In tp_call, if ctorInfo is not null, ctorBinder.InvokeRaw() is called. /// + [Serializable] internal class ConstructorBinding : ExtensionType { private Type type; // The managed Type being wrapped in a ClassObject private IntPtr pyTypeHndl; // The python type tells GetInstHandle which Type to create. private ConstructorBinder ctorBinder; + + [NonSerialized] private IntPtr repr; public ConstructorBinding(Type type, IntPtr pyTypeHndl, ConstructorBinder ctorBinder) { this.type = type; - Runtime.XIncref(pyTypeHndl); - this.pyTypeHndl = pyTypeHndl; + this.pyTypeHndl = pyTypeHndl; // steal a type reference this.ctorBinder = ctorBinder; repr = IntPtr.Zero; } @@ -144,8 +146,25 @@ public static IntPtr tp_repr(IntPtr ob) { var self = (ConstructorBinding)GetManagedObject(ob); Runtime.XDecref(self.repr); - Runtime.XDecref(self.pyTypeHndl); - ExtensionType.FinalizeObject(self); + self.Dealloc(); + } + + public static int tp_clear(IntPtr ob) + { + var self = (ConstructorBinding)GetManagedObject(ob); + Runtime.Py_CLEAR(ref self.repr); + return 0; + } + + public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) + { + var self = (ConstructorBinding)GetManagedObject(ob); + int res = PyVisit(self.pyTypeHndl, visit, arg); + if (res != 0) return res; + + res = PyVisit(self.repr, visit, arg); + if (res != 0) return res; + return 0; } } @@ -157,6 +176,7 @@ public static IntPtr tp_repr(IntPtr ob) /// An earlier implementation hung the __call__ on the ContructorBinding class and /// returned an Incref()ed self.pyHandle from the __get__ function. /// + [Serializable] internal class BoundContructor : ExtensionType { private Type type; // The managed Type being wrapped in a ClassObject @@ -168,8 +188,7 @@ internal class BoundContructor : ExtensionType public BoundContructor(Type type, IntPtr pyTypeHndl, ConstructorBinder ctorBinder, ConstructorInfo ci) { this.type = type; - Runtime.XIncref(pyTypeHndl); - this.pyTypeHndl = pyTypeHndl; + this.pyTypeHndl = pyTypeHndl; // steal a type reference this.ctorBinder = ctorBinder; ctorInfo = ci; repr = IntPtr.Zero; @@ -230,8 +249,25 @@ public static IntPtr tp_repr(IntPtr ob) { var self = (BoundContructor)GetManagedObject(ob); Runtime.XDecref(self.repr); - Runtime.XDecref(self.pyTypeHndl); - FinalizeObject(self); + self.Dealloc(); + } + + public static int tp_clear(IntPtr ob) + { + var self = (BoundContructor)GetManagedObject(ob); + Runtime.Py_CLEAR(ref self.repr); + return 0; + } + + public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) + { + var self = (BoundContructor)GetManagedObject(ob); + int res = PyVisit(self.pyTypeHndl, visit, arg); + if (res != 0) return res; + + res = PyVisit(self.repr, visit, arg); + if (res != 0) return res; + return 0; } } } diff --git a/src/runtime/delegateobject.cs b/src/runtime/delegateobject.cs index c9aad9898..c5078740f 100644 --- a/src/runtime/delegateobject.cs +++ b/src/runtime/delegateobject.cs @@ -8,6 +8,7 @@ namespace Python.Runtime /// Each of those type objects is associated an instance of this class, /// which provides its implementation. /// + [Serializable] internal class DelegateObject : ClassBase { private MethodBinder binder; diff --git a/src/runtime/eventbinding.cs b/src/runtime/eventbinding.cs index b8b4c82ad..581095185 100644 --- a/src/runtime/eventbinding.cs +++ b/src/runtime/eventbinding.cs @@ -5,6 +5,7 @@ namespace Python.Runtime /// /// Implements a Python event binding type, similar to a method binding. /// + [Serializable] internal class EventBinding : ExtensionType { private EventObject e; @@ -118,7 +119,14 @@ public static IntPtr tp_repr(IntPtr ob) { var self = (EventBinding)GetManagedObject(ob); Runtime.XDecref(self.target); - ExtensionType.FinalizeObject(self); + self.Dealloc(); + } + + public static int tp_clear(IntPtr ob) + { + var self = (EventBinding)GetManagedObject(ob); + Runtime.Py_CLEAR(ref self.target); + return 0; } } } diff --git a/src/runtime/eventobject.cs b/src/runtime/eventobject.cs index 5f18c4609..0f2796a14 100644 --- a/src/runtime/eventobject.cs +++ b/src/runtime/eventobject.cs @@ -7,6 +7,7 @@ namespace Python.Runtime /// /// Implements a Python descriptor type that provides access to CLR events. /// + [Serializable] internal class EventObject : ExtensionType { internal string name; @@ -202,7 +203,7 @@ public static IntPtr tp_repr(IntPtr ob) { Runtime.XDecref(self.unbound.pyHandle); } - FinalizeObject(self); + self.Dealloc(); } } diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index f4cb519a6..58506bfbb 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -15,6 +15,7 @@ namespace Python.Runtime /// it subclasses System.Object. Instead TypeManager.CreateType() uses /// Python's exception.Exception class as base class for System.Exception. /// + [Serializable] internal class ExceptionClassObject : ClassObject { internal ExceptionClassObject(Type tp) : base(tp) @@ -89,15 +90,11 @@ internal static Exception ToException(IntPtr ob) /// /// Readability of the Exceptions class improvements as we look toward version 2.7 ... /// - public class Exceptions + public static class Exceptions { internal static IntPtr warnings_module; internal static IntPtr exceptions_module; - private Exceptions() - { - } - /// /// Initialization performed on startup of the Python runtime. /// @@ -132,21 +129,23 @@ internal static void Initialize() /// internal static void Shutdown() { - if (Runtime.Py_IsInitialized() != 0) + if (Runtime.Py_IsInitialized() == 0) + { + return; + } + Type type = typeof(Exceptions); + foreach (FieldInfo fi in type.GetFields(BindingFlags.Public | BindingFlags.Static)) { - Type type = typeof(Exceptions); - foreach (FieldInfo fi in type.GetFields(BindingFlags.Public | BindingFlags.Static)) + var op = (IntPtr)fi.GetValue(type); + if (op == IntPtr.Zero) { - var op = (IntPtr)fi.GetValue(type); - if (op != IntPtr.Zero) - { - Runtime.XDecref(op); - } + continue; } - Runtime.XDecref(exceptions_module); - Runtime.PyObject_HasAttrString(warnings_module, "xx"); - Runtime.XDecref(warnings_module); + Runtime.XDecref(op); + fi.SetValue(null, IntPtr.Zero); } + Runtime.Py_CLEAR(ref exceptions_module); + Runtime.Py_CLEAR(ref warnings_module); } /// diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 6585180c1..a5f0f1219 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -8,6 +8,7 @@ namespace Python.Runtime /// type object, such as the types that represent CLR methods, fields, /// etc. Instances implemented by this class do not support sub-typing. /// + [Serializable] internal abstract class ExtensionType : ManagedType { public ExtensionType() @@ -28,20 +29,24 @@ public ExtensionType() IntPtr py = Runtime.PyType_GenericAlloc(tp, 0); - GCHandle gc = GCHandle.Alloc(this); - Marshal.WriteIntPtr(py, ObjectOffset.magic(tp), (IntPtr)gc); + // Steals a ref to tpHandle. + tpHandle = tp; + pyHandle = py; + + SetupGc(); + } + + void SetupGc () + { + GCHandle gc = AllocGCHandle(TrackTypes.Extension); + Marshal.WriteIntPtr(pyHandle, ObjectOffset.magic(tpHandle), (IntPtr)gc); // We have to support gc because the type machinery makes it very // hard not to - but we really don't have a need for it in most // concrete extension types, so untrack the object to save calls // from Python into the managed runtime that are pure overhead. - Runtime.PyObject_GC_UnTrack(py); - - // Steals a ref to tpHandle. - tpHandle = tp; - pyHandle = py; - gcHandle = gc; + Runtime.PyObject_GC_UnTrack(pyHandle); } @@ -50,11 +55,16 @@ public ExtensionType() /// public static void FinalizeObject(ManagedType self) { + ClearObjectDict(self.pyHandle); Runtime.PyObject_GC_Del(self.pyHandle); // Not necessary for decref of `tpHandle`. - self.gcHandle.Free(); + self.FreeGCHandle(); } + protected void Dealloc() + { + FinalizeObject(this); + } /// /// Type __setattr__ implementation. @@ -89,8 +99,14 @@ public static void tp_dealloc(IntPtr ob) { // Clean up a Python instance of this extension type. This // frees the allocated Python object and decrefs the type. - ManagedType self = GetManagedObject(ob); - FinalizeObject(self); + var self = (ExtensionType)GetManagedObject(ob); + self.Dealloc(); + } + + protected override void OnLoad(InterDomainContext context) + { + base.OnLoad(context); + SetupGc(); } } } diff --git a/src/runtime/fieldobject.cs b/src/runtime/fieldobject.cs index 7c9a466d5..86b93dd1b 100644 --- a/src/runtime/fieldobject.cs +++ b/src/runtime/fieldobject.cs @@ -6,6 +6,7 @@ namespace Python.Runtime /// /// Implements a Python descriptor type that provides access to CLR fields. /// + [Serializable] internal class FieldObject : ExtensionType { private FieldInfo info; @@ -55,6 +56,11 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) try { var co = (CLRObject)GetManagedObject(ob); + if (co == null) + { + Exceptions.SetError(Exceptions.TypeError, "instance is not a clr object"); + return IntPtr.Zero; + } result = info.GetValue(co.inst); return Converter.ToPython(result, info.FieldType); } @@ -115,6 +121,11 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) if (!is_static) { var co = (CLRObject)GetManagedObject(ob); + if (co == null) + { + Exceptions.SetError(Exceptions.TypeError, "instance is not a clr object"); + return -1; + } info.SetValue(co.inst, newval); } else diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index ba562cc26..70b69345b 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -57,7 +57,7 @@ public IncorrectRefCountException(IntPtr ptr) IntPtr pyname = Runtime.PyObject_Unicode(PyPtr); string name = Runtime.GetManagedString(pyname); Runtime.XDecref(pyname); - _message = $"{name} may has a incorrect ref count"; + _message = $"<{name}> may has a incorrect ref count"; } } diff --git a/src/runtime/generictype.cs b/src/runtime/generictype.cs index eeae801d2..76d2e9a5d 100644 --- a/src/runtime/generictype.cs +++ b/src/runtime/generictype.cs @@ -8,6 +8,7 @@ namespace Python.Runtime /// generic types. Both are essentially factories for creating closed /// types based on the required generic type parameters. /// + [Serializable] internal class GenericType : ClassBase { internal GenericType(Type tp) : base(tp) diff --git a/src/runtime/genericutil.cs b/src/runtime/genericutil.cs index 3a230e12c..df78d9899 100644 --- a/src/runtime/genericutil.cs +++ b/src/runtime/genericutil.cs @@ -16,11 +16,6 @@ private GenericUtil() { } - static GenericUtil() - { - mapping = new Dictionary>>(); - } - public static void Reset() { mapping = new Dictionary>>(); diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 96a8b7ebe..8cf57c85d 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -7,7 +7,7 @@ namespace Python.Runtime /// /// Implements the "import hook" used to integrate Python with the CLR. /// - internal class ImportHook + internal static class ImportHook { private static IntPtr py_import; private static CLRModule root; @@ -122,6 +122,24 @@ internal static void Shutdown() CLRModule.Reset(); } + internal static void SaveRuntimeData(RuntimeDataStorage storage) + { + // Increment the reference counts here so that the objects don't + // get freed in Shutdown. + Runtime.XIncref(py_clr_module); + Runtime.XIncref(root.pyHandle); + storage.AddValue("py_clr_module", py_clr_module); + storage.AddValue("root", root.pyHandle); + } + + internal static void RestoreRuntimeData(RuntimeDataStorage storage) + { + InitImport(); + storage.GetValue("py_clr_module", out py_clr_module); + var rootHandle = storage.GetValue("root"); + root = (CLRModule)ManagedType.GetManagedObject(rootHandle); + } + /// /// Return the clr python module (new reference) /// diff --git a/src/runtime/indexer.cs b/src/runtime/indexer.cs index 71f7e7aa1..0772b57c6 100644 --- a/src/runtime/indexer.cs +++ b/src/runtime/indexer.cs @@ -6,6 +6,7 @@ namespace Python.Runtime /// /// Bundles the information required to support an indexer property. /// + [Serializable] internal class Indexer { public MethodBinder GetterBinder; diff --git a/src/runtime/interfaceobject.cs b/src/runtime/interfaceobject.cs index 74396f50c..a2fa86479 100644 --- a/src/runtime/interfaceobject.cs +++ b/src/runtime/interfaceobject.cs @@ -10,6 +10,7 @@ namespace Python.Runtime /// Each of those type objects is associated with an instance of this /// class, which provides the implementation for the Python type. /// + [Serializable] internal class InterfaceObject : ClassBase { internal ConstructorInfo ctor; diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 95f3e5b9f..1caabab17 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -5,7 +5,6 @@ using System.Runtime.InteropServices; using System.Reflection; using System.Text; -using System.Collections.Generic; namespace Python.Runtime { @@ -69,39 +68,83 @@ public ModulePropertyAttribute() } } + internal static partial class TypeOffset + { + static TypeOffset() + { + Type type = typeof(TypeOffset); + FieldInfo[] fields = type.GetFields(); + int size = IntPtr.Size; + for (int i = 0; i < fields.Length; i++) + { + int offset = i * size; + FieldInfo fi = fields[i]; + fi.SetValue(null, offset); + } + } + + public static int magic() => ManagedDataOffsets.Magic; + } + internal static class ManagedDataOffsets { + public static int Magic { get; private set; } + public static readonly Dictionary NameMapping = new Dictionary(); + + static class DataOffsets + { + public static readonly int ob_data; + public static readonly int ob_dict; + + static DataOffsets() + { + FieldInfo[] fields = typeof(DataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); + for (int i = 0; i < fields.Length; i++) + { + fields[i].SetValue(null, -(i * IntPtr.Size) - IntPtr.Size); + } + } + } + static ManagedDataOffsets() { - FieldInfo[] fi = typeof(ManagedDataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); - for (int i = 0; i < fi.Length; i++) + Type type = typeof(TypeOffset); + foreach (FieldInfo fi in type.GetFields()) { - fi[i].SetValue(null, -(i * IntPtr.Size) - IntPtr.Size); + NameMapping[fi.Name] = (int)fi.GetValue(null); } + // XXX: Use the members after PyHeapTypeObject as magic slot + Magic = TypeOffset.members; - size = fi.Length * IntPtr.Size; + FieldInfo[] fields = typeof(DataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); + size = fields.Length * IntPtr.Size; } - public static readonly int ob_data; - public static readonly int ob_dict; + public static int GetSlotOffset(string name) + { + return NameMapping[name]; + } private static int BaseOffset(IntPtr type) { Debug.Assert(type != IntPtr.Zero); int typeSize = Marshal.ReadInt32(type, TypeOffset.tp_basicsize); - Debug.Assert(typeSize > 0 && typeSize <= ExceptionOffset.Size()); + Debug.Assert(typeSize > 0); return typeSize; } + public static int DataOffset(IntPtr type) { - return BaseOffset(type) + ob_data; + return BaseOffset(type) + DataOffsets.ob_data; } public static int DictOffset(IntPtr type) { - return BaseOffset(type) + ob_dict; + return BaseOffset(type) + DataOffsets.ob_dict; } + public static int ob_data => DataOffsets.ob_data; + public static int ob_dict => DataOffsets.ob_dict; public static int Size { get { return size; } } private static readonly int size; @@ -312,6 +355,7 @@ public static void FreeModuleDef(IntPtr ptr) public static int name = 0; } + /// /// TypeFlags(): The actual bit values for the Type Flags stored /// in a class. @@ -320,34 +364,34 @@ public static void FreeModuleDef(IntPtr ptr) /// internal class TypeFlags { - public static int HeapType = (1 << 9); - public static int BaseType = (1 << 10); - public static int Ready = (1 << 12); - public static int Readying = (1 << 13); - public static int HaveGC = (1 << 14); + public const int HeapType = (1 << 9); + public const int BaseType = (1 << 10); + public const int Ready = (1 << 12); + public const int Readying = (1 << 13); + public const int HaveGC = (1 << 14); // 15 and 16 are reserved for stackless - public static int HaveStacklessExtension = 0; + public const int HaveStacklessExtension = 0; /* XXX Reusing reserved constants */ - public static int Managed = (1 << 15); // PythonNet specific - public static int Subclass = (1 << 16); // PythonNet specific - public static int HaveIndex = (1 << 17); + public const int Managed = (1 << 15); // PythonNet specific + public const int Subclass = (1 << 16); // PythonNet specific + public const int HaveIndex = (1 << 17); /* Objects support nb_index in PyNumberMethods */ - public static int HaveVersionTag = (1 << 18); - public static int ValidVersionTag = (1 << 19); - public static int IsAbstract = (1 << 20); - public static int HaveNewBuffer = (1 << 21); + public const int HaveVersionTag = (1 << 18); + public const int ValidVersionTag = (1 << 19); + public const int IsAbstract = (1 << 20); + public const int HaveNewBuffer = (1 << 21); // TODO: Implement FastSubclass functions - public static int IntSubclass = (1 << 23); - public static int LongSubclass = (1 << 24); - public static int ListSubclass = (1 << 25); - public static int TupleSubclass = (1 << 26); - public static int StringSubclass = (1 << 27); - public static int UnicodeSubclass = (1 << 28); - public static int DictSubclass = (1 << 29); - public static int BaseExceptionSubclass = (1 << 30); - public static int TypeSubclass = (1 << 31); - - public static int Default = ( + public const int IntSubclass = (1 << 23); + public const int LongSubclass = (1 << 24); + public const int ListSubclass = (1 << 25); + public const int TupleSubclass = (1 << 26); + public const int StringSubclass = (1 << 27); + public const int UnicodeSubclass = (1 << 28); + public const int DictSubclass = (1 << 29); + public const int BaseExceptionSubclass = (1 << 30); + public const int TypeSubclass = (1 << 31); + + public const int Default = ( HaveStacklessExtension | HaveVersionTag); } @@ -360,7 +404,6 @@ internal class TypeFlags internal class Interop { - private static List keepAlive; private static Hashtable pmap; static Interop() @@ -377,7 +420,6 @@ static Interop() p[item.Name] = item; } - keepAlive = new List(); pmap = new Hashtable(); pmap["tp_dealloc"] = p["DestructorFunc"]; @@ -482,11 +524,10 @@ internal static ThunkInfo GetThunk(MethodInfo method, string funcType = null) } Delegate d = Delegate.CreateDelegate(dt, method); var info = new ThunkInfo(d); - // TODO: remove keepAlive when #958 merged, let the lifecycle of ThunkInfo transfer to caller. - keepAlive.Add(info); return info; } + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] public delegate IntPtr UnaryFunc(IntPtr ob); @@ -528,17 +569,6 @@ internal static ThunkInfo GetThunk(MethodInfo method, string funcType = null) } - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal struct Thunk - { - public Delegate fn; - - public Thunk(Delegate d) - { - fn = d; - } - } - internal class ThunkInfo { public readonly Delegate Target; @@ -556,4 +586,29 @@ public ThunkInfo(Delegate target) Address = Marshal.GetFunctionPointerForDelegate(target); } } + + [StructLayout(LayoutKind.Sequential)] + struct PyGC_Node + { + public IntPtr gc_next; + public IntPtr gc_prev; + public IntPtr gc_refs; + } + + [StructLayout(LayoutKind.Sequential)] + struct PyGC_Head + { + public PyGC_Node gc; + } + + + [StructLayout(LayoutKind.Sequential)] + struct PyMethodDef + { + public IntPtr ml_name; + public IntPtr ml_meth; + public int ml_flags; + public IntPtr ml_doc; + } + } diff --git a/src/runtime/interop36.cs b/src/runtime/interop36.cs index c46bcc2f5..d68539d56 100644 --- a/src/runtime/interop36.cs +++ b/src/runtime/interop36.cs @@ -1,5 +1,6 @@ + // Auto-generated by geninterop.py. -// DO NOT MODIFIY BY HAND. +// DO NOT MODIFY BY HAND. #if PYTHON36 @@ -12,25 +13,10 @@ namespace Python.Runtime { - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } + [StructLayout(LayoutKind.Sequential)] + internal static partial class TypeOffset + { // Auto-generated from PyHeapTypeObject in Python.h public static int ob_refcnt = 0; public static int ob_type = 0; @@ -144,6 +130,96 @@ public static int magic() /* here are optional user slots, followed by the members. */ public static int members = 0; } -} + [StructLayout(LayoutKind.Sequential)] + internal struct PyNumberMethods + { + public IntPtr nb_add; + public IntPtr nb_subtract; + public IntPtr nb_multiply; + public IntPtr nb_remainder; + public IntPtr nb_divmod; + public IntPtr nb_power; + public IntPtr nb_negative; + public IntPtr nb_positive; + public IntPtr nb_absolute; + public IntPtr nb_bool; + public IntPtr nb_invert; + public IntPtr nb_lshift; + public IntPtr nb_rshift; + public IntPtr nb_and; + public IntPtr nb_xor; + public IntPtr nb_or; + public IntPtr nb_int; + public IntPtr nb_reserved; + public IntPtr nb_float; + public IntPtr nb_inplace_add; + public IntPtr nb_inplace_subtract; + public IntPtr nb_inplace_multiply; + public IntPtr nb_inplace_remainder; + public IntPtr nb_inplace_power; + public IntPtr nb_inplace_lshift; + public IntPtr nb_inplace_rshift; + public IntPtr nb_inplace_and; + public IntPtr nb_inplace_xor; + public IntPtr nb_inplace_or; + public IntPtr nb_floor_divide; + public IntPtr nb_true_divide; + public IntPtr nb_inplace_floor_divide; + public IntPtr nb_inplace_true_divide; + public IntPtr nb_index; + public IntPtr nb_matrix_multiply; + public IntPtr nb_inplace_matrix_multiply; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PySequenceMethods + { + public IntPtr sq_length; + public IntPtr sq_concat; + public IntPtr sq_repeat; + public IntPtr sq_item; + public IntPtr was_sq_slice; + public IntPtr sq_ass_item; + public IntPtr was_sq_ass_slice; + public IntPtr sq_contains; + public IntPtr sq_inplace_concat; + public IntPtr sq_inplace_repeat; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyMappingMethods + { + public IntPtr mp_length; + public IntPtr mp_subscript; + public IntPtr mp_ass_subscript; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyAsyncMethods + { + public IntPtr am_await; + public IntPtr am_aiter; + public IntPtr am_anext; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyBufferProcs + { + public IntPtr bf_getbuffer; + public IntPtr bf_releasebuffer; + } + + internal static partial class SlotTypes + { + public static readonly Type[] Types = { + typeof(PyNumberMethods), + typeof(PySequenceMethods), + typeof(PyMappingMethods), + typeof(PyAsyncMethods), + typeof(PyBufferProcs), + }; + } + +} #endif diff --git a/src/runtime/interop37.cs b/src/runtime/interop37.cs index d5fc76ad3..c85d06525 100644 --- a/src/runtime/interop37.cs +++ b/src/runtime/interop37.cs @@ -1,5 +1,6 @@ + // Auto-generated by geninterop.py. -// DO NOT MODIFIY BY HAND. +// DO NOT MODIFY BY HAND. #if PYTHON37 @@ -12,25 +13,10 @@ namespace Python.Runtime { - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } + [StructLayout(LayoutKind.Sequential)] + internal static partial class TypeOffset + { // Auto-generated from PyHeapTypeObject in Python.h public static int ob_refcnt = 0; public static int ob_type = 0; @@ -144,6 +130,96 @@ public static int magic() /* here are optional user slots, followed by the members. */ public static int members = 0; } -} + [StructLayout(LayoutKind.Sequential)] + internal struct PyNumberMethods + { + public IntPtr nb_add; + public IntPtr nb_subtract; + public IntPtr nb_multiply; + public IntPtr nb_remainder; + public IntPtr nb_divmod; + public IntPtr nb_power; + public IntPtr nb_negative; + public IntPtr nb_positive; + public IntPtr nb_absolute; + public IntPtr nb_bool; + public IntPtr nb_invert; + public IntPtr nb_lshift; + public IntPtr nb_rshift; + public IntPtr nb_and; + public IntPtr nb_xor; + public IntPtr nb_or; + public IntPtr nb_int; + public IntPtr nb_reserved; + public IntPtr nb_float; + public IntPtr nb_inplace_add; + public IntPtr nb_inplace_subtract; + public IntPtr nb_inplace_multiply; + public IntPtr nb_inplace_remainder; + public IntPtr nb_inplace_power; + public IntPtr nb_inplace_lshift; + public IntPtr nb_inplace_rshift; + public IntPtr nb_inplace_and; + public IntPtr nb_inplace_xor; + public IntPtr nb_inplace_or; + public IntPtr nb_floor_divide; + public IntPtr nb_true_divide; + public IntPtr nb_inplace_floor_divide; + public IntPtr nb_inplace_true_divide; + public IntPtr nb_index; + public IntPtr nb_matrix_multiply; + public IntPtr nb_inplace_matrix_multiply; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PySequenceMethods + { + public IntPtr sq_length; + public IntPtr sq_concat; + public IntPtr sq_repeat; + public IntPtr sq_item; + public IntPtr was_sq_slice; + public IntPtr sq_ass_item; + public IntPtr was_sq_ass_slice; + public IntPtr sq_contains; + public IntPtr sq_inplace_concat; + public IntPtr sq_inplace_repeat; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyMappingMethods + { + public IntPtr mp_length; + public IntPtr mp_subscript; + public IntPtr mp_ass_subscript; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyAsyncMethods + { + public IntPtr am_await; + public IntPtr am_aiter; + public IntPtr am_anext; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyBufferProcs + { + public IntPtr bf_getbuffer; + public IntPtr bf_releasebuffer; + } + + internal static partial class SlotTypes + { + public static readonly Type[] Types = { + typeof(PyNumberMethods), + typeof(PySequenceMethods), + typeof(PyMappingMethods), + typeof(PyAsyncMethods), + typeof(PyBufferProcs), + }; + } + +} #endif diff --git a/src/runtime/interop38.cs b/src/runtime/interop38.cs index 9126bca6a..a87573e90 100644 --- a/src/runtime/interop38.cs +++ b/src/runtime/interop38.cs @@ -13,25 +13,10 @@ namespace Python.Runtime { - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } + [StructLayout(LayoutKind.Sequential)] + internal static partial class TypeOffset + { // Auto-generated from PyHeapTypeObject in Python.h public static int ob_refcnt = 0; public static int ob_type = 0; @@ -147,6 +132,96 @@ public static int magic() /* here are optional user slots, followed by the members. */ public static int members = 0; } -} + [StructLayout(LayoutKind.Sequential)] + internal struct PyNumberMethods + { + public IntPtr nb_add; + public IntPtr nb_subtract; + public IntPtr nb_multiply; + public IntPtr nb_remainder; + public IntPtr nb_divmod; + public IntPtr nb_power; + public IntPtr nb_negative; + public IntPtr nb_positive; + public IntPtr nb_absolute; + public IntPtr nb_bool; + public IntPtr nb_invert; + public IntPtr nb_lshift; + public IntPtr nb_rshift; + public IntPtr nb_and; + public IntPtr nb_xor; + public IntPtr nb_or; + public IntPtr nb_int; + public IntPtr nb_reserved; + public IntPtr nb_float; + public IntPtr nb_inplace_add; + public IntPtr nb_inplace_subtract; + public IntPtr nb_inplace_multiply; + public IntPtr nb_inplace_remainder; + public IntPtr nb_inplace_power; + public IntPtr nb_inplace_lshift; + public IntPtr nb_inplace_rshift; + public IntPtr nb_inplace_and; + public IntPtr nb_inplace_xor; + public IntPtr nb_inplace_or; + public IntPtr nb_floor_divide; + public IntPtr nb_true_divide; + public IntPtr nb_inplace_floor_divide; + public IntPtr nb_inplace_true_divide; + public IntPtr nb_index; + public IntPtr nb_matrix_multiply; + public IntPtr nb_inplace_matrix_multiply; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PySequenceMethods + { + public IntPtr sq_length; + public IntPtr sq_concat; + public IntPtr sq_repeat; + public IntPtr sq_item; + public IntPtr was_sq_slice; + public IntPtr sq_ass_item; + public IntPtr was_sq_ass_slice; + public IntPtr sq_contains; + public IntPtr sq_inplace_concat; + public IntPtr sq_inplace_repeat; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyMappingMethods + { + public IntPtr mp_length; + public IntPtr mp_subscript; + public IntPtr mp_ass_subscript; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyAsyncMethods + { + public IntPtr am_await; + public IntPtr am_aiter; + public IntPtr am_anext; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyBufferProcs + { + public IntPtr bf_getbuffer; + public IntPtr bf_releasebuffer; + } + + internal static partial class SlotTypes + { + public static readonly Type[] Types = { + typeof(PyNumberMethods), + typeof(PySequenceMethods), + typeof(PyMappingMethods), + typeof(PyAsyncMethods), + typeof(PyBufferProcs), + }; + } + +} #endif diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index afa3bf2d7..bc2805d80 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -1,5 +1,8 @@ using System; +using System.Collections.Generic; +using System.Diagnostics; using System.Runtime.InteropServices; +using System.Linq; namespace Python.Runtime { @@ -8,12 +11,69 @@ namespace Python.Runtime /// code. It defines the common fields that associate CLR and Python /// objects and common utilities to convert between those identities. /// + [Serializable] internal abstract class ManagedType { + internal enum TrackTypes + { + Untrack, + Extension, + Wrapper, + } + + [NonSerialized] internal GCHandle gcHandle; // Native handle + internal IntPtr pyHandle; // PyObject * internal IntPtr tpHandle; // PyType * + private static readonly Dictionary _managedObjs = new Dictionary(); + + internal void IncrRefCount() + { + Runtime.XIncref(pyHandle); + } + + internal void DecrRefCount() + { + Runtime.XDecref(pyHandle); + } + + internal long RefCount + { + get + { + var gs = Runtime.PyGILState_Ensure(); + try + { + return Runtime.Refcount(pyHandle); + } + finally + { + Runtime.PyGILState_Release(gs); + } + } + } + + internal GCHandle AllocGCHandle(TrackTypes track = TrackTypes.Untrack) + { + gcHandle = GCHandle.Alloc(this); + if (track != TrackTypes.Untrack) + { + _managedObjs.Add(this, track); + } + return gcHandle; + } + + internal void FreeGCHandle() + { + _managedObjs.Remove(this); + if (gcHandle.IsAllocated) + { + gcHandle.Free(); + gcHandle = default; + } + } /// /// Given a Python object, return the associated managed object or null. @@ -94,5 +154,108 @@ internal static bool IsManagedType(IntPtr ob) } return false; } + + public bool IsTypeObject() + { + return pyHandle == tpHandle; + } + + internal static IDictionary GetManagedObjects() + { + return _managedObjs; + } + + internal static void ClearTrackedObjects() + { + _managedObjs.Clear(); + } + + internal static int PyVisit(IntPtr ob, IntPtr visit, IntPtr arg) + { + if (ob == IntPtr.Zero) + { + return 0; + } + var visitFunc = (Interop.ObjObjFunc)Marshal.GetDelegateForFunctionPointer(visit, typeof(Interop.ObjObjFunc)); + return visitFunc(ob, arg); + } + + /// + /// Wrapper for calling tp_clear + /// + internal void CallTypeClear() + { + if (tpHandle == IntPtr.Zero || pyHandle == IntPtr.Zero) + { + return; + } + var clearPtr = Marshal.ReadIntPtr(tpHandle, TypeOffset.tp_clear); + if (clearPtr == IntPtr.Zero) + { + return; + } + var clearFunc = (Interop.InquiryFunc)Marshal.GetDelegateForFunctionPointer(clearPtr, typeof(Interop.InquiryFunc)); + clearFunc(pyHandle); + } + + /// + /// Wrapper for calling tp_traverse + /// + internal void CallTypeTraverse(Interop.ObjObjFunc visitproc, IntPtr arg) + { + if (tpHandle == IntPtr.Zero || pyHandle == IntPtr.Zero) + { + return; + } + var traversePtr = Marshal.ReadIntPtr(tpHandle, TypeOffset.tp_traverse); + if (traversePtr == IntPtr.Zero) + { + return; + } + var traverseFunc = (Interop.ObjObjArgFunc)Marshal.GetDelegateForFunctionPointer(traversePtr, typeof(Interop.ObjObjArgFunc)); + var visiPtr = Marshal.GetFunctionPointerForDelegate(visitproc); + traverseFunc(pyHandle, visiPtr, arg); + } + + protected void TypeClear() + { + ClearObjectDict(pyHandle); + } + + internal void Save(InterDomainContext context) + { + OnSave(context); + } + + internal void Load(InterDomainContext context) + { + OnLoad(context); + } + + protected virtual void OnSave(InterDomainContext context) { } + protected virtual void OnLoad(InterDomainContext context) { } + + protected static void ClearObjectDict(IntPtr ob) + { + IntPtr dict = GetObjectDict(ob); + if (dict == IntPtr.Zero) + { + return; + } + SetObjectDict(ob, IntPtr.Zero); + Runtime.XDecref(dict); + } + + protected static IntPtr GetObjectDict(IntPtr ob) + { + IntPtr type = Runtime.PyObject_TYPE(ob); + return Marshal.ReadIntPtr(ob, ObjectOffset.TypeDictOffset(type)); + } + + protected static void SetObjectDict(IntPtr ob, IntPtr value) + { + IntPtr type = Runtime.PyObject_TYPE(ob); + Marshal.WriteIntPtr(ob, ObjectOffset.TypeDictOffset(type), value); + } } } diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 5af2e1a7e..f7afd5d6d 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -1,4 +1,6 @@ using System; +using System.Collections; +using System.IO; using System.Runtime.InteropServices; namespace Python.Runtime @@ -11,17 +13,55 @@ namespace Python.Runtime internal class MetaType : ManagedType { private static IntPtr PyCLRMetaType; + private static SlotsHolder _metaSlotsHodler; + internal static readonly string[] CustomMethods = new string[] + { + "__instancecheck__", + "__subclasscheck__", + }; /// /// Metatype initialization. This bootstraps the CLR metatype to life. /// public static IntPtr Initialize() { - PyCLRMetaType = TypeManager.CreateMetaType(typeof(MetaType)); + PyCLRMetaType = TypeManager.CreateMetaType(typeof(MetaType), out _metaSlotsHodler); return PyCLRMetaType; } + public static void Release() + { + if (Runtime.Refcount(PyCLRMetaType) > 1) + { + _metaSlotsHodler.ResetSlots(); + } + Runtime.Py_CLEAR(ref PyCLRMetaType); + _metaSlotsHodler = null; + } + + internal static void SaveRuntimeData(RuntimeDataStorage storage) + { + Runtime.XIncref(PyCLRMetaType); + storage.PushValue(PyCLRMetaType); + } + + internal static IntPtr RestoreRuntimeData(RuntimeDataStorage storage) + { + PyCLRMetaType = storage.PopValue(); + _metaSlotsHodler = new SlotsHolder(PyCLRMetaType); + TypeManager.InitializeSlots(PyCLRMetaType, typeof(MetaType), _metaSlotsHodler); + + IntPtr mdef = Marshal.ReadIntPtr(PyCLRMetaType, TypeOffset.tp_methods); + foreach (var methodName in CustomMethods) + { + var mi = typeof(MetaType).GetMethod(methodName); + ThunkInfo thunkInfo = Interop.GetThunk(mi, "BinaryFunc"); + _metaSlotsHodler.KeeapAlive(thunkInfo); + mdef = TypeManager.WriteMethodDef(mdef, methodName, thunkInfo.Address); + } + return PyCLRMetaType; + } /// /// Metatype __new__ implementation. This is called to create a new diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index e2d8581b3..c0cc4c75c 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -13,6 +13,7 @@ namespace Python.Runtime /// a set of Python arguments. This is also used as a base class for the /// ConstructorBinder, a minor variation used to invoke constructors. /// + [Serializable] internal class MethodBinder { public ArrayList list; diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index f402f91f8..011d8217d 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -9,6 +9,7 @@ namespace Python.Runtime /// standard Python method bindings, but the same type is used to bind /// both static and instance methods. /// + [Serializable] internal class MethodBinding : ExtensionType { internal MethodInfo info; @@ -21,11 +22,15 @@ public MethodBinding(MethodObject m, IntPtr target, IntPtr targetType) Runtime.XIncref(target); this.target = target; - Runtime.XIncref(targetType); if (targetType == IntPtr.Zero) { targetType = Runtime.PyObject_Type(target); } + else + { + Runtime.XIncref(targetType); + } + this.targetType = targetType; this.info = null; @@ -36,6 +41,12 @@ public MethodBinding(MethodObject m, IntPtr target) : this(m, target, IntPtr.Zer { } + private void ClearMembers() + { + Runtime.Py_CLEAR(ref target); + Runtime.Py_CLEAR(ref targetType); + } + /// /// Implement binding of generic methods using the subscript syntax []. /// @@ -237,9 +248,22 @@ public static IntPtr tp_repr(IntPtr ob) public new static void tp_dealloc(IntPtr ob) { var self = (MethodBinding)GetManagedObject(ob); - Runtime.XDecref(self.target); - Runtime.XDecref(self.targetType); - FinalizeObject(self); + self.ClearMembers(); + self.Dealloc(); + } + + public static int tp_clear(IntPtr ob) + { + var self = (MethodBinding)GetManagedObject(ob); + self.ClearMembers(); + return 0; + } + + protected override void OnSave(InterDomainContext context) + { + base.OnSave(context); + Runtime.XIncref(target); + Runtime.XIncref(targetType); } } } diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 8df9c8029..eb3ce8a18 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -10,6 +10,7 @@ namespace Python.Runtime /// /// TODO: ForbidPythonThreadsAttribute per method info /// + [Serializable] internal class MethodObject : ExtensionType { internal MethodInfo[] info; @@ -17,6 +18,7 @@ internal class MethodObject : ExtensionType internal MethodBinding unbound; internal MethodBinder binder; internal bool is_static = false; + internal IntPtr doc; internal Type type; @@ -109,6 +111,16 @@ internal bool IsStatic() return is_static; } + private void ClearMembers() + { + Runtime.Py_CLEAR(ref doc); + if (unbound != null) + { + Runtime.XDecref(unbound.pyHandle); + unbound = null; + } + } + /// /// Descriptor __getattribute__ implementation. /// @@ -196,12 +208,27 @@ public static IntPtr tp_repr(IntPtr ob) public new static void tp_dealloc(IntPtr ob) { var self = (MethodObject)GetManagedObject(ob); - Runtime.XDecref(self.doc); - if (self.unbound != null) + self.ClearMembers(); + ClearObjectDict(ob); + self.Dealloc(); + } + + public static int tp_clear(IntPtr ob) + { + var self = (MethodObject)GetManagedObject(ob); + self.ClearMembers(); + ClearObjectDict(ob); + return 0; + } + + protected override void OnSave(InterDomainContext context) + { + base.OnSave(context); + if (unbound != null) { - Runtime.XDecref(self.unbound.pyHandle); + Runtime.XIncref(unbound.pyHandle); } - ExtensionType.FinalizeObject(self); + Runtime.XIncref(doc); } } } diff --git a/src/runtime/methodwrapper.cs b/src/runtime/methodwrapper.cs index bc7500dab..4caefccb6 100644 --- a/src/runtime/methodwrapper.cs +++ b/src/runtime/methodwrapper.cs @@ -12,9 +12,10 @@ internal class MethodWrapper { public IntPtr mdef; public IntPtr ptr; + private ThunkInfo _thunk; + private bool _disposed = false; - private ThunkInfo _thunk; public MethodWrapper(Type type, string name, string funcType = null) { @@ -30,6 +31,7 @@ public MethodWrapper(Type type, string name, string funcType = null) ptr = Runtime.PyCFunction_NewEx(mdef, IntPtr.Zero, IntPtr.Zero); } + public IntPtr Call(IntPtr args, IntPtr kw) { return Runtime.PyCFunction_Call(ptr, args, kw); diff --git a/src/runtime/modulefunctionobject.cs b/src/runtime/modulefunctionobject.cs index 8f8692af9..e7a2c515a 100644 --- a/src/runtime/modulefunctionobject.cs +++ b/src/runtime/modulefunctionobject.cs @@ -7,6 +7,7 @@ namespace Python.Runtime /// /// Module level functions /// + [Serializable] internal class ModuleFunctionObject : MethodObject { public ModuleFunctionObject(Type type, string name, MethodInfo[] info, bool allow_threads) diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index ea5bbbfa0..6313975da 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -11,9 +11,11 @@ namespace Python.Runtime /// Implements a Python type that provides access to CLR namespaces. The /// type behaves like a Python module, and can contain other sub-modules. /// + [Serializable] internal class ModuleObject : ExtensionType { private Dictionary cache; + internal string moduleName; internal IntPtr dict; protected string _namespace; @@ -54,7 +56,8 @@ public ModuleObject(string name) Runtime.XDecref(pyfilename); Runtime.XDecref(pydocstring); - Marshal.WriteIntPtr(pyHandle, ObjectOffset.TypeDictOffset(tpHandle), dict); + Runtime.XIncref(dict); + SetObjectDict(pyHandle, dict); InitializeModuleMembers(); } @@ -100,6 +103,7 @@ public ManagedType GetAttribute(string name, bool guess) { m = new ModuleObject(qname); StoreAttribute(name, m); + m.DecrRefCount(); return m; } @@ -125,6 +129,7 @@ public ManagedType GetAttribute(string name, bool guess) { m = new ModuleObject(qname); StoreAttribute(name, m); + m.DecrRefCount(); return m; } @@ -172,7 +177,11 @@ static void ImportWarning(Exception exception) /// private void StoreAttribute(string name, ManagedType ob) { - Runtime.PyDict_SetItemString(dict, name, ob.pyHandle); + if (Runtime.PyDict_SetItemString(dict, name, ob.pyHandle) != 0) + { + throw new PythonException(); + } + ob.IncrRefCount(); cache[name] = ob; } @@ -229,6 +238,7 @@ internal void InitializeModuleMembers() mi[0] = method; var m = new ModuleFunctionObject(type, name, mi, allow_threads); StoreAttribute(name, m); + m.DecrRefCount(); } } @@ -241,6 +251,7 @@ internal void InitializeModuleMembers() string name = property.Name; var p = new ModulePropertyObject(property); StoreAttribute(name, p); + p.DecrRefCount(); } } type = type.BaseType; @@ -309,6 +320,58 @@ public static IntPtr tp_repr(IntPtr ob) var self = (ModuleObject)GetManagedObject(ob); return Runtime.PyString_FromString($""); } + + public new static void tp_dealloc(IntPtr ob) + { + var self = (ModuleObject)GetManagedObject(ob); + tp_clear(ob); + self.Dealloc(); + } + + public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) + { + var self = (ModuleObject)GetManagedObject(ob); + int res = PyVisit(self.dict, visit, arg); + if (res != 0) return res; + foreach (var attr in self.cache.Values) + { + res = PyVisit(attr.pyHandle, visit, arg); + if (res != 0) return res; + } + return 0; + } + + public static int tp_clear(IntPtr ob) + { + var self = (ModuleObject)GetManagedObject(ob); + Runtime.Py_CLEAR(ref self.dict); + ClearObjectDict(ob); + foreach (var attr in self.cache.Values) + { + Runtime.XDecref(attr.pyHandle); + } + self.cache.Clear(); + return 0; + } + + protected override void OnSave(InterDomainContext context) + { + base.OnSave(context); + System.Diagnostics.Debug.Assert(dict == GetObjectDict(pyHandle)); + foreach (var attr in cache.Values) + { + Runtime.XIncref(attr.pyHandle); + } + // Decref twice in tp_clear, equilibrate them. + Runtime.XIncref(dict); + Runtime.XIncref(dict); + } + + protected override void OnLoad(InterDomainContext context) + { + base.OnLoad(context); + SetObjectDict(pyHandle, dict); + } } /// @@ -316,6 +379,7 @@ public static IntPtr tp_repr(IntPtr ob) /// to import assemblies. It has a fixed module name "clr" and doesn't /// provide a namespace. /// + [Serializable] internal class CLRModule : ModuleObject { protected static bool hacked = false; diff --git a/src/runtime/nativecall.cs b/src/runtime/nativecall.cs index 4a7bf05c8..ec0bf338c 100644 --- a/src/runtime/nativecall.cs +++ b/src/runtime/nativecall.cs @@ -23,7 +23,6 @@ namespace Python.Runtime /// internal class NativeCall { -#if NETSTANDARD [UnmanagedFunctionPointer(CallingConvention.Cdecl)] private delegate void Void_1_Delegate(IntPtr a1); @@ -32,148 +31,27 @@ internal class NativeCall public static void Void_Call_1(IntPtr fp, IntPtr a1) { - var d = Marshal.GetDelegateForFunctionPointer(fp); + var d = GetDelegate(fp); d(a1); } public static IntPtr Call_3(IntPtr fp, IntPtr a1, IntPtr a2, IntPtr a3) { - var d = Marshal.GetDelegateForFunctionPointer(fp); + var d = GetDelegate(fp); return d(a1, a2, a3); } public static int Int_Call_3(IntPtr fp, IntPtr a1, IntPtr a2, IntPtr a3) { - var d = Marshal.GetDelegateForFunctionPointer(fp); + var d = GetDelegate(fp); return d(a1, a2, a3); } -#else - private static AssemblyBuilder aBuilder; - private static ModuleBuilder mBuilder; - public static INativeCall Impl; - - static NativeCall() - { - // The static constructor is responsible for generating the - // assembly and the methods that implement the IJW thunks. - // - // To do this, we actually use reflection on the INativeCall - // interface (defined below) and generate the required thunk - // code based on the method signatures. - - var aname = new AssemblyName { Name = "e__NativeCall_Assembly" }; - var aa = AssemblyBuilderAccess.Run; - - aBuilder = Thread.GetDomain().DefineDynamicAssembly(aname, aa); - mBuilder = aBuilder.DefineDynamicModule("e__NativeCall_Module"); - - var ta = TypeAttributes.Public; - TypeBuilder tBuilder = mBuilder.DefineType("e__NativeCall", ta); - - Type iType = typeof(INativeCall); - tBuilder.AddInterfaceImplementation(iType); - - // Use reflection to loop over the INativeCall interface methods, - // calling GenerateThunk to create a managed thunk for each one. - - foreach (MethodInfo method in iType.GetMethods()) - { - GenerateThunk(tBuilder, method); - } - - Type theType = tBuilder.CreateType(); - - Impl = (INativeCall)Activator.CreateInstance(theType); - } - - private static void GenerateThunk(TypeBuilder tb, MethodInfo method) - { - ParameterInfo[] pi = method.GetParameters(); - int count = pi.Length; - int argc = count - 1; - - var args = new Type[count]; - for (var i = 0; i < count; i++) - { - args[i] = pi[i].ParameterType; - } - - MethodBuilder mb = tb.DefineMethod( - method.Name, - MethodAttributes.Public | - MethodAttributes.Virtual, - method.ReturnType, - args - ); - - // Build the method signature for the actual native function. - // This is essentially the signature of the wrapper method - // minus the first argument (the passed in function pointer). - - var nargs = new Type[argc]; - for (var i = 1; i < count; i++) - { - nargs[i - 1] = args[i]; - } - - // IL generation: the (implicit) first argument of the method - // is the 'this' pointer and the second is the function pointer. - // This code pushes the real args onto the stack, followed by - // the function pointer, then the calli opcode to make the call. - - ILGenerator il = mb.GetILGenerator(); - - for (var i = 0; i < argc; i++) - { - il.Emit(OpCodes.Ldarg_S, i + 2); - } - - il.Emit(OpCodes.Ldarg_1); - - il.EmitCalli(OpCodes.Calli, - CallingConvention.Cdecl, - method.ReturnType, - nargs - ); - - il.Emit(OpCodes.Ret); - - tb.DefineMethodOverride(mb, method); - } - - - public static void Void_Call_1(IntPtr fp, IntPtr a1) - { - Impl.Void_Call_1(fp, a1); - } - - public static IntPtr Call_3(IntPtr fp, IntPtr a1, IntPtr a2, IntPtr a3) - { - return Impl.Call_3(fp, a1, a2, a3); - } - - public static int Int_Call_3(IntPtr fp, IntPtr a1, IntPtr a2, IntPtr a3) + private static T GetDelegate(IntPtr fp) where T: Delegate { - return Impl.Int_Call_3(fp, a1, a2, a3); + // Use Marshal.GetDelegateForFunctionPointer<> directly after upgrade the framework + return (T)Marshal.GetDelegateForFunctionPointer(fp, typeof(T)); } -#endif - } - -#if !NETSTANDARD - /// - /// Defines native call signatures to be generated by NativeCall. - /// - public interface INativeCall - { - void Void_Call_0(IntPtr funcPtr); - - void Void_Call_1(IntPtr funcPtr, IntPtr arg1); - - int Int_Call_3(IntPtr funcPtr, IntPtr t, IntPtr n, IntPtr v); - - IntPtr Call_3(IntPtr funcPtr, IntPtr a1, IntPtr a2, IntPtr a3); } -#endif } diff --git a/src/runtime/overload.cs b/src/runtime/overload.cs index 67868a1b1..e9fa91d3b 100644 --- a/src/runtime/overload.cs +++ b/src/runtime/overload.cs @@ -65,7 +65,7 @@ public static IntPtr tp_repr(IntPtr op) { var self = (OverloadMapper)GetManagedObject(ob); Runtime.XDecref(self.target); - FinalizeObject(self); + self.Dealloc(); } } } diff --git a/src/runtime/platform/NativeCodePage.cs b/src/runtime/platform/NativeCodePage.cs new file mode 100644 index 000000000..ab2ee3bcf --- /dev/null +++ b/src/runtime/platform/NativeCodePage.cs @@ -0,0 +1,291 @@ +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace Python.Runtime.Platform +{ + class NativeCodePageHelper + { + /// + /// Gets the operating system as reported by python's platform.system(). + /// + public static OperatingSystemType OperatingSystem { get; private set; } + + /// + /// Gets the operating system as reported by python's platform.system(). + /// + [Obsolete] + public static string OperatingSystemName => PythonEngine.Platform; + + /// + /// Gets the machine architecture as reported by python's platform.machine(). + /// + public static MachineType Machine { get; private set; }/* set in Initialize using python's platform.machine */ + + /// + /// Gets the machine architecture as reported by python's platform.machine(). + /// + [Obsolete] + public static string MachineName { get; private set; } + + /// + /// Initialized by InitializeNativeCodePage. + /// + /// This points to a page of memory allocated using mmap or VirtualAlloc + /// (depending on the system), and marked read and execute (not write). + /// Very much on purpose, the page is *not* released on a shutdown and + /// is instead leaked. See the TestDomainReload test case. + /// + /// The contents of the page are two native functions: one that returns 0, + /// one that returns 1. + /// + /// If python didn't keep its gc list through a Py_Finalize we could remove + /// this entire section. + /// + internal static IntPtr NativeCodePage = IntPtr.Zero; + + + static readonly Dictionary OperatingSystemTypeMapping = new Dictionary() + { + { "Windows", OperatingSystemType.Windows }, + { "Darwin", OperatingSystemType.Darwin }, + { "Linux", OperatingSystemType.Linux }, + }; + + /// + /// Map lower-case version of the python machine name to the processor + /// type. There are aliases, e.g. x86_64 and amd64 are two names for + /// the same thing. Make sure to lower-case the search string, because + /// capitalization can differ. + /// + static readonly Dictionary MachineTypeMapping = new Dictionary() + { + ["i386"] = MachineType.i386, + ["i686"] = MachineType.i386, + ["x86"] = MachineType.i386, + ["x86_64"] = MachineType.x86_64, + ["amd64"] = MachineType.x86_64, + ["x64"] = MachineType.x86_64, + ["em64t"] = MachineType.x86_64, + ["armv7l"] = MachineType.armv7l, + ["armv8"] = MachineType.armv8, + ["aarch64"] = MachineType.aarch64, + }; + + /// + /// Structure to describe native code. + /// + /// Use NativeCode.Active to get the native code for the current platform. + /// + /// Generate the code by creating the following C code: + /// + /// int Return0() { return 0; } + /// int Return1() { return 1; } + /// + /// Then compiling on the target platform, e.g. with gcc or clang: + /// cc -c -fomit-frame-pointer -O2 foo.c + /// And then analyzing the resulting functions with a hex editor, e.g.: + /// objdump -disassemble foo.o + /// + internal class NativeCode + { + /// + /// The code, as a string of bytes. + /// + public byte[] Code { get; private set; } + + /// + /// Where does the "return 0" function start? + /// + public int Return0 { get; private set; } + + /// + /// Where does the "return 1" function start? + /// + public int Return1 { get; private set; } + + public static NativeCode Active + { + get + { + switch (Machine) + { + case MachineType.i386: + return I386; + case MachineType.x86_64: + return X86_64; + default: + return null; + } + } + } + + /// + /// Code for x86_64. See the class comment for how it was generated. + /// + public static readonly NativeCode X86_64 = new NativeCode() + { + Return0 = 0x10, + Return1 = 0, + Code = new byte[] + { + // First Return1: + 0xb8, 0x01, 0x00, 0x00, 0x00, // movl $1, %eax + 0xc3, // ret + + // Now some padding so that Return0 can be 16-byte-aligned. + // I put Return1 first so there's not as much padding to type in. + 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, // nop + + // Now Return0. + 0x31, 0xc0, // xorl %eax, %eax + 0xc3, // ret + } + }; + + /// + /// Code for X86. + /// + /// It's bitwise identical to X86_64, so we just point to it. + /// + /// + public static readonly NativeCode I386 = X86_64; + } + + /// + /// Platform-dependent mmap and mprotect. + /// + internal interface IMemoryMapper + { + /// + /// Map at least numBytes of memory. Mark the page read-write (but not exec). + /// + IntPtr MapWriteable(int numBytes); + + /// + /// Sets the mapped memory to be read-exec (but not write). + /// + void SetReadExec(IntPtr mappedMemory, int numBytes); + } + + class WindowsMemoryMapper : IMemoryMapper + { + const UInt32 MEM_COMMIT = 0x1000; + const UInt32 MEM_RESERVE = 0x2000; + const UInt32 PAGE_READWRITE = 0x04; + const UInt32 PAGE_EXECUTE_READ = 0x20; + + [DllImport("kernel32.dll")] + static extern IntPtr VirtualAlloc(IntPtr lpAddress, IntPtr dwSize, UInt32 flAllocationType, UInt32 flProtect); + + [DllImport("kernel32.dll")] + static extern bool VirtualProtect(IntPtr lpAddress, IntPtr dwSize, UInt32 flNewProtect, out UInt32 lpflOldProtect); + + public IntPtr MapWriteable(int numBytes) + { + return VirtualAlloc(IntPtr.Zero, new IntPtr(numBytes), + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + } + + public void SetReadExec(IntPtr mappedMemory, int numBytes) + { + UInt32 _; + VirtualProtect(mappedMemory, new IntPtr(numBytes), PAGE_EXECUTE_READ, out _); + } + } + + class UnixMemoryMapper : IMemoryMapper + { + const int PROT_READ = 0x1; + const int PROT_WRITE = 0x2; + const int PROT_EXEC = 0x4; + + const int MAP_PRIVATE = 0x2; + int MAP_ANONYMOUS + { + get + { + switch (OperatingSystem) + { + case OperatingSystemType.Darwin: + return 0x1000; + case OperatingSystemType.Linux: + return 0x20; + default: + throw new NotImplementedException($"mmap is not supported on {OperatingSystemName}"); + } + } + } + + [DllImport("libc")] + static extern IntPtr mmap(IntPtr addr, IntPtr len, int prot, int flags, int fd, IntPtr offset); + + [DllImport("libc")] + static extern int mprotect(IntPtr addr, IntPtr len, int prot); + + public IntPtr MapWriteable(int numBytes) + { + // MAP_PRIVATE must be set on linux, even though MAP_ANON implies it. + // It doesn't hurt on darwin, so just do it. + return mmap(IntPtr.Zero, new IntPtr(numBytes), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, IntPtr.Zero); + } + + public void SetReadExec(IntPtr mappedMemory, int numBytes) + { + mprotect(mappedMemory, new IntPtr(numBytes), PROT_READ | PROT_EXEC); + } + } + + /// + /// Initializes the data about platforms. + /// + /// This must be the last step when initializing the runtime: + /// GetManagedString needs to have the cached values for types. + /// But it must run before initializing anything outside the runtime + /// because those rely on the platform data. + /// + public static void InitializePlatformData() + { + MachineName = SystemInfo.GetArchitecture(); + Machine = SystemInfo.GetMachineType(); + OperatingSystem = SystemInfo.GetSystemType(); + } + + internal static IMemoryMapper CreateMemoryMapper() + { + switch (OperatingSystem) + { + case OperatingSystemType.Darwin: + case OperatingSystemType.Linux: + return new UnixMemoryMapper(); + case OperatingSystemType.Windows: + return new WindowsMemoryMapper(); + default: + throw new NotImplementedException($"No support for {OperatingSystemName}"); + } + } + + /// + /// Initializes the native code page. + /// + /// Safe to call if we already initialized (this function is idempotent). + /// + /// + internal static void InitializeNativeCodePage() + { + // Do nothing if we already initialized. + if (NativeCodePage != IntPtr.Zero) + { + return; + } + + // Allocate the page, write the native code into it, then set it + // to be executable. + IMemoryMapper mapper = CreateMemoryMapper(); + int codeLength = NativeCode.Active.Code.Length; + NativeCodePage = mapper.MapWriteable(codeLength); + Marshal.Copy(NativeCode.Active.Code, 0, NativeCodePage, codeLength); + mapper.SetReadExec(NativeCodePage, codeLength); + } + } +} diff --git a/src/runtime/platform/Types.cs b/src/runtime/platform/Types.cs index 62be0e421..15235da5a 100644 --- a/src/runtime/platform/Types.cs +++ b/src/runtime/platform/Types.cs @@ -1,3 +1,6 @@ +using System; +using System.Runtime.InteropServices; + namespace Python.Runtime.Platform { public enum MachineType @@ -20,4 +23,170 @@ public enum OperatingSystemType Linux, Other } + + + static class SystemInfo + { + public static MachineType GetMachineType() + { + return Runtime.IsWindows ? GetMachineType_Windows() : GetMachineType_Unix(); + } + + public static string GetArchitecture() + { + return Runtime.IsWindows ? GetArchName_Windows() : GetArchName_Unix(); + } + + public static OperatingSystemType GetSystemType() + { + if (Runtime.IsWindows) + { + return OperatingSystemType.Windows; + } + switch (PythonEngine.Platform) + { + case "linux": + return OperatingSystemType.Linux; + + case "darwin": + return OperatingSystemType.Darwin; + + default: + return OperatingSystemType.Other; + } + } + + #region WINDOWS + + static string GetArchName_Windows() + { + // https://docs.microsoft.com/en-us/windows/win32/winprog64/wow64-implementation-details + return Environment.GetEnvironmentVariable("PROCESSOR_ARCHITECTURE"); + } + + static MachineType GetMachineType_Windows() + { + if (Runtime.Is32Bit) + { + return MachineType.i386; + } + switch (GetArchName_Windows()) + { + case "AMD64": + return MachineType.x86_64; + case "ARM64": + return MachineType.aarch64; + default: + return MachineType.Other; + } + } + + #endregion + + #region UNIX + + + [StructLayout(LayoutKind.Sequential)] + unsafe struct utsname_linux + { + const int NameLength = 65; + + /* Name of the implementation of the operating system. */ + public fixed byte sysname[NameLength]; + + /* Name of this node on the network. */ + public fixed byte nodename[NameLength]; + + /* Current release level of this implementation. */ + public fixed byte release[NameLength]; + + /* Current version level of this release. */ + public fixed byte version[NameLength]; + + /* Name of the hardware type the system is running on. */ + public fixed byte machine[NameLength]; + + // GNU extension + fixed byte domainname[NameLength]; /* NIS or YP domain name */ + } + + [StructLayout(LayoutKind.Sequential)] + unsafe struct utsname_darwin + { + const int NameLength = 256; + + /* Name of the implementation of the operating system. */ + public fixed byte sysname[NameLength]; + + /* Name of this node on the network. */ + public fixed byte nodename[NameLength]; + + /* Current release level of this implementation. */ + public fixed byte release[NameLength]; + + /* Current version level of this release. */ + public fixed byte version[NameLength]; + + /* Name of the hardware type the system is running on. */ + public fixed byte machine[NameLength]; + } + + [DllImport("libc")] + static extern int uname(IntPtr buf); + + + static unsafe string GetArchName_Unix() + { + switch (GetSystemType()) + { + case OperatingSystemType.Linux: + { + var buf = stackalloc utsname_linux[1]; + if (uname((IntPtr)buf) != 0) + { + return null; + } + return Marshal.PtrToStringAnsi((IntPtr)buf->machine); + } + + case OperatingSystemType.Darwin: + { + var buf = stackalloc utsname_darwin[1]; + if (uname((IntPtr)buf) != 0) + { + return null; + } + return Marshal.PtrToStringAnsi((IntPtr)buf->machine); + } + + default: + return null; + } + } + + static unsafe MachineType GetMachineType_Unix() + { + switch (GetArchName_Unix()) + { + case "x86_64": + case "em64t": + return Runtime.Is32Bit ? MachineType.i386 : MachineType.x86_64; + case "i386": + case "i686": + return MachineType.i386; + + case "armv7l": + return MachineType.armv7l; + case "armv8": + return Runtime.Is32Bit ? MachineType.armv7l : MachineType.armv8; + case "aarch64": + return Runtime.Is32Bit ? MachineType.armv7l : MachineType.aarch64; + + default: + return MachineType.Other; + } + } + + #endregion + } } diff --git a/src/runtime/propertyobject.cs b/src/runtime/propertyobject.cs index f2c97f163..ac1d077f9 100644 --- a/src/runtime/propertyobject.cs +++ b/src/runtime/propertyobject.cs @@ -7,6 +7,7 @@ namespace Python.Runtime /// /// Implements a Python descriptor type that manages CLR properties. /// + [Serializable] internal class PropertyObject : ExtensionType { private PropertyInfo info; diff --git a/src/runtime/pylist.cs b/src/runtime/pylist.cs index dcb0ea78f..7f5566401 100644 --- a/src/runtime/pylist.cs +++ b/src/runtime/pylist.cs @@ -93,7 +93,6 @@ public PyList(PyObject[] items) : base(FromArray(items)) { } - /// /// IsListType Method /// diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index a7f5e8305..9328312ce 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -20,7 +20,8 @@ public interface IPyDisposable : IDisposable /// PY3: https://docs.python.org/3/c-api/object.html /// for details. /// - public class PyObject : DynamicObject, IEnumerable, IPyDisposable + [Serializable] + public partial class PyObject : DynamicObject, IEnumerable, IPyDisposable { #if TRACE_ALLOC /// diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs index 9b9c78bae..fee78b40a 100644 --- a/src/runtime/pyscope.cs +++ b/src/runtime/pyscope.cs @@ -278,10 +278,9 @@ public PyObject Eval(string code, PyDict locals = null) { Check(); IntPtr _locals = locals == null ? variables : locals.obj; - var flag = (IntPtr)Runtime.Py_eval_input; NewReference reference = Runtime.PyRun_String( - code, flag, variables, _locals + code, RunFlagType.Eval, variables, _locals ); PythonException.ThrowIfIsNull(reference); return reference.MoveToPyObject(); @@ -317,9 +316,8 @@ public void Exec(string code, PyDict locals = null) private void Exec(string code, IntPtr _globals, IntPtr _locals) { - var flag = (IntPtr)Runtime.Py_file_input; NewReference reference = Runtime.PyRun_String( - code, flag, _globals, _locals + code, RunFlagType.File, _globals, _locals ); PythonException.ThrowIfIsNull(reference); reference.Dispose(); diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index 38ef2ee48..1d688ef9a 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -12,6 +12,14 @@ namespace Python.Runtime /// public class PythonEngine : IDisposable { + public static ShutdownMode ShutdownMode + { + get => Runtime.ShutdownMode; + set => Runtime.ShutdownMode = value; + } + + public static ShutdownMode DefaultShutdownMode => Runtime.GetDefaultShutdownMode(); + private static DelegateManager delegateManager; private static bool initialized; private static IntPtr _pythonHome = IntPtr.Zero; @@ -147,9 +155,9 @@ public static void Initialize() Initialize(setSysArgv: true); } - public static void Initialize(bool setSysArgv = true, bool initSigs = false) + public static void Initialize(bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) { - Initialize(Enumerable.Empty(), setSysArgv: setSysArgv, initSigs: initSigs); + Initialize(Enumerable.Empty(), setSysArgv: setSysArgv, initSigs: initSigs, mode); } /// @@ -162,82 +170,85 @@ public static void Initialize(bool setSysArgv = true, bool initSigs = false) /// interpreter lock (GIL) to call this method. /// initSigs can be set to 1 to do default python signal configuration. This will override the way signals are handled by the application. /// - public static void Initialize(IEnumerable args, bool setSysArgv = true, bool initSigs = false) + public static void Initialize(IEnumerable args, bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) { - if (!initialized) + if (initialized) { - // Creating the delegateManager MUST happen before Runtime.Initialize - // is called. If it happens afterwards, DelegateManager's CodeGenerator - // throws an exception in its ctor. This exception is eaten somehow - // during an initial "import clr", and the world ends shortly thereafter. - // This is probably masking some bad mojo happening somewhere in Runtime.Initialize(). - delegateManager = new DelegateManager(); - Runtime.Initialize(initSigs); - initialized = true; - Exceptions.Clear(); - - // Make sure we clean up properly on app domain unload. - AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; - - // Remember to shut down the runtime. - AddShutdownHandler(Runtime.Shutdown); - - // The global scope gets used implicitly quite early on, remember - // to clear it out when we shut down. - AddShutdownHandler(PyScopeManager.Global.Clear); - - if (setSysArgv) - { - Py.SetArgv(args); - } + return; + } + // Creating the delegateManager MUST happen before Runtime.Initialize + // is called. If it happens afterwards, DelegateManager's CodeGenerator + // throws an exception in its ctor. This exception is eaten somehow + // during an initial "import clr", and the world ends shortly thereafter. + // This is probably masking some bad mojo happening somewhere in Runtime.Initialize(). + delegateManager = new DelegateManager(); + Runtime.Initialize(initSigs, mode); + initialized = true; + Exceptions.Clear(); + + // Make sure we clean up properly on app domain unload. + AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; + + // The global scope gets used implicitly quite early on, remember + // to clear it out when we shut down. + AddShutdownHandler(PyScopeManager.Global.Clear); + + if (setSysArgv) + { + Py.SetArgv(args); + } + if (mode == ShutdownMode.Normal) + { + // TOOD: Check if this can be remove completely or not. // register the atexit callback (this doesn't use Py_AtExit as the C atexit // callbacks are called after python is fully finalized but the python ones // are called while the python engine is still running). - string code = - "import atexit, clr\n" + - "atexit.register(clr._AtExit)\n"; - PythonEngine.Exec(code); + //string code = + // "import atexit, clr\n" + + // "atexit.register(clr._AtExit)\n"; + //PythonEngine.Exec(code); + } - // Load the clr.py resource into the clr module - IntPtr clr = Python.Runtime.ImportHook.GetCLRModule(); - IntPtr clr_dict = Runtime.PyModule_GetDict(clr); + // Load the clr.py resource into the clr module + IntPtr clr = Python.Runtime.ImportHook.GetCLRModule(); + IntPtr clr_dict = Runtime.PyModule_GetDict(clr); - var locals = new PyDict(); - try + var locals = new PyDict(); + try + { + IntPtr module = Runtime.PyImport_AddModule("clr._extras"); + IntPtr module_globals = Runtime.PyModule_GetDict(module); + IntPtr builtins = Runtime.PyEval_GetBuiltins(); + Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins); + + Assembly assembly = Assembly.GetExecutingAssembly(); + using (Stream stream = assembly.GetManifestResourceStream("clr.py")) + using (var reader = new StreamReader(stream)) { - IntPtr module = Runtime.PyImport_AddModule("clr._extras"); - IntPtr module_globals = Runtime.PyModule_GetDict(module); - IntPtr builtins = Runtime.PyEval_GetBuiltins(); - Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins); - - Assembly assembly = Assembly.GetExecutingAssembly(); - using (Stream stream = assembly.GetManifestResourceStream("clr.py")) - using (var reader = new StreamReader(stream)) - { - // add the contents of clr.py to the module - string clr_py = reader.ReadToEnd(); - Exec(clr_py, module_globals, locals.Handle); - } + // add the contents of clr.py to the module + string clr_py = reader.ReadToEnd(); + Exec(clr_py, module_globals, locals.Handle); + } - // add the imported module to the clr module, and copy the API functions - // and decorators into the main clr module. - Runtime.PyDict_SetItemString(clr_dict, "_extras", module); - foreach (PyObject key in locals.Keys()) + // add the imported module to the clr module, and copy the API functions + // and decorators into the main clr module. + Runtime.PyDict_SetItemString(clr_dict, "_extras", module); + using (var keys = locals.Keys()) + foreach (PyObject key in keys) + { + if (!key.ToString().StartsWith("_") || key.ToString().Equals("__version__")) { - if (!key.ToString().StartsWith("_") || key.ToString().Equals("__version__")) - { - PyObject value = locals[key]; - Runtime.PyDict_SetItem(clr_dict, key.Handle, value.Handle); - value.Dispose(); - } - key.Dispose(); + PyObject value = locals[key]; + Runtime.PyDict_SetItem(clr_dict, key.Handle, value.Handle); + value.Dispose(); } + key.Dispose(); } - finally - { - locals.Dispose(); - } + } + finally + { + locals.Dispose(); } } @@ -305,22 +316,37 @@ public static IntPtr InitExt() /// Python runtime can no longer be used in the current process /// after calling the Shutdown method. /// - public static void Shutdown() + /// The ShutdownMode to use when shutting down the Runtime + public static void Shutdown(ShutdownMode mode) { - if (initialized) + if (!initialized) { - PyScopeManager.Global.Clear(); - - // If the shutdown handlers trigger a domain unload, - // don't call shutdown again. - AppDomain.CurrentDomain.DomainUnload -= OnDomainUnload; + return; + } + // If the shutdown handlers trigger a domain unload, + // don't call shutdown again. + AppDomain.CurrentDomain.DomainUnload -= OnDomainUnload; - ExecuteShutdownHandlers(); + PyScopeManager.Global.Clear(); + ExecuteShutdownHandlers(); + // Remember to shut down the runtime. + Runtime.Shutdown(mode); + PyObjectConversions.Reset(); - PyObjectConversions.Reset(); + initialized = false; + } - initialized = false; - } + /// + /// Shutdown Method + /// + /// + /// Shutdown and release resources held by the Python runtime. The + /// Python runtime can no longer be used in the current process + /// after calling the Shutdown method. + /// + public static void Shutdown() + { + Shutdown(Runtime.ShutdownMode); } /// @@ -585,7 +611,7 @@ internal static PyObject RunString(string code, IntPtr? globals, IntPtr? locals, try { NewReference result = Runtime.PyRun_String( - code, (IntPtr)flag, globals.Value, locals.Value + code, flag, globals.Value, locals.Value ); PythonException.ThrowIfIsNull(result); return result.MoveToPyObject(); diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index b69ac34e4..893bd9491 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -187,6 +187,11 @@ public string Format() return res; } + public bool IsMatches(IntPtr exc) + { + return Runtime.PyErr_GivenExceptionMatches(PyType, exc) != 0; + } + /// /// Dispose Method /// diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 2d523758b..83d404f9d 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -7,6 +7,7 @@ using System.Threading; using System.Collections.Generic; using Python.Runtime.Platform; +using System.Linq; namespace Python.Runtime { @@ -86,7 +87,9 @@ public class Runtime internal static object IsFinalizingLock = new object(); internal static bool IsFinalizing; - internal static bool Is32Bit => IntPtr.Size == 4; + private static bool _isInitialized = false; + + internal static readonly bool Is32Bit = IntPtr.Size == 4; // .NET core: System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(OSPlatform.Windows) internal static bool IsWindows = Environment.OSVersion.Platform == PlatformID.Win32NT; @@ -94,49 +97,6 @@ public class Runtime internal static Version InteropVersion { get; } = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version; - static readonly Dictionary OperatingSystemTypeMapping = new Dictionary() - { - { "Windows", OperatingSystemType.Windows }, - { "Darwin", OperatingSystemType.Darwin }, - { "Linux", OperatingSystemType.Linux }, - }; - - [Obsolete] - public static string OperatingSystemName => OperatingSystem.ToString(); - - [Obsolete] - public static string MachineName => Machine.ToString(); - - /// - /// Gets the operating system as reported by python's platform.system(). - /// - public static OperatingSystemType OperatingSystem { get; private set; } - - /// - /// Map lower-case version of the python machine name to the processor - /// type. There are aliases, e.g. x86_64 and amd64 are two names for - /// the same thing. Make sure to lower-case the search string, because - /// capitalization can differ. - /// - static readonly Dictionary MachineTypeMapping = new Dictionary() - { - ["i386"] = MachineType.i386, - ["i686"] = MachineType.i386, - ["x86"] = MachineType.i386, - ["x86_64"] = MachineType.x86_64, - ["amd64"] = MachineType.x86_64, - ["x64"] = MachineType.x86_64, - ["em64t"] = MachineType.x86_64, - ["armv7l"] = MachineType.armv7l, - ["armv8"] = MachineType.armv8, - ["aarch64"] = MachineType.aarch64, - }; - - /// - /// Gets the machine architecture as reported by python's platform.machine(). - /// - public static MachineType Machine { get; private set; }/* set in Initialize using python's platform.machine */ - public static int MainManagedThreadId { get; private set; } /// @@ -144,6 +104,7 @@ public class Runtime /// internal static readonly Encoding PyEncoding = _UCS == 2 ? Encoding.Unicode : Encoding.UTF32; + public static ShutdownMode ShutdownMode { get; internal set; } private static PyReferenceCollection _pyRefs = new PyReferenceCollection(); internal static Version PyVersion @@ -162,18 +123,44 @@ internal static Version PyVersion /// /// Initialize the runtime... /// - internal static void Initialize(bool initSigs = false) + /// Always call this method from the Main thread. After the + /// first call to this method, the main thread has acquired the GIL. + internal static void Initialize(bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) { + if (_isInitialized) + { + return; + } + _isInitialized = true; + + if (mode == ShutdownMode.Default) + { + mode = GetDefaultShutdownMode(); + } + ShutdownMode = mode; + if (Py_IsInitialized() == 0) { Py_InitializeEx(initSigs ? 1 : 0); - MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; + if (PyEval_ThreadsInitialized() == 0) + { + PyEval_InitThreads(); + } + // XXX: Reload mode may reduct to Soft mode, + // so even on Reload mode it still needs to save the RuntimeState + if (mode == ShutdownMode.Soft || mode == ShutdownMode.Reload) + { + RuntimeState.Save(); + } } - - if (PyEval_ThreadsInitialized() == 0) + else { - PyEval_InitThreads(); + // If we're coming back from a domain reload or a soft shutdown, + // we have previously released the thread state. Restore the main + // thread state here. + PyGILState_Ensure(); } + MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; IsFinalizing = false; @@ -181,8 +168,43 @@ internal static void Initialize(bool initSigs = false) PyScopeManager.Reset(); ClassManager.Reset(); ClassDerivedObject.Reset(); - TypeManager.Reset(); + TypeManager.Initialize(); + InitPyMembers(); + + // Initialize data about the platform we're running on. We need + // this for the type manager and potentially other details. Must + // happen after caching the python types, above. + NativeCodePageHelper.InitializePlatformData(); + + // Initialize modules that depend on the runtime class. + AssemblyManager.Initialize(); + if (mode == ShutdownMode.Reload && RuntimeData.HasStashData()) + { + RuntimeData.RestoreRuntimeData(); + } + else + { + PyCLRMetaType = MetaType.Initialize(); // Steal a reference + ImportHook.Initialize(); + } + Exceptions.Initialize(); + + // Need to add the runtime directory to sys.path so that we + // can find built-in assemblies like System.Data, et. al. + string rtdir = RuntimeEnvironment.GetRuntimeDirectory(); + IntPtr path = PySys_GetObject("path").DangerousGetAddress(); + IntPtr item = PyString_FromString(rtdir); + if (PySequence_Contains(path, item) == 0) + { + PyList_Append(new BorrowedReference(path), item); + } + XDecref(item); + AssemblyManager.UpdatePath(); + } + + private static void InitPyMembers() + { IntPtr op; { var builtins = GetBuiltins(); @@ -276,129 +298,141 @@ internal static void Initialize(bool initSigs = false) Error = new IntPtr(-1); - // Initialize data about the platform we're running on. We need - // this for the type manager and potentially other details. Must - // happen after caching the python types, above. - InitializePlatformData(); + _PyObject_NextNotImplemented = Get_PyObject_NextNotImplemented(); + { + IntPtr sys = PyImport_ImportModule("sys"); + PyModuleType = PyObject_Type(sys); + XDecref(sys); + } + } - IntPtr dllLocal = IntPtr.Zero; - var loader = LibraryLoader.Get(OperatingSystem); + private static IntPtr Get_PyObject_NextNotImplemented() + { + IntPtr pyType = SlotHelper.CreateObjectType(); + IntPtr iternext = Marshal.ReadIntPtr(pyType, TypeOffset.tp_iternext); + Runtime.XDecref(pyType); + return iternext; + } - if (_PythonDll != "__Internal") + /// + /// Tries to downgrade the shutdown mode, if possible. + /// The only possibles downgrades are: + /// Soft -> Normal + /// Reload -> Soft + /// Reload -> Normal + /// + /// The desired shutdown mode + /// The `mode` parameter if the downgrade is supported, the ShutdownMode + /// set at initialization otherwise. + static ShutdownMode TryDowngradeShutdown(ShutdownMode mode) + { + if ( + mode == Runtime.ShutdownMode + || mode == ShutdownMode.Normal + || (mode == ShutdownMode.Soft && Runtime.ShutdownMode == ShutdownMode.Reload) + ) { - dllLocal = loader.Load(_PythonDll); + return mode; } - _PyObject_NextNotImplemented = loader.GetFunction(dllLocal, "_PyObject_NextNotImplemented"); - PyModuleType = loader.GetFunction(dllLocal, "PyModule_Type"); - - if (dllLocal != IntPtr.Zero) + else // we can't downgrade { - loader.Free(dllLocal); + return Runtime.ShutdownMode; } - - // Initialize modules that depend on the runtime class. - AssemblyManager.Initialize(); - PyCLRMetaType = MetaType.Initialize(); - Exceptions.Initialize(); - ImportHook.Initialize(); - - // Need to add the runtime directory to sys.path so that we - // can find built-in assemblies like System.Data, et. al. - string rtdir = RuntimeEnvironment.GetRuntimeDirectory(); - BorrowedReference path = PySys_GetObject("path"); - IntPtr item = PyString_FromString(rtdir); - PyList_Append(path, item); - XDecref(item); - AssemblyManager.UpdatePath(); } - /// - /// Initializes the data about platforms. - /// - /// This must be the last step when initializing the runtime: - /// GetManagedString needs to have the cached values for types. - /// But it must run before initializing anything outside the runtime - /// because those rely on the platform data. - /// - private static void InitializePlatformData() + internal static void Shutdown(ShutdownMode mode) { -#if !NETSTANDARD - IntPtr op; - IntPtr fn; - IntPtr platformModule = PyImport_ImportModule("platform"); - IntPtr emptyTuple = PyTuple_New(0); - - fn = PyObject_GetAttrString(platformModule, "system"); - op = PyObject_Call(fn, emptyTuple, IntPtr.Zero); - string operatingSystemName = GetManagedString(op); - XDecref(op); - XDecref(fn); + if (Py_IsInitialized() == 0 || !_isInitialized) + { + return; + } + _isInitialized = false; - fn = PyObject_GetAttrString(platformModule, "machine"); - op = PyObject_Call(fn, emptyTuple, IntPtr.Zero); - string machineName = GetManagedString(op); - XDecref(op); - XDecref(fn); + // If the shutdown mode specified is not the the same as the one specified + // during Initialization, we need to validate it; we can only downgrade, + // not upgrade the shutdown mode. + mode = TryDowngradeShutdown(mode); - XDecref(emptyTuple); - XDecref(platformModule); + var state = PyGILState_Ensure(); - // Now convert the strings into enum values so we can do switch - // statements rather than constant parsing. - OperatingSystemType OSType; - if (!OperatingSystemTypeMapping.TryGetValue(operatingSystemName, out OSType)) + if (mode == ShutdownMode.Soft) + { + RunExitFuncs(); + } + if (mode == ShutdownMode.Reload) { - OSType = OperatingSystemType.Other; + RuntimeData.Stash(); } - OperatingSystem = OSType; + AssemblyManager.Shutdown(); + ImportHook.Shutdown(); + + ClearClrModules(); + RemoveClrRootModule(); + + MoveClrInstancesOnwershipToPython(); + ClassManager.DisposePythonWrappersForClrTypes(); + TypeManager.RemoveTypes(); - MachineType MType; - if (!MachineTypeMapping.TryGetValue(machineName.ToLower(), out MType)) + MetaType.Release(); + PyCLRMetaType = IntPtr.Zero; + + Exceptions.Shutdown(); + Finalizer.Shutdown(); + + if (mode != ShutdownMode.Normal) { - MType = MachineType.Other; + PyGC_Collect(); + if (mode == ShutdownMode.Soft) + { + RuntimeState.Restore(); + } + ResetPyMembers(); + GC.Collect(); + try + { + GC.WaitForFullGCComplete(); + } + catch (NotImplementedException) + { + // Some clr runtime didn't implement GC.WaitForFullGCComplete yet. + } + GC.WaitForPendingFinalizers(); + PyGILState_Release(state); + // Then release the GIL for good, if there is somehting to release + // Use the unchecked version as the checked version calls `abort()` + // if the current state is NULL. + if (_PyThreadState_UncheckedGet() != IntPtr.Zero) + { + PyEval_SaveThread(); + } + } - Machine = MType; -#else - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - OperatingSystem = OperatingSystemType.Linux; - else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) - OperatingSystem = OperatingSystemType.Darwin; - else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - OperatingSystem = OperatingSystemType.Windows; else - OperatingSystem = OperatingSystemType.Other; - - switch (RuntimeInformation.ProcessArchitecture) { - case Architecture.X86: - Machine = MachineType.i386; - break; - case Architecture.X64: - Machine = MachineType.x86_64; - break; - case Architecture.Arm: - Machine = MachineType.armv7l; - break; - case Architecture.Arm64: - Machine = MachineType.aarch64; - break; - default: - Machine = MachineType.Other; - break; + ResetPyMembers(); + Py_Finalize(); } -#endif } internal static void Shutdown() { - AssemblyManager.Shutdown(); - Exceptions.Shutdown(); - ImportHook.Shutdown(); - Finalizer.Shutdown(); - // TOOD: PyCLRMetaType's release operation still in #958 - PyCLRMetaType = IntPtr.Zero; - ResetPyMembers(); - Py_Finalize(); + var mode = ShutdownMode; + Shutdown(mode); + } + + internal static ShutdownMode GetDefaultShutdownMode() + { + string modeEvn = Environment.GetEnvironmentVariable("PYTHONNET_SHUTDOWN_MODE"); + if (modeEvn == null) + { + return ShutdownMode.Normal; + } + ShutdownMode mode; + if (Enum.TryParse(modeEvn, true, out mode)) + { + return mode; + } + return ShutdownMode.Normal; } // called *without* the GIL acquired by clr._AtExit @@ -411,6 +445,37 @@ internal static int AtExit() return 0; } + private static void RunExitFuncs() + { + PyObject atexit; + try + { + atexit = Py.Import("atexit"); + } + catch (PythonException e) + { + if (!e.IsMatches(Exceptions.ImportError)) + { + throw; + } + e.Dispose(); + // The runtime may not provided `atexit` module. + return; + } + using (atexit) + { + try + { + atexit.InvokeMethod("_run_exitfuncs").Dispose(); + } + catch (PythonException e) + { + Console.Error.WriteLine(e); + e.Dispose(); + } + } + } + private static void SetPyMember(ref IntPtr obj, IntPtr value, Action onRelease) { // XXX: For current usages, value should not be null. @@ -424,9 +489,75 @@ private static void ResetPyMembers() _pyRefs.Release(); } - internal static IntPtr Py_single_input = (IntPtr)256; - internal static IntPtr Py_file_input = (IntPtr)257; - internal static IntPtr Py_eval_input = (IntPtr)258; + private static void ClearClrModules() + { + var modules = PyImport_GetModuleDict(); + var items = PyDict_Items(modules); + long length = PyList_Size(items); + for (long i = 0; i < length; i++) + { + var item = PyList_GetItem(items, i); + var name = PyTuple_GetItem(item.DangerousGetAddress(), 0); + var module = PyTuple_GetItem(item.DangerousGetAddress(), 1); + if (ManagedType.IsManagedType(module)) + { + PyDict_DelItem(modules, name); + } + } + items.Dispose(); + } + + private static void RemoveClrRootModule() + { + var modules = PyImport_GetModuleDict(); + PyDictTryDelItem(modules, "CLR"); + PyDictTryDelItem(modules, "clr"); + PyDictTryDelItem(modules, "clr._extra"); + } + + private static void PyDictTryDelItem(IntPtr dict, string key) + { + if (PyDict_DelItemString(dict, key) == 0) + { + return; + } + if (!PythonException.Matches(Exceptions.KeyError)) + { + throw new PythonException(); + } + PyErr_Clear(); + } + + private static void MoveClrInstancesOnwershipToPython() + { + var objs = ManagedType.GetManagedObjects(); + var copyObjs = objs.ToArray(); + foreach (var entry in copyObjs) + { + ManagedType obj = entry.Key; + if (!objs.ContainsKey(obj)) + { + System.Diagnostics.Debug.Assert(obj.gcHandle == default); + continue; + } + if (entry.Value == ManagedType.TrackTypes.Extension) + { + obj.CallTypeClear(); + // obj's tp_type will degenerate to a pure Python type after TypeManager.RemoveTypes(), + // thus just be safe to give it back to GC chain. + if (!_PyObject_GC_IS_TRACKED(obj.pyHandle)) + { + PyObject_GC_Track(obj.pyHandle); + } + } + if (obj.gcHandle.IsAllocated) + { + obj.gcHandle.Free(); + } + obj.gcHandle = default; + } + ManagedType.ClearTrackedObjects(); + } internal static IntPtr PyBaseObjectType; internal static IntPtr PyModuleType; @@ -647,7 +778,7 @@ internal static unsafe void XDecref(IntPtr op) { return; } - NativeCall.Impl.Void_Call_1(new IntPtr(f), op); + NativeCall.Void_Call_1(new IntPtr(f), op); } } #endif @@ -656,7 +787,11 @@ internal static unsafe void XDecref(IntPtr op) [Pure] internal static unsafe long Refcount(IntPtr op) { +#if PYTHON_WITH_PYDEBUG + var p = (void*)(op + TypeOffset.ob_refcnt); +#else var p = (void*)op; +#endif if ((void*)0 == p) { return 0; @@ -704,6 +839,9 @@ internal static unsafe long Refcount(IntPtr op) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyThreadState_Get(); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr _PyThreadState_UncheckedGet(); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyThread_get_key_value(IntPtr key); @@ -802,7 +940,7 @@ public static extern int Py_Main( internal static extern int PyRun_SimpleString(string code); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern NewReference PyRun_String([MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string code, IntPtr st, IntPtr globals, IntPtr locals); + internal static extern NewReference PyRun_String([MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string code, RunFlagType st, IntPtr globals, IntPtr locals); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyEval_EvalCode(IntPtr co, IntPtr globals, IntPtr locals); @@ -851,6 +989,7 @@ internal static IntPtr Py_CompileStringFlags(string str, string file, int start, //==================================================================== /// + /// Return value: Borrowed reference. /// A macro-like method to get the type of a Python object. This is /// designed to be lean and mean in IL & avoid managed <-> unmanaged /// transitions. Note that this does not incref the type object. @@ -907,6 +1046,9 @@ internal static bool PyObject_IsIterable(IntPtr pointer) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_GetAttrString(IntPtr pointer, string name); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyObject_GetAttrString(IntPtr pointer, IntPtr name); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyObject_SetAttrString(IntPtr pointer, string name, IntPtr value); @@ -1004,6 +1146,11 @@ internal static long PyObject_Size(IntPtr pointer) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_Dir(IntPtr pointer); +#if PYTHON_WITH_PYDEBUG + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern void _Py_NewReference(IntPtr ob); +#endif + //==================================================================== // Python buffer API //==================================================================== @@ -1152,6 +1299,19 @@ internal static bool PyFloat_Check(IntPtr ob) return PyObject_TYPE(ob) == PyFloatType; } + /// + /// Return value: New reference. + /// Create a Python integer from the pointer p. The pointer value can be retrieved from the resulting value using PyLong_AsVoidPtr(). + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyLong_FromVoidPtr(IntPtr p); + + /// + /// Convert a Python integer pylong to a C void pointer. If pylong cannot be converted, an OverflowError will be raised. This is only assured to produce a usable void pointer for values created with PyLong_FromVoidPtr(). + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyLong_AsVoidPtr(IntPtr ob); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyFloat_FromDouble(double value); @@ -1484,18 +1644,34 @@ internal static bool PyDict_Check(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDict_New(); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyDict_Next(IntPtr p, out IntPtr ppos, out IntPtr pkey, out IntPtr pvalue); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDictProxy_New(IntPtr dict); + /// + /// Return value: Borrowed reference. + /// Return NULL if the key key is not present, but without setting an exception. + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDict_GetItem(IntPtr pointer, IntPtr key); + /// + /// Return value: Borrowed reference. + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDict_GetItemString(IntPtr pointer, string key); + /// + /// Return 0 on success or -1 on failure. + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyDict_SetItem(IntPtr pointer, IntPtr key, IntPtr value); + /// + /// Return 0 on success or -1 on failure. + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyDict_SetItemString(IntPtr pointer, string key, IntPtr value); @@ -1535,6 +1711,21 @@ internal static long PyDict_Size(IntPtr pointer) internal static extern IntPtr _PyDict_Size(IntPtr pointer); + /// + /// Return value: New reference. + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PySet_New(IntPtr iterable); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PySet_Add(IntPtr set, IntPtr key); + + /// + /// Return 1 if found, 0 if not found, and -1 if an error is encountered. + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PySet_Contains(IntPtr anyset, IntPtr key); + //==================================================================== // Python list API //==================================================================== @@ -1693,7 +1884,11 @@ internal static bool PyIter_Check(IntPtr pointer) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern string PyModule_GetFilename(IntPtr module); +#if PYTHON_WITH_PYDEBUG + [DllImport(_PythonDll, EntryPoint = "PyModule_Create2TraceRefs", CallingConvention = CallingConvention.Cdecl)] +#else [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] +#endif internal static extern IntPtr PyModule_Create2(IntPtr module, int apiver); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] @@ -1721,6 +1916,10 @@ internal static extern void PySys_SetArgvEx( int updatepath ); + /// + /// Return value: Borrowed reference. + /// Return the object name from the sys module or NULL if it does not exist, without setting an exception. + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern BorrowedReference PySys_GetObject(string name); @@ -1765,6 +1964,9 @@ internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] private static extern IntPtr PyType_GenericAlloc(IntPtr type, IntPtr n); + /// + /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type’s base class. Return 0 on success, or return -1 and sets an exception on error. + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyType_Ready(IntPtr type); @@ -1789,6 +1991,8 @@ internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyObject_GC_UnTrack(IntPtr tp); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern void _PyObject_Dump(IntPtr ob); //==================================================================== // Python memory API @@ -1864,6 +2068,74 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyCell_Set(BorrowedReference cell, IntPtr value); + //==================================================================== + // Python GC API + //==================================================================== + + internal const int _PyGC_REFS_SHIFT = 1; + internal const long _PyGC_REFS_UNTRACKED = -2; + internal const long _PyGC_REFS_REACHABLE = -3; + internal const long _PyGC_REFS_TENTATIVELY_UNREACHABLE = -4; + + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyGC_Collect(); + + internal static IntPtr _Py_AS_GC(IntPtr ob) + { + // XXX: PyGC_Head has a force alignment depend on platform. + // See PyGC_Head in objimpl.h for more details. + return Is32Bit ? ob - 16 : ob - 24; + } + + internal static IntPtr _Py_FROM_GC(IntPtr gc) + { + return Is32Bit ? gc + 16 : gc + 24; + } + + internal static IntPtr _PyGCHead_REFS(IntPtr gc) + { + unsafe + { + var pGC = (PyGC_Head*)gc; + var refs = pGC->gc.gc_refs; + if (Is32Bit) + { + return new IntPtr(refs.ToInt32() >> _PyGC_REFS_SHIFT); + } + return new IntPtr(refs.ToInt64() >> _PyGC_REFS_SHIFT); + } + } + + internal static IntPtr _PyGC_REFS(IntPtr ob) + { + return _PyGCHead_REFS(_Py_AS_GC(ob)); + } + + internal static bool _PyObject_GC_IS_TRACKED(IntPtr ob) + { + return (long)_PyGC_REFS(ob) != _PyGC_REFS_UNTRACKED; + } + + internal static void Py_CLEAR(ref IntPtr ob) + { + XDecref(ob); + ob = IntPtr.Zero; + } + + //==================================================================== + // Python Capsules API + //==================================================================== + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern NewReference PyCapsule_New(IntPtr pointer, string name, IntPtr destructor); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyCapsule_GetPointer(BorrowedReference capsule, string name); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyCapsule_SetPointer(BorrowedReference capsule, IntPtr pointer); + //==================================================================== // Miscellaneous //==================================================================== @@ -1882,12 +2154,16 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) internal static void SetNoSiteFlag() { - var loader = LibraryLoader.Get(OperatingSystem); - - IntPtr dllLocal = _PythonDll != "__Internal" - ? loader.Load(_PythonDll) - : IntPtr.Zero; - + var loader = LibraryLoader.Get(NativeCodePageHelper.OperatingSystem); + IntPtr dllLocal; + if (_PythonDll != "__Internal") + { + dllLocal = loader.Load(_PythonDll); + if (dllLocal == IntPtr.Zero) + { + throw new Exception($"Cannot load {_PythonDll}"); + } + } try { Py_NoSiteFlag = loader.GetFunction(dllLocal, "Py_NoSiteFlag"); @@ -1912,6 +2188,15 @@ internal static IntPtr GetBuiltins() } + public enum ShutdownMode + { + Default, + Normal, + Soft, + Reload, + } + + class PyReferenceCollection { private List> _actions = new List>(); diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs new file mode 100644 index 000000000..060573db4 --- /dev/null +++ b/src/runtime/runtime_data.cs @@ -0,0 +1,447 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Formatters.Binary; + +using static Python.Runtime.Runtime; + +namespace Python.Runtime +{ + public static class RuntimeData + { + private static Type _formatterType; + public static Type FormatterType + { + get => _formatterType; + set + { + if (!typeof(IFormatter).IsAssignableFrom(value)) + { + throw new ArgumentException("Not a type implemented IFormatter"); + } + _formatterType = value; + } + } + + public static ICLRObjectStorer WrappersStorer { get; set; } + + /// + /// Clears the old "clr_data" entry if a previous one is present. + /// + static void ClearCLRData () + { + BorrowedReference capsule = PySys_GetObject("clr_data"); + if (!capsule.IsNull) + { + IntPtr oldData = PyCapsule_GetPointer(capsule, null); + PyMem_Free(oldData); + PyCapsule_SetPointer(capsule, IntPtr.Zero); + } + } + + internal static void Stash() + { + var metaStorage = new RuntimeDataStorage(); + MetaType.SaveRuntimeData(metaStorage); + + var importStorage = new RuntimeDataStorage(); + ImportHook.SaveRuntimeData(importStorage); + + var typeStorage = new RuntimeDataStorage(); + TypeManager.SaveRuntimeData(typeStorage); + + var clsStorage = new RuntimeDataStorage(); + ClassManager.SaveRuntimeData(clsStorage); + + var moduleStorage = new RuntimeDataStorage(); + SaveRuntimeDataModules(moduleStorage); + + var objStorage = new RuntimeDataStorage(); + SaveRuntimeDataObjects(objStorage); + + var runtimeStorage = new RuntimeDataStorage(); + runtimeStorage.AddValue("meta", metaStorage); + runtimeStorage.AddValue("import", importStorage); + runtimeStorage.AddValue("types", typeStorage); + runtimeStorage.AddValue("classes", clsStorage); + runtimeStorage.AddValue("modules", moduleStorage); + runtimeStorage.AddValue("objs", objStorage); + + IFormatter formatter = CreateFormatter(); + var ms = new MemoryStream(); + formatter.Serialize(ms, runtimeStorage); + + Debug.Assert(ms.Length <= int.MaxValue); + byte[] data = ms.GetBuffer(); + // TODO: use buffer api instead + IntPtr mem = PyMem_Malloc(ms.Length + IntPtr.Size); + Marshal.WriteIntPtr(mem, (IntPtr)ms.Length); + Marshal.Copy(data, 0, mem + IntPtr.Size, (int)ms.Length); + + ClearCLRData(); + NewReference capsule = PyCapsule_New(mem, null, IntPtr.Zero); + PySys_SetObject("clr_data", capsule.DangerousGetAddress()); + // Let the dictionary own the reference + capsule.Dispose(); + } + + internal static void RestoreRuntimeData() + { + try + { + RestoreRuntimeDataImpl(); + } + finally + { + ClearStash(); + } + } + + private static void RestoreRuntimeDataImpl() + { + BorrowedReference capsule = PySys_GetObject("clr_data"); + if (capsule.IsNull) + { + return; + } + IntPtr mem = PyCapsule_GetPointer(capsule, null); + int length = (int)Marshal.ReadIntPtr(mem); + byte[] data = new byte[length]; + Marshal.Copy(mem + IntPtr.Size, data, 0, length); + var ms = new MemoryStream(data); + var formatter = CreateFormatter(); + var storage = (RuntimeDataStorage)formatter.Deserialize(ms); + + var objs = RestoreRuntimeDataObjects(storage.GetStorage("objs")); + RestoreRuntimeDataModules(storage.GetStorage("modules")); + var clsObjs = ClassManager.RestoreRuntimeData(storage.GetStorage("classes")); + TypeManager.RestoreRuntimeData(storage.GetStorage("types")); + ImportHook.RestoreRuntimeData(storage.GetStorage("import")); + PyCLRMetaType = MetaType.RestoreRuntimeData(storage.GetStorage("meta")); + + foreach (var item in objs) + { + item.Value.ExecutePostActions(); + XDecref(item.Key.pyHandle); + } + foreach (var item in clsObjs) + { + item.Value.ExecutePostActions(); + } + } + + public static bool HasStashData() + { + return !PySys_GetObject("clr_data").IsNull; + } + + public static void ClearStash() + { + PySys_SetObject("clr_data", IntPtr.Zero); + } + + static bool CheckSerializable (object o) + { + Type type = o.GetType(); + do + { + if (!type.IsSerializable) + { + return false; + } + } while ((type = type.BaseType) != null); + return true; + } + + private static void SaveRuntimeDataObjects(RuntimeDataStorage storage) + { + var objs = ManagedType.GetManagedObjects(); + var extensionObjs = new List(); + var wrappers = new Dictionary>(); + var serializeObjs = new CLRWrapperCollection(); + var contexts = new Dictionary(); + foreach (var entry in objs) + { + var obj = entry.Key; + XIncref(obj.pyHandle); + switch (entry.Value) + { + case ManagedType.TrackTypes.Extension: + Debug.Assert(CheckSerializable(obj)); + var context = new InterDomainContext(); + contexts[obj.pyHandle] = context; + obj.Save(context); + extensionObjs.Add(obj); + break; + case ManagedType.TrackTypes.Wrapper: + // Wrapper must be the CLRObject + var clrObj = (CLRObject)obj; + object inst = clrObj.inst; + CLRMappedItem item; + List mappedObjs; + if (!serializeObjs.TryGetValue(inst, out item)) + { + item = new CLRMappedItem(inst) + { + Handles = new List() + }; + serializeObjs.Add(item); + + Debug.Assert(!wrappers.ContainsKey(inst)); + mappedObjs = new List(); + wrappers.Add(inst, mappedObjs); + } + else + { + mappedObjs = wrappers[inst]; + } + item.Handles.Add(clrObj.pyHandle); + mappedObjs.Add(clrObj); + break; + default: + break; + } + } + + var wrapperStorage = new RuntimeDataStorage(); + WrappersStorer?.Store(serializeObjs, wrapperStorage); + + var internalStores = new List(); + foreach (var item in serializeObjs) + { + if (!item.Stored) + { + if (!CheckSerializable(item.Instance)) + { + continue; + } + internalStores.AddRange(wrappers[item.Instance]); + } + foreach (var clrObj in wrappers[item.Instance]) + { + XIncref(clrObj.pyHandle); + var context = new InterDomainContext(); + contexts[clrObj.pyHandle] = context; + clrObj.Save(context); + } + } + storage.AddValue("internalStores", internalStores); + storage.AddValue("extensions", extensionObjs); + storage.AddValue("wrappers", wrapperStorage); + storage.AddValue("contexts", contexts); + } + + private static Dictionary RestoreRuntimeDataObjects(RuntimeDataStorage storage) + { + var extensions = storage.GetValue>("extensions"); + var internalStores = storage.GetValue>("internalStores"); + var contexts = storage.GetValue >("contexts"); + var storedObjs = new Dictionary(); + foreach (var obj in Enumerable.Union(extensions, internalStores)) + { + var context = contexts[obj.pyHandle]; + obj.Load(context); + storedObjs.Add(obj, context); + } + if (WrappersStorer != null) + { + var wrapperStorage = storage.GetStorage("wrappers"); + var handle2Obj = WrappersStorer.Restore(wrapperStorage); + foreach (var item in handle2Obj) + { + object obj = item.Instance; + foreach (var handle in item.Handles) + { + var context = contexts[handle]; + var co = CLRObject.Restore(obj, handle, context); + storedObjs.Add(co, context); + } + } + } + return storedObjs; + } + + private static void SaveRuntimeDataModules(RuntimeDataStorage storage) + { + var pyModules = PyImport_GetModuleDict(); + var items = PyDict_Items(pyModules); + long length = PyList_Size(items); + var modules = new Dictionary(); ; + for (long i = 0; i < length; i++) + { + var item = PyList_GetItem(items, i); + var name = PyTuple_GetItem(item.DangerousGetAddress(), 0); + var module = PyTuple_GetItem(item.DangerousGetAddress(), 1); + if (ManagedType.IsManagedType(module)) + { + XIncref(name); + XIncref(module); + modules.Add(name, module); + } + } + items.Dispose(); + storage.AddValue("modules", modules); + } + + private static void RestoreRuntimeDataModules(RuntimeDataStorage storage) + { + var modules = storage.GetValue>("modules"); + var pyMoudles = PyImport_GetModuleDict(); + foreach (var item in modules) + { + int res = PyDict_SetItem(pyMoudles, item.Key, item.Value); + PythonException.ThrowIfIsNotZero(res); + XDecref(item.Key); + XDecref(item.Value); + } + modules.Clear(); + } + + private static IFormatter CreateFormatter() + { + return FormatterType != null ? + (IFormatter)Activator.CreateInstance(FormatterType) + : new BinaryFormatter(); + } + } + + + [Serializable] + public class RuntimeDataStorage + { + private Stack _stack; + private Dictionary _namedValues; + + public T AddValue(string name, T value) + { + if (_namedValues == null) + { + _namedValues = new Dictionary(); + } + _namedValues.Add(name, value); + return value; + } + + public object GetValue(string name) + { + return _namedValues[name]; + } + + public T GetValue(string name) + { + return (T)GetValue(name); + } + + public T GetValue(string name, out T value) + { + value = GetValue(name); + return value; + } + + public RuntimeDataStorage GetStorage(string name) + { + return GetValue(name); + } + + public T PushValue(T value) + { + if (_stack == null) + { + _stack = new Stack(); + } + _stack.Push(value); + return value; + } + + public object PopValue() + { + return _stack.Pop(); + } + + public T PopValue() + { + return (T)PopValue(); + } + + public T PopValue(out T value) + { + return value = (T)PopValue(); + } + } + + + [Serializable] + class InterDomainContext + { + private RuntimeDataStorage _storage; + public RuntimeDataStorage Storage => _storage ?? (_storage = new RuntimeDataStorage()); + + /// + /// Actions after loaded. + /// + [NonSerialized] + private List _postActions; + public List PostActions => _postActions ?? (_postActions = new List()); + + public void AddPostAction(Action action) + { + PostActions.Add(action); + } + + public void ExecutePostActions() + { + if (_postActions == null) + { + return; + } + foreach (var action in _postActions) + { + action(); + } + } + } + + public class CLRMappedItem + { + public object Instance { get; private set; } + public IList Handles { get; set; } + public bool Stored { get; set; } + + public CLRMappedItem(object instance) + { + Instance = instance; + } + } + + + public interface ICLRObjectStorer + { + ICollection Store(CLRWrapperCollection wrappers, RuntimeDataStorage storage); + CLRWrapperCollection Restore(RuntimeDataStorage storage); + } + + + public class CLRWrapperCollection : KeyedCollection + { + public bool TryGetValue(object key, out CLRMappedItem value) + { + if (Dictionary == null) + { + value = null; + return false; + } + return Dictionary.TryGetValue(key, out value); + } + + protected override object GetKeyForItem(CLRMappedItem item) + { + return item.Instance; + } + } +} diff --git a/src/runtime/runtime_state.cs b/src/runtime/runtime_state.cs new file mode 100644 index 000000000..69acbcd31 --- /dev/null +++ b/src/runtime/runtime_state.cs @@ -0,0 +1,238 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.InteropServices; + +using static Python.Runtime.Runtime; + +namespace Python.Runtime +{ + class RuntimeState + { + public static bool ShouldRestoreObjects { get; set; } = false; + public static bool UseDummyGC { get; set; } = false; + + public static void Save() + { + if (!PySys_GetObject("dummy_gc").IsNull) + { + throw new Exception("Runtime State set already"); + } + + IntPtr objs = IntPtr.Zero; + if (ShouldRestoreObjects) + { + objs = PySet_New(IntPtr.Zero); + foreach (var obj in PyGCGetObjects()) + { + AddObjPtrToSet(objs, obj); + } + } + + var modules = PySet_New(IntPtr.Zero); + foreach (var name in GetModuleNames()) + { + int res = PySet_Add(modules, name); + PythonException.ThrowIfIsNotZero(res); + } + + + var dummyGCHead = PyMem_Malloc(Marshal.SizeOf(typeof(PyGC_Head))); + unsafe + { + var head = (PyGC_Head*)dummyGCHead; + head->gc.gc_next = dummyGCHead; + head->gc.gc_prev = dummyGCHead; + head->gc.gc_refs = IntPtr.Zero; + } + { + var pyDummyGC = PyLong_FromVoidPtr(dummyGCHead); + int res = PySys_SetObject("dummy_gc", pyDummyGC); + PythonException.ThrowIfIsNotZero(res); + XDecref(pyDummyGC); + + try + { + res = PySys_SetObject("initial_modules", modules); + PythonException.ThrowIfIsNotZero(res); + } + finally + { + XDecref(modules); + } + + if (ShouldRestoreObjects) + { + AddObjPtrToSet(objs, modules); + try + { + res = PySys_SetObject("initial_objs", objs); + PythonException.ThrowIfIsNotZero(res); + } + finally + { + XDecref(objs); + } + } + } + } + + public static void Restore() + { + var dummyGCAddr = PySys_GetObject("dummy_gc").DangerousGetAddress(); + if (dummyGCAddr == IntPtr.Zero) + { + throw new InvalidOperationException("Runtime state have not set"); + } + var dummyGC = PyLong_AsVoidPtr(dummyGCAddr); + ResotreModules(dummyGC); + if (ShouldRestoreObjects) + { + RestoreObjects(dummyGC); + } + } + + private static void ResotreModules(IntPtr dummyGC) + { + var intialModules = PySys_GetObject("initial_modules"); + Debug.Assert(!intialModules.IsNull); + var modules = PyImport_GetModuleDict(); + foreach (var name in GetModuleNames()) + { + if (PySet_Contains(intialModules.DangerousGetAddress(), name) == 1) + { + continue; + } + var module = PyDict_GetItem(modules, name); + + if (UseDummyGC && _PyObject_GC_IS_TRACKED(module)) + { + ExchangeGCChain(module, dummyGC); + } + if (PyDict_DelItem(modules, name) != 0) + { + PyErr_Print(); + } + } + } + + private static void RestoreObjects(IntPtr dummyGC) + { + if (!UseDummyGC) + { + throw new Exception("To prevent crash by _PyObject_GC_UNTRACK in Python internal, UseDummyGC should be enabled when using ResotreObjects"); + } + IntPtr intialObjs = PySys_GetObject("initial_objs").DangerousGetAddress(); + Debug.Assert(intialObjs != IntPtr.Zero); + foreach (var obj in PyGCGetObjects()) + { + var p = PyLong_FromVoidPtr(obj); + try + { + if (PySet_Contains(intialObjs, p) == 1) + { + continue; + } + } + finally + { + XDecref(p); + } + Debug.Assert(_PyObject_GC_IS_TRACKED(obj), "A GC object must be tracked"); + ExchangeGCChain(obj, dummyGC); + } + } + + public static IEnumerable PyGCGetObjects() + { + var gc = PyImport_ImportModule("gc"); + PythonException.ThrowIfIsNull(gc); + var get_objects = PyObject_GetAttrString(gc, "get_objects"); + var objs = PyObject_CallObject(get_objects, IntPtr.Zero); + var length = PyList_Size(new BorrowedReference(objs)); + for (long i = 0; i < length; i++) + { + var obj = PyList_GetItem(new BorrowedReference(objs), i); + yield return obj.DangerousGetAddress(); + } + XDecref(objs); + XDecref(gc); + } + + public static IEnumerable GetModuleNames() + { + var modules = PyImport_GetModuleDict(); + var names = PyDict_Keys(modules); + var length = PyList_Size(new BorrowedReference(names)); + for (int i = 0; i < length; i++) + { + var name = PyList_GetItem(new BorrowedReference(names), i); + yield return name.DangerousGetAddress(); + } + XDecref(names); + } + + private static void AddObjPtrToSet(IntPtr set, IntPtr obj) + { + var p = PyLong_FromVoidPtr(obj); + XIncref(obj); + try + { + int res = PySet_Add(set, p); + PythonException.ThrowIfIsNotZero(res); + } + finally + { + XDecref(p); + } + } + /// + /// Exchange gc to a dummy gc prevent nullptr error in _PyObject_GC_UnTrack macro. + /// + private static void ExchangeGCChain(IntPtr obj, IntPtr gc) + { + var head = _Py_AS_GC(obj); + if ((long)_PyGCHead_REFS(head) == _PyGC_REFS_UNTRACKED) + { + throw new ArgumentException("GC object untracked"); + } + unsafe + { + var g = (PyGC_Head*)head; + var newGCGen = (PyGC_Head*)gc; + + ((PyGC_Head*)g->gc.gc_prev)->gc.gc_next = g->gc.gc_next; + ((PyGC_Head*)g->gc.gc_next)->gc.gc_prev = g->gc.gc_prev; + + g->gc.gc_next = gc; + g->gc.gc_prev = newGCGen->gc.gc_prev; + ((PyGC_Head*)g->gc.gc_prev)->gc.gc_next = head; + newGCGen->gc.gc_prev = head; + } + } + + private static IEnumerable IterGCNodes(IntPtr gc) + { + var node = GetNextGCNode(gc); + while (node != gc) + { + var next = GetNextGCNode(node); + yield return node; + node = next; + } + } + + private static IEnumerable IterObjects(IntPtr gc) + { + foreach (var node in IterGCNodes(gc)) + { + yield return _Py_FROM_GC(node); + } + } + + private static unsafe IntPtr GetNextGCNode(IntPtr node) + { + return ((PyGC_Head*)node)->gc.gc_next; + } + } +} diff --git a/src/runtime/slots/mp_length.cs b/src/runtime/slots/mp_length.cs index b0a2e8d79..42448a2e9 100644 --- a/src/runtime/slots/mp_length.cs +++ b/src/runtime/slots/mp_length.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Reflection; @@ -8,11 +9,41 @@ namespace Python.Runtime.Slots { internal static class mp_length_slot { + private static MethodInfo _lengthMethod; + public static MethodInfo Method + { + get + { + if (_lengthMethod != null) + { + return _lengthMethod; + } + _lengthMethod = typeof(mp_length_slot).GetMethod( + nameof(mp_length_slot.mp_length), + BindingFlags.Static | BindingFlags.NonPublic); + Debug.Assert(_lengthMethod != null); + return _lengthMethod; + } + } + + public static bool CanAssgin(Type clrType) + { + if (typeof(ICollection).IsAssignableFrom(clrType)) + { + return true; + } + if (clrType.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(ICollection<>))) + { + return true; + } + return false; + } + /// /// Implements __len__ for classes that implement ICollection /// (this includes any IList implementer or Array subclass) /// - public static int mp_length(IntPtr ob) + private static int mp_length(IntPtr ob) { var co = ManagedType.GetManagedObject(ob) as CLRObject; if (co == null) diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index a64f91e75..c00247ca4 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -4,7 +4,7 @@ using System.Linq; using System.Reflection; using System.Runtime.InteropServices; -using Python.Runtime.Platform; +using System.Diagnostics; using Python.Runtime.Slots; namespace Python.Runtime @@ -16,21 +16,79 @@ namespace Python.Runtime /// internal class TypeManager { - private static BindingFlags tbFlags; - private static Dictionary cache; + internal static IntPtr subtype_traverse; + internal static IntPtr subtype_clear; - static TypeManager() + private const BindingFlags tbFlags = BindingFlags.Public | BindingFlags.Static; + private static Dictionary cache = new Dictionary(); + private static readonly Dictionary _slotsHolders = new Dictionary(); + private static Dictionary _slotsImpls = new Dictionary(); + + // Slots which must be set + private static readonly string[] _requiredSlots = new string[] + { + "tp_traverse", + "tp_clear", + }; + + internal static void Initialize() { - tbFlags = BindingFlags.Public | BindingFlags.Static; - cache = new Dictionary(128); + Debug.Assert(cache.Count == 0, "Cache should be empty", + "Some errors may occurred on last shutdown"); + IntPtr type = SlotHelper.CreateObjectType(); + subtype_traverse = Marshal.ReadIntPtr(type, TypeOffset.tp_traverse); + subtype_clear = Marshal.ReadIntPtr(type, TypeOffset.tp_clear); + Runtime.XDecref(type); } - public static void Reset() + internal static void RemoveTypes() { - cache = new Dictionary(128); + foreach (var tpHandle in cache.Values) + { + SlotsHolder holder; + if (_slotsHolders.TryGetValue(tpHandle, out holder)) + { + // If refcount > 1, it needs to reset the managed slot, + // otherwise it can dealloc without any trick. + if (Runtime.Refcount(tpHandle) > 1) + { + holder.ResetSlots(); + } + } + Runtime.XDecref(tpHandle); + } + cache.Clear(); + _slotsImpls.Clear(); + _slotsHolders.Clear(); + } + + internal static void SaveRuntimeData(RuntimeDataStorage storage) + { + foreach (var tpHandle in cache.Values) + { + Runtime.XIncref(tpHandle); + } + storage.AddValue("cache", cache); + storage.AddValue("slots", _slotsImpls); + } + + internal static void RestoreRuntimeData(RuntimeDataStorage storage) + { + Debug.Assert(cache == null || cache.Count == 0); + storage.GetValue("slots", out _slotsImpls); + storage.GetValue("cache", out cache); + foreach (var entry in cache) + { + Type type = entry.Key; + IntPtr handle = entry.Value; + SlotsHolder holder = CreateSolotsHolder(handle); + InitializeSlots(handle, _slotsImpls[type], holder); + // FIXME: mp_length_slot.CanAssgin(clrType) + } } /// + /// Return value: Borrowed reference. /// Given a managed Type derived from ExtensionType, get the handle to /// a Python type object that delegates its implementation to the Type /// object. These Python type instances are used to implement internal @@ -48,11 +106,13 @@ internal static IntPtr GetTypeHandle(Type type) } handle = CreateType(type); cache[type] = handle; + _slotsImpls.Add(type, type); return handle; } /// + /// Return value: Borrowed reference. /// Get the handle of a Python type that reflects the given CLR type. /// The given ManagedType instance is a managed object that implements /// the appropriate semantics in Python for the reflected managed type. @@ -67,6 +127,7 @@ internal static IntPtr GetTypeHandle(ManagedType obj, Type type) } handle = CreateType(obj, type); cache[type] = handle; + _slotsImpls.Add(type, obj.GetType()); return handle; } @@ -90,20 +151,24 @@ internal static IntPtr CreateType(Type impl) var offset = (IntPtr)ObjectOffset.TypeDictOffset(type); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); - InitializeSlots(type, impl); + SlotsHolder slotsHolder = CreateSolotsHolder(type); + InitializeSlots(type, impl, slotsHolder); int flags = TypeFlags.Default | TypeFlags.Managed | TypeFlags.HeapType | TypeFlags.HaveGC; Util.WriteCLong(type, TypeOffset.tp_flags, flags); - Runtime.PyType_Ready(type); + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); IntPtr mod = Runtime.PyString_FromString("CLR"); Runtime.PyDict_SetItemString(dict, "__module__", mod); + Runtime.XDecref(mod); InitMethods(type, impl); - return type; } @@ -155,21 +220,22 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, (IntPtr)tp_dictoffset); - // add a __len__ slot for inheritors of ICollection and ICollection<> - if (typeof(ICollection).IsAssignableFrom(clrType) || clrType.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(ICollection<>))) + // we want to do this after the slot stuff above in case the class itself implements a slot method + SlotsHolder slotsHolder = CreateSolotsHolder(type); + InitializeSlots(type, impl.GetType(), slotsHolder); + + if (Marshal.ReadIntPtr(type, TypeOffset.mp_length) == IntPtr.Zero + && mp_length_slot.CanAssgin(clrType)) { - InitializeSlot(type, TypeOffset.mp_length, typeof(mp_length_slot).GetMethod(nameof(mp_length_slot.mp_length))); + InitializeSlot(type, TypeOffset.mp_length, mp_length_slot.Method, slotsHolder); } - // we want to do this after the slot stuff above in case the class itself implements a slot method - InitializeSlots(type, impl.GetType()); - if (!typeof(IEnumerable).IsAssignableFrom(clrType) && !typeof(IEnumerator).IsAssignableFrom(clrType)) - { - // The tp_iter slot should only be set for enumerable types. - Marshal.WriteIntPtr(type, TypeOffset.tp_iter, IntPtr.Zero); - } + { + // The tp_iter slot should only be set for enumerable types. + Marshal.WriteIntPtr(type, TypeOffset.tp_iter, IntPtr.Zero); + } // Only set mp_subscript and mp_ass_subscript for types with indexers @@ -199,31 +265,34 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Runtime.XIncref(base_); } - int flags = TypeFlags.Default; - flags |= TypeFlags.Managed; - flags |= TypeFlags.HeapType; - flags |= TypeFlags.BaseType; - flags |= TypeFlags.HaveGC; + const int flags = TypeFlags.Default + | TypeFlags.Managed + | TypeFlags.HeapType + | TypeFlags.BaseType + | TypeFlags.HaveGC; Util.WriteCLong(type, TypeOffset.tp_flags, flags); // Leverage followup initialization from the Python runtime. Note // that the type of the new type must PyType_Type at the time we // call this, else PyType_Ready will skip some slot initialization. - Runtime.PyType_Ready(type); + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); string mn = clrType.Namespace ?? ""; IntPtr mod = Runtime.PyString_FromString(mn); Runtime.PyDict_SetItemString(dict, "__module__", mod); + Runtime.XDecref(mod); // Hide the gchandle of the implementation in a magic type slot. - GCHandle gc = GCHandle.Alloc(impl); + GCHandle gc = impl.AllocGCHandle(); Marshal.WriteIntPtr(type, TypeOffset.magic(), (IntPtr)gc); // Set the handle attributes on the implementing instance. - impl.tpHandle = Runtime.PyCLRMetaType; - impl.gcHandle = gc; + impl.tpHandle = type; impl.pyHandle = type; //DebugUtil.DumpType(type); @@ -231,12 +300,6 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) return type; } - static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method) - { - var thunk = Interop.GetThunk(method); - Marshal.WriteIntPtr(type, slotOffset, thunk.Address); - } - internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr py_dict) { // Utility to create a subtype of a managed type with the ability for the @@ -308,7 +371,7 @@ internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr // derived class we want the python overrides in there instead if they exist. IntPtr cls_dict = Marshal.ReadIntPtr(py_type, TypeOffset.tp_dict); Runtime.PyDict_Update(cls_dict, py_dict); - + Runtime.XIncref(py_type); // Update the __classcell__ if it exists var cell = new BorrowedReference(Runtime.PyDict_GetItemString(cls_dict, "__classcell__")); if (!cell.IsNull) @@ -348,7 +411,25 @@ internal static IntPtr WriteMethodDefSentinel(IntPtr mdef) return WriteMethodDef(mdef, IntPtr.Zero, IntPtr.Zero, 0, IntPtr.Zero); } - internal static IntPtr CreateMetaType(Type impl) + internal static void FreeMethodDef(IntPtr mdef) + { + unsafe + { + var def = (PyMethodDef*)mdef; + if (def->ml_name != IntPtr.Zero) + { + Marshal.FreeHGlobal(def->ml_name); + def->ml_name = IntPtr.Zero; + } + if (def->ml_doc != IntPtr.Zero) + { + Marshal.FreeHGlobal(def->ml_doc); + def->ml_doc = IntPtr.Zero; + } + } + } + + internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) { // The managed metatype is functionally little different than the // standard Python metatype (PyType_Type). It overrides certain of @@ -361,57 +442,90 @@ internal static IntPtr CreateMetaType(Type impl) Marshal.WriteIntPtr(type, TypeOffset.tp_base, py_type); Runtime.XIncref(py_type); + const int flags = TypeFlags.Default + | TypeFlags.Managed + | TypeFlags.HeapType + | TypeFlags.HaveGC; + Util.WriteCLong(type, TypeOffset.tp_flags, flags); + // Slots will inherit from TypeType, it's not neccesary for setting them. // Inheried slots: // tp_basicsize, tp_itemsize, // tp_dictoffset, tp_weaklistoffset, // tp_traverse, tp_clear, tp_is_gc, etc. + slotsHolder = SetupMetaSlots(impl, type); - // Override type slots with those of the managed implementation. + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } - InitializeSlots(type, impl); + IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); + IntPtr mod = Runtime.PyString_FromString("CLR"); + Runtime.PyDict_SetItemString(dict, "__module__", mod); - int flags = TypeFlags.Default; - flags |= TypeFlags.Managed; - flags |= TypeFlags.HeapType; - flags |= TypeFlags.HaveGC; - Util.WriteCLong(type, TypeOffset.tp_flags, flags); + //DebugUtil.DumpType(type); + + return type; + } - // We need space for 3 PyMethodDef structs, each of them - // 4 int-ptrs in size. - IntPtr mdef = Runtime.PyMem_Malloc(3 * 4 * IntPtr.Size); + internal static SlotsHolder SetupMetaSlots(Type impl, IntPtr type) + { + // Override type slots with those of the managed implementation. + SlotsHolder slotsHolder = new SlotsHolder(type); + InitializeSlots(type, impl, slotsHolder); + + // We need space for 3 PyMethodDef structs. + int mdefSize = (MetaType.CustomMethods.Length + 1) * Marshal.SizeOf(typeof(PyMethodDef)); + IntPtr mdef = Runtime.PyMem_Malloc(mdefSize); IntPtr mdefStart = mdef; - ThunkInfo thunkInfo = Interop.GetThunk(typeof(MetaType).GetMethod("__instancecheck__"), "BinaryFunc"); - mdef = WriteMethodDef( - mdef, - "__instancecheck__", - thunkInfo.Address - ); - - thunkInfo = Interop.GetThunk(typeof(MetaType).GetMethod("__subclasscheck__"), "BinaryFunc"); - mdef = WriteMethodDef( - mdef, - "__subclasscheck__", - thunkInfo.Address - ); - - // FIXME: mdef is not used + foreach (var methodName in MetaType.CustomMethods) + { + mdef = AddCustomMetaMethod(methodName, type, mdef, slotsHolder); + } mdef = WriteMethodDefSentinel(mdef); + Debug.Assert((long)(mdefStart + mdefSize) <= (long)mdef); Marshal.WriteIntPtr(type, TypeOffset.tp_methods, mdefStart); - Runtime.PyType_Ready(type); - - IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); - IntPtr mod = Runtime.PyString_FromString("CLR"); - Runtime.PyDict_SetItemString(dict, "__module__", mod); + // XXX: Hard code with mode check. + if (Runtime.ShutdownMode != ShutdownMode.Reload) + { + slotsHolder.Set(TypeOffset.tp_methods, (t, offset) => + { + var p = Marshal.ReadIntPtr(t, offset); + Runtime.PyMem_Free(p); + Marshal.WriteIntPtr(t, offset, IntPtr.Zero); + }); + } + return slotsHolder; + } - //DebugUtil.DumpType(type); + private static IntPtr AddCustomMetaMethod(string name, IntPtr type, IntPtr mdef, SlotsHolder slotsHolder) + { + MethodInfo mi = typeof(MetaType).GetMethod(name); + ThunkInfo thunkInfo = Interop.GetThunk(mi, "BinaryFunc"); + slotsHolder.KeeapAlive(thunkInfo); - return type; + // XXX: Hard code with mode check. + if (Runtime.ShutdownMode != ShutdownMode.Reload) + { + IntPtr mdefAddr = mdef; + slotsHolder.AddDealloctor(() => + { + IntPtr tp_dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); + if (Runtime.PyDict_DelItemString(tp_dict, name) != 0) + { + Runtime.PyErr_Print(); + Debug.Fail($"Cannot remove {name} from metatype"); + } + FreeMethodDef(mdefAddr); + }); + } + mdef = WriteMethodDef(mdef, name, thunkInfo.Address); + return mdef; } - internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) { // Utility to create a subtype of a std Python type, but with @@ -440,9 +554,13 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) CopySlot(base_, type, TypeOffset.tp_clear); CopySlot(base_, type, TypeOffset.tp_is_gc); - InitializeSlots(type, impl); + SlotsHolder slotsHolder = CreateSolotsHolder(type); + InitializeSlots(type, impl, slotsHolder); - Runtime.PyType_Ready(type); + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } IntPtr tp_dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); IntPtr mod = Runtime.PyString_FromString("CLR"); @@ -458,6 +576,10 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) internal static IntPtr AllocateTypeObject(string name) { IntPtr type = Runtime.PyType_GenericAlloc(Runtime.PyTypeType, 0); + // Clr type would not use __slots__, + // and the PyMemberDef after PyHeapTypeObject will have other uses(e.g. type handle), + // thus set the ob_size to 0 for avoiding slots iterations. + Marshal.WriteIntPtr(type, TypeOffset.ob_size, IntPtr.Zero); // Cheat a little: we'll set tp_name to the internal char * of // the Python version of the type name - otherwise we'd have to @@ -467,254 +589,28 @@ internal static IntPtr AllocateTypeObject(string name) Marshal.WriteIntPtr(type, TypeOffset.tp_name, raw); Marshal.WriteIntPtr(type, TypeOffset.name, temp); + Runtime.XIncref(temp); Marshal.WriteIntPtr(type, TypeOffset.qualname, temp); - - long ptr = type.ToInt64(); // 64-bit safe - - temp = new IntPtr(ptr + TypeOffset.nb_add); + temp = type + TypeOffset.nb_add; Marshal.WriteIntPtr(type, TypeOffset.tp_as_number, temp); - temp = new IntPtr(ptr + TypeOffset.sq_length); + temp = type + TypeOffset.sq_length; Marshal.WriteIntPtr(type, TypeOffset.tp_as_sequence, temp); - temp = new IntPtr(ptr + TypeOffset.mp_length); + temp = type + TypeOffset.mp_length; Marshal.WriteIntPtr(type, TypeOffset.tp_as_mapping, temp); - temp = new IntPtr(ptr + TypeOffset.bf_getbuffer); + temp = type + TypeOffset.bf_getbuffer; Marshal.WriteIntPtr(type, TypeOffset.tp_as_buffer, temp); return type; } - - #region Native Code Page - /// - /// Initialized by InitializeNativeCodePage. - /// - /// This points to a page of memory allocated using mmap or VirtualAlloc - /// (depending on the system), and marked read and execute (not write). - /// Very much on purpose, the page is *not* released on a shutdown and - /// is instead leaked. See the TestDomainReload test case. - /// - /// The contents of the page are two native functions: one that returns 0, - /// one that returns 1. - /// - /// If python didn't keep its gc list through a Py_Finalize we could remove - /// this entire section. - /// - internal static IntPtr NativeCodePage = IntPtr.Zero; - - /// - /// Structure to describe native code. - /// - /// Use NativeCode.Active to get the native code for the current platform. - /// - /// Generate the code by creating the following C code: - /// - /// int Return0() { return 0; } - /// int Return1() { return 1; } - /// - /// Then compiling on the target platform, e.g. with gcc or clang: - /// cc -c -fomit-frame-pointer -O2 foo.c - /// And then analyzing the resulting functions with a hex editor, e.g.: - /// objdump -disassemble foo.o - /// - internal class NativeCode - { - /// - /// The code, as a string of bytes. - /// - public byte[] Code { get; private set; } - - /// - /// Where does the "return 0" function start? - /// - public int Return0 { get; private set; } - - /// - /// Where does the "return 1" function start? - /// - public int Return1 { get; private set; } - - public static NativeCode Active - { - get - { - switch (Runtime.Machine) - { - case MachineType.i386: - return I386; - case MachineType.x86_64: - return X86_64; - default: - return null; - } - } - } - - /// - /// Code for x86_64. See the class comment for how it was generated. - /// - public static readonly NativeCode X86_64 = new NativeCode() - { - Return0 = 0x10, - Return1 = 0, - Code = new byte[] - { - // First Return1: - 0xb8, 0x01, 0x00, 0x00, 0x00, // movl $1, %eax - 0xc3, // ret - - // Now some padding so that Return0 can be 16-byte-aligned. - // I put Return1 first so there's not as much padding to type in. - 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, // nop - - // Now Return0. - 0x31, 0xc0, // xorl %eax, %eax - 0xc3, // ret - } - }; - - /// - /// Code for X86. - /// - /// It's bitwise identical to X86_64, so we just point to it. - /// - /// - public static readonly NativeCode I386 = X86_64; - } - - /// - /// Platform-dependent mmap and mprotect. - /// - internal interface IMemoryMapper - { - /// - /// Map at least numBytes of memory. Mark the page read-write (but not exec). - /// - IntPtr MapWriteable(int numBytes); - - /// - /// Sets the mapped memory to be read-exec (but not write). - /// - void SetReadExec(IntPtr mappedMemory, int numBytes); - } - - class WindowsMemoryMapper : IMemoryMapper - { - const UInt32 MEM_COMMIT = 0x1000; - const UInt32 MEM_RESERVE = 0x2000; - const UInt32 PAGE_READWRITE = 0x04; - const UInt32 PAGE_EXECUTE_READ = 0x20; - - [DllImport("kernel32.dll")] - static extern IntPtr VirtualAlloc(IntPtr lpAddress, IntPtr dwSize, UInt32 flAllocationType, UInt32 flProtect); - - [DllImport("kernel32.dll")] - static extern bool VirtualProtect(IntPtr lpAddress, IntPtr dwSize, UInt32 flNewProtect, out UInt32 lpflOldProtect); - - public IntPtr MapWriteable(int numBytes) - { - return VirtualAlloc(IntPtr.Zero, new IntPtr(numBytes), - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); - } - - public void SetReadExec(IntPtr mappedMemory, int numBytes) - { - UInt32 _; - VirtualProtect(mappedMemory, new IntPtr(numBytes), PAGE_EXECUTE_READ, out _); - } - } - - class UnixMemoryMapper : IMemoryMapper - { - const int PROT_READ = 0x1; - const int PROT_WRITE = 0x2; - const int PROT_EXEC = 0x4; - - const int MAP_PRIVATE = 0x2; - int MAP_ANONYMOUS - { - get - { - switch (Runtime.OperatingSystem) - { - case OperatingSystemType.Darwin: - return 0x1000; - case OperatingSystemType.Linux: - return 0x20; - default: - throw new NotImplementedException( - $"mmap is not supported on {Runtime.OperatingSystem}" - ); - } - } - } - - [DllImport("libc")] - static extern IntPtr mmap(IntPtr addr, IntPtr len, int prot, int flags, int fd, IntPtr offset); - - [DllImport("libc")] - static extern int mprotect(IntPtr addr, IntPtr len, int prot); - - public IntPtr MapWriteable(int numBytes) - { - // MAP_PRIVATE must be set on linux, even though MAP_ANON implies it. - // It doesn't hurt on darwin, so just do it. - return mmap(IntPtr.Zero, new IntPtr(numBytes), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, IntPtr.Zero); - } - - public void SetReadExec(IntPtr mappedMemory, int numBytes) - { - mprotect(mappedMemory, new IntPtr(numBytes), PROT_READ | PROT_EXEC); - } - } - - internal static IMemoryMapper CreateMemoryMapper() - { - switch (Runtime.OperatingSystem) - { - case OperatingSystemType.Darwin: - case OperatingSystemType.Linux: - return new UnixMemoryMapper(); - case OperatingSystemType.Windows: - return new WindowsMemoryMapper(); - default: - throw new NotImplementedException( - $"No support for {Runtime.OperatingSystem}" - ); - } - } - - /// - /// Initializes the native code page. - /// - /// Safe to call if we already initialized (this function is idempotent). - /// - /// - internal static void InitializeNativeCodePage() - { - // Do nothing if we already initialized. - if (NativeCodePage != IntPtr.Zero) - { - return; - } - - // Allocate the page, write the native code into it, then set it - // to be executable. - IMemoryMapper mapper = CreateMemoryMapper(); - int codeLength = NativeCode.Active.Code.Length; - NativeCodePage = mapper.MapWriteable(codeLength); - Marshal.Copy(NativeCode.Active.Code, 0, NativeCodePage, codeLength); - mapper.SetReadExec(NativeCodePage, codeLength); - } - #endregion - /// /// Given a newly allocated Python type object and a managed Type that /// provides the implementation for the type, connect the type slots of /// the Python object to the managed methods of the implementing Type. /// - internal static void InitializeSlots(IntPtr type, Type impl) + internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHolder = null) { // We work from the most-derived class up; make sure to get // the most-derived slot and not to override it with a base @@ -727,12 +623,7 @@ internal static void InitializeSlots(IntPtr type, Type impl) foreach (MethodInfo method in methods) { string name = method.Name; - if (!(name.StartsWith("tp_") || - name.StartsWith("nb_") || - name.StartsWith("sq_") || - name.StartsWith("mp_") || - name.StartsWith("bf_") - )) + if (!name.StartsWith("tp_") && !SlotTypes.IsSlotName(name)) { continue; } @@ -742,8 +633,7 @@ internal static void InitializeSlots(IntPtr type, Type impl) continue; } - var thunkInfo = Interop.GetThunk(method); - InitializeSlot(type, thunkInfo.Address, name); + InitializeSlot(type, Interop.GetThunk(method), name, slotsHolder); seen.Add(name); } @@ -751,40 +641,17 @@ internal static void InitializeSlots(IntPtr type, Type impl) impl = impl.BaseType; } - var native = NativeCode.Active; - - // The garbage collection related slots always have to return 1 or 0 - // since .NET objects don't take part in Python's gc: - // tp_traverse (returns 0) - // tp_clear (returns 0) - // tp_is_gc (returns 1) - // These have to be defined, though, so by default we fill these with - // static C# functions from this class. - - var ret0 = Interop.GetThunk(((Func)Return0).Method).Address; - var ret1 = Interop.GetThunk(((Func)Return1).Method).Address; - - if (native != null) + foreach (string slot in _requiredSlots) { - // If we want to support domain reload, the C# implementation - // cannot be used as the assembly may get released before - // CPython calls these functions. Instead, for amd64 and x86 we - // load them into a separate code page that is leaked - // intentionally. - InitializeNativeCodePage(); - ret1 = NativeCodePage + native.Return1; - ret0 = NativeCodePage + native.Return0; + if (seen.Contains(slot)) + { + continue; + } + var offset = ManagedDataOffsets.GetSlotOffset(slot); + Marshal.WriteIntPtr(type, offset, SlotsHolder.GetDefaultSlot(offset)); } - - InitializeSlot(type, ret0, "tp_traverse"); - InitializeSlot(type, ret0, "tp_clear"); - InitializeSlot(type, ret1, "tp_is_gc"); } - static int Return1(IntPtr _) => 1; - - static int Return0(IntPtr _) => 0; - /// /// Helper for InitializeSlots. /// @@ -795,13 +662,48 @@ internal static void InitializeSlots(IntPtr type, Type impl) /// Type being initialized. /// Function pointer. /// Name of the method. - static void InitializeSlot(IntPtr type, IntPtr slot, string name) + /// Can override the slot when it existed + static void InitializeSlot(IntPtr type, IntPtr slot, string name, bool canOverride = true) + { + var offset = ManagedDataOffsets.GetSlotOffset(name); + if (!canOverride && Marshal.ReadIntPtr(type, offset) != IntPtr.Zero) + { + return; + } + Marshal.WriteIntPtr(type, offset, slot); + } + + static void InitializeSlot(IntPtr type, ThunkInfo thunk, string name, SlotsHolder slotsHolder = null, bool canOverride = true) { Type typeOffset = typeof(TypeOffset); FieldInfo fi = typeOffset.GetField(name); var offset = (int)fi.GetValue(typeOffset); - Marshal.WriteIntPtr(type, offset, slot); + if (!canOverride && Marshal.ReadIntPtr(type, offset) != IntPtr.Zero) + { + return; + } + Marshal.WriteIntPtr(type, offset, thunk.Address); + if (slotsHolder != null) + { + slotsHolder.Set(offset, thunk); + } + } + + static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method, SlotsHolder slotsHolder = null) + { + var thunk = Interop.GetThunk(method); + Marshal.WriteIntPtr(type, slotOffset, thunk.Address); + if (slotsHolder != null) + { + slotsHolder.Set(slotOffset, thunk); + } + } + + static bool IsSlotSet(IntPtr type, string name) + { + int offset = ManagedDataOffsets.GetSlotOffset(name); + return Marshal.ReadIntPtr(type, offset) != IntPtr.Zero; } /// @@ -832,6 +734,7 @@ private static void InitMethods(IntPtr pytype, Type type) mi[0] = method; MethodObject m = new TypeMethod(type, method_name, mi); Runtime.PyDict_SetItemString(dict, method_name, m.pyHandle); + m.DecrRefCount(); addedMethods.Add(method_name); } } @@ -849,5 +752,204 @@ internal static void CopySlot(IntPtr from, IntPtr to, int offset) IntPtr fp = Marshal.ReadIntPtr(from, offset); Marshal.WriteIntPtr(to, offset, fp); } + + private static SlotsHolder CreateSolotsHolder(IntPtr type) + { + var holder = new SlotsHolder(type); + _slotsHolders.Add(type, holder); + return holder; + } + } + + + class SlotsHolder + { + public delegate void Resetor(IntPtr type, int offset); + + private readonly IntPtr _type; + private Dictionary _slots = new Dictionary(); + private List _keepalive = new List(); + private Dictionary _customResetors = new Dictionary(); + private List _deallocators = new List(); + private bool _alreadyReset = false; + + /// + /// Create slots holder for holding the delegate of slots and be able to reset them. + /// + /// Steals a reference to target type + public SlotsHolder(IntPtr type) + { + _type = type; + } + + public void Set(int offset, ThunkInfo thunk) + { + _slots[offset] = thunk; + } + + public void Set(int offset, Resetor resetor) + { + _customResetors[offset] = resetor; + } + + public void AddDealloctor(Action deallocate) + { + _deallocators.Add(deallocate); + } + + public void KeeapAlive(ThunkInfo thunk) + { + _keepalive.Add(thunk); + } + + public void ResetSlots() + { + if (_alreadyReset) + { + return; + } + _alreadyReset = true; +#if DEBUG + IntPtr tp_name = Marshal.ReadIntPtr(_type, TypeOffset.tp_name); + string typeName = Marshal.PtrToStringAnsi(tp_name); +#endif + foreach (var offset in _slots.Keys) + { + IntPtr ptr = GetDefaultSlot(offset); +#if DEBUG + //DebugUtil.Print($"Set slot<{TypeOffsetHelper.GetSlotNameByOffset(offset)}> to 0x{ptr.ToString("X")} at {typeName}<0x{_type}>"); +#endif + Marshal.WriteIntPtr(_type, offset, ptr); + } + + foreach (var action in _deallocators) + { + action(); + } + + foreach (var pair in _customResetors) + { + int offset = pair.Key; + var resetor = pair.Value; + resetor?.Invoke(_type, offset); + } + + _customResetors.Clear(); + _slots.Clear(); + _keepalive.Clear(); + _deallocators.Clear(); + + // Custom reset + IntPtr handlePtr = Marshal.ReadIntPtr(_type, TypeOffset.magic()); + if (handlePtr != IntPtr.Zero) + { + GCHandle handle = GCHandle.FromIntPtr(handlePtr); + if (handle.IsAllocated) + { + handle.Free(); + } + Marshal.WriteIntPtr(_type, TypeOffset.magic(), IntPtr.Zero); + } + } + + public static IntPtr GetDefaultSlot(int offset) + { + if (offset == TypeOffset.tp_clear) + { + return TypeManager.subtype_clear; + } + else if (offset == TypeOffset.tp_traverse) + { + return TypeManager.subtype_traverse; + } + else if (offset == TypeOffset.tp_dealloc) + { + // tp_free of PyTypeType is point to PyObejct_GC_Del. + return Marshal.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_free); + } + else if (offset == TypeOffset.tp_free) + { + // PyObject_GC_Del + return Marshal.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_free); + } + else if (offset == TypeOffset.tp_call) + { + return IntPtr.Zero; + } + else if (offset == TypeOffset.tp_new) + { + // PyType_GenericNew + return Marshal.ReadIntPtr(Runtime.PySuper_Type, TypeOffset.tp_new); + } + else if (offset == TypeOffset.tp_getattro) + { + // PyObject_GenericGetAttr + return Marshal.ReadIntPtr(Runtime.PyBaseObjectType, TypeOffset.tp_getattro); + } + else if (offset == TypeOffset.tp_setattro) + { + // PyObject_GenericSetAttr + return Marshal.ReadIntPtr(Runtime.PyBaseObjectType, TypeOffset.tp_setattro); + } + + return Marshal.ReadIntPtr(Runtime.PyTypeType, offset); + } + } + + + static class SlotHelper + { + public static IntPtr CreateObjectType() + { + IntPtr globals = Runtime.PyDict_New(); + if (Runtime.PyDict_SetItemString(globals, "__builtins__", Runtime.PyEval_GetBuiltins()) != 0) + { + Runtime.XDecref(globals); + throw new PythonException(); + } + const string code = "class A(object): pass"; + var resRef = Runtime.PyRun_String(code, RunFlagType.File, globals, globals); + IntPtr res = resRef.DangerousGetAddress(); + if (res == IntPtr.Zero) + { + try + { + throw new PythonException(); + } + finally + { + Runtime.XDecref(globals); + } + } + resRef.Dispose(); + IntPtr A = Runtime.PyDict_GetItemString(globals, "A"); + Debug.Assert(A != IntPtr.Zero); + Runtime.XIncref(A); + Runtime.XDecref(globals); + return A; + } + } + + + static partial class SlotTypes + { + private static Dictionary _nameMap = new Dictionary(); + + static SlotTypes() + { + foreach (var type in Types) + { + FieldInfo[] fields = type.GetFields(); + foreach (var fi in fields) + { + _nameMap[fi.Name] = type; + } + } + } + + public static bool IsSlotName(string name) + { + return _nameMap.ContainsKey(name); + } } } diff --git a/tools/geninterop/geninterop.py b/tools/geninterop/geninterop.py index 902296229..aacc4af65 100644 --- a/tools/geninterop/geninterop.py +++ b/tools/geninterop/geninterop.py @@ -21,6 +21,11 @@ import sysconfig import subprocess +if sys.version_info.major > 2: + from io import StringIO +else: + from StringIO import StringIO + from pycparser import c_ast, c_parser _log = logging.getLogger() @@ -55,15 +60,18 @@ def __init__(self): self.__struct_members_stack = [] self.__ptr_decl_depth = 0 self.__struct_members = {} + self.__decl_names = {} def get_struct_members(self, name): """return a list of (name, type) of struct members""" - if name in self.__typedefs: - node = self.__get_leaf_node(self.__typedefs[name]) - name = node.name - if name not in self.__struct_members: - raise Exception("Unknown struct '%s'" % name) - return self.__struct_members[name] + defs = self.__typedefs.get(name) + if defs is None: + return None + node = self.__get_leaf_node(defs) + name = node.name + if name is None: + name = defs.declname + return self.__struct_members.get(name) def visit(self, node): if isinstance(node, c_ast.FileAST): @@ -92,6 +100,7 @@ def visit_typedef(self, typedef): self.visit(typedef.type) def visit_typedecl(self, typedecl): + self.__decl_names[typedecl.type] = typedecl.declname self.visit(typedecl.type) def visit_struct(self, struct): @@ -160,7 +169,22 @@ def __get_leaf_node(self, node): return node def __get_struct_name(self, node): - return node.name or "_struct_%d" % id(node) + return node.name or self.__decl_names.get(node) or "_struct_%d" % id(node) + + +class Writer(object): + + def __init__(self): + self._stream = StringIO() + + def append(self, indent=0, code=""): + self._stream.write("%s%s\n" % (indent * " ", code)) + + def extend(self, s): + self._stream.write(s) + + def to_string(self): + return self._stream.getvalue() def preprocess_python_headers(): @@ -188,6 +212,7 @@ def preprocess_python_headers(): defines.extend([ "-D", "__inline=inline", "-D", "__ptr32=", + "-D", "__ptr64=", "-D", "__declspec(x)=", ]) @@ -211,9 +236,8 @@ def preprocess_python_headers(): return "\n".join(lines) -def gen_interop_code(members): - """Generate the TypeOffset C# class""" +def gen_interop_head(writer): defines = [ "PYTHON{0}{1}".format(PY_MAJOR, PY_MINOR) ] @@ -243,27 +267,26 @@ def gen_interop_code(members): namespace Python.Runtime { - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } +""" % (filename, defines_str) + writer.extend(class_definition) + + +def gen_interop_tail(writer): + tail = """} +#endif +""" + writer.extend(tail) + +def gen_heap_type_members(parser, writer): + """Generate the TypeOffset C# class""" + members = parser.get_struct_members("PyHeapTypeObject") + class_definition = """ + [StructLayout(LayoutKind.Sequential)] + internal static partial class TypeOffset + { // Auto-generated from PyHeapTypeObject in Python.h -""" % (filename, defines_str) +""" # All the members are sizeof(void*) so we don't need to do any # extra work to determine the size based on the type. @@ -275,11 +298,36 @@ def gen_interop_code(members): /* here are optional user slots, followed by the members. */ public static int members = 0; } -} -#endif """ - return class_definition + writer.extend(class_definition) + + +def gen_structure_code(parser, writer, type_name, indent): + members = parser.get_struct_members(type_name) + if members is None: + return False + out = writer.append + out(indent, "[StructLayout(LayoutKind.Sequential)]") + out(indent, "internal struct %s" % type_name) + out(indent, "{") + for name, tpy in members: + out(indent + 1, "public IntPtr %s;" % name) + out(indent, "}") + out() + return True + + +def gen_supported_slot_record(writer, types, indent): + out = writer.append + out(indent, "internal static partial class SlotTypes") + out(indent, "{") + out(indent + 1, "public static readonly Type[] Types = {") + for name in types: + out(indent + 2, "typeof(%s)," % name) + out(indent + 1, "};") + out(indent, "}") + out() def main(): @@ -292,10 +340,29 @@ def main(): ast_parser = AstParser() ast_parser.visit(ast) + writer = Writer() # generate the C# code - members = ast_parser.get_struct_members("PyHeapTypeObject") - interop_cs = gen_interop_code(members) + gen_interop_head(writer) + + gen_heap_type_members(ast_parser, writer) + slots_types = [ + "PyNumberMethods", + "PySequenceMethods", + "PyMappingMethods", + "PyAsyncMethods", + "PyBufferProcs", + ] + supported_types = [] + indent = 1 + for type_name in slots_types: + if not gen_structure_code(ast_parser, writer, type_name, indent): + continue + supported_types.append(type_name) + gen_supported_slot_record(writer, supported_types, indent) + + gen_interop_tail(writer) + interop_cs = writer.to_string() if len(sys.argv) > 1: with open(sys.argv[1], "w") as fh: fh.write(interop_cs)