From fc7da1de272b9112eb7c0764d335d76a4edfea0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 9 Nov 2020 15:25:28 -0500 Subject: [PATCH 01/51] Adds test cases for member changes during a domain reload The tests are marked as expected failures for now. --- src/domain_tests/App.config | 6 + .../Python.DomainReloadTests.15.csproj | 54 +++ .../Python.DomainReloadTests.csproj | 86 ++++ src/domain_tests/TestRunner.cs | 405 ++++++++++++++++++ src/domain_tests/test_domain_reload.py | 26 ++ 5 files changed, 577 insertions(+) create mode 100644 src/domain_tests/App.config create mode 100644 src/domain_tests/Python.DomainReloadTests.15.csproj create mode 100644 src/domain_tests/Python.DomainReloadTests.csproj create mode 100644 src/domain_tests/TestRunner.cs create mode 100644 src/domain_tests/test_domain_reload.py diff --git a/src/domain_tests/App.config b/src/domain_tests/App.config new file mode 100644 index 000000000..56efbc7b5 --- /dev/null +++ b/src/domain_tests/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/src/domain_tests/Python.DomainReloadTests.15.csproj b/src/domain_tests/Python.DomainReloadTests.15.csproj new file mode 100644 index 000000000..9613a95fc --- /dev/null +++ b/src/domain_tests/Python.DomainReloadTests.15.csproj @@ -0,0 +1,54 @@ + + + + + Debug + AnyCPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5} + Exe + Python.DomainReloadTests + Python.DomainReloadTests + bin\ + v4.0 + 512 + true + true + 7.3 + + + AnyCPU + true + full + false + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/domain_tests/Python.DomainReloadTests.csproj b/src/domain_tests/Python.DomainReloadTests.csproj new file mode 100644 index 000000000..0914070ce --- /dev/null +++ b/src/domain_tests/Python.DomainReloadTests.csproj @@ -0,0 +1,86 @@ + + + + + Debug + AnyCPU + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2} + Exe + Python.DomainReloadTests + Python.DomainReloadTests + v4.0 + bin\ + 512 + true + true + + + x86 + + + x64 + + + true + DEBUG;TRACE + full + + + + + true + pdbonly + + + true + DEBUG;TRACE + full + + + + + true + pdbonly + + + true + DEBUG;TRACE + full + + + + + true + pdbonly + + + true + DEBUG;TRACE + full + + + + + true + pdbonly + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/domain_tests/TestRunner.cs b/src/domain_tests/TestRunner.cs new file mode 100644 index 000000000..88ee460e4 --- /dev/null +++ b/src/domain_tests/TestRunner.cs @@ -0,0 +1,405 @@ +// We can't refer to or use Python.Runtime here. +// We want it to be loaded only inside the subdomains +using System; +using Microsoft.CSharp; +using System.CodeDom.Compiler; +using System.IO; +using System.Linq; + +namespace Python.DomainReloadTests +{ + /// + /// This class provides an executable that can run domain reload tests. + /// The setup is a bit complicated: + /// 1. pytest runs test_*.py in this directory. + /// 2. test_classname runs Python.DomainReloadTests.exe (this class) with an argument + /// 3. This class at runtime creates a directory that has both C# and + /// python code, and compiles the C#. + /// 4. This class then runs the C# code. + /// + /// But wait there's more indirection. The C# code that's run -- known as + /// the test runner -- + /// This class compiles a DLL that contains the class which code will change + /// and a runner executable that will run Python code referencing the class. + /// Each test case: + /// * Compiles some code, loads it into a domain, runs python that refers to it. + /// * Unload the domain. + /// * Compile a new piece of code, load it into a domain, run a new piece of python that accesses the code. + /// * Unload the domain. Reload the domain, run the same python again. + /// This class gets built into an executable which takes one argument: + /// which test case to run. That's because pytest assumes we'll run + /// everything in one process, but we really want a clean process on each + /// test case to test the init/reload/teardown parts of the domain reload + /// code. + /// + class TestRunner + { + const string TestAssemblyName = "DomainTests"; + + class TestCase + { + /// + /// The key to pass as an argument to choose this test. + /// + public string Name; + + /// + /// The C# code to run in the first domain. + /// + public string DotNetBefore; + + /// + /// The C# code to run in the second domain. + /// + public string DotNetAfter; + + /// + /// The Python code to run as a module that imports the C#. + /// It should have two functions: before() and after(). Before + /// will be called when DotNetBefore is loaded; after will be + /// called (twice) when DotNetAfter is loaded. + /// To make the test fail, have those functions raise exceptions. + /// + /// Make sure there's no leading spaces since Python cares. + /// + public string PythonCode; + } + + static TestCase[] Cases = new TestCase[] + { + new TestCase + { + Name = "class_rename", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Before { } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class After { } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Before + +def before_reload(): + sys.my_cls = Before + +def after_reload(): + try: + sys.my_cls.Member() + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + ", + }, + + new TestCase + { + Name = "static_member_rename", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls { public static int Before() { return 5; } } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls { public static int After() { return 10; } } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace + +def before_reload(): + sys.my_cls = TestNamespace.Cls + sys.my_fn = TestNamespace.Cls.Before + sys.my_fn() + TestNamespace.Cls.Before() + +def after_reload(): + + # We should have reloaded the class so we can access the new function. + assert 10 == sys.my_cls.After() + assert True is True + + try: + # We should have reloaded the class. The old function still exists, but is now invalid. + sys.my_cls.Before() + except TypeError: + print('Caught expected TypeError') + else: + raise AssertionError('Failed to throw exception: expected TypeError calling class member that no longer exists') + + try: + # We should have failed to reload the function which no longer exists. + sys.my_fn() + except TypeError: + print('Caught expected TypeError') + else: + raise AssertionError('Failed to throw exception: expected TypeError calling unbound .NET function that no longer exists') + ", + }, + + + new TestCase + { + Name = "member_rename", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls { public int Before() { return 5; } } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls { public int After() { return 10; } } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace + +def before_reload(): + sys.my_cls = TestNamespace.Cls() + sys.my_fn = TestNamespace.Cls().Before + sys.my_fn() + TestNamespace.Cls().Before() + +def after_reload(): + + # We should have reloaded the class so we can access the new function. + assert 10 == sys.my_cls.After() + assert True is True + + try: + # We should have reloaded the class. The old function still exists, but is now invalid. + sys.my_cls.Before() + except TypeError: + print('Caught expected TypeError') + else: + raise AssertionError('Failed to throw exception: expected TypeError calling class member that no longer exists') + + try: + # We should have failed to reload the function which no longer exists. + sys.my_fn() + except TypeError: + print('Caught expected TypeError') + else: + raise AssertionError('Failed to throw exception: expected TypeError calling unbound .NET function that no longer exists') + ", + }, + }; + + /// + /// The runner's code. Runs the python code + /// This is a template for string.Format + /// Arg 0 is the reload mode: ShutdownMode.Reload or other. + /// Arg 1 is the no-arg python function to run, before or after. + /// + const string CaseRunnerTemplate = @" +using System; +using System.IO; +using Python.Runtime; +namespace CaseRunner +{{ + class CaseRunner + {{ + public static int Main() + {{ + try + {{ + PythonEngine.Initialize(mode:{0}); + using (Py.GIL()) + {{ + // Because the generated assemblies are in the $TEMP folder, add it to the path + var temp = Path.GetTempPath(); + dynamic sys = Py.Import(""sys""); + sys.path.append(new PyString(temp)); + dynamic test_mod = Py.Import(""domain_test_module.mod""); + test_mod.{1}_reload(); + }} + PythonEngine.Shutdown(); + }} + catch (PythonException pe) + {{ + throw new ArgumentException(message:pe.Message+"" ""+pe.StackTrace); + }} + catch (Exception e) + {{ + Console.WriteLine(e.StackTrace); + throw; + }} + return 0; + }} + }} +}} +"; + readonly static string PythonDllLocation = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "../../runtime/bin/Python.Runtime.dll"); + + public static int Main(string[] args) + { + TestCase testCase; + if (args.Length < 1) + { + testCase = Cases[0]; + } + else + { + string testName = args[0]; + Console.WriteLine($"-- Looking for domain reload test case {testName}"); + testCase = Cases.First(c => c.Name == testName); + } + Console.WriteLine($"-- Running domain reload test case: {testCase.Name}"); + + var tempFolderPython = Path.Combine(Path.GetTempPath(), "Python.Runtime.dll"); + if (File.Exists(tempFolderPython)) + { + File.Delete(tempFolderPython); + } + + File.Copy(PythonDllLocation, tempFolderPython); + + CreatePythonModule(testCase); + { + var runnerAssembly = CreateCaseRunnerAssembly(verb:"before"); + CreateTestClassAssembly(testCase.DotNetBefore); + { + var runnerDomain = CreateDomain("case runner before"); + RunAndUnload(runnerDomain, runnerAssembly); + } + { + var runnerDomain = CreateDomain("case runner before (again)"); + RunAndUnload(runnerDomain, runnerAssembly); + } + } + + { + var runnerAssembly = CreateCaseRunnerAssembly(verb:"after"); + CreateTestClassAssembly(testCase.DotNetAfter); + + // Do it twice for good measure + { + var runnerDomain = CreateDomain("case runner after"); + RunAndUnload(runnerDomain, runnerAssembly); + } + { + var runnerDomain = CreateDomain("case runner after (again)"); + RunAndUnload(runnerDomain, runnerAssembly); + } + } + + return 0; + } + + static void RunAndUnload(AppDomain domain, string assemblyPath) + { + // Somehow the stack traces during execution sometimes have the wrong line numbers. + // Add some info for when debugging is required. + Console.WriteLine($"-- Running domain {domain.FriendlyName}"); + domain.ExecuteAssembly(assemblyPath); + AppDomain.Unload(domain); + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + } + + static string CreateTestClassAssembly(string code) + { + return CreateAssembly(TestAssemblyName + ".dll", code, exe: false); + } + + static string CreateCaseRunnerAssembly(string verb, string shutdownMode = "ShutdownMode.Reload") + { + var code = string.Format(CaseRunnerTemplate, shutdownMode, verb); + var name = "TestCaseRunner.exe"; + + return CreateAssembly(name, code, exe: true); + } + + static string CreateAssembly(string name, string code, bool exe = false) + { + // Never return or hold the Assembly instance. This will cause + // the assembly to be loaded into the current domain and this + // interferes with the tests. The Domain can execute fine from a + // path, so let's return that. + CSharpCodeProvider provider = new CSharpCodeProvider(); + CompilerParameters parameters = new CompilerParameters(); + parameters.GenerateExecutable = exe; + var assemblyName = name; + var assemblyFullPath = Path.Combine(Path.GetTempPath(), assemblyName); + parameters.OutputAssembly = assemblyFullPath; + parameters.ReferencedAssemblies.Add("System.dll"); + parameters.ReferencedAssemblies.Add("System.Core.dll"); + parameters.ReferencedAssemblies.Add("Microsoft.CSharp.dll"); + parameters.ReferencedAssemblies.Add(PythonDllLocation); + CompilerResults results = provider.CompileAssemblyFromSource(parameters, code); + if (results.NativeCompilerReturnValue != 0) + { + var stderr = System.Console.Error; + stderr.WriteLine($"Error in {name} compiling:\n{code}"); + foreach (var error in results.Errors) + { + stderr.WriteLine(error); + } + throw new ArgumentException("Error compiling code"); + } + + return assemblyFullPath; + } + + static AppDomain CreateDomain(string name) + { + // Create the domain. Make sure to set PrivateBinPath to a relative + // path from the CWD (namely, 'bin'). + // See https://stackoverflow.com/questions/24760543/createinstanceandunwrap-in-another-domain + var currentDomain = AppDomain.CurrentDomain; + var domainsetup = new AppDomainSetup() + { + ApplicationBase = Path.GetTempPath(), + ConfigurationFile = currentDomain.SetupInformation.ConfigurationFile, + LoaderOptimization = LoaderOptimization.SingleDomain, + PrivateBinPath = "." + }; + var domain = AppDomain.CreateDomain( + $"My Domain {name}", + currentDomain.Evidence, + domainsetup); + + return domain; + } + + static string CreatePythonModule(TestCase testCase) + { + var modulePath = Path.Combine(Path.GetTempPath(), "domain_test_module"); + if (Directory.Exists(modulePath)) + { + Directory.Delete(modulePath, recursive: true); + } + Directory.CreateDirectory(modulePath); + + File.Create(Path.Combine(modulePath, "__init__.py")).Close(); //Create and don't forget to close! + using (var writer = File.CreateText(Path.Combine(modulePath, "mod.py"))) + { + writer.Write(testCase.PythonCode); + } + + return null; + } + + } +} diff --git a/src/domain_tests/test_domain_reload.py b/src/domain_tests/test_domain_reload.py new file mode 100644 index 000000000..1b07b9b5a --- /dev/null +++ b/src/domain_tests/test_domain_reload.py @@ -0,0 +1,26 @@ +import subprocess +import os +import pytest + +def _run_test(testname): + dirname = os.path.split(__file__)[0] + exename = os.path.join(dirname, 'bin', 'Python.DomainReloadTests.exe'), + proc = subprocess.Popen([ + exename, + testname, + ]) + proc.wait() + + assert proc.returncode == 0 + +@pytest.mark.xfail(reason="Issue not yet fixed.") +def test_rename_class(): + _run_test('class_rename') + +@pytest.mark.xfail(reason="Issue not yet fixed.") +def test_rename_class_member_static_function(): + _run_test('static_member_rename') + +@pytest.mark.xfail(reason="Issue not yet fixed.") +def test_rename_class_member_function(): + _run_test('member_rename') From 72fafdd7a527ec4b5ac191879833b2e93a271aa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 9 Nov 2020 15:48:15 -0500 Subject: [PATCH 02/51] !fixup add the project to the solution files --- pythonnet.15.sln | 62 ++++++++++++++++++++++++++++++++++++++++++++++++ pythonnet.sln | 26 ++++++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/pythonnet.15.sln b/pythonnet.15.sln index ce863817f..cfe3807ae 100644 --- a/pythonnet.15.sln +++ b/pythonnet.15.sln @@ -14,6 +14,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Test.15", "src\testi EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.PerformanceTests", "src\perf_tests\Python.PerformanceTests.csproj", "{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Python.DomainReloadTests", "src\domain_tests\Python.DomainReloadTests.15.csproj", "{F2FB6DA3-318E-4F30-9A1F-932C667E38C5}" +EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Repo", "Repo", "{441A0123-F4C6-4EE4-9AEE-315FD79BE2D5}" ProjectSection(SolutionItems) = preProject .editorconfig = .editorconfig @@ -388,6 +390,66 @@ Global {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x64.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x64.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x86.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x86.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|Any CPU.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|Any CPU.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|x64.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|x64.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|x86.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|x86.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|Any CPU.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|Any CPU.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|x64.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|x64.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|x86.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|x86.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|Any CPU.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|Any CPU.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|x64.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|x64.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|x86.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|x86.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|Any CPU.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|Any CPU.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|x64.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|x64.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|x86.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|x86.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|Any CPU.Build.0 = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x64.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x64.Build.0 = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x86.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x86.Build.0 = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|Any CPU.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|Any CPU.Build.0 = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|x64.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|x64.Build.0 = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|x86.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|x86.Build.0 = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|Any CPU.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|Any CPU.Build.0 = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|x64.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|x64.Build.0 = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|x86.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|x86.Build.0 = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|Any CPU.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|Any CPU.Build.0 = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|x64.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|x64.Build.0 = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|x86.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|x86.Build.0 = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|Any CPU.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|Any CPU.Build.0 = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|x64.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|x64.Build.0 = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|x86.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/pythonnet.sln b/pythonnet.sln index c5afd66c3..7b198b336 100644 --- a/pythonnet.sln +++ b/pythonnet.sln @@ -12,6 +12,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Console", "src\console\Cons EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "clrmodule", "src\clrmodule\clrmodule.csproj", "{86E834DE-1139-4511-96CC-69636A56E7AC}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Python.DomainReloadTests", "src\domain_tests\Python.DomainReloadTests.csproj", "{7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution DebugMono|x64 = DebugMono|x64 @@ -184,6 +186,30 @@ Global {86E834DE-1139-4511-96CC-69636A56E7AC}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 {86E834DE-1139-4511-96CC-69636A56E7AC}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 {86E834DE-1139-4511-96CC-69636A56E7AC}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugMono|x64.ActiveCfg = DebugMono|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugMono|x86.ActiveCfg = DebugMono|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWin|x64.ActiveCfg = DebugWin|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWin|x64.Build.0 = DebugWin|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWin|x86.ActiveCfg = DebugWin|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWin|x86.Build.0 = DebugWin|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 20861b2e45ea2b2664b91766b265c885cf459cbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 18 Nov 2020 14:28:23 -0500 Subject: [PATCH 03/51] Add more test cases --- src/domain_tests/TestRunner.cs | 583 ++++++++++++++++++++++++- src/domain_tests/test_domain_reload.py | 47 ++ 2 files changed, 623 insertions(+), 7 deletions(-) diff --git a/src/domain_tests/TestRunner.cs b/src/domain_tests/TestRunner.cs index 88ee460e4..b518fcd32 100644 --- a/src/domain_tests/TestRunner.cs +++ b/src/domain_tests/TestRunner.cs @@ -86,14 +86,16 @@ public class After { } import clr import sys clr.AddReference('DomainTests') -from TestNamespace import Before +import TestNamespace def before_reload(): - sys.my_cls = Before + sys.my_cls = TestNamespace.Before + def after_reload(): + assert sys.my_cls is not None try: - sys.my_cls.Member() + foo = TestNamespace.Before except AttributeError: print('Caught expected exception') else: @@ -137,13 +139,15 @@ assert True is True try: # We should have reloaded the class. The old function still exists, but is now invalid. sys.my_cls.Before() - except TypeError: + except AttributeError: print('Caught expected TypeError') else: raise AssertionError('Failed to throw exception: expected TypeError calling class member that no longer exists') + assert sys.my_fn is not None + try: - # We should have failed to reload the function which no longer exists. + # Unbound functions still exist. They will error out when called though. sys.my_fn() except TypeError: print('Caught expected TypeError') @@ -189,13 +193,15 @@ assert True is True try: # We should have reloaded the class. The old function still exists, but is now invalid. sys.my_cls.Before() - except TypeError: + except AttributeError: print('Caught expected TypeError') else: raise AssertionError('Failed to throw exception: expected TypeError calling class member that no longer exists') + + assert sys.my_fn is not None try: - # We should have failed to reload the function which no longer exists. + # Unbound functions still exist. They will error out when called though. sys.my_fn() except TypeError: print('Caught expected TypeError') @@ -203,6 +209,568 @@ assert True is True raise AssertionError('Failed to throw exception: expected TypeError calling unbound .NET function that no longer exists') ", }, + + new TestCase + { + Name = "field_rename", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + static public int Before = 2; + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + static public int After = 4; + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + sys.my_int = Cls.Before + +def after_reload(): + print(sys.my_int) + try: + assert 2 == Cls.Before + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') +", + }, + new TestCase + { + Name = "property_rename", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + static public int Before { get { return 2; } } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + static public int After { get { return 4; } } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + sys.my_int = Cls.Before + +def after_reload(): + print(sys.my_int) + try: + assert 2 == Cls.Before + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') +", + }, + + new TestCase + { + Name = "event_rename", + DotNetBefore = @" + using System; + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static event Action Before; + public static void Call() + { + Before(); + } + } + }", + DotNetAfter = @" + using System; + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static event Action After; + public static void Call() + { + After(); + } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +called = False + +def callback_function(): + global called + called = True + +def before_reload(): + global called + called = False + Cls.Before += callback_function + Cls.Call() + assert called is True + +def after_reload(): + global called + assert called is True + called = False + Cls.Call() + assert called is False + #try: + # assert 2 == Cls.Before + #except TypeError: + # print('Caught expected exception') + #else: + # raise AssertionError('Failed to throw exception') +", + }, + + new TestCase + { + Name = "namespace_rename", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public int Foo; + public Cls(int i) + { + Foo = i; + } + } + }", + DotNetAfter = @" + namespace NewTestNamespace + { + [System.Serializable] + public class Cls + { + public int Foo; + public Cls(int i) + { + Foo = i; + } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace + +def before_reload(): + sys.my_cls = TestNamespace.Cls + sys.my_inst = TestNamespace.Cls(1) + +def after_reload(): + try: + TestNamespace.Cls(2) + sys.my_cls.Member() + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + ", + }, + + new TestCase + { + Name = "field_visibility_change", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static int Foo = 1; + public static int Field = 2; + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static int Foo = 1; + private static int Field = 2; + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + assert 2 == Cls.Field + assert 1 == Cls.Foo + +def after_reload(): + assert 1 == Cls.Foo + try: + assert 1 == Cls.Field + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + ", + }, + + new TestCase + { + Name = "method_visibility_change", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static int Foo() { return 1; } + public static int Function() { return 2; } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static int Foo() { return 1; } + private static int Function() { return 2; } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + sys.my_func = Cls.Function + assert 1 == Cls.Foo() + assert 2 == Cls.Function() + +def after_reload(): + assert 1 == Cls.Foo() + try: + assert 2 == Cls.Function() + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + + try: + assert 2 == sys.my_func() + except TypeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + ", + }, + + new TestCase + { + Name = "property_visibility_change", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static int Foo { get { return 1; } } + public static int Property { get { return 2; } } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static int Foo { get { return 1; } } + private static int Property { get { return 2; } } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + assert 1 == Cls.Foo + assert 2 == Cls.Property + +def after_reload(): + assert 1 == Cls.Foo + try: + assert 2 == Cls.Property + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + ", + }, + + new TestCase + { + Name = "class_visibility_change", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class PublicClass { } + + [System.Serializable] + public class Cls { } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + internal class Cls { } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace + +def before_reload(): + sys.my_cls = TestNamespace.Cls + +def after_reload(): + sys.my_cls() + + try: + TestNamespace.Cls() + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + ", + }, + + new TestCase + { + Name = "method_parameters_change", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static void MyFunction(int a) + { + System.Console.WriteLine(string.Format(""MyFunction says: {0}"", a)); + } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static void MyFunction(string a) + { + System.Console.WriteLine(string.Format(""MyFunction says: {0}"", a)); + } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + sys.my_cls = Cls + sys.my_func = Cls.MyFunction + sys.my_cls.MyFunction(1) + sys.my_func(2) + +def after_reload(): + try: + sys.my_cls.MyFunction(1) + except TypeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + + try: + sys.my_func(2) + except TypeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + + # Calling the function from the class passes + sys.my_cls.MyFunction('test') + + try: + # calling the callable directly fails + sys.my_func('test') + except TypeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + + Cls.MyFunction('another test') + + ", + }, + + new TestCase + { + Name = "method_return_type_change", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static int MyFunction() + { + return 2; + } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static string MyFunction() + { + return ""22""; + } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + sys.my_cls = Cls + sys.my_func = Cls.MyFunction + assert 2 == sys.my_cls.MyFunction() + assert 2 == sys.my_func() + +def after_reload(): + assert '22' == sys.my_cls.MyFunction() + assert '22' == sys.my_func() + assert '22' == Cls.MyFunction() + ", + }, + + new TestCase + { + Name = "field_type_change", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + static public int Field = 2; + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + static public string Field = ""22""; + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + sys.my_cls = Cls + assert 2 == sys.my_cls.Field + +def after_reload(): + assert '22' == Cls.Field + assert '22' == sys.my_cls.Field + ", + }, + + new TestCase + { + Name = "construct_removed_class", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Before { } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class After { } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace + +def before_reload(): + sys.my_cls = TestNamespace.Before + +def after_reload(): + bar = sys.my_cls() + + # Don't crash! + print(bar) + print(bar.__str__()) + print(bar.__repr__()) + ", + }, }; /// @@ -333,6 +901,7 @@ static string CreateCaseRunnerAssembly(string verb, string shutdownMode = "Shutd static string CreateAssembly(string name, string code, bool exe = false) { + // Console.WriteLine(code); // Never return or hold the Assembly instance. This will cause // the assembly to be loaded into the current domain and this // interferes with the tests. The Domain can execute fine from a diff --git a/src/domain_tests/test_domain_reload.py b/src/domain_tests/test_domain_reload.py index 1b07b9b5a..20b229e95 100644 --- a/src/domain_tests/test_domain_reload.py +++ b/src/domain_tests/test_domain_reload.py @@ -1,5 +1,6 @@ import subprocess import os + import pytest def _run_test(testname): @@ -24,3 +25,49 @@ def test_rename_class_member_static_function(): @pytest.mark.xfail(reason="Issue not yet fixed.") def test_rename_class_member_function(): _run_test('member_rename') + +@pytest.mark.xfail(reason="Issue not yet fixed.") +def test_rename_class_member_field(): + _run_test('field_rename') + +@pytest.mark.xfail(reason="Issue not yet fixed.") +def test_rename_class_member_property(): + _run_test('property_rename') + +@pytest.mark.xfail(reason="Issue not yet fixed.") +def test_rename_namespace(): + _run_test('namespace_rename') + +@pytest.mark.xfail(reason="Issue not yet fixed.") +def test_field_visibility_change(): + _run_test("field_visibility_change") + +@pytest.mark.xfail(reason="Issue not yet fixed.") +def test_method_visibility_change(): + _run_test("method_visibility_change") + +@pytest.mark.xfail(reason="Issue not yet fixed.") +def test_property_visibility_change(): + _run_test("property_visibility_change") + +@pytest.mark.xfail(reason="Issue not yet fixed.") +def test_class_visibility_change(): + _run_test("class_visibility_change") + +@pytest.mark.xfail(reason="Issue not yet fixed.") +def test_method_parameters_change(): + _run_test("method_parameters_change") + +def test_method_return_type_change(): + _run_test("method_return_type_change") + +def test_field_type_change(): + _run_test("field_type_change") + +@pytest.mark.xfail(reason="Events not yet serializable") +def test_rename_event(): + _run_test('event_rename') + +@pytest.mark.xfail(reason="newly instanced object uses PyType_GenericAlloc") +def test_construct_removed_class(): + _run_test("construct_removed_class") \ No newline at end of file From 635edac0d9969312ab96828e4046fc83deb7b3a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 19 Nov 2020 15:42:53 -0500 Subject: [PATCH 04/51] Make the tests run as netcoreapp Even though it tests nothing. --- pythonnet.15.sln | 113 ++++++++---------- .../Python.DomainReloadTests.15.csproj | 110 ++++++++++++----- src/domain_tests/TestRunner.cs | 8 +- 3 files changed, 136 insertions(+), 95 deletions(-) diff --git a/pythonnet.15.sln b/pythonnet.15.sln index cfe3807ae..4efba4495 100644 --- a/pythonnet.15.sln +++ b/pythonnet.15.sln @@ -390,66 +390,59 @@ Global {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|Any CPU.Build.0 = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x64.ActiveCfg = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x64.Build.0 = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x86.ActiveCfg = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x86.Build.0 = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|Any CPU.ActiveCfg = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|Any CPU.Build.0 = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|x64.ActiveCfg = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|x64.Build.0 = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|x86.ActiveCfg = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|x86.Build.0 = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|Any CPU.ActiveCfg = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|Any CPU.Build.0 = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|x64.ActiveCfg = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|x64.Build.0 = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|x86.ActiveCfg = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|x86.Build.0 = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|Any CPU.ActiveCfg = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|Any CPU.Build.0 = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|x64.ActiveCfg = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|x64.Build.0 = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|x86.ActiveCfg = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|x86.Build.0 = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|Any CPU.ActiveCfg = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|Any CPU.Build.0 = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|x64.ActiveCfg = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|x64.Build.0 = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|x86.ActiveCfg = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|x86.Build.0 = Debug|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|Any CPU.ActiveCfg = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|Any CPU.Build.0 = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x64.ActiveCfg = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x64.Build.0 = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x86.ActiveCfg = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x86.Build.0 = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|Any CPU.ActiveCfg = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|Any CPU.Build.0 = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|x64.ActiveCfg = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|x64.Build.0 = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|x86.ActiveCfg = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|x86.Build.0 = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|Any CPU.ActiveCfg = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|Any CPU.Build.0 = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|x64.ActiveCfg = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|x64.Build.0 = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|x86.ActiveCfg = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|x86.Build.0 = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|Any CPU.ActiveCfg = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|Any CPU.Build.0 = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|x64.ActiveCfg = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|x64.Build.0 = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|x86.ActiveCfg = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|x86.Build.0 = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|Any CPU.ActiveCfg = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|Any CPU.Build.0 = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|x64.ActiveCfg = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|x64.Build.0 = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|x86.ActiveCfg = Release|Any CPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|x86.Build.0 = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|Any CPU.ActiveCfg = ReleaseWin|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|Any CPU.Build.0 = ReleaseWin|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x64.ActiveCfg = DebugWinPY3|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x64.Build.0 = DebugWinPY3|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x86.ActiveCfg = DebugWinPY3|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x86.Build.0 = DebugWinPY3|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|Any CPU.ActiveCfg = DebugMono|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|x64.ActiveCfg = DebugMono|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|x64.Build.0 = DebugMono|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|x86.ActiveCfg = DebugMono|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|x86.Build.0 = DebugMono|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|Any CPU.ActiveCfg = DebugWin|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|x64.ActiveCfg = DebugWin|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|x64.Build.0 = DebugWin|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|x86.ActiveCfg = DebugWin|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|x86.Build.0 = DebugWin|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|Any CPU.ActiveCfg = ReleaseWin|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|Any CPU.Build.0 = ReleaseWin|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x64.ActiveCfg = ReleaseWinPY3|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x64.Build.0 = ReleaseWinPY3|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x86.ActiveCfg = ReleaseWinPY3|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x86.Build.0 = ReleaseWinPY3|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|x64.Build.0 = ReleaseMono|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|x86.Build.0 = ReleaseMono|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 + EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/domain_tests/Python.DomainReloadTests.15.csproj b/src/domain_tests/Python.DomainReloadTests.15.csproj index 9613a95fc..d8d489cbf 100644 --- a/src/domain_tests/Python.DomainReloadTests.15.csproj +++ b/src/domain_tests/Python.DomainReloadTests.15.csproj @@ -1,54 +1,98 @@ - - - + + + - Debug - AnyCPU - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5} + net40;netcoreapp3.1 + x64;x86 + DebugMono;DebugMonoPY3;ReleaseMono;ReleaseMonoPY3;DebugWin;DebugWinPY3;ReleaseWin;ReleaseWinPY3 Exe - Python.DomainReloadTests + false Python.DomainReloadTests + Python.DomainReloadTests + Python.DomainReloadTests + 2.5.0 + false + false + false + false bin\ - v4.0 - 512 - true - true + false + $(OutputPath)\$(AssemblyName).xml + $(OutputPath)\$(TargetFramework)\$(AssemblyName).xml + 1591 + ..\..\ + $(SolutionDir)\bin\ + $(OutputPath)\$(TargetFramework)_publish 7.3 + prompt + $(PYTHONNET_DEFINE_CONSTANTS) + XPLAT + $(DefineConstants);$(CustomDefineConstants);$(BaseDefineConstants); + $(DefineConstants);NETCOREAPP + $(DefineConstants);NETSTANDARD + $(DefineConstants);TRACE;DEBUG + $(NuGetPackageRoot)\microsoft.targetingpack.netframework.v4.5\1.0.1\lib\net45\ - - AnyCPU - true - full + + x86 + + + x64 + + + false - DEBUG;TRACE - prompt - 4 + full - - AnyCPU + + true pdbonly + + + true + false + full + + + true true - TRACE - prompt - 4 + portable + + + + $(DefineConstants);DEBUG;TRACE + + $(DefineConstants) + + - - - + + + - - - - - + + + - - \ No newline at end of file + + + + + $(TargetPath) + $(TargetDir)$(TargetName).pdb + + + + + + + + diff --git a/src/domain_tests/TestRunner.cs b/src/domain_tests/TestRunner.cs index b518fcd32..f8474d99b 100644 --- a/src/domain_tests/TestRunner.cs +++ b/src/domain_tests/TestRunner.cs @@ -821,6 +821,9 @@ public static int Main() public static int Main(string[] args) { +// We require this slightly convoluted way of ifdef'ing because the Python +// comments '#' are mistaken as C# preprocessior directive +#if !NETCOREAPP TestCase testCase; if (args.Length < 1) { @@ -870,9 +873,10 @@ public static int Main(string[] args) RunAndUnload(runnerDomain, runnerAssembly); } } - +#endif return 0; } +#if !NETCOREAPP static void RunAndUnload(AppDomain domain, string assemblyPath) { @@ -898,7 +902,6 @@ static string CreateCaseRunnerAssembly(string verb, string shutdownMode = "Shutd return CreateAssembly(name, code, exe: true); } - static string CreateAssembly(string name, string code, bool exe = false) { // Console.WriteLine(code); @@ -969,6 +972,7 @@ static string CreatePythonModule(TestCase testCase) return null; } +#endif } } From 0ee931eb2934b0567ff0b7744cf31209a11966f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 19 Nov 2020 15:57:41 -0500 Subject: [PATCH 05/51] Remove stray colon --- src/domain_tests/test_domain_reload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain_tests/test_domain_reload.py b/src/domain_tests/test_domain_reload.py index 20b229e95..e4d6bf6d7 100644 --- a/src/domain_tests/test_domain_reload.py +++ b/src/domain_tests/test_domain_reload.py @@ -5,7 +5,7 @@ def _run_test(testname): dirname = os.path.split(__file__)[0] - exename = os.path.join(dirname, 'bin', 'Python.DomainReloadTests.exe'), + exename = os.path.join(dirname, 'bin', 'Python.DomainReloadTests.exe') proc = subprocess.Popen([ exename, testname, From 3dad96eef4c01c01027a29899a5ea2d830e88b63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 19 Nov 2020 11:43:42 -0500 Subject: [PATCH 06/51] Rework the serialization of reflected types Serialization of System.Type, MemberInfo and MethodBase is now string based. At deserialization, use reflection to attempt to recreate the object, which may fail safely instead of throwing a SerializaitonException during the deserialization of the whoie data stream. Appropriate Exceptions will now be raised when the Maybe*'s Value property. ClasseBase objects are now de-initialized and re-initialized in Reload mode so that it's tp_dict picks up newly added members and removed members no longer linger. ModuleObject clears it's cache and remove cached members from it's tp_dict. Minor refactoring and modernization of MethodObject and MethodBinder --- src/domain_tests/test_domain_reload.py | 11 - src/runtime/Python.Runtime.csproj | 383 +++++++++++++------------ src/runtime/arrayobject.cs | 12 +- src/runtime/classbase.cs | 14 +- src/runtime/classmanager.cs | 59 +++- src/runtime/classobject.cs | 27 +- src/runtime/constructorbinder.cs | 12 +- src/runtime/constructorbinding.cs | 12 +- src/runtime/converter.cs | 2 +- src/runtime/delegateobject.cs | 7 +- src/runtime/fieldobject.cs | 19 +- src/runtime/interfaceobject.cs | 6 +- src/runtime/maybeserialize.cs | 227 +++++++++++++++ src/runtime/metatype.cs | 18 +- src/runtime/methodbinder.cs | 41 ++- src/runtime/methodbinding.cs | 16 +- src/runtime/methodobject.cs | 40 ++- src/runtime/moduleobject.cs | 11 + src/runtime/propertyobject.cs | 37 ++- src/runtime/runtime.cs | 3 +- src/runtime/runtime_data.cs | 2 +- src/runtime/typemanager.cs | 24 +- 22 files changed, 681 insertions(+), 302 deletions(-) create mode 100644 src/runtime/maybeserialize.cs diff --git a/src/domain_tests/test_domain_reload.py b/src/domain_tests/test_domain_reload.py index e4d6bf6d7..7d2ce14c7 100644 --- a/src/domain_tests/test_domain_reload.py +++ b/src/domain_tests/test_domain_reload.py @@ -14,47 +14,36 @@ def _run_test(testname): assert proc.returncode == 0 -@pytest.mark.xfail(reason="Issue not yet fixed.") def test_rename_class(): _run_test('class_rename') -@pytest.mark.xfail(reason="Issue not yet fixed.") def test_rename_class_member_static_function(): _run_test('static_member_rename') -@pytest.mark.xfail(reason="Issue not yet fixed.") def test_rename_class_member_function(): _run_test('member_rename') -@pytest.mark.xfail(reason="Issue not yet fixed.") def test_rename_class_member_field(): _run_test('field_rename') -@pytest.mark.xfail(reason="Issue not yet fixed.") def test_rename_class_member_property(): _run_test('property_rename') -@pytest.mark.xfail(reason="Issue not yet fixed.") def test_rename_namespace(): _run_test('namespace_rename') -@pytest.mark.xfail(reason="Issue not yet fixed.") def test_field_visibility_change(): _run_test("field_visibility_change") -@pytest.mark.xfail(reason="Issue not yet fixed.") def test_method_visibility_change(): _run_test("method_visibility_change") -@pytest.mark.xfail(reason="Issue not yet fixed.") def test_property_visibility_change(): _run_test("property_visibility_change") -@pytest.mark.xfail(reason="Issue not yet fixed.") def test_class_visibility_change(): _run_test("class_visibility_change") -@pytest.mark.xfail(reason="Issue not yet fixed.") def test_method_parameters_change(): _run_test("method_parameters_change") diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 418620136..3d7913dbd 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,191 +1,192 @@ - - - - Debug - AnyCPU - {097B4AC0-74E9-4C58-BCF8-C69746EC8271} - Library - Python.Runtime - Python.Runtime - bin\Python.Runtime.xml - bin\ - v4.0 - - 1591 - ..\..\ - $(SolutionDir)\bin\ - Properties - 7.3 - true - false - ..\pythonnet.snk - - - - - - PYTHON2;PYTHON27;UCS4 - true - pdbonly - - - PYTHON3;PYTHON38;UCS4 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS4;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON38;UCS4;TRACE;DEBUG - false - full - - - PYTHON2;PYTHON27;UCS2 - true - pdbonly - - - PYTHON3;PYTHON38;UCS2 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS2;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON38;UCS2;TRACE;DEBUG - false - full - - - - - - - - - - - - - - - - Properties\SharedAssemblyInfo.cs - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - clr.py - - - - - $(TargetPath) - $(TargetDir)$(TargetName).pdb - - - - - - + + + + Debug + AnyCPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271} + Library + Python.Runtime + Python.Runtime + bin\Python.Runtime.xml + bin\ + v4.0 + + 1591 + ..\..\ + $(SolutionDir)\bin\ + Properties + 7.3 + true + false + ..\pythonnet.snk + + + + + + PYTHON2;PYTHON27;UCS4 + true + pdbonly + + + PYTHON3;PYTHON38;UCS4 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS4;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON38;UCS4;TRACE;DEBUG + false + full + + + PYTHON2;PYTHON27;UCS2 + true + pdbonly + + + PYTHON3;PYTHON38;UCS2 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS2;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON38;UCS2;TRACE;DEBUG + false + full + + + + + + + + + + + + + + + + Properties\SharedAssemblyInfo.cs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + clr.py + + + + + $(TargetPath) + $(TargetDir)$(TargetName).pdb + + + + + + diff --git a/src/runtime/arrayobject.cs b/src/runtime/arrayobject.cs index 0db84dd90..071ba40e5 100644 --- a/src/runtime/arrayobject.cs +++ b/src/runtime/arrayobject.cs @@ -23,6 +23,10 @@ internal override bool CanSubclass() public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { var self = GetManagedObject(tp) as ArrayObject; + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } if (Runtime.PyTuple_Size(args) != 1) { return Exceptions.RaiseTypeError("array expects 1 argument"); @@ -30,7 +34,7 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) IntPtr op = Runtime.PyTuple_GetItem(args, 0); object result; - if (!Converter.ToManaged(op, self.type, out result, true)) + if (!Converter.ToManaged(op, self.type.Value, out result, true)) { return IntPtr.Zero; } @@ -45,8 +49,12 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { var obj = (CLRObject)GetManagedObject(ob); var arrObj = (ArrayObject)GetManagedObjectType(ob); + if (!arrObj.type.Valid) + { + return Exceptions.RaiseTypeError(arrObj.type.DeletedMessage); + } var items = obj.inst as Array; - Type itemType = arrObj.type.GetElementType(); + Type itemType = arrObj.type.Value.GetElementType(); int rank = items.Rank; int index; object value; diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index a62e76050..a5c2932f4 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -18,18 +18,21 @@ namespace Python.Runtime [Serializable] internal class ClassBase : ManagedType { + [NonSerialized] + internal List dotNetMembers; internal Indexer indexer; - internal Type type; + internal MaybeType type; internal ClassBase(Type tp) { + dotNetMembers = new List(); indexer = null; type = tp; } internal virtual bool CanSubclass() { - return !type.IsEnum; + return !type.Value.IsEnum; } @@ -44,7 +47,12 @@ public virtual IntPtr type_subscript(IntPtr idx) return Exceptions.RaiseTypeError("type(s) expected"); } - Type target = GenericUtil.GenericForType(type, types.Length); + if (!type.Valid) + { + return Exceptions.RaiseTypeError(type.DeletedMessage); + } + + Type target = GenericUtil.GenericForType(type.Value, types.Length); if (target != null) { diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index c8bed6bc4..250a2a449 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -18,7 +18,7 @@ namespace Python.Runtime /// internal class ClassManager { - private static Dictionary cache; + private static Dictionary cache; private static readonly Type dtype; private ClassManager() @@ -36,7 +36,7 @@ static ClassManager() public static void Reset() { - cache = new Dictionary(128); + cache = new Dictionary(128); } internal static void DisposePythonWrappersForClrTypes() @@ -85,26 +85,56 @@ internal static void SaveRuntimeData(RuntimeDataStorage storage) var contexts = storage.AddValue("contexts", new Dictionary()); storage.AddValue("cache", cache); - foreach (var cls in cache.Values) + foreach (var cls in cache) { + if (!cls.Key.Valid) + { + // Don't serialize an invalid class + continue; + } // This incref is for cache to hold the cls, // thus no need for decreasing it at RestoreRuntimeData. - Runtime.XIncref(cls.pyHandle); - var context = contexts[cls.pyHandle] = new InterDomainContext(); - cls.Save(context); + Runtime.XIncref(cls.Value.pyHandle); + var context = contexts[cls.Value.pyHandle] = new InterDomainContext(); + cls.Value.Save(context); + + // Remove all members added in InitBaseClass. + // this is done so that if domain reloads and a member of a + // reflected dotnet class is removed, it is removed from the + // Python object's dictionary tool; thus raising an AttributeError + // instead of a TypeError. + // Classes are re-initialized on in RestoreRuntimeData. + IntPtr dict = Marshal.ReadIntPtr(cls.Value.tpHandle, TypeOffset.tp_dict); + foreach (var member in cls.Value.dotNetMembers) + { + // No need to decref the member, the ClassBase instance does + // not own the reference. + Runtime.PyDict_DelItemString(dict, member); + } + // Trying to remove a key that's not in the dictionary may + // raise an error. We don't care about it. + Runtime.PyErr_Clear(); } } internal static Dictionary RestoreRuntimeData(RuntimeDataStorage storage) { - cache = storage.GetValue>("cache"); + var _cache = storage.GetValue>("cache"); var contexts = storage.GetValue >("contexts"); var loadedObjs = new Dictionary(); - foreach (var cls in cache.Values) + foreach (var pair in _cache) { - var context = contexts[cls.pyHandle]; - cls.Load(context); - loadedObjs.Add(cls, context); + if (!pair.Key.Valid) + { + Runtime.XDecref(pair.Value.pyHandle); + continue; + } + // re-init the class + InitClassBase(pair.Key.Value, pair.Value); + cache.Add(pair.Key, pair.Value); + var context = contexts[pair.Value.pyHandle]; + pair.Value.Load(context); + loadedObjs.Add(pair.Value, context); } return loadedObjs; } @@ -209,11 +239,16 @@ private static void InitClassBase(Type type, ClassBase impl) IntPtr dict = Marshal.ReadIntPtr(tp, TypeOffset.tp_dict); + if (impl.dotNetMembers == null) + { + impl.dotNetMembers = new List(); + } IDictionaryEnumerator iter = info.members.GetEnumerator(); while (iter.MoveNext()) { var item = (ManagedType)iter.Value; var name = (string)iter.Key; + impl.dotNetMembers.Add(name); Runtime.PyDict_SetItemString(dict, name, item.pyHandle); // Decref the item now that it's been used. item.DecrRefCount(); @@ -241,7 +276,7 @@ private static void InitClassBase(Type type, ClassBase impl) // required that the ClassObject.ctors be changed to internal if (co != null) { - if (co.ctors.Length > 0) + if (co.NumCtors > 0) { // Implement Overloads on the class object if (!CLRModule._SuppressOverloads) diff --git a/src/runtime/classobject.cs b/src/runtime/classobject.cs index 18816781f..9a19442dc 100644 --- a/src/runtime/classobject.cs +++ b/src/runtime/classobject.cs @@ -13,14 +13,14 @@ namespace Python.Runtime internal class ClassObject : ClassBase { internal ConstructorBinder binder; - internal ConstructorInfo[] ctors; + internal int NumCtors = 0; internal ClassObject(Type tp) : base(tp) { - ctors = type.GetConstructors(); - binder = new ConstructorBinder(type); - - foreach (ConstructorInfo t in ctors) + var _ctors = type.Value.GetConstructors(); + NumCtors = _ctors.Length; + binder = new ConstructorBinder(type.Value); + foreach (ConstructorInfo t in _ctors) { binder.AddMethod(t); } @@ -61,7 +61,11 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) return Exceptions.RaiseTypeError("invalid object"); } - Type type = self.type; + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } + Type type = self.type.Value; // Primitive types do not have constructors, but they look like // they do from Python. If the ClassObject represents one of the @@ -114,16 +118,21 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) /// public override IntPtr type_subscript(IntPtr idx) { + if (!type.Valid) + { + return Exceptions.RaiseTypeError(type.DeletedMessage); + } + // If this type is the Array type, the [] means we need to // construct and return an array type of the given element type. - if (type == typeof(Array)) + if (type.Value == typeof(Array)) { if (Runtime.PyTuple_Check(idx)) { return Exceptions.RaiseTypeError("type expected"); } var c = GetManagedObject(idx) as ClassBase; - Type t = c != null ? c.type : Converter.GetTypeByAlias(idx); + Type t = c != null ? c.type.Value : Converter.GetTypeByAlias(idx); if (t == null) { return Exceptions.RaiseTypeError("type expected"); @@ -143,7 +152,7 @@ public override IntPtr type_subscript(IntPtr idx) return Exceptions.RaiseTypeError("type(s) expected"); } - Type gtype = AssemblyManager.LookupType($"{type.FullName}`{types.Length}"); + Type gtype = AssemblyManager.LookupType($"{type.Value.FullName}`{types.Length}"); if (gtype != null) { var g = ClassManager.GetClass(gtype) as GenericType; diff --git a/src/runtime/constructorbinder.cs b/src/runtime/constructorbinder.cs index 0cda3a3d9..52e133983 100644 --- a/src/runtime/constructorbinder.cs +++ b/src/runtime/constructorbinder.cs @@ -14,7 +14,7 @@ namespace Python.Runtime [Serializable] internal class ConstructorBinder : MethodBinder { - private Type _containingType; + private MaybeType _containingType; internal ConstructorBinder(Type containingType) { @@ -51,10 +51,14 @@ internal object InvokeRaw(IntPtr inst, IntPtr args, IntPtr kw) /// internal object InvokeRaw(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info) { + if (!_containingType.Valid) + { + return Exceptions.RaiseTypeError(_containingType.DeletedMessage); + } object result; - if (_containingType.IsValueType && !_containingType.IsPrimitive && - !_containingType.IsEnum && _containingType != typeof(decimal) && + if (_containingType.Value.IsValueType && !_containingType.Value.IsPrimitive && + !_containingType.Value.IsEnum && _containingType.Value != typeof(decimal) && Runtime.PyTuple_Size(args) == 0) { // If you are trying to construct an instance of a struct by @@ -64,7 +68,7 @@ internal object InvokeRaw(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info) // Activator.CreateInstance(). try { - result = Activator.CreateInstance(_containingType); + result = Activator.CreateInstance(_containingType.Value); } catch (Exception e) { diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index 0c81c0a93..354ed4109 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -22,7 +22,7 @@ namespace Python.Runtime [Serializable] internal class ConstructorBinding : ExtensionType { - private Type type; // The managed Type being wrapped in a ClassObject + private MaybeType type; // The managed Type being wrapped in a ClassObject private IntPtr pyTypeHndl; // The python type tells GetInstHandle which Type to create. private ConstructorBinder ctorBinder; @@ -92,6 +92,10 @@ public static IntPtr tp_descr_get(IntPtr op, IntPtr instance, IntPtr owner) public static IntPtr mp_subscript(IntPtr op, IntPtr key) { var self = (ConstructorBinding)GetManagedObject(op); + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } Type[] types = Runtime.PythonArgsToTypeArray(key); if (types == null) @@ -100,12 +104,12 @@ public static IntPtr mp_subscript(IntPtr op, IntPtr key) } //MethodBase[] methBaseArray = self.ctorBinder.GetMethods(); //MethodBase ci = MatchSignature(methBaseArray, types); - ConstructorInfo ci = self.type.GetConstructor(types); + ConstructorInfo ci = self.type.Value.GetConstructor(types); if (ci == null) { return Exceptions.RaiseTypeError("No match found for constructor signature"); } - var boundCtor = new BoundContructor(self.type, self.pyTypeHndl, self.ctorBinder, ci); + var boundCtor = new BoundContructor(self.type.Value, self.pyTypeHndl, self.ctorBinder, ci); return boundCtor.pyHandle; } @@ -122,7 +126,7 @@ public static IntPtr tp_repr(IntPtr ob) return self.repr; } MethodBase[] methods = self.ctorBinder.GetMethods(); - string name = self.type.FullName; + string name = self.type.Value.FullName; var doc = ""; foreach (MethodBase t in methods) { diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 98fe99141..5b7998015 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -344,7 +344,7 @@ internal static bool ToManagedValue(IntPtr value, Type obType, } if (mt is ClassBase) { - result = ((ClassBase)mt).type; + result = ((ClassBase)mt).type.Value; return true; } // shouldn't happen diff --git a/src/runtime/delegateobject.cs b/src/runtime/delegateobject.cs index c5078740f..8379958f9 100644 --- a/src/runtime/delegateobject.cs +++ b/src/runtime/delegateobject.cs @@ -52,6 +52,11 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { var self = (DelegateObject)GetManagedObject(tp); + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } + if (Runtime.PyTuple_Size(args) != 1) { return Exceptions.RaiseTypeError("class takes exactly one argument"); @@ -64,7 +69,7 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) return Exceptions.RaiseTypeError("argument must be callable"); } - Delegate d = PythonEngine.DelegateManager.GetDelegate(self.type, method); + Delegate d = PythonEngine.DelegateManager.GetDelegate(self.type.Value, method); return CLRObject.GetInstHandle(d, self.pyHandle); } diff --git a/src/runtime/fieldobject.cs b/src/runtime/fieldobject.cs index 86b93dd1b..2850ac6e1 100644 --- a/src/runtime/fieldobject.cs +++ b/src/runtime/fieldobject.cs @@ -3,13 +3,14 @@ namespace Python.Runtime { + using MaybeFieldInfo = MaybeMemberInfo; /// /// Implements a Python descriptor type that provides access to CLR fields. /// [Serializable] internal class FieldObject : ExtensionType { - private FieldInfo info; + private MaybeFieldInfo info; public FieldObject(FieldInfo info) { @@ -30,8 +31,13 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) { return IntPtr.Zero; } + else if (!self.info.Valid) + { + Exceptions.SetError(Exceptions.AttributeError, self.info.DeletedMessage); + return IntPtr.Zero; + } - FieldInfo info = self.info; + FieldInfo info = self.info.Value; if (ob == IntPtr.Zero || ob == Runtime.PyNone) { @@ -85,6 +91,11 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) { return -1; } + else if (!self.info.Valid) + { + Exceptions.SetError(Exceptions.AttributeError, self.info.DeletedMessage); + return -1; + } if (val == IntPtr.Zero) { @@ -92,7 +103,7 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) return -1; } - FieldInfo info = self.info; + FieldInfo info = self.info.Value; if (info.IsLiteral || info.IsInitOnly) { @@ -147,7 +158,7 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) public static IntPtr tp_repr(IntPtr ob) { var self = (FieldObject)GetManagedObject(ob); - return Runtime.PyString_FromString($""); + return Runtime.PyString_FromString($""); } } } diff --git a/src/runtime/interfaceobject.cs b/src/runtime/interfaceobject.cs index a2fa86479..976c09be0 100644 --- a/src/runtime/interfaceobject.cs +++ b/src/runtime/interfaceobject.cs @@ -37,8 +37,12 @@ static InterfaceObject() public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { var self = (InterfaceObject)GetManagedObject(tp); + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } var nargs = Runtime.PyTuple_Size(args); - Type type = self.type; + Type type = self.type.Value; object obj; if (nargs == 1) diff --git a/src/runtime/maybeserialize.cs b/src/runtime/maybeserialize.cs new file mode 100644 index 000000000..ec59596e0 --- /dev/null +++ b/src/runtime/maybeserialize.cs @@ -0,0 +1,227 @@ +using System; +using System.Reflection; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Formatters.Binary; +using System.IO; + +namespace Python.Runtime +{ + [Serializable] + internal struct MaybeType : ISerializable + { + public static implicit operator MaybeType (Type ob) => new MaybeType(ob); + + string m_name; + Type m_type; + + public string DeletedMessage + { + get + { + return $"The .NET Type {m_name} no longer exists"; + } + } + + public Type Value + { + get + { + if (m_type == null) + { + throw new SerializationException(DeletedMessage); + } + return m_type; + } + } + + public string Name {get{return m_name;}} + public bool Valid => m_type != null; + + public override string ToString() + { + return (m_type != null ? m_type.ToString() : $"missing type: {m_name}") + Valid.ToString(); + } + + public MaybeType(Type tp) + { + m_type = tp; + m_name = tp.AssemblyQualifiedName; + } + + private MaybeType(SerializationInfo info, StreamingContext context) + { + m_name = (string)info.GetValue("n", typeof(string)); + m_type = Type.GetType(m_name, throwOnError:false); + } + + public void GetObjectData(SerializationInfo info, StreamingContext context) + { + info.AddValue("n", m_name); + } + } + + [Serializable] + internal struct MaybeMethod : ISerializable where T: MethodBase//, MethodInfo, ConstructorInfo + { + + public static implicit operator MaybeMethod (T ob) => new MaybeMethod(ob); + + string m_name; + MethodBase m_info; + // As seen in ClassManager.GetClassInfo + const BindingFlags k_flags = BindingFlags.Static | + BindingFlags.Instance | + BindingFlags.Public ; + + public T Value + { + get + { + if (m_info == null) + { + throw new SerializationException($"The .NET {typeof(T)} {m_name} no longer exists"); + } + return (T)m_info; + } + } + + public T UnsafeValue { get { return (T)m_info; } } + public string Name {get{return m_name;}} + public bool Valid => m_info != null; + + public override string ToString() + { + return (m_info != null ? m_info.ToString() : $"missing method info: {m_name}"); + } + + public MaybeMethod(T mi) + { + m_info = mi; + m_name = mi?.ToString(); + } + + internal MaybeMethod(SerializationInfo info, StreamingContext context) + { + m_name = info.GetString("s"); + m_info = null; + try + { + // Retrive the reflected type of the method; + var tp = Type.GetType(info.GetString("t")); + // Get the method's parameters types + var field_name = info.GetString("f"); + var param = (string[])info.GetValue("p", typeof(string[])); + Type[] types = new Type[param.Length]; + for (int i = 0; i < param.Length; i++) + { + types[i] = Type.GetType(param[i]); + } + // Try to get the method + m_info = tp.GetMethod(field_name, k_flags, binder:null, types:types, modifiers:null); + // Try again, may be a constructor + if (m_info == null && m_name.Contains(".ctor")) + { + m_info = tp.GetConstructor(k_flags, binder:null, types:types, modifiers:null); + } + } + catch + { + } + } + + public void GetObjectData(SerializationInfo info, StreamingContext context) + { + info.AddValue("s", m_name); + if (Valid) + { + info.AddValue("f", m_info.Name); + info.AddValue("t", m_info.ReflectedType.AssemblyQualifiedName); + var p = m_info.GetParameters(); + string[] types = new string[p.Length]; + for (int i = 0; i < p.Length; i++) + { + types[i] = p[i].ParameterType.AssemblyQualifiedName; + } + info.AddValue("p", types, typeof(string[])); + } + } + } + + [Serializable] + internal struct MaybeMemberInfo : ISerializable where T: MemberInfo + { + public static implicit operator MaybeMemberInfo (T ob) => new MaybeMemberInfo(ob); + + string m_name; + MemberInfo m_info; + + // As seen in ClassManager.GetClassInfo + const BindingFlags k_flags = BindingFlags.Static | + BindingFlags.Instance | + BindingFlags.Public ; + + public string DeletedMessage + { + get + { + return $"The .NET {typeof(T)} {m_name} no longer exists"; + } + } + + public T Value + { + get + { + if (m_info == null) + { + throw new SerializationException(DeletedMessage); + } + return (T)m_info; + } + } + + public string Name {get{return m_name;}} + public bool Valid => m_info != null; + + public override string ToString() + { + return (m_info != null ? m_info.ToString() : $"missing type: {m_name} ") + Valid.ToString(); + } + + public MaybeMemberInfo(T fi) + { + m_info = fi; + m_name = m_info?.ToString(); + } + + internal MaybeMemberInfo(SerializationInfo info, StreamingContext context) + { + // Assumption: name is always stored in "s" + m_name = info.GetString("s"); + m_info = null; + try + { + var tp = Type.GetType(info.GetString("t")); + if (tp != null) + { + var field_name = info.GetString("f"); + m_info = tp.GetField(field_name, k_flags); + } + } + catch + { + } + } + + public void GetObjectData(SerializationInfo info, StreamingContext context) + { + info.AddValue("s", m_name); + if (Valid) + { + info.AddValue("f", m_info.Name); + info.AddValue("t", m_info.ReflectedType.AssemblyQualifiedName); + } + } + } + +} diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 84abe28b9..36b406c7b 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -2,6 +2,7 @@ using System.Collections; using System.IO; using System.Runtime.InteropServices; +using System.Runtime.Serialization; namespace Python.Runtime { @@ -103,9 +104,16 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) var cb = GetManagedObject(base_type) as ClassBase; if (cb != null) { - if (!cb.CanSubclass()) + try { - return Exceptions.RaiseTypeError("delegates, enums and array types cannot be subclassed"); + if (!cb.CanSubclass()) + { + return Exceptions.RaiseTypeError("delegates, enums and array types cannot be subclassed"); + } + } + catch (SerializationException) + { + return Exceptions.RaiseTypeError($"Underlying C# Base class {cb.type} has been deleted"); } } @@ -300,7 +308,7 @@ private static IntPtr DoInstanceCheck(IntPtr tp, IntPtr args, bool checkType) { var cb = GetManagedObject(tp) as ClassBase; - if (cb == null) + if (cb == null || !cb.type.Valid) { Runtime.XIncref(Runtime.PyFalse); return Runtime.PyFalse; @@ -332,13 +340,13 @@ private static IntPtr DoInstanceCheck(IntPtr tp, IntPtr args, bool checkType) } var otherCb = GetManagedObject(otherType.Handle) as ClassBase; - if (otherCb == null) + if (otherCb == null || !otherCb.type.Valid) { Runtime.XIncref(Runtime.PyFalse); return Runtime.PyFalse; } - return Converter.ToPython(cb.type.IsAssignableFrom(otherCb.type)); + return Converter.ToPython(cb.type.Value.IsAssignableFrom(otherCb.type.Value)); } } diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index bd6eb32ba..b94cd86b5 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -7,6 +7,7 @@ namespace Python.Runtime { + using MaybeMethodBase = MaybeMethod; /// /// A MethodBinder encapsulates information about a (possibly overloaded) /// managed method, and is responsible for selecting the right method given @@ -16,19 +17,23 @@ namespace Python.Runtime [Serializable] internal class MethodBinder { - public ArrayList list; + public List list; + + [NonSerialized] public MethodBase[] methods; + + [NonSerialized] public bool init = false; public bool allow_threads = true; internal MethodBinder() { - list = new ArrayList(); + list = new List(); } internal MethodBinder(MethodInfo mi) { - list = new ArrayList { mi }; + list = new List { new MaybeMethodBase(mi) }; } public int Count @@ -164,7 +169,7 @@ internal MethodBase[] GetMethods() { // I'm sure this could be made more efficient. list.Sort(new MethodSorter()); - methods = (MethodBase[])list.ToArray(typeof(MethodBase)); + methods = (from method in list where method.Valid select method.Value).ToArray(); init = true; } return methods; @@ -180,6 +185,11 @@ internal MethodBase[] GetMethods() /// internal static int GetPrecedence(MethodBase mi) { + if (mi == null) + { + return -1; + } + ParameterInfo[] pi = mi.GetParameters(); int val = mi.IsStatic ? 3000 : 0; int num = pi.Length; @@ -740,6 +750,17 @@ protected static void AppendArgumentTypes(StringBuilder to, IntPtr args) internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, MethodInfo[] methodinfo) { + // No valid methods, nothing to bind. + if (GetMethods().Length == 0) + { + var msg = new StringBuilder("The underlying C# method(s) have been deleted"); + if (list.Count > 0 && list[0].Name != null) + { + msg.Append($": {list[0].ToString()}"); + } + return Exceptions.RaiseTypeError(msg.ToString());; + } + Binding binding = Bind(inst, args, kw, info, methodinfo); object result; IntPtr ts = IntPtr.Zero; @@ -839,12 +860,12 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i /// /// Utility class to sort method info by parameter type precedence. /// - internal class MethodSorter : IComparer + internal class MethodSorter : IComparer { - int IComparer.Compare(object m1, object m2) + int IComparer.Compare(MaybeMethodBase m1, MaybeMethodBase m2) { - var me1 = (MethodBase)m1; - var me2 = (MethodBase)m2; + MethodBase me1 = m1.Valid ? m1.Value : null; + MethodBase me2 = m2.Valid ? m2.Value : null; if (me1.DeclaringType != me2.DeclaringType) { // m2's type derives from m1's type, favor m2 @@ -856,8 +877,8 @@ int IComparer.Compare(object m1, object m2) return -1; } - int p1 = MethodBinder.GetPrecedence((MethodBase)m1); - int p2 = MethodBinder.GetPrecedence((MethodBase)m2); + int p1 = MethodBinder.GetPrecedence(me1); + int p2 = MethodBinder.GetPrecedence(me2); if (p1 < p2) { return -1; diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index 7a10fcdef..0fbdfe4b8 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -4,6 +4,7 @@ namespace Python.Runtime { + using MaybeMethodInfo = MaybeMethod; /// /// Implements a Python binding type for CLR methods. These work much like /// standard Python method bindings, but the same type is used to bind @@ -12,7 +13,7 @@ namespace Python.Runtime [Serializable] internal class MethodBinding : ExtensionType { - internal MethodInfo info; + internal MaybeMethodInfo info; internal MethodObject m; internal IntPtr target; internal IntPtr targetType; @@ -111,15 +112,15 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) // This works around a situation where the wrong generic method is picked, // for example this method in the tests: string Overloaded(int arg1, int arg2, string arg3) - if (self.info != null) + if (self.info.Valid) { - if (self.info.IsGenericMethod) + if (self.info.Value.IsGenericMethod) { var len = Runtime.PyTuple_Size(args); //FIXME: Never used Type[] sigTp = Runtime.PythonArgsToTypeArray(args, true); if (sigTp != null) { - Type[] genericTp = self.info.GetGenericArguments(); + Type[] genericTp = self.info.Value.GetGenericArguments(); MethodInfo betterMatch = MethodBinder.MatchSignatureAndParameters(self.m.info, genericTp, sigTp); if (betterMatch != null) { @@ -164,9 +165,9 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) if (inst?.inst is IPythonDerivedType) { var baseType = GetManagedObject(self.targetType) as ClassBase; - if (baseType != null) + if (baseType != null && baseType.type.Valid) { - string baseMethodName = "_" + baseType.type.Name + "__" + self.m.name; + string baseMethodName = "_" + baseType.type.Value.Name + "__" + self.m.name; IntPtr baseMethod = Runtime.PyObject_GetAttrString(target, baseMethodName); if (baseMethod != IntPtr.Zero) { @@ -184,8 +185,7 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) } } } - - return self.m.Invoke(target, args, kw, self.info); + return self.m.Invoke(target, args, kw, self.info.UnsafeValue); } finally { diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index dc23e3ce5..883e17043 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -1,8 +1,12 @@ using System; +using System.Collections.Generic; using System.Reflection; +using System.Linq; namespace Python.Runtime { + using MaybeMethodInfo = MaybeMethod; + /// /// Implements a Python type that represents a CLR method. Method objects /// support a subscript syntax [] to allow explicit overload selection. @@ -13,40 +17,46 @@ namespace Python.Runtime [Serializable] internal class MethodObject : ExtensionType { - internal MethodInfo[] info; + [NonSerialized] + private MethodInfo[] _info = null; + private readonly List infoList; internal string name; internal MethodBinding unbound; - internal MethodBinder binder; + internal readonly MethodBinder binder; internal bool is_static = false; internal IntPtr doc; internal Type type; - public MethodObject(Type type, string name, MethodInfo[] info) - { - _MethodObject(type, name, info); - } - - public MethodObject(Type type, string name, MethodInfo[] info, bool allow_threads) - { - _MethodObject(type, name, info); - binder.allow_threads = allow_threads; - } - - private void _MethodObject(Type type, string name, MethodInfo[] info) + // `allow_threads = true`: True being the default value of MethodBinder.allow_threads + public MethodObject(Type type, string name, MethodInfo[] info, bool allow_threads = true) { this.type = type; this.name = name; - this.info = info; + this.infoList = new List(); binder = new MethodBinder(); foreach (MethodInfo item in info) { + this.infoList.Add(item); binder.AddMethod(item); if (item.IsStatic) { this.is_static = true; } } + binder.allow_threads = allow_threads; + } + + internal MethodInfo[] info + { + get + { + if (_info == null) + { + _info = (from i in infoList where i.Valid select i.Value).ToArray(); + } + return _info; + } } public virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw) diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 334c5c2f3..1a1a2d73f 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -341,6 +341,17 @@ protected override void OnSave(InterDomainContext context) // Decref twice in tp_clear, equilibrate them. Runtime.XIncref(dict); Runtime.XIncref(dict); + // destroy the cache(s) + foreach (var pair in cache) + { + Runtime.PyDict_DelItemString(dict, pair.Key); + pair.Value.DecrRefCount(); + } + // Trying to remove a key that's not in the dictionary may + // raise an error. We don't care about it. + Runtime.PyErr_Clear(); + + cache.Clear(); } protected override void OnLoad(InterDomainContext context) diff --git a/src/runtime/propertyobject.cs b/src/runtime/propertyobject.cs index ac1d077f9..a5bc373a4 100644 --- a/src/runtime/propertyobject.cs +++ b/src/runtime/propertyobject.cs @@ -4,15 +4,16 @@ namespace Python.Runtime { + using MaybeMethodInfo = MaybeMethod; /// /// Implements a Python descriptor type that manages CLR properties. /// [Serializable] internal class PropertyObject : ExtensionType { - private PropertyInfo info; - private MethodInfo getter; - private MethodInfo setter; + private MaybeMemberInfo info; + private MaybeMethodInfo getter; + private MaybeMethodInfo setter; [StrongNameIdentityPermission(SecurityAction.Assert)] public PropertyObject(PropertyInfo md) @@ -31,7 +32,11 @@ public PropertyObject(PropertyInfo md) public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) { var self = (PropertyObject)GetManagedObject(ds); - MethodInfo getter = self.getter; + if (!self.info.Valid) + { + return Exceptions.RaiseTypeError(self.info.DeletedMessage); + } + MethodInfo getter = self.getter.Value; object result; @@ -51,8 +56,8 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) try { - result = self.info.GetValue(null, null); - return Converter.ToPython(result, self.info.PropertyType); + result = self.info.Value.GetValue(null, null); + return Converter.ToPython(result, self.info.Value.PropertyType); } catch (Exception e) { @@ -68,8 +73,8 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) try { - result = self.info.GetValue(co.inst, null); - return Converter.ToPython(result, self.info.PropertyType); + result = self.info.Value.GetValue(co.inst, null); + return Converter.ToPython(result, self.info.Value.PropertyType); } catch (Exception e) { @@ -91,7 +96,13 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) public new static int tp_descr_set(IntPtr ds, IntPtr ob, IntPtr val) { var self = (PropertyObject)GetManagedObject(ds); - MethodInfo setter = self.setter; + if (!self.info.Valid) + { + Exceptions.RaiseTypeError(self.info.DeletedMessage); + return -1; + } + + MethodInfo setter = self.setter.Value; object newval; if (val == IntPtr.Zero) @@ -107,7 +118,7 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) } - if (!Converter.ToManaged(val, self.info.PropertyType, out newval, true)) + if (!Converter.ToManaged(val, self.info.Value.PropertyType, out newval, true)) { return -1; } @@ -133,11 +144,11 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) Exceptions.RaiseTypeError("invalid target"); return -1; } - self.info.SetValue(co.inst, newval, null); + self.info.Value.SetValue(co.inst, newval, null); } else { - self.info.SetValue(null, newval, null); + self.info.Value.SetValue(null, newval, null); } return 0; } @@ -159,7 +170,7 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) public static IntPtr tp_repr(IntPtr ob) { var self = (PropertyObject)GetManagedObject(ob); - return Runtime.PyString_FromString($""); + return Runtime.PyString_FromString($""); } } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index a11e9002e..4592745ca 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -686,7 +686,8 @@ internal static Type[] PythonArgsToTypeArray(IntPtr arg, bool mangleObjects) if (mt is ClassBase) { - t = ((ClassBase)mt).type; + MaybeType _type = ((ClassBase)mt).type; + t = _type.Valid ? _type.Value : null; } else if (mt is CLRObject) { diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index 060573db4..f45e76db4 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -120,8 +120,8 @@ private static void RestoreRuntimeDataImpl() var objs = RestoreRuntimeDataObjects(storage.GetStorage("objs")); RestoreRuntimeDataModules(storage.GetStorage("modules")); - var clsObjs = ClassManager.RestoreRuntimeData(storage.GetStorage("classes")); TypeManager.RestoreRuntimeData(storage.GetStorage("types")); + var clsObjs = ClassManager.RestoreRuntimeData(storage.GetStorage("classes")); ImportHook.RestoreRuntimeData(storage.GetStorage("import")); PyCLRMetaType = MetaType.RestoreRuntimeData(storage.GetStorage("meta")); diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 43c160bc3..564da983d 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -20,9 +20,10 @@ internal class TypeManager internal static IntPtr subtype_clear; private const BindingFlags tbFlags = BindingFlags.Public | BindingFlags.Static; - private static Dictionary cache = new Dictionary(); + private static Dictionary cache = new Dictionary(); + private static readonly Dictionary _slotsHolders = new Dictionary(); - private static Dictionary _slotsImpls = new Dictionary(); + private static Dictionary _slotsImpls = new Dictionary(); // Slots which must be set private static readonly string[] _requiredSlots = new string[] @@ -76,11 +77,22 @@ internal static void RestoreRuntimeData(RuntimeDataStorage storage) { Debug.Assert(cache == null || cache.Count == 0); storage.GetValue("slots", out _slotsImpls); - storage.GetValue("cache", out cache); - foreach (var entry in cache) + var _cache = new Dictionary(); + storage.GetValue("cache", out _cache); + foreach (var entry in _cache) { - Type type = entry.Key; + Type type = null; + try + { + type = entry.Key.Value; + } + catch + { + Runtime.XDecref(entry.Value); + continue; + } IntPtr handle = entry.Value; + cache[type] = handle; SlotsHolder holder = CreateSolotsHolder(handle); InitializeSlots(handle, _slotsImpls[type], holder); // FIXME: mp_length_slot.CanAssgin(clrType) @@ -358,7 +370,7 @@ internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr try { Type subType = ClassDerivedObject.CreateDerivedType(name, - baseClass.type, + baseClass.type.Value, py_dict, (string)namespaceStr, (string)assembly); From 90a81f39c86b4e68308e324f8f5668c71b378ca9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Fri, 20 Nov 2020 15:37:26 -0500 Subject: [PATCH 07/51] Call PyType_Modified after modifying the type Changing a type's attribute causes problem with it's cache. Force the type to refresh itself when modifying it. --- src/runtime/classmanager.cs | 7 +++++++ src/runtime/typemanager.cs | 11 +++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 250a2a449..6d62b3941 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -114,6 +114,8 @@ internal static void SaveRuntimeData(RuntimeDataStorage storage) // Trying to remove a key that's not in the dictionary may // raise an error. We don't care about it. Runtime.PyErr_Clear(); + // We modified the Type object, notify it we did. + Runtime.PyType_Modified(cls.Value.tpHandle); } } @@ -131,6 +133,8 @@ internal static Dictionary RestoreRuntimeData(R } // re-init the class InitClassBase(pair.Key.Value, pair.Value); + // We modified the Type object, notify it we did. + Runtime.PyType_Modified(pair.Value.tpHandle); cache.Add(pair.Key, pair.Value); var context = contexts[pair.Value.pyHandle]; pair.Value.Load(context); @@ -298,6 +302,9 @@ private static void InitClassBase(Type type, ClassBase impl) } } } + // The type has been modified after PyType_Ready has been called + // Refresh the type + Runtime.PyType_Modified(tp); } private static ClassInfo GetClassInfo(Type type) diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 564da983d..3682cbb7d 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -181,6 +181,10 @@ internal static IntPtr CreateType(Type impl) Runtime.XDecref(mod); InitMethods(type, impl); + + // The type has been modified after PyType_Ready has been called + // Refresh the type + Runtime.PyType_Modified(type); return type; } @@ -476,6 +480,9 @@ internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) IntPtr mod = Runtime.PyString_FromString("CLR"); Runtime.PyDict_SetItemString(dict, "__module__", mod); + // The type has been modified after PyType_Ready has been called + // Refresh the type + Runtime.PyType_Modified(type); //DebugUtil.DumpType(type); return type; @@ -577,6 +584,10 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) IntPtr tp_dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); IntPtr mod = Runtime.PyString_FromString("CLR"); Runtime.PyDict_SetItem(tp_dict, PyIdentifier.__module__, mod); + + // The type has been modified after PyType_Ready has been called + // Refresh the type + Runtime.PyType_Modified(type); return type; } From 10276f1965bab5c11634d16e0ee2f01eeca98077 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Fri, 20 Nov 2020 15:48:20 -0500 Subject: [PATCH 08/51] Some code review changes * Revert line endings change in Python.Runtime.csproj * Split maybe serialize into respective class files * Name changes for consistency --- src/runtime/Python.Runtime.csproj | 386 +++++++++--------- .../StateSerialization/MaybeMemberInfo.cs | 85 ++++ .../StateSerialization/MaybeMethodBase.cs | 94 +++++ src/runtime/StateSerialization/MaybeType.cs | 62 +++ src/runtime/maybeserialize.cs | 227 ---------- src/runtime/methodbinder.cs | 2 +- src/runtime/methodbinding.cs | 2 +- src/runtime/methodobject.cs | 2 +- src/runtime/propertyobject.cs | 2 +- 9 files changed, 439 insertions(+), 423 deletions(-) create mode 100644 src/runtime/StateSerialization/MaybeMemberInfo.cs create mode 100644 src/runtime/StateSerialization/MaybeMethodBase.cs create mode 100644 src/runtime/StateSerialization/MaybeType.cs delete mode 100644 src/runtime/maybeserialize.cs diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 3d7913dbd..bb4f190ff 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,192 +1,194 @@ - - - - Debug - AnyCPU - {097B4AC0-74E9-4C58-BCF8-C69746EC8271} - Library - Python.Runtime - Python.Runtime - bin\Python.Runtime.xml - bin\ - v4.0 - - 1591 - ..\..\ - $(SolutionDir)\bin\ - Properties - 7.3 - true - false - ..\pythonnet.snk - - - - - - PYTHON2;PYTHON27;UCS4 - true - pdbonly - - - PYTHON3;PYTHON38;UCS4 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS4;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON38;UCS4;TRACE;DEBUG - false - full - - - PYTHON2;PYTHON27;UCS2 - true - pdbonly - - - PYTHON3;PYTHON38;UCS2 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS2;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON38;UCS2;TRACE;DEBUG - false - full - - - - - - - - - - - - - - - - Properties\SharedAssemblyInfo.cs - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - clr.py - - - - - $(TargetPath) - $(TargetDir)$(TargetName).pdb - - - - - - + + + + Debug + AnyCPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271} + Library + Python.Runtime + Python.Runtime + bin\Python.Runtime.xml + bin\ + v4.0 + + 1591 + ..\..\ + $(SolutionDir)\bin\ + Properties + 7.3 + true + false + ..\pythonnet.snk + + + + + + PYTHON2;PYTHON27;UCS4 + true + pdbonly + + + PYTHON3;PYTHON38;UCS4 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS4;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON38;UCS4;TRACE;DEBUG + false + full + + + PYTHON2;PYTHON27;UCS2 + true + pdbonly + + + PYTHON3;PYTHON38;UCS2 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS2;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON38;UCS2;TRACE;DEBUG + false + full + + + + + + + + + + + + + + + + Properties\SharedAssemblyInfo.cs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + clr.py + + + + + $(TargetPath) + $(TargetDir)$(TargetName).pdb + + + + + + diff --git a/src/runtime/StateSerialization/MaybeMemberInfo.cs b/src/runtime/StateSerialization/MaybeMemberInfo.cs new file mode 100644 index 000000000..518911de2 --- /dev/null +++ b/src/runtime/StateSerialization/MaybeMemberInfo.cs @@ -0,0 +1,85 @@ +using System; +using System.Reflection; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Formatters.Binary; +using System.IO; + +namespace Python.Runtime +{ + [Serializable] + internal struct MaybeMemberInfo : ISerializable where T: MemberInfo + { + public static implicit operator MaybeMemberInfo (T ob) => new MaybeMemberInfo(ob); + + string m_name; + MemberInfo m_info; + + // As seen in ClassManager.GetClassInfo + const BindingFlags k_flags = BindingFlags.Static | + BindingFlags.Instance | + BindingFlags.Public ; + + public string DeletedMessage + { + get + { + return $"The .NET {typeof(T)} {m_name} no longer exists"; + } + } + + public T Value + { + get + { + if (m_info == null) + { + throw new SerializationException(DeletedMessage); + } + return (T)m_info; + } + } + + public string Name {get{return m_name;}} + public bool Valid => m_info != null; + + public override string ToString() + { + return (m_info != null ? m_info.ToString() : $"missing type: {m_name}"); + } + + public MaybeMemberInfo(T fi) + { + m_info = fi; + m_name = m_info?.ToString(); + } + + internal MaybeMemberInfo(SerializationInfo info, StreamingContext context) + { + // Assumption: name is always stored in "s" + m_name = info.GetString("s"); + m_info = null; + try + { + var tp = Type.GetType(info.GetString("t")); + if (tp != null) + { + var field_name = info.GetString("f"); + m_info = tp.GetField(field_name, k_flags); + } + } + catch + { + } + } + + public void GetObjectData(SerializationInfo info, StreamingContext context) + { + info.AddValue("s", m_name); + if (Valid) + { + info.AddValue("f", m_info.Name); + info.AddValue("t", m_info.ReflectedType.AssemblyQualifiedName); + } + } + } +} diff --git a/src/runtime/StateSerialization/MaybeMethodBase.cs b/src/runtime/StateSerialization/MaybeMethodBase.cs new file mode 100644 index 000000000..359577e3b --- /dev/null +++ b/src/runtime/StateSerialization/MaybeMethodBase.cs @@ -0,0 +1,94 @@ +using System; +using System.Reflection; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Formatters.Binary; +using System.IO; + +namespace Python.Runtime +{ + [Serializable] + internal struct MaybeMethodBase : ISerializable where T: MethodBase + { + public static implicit operator MaybeMethodBase (T ob) => new MaybeMethodBase(ob); + + string m_name; + MethodBase m_info; + // As seen in ClassManager.GetClassInfo + const BindingFlags k_flags = BindingFlags.Static | + BindingFlags.Instance | + BindingFlags.Public ; + + public T Value + { + get + { + if (m_info == null) + { + throw new SerializationException($"The .NET {typeof(T)} {m_name} no longer exists"); + } + return (T)m_info; + } + } + + public T UnsafeValue { get { return (T)m_info; } } + public string Name {get{return m_name;}} + public bool Valid => m_info != null; + + public override string ToString() + { + return (m_info != null ? m_info.ToString() : $"missing method info: {m_name}"); + } + + public MaybeMethod(T mi) + { + m_info = mi; + m_name = mi?.ToString(); + } + + internal MaybeMethod(SerializationInfo info, StreamingContext context) + { + m_name = info.GetString("s"); + m_info = null; + try + { + // Retrive the reflected type of the method; + var tp = Type.GetType(info.GetString("t")); + // Get the method's parameters types + var field_name = info.GetString("f"); + var param = (string[])info.GetValue("p", typeof(string[])); + Type[] types = new Type[param.Length]; + for (int i = 0; i < param.Length; i++) + { + types[i] = Type.GetType(param[i]); + } + // Try to get the method + m_info = tp.GetMethod(field_name, k_flags, binder:null, types:types, modifiers:null); + // Try again, may be a constructor + if (m_info == null && m_name.Contains(".ctor")) + { + m_info = tp.GetConstructor(k_flags, binder:null, types:types, modifiers:null); + } + } + catch + { + } + } + + public void GetObjectData(SerializationInfo info, StreamingContext context) + { + info.AddValue("s", m_name); + if (Valid) + { + info.AddValue("f", m_info.Name); + info.AddValue("t", m_info.ReflectedType.AssemblyQualifiedName); + var p = m_info.GetParameters(); + string[] types = new string[p.Length]; + for (int i = 0; i < p.Length; i++) + { + types[i] = p[i].ParameterType.AssemblyQualifiedName; + } + info.AddValue("p", types, typeof(string[])); + } + } + } +} \ No newline at end of file diff --git a/src/runtime/StateSerialization/MaybeType.cs b/src/runtime/StateSerialization/MaybeType.cs new file mode 100644 index 000000000..f8e243e29 --- /dev/null +++ b/src/runtime/StateSerialization/MaybeType.cs @@ -0,0 +1,62 @@ +using System; +using System.Reflection; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Formatters.Binary; +using System.IO; + +namespace Python.Runtime +{ + [Serializable] + internal struct MaybeType : ISerializable + { + public static implicit operator MaybeType (Type ob) => new MaybeType(ob); + + string m_name; + Type m_type; + + public string DeletedMessage + { + get + { + return $"The .NET Type {m_name} no longer exists"; + } + } + + public Type Value + { + get + { + if (m_type == null) + { + throw new SerializationException(DeletedMessage); + } + return m_type; + } + } + + public string Name {get{return m_name;}} + public bool Valid => m_type != null; + + public override string ToString() + { + return (m_type != null ? m_type.ToString() : $"missing type: {m_name}"); + } + + public MaybeType(Type tp) + { + m_type = tp; + m_name = tp.AssemblyQualifiedName; + } + + private MaybeType(SerializationInfo info, StreamingContext context) + { + m_name = (string)info.GetValue("n", typeof(string)); + m_type = Type.GetType(m_name, throwOnError:false); + } + + public void GetObjectData(SerializationInfo info, StreamingContext context) + { + info.AddValue("n", m_name); + } + } +} \ No newline at end of file diff --git a/src/runtime/maybeserialize.cs b/src/runtime/maybeserialize.cs deleted file mode 100644 index ec59596e0..000000000 --- a/src/runtime/maybeserialize.cs +++ /dev/null @@ -1,227 +0,0 @@ -using System; -using System.Reflection; -using System.Runtime.Serialization; -using System.Runtime.Serialization.Formatters.Binary; -using System.IO; - -namespace Python.Runtime -{ - [Serializable] - internal struct MaybeType : ISerializable - { - public static implicit operator MaybeType (Type ob) => new MaybeType(ob); - - string m_name; - Type m_type; - - public string DeletedMessage - { - get - { - return $"The .NET Type {m_name} no longer exists"; - } - } - - public Type Value - { - get - { - if (m_type == null) - { - throw new SerializationException(DeletedMessage); - } - return m_type; - } - } - - public string Name {get{return m_name;}} - public bool Valid => m_type != null; - - public override string ToString() - { - return (m_type != null ? m_type.ToString() : $"missing type: {m_name}") + Valid.ToString(); - } - - public MaybeType(Type tp) - { - m_type = tp; - m_name = tp.AssemblyQualifiedName; - } - - private MaybeType(SerializationInfo info, StreamingContext context) - { - m_name = (string)info.GetValue("n", typeof(string)); - m_type = Type.GetType(m_name, throwOnError:false); - } - - public void GetObjectData(SerializationInfo info, StreamingContext context) - { - info.AddValue("n", m_name); - } - } - - [Serializable] - internal struct MaybeMethod : ISerializable where T: MethodBase//, MethodInfo, ConstructorInfo - { - - public static implicit operator MaybeMethod (T ob) => new MaybeMethod(ob); - - string m_name; - MethodBase m_info; - // As seen in ClassManager.GetClassInfo - const BindingFlags k_flags = BindingFlags.Static | - BindingFlags.Instance | - BindingFlags.Public ; - - public T Value - { - get - { - if (m_info == null) - { - throw new SerializationException($"The .NET {typeof(T)} {m_name} no longer exists"); - } - return (T)m_info; - } - } - - public T UnsafeValue { get { return (T)m_info; } } - public string Name {get{return m_name;}} - public bool Valid => m_info != null; - - public override string ToString() - { - return (m_info != null ? m_info.ToString() : $"missing method info: {m_name}"); - } - - public MaybeMethod(T mi) - { - m_info = mi; - m_name = mi?.ToString(); - } - - internal MaybeMethod(SerializationInfo info, StreamingContext context) - { - m_name = info.GetString("s"); - m_info = null; - try - { - // Retrive the reflected type of the method; - var tp = Type.GetType(info.GetString("t")); - // Get the method's parameters types - var field_name = info.GetString("f"); - var param = (string[])info.GetValue("p", typeof(string[])); - Type[] types = new Type[param.Length]; - for (int i = 0; i < param.Length; i++) - { - types[i] = Type.GetType(param[i]); - } - // Try to get the method - m_info = tp.GetMethod(field_name, k_flags, binder:null, types:types, modifiers:null); - // Try again, may be a constructor - if (m_info == null && m_name.Contains(".ctor")) - { - m_info = tp.GetConstructor(k_flags, binder:null, types:types, modifiers:null); - } - } - catch - { - } - } - - public void GetObjectData(SerializationInfo info, StreamingContext context) - { - info.AddValue("s", m_name); - if (Valid) - { - info.AddValue("f", m_info.Name); - info.AddValue("t", m_info.ReflectedType.AssemblyQualifiedName); - var p = m_info.GetParameters(); - string[] types = new string[p.Length]; - for (int i = 0; i < p.Length; i++) - { - types[i] = p[i].ParameterType.AssemblyQualifiedName; - } - info.AddValue("p", types, typeof(string[])); - } - } - } - - [Serializable] - internal struct MaybeMemberInfo : ISerializable where T: MemberInfo - { - public static implicit operator MaybeMemberInfo (T ob) => new MaybeMemberInfo(ob); - - string m_name; - MemberInfo m_info; - - // As seen in ClassManager.GetClassInfo - const BindingFlags k_flags = BindingFlags.Static | - BindingFlags.Instance | - BindingFlags.Public ; - - public string DeletedMessage - { - get - { - return $"The .NET {typeof(T)} {m_name} no longer exists"; - } - } - - public T Value - { - get - { - if (m_info == null) - { - throw new SerializationException(DeletedMessage); - } - return (T)m_info; - } - } - - public string Name {get{return m_name;}} - public bool Valid => m_info != null; - - public override string ToString() - { - return (m_info != null ? m_info.ToString() : $"missing type: {m_name} ") + Valid.ToString(); - } - - public MaybeMemberInfo(T fi) - { - m_info = fi; - m_name = m_info?.ToString(); - } - - internal MaybeMemberInfo(SerializationInfo info, StreamingContext context) - { - // Assumption: name is always stored in "s" - m_name = info.GetString("s"); - m_info = null; - try - { - var tp = Type.GetType(info.GetString("t")); - if (tp != null) - { - var field_name = info.GetString("f"); - m_info = tp.GetField(field_name, k_flags); - } - } - catch - { - } - } - - public void GetObjectData(SerializationInfo info, StreamingContext context) - { - info.AddValue("s", m_name); - if (Valid) - { - info.AddValue("f", m_info.Name); - info.AddValue("t", m_info.ReflectedType.AssemblyQualifiedName); - } - } - } - -} diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index b94cd86b5..f46bc63c1 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -7,7 +7,7 @@ namespace Python.Runtime { - using MaybeMethodBase = MaybeMethod; + using MaybeMethodBase = MaybeMethodBase; /// /// A MethodBinder encapsulates information about a (possibly overloaded) /// managed method, and is responsible for selecting the right method given diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index 0fbdfe4b8..d94bd2f92 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -4,7 +4,7 @@ namespace Python.Runtime { - using MaybeMethodInfo = MaybeMethod; + using MaybeMethodInfo = MaybeMethodBase; /// /// Implements a Python binding type for CLR methods. These work much like /// standard Python method bindings, but the same type is used to bind diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 883e17043..0c8e3d4ec 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -5,7 +5,7 @@ namespace Python.Runtime { - using MaybeMethodInfo = MaybeMethod; + using MaybeMethodInfo = MaybeMethodBase; /// /// Implements a Python type that represents a CLR method. Method objects diff --git a/src/runtime/propertyobject.cs b/src/runtime/propertyobject.cs index a5bc373a4..e01a0d877 100644 --- a/src/runtime/propertyobject.cs +++ b/src/runtime/propertyobject.cs @@ -4,7 +4,7 @@ namespace Python.Runtime { - using MaybeMethodInfo = MaybeMethod; + using MaybeMethodInfo = MaybeMethodBase; /// /// Implements a Python descriptor type that manages CLR properties. /// From 4d0e2ce76477f36e6bb16bcc3bf0a2e0867c7687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 23 Nov 2020 10:40:55 -0500 Subject: [PATCH 09/51] fixup! Some code review changes --- src/runtime/Python.Runtime.csproj | 2 +- src/runtime/StateSerialization/MaybeMethodBase.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index bb4f190ff..bf07cebea 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -162,7 +162,7 @@ - + diff --git a/src/runtime/StateSerialization/MaybeMethodBase.cs b/src/runtime/StateSerialization/MaybeMethodBase.cs index 359577e3b..2d06de2d5 100644 --- a/src/runtime/StateSerialization/MaybeMethodBase.cs +++ b/src/runtime/StateSerialization/MaybeMethodBase.cs @@ -39,13 +39,13 @@ public override string ToString() return (m_info != null ? m_info.ToString() : $"missing method info: {m_name}"); } - public MaybeMethod(T mi) + public MaybeMethodBase(T mi) { m_info = mi; m_name = mi?.ToString(); } - internal MaybeMethod(SerializationInfo info, StreamingContext context) + internal MaybeMethodBase(SerializationInfo info, StreamingContext context) { m_name = info.GetString("s"); m_info = null; From 02fa245eed021c36ab211d7969766d5c871b2c5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 23 Nov 2020 11:08:06 -0500 Subject: [PATCH 10/51] Remove hungarian notation from Maybe* types --- .../StateSerialization/MaybeMemberInfo.cs | 40 +++++++------- .../StateSerialization/MaybeMethodBase.cs | 52 +++++++++---------- src/runtime/StateSerialization/MaybeType.cs | 30 +++++------ 3 files changed, 61 insertions(+), 61 deletions(-) diff --git a/src/runtime/StateSerialization/MaybeMemberInfo.cs b/src/runtime/StateSerialization/MaybeMemberInfo.cs index 518911de2..14f6ded3c 100644 --- a/src/runtime/StateSerialization/MaybeMemberInfo.cs +++ b/src/runtime/StateSerialization/MaybeMemberInfo.cs @@ -11,8 +11,8 @@ internal struct MaybeMemberInfo : ISerializable where T: MemberInfo { public static implicit operator MaybeMemberInfo (T ob) => new MaybeMemberInfo(ob); - string m_name; - MemberInfo m_info; + string name; + MemberInfo info; // As seen in ClassManager.GetClassInfo const BindingFlags k_flags = BindingFlags.Static | @@ -23,7 +23,7 @@ public string DeletedMessage { get { - return $"The .NET {typeof(T)} {m_name} no longer exists"; + return $"The .NET {typeof(T)} {name} no longer exists"; } } @@ -31,40 +31,40 @@ public T Value { get { - if (m_info == null) + if (info == null) { throw new SerializationException(DeletedMessage); } - return (T)m_info; + return (T)info; } } - public string Name {get{return m_name;}} - public bool Valid => m_info != null; + public string Name {get{return name;}} + public bool Valid => info != null; public override string ToString() { - return (m_info != null ? m_info.ToString() : $"missing type: {m_name}"); + return (info != null ? info.ToString() : $"missing type: {name}"); } public MaybeMemberInfo(T fi) { - m_info = fi; - m_name = m_info?.ToString(); + info = fi; + name = info?.ToString(); } - internal MaybeMemberInfo(SerializationInfo info, StreamingContext context) + internal MaybeMemberInfo(SerializationInfo serializationInfo, StreamingContext context) { // Assumption: name is always stored in "s" - m_name = info.GetString("s"); - m_info = null; + name = serializationInfo.GetString("s"); + info = null; try { - var tp = Type.GetType(info.GetString("t")); + var tp = Type.GetType(serializationInfo.GetString("t")); if (tp != null) { - var field_name = info.GetString("f"); - m_info = tp.GetField(field_name, k_flags); + var field_name = serializationInfo.GetString("f"); + info = tp.GetField(field_name, k_flags); } } catch @@ -72,13 +72,13 @@ internal MaybeMemberInfo(SerializationInfo info, StreamingContext context) } } - public void GetObjectData(SerializationInfo info, StreamingContext context) + public void GetObjectData(SerializationInfo serializationInfo, StreamingContext context) { - info.AddValue("s", m_name); + serializationInfo.AddValue("s", name); if (Valid) { - info.AddValue("f", m_info.Name); - info.AddValue("t", m_info.ReflectedType.AssemblyQualifiedName); + serializationInfo.AddValue("f", info.Name); + serializationInfo.AddValue("t", info.ReflectedType.AssemblyQualifiedName); } } } diff --git a/src/runtime/StateSerialization/MaybeMethodBase.cs b/src/runtime/StateSerialization/MaybeMethodBase.cs index 2d06de2d5..3501c1c45 100644 --- a/src/runtime/StateSerialization/MaybeMethodBase.cs +++ b/src/runtime/StateSerialization/MaybeMethodBase.cs @@ -11,8 +11,8 @@ internal struct MaybeMethodBase : ISerializable where T: MethodBase { public static implicit operator MaybeMethodBase (T ob) => new MaybeMethodBase(ob); - string m_name; - MethodBase m_info; + string name; + MethodBase info; // As seen in ClassManager.GetClassInfo const BindingFlags k_flags = BindingFlags.Static | BindingFlags.Instance | @@ -22,51 +22,51 @@ public T Value { get { - if (m_info == null) + if (info == null) { - throw new SerializationException($"The .NET {typeof(T)} {m_name} no longer exists"); + throw new SerializationException($"The .NET {typeof(T)} {name} no longer exists"); } - return (T)m_info; + return (T)info; } } - public T UnsafeValue { get { return (T)m_info; } } - public string Name {get{return m_name;}} - public bool Valid => m_info != null; + public T UnsafeValue { get { return (T)info; } } + public string Name {get{return name;}} + public bool Valid => info != null; public override string ToString() { - return (m_info != null ? m_info.ToString() : $"missing method info: {m_name}"); + return (info != null ? info.ToString() : $"missing method info: {name}"); } public MaybeMethodBase(T mi) { - m_info = mi; - m_name = mi?.ToString(); + info = mi; + name = mi?.ToString(); } - internal MaybeMethodBase(SerializationInfo info, StreamingContext context) + internal MaybeMethodBase(SerializationInfo serializationInfo, StreamingContext context) { - m_name = info.GetString("s"); - m_info = null; + name = serializationInfo.GetString("s"); + info = null; try { // Retrive the reflected type of the method; - var tp = Type.GetType(info.GetString("t")); + var tp = Type.GetType(serializationInfo.GetString("t")); // Get the method's parameters types - var field_name = info.GetString("f"); - var param = (string[])info.GetValue("p", typeof(string[])); + var field_name = serializationInfo.GetString("f"); + var param = (string[])serializationInfo.GetValue("p", typeof(string[])); Type[] types = new Type[param.Length]; for (int i = 0; i < param.Length; i++) { types[i] = Type.GetType(param[i]); } // Try to get the method - m_info = tp.GetMethod(field_name, k_flags, binder:null, types:types, modifiers:null); + info = tp.GetMethod(field_name, k_flags, binder:null, types:types, modifiers:null); // Try again, may be a constructor - if (m_info == null && m_name.Contains(".ctor")) + if (info == null && name.Contains(".ctor")) { - m_info = tp.GetConstructor(k_flags, binder:null, types:types, modifiers:null); + info = tp.GetConstructor(k_flags, binder:null, types:types, modifiers:null); } } catch @@ -74,20 +74,20 @@ internal MaybeMethodBase(SerializationInfo info, StreamingContext context) } } - public void GetObjectData(SerializationInfo info, StreamingContext context) + public void GetObjectData(SerializationInfo serializationInfo, StreamingContext context) { - info.AddValue("s", m_name); + serializationInfo.AddValue("s", name); if (Valid) { - info.AddValue("f", m_info.Name); - info.AddValue("t", m_info.ReflectedType.AssemblyQualifiedName); - var p = m_info.GetParameters(); + serializationInfo.AddValue("f", info.Name); + serializationInfo.AddValue("t", info.ReflectedType.AssemblyQualifiedName); + var p = info.GetParameters(); string[] types = new string[p.Length]; for (int i = 0; i < p.Length; i++) { types[i] = p[i].ParameterType.AssemblyQualifiedName; } - info.AddValue("p", types, typeof(string[])); + serializationInfo.AddValue("p", types, typeof(string[])); } } } diff --git a/src/runtime/StateSerialization/MaybeType.cs b/src/runtime/StateSerialization/MaybeType.cs index f8e243e29..4ce5a3827 100644 --- a/src/runtime/StateSerialization/MaybeType.cs +++ b/src/runtime/StateSerialization/MaybeType.cs @@ -11,14 +11,14 @@ internal struct MaybeType : ISerializable { public static implicit operator MaybeType (Type ob) => new MaybeType(ob); - string m_name; - Type m_type; + string name; + Type type; public string DeletedMessage { get { - return $"The .NET Type {m_name} no longer exists"; + return $"The .NET Type {name} no longer exists"; } } @@ -26,37 +26,37 @@ public Type Value { get { - if (m_type == null) + if (type == null) { throw new SerializationException(DeletedMessage); } - return m_type; + return type; } } - public string Name {get{return m_name;}} - public bool Valid => m_type != null; + public string Name {get{return name;}} + public bool Valid => type != null; public override string ToString() { - return (m_type != null ? m_type.ToString() : $"missing type: {m_name}"); + return (type != null ? type.ToString() : $"missing type: {name}"); } public MaybeType(Type tp) { - m_type = tp; - m_name = tp.AssemblyQualifiedName; + type = tp; + name = tp.AssemblyQualifiedName; } - private MaybeType(SerializationInfo info, StreamingContext context) + private MaybeType(SerializationInfo serializationInfo, StreamingContext context) { - m_name = (string)info.GetValue("n", typeof(string)); - m_type = Type.GetType(m_name, throwOnError:false); + name = (string)serializationInfo.GetValue("n", typeof(string)); + type = Type.GetType(name, throwOnError:false); } - public void GetObjectData(SerializationInfo info, StreamingContext context) + public void GetObjectData(SerializationInfo serializationInfo, StreamingContext context) { - info.AddValue("n", m_name); + serializationInfo.AddValue("n", name); } } } \ No newline at end of file From 91c881c733b1d322faf933a54bd1b2b01009d146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 23 Nov 2020 11:09:10 -0500 Subject: [PATCH 11/51] Check the type of the exception before ignoring it --- src/runtime/classmanager.cs | 9 ++++++--- src/runtime/moduleobject.cs | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 6d62b3941..fef01f09a 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -110,10 +110,13 @@ internal static void SaveRuntimeData(RuntimeDataStorage storage) // No need to decref the member, the ClassBase instance does // not own the reference. Runtime.PyDict_DelItemString(dict, member); + if (Exceptions.ExceptionMatches(Exceptions.KeyError)) + { + // Trying to remove a key that's not in the dictionary + // raises an error. We don't care about it. + Runtime.PyErr_Clear(); + } } - // Trying to remove a key that's not in the dictionary may - // raise an error. We don't care about it. - Runtime.PyErr_Clear(); // We modified the Type object, notify it we did. Runtime.PyType_Modified(cls.Value.tpHandle); } diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 1a1a2d73f..def33b6c5 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -346,10 +346,13 @@ protected override void OnSave(InterDomainContext context) { Runtime.PyDict_DelItemString(dict, pair.Key); pair.Value.DecrRefCount(); + if (Exceptions.ExceptionMatches(Exceptions.KeyError)) + { + // Trying to remove a key that's not in the dictionary + // raises an error. We don't care about it. + Runtime.PyErr_Clear(); + } } - // Trying to remove a key that's not in the dictionary may - // raise an error. We don't care about it. - Runtime.PyErr_Clear(); cache.Clear(); } From 284e8e1d66e41e70c4f1730ff21be44e188bcc3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 23 Nov 2020 11:09:28 -0500 Subject: [PATCH 12/51] Check for validity, don't throw --- src/runtime/typemanager.cs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 3682cbb7d..d8b9ca70c 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -77,20 +77,15 @@ internal static void RestoreRuntimeData(RuntimeDataStorage storage) { Debug.Assert(cache == null || cache.Count == 0); storage.GetValue("slots", out _slotsImpls); - var _cache = new Dictionary(); - storage.GetValue("cache", out _cache); + storage.GetValue>("cache", out var _cache); foreach (var entry in _cache) { - Type type = null; - try - { - type = entry.Key.Value; - } - catch + if (!entry.Key.Valid) { Runtime.XDecref(entry.Value); continue; } + Type type = entry.Key.Value;; IntPtr handle = entry.Value; cache[type] = handle; SlotsHolder holder = CreateSolotsHolder(handle); From fe9678166f21019675cbc4ffb467c46267296688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 23 Nov 2020 12:18:07 -0500 Subject: [PATCH 13/51] Refactor the member binding logic of ClassManager So that we can use that same logic when deserializing Maybe* types --- .../StateSerialization/MaybeMemberInfo.cs | 34 ++++-- .../StateSerialization/MaybeMethodBase.cs | 16 +-- src/runtime/classmanager.cs | 100 +++++++++++------- 3 files changed, 97 insertions(+), 53 deletions(-) diff --git a/src/runtime/StateSerialization/MaybeMemberInfo.cs b/src/runtime/StateSerialization/MaybeMemberInfo.cs index 14f6ded3c..8e5400b39 100644 --- a/src/runtime/StateSerialization/MaybeMemberInfo.cs +++ b/src/runtime/StateSerialization/MaybeMemberInfo.cs @@ -13,11 +13,6 @@ internal struct MaybeMemberInfo : ISerializable where T: MemberInfo string name; MemberInfo info; - - // As seen in ClassManager.GetClassInfo - const BindingFlags k_flags = BindingFlags.Static | - BindingFlags.Instance | - BindingFlags.Public ; public string DeletedMessage { @@ -64,7 +59,11 @@ internal MaybeMemberInfo(SerializationInfo serializationInfo, StreamingContext c if (tp != null) { var field_name = serializationInfo.GetString("f"); - info = tp.GetField(field_name, k_flags); + MemberInfo mi = tp.GetField(field_name, ClassManager.BindingFlags); + if (mi != null && ShouldBindMember(mi)) + { + info = mi; + } } } catch @@ -72,6 +71,29 @@ internal MaybeMemberInfo(SerializationInfo serializationInfo, StreamingContext c } } + // This is complicated because we bind fields + // based on the visibility of the field, properties + // based on it's setter/getter (which is a method + // info) visibility and events based on their + // AddMethod visibility. + static bool ShouldBindMember(MemberInfo mi) + { + if (mi is PropertyInfo pi) + { + return ClassManager.ShouldBindProperty(pi); + } + else if (mi is FieldInfo fi) + { + return ClassManager.ShouldBindField(fi); + } + else if (mi is EventInfo ei) + { + return ClassManager.ShouldBindEvent(ei); + } + + return false; + } + public void GetObjectData(SerializationInfo serializationInfo, StreamingContext context) { serializationInfo.AddValue("s", name); diff --git a/src/runtime/StateSerialization/MaybeMethodBase.cs b/src/runtime/StateSerialization/MaybeMethodBase.cs index 3501c1c45..12fbb89a1 100644 --- a/src/runtime/StateSerialization/MaybeMethodBase.cs +++ b/src/runtime/StateSerialization/MaybeMethodBase.cs @@ -13,10 +13,6 @@ internal struct MaybeMethodBase : ISerializable where T: MethodBase string name; MethodBase info; - // As seen in ClassManager.GetClassInfo - const BindingFlags k_flags = BindingFlags.Static | - BindingFlags.Instance | - BindingFlags.Public ; public T Value { @@ -62,11 +58,17 @@ internal MaybeMethodBase(SerializationInfo serializationInfo, StreamingContext c types[i] = Type.GetType(param[i]); } // Try to get the method - info = tp.GetMethod(field_name, k_flags, binder:null, types:types, modifiers:null); + MethodBase mb = tp.GetMethod(field_name, ClassManager.BindingFlags, binder:null, types:types, modifiers:null); // Try again, may be a constructor - if (info == null && name.Contains(".ctor")) + if (mb == null && name.Contains(".ctor")) { - info = tp.GetConstructor(k_flags, binder:null, types:types, modifiers:null); + mb = tp.GetConstructor(ClassManager.BindingFlags, binder:null, types:types, modifiers:null); + } + + // Do like in ClassManager.GetClassInfo + if(mb != null && ClassManager.ShouldBindMethod(mb)) + { + info = mb; } } catch diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index fef01f09a..9da40627c 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -18,6 +18,19 @@ namespace Python.Runtime /// internal class ClassManager { + + // Binding flags to determine which members to expose in Python. + // This is complicated because inheritance in Python is name + // based. We can't just find DeclaredOnly members, because we + // could have a base class A that defines two overloads of a + // method and a class B that defines two more. The name-based + // descriptor Python will find needs to know about inherited + // overloads as well as those declared on the sub class. + internal static readonly BindingFlags BindingFlags = BindingFlags.Static | + BindingFlags.Instance | + BindingFlags.Public | + BindingFlags.NonPublic; + private static Dictionary cache; private static readonly Type dtype; @@ -310,6 +323,47 @@ private static void InitClassBase(Type type, ClassBase impl) Runtime.PyType_Modified(tp); } + internal static bool ShouldBindMethod(MethodBase mb) + { + return (mb.IsPublic || mb.IsFamily || mb.IsFamilyOrAssembly); + } + + internal static bool ShouldBindField(FieldInfo fi) + { + return (fi.IsPublic || fi.IsFamily || fi.IsFamilyOrAssembly); + } + + internal static bool ShouldBindProperty(PropertyInfo pi) + { + MethodInfo mm = null; + try + { + mm = pi.GetGetMethod(true); + if (mm == null) + { + mm = pi.GetSetMethod(true); + } + } + catch (SecurityException) + { + // GetGetMethod may try to get a method protected by + // StrongNameIdentityPermission - effectively private. + return false; + } + + if (mm == null) + { + return false; + } + + return ShouldBindMethod(mm); + } + + internal static bool ShouldBindEvent(EventInfo ei) + { + return ShouldBindMethod(ei.GetAddMethod(true)); + } + private static ClassInfo GetClassInfo(Type type) { var ci = new ClassInfo(); @@ -322,18 +376,7 @@ private static ClassInfo GetClassInfo(Type type) Type tp; int i, n; - // This is complicated because inheritance in Python is name - // based. We can't just find DeclaredOnly members, because we - // could have a base class A that defines two overloads of a - // method and a class B that defines two more. The name-based - // descriptor Python will find needs to know about inherited - // overloads as well as those declared on the sub class. - BindingFlags flags = BindingFlags.Static | - BindingFlags.Instance | - BindingFlags.Public | - BindingFlags.NonPublic; - - MemberInfo[] info = type.GetMembers(flags); + MemberInfo[] info = type.GetMembers(BindingFlags); var local = new Hashtable(); var items = new ArrayList(); MemberInfo m; @@ -376,7 +419,7 @@ private static ClassInfo GetClassInfo(Type type) for (i = 0; i < inheritedInterfaces.Length; ++i) { Type inheritedType = inheritedInterfaces[i]; - MemberInfo[] imembers = inheritedType.GetMembers(flags); + MemberInfo[] imembers = inheritedType.GetMembers(BindingFlags); for (n = 0; n < imembers.Length; n++) { m = imembers[n]; @@ -407,8 +450,7 @@ private static ClassInfo GetClassInfo(Type type) { case MemberTypes.Method: meth = (MethodInfo)mi; - if (!(meth.IsPublic || meth.IsFamily || - meth.IsFamilyOrAssembly)) + if (!ShouldBindMethod(meth)) { continue; } @@ -425,28 +467,7 @@ private static ClassInfo GetClassInfo(Type type) case MemberTypes.Property: var pi = (PropertyInfo)mi; - MethodInfo mm = null; - try - { - mm = pi.GetGetMethod(true); - if (mm == null) - { - mm = pi.GetSetMethod(true); - } - } - catch (SecurityException) - { - // GetGetMethod may try to get a method protected by - // StrongNameIdentityPermission - effectively private. - continue; - } - - if (mm == null) - { - continue; - } - - if (!(mm.IsPublic || mm.IsFamily || mm.IsFamilyOrAssembly)) + if(!ShouldBindProperty(pi)) { continue; } @@ -471,7 +492,7 @@ private static ClassInfo GetClassInfo(Type type) case MemberTypes.Field: var fi = (FieldInfo)mi; - if (!(fi.IsPublic || fi.IsFamily || fi.IsFamilyOrAssembly)) + if (!ShouldBindField(fi)) { continue; } @@ -481,8 +502,7 @@ private static ClassInfo GetClassInfo(Type type) case MemberTypes.Event: var ei = (EventInfo)mi; - MethodInfo me = ei.GetAddMethod(true); - if (!(me.IsPublic || me.IsFamily || me.IsFamilyOrAssembly)) + if (!ShouldBindEvent(ei)) { continue; } From 6ff9e0b6c73036b352f0a46395da8e0ae4a4ade8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 23 Nov 2020 13:26:11 -0500 Subject: [PATCH 14/51] Include info about why deserialization failed in Maybe* --- .../StateSerialization/MaybeMemberInfo.cs | 12 +++++++++--- .../StateSerialization/MaybeMethodBase.cs | 18 ++++++++++++++++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/runtime/StateSerialization/MaybeMemberInfo.cs b/src/runtime/StateSerialization/MaybeMemberInfo.cs index 8e5400b39..610bc967d 100644 --- a/src/runtime/StateSerialization/MaybeMemberInfo.cs +++ b/src/runtime/StateSerialization/MaybeMemberInfo.cs @@ -14,11 +14,14 @@ internal struct MaybeMemberInfo : ISerializable where T: MemberInfo string name; MemberInfo info; + [NonSerialized] + Exception deserializationException; + public string DeletedMessage { get { - return $"The .NET {typeof(T)} {name} no longer exists"; + return $"The .NET {typeof(T)} {name} no longer exists. Cause: " + deserializationException?.Message ; } } @@ -28,7 +31,7 @@ public T Value { if (info == null) { - throw new SerializationException(DeletedMessage); + throw new SerializationException(DeletedMessage, innerException: deserializationException); } return (T)info; } @@ -46,6 +49,7 @@ public MaybeMemberInfo(T fi) { info = fi; name = info?.ToString(); + deserializationException = null; } internal MaybeMemberInfo(SerializationInfo serializationInfo, StreamingContext context) @@ -53,6 +57,7 @@ internal MaybeMemberInfo(SerializationInfo serializationInfo, StreamingContext c // Assumption: name is always stored in "s" name = serializationInfo.GetString("s"); info = null; + deserializationException = null; try { var tp = Type.GetType(serializationInfo.GetString("t")); @@ -66,8 +71,9 @@ internal MaybeMemberInfo(SerializationInfo serializationInfo, StreamingContext c } } } - catch + catch (Exception e) { + deserializationException = e; } } diff --git a/src/runtime/StateSerialization/MaybeMethodBase.cs b/src/runtime/StateSerialization/MaybeMethodBase.cs index 12fbb89a1..7e13129e6 100644 --- a/src/runtime/StateSerialization/MaybeMethodBase.cs +++ b/src/runtime/StateSerialization/MaybeMethodBase.cs @@ -14,13 +14,24 @@ internal struct MaybeMethodBase : ISerializable where T: MethodBase string name; MethodBase info; + [NonSerialized] + Exception deserializationException; + + public string DeletedMessage + { + get + { + return $"The .NET {typeof(T)} {name} no longer exists. Cause: " + deserializationException?.Message ; + } + } + public T Value { get { if (info == null) { - throw new SerializationException($"The .NET {typeof(T)} {name} no longer exists"); + throw new SerializationException(DeletedMessage, innerException: deserializationException); } return (T)info; } @@ -39,12 +50,14 @@ public MaybeMethodBase(T mi) { info = mi; name = mi?.ToString(); + deserializationException = null; } internal MaybeMethodBase(SerializationInfo serializationInfo, StreamingContext context) { name = serializationInfo.GetString("s"); info = null; + deserializationException = null; try { // Retrive the reflected type of the method; @@ -71,8 +84,9 @@ internal MaybeMethodBase(SerializationInfo serializationInfo, StreamingContext c info = mb; } } - catch + catch (Exception e) { + deserializationException = e; } } From 4d2d05b34b6890c96b661dad89ca6824b73902d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Tue, 24 Nov 2020 13:14:19 -0500 Subject: [PATCH 15/51] improve deserialization resolution of ref, out and in parameters Also add a test --- src/domain_tests/TestRunner.cs | 78 +++++++++++++++++++ src/domain_tests/test_domain_reload.py | 5 +- .../StateSerialization/MaybeMethodBase.cs | 77 +++++++++++++++--- src/runtime/methodbinder.cs | 13 ++++ 4 files changed, 160 insertions(+), 13 deletions(-) diff --git a/src/domain_tests/TestRunner.cs b/src/domain_tests/TestRunner.cs index f8474d99b..51ec73fcb 100644 --- a/src/domain_tests/TestRunner.cs +++ b/src/domain_tests/TestRunner.cs @@ -771,6 +771,84 @@ def after_reload(): print(bar.__repr__()) ", }, + + new TestCase + { + Name = "out_to_ref_param", + DotNetBefore = @" + namespace TestNamespace + { + + [System.Serializable] + public class Data + { + public int num = -1; + } + + [System.Serializable] + public class Cls + { + public static void MyFn (out Data a) + { + a = new Data(); + a.num = 9001; + } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + + [System.Serializable] + public class Data + { + public int num = -1; + } + + [System.Serializable] + public class Cls + { + public static void MyFn (ref Data a) + { + a.num = 7; + } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace +import System + +def before_reload(): + + foo = TestNamespace.Data() + bar = TestNamespace.Cls.MyFn(foo) + assert bar.num == 9001 + # foo shouldn't have changed. + assert foo.num == -1 + + +def after_reload(): + + try: + # Now that the function takes a ref type, we must pass a valid object. + bar = TestNamespace.Cls.MyFn(None) + except System.NullReferenceException as e: + print('caught expected exception') + else: + raise AssertionError('failed to raise') + + foo = TestNamespace.Data() + bar = TestNamespace.Cls.MyFn(foo) + # foo should have changed + assert foo.num == 7 + assert bar.num == 7 + # Pythonnet also returns a new object with `ref`-quialified parameters + assert foo is not bar + ", + }, }; /// diff --git a/src/domain_tests/test_domain_reload.py b/src/domain_tests/test_domain_reload.py index 7d2ce14c7..cca3e0232 100644 --- a/src/domain_tests/test_domain_reload.py +++ b/src/domain_tests/test_domain_reload.py @@ -59,4 +59,7 @@ def test_rename_event(): @pytest.mark.xfail(reason="newly instanced object uses PyType_GenericAlloc") def test_construct_removed_class(): - _run_test("construct_removed_class") \ No newline at end of file + _run_test("construct_removed_class") + +def test_out_to_ref_param(): + _run_test("out_to_ref_param") \ No newline at end of file diff --git a/src/runtime/StateSerialization/MaybeMethodBase.cs b/src/runtime/StateSerialization/MaybeMethodBase.cs index 7e13129e6..3f8e0ff72 100644 --- a/src/runtime/StateSerialization/MaybeMethodBase.cs +++ b/src/runtime/StateSerialization/MaybeMethodBase.cs @@ -1,14 +1,50 @@ using System; using System.Reflection; using System.Runtime.Serialization; -using System.Runtime.Serialization.Formatters.Binary; -using System.IO; +using System.Linq; namespace Python.Runtime { [Serializable] internal struct MaybeMethodBase : ISerializable where T: MethodBase { + [Serializable] + struct ParameterHelper : IEquatable + { + public enum TypeModifier + { + None, + In, + Out, + Ref + } + public readonly string Name; + public readonly TypeModifier Modifier; + + public ParameterHelper(ParameterInfo tp) + { + Name = tp.ParameterType.AssemblyQualifiedName; + Modifier = TypeModifier.None; + + if (tp.IsIn) + { + Modifier = TypeModifier.In; + } + else if (tp.IsOut) + { + Modifier = TypeModifier.Out; + } + else if (tp.ParameterType.IsByRef) + { + Modifier = TypeModifier.Ref; + } + } + + public bool Equals(ParameterInfo other) + { + return this.Equals(new ParameterHelper(other)); + } + } public static implicit operator MaybeMethodBase (T ob) => new MaybeMethodBase(ob); string name; @@ -64,11 +100,11 @@ internal MaybeMethodBase(SerializationInfo serializationInfo, StreamingContext c var tp = Type.GetType(serializationInfo.GetString("t")); // Get the method's parameters types var field_name = serializationInfo.GetString("f"); - var param = (string[])serializationInfo.GetValue("p", typeof(string[])); + var param = (ParameterHelper[])serializationInfo.GetValue("p", typeof(ParameterHelper[])); Type[] types = new Type[param.Length]; for (int i = 0; i < param.Length; i++) { - types[i] = Type.GetType(param[i]); + types[i] = Type.GetType(param[i].Name); } // Try to get the method MethodBase mb = tp.GetMethod(field_name, ClassManager.BindingFlags, binder:null, types:types, modifiers:null); @@ -81,7 +117,29 @@ internal MaybeMethodBase(SerializationInfo serializationInfo, StreamingContext c // Do like in ClassManager.GetClassInfo if(mb != null && ClassManager.ShouldBindMethod(mb)) { - info = mb; + // One more step: Changing: + // void MyFn (ref int a) + // to: + // void MyFn (out int a) + // will still find the fucntion correctly as, `in`, `out` and `ref` + // are all represented as a reference type. Query the method we got + // and validate the parameters + bool matches = true; + if (param.Length != 0) + { + foreach (var item in Enumerable.Zip(param, mb.GetParameters(), (x, y) => new {x, y})) + { + if (!item.x.Equals(item.y)) + { + matches = false; + break; + } + } + } + if (matches) + { + info = mb; + } } } catch (Exception e) @@ -97,13 +155,8 @@ public void GetObjectData(SerializationInfo serializationInfo, StreamingContext { serializationInfo.AddValue("f", info.Name); serializationInfo.AddValue("t", info.ReflectedType.AssemblyQualifiedName); - var p = info.GetParameters(); - string[] types = new string[p.Length]; - for (int i = 0; i < p.Length; i++) - { - types[i] = p[i].ParameterType.AssemblyQualifiedName; - } - serializationInfo.AddValue("p", types, typeof(string[])); + ParameterHelper[] parameters = (from p in info.GetParameters() select new ParameterHelper(p)).ToArray(); + serializationInfo.AddValue("p", parameters, typeof(ParameterHelper[])); } } } diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index f46bc63c1..9abb20ac7 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -866,6 +866,19 @@ int IComparer.Compare(MaybeMethodBase m1, MaybeMethodBase m2) { MethodBase me1 = m1.Valid ? m1.Value : null; MethodBase me2 = m2.Valid ? m2.Value : null; + if (me1 == null && me2 == null) + { + return 0; + } + else if (me1 == null) + { + return -1; + } + else if (me2 == null) + { + return 1; + } + if (me1.DeclaringType != me2.DeclaringType) { // m2's type derives from m1's type, favor m2 From 5f061bc56be37fd7cdef4ecd8ebdfc4158a7084b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 26 Nov 2020 14:34:01 -0500 Subject: [PATCH 16/51] Clean up the project and solution files --- pythonnet.15.sln | 114 +++++++++--------- pythonnet.sln | 48 ++++---- .../Python.DomainReloadTests.15.csproj | 38 +----- .../Python.DomainReloadTests.csproj | 47 -------- 4 files changed, 87 insertions(+), 160 deletions(-) diff --git a/pythonnet.15.sln b/pythonnet.15.sln index 4efba4495..3f41b7abe 100644 --- a/pythonnet.15.sln +++ b/pythonnet.15.sln @@ -14,7 +14,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Test.15", "src\testi EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.PerformanceTests", "src\perf_tests\Python.PerformanceTests.csproj", "{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Python.DomainReloadTests", "src\domain_tests\Python.DomainReloadTests.15.csproj", "{F2FB6DA3-318E-4F30-9A1F-932C667E38C5}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.DomainReloadTests.15", "src\domain_tests\Python.DomainReloadTests.15.csproj", "{F2FB6DA3-318E-4F30-9A1F-932C667E38C5}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Repo", "Repo", "{441A0123-F4C6-4EE4-9AEE-315FD79BE2D5}" ProjectSection(SolutionItems) = preProject @@ -390,59 +390,65 @@ Global {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|Any CPU.ActiveCfg = ReleaseWin|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|Any CPU.Build.0 = ReleaseWin|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x64.ActiveCfg = DebugWinPY3|x64 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x64.Build.0 = DebugWinPY3|x64 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x86.ActiveCfg = DebugWinPY3|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x86.Build.0 = DebugWinPY3|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|Any CPU.ActiveCfg = DebugMono|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|x64.ActiveCfg = DebugMono|x64 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|x64.Build.0 = DebugMono|x64 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|x86.ActiveCfg = DebugMono|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|x86.Build.0 = DebugMono|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|x64 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|Any CPU.ActiveCfg = DebugWin|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|x64.ActiveCfg = DebugWin|x64 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|x64.Build.0 = DebugWin|x64 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|x86.ActiveCfg = DebugWin|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|x86.Build.0 = DebugWin|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|Any CPU.ActiveCfg = ReleaseWin|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|Any CPU.Build.0 = ReleaseWin|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x64.ActiveCfg = ReleaseWinPY3|x64 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x64.Build.0 = ReleaseWinPY3|x64 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x86.ActiveCfg = ReleaseWinPY3|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x86.Build.0 = ReleaseWinPY3|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|x64.Build.0 = ReleaseMono|x64 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|x86.Build.0 = ReleaseMono|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|x64 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 - {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 - + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|Any CPU.ActiveCfg = Debug|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|Any CPU.Build.0 = Debug|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x64.ActiveCfg = Debug|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x64.Build.0 = Debug|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x86.ActiveCfg = Debug|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x86.Build.0 = Debug|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|Any CPU.ActiveCfg = Debug|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|Any CPU.Build.0 = Debug|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|x64.ActiveCfg = Debug|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|x64.Build.0 = Debug|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|x86.ActiveCfg = Debug|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMono|x86.Build.0 = Debug|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|Any CPU.ActiveCfg = Debug|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|Any CPU.Build.0 = Debug|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|x64.ActiveCfg = Debug|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|x64.Build.0 = Debug|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|x86.ActiveCfg = Debug|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugMonoPY3|x86.Build.0 = Debug|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|Any CPU.ActiveCfg = Debug|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|Any CPU.Build.0 = Debug|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|x64.ActiveCfg = Debug|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|x64.Build.0 = Debug|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|x86.ActiveCfg = Debug|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWin|x86.Build.0 = Debug|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|Any CPU.ActiveCfg = Debug|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|Any CPU.Build.0 = Debug|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|x64.ActiveCfg = Debug|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|x64.Build.0 = Debug|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|x86.ActiveCfg = Debug|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.DebugWinPY3|x86.Build.0 = Debug|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|Any CPU.ActiveCfg = Release|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x64.ActiveCfg = Release|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x64.Build.0 = Release|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x86.ActiveCfg = Release|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x86.Build.0 = Release|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|Any CPU.ActiveCfg = Release|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|Any CPU.Build.0 = Release|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|x64.ActiveCfg = Release|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|x64.Build.0 = Release|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|x86.ActiveCfg = Release|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMono|x86.Build.0 = Release|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|Any CPU.ActiveCfg = Release|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|Any CPU.Build.0 = Release|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|x64.ActiveCfg = Release|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|x64.Build.0 = Release|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|x86.ActiveCfg = Release|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseMonoPY3|x86.Build.0 = Release|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|Any CPU.ActiveCfg = Release|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|Any CPU.Build.0 = Release|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|x64.ActiveCfg = Release|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|x64.Build.0 = Release|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|x86.ActiveCfg = Release|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWin|x86.Build.0 = Release|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|Any CPU.ActiveCfg = Release|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|Any CPU.Build.0 = Release|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|x64.ActiveCfg = Release|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|x64.Build.0 = Release|x64 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|x86.ActiveCfg = Release|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.ReleaseWinPY3|x86.Build.0 = Release|x86 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/pythonnet.sln b/pythonnet.sln index 7b198b336..fdd140003 100644 --- a/pythonnet.sln +++ b/pythonnet.sln @@ -186,30 +186,30 @@ Global {86E834DE-1139-4511-96CC-69636A56E7AC}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 {86E834DE-1139-4511-96CC-69636A56E7AC}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 {86E834DE-1139-4511-96CC-69636A56E7AC}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugMono|x64.ActiveCfg = DebugMono|x64 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugMono|x86.ActiveCfg = DebugMono|x86 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWin|x64.ActiveCfg = DebugWin|x64 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWin|x64.Build.0 = DebugWin|x64 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWin|x86.ActiveCfg = DebugWin|x86 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWin|x86.Build.0 = DebugWin|x86 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugMono|x64.ActiveCfg = Debug|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugMono|x86.ActiveCfg = Debug|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugMonoPY3|x64.ActiveCfg = Debug|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugMonoPY3|x86.ActiveCfg = Debug|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWin|x64.ActiveCfg = Debug|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWin|x64.Build.0 = Debug|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWin|x86.ActiveCfg = Debug|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWin|x86.Build.0 = Debug|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWinPY3|x64.ActiveCfg = Debug|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWinPY3|x64.Build.0 = Debug|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWinPY3|x86.ActiveCfg = Debug|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWinPY3|x86.Build.0 = Debug|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMono|x64.ActiveCfg = Release|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMono|x86.ActiveCfg = Release|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMon|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMon|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWin|x64.ActiveCfg = Release|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWin|x64.Build.0 = Release|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWin|x86.ActiveCfg = Release|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWin|x86.Build.0 = Release|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWinPY3|x64.ActiveCfg = Release|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWinPY3|x64.Build.0 = Release|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWinPY3|x86.ActiveCfg = Release|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWinPY3|x86.Build.0 = Release|x86 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/domain_tests/Python.DomainReloadTests.15.csproj b/src/domain_tests/Python.DomainReloadTests.15.csproj index d8d489cbf..a6953ca64 100644 --- a/src/domain_tests/Python.DomainReloadTests.15.csproj +++ b/src/domain_tests/Python.DomainReloadTests.15.csproj @@ -4,7 +4,7 @@ net40;netcoreapp3.1 x64;x86 - DebugMono;DebugMonoPY3;ReleaseMono;ReleaseMonoPY3;DebugWin;DebugWinPY3;ReleaseWin;ReleaseWinPY3 + Debug;Release Exe false Python.DomainReloadTests @@ -25,12 +25,11 @@ $(OutputPath)\$(TargetFramework)_publish 7.3 prompt - $(PYTHONNET_DEFINE_CONSTANTS) XPLAT - $(DefineConstants);$(CustomDefineConstants);$(BaseDefineConstants); + $(DefineConstants);$(BaseDefineConstants); $(DefineConstants);NETCOREAPP $(DefineConstants);NETSTANDARD - $(DefineConstants);TRACE;DEBUG + $(DefineConstants);TRACE;DEBUG $(NuGetPackageRoot)\microsoft.targetingpack.netframework.v4.5\1.0.1\lib\net45\ @@ -40,32 +39,6 @@ x64 - - false - full - - - true - pdbonly - - - true - false - full - - - true - true - portable - - - - $(DefineConstants);DEBUG;TRACE - - - $(DefineConstants) - - @@ -90,9 +63,4 @@ $(TargetDir)$(TargetName).pdb - - - - - diff --git a/src/domain_tests/Python.DomainReloadTests.csproj b/src/domain_tests/Python.DomainReloadTests.csproj index 0914070ce..af454c89d 100644 --- a/src/domain_tests/Python.DomainReloadTests.csproj +++ b/src/domain_tests/Python.DomainReloadTests.csproj @@ -20,50 +20,6 @@ x64 - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - @@ -79,8 +35,5 @@ - - - \ No newline at end of file From e8543cf32bad1e19c648df9c9b6a886c0caaee4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 19 Nov 2020 11:43:42 -0500 Subject: [PATCH 17/51] Rework the serialization of reflected types Serialization of System.Type, MemberInfo and MethodBase is now string based. At deserialization, use reflection to attempt to recreate the object, which may fail safely instead of throwing a SerializaitonException during the deserialization of the whoie data stream. Appropriate Exceptions will now be raised when the Maybe*'s Value property. ClasseBase objects are now de-initialized and re-initialized in Reload mode so that it's tp_dict picks up newly added members and removed members no longer linger. ModuleObject clears it's cache and remove cached members from it's tp_dict. Minor refactoring and modernization of MethodObject and MethodBinder --- src/domain_tests/test_domain_reload.py | 11 - src/runtime/Python.Runtime.csproj | 383 +++++++++++++------------ src/runtime/arrayobject.cs | 13 +- src/runtime/classbase.cs | 14 +- src/runtime/classmanager.cs | 59 +++- src/runtime/classobject.cs | 27 +- src/runtime/constructorbinder.cs | 12 +- src/runtime/constructorbinding.cs | 12 +- src/runtime/converter.cs | 2 +- src/runtime/delegateobject.cs | 7 +- src/runtime/fieldobject.cs | 19 +- src/runtime/interfaceobject.cs | 6 +- src/runtime/maybeserialize.cs | 227 +++++++++++++++ src/runtime/metatype.cs | 18 +- src/runtime/methodbinder.cs | 41 ++- src/runtime/methodbinding.cs | 16 +- src/runtime/methodobject.cs | 40 ++- src/runtime/moduleobject.cs | 11 + src/runtime/propertyobject.cs | 37 ++- src/runtime/runtime.cs | 3 +- src/runtime/runtime_data.cs | 2 +- src/runtime/typemanager.cs | 24 +- 22 files changed, 682 insertions(+), 302 deletions(-) create mode 100644 src/runtime/maybeserialize.cs diff --git a/src/domain_tests/test_domain_reload.py b/src/domain_tests/test_domain_reload.py index e4d6bf6d7..7d2ce14c7 100644 --- a/src/domain_tests/test_domain_reload.py +++ b/src/domain_tests/test_domain_reload.py @@ -14,47 +14,36 @@ def _run_test(testname): assert proc.returncode == 0 -@pytest.mark.xfail(reason="Issue not yet fixed.") def test_rename_class(): _run_test('class_rename') -@pytest.mark.xfail(reason="Issue not yet fixed.") def test_rename_class_member_static_function(): _run_test('static_member_rename') -@pytest.mark.xfail(reason="Issue not yet fixed.") def test_rename_class_member_function(): _run_test('member_rename') -@pytest.mark.xfail(reason="Issue not yet fixed.") def test_rename_class_member_field(): _run_test('field_rename') -@pytest.mark.xfail(reason="Issue not yet fixed.") def test_rename_class_member_property(): _run_test('property_rename') -@pytest.mark.xfail(reason="Issue not yet fixed.") def test_rename_namespace(): _run_test('namespace_rename') -@pytest.mark.xfail(reason="Issue not yet fixed.") def test_field_visibility_change(): _run_test("field_visibility_change") -@pytest.mark.xfail(reason="Issue not yet fixed.") def test_method_visibility_change(): _run_test("method_visibility_change") -@pytest.mark.xfail(reason="Issue not yet fixed.") def test_property_visibility_change(): _run_test("property_visibility_change") -@pytest.mark.xfail(reason="Issue not yet fixed.") def test_class_visibility_change(): _run_test("class_visibility_change") -@pytest.mark.xfail(reason="Issue not yet fixed.") def test_method_parameters_change(): _run_test("method_parameters_change") diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 418620136..3d7913dbd 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,191 +1,192 @@ - - - - Debug - AnyCPU - {097B4AC0-74E9-4C58-BCF8-C69746EC8271} - Library - Python.Runtime - Python.Runtime - bin\Python.Runtime.xml - bin\ - v4.0 - - 1591 - ..\..\ - $(SolutionDir)\bin\ - Properties - 7.3 - true - false - ..\pythonnet.snk - - - - - - PYTHON2;PYTHON27;UCS4 - true - pdbonly - - - PYTHON3;PYTHON38;UCS4 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS4;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON38;UCS4;TRACE;DEBUG - false - full - - - PYTHON2;PYTHON27;UCS2 - true - pdbonly - - - PYTHON3;PYTHON38;UCS2 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS2;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON38;UCS2;TRACE;DEBUG - false - full - - - - - - - - - - - - - - - - Properties\SharedAssemblyInfo.cs - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - clr.py - - - - - $(TargetPath) - $(TargetDir)$(TargetName).pdb - - - - - - + + + + Debug + AnyCPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271} + Library + Python.Runtime + Python.Runtime + bin\Python.Runtime.xml + bin\ + v4.0 + + 1591 + ..\..\ + $(SolutionDir)\bin\ + Properties + 7.3 + true + false + ..\pythonnet.snk + + + + + + PYTHON2;PYTHON27;UCS4 + true + pdbonly + + + PYTHON3;PYTHON38;UCS4 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS4;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON38;UCS4;TRACE;DEBUG + false + full + + + PYTHON2;PYTHON27;UCS2 + true + pdbonly + + + PYTHON3;PYTHON38;UCS2 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS2;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON38;UCS2;TRACE;DEBUG + false + full + + + + + + + + + + + + + + + + Properties\SharedAssemblyInfo.cs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + clr.py + + + + + $(TargetPath) + $(TargetDir)$(TargetName).pdb + + + + + + diff --git a/src/runtime/arrayobject.cs b/src/runtime/arrayobject.cs index e6a4bee19..7b7adf792 100644 --- a/src/runtime/arrayobject.cs +++ b/src/runtime/arrayobject.cs @@ -30,9 +30,14 @@ public static IntPtr tp_new(IntPtr tpRaw, IntPtr args, IntPtr kw) var tp = new BorrowedReference(tpRaw); var self = GetManagedObject(tp) as ArrayObject; + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } long[] dimensions = new long[Runtime.PyTuple_Size(args)]; if (dimensions.Length == 0) + if (Runtime.PyTuple_Size(args) != 1) { return Exceptions.RaiseTypeError("array constructor requires at least one integer argument or an object convertible to array"); } @@ -63,7 +68,7 @@ public static IntPtr tp_new(IntPtr tpRaw, IntPtr args, IntPtr kw) object result; // this implements casting to Array[T] - if (!Converter.ToManaged(op, self.type, out result, true)) + if (!Converter.ToManaged(op, self.type.Value, out result, true)) { return IntPtr.Zero; } @@ -133,8 +138,12 @@ static NewReference NewInstance(Type elementType, BorrowedReference arrayPyType, { var obj = (CLRObject)GetManagedObject(ob); var arrObj = (ArrayObject)GetManagedObjectType(ob); + if (!arrObj.type.Valid) + { + return Exceptions.RaiseTypeError(arrObj.type.DeletedMessage); + } var items = obj.inst as Array; - Type itemType = arrObj.type.GetElementType(); + Type itemType = arrObj.type.Value.GetElementType(); int rank = items.Rank; int index; object value; diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index a62e76050..a5c2932f4 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -18,18 +18,21 @@ namespace Python.Runtime [Serializable] internal class ClassBase : ManagedType { + [NonSerialized] + internal List dotNetMembers; internal Indexer indexer; - internal Type type; + internal MaybeType type; internal ClassBase(Type tp) { + dotNetMembers = new List(); indexer = null; type = tp; } internal virtual bool CanSubclass() { - return !type.IsEnum; + return !type.Value.IsEnum; } @@ -44,7 +47,12 @@ public virtual IntPtr type_subscript(IntPtr idx) return Exceptions.RaiseTypeError("type(s) expected"); } - Type target = GenericUtil.GenericForType(type, types.Length); + if (!type.Valid) + { + return Exceptions.RaiseTypeError(type.DeletedMessage); + } + + Type target = GenericUtil.GenericForType(type.Value, types.Length); if (target != null) { diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index c8bed6bc4..250a2a449 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -18,7 +18,7 @@ namespace Python.Runtime /// internal class ClassManager { - private static Dictionary cache; + private static Dictionary cache; private static readonly Type dtype; private ClassManager() @@ -36,7 +36,7 @@ static ClassManager() public static void Reset() { - cache = new Dictionary(128); + cache = new Dictionary(128); } internal static void DisposePythonWrappersForClrTypes() @@ -85,26 +85,56 @@ internal static void SaveRuntimeData(RuntimeDataStorage storage) var contexts = storage.AddValue("contexts", new Dictionary()); storage.AddValue("cache", cache); - foreach (var cls in cache.Values) + foreach (var cls in cache) { + if (!cls.Key.Valid) + { + // Don't serialize an invalid class + continue; + } // This incref is for cache to hold the cls, // thus no need for decreasing it at RestoreRuntimeData. - Runtime.XIncref(cls.pyHandle); - var context = contexts[cls.pyHandle] = new InterDomainContext(); - cls.Save(context); + Runtime.XIncref(cls.Value.pyHandle); + var context = contexts[cls.Value.pyHandle] = new InterDomainContext(); + cls.Value.Save(context); + + // Remove all members added in InitBaseClass. + // this is done so that if domain reloads and a member of a + // reflected dotnet class is removed, it is removed from the + // Python object's dictionary tool; thus raising an AttributeError + // instead of a TypeError. + // Classes are re-initialized on in RestoreRuntimeData. + IntPtr dict = Marshal.ReadIntPtr(cls.Value.tpHandle, TypeOffset.tp_dict); + foreach (var member in cls.Value.dotNetMembers) + { + // No need to decref the member, the ClassBase instance does + // not own the reference. + Runtime.PyDict_DelItemString(dict, member); + } + // Trying to remove a key that's not in the dictionary may + // raise an error. We don't care about it. + Runtime.PyErr_Clear(); } } internal static Dictionary RestoreRuntimeData(RuntimeDataStorage storage) { - cache = storage.GetValue>("cache"); + var _cache = storage.GetValue>("cache"); var contexts = storage.GetValue >("contexts"); var loadedObjs = new Dictionary(); - foreach (var cls in cache.Values) + foreach (var pair in _cache) { - var context = contexts[cls.pyHandle]; - cls.Load(context); - loadedObjs.Add(cls, context); + if (!pair.Key.Valid) + { + Runtime.XDecref(pair.Value.pyHandle); + continue; + } + // re-init the class + InitClassBase(pair.Key.Value, pair.Value); + cache.Add(pair.Key, pair.Value); + var context = contexts[pair.Value.pyHandle]; + pair.Value.Load(context); + loadedObjs.Add(pair.Value, context); } return loadedObjs; } @@ -209,11 +239,16 @@ private static void InitClassBase(Type type, ClassBase impl) IntPtr dict = Marshal.ReadIntPtr(tp, TypeOffset.tp_dict); + if (impl.dotNetMembers == null) + { + impl.dotNetMembers = new List(); + } IDictionaryEnumerator iter = info.members.GetEnumerator(); while (iter.MoveNext()) { var item = (ManagedType)iter.Value; var name = (string)iter.Key; + impl.dotNetMembers.Add(name); Runtime.PyDict_SetItemString(dict, name, item.pyHandle); // Decref the item now that it's been used. item.DecrRefCount(); @@ -241,7 +276,7 @@ private static void InitClassBase(Type type, ClassBase impl) // required that the ClassObject.ctors be changed to internal if (co != null) { - if (co.ctors.Length > 0) + if (co.NumCtors > 0) { // Implement Overloads on the class object if (!CLRModule._SuppressOverloads) diff --git a/src/runtime/classobject.cs b/src/runtime/classobject.cs index 18816781f..9a19442dc 100644 --- a/src/runtime/classobject.cs +++ b/src/runtime/classobject.cs @@ -13,14 +13,14 @@ namespace Python.Runtime internal class ClassObject : ClassBase { internal ConstructorBinder binder; - internal ConstructorInfo[] ctors; + internal int NumCtors = 0; internal ClassObject(Type tp) : base(tp) { - ctors = type.GetConstructors(); - binder = new ConstructorBinder(type); - - foreach (ConstructorInfo t in ctors) + var _ctors = type.Value.GetConstructors(); + NumCtors = _ctors.Length; + binder = new ConstructorBinder(type.Value); + foreach (ConstructorInfo t in _ctors) { binder.AddMethod(t); } @@ -61,7 +61,11 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) return Exceptions.RaiseTypeError("invalid object"); } - Type type = self.type; + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } + Type type = self.type.Value; // Primitive types do not have constructors, but they look like // they do from Python. If the ClassObject represents one of the @@ -114,16 +118,21 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) /// public override IntPtr type_subscript(IntPtr idx) { + if (!type.Valid) + { + return Exceptions.RaiseTypeError(type.DeletedMessage); + } + // If this type is the Array type, the [] means we need to // construct and return an array type of the given element type. - if (type == typeof(Array)) + if (type.Value == typeof(Array)) { if (Runtime.PyTuple_Check(idx)) { return Exceptions.RaiseTypeError("type expected"); } var c = GetManagedObject(idx) as ClassBase; - Type t = c != null ? c.type : Converter.GetTypeByAlias(idx); + Type t = c != null ? c.type.Value : Converter.GetTypeByAlias(idx); if (t == null) { return Exceptions.RaiseTypeError("type expected"); @@ -143,7 +152,7 @@ public override IntPtr type_subscript(IntPtr idx) return Exceptions.RaiseTypeError("type(s) expected"); } - Type gtype = AssemblyManager.LookupType($"{type.FullName}`{types.Length}"); + Type gtype = AssemblyManager.LookupType($"{type.Value.FullName}`{types.Length}"); if (gtype != null) { var g = ClassManager.GetClass(gtype) as GenericType; diff --git a/src/runtime/constructorbinder.cs b/src/runtime/constructorbinder.cs index 0cda3a3d9..52e133983 100644 --- a/src/runtime/constructorbinder.cs +++ b/src/runtime/constructorbinder.cs @@ -14,7 +14,7 @@ namespace Python.Runtime [Serializable] internal class ConstructorBinder : MethodBinder { - private Type _containingType; + private MaybeType _containingType; internal ConstructorBinder(Type containingType) { @@ -51,10 +51,14 @@ internal object InvokeRaw(IntPtr inst, IntPtr args, IntPtr kw) /// internal object InvokeRaw(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info) { + if (!_containingType.Valid) + { + return Exceptions.RaiseTypeError(_containingType.DeletedMessage); + } object result; - if (_containingType.IsValueType && !_containingType.IsPrimitive && - !_containingType.IsEnum && _containingType != typeof(decimal) && + if (_containingType.Value.IsValueType && !_containingType.Value.IsPrimitive && + !_containingType.Value.IsEnum && _containingType.Value != typeof(decimal) && Runtime.PyTuple_Size(args) == 0) { // If you are trying to construct an instance of a struct by @@ -64,7 +68,7 @@ internal object InvokeRaw(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info) // Activator.CreateInstance(). try { - result = Activator.CreateInstance(_containingType); + result = Activator.CreateInstance(_containingType.Value); } catch (Exception e) { diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index 0c81c0a93..354ed4109 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -22,7 +22,7 @@ namespace Python.Runtime [Serializable] internal class ConstructorBinding : ExtensionType { - private Type type; // The managed Type being wrapped in a ClassObject + private MaybeType type; // The managed Type being wrapped in a ClassObject private IntPtr pyTypeHndl; // The python type tells GetInstHandle which Type to create. private ConstructorBinder ctorBinder; @@ -92,6 +92,10 @@ public static IntPtr tp_descr_get(IntPtr op, IntPtr instance, IntPtr owner) public static IntPtr mp_subscript(IntPtr op, IntPtr key) { var self = (ConstructorBinding)GetManagedObject(op); + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } Type[] types = Runtime.PythonArgsToTypeArray(key); if (types == null) @@ -100,12 +104,12 @@ public static IntPtr mp_subscript(IntPtr op, IntPtr key) } //MethodBase[] methBaseArray = self.ctorBinder.GetMethods(); //MethodBase ci = MatchSignature(methBaseArray, types); - ConstructorInfo ci = self.type.GetConstructor(types); + ConstructorInfo ci = self.type.Value.GetConstructor(types); if (ci == null) { return Exceptions.RaiseTypeError("No match found for constructor signature"); } - var boundCtor = new BoundContructor(self.type, self.pyTypeHndl, self.ctorBinder, ci); + var boundCtor = new BoundContructor(self.type.Value, self.pyTypeHndl, self.ctorBinder, ci); return boundCtor.pyHandle; } @@ -122,7 +126,7 @@ public static IntPtr tp_repr(IntPtr ob) return self.repr; } MethodBase[] methods = self.ctorBinder.GetMethods(); - string name = self.type.FullName; + string name = self.type.Value.FullName; var doc = ""; foreach (MethodBase t in methods) { diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 98fe99141..5b7998015 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -344,7 +344,7 @@ internal static bool ToManagedValue(IntPtr value, Type obType, } if (mt is ClassBase) { - result = ((ClassBase)mt).type; + result = ((ClassBase)mt).type.Value; return true; } // shouldn't happen diff --git a/src/runtime/delegateobject.cs b/src/runtime/delegateobject.cs index c5078740f..8379958f9 100644 --- a/src/runtime/delegateobject.cs +++ b/src/runtime/delegateobject.cs @@ -52,6 +52,11 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { var self = (DelegateObject)GetManagedObject(tp); + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } + if (Runtime.PyTuple_Size(args) != 1) { return Exceptions.RaiseTypeError("class takes exactly one argument"); @@ -64,7 +69,7 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) return Exceptions.RaiseTypeError("argument must be callable"); } - Delegate d = PythonEngine.DelegateManager.GetDelegate(self.type, method); + Delegate d = PythonEngine.DelegateManager.GetDelegate(self.type.Value, method); return CLRObject.GetInstHandle(d, self.pyHandle); } diff --git a/src/runtime/fieldobject.cs b/src/runtime/fieldobject.cs index 86b93dd1b..2850ac6e1 100644 --- a/src/runtime/fieldobject.cs +++ b/src/runtime/fieldobject.cs @@ -3,13 +3,14 @@ namespace Python.Runtime { + using MaybeFieldInfo = MaybeMemberInfo; /// /// Implements a Python descriptor type that provides access to CLR fields. /// [Serializable] internal class FieldObject : ExtensionType { - private FieldInfo info; + private MaybeFieldInfo info; public FieldObject(FieldInfo info) { @@ -30,8 +31,13 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) { return IntPtr.Zero; } + else if (!self.info.Valid) + { + Exceptions.SetError(Exceptions.AttributeError, self.info.DeletedMessage); + return IntPtr.Zero; + } - FieldInfo info = self.info; + FieldInfo info = self.info.Value; if (ob == IntPtr.Zero || ob == Runtime.PyNone) { @@ -85,6 +91,11 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) { return -1; } + else if (!self.info.Valid) + { + Exceptions.SetError(Exceptions.AttributeError, self.info.DeletedMessage); + return -1; + } if (val == IntPtr.Zero) { @@ -92,7 +103,7 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) return -1; } - FieldInfo info = self.info; + FieldInfo info = self.info.Value; if (info.IsLiteral || info.IsInitOnly) { @@ -147,7 +158,7 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) public static IntPtr tp_repr(IntPtr ob) { var self = (FieldObject)GetManagedObject(ob); - return Runtime.PyString_FromString($""); + return Runtime.PyString_FromString($""); } } } diff --git a/src/runtime/interfaceobject.cs b/src/runtime/interfaceobject.cs index a2fa86479..976c09be0 100644 --- a/src/runtime/interfaceobject.cs +++ b/src/runtime/interfaceobject.cs @@ -37,8 +37,12 @@ static InterfaceObject() public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { var self = (InterfaceObject)GetManagedObject(tp); + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } var nargs = Runtime.PyTuple_Size(args); - Type type = self.type; + Type type = self.type.Value; object obj; if (nargs == 1) diff --git a/src/runtime/maybeserialize.cs b/src/runtime/maybeserialize.cs new file mode 100644 index 000000000..ec59596e0 --- /dev/null +++ b/src/runtime/maybeserialize.cs @@ -0,0 +1,227 @@ +using System; +using System.Reflection; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Formatters.Binary; +using System.IO; + +namespace Python.Runtime +{ + [Serializable] + internal struct MaybeType : ISerializable + { + public static implicit operator MaybeType (Type ob) => new MaybeType(ob); + + string m_name; + Type m_type; + + public string DeletedMessage + { + get + { + return $"The .NET Type {m_name} no longer exists"; + } + } + + public Type Value + { + get + { + if (m_type == null) + { + throw new SerializationException(DeletedMessage); + } + return m_type; + } + } + + public string Name {get{return m_name;}} + public bool Valid => m_type != null; + + public override string ToString() + { + return (m_type != null ? m_type.ToString() : $"missing type: {m_name}") + Valid.ToString(); + } + + public MaybeType(Type tp) + { + m_type = tp; + m_name = tp.AssemblyQualifiedName; + } + + private MaybeType(SerializationInfo info, StreamingContext context) + { + m_name = (string)info.GetValue("n", typeof(string)); + m_type = Type.GetType(m_name, throwOnError:false); + } + + public void GetObjectData(SerializationInfo info, StreamingContext context) + { + info.AddValue("n", m_name); + } + } + + [Serializable] + internal struct MaybeMethod : ISerializable where T: MethodBase//, MethodInfo, ConstructorInfo + { + + public static implicit operator MaybeMethod (T ob) => new MaybeMethod(ob); + + string m_name; + MethodBase m_info; + // As seen in ClassManager.GetClassInfo + const BindingFlags k_flags = BindingFlags.Static | + BindingFlags.Instance | + BindingFlags.Public ; + + public T Value + { + get + { + if (m_info == null) + { + throw new SerializationException($"The .NET {typeof(T)} {m_name} no longer exists"); + } + return (T)m_info; + } + } + + public T UnsafeValue { get { return (T)m_info; } } + public string Name {get{return m_name;}} + public bool Valid => m_info != null; + + public override string ToString() + { + return (m_info != null ? m_info.ToString() : $"missing method info: {m_name}"); + } + + public MaybeMethod(T mi) + { + m_info = mi; + m_name = mi?.ToString(); + } + + internal MaybeMethod(SerializationInfo info, StreamingContext context) + { + m_name = info.GetString("s"); + m_info = null; + try + { + // Retrive the reflected type of the method; + var tp = Type.GetType(info.GetString("t")); + // Get the method's parameters types + var field_name = info.GetString("f"); + var param = (string[])info.GetValue("p", typeof(string[])); + Type[] types = new Type[param.Length]; + for (int i = 0; i < param.Length; i++) + { + types[i] = Type.GetType(param[i]); + } + // Try to get the method + m_info = tp.GetMethod(field_name, k_flags, binder:null, types:types, modifiers:null); + // Try again, may be a constructor + if (m_info == null && m_name.Contains(".ctor")) + { + m_info = tp.GetConstructor(k_flags, binder:null, types:types, modifiers:null); + } + } + catch + { + } + } + + public void GetObjectData(SerializationInfo info, StreamingContext context) + { + info.AddValue("s", m_name); + if (Valid) + { + info.AddValue("f", m_info.Name); + info.AddValue("t", m_info.ReflectedType.AssemblyQualifiedName); + var p = m_info.GetParameters(); + string[] types = new string[p.Length]; + for (int i = 0; i < p.Length; i++) + { + types[i] = p[i].ParameterType.AssemblyQualifiedName; + } + info.AddValue("p", types, typeof(string[])); + } + } + } + + [Serializable] + internal struct MaybeMemberInfo : ISerializable where T: MemberInfo + { + public static implicit operator MaybeMemberInfo (T ob) => new MaybeMemberInfo(ob); + + string m_name; + MemberInfo m_info; + + // As seen in ClassManager.GetClassInfo + const BindingFlags k_flags = BindingFlags.Static | + BindingFlags.Instance | + BindingFlags.Public ; + + public string DeletedMessage + { + get + { + return $"The .NET {typeof(T)} {m_name} no longer exists"; + } + } + + public T Value + { + get + { + if (m_info == null) + { + throw new SerializationException(DeletedMessage); + } + return (T)m_info; + } + } + + public string Name {get{return m_name;}} + public bool Valid => m_info != null; + + public override string ToString() + { + return (m_info != null ? m_info.ToString() : $"missing type: {m_name} ") + Valid.ToString(); + } + + public MaybeMemberInfo(T fi) + { + m_info = fi; + m_name = m_info?.ToString(); + } + + internal MaybeMemberInfo(SerializationInfo info, StreamingContext context) + { + // Assumption: name is always stored in "s" + m_name = info.GetString("s"); + m_info = null; + try + { + var tp = Type.GetType(info.GetString("t")); + if (tp != null) + { + var field_name = info.GetString("f"); + m_info = tp.GetField(field_name, k_flags); + } + } + catch + { + } + } + + public void GetObjectData(SerializationInfo info, StreamingContext context) + { + info.AddValue("s", m_name); + if (Valid) + { + info.AddValue("f", m_info.Name); + info.AddValue("t", m_info.ReflectedType.AssemblyQualifiedName); + } + } + } + +} diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 84abe28b9..36b406c7b 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -2,6 +2,7 @@ using System.Collections; using System.IO; using System.Runtime.InteropServices; +using System.Runtime.Serialization; namespace Python.Runtime { @@ -103,9 +104,16 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) var cb = GetManagedObject(base_type) as ClassBase; if (cb != null) { - if (!cb.CanSubclass()) + try { - return Exceptions.RaiseTypeError("delegates, enums and array types cannot be subclassed"); + if (!cb.CanSubclass()) + { + return Exceptions.RaiseTypeError("delegates, enums and array types cannot be subclassed"); + } + } + catch (SerializationException) + { + return Exceptions.RaiseTypeError($"Underlying C# Base class {cb.type} has been deleted"); } } @@ -300,7 +308,7 @@ private static IntPtr DoInstanceCheck(IntPtr tp, IntPtr args, bool checkType) { var cb = GetManagedObject(tp) as ClassBase; - if (cb == null) + if (cb == null || !cb.type.Valid) { Runtime.XIncref(Runtime.PyFalse); return Runtime.PyFalse; @@ -332,13 +340,13 @@ private static IntPtr DoInstanceCheck(IntPtr tp, IntPtr args, bool checkType) } var otherCb = GetManagedObject(otherType.Handle) as ClassBase; - if (otherCb == null) + if (otherCb == null || !otherCb.type.Valid) { Runtime.XIncref(Runtime.PyFalse); return Runtime.PyFalse; } - return Converter.ToPython(cb.type.IsAssignableFrom(otherCb.type)); + return Converter.ToPython(cb.type.Value.IsAssignableFrom(otherCb.type.Value)); } } diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index bd6eb32ba..b94cd86b5 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -7,6 +7,7 @@ namespace Python.Runtime { + using MaybeMethodBase = MaybeMethod; /// /// A MethodBinder encapsulates information about a (possibly overloaded) /// managed method, and is responsible for selecting the right method given @@ -16,19 +17,23 @@ namespace Python.Runtime [Serializable] internal class MethodBinder { - public ArrayList list; + public List list; + + [NonSerialized] public MethodBase[] methods; + + [NonSerialized] public bool init = false; public bool allow_threads = true; internal MethodBinder() { - list = new ArrayList(); + list = new List(); } internal MethodBinder(MethodInfo mi) { - list = new ArrayList { mi }; + list = new List { new MaybeMethodBase(mi) }; } public int Count @@ -164,7 +169,7 @@ internal MethodBase[] GetMethods() { // I'm sure this could be made more efficient. list.Sort(new MethodSorter()); - methods = (MethodBase[])list.ToArray(typeof(MethodBase)); + methods = (from method in list where method.Valid select method.Value).ToArray(); init = true; } return methods; @@ -180,6 +185,11 @@ internal MethodBase[] GetMethods() /// internal static int GetPrecedence(MethodBase mi) { + if (mi == null) + { + return -1; + } + ParameterInfo[] pi = mi.GetParameters(); int val = mi.IsStatic ? 3000 : 0; int num = pi.Length; @@ -740,6 +750,17 @@ protected static void AppendArgumentTypes(StringBuilder to, IntPtr args) internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, MethodInfo[] methodinfo) { + // No valid methods, nothing to bind. + if (GetMethods().Length == 0) + { + var msg = new StringBuilder("The underlying C# method(s) have been deleted"); + if (list.Count > 0 && list[0].Name != null) + { + msg.Append($": {list[0].ToString()}"); + } + return Exceptions.RaiseTypeError(msg.ToString());; + } + Binding binding = Bind(inst, args, kw, info, methodinfo); object result; IntPtr ts = IntPtr.Zero; @@ -839,12 +860,12 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i /// /// Utility class to sort method info by parameter type precedence. /// - internal class MethodSorter : IComparer + internal class MethodSorter : IComparer { - int IComparer.Compare(object m1, object m2) + int IComparer.Compare(MaybeMethodBase m1, MaybeMethodBase m2) { - var me1 = (MethodBase)m1; - var me2 = (MethodBase)m2; + MethodBase me1 = m1.Valid ? m1.Value : null; + MethodBase me2 = m2.Valid ? m2.Value : null; if (me1.DeclaringType != me2.DeclaringType) { // m2's type derives from m1's type, favor m2 @@ -856,8 +877,8 @@ int IComparer.Compare(object m1, object m2) return -1; } - int p1 = MethodBinder.GetPrecedence((MethodBase)m1); - int p2 = MethodBinder.GetPrecedence((MethodBase)m2); + int p1 = MethodBinder.GetPrecedence(me1); + int p2 = MethodBinder.GetPrecedence(me2); if (p1 < p2) { return -1; diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index 7a10fcdef..0fbdfe4b8 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -4,6 +4,7 @@ namespace Python.Runtime { + using MaybeMethodInfo = MaybeMethod; /// /// Implements a Python binding type for CLR methods. These work much like /// standard Python method bindings, but the same type is used to bind @@ -12,7 +13,7 @@ namespace Python.Runtime [Serializable] internal class MethodBinding : ExtensionType { - internal MethodInfo info; + internal MaybeMethodInfo info; internal MethodObject m; internal IntPtr target; internal IntPtr targetType; @@ -111,15 +112,15 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) // This works around a situation where the wrong generic method is picked, // for example this method in the tests: string Overloaded(int arg1, int arg2, string arg3) - if (self.info != null) + if (self.info.Valid) { - if (self.info.IsGenericMethod) + if (self.info.Value.IsGenericMethod) { var len = Runtime.PyTuple_Size(args); //FIXME: Never used Type[] sigTp = Runtime.PythonArgsToTypeArray(args, true); if (sigTp != null) { - Type[] genericTp = self.info.GetGenericArguments(); + Type[] genericTp = self.info.Value.GetGenericArguments(); MethodInfo betterMatch = MethodBinder.MatchSignatureAndParameters(self.m.info, genericTp, sigTp); if (betterMatch != null) { @@ -164,9 +165,9 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) if (inst?.inst is IPythonDerivedType) { var baseType = GetManagedObject(self.targetType) as ClassBase; - if (baseType != null) + if (baseType != null && baseType.type.Valid) { - string baseMethodName = "_" + baseType.type.Name + "__" + self.m.name; + string baseMethodName = "_" + baseType.type.Value.Name + "__" + self.m.name; IntPtr baseMethod = Runtime.PyObject_GetAttrString(target, baseMethodName); if (baseMethod != IntPtr.Zero) { @@ -184,8 +185,7 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) } } } - - return self.m.Invoke(target, args, kw, self.info); + return self.m.Invoke(target, args, kw, self.info.UnsafeValue); } finally { diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index dc23e3ce5..883e17043 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -1,8 +1,12 @@ using System; +using System.Collections.Generic; using System.Reflection; +using System.Linq; namespace Python.Runtime { + using MaybeMethodInfo = MaybeMethod; + /// /// Implements a Python type that represents a CLR method. Method objects /// support a subscript syntax [] to allow explicit overload selection. @@ -13,40 +17,46 @@ namespace Python.Runtime [Serializable] internal class MethodObject : ExtensionType { - internal MethodInfo[] info; + [NonSerialized] + private MethodInfo[] _info = null; + private readonly List infoList; internal string name; internal MethodBinding unbound; - internal MethodBinder binder; + internal readonly MethodBinder binder; internal bool is_static = false; internal IntPtr doc; internal Type type; - public MethodObject(Type type, string name, MethodInfo[] info) - { - _MethodObject(type, name, info); - } - - public MethodObject(Type type, string name, MethodInfo[] info, bool allow_threads) - { - _MethodObject(type, name, info); - binder.allow_threads = allow_threads; - } - - private void _MethodObject(Type type, string name, MethodInfo[] info) + // `allow_threads = true`: True being the default value of MethodBinder.allow_threads + public MethodObject(Type type, string name, MethodInfo[] info, bool allow_threads = true) { this.type = type; this.name = name; - this.info = info; + this.infoList = new List(); binder = new MethodBinder(); foreach (MethodInfo item in info) { + this.infoList.Add(item); binder.AddMethod(item); if (item.IsStatic) { this.is_static = true; } } + binder.allow_threads = allow_threads; + } + + internal MethodInfo[] info + { + get + { + if (_info == null) + { + _info = (from i in infoList where i.Valid select i.Value).ToArray(); + } + return _info; + } } public virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw) diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 334c5c2f3..1a1a2d73f 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -341,6 +341,17 @@ protected override void OnSave(InterDomainContext context) // Decref twice in tp_clear, equilibrate them. Runtime.XIncref(dict); Runtime.XIncref(dict); + // destroy the cache(s) + foreach (var pair in cache) + { + Runtime.PyDict_DelItemString(dict, pair.Key); + pair.Value.DecrRefCount(); + } + // Trying to remove a key that's not in the dictionary may + // raise an error. We don't care about it. + Runtime.PyErr_Clear(); + + cache.Clear(); } protected override void OnLoad(InterDomainContext context) diff --git a/src/runtime/propertyobject.cs b/src/runtime/propertyobject.cs index ac1d077f9..a5bc373a4 100644 --- a/src/runtime/propertyobject.cs +++ b/src/runtime/propertyobject.cs @@ -4,15 +4,16 @@ namespace Python.Runtime { + using MaybeMethodInfo = MaybeMethod; /// /// Implements a Python descriptor type that manages CLR properties. /// [Serializable] internal class PropertyObject : ExtensionType { - private PropertyInfo info; - private MethodInfo getter; - private MethodInfo setter; + private MaybeMemberInfo info; + private MaybeMethodInfo getter; + private MaybeMethodInfo setter; [StrongNameIdentityPermission(SecurityAction.Assert)] public PropertyObject(PropertyInfo md) @@ -31,7 +32,11 @@ public PropertyObject(PropertyInfo md) public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) { var self = (PropertyObject)GetManagedObject(ds); - MethodInfo getter = self.getter; + if (!self.info.Valid) + { + return Exceptions.RaiseTypeError(self.info.DeletedMessage); + } + MethodInfo getter = self.getter.Value; object result; @@ -51,8 +56,8 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) try { - result = self.info.GetValue(null, null); - return Converter.ToPython(result, self.info.PropertyType); + result = self.info.Value.GetValue(null, null); + return Converter.ToPython(result, self.info.Value.PropertyType); } catch (Exception e) { @@ -68,8 +73,8 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) try { - result = self.info.GetValue(co.inst, null); - return Converter.ToPython(result, self.info.PropertyType); + result = self.info.Value.GetValue(co.inst, null); + return Converter.ToPython(result, self.info.Value.PropertyType); } catch (Exception e) { @@ -91,7 +96,13 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) public new static int tp_descr_set(IntPtr ds, IntPtr ob, IntPtr val) { var self = (PropertyObject)GetManagedObject(ds); - MethodInfo setter = self.setter; + if (!self.info.Valid) + { + Exceptions.RaiseTypeError(self.info.DeletedMessage); + return -1; + } + + MethodInfo setter = self.setter.Value; object newval; if (val == IntPtr.Zero) @@ -107,7 +118,7 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) } - if (!Converter.ToManaged(val, self.info.PropertyType, out newval, true)) + if (!Converter.ToManaged(val, self.info.Value.PropertyType, out newval, true)) { return -1; } @@ -133,11 +144,11 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) Exceptions.RaiseTypeError("invalid target"); return -1; } - self.info.SetValue(co.inst, newval, null); + self.info.Value.SetValue(co.inst, newval, null); } else { - self.info.SetValue(null, newval, null); + self.info.Value.SetValue(null, newval, null); } return 0; } @@ -159,7 +170,7 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) public static IntPtr tp_repr(IntPtr ob) { var self = (PropertyObject)GetManagedObject(ob); - return Runtime.PyString_FromString($""); + return Runtime.PyString_FromString($""); } } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 10aa165c8..90b1519bb 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -686,7 +686,8 @@ internal static Type[] PythonArgsToTypeArray(IntPtr arg, bool mangleObjects) if (mt is ClassBase) { - t = ((ClassBase)mt).type; + MaybeType _type = ((ClassBase)mt).type; + t = _type.Valid ? _type.Value : null; } else if (mt is CLRObject) { diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index 060573db4..f45e76db4 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -120,8 +120,8 @@ private static void RestoreRuntimeDataImpl() var objs = RestoreRuntimeDataObjects(storage.GetStorage("objs")); RestoreRuntimeDataModules(storage.GetStorage("modules")); - var clsObjs = ClassManager.RestoreRuntimeData(storage.GetStorage("classes")); TypeManager.RestoreRuntimeData(storage.GetStorage("types")); + var clsObjs = ClassManager.RestoreRuntimeData(storage.GetStorage("classes")); ImportHook.RestoreRuntimeData(storage.GetStorage("import")); PyCLRMetaType = MetaType.RestoreRuntimeData(storage.GetStorage("meta")); diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 43c160bc3..564da983d 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -20,9 +20,10 @@ internal class TypeManager internal static IntPtr subtype_clear; private const BindingFlags tbFlags = BindingFlags.Public | BindingFlags.Static; - private static Dictionary cache = new Dictionary(); + private static Dictionary cache = new Dictionary(); + private static readonly Dictionary _slotsHolders = new Dictionary(); - private static Dictionary _slotsImpls = new Dictionary(); + private static Dictionary _slotsImpls = new Dictionary(); // Slots which must be set private static readonly string[] _requiredSlots = new string[] @@ -76,11 +77,22 @@ internal static void RestoreRuntimeData(RuntimeDataStorage storage) { Debug.Assert(cache == null || cache.Count == 0); storage.GetValue("slots", out _slotsImpls); - storage.GetValue("cache", out cache); - foreach (var entry in cache) + var _cache = new Dictionary(); + storage.GetValue("cache", out _cache); + foreach (var entry in _cache) { - Type type = entry.Key; + Type type = null; + try + { + type = entry.Key.Value; + } + catch + { + Runtime.XDecref(entry.Value); + continue; + } IntPtr handle = entry.Value; + cache[type] = handle; SlotsHolder holder = CreateSolotsHolder(handle); InitializeSlots(handle, _slotsImpls[type], holder); // FIXME: mp_length_slot.CanAssgin(clrType) @@ -358,7 +370,7 @@ internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr try { Type subType = ClassDerivedObject.CreateDerivedType(name, - baseClass.type, + baseClass.type.Value, py_dict, (string)namespaceStr, (string)assembly); From 61b0d8cb2bdb102e2b444bf9a22e5b97404959ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Fri, 20 Nov 2020 15:37:26 -0500 Subject: [PATCH 18/51] Call PyType_Modified after modifying the type Changing a type's attribute causes problem with it's cache. Force the type to refresh itself when modifying it. --- src/runtime/classmanager.cs | 7 +++++++ src/runtime/typemanager.cs | 11 +++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 250a2a449..6d62b3941 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -114,6 +114,8 @@ internal static void SaveRuntimeData(RuntimeDataStorage storage) // Trying to remove a key that's not in the dictionary may // raise an error. We don't care about it. Runtime.PyErr_Clear(); + // We modified the Type object, notify it we did. + Runtime.PyType_Modified(cls.Value.tpHandle); } } @@ -131,6 +133,8 @@ internal static Dictionary RestoreRuntimeData(R } // re-init the class InitClassBase(pair.Key.Value, pair.Value); + // We modified the Type object, notify it we did. + Runtime.PyType_Modified(pair.Value.tpHandle); cache.Add(pair.Key, pair.Value); var context = contexts[pair.Value.pyHandle]; pair.Value.Load(context); @@ -298,6 +302,9 @@ private static void InitClassBase(Type type, ClassBase impl) } } } + // The type has been modified after PyType_Ready has been called + // Refresh the type + Runtime.PyType_Modified(tp); } private static ClassInfo GetClassInfo(Type type) diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 564da983d..3682cbb7d 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -181,6 +181,10 @@ internal static IntPtr CreateType(Type impl) Runtime.XDecref(mod); InitMethods(type, impl); + + // The type has been modified after PyType_Ready has been called + // Refresh the type + Runtime.PyType_Modified(type); return type; } @@ -476,6 +480,9 @@ internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) IntPtr mod = Runtime.PyString_FromString("CLR"); Runtime.PyDict_SetItemString(dict, "__module__", mod); + // The type has been modified after PyType_Ready has been called + // Refresh the type + Runtime.PyType_Modified(type); //DebugUtil.DumpType(type); return type; @@ -577,6 +584,10 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) IntPtr tp_dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); IntPtr mod = Runtime.PyString_FromString("CLR"); Runtime.PyDict_SetItem(tp_dict, PyIdentifier.__module__, mod); + + // The type has been modified after PyType_Ready has been called + // Refresh the type + Runtime.PyType_Modified(type); return type; } From c8bacf3f964012ede17e17d20497c8bb99af1726 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Fri, 20 Nov 2020 15:48:20 -0500 Subject: [PATCH 19/51] Some code review changes * Revert line endings change in Python.Runtime.csproj * Split maybe serialize into respective class files * Name changes for consistency --- src/runtime/Python.Runtime.csproj | 386 +++++++++--------- .../StateSerialization/MaybeMemberInfo.cs | 85 ++++ .../StateSerialization/MaybeMethodBase.cs | 94 +++++ src/runtime/StateSerialization/MaybeType.cs | 62 +++ src/runtime/maybeserialize.cs | 227 ---------- src/runtime/methodbinder.cs | 2 +- src/runtime/methodbinding.cs | 2 +- src/runtime/methodobject.cs | 2 +- src/runtime/propertyobject.cs | 2 +- 9 files changed, 439 insertions(+), 423 deletions(-) create mode 100644 src/runtime/StateSerialization/MaybeMemberInfo.cs create mode 100644 src/runtime/StateSerialization/MaybeMethodBase.cs create mode 100644 src/runtime/StateSerialization/MaybeType.cs delete mode 100644 src/runtime/maybeserialize.cs diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 3d7913dbd..bb4f190ff 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,192 +1,194 @@ - - - - Debug - AnyCPU - {097B4AC0-74E9-4C58-BCF8-C69746EC8271} - Library - Python.Runtime - Python.Runtime - bin\Python.Runtime.xml - bin\ - v4.0 - - 1591 - ..\..\ - $(SolutionDir)\bin\ - Properties - 7.3 - true - false - ..\pythonnet.snk - - - - - - PYTHON2;PYTHON27;UCS4 - true - pdbonly - - - PYTHON3;PYTHON38;UCS4 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS4;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON38;UCS4;TRACE;DEBUG - false - full - - - PYTHON2;PYTHON27;UCS2 - true - pdbonly - - - PYTHON3;PYTHON38;UCS2 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS2;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON38;UCS2;TRACE;DEBUG - false - full - - - - - - - - - - - - - - - - Properties\SharedAssemblyInfo.cs - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - clr.py - - - - - $(TargetPath) - $(TargetDir)$(TargetName).pdb - - - - - - + + + + Debug + AnyCPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271} + Library + Python.Runtime + Python.Runtime + bin\Python.Runtime.xml + bin\ + v4.0 + + 1591 + ..\..\ + $(SolutionDir)\bin\ + Properties + 7.3 + true + false + ..\pythonnet.snk + + + + + + PYTHON2;PYTHON27;UCS4 + true + pdbonly + + + PYTHON3;PYTHON38;UCS4 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS4;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON38;UCS4;TRACE;DEBUG + false + full + + + PYTHON2;PYTHON27;UCS2 + true + pdbonly + + + PYTHON3;PYTHON38;UCS2 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS2;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON38;UCS2;TRACE;DEBUG + false + full + + + + + + + + + + + + + + + + Properties\SharedAssemblyInfo.cs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + clr.py + + + + + $(TargetPath) + $(TargetDir)$(TargetName).pdb + + + + + + diff --git a/src/runtime/StateSerialization/MaybeMemberInfo.cs b/src/runtime/StateSerialization/MaybeMemberInfo.cs new file mode 100644 index 000000000..518911de2 --- /dev/null +++ b/src/runtime/StateSerialization/MaybeMemberInfo.cs @@ -0,0 +1,85 @@ +using System; +using System.Reflection; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Formatters.Binary; +using System.IO; + +namespace Python.Runtime +{ + [Serializable] + internal struct MaybeMemberInfo : ISerializable where T: MemberInfo + { + public static implicit operator MaybeMemberInfo (T ob) => new MaybeMemberInfo(ob); + + string m_name; + MemberInfo m_info; + + // As seen in ClassManager.GetClassInfo + const BindingFlags k_flags = BindingFlags.Static | + BindingFlags.Instance | + BindingFlags.Public ; + + public string DeletedMessage + { + get + { + return $"The .NET {typeof(T)} {m_name} no longer exists"; + } + } + + public T Value + { + get + { + if (m_info == null) + { + throw new SerializationException(DeletedMessage); + } + return (T)m_info; + } + } + + public string Name {get{return m_name;}} + public bool Valid => m_info != null; + + public override string ToString() + { + return (m_info != null ? m_info.ToString() : $"missing type: {m_name}"); + } + + public MaybeMemberInfo(T fi) + { + m_info = fi; + m_name = m_info?.ToString(); + } + + internal MaybeMemberInfo(SerializationInfo info, StreamingContext context) + { + // Assumption: name is always stored in "s" + m_name = info.GetString("s"); + m_info = null; + try + { + var tp = Type.GetType(info.GetString("t")); + if (tp != null) + { + var field_name = info.GetString("f"); + m_info = tp.GetField(field_name, k_flags); + } + } + catch + { + } + } + + public void GetObjectData(SerializationInfo info, StreamingContext context) + { + info.AddValue("s", m_name); + if (Valid) + { + info.AddValue("f", m_info.Name); + info.AddValue("t", m_info.ReflectedType.AssemblyQualifiedName); + } + } + } +} diff --git a/src/runtime/StateSerialization/MaybeMethodBase.cs b/src/runtime/StateSerialization/MaybeMethodBase.cs new file mode 100644 index 000000000..359577e3b --- /dev/null +++ b/src/runtime/StateSerialization/MaybeMethodBase.cs @@ -0,0 +1,94 @@ +using System; +using System.Reflection; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Formatters.Binary; +using System.IO; + +namespace Python.Runtime +{ + [Serializable] + internal struct MaybeMethodBase : ISerializable where T: MethodBase + { + public static implicit operator MaybeMethodBase (T ob) => new MaybeMethodBase(ob); + + string m_name; + MethodBase m_info; + // As seen in ClassManager.GetClassInfo + const BindingFlags k_flags = BindingFlags.Static | + BindingFlags.Instance | + BindingFlags.Public ; + + public T Value + { + get + { + if (m_info == null) + { + throw new SerializationException($"The .NET {typeof(T)} {m_name} no longer exists"); + } + return (T)m_info; + } + } + + public T UnsafeValue { get { return (T)m_info; } } + public string Name {get{return m_name;}} + public bool Valid => m_info != null; + + public override string ToString() + { + return (m_info != null ? m_info.ToString() : $"missing method info: {m_name}"); + } + + public MaybeMethod(T mi) + { + m_info = mi; + m_name = mi?.ToString(); + } + + internal MaybeMethod(SerializationInfo info, StreamingContext context) + { + m_name = info.GetString("s"); + m_info = null; + try + { + // Retrive the reflected type of the method; + var tp = Type.GetType(info.GetString("t")); + // Get the method's parameters types + var field_name = info.GetString("f"); + var param = (string[])info.GetValue("p", typeof(string[])); + Type[] types = new Type[param.Length]; + for (int i = 0; i < param.Length; i++) + { + types[i] = Type.GetType(param[i]); + } + // Try to get the method + m_info = tp.GetMethod(field_name, k_flags, binder:null, types:types, modifiers:null); + // Try again, may be a constructor + if (m_info == null && m_name.Contains(".ctor")) + { + m_info = tp.GetConstructor(k_flags, binder:null, types:types, modifiers:null); + } + } + catch + { + } + } + + public void GetObjectData(SerializationInfo info, StreamingContext context) + { + info.AddValue("s", m_name); + if (Valid) + { + info.AddValue("f", m_info.Name); + info.AddValue("t", m_info.ReflectedType.AssemblyQualifiedName); + var p = m_info.GetParameters(); + string[] types = new string[p.Length]; + for (int i = 0; i < p.Length; i++) + { + types[i] = p[i].ParameterType.AssemblyQualifiedName; + } + info.AddValue("p", types, typeof(string[])); + } + } + } +} \ No newline at end of file diff --git a/src/runtime/StateSerialization/MaybeType.cs b/src/runtime/StateSerialization/MaybeType.cs new file mode 100644 index 000000000..f8e243e29 --- /dev/null +++ b/src/runtime/StateSerialization/MaybeType.cs @@ -0,0 +1,62 @@ +using System; +using System.Reflection; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Formatters.Binary; +using System.IO; + +namespace Python.Runtime +{ + [Serializable] + internal struct MaybeType : ISerializable + { + public static implicit operator MaybeType (Type ob) => new MaybeType(ob); + + string m_name; + Type m_type; + + public string DeletedMessage + { + get + { + return $"The .NET Type {m_name} no longer exists"; + } + } + + public Type Value + { + get + { + if (m_type == null) + { + throw new SerializationException(DeletedMessage); + } + return m_type; + } + } + + public string Name {get{return m_name;}} + public bool Valid => m_type != null; + + public override string ToString() + { + return (m_type != null ? m_type.ToString() : $"missing type: {m_name}"); + } + + public MaybeType(Type tp) + { + m_type = tp; + m_name = tp.AssemblyQualifiedName; + } + + private MaybeType(SerializationInfo info, StreamingContext context) + { + m_name = (string)info.GetValue("n", typeof(string)); + m_type = Type.GetType(m_name, throwOnError:false); + } + + public void GetObjectData(SerializationInfo info, StreamingContext context) + { + info.AddValue("n", m_name); + } + } +} \ No newline at end of file diff --git a/src/runtime/maybeserialize.cs b/src/runtime/maybeserialize.cs deleted file mode 100644 index ec59596e0..000000000 --- a/src/runtime/maybeserialize.cs +++ /dev/null @@ -1,227 +0,0 @@ -using System; -using System.Reflection; -using System.Runtime.Serialization; -using System.Runtime.Serialization.Formatters.Binary; -using System.IO; - -namespace Python.Runtime -{ - [Serializable] - internal struct MaybeType : ISerializable - { - public static implicit operator MaybeType (Type ob) => new MaybeType(ob); - - string m_name; - Type m_type; - - public string DeletedMessage - { - get - { - return $"The .NET Type {m_name} no longer exists"; - } - } - - public Type Value - { - get - { - if (m_type == null) - { - throw new SerializationException(DeletedMessage); - } - return m_type; - } - } - - public string Name {get{return m_name;}} - public bool Valid => m_type != null; - - public override string ToString() - { - return (m_type != null ? m_type.ToString() : $"missing type: {m_name}") + Valid.ToString(); - } - - public MaybeType(Type tp) - { - m_type = tp; - m_name = tp.AssemblyQualifiedName; - } - - private MaybeType(SerializationInfo info, StreamingContext context) - { - m_name = (string)info.GetValue("n", typeof(string)); - m_type = Type.GetType(m_name, throwOnError:false); - } - - public void GetObjectData(SerializationInfo info, StreamingContext context) - { - info.AddValue("n", m_name); - } - } - - [Serializable] - internal struct MaybeMethod : ISerializable where T: MethodBase//, MethodInfo, ConstructorInfo - { - - public static implicit operator MaybeMethod (T ob) => new MaybeMethod(ob); - - string m_name; - MethodBase m_info; - // As seen in ClassManager.GetClassInfo - const BindingFlags k_flags = BindingFlags.Static | - BindingFlags.Instance | - BindingFlags.Public ; - - public T Value - { - get - { - if (m_info == null) - { - throw new SerializationException($"The .NET {typeof(T)} {m_name} no longer exists"); - } - return (T)m_info; - } - } - - public T UnsafeValue { get { return (T)m_info; } } - public string Name {get{return m_name;}} - public bool Valid => m_info != null; - - public override string ToString() - { - return (m_info != null ? m_info.ToString() : $"missing method info: {m_name}"); - } - - public MaybeMethod(T mi) - { - m_info = mi; - m_name = mi?.ToString(); - } - - internal MaybeMethod(SerializationInfo info, StreamingContext context) - { - m_name = info.GetString("s"); - m_info = null; - try - { - // Retrive the reflected type of the method; - var tp = Type.GetType(info.GetString("t")); - // Get the method's parameters types - var field_name = info.GetString("f"); - var param = (string[])info.GetValue("p", typeof(string[])); - Type[] types = new Type[param.Length]; - for (int i = 0; i < param.Length; i++) - { - types[i] = Type.GetType(param[i]); - } - // Try to get the method - m_info = tp.GetMethod(field_name, k_flags, binder:null, types:types, modifiers:null); - // Try again, may be a constructor - if (m_info == null && m_name.Contains(".ctor")) - { - m_info = tp.GetConstructor(k_flags, binder:null, types:types, modifiers:null); - } - } - catch - { - } - } - - public void GetObjectData(SerializationInfo info, StreamingContext context) - { - info.AddValue("s", m_name); - if (Valid) - { - info.AddValue("f", m_info.Name); - info.AddValue("t", m_info.ReflectedType.AssemblyQualifiedName); - var p = m_info.GetParameters(); - string[] types = new string[p.Length]; - for (int i = 0; i < p.Length; i++) - { - types[i] = p[i].ParameterType.AssemblyQualifiedName; - } - info.AddValue("p", types, typeof(string[])); - } - } - } - - [Serializable] - internal struct MaybeMemberInfo : ISerializable where T: MemberInfo - { - public static implicit operator MaybeMemberInfo (T ob) => new MaybeMemberInfo(ob); - - string m_name; - MemberInfo m_info; - - // As seen in ClassManager.GetClassInfo - const BindingFlags k_flags = BindingFlags.Static | - BindingFlags.Instance | - BindingFlags.Public ; - - public string DeletedMessage - { - get - { - return $"The .NET {typeof(T)} {m_name} no longer exists"; - } - } - - public T Value - { - get - { - if (m_info == null) - { - throw new SerializationException(DeletedMessage); - } - return (T)m_info; - } - } - - public string Name {get{return m_name;}} - public bool Valid => m_info != null; - - public override string ToString() - { - return (m_info != null ? m_info.ToString() : $"missing type: {m_name} ") + Valid.ToString(); - } - - public MaybeMemberInfo(T fi) - { - m_info = fi; - m_name = m_info?.ToString(); - } - - internal MaybeMemberInfo(SerializationInfo info, StreamingContext context) - { - // Assumption: name is always stored in "s" - m_name = info.GetString("s"); - m_info = null; - try - { - var tp = Type.GetType(info.GetString("t")); - if (tp != null) - { - var field_name = info.GetString("f"); - m_info = tp.GetField(field_name, k_flags); - } - } - catch - { - } - } - - public void GetObjectData(SerializationInfo info, StreamingContext context) - { - info.AddValue("s", m_name); - if (Valid) - { - info.AddValue("f", m_info.Name); - info.AddValue("t", m_info.ReflectedType.AssemblyQualifiedName); - } - } - } - -} diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index b94cd86b5..f46bc63c1 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -7,7 +7,7 @@ namespace Python.Runtime { - using MaybeMethodBase = MaybeMethod; + using MaybeMethodBase = MaybeMethodBase; /// /// A MethodBinder encapsulates information about a (possibly overloaded) /// managed method, and is responsible for selecting the right method given diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index 0fbdfe4b8..d94bd2f92 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -4,7 +4,7 @@ namespace Python.Runtime { - using MaybeMethodInfo = MaybeMethod; + using MaybeMethodInfo = MaybeMethodBase; /// /// Implements a Python binding type for CLR methods. These work much like /// standard Python method bindings, but the same type is used to bind diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 883e17043..0c8e3d4ec 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -5,7 +5,7 @@ namespace Python.Runtime { - using MaybeMethodInfo = MaybeMethod; + using MaybeMethodInfo = MaybeMethodBase; /// /// Implements a Python type that represents a CLR method. Method objects diff --git a/src/runtime/propertyobject.cs b/src/runtime/propertyobject.cs index a5bc373a4..e01a0d877 100644 --- a/src/runtime/propertyobject.cs +++ b/src/runtime/propertyobject.cs @@ -4,7 +4,7 @@ namespace Python.Runtime { - using MaybeMethodInfo = MaybeMethod; + using MaybeMethodInfo = MaybeMethodBase; /// /// Implements a Python descriptor type that manages CLR properties. /// From cde5c2309f0ab7cfa3db2aee4a8107156ab3e59a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 23 Nov 2020 10:40:55 -0500 Subject: [PATCH 20/51] fixup! Some code review changes --- src/runtime/Python.Runtime.csproj | 2 +- src/runtime/StateSerialization/MaybeMethodBase.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index bb4f190ff..bf07cebea 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -162,7 +162,7 @@ - + diff --git a/src/runtime/StateSerialization/MaybeMethodBase.cs b/src/runtime/StateSerialization/MaybeMethodBase.cs index 359577e3b..2d06de2d5 100644 --- a/src/runtime/StateSerialization/MaybeMethodBase.cs +++ b/src/runtime/StateSerialization/MaybeMethodBase.cs @@ -39,13 +39,13 @@ public override string ToString() return (m_info != null ? m_info.ToString() : $"missing method info: {m_name}"); } - public MaybeMethod(T mi) + public MaybeMethodBase(T mi) { m_info = mi; m_name = mi?.ToString(); } - internal MaybeMethod(SerializationInfo info, StreamingContext context) + internal MaybeMethodBase(SerializationInfo info, StreamingContext context) { m_name = info.GetString("s"); m_info = null; From a95677319508dca327424bebed6e8852e5a1cb46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 23 Nov 2020 11:08:06 -0500 Subject: [PATCH 21/51] Remove hungarian notation from Maybe* types --- .../StateSerialization/MaybeMemberInfo.cs | 40 +++++++------- .../StateSerialization/MaybeMethodBase.cs | 52 +++++++++---------- src/runtime/StateSerialization/MaybeType.cs | 30 +++++------ 3 files changed, 61 insertions(+), 61 deletions(-) diff --git a/src/runtime/StateSerialization/MaybeMemberInfo.cs b/src/runtime/StateSerialization/MaybeMemberInfo.cs index 518911de2..14f6ded3c 100644 --- a/src/runtime/StateSerialization/MaybeMemberInfo.cs +++ b/src/runtime/StateSerialization/MaybeMemberInfo.cs @@ -11,8 +11,8 @@ internal struct MaybeMemberInfo : ISerializable where T: MemberInfo { public static implicit operator MaybeMemberInfo (T ob) => new MaybeMemberInfo(ob); - string m_name; - MemberInfo m_info; + string name; + MemberInfo info; // As seen in ClassManager.GetClassInfo const BindingFlags k_flags = BindingFlags.Static | @@ -23,7 +23,7 @@ public string DeletedMessage { get { - return $"The .NET {typeof(T)} {m_name} no longer exists"; + return $"The .NET {typeof(T)} {name} no longer exists"; } } @@ -31,40 +31,40 @@ public T Value { get { - if (m_info == null) + if (info == null) { throw new SerializationException(DeletedMessage); } - return (T)m_info; + return (T)info; } } - public string Name {get{return m_name;}} - public bool Valid => m_info != null; + public string Name {get{return name;}} + public bool Valid => info != null; public override string ToString() { - return (m_info != null ? m_info.ToString() : $"missing type: {m_name}"); + return (info != null ? info.ToString() : $"missing type: {name}"); } public MaybeMemberInfo(T fi) { - m_info = fi; - m_name = m_info?.ToString(); + info = fi; + name = info?.ToString(); } - internal MaybeMemberInfo(SerializationInfo info, StreamingContext context) + internal MaybeMemberInfo(SerializationInfo serializationInfo, StreamingContext context) { // Assumption: name is always stored in "s" - m_name = info.GetString("s"); - m_info = null; + name = serializationInfo.GetString("s"); + info = null; try { - var tp = Type.GetType(info.GetString("t")); + var tp = Type.GetType(serializationInfo.GetString("t")); if (tp != null) { - var field_name = info.GetString("f"); - m_info = tp.GetField(field_name, k_flags); + var field_name = serializationInfo.GetString("f"); + info = tp.GetField(field_name, k_flags); } } catch @@ -72,13 +72,13 @@ internal MaybeMemberInfo(SerializationInfo info, StreamingContext context) } } - public void GetObjectData(SerializationInfo info, StreamingContext context) + public void GetObjectData(SerializationInfo serializationInfo, StreamingContext context) { - info.AddValue("s", m_name); + serializationInfo.AddValue("s", name); if (Valid) { - info.AddValue("f", m_info.Name); - info.AddValue("t", m_info.ReflectedType.AssemblyQualifiedName); + serializationInfo.AddValue("f", info.Name); + serializationInfo.AddValue("t", info.ReflectedType.AssemblyQualifiedName); } } } diff --git a/src/runtime/StateSerialization/MaybeMethodBase.cs b/src/runtime/StateSerialization/MaybeMethodBase.cs index 2d06de2d5..3501c1c45 100644 --- a/src/runtime/StateSerialization/MaybeMethodBase.cs +++ b/src/runtime/StateSerialization/MaybeMethodBase.cs @@ -11,8 +11,8 @@ internal struct MaybeMethodBase : ISerializable where T: MethodBase { public static implicit operator MaybeMethodBase (T ob) => new MaybeMethodBase(ob); - string m_name; - MethodBase m_info; + string name; + MethodBase info; // As seen in ClassManager.GetClassInfo const BindingFlags k_flags = BindingFlags.Static | BindingFlags.Instance | @@ -22,51 +22,51 @@ public T Value { get { - if (m_info == null) + if (info == null) { - throw new SerializationException($"The .NET {typeof(T)} {m_name} no longer exists"); + throw new SerializationException($"The .NET {typeof(T)} {name} no longer exists"); } - return (T)m_info; + return (T)info; } } - public T UnsafeValue { get { return (T)m_info; } } - public string Name {get{return m_name;}} - public bool Valid => m_info != null; + public T UnsafeValue { get { return (T)info; } } + public string Name {get{return name;}} + public bool Valid => info != null; public override string ToString() { - return (m_info != null ? m_info.ToString() : $"missing method info: {m_name}"); + return (info != null ? info.ToString() : $"missing method info: {name}"); } public MaybeMethodBase(T mi) { - m_info = mi; - m_name = mi?.ToString(); + info = mi; + name = mi?.ToString(); } - internal MaybeMethodBase(SerializationInfo info, StreamingContext context) + internal MaybeMethodBase(SerializationInfo serializationInfo, StreamingContext context) { - m_name = info.GetString("s"); - m_info = null; + name = serializationInfo.GetString("s"); + info = null; try { // Retrive the reflected type of the method; - var tp = Type.GetType(info.GetString("t")); + var tp = Type.GetType(serializationInfo.GetString("t")); // Get the method's parameters types - var field_name = info.GetString("f"); - var param = (string[])info.GetValue("p", typeof(string[])); + var field_name = serializationInfo.GetString("f"); + var param = (string[])serializationInfo.GetValue("p", typeof(string[])); Type[] types = new Type[param.Length]; for (int i = 0; i < param.Length; i++) { types[i] = Type.GetType(param[i]); } // Try to get the method - m_info = tp.GetMethod(field_name, k_flags, binder:null, types:types, modifiers:null); + info = tp.GetMethod(field_name, k_flags, binder:null, types:types, modifiers:null); // Try again, may be a constructor - if (m_info == null && m_name.Contains(".ctor")) + if (info == null && name.Contains(".ctor")) { - m_info = tp.GetConstructor(k_flags, binder:null, types:types, modifiers:null); + info = tp.GetConstructor(k_flags, binder:null, types:types, modifiers:null); } } catch @@ -74,20 +74,20 @@ internal MaybeMethodBase(SerializationInfo info, StreamingContext context) } } - public void GetObjectData(SerializationInfo info, StreamingContext context) + public void GetObjectData(SerializationInfo serializationInfo, StreamingContext context) { - info.AddValue("s", m_name); + serializationInfo.AddValue("s", name); if (Valid) { - info.AddValue("f", m_info.Name); - info.AddValue("t", m_info.ReflectedType.AssemblyQualifiedName); - var p = m_info.GetParameters(); + serializationInfo.AddValue("f", info.Name); + serializationInfo.AddValue("t", info.ReflectedType.AssemblyQualifiedName); + var p = info.GetParameters(); string[] types = new string[p.Length]; for (int i = 0; i < p.Length; i++) { types[i] = p[i].ParameterType.AssemblyQualifiedName; } - info.AddValue("p", types, typeof(string[])); + serializationInfo.AddValue("p", types, typeof(string[])); } } } diff --git a/src/runtime/StateSerialization/MaybeType.cs b/src/runtime/StateSerialization/MaybeType.cs index f8e243e29..4ce5a3827 100644 --- a/src/runtime/StateSerialization/MaybeType.cs +++ b/src/runtime/StateSerialization/MaybeType.cs @@ -11,14 +11,14 @@ internal struct MaybeType : ISerializable { public static implicit operator MaybeType (Type ob) => new MaybeType(ob); - string m_name; - Type m_type; + string name; + Type type; public string DeletedMessage { get { - return $"The .NET Type {m_name} no longer exists"; + return $"The .NET Type {name} no longer exists"; } } @@ -26,37 +26,37 @@ public Type Value { get { - if (m_type == null) + if (type == null) { throw new SerializationException(DeletedMessage); } - return m_type; + return type; } } - public string Name {get{return m_name;}} - public bool Valid => m_type != null; + public string Name {get{return name;}} + public bool Valid => type != null; public override string ToString() { - return (m_type != null ? m_type.ToString() : $"missing type: {m_name}"); + return (type != null ? type.ToString() : $"missing type: {name}"); } public MaybeType(Type tp) { - m_type = tp; - m_name = tp.AssemblyQualifiedName; + type = tp; + name = tp.AssemblyQualifiedName; } - private MaybeType(SerializationInfo info, StreamingContext context) + private MaybeType(SerializationInfo serializationInfo, StreamingContext context) { - m_name = (string)info.GetValue("n", typeof(string)); - m_type = Type.GetType(m_name, throwOnError:false); + name = (string)serializationInfo.GetValue("n", typeof(string)); + type = Type.GetType(name, throwOnError:false); } - public void GetObjectData(SerializationInfo info, StreamingContext context) + public void GetObjectData(SerializationInfo serializationInfo, StreamingContext context) { - info.AddValue("n", m_name); + serializationInfo.AddValue("n", name); } } } \ No newline at end of file From ee3b391bf5d95c93f4d34e89fca8877b59d5db14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 23 Nov 2020 11:09:10 -0500 Subject: [PATCH 22/51] Check the type of the exception before ignoring it --- src/runtime/classmanager.cs | 9 ++++++--- src/runtime/moduleobject.cs | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 6d62b3941..fef01f09a 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -110,10 +110,13 @@ internal static void SaveRuntimeData(RuntimeDataStorage storage) // No need to decref the member, the ClassBase instance does // not own the reference. Runtime.PyDict_DelItemString(dict, member); + if (Exceptions.ExceptionMatches(Exceptions.KeyError)) + { + // Trying to remove a key that's not in the dictionary + // raises an error. We don't care about it. + Runtime.PyErr_Clear(); + } } - // Trying to remove a key that's not in the dictionary may - // raise an error. We don't care about it. - Runtime.PyErr_Clear(); // We modified the Type object, notify it we did. Runtime.PyType_Modified(cls.Value.tpHandle); } diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 1a1a2d73f..def33b6c5 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -346,10 +346,13 @@ protected override void OnSave(InterDomainContext context) { Runtime.PyDict_DelItemString(dict, pair.Key); pair.Value.DecrRefCount(); + if (Exceptions.ExceptionMatches(Exceptions.KeyError)) + { + // Trying to remove a key that's not in the dictionary + // raises an error. We don't care about it. + Runtime.PyErr_Clear(); + } } - // Trying to remove a key that's not in the dictionary may - // raise an error. We don't care about it. - Runtime.PyErr_Clear(); cache.Clear(); } From 9b4d5f985d765d3bbd6f89782ae037dc896c06ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 23 Nov 2020 11:09:28 -0500 Subject: [PATCH 23/51] Check for validity, don't throw --- src/runtime/typemanager.cs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 3682cbb7d..d8b9ca70c 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -77,20 +77,15 @@ internal static void RestoreRuntimeData(RuntimeDataStorage storage) { Debug.Assert(cache == null || cache.Count == 0); storage.GetValue("slots", out _slotsImpls); - var _cache = new Dictionary(); - storage.GetValue("cache", out _cache); + storage.GetValue>("cache", out var _cache); foreach (var entry in _cache) { - Type type = null; - try - { - type = entry.Key.Value; - } - catch + if (!entry.Key.Valid) { Runtime.XDecref(entry.Value); continue; } + Type type = entry.Key.Value;; IntPtr handle = entry.Value; cache[type] = handle; SlotsHolder holder = CreateSolotsHolder(handle); From a2f329446b013fd4a5270af1fca1f3e0331bc40e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 23 Nov 2020 12:18:07 -0500 Subject: [PATCH 24/51] Refactor the member binding logic of ClassManager So that we can use that same logic when deserializing Maybe* types --- .../StateSerialization/MaybeMemberInfo.cs | 34 ++++-- .../StateSerialization/MaybeMethodBase.cs | 16 +-- src/runtime/classmanager.cs | 100 +++++++++++------- 3 files changed, 97 insertions(+), 53 deletions(-) diff --git a/src/runtime/StateSerialization/MaybeMemberInfo.cs b/src/runtime/StateSerialization/MaybeMemberInfo.cs index 14f6ded3c..8e5400b39 100644 --- a/src/runtime/StateSerialization/MaybeMemberInfo.cs +++ b/src/runtime/StateSerialization/MaybeMemberInfo.cs @@ -13,11 +13,6 @@ internal struct MaybeMemberInfo : ISerializable where T: MemberInfo string name; MemberInfo info; - - // As seen in ClassManager.GetClassInfo - const BindingFlags k_flags = BindingFlags.Static | - BindingFlags.Instance | - BindingFlags.Public ; public string DeletedMessage { @@ -64,7 +59,11 @@ internal MaybeMemberInfo(SerializationInfo serializationInfo, StreamingContext c if (tp != null) { var field_name = serializationInfo.GetString("f"); - info = tp.GetField(field_name, k_flags); + MemberInfo mi = tp.GetField(field_name, ClassManager.BindingFlags); + if (mi != null && ShouldBindMember(mi)) + { + info = mi; + } } } catch @@ -72,6 +71,29 @@ internal MaybeMemberInfo(SerializationInfo serializationInfo, StreamingContext c } } + // This is complicated because we bind fields + // based on the visibility of the field, properties + // based on it's setter/getter (which is a method + // info) visibility and events based on their + // AddMethod visibility. + static bool ShouldBindMember(MemberInfo mi) + { + if (mi is PropertyInfo pi) + { + return ClassManager.ShouldBindProperty(pi); + } + else if (mi is FieldInfo fi) + { + return ClassManager.ShouldBindField(fi); + } + else if (mi is EventInfo ei) + { + return ClassManager.ShouldBindEvent(ei); + } + + return false; + } + public void GetObjectData(SerializationInfo serializationInfo, StreamingContext context) { serializationInfo.AddValue("s", name); diff --git a/src/runtime/StateSerialization/MaybeMethodBase.cs b/src/runtime/StateSerialization/MaybeMethodBase.cs index 3501c1c45..12fbb89a1 100644 --- a/src/runtime/StateSerialization/MaybeMethodBase.cs +++ b/src/runtime/StateSerialization/MaybeMethodBase.cs @@ -13,10 +13,6 @@ internal struct MaybeMethodBase : ISerializable where T: MethodBase string name; MethodBase info; - // As seen in ClassManager.GetClassInfo - const BindingFlags k_flags = BindingFlags.Static | - BindingFlags.Instance | - BindingFlags.Public ; public T Value { @@ -62,11 +58,17 @@ internal MaybeMethodBase(SerializationInfo serializationInfo, StreamingContext c types[i] = Type.GetType(param[i]); } // Try to get the method - info = tp.GetMethod(field_name, k_flags, binder:null, types:types, modifiers:null); + MethodBase mb = tp.GetMethod(field_name, ClassManager.BindingFlags, binder:null, types:types, modifiers:null); // Try again, may be a constructor - if (info == null && name.Contains(".ctor")) + if (mb == null && name.Contains(".ctor")) { - info = tp.GetConstructor(k_flags, binder:null, types:types, modifiers:null); + mb = tp.GetConstructor(ClassManager.BindingFlags, binder:null, types:types, modifiers:null); + } + + // Do like in ClassManager.GetClassInfo + if(mb != null && ClassManager.ShouldBindMethod(mb)) + { + info = mb; } } catch diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index fef01f09a..9da40627c 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -18,6 +18,19 @@ namespace Python.Runtime /// internal class ClassManager { + + // Binding flags to determine which members to expose in Python. + // This is complicated because inheritance in Python is name + // based. We can't just find DeclaredOnly members, because we + // could have a base class A that defines two overloads of a + // method and a class B that defines two more. The name-based + // descriptor Python will find needs to know about inherited + // overloads as well as those declared on the sub class. + internal static readonly BindingFlags BindingFlags = BindingFlags.Static | + BindingFlags.Instance | + BindingFlags.Public | + BindingFlags.NonPublic; + private static Dictionary cache; private static readonly Type dtype; @@ -310,6 +323,47 @@ private static void InitClassBase(Type type, ClassBase impl) Runtime.PyType_Modified(tp); } + internal static bool ShouldBindMethod(MethodBase mb) + { + return (mb.IsPublic || mb.IsFamily || mb.IsFamilyOrAssembly); + } + + internal static bool ShouldBindField(FieldInfo fi) + { + return (fi.IsPublic || fi.IsFamily || fi.IsFamilyOrAssembly); + } + + internal static bool ShouldBindProperty(PropertyInfo pi) + { + MethodInfo mm = null; + try + { + mm = pi.GetGetMethod(true); + if (mm == null) + { + mm = pi.GetSetMethod(true); + } + } + catch (SecurityException) + { + // GetGetMethod may try to get a method protected by + // StrongNameIdentityPermission - effectively private. + return false; + } + + if (mm == null) + { + return false; + } + + return ShouldBindMethod(mm); + } + + internal static bool ShouldBindEvent(EventInfo ei) + { + return ShouldBindMethod(ei.GetAddMethod(true)); + } + private static ClassInfo GetClassInfo(Type type) { var ci = new ClassInfo(); @@ -322,18 +376,7 @@ private static ClassInfo GetClassInfo(Type type) Type tp; int i, n; - // This is complicated because inheritance in Python is name - // based. We can't just find DeclaredOnly members, because we - // could have a base class A that defines two overloads of a - // method and a class B that defines two more. The name-based - // descriptor Python will find needs to know about inherited - // overloads as well as those declared on the sub class. - BindingFlags flags = BindingFlags.Static | - BindingFlags.Instance | - BindingFlags.Public | - BindingFlags.NonPublic; - - MemberInfo[] info = type.GetMembers(flags); + MemberInfo[] info = type.GetMembers(BindingFlags); var local = new Hashtable(); var items = new ArrayList(); MemberInfo m; @@ -376,7 +419,7 @@ private static ClassInfo GetClassInfo(Type type) for (i = 0; i < inheritedInterfaces.Length; ++i) { Type inheritedType = inheritedInterfaces[i]; - MemberInfo[] imembers = inheritedType.GetMembers(flags); + MemberInfo[] imembers = inheritedType.GetMembers(BindingFlags); for (n = 0; n < imembers.Length; n++) { m = imembers[n]; @@ -407,8 +450,7 @@ private static ClassInfo GetClassInfo(Type type) { case MemberTypes.Method: meth = (MethodInfo)mi; - if (!(meth.IsPublic || meth.IsFamily || - meth.IsFamilyOrAssembly)) + if (!ShouldBindMethod(meth)) { continue; } @@ -425,28 +467,7 @@ private static ClassInfo GetClassInfo(Type type) case MemberTypes.Property: var pi = (PropertyInfo)mi; - MethodInfo mm = null; - try - { - mm = pi.GetGetMethod(true); - if (mm == null) - { - mm = pi.GetSetMethod(true); - } - } - catch (SecurityException) - { - // GetGetMethod may try to get a method protected by - // StrongNameIdentityPermission - effectively private. - continue; - } - - if (mm == null) - { - continue; - } - - if (!(mm.IsPublic || mm.IsFamily || mm.IsFamilyOrAssembly)) + if(!ShouldBindProperty(pi)) { continue; } @@ -471,7 +492,7 @@ private static ClassInfo GetClassInfo(Type type) case MemberTypes.Field: var fi = (FieldInfo)mi; - if (!(fi.IsPublic || fi.IsFamily || fi.IsFamilyOrAssembly)) + if (!ShouldBindField(fi)) { continue; } @@ -481,8 +502,7 @@ private static ClassInfo GetClassInfo(Type type) case MemberTypes.Event: var ei = (EventInfo)mi; - MethodInfo me = ei.GetAddMethod(true); - if (!(me.IsPublic || me.IsFamily || me.IsFamilyOrAssembly)) + if (!ShouldBindEvent(ei)) { continue; } From 46dcb9d6e67bca1b6681cdb5d3998144e9542c16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 23 Nov 2020 13:26:11 -0500 Subject: [PATCH 25/51] Include info about why deserialization failed in Maybe* --- .../StateSerialization/MaybeMemberInfo.cs | 12 +++++++++--- .../StateSerialization/MaybeMethodBase.cs | 18 ++++++++++++++++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/runtime/StateSerialization/MaybeMemberInfo.cs b/src/runtime/StateSerialization/MaybeMemberInfo.cs index 8e5400b39..610bc967d 100644 --- a/src/runtime/StateSerialization/MaybeMemberInfo.cs +++ b/src/runtime/StateSerialization/MaybeMemberInfo.cs @@ -14,11 +14,14 @@ internal struct MaybeMemberInfo : ISerializable where T: MemberInfo string name; MemberInfo info; + [NonSerialized] + Exception deserializationException; + public string DeletedMessage { get { - return $"The .NET {typeof(T)} {name} no longer exists"; + return $"The .NET {typeof(T)} {name} no longer exists. Cause: " + deserializationException?.Message ; } } @@ -28,7 +31,7 @@ public T Value { if (info == null) { - throw new SerializationException(DeletedMessage); + throw new SerializationException(DeletedMessage, innerException: deserializationException); } return (T)info; } @@ -46,6 +49,7 @@ public MaybeMemberInfo(T fi) { info = fi; name = info?.ToString(); + deserializationException = null; } internal MaybeMemberInfo(SerializationInfo serializationInfo, StreamingContext context) @@ -53,6 +57,7 @@ internal MaybeMemberInfo(SerializationInfo serializationInfo, StreamingContext c // Assumption: name is always stored in "s" name = serializationInfo.GetString("s"); info = null; + deserializationException = null; try { var tp = Type.GetType(serializationInfo.GetString("t")); @@ -66,8 +71,9 @@ internal MaybeMemberInfo(SerializationInfo serializationInfo, StreamingContext c } } } - catch + catch (Exception e) { + deserializationException = e; } } diff --git a/src/runtime/StateSerialization/MaybeMethodBase.cs b/src/runtime/StateSerialization/MaybeMethodBase.cs index 12fbb89a1..7e13129e6 100644 --- a/src/runtime/StateSerialization/MaybeMethodBase.cs +++ b/src/runtime/StateSerialization/MaybeMethodBase.cs @@ -14,13 +14,24 @@ internal struct MaybeMethodBase : ISerializable where T: MethodBase string name; MethodBase info; + [NonSerialized] + Exception deserializationException; + + public string DeletedMessage + { + get + { + return $"The .NET {typeof(T)} {name} no longer exists. Cause: " + deserializationException?.Message ; + } + } + public T Value { get { if (info == null) { - throw new SerializationException($"The .NET {typeof(T)} {name} no longer exists"); + throw new SerializationException(DeletedMessage, innerException: deserializationException); } return (T)info; } @@ -39,12 +50,14 @@ public MaybeMethodBase(T mi) { info = mi; name = mi?.ToString(); + deserializationException = null; } internal MaybeMethodBase(SerializationInfo serializationInfo, StreamingContext context) { name = serializationInfo.GetString("s"); info = null; + deserializationException = null; try { // Retrive the reflected type of the method; @@ -71,8 +84,9 @@ internal MaybeMethodBase(SerializationInfo serializationInfo, StreamingContext c info = mb; } } - catch + catch (Exception e) { + deserializationException = e; } } From 10116bb43a706f26117ad1b3f2cab5c3b6a22616 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Tue, 24 Nov 2020 13:14:19 -0500 Subject: [PATCH 26/51] improve deserialization resolution of ref, out and in parameters Also add a test --- src/domain_tests/TestRunner.cs | 78 +++++++++++++++++++ src/domain_tests/test_domain_reload.py | 5 +- .../StateSerialization/MaybeMethodBase.cs | 77 +++++++++++++++--- src/runtime/methodbinder.cs | 13 ++++ 4 files changed, 160 insertions(+), 13 deletions(-) diff --git a/src/domain_tests/TestRunner.cs b/src/domain_tests/TestRunner.cs index f8474d99b..51ec73fcb 100644 --- a/src/domain_tests/TestRunner.cs +++ b/src/domain_tests/TestRunner.cs @@ -771,6 +771,84 @@ def after_reload(): print(bar.__repr__()) ", }, + + new TestCase + { + Name = "out_to_ref_param", + DotNetBefore = @" + namespace TestNamespace + { + + [System.Serializable] + public class Data + { + public int num = -1; + } + + [System.Serializable] + public class Cls + { + public static void MyFn (out Data a) + { + a = new Data(); + a.num = 9001; + } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + + [System.Serializable] + public class Data + { + public int num = -1; + } + + [System.Serializable] + public class Cls + { + public static void MyFn (ref Data a) + { + a.num = 7; + } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace +import System + +def before_reload(): + + foo = TestNamespace.Data() + bar = TestNamespace.Cls.MyFn(foo) + assert bar.num == 9001 + # foo shouldn't have changed. + assert foo.num == -1 + + +def after_reload(): + + try: + # Now that the function takes a ref type, we must pass a valid object. + bar = TestNamespace.Cls.MyFn(None) + except System.NullReferenceException as e: + print('caught expected exception') + else: + raise AssertionError('failed to raise') + + foo = TestNamespace.Data() + bar = TestNamespace.Cls.MyFn(foo) + # foo should have changed + assert foo.num == 7 + assert bar.num == 7 + # Pythonnet also returns a new object with `ref`-quialified parameters + assert foo is not bar + ", + }, }; /// diff --git a/src/domain_tests/test_domain_reload.py b/src/domain_tests/test_domain_reload.py index 7d2ce14c7..cca3e0232 100644 --- a/src/domain_tests/test_domain_reload.py +++ b/src/domain_tests/test_domain_reload.py @@ -59,4 +59,7 @@ def test_rename_event(): @pytest.mark.xfail(reason="newly instanced object uses PyType_GenericAlloc") def test_construct_removed_class(): - _run_test("construct_removed_class") \ No newline at end of file + _run_test("construct_removed_class") + +def test_out_to_ref_param(): + _run_test("out_to_ref_param") \ No newline at end of file diff --git a/src/runtime/StateSerialization/MaybeMethodBase.cs b/src/runtime/StateSerialization/MaybeMethodBase.cs index 7e13129e6..3f8e0ff72 100644 --- a/src/runtime/StateSerialization/MaybeMethodBase.cs +++ b/src/runtime/StateSerialization/MaybeMethodBase.cs @@ -1,14 +1,50 @@ using System; using System.Reflection; using System.Runtime.Serialization; -using System.Runtime.Serialization.Formatters.Binary; -using System.IO; +using System.Linq; namespace Python.Runtime { [Serializable] internal struct MaybeMethodBase : ISerializable where T: MethodBase { + [Serializable] + struct ParameterHelper : IEquatable + { + public enum TypeModifier + { + None, + In, + Out, + Ref + } + public readonly string Name; + public readonly TypeModifier Modifier; + + public ParameterHelper(ParameterInfo tp) + { + Name = tp.ParameterType.AssemblyQualifiedName; + Modifier = TypeModifier.None; + + if (tp.IsIn) + { + Modifier = TypeModifier.In; + } + else if (tp.IsOut) + { + Modifier = TypeModifier.Out; + } + else if (tp.ParameterType.IsByRef) + { + Modifier = TypeModifier.Ref; + } + } + + public bool Equals(ParameterInfo other) + { + return this.Equals(new ParameterHelper(other)); + } + } public static implicit operator MaybeMethodBase (T ob) => new MaybeMethodBase(ob); string name; @@ -64,11 +100,11 @@ internal MaybeMethodBase(SerializationInfo serializationInfo, StreamingContext c var tp = Type.GetType(serializationInfo.GetString("t")); // Get the method's parameters types var field_name = serializationInfo.GetString("f"); - var param = (string[])serializationInfo.GetValue("p", typeof(string[])); + var param = (ParameterHelper[])serializationInfo.GetValue("p", typeof(ParameterHelper[])); Type[] types = new Type[param.Length]; for (int i = 0; i < param.Length; i++) { - types[i] = Type.GetType(param[i]); + types[i] = Type.GetType(param[i].Name); } // Try to get the method MethodBase mb = tp.GetMethod(field_name, ClassManager.BindingFlags, binder:null, types:types, modifiers:null); @@ -81,7 +117,29 @@ internal MaybeMethodBase(SerializationInfo serializationInfo, StreamingContext c // Do like in ClassManager.GetClassInfo if(mb != null && ClassManager.ShouldBindMethod(mb)) { - info = mb; + // One more step: Changing: + // void MyFn (ref int a) + // to: + // void MyFn (out int a) + // will still find the fucntion correctly as, `in`, `out` and `ref` + // are all represented as a reference type. Query the method we got + // and validate the parameters + bool matches = true; + if (param.Length != 0) + { + foreach (var item in Enumerable.Zip(param, mb.GetParameters(), (x, y) => new {x, y})) + { + if (!item.x.Equals(item.y)) + { + matches = false; + break; + } + } + } + if (matches) + { + info = mb; + } } } catch (Exception e) @@ -97,13 +155,8 @@ public void GetObjectData(SerializationInfo serializationInfo, StreamingContext { serializationInfo.AddValue("f", info.Name); serializationInfo.AddValue("t", info.ReflectedType.AssemblyQualifiedName); - var p = info.GetParameters(); - string[] types = new string[p.Length]; - for (int i = 0; i < p.Length; i++) - { - types[i] = p[i].ParameterType.AssemblyQualifiedName; - } - serializationInfo.AddValue("p", types, typeof(string[])); + ParameterHelper[] parameters = (from p in info.GetParameters() select new ParameterHelper(p)).ToArray(); + serializationInfo.AddValue("p", parameters, typeof(ParameterHelper[])); } } } diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index f46bc63c1..9abb20ac7 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -866,6 +866,19 @@ int IComparer.Compare(MaybeMethodBase m1, MaybeMethodBase m2) { MethodBase me1 = m1.Valid ? m1.Value : null; MethodBase me2 = m2.Valid ? m2.Value : null; + if (me1 == null && me2 == null) + { + return 0; + } + else if (me1 == null) + { + return -1; + } + else if (me2 == null) + { + return 1; + } + if (me1.DeclaringType != me2.DeclaringType) { // m2's type derives from m1's type, favor m2 From 102054e87dd22f0fba2573582788ee088b9a2250 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 26 Nov 2020 15:17:35 -0500 Subject: [PATCH 27/51] !fixup leftover error from the rebase --- src/runtime/arrayobject.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/runtime/arrayobject.cs b/src/runtime/arrayobject.cs index 7b7adf792..049d2b8bb 100644 --- a/src/runtime/arrayobject.cs +++ b/src/runtime/arrayobject.cs @@ -43,7 +43,7 @@ public static IntPtr tp_new(IntPtr tpRaw, IntPtr args, IntPtr kw) } if (dimensions.Length != 1) { - return CreateMultidimensional(self.type.GetElementType(), dimensions, + return CreateMultidimensional(self.type.Value.GetElementType(), dimensions, shapeTuple: new BorrowedReference(args), pyType: tp) .DangerousMoveToPointerOrNull(); @@ -61,7 +61,7 @@ public static IntPtr tp_new(IntPtr tpRaw, IntPtr args, IntPtr kw) } else { - return NewInstance(self.type.GetElementType(), tp, dimensions) + return NewInstance(self.type.Value.GetElementType(), tp, dimensions) .DangerousMoveToPointerOrNull(); } } From 329de5df8a6b7c30162ecba2c38b5556d14bba0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Fri, 27 Nov 2020 10:51:07 -0500 Subject: [PATCH 28/51] Add fixes and test for nested classes --- src/domain_tests/TestRunner.cs | 47 +++++++++ src/domain_tests/test_domain_reload.py | 5 +- .../StateSerialization/MaybeMethodBase.cs | 99 +++++++++++++------ src/runtime/classmanager.cs | 18 +++- 4 files changed, 136 insertions(+), 33 deletions(-) diff --git a/src/domain_tests/TestRunner.cs b/src/domain_tests/TestRunner.cs index 51ec73fcb..d5fa083e3 100644 --- a/src/domain_tests/TestRunner.cs +++ b/src/domain_tests/TestRunner.cs @@ -849,6 +849,53 @@ raise AssertionError('failed to raise') assert foo is not bar ", }, + new TestCase + { + Name = "nested_type", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class WithNestedType + { + [System.Serializable] + public class Inner + { + public static int Value = -1; + } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class WithNestedType + { + [System.Serializable] + public class Inner + { + public static int Value = -1; + } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace + +def before_reload(): + + sys.my_obj = TestNamespace.WithNestedType + +def after_reload(): + + assert sys.my_obj is not None + foo = sys.my_obj.Inner() + print(foo) + + ", + }, }; /// diff --git a/src/domain_tests/test_domain_reload.py b/src/domain_tests/test_domain_reload.py index cca3e0232..c040931b2 100644 --- a/src/domain_tests/test_domain_reload.py +++ b/src/domain_tests/test_domain_reload.py @@ -62,4 +62,7 @@ def test_construct_removed_class(): _run_test("construct_removed_class") def test_out_to_ref_param(): - _run_test("out_to_ref_param") \ No newline at end of file + _run_test("out_to_ref_param") + +def test_nested_type(): + _run_test("nested_type") diff --git a/src/runtime/StateSerialization/MaybeMethodBase.cs b/src/runtime/StateSerialization/MaybeMethodBase.cs index 3f8e0ff72..695819561 100644 --- a/src/runtime/StateSerialization/MaybeMethodBase.cs +++ b/src/runtime/StateSerialization/MaybeMethodBase.cs @@ -18,6 +18,7 @@ public enum TypeModifier Out, Ref } + public readonly string Name; public readonly TypeModifier Modifier; @@ -45,6 +46,7 @@ public bool Equals(ParameterInfo other) return this.Equals(new ParameterHelper(other)); } } + public static implicit operator MaybeMethodBase (T ob) => new MaybeMethodBase(ob); string name; @@ -102,44 +104,36 @@ internal MaybeMethodBase(SerializationInfo serializationInfo, StreamingContext c var field_name = serializationInfo.GetString("f"); var param = (ParameterHelper[])serializationInfo.GetValue("p", typeof(ParameterHelper[])); Type[] types = new Type[param.Length]; + bool hasRefType = false; for (int i = 0; i < param.Length; i++) { types[i] = Type.GetType(param[i].Name); + if (types[i].IsByRef) + { + hasRefType = true; + } + } + + MethodBase mb = null; + var isCtor = serializationInfo.GetBoolean("c"); + if (isCtor) + { + mb = ResolveConstructor(tp, types); + } + else + { + mb = tp.GetMethod(field_name, ClassManager.BindingFlags, binder:null, types:types, modifiers:null); } - // Try to get the method - MethodBase mb = tp.GetMethod(field_name, ClassManager.BindingFlags, binder:null, types:types, modifiers:null); - // Try again, may be a constructor - if (mb == null && name.Contains(".ctor")) + + if (mb != null && hasRefType) { - mb = tp.GetConstructor(ClassManager.BindingFlags, binder:null, types:types, modifiers:null); + CheckRefTypes(mb, param); } // Do like in ClassManager.GetClassInfo if(mb != null && ClassManager.ShouldBindMethod(mb)) { - // One more step: Changing: - // void MyFn (ref int a) - // to: - // void MyFn (out int a) - // will still find the fucntion correctly as, `in`, `out` and `ref` - // are all represented as a reference type. Query the method we got - // and validate the parameters - bool matches = true; - if (param.Length != 0) - { - foreach (var item in Enumerable.Zip(param, mb.GetParameters(), (x, y) => new {x, y})) - { - if (!item.x.Equals(item.y)) - { - matches = false; - break; - } - } - } - if (matches) - { - info = mb; - } + info = mb; } } catch (Exception e) @@ -148,6 +142,54 @@ internal MaybeMethodBase(SerializationInfo serializationInfo, StreamingContext c } } + MethodBase ResolveConstructor (Type tp, Type[] parameters) + { + MethodBase mb = null; + try + { + mb = tp.GetConstructor(ClassManager.BindingFlags, binder:null, types:parameters, modifiers:null); + } + catch (AmbiguousMatchException) + { + // The static constructor and non-static constructor may + // have the same signature. Use ToString to disambiguate. + var mbs = tp.GetConstructors(ClassManager.BindingFlags); + foreach (var ctor in mbs) + { + if (name == ctor.ToString()) + { + mb = ctor; + break; + } + } + } + return mb; + } + + MethodBase CheckRefTypes(MethodBase mb, ParameterHelper[] ph) + { + // One more step: Changing: + // void MyFn (ref int a) + // to: + // void MyFn (out int a) + // will still find the fucntion correctly as, `in`, `out` and `ref` + // are all represented as a reference type. Query the method we got + // and validate the parameters + if (ph.Length != 0) + { + foreach (var item in Enumerable.Zip(ph, mb.GetParameters(), (orig, current) => new {orig, current})) + { + if (!item.current.Equals(item.orig)) + { + // False positive + return null; + } + } + } + + return mb; + } + public void GetObjectData(SerializationInfo serializationInfo, StreamingContext context) { serializationInfo.AddValue("s", name); @@ -157,6 +199,7 @@ public void GetObjectData(SerializationInfo serializationInfo, StreamingContext serializationInfo.AddValue("t", info.ReflectedType.AssemblyQualifiedName); ParameterHelper[] parameters = (from p in info.GetParameters() select new ParameterHelper(p)).ToArray(); serializationInfo.AddValue("p", parameters, typeof(ParameterHelper[])); + serializationInfo.AddValue("c", info.IsConstructor); } } } diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 9da40627c..8cfaaad2f 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -137,25 +137,32 @@ internal static void SaveRuntimeData(RuntimeDataStorage storage) internal static Dictionary RestoreRuntimeData(RuntimeDataStorage storage) { - var _cache = storage.GetValue>("cache"); + cache = storage.GetValue>("cache"); + var invalidClasses = new List>(); var contexts = storage.GetValue >("contexts"); var loadedObjs = new Dictionary(); - foreach (var pair in _cache) + foreach (var pair in cache) { if (!pair.Key.Valid) { - Runtime.XDecref(pair.Value.pyHandle); + invalidClasses.Add(pair); continue; } // re-init the class InitClassBase(pair.Key.Value, pair.Value); // We modified the Type object, notify it we did. Runtime.PyType_Modified(pair.Value.tpHandle); - cache.Add(pair.Key, pair.Value); var context = contexts[pair.Value.pyHandle]; pair.Value.Load(context); loadedObjs.Add(pair.Value, context); } + + foreach (var pair in invalidClasses) + { + cache.Remove(pair.Key); + Runtime.XDecref(pair.Value.pyHandle); + } + return loadedObjs; } @@ -163,6 +170,7 @@ internal static Dictionary RestoreRuntimeData(R /// Return the ClassBase-derived instance that implements a particular /// reflected managed type, creating it if it doesn't yet exist. /// + /// A Borrowed reference to the ClassBase object internal static ClassBase GetClass(Type type) { ClassBase cb = null; @@ -519,6 +527,8 @@ private static ClassInfo GetClassInfo(Type type) } // Note the given instance might be uninitialized ob = GetClass(tp); + // GetClass returns a Borrowed ref. ci.members owns the reference. + ob.IncrRefCount(); ci.members[mi.Name] = ob; continue; } From f97262b4ba6a62eda2dae96bef200ae81e9340c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 2 Dec 2020 14:37:20 -0500 Subject: [PATCH 29/51] Code review fixes * csproj and sln files fixes * change the serialization names in Maybe* types to const strings to improve readability * make MyabeMethodBase ignore the static constructor * fix bad merge artifacts * give more information on deserialization failure of MaybeMethodBase --- pythonnet.sln | 4 +- .../Python.DomainReloadTests.15.csproj | 2 - .../Python.DomainReloadTests.csproj | 3 + .../StateSerialization/MaybeMemberInfo.cs | 17 +++-- .../StateSerialization/MaybeMethodBase.cs | 71 +++++++++---------- src/runtime/StateSerialization/MaybeType.cs | 6 +- src/runtime/arrayobject.cs | 1 - 7 files changed, 53 insertions(+), 51 deletions(-) diff --git a/pythonnet.sln b/pythonnet.sln index fdd140003..3e8a52ac5 100644 --- a/pythonnet.sln +++ b/pythonnet.sln @@ -200,8 +200,8 @@ Global {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWinPY3|x86.Build.0 = Debug|x86 {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMono|x64.ActiveCfg = Release|x64 {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMono|x86.ActiveCfg = Release|x86 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMon|x64 - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMon|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMonoPY3|x64.ActiveCfg = Release|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMonoPY3|x86.ActiveCfg = Release|x86 {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWin|x64.ActiveCfg = Release|x64 {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWin|x64.Build.0 = Release|x64 {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWin|x86.ActiveCfg = Release|x86 diff --git a/src/domain_tests/Python.DomainReloadTests.15.csproj b/src/domain_tests/Python.DomainReloadTests.15.csproj index a6953ca64..fead26717 100644 --- a/src/domain_tests/Python.DomainReloadTests.15.csproj +++ b/src/domain_tests/Python.DomainReloadTests.15.csproj @@ -28,8 +28,6 @@ XPLAT $(DefineConstants);$(BaseDefineConstants); $(DefineConstants);NETCOREAPP - $(DefineConstants);NETSTANDARD - $(DefineConstants);TRACE;DEBUG $(NuGetPackageRoot)\microsoft.targetingpack.netframework.v4.5\1.0.1\lib\net45\ diff --git a/src/domain_tests/Python.DomainReloadTests.csproj b/src/domain_tests/Python.DomainReloadTests.csproj index af454c89d..1777bf7ce 100644 --- a/src/domain_tests/Python.DomainReloadTests.csproj +++ b/src/domain_tests/Python.DomainReloadTests.csproj @@ -35,5 +35,8 @@ + + + \ No newline at end of file diff --git a/src/runtime/StateSerialization/MaybeMemberInfo.cs b/src/runtime/StateSerialization/MaybeMemberInfo.cs index 610bc967d..4305581b6 100644 --- a/src/runtime/StateSerialization/MaybeMemberInfo.cs +++ b/src/runtime/StateSerialization/MaybeMemberInfo.cs @@ -11,6 +11,11 @@ internal struct MaybeMemberInfo : ISerializable where T: MemberInfo { public static implicit operator MaybeMemberInfo (T ob) => new MaybeMemberInfo(ob); + // .ToString() of the serialized object + const string SerializationName = "s"; + // The ReflectedType of the object + const string SerializationType = "t"; + const string SerializationFieldName = "f"; string name; MemberInfo info; @@ -55,15 +60,15 @@ public MaybeMemberInfo(T fi) internal MaybeMemberInfo(SerializationInfo serializationInfo, StreamingContext context) { // Assumption: name is always stored in "s" - name = serializationInfo.GetString("s"); + name = serializationInfo.GetString(SerializationName); info = null; deserializationException = null; try { - var tp = Type.GetType(serializationInfo.GetString("t")); + var tp = Type.GetType(serializationInfo.GetString(SerializationType)); if (tp != null) { - var field_name = serializationInfo.GetString("f"); + var field_name = serializationInfo.GetString(SerializationFieldName); MemberInfo mi = tp.GetField(field_name, ClassManager.BindingFlags); if (mi != null && ShouldBindMember(mi)) { @@ -102,11 +107,11 @@ static bool ShouldBindMember(MemberInfo mi) public void GetObjectData(SerializationInfo serializationInfo, StreamingContext context) { - serializationInfo.AddValue("s", name); + serializationInfo.AddValue(SerializationName, name); if (Valid) { - serializationInfo.AddValue("f", info.Name); - serializationInfo.AddValue("t", info.ReflectedType.AssemblyQualifiedName); + serializationInfo.AddValue(SerializationFieldName, info.Name); + serializationInfo.AddValue(SerializationType, info.ReflectedType.AssemblyQualifiedName); } } } diff --git a/src/runtime/StateSerialization/MaybeMethodBase.cs b/src/runtime/StateSerialization/MaybeMethodBase.cs index 695819561..efde4bb23 100644 --- a/src/runtime/StateSerialization/MaybeMethodBase.cs +++ b/src/runtime/StateSerialization/MaybeMethodBase.cs @@ -8,6 +8,15 @@ namespace Python.Runtime [Serializable] internal struct MaybeMethodBase : ISerializable where T: MethodBase { + // .ToString() of the serialized object + const string SerializationName = "s"; + // The ReflectedType of the object + const string SerializationType = "t"; + // Fhe parameters of the MethodBase + const string SerializationParameters = "p"; + const string SerializationIsCtor = "c"; + const string SerializationMethodName = "n"; + [Serializable] struct ParameterHelper : IEquatable { @@ -93,32 +102,42 @@ public MaybeMethodBase(T mi) internal MaybeMethodBase(SerializationInfo serializationInfo, StreamingContext context) { - name = serializationInfo.GetString("s"); + name = serializationInfo.GetString(SerializationName); info = null; deserializationException = null; try { // Retrive the reflected type of the method; - var tp = Type.GetType(serializationInfo.GetString("t")); + var typeName = serializationInfo.GetString(SerializationType); + var tp = Type.GetType(typeName); + if (tp == null) + { + throw new SerializationException($"The underlying type {typeName} can't be found"); + } // Get the method's parameters types - var field_name = serializationInfo.GetString("f"); - var param = (ParameterHelper[])serializationInfo.GetValue("p", typeof(ParameterHelper[])); + var field_name = serializationInfo.GetString(SerializationMethodName); + var param = (ParameterHelper[])serializationInfo.GetValue(SerializationParameters, typeof(ParameterHelper[])); Type[] types = new Type[param.Length]; bool hasRefType = false; for (int i = 0; i < param.Length; i++) { - types[i] = Type.GetType(param[i].Name); - if (types[i].IsByRef) + var paramTypeName = param[i].Name; + types[i] = Type.GetType(paramTypeName); + if (types[i] == null) + { + throw new SerializationException($"The parameter of type {paramTypeName} can't be found"); + } + else if (types[i].IsByRef) { hasRefType = true; } } MethodBase mb = null; - var isCtor = serializationInfo.GetBoolean("c"); - if (isCtor) + if (serializationInfo.GetBoolean(SerializationIsCtor)) { - mb = ResolveConstructor(tp, types); + // We never want the static constructor. + mb = tp.GetConstructor(ClassManager.BindingFlags&(~BindingFlags.Instance), binder:null, types:types, modifiers:null); } else { @@ -142,30 +161,6 @@ internal MaybeMethodBase(SerializationInfo serializationInfo, StreamingContext c } } - MethodBase ResolveConstructor (Type tp, Type[] parameters) - { - MethodBase mb = null; - try - { - mb = tp.GetConstructor(ClassManager.BindingFlags, binder:null, types:parameters, modifiers:null); - } - catch (AmbiguousMatchException) - { - // The static constructor and non-static constructor may - // have the same signature. Use ToString to disambiguate. - var mbs = tp.GetConstructors(ClassManager.BindingFlags); - foreach (var ctor in mbs) - { - if (name == ctor.ToString()) - { - mb = ctor; - break; - } - } - } - return mb; - } - MethodBase CheckRefTypes(MethodBase mb, ParameterHelper[] ph) { // One more step: Changing: @@ -192,14 +187,14 @@ MethodBase CheckRefTypes(MethodBase mb, ParameterHelper[] ph) public void GetObjectData(SerializationInfo serializationInfo, StreamingContext context) { - serializationInfo.AddValue("s", name); + serializationInfo.AddValue(SerializationName, name); if (Valid) { - serializationInfo.AddValue("f", info.Name); - serializationInfo.AddValue("t", info.ReflectedType.AssemblyQualifiedName); + serializationInfo.AddValue(SerializationMethodName, info.Name); + serializationInfo.AddValue(SerializationType, info.ReflectedType.AssemblyQualifiedName); ParameterHelper[] parameters = (from p in info.GetParameters() select new ParameterHelper(p)).ToArray(); - serializationInfo.AddValue("p", parameters, typeof(ParameterHelper[])); - serializationInfo.AddValue("c", info.IsConstructor); + serializationInfo.AddValue(SerializationParameters, parameters, typeof(ParameterHelper[])); + serializationInfo.AddValue(SerializationIsCtor, info.IsConstructor); } } } diff --git a/src/runtime/StateSerialization/MaybeType.cs b/src/runtime/StateSerialization/MaybeType.cs index 4ce5a3827..a12d2d6e0 100644 --- a/src/runtime/StateSerialization/MaybeType.cs +++ b/src/runtime/StateSerialization/MaybeType.cs @@ -11,6 +11,8 @@ internal struct MaybeType : ISerializable { public static implicit operator MaybeType (Type ob) => new MaybeType(ob); + // The AssemblyQualifiedName of the serialized Type + const string SerializationName = "n"; string name; Type type; @@ -50,13 +52,13 @@ public MaybeType(Type tp) private MaybeType(SerializationInfo serializationInfo, StreamingContext context) { - name = (string)serializationInfo.GetValue("n", typeof(string)); + name = (string)serializationInfo.GetValue(SerializationName, typeof(string)); type = Type.GetType(name, throwOnError:false); } public void GetObjectData(SerializationInfo serializationInfo, StreamingContext context) { - serializationInfo.AddValue("n", name); + serializationInfo.AddValue(SerializationName, name); } } } \ No newline at end of file diff --git a/src/runtime/arrayobject.cs b/src/runtime/arrayobject.cs index 049d2b8bb..ff745a270 100644 --- a/src/runtime/arrayobject.cs +++ b/src/runtime/arrayobject.cs @@ -37,7 +37,6 @@ public static IntPtr tp_new(IntPtr tpRaw, IntPtr args, IntPtr kw) long[] dimensions = new long[Runtime.PyTuple_Size(args)]; if (dimensions.Length == 0) - if (Runtime.PyTuple_Size(args) != 1) { return Exceptions.RaiseTypeError("array constructor requires at least one integer argument or an object convertible to array"); } From 2d6ae4c12ae403ccce5dd92eb934d7d01749d3a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 2 Dec 2020 15:00:03 -0500 Subject: [PATCH 30/51] fixup! Merge branch 'master' into domain-reload-test-cases-fixes --- src/domain_tests/Python.DomainReloadTests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain_tests/Python.DomainReloadTests.csproj b/src/domain_tests/Python.DomainReloadTests.csproj index 1777bf7ce..9107c4111 100644 --- a/src/domain_tests/Python.DomainReloadTests.csproj +++ b/src/domain_tests/Python.DomainReloadTests.csproj @@ -36,7 +36,7 @@ - + \ No newline at end of file From 16a39a6e27a618f0ad2679bdc7b8df7dda817ced Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 2 Dec 2020 15:03:42 -0500 Subject: [PATCH 31/51] fixup! Code review fixes --- src/runtime/StateSerialization/MaybeMethodBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/StateSerialization/MaybeMethodBase.cs b/src/runtime/StateSerialization/MaybeMethodBase.cs index efde4bb23..e939cf416 100644 --- a/src/runtime/StateSerialization/MaybeMethodBase.cs +++ b/src/runtime/StateSerialization/MaybeMethodBase.cs @@ -137,7 +137,7 @@ internal MaybeMethodBase(SerializationInfo serializationInfo, StreamingContext c if (serializationInfo.GetBoolean(SerializationIsCtor)) { // We never want the static constructor. - mb = tp.GetConstructor(ClassManager.BindingFlags&(~BindingFlags.Instance), binder:null, types:types, modifiers:null); + mb = tp.GetConstructor(ClassManager.BindingFlags&(~BindingFlags.Static), binder:null, types:types, modifiers:null); } else { From ace340d12f8d43c4d073a7e1a53474282656c3c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Tue, 8 Dec 2020 09:53:54 -0500 Subject: [PATCH 32/51] Fix build failures on non-windows --- pythonnet.sln | 8 ++++++++ src/domain_tests/Python.DomainReloadTests.15.csproj | 4 ++++ src/domain_tests/test_domain_reload.py | 11 +++++++---- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/pythonnet.sln b/pythonnet.sln index 3e8a52ac5..42cf14b05 100644 --- a/pythonnet.sln +++ b/pythonnet.sln @@ -187,9 +187,13 @@ Global {86E834DE-1139-4511-96CC-69636A56E7AC}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 {86E834DE-1139-4511-96CC-69636A56E7AC}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugMono|x64.ActiveCfg = Debug|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugMono|x64.Build.0 = Debug|x64 {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugMono|x86.ActiveCfg = Debug|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugMono|x86.Build.0 = Debug|x86 {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugMonoPY3|x64.ActiveCfg = Debug|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugMonoPY3|x64.Build.0 = Debug|x64 {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugMonoPY3|x86.ActiveCfg = Debug|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugMonoPY3|x86.Build.0 = Debug|x86 {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWin|x64.ActiveCfg = Debug|x64 {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWin|x64.Build.0 = Debug|x64 {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWin|x86.ActiveCfg = Debug|x86 @@ -199,9 +203,13 @@ Global {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWinPY3|x86.ActiveCfg = Debug|x86 {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.DebugWinPY3|x86.Build.0 = Debug|x86 {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMono|x64.ActiveCfg = Release|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMono|x64.Build.0 = Release|x64 {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMono|x86.ActiveCfg = Release|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMono|x86.Build.0 = Release|x86 {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMonoPY3|x64.ActiveCfg = Release|x64 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMonoPY3|x64.Build.0 = Release|x64 {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMonoPY3|x86.ActiveCfg = Release|x86 + {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseMonoPY3|x86.Build.0 = Release|x86 {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWin|x64.ActiveCfg = Release|x64 {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWin|x64.Build.0 = Release|x64 {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}.ReleaseWin|x86.ActiveCfg = Release|x86 diff --git a/src/domain_tests/Python.DomainReloadTests.15.csproj b/src/domain_tests/Python.DomainReloadTests.15.csproj index fead26717..4af49b8ca 100644 --- a/src/domain_tests/Python.DomainReloadTests.15.csproj +++ b/src/domain_tests/Python.DomainReloadTests.15.csproj @@ -50,6 +50,10 @@ + + + + diff --git a/src/domain_tests/test_domain_reload.py b/src/domain_tests/test_domain_reload.py index c040931b2..0e66bc2c8 100644 --- a/src/domain_tests/test_domain_reload.py +++ b/src/domain_tests/test_domain_reload.py @@ -1,15 +1,18 @@ import subprocess import os +import platform import pytest def _run_test(testname): dirname = os.path.split(__file__)[0] exename = os.path.join(dirname, 'bin', 'Python.DomainReloadTests.exe') - proc = subprocess.Popen([ - exename, - testname, - ]) + args = [exename, testname] + + if platform.system() != 'Windows': + args = ['mono'] + args + + proc = subprocess.Popen(args) proc.wait() assert proc.returncode == 0 From 78a808813e8634c050fbc511b60a52bc2dea813f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Tue, 8 Dec 2020 10:06:05 -0500 Subject: [PATCH 33/51] fixup! Fix build failures on non-windows --- src/domain_tests/Python.DomainReloadTests.csproj | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/domain_tests/Python.DomainReloadTests.csproj b/src/domain_tests/Python.DomainReloadTests.csproj index 9107c4111..af454c89d 100644 --- a/src/domain_tests/Python.DomainReloadTests.csproj +++ b/src/domain_tests/Python.DomainReloadTests.csproj @@ -35,8 +35,5 @@ - - - \ No newline at end of file From 79516f1984ae310d412c62cf6cb4c01655ea7c49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 10 Dec 2020 11:55:15 -0500 Subject: [PATCH 34/51] Code review fixes --- src/domain_tests/TestRunner.cs | 30 +++++++------------ .../StateSerialization/MaybeMethodBase.cs | 6 ++-- src/runtime/arrayobject.cs | 7 +++-- src/runtime/constructorbinder.cs | 7 +++-- src/runtime/constructorbinding.cs | 10 +++++-- src/runtime/converter.cs | 8 ++++- src/runtime/delegateobject.cs | 3 +- src/runtime/methodbinding.cs | 5 ++-- src/runtime/moduleobject.cs | 2 +- src/runtime/propertyobject.cs | 20 +++++++------ 10 files changed, 54 insertions(+), 44 deletions(-) diff --git a/src/domain_tests/TestRunner.cs b/src/domain_tests/TestRunner.cs index d5fa083e3..96aadca05 100644 --- a/src/domain_tests/TestRunner.cs +++ b/src/domain_tests/TestRunner.cs @@ -16,22 +16,22 @@ namespace Python.DomainReloadTests /// 3. This class at runtime creates a directory that has both C# and /// python code, and compiles the C#. /// 4. This class then runs the C# code. - /// - /// But wait there's more indirection. The C# code that's run -- known as - /// the test runner -- - /// This class compiles a DLL that contains the class which code will change - /// and a runner executable that will run Python code referencing the class. - /// Each test case: + /// + /// But there's a bit more indirection. This class compiles a DLL that + /// contains code that will change. + /// Then, the test case: /// * Compiles some code, loads it into a domain, runs python that refers to it. - /// * Unload the domain. - /// * Compile a new piece of code, load it into a domain, run a new piece of python that accesses the code. + /// * Unload the domain, re-runs the domain to make sure domain reload happens correctly. + /// * Compile a new piece of code, load it into a new domain, run a new piece of + /// Python code to test the objects after they've been deleted or modified in C#. /// * Unload the domain. Reload the domain, run the same python again. + /// /// This class gets built into an executable which takes one argument: /// which test case to run. That's because pytest assumes we'll run /// everything in one process, but we really want a clean process on each - /// test case to test the init/reload/teardown parts of the domain reload - /// code. + /// test case to test the init/reload/teardown parts of the domain reload. /// + /// class TestRunner { const string TestAssemblyName = "DomainTests"; @@ -347,12 +347,6 @@ assert called is True called = False Cls.Call() assert called is False - #try: - # assert 2 == Cls.Before - #except TypeError: - # print('Caught expected exception') - #else: - # raise AssertionError('Failed to throw exception') ", }, @@ -398,7 +392,6 @@ def before_reload(): def after_reload(): try: TestNamespace.Cls(2) - sys.my_cls.Member() except AttributeError: print('Caught expected exception') else: @@ -845,7 +838,7 @@ raise AssertionError('failed to raise') # foo should have changed assert foo.num == 7 assert bar.num == 7 - # Pythonnet also returns a new object with `ref`-quialified parameters + # Pythonnet also returns a new object with `ref`-qualified parameters assert foo is not bar ", }, @@ -1029,7 +1022,6 @@ static string CreateCaseRunnerAssembly(string verb, string shutdownMode = "Shutd } static string CreateAssembly(string name, string code, bool exe = false) { - // Console.WriteLine(code); // Never return or hold the Assembly instance. This will cause // the assembly to be loaded into the current domain and this // interferes with the tests. The Domain can execute fine from a diff --git a/src/runtime/StateSerialization/MaybeMethodBase.cs b/src/runtime/StateSerialization/MaybeMethodBase.cs index dcc9fb591..3f57f0d8a 100644 --- a/src/runtime/StateSerialization/MaybeMethodBase.cs +++ b/src/runtime/StateSerialization/MaybeMethodBase.cs @@ -105,7 +105,7 @@ internal MaybeMethodBase(SerializationInfo serializationInfo, StreamingContext c deserializationException = null; try { - // Retrive the reflected type of the method; + // Retrieve the reflected type of the method; var typeName = serializationInfo.GetString(SerializationType); var tp = Type.GetType(typeName); if (tp == null) @@ -144,7 +144,7 @@ internal MaybeMethodBase(SerializationInfo serializationInfo, StreamingContext c if (mb != null && hasRefType) { - CheckRefTypes(mb, param); + mb = CheckRefTypes(mb, param); } // Do like in ClassManager.GetClassInfo @@ -165,7 +165,7 @@ MethodBase CheckRefTypes(MethodBase mb, ParameterHelper[] ph) // void MyFn (ref int a) // to: // void MyFn (out int a) - // will still find the fucntion correctly as, `in`, `out` and `ref` + // will still find the function correctly as, `in`, `out` and `ref` // are all represented as a reference type. Query the method we got // and validate the parameters if (ph.Length != 0) diff --git a/src/runtime/arrayobject.cs b/src/runtime/arrayobject.cs index ff745a270..262e521a5 100644 --- a/src/runtime/arrayobject.cs +++ b/src/runtime/arrayobject.cs @@ -34,6 +34,7 @@ public static IntPtr tp_new(IntPtr tpRaw, IntPtr args, IntPtr kw) { return Exceptions.RaiseTypeError(self.type.DeletedMessage); } + Type arrType = self.type.Value; long[] dimensions = new long[Runtime.PyTuple_Size(args)]; if (dimensions.Length == 0) @@ -42,7 +43,7 @@ public static IntPtr tp_new(IntPtr tpRaw, IntPtr args, IntPtr kw) } if (dimensions.Length != 1) { - return CreateMultidimensional(self.type.Value.GetElementType(), dimensions, + return CreateMultidimensional(arrType.GetElementType(), dimensions, shapeTuple: new BorrowedReference(args), pyType: tp) .DangerousMoveToPointerOrNull(); @@ -60,14 +61,14 @@ public static IntPtr tp_new(IntPtr tpRaw, IntPtr args, IntPtr kw) } else { - return NewInstance(self.type.Value.GetElementType(), tp, dimensions) + return NewInstance(arrType.GetElementType(), tp, dimensions) .DangerousMoveToPointerOrNull(); } } object result; // this implements casting to Array[T] - if (!Converter.ToManaged(op, self.type.Value, out result, true)) + if (!Converter.ToManaged(op, arrType, out result, true)) { return IntPtr.Zero; } diff --git a/src/runtime/constructorbinder.cs b/src/runtime/constructorbinder.cs index 52e133983..83f2c81e4 100644 --- a/src/runtime/constructorbinder.cs +++ b/src/runtime/constructorbinder.cs @@ -56,9 +56,10 @@ internal object InvokeRaw(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info) return Exceptions.RaiseTypeError(_containingType.DeletedMessage); } object result; + Type tp = _containingType.Value; - if (_containingType.Value.IsValueType && !_containingType.Value.IsPrimitive && - !_containingType.Value.IsEnum && _containingType.Value != typeof(decimal) && + if (tp.IsValueType && !tp.IsPrimitive && + !tp.IsEnum && tp != typeof(decimal) && Runtime.PyTuple_Size(args) == 0) { // If you are trying to construct an instance of a struct by @@ -68,7 +69,7 @@ internal object InvokeRaw(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info) // Activator.CreateInstance(). try { - result = Activator.CreateInstance(_containingType.Value); + result = Activator.CreateInstance(tp); } catch (Exception e) { diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index 354ed4109..803823e39 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -96,6 +96,7 @@ public static IntPtr mp_subscript(IntPtr op, IntPtr key) { return Exceptions.RaiseTypeError(self.type.DeletedMessage); } + Type tp = self.type.Value; Type[] types = Runtime.PythonArgsToTypeArray(key); if (types == null) @@ -104,12 +105,12 @@ public static IntPtr mp_subscript(IntPtr op, IntPtr key) } //MethodBase[] methBaseArray = self.ctorBinder.GetMethods(); //MethodBase ci = MatchSignature(methBaseArray, types); - ConstructorInfo ci = self.type.Value.GetConstructor(types); + ConstructorInfo ci = tp.GetConstructor(types); if (ci == null) { return Exceptions.RaiseTypeError("No match found for constructor signature"); } - var boundCtor = new BoundContructor(self.type.Value, self.pyTypeHndl, self.ctorBinder, ci); + var boundCtor = new BoundContructor(tp, self.pyTypeHndl, self.ctorBinder, ci); return boundCtor.pyHandle; } @@ -126,6 +127,11 @@ public static IntPtr tp_repr(IntPtr ob) return self.repr; } MethodBase[] methods = self.ctorBinder.GetMethods(); + + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } string name = self.type.Value.FullName; var doc = ""; foreach (MethodBase t in methods) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 5b7998015..6c22484d7 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -344,7 +344,13 @@ internal static bool ToManagedValue(IntPtr value, Type obType, } if (mt is ClassBase) { - result = ((ClassBase)mt).type.Value; + var cb = (ClassBase)mt; + if (!cb.type.Valid) + { + Exceptions.SetError(Exceptions.TypeError, cb.type.DeletedMessage); + return false; + } + result = cb.type.Value; return true; } // shouldn't happen diff --git a/src/runtime/delegateobject.cs b/src/runtime/delegateobject.cs index 8379958f9..e0d29f1a0 100644 --- a/src/runtime/delegateobject.cs +++ b/src/runtime/delegateobject.cs @@ -56,6 +56,7 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { return Exceptions.RaiseTypeError(self.type.DeletedMessage); } + Type type = self.type.Value; if (Runtime.PyTuple_Size(args) != 1) { @@ -69,7 +70,7 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) return Exceptions.RaiseTypeError("argument must be callable"); } - Delegate d = PythonEngine.DelegateManager.GetDelegate(self.type.Value, method); + Delegate d = PythonEngine.DelegateManager.GetDelegate(type, method); return CLRObject.GetInstHandle(d, self.pyHandle); } diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index d94bd2f92..46b62807d 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -114,13 +114,14 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) // for example this method in the tests: string Overloaded(int arg1, int arg2, string arg3) if (self.info.Valid) { - if (self.info.Value.IsGenericMethod) + var info = self.info.Value; + if (info.IsGenericMethod) { var len = Runtime.PyTuple_Size(args); //FIXME: Never used Type[] sigTp = Runtime.PythonArgsToTypeArray(args, true); if (sigTp != null) { - Type[] genericTp = self.info.Value.GetGenericArguments(); + Type[] genericTp = info.GetGenericArguments(); MethodInfo betterMatch = MethodBinder.MatchSignatureAndParameters(self.m.info, genericTp, sigTp); if (betterMatch != null) { diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index def33b6c5..941b032a0 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -346,7 +346,7 @@ protected override void OnSave(InterDomainContext context) { Runtime.PyDict_DelItemString(dict, pair.Key); pair.Value.DecrRefCount(); - if (Exceptions.ExceptionMatches(Exceptions.KeyError)) + if (Exceptions.ExceptionMatches(Exceptions.KeyError)) { // Trying to remove a key that's not in the dictionary // raises an error. We don't care about it. diff --git a/src/runtime/propertyobject.cs b/src/runtime/propertyobject.cs index e01a0d877..20061b358 100644 --- a/src/runtime/propertyobject.cs +++ b/src/runtime/propertyobject.cs @@ -36,7 +36,8 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) { return Exceptions.RaiseTypeError(self.info.DeletedMessage); } - MethodInfo getter = self.getter.Value; + var info = self.info.Value; + MethodInfo getter = self.getter.UnsafeValue; object result; @@ -56,8 +57,8 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) try { - result = self.info.Value.GetValue(null, null); - return Converter.ToPython(result, self.info.Value.PropertyType); + result = info.GetValue(null, null); + return Converter.ToPython(result, info.PropertyType); } catch (Exception e) { @@ -73,8 +74,8 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) try { - result = self.info.Value.GetValue(co.inst, null); - return Converter.ToPython(result, self.info.Value.PropertyType); + result = info.GetValue(co.inst, null); + return Converter.ToPython(result, info.PropertyType); } catch (Exception e) { @@ -101,8 +102,9 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) Exceptions.RaiseTypeError(self.info.DeletedMessage); return -1; } + var info = self.info.Value; - MethodInfo setter = self.setter.Value; + MethodInfo setter = self.setter.UnsafeValue; object newval; if (val == IntPtr.Zero) @@ -118,7 +120,7 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) } - if (!Converter.ToManaged(val, self.info.Value.PropertyType, out newval, true)) + if (!Converter.ToManaged(val, info.PropertyType, out newval, true)) { return -1; } @@ -144,11 +146,11 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) Exceptions.RaiseTypeError("invalid target"); return -1; } - self.info.Value.SetValue(co.inst, newval, null); + info.SetValue(co.inst, newval, null); } else { - self.info.Value.SetValue(null, newval, null); + info.SetValue(null, newval, null); } return 0; } From 5e4c9761737ef7e612911febe0bc4481b7ee35aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Fri, 11 Dec 2020 11:43:15 -0500 Subject: [PATCH 35/51] (WIP) rework project structure --- pythonnet.sln | 16 ++++- .../Python.DomainReloadTests.15.csproj | 68 ------------------- .../Python.DomainReloadTests.csproj | 41 +++++------ 3 files changed, 30 insertions(+), 95 deletions(-) delete mode 100644 src/domain_tests/Python.DomainReloadTests.15.csproj diff --git a/pythonnet.sln b/pythonnet.sln index 9379cc5f5..b1d50ec7d 100644 --- a/pythonnet.sln +++ b/pythonnet.sln @@ -14,6 +14,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Test", "src\testing\ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.PerformanceTests", "src\perf_tests\Python.PerformanceTests.csproj", "{4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.DomainReloadTests", "src\domain_tests\Python.DomainReloadTests.csproj", "{F2FB6DA3-318E-4F30-9A1F-932C667E38C5}" +EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Repo", "Repo", "{441A0123-F4C6-4EE4-9AEE-315FD79BE2D5}" ProjectSection(SolutionItems) = preProject .editorconfig = .editorconfig @@ -47,8 +49,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tools", "Tools", "{BC426F42 tools\geninterop\geninterop.py = tools\geninterop\geninterop.py EndProjectSection EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Python.DomainReloadTests", "src\domain_tests\Python.DomainReloadTests.csproj", "{7539EBD5-7A15-4513-BCB0-1D9BEF112BC2}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -131,6 +131,18 @@ Global {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Release|x64.Build.0 = Release|x64 {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Release|x86.ActiveCfg = Release|x86 {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Release|x86.Build.0 = Release|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x64.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x64.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x86.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x86.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|Any CPU.Build.0 = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x64.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x64.Build.0 = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x86.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/domain_tests/Python.DomainReloadTests.15.csproj b/src/domain_tests/Python.DomainReloadTests.15.csproj deleted file mode 100644 index 4af49b8ca..000000000 --- a/src/domain_tests/Python.DomainReloadTests.15.csproj +++ /dev/null @@ -1,68 +0,0 @@ - - - - - net40;netcoreapp3.1 - x64;x86 - Debug;Release - Exe - false - Python.DomainReloadTests - Python.DomainReloadTests - Python.DomainReloadTests - 2.5.0 - false - false - false - false - bin\ - false - $(OutputPath)\$(AssemblyName).xml - $(OutputPath)\$(TargetFramework)\$(AssemblyName).xml - 1591 - ..\..\ - $(SolutionDir)\bin\ - $(OutputPath)\$(TargetFramework)_publish - 7.3 - prompt - XPLAT - $(DefineConstants);$(BaseDefineConstants); - $(DefineConstants);NETCOREAPP - $(NuGetPackageRoot)\microsoft.targetingpack.netframework.v4.5\1.0.1\lib\net45\ - - - x86 - - - x64 - - - - - - - - - - - - - - - - - - - - - - - - - - - $(TargetPath) - $(TargetDir)$(TargetName).pdb - - - diff --git a/src/domain_tests/Python.DomainReloadTests.csproj b/src/domain_tests/Python.DomainReloadTests.csproj index af454c89d..db68267f7 100644 --- a/src/domain_tests/Python.DomainReloadTests.csproj +++ b/src/domain_tests/Python.DomainReloadTests.csproj @@ -1,39 +1,30 @@ - - - + + - Debug - AnyCPU - {7539EBD5-7A15-4513-BCB0-1D9BEF112BC2} - Exe - Python.DomainReloadTests - Python.DomainReloadTests - v4.0 + net472;netcoreapp3.1 bin\ - 512 - true - true - - - x86 - - - x64 + - + + + - + - - + + + - - \ No newline at end of file + + + From bcf0cd6d74e6ef61e7ece033aebd3c7fa7afefe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 14 Dec 2020 12:50:21 -0500 Subject: [PATCH 36/51] Rework the projects file structure Reworks the domain reload tests project files to fit the new sdk style. --- src/domain_tests/Python.DomainReloadTests.csproj | 10 +++++----- src/domain_tests/TestRunner.cs | 8 +++----- src/domain_tests/conftest.py | 7 +++++++ 3 files changed, 15 insertions(+), 10 deletions(-) create mode 100644 src/domain_tests/conftest.py diff --git a/src/domain_tests/Python.DomainReloadTests.csproj b/src/domain_tests/Python.DomainReloadTests.csproj index db68267f7..263f83663 100644 --- a/src/domain_tests/Python.DomainReloadTests.csproj +++ b/src/domain_tests/Python.DomainReloadTests.csproj @@ -1,8 +1,9 @@ - net472;netcoreapp3.1 + net472 bin\ + Exe @@ -14,9 +15,9 @@ - + + + @@ -26,5 +27,4 @@ - diff --git a/src/domain_tests/TestRunner.cs b/src/domain_tests/TestRunner.cs index 96aadca05..c734dac88 100644 --- a/src/domain_tests/TestRunner.cs +++ b/src/domain_tests/TestRunner.cs @@ -935,13 +935,11 @@ public static int Main() }} }} "; - readonly static string PythonDllLocation = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "../../runtime/bin/Python.Runtime.dll"); + // readonly static string PythonDllLocation = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "../../runtime/bin/Python.Runtime.dll"); + readonly static string PythonDllLocation = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Python.Runtime.dll"); public static int Main(string[] args) { -// We require this slightly convoluted way of ifdef'ing because the Python -// comments '#' are mistaken as C# preprocessior directive -#if !NETCOREAPP TestCase testCase; if (args.Length < 1) { @@ -991,7 +989,6 @@ public static int Main(string[] args) RunAndUnload(runnerDomain, runnerAssembly); } } -#endif return 0; } #if !NETCOREAPP @@ -1035,6 +1032,7 @@ static string CreateAssembly(string name, string code, bool exe = false) parameters.ReferencedAssemblies.Add("System.dll"); parameters.ReferencedAssemblies.Add("System.Core.dll"); parameters.ReferencedAssemblies.Add("Microsoft.CSharp.dll"); + parameters.ReferencedAssemblies.Add("netstandard.dll"); parameters.ReferencedAssemblies.Add(PythonDllLocation); CompilerResults results = provider.CompileAssemblyFromSource(parameters, code); if (results.NativeCompilerReturnValue != 0) diff --git a/src/domain_tests/conftest.py b/src/domain_tests/conftest.py new file mode 100644 index 000000000..5f0d52e10 --- /dev/null +++ b/src/domain_tests/conftest.py @@ -0,0 +1,7 @@ +import os + +from subprocess import check_call +# test_proj_path = os.path.join(cwd, "..", "testing") +cfd = os.path.dirname(__file__) +bin_path = os.path.join(cfd, 'bin') +check_call(["dotnet", "build", cfd, '-o', bin_path]) \ No newline at end of file From 73e5a6b33389405c48236447f9b61ceb3560db43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 14 Dec 2020 12:58:06 -0500 Subject: [PATCH 37/51] Check teh return value of PyDict_DelItemString --- src/runtime/classmanager.cs | 4 ++-- src/runtime/moduleobject.cs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 8cfaaad2f..650ba5ba3 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -122,8 +122,8 @@ internal static void SaveRuntimeData(RuntimeDataStorage storage) { // No need to decref the member, the ClassBase instance does // not own the reference. - Runtime.PyDict_DelItemString(dict, member); - if (Exceptions.ExceptionMatches(Exceptions.KeyError)) + if ((Runtime.PyDict_DelItemString(dict, member) == -1) && + (Exceptions.ExceptionMatches(Exceptions.KeyError))) { // Trying to remove a key that's not in the dictionary // raises an error. We don't care about it. diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 941b032a0..0dbad6daf 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -344,14 +344,14 @@ protected override void OnSave(InterDomainContext context) // destroy the cache(s) foreach (var pair in cache) { - Runtime.PyDict_DelItemString(dict, pair.Key); - pair.Value.DecrRefCount(); - if (Exceptions.ExceptionMatches(Exceptions.KeyError)) + if ((Runtime.PyDict_DelItemString(dict, pair.Key) == -1) && + (Exceptions.ExceptionMatches(Exceptions.KeyError))) { // Trying to remove a key that's not in the dictionary // raises an error. We don't care about it. Runtime.PyErr_Clear(); } + pair.Value.DecrRefCount(); } cache.Clear(); From 510a7aea981de00e0e49334364ba05192e646713 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 16 Dec 2020 15:30:36 -0500 Subject: [PATCH 38/51] netstandard.dll is a Facade Library --- src/domain_tests/TestRunner.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/domain_tests/TestRunner.cs b/src/domain_tests/TestRunner.cs index c734dac88..84bbe3420 100644 --- a/src/domain_tests/TestRunner.cs +++ b/src/domain_tests/TestRunner.cs @@ -1032,7 +1032,12 @@ static string CreateAssembly(string name, string code, bool exe = false) parameters.ReferencedAssemblies.Add("System.dll"); parameters.ReferencedAssemblies.Add("System.Core.dll"); parameters.ReferencedAssemblies.Add("Microsoft.CSharp.dll"); - parameters.ReferencedAssemblies.Add("netstandard.dll"); + var netstandard = "netstandard.dll"; + if (Type.GetType("Mono.Runtime") != null) + { + netstandard = "Facades/" + netstandard; + } + parameters.ReferencedAssemblies.Add(netstandard); parameters.ReferencedAssemblies.Add(PythonDllLocation); CompilerResults results = provider.CompileAssemblyFromSource(parameters, code); if (results.NativeCompilerReturnValue != 0) From 21eb14cb88ce621e601fac12937812d8735c34a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 17 Dec 2020 11:42:16 -0500 Subject: [PATCH 39/51] Remove TestClassReference The test case is covered in test_domain_relaod --- src/domain_tests/TestRunner.cs | 9 +-- src/embed_tests/TestDomainReload.cs | 91 ----------------------------- 2 files changed, 5 insertions(+), 95 deletions(-) diff --git a/src/domain_tests/TestRunner.cs b/src/domain_tests/TestRunner.cs index 84bbe3420..e494992a5 100644 --- a/src/domain_tests/TestRunner.cs +++ b/src/domain_tests/TestRunner.cs @@ -125,10 +125,11 @@ import sys import TestNamespace def before_reload(): - sys.my_cls = TestNamespace.Cls - sys.my_fn = TestNamespace.Cls.Before - sys.my_fn() - TestNamespace.Cls.Before() + if not hasattr(sys, 'my_cls'): + sys.my_cls = TestNamespace.Cls + sys.my_fn = TestNamespace.Cls.Before + assert 5 == sys.my_fn() + assert 5 == TestNamespace.Cls.Before() def after_reload(): diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 3e0a18c70..f8445edb4 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -179,97 +179,6 @@ public static void CrossDomainObject() #endregion - #region TestClassReference - - class ReloadClassRefStep1 : CrossCaller - { - public override ValueType Execute(ValueType arg) - { - const string code = @" -from Python.EmbeddingTest.Domain import MyClass - -def test_obj_call(): - obj = MyClass() - obj.Method() - MyClass.StaticMethod() - obj.Property = 1 - obj.Field = 10 - -test_obj_call() -"; - const string name = "test_domain_reload_mod"; - using (Py.GIL()) - { - // Create a new module - IntPtr module = PyRuntime.PyModule_New(name); - Assert.That(module != IntPtr.Zero); - IntPtr globals = PyRuntime.PyObject_GetAttr(module, PyIdentifier.__dict__); - Assert.That(globals != IntPtr.Zero); - try - { - // import builtins - // module.__dict__[__builtins__] = builtins - int res = PyRuntime.PyDict_SetItem(globals, PyIdentifier.__builtins__, - PyRuntime.PyEval_GetBuiltins()); - PythonException.ThrowIfIsNotZero(res); - - // Execute the code in the module's scope - PythonEngine.Exec(code, globals); - // import sys - // modules = sys.modules - IntPtr modules = PyRuntime.PyImport_GetModuleDict(); - // modules[name] = module - res = PyRuntime.PyDict_SetItemString(modules, name, module); - PythonException.ThrowIfIsNotZero(res); - } - catch - { - PyRuntime.XDecref(module); - throw; - } - finally - { - PyRuntime.XDecref(globals); - } - return module; - } - } - } - - class ReloadClassRefStep2 : CrossCaller - { - public override ValueType Execute(ValueType arg) - { - var module = (IntPtr)arg; - using (Py.GIL()) - { - var test_obj_call = PyRuntime.PyObject_GetAttrString(module, "test_obj_call"); - PythonException.ThrowIfIsNull(test_obj_call); - var args = PyRuntime.PyTuple_New(0); - var res = PyRuntime.PyObject_CallObject(test_obj_call, args); - PythonException.ThrowIfIsNull(res); - - PyRuntime.XDecref(args); - PyRuntime.XDecref(res); - } - return 0; - } - } - - - [Test] - /// - /// Create a new Python module, define a function in it. - /// Unload the domain, load a new one. - /// Make sure the function (and module) still exists. - /// - public void TestClassReference() - { - RunDomainReloadSteps(); - } - - #endregion - #region Tempary tests // https://github.com/pythonnet/pythonnet/pull/1074#issuecomment-596139665 From 59e81e2823ec1d0bc6d9587ca1a2ed5406728d88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 17 Dec 2020 11:42:50 -0500 Subject: [PATCH 40/51] Test runner docs fixes --- src/domain_tests/TestRunner.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/domain_tests/TestRunner.cs b/src/domain_tests/TestRunner.cs index e494992a5..94f6b335c 100644 --- a/src/domain_tests/TestRunner.cs +++ b/src/domain_tests/TestRunner.cs @@ -55,9 +55,9 @@ class TestCase /// /// The Python code to run as a module that imports the C#. - /// It should have two functions: before() and after(). Before - /// will be called when DotNetBefore is loaded; after will be - /// called (twice) when DotNetAfter is loaded. + /// It should have two functions: before_reload() and after_reload(). + /// Before will be called twice when DotNetBefore is loaded; + /// after will also be called twice when DotNetAfter is loaded. /// To make the test fail, have those functions raise exceptions. /// /// Make sure there's no leading spaces since Python cares. @@ -936,7 +936,6 @@ public static int Main() }} }} "; - // readonly static string PythonDllLocation = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "../../runtime/bin/Python.Runtime.dll"); readonly static string PythonDllLocation = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Python.Runtime.dll"); public static int Main(string[] args) From 1383b5ae9b08eae01658a342c724bb57d32fe355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 17 Dec 2020 15:39:14 -0500 Subject: [PATCH 41/51] Skip the domain reload tests on macos Because it can't find the python library --- src/domain_tests/test_domain_reload.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/domain_tests/test_domain_reload.py b/src/domain_tests/test_domain_reload.py index 0e66bc2c8..9a1263aa5 100644 --- a/src/domain_tests/test_domain_reload.py +++ b/src/domain_tests/test_domain_reload.py @@ -17,55 +17,72 @@ def _run_test(testname): assert proc.returncode == 0 +@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_rename_class(): _run_test('class_rename') +@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_rename_class_member_static_function(): _run_test('static_member_rename') +@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_rename_class_member_function(): _run_test('member_rename') +@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_rename_class_member_field(): _run_test('field_rename') +@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_rename_class_member_property(): _run_test('property_rename') +@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_rename_namespace(): _run_test('namespace_rename') +@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_field_visibility_change(): _run_test("field_visibility_change") +@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_method_visibility_change(): _run_test("method_visibility_change") +@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_property_visibility_change(): _run_test("property_visibility_change") +@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_class_visibility_change(): _run_test("class_visibility_change") +@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_method_parameters_change(): _run_test("method_parameters_change") +@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_method_return_type_change(): _run_test("method_return_type_change") +@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_field_type_change(): _run_test("field_type_change") +@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') @pytest.mark.xfail(reason="Events not yet serializable") def test_rename_event(): _run_test('event_rename') +@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') @pytest.mark.xfail(reason="newly instanced object uses PyType_GenericAlloc") def test_construct_removed_class(): _run_test("construct_removed_class") +@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_out_to_ref_param(): _run_test("out_to_ref_param") +@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_nested_type(): _run_test("nested_type") From fbc06ef2d68a559a1b44e1bb9d85d5887bf37723 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 17 Dec 2020 15:48:22 -0500 Subject: [PATCH 42/51] fixup! Skip the domain reload tests on macos --- src/domain_tests/test_domain_reload.py | 34 +++++++++++++------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/domain_tests/test_domain_reload.py b/src/domain_tests/test_domain_reload.py index 9a1263aa5..2840cdd58 100644 --- a/src/domain_tests/test_domain_reload.py +++ b/src/domain_tests/test_domain_reload.py @@ -17,72 +17,72 @@ def _run_test(testname): assert proc.returncode == 0 -@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_rename_class(): _run_test('class_rename') -@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_rename_class_member_static_function(): _run_test('static_member_rename') -@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_rename_class_member_function(): _run_test('member_rename') -@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_rename_class_member_field(): _run_test('field_rename') -@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_rename_class_member_property(): _run_test('property_rename') -@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_rename_namespace(): _run_test('namespace_rename') -@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_field_visibility_change(): _run_test("field_visibility_change") -@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_method_visibility_change(): _run_test("method_visibility_change") -@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_property_visibility_change(): _run_test("property_visibility_change") -@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_class_visibility_change(): _run_test("class_visibility_change") -@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_method_parameters_change(): _run_test("method_parameters_change") -@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_method_return_type_change(): _run_test("method_return_type_change") -@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_field_type_change(): _run_test("field_type_change") -@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') @pytest.mark.xfail(reason="Events not yet serializable") def test_rename_event(): _run_test('event_rename') -@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') @pytest.mark.xfail(reason="newly instanced object uses PyType_GenericAlloc") def test_construct_removed_class(): _run_test("construct_removed_class") -@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_out_to_ref_param(): _run_test("out_to_ref_param") -@pytest.mark.skipif(platform.system == 'Darwin', reason='FIXME: macos can\'t find the python library') +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') def test_nested_type(): _run_test("nested_type") From b3e86da85970d9c88116fe7a2b850068c1911fc6 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Thu, 17 Dec 2020 18:09:36 +0100 Subject: [PATCH 43/51] Update ISSUE_TEMPLATE.md --- .github/ISSUE_TEMPLATE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 2758080dc..c232a1186 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -3,6 +3,7 @@ - Pythonnet version: - Python version: - Operating System: +- .NET Runtime: ### Details From 0b027c63ef0b4f3db69b8720335e76fad154ce02 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 17 Dec 2020 15:55:36 -0800 Subject: [PATCH 44/51] refactoring in CreateSubType --- src/runtime/converter.cs | 4 ++- src/runtime/runtime.cs | 2 ++ src/runtime/typemanager.cs | 51 ++++++++++++++++---------------------- 3 files changed, 26 insertions(+), 31 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 6c22484d7..e1b689cf3 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -313,7 +313,9 @@ internal static bool ToManaged(IntPtr value, Type type, return Converter.ToManagedValue(value, type, out result, setError); } - + internal static bool ToManagedValue(BorrowedReference value, Type obType, + out object result, bool setError) + => ToManagedValue(value.DangerousGetAddress(), obType, out result, setError); internal static bool ToManagedValue(IntPtr value, Type obType, out object result, bool setError) { diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index c4dd89c36..ff73425c7 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1642,6 +1642,8 @@ internal static bool PyDict_Check(IntPtr ob) /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDict_GetItem(IntPtr pointer, IntPtr key); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern BorrowedReference PyDict_GetItemWithError(BorrowedReference pointer, BorrowedReference key); /// /// Return value: Borrowed reference. diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 7e82360e9..d4b7db399 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -6,6 +6,7 @@ using System.Runtime.InteropServices; using System.Diagnostics; using Python.Runtime.Slots; +using static Python.Runtime.PythonException; namespace Python.Runtime { @@ -313,6 +314,7 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr py_dict) { + var dictRef = new BorrowedReference(py_dict); // Utility to create a subtype of a managed type with the ability for the // a python subtype able to override the managed implementation string name = Runtime.GetManagedString(py_name); @@ -322,40 +324,29 @@ internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr object assembly = null; object namespaceStr = null; - var disposeList = new List(); - try + using (var assemblyKey = new PyString("__assembly__")) { - var assemblyKey = new PyObject(Converter.ToPython("__assembly__", typeof(string))); - disposeList.Add(assemblyKey); - if (0 != Runtime.PyMapping_HasKey(py_dict, assemblyKey.Handle)) + var assemblyPtr = Runtime.PyDict_GetItemWithError(dictRef, assemblyKey.Reference); + if (assemblyPtr.IsNull) { - var pyAssembly = new PyObject(Runtime.PyDict_GetItem(py_dict, assemblyKey.Handle)); - Runtime.XIncref(pyAssembly.Handle); - disposeList.Add(pyAssembly); - if (!Converter.ToManagedValue(pyAssembly.Handle, typeof(string), out assembly, false)) - { - throw new InvalidCastException("Couldn't convert __assembly__ value to string"); - } + if (Exceptions.ErrorOccurred()) return IntPtr.Zero; + } + else if (!Converter.ToManagedValue(assemblyPtr, typeof(string), out assembly, false)) + { + return Exceptions.RaiseTypeError("Couldn't convert __assembly__ value to string"); } - var namespaceKey = new PyObject(Converter.ToPythonImplicit("__namespace__")); - disposeList.Add(namespaceKey); - if (0 != Runtime.PyMapping_HasKey(py_dict, namespaceKey.Handle)) + using (var namespaceKey = new PyString("__namespace__")) { - var pyNamespace = new PyObject(Runtime.PyDict_GetItem(py_dict, namespaceKey.Handle)); - Runtime.XIncref(pyNamespace.Handle); - disposeList.Add(pyNamespace); - if (!Converter.ToManagedValue(pyNamespace.Handle, typeof(string), out namespaceStr, false)) + var pyNamespace = Runtime.PyDict_GetItemWithError(dictRef, namespaceKey.Reference); + if (pyNamespace.IsNull) { - throw new InvalidCastException("Couldn't convert __namespace__ value to string"); + if (Exceptions.ErrorOccurred()) return IntPtr.Zero; + } + else if (!Converter.ToManagedValue(pyNamespace, typeof(string), out namespaceStr, false)) + { + return Exceptions.RaiseTypeError("Couldn't convert __namespace__ value to string"); } - } - } - finally - { - foreach (PyObject o in disposeList) - { - o.Dispose(); } } @@ -381,14 +372,14 @@ internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr // by default the class dict will have all the C# methods in it, but as this is a // derived class we want the python overrides in there instead if they exist. IntPtr cls_dict = Marshal.ReadIntPtr(py_type, TypeOffset.tp_dict); - Runtime.PyDict_Update(cls_dict, py_dict); + ThrowIfIsNotZero(Runtime.PyDict_Update(cls_dict, py_dict)); Runtime.XIncref(py_type); // Update the __classcell__ if it exists var cell = new BorrowedReference(Runtime.PyDict_GetItemString(cls_dict, "__classcell__")); if (!cell.IsNull) { - Runtime.PyCell_Set(cell, py_type); - Runtime.PyDict_DelItemString(cls_dict, "__classcell__"); + ThrowIfIsNotZero(Runtime.PyCell_Set(cell, py_type)); + ThrowIfIsNotZero(Runtime.PyDict_DelItemString(cls_dict, "__classcell__")); } return py_type; From b4533c4d5b9c98bbc80bc81e863300f64085a419 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 17 Dec 2020 19:46:00 -0800 Subject: [PATCH 45/51] allocate space for GCHandle in instances of CLR Metatype (which are types themselves) --- src/runtime/typemanager.cs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index d4b7db399..8ab5afa1b 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -150,7 +150,7 @@ internal static IntPtr GetTypeHandle(ManagedType obj, Type type) /// internal static IntPtr CreateType(Type impl) { - IntPtr type = AllocateTypeObject(impl.Name); + IntPtr type = AllocateTypeObject(impl.Name, metatype: Runtime.PyTypeType); int ob_size = ObjectOffset.Size(type); // Set tp_basicsize to the size of our managed instance objects. @@ -223,7 +223,7 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) base_ = bc.pyHandle; } - IntPtr type = AllocateTypeObject(name); + IntPtr type = AllocateTypeObject(name, Runtime.PyCLRMetaType); Marshal.WriteIntPtr(type, TypeOffset.ob_type, Runtime.PyCLRMetaType); Runtime.XIncref(Runtime.PyCLRMetaType); @@ -438,12 +438,15 @@ internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) // the standard type slots, and has to subclass PyType_Type for // certain functions in the C runtime to work correctly with it. - IntPtr type = AllocateTypeObject("CLR Metatype"); - IntPtr py_type = Runtime.PyTypeType; + IntPtr type = AllocateTypeObject("CLR Metatype", metatype: Runtime.PyTypeType); + IntPtr py_type = Runtime.PyTypeType; Marshal.WriteIntPtr(type, TypeOffset.tp_base, py_type); Runtime.XIncref(py_type); + int size = TypeOffset.magic() + IntPtr.Size; + Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, new IntPtr(size)); + const int flags = TypeFlags.Default | TypeFlags.Managed | TypeFlags.HeapType @@ -536,7 +539,7 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) // Utility to create a subtype of a std Python type, but with // a managed type able to override implementation - IntPtr type = AllocateTypeObject(name); + IntPtr type = AllocateTypeObject(name, metatype: Runtime.PyTypeType); //Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)obSize); //Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); @@ -582,9 +585,9 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) /// /// Utility method to allocate a type object & do basic initialization. /// - internal static IntPtr AllocateTypeObject(string name) + internal static IntPtr AllocateTypeObject(string name, IntPtr metatype) { - IntPtr type = Runtime.PyType_GenericAlloc(Runtime.PyTypeType, 0); + IntPtr type = Runtime.PyType_GenericAlloc(metatype, 0); // Clr type would not use __slots__, // and the PyMemberDef after PyHeapTypeObject will have other uses(e.g. type handle), // thus set the ob_size to 0 for avoiding slots iterations. From 0a3f044a568964d0ac1bd884fe5ac489b90f8a50 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 17 Dec 2020 22:29:47 -0800 Subject: [PATCH 46/51] classderived: handle tp_dealloc called after tp_clear Because tp_clear sets tpHandle to NULL, it can't be used. Fortunately, we can simply read object's type from pyHandle. --- src/runtime/classderived.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index e55e89240..dad9b039d 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -75,7 +75,8 @@ internal ClassDerivedObject(Type tp) : base(tp) // So we don't call PyObject_GC_Del here and instead we set the python // reference to a weak reference so that the C# object can be collected. GCHandle gc = GCHandle.Alloc(self, GCHandleType.Weak); - Marshal.WriteIntPtr(self.pyHandle, ObjectOffset.magic(self.tpHandle), (IntPtr)gc); + int gcOffset = ObjectOffset.magic(Runtime.PyObject_TYPE(self.pyHandle)); + Marshal.WriteIntPtr(self.pyHandle, gcOffset, (IntPtr)gc); self.gcHandle.Free(); self.gcHandle = gc; } From 639236a5b8bb2067d7182c184036da9b8ce4f158 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 17 Dec 2020 22:44:26 -0800 Subject: [PATCH 47/51] a few extra assertions --- src/runtime/clrobject.cs | 2 ++ src/runtime/native/TypeOffset.cs | 8 +++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index a79662ccc..0a352b381 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics; using System.Runtime.InteropServices; namespace Python.Runtime @@ -85,6 +86,7 @@ internal static CLRObject Restore(object ob, IntPtr pyHandle, InterDomainContext pyHandle = pyHandle, tpHandle = Runtime.PyObject_TYPE(pyHandle) }; + Debug.Assert(co.tpHandle != IntPtr.Zero); co.Load(context); return co; } diff --git a/src/runtime/native/TypeOffset.cs b/src/runtime/native/TypeOffset.cs index 0c51110da..bca191565 100644 --- a/src/runtime/native/TypeOffset.cs +++ b/src/runtime/native/TypeOffset.cs @@ -85,9 +85,11 @@ internal static void Use(ITypeOffsets offsets) internal static Dictionary GetOffsets() { var properties = typeof(TypeOffset).GetProperties(FieldFlags); - return properties.ToDictionary( - keySelector: p => p.Name, - elementSelector: p => (int)p.GetValue(obj: null, index: null)); + var result = properties.ToDictionary( + keySelector: p => p.Name, + elementSelector: p => (int)p.GetValue(obj: null, index: null)); + Debug.Assert(result.Values.Any(v => v != 0)); + return result; } internal static int GetOffsetUncached(string name) From 3069285b579998696969312e3c035adfff693ce2 Mon Sep 17 00:00:00 2001 From: Victor Date: Fri, 18 Dec 2020 04:30:06 -0800 Subject: [PATCH 48/51] fixed crash in finalizer of CLR types defined in Python, that survive engine shutdown (#1260) https://github.com/pythonnet/pythonnet/issues/1256 #1256 During engine shutdown all links from Python to .NET instances are severed. If an instance of CLR class defined in Python survives the shutdown (for example, a reference is stored in static field) and later gets finalized, it will attempt to severe link again, which is an invalid operation. The fix is to check if the link has already been severed and skip that step during finalization. --- src/runtime/classderived.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index dad9b039d..4e8e88bf3 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -858,7 +858,7 @@ public static void Finalize(IPythonDerivedType obj) { if (0 == Runtime.Py_IsInitialized() || Runtime.IsFinalizing) { - self.gcHandle.Free(); + if (self.gcHandle.IsAllocated) self.gcHandle.Free(); return; } } @@ -873,7 +873,7 @@ public static void Finalize(IPythonDerivedType obj) // If python's been terminated then just free the gchandle. if (0 == Runtime.Py_IsInitialized() || Runtime.IsFinalizing) { - self.gcHandle.Free(); + if (self.gcHandle.IsAllocated) self.gcHandle.Free(); return; } From 833e836b52cad11e3acde98751a5d6d442249094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 4 Jan 2021 10:48:58 -0500 Subject: [PATCH 49/51] Fix break introduced by merge --- src/runtime/classbase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 781305f9f..0ff4ba154 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -62,7 +62,7 @@ public virtual IntPtr type_subscript(IntPtr idx) return c.pyHandle; } - return Exceptions.RaiseTypeError($"{type.Namespace}.{type.Name} does not accept {types.Length} generic parameters"); + return Exceptions.RaiseTypeError($"{type.Value.Namespace}.{type.Name} does not accept {types.Length} generic parameters"); } /// From 62ae10749aea3512e0b8c45730e1a7692817d8b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Mon, 4 Jan 2021 15:46:48 -0500 Subject: [PATCH 50/51] Code review fixes --- .../Python.DomainReloadTests.csproj | 4 - src/domain_tests/TestRunner.cs | 47 +++++++---- src/runtime/Python.Runtime.csproj | 78 +++++++++---------- .../StateSerialization/MaybeMemberInfo.cs | 10 +-- src/runtime/StateSerialization/MaybeType.cs | 2 +- src/runtime/classmanager.cs | 4 + src/runtime/methodbinder.cs | 9 ++- src/runtime/methodobject.cs | 3 +- src/runtime/moduleobject.cs | 4 + 9 files changed, 91 insertions(+), 70 deletions(-) diff --git a/src/domain_tests/Python.DomainReloadTests.csproj b/src/domain_tests/Python.DomainReloadTests.csproj index 263f83663..54196f210 100644 --- a/src/domain_tests/Python.DomainReloadTests.csproj +++ b/src/domain_tests/Python.DomainReloadTests.csproj @@ -23,8 +23,4 @@ - - - - diff --git a/src/domain_tests/TestRunner.cs b/src/domain_tests/TestRunner.cs index 94f6b335c..6fafd3c6d 100644 --- a/src/domain_tests/TestRunner.cs +++ b/src/domain_tests/TestRunner.cs @@ -913,8 +913,7 @@ public static int Main() PythonEngine.Initialize(mode:{0}); using (Py.GIL()) {{ - // Because the generated assemblies are in the $TEMP folder, add it to the path - var temp = Path.GetTempPath(); + var temp = AppDomain.CurrentDomain.BaseDirectory; dynamic sys = Py.Import(""sys""); sys.path.append(new PyString(temp)); dynamic test_mod = Py.Import(""domain_test_module.mod""); @@ -938,6 +937,8 @@ public static int Main() "; readonly static string PythonDllLocation = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Python.Runtime.dll"); + static string TestPath = null; + public static int Main(string[] args) { TestCase testCase; @@ -951,16 +952,11 @@ public static int Main(string[] args) Console.WriteLine($"-- Looking for domain reload test case {testName}"); testCase = Cases.First(c => c.Name == testName); } + Console.WriteLine($"-- Running domain reload test case: {testCase.Name}"); - var tempFolderPython = Path.Combine(Path.GetTempPath(), "Python.Runtime.dll"); - if (File.Exists(tempFolderPython)) - { - File.Delete(tempFolderPython); - } + SetupTestFolder(testCase.Name); - File.Copy(PythonDllLocation, tempFolderPython); - CreatePythonModule(testCase); { var runnerAssembly = CreateCaseRunnerAssembly(verb:"before"); @@ -989,9 +985,32 @@ public static int Main(string[] args) RunAndUnload(runnerDomain, runnerAssembly); } } + + // Don't delete unconditionally. It's sometimes useful to leave the + // folder behind to debug failing tests. + TeardownTestFolder(); + return 0; } -#if !NETCOREAPP + + static void SetupTestFolder(string testCaseName) + { + TestPath = Path.Combine(Path.GetTempPath(), $"Python.TestRunner.{testCaseName}"); + if (Directory.Exists(TestPath)) + { + Directory.Delete(TestPath, recursive: true); + } + Directory.CreateDirectory(TestPath); + File.Copy(PythonDllLocation, Path.Combine(TestPath, "Python.Runtime.dll")); + } + + static void TeardownTestFolder() + { + if (Directory.Exists(TestPath)) + { + Directory.Delete(TestPath, recursive: true); + } + } static void RunAndUnload(AppDomain domain, string assemblyPath) { @@ -1027,7 +1046,7 @@ static string CreateAssembly(string name, string code, bool exe = false) CompilerParameters parameters = new CompilerParameters(); parameters.GenerateExecutable = exe; var assemblyName = name; - var assemblyFullPath = Path.Combine(Path.GetTempPath(), assemblyName); + var assemblyFullPath = Path.Combine(TestPath, assemblyName); parameters.OutputAssembly = assemblyFullPath; parameters.ReferencedAssemblies.Add("System.dll"); parameters.ReferencedAssemblies.Add("System.Core.dll"); @@ -1062,7 +1081,7 @@ static AppDomain CreateDomain(string name) var currentDomain = AppDomain.CurrentDomain; var domainsetup = new AppDomainSetup() { - ApplicationBase = Path.GetTempPath(), + ApplicationBase = TestPath, ConfigurationFile = currentDomain.SetupInformation.ConfigurationFile, LoaderOptimization = LoaderOptimization.SingleDomain, PrivateBinPath = "." @@ -1077,7 +1096,7 @@ static AppDomain CreateDomain(string name) static string CreatePythonModule(TestCase testCase) { - var modulePath = Path.Combine(Path.GetTempPath(), "domain_test_module"); + var modulePath = Path.Combine(TestPath, "domain_test_module"); if (Directory.Exists(modulePath)) { Directory.Delete(modulePath, recursive: true); @@ -1092,7 +1111,5 @@ static string CreatePythonModule(TestCase testCase) return null; } -#endif - } } diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index c25ebfc92..acb8efc4e 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,39 +1,39 @@ - - - netstandard2.0 - AnyCPU - Python.Runtime - Python.Runtime - pythonnet - https://github.com/pythonnet/pythonnet/blob/master/LICENSE - https://github.com/pythonnet/pythonnet - git - python interop dynamic dlr Mono pinvoke - https://raw.githubusercontent.com/pythonnet/pythonnet/master/src/console/python-clear.ico - https://pythonnet.github.io/ - 1591;NU1701 - True - - - - $(DefineConstants);$(ConfiguredConstants) - - - - - - - - - - - - clr.py - - - - - - - - + + + netstandard2.0 + AnyCPU + Python.Runtime + Python.Runtime + pythonnet + https://github.com/pythonnet/pythonnet/blob/master/LICENSE + https://github.com/pythonnet/pythonnet + git + python interop dynamic dlr Mono pinvoke + https://raw.githubusercontent.com/pythonnet/pythonnet/master/src/console/python-clear.ico + https://pythonnet.github.io/ + 1591;NU1701 + True + + + + $(DefineConstants);$(ConfiguredConstants) + + + + + + + + + + + + clr.py + + + + + + + + diff --git a/src/runtime/StateSerialization/MaybeMemberInfo.cs b/src/runtime/StateSerialization/MaybeMemberInfo.cs index 4305581b6..e14e74bbc 100644 --- a/src/runtime/StateSerialization/MaybeMemberInfo.cs +++ b/src/runtime/StateSerialization/MaybeMemberInfo.cs @@ -6,10 +6,10 @@ namespace Python.Runtime { - [Serializable] - internal struct MaybeMemberInfo : ISerializable where T: MemberInfo + [Serializable] + internal struct MaybeMemberInfo : ISerializable where T : MemberInfo { - public static implicit operator MaybeMemberInfo (T ob) => new MaybeMemberInfo(ob); + public static implicit operator MaybeMemberInfo(T ob) => new MaybeMemberInfo(ob); // .ToString() of the serialized object const string SerializationName = "s"; @@ -22,7 +22,7 @@ internal struct MaybeMemberInfo : ISerializable where T: MemberInfo [NonSerialized] Exception deserializationException; - public string DeletedMessage + public string DeletedMessage { get { @@ -42,7 +42,7 @@ public T Value } } - public string Name {get{return name;}} + public string Name => name; public bool Valid => info != null; public override string ToString() diff --git a/src/runtime/StateSerialization/MaybeType.cs b/src/runtime/StateSerialization/MaybeType.cs index a12d2d6e0..abb3a8fb6 100644 --- a/src/runtime/StateSerialization/MaybeType.cs +++ b/src/runtime/StateSerialization/MaybeType.cs @@ -36,7 +36,7 @@ public Type Value } } - public string Name {get{return name;}} + public string Name => name; public bool Valid => type != null; public override string ToString() diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 650ba5ba3..e1950bb9c 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -129,6 +129,10 @@ internal static void SaveRuntimeData(RuntimeDataStorage storage) // raises an error. We don't care about it. Runtime.PyErr_Clear(); } + else if (Exceptions.ErrorOccurred()) + { + throw new PythonException(); + } } // We modified the Type object, notify it we did. Runtime.PyType_Modified(cls.Value.tpHandle); diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index b360e5250..6eb3331b3 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -24,7 +24,8 @@ internal class MethodBinder [NonSerialized] public bool init = false; - public bool allow_threads = true; + public const bool DefaultAllowThreads = true; + public bool allow_threads = DefaultAllowThreads; internal MethodBinder() { @@ -187,7 +188,7 @@ internal static int GetPrecedence(MethodBase mi) { if (mi == null) { - return -1; + return int.MaxValue; } ParameterInfo[] pi = mi.GetParameters(); @@ -864,8 +865,8 @@ internal class MethodSorter : IComparer { int IComparer.Compare(MaybeMethodBase m1, MaybeMethodBase m2) { - MethodBase me1 = m1.Valid ? m1.Value : null; - MethodBase me2 = m2.Valid ? m2.Value : null; + MethodBase me1 = m1.UnsafeValue; + MethodBase me2 = m2.UnsafeValue; if (me1 == null && me2 == null) { return 0; diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 0c8e3d4ec..37c01f5c5 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -28,8 +28,7 @@ internal class MethodObject : ExtensionType internal IntPtr doc; internal Type type; - // `allow_threads = true`: True being the default value of MethodBinder.allow_threads - public MethodObject(Type type, string name, MethodInfo[] info, bool allow_threads = true) + public MethodObject(Type type, string name, MethodInfo[] info, bool allow_threads = MethodBinder.DefaultAllowThreads) { this.type = type; this.name = name; diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 0dbad6daf..07dd20e55 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -351,6 +351,10 @@ protected override void OnSave(InterDomainContext context) // raises an error. We don't care about it. Runtime.PyErr_Clear(); } + else if (Exceptions.ErrorOccurred()) + { + throw new PythonException(); + } pair.Value.DecrRefCount(); } From 73f39bc38e24b4d39fdca410cdf3dcde3c387ab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 6 Jan 2021 11:03:48 -0500 Subject: [PATCH 51/51] Add the PID of the test runner To the test folder name --- src/domain_tests/TestRunner.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/domain_tests/TestRunner.cs b/src/domain_tests/TestRunner.cs index 6fafd3c6d..924b622c6 100644 --- a/src/domain_tests/TestRunner.cs +++ b/src/domain_tests/TestRunner.cs @@ -995,12 +995,14 @@ public static int Main(string[] args) static void SetupTestFolder(string testCaseName) { - TestPath = Path.Combine(Path.GetTempPath(), $"Python.TestRunner.{testCaseName}"); + var pid = System.Diagnostics.Process.GetCurrentProcess().Id; + TestPath = Path.Combine(Path.GetTempPath(), $"Python.TestRunner.{testCaseName}-{pid}"); if (Directory.Exists(TestPath)) { Directory.Delete(TestPath, recursive: true); } Directory.CreateDirectory(TestPath); + Console.WriteLine($"Using directory: {TestPath}"); File.Copy(PythonDllLocation, Path.Combine(TestPath, "Python.Runtime.dll")); }