diff --git a/pythonnet.sln b/pythonnet.sln index e0fbeede7..4da4d7e99 100644 --- a/pythonnet.sln +++ b/pythonnet.sln @@ -14,6 +14,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Test", "src\testing\ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.PerformanceTests", "src\perf_tests\Python.PerformanceTests.csproj", "{4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.DomainReloadTests", "src\domain_tests\Python.DomainReloadTests.csproj", "{F2FB6DA3-318E-4F30-9A1F-932C667E38C5}" +EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Repo", "Repo", "{441A0123-F4C6-4EE4-9AEE-315FD79BE2D5}" ProjectSection(SolutionItems) = preProject .editorconfig = .editorconfig @@ -131,6 +133,18 @@ Global {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Release|x64.Build.0 = Release|x64 {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Release|x86.ActiveCfg = Release|x86 {4F2EA4A1-7ECA-48B5-8077-7A3C366F9931}.Release|x86.Build.0 = Release|x86 + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x64.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x64.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x86.ActiveCfg = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Debug|x86.Build.0 = Debug|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|Any CPU.Build.0 = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x64.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x64.Build.0 = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x86.ActiveCfg = Release|Any CPU + {F2FB6DA3-318E-4F30-9A1F-932C667E38C5}.Release|x86.Build.0 = Release|Any CPU {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Debug|Any CPU.Build.0 = Debug|Any CPU {6CF9EEA0-F865-4536-AABA-739AE3DA971E}.Debug|x64.ActiveCfg = Debug|Any CPU diff --git a/src/domain_tests/App.config b/src/domain_tests/App.config new file mode 100644 index 000000000..56efbc7b5 --- /dev/null +++ b/src/domain_tests/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/src/domain_tests/Python.DomainReloadTests.csproj b/src/domain_tests/Python.DomainReloadTests.csproj new file mode 100644 index 000000000..54196f210 --- /dev/null +++ b/src/domain_tests/Python.DomainReloadTests.csproj @@ -0,0 +1,26 @@ + + + + net472 + bin\ + Exe + + + + + + + + + + + + + + + + + + + + diff --git a/src/domain_tests/TestRunner.cs b/src/domain_tests/TestRunner.cs new file mode 100644 index 000000000..924b622c6 --- /dev/null +++ b/src/domain_tests/TestRunner.cs @@ -0,0 +1,1117 @@ +// We can't refer to or use Python.Runtime here. +// We want it to be loaded only inside the subdomains +using System; +using Microsoft.CSharp; +using System.CodeDom.Compiler; +using System.IO; +using System.Linq; + +namespace Python.DomainReloadTests +{ + /// + /// This class provides an executable that can run domain reload tests. + /// The setup is a bit complicated: + /// 1. pytest runs test_*.py in this directory. + /// 2. test_classname runs Python.DomainReloadTests.exe (this class) with an argument + /// 3. This class at runtime creates a directory that has both C# and + /// python code, and compiles the C#. + /// 4. This class then runs the C# code. + /// + /// But there's a bit more indirection. This class compiles a DLL that + /// contains code that will change. + /// Then, the test case: + /// * Compiles some code, loads it into a domain, runs python that refers to it. + /// * Unload the domain, re-runs the domain to make sure domain reload happens correctly. + /// * Compile a new piece of code, load it into a new domain, run a new piece of + /// Python code to test the objects after they've been deleted or modified in C#. + /// * Unload the domain. Reload the domain, run the same python again. + /// + /// This class gets built into an executable which takes one argument: + /// which test case to run. That's because pytest assumes we'll run + /// everything in one process, but we really want a clean process on each + /// test case to test the init/reload/teardown parts of the domain reload. + /// + /// + class TestRunner + { + const string TestAssemblyName = "DomainTests"; + + class TestCase + { + /// + /// The key to pass as an argument to choose this test. + /// + public string Name; + + /// + /// The C# code to run in the first domain. + /// + public string DotNetBefore; + + /// + /// The C# code to run in the second domain. + /// + public string DotNetAfter; + + /// + /// The Python code to run as a module that imports the C#. + /// It should have two functions: before_reload() and after_reload(). + /// Before will be called twice when DotNetBefore is loaded; + /// after will also be called twice when DotNetAfter is loaded. + /// To make the test fail, have those functions raise exceptions. + /// + /// Make sure there's no leading spaces since Python cares. + /// + public string PythonCode; + } + + static TestCase[] Cases = new TestCase[] + { + new TestCase + { + Name = "class_rename", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Before { } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class After { } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace + +def before_reload(): + sys.my_cls = TestNamespace.Before + + +def after_reload(): + assert sys.my_cls is not None + try: + foo = TestNamespace.Before + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + ", + }, + + new TestCase + { + Name = "static_member_rename", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls { public static int Before() { return 5; } } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls { public static int After() { return 10; } } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace + +def before_reload(): + if not hasattr(sys, 'my_cls'): + sys.my_cls = TestNamespace.Cls + sys.my_fn = TestNamespace.Cls.Before + assert 5 == sys.my_fn() + assert 5 == TestNamespace.Cls.Before() + +def after_reload(): + + # We should have reloaded the class so we can access the new function. + assert 10 == sys.my_cls.After() + assert True is True + + try: + # We should have reloaded the class. The old function still exists, but is now invalid. + sys.my_cls.Before() + except AttributeError: + print('Caught expected TypeError') + else: + raise AssertionError('Failed to throw exception: expected TypeError calling class member that no longer exists') + + assert sys.my_fn is not None + + try: + # Unbound functions still exist. They will error out when called though. + sys.my_fn() + except TypeError: + print('Caught expected TypeError') + else: + raise AssertionError('Failed to throw exception: expected TypeError calling unbound .NET function that no longer exists') + ", + }, + + + new TestCase + { + Name = "member_rename", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls { public int Before() { return 5; } } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls { public int After() { return 10; } } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace + +def before_reload(): + sys.my_cls = TestNamespace.Cls() + sys.my_fn = TestNamespace.Cls().Before + sys.my_fn() + TestNamespace.Cls().Before() + +def after_reload(): + + # We should have reloaded the class so we can access the new function. + assert 10 == sys.my_cls.After() + assert True is True + + try: + # We should have reloaded the class. The old function still exists, but is now invalid. + sys.my_cls.Before() + except AttributeError: + print('Caught expected TypeError') + else: + raise AssertionError('Failed to throw exception: expected TypeError calling class member that no longer exists') + + assert sys.my_fn is not None + + try: + # Unbound functions still exist. They will error out when called though. + sys.my_fn() + except TypeError: + print('Caught expected TypeError') + else: + raise AssertionError('Failed to throw exception: expected TypeError calling unbound .NET function that no longer exists') + ", + }, + + new TestCase + { + Name = "field_rename", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + static public int Before = 2; + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + static public int After = 4; + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + sys.my_int = Cls.Before + +def after_reload(): + print(sys.my_int) + try: + assert 2 == Cls.Before + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') +", + }, + new TestCase + { + Name = "property_rename", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + static public int Before { get { return 2; } } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + static public int After { get { return 4; } } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + sys.my_int = Cls.Before + +def after_reload(): + print(sys.my_int) + try: + assert 2 == Cls.Before + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') +", + }, + + new TestCase + { + Name = "event_rename", + DotNetBefore = @" + using System; + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static event Action Before; + public static void Call() + { + Before(); + } + } + }", + DotNetAfter = @" + using System; + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static event Action After; + public static void Call() + { + After(); + } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +called = False + +def callback_function(): + global called + called = True + +def before_reload(): + global called + called = False + Cls.Before += callback_function + Cls.Call() + assert called is True + +def after_reload(): + global called + assert called is True + called = False + Cls.Call() + assert called is False +", + }, + + new TestCase + { + Name = "namespace_rename", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public int Foo; + public Cls(int i) + { + Foo = i; + } + } + }", + DotNetAfter = @" + namespace NewTestNamespace + { + [System.Serializable] + public class Cls + { + public int Foo; + public Cls(int i) + { + Foo = i; + } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace + +def before_reload(): + sys.my_cls = TestNamespace.Cls + sys.my_inst = TestNamespace.Cls(1) + +def after_reload(): + try: + TestNamespace.Cls(2) + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + ", + }, + + new TestCase + { + Name = "field_visibility_change", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static int Foo = 1; + public static int Field = 2; + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static int Foo = 1; + private static int Field = 2; + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + assert 2 == Cls.Field + assert 1 == Cls.Foo + +def after_reload(): + assert 1 == Cls.Foo + try: + assert 1 == Cls.Field + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + ", + }, + + new TestCase + { + Name = "method_visibility_change", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static int Foo() { return 1; } + public static int Function() { return 2; } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static int Foo() { return 1; } + private static int Function() { return 2; } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + sys.my_func = Cls.Function + assert 1 == Cls.Foo() + assert 2 == Cls.Function() + +def after_reload(): + assert 1 == Cls.Foo() + try: + assert 2 == Cls.Function() + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + + try: + assert 2 == sys.my_func() + except TypeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + ", + }, + + new TestCase + { + Name = "property_visibility_change", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static int Foo { get { return 1; } } + public static int Property { get { return 2; } } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static int Foo { get { return 1; } } + private static int Property { get { return 2; } } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + assert 1 == Cls.Foo + assert 2 == Cls.Property + +def after_reload(): + assert 1 == Cls.Foo + try: + assert 2 == Cls.Property + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + ", + }, + + new TestCase + { + Name = "class_visibility_change", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class PublicClass { } + + [System.Serializable] + public class Cls { } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + internal class Cls { } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace + +def before_reload(): + sys.my_cls = TestNamespace.Cls + +def after_reload(): + sys.my_cls() + + try: + TestNamespace.Cls() + except AttributeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + ", + }, + + new TestCase + { + Name = "method_parameters_change", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static void MyFunction(int a) + { + System.Console.WriteLine(string.Format(""MyFunction says: {0}"", a)); + } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static void MyFunction(string a) + { + System.Console.WriteLine(string.Format(""MyFunction says: {0}"", a)); + } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + sys.my_cls = Cls + sys.my_func = Cls.MyFunction + sys.my_cls.MyFunction(1) + sys.my_func(2) + +def after_reload(): + try: + sys.my_cls.MyFunction(1) + except TypeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + + try: + sys.my_func(2) + except TypeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + + # Calling the function from the class passes + sys.my_cls.MyFunction('test') + + try: + # calling the callable directly fails + sys.my_func('test') + except TypeError: + print('Caught expected exception') + else: + raise AssertionError('Failed to throw exception') + + Cls.MyFunction('another test') + + ", + }, + + new TestCase + { + Name = "method_return_type_change", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static int MyFunction() + { + return 2; + } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + public static string MyFunction() + { + return ""22""; + } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + sys.my_cls = Cls + sys.my_func = Cls.MyFunction + assert 2 == sys.my_cls.MyFunction() + assert 2 == sys.my_func() + +def after_reload(): + assert '22' == sys.my_cls.MyFunction() + assert '22' == sys.my_func() + assert '22' == Cls.MyFunction() + ", + }, + + new TestCase + { + Name = "field_type_change", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + static public int Field = 2; + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class Cls + { + static public string Field = ""22""; + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +from TestNamespace import Cls + +def before_reload(): + sys.my_cls = Cls + assert 2 == sys.my_cls.Field + +def after_reload(): + assert '22' == Cls.Field + assert '22' == sys.my_cls.Field + ", + }, + + new TestCase + { + Name = "construct_removed_class", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class Before { } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class After { } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace + +def before_reload(): + sys.my_cls = TestNamespace.Before + +def after_reload(): + bar = sys.my_cls() + + # Don't crash! + print(bar) + print(bar.__str__()) + print(bar.__repr__()) + ", + }, + + new TestCase + { + Name = "out_to_ref_param", + DotNetBefore = @" + namespace TestNamespace + { + + [System.Serializable] + public class Data + { + public int num = -1; + } + + [System.Serializable] + public class Cls + { + public static void MyFn (out Data a) + { + a = new Data(); + a.num = 9001; + } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + + [System.Serializable] + public class Data + { + public int num = -1; + } + + [System.Serializable] + public class Cls + { + public static void MyFn (ref Data a) + { + a.num = 7; + } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace +import System + +def before_reload(): + + foo = TestNamespace.Data() + bar = TestNamespace.Cls.MyFn(foo) + assert bar.num == 9001 + # foo shouldn't have changed. + assert foo.num == -1 + + +def after_reload(): + + try: + # Now that the function takes a ref type, we must pass a valid object. + bar = TestNamespace.Cls.MyFn(None) + except System.NullReferenceException as e: + print('caught expected exception') + else: + raise AssertionError('failed to raise') + + foo = TestNamespace.Data() + bar = TestNamespace.Cls.MyFn(foo) + # foo should have changed + assert foo.num == 7 + assert bar.num == 7 + # Pythonnet also returns a new object with `ref`-qualified parameters + assert foo is not bar + ", + }, + new TestCase + { + Name = "nested_type", + DotNetBefore = @" + namespace TestNamespace + { + [System.Serializable] + public class WithNestedType + { + [System.Serializable] + public class Inner + { + public static int Value = -1; + } + } + }", + DotNetAfter = @" + namespace TestNamespace + { + [System.Serializable] + public class WithNestedType + { + [System.Serializable] + public class Inner + { + public static int Value = -1; + } + } + }", + PythonCode = @" +import clr +import sys +clr.AddReference('DomainTests') +import TestNamespace + +def before_reload(): + + sys.my_obj = TestNamespace.WithNestedType + +def after_reload(): + + assert sys.my_obj is not None + foo = sys.my_obj.Inner() + print(foo) + + ", + }, + }; + + /// + /// The runner's code. Runs the python code + /// This is a template for string.Format + /// Arg 0 is the reload mode: ShutdownMode.Reload or other. + /// Arg 1 is the no-arg python function to run, before or after. + /// + const string CaseRunnerTemplate = @" +using System; +using System.IO; +using Python.Runtime; +namespace CaseRunner +{{ + class CaseRunner + {{ + public static int Main() + {{ + try + {{ + PythonEngine.Initialize(mode:{0}); + using (Py.GIL()) + {{ + var temp = AppDomain.CurrentDomain.BaseDirectory; + dynamic sys = Py.Import(""sys""); + sys.path.append(new PyString(temp)); + dynamic test_mod = Py.Import(""domain_test_module.mod""); + test_mod.{1}_reload(); + }} + PythonEngine.Shutdown(); + }} + catch (PythonException pe) + {{ + throw new ArgumentException(message:pe.Message+"" ""+pe.StackTrace); + }} + catch (Exception e) + {{ + Console.WriteLine(e.StackTrace); + throw; + }} + return 0; + }} + }} +}} +"; + readonly static string PythonDllLocation = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Python.Runtime.dll"); + + static string TestPath = null; + + public static int Main(string[] args) + { + TestCase testCase; + if (args.Length < 1) + { + testCase = Cases[0]; + } + else + { + string testName = args[0]; + Console.WriteLine($"-- Looking for domain reload test case {testName}"); + testCase = Cases.First(c => c.Name == testName); + } + + Console.WriteLine($"-- Running domain reload test case: {testCase.Name}"); + + SetupTestFolder(testCase.Name); + + CreatePythonModule(testCase); + { + var runnerAssembly = CreateCaseRunnerAssembly(verb:"before"); + CreateTestClassAssembly(testCase.DotNetBefore); + { + var runnerDomain = CreateDomain("case runner before"); + RunAndUnload(runnerDomain, runnerAssembly); + } + { + var runnerDomain = CreateDomain("case runner before (again)"); + RunAndUnload(runnerDomain, runnerAssembly); + } + } + + { + var runnerAssembly = CreateCaseRunnerAssembly(verb:"after"); + CreateTestClassAssembly(testCase.DotNetAfter); + + // Do it twice for good measure + { + var runnerDomain = CreateDomain("case runner after"); + RunAndUnload(runnerDomain, runnerAssembly); + } + { + var runnerDomain = CreateDomain("case runner after (again)"); + RunAndUnload(runnerDomain, runnerAssembly); + } + } + + // Don't delete unconditionally. It's sometimes useful to leave the + // folder behind to debug failing tests. + TeardownTestFolder(); + + return 0; + } + + static void SetupTestFolder(string testCaseName) + { + var pid = System.Diagnostics.Process.GetCurrentProcess().Id; + TestPath = Path.Combine(Path.GetTempPath(), $"Python.TestRunner.{testCaseName}-{pid}"); + if (Directory.Exists(TestPath)) + { + Directory.Delete(TestPath, recursive: true); + } + Directory.CreateDirectory(TestPath); + Console.WriteLine($"Using directory: {TestPath}"); + File.Copy(PythonDllLocation, Path.Combine(TestPath, "Python.Runtime.dll")); + } + + static void TeardownTestFolder() + { + if (Directory.Exists(TestPath)) + { + Directory.Delete(TestPath, recursive: true); + } + } + + static void RunAndUnload(AppDomain domain, string assemblyPath) + { + // Somehow the stack traces during execution sometimes have the wrong line numbers. + // Add some info for when debugging is required. + Console.WriteLine($"-- Running domain {domain.FriendlyName}"); + domain.ExecuteAssembly(assemblyPath); + AppDomain.Unload(domain); + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + } + + static string CreateTestClassAssembly(string code) + { + return CreateAssembly(TestAssemblyName + ".dll", code, exe: false); + } + + static string CreateCaseRunnerAssembly(string verb, string shutdownMode = "ShutdownMode.Reload") + { + var code = string.Format(CaseRunnerTemplate, shutdownMode, verb); + var name = "TestCaseRunner.exe"; + + return CreateAssembly(name, code, exe: true); + } + static string CreateAssembly(string name, string code, bool exe = false) + { + // Never return or hold the Assembly instance. This will cause + // the assembly to be loaded into the current domain and this + // interferes with the tests. The Domain can execute fine from a + // path, so let's return that. + CSharpCodeProvider provider = new CSharpCodeProvider(); + CompilerParameters parameters = new CompilerParameters(); + parameters.GenerateExecutable = exe; + var assemblyName = name; + var assemblyFullPath = Path.Combine(TestPath, assemblyName); + parameters.OutputAssembly = assemblyFullPath; + parameters.ReferencedAssemblies.Add("System.dll"); + parameters.ReferencedAssemblies.Add("System.Core.dll"); + parameters.ReferencedAssemblies.Add("Microsoft.CSharp.dll"); + var netstandard = "netstandard.dll"; + if (Type.GetType("Mono.Runtime") != null) + { + netstandard = "Facades/" + netstandard; + } + parameters.ReferencedAssemblies.Add(netstandard); + parameters.ReferencedAssemblies.Add(PythonDllLocation); + CompilerResults results = provider.CompileAssemblyFromSource(parameters, code); + if (results.NativeCompilerReturnValue != 0) + { + var stderr = System.Console.Error; + stderr.WriteLine($"Error in {name} compiling:\n{code}"); + foreach (var error in results.Errors) + { + stderr.WriteLine(error); + } + throw new ArgumentException("Error compiling code"); + } + + return assemblyFullPath; + } + + static AppDomain CreateDomain(string name) + { + // Create the domain. Make sure to set PrivateBinPath to a relative + // path from the CWD (namely, 'bin'). + // See https://stackoverflow.com/questions/24760543/createinstanceandunwrap-in-another-domain + var currentDomain = AppDomain.CurrentDomain; + var domainsetup = new AppDomainSetup() + { + ApplicationBase = TestPath, + ConfigurationFile = currentDomain.SetupInformation.ConfigurationFile, + LoaderOptimization = LoaderOptimization.SingleDomain, + PrivateBinPath = "." + }; + var domain = AppDomain.CreateDomain( + $"My Domain {name}", + currentDomain.Evidence, + domainsetup); + + return domain; + } + + static string CreatePythonModule(TestCase testCase) + { + var modulePath = Path.Combine(TestPath, "domain_test_module"); + if (Directory.Exists(modulePath)) + { + Directory.Delete(modulePath, recursive: true); + } + Directory.CreateDirectory(modulePath); + + File.Create(Path.Combine(modulePath, "__init__.py")).Close(); //Create and don't forget to close! + using (var writer = File.CreateText(Path.Combine(modulePath, "mod.py"))) + { + writer.Write(testCase.PythonCode); + } + + return null; + } + } +} diff --git a/src/domain_tests/conftest.py b/src/domain_tests/conftest.py new file mode 100644 index 000000000..5f0d52e10 --- /dev/null +++ b/src/domain_tests/conftest.py @@ -0,0 +1,7 @@ +import os + +from subprocess import check_call +# test_proj_path = os.path.join(cwd, "..", "testing") +cfd = os.path.dirname(__file__) +bin_path = os.path.join(cfd, 'bin') +check_call(["dotnet", "build", cfd, '-o', bin_path]) \ No newline at end of file diff --git a/src/domain_tests/test_domain_reload.py b/src/domain_tests/test_domain_reload.py new file mode 100644 index 000000000..2840cdd58 --- /dev/null +++ b/src/domain_tests/test_domain_reload.py @@ -0,0 +1,88 @@ +import subprocess +import os +import platform + +import pytest + +def _run_test(testname): + dirname = os.path.split(__file__)[0] + exename = os.path.join(dirname, 'bin', 'Python.DomainReloadTests.exe') + args = [exename, testname] + + if platform.system() != 'Windows': + args = ['mono'] + args + + proc = subprocess.Popen(args) + proc.wait() + + assert proc.returncode == 0 + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_rename_class(): + _run_test('class_rename') + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_rename_class_member_static_function(): + _run_test('static_member_rename') + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_rename_class_member_function(): + _run_test('member_rename') + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_rename_class_member_field(): + _run_test('field_rename') + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_rename_class_member_property(): + _run_test('property_rename') + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_rename_namespace(): + _run_test('namespace_rename') + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_field_visibility_change(): + _run_test("field_visibility_change") + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_method_visibility_change(): + _run_test("method_visibility_change") + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_property_visibility_change(): + _run_test("property_visibility_change") + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_class_visibility_change(): + _run_test("class_visibility_change") + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_method_parameters_change(): + _run_test("method_parameters_change") + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_method_return_type_change(): + _run_test("method_return_type_change") + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_field_type_change(): + _run_test("field_type_change") + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +@pytest.mark.xfail(reason="Events not yet serializable") +def test_rename_event(): + _run_test('event_rename') + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +@pytest.mark.xfail(reason="newly instanced object uses PyType_GenericAlloc") +def test_construct_removed_class(): + _run_test("construct_removed_class") + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_out_to_ref_param(): + _run_test("out_to_ref_param") + +@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library') +def test_nested_type(): + _run_test("nested_type") diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 3e0a18c70..f8445edb4 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -179,97 +179,6 @@ public static void CrossDomainObject() #endregion - #region TestClassReference - - class ReloadClassRefStep1 : CrossCaller - { - public override ValueType Execute(ValueType arg) - { - const string code = @" -from Python.EmbeddingTest.Domain import MyClass - -def test_obj_call(): - obj = MyClass() - obj.Method() - MyClass.StaticMethod() - obj.Property = 1 - obj.Field = 10 - -test_obj_call() -"; - const string name = "test_domain_reload_mod"; - using (Py.GIL()) - { - // Create a new module - IntPtr module = PyRuntime.PyModule_New(name); - Assert.That(module != IntPtr.Zero); - IntPtr globals = PyRuntime.PyObject_GetAttr(module, PyIdentifier.__dict__); - Assert.That(globals != IntPtr.Zero); - try - { - // import builtins - // module.__dict__[__builtins__] = builtins - int res = PyRuntime.PyDict_SetItem(globals, PyIdentifier.__builtins__, - PyRuntime.PyEval_GetBuiltins()); - PythonException.ThrowIfIsNotZero(res); - - // Execute the code in the module's scope - PythonEngine.Exec(code, globals); - // import sys - // modules = sys.modules - IntPtr modules = PyRuntime.PyImport_GetModuleDict(); - // modules[name] = module - res = PyRuntime.PyDict_SetItemString(modules, name, module); - PythonException.ThrowIfIsNotZero(res); - } - catch - { - PyRuntime.XDecref(module); - throw; - } - finally - { - PyRuntime.XDecref(globals); - } - return module; - } - } - } - - class ReloadClassRefStep2 : CrossCaller - { - public override ValueType Execute(ValueType arg) - { - var module = (IntPtr)arg; - using (Py.GIL()) - { - var test_obj_call = PyRuntime.PyObject_GetAttrString(module, "test_obj_call"); - PythonException.ThrowIfIsNull(test_obj_call); - var args = PyRuntime.PyTuple_New(0); - var res = PyRuntime.PyObject_CallObject(test_obj_call, args); - PythonException.ThrowIfIsNull(res); - - PyRuntime.XDecref(args); - PyRuntime.XDecref(res); - } - return 0; - } - } - - - [Test] - /// - /// Create a new Python module, define a function in it. - /// Unload the domain, load a new one. - /// Make sure the function (and module) still exists. - /// - public void TestClassReference() - { - RunDomainReloadSteps(); - } - - #endregion - #region Tempary tests // https://github.com/pythonnet/pythonnet/pull/1074#issuecomment-596139665 diff --git a/src/runtime/StateSerialization/MaybeMemberInfo.cs b/src/runtime/StateSerialization/MaybeMemberInfo.cs new file mode 100644 index 000000000..e14e74bbc --- /dev/null +++ b/src/runtime/StateSerialization/MaybeMemberInfo.cs @@ -0,0 +1,118 @@ +using System; +using System.Reflection; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Formatters.Binary; +using System.IO; + +namespace Python.Runtime +{ + [Serializable] + internal struct MaybeMemberInfo : ISerializable where T : MemberInfo + { + public static implicit operator MaybeMemberInfo(T ob) => new MaybeMemberInfo(ob); + + // .ToString() of the serialized object + const string SerializationName = "s"; + // The ReflectedType of the object + const string SerializationType = "t"; + const string SerializationFieldName = "f"; + string name; + MemberInfo info; + + [NonSerialized] + Exception deserializationException; + + public string DeletedMessage + { + get + { + return $"The .NET {typeof(T)} {name} no longer exists. Cause: " + deserializationException?.Message ; + } + } + + public T Value + { + get + { + if (info == null) + { + throw new SerializationException(DeletedMessage, innerException: deserializationException); + } + return (T)info; + } + } + + public string Name => name; + public bool Valid => info != null; + + public override string ToString() + { + return (info != null ? info.ToString() : $"missing type: {name}"); + } + + public MaybeMemberInfo(T fi) + { + info = fi; + name = info?.ToString(); + deserializationException = null; + } + + internal MaybeMemberInfo(SerializationInfo serializationInfo, StreamingContext context) + { + // Assumption: name is always stored in "s" + name = serializationInfo.GetString(SerializationName); + info = null; + deserializationException = null; + try + { + var tp = Type.GetType(serializationInfo.GetString(SerializationType)); + if (tp != null) + { + var field_name = serializationInfo.GetString(SerializationFieldName); + MemberInfo mi = tp.GetField(field_name, ClassManager.BindingFlags); + if (mi != null && ShouldBindMember(mi)) + { + info = mi; + } + } + } + catch (Exception e) + { + deserializationException = e; + } + } + + // This is complicated because we bind fields + // based on the visibility of the field, properties + // based on it's setter/getter (which is a method + // info) visibility and events based on their + // AddMethod visibility. + static bool ShouldBindMember(MemberInfo mi) + { + if (mi is PropertyInfo pi) + { + return ClassManager.ShouldBindProperty(pi); + } + else if (mi is FieldInfo fi) + { + return ClassManager.ShouldBindField(fi); + } + else if (mi is EventInfo ei) + { + return ClassManager.ShouldBindEvent(ei); + } + + return false; + } + + public void GetObjectData(SerializationInfo serializationInfo, StreamingContext context) + { + serializationInfo.AddValue(SerializationName, name); + if (Valid) + { + serializationInfo.AddValue(SerializationFieldName, info.Name); + serializationInfo.AddValue(SerializationType, info.ReflectedType.AssemblyQualifiedName); + } + } + } +} diff --git a/src/runtime/StateSerialization/MaybeMethodBase.cs b/src/runtime/StateSerialization/MaybeMethodBase.cs new file mode 100644 index 000000000..3f57f0d8a --- /dev/null +++ b/src/runtime/StateSerialization/MaybeMethodBase.cs @@ -0,0 +1,199 @@ +using System; +using System.Reflection; +using System.Runtime.Serialization; +using System.Linq; + +namespace Python.Runtime +{ + [Serializable] + internal struct MaybeMethodBase : ISerializable where T: MethodBase + { + // .ToString() of the serialized object + const string SerializationName = "s"; + // The ReflectedType of the object + const string SerializationType = "t"; + // Fhe parameters of the MethodBase + const string SerializationParameters = "p"; + const string SerializationIsCtor = "c"; + const string SerializationMethodName = "n"; + + [Serializable] + struct ParameterHelper : IEquatable + { + public enum TypeModifier + { + None, + In, + Out, + Ref + } + public readonly string Name; + public readonly TypeModifier Modifier; + + public ParameterHelper(ParameterInfo tp) + { + Name = tp.ParameterType.AssemblyQualifiedName; + Modifier = TypeModifier.None; + + if (tp.IsIn) + { + Modifier = TypeModifier.In; + } + else if (tp.IsOut) + { + Modifier = TypeModifier.Out; + } + else if (tp.ParameterType.IsByRef) + { + Modifier = TypeModifier.Ref; + } + } + + public bool Equals(ParameterInfo other) + { + return this.Equals(new ParameterHelper(other)); + } + } + public static implicit operator MaybeMethodBase (T ob) => new MaybeMethodBase(ob); + + string name; + MethodBase info; + + [NonSerialized] + Exception deserializationException; + + public string DeletedMessage + { + get + { + return $"The .NET {typeof(T)} {name} no longer exists. Cause: " + deserializationException?.Message ; + } + } + + public T Value + { + get + { + if (info == null) + { + throw new SerializationException(DeletedMessage, innerException: deserializationException); + } + return (T)info; + } + } + + public T UnsafeValue { get { return (T)info; } } + public string Name {get{return name;}} + public bool Valid => info != null; + + public override string ToString() + { + return (info != null ? info.ToString() : $"missing method info: {name}"); + } + + public MaybeMethodBase(T mi) + { + info = mi; + name = mi?.ToString(); + deserializationException = null; + } + + internal MaybeMethodBase(SerializationInfo serializationInfo, StreamingContext context) + { + name = serializationInfo.GetString(SerializationName); + info = null; + deserializationException = null; + try + { + // Retrieve the reflected type of the method; + var typeName = serializationInfo.GetString(SerializationType); + var tp = Type.GetType(typeName); + if (tp == null) + { + throw new SerializationException($"The underlying type {typeName} can't be found"); + } + // Get the method's parameters types + var field_name = serializationInfo.GetString(SerializationMethodName); + var param = (ParameterHelper[])serializationInfo.GetValue(SerializationParameters, typeof(ParameterHelper[])); + Type[] types = new Type[param.Length]; + bool hasRefType = false; + for (int i = 0; i < param.Length; i++) + { + var paramTypeName = param[i].Name; + types[i] = Type.GetType(paramTypeName); + if (types[i] == null) + { + throw new SerializationException($"The parameter of type {paramTypeName} can't be found"); + } + else if (types[i].IsByRef) + { + hasRefType = true; + } + } + + MethodBase mb = null; + if (serializationInfo.GetBoolean(SerializationIsCtor)) + { + // We never want the static constructor. + mb = tp.GetConstructor(ClassManager.BindingFlags&(~BindingFlags.Static), binder:null, types:types, modifiers:null); + } + else + { + mb = tp.GetMethod(field_name, ClassManager.BindingFlags, binder:null, types:types, modifiers:null); + } + + if (mb != null && hasRefType) + { + mb = CheckRefTypes(mb, param); + } + + // Do like in ClassManager.GetClassInfo + if(mb != null && ClassManager.ShouldBindMethod(mb)) + { + info = mb; + } + } + catch (Exception e) + { + deserializationException = e; + } + } + + MethodBase CheckRefTypes(MethodBase mb, ParameterHelper[] ph) + { + // One more step: Changing: + // void MyFn (ref int a) + // to: + // void MyFn (out int a) + // will still find the function correctly as, `in`, `out` and `ref` + // are all represented as a reference type. Query the method we got + // and validate the parameters + if (ph.Length != 0) + { + foreach (var item in Enumerable.Zip(ph, mb.GetParameters(), (orig, current) => new {orig, current})) + { + if (!item.current.Equals(item.orig)) + { + // False positive + return null; + } + } + } + + return mb; + } + + public void GetObjectData(SerializationInfo serializationInfo, StreamingContext context) + { + serializationInfo.AddValue(SerializationName, name); + if (Valid) + { + serializationInfo.AddValue(SerializationMethodName, info.Name); + serializationInfo.AddValue(SerializationType, info.ReflectedType.AssemblyQualifiedName); + ParameterHelper[] parameters = (from p in info.GetParameters() select new ParameterHelper(p)).ToArray(); + serializationInfo.AddValue(SerializationParameters, parameters, typeof(ParameterHelper[])); + serializationInfo.AddValue(SerializationIsCtor, info.IsConstructor); + } + } + } +} \ No newline at end of file diff --git a/src/runtime/StateSerialization/MaybeType.cs b/src/runtime/StateSerialization/MaybeType.cs new file mode 100644 index 000000000..abb3a8fb6 --- /dev/null +++ b/src/runtime/StateSerialization/MaybeType.cs @@ -0,0 +1,64 @@ +using System; +using System.Reflection; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Formatters.Binary; +using System.IO; + +namespace Python.Runtime +{ + [Serializable] + internal struct MaybeType : ISerializable + { + public static implicit operator MaybeType (Type ob) => new MaybeType(ob); + + // The AssemblyQualifiedName of the serialized Type + const string SerializationName = "n"; + string name; + Type type; + + public string DeletedMessage + { + get + { + return $"The .NET Type {name} no longer exists"; + } + } + + public Type Value + { + get + { + if (type == null) + { + throw new SerializationException(DeletedMessage); + } + return type; + } + } + + public string Name => name; + public bool Valid => type != null; + + public override string ToString() + { + return (type != null ? type.ToString() : $"missing type: {name}"); + } + + public MaybeType(Type tp) + { + type = tp; + name = tp.AssemblyQualifiedName; + } + + private MaybeType(SerializationInfo serializationInfo, StreamingContext context) + { + name = (string)serializationInfo.GetValue(SerializationName, typeof(string)); + type = Type.GetType(name, throwOnError:false); + } + + public void GetObjectData(SerializationInfo serializationInfo, StreamingContext context) + { + serializationInfo.AddValue(SerializationName, name); + } + } +} \ No newline at end of file diff --git a/src/runtime/arrayobject.cs b/src/runtime/arrayobject.cs index e6a4bee19..262e521a5 100644 --- a/src/runtime/arrayobject.cs +++ b/src/runtime/arrayobject.cs @@ -30,6 +30,11 @@ public static IntPtr tp_new(IntPtr tpRaw, IntPtr args, IntPtr kw) var tp = new BorrowedReference(tpRaw); var self = GetManagedObject(tp) as ArrayObject; + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } + Type arrType = self.type.Value; long[] dimensions = new long[Runtime.PyTuple_Size(args)]; if (dimensions.Length == 0) @@ -38,7 +43,7 @@ public static IntPtr tp_new(IntPtr tpRaw, IntPtr args, IntPtr kw) } if (dimensions.Length != 1) { - return CreateMultidimensional(self.type.GetElementType(), dimensions, + return CreateMultidimensional(arrType.GetElementType(), dimensions, shapeTuple: new BorrowedReference(args), pyType: tp) .DangerousMoveToPointerOrNull(); @@ -56,14 +61,14 @@ public static IntPtr tp_new(IntPtr tpRaw, IntPtr args, IntPtr kw) } else { - return NewInstance(self.type.GetElementType(), tp, dimensions) + return NewInstance(arrType.GetElementType(), tp, dimensions) .DangerousMoveToPointerOrNull(); } } object result; // this implements casting to Array[T] - if (!Converter.ToManaged(op, self.type, out result, true)) + if (!Converter.ToManaged(op, arrType, out result, true)) { return IntPtr.Zero; } @@ -133,8 +138,12 @@ static NewReference NewInstance(Type elementType, BorrowedReference arrayPyType, { var obj = (CLRObject)GetManagedObject(ob); var arrObj = (ArrayObject)GetManagedObjectType(ob); + if (!arrObj.type.Valid) + { + return Exceptions.RaiseTypeError(arrObj.type.DeletedMessage); + } var items = obj.inst as Array; - Type itemType = arrObj.type.GetElementType(); + Type itemType = arrObj.type.Value.GetElementType(); int rank = items.Rank; int index; object value; diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 7cb6938bc..0ff4ba154 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -18,18 +18,21 @@ namespace Python.Runtime [Serializable] internal class ClassBase : ManagedType { + [NonSerialized] + internal List dotNetMembers; internal Indexer indexer; - internal Type type; + internal MaybeType type; internal ClassBase(Type tp) { + dotNetMembers = new List(); indexer = null; type = tp; } internal virtual bool CanSubclass() { - return !type.IsEnum; + return !type.Value.IsEnum; } @@ -44,7 +47,12 @@ public virtual IntPtr type_subscript(IntPtr idx) return Exceptions.RaiseTypeError("type(s) expected"); } - Type target = GenericUtil.GenericForType(type, types.Length); + if (!type.Valid) + { + return Exceptions.RaiseTypeError(type.DeletedMessage); + } + + Type target = GenericUtil.GenericForType(type.Value, types.Length); if (target != null) { @@ -54,7 +62,7 @@ public virtual IntPtr type_subscript(IntPtr idx) return c.pyHandle; } - return Exceptions.RaiseTypeError($"{type.Namespace}.{type.Name} does not accept {types.Length} generic parameters"); + return Exceptions.RaiseTypeError($"{type.Value.Namespace}.{type.Name} does not accept {types.Length} generic parameters"); } /// diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index db4146722..64c985ce7 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -18,7 +18,20 @@ namespace Python.Runtime /// internal class ClassManager { - private static Dictionary cache; + + // Binding flags to determine which members to expose in Python. + // This is complicated because inheritance in Python is name + // based. We can't just find DeclaredOnly members, because we + // could have a base class A that defines two overloads of a + // method and a class B that defines two more. The name-based + // descriptor Python will find needs to know about inherited + // overloads as well as those declared on the sub class. + internal static readonly BindingFlags BindingFlags = BindingFlags.Static | + BindingFlags.Instance | + BindingFlags.Public | + BindingFlags.NonPublic; + + private static Dictionary cache; private static readonly Type dtype; private ClassManager() @@ -36,7 +49,7 @@ static ClassManager() public static void Reset() { - cache = new Dictionary(128); + cache = new Dictionary(128); } internal static void DisposePythonWrappersForClrTypes() @@ -85,27 +98,75 @@ internal static void SaveRuntimeData(RuntimeDataStorage storage) var contexts = storage.AddValue("contexts", new Dictionary()); storage.AddValue("cache", cache); - foreach (var cls in cache.Values) + foreach (var cls in cache) { + if (!cls.Key.Valid) + { + // Don't serialize an invalid class + continue; + } // This incref is for cache to hold the cls, // thus no need for decreasing it at RestoreRuntimeData. - Runtime.XIncref(cls.pyHandle); - var context = contexts[cls.pyHandle] = new InterDomainContext(); - cls.Save(context); + Runtime.XIncref(cls.Value.pyHandle); + var context = contexts[cls.Value.pyHandle] = new InterDomainContext(); + cls.Value.Save(context); + + // Remove all members added in InitBaseClass. + // this is done so that if domain reloads and a member of a + // reflected dotnet class is removed, it is removed from the + // Python object's dictionary tool; thus raising an AttributeError + // instead of a TypeError. + // Classes are re-initialized on in RestoreRuntimeData. + IntPtr dict = Marshal.ReadIntPtr(cls.Value.tpHandle, TypeOffset.tp_dict); + foreach (var member in cls.Value.dotNetMembers) + { + // No need to decref the member, the ClassBase instance does + // not own the reference. + if ((Runtime.PyDict_DelItemString(dict, member) == -1) && + (Exceptions.ExceptionMatches(Exceptions.KeyError))) + { + // Trying to remove a key that's not in the dictionary + // raises an error. We don't care about it. + Runtime.PyErr_Clear(); + } + else if (Exceptions.ErrorOccurred()) + { + throw new PythonException(); + } + } + // We modified the Type object, notify it we did. + Runtime.PyType_Modified(cls.Value.tpHandle); } } internal static Dictionary RestoreRuntimeData(RuntimeDataStorage storage) { - cache = storage.GetValue>("cache"); + cache = storage.GetValue>("cache"); + var invalidClasses = new List>(); var contexts = storage.GetValue >("contexts"); var loadedObjs = new Dictionary(); - foreach (var cls in cache.Values) + foreach (var pair in cache) + { + if (!pair.Key.Valid) + { + invalidClasses.Add(pair); + continue; + } + // re-init the class + InitClassBase(pair.Key.Value, pair.Value); + // We modified the Type object, notify it we did. + Runtime.PyType_Modified(pair.Value.tpHandle); + var context = contexts[pair.Value.pyHandle]; + pair.Value.Load(context); + loadedObjs.Add(pair.Value, context); + } + + foreach (var pair in invalidClasses) { - var context = contexts[cls.pyHandle]; - cls.Load(context); - loadedObjs.Add(cls, context); + cache.Remove(pair.Key); + Runtime.XDecref(pair.Value.pyHandle); } + return loadedObjs; } @@ -113,6 +174,7 @@ internal static Dictionary RestoreRuntimeData(R /// Return the ClassBase-derived instance that implements a particular /// reflected managed type, creating it if it doesn't yet exist. /// + /// A Borrowed reference to the ClassBase object internal static ClassBase GetClass(Type type) { ClassBase cb = null; @@ -209,11 +271,16 @@ private static void InitClassBase(Type type, ClassBase impl) IntPtr dict = Marshal.ReadIntPtr(tp, TypeOffset.tp_dict); + if (impl.dotNetMembers == null) + { + impl.dotNetMembers = new List(); + } IDictionaryEnumerator iter = info.members.GetEnumerator(); while (iter.MoveNext()) { var item = (ManagedType)iter.Value; var name = (string)iter.Key; + impl.dotNetMembers.Add(name); Runtime.PyDict_SetItemString(dict, name, item.pyHandle); // Decref the item now that it's been used. item.DecrRefCount(); @@ -241,7 +308,7 @@ private static void InitClassBase(Type type, ClassBase impl) // required that the ClassObject.ctors be changed to internal if (co != null) { - if (co.ctors.Length > 0) + if (co.NumCtors > 0) { // Implement Overloads on the class object if (!CLRModule._SuppressOverloads) @@ -263,6 +330,50 @@ private static void InitClassBase(Type type, ClassBase impl) } } } + // The type has been modified after PyType_Ready has been called + // Refresh the type + Runtime.PyType_Modified(tp); + } + + internal static bool ShouldBindMethod(MethodBase mb) + { + return (mb.IsPublic || mb.IsFamily || mb.IsFamilyOrAssembly); + } + + internal static bool ShouldBindField(FieldInfo fi) + { + return (fi.IsPublic || fi.IsFamily || fi.IsFamilyOrAssembly); + } + + internal static bool ShouldBindProperty(PropertyInfo pi) + { + MethodInfo mm = null; + try + { + mm = pi.GetGetMethod(true); + if (mm == null) + { + mm = pi.GetSetMethod(true); + } + } + catch (SecurityException) + { + // GetGetMethod may try to get a method protected by + // StrongNameIdentityPermission - effectively private. + return false; + } + + if (mm == null) + { + return false; + } + + return ShouldBindMethod(mm); + } + + internal static bool ShouldBindEvent(EventInfo ei) + { + return ShouldBindMethod(ei.GetAddMethod(true)); } private static ClassInfo GetClassInfo(Type type) @@ -277,18 +388,7 @@ private static ClassInfo GetClassInfo(Type type) Type tp; int i, n; - // This is complicated because inheritance in Python is name - // based. We can't just find DeclaredOnly members, because we - // could have a base class A that defines two overloads of a - // method and a class B that defines two more. The name-based - // descriptor Python will find needs to know about inherited - // overloads as well as those declared on the sub class. - BindingFlags flags = BindingFlags.Static | - BindingFlags.Instance | - BindingFlags.Public | - BindingFlags.NonPublic; - - MemberInfo[] info = type.GetMembers(flags); + MemberInfo[] info = type.GetMembers(BindingFlags); var local = new Hashtable(); var items = new ArrayList(); MemberInfo m; @@ -331,7 +431,7 @@ private static ClassInfo GetClassInfo(Type type) for (i = 0; i < inheritedInterfaces.Length; ++i) { Type inheritedType = inheritedInterfaces[i]; - MemberInfo[] imembers = inheritedType.GetMembers(flags); + MemberInfo[] imembers = inheritedType.GetMembers(BindingFlags); for (n = 0; n < imembers.Length; n++) { m = imembers[n]; @@ -362,8 +462,7 @@ private static ClassInfo GetClassInfo(Type type) { case MemberTypes.Method: meth = (MethodInfo)mi; - if (!(meth.IsPublic || meth.IsFamily || - meth.IsFamilyOrAssembly)) + if (!ShouldBindMethod(meth)) { continue; } @@ -380,28 +479,7 @@ private static ClassInfo GetClassInfo(Type type) case MemberTypes.Property: var pi = (PropertyInfo)mi; - MethodInfo mm = null; - try - { - mm = pi.GetGetMethod(true); - if (mm == null) - { - mm = pi.GetSetMethod(true); - } - } - catch (SecurityException) - { - // GetGetMethod may try to get a method protected by - // StrongNameIdentityPermission - effectively private. - continue; - } - - if (mm == null) - { - continue; - } - - if (!(mm.IsPublic || mm.IsFamily || mm.IsFamilyOrAssembly)) + if(!ShouldBindProperty(pi)) { continue; } @@ -426,7 +504,7 @@ private static ClassInfo GetClassInfo(Type type) case MemberTypes.Field: var fi = (FieldInfo)mi; - if (!(fi.IsPublic || fi.IsFamily || fi.IsFamilyOrAssembly)) + if (!ShouldBindField(fi)) { continue; } @@ -436,8 +514,7 @@ private static ClassInfo GetClassInfo(Type type) case MemberTypes.Event: var ei = (EventInfo)mi; - MethodInfo me = ei.GetAddMethod(true); - if (!(me.IsPublic || me.IsFamily || me.IsFamilyOrAssembly)) + if (!ShouldBindEvent(ei)) { continue; } @@ -454,6 +531,8 @@ private static ClassInfo GetClassInfo(Type type) } // Note the given instance might be uninitialized ob = GetClass(tp); + // GetClass returns a Borrowed ref. ci.members owns the reference. + ob.IncrRefCount(); ci.members[mi.Name] = ob; continue; } diff --git a/src/runtime/classobject.cs b/src/runtime/classobject.cs index 355cf744a..826ae5c54 100644 --- a/src/runtime/classobject.cs +++ b/src/runtime/classobject.cs @@ -14,14 +14,14 @@ namespace Python.Runtime internal class ClassObject : ClassBase { internal ConstructorBinder binder; - internal ConstructorInfo[] ctors; + internal int NumCtors = 0; internal ClassObject(Type tp) : base(tp) { - ctors = type.GetConstructors(); - binder = new ConstructorBinder(type); - - foreach (ConstructorInfo t in ctors) + var _ctors = type.Value.GetConstructors(); + NumCtors = _ctors.Length; + binder = new ConstructorBinder(type.Value); + foreach (ConstructorInfo t in _ctors) { binder.AddMethod(t); } @@ -62,7 +62,11 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) return Exceptions.RaiseTypeError("invalid object"); } - Type type = self.type; + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } + Type type = self.type.Value; // Primitive types do not have constructors, but they look like // they do from Python. If the ClassObject represents one of the @@ -115,16 +119,21 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) /// public override IntPtr type_subscript(IntPtr idx) { + if (!type.Valid) + { + return Exceptions.RaiseTypeError(type.DeletedMessage); + } + // If this type is the Array type, the [] means we need to // construct and return an array type of the given element type. - if (type == typeof(Array)) + if (type.Value == typeof(Array)) { if (Runtime.PyTuple_Check(idx)) { return Exceptions.RaiseTypeError("type expected"); } var c = GetManagedObject(idx) as ClassBase; - Type t = c != null ? c.type : Converter.GetTypeByAlias(idx); + Type t = c != null ? c.type.Value : Converter.GetTypeByAlias(idx); if (t == null) { return Exceptions.RaiseTypeError("type expected"); @@ -144,7 +153,7 @@ public override IntPtr type_subscript(IntPtr idx) return Exceptions.RaiseTypeError("type(s) expected"); } - Type gtype = AssemblyManager.LookupTypes($"{type.FullName}`{types.Length}").FirstOrDefault(); + Type gtype = AssemblyManager.LookupTypes($"{type.Value.FullName}`{types.Length}").FirstOrDefault(); if (gtype != null) { var g = ClassManager.GetClass(gtype) as GenericType; diff --git a/src/runtime/constructorbinder.cs b/src/runtime/constructorbinder.cs index 0cda3a3d9..83f2c81e4 100644 --- a/src/runtime/constructorbinder.cs +++ b/src/runtime/constructorbinder.cs @@ -14,7 +14,7 @@ namespace Python.Runtime [Serializable] internal class ConstructorBinder : MethodBinder { - private Type _containingType; + private MaybeType _containingType; internal ConstructorBinder(Type containingType) { @@ -51,10 +51,15 @@ internal object InvokeRaw(IntPtr inst, IntPtr args, IntPtr kw) /// internal object InvokeRaw(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info) { + if (!_containingType.Valid) + { + return Exceptions.RaiseTypeError(_containingType.DeletedMessage); + } object result; + Type tp = _containingType.Value; - if (_containingType.IsValueType && !_containingType.IsPrimitive && - !_containingType.IsEnum && _containingType != typeof(decimal) && + if (tp.IsValueType && !tp.IsPrimitive && + !tp.IsEnum && tp != typeof(decimal) && Runtime.PyTuple_Size(args) == 0) { // If you are trying to construct an instance of a struct by @@ -64,7 +69,7 @@ internal object InvokeRaw(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info) // Activator.CreateInstance(). try { - result = Activator.CreateInstance(_containingType); + result = Activator.CreateInstance(tp); } catch (Exception e) { diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index 0c81c0a93..803823e39 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -22,7 +22,7 @@ namespace Python.Runtime [Serializable] internal class ConstructorBinding : ExtensionType { - private Type type; // The managed Type being wrapped in a ClassObject + private MaybeType type; // The managed Type being wrapped in a ClassObject private IntPtr pyTypeHndl; // The python type tells GetInstHandle which Type to create. private ConstructorBinder ctorBinder; @@ -92,6 +92,11 @@ public static IntPtr tp_descr_get(IntPtr op, IntPtr instance, IntPtr owner) public static IntPtr mp_subscript(IntPtr op, IntPtr key) { var self = (ConstructorBinding)GetManagedObject(op); + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } + Type tp = self.type.Value; Type[] types = Runtime.PythonArgsToTypeArray(key); if (types == null) @@ -100,12 +105,12 @@ public static IntPtr mp_subscript(IntPtr op, IntPtr key) } //MethodBase[] methBaseArray = self.ctorBinder.GetMethods(); //MethodBase ci = MatchSignature(methBaseArray, types); - ConstructorInfo ci = self.type.GetConstructor(types); + ConstructorInfo ci = tp.GetConstructor(types); if (ci == null) { return Exceptions.RaiseTypeError("No match found for constructor signature"); } - var boundCtor = new BoundContructor(self.type, self.pyTypeHndl, self.ctorBinder, ci); + var boundCtor = new BoundContructor(tp, self.pyTypeHndl, self.ctorBinder, ci); return boundCtor.pyHandle; } @@ -122,7 +127,12 @@ public static IntPtr tp_repr(IntPtr ob) return self.repr; } MethodBase[] methods = self.ctorBinder.GetMethods(); - string name = self.type.FullName; + + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } + string name = self.type.Value.FullName; var doc = ""; foreach (MethodBase t in methods) { diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 2f3810c58..e1b689cf3 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -346,7 +346,13 @@ internal static bool ToManagedValue(IntPtr value, Type obType, } if (mt is ClassBase) { - result = ((ClassBase)mt).type; + var cb = (ClassBase)mt; + if (!cb.type.Valid) + { + Exceptions.SetError(Exceptions.TypeError, cb.type.DeletedMessage); + return false; + } + result = cb.type.Value; return true; } // shouldn't happen diff --git a/src/runtime/delegateobject.cs b/src/runtime/delegateobject.cs index c5078740f..e0d29f1a0 100644 --- a/src/runtime/delegateobject.cs +++ b/src/runtime/delegateobject.cs @@ -52,6 +52,12 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { var self = (DelegateObject)GetManagedObject(tp); + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } + Type type = self.type.Value; + if (Runtime.PyTuple_Size(args) != 1) { return Exceptions.RaiseTypeError("class takes exactly one argument"); @@ -64,7 +70,7 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) return Exceptions.RaiseTypeError("argument must be callable"); } - Delegate d = PythonEngine.DelegateManager.GetDelegate(self.type, method); + Delegate d = PythonEngine.DelegateManager.GetDelegate(type, method); return CLRObject.GetInstHandle(d, self.pyHandle); } diff --git a/src/runtime/fieldobject.cs b/src/runtime/fieldobject.cs index 86b93dd1b..2850ac6e1 100644 --- a/src/runtime/fieldobject.cs +++ b/src/runtime/fieldobject.cs @@ -3,13 +3,14 @@ namespace Python.Runtime { + using MaybeFieldInfo = MaybeMemberInfo; /// /// Implements a Python descriptor type that provides access to CLR fields. /// [Serializable] internal class FieldObject : ExtensionType { - private FieldInfo info; + private MaybeFieldInfo info; public FieldObject(FieldInfo info) { @@ -30,8 +31,13 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) { return IntPtr.Zero; } + else if (!self.info.Valid) + { + Exceptions.SetError(Exceptions.AttributeError, self.info.DeletedMessage); + return IntPtr.Zero; + } - FieldInfo info = self.info; + FieldInfo info = self.info.Value; if (ob == IntPtr.Zero || ob == Runtime.PyNone) { @@ -85,6 +91,11 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) { return -1; } + else if (!self.info.Valid) + { + Exceptions.SetError(Exceptions.AttributeError, self.info.DeletedMessage); + return -1; + } if (val == IntPtr.Zero) { @@ -92,7 +103,7 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) return -1; } - FieldInfo info = self.info; + FieldInfo info = self.info.Value; if (info.IsLiteral || info.IsInitOnly) { @@ -147,7 +158,7 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) public static IntPtr tp_repr(IntPtr ob) { var self = (FieldObject)GetManagedObject(ob); - return Runtime.PyString_FromString($""); + return Runtime.PyString_FromString($""); } } } diff --git a/src/runtime/interfaceobject.cs b/src/runtime/interfaceobject.cs index a2fa86479..976c09be0 100644 --- a/src/runtime/interfaceobject.cs +++ b/src/runtime/interfaceobject.cs @@ -37,8 +37,12 @@ static InterfaceObject() public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { var self = (InterfaceObject)GetManagedObject(tp); + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } var nargs = Runtime.PyTuple_Size(args); - Type type = self.type; + Type type = self.type.Value; object obj; if (nargs == 1) diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 84abe28b9..36b406c7b 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -2,6 +2,7 @@ using System.Collections; using System.IO; using System.Runtime.InteropServices; +using System.Runtime.Serialization; namespace Python.Runtime { @@ -103,9 +104,16 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) var cb = GetManagedObject(base_type) as ClassBase; if (cb != null) { - if (!cb.CanSubclass()) + try { - return Exceptions.RaiseTypeError("delegates, enums and array types cannot be subclassed"); + if (!cb.CanSubclass()) + { + return Exceptions.RaiseTypeError("delegates, enums and array types cannot be subclassed"); + } + } + catch (SerializationException) + { + return Exceptions.RaiseTypeError($"Underlying C# Base class {cb.type} has been deleted"); } } @@ -300,7 +308,7 @@ private static IntPtr DoInstanceCheck(IntPtr tp, IntPtr args, bool checkType) { var cb = GetManagedObject(tp) as ClassBase; - if (cb == null) + if (cb == null || !cb.type.Valid) { Runtime.XIncref(Runtime.PyFalse); return Runtime.PyFalse; @@ -332,13 +340,13 @@ private static IntPtr DoInstanceCheck(IntPtr tp, IntPtr args, bool checkType) } var otherCb = GetManagedObject(otherType.Handle) as ClassBase; - if (otherCb == null) + if (otherCb == null || !otherCb.type.Valid) { Runtime.XIncref(Runtime.PyFalse); return Runtime.PyFalse; } - return Converter.ToPython(cb.type.IsAssignableFrom(otherCb.type)); + return Converter.ToPython(cb.type.Value.IsAssignableFrom(otherCb.type.Value)); } } diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 47883f0e6..ba37c19c1 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -7,6 +7,7 @@ namespace Python.Runtime { + using MaybeMethodBase = MaybeMethodBase; /// /// A MethodBinder encapsulates information about a (possibly overloaded) /// managed method, and is responsible for selecting the right method given @@ -16,19 +17,24 @@ namespace Python.Runtime [Serializable] internal class MethodBinder { - public ArrayList list; + public List list; + + [NonSerialized] public MethodBase[] methods; + + [NonSerialized] public bool init = false; - public bool allow_threads = true; + public const bool DefaultAllowThreads = true; + public bool allow_threads = DefaultAllowThreads; internal MethodBinder() { - list = new ArrayList(); + list = new List(); } internal MethodBinder(MethodInfo mi) { - list = new ArrayList { mi }; + list = new List { new MaybeMethodBase(mi) }; } public int Count @@ -164,7 +170,7 @@ internal MethodBase[] GetMethods() { // I'm sure this could be made more efficient. list.Sort(new MethodSorter()); - methods = (MethodBase[])list.ToArray(typeof(MethodBase)); + methods = (from method in list where method.Valid select method.Value).ToArray(); init = true; } return methods; @@ -180,6 +186,11 @@ internal MethodBase[] GetMethods() /// internal static int GetPrecedence(MethodBase mi) { + if (mi == null) + { + return int.MaxValue; + } + ParameterInfo[] pi = mi.GetParameters(); int val = mi.IsStatic ? 3000 : 0; int num = pi.Length; @@ -797,6 +808,17 @@ protected static void AppendArgumentTypes(StringBuilder to, IntPtr args) internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, MethodInfo[] methodinfo) { + // No valid methods, nothing to bind. + if (GetMethods().Length == 0) + { + var msg = new StringBuilder("The underlying C# method(s) have been deleted"); + if (list.Count > 0 && list[0].Name != null) + { + msg.Append($": {list[0].ToString()}"); + } + return Exceptions.RaiseTypeError(msg.ToString());; + } + Binding binding = Bind(inst, args, kw, info, methodinfo); object result; IntPtr ts = IntPtr.Zero; @@ -896,12 +918,25 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i /// /// Utility class to sort method info by parameter type precedence. /// - internal class MethodSorter : IComparer + internal class MethodSorter : IComparer { - int IComparer.Compare(object m1, object m2) + int IComparer.Compare(MaybeMethodBase m1, MaybeMethodBase m2) { - var me1 = (MethodBase)m1; - var me2 = (MethodBase)m2; + MethodBase me1 = m1.UnsafeValue; + MethodBase me2 = m2.UnsafeValue; + if (me1 == null && me2 == null) + { + return 0; + } + else if (me1 == null) + { + return -1; + } + else if (me2 == null) + { + return 1; + } + if (me1.DeclaringType != me2.DeclaringType) { // m2's type derives from m1's type, favor m2 @@ -913,8 +948,8 @@ int IComparer.Compare(object m1, object m2) return -1; } - int p1 = MethodBinder.GetPrecedence((MethodBase)m1); - int p2 = MethodBinder.GetPrecedence((MethodBase)m2); + int p1 = MethodBinder.GetPrecedence(me1); + int p2 = MethodBinder.GetPrecedence(me2); if (p1 < p2) { return -1; diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index 7a10fcdef..46b62807d 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -4,6 +4,7 @@ namespace Python.Runtime { + using MaybeMethodInfo = MaybeMethodBase; /// /// Implements a Python binding type for CLR methods. These work much like /// standard Python method bindings, but the same type is used to bind @@ -12,7 +13,7 @@ namespace Python.Runtime [Serializable] internal class MethodBinding : ExtensionType { - internal MethodInfo info; + internal MaybeMethodInfo info; internal MethodObject m; internal IntPtr target; internal IntPtr targetType; @@ -111,15 +112,16 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) // This works around a situation where the wrong generic method is picked, // for example this method in the tests: string Overloaded(int arg1, int arg2, string arg3) - if (self.info != null) + if (self.info.Valid) { - if (self.info.IsGenericMethod) + var info = self.info.Value; + if (info.IsGenericMethod) { var len = Runtime.PyTuple_Size(args); //FIXME: Never used Type[] sigTp = Runtime.PythonArgsToTypeArray(args, true); if (sigTp != null) { - Type[] genericTp = self.info.GetGenericArguments(); + Type[] genericTp = info.GetGenericArguments(); MethodInfo betterMatch = MethodBinder.MatchSignatureAndParameters(self.m.info, genericTp, sigTp); if (betterMatch != null) { @@ -164,9 +166,9 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) if (inst?.inst is IPythonDerivedType) { var baseType = GetManagedObject(self.targetType) as ClassBase; - if (baseType != null) + if (baseType != null && baseType.type.Valid) { - string baseMethodName = "_" + baseType.type.Name + "__" + self.m.name; + string baseMethodName = "_" + baseType.type.Value.Name + "__" + self.m.name; IntPtr baseMethod = Runtime.PyObject_GetAttrString(target, baseMethodName); if (baseMethod != IntPtr.Zero) { @@ -184,8 +186,7 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) } } } - - return self.m.Invoke(target, args, kw, self.info); + return self.m.Invoke(target, args, kw, self.info.UnsafeValue); } finally { diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index dc23e3ce5..37c01f5c5 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -1,8 +1,12 @@ using System; +using System.Collections.Generic; using System.Reflection; +using System.Linq; namespace Python.Runtime { + using MaybeMethodInfo = MaybeMethodBase; + /// /// Implements a Python type that represents a CLR method. Method objects /// support a subscript syntax [] to allow explicit overload selection. @@ -13,40 +17,45 @@ namespace Python.Runtime [Serializable] internal class MethodObject : ExtensionType { - internal MethodInfo[] info; + [NonSerialized] + private MethodInfo[] _info = null; + private readonly List infoList; internal string name; internal MethodBinding unbound; - internal MethodBinder binder; + internal readonly MethodBinder binder; internal bool is_static = false; internal IntPtr doc; internal Type type; - public MethodObject(Type type, string name, MethodInfo[] info) - { - _MethodObject(type, name, info); - } - - public MethodObject(Type type, string name, MethodInfo[] info, bool allow_threads) - { - _MethodObject(type, name, info); - binder.allow_threads = allow_threads; - } - - private void _MethodObject(Type type, string name, MethodInfo[] info) + public MethodObject(Type type, string name, MethodInfo[] info, bool allow_threads = MethodBinder.DefaultAllowThreads) { this.type = type; this.name = name; - this.info = info; + this.infoList = new List(); binder = new MethodBinder(); foreach (MethodInfo item in info) { + this.infoList.Add(item); binder.AddMethod(item); if (item.IsStatic) { this.is_static = true; } } + binder.allow_threads = allow_threads; + } + + internal MethodInfo[] info + { + get + { + if (_info == null) + { + _info = (from i in infoList where i.Valid select i.Value).ToArray(); + } + return _info; + } } public virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw) diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 334c5c2f3..07dd20e55 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -341,6 +341,24 @@ protected override void OnSave(InterDomainContext context) // Decref twice in tp_clear, equilibrate them. Runtime.XIncref(dict); Runtime.XIncref(dict); + // destroy the cache(s) + foreach (var pair in cache) + { + if ((Runtime.PyDict_DelItemString(dict, pair.Key) == -1) && + (Exceptions.ExceptionMatches(Exceptions.KeyError))) + { + // Trying to remove a key that's not in the dictionary + // raises an error. We don't care about it. + Runtime.PyErr_Clear(); + } + else if (Exceptions.ErrorOccurred()) + { + throw new PythonException(); + } + pair.Value.DecrRefCount(); + } + + cache.Clear(); } protected override void OnLoad(InterDomainContext context) diff --git a/src/runtime/propertyobject.cs b/src/runtime/propertyobject.cs index ac1d077f9..20061b358 100644 --- a/src/runtime/propertyobject.cs +++ b/src/runtime/propertyobject.cs @@ -4,15 +4,16 @@ namespace Python.Runtime { + using MaybeMethodInfo = MaybeMethodBase; /// /// Implements a Python descriptor type that manages CLR properties. /// [Serializable] internal class PropertyObject : ExtensionType { - private PropertyInfo info; - private MethodInfo getter; - private MethodInfo setter; + private MaybeMemberInfo info; + private MaybeMethodInfo getter; + private MaybeMethodInfo setter; [StrongNameIdentityPermission(SecurityAction.Assert)] public PropertyObject(PropertyInfo md) @@ -31,7 +32,12 @@ public PropertyObject(PropertyInfo md) public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) { var self = (PropertyObject)GetManagedObject(ds); - MethodInfo getter = self.getter; + if (!self.info.Valid) + { + return Exceptions.RaiseTypeError(self.info.DeletedMessage); + } + var info = self.info.Value; + MethodInfo getter = self.getter.UnsafeValue; object result; @@ -51,8 +57,8 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) try { - result = self.info.GetValue(null, null); - return Converter.ToPython(result, self.info.PropertyType); + result = info.GetValue(null, null); + return Converter.ToPython(result, info.PropertyType); } catch (Exception e) { @@ -68,8 +74,8 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) try { - result = self.info.GetValue(co.inst, null); - return Converter.ToPython(result, self.info.PropertyType); + result = info.GetValue(co.inst, null); + return Converter.ToPython(result, info.PropertyType); } catch (Exception e) { @@ -91,7 +97,14 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) public new static int tp_descr_set(IntPtr ds, IntPtr ob, IntPtr val) { var self = (PropertyObject)GetManagedObject(ds); - MethodInfo setter = self.setter; + if (!self.info.Valid) + { + Exceptions.RaiseTypeError(self.info.DeletedMessage); + return -1; + } + var info = self.info.Value; + + MethodInfo setter = self.setter.UnsafeValue; object newval; if (val == IntPtr.Zero) @@ -107,7 +120,7 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) } - if (!Converter.ToManaged(val, self.info.PropertyType, out newval, true)) + if (!Converter.ToManaged(val, info.PropertyType, out newval, true)) { return -1; } @@ -133,11 +146,11 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) Exceptions.RaiseTypeError("invalid target"); return -1; } - self.info.SetValue(co.inst, newval, null); + info.SetValue(co.inst, newval, null); } else { - self.info.SetValue(null, newval, null); + info.SetValue(null, newval, null); } return 0; } @@ -159,7 +172,7 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) public static IntPtr tp_repr(IntPtr ob) { var self = (PropertyObject)GetManagedObject(ob); - return Runtime.PyString_FromString($""); + return Runtime.PyString_FromString($""); } } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 1e8db8278..63467c917 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -666,7 +666,8 @@ internal static Type[] PythonArgsToTypeArray(IntPtr arg, bool mangleObjects) if (mt is ClassBase) { - t = ((ClassBase)mt).type; + MaybeType _type = ((ClassBase)mt).type; + t = _type.Valid ? _type.Value : null; } else if (mt is CLRObject) { diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index 060573db4..f45e76db4 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -120,8 +120,8 @@ private static void RestoreRuntimeDataImpl() var objs = RestoreRuntimeDataObjects(storage.GetStorage("objs")); RestoreRuntimeDataModules(storage.GetStorage("modules")); - var clsObjs = ClassManager.RestoreRuntimeData(storage.GetStorage("classes")); TypeManager.RestoreRuntimeData(storage.GetStorage("types")); + var clsObjs = ClassManager.RestoreRuntimeData(storage.GetStorage("classes")); ImportHook.RestoreRuntimeData(storage.GetStorage("import")); PyCLRMetaType = MetaType.RestoreRuntimeData(storage.GetStorage("meta")); diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 31682c519..973a5aea2 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -21,9 +21,10 @@ internal class TypeManager internal static IntPtr subtype_clear; private const BindingFlags tbFlags = BindingFlags.Public | BindingFlags.Static; - private static Dictionary cache = new Dictionary(); + private static Dictionary cache = new Dictionary(); + private static readonly Dictionary _slotsHolders = new Dictionary(); - private static Dictionary _slotsImpls = new Dictionary(); + private static Dictionary _slotsImpls = new Dictionary(); // Slots which must be set private static readonly string[] _requiredSlots = new string[] @@ -77,11 +78,17 @@ internal static void RestoreRuntimeData(RuntimeDataStorage storage) { Debug.Assert(cache == null || cache.Count == 0); storage.GetValue("slots", out _slotsImpls); - storage.GetValue("cache", out cache); - foreach (var entry in cache) + storage.GetValue>("cache", out var _cache); + foreach (var entry in _cache) { - Type type = entry.Key; + if (!entry.Key.Valid) + { + Runtime.XDecref(entry.Value); + continue; + } + Type type = entry.Key.Value;; IntPtr handle = entry.Value; + cache[type] = handle; SlotsHolder holder = CreateSolotsHolder(handle); InitializeSlots(handle, _slotsImpls[type], holder); // FIXME: mp_length_slot.CanAssgin(clrType) @@ -170,6 +177,10 @@ internal static IntPtr CreateType(Type impl) Runtime.XDecref(mod); InitMethods(type, impl); + + // The type has been modified after PyType_Ready has been called + // Refresh the type + Runtime.PyType_Modified(type); return type; } @@ -350,7 +361,7 @@ internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr try { Type subType = ClassDerivedObject.CreateDerivedType(name, - baseClass.type, + baseClass.type.Value, py_dict, (string)namespaceStr, (string)assembly); @@ -459,6 +470,9 @@ internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) IntPtr mod = Runtime.PyString_FromString("CLR"); Runtime.PyDict_SetItemString(dict, "__module__", mod); + // The type has been modified after PyType_Ready has been called + // Refresh the type + Runtime.PyType_Modified(type); //DebugUtil.DumpType(type); return type; @@ -560,6 +574,10 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) IntPtr tp_dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); IntPtr mod = Runtime.PyString_FromString("CLR"); Runtime.PyDict_SetItem(tp_dict, PyIdentifier.__module__, mod); + + // The type has been modified after PyType_Ready has been called + // Refresh the type + Runtime.PyType_Modified(type); return type; }