UNI-63112: implement platform-aware native code for tp_traverse et al

1. Implement tp_traverse, tp_clear, and tp_is_gc in native code in memory that is *not* released upon domain reload. Effect: fixes the crash on domain reload. Side effect: leaks a page every domain reload. 2. Use python's platform package to determine what we're running on, so we can use the right mmap/mprotect or VirtualAlloc/VirtualProtect routines, the right flags for mmap, and (theoretically) the right native code. It should be easy to add another system, as long as it has the same kinds of functionality. 3. Enable the domain reload test. Added some unit tests for the new stuff. 4. Remove tp_traverse, tp_clear, and tp_is_gc where it was implemented. Note: I haven't actually tested on windows and linux yet, I'm checking in so I can do that easily.
pythonnet · benoithudson · Sep 14, 2018 · Sep 14, 2018 · Sep 26, 2018 · Sep 26, 2018
commit 9ae91baa163a1c8ce4f78d2fcb6b45bd969f2386
diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj
@@ -104,6 +104,7 @@
     <Compile Include="TestPyWith.cs" />
     <Compile Include="TestRuntime.cs" />
     <Compile Include="TestPyScope.cs" />
+    <Compile Include="TestTypeManager.cs" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\runtime\Python.Runtime.csproj">

diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs
@@ -40,7 +40,6 @@ class TestDomainReload
         /// a hot reload).
         /// </summary>
         [Test]
-        [Ignore("Test crashes")]
         public static void DomainReloadAndGC()
         {
             // We're set up to run in the directory that includes the bin directory.

diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs
@@ -6,6 +6,25 @@ namespace Python.EmbeddingTest
 {
     public class TestRuntime
     {
+        /// <summary>
+        /// Test the cache of the information from the platform module.
+        ///
+        /// Test fails on platforms we haven't implemented yet.
+        /// </summary>
+        [Test]
+        public static void PlatformCache()
+        {
+            Runtime.Runtime.Initialize();
+
+            Assert.That(Runtime.Runtime.Machine, Is.Not.EqualTo(Runtime.Runtime.MachineType.Other));
+            Assert.That(!string.IsNullOrEmpty(Runtime.Runtime.MachineName));
+
+            Assert.That(Runtime.Runtime.OperatingSystem, Is.Not.EqualTo(Runtime.Runtime.OperatingSystemType.Other));
+            Assert.That(!string.IsNullOrEmpty(Runtime.Runtime.OperatingSystemName));
+
+            Runtime.Runtime.Shutdown();
+        }
+
         [Test]
         public static void Py_IsInitializedValue()
         {

diff --git a/src/embed_tests/TestTypeManager.cs b/src/embed_tests/TestTypeManager.cs
@@ -0,0 +1,45 @@
+using NUnit.Framework;
+using Python.Runtime;
+using System.Runtime.InteropServices;
+
+namespace Python.EmbeddingTest
+{
+    class TestTypeManager
+    {
+        [Test]
+        public static void TestNativeCode()
+        {
+            Runtime.Runtime.Initialize();
+
+            Assert.That(() => { var _ = TypeManager.NativeCode.Active; }, Throws.Nothing);
+            Assert.That(TypeManager.NativeCode.Active.Code.Length, Is.GreaterThan(0));
+
+            Runtime.Runtime.Shutdown();
+        }
+
+        [Test]
+        public static void TestMemoryMapping()
+        {
+            Runtime.Runtime.Initialize();
+
+            Assert.That(() => { var _ = TypeManager.CreateMemoryMapper(); }, Throws.Nothing);
+            var mapper = TypeManager.CreateMemoryMapper();
+
+            // Allocate a read-write page.
+            int len = 12;
+            var page = mapper.MapWriteable(len);
+            Assert.That(() => { Marshal.WriteInt64(page, 17); }, Throws.Nothing);
+            Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17));
+
+            // Mark it read-execute, now we can't write anymore (I'm not testing we can execute).
+            // We should be getting AccessViolationException, but Mono translates
+            // SIGSEGV to NullReferenceException instead, so just check for some exception.
+            mapper.SetReadExec(page, len);
+            Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17));
+            Assert.That(() => { Marshal.WriteInt64(page, 18); }, Throws.Exception);
+            Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17));
+
+            Runtime.Runtime.Shutdown();
+        }
+    }
+}
diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs
@@ -247,24 +247,6 @@ public static IntPtr tp_str(IntPtr ob)
         }
 
 
-        /// <summary>
-        /// Default implementations for required Python GC support.
-        /// </summary>
-        public static int tp_traverse(IntPtr ob, IntPtr func, IntPtr args)
-        {
-            return 0;
-        }
-
-        public static int tp_clear(IntPtr ob)
-        {
-            return 0;
-        }
-
-        public static int tp_is_gc(IntPtr type)
-        {
-            return 1;
-        }
-
         /// <summary>
         /// Standard dealloc implementation for instances of reflected types.
         /// </summary>

diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs
@@ -81,27 +81,6 @@ public static int tp_descr_set(IntPtr ds, IntPtr ob, IntPtr val)
         }
 
 
-        /// <summary>
-        /// Required Python GC support.
-        /// </summary>
-        public static int tp_traverse(IntPtr ob, IntPtr func, IntPtr args)
-        {
-            return 0;
-        }
-
-
-        public static int tp_clear(IntPtr ob)
-        {
-            return 0;
-        }
-
-
-        public static int tp_is_gc(IntPtr type)
-        {
-            return 1;
-        }
-
-
         /// <summary>
         /// Default dealloc implementation.
         /// </summary>

diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs
@@ -2,6 +2,7 @@
 using System.Runtime.InteropServices;
 using System.Security;
 using System.Text;
+using System.Collections.Generic;
 
 namespace Python.Runtime
 {
@@ -195,6 +196,57 @@ public class Runtime
         // .NET core: System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
         internal static bool IsWindows = Environment.OSVersion.Platform == PlatformID.Win32NT;
 
+        /// <summary>
+        /// Operating system type as reported by Python.
+        /// </summary>
+        public enum OperatingSystemType
+        {
+            Windows,
+            Darwin,
+            Linux,
+            Other
+        }
+
+        static readonly Dictionary<string, OperatingSystemType> OperatingSystemTypeMapping = new Dictionary<string, OperatingSystemType>()
+        {
+            { "Windows", OperatingSystemType.Windows },
+            { "Darwin", OperatingSystemType.Darwin },
+            { "Linux", OperatingSystemType.Linux },
+        };
+
+        /// <summary>
+        /// Gets the operating system as reported by python's platform.system().
+        /// </summary>
+        public static OperatingSystemType OperatingSystem { get; private set; }
+
+        /// <summary>
+        /// Gets the operating system as reported by python's platform.system().
+        /// </summary>
+            public static string OperatingSystemName { get; private set; }
+
+        public enum MachineType
+        {
+            i386,
+            x86_64,
+            Other
+        };
+
+        static readonly Dictionary<string, MachineType> MachineTypeMapping = new Dictionary<string, MachineType>()
+        {
+            { "i386", MachineType.i386 },
+            { "x86_64", MachineType.x86_64 },
+        };
+
+        /// <summary>
+        /// Gets the machine architecture as reported by python's platform.machine().
+        /// </summary>
+        public static MachineType Machine { get; private set; }/* set in Initialize using python's platform.machine */
+
+        /// <summary>
+        /// Gets the machine architecture as reported by python's platform.machine().
+        /// </summary>
+        public static string MachineName { get; private set; }
+
         internal static bool IsPython2 = pyversionnumber < 30;
         internal static bool IsPython3 = pyversionnumber >= 30;
 
@@ -331,6 +383,10 @@ internal static void Initialize()
                 NativeMethods.FreeLibrary(dllLocal);
             }
 #endif
+            // Initialize data about the platform we're running on. We need
+            // this for the type manager and potentially other details. Must
+            // happen after caching the python types, above.
+            InitializePlatformData();
 
             // Initialize modules that depend on the runtime class.
             AssemblyManager.Initialize();
@@ -348,6 +404,53 @@ internal static void Initialize()
             AssemblyManager.UpdatePath();
         }
 
+        /// <summary>
+        /// Initializes the data about platforms.
+        ///
+        /// This must be the last step when initializing the runtime:
+        /// GetManagedString needs to have the cached values for types.
+        /// But it must run before initializing anything outside the runtime
+        /// because those rely on the platform data.
+        /// </summary>
+        private static void InitializePlatformData()
+        {
+            IntPtr op;
+            IntPtr fn;
+            IntPtr platformModule = PyImport_ImportModule("platform");
+            IntPtr emptyTuple = PyTuple_New(0);
+
+            fn = PyObject_GetAttrString(platformModule, "system");
+            op = PyObject_Call(fn, emptyTuple, IntPtr.Zero);
+            OperatingSystemName = GetManagedString(op);
+            XDecref(op);
+            XDecref(fn);
+
+            fn = PyObject_GetAttrString(platformModule, "machine");
+            op = PyObject_Call(fn, emptyTuple, IntPtr.Zero);
+            MachineName = GetManagedString(op);
+            XDecref(op);
+            XDecref(fn);
+
+            XDecref(emptyTuple);
+            XDecref(platformModule);
+
+            // Now convert the strings into enum values so we can do switch
+            // statements rather than constant parsing.
+            OperatingSystemType OSType;
+            if (!OperatingSystemTypeMapping.TryGetValue(OperatingSystemName, out OSType))
+            {
+                OSType = OperatingSystemType.Other;
+            }
+            OperatingSystem = OSType;
+
+            MachineType MType;
+            if (!MachineTypeMapping.TryGetValue(MachineName, out MType))
+            {
+                MType = MachineType.Other;
+            }
+            Machine = MType;
+        }
+
         internal static void Shutdown()
         {
             AssemblyManager.Shutdown();