From b2a911d2bc63b8fb24591a898ee4f3f379be6d68 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Mon, 12 Apr 2021 03:30:04 -0700 Subject: [PATCH] Replaced magic offsets with per-type calculation removed some macro-like method copy-pastes from CPython and bits of dead code All Python types created to represent CLR concepts derive from `CLR MetaType` (as before), which now has two new fields: - `tp_clr_inst_offset`, which is similar to `tp_dictoffset` in that it tells where to find `GCHandle` in instances of this type (e.g. where to find `GCHandle` pointing to `System.Uri` in corresponding Python object) - `tp_clr_inst`, which holds an optional instance of `ManagedType`, that implements the behavior of the type itself (e.g. `GCHandle` pointing to `ClassObject(type = System.Uri)`) So the layout of all Python types created by Python.NET is PyType type; nint tp_clr_inst_offset; GCHandel tp_clr_inst; (points, for example, to an instance of `ClassObject`) When a Python type, that reflects CLR type is created, the layout of instances will be: BaseTypeFields base; optional (if not in base): IntPtr dict; optional (if not in base): IntPtr weaklist; GCHandle gcHandle; (points to `CLRObject` instance, which in turn, for example, points to the actual instance of `System.Uri`) The offset to GC handle is recorded in the Python type's `tp_clr_inst_offset`, and can be found as `PyObject_Type(inst).tp_clr_inst_offset`. Or, preferably, accessor functions in `ManagedType` should be used. --- src/embed_tests/TestPyType.cs | 3 +- src/runtime/classbase.cs | 17 +- src/runtime/classderived.cs | 12 +- src/runtime/clrobject.cs | 23 +-- src/runtime/converter.cs | 6 +- src/runtime/exceptions.cs | 16 +- src/runtime/extensiontype.cs | 6 +- src/runtime/importhook.cs | 28 +-- src/runtime/interop.cs | 272 +---------------------------- src/runtime/managedtype.cs | 108 +++++++++--- src/runtime/metatype.cs | 27 ++- src/runtime/moduleobject.cs | 8 +- src/runtime/native/ABI.cs | 3 - src/runtime/native/ITypeOffsets.cs | 1 + src/runtime/native/TypeOffset.cs | 10 +- src/runtime/runtime.cs | 29 +-- src/runtime/runtime_data.cs | 2 +- src/runtime/typemanager.cs | 195 ++++++++++----------- 18 files changed, 264 insertions(+), 502 deletions(-) diff --git a/src/embed_tests/TestPyType.cs b/src/embed_tests/TestPyType.cs index 02142b782..f70a54c99 100644 --- a/src/embed_tests/TestPyType.cs +++ b/src/embed_tests/TestPyType.cs @@ -1,3 +1,4 @@ +using System.Runtime.InteropServices; using System.Text; using NUnit.Framework; @@ -30,7 +31,7 @@ public void CanCreateHeapType() using var doc = new StrPtr(docStr, Encoding.UTF8); var spec = new TypeSpec( name: name, - basicSize: ObjectOffset.Size(Runtime.Runtime.PyTypeType), + basicSize: Marshal.ReadInt32(Runtime.Runtime.PyBaseObjectType, TypeOffset.tp_basicsize), slots: new TypeSpec.Slot[] { new (TypeSlotID.tp_doc, doc.RawPointer), }, diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 8b96a96da..bf6a8034d 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -1,9 +1,6 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Diagnostics; -using System.Runtime.InteropServices; -using System.Runtime.Serialization; namespace Python.Runtime { @@ -355,19 +352,21 @@ public static void tp_dealloc(IntPtr ob) { ManagedType self = GetManagedObject(ob); tp_clear(ob); - Runtime.PyObject_GC_UnTrack(self.pyHandle); - Runtime.PyObject_GC_Del(self.pyHandle); - self.FreeGCHandle(); + Runtime.PyObject_GC_UnTrack(ob); + Runtime.PyObject_GC_Del(ob); + self?.FreeGCHandle(); } public static int tp_clear(IntPtr ob) { ManagedType self = GetManagedObject(ob); - if (!self.IsTypeObject()) + + bool isTypeObject = Runtime.PyObject_TYPE(ob) == Runtime.PyCLRMetaType; + if (!isTypeObject) { ClearObjectDict(ob); } - self.tpHandle = IntPtr.Zero; + if (self is not null) self.tpHandle = IntPtr.Zero; return 0; } @@ -391,7 +390,7 @@ protected override void OnLoad(InterDomainContext context) SetObjectDict(pyHandle, dict); } gcHandle = AllocGCHandle(); - Marshal.WriteIntPtr(pyHandle, TypeOffset.magic(), (IntPtr)gcHandle); + SetGCHandle(ObjectReference, gcHandle); } diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index 4e8e88bf3..8b15213c3 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Reflection; using System.Reflection.Emit; @@ -75,8 +76,8 @@ internal ClassDerivedObject(Type tp) : base(tp) // So we don't call PyObject_GC_Del here and instead we set the python // reference to a weak reference so that the C# object can be collected. GCHandle gc = GCHandle.Alloc(self, GCHandleType.Weak); - int gcOffset = ObjectOffset.magic(Runtime.PyObject_TYPE(self.pyHandle)); - Marshal.WriteIntPtr(self.pyHandle, gcOffset, (IntPtr)gc); + Debug.Assert(self.TypeReference == Runtime.PyObject_TYPE(self.ObjectReference)); + SetGCHandle(self.ObjectReference, self.TypeReference, gc); self.gcHandle.Free(); self.gcHandle = gc; } @@ -106,7 +107,7 @@ internal static IntPtr ToPython(IPythonDerivedType obj) Runtime._Py_NewReference(self.pyHandle); #endif GCHandle gc = GCHandle.Alloc(self, GCHandleType.Normal); - Marshal.WriteIntPtr(self.pyHandle, ObjectOffset.magic(self.tpHandle), (IntPtr)gc); + SetGCHandle(self.ObjectReference, self.TypeReference, gc); self.gcHandle.Free(); self.gcHandle = gc; @@ -883,11 +884,6 @@ public static void Finalize(IPythonDerivedType obj) // the C# object is being destroyed which must mean there are no more // references to the Python object as well so now we can dealloc the // python object. - IntPtr dict = Marshal.ReadIntPtr(self.pyHandle, ObjectOffset.TypeDictOffset(self.tpHandle)); - if (dict != IntPtr.Zero) - { - Runtime.XDecref(dict); - } Runtime.PyObject_GC_Del(self.pyHandle); self.gcHandle.Free(); } diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 0aa829ee6..46cd896e2 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -14,26 +14,16 @@ internal CLRObject(object ob, IntPtr tp) System.Diagnostics.Debug.Assert(tp != IntPtr.Zero); IntPtr py = Runtime.PyType_GenericAlloc(tp, 0); - var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); - if ((flags & TypeFlags.Subclass) != 0) - { - IntPtr dict = Marshal.ReadIntPtr(py, ObjectOffset.TypeDictOffset(tp)); - if (dict == IntPtr.Zero) - { - dict = Runtime.PyDict_New(); - Marshal.WriteIntPtr(py, ObjectOffset.TypeDictOffset(tp), dict); - } - } - - GCHandle gc = AllocGCHandle(TrackTypes.Wrapper); - Marshal.WriteIntPtr(py, ObjectOffset.magic(tp), (IntPtr)gc); tpHandle = tp; pyHandle = py; inst = ob; + GCHandle gc = AllocGCHandle(TrackTypes.Wrapper); + InitGCHandle(ObjectReference, type: TypeReference, gc); + // Fix the BaseException args (and __cause__ in case of Python 3) // slot if wrapping a CLR exception - Exceptions.SetArgsAndCause(py); + if (ob is Exception e) Exceptions.SetArgsAndCause(e, py); } protected CLRObject() @@ -78,6 +68,9 @@ internal static IntPtr GetInstHandle(object ob) return co.pyHandle; } + internal static NewReference GetReference(object ob) + => NewReference.DangerousFromPointer(GetInstHandle(ob)); + internal static CLRObject Restore(object ob, IntPtr pyHandle, InterDomainContext context) { CLRObject co = new CLRObject() @@ -101,7 +94,7 @@ protected override void OnLoad(InterDomainContext context) { base.OnLoad(context); GCHandle gc = AllocGCHandle(TrackTypes.Wrapper); - Marshal.WriteIntPtr(pyHandle, ObjectOffset.magic(tpHandle), (IntPtr)gc); + SetGCHandle(ObjectReference, TypeReference, gc); } } } diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 6b2e0f648..2e10e9041 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -580,7 +580,7 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo { if (Runtime.PyBytes_Size(value) == 1) { - op = Runtime.PyBytes_AS_STRING(value); + op = Runtime.PyBytes_AsString(value); result = (byte)Marshal.ReadByte(op); return true; } @@ -606,7 +606,7 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo { if (Runtime.PyBytes_Size(value) == 1) { - op = Runtime.PyBytes_AS_STRING(value); + op = Runtime.PyBytes_AsString(value); result = (byte)Marshal.ReadByte(op); return true; } @@ -632,7 +632,7 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo { if (Runtime.PyBytes_Size(value) == 1) { - op = Runtime.PyBytes_AS_STRING(value); + op = Runtime.PyBytes_AsString(value); result = (byte)Marshal.ReadByte(op); return true; } diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index 06d2d55b5..40e018fe6 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -156,15 +156,8 @@ internal static void Shutdown() /// pointer. /// /// The python object wrapping - internal static void SetArgsAndCause(IntPtr ob) + internal static void SetArgsAndCause(Exception e, IntPtr ob) { - // e: A CLR Exception - Exception e = ExceptionClassObject.ToException(ob); - if (e == null) - { - return; - } - IntPtr args; if (!string.IsNullOrEmpty(e.Message)) { @@ -177,13 +170,14 @@ internal static void SetArgsAndCause(IntPtr ob) args = Runtime.PyTuple_New(0); } - Marshal.WriteIntPtr(ob, ExceptionOffset.args, args); + if (Runtime.PyObject_SetAttrString(ob, "args", args) != 0) + throw new PythonException(); if (e.InnerException != null) { // Note: For an AggregateException, InnerException is only the first of the InnerExceptions. - IntPtr cause = CLRObject.GetInstHandle(e.InnerException); - Marshal.WriteIntPtr(ob, ExceptionOffset.cause, cause); + using var cause = CLRObject.GetReference(e.InnerException); + Runtime.PyException_SetCause(ob, cause.DangerousMoveToPointer()); } } diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index a5f0f1219..554837c46 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -33,13 +33,17 @@ public ExtensionType() tpHandle = tp; pyHandle = py; +#if DEBUG + GetGCHandle(ObjectReference, TypeReference, out var existing); + System.Diagnostics.Debug.Assert(existing == IntPtr.Zero); +#endif SetupGc(); } void SetupGc () { GCHandle gc = AllocGCHandle(TrackTypes.Extension); - Marshal.WriteIntPtr(pyHandle, ObjectOffset.magic(tpHandle), (IntPtr)gc); + InitGCHandle(ObjectReference, TypeReference, gc); // We have to support gc because the type machinery makes it very // hard not to - but we really don't have a need for it in most diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 184b588ad..be2281c8f 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -15,26 +15,6 @@ internal static class ImportHook private static IntPtr py_clr_module; static BorrowedReference ClrModuleReference => new BorrowedReference(py_clr_module); - private static IntPtr module_def = IntPtr.Zero; - - internal static void InitializeModuleDef() - { - if (module_def == IntPtr.Zero) - { - module_def = ModuleDefOffset.AllocModuleDef("clr"); - } - } - - internal static void ReleaseModuleDef() - { - if (module_def == IntPtr.Zero) - { - return; - } - ModuleDefOffset.FreeModuleDef(module_def); - module_def = IntPtr.Zero; - } - /// /// Initialize just the __import__ hook itself. /// @@ -90,8 +70,7 @@ internal static unsafe void Initialize() root = new CLRModule(); // create a python module with the same methods as the clr module-like object - InitializeModuleDef(); - py_clr_module = Runtime.PyModule_Create2(module_def, 3); + py_clr_module = Runtime.PyModule_New("clr").DangerousMoveToPointer(); // both dicts are borrowed references BorrowedReference mod_dict = Runtime.PyModule_GetDict(ClrModuleReference); @@ -116,13 +95,8 @@ internal static void Shutdown() RestoreImport(); - bool shouldFreeDef = Runtime.Refcount(py_clr_module) == 1; Runtime.XDecref(py_clr_module); py_clr_module = IntPtr.Zero; - if (shouldFreeDef) - { - ReleaseModuleDef(); - } Runtime.XDecref(root.pyHandle); root = null; diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index c5958e0f7..188db3a58 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -68,276 +68,6 @@ public ModulePropertyAttribute() } } - internal static partial class TypeOffset - { - public static int magic() => ManagedDataOffsets.Magic; - } - - internal static class ManagedDataOffsets - { - public static int Magic { get; internal set; } - public static readonly Dictionary NameMapping = new Dictionary(); - - static class DataOffsets - { - public static readonly int ob_data = 0; - public static readonly int ob_dict = 0; - - static DataOffsets() - { - FieldInfo[] fields = typeof(DataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); - for (int i = 0; i < fields.Length; i++) - { - fields[i].SetValue(null, -(i * IntPtr.Size) - IntPtr.Size); - } - } - } - - static ManagedDataOffsets() - { - NameMapping = TypeOffset.GetOffsets(); - - FieldInfo[] fields = typeof(DataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); - size = fields.Length * IntPtr.Size; - } - - public static int GetSlotOffset(string name) - { - return NameMapping[name]; - } - - private static int BaseOffset(IntPtr type) - { - Debug.Assert(type != IntPtr.Zero); - int typeSize = Marshal.ReadInt32(type, TypeOffset.tp_basicsize); - Debug.Assert(typeSize > 0); - return typeSize; - } - - public static int DataOffset(IntPtr type) - { - return BaseOffset(type) + DataOffsets.ob_data; - } - - public static int DictOffset(IntPtr type) - { - return BaseOffset(type) + DataOffsets.ob_dict; - } - - public static int ob_data => DataOffsets.ob_data; - public static int ob_dict => DataOffsets.ob_dict; - public static int Size { get { return size; } } - - private static readonly int size; - } - - internal static class OriginalObjectOffsets - { - static OriginalObjectOffsets() - { - int size = IntPtr.Size; - var n = 0; // Py_TRACE_REFS add two pointers to PyObject_HEAD -#if PYTHON_WITH_PYDEBUG - _ob_next = 0; - _ob_prev = 1 * size; - n = 2; -#endif - ob_refcnt = (n + 0) * size; - ob_type = (n + 1) * size; - } - - public static int Size { get { return size; } } - - private static readonly int size = -#if PYTHON_WITH_PYDEBUG - 4 * IntPtr.Size; -#else - 2 * IntPtr.Size; -#endif - -#if PYTHON_WITH_PYDEBUG - public static int _ob_next; - public static int _ob_prev; -#endif - public static int ob_refcnt; - public static int ob_type; - } - - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class ObjectOffset - { - static ObjectOffset() - { -#if PYTHON_WITH_PYDEBUG - _ob_next = OriginalObjectOffsets._ob_next; - _ob_prev = OriginalObjectOffsets._ob_prev; -#endif - ob_refcnt = OriginalObjectOffsets.ob_refcnt; - ob_type = OriginalObjectOffsets.ob_type; - - size = OriginalObjectOffsets.Size + ManagedDataOffsets.Size; - } - - public static int magic(IntPtr type) - { - return ManagedDataOffsets.DataOffset(type); - } - - public static int TypeDictOffset(IntPtr type) - { - return ManagedDataOffsets.DictOffset(type); - } - - public static int Size(IntPtr pyType) - { - if (IsException(pyType)) - { - return ExceptionOffset.Size(); - } - - return size; - } - -#if PYTHON_WITH_PYDEBUG - public static int _ob_next; - public static int _ob_prev; -#endif - public static int ob_refcnt; - public static int ob_type; - private static readonly int size; - - private static bool IsException(IntPtr pyObjectPtr) - { - var pyObject = new BorrowedReference(pyObjectPtr); - var type = Runtime.PyObject_TYPE(pyObject); - return Runtime.PyType_IsSameAsOrSubtype(type, ofType: Exceptions.BaseException) - || Runtime.PyType_IsSameAsOrSubtype(type, ofType: Runtime.PyTypeType) - && Runtime.PyType_IsSubtype(pyObject, Exceptions.BaseException); - } - } - - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class ExceptionOffset - { - static ExceptionOffset() - { - Type type = typeof(ExceptionOffset); - FieldInfo[] fi = type.GetFields(BindingFlags.Static | BindingFlags.Public); - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, (i * IntPtr.Size) + OriginalObjectOffsets.Size); - } - - size = fi.Length * IntPtr.Size + OriginalObjectOffsets.Size + ManagedDataOffsets.Size; - } - - public static int Size() { return size; } - - // PyException_HEAD - // (start after PyObject_HEAD) - public static int dict = 0; - public static int args = 0; - public static int traceback = 0; - public static int context = 0; - public static int cause = 0; - public static int suppress_context = 0; - - private static readonly int size; - } - - - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class BytesOffset - { - static BytesOffset() - { - Type type = typeof(BytesOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - /* The *real* layout of a type object when allocated on the heap */ - //typedef struct _heaptypeobject { -#if PYTHON_WITH_PYDEBUG -/* _PyObject_HEAD_EXTRA defines pointers to support a doubly-linked list of all live heap objects. */ - public static int _ob_next = 0; - public static int _ob_prev = 0; -#endif - // PyObject_VAR_HEAD { - // PyObject_HEAD { - public static int ob_refcnt = 0; - public static int ob_type = 0; - // } - public static int ob_size = 0; /* Number of items in _VAR_iable part */ - // } - public static int ob_shash = 0; - public static int ob_sval = 0; /* start of data */ - - /* Invariants: - * ob_sval contains space for 'ob_size+1' elements. - * ob_sval[ob_size] == 0. - * ob_shash is the hash of the string or -1 if not computed yet. - */ - //} PyBytesObject; - } - - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class ModuleDefOffset - { - static ModuleDefOffset() - { - Type type = typeof(ModuleDefOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, (i * size) + TypeOffset.ob_size); - } - } - - public static IntPtr AllocModuleDef(string modulename) - { - byte[] ascii = Encoding.ASCII.GetBytes(modulename); - int size = name + ascii.Length + 1; - IntPtr ptr = Marshal.AllocHGlobal(size); - for (int i = 0; i <= m_free; i += IntPtr.Size) - Marshal.WriteIntPtr(ptr, i, IntPtr.Zero); - Marshal.Copy(ascii, 0, (IntPtr)(ptr + name), ascii.Length); - Marshal.WriteIntPtr(ptr, m_name, (IntPtr)(ptr + name)); - Marshal.WriteByte(ptr, name + ascii.Length, 0); - return ptr; - } - - public static void FreeModuleDef(IntPtr ptr) - { - Marshal.FreeHGlobal(ptr); - } - - // typedef struct PyModuleDef{ - // typedef struct PyModuleDef_Base { - // starts after PyObject_HEAD (TypeOffset.ob_type + 1) - public static int m_init = 0; - public static int m_index = 0; - public static int m_copy = 0; - // } PyModuleDef_Base - public static int m_name = 0; - public static int m_doc = 0; - public static int m_size = 0; - public static int m_methods = 0; - public static int m_reload = 0; - public static int m_traverse = 0; - public static int m_clear = 0; - public static int m_free = 0; - // } PyModuleDef - - public static int name = 0; - } - - /// /// TypeFlags(): The actual bit values for the Type Flags stored /// in a class. @@ -357,7 +87,7 @@ public enum TypeFlags: int HaveStacklessExtension = 0, /* XXX Reusing reserved constants */ /// PythonNet specific - Managed = (1 << 15), + HasClrInstance = (1 << 15), /// PythonNet specific Subclass = (1 << 16), HaveIndex = (1 << 17), diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index d3ee697fd..41408abc7 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -28,6 +28,7 @@ internal enum TrackTypes internal IntPtr tpHandle; // PyType * internal BorrowedReference ObjectReference => new BorrowedReference(pyHandle); + internal BorrowedReference TypeReference => new BorrowedReference(tpHandle); private static readonly Dictionary _managedObjs = new Dictionary(); @@ -93,17 +94,10 @@ internal static ManagedType GetManagedObject(IntPtr ob) } var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); - if ((flags & TypeFlags.Managed) != 0) + if ((flags & TypeFlags.HasClrInstance) != 0) { - IntPtr op = tp == ob - ? Marshal.ReadIntPtr(tp, TypeOffset.magic()) - : Marshal.ReadIntPtr(ob, ObjectOffset.magic(tp)); - if (op == IntPtr.Zero) - { - return null; - } - var gc = (GCHandle)op; - return (ManagedType)gc.Target; + var gc = TryGetGCHandle(new BorrowedReference(ob)); + return (ManagedType)gc?.Target; } } return null; @@ -118,10 +112,9 @@ internal static ManagedType GetManagedObjectType(IntPtr ob) { IntPtr tp = Runtime.PyObject_TYPE(ob); var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); - if ((flags & TypeFlags.Managed) != 0) + if ((flags & TypeFlags.HasClrInstance) != 0) { - tp = Marshal.ReadIntPtr(tp, TypeOffset.magic()); - var gc = (GCHandle)tp; + var gc = GetGCHandle(new BorrowedReference(tp), Runtime.CLRMetaType); return (ManagedType)gc.Target; } } @@ -140,9 +133,9 @@ internal static ManagedType GetManagedObjectErr(IntPtr ob) } - internal static bool IsManagedType(BorrowedReference ob) - => IsManagedType(ob.DangerousGetAddressOrNull()); - internal static bool IsManagedType(IntPtr ob) + internal static bool IsInstanceOfManagedType(BorrowedReference ob) + => IsInstanceOfManagedType(ob.DangerousGetAddressOrNull()); + internal static bool IsInstanceOfManagedType(IntPtr ob) { if (ob != IntPtr.Zero) { @@ -152,18 +145,15 @@ internal static bool IsManagedType(IntPtr ob) tp = ob; } - var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); - if ((flags & TypeFlags.Managed) != 0) - { - return true; - } + return IsManagedType(new BorrowedReference(tp)); } return false; } - public bool IsTypeObject() + internal static bool IsManagedType(BorrowedReference type) { - return pyHandle == tpHandle; + var flags = (TypeFlags)Util.ReadCLong(type.DangerousGetAddress(), TypeOffset.tp_flags); + return (flags & TypeFlags.HasClrInstance) != 0; } internal static IDictionary GetManagedObjects() @@ -256,13 +246,81 @@ protected static void ClearObjectDict(IntPtr ob) protected static IntPtr GetObjectDict(IntPtr ob) { IntPtr type = Runtime.PyObject_TYPE(ob); - return Marshal.ReadIntPtr(ob, ObjectOffset.TypeDictOffset(type)); + int instanceDictOffset = Marshal.ReadInt32(type, TypeOffset.tp_dictoffset); + Debug.Assert(instanceDictOffset > 0); + return Marshal.ReadIntPtr(ob, instanceDictOffset); } protected static void SetObjectDict(IntPtr ob, IntPtr value) { IntPtr type = Runtime.PyObject_TYPE(ob); - Marshal.WriteIntPtr(ob, ObjectOffset.TypeDictOffset(type), value); + int instanceDictOffset = Marshal.ReadInt32(type, TypeOffset.tp_dictoffset); + Debug.Assert(instanceDictOffset > 0); + Marshal.WriteIntPtr(ob, instanceDictOffset, value); + } + + internal static void GetGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type, out IntPtr handle) + { + Debug.Assert(reflectedClrObject != null); + Debug.Assert(IsManagedType(type) || type == Runtime.CLRMetaType); + Debug.Assert(Runtime.PyObject_TypeCheck(reflectedClrObject, type)); + + int gcHandleOffset = Marshal.ReadInt32(type.DangerousGetAddress(), Offsets.tp_clr_inst_offset); + Debug.Assert(gcHandleOffset > 0); + + handle = Marshal.ReadIntPtr(reflectedClrObject.DangerousGetAddress(), gcHandleOffset); + } + + internal static GCHandle? TryGetGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type) + { + GetGCHandle(reflectedClrObject, type, out IntPtr handle); + return handle == IntPtr.Zero ? null : (GCHandle)handle; + } + internal static GCHandle? TryGetGCHandle(BorrowedReference reflectedClrObject) + { + BorrowedReference reflectedType = Runtime.PyObject_TYPE(reflectedClrObject); + + return TryGetGCHandle(reflectedClrObject, reflectedType); + } + + internal static GCHandle GetGCHandle(BorrowedReference reflectedClrObject) + => TryGetGCHandle(reflectedClrObject) ?? throw new InvalidOperationException(); + internal static GCHandle GetGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type) + => TryGetGCHandle(reflectedClrObject, type) ?? throw new InvalidOperationException(); + + internal static void InitGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type, GCHandle handle) + { + Debug.Assert(TryGetGCHandle(reflectedClrObject) == null); + + SetGCHandle(reflectedClrObject, type: type, handle); + } + internal static void InitGCHandle(BorrowedReference reflectedClrObject, GCHandle handle) + => InitGCHandle(reflectedClrObject, Runtime.PyObject_TYPE(reflectedClrObject), handle); + + internal static void SetGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type, GCHandle newHandle) + { + Debug.Assert(Runtime.PyObject_TypeCheck(reflectedClrObject, type)); + + int offset = Marshal.ReadInt32(type.DangerousGetAddress(), Offsets.tp_clr_inst_offset); + Debug.Assert(offset > 0); + + Marshal.WriteIntPtr(reflectedClrObject.DangerousGetAddress(), offset, (IntPtr)newHandle); + } + internal static void SetGCHandle(BorrowedReference reflectedClrObject, GCHandle newHandle) + => SetGCHandle(reflectedClrObject, Runtime.PyObject_TYPE(reflectedClrObject), newHandle); + + internal static class Offsets + { + static Offsets() + { + int pyTypeSize = Marshal.ReadInt32(Runtime.PyTypeType, TypeOffset.tp_basicsize); + if (pyTypeSize < 0) throw new InvalidOperationException(); + + tp_clr_inst_offset = pyTypeSize; + tp_clr_inst = tp_clr_inst_offset + IntPtr.Size; + } + public static int tp_clr_inst_offset { get; } + public static int tp_clr_inst { get; } } } } diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 68dae2508..1fde7dd78 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -1,6 +1,4 @@ using System; -using System.Collections; -using System.IO; using System.Runtime.InteropServices; using System.Runtime.Serialization; @@ -148,7 +146,7 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) } var flags = TypeFlags.Default; - flags |= TypeFlags.Managed; + flags |= TypeFlags.HasClrInstance; flags |= TypeFlags.HeapType; flags |= TypeFlags.BaseType; flags |= TypeFlags.Subclass; @@ -164,10 +162,16 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) TypeManager.CopySlot(base_type, type, TypeOffset.tp_traverse); TypeManager.CopySlot(base_type, type, TypeOffset.tp_clear); + // derived types must have their GCHandle at the same offset as the base types + int clrInstOffset = Marshal.ReadInt32(base_type, Offsets.tp_clr_inst_offset); + Marshal.WriteInt32(type, Offsets.tp_clr_inst_offset, clrInstOffset); // for now, move up hidden handle... - IntPtr gc = Marshal.ReadIntPtr(base_type, TypeOffset.magic()); - Marshal.WriteIntPtr(type, TypeOffset.magic(), gc); + IntPtr gc = Marshal.ReadIntPtr(base_type, Offsets.tp_clr_inst); + Marshal.WriteIntPtr(type, Offsets.tp_clr_inst, gc); + + if (Runtime.PyType_Ready(type) != 0) + throw new PythonException(); return type; } @@ -205,6 +209,11 @@ public static IntPtr tp_call(IntPtr tp, IntPtr args, IntPtr kw) return IntPtr.Zero; } + return CallInit(obj, args, kw); + } + + private static IntPtr CallInit(IntPtr obj, IntPtr args, IntPtr kw) + { var init = Runtime.PyObject_GetAttr(obj, PyIdentifier.__init__); Runtime.PyErr_Clear(); @@ -288,8 +297,12 @@ public static void tp_dealloc(IntPtr tp) var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Subclass) == 0) { - IntPtr gc = Marshal.ReadIntPtr(tp, TypeOffset.magic()); - ((GCHandle)gc).Free(); + GetGCHandle(new BorrowedReference(tp)).Free(); +#if DEBUG + // prevent ExecutionEngineException in debug builds in case we have a bug + // this would allow using managed debugger to investigate the issue + SetGCHandle(new BorrowedReference(tp), Runtime.CLRMetaType, default); +#endif } IntPtr op = Marshal.ReadIntPtr(tp, TypeOffset.ob_type); diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 41167e322..3c4e02a23 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -44,7 +44,10 @@ public ModuleObject(string name) docstring += "- " + a.FullName + "\n"; } - dict = Runtime.PyDict_New(); + var dictRef = Runtime.PyObject_GenericGetDict(ObjectReference); + PythonException.ThrowIfIsNull(dictRef); + dict = dictRef.DangerousMoveToPointer(); + using var pyname = NewReference.DangerousFromPointer(Runtime.PyString_FromString(moduleName)); using var pyfilename = NewReference.DangerousFromPointer(Runtime.PyString_FromString(filename)); using var pydocstring = NewReference.DangerousFromPointer(Runtime.PyString_FromString(docstring)); @@ -54,9 +57,6 @@ public ModuleObject(string name) Runtime.PyDict_SetItem(DictRef, PyIdentifier.__doc__, pydocstring); Runtime.PyDict_SetItem(DictRef, PyIdentifier.__class__, pycls); - Runtime.XIncref(dict); - SetObjectDict(pyHandle, dict); - InitializeModuleMembers(); } diff --git a/src/runtime/native/ABI.cs b/src/runtime/native/ABI.cs index 3264531de..339919dee 100644 --- a/src/runtime/native/ABI.cs +++ b/src/runtime/native/ABI.cs @@ -4,7 +4,6 @@ namespace Python.Runtime.Native using System.Globalization; using System.Linq; using System.Reflection; - using System.Runtime.InteropServices; static class ABI { @@ -29,8 +28,6 @@ internal static void Initialize(Version version, BorrowedReference pyType) } var typeOffsets = (ITypeOffsets)Activator.CreateInstance(typeOffsetsClass); TypeOffset.Use(typeOffsets); - - ManagedDataOffsets.Magic = Marshal.ReadInt32(pyType.DangerousGetAddress(), TypeOffset.tp_basicsize); } } } diff --git a/src/runtime/native/ITypeOffsets.cs b/src/runtime/native/ITypeOffsets.cs index 485c041f8..0829e5bc9 100644 --- a/src/runtime/native/ITypeOffsets.cs +++ b/src/runtime/native/ITypeOffsets.cs @@ -63,6 +63,7 @@ interface ITypeOffsets int tp_new { get; } int tp_repr { get; } int tp_richcompare { get; } + int tp_weaklistoffset { get; } int tp_setattro { get; } int tp_str { get; } int tp_traverse { get; } diff --git a/src/runtime/native/TypeOffset.cs b/src/runtime/native/TypeOffset.cs index 9f5ed671b..edbbe3b2c 100644 --- a/src/runtime/native/TypeOffset.cs +++ b/src/runtime/native/TypeOffset.cs @@ -70,6 +70,7 @@ static partial class TypeOffset internal static int tp_new { get; private set; } internal static int tp_repr { get; private set; } internal static int tp_richcompare { get; private set; } + internal static int tp_weaklistoffset { get; private set; } internal static int tp_setattro { get; private set; } internal static int tp_str { get; private set; } internal static int tp_traverse { get; private set; } @@ -91,9 +92,13 @@ internal static void Use(ITypeOffsets offsets) ValidateUnusedTypeOffsetProperties(offsetProperties); ValidateRequiredOffsetsPresent(offsetProperties); + + SlotOffsets = GetOffsets(); } static readonly BindingFlags FieldFlags = BindingFlags.NonPublic | BindingFlags.Static; + + static Dictionary SlotOffsets; internal static Dictionary GetOffsets() { var properties = typeof(TypeOffset).GetProperties(FieldFlags); @@ -104,10 +109,9 @@ internal static Dictionary GetOffsets() return result; } - internal static int GetOffsetUncached(string name) + public static int GetSlotOffset(string slotName) { - var property = typeof(TypeOffset).GetProperty(name, FieldFlags); - return (int)property.GetValue(obj: null, index: null); + return SlotOffsets[slotName]; } static readonly HashSet slotNames = new HashSet(); diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 4a8d01dd8..1b1a7eccc 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -470,7 +470,7 @@ private static void ClearClrModules() var item = PyList_GetItem(items, i); var name = PyTuple_GetItem(item, 0); var module = PyTuple_GetItem(item, 1); - if (ManagedType.IsManagedType(module)) + if (ManagedType.IsInstanceOfManagedType(module)) { PyDict_DelItem(modules, name); } @@ -523,6 +523,7 @@ private static void MoveClrInstancesOnwershipToPython() if (obj.gcHandle.IsAllocated) { obj.gcHandle.Free(); + ManagedType.SetGCHandle(obj.ObjectReference, default); } obj.gcHandle = default; } @@ -568,6 +569,8 @@ private static void MoveClrInstancesOnwershipToPython() internal static IntPtr PyNone; internal static IntPtr Error; + internal static BorrowedReference CLRMetaType => new BorrowedReference(PyCLRMetaType); + public static PyObject None { get @@ -1000,7 +1003,7 @@ internal static IntPtr PyObject_Type(IntPtr op) internal static string PyObject_GetTypeName(IntPtr op) { - IntPtr pyType = Marshal.ReadIntPtr(op, ObjectOffset.ob_type); + IntPtr pyType = PyObject_TYPE(op); IntPtr ppName = Marshal.ReadIntPtr(pyType, TypeOffset.tp_name); return Marshal.PtrToStringAnsi(ppName); } @@ -1010,7 +1013,7 @@ internal static string PyObject_GetTypeName(IntPtr op) /// internal static bool PyObject_IsIterable(IntPtr pointer) { - var ob_type = Marshal.ReadIntPtr(pointer, ObjectOffset.ob_type); + var ob_type = PyObject_TYPE(pointer); IntPtr tp_iter = Marshal.ReadIntPtr(ob_type, TypeOffset.tp_iter); return tp_iter != IntPtr.Zero; } @@ -1508,6 +1511,13 @@ internal static IntPtr EmptyPyBytes() return Delegates.PyBytes_FromString((IntPtr)bytes); } + internal static IntPtr PyBytes_AsString(IntPtr ob) => PyBytes_AsString(new BorrowedReference(ob)); + internal static IntPtr PyBytes_AsString(BorrowedReference ob) + { + Debug.Assert(ob != null); + return Delegates.PyBytes_AsString(ob); + } + internal static long PyBytes_Size(IntPtr op) { return (long)_PyBytes_Size(op); @@ -1516,11 +1526,6 @@ internal static long PyBytes_Size(IntPtr op) private static IntPtr _PyBytes_Size(IntPtr op) => Delegates._PyBytes_Size(op); - internal static IntPtr PyBytes_AS_STRING(IntPtr ob) - { - return ob + BytesOffset.ob_sval; - } - internal static IntPtr PyUnicode_FromStringAndSize(IntPtr value, long size) { @@ -1615,7 +1620,7 @@ internal static string GetManagedString(IntPtr op) { using var p = PyUnicode_AsUTF16String(new BorrowedReference(op)); int length = (int)PyUnicode_GetSize(op); - char* codePoints = (char*)PyBytes_AS_STRING(p.DangerousGetAddress()); + char* codePoints = (char*)PyBytes_AsString(p.DangerousGetAddress()); return new string(codePoints, startIndex: 1, // skip BOM length: length); @@ -1876,7 +1881,7 @@ internal static IntPtr PyTuple_GetSlice(IntPtr pointer, long start, long end) internal static bool PyIter_Check(IntPtr pointer) { - var ob_type = Marshal.ReadIntPtr(pointer, ObjectOffset.ob_type); + var ob_type = PyObject_TYPE(pointer); IntPtr tp_iternext = Marshal.ReadIntPtr(ob_type, TypeOffset.tp_iternext); return tp_iternext != IntPtr.Zero && tp_iternext != _PyObject_NextNotImplemented; } @@ -2248,7 +2253,7 @@ internal static IntPtr GetBuiltins() return PyImport_Import(PyIdentifier.builtins); } - private static class Delegates + internal static class Delegates { static readonly ILibraryLoader libraryLoader = LibraryLoader.Instance; @@ -2406,6 +2411,7 @@ static Delegates() _PySequence_Count = (delegate* unmanaged[Cdecl])GetFunctionByName("PySequence_Count", GetUnmanagedDll(_PythonDll)); PySequence_Tuple = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_Tuple), GetUnmanagedDll(_PythonDll)); PySequence_List = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_List), GetUnmanagedDll(_PythonDll)); + PyBytes_AsString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBytes_AsString), GetUnmanagedDll(_PythonDll)); PyBytes_FromString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBytes_FromString), GetUnmanagedDll(_PythonDll)); _PyBytes_Size = (delegate* unmanaged[Cdecl])GetFunctionByName("PyBytes_Size", GetUnmanagedDll(_PythonDll)); PyUnicode_FromStringAndSize = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromStringAndSize), GetUnmanagedDll(_PythonDll)); @@ -2683,6 +2689,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] _PySequence_Count { get; } internal static delegate* unmanaged[Cdecl] PySequence_Tuple { get; } internal static delegate* unmanaged[Cdecl] PySequence_List { get; } + internal static delegate* unmanaged[Cdecl] PyBytes_AsString { get; } internal static delegate* unmanaged[Cdecl] PyBytes_FromString { get; } internal static delegate* unmanaged[Cdecl] _PyBytes_Size { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_FromStringAndSize { get; } diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index 0b3bf3017..29cea4181 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -279,7 +279,7 @@ private static void SaveRuntimeDataModules(RuntimeDataStorage storage) var item = PyList_GetItem(items, i); var name = PyTuple_GetItem(item.DangerousGetAddress(), 0); var module = PyTuple_GetItem(item.DangerousGetAddress(), 1); - if (ManagedType.IsManagedType(module)) + if (ManagedType.IsInstanceOfManagedType(module)) { XIncref(name); XIncref(module); diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 01aceb656..20b95d85c 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -134,7 +134,7 @@ internal static BorrowedReference GetTypeReference(Type type) /// The given ManagedType instance is a managed object that implements /// the appropriate semantics in Python for the reflected managed type. /// - internal static IntPtr GetTypeHandle(ManagedType obj, Type type) + internal static IntPtr GetTypeHandle(ClassBase obj, Type type) { IntPtr handle; cache.TryGetValue(type, out handle); @@ -157,21 +157,28 @@ internal static IntPtr GetTypeHandle(ManagedType obj, Type type) /// behavior needed and the desire to have the existing Python runtime /// do as much of the allocation and initialization work as possible. /// - internal static IntPtr CreateType(Type impl) + internal static unsafe IntPtr CreateType(Type impl) { - IntPtr type = AllocateTypeObject(impl.Name, metatype: Runtime.PyTypeType); - int ob_size = ObjectOffset.Size(type); + IntPtr type = AllocateTypeObject(impl.Name, metatype: Runtime.PyCLRMetaType); + IntPtr base_ = impl == typeof(CLRModule) + ? Runtime.PyModuleType + : Runtime.PyBaseObjectType; + int newFieldOffset = InheritOrAllocateStandardFields(type, base_); + + int tp_clr_inst_offset = newFieldOffset; + newFieldOffset += IntPtr.Size; + + int ob_size = newFieldOffset; // Set tp_basicsize to the size of our managed instance objects. Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); - - var offset = (IntPtr)ObjectOffset.TypeDictOffset(type); - Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); + Marshal.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, tp_clr_inst_offset); + Marshal.WriteIntPtr(type, TypeOffset.tp_new, (IntPtr)Runtime.Delegates.PyType_GenericNew); SlotsHolder slotsHolder = CreateSolotsHolder(type); InitializeSlots(type, impl, slotsHolder); - var flags = TypeFlags.Default | TypeFlags.Managed | + var flags = TypeFlags.Default | TypeFlags.HasClrInstance | TypeFlags.HeapType | TypeFlags.HaveGC; Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); @@ -194,7 +201,7 @@ internal static IntPtr CreateType(Type impl) } - internal static IntPtr CreateType(ManagedType impl, Type clrType) + internal static IntPtr CreateType(ClassBase impl, Type clrType) { // Cleanup the type name to get rid of funny nested type names. string name = $"clr.{clrType.FullName}"; @@ -209,19 +216,7 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) name = name.Substring(i + 1); } - IntPtr base_ = IntPtr.Zero; - int ob_size = ObjectOffset.Size(Runtime.PyTypeType); - - // XXX Hack, use a different base class for System.Exception - // Python 2.5+ allows new style class exceptions but they *must* - // subclass BaseException (or better Exception). - if (typeof(Exception).IsAssignableFrom(clrType)) - { - ob_size = ObjectOffset.Size(Exceptions.Exception); - } - - int tp_dictoffset = ob_size + ManagedDataOffsets.ob_dict; - + IntPtr base_ = Runtime.PyBaseObjectType; if (clrType == typeof(Exception)) { base_ = Exceptions.Exception; @@ -229,17 +224,32 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) else if (clrType.BaseType != null) { ClassBase bc = ClassManager.GetClass(clrType.BaseType); - base_ = bc.pyHandle; + if (bc.ObjectReference != null) + { + // there are cases when base class has not been fully initialized yet (nested types) + base_ = bc.pyHandle; + } } IntPtr type = AllocateTypeObject(name, Runtime.PyCLRMetaType); - Marshal.WriteIntPtr(type, TypeOffset.ob_type, Runtime.PyCLRMetaType); - Runtime.XIncref(Runtime.PyCLRMetaType); + int newFieldOffset = InheritOrAllocateStandardFields(type, base_); + + if (ManagedType.IsManagedType(new BorrowedReference(base_))) + { + int baseClrInstOffset = Marshal.ReadInt32(base_, ManagedType.Offsets.tp_clr_inst_offset); + Marshal.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, baseClrInstOffset); + } + else + { + Marshal.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, newFieldOffset); + newFieldOffset += IntPtr.Size; + } + + int ob_size = newFieldOffset; Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); - Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, (IntPtr)tp_dictoffset); // we want to do this after the slot stuff above in case the class itself implements a slot method SlotsHolder slotsHolder = CreateSolotsHolder(type); @@ -260,24 +270,16 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) // Only set mp_subscript and mp_ass_subscript for types with indexers - if (impl is ClassBase cb) + if (!(impl is ArrayObject)) { - if (!(impl is ArrayObject)) + if (impl.indexer == null || !impl.indexer.CanGet) { - if (cb.indexer == null || !cb.indexer.CanGet) - { - Marshal.WriteIntPtr(type, TypeOffset.mp_subscript, IntPtr.Zero); - } - if (cb.indexer == null || !cb.indexer.CanSet) - { - Marshal.WriteIntPtr(type, TypeOffset.mp_ass_subscript, IntPtr.Zero); - } + Marshal.WriteIntPtr(type, TypeOffset.mp_subscript, IntPtr.Zero); + } + if (impl.indexer == null || !impl.indexer.CanSet) + { + Marshal.WriteIntPtr(type, TypeOffset.mp_ass_subscript, IntPtr.Zero); } - } - else - { - Marshal.WriteIntPtr(type, TypeOffset.mp_subscript, IntPtr.Zero); - Marshal.WriteIntPtr(type, TypeOffset.mp_ass_subscript, IntPtr.Zero); } if (base_ != IntPtr.Zero) @@ -287,7 +289,7 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) } const TypeFlags flags = TypeFlags.Default - | TypeFlags.Managed + | TypeFlags.HasClrInstance | TypeFlags.HeapType | TypeFlags.BaseType | TypeFlags.HaveGC; @@ -309,9 +311,11 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Runtime.PyDict_SetItem(dict, PyIdentifier.__module__, mod); mod.Dispose(); + var typeRef = new BorrowedReference(type); + // Hide the gchandle of the implementation in a magic type slot. GCHandle gc = impl.AllocGCHandle(); - Marshal.WriteIntPtr(type, TypeOffset.magic(), (IntPtr)gc); + ManagedType.InitGCHandle(typeRef, Runtime.CLRMetaType, gc); // Set the handle attributes on the implementing instance. impl.tpHandle = type; @@ -322,6 +326,31 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) return type; } + static int InheritOrAllocateStandardFields(IntPtr type, IntPtr @base) + { + int baseSize = Marshal.ReadInt32(@base, TypeOffset.tp_basicsize); + int newFieldOffset = baseSize; + + void InheritOrAllocate(int typeField) + { + int value = Marshal.ReadInt32(@base, typeField); + if (value == 0) + { + Marshal.WriteIntPtr(type, typeField, new IntPtr(newFieldOffset)); + newFieldOffset += IntPtr.Size; + } + else + { + Marshal.WriteIntPtr(type, typeField, new IntPtr(value)); + } + } + + InheritOrAllocate(TypeOffset.tp_dictoffset); + InheritOrAllocate(TypeOffset.tp_weaklistoffset); + + return newFieldOffset; + } + internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr py_dict) { var dictRef = new BorrowedReference(py_dict); @@ -454,11 +483,14 @@ internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) Marshal.WriteIntPtr(type, TypeOffset.tp_base, py_type); Runtime.XIncref(py_type); - int size = TypeOffset.magic() + IntPtr.Size; + int size = Marshal.ReadInt32(Runtime.PyTypeType, TypeOffset.tp_basicsize) + + IntPtr.Size // tp_clr_inst_offset + + IntPtr.Size // tp_clr_inst + ; Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, new IntPtr(size)); + Marshal.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, ManagedType.Offsets.tp_clr_inst); const TypeFlags flags = TypeFlags.Default - | TypeFlags.Managed | TypeFlags.HeapType | TypeFlags.HaveGC; Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); @@ -544,53 +576,6 @@ private static IntPtr AddCustomMetaMethod(string name, IntPtr type, IntPtr mdef, return mdef; } - internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) - { - // Utility to create a subtype of a std Python type, but with - // a managed type able to override implementation - - IntPtr type = AllocateTypeObject(name, metatype: Runtime.PyTypeType); - //Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)obSize); - //Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); - - //IntPtr offset = (IntPtr)ObjectOffset.ob_dict; - //Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); - - //IntPtr dc = Runtime.PyDict_Copy(dict); - //Marshal.WriteIntPtr(type, TypeOffset.tp_dict, dc); - - Marshal.WriteIntPtr(type, TypeOffset.tp_base, base_); - Runtime.XIncref(base_); - - var flags = TypeFlags.Default; - flags |= TypeFlags.Managed; - flags |= TypeFlags.HeapType; - flags |= TypeFlags.HaveGC; - Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); - - CopySlot(base_, type, TypeOffset.tp_traverse); - CopySlot(base_, type, TypeOffset.tp_clear); - CopySlot(base_, type, TypeOffset.tp_is_gc); - - SlotsHolder slotsHolder = CreateSolotsHolder(type); - InitializeSlots(type, impl, slotsHolder); - - if (Runtime.PyType_Ready(type) != 0) - { - throw new PythonException(); - } - - IntPtr tp_dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); - IntPtr mod = Runtime.PyString_FromString("CLR"); - Runtime.PyDict_SetItem(tp_dict, PyIdentifier.__module__, mod); - - // The type has been modified after PyType_Ready has been called - // Refresh the type - Runtime.PyType_Modified(type); - - return type; - } - /// /// Utility method to allocate a type object & do basic initialization. @@ -598,6 +583,7 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) internal static IntPtr AllocateTypeObject(string name, IntPtr metatype) { IntPtr type = Runtime.PyType_GenericAlloc(metatype, 0); + PythonException.ThrowIfIsNull(type); // Clr type would not use __slots__, // and the PyMemberDef after PyHeapTypeObject will have other uses(e.g. type handle), // thus set the ob_size to 0 for avoiding slots iterations. @@ -670,7 +656,7 @@ internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHo { continue; } - var offset = ManagedDataOffsets.GetSlotOffset(slot); + var offset = TypeOffset.GetSlotOffset(slot); Marshal.WriteIntPtr(type, offset, SlotsHolder.GetDefaultSlot(offset)); } } @@ -688,7 +674,7 @@ internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHo /// Can override the slot when it existed static void InitializeSlot(IntPtr type, IntPtr slot, string name, bool canOverride = true) { - var offset = ManagedDataOffsets.GetSlotOffset(name); + var offset = TypeOffset.GetSlotOffset(name); if (!canOverride && Marshal.ReadIntPtr(type, offset) != IntPtr.Zero) { return; @@ -698,7 +684,7 @@ static void InitializeSlot(IntPtr type, IntPtr slot, string name, bool canOverri static void InitializeSlot(IntPtr type, ThunkInfo thunk, string name, SlotsHolder slotsHolder = null, bool canOverride = true) { - int offset = ManagedDataOffsets.GetSlotOffset(name); + int offset = TypeOffset.GetSlotOffset(name); if (!canOverride && Marshal.ReadIntPtr(type, offset) != IntPtr.Zero) { @@ -723,7 +709,7 @@ static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method, Slots static bool IsSlotSet(IntPtr type, string name) { - int offset = ManagedDataOffsets.GetSlotOffset(name); + int offset = TypeOffset.GetSlotOffset(name); return Marshal.ReadIntPtr(type, offset) != IntPtr.Zero; } @@ -794,6 +780,8 @@ class SlotsHolder private List _deallocators = new List(); private bool _alreadyReset = false; + BorrowedReference Type => new BorrowedReference(_type); + /// /// Create slots holder for holding the delegate of slots and be able to reset them. /// @@ -861,15 +849,18 @@ public void ResetSlots() _deallocators.Clear(); // Custom reset - IntPtr handlePtr = Marshal.ReadIntPtr(_type, TypeOffset.magic()); - if (handlePtr != IntPtr.Zero) + if (Type != Runtime.CLRMetaType) { - GCHandle handle = GCHandle.FromIntPtr(handlePtr); - if (handle.IsAllocated) + var metatype = Runtime.PyObject_TYPE(Type); + if (ManagedType.TryGetGCHandle(Type, metatype) is { } handle) { - handle.Free(); + if (handle.IsAllocated) + { + handle.Free(); + } + + ManagedType.SetGCHandle(Type, metatype, default); } - Marshal.WriteIntPtr(_type, TypeOffset.magic(), IntPtr.Zero); } }