From 5c37f3329896679d13d7b96683930a5e9d77ed25 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 14 Oct 2020 13:50:19 +0200 Subject: [PATCH 1/4] Add Python 3.9 to CI --- .travis.yml | 1 + appveyor.yml | 5 +++++ src/runtime/runtime.cs | 2 ++ 3 files changed, 8 insertions(+) diff --git a/.travis.yml b/.travis.yml index e664a4696..87dcf97af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ dist: xenial sudo: false language: python python: + - 3.9 - 3.8 - 3.7 - 3.6 diff --git a/appveyor.yml b/appveyor.yml index 507c3cab2..153629736 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -15,15 +15,20 @@ environment: CODECOV_ENV: PYTHON_VERSION, PLATFORM matrix: + - PYTHON_VERSION: 3.9 + BUILD_OPTS: --xplat - PYTHON_VERSION: 3.8 BUILD_OPTS: --xplat - PYTHON_VERSION: 3.7 BUILD_OPTS: --xplat - PYTHON_VERSION: 3.6 BUILD_OPTS: --xplat + - PYTHON_VERSION: 3.9 - PYTHON_VERSION: 3.8 - PYTHON_VERSION: 3.7 - PYTHON_VERSION: 3.6 + - PYTHON_VERSION: 3.9 + PYTHONNET_SHUTDOWN_MODE: Soft - PYTHON_VERSION: 3.8 PYTHONNET_SHUTDOWN_MODE: Soft - PYTHON_VERSION: 3.7 diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 915e1db00..ddeec15a2 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -50,6 +50,8 @@ public class Runtime const string _minor = "7"; #elif PYTHON38 const string _minor = "8"; +#elif PYTHON39 + const string _minor = "9"; #else #error You must define one of PYTHON36 to PYTHON38 #endif From 34348aaff8c1f80ba4b42cc82003fdbc551f2c88 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 11 Nov 2020 20:19:19 +0100 Subject: [PATCH 2/4] Update AppVeyor image and always install all requirements --- appveyor.yml | 2 +- requirements.txt | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 153629736..a91afdcba 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,7 +2,7 @@ version: '{branch}-{build}' build: off image: - - Visual Studio 2017 + - Visual Studio 2019 platform: - x86 diff --git a/requirements.txt b/requirements.txt index 29c2e4566..78570cb95 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # Requirements for both Travis and AppVeyor -pytest==3.2.5 +pytest coverage psutil @@ -7,6 +7,5 @@ psutil codecov # Platform specific requirements -# pip; sys_platform == 'win32' -wheel; sys_platform == 'win32' -pycparser; sys_platform != 'win32' +wheel +pycparser From fd4f9bc4f26ba82fdf63db375d5ff76d4f9dc07a Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 11 Nov 2020 20:23:39 +0100 Subject: [PATCH 3/4] Add Python 3.9 interop file --- src/runtime/Python.Runtime.csproj | 3 +- src/runtime/interop39.cs | 177 ++++++++++++++++++++++++++++++ src/runtime/runtime.cs | 8 +- 3 files changed, 183 insertions(+), 5 deletions(-) create mode 100644 src/runtime/interop39.cs diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index a448e2bbd..672eff96a 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,4 +1,4 @@ - + Debug @@ -167,6 +167,7 @@ + diff --git a/src/runtime/interop39.cs b/src/runtime/interop39.cs new file mode 100644 index 000000000..30635d9fd --- /dev/null +++ b/src/runtime/interop39.cs @@ -0,0 +1,177 @@ + +// Auto-generated by geninterop.py. +// DO NOT MODIFY BY HAND. + + +#if PYTHON39 +using System; +using System.Collections; +using System.Collections.Specialized; +using System.Runtime.InteropServices; +using System.Reflection; +using System.Text; + +namespace Python.Runtime +{ + + [StructLayout(LayoutKind.Sequential)] + internal static partial class TypeOffset + { + // Auto-generated from PyHeapTypeObject in Python.h + public static int am_await = 0; + public static int am_aiter = 0; + public static int am_anext = 0; + public static int nb_add = 0; + public static int nb_subtract = 0; + public static int nb_multiply = 0; + public static int nb_remainder = 0; + public static int nb_divmod = 0; + public static int nb_power = 0; + public static int nb_negative = 0; + public static int nb_positive = 0; + public static int nb_absolute = 0; + public static int nb_bool = 0; + public static int nb_invert = 0; + public static int nb_lshift = 0; + public static int nb_rshift = 0; + public static int nb_and = 0; + public static int nb_xor = 0; + public static int nb_or = 0; + public static int nb_int = 0; + public static int nb_reserved = 0; + public static int nb_float = 0; + public static int nb_inplace_add = 0; + public static int nb_inplace_subtract = 0; + public static int nb_inplace_multiply = 0; + public static int nb_inplace_remainder = 0; + public static int nb_inplace_power = 0; + public static int nb_inplace_lshift = 0; + public static int nb_inplace_rshift = 0; + public static int nb_inplace_and = 0; + public static int nb_inplace_xor = 0; + public static int nb_inplace_or = 0; + public static int nb_floor_divide = 0; + public static int nb_true_divide = 0; + public static int nb_inplace_floor_divide = 0; + public static int nb_inplace_true_divide = 0; + public static int nb_index = 0; + public static int nb_matrix_multiply = 0; + public static int nb_inplace_matrix_multiply = 0; + public static int mp_length = 0; + public static int mp_subscript = 0; + public static int mp_ass_subscript = 0; + public static int sq_length = 0; + public static int sq_concat = 0; + public static int sq_repeat = 0; + public static int sq_item = 0; + public static int was_sq_slice = 0; + public static int sq_ass_item = 0; + public static int was_sq_ass_slice = 0; + public static int sq_contains = 0; + public static int sq_inplace_concat = 0; + public static int sq_inplace_repeat = 0; + public static int bf_getbuffer = 0; + public static int bf_releasebuffer = 0; + public static int name = 0; + public static int ht_slots = 0; + public static int qualname = 0; + public static int ht_cached_keys = 0; + public static int ht_module = 0; + + /* here are optional user slots, followed by the members. */ + public static int members = 0; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyNumberMethods + { + public IntPtr nb_add; + public IntPtr nb_subtract; + public IntPtr nb_multiply; + public IntPtr nb_remainder; + public IntPtr nb_divmod; + public IntPtr nb_power; + public IntPtr nb_negative; + public IntPtr nb_positive; + public IntPtr nb_absolute; + public IntPtr nb_bool; + public IntPtr nb_invert; + public IntPtr nb_lshift; + public IntPtr nb_rshift; + public IntPtr nb_and; + public IntPtr nb_xor; + public IntPtr nb_or; + public IntPtr nb_int; + public IntPtr nb_reserved; + public IntPtr nb_float; + public IntPtr nb_inplace_add; + public IntPtr nb_inplace_subtract; + public IntPtr nb_inplace_multiply; + public IntPtr nb_inplace_remainder; + public IntPtr nb_inplace_power; + public IntPtr nb_inplace_lshift; + public IntPtr nb_inplace_rshift; + public IntPtr nb_inplace_and; + public IntPtr nb_inplace_xor; + public IntPtr nb_inplace_or; + public IntPtr nb_floor_divide; + public IntPtr nb_true_divide; + public IntPtr nb_inplace_floor_divide; + public IntPtr nb_inplace_true_divide; + public IntPtr nb_index; + public IntPtr nb_matrix_multiply; + public IntPtr nb_inplace_matrix_multiply; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PySequenceMethods + { + public IntPtr sq_length; + public IntPtr sq_concat; + public IntPtr sq_repeat; + public IntPtr sq_item; + public IntPtr was_sq_slice; + public IntPtr sq_ass_item; + public IntPtr was_sq_ass_slice; + public IntPtr sq_contains; + public IntPtr sq_inplace_concat; + public IntPtr sq_inplace_repeat; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyMappingMethods + { + public IntPtr mp_length; + public IntPtr mp_subscript; + public IntPtr mp_ass_subscript; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyAsyncMethods + { + public IntPtr am_await; + public IntPtr am_aiter; + public IntPtr am_anext; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyBufferProcs + { + public IntPtr bf_getbuffer; + public IntPtr bf_releasebuffer; + } + + internal static partial class SlotTypes + { + public static readonly Type[] Types = { + typeof(PyNumberMethods), + typeof(PySequenceMethods), + typeof(PyMappingMethods), + typeof(PyAsyncMethods), + typeof(PyBufferProcs), + }; + } + +} +#endif + diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index ddeec15a2..af273c0d3 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -53,7 +53,7 @@ public class Runtime #elif PYTHON39 const string _minor = "9"; #else -#error You must define one of PYTHON36 to PYTHON38 +#error You must define one of PYTHON36 to PYTHON39 #endif #if WINDOWS @@ -125,7 +125,7 @@ internal static Version PyVersion /// /// Initialize the runtime... /// - /// Always call this method from the Main thread. After the + /// Always call this method from the Main thread. After the /// first call to this method, the main thread has acquired the GIL. internal static void Initialize(bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) { @@ -407,7 +407,7 @@ internal static void Shutdown(ShutdownMode mode) { PyEval_SaveThread(); } - + } else { @@ -1723,7 +1723,7 @@ internal static long PyDict_Size(IntPtr pointer) internal static extern int PySet_Add(IntPtr set, IntPtr key); /// - /// Return 1 if found, 0 if not found, and -1 if an error is encountered. + /// Return 1 if found, 0 if not found, and -1 if an error is encountered. /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PySet_Contains(IntPtr anyset, IntPtr key); From 094bf2cee96e1996241db92a41b6448e4000bd01 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Fri, 13 Nov 2020 08:06:16 +0100 Subject: [PATCH 4/4] Fix geninterop script and regenerate interop39.cs - Only record structs when they are defined, not when they are declared - If a struct was only declared when a typedef was created, it won't contain its member declarations. Those have to be drawn from the recorded structs instead. - Rename internal members of AstParser to make it easier to debug --- src/runtime/interop39.cs | 52 +++++++++++++++++++- tools/geninterop/geninterop.py | 88 ++++++++++++++++++---------------- 2 files changed, 98 insertions(+), 42 deletions(-) diff --git a/src/runtime/interop39.cs b/src/runtime/interop39.cs index 30635d9fd..cb2b5b49a 100644 --- a/src/runtime/interop39.cs +++ b/src/runtime/interop39.cs @@ -18,6 +18,57 @@ namespace Python.Runtime internal static partial class TypeOffset { // Auto-generated from PyHeapTypeObject in Python.h + public static int ob_refcnt = 0; + public static int ob_type = 0; + public static int ob_size = 0; + public static int tp_name = 0; + public static int tp_basicsize = 0; + public static int tp_itemsize = 0; + public static int tp_dealloc = 0; + public static int tp_vectorcall_offset = 0; + public static int tp_getattr = 0; + public static int tp_setattr = 0; + public static int tp_as_async = 0; + public static int tp_repr = 0; + public static int tp_as_number = 0; + public static int tp_as_sequence = 0; + public static int tp_as_mapping = 0; + public static int tp_hash = 0; + public static int tp_call = 0; + public static int tp_str = 0; + public static int tp_getattro = 0; + public static int tp_setattro = 0; + public static int tp_as_buffer = 0; + public static int tp_flags = 0; + public static int tp_doc = 0; + public static int tp_traverse = 0; + public static int tp_clear = 0; + public static int tp_richcompare = 0; + public static int tp_weaklistoffset = 0; + public static int tp_iter = 0; + public static int tp_iternext = 0; + public static int tp_methods = 0; + public static int tp_members = 0; + public static int tp_getset = 0; + public static int tp_base = 0; + public static int tp_dict = 0; + public static int tp_descr_get = 0; + public static int tp_descr_set = 0; + public static int tp_dictoffset = 0; + public static int tp_init = 0; + public static int tp_alloc = 0; + public static int tp_new = 0; + public static int tp_free = 0; + public static int tp_is_gc = 0; + public static int tp_bases = 0; + public static int tp_mro = 0; + public static int tp_cache = 0; + public static int tp_subclasses = 0; + public static int tp_weaklist = 0; + public static int tp_del = 0; + public static int tp_version_tag = 0; + public static int tp_finalize = 0; + public static int tp_vectorcall = 0; public static int am_await = 0; public static int am_aiter = 0; public static int am_anext = 0; @@ -174,4 +225,3 @@ internal static partial class SlotTypes } #endif - diff --git a/tools/geninterop/geninterop.py b/tools/geninterop/geninterop.py index aacc4af65..e74221e46 100644 --- a/tools/geninterop/geninterop.py +++ b/tools/geninterop/geninterop.py @@ -53,25 +53,25 @@ class AstParser(object): """Walk an AST and determine the members of all structs""" def __init__(self): - self.__typedefs = {} - self.__typedecls = {} - self.__structs = {} - self.__struct_stack = [] - self.__struct_members_stack = [] - self.__ptr_decl_depth = 0 - self.__struct_members = {} - self.__decl_names = {} + self._typedefs = {} + self._typedecls = {} + self._structs = {} + self._struct_stack = [] + self._struct_members_stack = [] + self._ptr_decl_depth = 0 + self._struct_members = {} + self._decl_names = {} def get_struct_members(self, name): """return a list of (name, type) of struct members""" - defs = self.__typedefs.get(name) + defs = self._typedefs.get(name) if defs is None: return None - node = self.__get_leaf_node(defs) + node = self._get_leaf_node(defs) name = node.name if name is None: name = defs.declname - return self.__struct_members.get(name) + return self._struct_members.get(name) def visit(self, node): if isinstance(node, c_ast.FileAST): @@ -96,27 +96,27 @@ def visit_ast(self, ast): self.visit(node) def visit_typedef(self, typedef): - self.__typedefs[typedef.name] = typedef.type + self._typedefs[typedef.name] = typedef.type self.visit(typedef.type) def visit_typedecl(self, typedecl): - self.__decl_names[typedecl.type] = typedecl.declname + self._decl_names[typedecl.type] = typedecl.declname self.visit(typedecl.type) def visit_struct(self, struct): - self.__structs[self.__get_struct_name(struct)] = struct if struct.decls: + self._structs[self._get_struct_name(struct)] = struct # recurse into the struct - self.__struct_stack.insert(0, struct) + self._struct_stack.insert(0, struct) for decl in struct.decls: - self.__struct_members_stack.insert(0, decl.name) + self._struct_members_stack.insert(0, decl.name) self.visit(decl) - self.__struct_members_stack.pop(0) - self.__struct_stack.pop(0) - elif self.__ptr_decl_depth: + self._struct_members_stack.pop(0) + self._struct_stack.pop(0) + elif self._ptr_decl_depth: # the struct is empty, but add it as a member to the current # struct as the current member maybe a pointer to it. - self.__add_struct_member(struct.name) + self._add_struct_member(struct.name) def visit_decl(self, decl): self.visit(decl.type) @@ -125,51 +125,57 @@ def visit_funcdecl(self, funcdecl): self.visit(funcdecl.type) def visit_ptrdecl(self, ptrdecl): - self.__ptr_decl_depth += 1 + self._ptr_decl_depth += 1 self.visit(ptrdecl.type) - self.__ptr_decl_depth -= 1 + self._ptr_decl_depth -= 1 def visit_identifier(self, identifier): type_name = " ".join(identifier.names) - self.__add_struct_member(type_name) + self._add_struct_member(type_name) - def __add_struct_member(self, type_name): - if not (self.__struct_stack and self.__struct_members_stack): + def _add_struct_member(self, type_name): + if not (self._struct_stack and self._struct_members_stack): return # add member to current struct - current_struct = self.__struct_stack[0] - member_name = self.__struct_members_stack[0] - struct_members = self.__struct_members.setdefault( - self.__get_struct_name(current_struct), []) + current_struct = self._struct_stack[0] + member_name = self._struct_members_stack[0] + struct_members = self._struct_members.setdefault( + self._get_struct_name(current_struct), []) # get the node associated with this type node = None - if type_name in self.__typedefs: - node = self.__get_leaf_node(self.__typedefs[type_name]) - elif type_name in self.__structs: - node = self.__structs[type_name] + if type_name in self._typedefs: + node = self._get_leaf_node(self._typedefs[type_name]) + # If the struct was only declared when the typedef was created, its member + # information will not have been recorded and we have to look it up in the + # structs + if isinstance(node, c_ast.Struct) and node.decls is None: + if node.name in self._structs: + node = self._structs[node.name] + elif type_name in self._structs: + node = self._structs[type_name] # If it's a struct (and not a pointer to a struct) expand # it into the current struct definition - if not self.__ptr_decl_depth and isinstance(node, c_ast.Struct): + if not self._ptr_decl_depth and isinstance(node, c_ast.Struct): for decl in node.decls or []: - self.__struct_members_stack.insert(0, decl.name) + self._struct_members_stack.insert(0, decl.name) self.visit(decl) - self.__struct_members_stack.pop(0) + self._struct_members_stack.pop(0) else: # otherwise add it as a single member struct_members.append((member_name, type_name)) - def __get_leaf_node(self, node): + def _get_leaf_node(self, node): if isinstance(node, c_ast.Typedef): - return self.__get_leaf_node(node.type) + return self._get_leaf_node(node.type) if isinstance(node, c_ast.TypeDecl): - return self.__get_leaf_node(node.type) + return self._get_leaf_node(node.type) return node - def __get_struct_name(self, node): - return node.name or self.__decl_names.get(node) or "_struct_%d" % id(node) + def _get_struct_name(self, node): + return node.name or self._decl_names.get(node) or "_struct_%d" % id(node) class Writer(object):