10000 [WIP] Automated 3.9 update by emmatyping · Pull Request #138 · python/typed_ast · GitHub
[go: up one dir, main page]
Skip to content
This repository was archived by the owner on Jul 5, 2023. It is now read-only.

[WIP] Automated 3.9 update #138

Closed
wants to merge 7 commits into from
Closed

[WIP] Automated 3.9 update #138

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
2b5b82c
Import tools for new Python version
emmatyping May 10, 2020
a6f6b72
Import files for Python 3.9
emmatyping May 10, 2020
1791235
Update header guards
emmatyping May 10, 2020
80f3bbb
Add custom files
emmatyping May 10, 2020
5be6a5b
Update exported symbols
emmatyping May 10, 2020
a647bb4
Update typed_ast version
emmatyping May 10, 2020
752e47e
Fix small issue
emmatyping May 10, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Update exported symbols
  • Loading branch information
@emmatyping
emmatyping committed May 10, 2020
commit 5be6a5b9cf71c27747387be585e9ee06ad1df4b9
438 changes: 219 additions & 219 deletions ast3/Grammar/python.gram
View file
Open in desktop

Large diffs are not rendered by default.

439 changes: 221 additions & 218 deletions ast3/Include/Python-ast.h
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions ast3/Include/asdl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ typedef struct {
int elements[1];
} asdl_int_seq;

asdl_seq *_Py_asdl_seq_new(Py_ssize_t size, PyArena *arena);
asdl_int_seq *_Py_asdl_int_seq_new(Py_ssize_t size, PyArena *arena);
asdl_seq *_Ta3_asdl_seq_new(Py_ssize_t size, PyArena *arena);
asdl_int_seq *_Ta3_asdl_int_seq_new(Py_ssize_t size, PyArena *arena);

#define asdl_seq_GET(S, I) (S)->elements[(I)]
#define asdl_seq_LEN(S) ((S) == NULL ? 0 : (S)->size)
Expand Down
16 changes: 8 additions & 8 deletions ast3/Include/pegen_interface.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,32 +11,32 @@ extern "C" {
#include "Python.h"
#include "Python-ast.h"

PyAPI_FUNC(mod_ty) PyPegen_ASTFromString(
PyAPI_FUNC(mod_ty) Ta3Pegen_ASTFromString(
const char *str,
const char *filename,
int mode,
PyCompilerFlags *flags,
PegenCompilerFlags *flags,
PyArena *arena);
PyAPI_FUNC(mod_ty) PyPegen_ASTFromStringObject(
PyAPI_FUNC(mod_ty) Ta3Pegen_ASTFromStringObject(
const char *str,
PyObject* filename,
int mode,
PyCompilerFlags *flags,
PegenCompilerFlags *flags,
PyArena *arena);
PyAPI_FUNC(mod_ty) PyPegen_ASTFromFileObject(
PyAPI_FUNC(mod_ty) Ta3Pegen_ASTFromFileObject(
FILE *fp,
PyObject *filename_ob,
int mode,
const char *enc,
const char *ps1,
const char *ps2,
PyCompilerFlags *flags,
PegenCompilerFlags *flags,
int *errcode,
PyArena *arena);
PyAPI_FUNC(mod_ty) PyPegen_ASTFromFilename(
PyAPI_FUNC(mod_ty) Ta3Pegen_ASTFromFilename(
const char *filename,
int mode,
PyCompilerFlags *flags,
PegenCompilerFlags *flags,
PyArena *arena);


Expand Down
78 changes: 78 additions & 0 deletions ast3/Include/ta3_compat.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
#ifndef Ta3_COMPAT_H
#define Ta3_COMPAT_H

// Various workarounds to make the parser compatible with older 3.x
#include <Python.h>

// f-string input is new in 3.9
#if PY_MINOR_VERSION < 9
#define Py_fstring_input 800
#endif

// top level await new in 3.9
#if PY_MINOR_VERSION < 8
#define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000
#endif

// typed_ast was merged into Python in 3.8
#if PY_MINOR_VERSION < 8
#define PyCF_TYPE_COMMENTS 0x1000
#endif

typedef struct {
int cf_flags; /* bitmask of CO_xxx flags relevant to future */
int cf_feature_version; /* minor Python version (PyCF_ONLY_AST) */
} PegenCompilerFlags;

#define _PegenCompilerFlags_INIT \
(PegenCompilerFlags){.cf_flags = 0, .cf_feature_version = PY_MINOR_VERSION}


// Define 3.7+ input types
#if PY_MINOR_VERSION < 8
/* These definitions must match corresponding definitions in graminit.h. */
#define Py_single_input 256
#define Py_file_input 257
#define Py_eval_input 258
#define Py_func_type_input 345
#endif

// Py_UNREACHABLE was defined in 3.7
#if PY_MINOR_VERSION < 7
#if defined(RANDALL_WAS_HERE)
# define Py_UNREACHABLE() \
Py_FatalError( \
"If you're seeing this, the code is in what I thought was\n" \
"an unreachable state.\n\n" \
"I could give you advice for what to do, but honestly, why\n" \
"should you trust me? I clearly screwed this up. I'm writing\n" \
"a message that should never appear, yet I know it will\n" \
"probably appear someday.\n\n" \
"On a deep level, I know I'm not up to this task.\n" \
"I'm so sorry.\n" \
"https://xkcd.com/2200")
#elif defined(Py_DEBUG)
# define Py_UNREACHABLE() \
Py_FatalError( \
"We've reached an unreachable state. Anything is possible.\n" \
"The limits were in our heads all along. Follow your dreams.\n" \
"https://xkcd.com/2200")
#elif defined(__GNUC__) || defined(__clang__) || defined(__INTEL_COMPILER)
# define Py_UNREACHABLE() __builtin_unreachable()
#elif defined(_MSC_VER)
# define Py_UNREACHABLE() __assume(0)
#else
# define Py_UNREACHABLE() \
Py_FatalError("Unreachable C code path reached")
#endif
#endif

// 3.7 also introduced _Py_TypeName, which we provide an alternative for it here
const char * _Ta3Type_Name(PyTypeObject *);

int _Pegen_PyObject_LookupAttr(PyObject *, PyObject *, PyObject **);

const char *
_Pegen_SourceAsString(PyObject *cmd, const char *funcname, const char *what, PegenCompilerFlags *cf, PyObject **cmd_copy);

#endif // Ta3_COMPAT_H
56 changes: 29 additions & 27 deletions ast3/Parser/asdl_c.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -292,9 +292,9 @@ def emit_function(self, name, ctype, args, attrs, union=True):
margs = "a0"
for i in range(1, len(args)+1):
margs += ", a%d" % i
self.emit("#define %s(%s) _Py_%s(%s)" % (name, margs, name, margs), 0,
self.emit("#define %s(%s) _Ta3_%s(%s)" % (name, margs, name, margs), 0,
reflow=False)
self.emit("%s _Py_%s(%s);" % (ctype, name, argstr), False)
self.emit("%s _Ta3_%s(%s);" % (ctype, name, argstr), False)

def visitProduct(self, prod, name):
self.emit_function(name, get_c_type(name),
Expand Down Expand Up @@ -520,7 +520,7 @@ def isSimpleType(self, field):

def visitField(self, field, name, sum=None, prod=None, depth=0):
ctype = get_c_type(field.type)
line = "if (_PyObject_LookupAttr(obj, astmodulestate_global->%s, &tmp) < 0) {"
line = "if (_Pegen_PyObject_LookupAttr(obj, astmodulestate_global->%s, &tmp) < 0) {"
self.emit(line % field.name, depth)
self.emit("return 1;", depth+1)
self.emit("}", depth)
Expand Down Expand Up @@ -548,16 +548,16 @@ def visitField(self, field, name, sum=None, prod=None, depth=0):
self.emit("Py_ssize_t i;", depth+1)
self.emit("if (!PyList_Check(tmp)) {", depth+1)
self.emit("PyErr_Format(PyExc_TypeError, \"%s field \\\"%s\\\" must "
"be a list, not a %%.200s\", _PyType_Name(Py_TYPE(tmp)));" %
"be a list, not a %%.200s\", _Ta3Type_Name(Py_TYPE(tmp)));" %
(name, field.name),
depth+2, reflow=False)
self.emit("goto failed;", depth+2)
self.emit("}", depth+1)
self.emit("len = PyList_GET_SIZE(tmp);", depth+1)
if self.isSimpleType(field):
self.emit("%s = _Py_asdl_int_seq_new(len, arena);" % field.name, depth+1)
self.emit("%s = _Ta3_asdl_int_seq_new(len, arena);" % field.name, depth+1)
else:
self.emit("%s = _Py_asdl_seq_new(len, arena);" % field.name, depth+1)
self.emit("%s = _Ta3_asdl_seq_new(len, arena);" % field.name, depth+1)
self.emit("if (%s == NULL) goto failed;" % field.name, depth+1)
self.emit("for (i = 0; i < len; i++) {", depth+1)
self.emit("%s val;" % ctype, depth+2)
Expand Down Expand Up @@ -685,7 +685,7 @@ def visitModule(self, mod):
Py_ssize_t i, numfields = 0;
int res = -1;
PyObject *key, *value, *fields;
if (_PyObject_LookupAttr((PyObject*)Py_TYPE(self), astmodulestate_global->_fields, &fields) < 0) {
if (_Pegen_PyObject_LookupAttr((PyObject*)Py_TYPE(self), astmodulestate_global->_fields, &fields) < 0) {
goto cleanup;
}
if (fields) {
Expand All @@ -698,7 +698,7 @@ def visitModule(self, mod):
if (numfields < PyTuple_GET_SIZE(args)) {
PyErr_Format(PyExc_TypeError, "%.400s constructor takes at most "
"%zd positional argument%s",
_PyType_Name(Py_TYPE(self)),
_Ta3Type_Name(Py_TYPE(self)),
numfields, numfields == 1 ? "" : "s") F438 ;
res = -1;
goto cleanup;
Expand Down Expand Up @@ -733,7 +733,7 @@ def visitModule(self, mod):
ast_type_reduce(PyObject *self, PyObject *unused)
{
PyObject *dict;
if (_PyObject_LookupAttr(self, astmodulestate_global->__dict__, &dict) < 0) {
if (_Pegen_PyObject_LookupAttr(self, astmodulestate_global->__dict__, &dict) < 0) {
return NULL;
}
if (dict) {
Expand Down Expand Up @@ -1023,7 +1023,7 @@ class ASTModuleVisitor(PickleVisitor):

def visitModule(self, mod):
self.emit("PyMODINIT_FUNC", 0)
self.emit("PyInit__ast(void)", 0)
self.emit("PyInit__ast3(void)", 0)
self.emit("{", 0)
self.emit("PyObject *m;", 1)
self.emit("if (!init_types()) return NULL;", 1)
Expand Down Expand Up @@ -1223,24 +1223,20 @@ def set(self, field, value, depth):
class PartingShots(StaticVisitor):

CODE = """
PyObject* PyAST_mod2obj(mod_ty t)
PyObject* Ta3AST_mod2obj(mod_ty t)
{
if (!init_types())
return NULL;
return ast2obj_mod(t);
}

/* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */
mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
mod_ty Ta3AST_obj2mod(PyObject* ast, PyArena* arena, int mode)
{
PyObject *req_type[3];
const char * const req_name[] = {"Module", "Expression", "Interactive"};
int isinstance;

if (PySys_Audit("compile", "OO", ast, Py_None) < 0) {
return NULL;
}

req_type[0] = astmodulestate_global->Module_type;
req_type[1] = astmodulestate_global->Expression_type;
req_type[2] = astmodulestate_global->Interactive_type;
Expand All @@ -1255,7 +1251,7 @@ class PartingShots(StaticVisitor):
return NULL;
if (!isinstance) {
PyErr_Format(PyExc_TypeError, "expected %s node, got %.400s",
req_name[mode], _PyType_Name(Py_TYPE(ast)));
req_name[mode], _Ta3Type_Name(Py_TYPE(ast)));
return NULL;
}

Expand All @@ -1266,7 +1262,7 @@ class PartingShots(StaticVisitor):
return res;
}

int PyAST_Check(PyObject* obj)
int Ta3AST_Check(PyObject* obj)
{
if (!init_types())
return -1;
Expand Down Expand Up @@ -1346,10 +1342,10 @@ def generate_module_def(f, mod):

static struct PyModuleDef _astmodule = {
PyModuleDef_HEAD_INIT,
"_ast",
"_ast3",
NULL,
sizeof(astmodulestate),
NULL,
ast3_methods,
NULL,
astmodule_traverse,
astmodule_clear,
Expand Down Expand Up @@ -1387,14 +1383,15 @@ def main(srcfile, dump_module=False):
if H_FILE:
with open(H_FILE, "w") as f:
f.write(auto_gen_msg)
f.write('#ifndef Py_PYTHON_AST_H\n')
f.write('#define Py_PYTHON_AST_H\n')
f.write('#ifndef Ta3_PYTHON_AST_H\n')
f.write('#define Ta3_PYTHON_AST_H\n')
f.write('#ifdef __cplusplus\n')
f.write('extern "C" {\n')
f.write('#endif\n')
f.write('\n')
f.write('#ifndef Py_LIMITED_API\n')
f.write('#include "asdl.h"\n')
f.write('#include "../Include/ta3_compat.h"')
f.write('\n')
f.write('#undef Yield /* undefine macro conflicting with <winbase.h> */\n')
f.write('\n')
Expand All @@ -1405,25 +1402,30 @@ def main(srcfile, dump_module=False):
f.write("// Note: these macros affect function definitions, not only call sites.\n")
PrototypeVisitor(f).visit(mod)
f.write("\n")
f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
f.write("int PyAST_Check(PyObject* obj);\n")
f.write("PyObject* Ta3AST_mod2obj(mod_ty t);\n")
f.write("mod_ty Ta3AST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
f.write("int Ta3AST_Check(PyObject* obj);\n")
f.write("#endif /* !Py_LIMITED_API */\n")
f.write('\n')
f.write('#ifdef __cplusplus\n')
f.write('}\n')
f.write('#endif\n')
f.write('#endif /* !Py_PYTHON_AST_H */\n')
f.write('#endif /* !Ta3_PYTHON_AST_H */\n')

if C_FILE:
with open(C_FILE, "w") as f:
f.write(auto_gen_msg)
f.write('#include <stddef.h>\n')
f.write('\n')
f.write('#include "Python.h"\n')
f.write('#include "%s-ast.h"\n' % mod.name)
f.write('#include "../Include/%s-ast.h"\n' % mod.name)
f.write('#include "structmember.h" // PyMemberDef\n')
f.write('\n')
f.write("PyObject *ast3_parse(PyObject *self, PyObject *args);\n")
f.write("static PyMethodDef ast3_methods[] = {\n")
f.write(' {"_parse", ast3_parse, METH_VARARGS, "Parse string into typed AST."},\n')
f.write(" {NULL, NULL, NULL}\n")
f.write("};\n")

generate_module_def(f, mod)

Expand Down
Loading
0