8000 py/runtime: Allow multiple *args in function call. · micropython/micropython@d6aa8de · GitHub
[go: up one dir, main page]
Skip to content

Commit d6aa8de

Browse files
dlechdlech
committed
py/runtime: Allow multiple *args in function call.
This is a partial implementation of PEP 448 to allow unmacking multiple star args in a function or method call. This is implemented by changing the emitted bytecodes so that both positoinal args and star args are stored as positoinal args. A bitmap is added to indicate if an argument at a given position is a positoinal argument or a star arg. In the bytecodes, this new bitmap takes the place of the old star arg. It is stored as a small int, so this means only the first N arguments can be star args where N is the number of bits in a small int. The runtime is modified to interpret this new bytecode format while still trying to perform as few memory reallocations as possible. Signed-off-by: David Lechner <david@pybricks.com>
1 parent 2e5e9da commit d6aa8de

File tree

6 files changed

+101
-67
lines changed

6 files changed

+101
-67
lines changed

‎py/compile.c‎

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2397,17 +2397,19 @@ STATIC void compile_trailer_paren_helper(compiler_t *comp, mp_parse_node_t pn_ar
23972397
int n_positional = n_positional_extra;
23982398
uint n_keyword = 0;
23992399
uint star_flags = 0;
2400-
mp_parse_node_struct_t *star_args_node = NULL;
2400+
mp_uint_t star_args = 0;
24012401
for (size_t i = 0; i < n_args; i++) {
24022402
if (MP_PARSE_NODE_IS_STRUCT(args[i])) {
24032403
mp_parse_node_struct_t *pns_arg = (mp_parse_node_struct_t *)args[i];
24042404
if (MP_PARSE_NODE_STRUCT_KIND(pns_arg) == PN_arglist_star) {
2405-
if (star_flags & MP_EMIT_STAR_FLAG_SINGLE) {
2406-
compile_syntax_error(comp, (mp_parse_node_t)pns_arg, MP_ERROR_TEXT("can't have multiple *x"));
2405+
if (star_flags & MP_EMIT_STAR_FLAG_DOUBLE) {
2406+
compile_syntax_error(comp, (mp_parse_node_t)pns_arg, MP_ERROR_TEXT("* arg after **"));
24072407
return;
24082408
}
24092409
star_flags |= MP_EMIT_STAR_FLAG_SINGLE;
2410-
star_args_node = pns_arg;
2410+
star_args |= 1 << i;
2411+
compile_node(comp, pns_arg->nodes[0]);
2412+
n_positional++;
24112413
} else if (MP_PARSE_NODE_STRUCT_KIND(pns_arg) == PN_arglist_dbl_star) {
24122414
star_flags |= MP_EMIT_STAR_FLAG_DOUBLE;
24132415
// double-star args are stored as kw arg with key of None
@@ -2438,27 +2440,22 @@ STATIC void compile_trailer_paren_helper(compiler_t *comp, mp_parse_node_t pn_ar
24382440
}
24392441
} else {
24402442
normal_argument:
2441-
if (star_flags) {
2442-
compile_syntax_error(comp, args[i], MP_ERROR_TEXT("non-keyword arg after */**"));
2443+
if (star_flags & MP_EMIT_STAR_FLAG_DOUBLE) {
2444+
compile_syntax_error(comp, args[i], MP_ERROR_TEXT("positional arg after **"));
24432445
return;
24442446
}
24452447
if (n_keyword > 0) {
2446-
compile_syntax_error(comp, args[i], MP_ERROR_TEXT("non-keyword arg after keyword arg"));
2448+
compile_syntax_error(comp, args[i], MP_ERROR_TEXT("positional arg after keyword arg"));
24472449
return;
24482450
}
24492451
compile_node(comp, args[i]);
24502452
n_positional++;
24512453
}
24522454
}
24532455

2454-
// compile the star/double-star arguments if we had them
2455-
// if we had one but not the other then we load "null" as a place holder
24562456
if (star_flags != 0) {
2457-
if (star_args_node == NULL) {
2458-
EMIT(load_null);
2459-
} else {
2460-
compile_node(comp, star_args_node->nodes[0]);
2461-
}
2457+
// one extra object that contains the star_args map
2458+
EMIT_ARG(load_const_small_int, star_args);
24622459
// FIXME: delete this and update interpreter
24632460
EMIT(load_null);
24642461
}

‎py/runtime.c‎

Lines changed: 64 additions & 41 deletions
< 9B60 td data-grid-cell-id="diff-eca4e26b5e481a060f702d7f8e7bc9e704e388f55415fce4273c9ad7f61b50fc-720-734-2" data-line-anchor="diff-eca4e26b5e481a060f702d7f8e7bc9e704e388f55415fce4273c9ad7f61b50fcR734" data-selected="false" role="gridcell" style="background-color:var(--bgColor-default);padding-right:24px" tabindex="-1" valign="top" class="focusable-grid-cell diff-text-cell right-side-diff-cell left-side">
// Note that it can be arbitrary iterator.
Original file line numberDiff line numberDiff line change
@@ -692,9 +692,9 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_
692692
}
693693
uint n_args = n_args_n_kw & 0xff;
694694
uint n_kw = (n_args_n_kw >> 8) & 0xff;
695-
mp_obj_t pos_seq = args[n_args + 2 * n_kw]; // may be MP_OBJ_NULL
695+
mp_uint_t star_args = mp_obj_get_int_truncated(args[n_args + 2 * n_kw]);
696696

697-
DEBUG_OP_printf("call method var (fun=%p, self=%p, n_args=%u, n_kw=%u, args=%p, seq=%p)\n", fun, self, n_args, n_kw, args, pos_seq);
697+
DEBUG_OP_printf("call method var (fun=%p, self=%p, n_args=%u, n_kw=%u, args=%p, map=%u)\n", fun, self, n_args, n_kw, args, star_args);
698698

699699
// We need to create the following array of objects:
700700
// args[0 .. n_args] unpacked(pos_seq) args[n_args .. n_args + 2 * n_kw] unpacked(kw_dict)
@@ -705,6 +705,20 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_
705705
uint args2_alloc;
706706
uint args2_len = 0;
707707

708+
// Try to get a hint for unpacked * args length
709+
uint list_len = 0;
710+
711+
if (star_args != 0) {
712+
for (uint i = 0; i < n_args; i++) {
713+
if (star_args & (1 << i)) {
714+
mp_obj_t len = mp_obj_len_maybe(args[i]);
715+
if (len != MP_OBJ_NULL) {
716+
list_len += mp_obj_get_int(len);
717+
}
718+
}
719+
}
720+
}
721+
708722
// Try to get a hint for the size of the kw_dict
709723
uint kw_dict_len = 0;
710724

@@ -718,8 +732,8 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_
718732

719733
// Extract the pos_seq sequence to the new args array.
720734
721-
if (pos_seq == MP_OBJ_NULL) {
722-
// no sequence
735+
if (star_args == 0) {
736+
// no star args to unpack
723737

724738
// allocate memory for the new array of args
725739
args2_alloc = 1 + n_args + 2 * (n_kw + kw_dict_len);
@@ -733,60 +747,69 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_
733747
// copy the fixed pos args
734748
mp_seq_copy(args2 + args2_len, args, n_args, mp_obj_t);
735749
args2_len += n_args;
736-
737-
} else if (mp_obj_is_type(pos_seq, &mp_type_tuple) || mp_obj_is_type(pos_seq, &mp_type_list)) {
738-
// optimise the case of a tuple and list
739-
740-
// get the items
741-
size_t len;
742-
mp_obj_t *items;
743-
mp_obj_get_array(pos_seq, &len, &items);
744-
745-
// allocate memory for the new array of args
746-
args2_alloc = 1 + n_args + len + 2 * (n_kw + kw_dict_len);
747-
args2 = mp_nonlocal_alloc(args2_alloc * sizeof(mp_obj_t));
748-
749-
// copy the self
750-
if (self != MP_OBJ_NULL) {
751-
args2[args2_len++] = self;
752-
}
753-
754-
// copy the fixed and variable position args
755-
mp_seq_cat(args2 + args2_len, args, n_args, items, len, mp_obj_t);
756-
args2_len += n_args + len;
757-
758750
} else {
759-
// generic iterator
751+
// at least one star arg to unpack
760752

761753
// allocate memory for the new array of args
762-
args2_alloc = 1 + n_args + 2 * (n_kw + kw_dict_len) + 3;
754+
args2_alloc = 1 + n_args + list_len + 2 * (n_kw + kw_dict_len);
763755
args2 = mp_nonlocal_alloc(args2_alloc * sizeof(mp_obj_t));
764756

765757
// copy the self
766758
if (self != MP_OBJ_NULL) {
767759
args2[args2_len++] = self;
768760
}
769761

770-
// copy the fixed position args
771-
mp_seq_copy(args2 + args2_len, args, n_args, mp_obj_t);
772-
args2_len += n_args;
773-
774-
// extract the variable position args from the iterator
775-
mp_obj_iter_buf_t iter_buf;
776-
mp_obj_t iterable = mp_getiter(pos_seq, &iter_buf);
777-
mp_obj_t item;
778-
while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) {
779-
if (args2_len >= args2_alloc) {
780-
args2 = mp_nonlocal_realloc(args2, args2_alloc * sizeof(mp_obj_t), args2_alloc * 2 * sizeof(mp_obj_t));
781-
args2_alloc *= 2;
762+
for (uint i = 0; i < n_args; i++) {
763+
mp_obj_t arg = args[i];
764+
if (star_args & (1 << i)) {
765+
// star arg
766+
if (mp_obj_is_type(arg, &mp_type_tuple) || mp_obj_is_type(arg, &mp_type_list)) {
767+
// optimise the case of a tuple and list
768+
769+
// get the items
770+
size_t len;
771+
mp_obj_t *items;
772+
mp_obj_get_array(arg, &len, &items);
773+
774+
// copy the items
775+
assert(args2_len + len <= args2_alloc);
776+
mp_seq_copy(args2 + args2_len, items, len, mp_obj_t);
777+
args2_len += len;
778+
} else {
779+
// generic iterator
780+
781+
// extract the variable position args from the iterator
782+
mp_obj_iter_buf_t iter_buf;
783+
mp_obj_t iterable = mp_getiter(arg, &iter_buf);
784+
mp_obj_t item;
785+
while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) {
786+
if (args2_len >= args2_alloc) {
787+
args2 = mp_nonlocal_realloc(args2, args2_alloc * sizeof(mp_obj_t),
788+
args2_alloc * 2 * sizeof(mp_obj_t));
789+
args2_alloc *= 2;
790+
}
791+
args2[args2_len++] = item;
792+
}
793+
}
794+
} else {
795+
// normal argument
796+
assert(args2_len < args2_alloc);
797+
args2[args2_len++] = arg;
782798
}
783-
args2[args2_len++] = item;
784799
}
785800
}
786801

787802
// The size of the args2 array now is the number of positional args.
788803
uint pos_args_len = args2_len;
789804

805+
// ensure there is still enough room for kw args
806+
if (args2_len + 2 * (n_kw + kw_dict_len) > args2_alloc) {
807+
uint new_alloc = args2_len + 2 * (n_kw + kw_dict_len);
808+
args2 = mp_nonlocal_realloc(args2, args2_alloc * sizeof(mp_obj_t),
809+
new_alloc * sizeof(mp_obj_t));
810+
args2_alloc = new_alloc;
811+
}
812+
790813
// Copy the kw args.
791814
for (uint i = 0; i < n_kw; i++) {
792815
mp_obj_t kw_key = args[n_args + i * 2];

‎tests/basics/fun_callstar.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,16 @@
33
def foo(a, b, c):
44
print(a, b, c)
55

6+
foo(*(), 1, 2, 3)
7+
foo(*(1,), 2, 3)
8+
foo(*(1, 2), 3)
69
foo(*(1, 2, 3))
710
foo(1, *(2, 3))
811
foo(1, 2, *(3,))
912
foo(1, 2, 3, *())
13+
foo(*(1,), 2, *(3,))
14+
foo(*(1, 2), *(3,))
15+
foo(*(1,), *(2, 3))
1016

1117
# Another sequence type
1218
foo(1, 2, *[100])
@@ -29,10 +35,16 @@ def foo(self, a, b, c):
2935
print(a, b, c)
3036

3137
a = A()
38+
a.foo(*(), 1, 2, 3)
39+
a.foo(*(1,), 2, 3)
40+
a.foo(*(1, 2), 3)
3241
a.foo(*(1, 2, 3))
3342
a.foo(1, *(2, 3))
3443
a.foo(1, 2, *(3,))
3544
a.foo(1, 2, 3, *())
45+
a.foo(*(1,), 2, *(3,))
46+
a.foo(*(1, 2), *(3,))
47+
a.foo(*(1,), *(2, 3))
3648

3749
# Another sequence type
3850
a.foo(1, 2, *[100])

‎tests/basics/fun_callstardblstar.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,11 @@ def f(a, b, c, d):
66
f(*(1, 2), **{'c':3, 'd':4})
77
f(*(1, 2), **{['c', 'd'][i]:(3 + i) for i in range(2)})
88

9+
try:
10+
eval("f(**{'a': 1}, *(2, 3, 4))")
11+
except SyntaxError:
12+
print("SyntaxError")
13+
914
# test calling a method with *tuple and **dict
1015

1116
class A:
@@ -15,3 +20,8 @@ def f(self, a, b, c, d):
1520
a = A()
1621
a.f(*(1, 2), **{'c':3, 'd':4})
1722
a.f(*(1, 2), **{['c', 'd'][i]:(3 + i) for i in range(2)})
23+
24+
try:
25+
eval("a.f(**{'a': 1}, *(2, 3, 4))")
26+
except SyntaxError:
27+
print("SyntaxError")

‎tests/basics/python34.py‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,26 +6,23 @@
66
print("SKIP")
77
raise SystemExit
88

9-
# from basics/fun_kwvarargs.py
10-
# test evaluation order of arguments (in 3.4 it's backwards, 3.5 it's fixed)
11-
def f4(*vargs, **kwargs):
12-
print(vargs, kwargs)
9+
1310
def print_ret(x):
1411
print(x)
1512
return x
16-
f4(*print_ret(['a', 'b']), kw_arg=print_ret(None))
1713

1814
# test evaluation order of dictionary key/value pair (in 3.4 it's backwards)
1915
{print_ret(1):print_ret(2)}
2016

17+
2118
# from basics/syntaxerror.py
2219
def test_syntax(code):
2320
try:
2421
exec(code)
2522
except SyntaxError:
2623
print("SyntaxError")
27-
test_syntax("f(*a, *b)") # can't have multiple * (in 3.5 we can)
28-
test_syntax("f(*a, b)") # can't have positional after *
24+
25+
2926
test_syntax("f(**a, b)") # can't have positional after **
3027
test_syntax("() = []") # can't assign to empty tuple (in 3.6 we can)
3128
test_syntax("del ()") # can't delete empty tuple (in 3.6 we can)

‎tests/basics/python34.py.exp‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,8 @@
1-
None
2-
['a', 'b']
3-
('a', 'b') {'kw_arg': None}
41
2
52
1
63
SyntaxError
74
SyntaxError
85
SyntaxError
9-
SyntaxError
10-
SyntaxError
116
3.4
127
3 4
138
IndexError('foo',)

0 commit comments

Comments
 (0)
0