10000 py/runtime: Allow multiple *args in function call · micropython/micropython@129849f · GitHub
[go: up one dir, main page]
Skip to content

Commit 129849f

Browse files
dlechdlech
committed
py/runtime: Allow multiple *args in function call
This is a partial implementation of PEP 448 to allow unmacking multiple star args in a function or method call. This is implemented by changing the emitted bytecodes so that both positoinal args and star args are stored as positoinal args. A bitmap is added to indicate if an argument at a given position is a positoinal argument or a star arg. In the bytecodes, this new bitmap takes the place of the old star arg. It is stored as a small int, so this means only the first N arguments can be star args where N is the number of bits in a small int. The runtime is modified to interpret this new bytecode format while still trying to perform as few memory reallocations as possible.
1 parent 67060ab commit 129849f

File tree

7 files changed

+105
-69
lines changed

7 files changed

+105
-69
lines changed

‎py/compile.c

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2334,17 +2334,19 @@ STATIC void compile_trailer_paren_helper(compiler_t *comp, mp_parse_node_t pn_ar
23342334
int n_positional = n_positional_extra;
23352335
uint n_keyword = 0;
23362336
uint star_flags = 0;
2337-
mp_parse_node_struct_t *star_args_node = NULL;
2337+
mp_uint_t star_args = 0;
23382338
for (int i = 0; i < n_args; i++) {
23392339
if (MP_PARSE_NODE_IS_STRUCT(args[i])) {
23402340
mp_parse_node_struct_t *pns_arg = (mp_parse_node_struct_t *)args[i];
23412341
if (MP_PARSE_NODE_STRUCT_KIND(pns_arg) == PN_arglist_star) {
2342-
if (star_flags & MP_EMIT_STAR_FLAG_SINGLE) {
2343-
compile_syntax_error(comp, (mp_parse_node_t)pns_arg, "can't have multiple *x");
2342+
if (star_flags & MP_EMIT_STAR_FLAG_DOUBLE) {
2343+
compile_syntax_error(comp, (mp_parse_node_t)pns_arg, "* arg after **");
23442344
return;
23452345
}
23462346
star_flags |= MP_EMIT_STAR_FLAG_SINGLE;
2347-
star_args_node = pns_arg;
2347+
star_args |= 1 << i;
2348+
compile_node(comp, pns_arg->nodes[0]);
2349+
n_positional++;
23482350
} else if (MP_PARSE_NODE_STRUCT_KIND(pns_arg) == PN_arglist_dbl_star) {
23492351
star_flags |= MP_EMIT_STAR_FLAG_DOUBLE;
23502352
// double-star args are stored as kw arg with key of None
@@ -2369,27 +2371,22 @@ STATIC void compile_trailer_paren_helper(compiler_t *comp, mp_parse_node_t pn_ar
23692371
}
23702372
} else {
23712373
normal_argument:
2372-
if (star_flags) {
2373-
compile_syntax_error(comp, args[i], "non-keyword arg after */**");
2374+
if (star_flags & MP_EMIT_STAR_FLAG_DOUBLE) {
2375+
compile_syntax_error(comp, args[i], "positional arg after **");
23742376
return;
23752377
}
23762378
if (n_keyword > 0) {
2377-
compile_syntax_error(comp, args[i], "non-keyword arg after keyword arg");
2379+
compile_syntax_error(comp, args[i], "positional arg after keyword arg");
23782380
return;
23792381
}
23802382
compile_node(comp, args[i]);
23812383
n_positional++;
23822384
}
23832385
}
23842386

2385-
// compile the star/double-star arguments if we had them
2386-
// if we had one but not the other then we load "null" as a place holder
23872387
if (star_flags != 0) {
2388-
if (star_args_node == NULL) {
2389-
EMIT(load_null);
2390-
} else {
2391-
compile_node(comp, star_args_node->nodes[0]);
2392-
}
2388+
// one extra object that contains the star_args map
2389+
EMIT_ARG(load_const_small_int, star_args);
23932390
// FIXME: delete this and update interpreter
23942391
EMIT(load_null);
23952392
}

‎py/runtime.c

Lines changed: 64 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -674,9 +674,9 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_
674674
}
675675
uint n_args = n_args_n_kw & 0xff;
676676
uint n_kw = (n_args_n_kw >> 8) & 0xff;
677-
mp_obj_t pos_seq = args[n_args + 2 * n_kw]; // may be MP_OBJ_NULL
677+
mp_uint_t star_args = mp_obj_get_int_truncated(args[n_args + 2 * n_kw]);
678678

679-
DEBUG_OP_printf("call method var (fun=%p, self=%p, n_args=%u, n_kw=%u, args=%p, seq=%p)\n", fun, self, n_args, n_kw, args, pos_seq);
679+
DEBUG_OP_printf("call method var (fun=%p, self=%p, n_args=%u, n_kw=%u, args=%p, map=%u)\n", fun, self, n_args, n_kw, args, star_args);
680680

681681
// We need to create the following array of objects:
682682
// args[0 .. n_args] unpacked(pos_seq) args[n_args .. n_args + 2 * n_kw] unpacked(kw_dict)
@@ -687,6 +687,20 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_
687687
uint args2_alloc;
688688
uint args2_len = 0;
689689

690+
// Try to get a hint for unpacked * args length
691+
uint list_len = 0;
692+
693+
if (star_args != 0) {
694+
for (uint i = 0; i < n_args; i++) {
695+
if (star_args & (1 << i)) {
696+
mp_obj_t len = mp_obj_len_maybe(args[i]);
697+
if (len != MP_OBJ_NULL) {
698+
list_len += mp_obj_get_int(len);
699+
}
700+
}
701+
}
702+
}
703+
690704
// Try to get a hint for the size of the kw_dict
691705
uint kw_dict_len = 0;
692706

@@ -700,8 +714,8 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_
700714

701715
// Extract the pos_seq sequence to the new args array.
702716
// Note that it can be arbitrary iterator.
703-
if (pos_seq == MP_OBJ_NULL) {
704-
// no sequence
717+
if (star_args == 0) {
718+
// no star args to unpack
705719

706720
// allocate memory for the new array of args
707721
args2_alloc = 1 + n_args + 2 * (n_kw + kw_dict_len);
@@ -715,60 +729,69 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_
715729
// copy the fixed pos args
716730
mp_seq_copy(args2 + args2_len, args, n_args, mp_obj_t);
717731
args2_len += n_args;
718-
719-
} else if (mp_obj_is_type(pos_seq, &mp_type_tuple) || mp_obj_is_type(pos_seq, &mp_type_list)) {
720-
// optimise the case of a tuple and list
721-
722-
// get the items
723-
size_t len;
724-
mp_obj_t *items;
725-
mp_obj_get_array(pos_seq, &len, &items);
726-
727-
// allocate memory for the new array of args
728-
args2_alloc = 1 + n_args + len + 2 * (n_kw + kw_dict_len);
729-
args2 = mp_nonlocal_alloc(args2_alloc * sizeof(mp_obj_t));
730-
731-
// copy the self
732-
if (self != MP_OBJ_NULL) {
733-
args2[args2_len++] = self;
734-
}
735-
736-
// copy the fixed and variable position args
737-
mp_seq_cat(args2 + args2_len, args, n_args, items, len, mp_obj_t);
738-
args2_len += n_args + len;
739-
740732
} else {
741-
// generic iterator
733+
// at least one star arg to unpack
742734

743735
// allocate memory for the new array of args
744-
args2_alloc = 1 + n_args + 2 * (n_kw + kw_dict_len) + 3;
736+
args2_alloc = 1 + n_args + list_len + 2 * (n_kw + kw_dict_len);
745737
args2 = mp_nonlocal_alloc(args2_alloc * sizeof(mp_obj_t));
746738

747739
// copy the self
748740
if (self != MP_OBJ_NULL) {
749741
args2[args2_len++] = self;
750742
}
751743

752-
// copy the fixed position args
753-
mp_seq_copy(args2 + args2_len, args, n_args, mp_obj_t);
754-
args2_len += n_args;
755-
756-
// extract the variable position args from the iterator
757-
mp_obj_iter_buf_t iter_buf;
758-
mp_obj_t iterable = mp_getiter(pos_seq, &iter_buf);
759-
mp_obj_t item;
760-
while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) {
761-
if (args2_len >= args2_alloc) {
762-
args2 = mp_nonlocal_realloc(args2, args2_alloc * sizeof(mp_obj_t), args2_alloc * 2 * sizeof(mp_obj_t));
763-
args2_alloc *= 2;
744+
for (uint i = 0; i < n_args; i++) {
745+
mp_obj_t arg = args[i];
746+
if (star_args & (1 << i)) {
747+
// star arg
748+
if (mp_obj_is_type(arg, &mp_type_tuple) || mp_obj_is_type(arg, &mp_type_list)) {
749+
// optimise the case of a tuple and list
750+
751+
// get the items
752+
size_t len;
753+
mp_obj_t *items;
754+
mp_obj_get_array(arg, &len, &items);
755+
756+
// copy the items
757+
assert(args2_len + len <= args2_alloc);
758+
mp_seq_copy(args2 + args2_len, items, len, mp_obj_t);
759+
args2_len += len;
760+
} else {
761+
// generic iterator
762+
763+
// extract the variable position args from the iterator
764+
mp_obj_iter_buf_t iter_buf;
765+
mp_obj_t iterable = mp_getiter(arg, &iter_buf);
766+
mp_obj_t item;
767+
while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) {
768+
if (args2_len >= args2_alloc) {
769+
args2 = mp_nonlocal_realloc(args2, args2_alloc * sizeof(mp_obj_t),
770+
args2_alloc * 2 * sizeof(mp_obj_t));
771+
args2_alloc *= 2;
772+
}
773+
args2[args2_len++] = item;
774+
}
775+
}
776+
} else {
777+
// normal argument
778+
assert(args2_len < args2_alloc);
779+
args2[args2_len++] = arg;
764780
}
765-
args2[args2_len++] = item;
766781
}
767782
}
768783

769784
// The size of the args2 array now is the number of positional args.
770785
uint pos_args_len = args2_len;
771786

787+
// ensure there is still enough room for kw args
788+
if (args2_len + 2 * (n_kw + kw_dict_len) > args2_alloc) {
789+
uint new_alloc = args2_len + 2 * (n_kw + kw_dict_len);
790+
args2 = mp_nonlocal_realloc(args2, args2_alloc * sizeof(mp_obj_t),
791+
new_alloc * sizeof(mp_obj_t));
792+
args2_alloc = new_alloc;
793+
}
794+
772795
// Copy the kw args.
773796
for (uint i = 0; i < n_kw; i++) {
774797
mp_obj_t kw_key = args[n_args + i * 2];

‎tests/basics/fun_callstar.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,16 @@
33
def foo(a, b, c):
44
print(a, b, c)
55

6+
foo(*(), 1, 2, 3)
7+
foo(*(1,), 2, 3)
8+
foo(*(1, 2), 3)
69
foo(*(1, 2, 3))
710
foo(1, *(2, 3))
811
foo(1, 2, *(3,))
912
foo(1, 2, 3, *())
13+
foo(*(1,), 2, *(3,))
14+
foo(*(1, 2), *(3,))
15+
foo(*(1,), *(2, 3))
1016

1117
# Another sequence type
1218
foo(1, 2, *[100])
@@ -29,10 +35,16 @@ def foo(self, a, b, c):
2935
print(a, b, c)
3036

3137
a = A()
38+
a.foo(*(), 1, 2, 3)
39+
a.foo(*(1,), 2, 3)
40+
a.foo(*(1, 2), 3)
3241
a.foo(*(1, 2, 3))
3342
a.foo(1, *(2, 3))
3443
a.foo(1, 2, *(3,))
3544
a.foo(1, 2, 3, *())
45+
a.foo(*(1,), 2, *(3,))
46+
a.foo(*(1, 2), *(3,))
47+
a.foo(*(1,), *(2, 3))
3648

3749
# Another sequence type
3850
a.foo(1, 2, *[100])

‎tests/basics/fun_callstardblstar.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,11 @@ def f(a, b, c, d):
66
f(*(1, 2), **{'c':3, 'd':4})
77
f(*(1, 2), **{['c', 'd'][i]:(3 + i) for i in range(2)})
88

9+
try:
10+
eval("f(**{'a': 1}, *(2, 3, 4))")
11+
except SyntaxError:
12+
print("SyntaxError")
13+
914
# test calling a method with *tuple and **dict
1015

1116
class A:
@@ -15,3 +20,8 @@ def f(self, a, b, c, d):
1520
a = A()
1621
a.f(*(1, 2), **{'c':3, 'd':4})
1722
a.f(*(1, 2), **{['c', 'd'][i]:(3 + i) for i in range(2)})
23+
24+
try:
25+
eval("a.f(**{'a': 1}, *(2, 3, 4))")
26+
except SyntaxError:
27+
print("SyntaxError")

‎tests/basics/python34.py

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,26 +6,23 @@
66
print("SKIP")
77
raise SystemExit
88

9-
# from basics/fun_kwvarargs.py
10-
# test evaluation order of arguments (in 3.4 it's backwards, 3.5 it's fixed)
11-
def f4(*vargs, **kwargs):
12-
print(vargs, kwargs)
9+
1310
def print_ret(x):
1411
print(x)
1512
return x
16-
f4(*print_ret(['a', 'b']), kw_arg=print_ret(None))
1713

1814
# test evaluation order of dictionary key/value pair (in 3.4 it's backwards)
1915
{print_ret(1):print_ret(2)}
2016

17+
2118
# from basics/syntaxerror.py
2219
def test_syntax(code):
2320
try:
2421
exec(code)
2522
except SyntaxError:
2623
print("SyntaxError")
27-
test_syntax("f(*a, *b)") # can't have multiple * (in 3.5 we can)
28-
test_syntax("f(*a, b)") # can't have positional after *
24+
25+
2926
test_syntax("f(**a, b)") # can't have positional after **
3027
test_syntax("() = []") # can't assign to empty tuple (in 3.6 we can)
3128
test_syntax("del ()") # can't delete empty tuple (in 3.6 we can)

‎tests/basics/python34.py.exp

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,8 @@
1-
None
2-
['a', 'b']
3-
('a', 'b') {'kw_arg': None}
41
2
52
1
63
SyntaxError
74
SyntaxError
85
SyntaxError
9-
SyntaxError
10-
SyntaxError
116
3.4
127
3 4
138
IndexError('foo',)

‎tests/cmdline/cmd_showbc.py.exp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -204,8 +204,9 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+):
204204
\\d\+ POP_TOP
205205
\\d\+ LOAD_FAST 0
206206
\\d\+ LOAD_DEREF 14
207+
\\d\+ LOAD_CONST_SMALL_INT 1
207208
\\d\+ LOAD_NULL
208-
\\d\+ CALL_FUNCTION_VAR_KW n=0 nkw=0
209+
\\d\+ CALL_FUNCTION_VAR_KW n=1 nkw=0
209210
\\d\+ POP_TOP
210211
\\d\+ LOAD_FAST 0
211212
\\d\+ LOAD_METHOD b
@@ -225,8 +226,9 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+):
225226
\\d\+ LOAD_FAST 0
226227
\\d\+ LOAD_METHOD b
227228
\\d\+ LOAD_FAST 1
229+
\\d\+ LOAD_CONST_SMALL_INT 1
228230
\\d\+ LOAD_NULL
229-
\\d\+ CALL_METHOD_VAR_KW n=0 nkw=0
231+
\\d\+ CALL_METHOD_VAR_KW n=1 nkw=0
230232
\\d\+ POP_TOP
231233
\\d\+ LOAD_FAST 0
232234
\\d\+ POP_JUMP_IF_FALSE \\d\+

0 commit comments

Comments
 (0)
0