10000 py: Add basic support for dynamically loadable native modules. · mimoccc/circuitpython@e06aed4 · GitHub
[go: up one dir, main page]
Skip to content

Commit e06aed4

Browse files
dpgeorgedpgeorge
committed
py: Add basic support for dynamically loadable native modules.
Using the import system you can now import native modules dynamically, as you would a normal .py module. Includes a simple example in extmod/modx. To test: >>> import modx >>> modx.data >>> modx.add1(1) where modx.mpy (compiled from extmod/modx) is in your path.
1 parent 50f5622 commit e06aed4

14 files changed

+426
-1
lines changed

‎extmod/modx/Makefile

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
include ../../py/mkenv.mk
2+
3+
CROSS = 0
4+
5+
ifeq ($(CROSS), 1)
6+
CROSS_COMPILE = arm-none-eabi-
7+
endif
8+
9+
INC = -I. -I../..
10+
11+
ifeq ($(CROSS), 1)
12+
CFLAGS_CORTEX_M4 = -mthumb -mtune=cortex-m4 -mabi=aapcs-linux -mcpu=cortex-m4 -mfpu=fpv4-sp-d16 -mfloat-abi=hard -fsingle-precision-constant -Wdouble-promotion
13+
CFLAGS = $(INC) -Wall -Werror -ansi -std=gnu99 -nostdlib -fPIC -Os $(CFLAGS_CORTEX_M4)
14+
else
15+
CFLAGS = $(INC) -Wall -Werror -ansi -std=gnu99 -fPIC -Os
16+
endif
17+
18+
all: modx.mpy
19+
20+
modx.mpy: modx.elf
21+
$(OBJCOPY) -O binary -j .all $< $@
22+
23+
modx.elf: modx.o
24+
$(LD) -T mpextern.ld -o $@ $<

‎extmod/modx/genhdr/qstrdefs.generated.h

Whitespace-only changes.

‎extmod/modx/modx.c

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#include "py/mpextern.h"
2+
3+
STATIC mp_obj_t modx_add1(const mp_ext_table_t *et, mp_obj_t x) {
4+
return et->mp_binary_op(MP_BINARY_OP_ADD, x, MP_OBJ_NEW_SMALL_INT(1));
5+
}
6+
7+
MP_EXT_HEADER
8+
9+
MP_EXT_INIT
10+
void init(const mp_ext_table_t *et) {
11+
mp_obj_t f_add1 = et->mp_obj_new_fun_extern(false, 1, 1, modx_add1);
12+
mp_obj_t list[6] = {MP_OBJ_NEW_SMALL_INT(1), MP_OBJ_NEW_SMALL_INT(2), MP_OBJ_NEW_SMALL_INT(3), et->mp_const_true_, MP_OBJ_NEW_QSTR(et->qstr_from_str("modx")), f_add1};
13+
et->mp_store_global(et->qstr_from_str("data"), et->mp_obj_new_list(6, list));
14+
et->mp_store_global(et->qstr_from_str("add1"), f_add1);
15+
}

‎extmod/modx/mpconfigport.h

Lines changed: 81 additions & 0 deletions
10000
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
/*
2+
* This file is part of the Micro Python project, http://micropython.org/
3+
*
4+
* The MIT License (MIT)
5+
*
6+
* Copyright (c) 2013, 2014 Damien P. George
7+
*
8+
* Permission is hereby granted, free of charge, to any person obtaining a copy
9+
* of this software and associated documentation files (the "Software"), to deal
10+
* in the Software without restriction, including without limitation the rights
11+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12+
* copies of the Software, and to permit persons to whom the Software is
13+
* furnished to do so, subject to the following conditions:
14+
*
15+
* The above copyright notice and this permission notice shall be included in
16+
* all copies or substantial portions of the Software.
17+
*
18+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
24+
* THE SOFTWARE.
25+
*/
26+
27+
// options to control how Micro Python is built
28+
29+
#define MICROPY_ALLOC_PATH_MAX (PATH_MAX)
30+
#define MICROPY_COMP_MODULE_CONST (1)
31+
#define MICROPY_ENABLE_GC (1)
32+
#define MICROPY_ENABLE_FINALISER (1)
33+
#define MICROPY_STACK_CHECK (1)
34+
#define MICROPY_MEM_STATS (1)
35+
#define MICROPY_DEBUG_PRINTERS (1)
36+
#define MICROPY_HELPER_REPL (1)
37+
#define MICROPY_HELPER_LEXER_UNIX (1)
38+
#define MICROPY_ENABLE_SOURCE_LINE (1)
39+
#define MICROPY_FLOAT_IMPL (MICROPY_FLOAT_IMPL_DOUBLE)
40+
#define MICROPY_LONGINT_IMPL (MICROPY_LONGINT_IMPL_MPZ)
41+
#define MICROPY_STREAMS_NON_BLOCK (1)
42+
#define MICROPY_OPT_COMPUTED_GOTO (1)
43+
#define MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE (1)
44+
#define MICROPY_CAN_OVERRIDE_BUILTINS (1)
45+
#define MICROPY_MODULE_EXTERN (1)
46+
#define MICROPY_PY_BUILTINS_STR_UNICODE (1)
47+
#define MICROPY_PY_BUILTINS_MEMORYVIEW (1)
48+
#define MICROPY_PY_BUILTINS_FROZENSET (1)
49+
#define MICROPY_PY_BUILTINS_COMPILE (1)
50+
#define MICROPY_PY_MICROPYTHON_MEM_INFO (1)
51+
#define MICROPY_PY_SYS_EXIT (1)
52+
#define MICROPY_PY_SYS_PLATFORM "linux"
53+
#define MICROPY_PY_SYS_MAXSIZE (1)
54+
#define MICROPY_PY_SYS_STDFILES (1)
55+
#define MICROPY_PY_CMATH (1)
56+
#define MICROPY_PY_IO_FILEIO (1)
57+
#define MICROPY_PY_GC_COLLECT_RETVAL (1)
58+
59+
// type definitions for the specific machine
60+
61+
#ifdef __LP64__
62+
typedef long mp_int_t; // must be pointer size
63+
typedef unsigned long mp_uint_t; // must be pointer size
64+
#else
65+
// These are definitions for machines where sizeof(int) == sizeof(void*),
66+
// regardless for actual size.
67+
typedef int mp_int_t; // must be pointer size
68+
typedef unsigned int mp_uint_t; // must be pointer size
69+
#endif
70+
71+
#define BYTES_PER_WORD sizeof(mp_int_t)
72+
73+
// Cannot include <sys/types.h>, as it may lead to symbol name clashes
74+
#if _FILE_OFFSET_BITS == 64 && !defined(__LP64__)
75+
typedef long long mp_off_t;
76+
#else
77+
typedef long mp_off_t;
78+
#endif
79+
80+
typedef void *machine_ptr_t; // must be of pointer size
81+
typedef const void *machine_const_ptr_t; // must be of pointer size

‎extmod/modx/mpextern.ld

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
/*
2+
GNU linker script for MPY external modules
3+
*/
4+
5+
SECTIONS
6+
{
7+
.all :
8+
{
9+
. = ALIGN(4);
10+
KEEP(*(.mpyheader))
11+
KEEP(*(.mpytext))
12+
*(.text*)
13+
*(.rodata*)
14+
. = ALIGN(4);
15+
}
16+
}

‎py/builtinimport.c

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
#include "py/runtime.h"
3636
#include "py/builtin.h"
3737
#include "py/frozenmod.h"
38+
#include "py/mpextern.h"
3839

3940
#if 0 // print debugging info
4041
#define DEBUG_PRINT (1)
@@ -71,6 +72,14 @@ STATIC mp_import_stat_t stat_dir_or_file(vstr_t *path) {
7172
if (stat == MP_IMPORT_STAT_FILE) {
7273
return stat;
7374
}
75+
#if MICROPY_MODULE_EXTERN
76+
vstr_cut_tail_bytes(path, 2);
77+
vstr_add_str(path, "mpy");
78+
stat = mp_import_stat(vstr_null_terminated_str(path));
79+
if (stat == MP_IMPORT_STAT_FILE) {
80+
return stat;
81+
}
82+
#endif
7483
return MP_IMPORT_STAT_NO_EXIST;
7584
}
7685

@@ -150,6 +159,18 @@ STATIC void chop_component(const char *start, const char **end) {
150159
*end = p;
151160
}
152161

162+
#if MICROPY_MODULE_EXTERN
163+
STATIC void do_load_extern(mp_obj_t module_obj, vstr_t *file) {
164+
#if MICROPY_PY___FILE__
165+
mp_store_attr(module_obj, MP_QSTR___file__, MP_OBJ_NEW_QSTR(qstr_from_str(vstr_str(file))));
166+
#endif
167+
168+
// load the extern module in its context
169+
mp_obj_dict_t *mod_globals = mp_obj_module_get_globals(module_obj);
170+
mp_extern_load(vstr_str(file), mod_globals);
171+
}
172+
#endif
173+
153174
mp_obj_t mp_builtin___import__(mp_uint_t n_args, const mp_obj_t *args) {
154175
#if DEBUG_PRINT
155176
DEBUG_printf("__import__:\n");
@@ -377,7 +398,14 @@ mp_obj_t mp_builtin___import__(mp_uint_t n_args, const mp_obj_t *args) {
377398
vstr_cut_tail_bytes(&path, sizeof("/__init__.py") - 1); // cut off /__init__.py
378399
}
379400
} else { // MP_IMPORT_STAT_FILE
380-
do_load(module_obj, &path);
401+
#if MICROPY_MODULE_EXTERN
402+
if (path.buf[path.len - 3] == 'm') {
403+
do_load_extern(module_obj, &path);
404+
} else
405+
#endif
406+
{
407+
do_load(module_obj, &path);
408+
}
381409
// TODO: We cannot just break here, at the very least, we must execute
382410
// trailer code below. But otherwise if there're remaining components,
383411
// that would be (??) object path within module, not modules path within FS.

‎py/mpconfig.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -439,6 +439,11 @@ typedef double mp_float_t;
439439
#define MICROPY_MODULE_FROZEN (0)
440440
#endif
441441

442+
// Whether to support loading external native modules
443+
#ifndef MICROPY_MODULE_EXTERN
444+
#define MICROPY_MODULE_EXTERN (0)
445+
#endif
446+
442447
// Whether you can override builtins in the builtins module
443448
#ifndef MICROPY_CAN_OVERRIDE_BUILTINS
444449
#define MICROPY_CAN_OVERRIDE_BUILTINS (0)

‎py/mpextern.c

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
#include "mpextern.h"
2+
3+
#if MICROPY_MODULE_EXTERN
4+
5+
typedef mp_obj_t (*mp_fun_ext_0_t)(const mp_ext_table_t *et);
6+
typedef mp_obj_t (*mp_fun_ext_1_t)(const mp_ext_table_t *et, mp_obj_t);
7+
typedef mp_obj_t (*mp_fun_ext_2_t)(const mp_ext_table_t *et, mp_obj_t, mp_obj_t);
8+
typedef mp_obj_t (*mp_fun_ext_3_t)(const mp_ext_table_t *et, mp_obj_t, mp_obj_t, mp_obj_t);
9+
typedef mp_obj_t (*mp_fun_ext_var_t)(const mp_ext_table_t *et, mp_uint_t n, const mp_obj_t *);
10+
typedef mp_obj_t (*mp_fun_ext_kw_t)(const mp_ext_table_t *et, mp_uint_t n, const mp_obj_t *, mp_map_t *);
11+
12+
STATIC const mp_obj_type_t mp_type_fun_extern;
13+
STATIC const mp_ext_table_t mp_ext_table;
14+
15+
typedef struct _mp_obj_fun_extern_t { // use this to make const objects that go in ROM
16+
mp_obj_base_t base;
17+
bool is_kw : 1;
18+
mp_uint_t n_args_min : 15; // inclusive
19+
mp_uint_t n_args_max : 16; // inclusive
20+
void *fun; // must be a pointer to a callable function in ROM
21+
mp_obj_dict_t *dict_globals;
22+
} mp_obj_fun_extern_t;
23+
24+
STATIC mp_obj_t fun_extern_call(mp_obj_t self_in, mp_uint_t n_args, mp_uint_t n_kw, const mp_obj_t *args) {
25+
assert(MP_OBJ_IS_TYPE(self_in, &mp_type_fun_extern));
26+
mp_obj_fun_extern_t *self = self_in;
27+
28+
// check number of arguments
29+
mp_arg_check_num(n_args, n_kw, self->n_args_min, self->n_args_max, self->is_kw);
30+
31+
if (self->is_kw) {
32+
// function allows keywords
33+
34+
// we create a map directly from the given args array
35+
mp_map_t kw_args;
36+
mp_map_init_fixed_table(&kw_args, n_kw, args + n_args);
37+
38+
return ((mp_fun_ext_kw_t)self->fun)(&mp_ext_table, n_args, args, &kw_args);
39+
40+
} else if (self->n_args_min <= 3 && self->n_args_min == self->n_args_max) {
41+
// function requires a fixed number of arguments
42+
43+
// dispatch function call
44+
switch (self->n_args_min) {
45+
case 0:
46+
return ((mp_fun_ext_0_t)self->fun)(&mp_ext_table);
47+
48+
case 1:
49+
return ((mp_fun_ext_1_t)self->fun)(&mp_ext_table, args[0]);
50+
51+
case 2:
52+
return ((mp_fun_ext_2_t)self->fun)(&mp_ext_table, args[0], args[1]);
53+
54+
case 3:
55+
default:
56+
return ((mp_fun_ext_3_t)self->fun)(&mp_ext_table, args[0], args[1], args[2]);
57+
}
58+
59+
} else {
60+
// function takes a variable number of arguments, but no keywords
61+
62+
return ((mp_fun_ext_var_t)self->fun)(&mp_ext_table, n_args, args);
63+
}
64+
}
65+
66+
STATIC const mp_obj_type_t mp_type_fun_extern = {
67+
{ &mp_type_type },
68+
.name = MP_QSTR_function,
69+
.call = fun_extern_call,
70+
};
71+
72+
STATIC mp_obj_t mp_obj_new_fun_extern(bool is_kw, mp_uint_t n_args_min, mp_uint_t n_args_max, void *f) {
73+
mp_obj_fun_extern_t *self = m_new_obj(mp_obj_fun_extern_t);
74+
self->base.type = &mp_type_fun_extern;
75+
self->is_kw = is_kw;
76+
self->n_args_min = n_args_min;
77+
self->n_args_max = n_args_max;
78+
self->fun = f;
79+
self->dict_globals = mp_globals_get();
80+
return self;
81+
}
82+
83+
STATIC const mp_ext_table_t mp_ext_table = {
84+
.mp_const_none_ = mp_const_none,
85+
.mp_const_false_ = mp_const_false,
86+
.mp_const_true_ = mp_const_true,
87+
.mp_obj_new_fun_extern = mp_obj_new_fun_extern,
88+
.qstr_from_str = qstr_from_str,
89+
.mp_store_global = mp_store_global,
90+
.mp_obj_new_list = mp_obj_new_list,
91+
.mp_binary_op = mp_binary_op,
92+
};
93+
94+
void mp_extern_load(const char *ext_name, mp_obj_dict_t *globals) {
95+
const byte *buf = mp_extern_load_binary(ext_name);
96+
97+
if (buf == NULL) {
98+
nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_OSError, "could not load MPY binary"));
99+
}
100+
101+
if (!(buf[0] == 'M' && buf[1] == 'P' && buf[2] == 'Y')) {
102+
nlr_raise(mp_obj_new_exception_msg(&mp_type_OSError, "not a valid MPY binary"));
103+
}
104+
105+
if (buf[3] != MP_EXT_VERSION_MAJOR || buf[4] > MP_EXT_VERSION_MINOR) {
106+
nlr_raise(mp_obj_new_exception_msg(&mp_type_OSError, "MPY binary has wrong version"));
107+
}
108+
109+
if (buf[6] != MP_EXT_ARCH_CURRENT) {
110+
nlr_raise(mp_obj_new_exception_msg(&mp_type_OSError, "MPY binary has wrong arch"));
111+
}
112+
113+
// push context
114+
mp_obj_dict_t *old_locals = mp_locals_get();
115+
mp_obj_dict_t *old_globals = mp_globals_get();
116+
mp_locals_set(globals);
117+
mp_globals_set(globals);
118+
119+
// call extern init
120+
mp_obj_t (*f)(const mp_ext_table_t*) = (mp_obj_t(*)(const mp_ext_table_t*))MICROPY_MAKE_POINTER_CALLABLE(buf + 8);
121+
f(&mp_ext_table);
122+
123+
// pop context
124+
mp_globals_set(old_globals);
125+
mp_locals_set(old_locals);
126+
}
127+
128+
#endif // MICROPY_MODULE_EXTERN

0 commit comments

Comments
 (0)
0