8000 py: Support import of native modules · micropython/micropython@2ff4c6b · GitHub
[go: up one dir, main page]
Skip to content

Commit 2ff4c6b

Browse files
stinosstinos
committed
py: Support import of native modules
Introduce CPython-style dynamic module loading: upon import look for .so or .pyd files on the module search path and load it using the dynamic loader, lookup the symbol 'init_<modulename>' and call it as a function which returns the actual uPy module.
1 parent 49dd9ba commit 2ff4c6b

File tree

12 files changed

+194
-3
lines changed

12 files changed

+194
-3
lines changed

‎examples/unix/Makefile

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
VARIANT ?= standard
2+
BUILD ?= build-$(VARIANT)
3+
include ../../py/mkenv.mk
4+
PORT_DIR ?= $(TOP)/ports/unix
5+
BUILD_DIR = $(PORT_DIR)/$(BUILD)
6+
7+
CFLAGS = \
8+
-Wall -Werror -I$(TOP) -I$(TOP)/py -I$(PORT_DIR) -I$(PORT_DIR)/variants/$(VARIANT) -I$(BUILD_DIR)
9+
10+
ifeq ($(MICROPY_FORCE_32BIT),1)
11+
CC += -m32
12+
endif
13+
14+
all:
15+
$(CC) -fPIC $(CFLAGS) -c dynmod.c -o $(BUILD_DIR)/dynmod.o
16+
$(CC) -shared -o $(BUILD_DIR)/dynmod.so $(BUILD_DIR)/dynmod.o
17+
18+
clean:
19+
$(RM) $(BUILD_DIR)/dynmod.*

‎examples/unix/dynmod.c

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#include <py/obj.h>
2+
#include <py/runtime.h>
3+
4+
//Convert const char* -> qstr -> mp_objt_t.
5+
mp_obj_t new_qstr_obj(const char *str) {
6+
return MP_OBJ_NEW_QSTR(qstr_from_str(str));
7+
}
8+
9+
//Example function which adds 1 to the integer passed.
10+
mp_obj_t add_one(mp_obj_t arg) {
11+
return mp_obj_new_int(mp_obj_get_int(arg) + 1);
12+
}
13+
MP_DEFINE_CONST_FUN_OBJ_1(add_one_obj, add_one);
14+
15+
//Example class which wraps an integer.
16+
typedef struct _mp_obj_number_t {
17+
mp_obj_base_t base;
18+
int number;
19+
} mp_obj_number_t;
20+
21+
mp_obj_t number_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) {
22+
mp_arg_check_num(n_args, n_kw, 1, 1, false);
23+
mp_obj_number_t *o = m_new_obj(mp_obj_number_t);
24+
o->base.type = type;
25+
o->number = mp_obj_get_int(args[0]);
26+
return MP_OBJ_FROM_PTR(o);
27+
}
28+
29+
void number_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) {
30+
(void)kind;
31+
mp_obj_number_t *self = MP_OBJ_TO_PTR(self_in);
32+
mp_printf(print, "number(%d)", self->number);
33+
}
34+
35+
mp_obj_t number_add(mp_obj_t lhs, mp_obj_t rhs) {
36+
mp_obj_number_t *self = MP_OBJ_TO_PTR(lhs);
37+
mp_obj_number_t *rhs_in = MP_OBJ_TO_PTR(rhs);
38+
if (!mp_obj_is_type(rhs_in, self->base.type)) {
39+
mp_raise_TypeError(MP_ERROR_TEXT("right-hand side must be a <number>"));
40+
}
41+
self->number += rhs_in->number;
42+
return mp_const_none;
43+
}
44+
MP_DEFINE_CONST_FUN_OBJ_2(number_add_obj, number_add);
45+
46+
//Module initialization funtion called upon 'import dynmod'.
47+
mp_obj_module_t* init_dynmod() {
48+
mp_obj_module_t *mod = (mp_obj_module_t*) mp_obj_new_module(qstr_from_str("dynmod"));
49+
//Add the function.
50+
mp_obj_dict_store(MP_OBJ_FROM_PTR(mod->globals), new_qstr_obj("add_one"), MP_OBJ_FROM_PTR(&add_one_obj));
51+
//Create clas definition and add it.
52+
mp_obj_type_t *number_type = m_new0(mp_obj_type_t, 1);
53+
number_type->base.type = &mp_type_type;
54+
number_type->name = qstr_from_str("number");
55+
number_type->make_new = number_make_new;
56+
number_type->print = number_print;
57+
number_type->locals_dict = mp_obj_new_dict(0);
58+
mp_obj_dict_store(MP_OBJ_FROM_PTR(number_type->locals_dict), new_qstr_obj("add"), MP_OBJ_FROM_PTR(&number_add_obj));
59+
mp_obj_dict_store(MP_OBJ_FROM_PTR(mod->globals), new_qstr_obj("number"), MP_OBJ_FROM_PTR(number_type));
60+
return mod;
61+
}

‎ports/unix/Makefile

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,11 @@ ifeq ($(MICROPY_PY_THREAD),1)
133133
CFLAGS_MOD += -DMICROPY_PY_THREAD=1 -DMICROPY_PY_THREAD_GIL=0
134134
LDFLAGS_MOD += $(LIBPTHREAD)
135135
endif
136+
ifeq ($(MICROPY_PY_DYNLOAD),1)
137+
CFLAGS_MOD += -DMICROPY_MODULE_LOADDYNLIB=1
138+
LDFLAGS_MOD += -ldl -rdynamic
139+
SRC_MOD += loaddynlib.c
140+
endif
136141

137142
# If the variant enables it, enable modbluetooth.
138143
ifeq ($(MICROPY_PY_BLUETOOTH),1)
@@ -298,9 +303,9 @@ test: $(PROG) $(TOP)/tests/run-tests
298303

299304
test_full: $(PROG) $(TOP)/tests/run-tests
300305
$(eval DIRNAME=ports/$(notdir $(CURDIR)))
301-
cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests
306+
cd $(TOP)/tests && MICROPYPATH=:../$(DIRNAME)/$(BUILD) MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests --keep-path
302307
cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests -d thread
303-
cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests --emit native
308+
cd $(TOP)/tests && MICROPYPATH=:../$(DIRNAME)/$(BUILD) MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests --emit native --keep-path
304309
cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests --via-mpy $(RUN_TESTS_MPY_CROSS_FLAGS) -d basics float micropython
305310
cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests --via-mpy $(RUN_TESTS_MPY_CROSS_FLAGS) --emit native -d basics float micropython
306311
cat $(TOP)/tests/basics/0prelim.py | ./$(PROG) | grep -q 'abc'

‎ports/unix/loaddynlib.c

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#include "py/mpconfig.h"
2+
#include "py/obj.h"
3+
#include <dlfcn.h>
4+
#include <string.h>
5+
6+
#if 0 // print debugging info
7+
#define DEBUG_PRINT (1)
8+
#define DEBUG_printf DEBUG_printf
9+
#else // don't print debugging info
10+
#define DEBUG_PRINT (0)
11+
#define DEBUG_printf(...) (void)0
12+
#endif
13+
14+
typedef mp_obj_module_t * (*fun)();
15+
16+
#if defined(MICROPY_UNIX_COVERAGE)
17+
mp_obj_module_t *mp_load_dynlib(const char *mod_name, vstr_t *path);
18+
#endif
19+
20+
mp_obj_module_t *mp_load_dynlib(const char *mod_name, vstr_t *path) {
21+
void *lib = dlopen(vstr_null_terminated_str(path), RTLD_LAZY | RTLD_LOCAL);
22+
if (!lib) {
23+
DEBUG_printf("dlopen %s failed: %s\n", vstr_null_terminated_str(path), dlerror());
24+
return NULL;
25+
}
26+
const size_t nameLen = strlen(mod_name);
27+
char *initFunc = (char *)alloca(nameLen + 6);
28+
memcpy(initFunc, "init_", 5);
29+
strcpy(initFunc + 5, mod_name);
30+
fun f = dlsym(lib, initFunc);
31+
if (!f) {
32+
DEBUG_printf("dlsym %s failed: %s\n", vstr_null_terminated_str(path), dlerror());
33+
dlclose(lib);
34+
return NULL;
35+
}
36+
return f();
37+
}

‎ports/unix/mpconfigport.mk

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,6 @@ MICROPY_PY_JNI = 0
3838
# Avoid using system libraries, use copies bundled with MicroPython
3939
# as submodules (currently affects only libffi).
4040
MICROPY_STANDALONE = 0
41+
42+
# Allow import from .so files.
43+
MICROPY_PY_DYNLOAD = 0

‎ports/unix/variants/coverage/mpconfigvariant.mk

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ MICROPY_ROM_TEXT_COMPRESSION = 1
1919
MICROPY_VFS_FAT = 1
2020
MICROPY_VFS_LFS1 = 1
2121
MICROPY_VFS_LFS2 = 1
22+
MICROPY_PY_DYNLOAD = 1
2223

2324
SRC_C += coverage.c
2425
SRC_CXX += coveragecpp.cpp

‎py/builtinimport.c

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,21 @@
4848

4949
#define PATH_SEP_CHAR '/'
5050

51+
#if MICROPY_MODULE_LOADDYNLIB
52+
// match CPython's native module naming
53+
#ifdef _WIN32
54+
#ifdef _DEBUG
55+
#define PYD_EXT "_d.pyd"
56+
#else
57+
#define PYD_EXT ".pyd"
58+
#endif
59+
#else
60+
#define PYD_EXT ".so"
61+
#endif
62+
63+
extern mp_obj_module_t *mp_load_dynlib(const char *mod_name,vstr_t *name);
64+
#endif
65+
5166
bool mp_obj_is_package(mp_obj_t module) {
5267
mp_obj_t dest[2];
5368
mp_load_method_maybe(module, MP_QSTR___path__, dest);
@@ -80,6 +95,18 @@ STATIC mp_import_stat_t stat_file_py_or_mpy(vstr_t *path) {
8095
}
8196
#endif
8297

98+
#if MICROPY_MODULE_LOADDYNLIB
99+
#if MICROPY_PERSISTENT_CODE_LOAD
100+
vstr_cut_tail_bytes(path, 1);
101+
#endif
102+
vstr_cut_tail_bytes(path, 3);
103+
vstr_add_str(path, PYD_EXT);
104+
stat = mp_import_stat(vstr_null_terminated_str(path));
105+
if (stat == MP_IMPORT_STAT_FILE) {
106+
return MP_IMPORT_STAT_PYD;
107+
}
108+
#endif
109+
83110
return MP_IMPORT_STAT_NO_EXIST;
84111
}
85112

@@ -402,7 +429,19 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) {
402429
mp_raise_msg_varg(&mp_type_ImportError, MP_ERROR_TEXT("no module named '%q'"), mod_name);
403430
#endif
404431
}
405-
} else {
432+
}
433+
#if MICROPY_MODULE_LOADDYNLIB
434+
else if (stat == MP_IMPORT_STAT_PYD) {
435+
// found the file, so try to load it and get the module
436+
mp_obj_module_t *module_ptr = mp_load_dynlib(mod_str + last, &path);
437+
if (module_ptr == NULL) {
438+
mp_raise_msg_varg(&mp_type_ImportError, MP_ERROR_TEXT("dynamic module load failed for '%q'"), mod_name);
439+
}
440+
mp_obj_dict_store(MP_OBJ_FROM_PTR(module_ptr->globals), MP_OBJ_NEW_QSTR(MP_QSTR___name__), MP_OBJ_NEW_QSTR(mod_name));
441+
module_obj = MP_OBJ_FROM_PTR(module_ptr);
442+
}
443+
#endif
444+
else {
406445
// found the file, so get the module
407446
module_obj = mp_module_get(mod_name);
408447
}

‎py/lexer.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,9 @@ typedef enum {
189189
MP_IMPORT_STAT_NO_EXIST,
190190
MP_IMPORT_STAT_DIR,
191191
MP_IMPORT_STAT_FILE,
192+
#if MICROPY_MODULE_LOADDYNLIB
193+
MP_IMPORT_STAT_PYD
194+
#endif
192195
} mp_import_stat_t;
193196

194197
mp_import_stat_t mp_import_stat(const char *path);

‎py/mpconfig.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -769,6 +769,11 @@ typedef double mp_float_t;
769769
#define MICROPY_MODULE_FROZEN (MICROPY_MODULE_FROZEN_STR || MICROPY_MODULE_FROZEN_MPY)
770770
#endif
771771

772+
// Whether loading of importing dynamic libraries is supported
773+
#ifndef MICROPY_MODULE_LOADDYNLIB
774+
#define MICROPY_MODULE_LOADDYNLIB (0)
775+
#endif
776+
772777
// Whether you can override builtins in the builtins module
773778
#ifndef MICROPY_CAN_OVERRIDE_BUILTINS
774779
#define MICROPY_CAN_OVERRIDE_BUILTINS (0)

‎tests/unix/extra_coverage.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,15 @@
5959

6060
print(cppexample.cppfunc(1, 2))
6161

62+
# test import of module from .so file
63+
import dynmod
64+
65+
print(dynmod.__name__)
66+
print(dynmod.add_one(3))
67+
number = dynmod.number(1)
68+
number.add(dynmod.number(2))
69+
print(number)
70+
6271
# test basic import of frozen scripts
6372
import frzstr1
6473

‎tests/unix/extra_coverage.py.exp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,9 @@ None
147147
cpp None
148148
5
149149
(3, 'hellocpp')
150+
dynmod
151+
4
152+
number(3)
150153
frzstr1
151154
frzstr1.py
152155
frzmpy1

‎tools/ci.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -288,6 +288,10 @@ function ci_native_mpy_modules_32bit_build {
288288
ci_native_mpy_modules_build x86
289289
}
290290

291+
function ci_unix_example_build {
292+
make ${MAKEOPTS} -C examples/unix "$@"
293+
}
294+
291295
function ci_unix_minimal_build {
292296
make ${MAKEOPTS} -C ports/unix VARIANT=minimal
293297
}
@@ -318,6 +322,7 @@ function ci_unix_coverage_setup {
318322

319323
function ci_unix_coverage_build {
320324
ci_unix_build_helper VARIANT=coverage
325+
ci_unix_example_build VARIANT=coverage
321326
}
322327

323328
function ci_unix_coverage_run_tests {
@@ -342,6 +347,7 @@ function ci_unix_32bit_setup {
342347

343348
function ci_unix_coverage_32bit_build {
344349
ci_unix_build_helper VARIANT=coverage MICROPY_FORCE_32BIT=1
350+
ci_unix_example_build VARIANT=coverage MICROPY_FORCE_32BIT=1
345351
}
346352

347353
function ci_unix_coverage_32bit_run_tests {

0 commit comments

Comments
 (0)
0