8000 py: Support import of native modules · micropython/micropython@7c5147c · GitHub
[go: up one dir, main page]
Skip to content

Commit 7c5147c

Browse files
stinosstinos
committed
py: Support import of native modules
Introduce CPython-style dynamic module loading: upon import look for .so or .pyd files on the module search path and load it using the dynamic loader, lookup the symbol 'init_<modulename>' and call it as a function which returns the actual uPy module.
1 parent f7aafc0 commit 7c5147c

File tree

12 files changed

+132
-3
lines changed

12 files changed

+132
-3
lines changed

‎examples/unix/Makefile

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
VARIANT ?= standard
2+
BUILD ?= build-$(VARIANT)
3+
include ../../py/mkenv.mk
4+
PORT_DIR ?= $(TOP)/ports/unix
5+
BUILD_DIR = $(PORT_DIR)/$(BUILD)
6+
7+
CFLAGS = \
8+
-Wall -Werror -I$(TOP) -I$(TOP)/py -I$(PORT_DIR) -I$(PORT_DIR)/variants/$(VARIANT) -I$(BUILD_DIR)
9+
10+
ifeq ($(MICROPY_FORCE_32BIT),1)
11+
CC += -m32
12+
endif
13+
14+
all:
15+
$(CC) -fPIC $(CFLAGS) -c dynmod.c -o $(BUILD_DIR)/dynmod.o
16+
$(CC) -shared -o $(BUILD_DIR)/dynmod.so $(BUILD_DIR)/dynmod.o
17+
18+
clean:
19+
$(RM) $(BUILD_DIR)/dynmod.*

‎examples/unix/dynmod.c

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#include <py/runtime.h>
2+
3+
mp_obj_module_t* init_dynmod() {
4+
return (mp_obj_module_t*) mp_obj_new_module(qstr_from_str("dynmod"));
5+
}

‎ports/unix/Makefile

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,11 @@ ifeq ($(MICROPY_PY_THREAD),1)
133133
CFLAGS_MOD += -DMICROPY_PY_THREAD=1 -DMICROPY_PY_THREAD_GIL=0
134134
LDFLAGS_MOD += $(LIBPTHREAD)
135135
endif
136+
ifeq ($(MICROPY_PY_DYNLOAD),1)
137+
CFLAGS_MOD += -DMICROPY_MODULE_LOADDYNLIB=1
138+
LDFLAGS_MOD += -ldl -rdynamic
139+
SRC_MOD += loaddynlib.c
140+
endif
136141

137142
# If the variant enables it, enable modbluetooth.
138143
ifeq ($(MICROPY_PY_BLUETOOTH),1)
@@ -298,9 +303,9 @@ test: $(PROG) $(TOP)/tests/run-tests
298303

299304
test_full: $(PROG) $(TOP)/tests/run-tests
300305
$(eval DIRNAME=ports/$(notdir $(CURDIR)))
301-
cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests
306+
cd $(TOP)/tests && MICROPYPATH=:../$(DIRNAME)/$(BUILD) MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests --keep-path
302307
cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests -d thread
303-
cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests --emit native
308+
cd $(TOP)/tests && MICROPYPATH=:../$(DIRNAME)/$(BUILD) MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests --emit native --keep-path
304309
cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests --via-mpy $(RUN_TESTS_MPY_CROSS_FLAGS) -d basics float micropython
305310
cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests --via-mpy $(RUN_TESTS_MPY_CROSS_FLAGS) --emit native -d basics float micropython
306311
cat $(TOP)/tests/basics/0prelim.py | ./$(PROG) | grep -q 'abc'

‎ports/unix/loaddynlib.c

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#include "py/mpconfig.h"
2+
#include "py/obj.h"
3+
#include <dlfcn.h>
4+
#include <string.h>
5+
6+
#if 0 // print debugging info
7+
#define DEBUG_PRINT (1)
8+
#define DEBUG_printf DEBUG_printf
9+
#else // don't print debugging info
10+
#define DEBUG_PRINT (0)
11+
#define DEBUG_printf(...) (void)0
12+
#endif
13+
14+
typedef mp_obj_module_t * (*fun)();
15+
16+
#if defined(MICROPY_UNIX_COVERAGE)
17+
mp_obj_module_t *mp_load_dynlib(const char *mod_name, vstr_t *path);
18+
#endif
19+
20+
mp_obj_module_t *mp_load_dynlib(const char *mod_name, vstr_t *path) {
21+
void *lib = dlopen(vstr_null_terminated_str(path), RTLD_LAZY | RTLD_LOCAL);
22+
if (!lib) {
23+
DEBUG_printf("dlopen %s failed: %s\n", vstr_null_terminated_str(path), dlerror());
24+
return NULL;
25+
}
26+
const size_t nameLen = strlen(mod_name);
27+
char *initFunc = (char *)alloca(nameLen + 6);
28+
memcpy(initFunc, "init_", 5);
29+
strcpy(initFunc + 5, mod_name);
30+
fun f = dlsym(lib, initFunc);
31+
if (!f) {
32+
DEBUG_printf("dlsym %s failed: %s\n", vstr_null_terminated_str(path), dlerror());
33+
dlclose(lib);
34+
return NULL;
35+
}
36+
return f();
37+
}

‎ports/unix/mpconfigport.mk

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,6 @@ MICROPY_PY_JNI = 0
3838
# Avoid using system libraries, use copies bundled with MicroPython
3939
# as submodules (currently affects only libffi).
4040
MICROPY_STANDALONE = 0
41+
42+
# Allow import from .so files.
43+
MICROPY_PY_DYNLOAD = 0

‎ports/unix/variants/coverage/mpconfigvariant.mk

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ MICROPY_ROM_TEXT_COMPRESSION = 1
1919
MICROPY_VFS_FAT = 1
2020
MICROPY_VFS_LFS1 = 1
2121
MICROPY_VFS_LFS2 = 1
22+
MICROPY_PY_DYNLOAD = 1
2223

2324
SRC_C += coverage.c
2425
SRC_CXX += coveragecpp.cpp

‎py/builtinimport.c

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,21 @@
4848

4949
#define PATH_SEP_CHAR '/'
5050

51+
#if MICROPY_MODULE_LOADDYNLIB
52+
// match CPython's native module naming
53+
#ifdef _WIN32
54+
#ifdef _DEBUG
55+
#define PYD_EXT "_d.pyd"
56+
#else
57+
#define PYD_EXT ".pyd"
58+
#endif
59+
#else
60+
#define PYD_EXT ".so"
61+
#endif
62+
63+
extern mp_obj_module_t *mp_load_dynlib(const char *mod_name,vstr_t *name);
64+
#endif
65+
5166
bool mp_obj_is_package(mp_obj_t module) {
5267
mp_obj_t dest[2];
5368
mp_load_method_maybe(module, MP_QSTR___path__, dest);
@@ -80,6 +95,18 @@ STATIC mp_import_stat_t stat_file_py_or_mpy(vstr_t *path) {
8095
}
8196
#endif
8297

98+
#if MICROPY_MODULE_LOADDYNLIB
99+
#if MICROPY_PERSISTENT_CODE_LOAD
100+
vstr_cut_tail_bytes(path, 1);
101+
#endif
102+
vstr_cut_tail_bytes(path, 3);
103+
vstr_add_str(path, PYD_EXT);
104+
stat = mp_import_stat(vstr_null_terminated_str(path));
105+
if (stat == MP_IMPORT_STAT_FILE) {
106+
return MP_IMPORT_STAT_PYD;
107+
}
108+
#endif
109+
83110
return MP_IMPORT_STAT_NO_EXIST;
84111
}
85112

@@ -402,7 +429,19 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) {
402429
mp_raise_msg_varg(&mp_type_ImportError, MP_ERROR_TEXT("no module named '%q'"), mod_name);
403430
#endif
404431
}
405-
} else {
432+
}
433+
#if MICROPY_MODULE_LOADDYNLIB
434+
else if (stat == MP_IMPORT_STAT_PYD) {
435+
// found the file, so try to load it and get the module
436+
mp_obj_module_t *module_ptr = mp_load_dynlib(mod_str + last, &path);
437+
if (module_ptr == NULL) {
438+
mp_raise_msg_varg(&mp_type_ImportError, MP_ERROR_TEXT("dynamic module load failed for '%q'"), mod_name);
439+
}
440+
mp_obj_dict_store(MP_OBJ_FROM_PTR(module_ptr->globals), MP_OBJ_NEW_QSTR(MP_QSTR___name__), MP_OBJ_NEW_QSTR(mod_name));
441+
module_obj = MP_OBJ_FROM_PTR(module_ptr);
442+
}
443+
#endif
444+
else {
406445
// found the file, so get the module
407446
module_obj = mp_module_get(mod_name);
408447
}

‎py/lexer.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,9 @@ typedef enum {
189189
MP_IMPORT_STAT_NO_EXIST,
190190
MP_IMPORT_STAT_DIR,
191191
MP_IMPORT_STAT_FILE,
192+
#if MICROPY_MODULE_LOADDYNLIB
193+
MP_IMPORT_STAT_PYD
194+
#endif
192195
} mp_import_stat_t;
193196

194197
mp_import_stat_t mp_import_stat(const char *path);

‎py/mpconfig.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -769,6 +769,11 @@ typedef double mp_float_t;
769769
#define MICROPY_MODULE_FROZEN (MICROPY_MODULE_FROZEN_STR || MICROPY_MODULE_FROZEN_MPY)
770770
#endif
771771

772+
// Whether loading of importing dynamic libraries is supported
773+
#ifndef MICROPY_MODULE_LOADDYNLIB
774+
#define MICROPY_MODULE_LOADDYNLIB (0)
775+
#endif
776+
772777
// Whether you can override builtins in the builtins module
773778
#ifndef MICROPY_CAN_OVERRIDE_BUILTINS
774779
#define MICROPY_CAN_OVERRIDE_BUILTINS (0)

‎tests/unix/extra_coverage.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,11 @@
5959

6060
print(cppexample.cppfunc(1, 2))
6161

62+
# test import of module from .so file
63+
import dynmod
64+
65+
print(dynmod.__name__)
66+
6267
# test basic import of frozen scripts
6368
import frzstr1
6469

‎tests/unix/extra_coverage.py.exp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,7 @@ None
147147
cpp None
148148
5
149149
(3, 'hellocpp')
150+
dynmod
150151
frzstr1
151152
frzstr1.py
152153
frzmpy1

‎tools/ci.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -287,6 +287,10 @@ function ci_native_mpy_modules_32bit_build {
287287
ci_native_mpy_modules_build x86
288288
}
289289

290+
function ci_unix_example_build {
291+
make ${MAKEOPTS} -C examples/unix "$@"
292+
}
293+
290294
< 9AFA span class="pl-k">function ci_unix_minimal_build {
291295
make ${MAKEOPTS} -C ports/unix VARIANT=minimal
292296
}
@@ -317,6 +321,7 @@ function ci_unix_coverage_setup {
317321

318322
function ci_unix_coverage_build {
319323
ci_unix_build_helper VARIANT=coverage
324+
ci_unix_example_build VARIANT=coverage
320325
}
321326

322327
function ci_unix_coverage_run_tests {
@@ -341,6 +346,7 @@ function ci_unix_32bit_setup {
341346

342347
function ci_unix_coverage_32bit_build {
343348
ci_unix_build_helper VARIANT=coverage MICROPY_FORCE_32BIT=1
349+
ci_unix_example_build VARIANT=coverage MICROPY_FORCE_32BIT=1
344350
}
345351

346352
function ci_unix_coverage_32bit_run_tests {

0 commit comments

Comments
 (0)
0