micropython · dpgeorge · Dec 12, 2019 · Oct 6, 2019 · Oct 7, 2019 · Oct 8, 2019
diff --git a/.travis.yml b/.travis.yml
@@ -62,7 +62,10 @@ jobs:
     - stage: test
       env: NAME="unix coverage build and tests"
       install:
+        - sudo apt-get install python3-pip
         - sudo pip install cpp-coveralls
+        - sudo pip3 install setuptools
+        - sudo pip3 install pyelftools
         - gcc --version
         - python3 --version
       script:
@@ -78,6 +81,17 @@ jobs:
         - (cd tests && MICROPY_CPYTHON3=python3 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests --via-mpy --emit native -d basics float micropython)
         # test when input script comes from stdin
         - cat tests/basics/0prelim.py | ports/unix/micropython_coverage | grep -q 'abc'
+        # test building native mpy modules
+        - make -C examples/natmod/features1 ARCH=x64
+        - make -C examples/natmod/features2 ARCH=x64
+        - make -C examples/natmod/btree ARCH=x64
+        - make -C examples/natmod/framebuf ARCH=x64
+        - make -C examples/natmod/uheapq ARCH=x64
+        - make -C examples/natmod/ure ARCH=x64
+        - make -C examples/natmod/uzlib ARCH=x64
+        # test importing .mpy generated by mpy_ld.py
+        - MICROPYPATH=examples/natmod/features2 ./ports/unix/micropython_coverage -m features2
+        - (cd tests && ./run-natmodtests.py extmod/{btree*,framebuf*,uheapq*,ure*,uzlib*}.py)
         # run coveralls coverage analysis (try to, even if some builds/tests failed)
         - (cd ports/unix && coveralls --root ../.. --build-root . --gcov $(which gcov) --gcov-options '\-o build-coverage/' --include py --include extmod)
       after_failure:

diff --git a/examples/natmod/btree/Makefile b/examples/natmod/btree/Makefile
@@ -0,0 +1,37 @@
+# Location of top-level MicroPython directory
+MPY_DIR = ../../..
+
+# Name of module (different to built-in btree so it can coexist)
+    MOD = btree_$(ARCH)
+
+# Source files (.c or .py)
+SRC = btree_c.c btree_py.py
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
+ARCH = x64
+
+BTREE_DIR = $(MPY_DIR)/lib/berkeley-db-1.xx
+BTREE_DEFS = -D__DBINTERFACE_PRIVATE=1 -Dmpool_error="(void)" -Dabort=abort_ "-Dvirt_fd_t=void*" $(BTREE_DEFS_EXTRA)
+CFLAGS += -I$(BTREE_DIR)/PORT/include
+CFLAGS += -Wno-old-style-definition -Wno-sign-compare -Wno-unused-parameter $(BTREE_DEFS)
+
+SRC += $(addprefix $(realpath $(BTREE_DIR))/,\
+	btree/bt_close.c \
+	btree/bt_conv.c \
+	btree/bt_delete.c \
+	btree/bt_get.c \
+	btree/bt_open.c \
+	btree/bt_overflow.c \
+	btree/bt_page.c \
+	btree/bt_put.c \
+	btree/bt_search.c \
+	btree/bt_seq.c \
+	btree/bt_split.c \
+	btree/bt_utils.c \
+	mpool/mpool.c \
+	)
+
+include $(MPY_DIR)/py/dynruntime.mk
+
+# btree needs gnu99 defined
+CFLAGS += -std=gnu99
diff --git a/examples/natmod/btree/btree_c.c b/examples/natmod/btree/btree_c.c
@@ -0,0 +1,148 @@
+#define MICROPY_ENABLE_DYNRUNTIME (1)
+#define MICROPY_PY_BTREE (1)
+
+#include "py/dynruntime.h"
+
+#include <unistd.h>
+
+#if !defined(__linux__)
+void *memcpy(void *dst, const void *src, size_t n) {
+    return mp_fun_table.memmove_(dst, src, n);
+}
+void *memset(void *s, int c, size_t n) {
+    return mp_fun_table.memset_(s, c, n);
+}
+#endif
+
+void *memmove(void *dest, const void *src, size_t n) {
+    return mp_fun_table.memmove_(dest, src, n);
+}
+
+void *malloc(size_t n) {
+    void *ptr = m_malloc(n);
+    return ptr;
+}
+void *realloc(void *ptr, size_t n) {
+    mp_printf(&mp_plat_print, "UNDEF %d\n", __LINE__);
+    return NULL;
+}
+void *calloc(size_t n, size_t m) {
+    void *ptr = m_malloc(n * m);
+    // memory already cleared by conservative GC
+    return ptr;
+}
+
+void free(void *ptr) {
+    m_free(ptr);
+}
+
+void abort_(void) {
+    nlr_raise(mp_obj_new_exception(mp_load_global(MP_QSTR_RuntimeError)));
+}
+
+int native_errno;
+#if defined(__linux__)
+int *__errno_location (void)
+#else
+int *__errno (void)
+#endif
+{
+    return &native_errno;
+}
+
+ssize_t mp_stream_posix_write(void *stream, const void *buf, size_t len) {
+    mp_obj_base_t* o = stream;
+    const mp_stream_p_t *stream_p = o->type->protocol;
+    mp_uint_t out_sz = stream_p->write(MP_OBJ_FROM_PTR(stream), buf, len, &native_errno);
+    if (out_sz == MP_STREAM_ERROR) {
+        return -1;
+    } else {
+        return out_sz;
+    }
+}
+
+ssize_t mp_stream_posix_read(void *stream, void *buf, size_t len) {
+    mp_obj_base_t* o = stream;
+    const mp_stream_p_t *stream_p = o->type->protocol;
+    mp_uint_t out_sz = stream_p->read(MP_OBJ_FROM_PTR(stream), buf, len, &native_errno);
+    if (out_sz == MP_STREAM_ERROR) {
+        return -1;
+    } else {
+        return out_sz;
+    }
+}
+
+off_t mp_stream_posix_lseek(void *stream, off_t offset, int whence) {
+    const mp_obj_base_t* o = stream;
+    const mp_stream_p_t *stream_p = o->type->protocol;
+    struct mp_stream_seek_t seek_s;
+    seek_s.offset = offset;
+    seek_s.whence = whence;
+    mp_uint_t res = stream_p->ioctl(MP_OBJ_FROM_PTR(stream), MP_STREAM_SEEK, (mp_uint_t)(uintptr_t)&seek_s, &native_errno);
+    if (res == MP_STREAM_ERROR) {
+        return -1;
+    }
+    return seek_s.offset;
+}
+
+int mp_stream_posix_fsync(void *stream) {
+    mp_obj_base_t* o = stream;
+    const mp_stream_p_t *stream_p = o->type->protocol;
+    mp_uint_t res = stream_p->ioctl(MP_OBJ_FROM_PTR(stream), MP_STREAM_FLUSH, 0, &native_errno);
+    if (res == MP_STREAM_ERROR) {
+        return -1;
+    }
+    return res;
+}
+
+mp_obj_type_t btree_type;
+
+#include "extmod/modbtree.c"
+
+mp_map_elem_t btree_locals_dict_table[8];
+STATIC MP_DEFINE_CONST_DICT(btree_locals_dict, btree_locals_dict_table);
+
+STATIC mp_obj_t btree_open(size_t n_args, const mp_obj_t *args) {
+    // Make sure we got a stream object
+    mp_get_stream_raise(args[0], MP_STREAM_OP_READ | MP_STREAM_OP_WRITE | MP_STREAM_OP_IOCTL);
+
+    BTREEINFO openinfo = {0};
+    openinfo.flags = mp_obj_get_int(args[1]);
+    openinfo.cachesize = mp_obj_get_int(args[2]);
+    openinfo.psize = mp_obj_get_int(args[3]);
+    openinfo.minkeypage = mp_obj_get_int(args[4]);
+    DB *db = __bt_open(MP_OBJ_TO_PTR(args[0]), &btree_stream_fvtable, &openinfo, 0);
+    if (db == NULL) {
+        mp_raise_OSError(native_errno);
+    }
+
+    return MP_OBJ_FROM_PTR(btree_new(db));
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(btree_open_obj, 5, 5, btree_open);
+
+mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    MP_DYNRUNTIME_INIT_ENTRY
+
+    btree_type.base.type = (void*)&mp_fun_table.type_type;
+    btree_type.name = MP_QSTR_btree;
+    btree_type.print = btree_print;
+    btree_type.getiter = btree_getiter;
+    btree_type.iternext = btree_iternext;
+    btree_type.binary_op = btree_binary_op;
+    btree_type.subscr = btree_subscr;
+    btree_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_close), MP_OBJ_FROM_PTR(&btree_close_obj) };
+    btree_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_flush), MP_OBJ_FROM_PTR(&btree_flush_obj) };
+    btree_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_get), MP_OBJ_FROM_PTR(&btree_get_obj) };
+    btree_locals_dict_table[3] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_put), MP_OBJ_FROM_PTR(&btree_put_obj) };
+    btree_locals_dict_table[4] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_seq), MP_OBJ_FROM_PTR(&btree_seq_obj) };
+    btree_locals_dict_table[5] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_keys), MP_OBJ_FROM_PTR(&btree_keys_obj) };
+    btree_locals_dict_table[6] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_values), MP_OBJ_FROM_PTR(&btree_values_obj) };
+    btree_locals_dict_table[7] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_items), MP_OBJ_FROM_PTR(&btree_items_obj) };
+    btree_type.locals_dict = (void*)&btree_locals_dict;
+
+    mp_store_global(MP_QSTR__open, MP_OBJ_FROM_PTR(&btree_open_obj));
+    mp_store_global(MP_QSTR_INCL, MP_OBJ_NEW_SMALL_INT(FLAG_END_KEY_INCL));
+    mp_store_global(MP_QSTR_DESC, MP_OBJ_NEW_SMALL_INT(FLAG_DESC));
+
+    MP_DYNRUNTIME_INIT_EXIT
+}
diff --git a/examples/natmod/btree/btree_py.py b/examples/natmod/btree/btree_py.py
@@ -0,0 +1,3 @@
+# Implemented in Python to support keyword arguments
+def open(stream, *, flags=0, cachesize=0, pagesize=0, minkeypage=0):
+    return _open(stream, flags, cachesize, pagesize, minkeypage)
diff --git a/examples/natmod/features1/Makefile b/examples/natmod/features1/Makefile
@@ -0,0 +1,14 @@
+# Location of top-level MicroPython directory
+MPY_DIR = ../../..
+
+# Name of module
+MOD = features1
+
+# Source files (.c or .py)
+SRC = features1.c
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
+ARCH = x64
+
+# Include to get the rules for compiling and linking the module
+include $(MPY_DIR)/py/dynruntime.mk
diff --git a/examples/natmod/features1/features1.c b/examples/natmod/features1/features1.c
@@ -0,0 +1,106 @@
+/* This example demonstrates the following features in a native module:
+    - defining simple functions exposed to Python
+    - defining local, helper C functions
+    - defining constant integers and strings exposed to Python
+    - getting and creating integer objects
+    - creating Python lists
+    - raising exceptions
+    - allocating memory
+    - BSS and constant data (rodata)
+    - relocated pointers in rodata
+*/
+
+// Include the header file to get access to the MicroPython API
+#include "py/dynruntime.h"
+
+// BSS (zero) data
+uint16_t data16[4];
+
+// Constant data (rodata)
+const uint8_t table8[] = { 0, 1, 1, 2, 3, 5, 8, 13 };
+const uint16_t table16[] = { 0x1000, 0x2000 };
+
+// Constant data pointing to BSS/constant data
+uint16_t *const table_ptr16a[] = { &data16[0], &data16[1], &data16[2], &data16[3] };
+const uint16_t *const table_ptr16b[] = { &table16[0], &table16[1] };
+
+// A simple function that adds its 2 arguments (must be integers)
+STATIC mp_obj_t add(mp_obj_t x_in, mp_obj_t y_in) {
+    mp_int_t x = mp_obj_get_int(x_in);
+    mp_int_t y = mp_obj_get_int(y_in);
+    return mp_obj_new_int(x + y);
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_2(add_obj, add);
+
+// A local helper function (not exposed to Python)
+STATIC mp_int_t fibonacci_helper(mp_int_t x) {
+    if (x < MP_ARRAY_SIZE(table8)) {
+        return table8[x];
+    } else {
+        return fibonacci_helper(x - 1) + fibonacci_helper(x - 2);
+    }
+}
+
+// A function which computes Fibonacci numbers
+STATIC mp_obj_t fibonacci(mp_obj_t x_in) {
+    mp_int_t x = mp_obj_get_int(x_in);
+    if (x < 0) {
+        mp_raise_ValueError("can't compute negative Fibonacci number");
+    }
+    return mp_obj_new_int(fibonacci_helper(x));
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_1(fibonacci_obj, fibonacci);
+
+// A function that accesses the BSS data
+STATIC mp_obj_t access(size_t n_args, const mp_obj_t *args) {
+    if (n_args == 0) {
+        // Create a list holding all items from data16
+        mp_obj_list_t *lst = MP_OBJ_TO_PTR(mp_obj_new_list(MP_ARRAY_SIZE(data16), NULL));
+        for (int i = 0; i < MP_ARRAY_SIZE(data16); ++i) {
+            lst->items[i] = mp_obj_new_int(data16[i]);
+        }
+        return MP_OBJ_FROM_PTR(lst);
+    } else if (n_args == 1) {
+        // Get one item from data16
+        mp_int_t idx = mp_obj_get_int(args[0]) & 3;
+        return mp_obj_new_int(data16[idx]);
+    } else {
+        // Set one item in data16 (via table_ptr16a)
+        mp_int_t idx = mp_obj_get_int(args[0]) & 3;
+        *table_ptr16a[idx] = mp_obj_get_int(args[1]);
+        return mp_const_none;
+    }
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(access_obj, 0, 2, access);
+
+// A function that allocates memory and creates a bytearray
+STATIC mp_obj_t make_array(void) {
+    uint16_t *ptr = m_new(uint16_t, MP_ARRAY_SIZE(table_ptr16b));
+    for (int i = 0; i < MP_ARRAY_SIZE(table_ptr16b); ++i) {
+        ptr[i] = *table_ptr16b[i];
+    }
+    return mp_obj_new_bytearray_by_ref(sizeof(uint16_t) * MP_ARRAY_SIZE(table_ptr16b), ptr);
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_0(make_array_obj, make_array);
+
+// This is the entry point and is called when the module is imported
+mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    // This must be first, it sets up the globals dict and other things
+    MP_DYNRUNTIME_INIT_ENTRY
+
+    // Messages can be printed as usualy
+    mp_printf(&mp_plat_print, "initialising module self=%p\n", self);
+
+    // Make the functions available in the module's namespace
+    mp_store_global(MP_QSTR_add, MP_OBJ_FROM_PTR(&add_obj));
+    mp_store_global(MP_QSTR_fibonacci, MP_OBJ_FROM_PTR(&fibonacci_obj));
+    mp_store_global(MP_QSTR_access, MP_OBJ_FROM_PTR(&access_obj));
+    mp_store_global(MP_QSTR_make_array, MP_OBJ_FROM_PTR(&make_array_obj));
+
+    // Add some constants to the module's namespace
+    mp_store_global(MP_QSTR_VAL, MP_OBJ_NEW_SMALL_INT(42));
+    mp_store_global(MP_QSTR_MSG, MP_OBJ_NEW_QSTR(MP_QSTR_HELLO_MICROPYTHON));
+
+    // This must be last, it restores the globals dict
+    MP_DYNRUNTIME_INIT_EXIT
+}
diff --git a/examples/natmod/features2/Makefile b/examples/natmod/features2/Makefile
@@ -0,0 +1,14 @@
+# Location of top-level MicroPython directory
+MPY_DIR = ../../..
+
+# Name of module
+MOD = features2
+
+# Source files (.c or .py)
+SRC = main.c prod.c test.py
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
+ARCH = x64
+
+# Include to get the rules for compiling and linking the module
+include $(MPY_DIR)/py/dynruntime.mk