micropython · tve · Apr 10, 2020 · Apr 10, 2020 · May 7, 2020 · May 8, 2020
diff --git a/docs/develop/natmod.rst b/docs/develop/natmod.rst
@@ -68,14 +68,36 @@ So, if your C code has writable data, make sure the data is defined globally,
 without an initialiser, and only written to within functions.
 
 Linker limitation: the native module is not linked against the symbol table of the
-full MicroPython firmware.  Rather, it is linked against an explicit table of exported
-symbols found in ``mp_fun_table`` (in ``py/nativeglue.h``), that is fixed at firmware
-build time.  It is thus not possible to simply call some arbitrary HAL/OS/RTOS/system
-function, for example.
-
-New symbols can be added to the end of the table and the firmware rebuilt.
-The symbols also need to be added to ``tools/mpy_ld.py``'s ``fun_table`` dict in the
-same location.  This allows ``mpy_ld.py`` to be able to pick the new symbols up and
+full MicroPython firmware.
+It is thus not possible to simply call some arbitrary HAL/OS/RTOS/system
+function. However, there are two limited linking mechanisms available:
+via the ``mp_fun_table`` and on some ports (initially esp32 only) the
+``mp_port_fun_table``.
+
+The ``mp_fun_table`` is the same on all ports and is defined in ``py/nativeglue.h``.
+It contains references to close to a hundred functions in MicroPython's core
+that can be used by native code. The functions are defined as a set of macros
+that turn what looks like a direct call (e.g. ``mp_printf(...)``) into
+indirect calls via ``mp_fun_table`` (e.g. ``(*mp_fun_table[72])(...)``).
+
+The ``mp_port_fun_table`` is a table available on some ports that
+exposes additional port-specific functions, typically ones coming from large
+RTOS or networking libraries, such as ESP-IDF in the esp32 case.
+The ``mp_port_fun_table`` may also expose some commonly used functions from
+the standard C library (libc).
+The ``mp_port_fun_table`` contains the addresses of the exposed functions
+and references in the native modules are "fixed-up" as part of the dynamic
+linking process.
+If a native module calls a function that does not exist (for example
+due to varying sets of libraries built into the firmware) the call
+will raise a ``RunTimeError``.
+A sample native module using the ``mp_port_fun_table`` can be found in
+``examples/natmod/esp32-heap``.
+
+New symbols can be added to the end of either table and the firmware rebuilt.
+In the case of the ``mp_fun_table`` the symbols also need to be added to
+``tools/mpy_ld.py``'s ``fun_table`` dict in the same location.
+This allows ``mpy_ld.py`` to be able to pick the new symbols up and
 provide relocations for them when the mpy is imported.  Finally, if the symbol is a
 function, a macro or stub should be added to ``py/dynruntime.h`` to make it easy to
 call the function.

diff --git a/examples/natmod/esp32-heap/Makefile b/examples/natmod/esp32-heap/Makefile
@@ -0,0 +1,29 @@
+# Location of top-level MicroPython directory
+MPY_DIR = $(abspath ../../..)
+
+# Name of module
+MOD = esp32heap
+
+# Source files (.c or .py)
+SRC = features0.c
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
+ARCH = xtensawin
+
+PORT = esp32
+
+# Location of MicroPython source tree
+TOP := $(abspath ../../..)
+
+# Espressif ESP-IDF path
+IDF_PATH ?= $(abspath $(TOP)/../esp-idf-micropython)
+
+# xtensa toolchain bin dir
+PATH := $(abspath $(TOP)/../xtensa-esp32-elf-8.2.0/bin):$(PATH)
+
+# Board to get correct ESP-IDF config
+BOARD ?= GENERIC
+
+# Include to get the rules for compiling and linking the module
+include $(MPY_DIR)/py/dynruntime.mk
+include $(MPY_DIR)/ports/$(PORT)/dynruntime.mk
diff --git a/examples/natmod/esp32-heap/features0.c b/examples/natmod/esp32-heap/features0.c
@@ -0,0 +1,46 @@
+/* This example demonstrates the following features in a native module:
+    - defining a simple function exposed to Python
+    - defining a local, helper C function
+    - getting and creating integer objects
+*/
+
+// Include the header file to get access to the MicroPython API
+#include "py/dynruntime.h"
+// Include esp-idf include files
+#include "esp_system.h"
+
+// This is the function which will be called from Python, as free_heap()
+STATIC mp_obj_t free_heap() {
+    // calling an ESP-IDF function
+    uint32_t f = esp_get_free_heap_size();
+    // sample debug printf
+    mp_printf(&mp_plat_print, "esp_get_free_heap_size returned %d\n", f);
+    // return integer as MP small int object
+    return mp_obj_new_int(f);
+}
+// Define a Python reference to the function above
+STATIC MP_DEFINE_CONST_FUN_OBJ_0(free_heap_obj, free_heap);
+
+// This a function that only exists in esp-idf-v4, it will raise in esp-idf-v3.
+STATIC mp_obj_t flash_encryption_mode() {
+    // calling an ESP-IDF function that doesn't exist in v3
+    extern uint32_t esp_get_flash_encryption_mode();
+    uint32_t f = esp_get_flash_encryption_mode();
+    // return integer as MP small int object
+    return mp_obj_new_int(f);
+}
+// Define a Python reference to the function above
+STATIC MP_DEFINE_CONST_FUN_OBJ_0(flash_encryption_mode_obj, flash_encryption_mode);
+
+// This is the entry point and is called when the module is imported
+mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    // This must be first, it sets up the globals dict and other things
+    MP_DYNRUNTIME_INIT_ENTRY
+
+    // Make the functions available in the module's namespace
+    mp_store_global(MP_QSTR_free_heap, MP_OBJ_FROM_PTR(&free_heap_obj));
+    mp_store_global(MP_QSTR_flash_encryption_mode, MP_OBJ_FROM_PTR(&flash_encryption_mode_obj));
+
+    // This must be last, it restores the globals dict
+    MP_DYNRUNTIME_INIT_EXIT
+}
diff --git a/lib/mynewt-nimble b/lib/mynewt-nimble
diff --git a/ports/esp32/Makefile b/ports/esp32/Makefile
@@ -1,19 +1,30 @@
+# get the directory where this makefile resides
+# fails if the path has a space :-(, if that happens define PORT_DIR in the parent makefile
+PORT_DIR ?= $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+PORT_DIR := $(PORT_DIR)
+$(info PORT_DIR=$(PORT_DIR))
+
+# if the parent makefile defines MPY_DIR then use that also for TOP (normally generated in mkenv.mk)
+ifneq ($(MPY_DIR),)
+TOP ?= $(MPY_DIR)
+endif
+
 # Select the board to build for: if not given on the command line,
 # then default to GENERIC.
 BOARD ?= GENERIC
 
 # If the build directory is not given, make it reflect the board name.
 BUILD ?= build-$(BOARD)
 
-BOARD_DIR ?= boards/$(BOARD)
+BOARD_DIR ?= $(PORT_DIR)/boards/$(BOARD)
 ifeq ($(wildcard $(BOARD_DIR)/.),)
 $(error Invalid BOARD specified: $(BOARD_DIR))
 endif
 
-include ../../py/mkenv.mk
+include $(PORT_DIR)/../../py/mkenv.mk
 
 # Optional (not currently used for ESP32)
--include mpconfigport.mk
+-include $(PORT_DIR)/mpconfigport.mk
 
 ifneq ($(SDKCONFIG),)
 $(error Use the BOARD variable instead of SDKCONFIG)
@@ -129,6 +140,7 @@ include $(TOP)/extmod/nimble/nimble.mk
 endif
 
 # include sdkconfig to get needed configuration values
+SDKCONFIG := $(addprefix $(PORT_DIR)/, $(SDKCONFIG))
 include $(SDKCONFIG)
 
 ################################################################################
@@ -253,6 +265,32 @@ CFLAGS_MOD += -DMICROPY_PY_BLUETOOTH=1
 CFLAGS_MOD += -DMICROPY_PY_BLUETOOTH_ENABLE_CENTRAL_MODE=1
 endif
 
+################################################################################
+# Generate sdkconfig.h from sdkconfig
+
+$(SDKCONFIG_COMBINED): $(SDKCONFIG)
+	$(Q)$(MKDIR) -p $(dir $@)
+	$(Q)$(CAT) $^ > $@
+
+$(SDKCONFIG_H): $(SDKCONFIG_COMBINED)
+	$(ECHO) "GEN $@"
+	$(Q)$(MKDIR) -p $(dir $@)
+	$(Q)$(PYTHON) $(ESPIDF)/tools/kconfig_new/confgen.py \
+		--output header $@ \
+		--config $< \
+		--kconfig $(ESPIDF)/Kconfig \
+		--env "IDF_TARGET=esp32" \
+		--env "IDF_CMAKE=n" \
+		--env "COMPONENT_KCONFIGS=$(ESPCOMP_KCONFIGS)" \
+		--env "COMPONENT_KCONFIGS_PROJBUILD=$(ESPCOMP_KCONFIGS_PROJBUILD)" \
+		--env "IDF_PATH=$(ESPIDF)"
+	$(Q)touch $@
+
+$(HEADER_BUILD)/qstrdefs.generated.h: $(SDKCONFIG_H) $(BOARD_DIR)/mpconfigboard.h
+
+# The rest of this Makefile is not needed to build dynamic native modules
+ifndef MPY_DYNRUNTIME
+
 # these flags are common to C and C++ compilation
 CFLAGS_COMMON = -Os -ffunction-sections -fdata-sections -fstrict-volatile-bitfields \
 	-mlongcalls -nostdlib \
@@ -354,6 +392,7 @@ SRC_C = \
 	mpthreadport.c \
 	machine_rtc.c \
 	machine_sdcard.c \
+	portnativeglue.c \
 	$(wildcard $(BOARD_DIR)/*.c) \
 	$(SRC_MOD)
 
@@ -387,29 +426,6 @@ SRC_QSTR += $(SRC_C) $(EXTMOD_SRC_C) $(LIB_SRC_C) $(DRIVERS_SRC_C)
 # Append any auto-generated sources that are needed by sources listed in SRC_QSTR
 SRC_QSTR_AUTO_DEPS +=
 
-################################################################################
-# Generate sdkconfig.h from sdkconfig
-
-$(SDKCONFIG_COMBINED): $(SDKCONFIG)
-	$(Q)$(MKDIR) -p $(dir $@)
-	$(Q)$(CAT) $^ > $@
-
-$(SDKCONFIG_H): $(SDKCONFIG_COMBINED)
-	$(ECHO) "GEN $@"
-	$(Q)$(MKDIR) -p $(dir $@)
-	$(Q)$(PYTHON) $(ESPIDF)/tools/kconfig_new/confgen.py \
-		--output header $@ \
-		--config $< \
-		--kconfig $(ESPIDF)/Kconfig \
-		--env "IDF_TARGET=esp32" \
-		--env "IDF_CMAKE=n" \
-		--env "COMPONENT_KCONFIGS=$(ESPCOMP_KCONFIGS)" \
-		--env "COMPONENT_KCONFIGS_PROJBUILD=$(ESPCOMP_KCONFIGS_PROJBUILD)" \
-		--env "IDF_PATH=$(ESPIDF)"
-	$(Q)touch $@
-
-$(HEADER_BUILD)/qstrdefs.generated.h: $(SDKCONFIG_H) $(BOARD_DIR)/mpconfigboard.h
-
 ################################################################################
 # List of object files from the ESP32 IDF components
 
@@ -777,6 +793,7 @@ $(BUILD)/application.elf: $(OBJ) $(LIB) $(BUILD)/esp32_out.ld $(BUILD)/esp32.pro
 	$(ECHO) "LINK $@"
 	$(Q)$(LD) $(LDFLAGS) -o $@ $(APP_LD_ARGS)
 	$(Q)$(SIZE) $@
+	$(Q)$(SIZE) -A $@ | egrep iram0.text | sed -e 's/  */: /' -e 's;  *[^ ]*$$; of 131072;'
 
 define compile_cxx
 $(ECHO) "CXX $<"
@@ -794,6 +811,12 @@ vpath %.cpp . $(TOP)
 $(BUILD)/%.o: %.cpp
 	$(call compile_cxx)
 
+ifeq ($(ESPIDF_CURHASH),$(ESPIDF_SUPHASH_V3))
+$(BUILD)/portnativeglue.o: CFLAGS := $(filter-out -Werror, $(CFLAGS))
+else
+$(BUILD)/portnativeglue.o: CFLAGS += -Wno-builtin-declaration-mismatch
+endif
+
 ################################################################################
 # Declarations to build the bootloader
 
@@ -979,3 +1002,5 @@ $(BUILD)/partitions.bin: $(PART_SRC)
 ################################################################################
 
 include $(TOP)/py/mkrules.mk
+
+endif # MPY_DYNRUNTIME
diff --git a/ports/esp32/dynruntime.mk b/ports/esp32/dynruntime.mk
@@ -0,0 +1,16 @@
+# Makefile to include after py/dynruntime.mk in order to add esp32-specific rules
+# for building native modules that include esp-idf header files and call esp-idf functions.
+
+PORT_DIR := $(MPY_DIR)/ports/esp32
+
+MPY_DYNRUNTIME := 1
+include $(PORT_DIR)/Makefile
+
+CFLAGS += -std=gnu99 # override -std=c99 in dynruntime.mk
+CFLAGS += $(INC) $(INC_ESPCOMP) $(CFLAGS_MOD)
+
+# Make all module object files depend on sdkconfig
+$(SRC_O): $(SDKCONFIG_H)
+
+# Tell mpy_ld to load port-specific function table
+PORT_FUN=--portfun $(PORT_DIR)/port_fun.inc
diff --git a/ports/esp32/mpconfigport.h b/ports/esp32/mpconfigport.h
@@ -59,6 +59,7 @@
 #define MICROPY_ENABLE_SCHEDULER            (1)
 #define MICROPY_SCHEDULER_DEPTH             (8)
 #define MICROPY_VFS                         (1)
+#define MICROPY_PORT_FUN_TABLE              (1)
 
 // control over Python builtins
 #define MICROPY_PY_FUNCTION_ATTRS           (1)