micropython · andrewleech · Feb 18, 2019 · Jun 14, 2018 · Dec 12, 2018 · stinos
diff --git a/docs/develop/cmodules.rst b/docs/develop/cmodules.rst
@@ -0,0 +1,164 @@
+MicroPython external C modules
+==============================
+
+When developing modules for use with micropython you may find you run into
+limitations with the python environment, often due to an inability to access
+certain hardware resources or python speed limitations.
+
+    If your limitations can't be resolved with suggestions in :ref:`speed_python`,
+writing some or all of your module in C is a viable option.
+
+If your module is designed to access or work with commonly available 
+hardware or libraries please consider implementing it inside the micropython
+source tree alongside similar modules and submitting it as a pull request. 
+If however you're targeting obscure or proprietary systems it may make 
+more sense to keep this external to the main micropython repository. 
+
+This chapter describes how to compile such external modules into the 
+micropython executable or firmware image.
+
+
+Structure of an external C module
+---------------------------------
+
+A micropython user c module is a directory with the following files:
+
+* ``*.c`` and/or ``*.h`` source code files for your module.
+
+  These will typically include the low level functionality being implemented and 
+  the micropython binding functions to expose the functions and module(s). 
+
+  Currently the best reference for writing these functions/modules is 
+  to find similar modules within the micropython tree and use them as examples.
+
+* ``micropython.mk`` contains the Makefile fragment for this module.
+
+  ``$(USERMOD_DIR)`` is available in ``micropython.mk`` as the path to your
+  module directory. As it's redefined for each c module, is should be expanded
+  in your ``micropython.mk`` to a local make variable,
+  eg ``EXAMPLE_MOD_DIR := $(USERMOD_DIR)``
+
+  Your ``micropython.mk`` must add your modules C files relative to your
+  expanded copy of ``$(USERMOD_DIR)`` to ``SRC_USERMOD``, eg
+  ``SRC_USERMOD += $(EXAMPLE_MOD_DIR)/example.c``
+
+  If you have custom ``CFLAGS`` settings or include folders to define, these
+  should be added to ``CFLAGS_USERMOD``.
+
+  See below for full usage example.
+
+
+Basic Example
+-------------
+
+This simple module named ``example`` provides a single function 
+``example.add_ints(a, b)`` which adds the two integer args together and returns 
+the result.
+
+Directory::
+
+    example/
+    ├── example.c
+    └── micropython.mk
+
+
+``example.c``
+
+.. code-block:: c
+
+    // Include required definitions first.
+    #include "py/obj.h"
+    #include "py/runtime.h"
+    #include "py/builtin.h"
+
+    #define MODULE_EXAMPLE_ENABLED (1)
+
+    // This is the function which will be called from python as example.add_ints(a, b).
+    STATIC mp_obj_t example_add_ints(mp_obj_t a_obj, mp_obj_tab_obj) {
+        // Extract the ints from the micropython input objects
+        int a = mp_obj_get_int(a_obj);
+        int b = mp_obj_get_int(b_obj);
+
+        // Calculate the addition and convert to MicroPython object.
+        return mp_obj_new_int(a + b);
+    }
+    // Define a python reference to the function above
+    STATIC MP_DEFINE_CONST_FUN_OBJ_1(example_add_ints_obj, example_add_ints);
+
+    // Define all properties of the example module. 
+    // Table entries are key/value pairs of the attribute name (a string) 
+    // and the micropython object reference.
+    // All identifiers and strings are written as MP_QSTR_xxx and will be
+    // optimized to word-sized integers by the build system (interned strings).
+    STATIC const mp_rom_map_elem_t example_module_globals_table[] = {
+        { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_example) },
+        { MP_ROM_QSTR(MP_QSTR_add_ints), MP_ROM_PTR(&example_add_ints_obj) },
+    };
+    STATIC MP_DEFINE_CONST_DICT(example_module_globals, example_module_globals_table);
+
+    // Define module object.
+    const mp_obj_module_t example_user_cmodule = {
+        .base = { &mp_type_module },
+        .globals = (mp_obj_dict_t*)&example_module_globals,
+    };
+
+    // Register the module to make it available in python
+    MP_REGISTER_MODULE(MP_QSTR_example, example_user_cmodule, MODULE_EXAMPLE_ENABLED);
+
+
+``micropython.mk``
+
+.. code-block:: make
+
+    EXAMPLE_MOD_DIR := $(USERMOD_DIR)
+
+    # Add all C files to SRC_USERMOD.
+    SRC_USERMOD += $(EXAMPLE_MOD_DIR)/example.c
+
+    # We can add our module folder to include paths if needed
+    # This is not actually needed in this example.
+    CFLAGS_USERMOD += -I$(EXAMPLE_MOD_DIR)
+
+
+Compiling the cmodule into micropython
+--------------------------------------
+
+To build such a module, compile MicroPython (see `getting started
+<https://github.com/micropython/micropython/wiki/Getting-Started>`_) with an
+extra ``make`` flag named ``USER_C_MODULES`` set to the directory containing
+all modules you want included (not to the module itself). For example:
+
+
+Directory::
+
+    my_project/
+    ├── modules/
+    │   └──example/
+    │       ├──example.c
+    │       └──micropython.mk
+    └── micropython/
+        ├──ports/
+       ... ├──stm32/
+          ...
+
+Building for stm32 port:
+
+.. code-block:: bash
+
+    cd my_project/micropython/ports/stm32
+    make USER_C_MODULES=../../../modules all
+
+
+
+Module usage in micropython
+---------------------------
+
+Once built into your copy of micropython, the module implemented 
+in ``example.c`` above can now be accessed in python just 
+like any other builtin module, eg
+
+.. code-block:: python
+
+    import example
+    print(example.add_ints(1, 3))
+    # should display 4
diff --git a/docs/develop/index.rst b/docs/develop/index.rst
@@ -0,0 +1,15 @@
+Developing and building MicroPython
+===================================
+
+This chapter describes modules (function and class libraries) which are built
+into MicroPython. There are a few categories of such modules:
+
+This chapter describes some options for extending MicroPython in C. Note
+that it doesn't aim to be a complete guide for developing with MicroPython.
+See the `getting started guide
+<https://github.com/micropython/micropython/wiki/Getting-Started>`_ for further information.
+
+.. toctree::
+   :maxdepth: 1
+
+   cmodules.rst
diff --git a/docs/index.rst b/docs/index.rst
@@ -6,6 +6,7 @@ MicroPython documentation and references
     library/index.rst
     reference/index.rst
     genrst/index.rst
+    develop/index.rst
     license.rst
     pyboard/quickref.rst
     esp8266/quickref.rst

diff --git a/docs/reference/speed_python.rst b/docs/reference/speed_python.rst
@@ -1,3 +1,5 @@
+.. _speed_python:
+
 Maximising MicroPython Speed
 ============================
 

diff --git a/mpy-cross/Makefile b/mpy-cross/Makefile
@@ -9,6 +9,8 @@ override undefine MICROPY_FORCE_32BIT
 override undefine CROSS_COMPILE
 override undefine FROZEN_DIR
 override undefine FROZEN_MPY_DIR
+override undefine USER_C_MODULES
+override undefine SRC_MOD
 override undefine BUILD
 override undefine PROG
 endif

diff --git a/ports/stm32/Makefile b/ports/stm32/Makefile
@@ -530,7 +530,7 @@ $(BUILD)/firmware.hex: $(BUILD)/firmware.elf
 
 $(BUILD)/firmware.elf: $(OBJ)
 	$(ECHO) "LINK $@"
-	$(Q)$(LD) $(LDFLAGS) -o $@ $^ $(LIBS)
+	$(Q)$(LD) $(LDFLAGS) -o $@ $^ $(LDFLAGS_MOD) $(LIBS)
 	$(Q)$(SIZE) $@
 
 PLLVALUES = boards/pllvalues.py

diff --git a/ports/windows/msvc/genhdr.targets b/ports/windows/msvc/genhdr.targets
@@ -4,7 +4,7 @@
   <Import Project="paths.props" Condition="'$(PyPathsIncluded)' != 'True'"/>
 
   <!--Generate qstrdefs.generated.h and mpversion.h similar to what is done in py/mkrules.mk and py/py.mk-->
-  <Target Name="GenerateHeaders" DependsOnTargets="MakeVersionHdr;MakeQstrData">
+  <Target Name="GenerateHeaders" DependsOnTargets="MakeVersionHdr;MakeModuleDefs;MakeQstrData">
   </Target>
 
   <PropertyGroup>
@@ -83,6 +83,20 @@ using(var outFile = System.IO.File.CreateText(OutputFile)) {
     <Exec Command="$(PyPython) $(PySrcDir)makeqstrdefs.py cat $(DestDir)qstr.i.last $(DestDir)qstr $(QstrDefsCollected)"/>
   </Target>
 
+  <Target Name="MakeModuleDefs" DependsOnTargets="MakeDestDir">
+    <PropertyGroup>
+      <DestFile>$(DestDir)moduledefs.h</DestFile>
+      <TmpFile>$(DestFile).tmp</TmpFile>
+    </PropertyGroup>
+    <ItemGroup>
+      <PyUserModuleFiles Include="@(ClCompile)">
+        <Path>$([System.String]::new('%(FullPath)').Replace('$(PyBaseDir)', ''))</Path>
+      </PyUserModuleFiles>
+    </ItemGroup>
+    <Exec Command="$(PyPython) $(PySrcDir)makemoduledefs.py --vpath=&quot;., $(PyBaseDir), $(PyUserCModules)&quot; @(PyUserModuleFiles->'%(Path)', ' ') > $(TmpFile)"/>
+    <MSBuild Projects="$(MSBuildThisFileFullPath)" Targets="CopyFileIfDifferent" Properties="SourceFile=$(TmpFile);DestFile=$(DestFile)"/>
+  </Target>
+
   <Target Name="MakeQstrData" DependsOnTargets="MakeQstrDefs" Inputs="$(QstrDefsCollected);$(PyQstrDefs);$(QstrDefs)" Outputs="$(QstrGen)">
     <PropertyGroup>
       <TmpFile>$(QstrGen).tmp</TmpFile>

diff --git a/py/makemoduledefs.py b/py/makemoduledefs.py
@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+
+# This pre-processor parses provided objects' c files for 
+# MP_REGISTER_MODULE(module_name, obj_module, enabled_define)
+# These are used to generate a header with the required entries for 
+# "mp_rom_map_elem_t mp_builtin_module_table[]" in py/objmodule.c
+
+from __future__ import print_function
+
+import re
+import os
+import argparse
+
+
+pattern = re.compile(
+    r"[\n;]\s*MP_REGISTER_MODULE\((.*?),\s*(.*?),\s*(.*?)\);",
+    flags=re.DOTALL
+)
+
+
+def find_c_file(obj_file, vpath):
+    """ Search vpaths for the c file that matches the provided object_file.
+
+    :param str obj_file: object file to find the matching c file for
+    :param List[str] vpath: List of base paths, similar to gcc vpath
+    :return: str path to c file or None
+    """
+    c_file = None
+    relative_c_file = os.path.splitext(obj_file)[0] + ".c"
+    relative_c_file = relative_c_file.lstrip('/\\')
+    for p in vpath:
+        possible_c_file = os.path.join(p, relative_c_file)
+        if os.path.exists(possible_c_file):
+            c_file = possible_c_file
+            break
+
+    return c_file
+
+
+def find_module_registrations(c_file):
+    """ Find any MP_REGISTER_MODULE definitions in the provided c file.
+
+    :param str c_file: path to c file to check
+    :return: List[(module_name, obj_module, enabled_define)]
+    """
+    global pattern
+
+    if c_file is None:
+        # No c file to match the object file, skip
+        return set()
+
+    with open(c_file) as c_file_obj:
+        return set(re.findall(pattern, c_file_obj.read()))
+
+
+
+
+def generate_module_table_header(modules):
+    """ Generate header with module table entries for builtin modules.
+
+    :param List[(module_name, obj_module, enabled_define)] modules: module defs
+    :return: None
+    """
+
+    # Print header file for all external modules.
+    mod_defs = []
+    print("// Automatically generated by makemoduledefs.py.\n")
+    for module_name, obj_module, enabled_define in modules:
+        mod_def = "MODULE_DEF_{}".format(module_name.upper())
+        mod_defs.append(mod_def)
+        print((
+            "#if ({enabled_define})\n"
+            "    extern const struct _mp_obj_module_t {obj_module};\n"
+            "    #define {mod_def} {{ MP_ROM_QSTR({module_name}), MP_ROM_PTR(&{obj_module}) }},\n"
+            "#else\n"
+            "    #define {mod_def}\n"
+            "#endif\n"
+            ).format(module_name=module_name, obj_module=obj_module,
+                     enabled_define=enabled_define, mod_def=mod_def)
+        )
+
+    print("\n#define MICROPY_REGISTERED_MODULES \\")
+
+    for mod_def in mod_defs:
+        print("    {mod_def} \\".format(mod_def=mod_def))
+
+    print("// MICROPY_REGISTERED_MODULES")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--vpath", default=".", 
+                        help="comma separated list of folders to search for c files in")
+    parser.add_argument("files", nargs="*",
+                        help="list of c files to search")
+    args = parser.parse_args()
+
+    vpath = [p.strip() for p in args.vpath.split(',')]
+
+    modules = set()
+    for obj_file in args.files:
+        c_file = find_c_file(obj_
F987
file, vpath)
+        modules |= find_module_registrations(c_file)
+
+    generate_module_table_header(sorted(modules))
+
+
+if __name__ == '__main__':
+    main()
diff --git a/py/mkrules.mk b/py/mkrules.mk
@@ -20,12 +20,12 @@ endif
 # can be located. By following this scheme, it allows a single build rule
 # to be used to compile all .c files.
 
-vpath %.S . $(TOP)
+vpath %.S . $(TOP) $(USER_C_MODULES)
 $(BUILD)/%.o: %.S
 	$(ECHO) "CC $<"
 	$(Q)$(CC) $(CFLAGS) -c -o $@ $<
 
-vpath %.s . $(TOP)
+vpath %.s . $(TOP) $(USER_C_MODULES)
 $(BUILD)/%.o: %.s
 	$(ECHO) "AS $<"
 	$(Q)$(AS) -o $@ $<
@@ -42,14 +42,14 @@ $(Q)$(CC) $(CFLAGS) -c -MD -o $@ $<
   $(RM) -f $(@:.o=.d)
 endef
 
-vpath %.c . $(TOP)
+vpath %.c . $(TOP) $(USER_C_MODULES)
 $(BUILD)/%.o: %.c
 	$(call compile_c)
 
 QSTR_GEN_EXTRA_CFLAGS += -DNO_QSTR
 QSTR_GEN_EXTRA_CFLAGS += -I$(BUILD)/tmp
 
-vpath %.c . $(TOP)
+vpath %.c . $(TOP) $(USER_C_MODULES)
 
 $(BUILD)/%.pp: %.c
 	$(ECHO) "PreProcess $<"

diff --git a/py/modarray.c b/py/modarray.c
@@ -40,4 +40,6 @@ const mp_obj_module_t mp_module_array = {
     .globals = (mp_obj_dict_t*)&mp_module_array_globals,
 };
 
+MP_REGISTER_MODULE(MP_QSTR_array, mp_module_array, MICROPY_PY_ARRAY);
+
 #endif
-Original file line number
+Diff line change
@@ Expand Up / @@ -40,4 +40,6 @@ const mp_obj_module_t mp_module_array = { @@
         .globals = (mp_obj_dict_t*)&mp_module_array_globals,
     };
+    MP_REGISTER_MODULE(MP_QSTR_array, mp_module_array, MICROPY_PY_ARRAY);
     #endif