8000 tools/mpy_ld.py: Add new mpy_ld.py tool and associated build files. · micropython/micropython@aad79ad · GitHub
[go: up one dir, main page]
Skip to content
8000

Commit aad79ad

Browse files
dpgeorgedpgeorge
committed
tools/mpy_ld.py: Add new mpy_ld.py tool and associated build files.
This commit adds a new tool called mpy_ld.py which is essentially a linker that builds .mpy files directly from .o files. A new header file (dynruntime.h) and makefile fragment (dynruntime.mk) are also included which allow building .mpy files from C source code. Such .mpy files can then be dynamically imported as though they were a normal Python module, even though they are implemented in C. Converting .o files directly (rather than pre-linked .elf files) allows the resulting .mpy to be more efficient because it has more control over the relocations; for example it can skip PLT indirection. Doing it this way also allows supporting more architectures, such as Xtensa which has specific needs for position-independent code and the GOT. The tool supports targets of x86, x86-64, ARM Thumb and Xtensa (windowed and non-windowed). BSS, text and rodata sections are supported, with relocations to all internal sections and symbols, as well as relocations to some external symbols (defined by dynruntime.h), and linking of qstrs.
1 parent 2787984 commit aad79ad

File tree

3 files changed

+1305
-0
lines changed

3 files changed

+1305
-0
lines changed

‎py/dynruntime.h

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
/*
2+
* This file is part of the MicroPython project, http://micropython.org/
3+
*
4+
* The MIT License (MIT)
5+
*
6+
* Copyright (c) 2019 Damien P. George
7+
*
8+
* Permission is hereby granted, free of charge, to any person obtaining a copy
9+
* of this software and associated documentation files (the "Software"), to deal
10+
* in the Software without restriction, including without limitation the rights
11+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12+
* copies of the Software, and to permit persons to whom the Software is
13+
* furnished to do so, subject to the following conditions:
14+
*
15+
* The above copyright notice and this permission notice shall be included in
16+
* all copies or substantial portions of the Software.
17+
*
18+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
24+
* THE SOFTWARE.
25+
*/
26+
#ifndef MICROPY_INCLUDED_PY_DYNRUNTIME_H
27+
#define MICROPY_INCLUDED_PY_DYNRUNTIME_H
28+
29+
// This header file contains definitions to dynamically implement the static
30+
// MicroPython runtime API defined in py/obj.h and py/runtime.h.
31+
32+
#include "py/nativeglue.h"
33+
#include "py/objstr.h"
34+
35+
#undef MP_ROM_QSTR
36+
#undef MP_OBJ_QSTR_VALUE
37+
#undef MP_OBJ_NEW_QSTR
38+
#undef mp_const_none
39+
#undef mp_const_false
40+
#undef mp_const_true
41+
#undef mp_const_empty_tuple
42+
#undef nlr_raise
43+
44+
/******************************************************************************/
45+
// Memory allocation
46+
47+
#define m_malloc(n) (m_malloc_dyn((n)))
48+
#define m_free(ptr) (m_free_dyn((ptr)))
49+
#define m_realloc(ptr, new_num_bytes) (m_realloc_dyn((ptr), (new_num_bytes)))
50+
51+
static inline void *m_malloc_dyn(size_t n) {
52+
// TODO won't raise on OOM
53+
return mp_fun_table.realloc_(NULL, n, false);
54+
}
55+
56+
static inline void m_free_dyn(void *ptr) {
57+
mp_fun_table.realloc_(ptr, 0, false);
58+
}
59+
60+
static inline void *m_realloc_dyn(void *ptr, size_t new_num_bytes) {
61+
// TODO won't raise on OOM
62+
return mp_fun_table.realloc_(ptr, new_num_bytes, true);
63+
}
64+
65+
/******************************************************************************/
66+
// Printing
67+
68+
#define mp_plat_print (*mp_fun_table.plat_print)
69+
#define mp_printf(p, ...) (mp_fun_table.printf_((p), __VA_ARGS__))
70+
#define mp_vprintf(p, fmt, args) (mp_fun_table.vprintf_((p), (fmt), (args)))
71+
72+
/******************************************************************************/
73+
// Types and objects
74+
75+
#define MP_OBJ_NEW_QSTR(x) MP_OBJ_NEW_QSTR_ ## x
76+
77+
#define mp_type_type (*mp_fun_table.type_type)
78+
#define mp_type_str (*mp_fun_table.type_str)
79+
#define mp_type_list (*mp_fun_table.type_list)
80+
#define mp_type_EOFError (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_EOFError)))
81+
#define mp_type_IndexError (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_IndexError)))
82+
#define mp_type_KeyError (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_KeyError)))
83+
#define mp_type_NotImplementedError (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_NotImplementedError)))
84+
#define mp_type_RuntimeError (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_RuntimeError)))
85+
#define mp_type_TypeError (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_TypeError)))
86+
#define mp_type_ValueError (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_ValueError)))
87+
88+
#define mp_stream_read_obj (*mp_fun_table.stream_read_obj)
89+
#define mp_stream_readinto_obj (*mp_fun_table.stream_readinto_obj)
90+
#define mp_stream_unbuffered_readline_obj (*mp_fun_table.stream_unbuffered_readline_obj)
91+
#define mp_stream_write_obj (*mp_fun_table.stream_write_obj)
92+
93+
#define mp_const_none ((mp_obj_t)mp_fun_table.const_none)
94+
#define mp_const_false ((mp_obj_t)mp_fun_table.const_false)
95+
#define mp_const_true ((mp_obj_t)mp_fun_table.const_true)
96+
#define mp_const_empty_tuple (mp_fun_table.new_tuple(0, NULL))
97+
98+
#define mp_obj_new_bool(b) ((b) ? (mp_obj_t)mp_fun_table.const_true : (mp_obj_t)mp_fun_table.const_false)
99+
#define mp_obj_new_int(i) (mp_fun_table.native_to_obj(i, MP_NATIVE_TYPE_INT))
100+
#define mp_obj_new_str(data, len) (mp_fun_table.obj_new_str((data), (len)))
101+
#define mp_obj_new_str_of_type(t, d, l) (mp_obj_new_str_of_type_dyn((t), (d), (l)))
102+
#define mp_obj_new_bytes(data, len) (mp_fun_table.obj_new_bytes((data), (len)))
103+
#define mp_obj_new_bytearray_by_ref(n, i) (mp_fun_table.obj_new_bytearray_by_ref((n), (i)))
104+
#define mp_obj_new_tuple(n, items) (mp_fun_table.new_tuple((n), (items)))
105+
#define mp_obj_new_list(n, items) (mp_fun_table.new_list((n), (items)))
106+
107+
#define mp_obj_get_type(o) (mp_fun_table.obj_get_type((o)))
108+
#define mp_obj_get_int(o) (mp_fun_table.native_from_obj(o, MP_NATIVE_TYPE_INT))
109+
#define mp_obj_str_get_str(s) ((void*)mp_fun_table.native_from_obj(s, MP_NATIVE_TYPE_PTR))
110+
#define mp_obj_str_get_data(o, len) (mp_obj_str_get_data_dyn((o), (len)))
111+
#define mp_get_buffer_raise(o, bufinfo, fl) (mp_fun_table.get_buffer_raise((o), (bufinfo), (fl)))
112+
#define mp_get_stream_raise(s, flags) (mp_fun_table.get_stream_raise((s), (flags)))
113+
114+
#define mp_obj_list_append(list, item) (mp_fun_table.list_append((list), (item)))
115+
116+
static inline mp_obj_t mp_obj_new_str_of_type_dyn(const mp_obj_type_t *type, const byte* data, size_t len) {
117+
if (type == &mp_type_str) {
118+
return mp_obj_new_str((const char*)data, len);
119+
} else {
120+
return mp_obj_new_bytes(data, len);
121+
}
122+
}
123+
124+
static inline void *mp_obj_str_get_data_dyn(mp_obj_t o, size_t *l) {
125+
mp_buffer_info_t bufinfo;
126+
mp_get_buffer_raise(o, &bufinfo, MP_BUFFER_READ);
127+
*l = bufinfo.len;
128+
return bufinfo.buf;
129+
}
130+
131+
/******************************************************************************/
132+
// General runtime functions
133+
134+
#define mp_load_name(qst) (mp_fun_table.load_name(qst))
135+
#define mp_load_global(qst) (mp_fun_table.load_global(qst))
136+
#define mp_store_global(qst, obj) (mp_fun_table.store_global((qst), (obj)))
137+
#define mp_unary_op(op, obj) (mp_fun_table.unary_op((op), (obj)))
138+
#define mp_binary_op(op, lhs, rhs) (mp_fun_table.binary_op((op), (lhs), (rhs)))
139+
140+
#define mp_make_function_from_raw_code(rc, def_args, def_kw_args) \
141+
(mp_fun_table.make_function_from_raw_code((rc), (def_args), (def_kw_args)))
142+
143+
#define mp_call_function_n_kw(fun, n_args, n_kw, args) \
144+
(mp_fun_table.call_function_n_kw((fun), (n_args) | ((n_kw) << 8), args))
145+
146+
#define mp_arg_check_num(n_args, n_kw, n_args_min, n_args_max, takes_kw) \
147+
(mp_fun_table.arg_check_num_sig((n_args), (n_kw), MP_OBJ_FUN_MAKE_SIG((n_args_min), (n_args_max), (takes_kw))))
148+
149+
#define MP_DYNRUNTIME_INIT_ENTRY \
150+
mp_obj_t old_globals = mp_fun_table.swap_globals(self->globals); \
151+
mp_raw_code_t rc; \
152+
rc.kind = MP_CODE_NATIVE_VIPER; \
153+
rc.scope_flags = 0; \
154+
rc.const_table = (void*)self->const_table; \
155+
(void)rc;
156+
157+
#define MP_DYNRUNTIME_INIT_EXIT \
158+
mp_fun_table.swap_globals(old_globals); \
159+
return mp_const_none;
160+
161+
#define MP_DYNRUNTIME_MAKE_FUNCTION(f) \
162+
(mp_make_function_from_raw_code((rc.fun_data = (f), &rc), MP_OBJ_NULL, MP_OBJ_NULL))
163+
164+
/******************************************************************************/
165+
// Exceptions
166+
167+
#define mp_obj_new_exception(o) ((mp_obj_t)(o)) // Assumes returned object will be raised, will create instance then
168+
#define mp_obj_new_exception_arg1(e_type, arg) (mp_obj_new_exception_arg1_dyn((e_type), (arg)))
169+
170+
#define nlr_raise(o) (mp_raise_dyn(o))
171+
#define mp_raise_msg(type, msg) (mp_fun_table.raise_msg((type), (msg)))
172+
#define mp_raise_OSError(er) (mp_raise_OSError_dyn(er))
173+
#define mp_raise_NotImplementedError(msg) (mp_raise_msg(&mp_type_NotImplementedError, (msg)))
174+
#define mp_raise_TypeError(msg) (mp_raise_msg(&mp_type_TypeError, (msg)))
175+
#define mp_raise_ValueError(msg) (mp_raise_msg(&mp_type_ValueError, (msg)))
176+
177+
static inline mp_obj_t mp_obj_new_exception_arg1_dyn(const mp_obj_type_t *exc_type, mp_obj_t arg) {
178+
mp_obj_t args[1] = { arg };
179+
return mp_call_function_n_kw(MP_OBJ_FROM_PTR(exc_type), 1, 0, &args[0]);
180+
}
181+
182+
static NORETURN inline void mp_raise_dyn(mp_obj_t o) {
183+
mp_fun_table.raise(o);
184+
for (;;) {
185+
}
186+
}
187+
188+
static inline void mp_raise_OSError_dyn(int er) {
189+
mp_obj_t args[1] = { MP_OBJ_NEW_SMALL_INT(er) };
190+
nlr_raise(mp_call_function_n_kw(mp_load_global(MP_QSTR_OSError), 1, 0, &args[0]));
191+
}
192+
193+
#endif // MICROPY_INCLUDED_PY_DYNRUNTIME_H

‎py/dynruntime.mk

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
# Makefile fragment for generating native .mpy files from C source
2+
# MPY_DIR must be set to the top of the MicroPython source tree
3+
4+
BUILD ?= build
5+
6+
ECHO = @echo
7+
RM = /bin/rm
8+
MKDIR = /bin/mkdir
9+
PYTHON = python3
10+
MPY_CROSS = $(MPY_DIR)/mpy-cross/mpy-cross
11+
MPY_TOOL = $(PYTHON) $(MPY_DIR)/tools/mpy-tool.py
12+
MPY_LD = $(PYTHON) $(MPY_DIR)/tools/mpy_ld.py
13+
14+
Q = @
15+
ifeq ("$(origin V)", "command line")
16+
ifeq ($(V),1)
17+
Q =
18+
MPY_LD += '-vvv'
19+
endif
20+
endif
21+
22+
ARCH_UPPER = $(shell echo $(ARCH) | tr '[:lower:]' '[:upper:]')
23+
CONFIG_H = $(BUILD)/$(MOD).config.h
24+
25+
CFLAGS += -I. -I$(MPY_DIR)
26+
CFLAGS += -std=c99
27+
CFLAGS += -Os
28+
CFLAGS += -Wall -Werror -DNDEBUG
29+
CFLAGS += -DNO_QSTR
30+
CFLAGS += -DMP_CONFIGFILE='<$(CONFIG_H)>'
31+
CFLAGS += -fpic -fno-common
32+
CFLAGS += -U _FORTIFY_SOURCE # prevent use of __*_chk libc functions
33+
#CFLAGS += -fdata-sections -ffunction-sections
34+
35+
MPY_CROSS_FLAGS += -march=$(ARCH)
36+
37+
SRC_O += $(addprefix $(BUILD)/, $(patsubst %.c,%.o,$(filter %.c,$(SRC))))
38+
SRC_MPY += $(addprefix $(BUILD)/, $(patsubst %.py,%.mpy,$(filter %.py,$(SRC))))
39+
40+
################################################################################
41+
# Architecture configuration
42+
43+
ifeq ($(ARCH),x86)
44+
45+
# x86
46+
CROSS =
47+
CFLAGS += -m32 -fno-stack-protector
48+
MPY_CROSS_FLAGS += -mcache-lookup-bc
49+
50+
else ifeq ($(ARCH),x64)
51+
52+
# x64
53+
CROSS =
54+
CFLAGS += -fno-stack-protector
55+
MPY_CROSS_FLAGS += -mcache-lookup-bc
56+
57+
else ifeq ($(ARCH),armv7m)
58+
59+
# thumb
60+
CROSS = arm-none-eabi-
61+
CFLAGS += -mthumb -mcpu=cortex-m3
62+
63+
else ifeq ($(ARCH),armv7emsp)
64+
65+
# thumb
66+
CROSS = arm-none-eabi-
67+
CFLAGS += -mthumb -mcpu=cortex-m4
68+
CFLAGS += -mfpu=fpv4-sp-d16 -mfloat-abi=hard
69+
70+
else ifeq ($(ARCH),armv7emdp)
71+
72+
# thumb
73+
CROSS = arm-none-eabi-
74+
CFLAGS += -mthumb -mcpu=cortex-m7
75+
CFLAGS += -mfpu=fpv5-d16 -mfloat-abi=hard
76+
77+
else ifeq ($(ARCH),xtensa)
78+
79+
# xtensa
80+
CROSS = xtensa-lx106-elf-
81+
CFLAGS += -mforce-l32
82+
83+
else ifeq ($(ARCH),xtensawin)
84+
85+
# xtensawin
86+
CROSS = xtensa-esp32-elf-
87+
CFLAGS +=
88+
89+
else
90+
$(error architecture '$(ARCH)' not supported)
91+
endif
92+
93+
CFLAGS += $(CFLAGS_EXTRA)
94+
95+
################################################################################
96+
# Build rules
97+
98+
.PHONY: all clean
99+
100+
all: $(MOD).mpy
101+
102+
clean:
103+
$(RM) -rf $(BUILD) $(CLEAN_EXTRA)
104+
105+
# Create build destination directories first
106+
BUILD_DIRS = $(sort $(dir $(CONFIG_H) $(SRC_O) $(SRC_MPY)))
107+
$(CONFIG_H) $(SRC_O) $(SRC_MPY): | $(BUILD_DIRS)
108+
$(BUILD_DIRS):
109+
$(Q)$(MKDIR) -p $@
110+
111+
# Preprocess all source files to generate $(CONFIG_H)
112+
$(CONFIG_H): $(SRC)
113+
$(ECHO) "GEN $@"
114+
$(Q)$(MPY_LD) --arch $(ARCH) --preprocess -o $@ $^
115+
116+
# Build .o from .c source files
117+
$(BUILD)/%.o: %.c $(CONFIG_H) Makefile
118+
$(ECHO) "CC $<"
119+
$(Q)$(CROSS)gcc $(CFLAGS) -o $@ -c $<
120+
121+
# Build .mpy from .py source files
122+
$(BUILD)/%.mpy: %.py
123+
$(ECHO) "MPY $<"
124+
$(Q)$(MPY_CROSS) $(MPY_CROSS_FLAGS) -o $@ $<
125+
126+
# Build native .mpy from object files
127+
$(BUILD)/$(MOD).native.mpy: $(SRC_O)
128+
$(ECHO) "LINK $<"
129+
$(Q)$(MPY_LD) --arch $(ARCH) --qstrs $(CONFIG_H) -o $@ $^
130+
131+
# Build final .mpy from all intermediate .mpy files
132+
$(MOD).mpy: $(BUILD)/$(MOD).native.mpy $(SRC_MPY)
133+
$(ECHO) "GEN $@"
134+
$(Q)$(MPY_TOOL) --merge -o $@ $^

0 commit comments

Comments
 (0)
0