8000 py/persistentcode: Add mp_raw_code_save_fun_to_bytes. · sparkfun/micropython@a11ba77 · GitHub
[go: up one dir, main page]
Skip to content < 8000 script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/keyboard-shortcuts-dialog-cf9f9950f389.js" defer="defer">

Commit a11ba77

Browse files
dpgeorgedpgeorge
committed
py/persistentcode: Add mp_raw_code_save_fun_to_bytes.
Serialises a bytecode function/generator to a valid .mpy as bytes. Signed-off-by: Damien George <damien@micropython.org>
1 parent ceb8ba6 commit a11ba77

File tree

3 files changed

+194
-2
lines changed

3 files changed

+194
-2
lines changed

‎py/mpconfig.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,11 @@
342342
#define MICROPY_PERSISTENT_CODE_SAVE_FILE (0)
343343
#endif
344344

345+
// Whether to support converting functions to persistent code (bytes)
346+
#ifndef MICROPY_PERSISTENT_CODE_SAVE_FUN
347+
#define MICROPY_PERSISTENT_CODE_SAVE_FUN (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING)
348+
#endif
349+
345350
// Whether generated code can persist independently of the VM/runtime instance
346351
// This is enabled automatically when needed by other features
347352
#ifndef MICROPY_PERSISTENT_CODE

‎py/persistentcode.c

Lines changed: 188 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -526,7 +526,7 @@ void mp_raw_code_load_file(qstr filename, mp_compiled_module_t *context) {
526526

527527
#endif // MICROPY_PERSISTENT_CODE_LOAD
528528

529-
#if MICROPY_PERSISTENT_CODE_SAVE
529+
#if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_PERSISTENT_CODE_SAVE_FUN
530530

531531
#include "py/objstr.h"
532532

@@ -624,6 +624,10 @@ static void save_obj(mp_print_t *print, mp_obj_t o) {
624624
}
625625
}
626626

627+
#endif // MICROPY_PERSISTENT_CODE_SAVE || MICROPY_PERSISTENT_CODE_SAVE_FUN
628+
629+
#if MICROPY_PERSISTENT_CODE_SAVE
630+
627631
static void save_raw_code(mp_print_t *print, const mp_raw_code_t *rc) {
628632
// Save function kind and data length
629633
mp_print_uint(print, (rc->fun_data_len << 3) | ((rc->n_children != 0) << 2) | (rc->kind - MP_CODE_BYTECODE));
@@ -693,6 +697,8 @@ void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print) {
693697
save_raw_code(print, cm->rc);
694698
}
695699

700+
#endif // MICROPY_PERSISTENT_CODE_SAVE
701+
696702
#if MICROPY_PERSISTENT_CODE_SAVE_FILE
697703

698704
#include <unistd.h>
@@ -723,4 +729,184 @@ void mp_raw_code_save_file(mp_compiled_module_t *cm, qstr filename) {
723729

724730
#endif // MICROPY_PERSISTENT_CODE_SAVE_FILE
725731

726-
#endif // MICROPY_PERSISTENT_CODE_SAVE
732+
#if MICROPY_PERSISTENT_CODE_SAVE_FUN
733+
734+
#include "py/bc0.h"
735+
#include "py/objfun.h"
736+
#include "py/smallint.h"
737+
#include "py/gc.h"
738+
739+
#define MP_BC_OPCODE_HAS_SIGNED_OFFSET(opcode) (MP_BC_UNWIND_JUMP <= (opcode) && (opcode) <= MP_BC_POP_JUMP_IF_FALSE)
740+
741+
typedef struct _bit_vector_t {
742+
size_t max_bit_set;
743+
size_t alloc;
744+
uintptr_t *bits;
745+
} bit_vector_t;
746+
747+
static void bit_vector_init(bit_vector_t *self) {
748+
self->max_bit_set = 0;
749+
self->alloc = 1;
750+
self->bits = m_new(uintptr_t, self->alloc);
751+
}
752+
753+
static void bit_vector_clear(bit_vector_t *self) {
754+
m_del(uintptr_t, self->bits, self->alloc);
755+
}
756+
757+
static bool bit_vector_is_set(bit_vector_t *self, size_t index) {
758+
const size_t bits_size = sizeof(*self->bits) * MP_BITS_PER_BYTE;
759+
return index / bits_size < self->alloc
760+
&& (self->bits[index / bits_size] & (1 << (index % bits_size))) != 0;
761+
}
762+
763+
static void bit_vector_set(bit_vector_t *self, size_t index) {
764+
const size_t bits_size = sizeof(*self->bits) * MP_BITS_PER_BYTE;
765+
self->max_bit_set = MAX(self->max_bit_set, index);
766+
if (index / bits_size >= self->alloc) {
767+
size_t new_alloc = self->alloc * 2;
768+
self->bits = m_renew(uintptr_t, self->bits, self->alloc, new_alloc);
769+
self->alloc = new_alloc;
770+
}
771+
self->bits[index / bits_size] |= 1 << (index % bits_size);
772+
}
773+
774+
typedef struct _mp_opcode_t {
775+
uint8_t opcode;
776+
uint8_t format;
777+
uint8_t size;
778+
mp_int_t arg;
779+
uint8_t extra_arg;
780+
} mp_opcode_t;
781+
782+
static mp_opcode_t mp_opcode_decode(const uint8_t *ip) {
783+
const uint8_t *ip_start = ip;
784+
uint8_t opcode = *ip++;
785+
uint8_t opcode_format = MP_BC_FORMAT(opcode);
786+
mp_uint_t arg = 0;
787+
uint8_t extra_arg = 0;
788+
if (opcode_format == MP_BC_FORMAT_QSTR || opcode_format == MP_BC_FORMAT_VAR_UINT) {
789+
arg = *ip & 0x7f;
790+
if (opcode == MP_BC_LOAD_CONST_SMALL_INT && (arg & 0x40) != 0) {
791+
arg |= (mp_uint_t)(-1) << 7;
792+
}
793+
while ((*ip & 0x80) != 0) {
794+
arg = (arg << 7) | (*++ip & 0x7f);
795+
}
796+
++ip;
797+
} else if (opcode_format == MP_BC_FORMAT_OFFSET) {
798+
if ((*ip & 0x80) == 0) {
799+
arg = *ip++;
800+
if (MP_BC_OPCODE_HAS_SIGNED_OFFSET(opcode)) {
801+
arg -= 0x40;
802+
}
803+
} else {
804+
arg = (ip[0] & 0x7f) | (ip[1] << 7);
805+
ip += 2;
806+
if (MP_BC_OPCODE_HAS_SIGNED_OFFSET(opcode)) {
807+
arg -= 0x4000;
808+
}
809+
}
810+
}
811+
if ((opcode & MP_BC_MASK_EXTRA_BYTE) == 0) {
812+
extra_arg = *ip++;
813+
}
814+
815+
mp_opcode_t op = { opcode, opcode_format, ip - ip_start, arg, extra_arg };
816+
return op;
817+
}
818+
819+
mp_obj_t mp_raw_code_save_fun_to_bytes(const mp_module_constants_t *consts, const uint8_t *bytecode) {
820+
const uint8_t *fun_data = bytecode;
821+
const uint8_t *fun_data_top = fun_data + gc_nbytes(fun_data);
822+
823+
// Extract function information.
824+
const byte *ip = fun_data;
825+
MP_BC_PRELUDE_SIG_DECODE(ip);
826+
MP_BC_PRELUDE_SIZE_DECODE(ip);
827+
828+
// Track the qstrs used by the function.
829+
bit_vector_t qstr_table_used;
830+
bit_vector_init(&qstr_table_used);
831+
832+
// Track the objects used by the function.
833+
bit_vector_t obj_table_used;
834+
bit_vector_init(&obj_table_used);
835+
836+
const byte *ip_names = ip;
837+
mp_uint_t simple_name = mp_decode_uint(&ip_names);
838+
bit_vector_set(&qstr_table_used, simple_name);
839+
for (size_t i = 0; i < n_pos_args + n_kwonly_args; ++i) {
840+
mp_uint_t arg_name = mp_decode_uint(&ip_names);
841+
bit_vector_set(&qstr_table_used, arg_name);
842+
}
843+
844+
// Skip pass source code info and cell info.
845+
// Then ip points to the start of the opcodes.
846+
ip += n_info + n_cell;
847+
848+
// Decode bytecode.
849+
while (ip < fun_data_top) {
850+
mp_opcode_t op = mp_opcode_decode(ip);
851+
if (op.opcode == MP_BC_BASE_RESERVED) {
852+
// End of opcodes.
853+
fun_data_top = ip;
854+
} else if (op.opcode == MP_BC_LOAD_CONST_OBJ) {
855+
bit_vector_set(&obj_table_used, op.arg);
856+
} else if (op.format == MP_BC_FORMAT_QSTR) {
857+
bit_vector_set(&qstr_table_used, op.arg);
858+
}
859+
ip += op.size;
860+
}
861+
862+
mp_uint_t fun_data_len = fun_data_top - fun_data;
863+
864+
mp_print_t print;
865+
vstr_t vstr;
866+
vstr_init_print(&vstr, 64, &print);
867+
868+
// Start with .mpy header.
869+
const uint8_t header[4] = { 'M', MPY_VERSION, 0, MP_SMALL_INT_BITS };
870+
mp_print_bytes(&print, header, sizeof(header));
871+
872+
// Number of entries in constant table.
873+
mp_print_uint(&print, qstr_table_used.max_bit_set + 1);
874+
mp_print_uint(&print, obj_table_used.max_bit_set + 1);
875+
876+
// Save qstrs.
877+
for (size_t i = 0; i <= qstr_table_used.max_bit_set; ++i) {
878+
if (bit_vector_is_set(&qstr_table_used, i)) {
879+
save_qstr(&print, consts->qstr_table[i]);
880+
} else {
881+
save_qstr(&print, MP_QSTR_);
882+
}
883+
}
884+
885+
// Save constant objects.
886+
for (size_t i = 0; i <= obj_table_used.max_bit_set; ++i) {
887+
if (bit_vector_is_set(&obj_table_used, i)) {
888+
save_obj(&print, consts->obj_table[i]);
889+
} else {
890+
save_obj(&print, mp_const_none);
891+
}
892+
}
893+
894+
bit_vector_clear(&qstr_table_used);
895+
bit_vector_clear(&obj_table_used);
896+
897+
// Save function kind and data length.
898+
mp_print_uint(&print, fun_data_len << 3);
899+
900+
// Save function code.
901+
mp_print_bytes(&print, fun_data, fun_data_len);
902+
903+
// Create and return bytes representing the .mpy data.
904+
return mp_obj_new_bytes_from_vstr(&vstr);
905+
}
906+
907+
#endif // MICROPY_PERSISTENT_CODE_SAVE_FUN
908+
909+
#if MICROPY_PERSISTENT_CODE_TRACK_RELOC_CODE
910+
// An mp_obj_list_t that tracks relocated native code to prevent the GC from reclaiming them.
911+
MP_REGISTER_ROOT_POINTER(mp_obj_t track_reloc_code_list);
912+
#endif

‎py/persistentcode.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@ void mp_raw_code_load_file(qstr filename, mp_compiled_module_t *ctx);
121121

122122
void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print);
123123
void mp_raw_code_save_file(mp_compiled_module_t *cm, qstr filename);
124+
mp_obj_t mp_raw_code_save_fun_to_bytes(const mp_module_constants_t *consts, const uint8_t *bytecode);
124125

125126
void mp_native_relocate(void *reloc, uint8_t *text, uintptr_t reloc_text);
126127

0 commit comments

Comments
 (0)
0