8000 WIP: py/persistentcode: Run .mpy from RAM without copying. · pybricks/micropython@f367bb7 · GitHub
[go: up one dir, main page]
Skip to content

Commit f367bb7

Browse files
laurensvalklaurensvalk
committed
WIP: py/persistentcode: Run .mpy from RAM without copying.
This is a partial implementation of MicroPython micropython#8381. We could use it until that PR gets merged upstream, or use this as a smaller solution since it doesn't require the various extmod modules. We can use something like this in our minimal port-variant. This allows us to run an .mpy blob from RAM without making a near-copy. This way we don't have to use free_len to free up the original blob, so the user can run it again without re-uploading.
1 parent 0237366 commit f367bb7

File tree

4 files changed

+103
-48
lines changed

4 files changed

+103
-48
lines changed

‎py/persistentcode.c

Lines changed: 52 additions & 12 deletions
10000
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,15 @@ void mp_native_relocate(void *ri_in, uint8_t *text, uintptr_t reloc_text) {
133133

134134
#endif
135135

136+
#if MICROPY_VFS_BLOB
137+
STATIC inline bool reader_is_map(mp_reader_t *reader) {
138+
return reader->readchunk != NULL;
139+
}
140+
STATIC inline const uint8_t *map_read_bytes(mp_reader_t *reader, size_t len) {
141+
return reader->readchunk(reader->data, len);
142+
}
143+
#endif
144+
136145
STATIC int read_byte(mp_reader_t *reader) {
137146
return reader->readbyte(reader->data);
138147
}
@@ -162,6 +171,11 @@ STATIC qstr load_qstr(mp_reader_t *reader) {
162171
return len >> 1;
163172
}
164173
len >>= 1;
174+
#if MICROPY_VFS_BLOB
175+
if (reader_is_map(reader)) {
176+
return qstr_from_strn_static((const char *)map_read_bytes(reader, len + 1), len);
177+
}
178+
#endif
165179
char *str = m_new(char, len);
166180
read_bytes(reader, (byte *)str, len);
167181
read_byte(reader); // read and discard null terminator
@@ -170,6 +184,23 @@ STATIC qstr load_qstr(mp_reader_t *reader) {
170184
return qst;
171185
}
172186

187+
#if MICROPY_VFS_BLOB
188+
STATIC mp_obj_t mp_obj_new_str_static(const mp_obj_type_t *type, const byte *data, size_t len) {
189+
if (type == &mp_type_str) {
190+
qstr q = qstr_find_strn((const char *)data, len);
191+
if (q != MP_QSTRnull) {
192+
return MP_OBJ_NEW_QSTR(q);
193+
}
194+
}
195+
mp_obj_str_t *o = m_new_obj(mp_obj_str_t);
196+
o->base.type = type;
197+
o->len = len;
198+
o->hash = qstr_compute_hash(data, len);
199+
o->data = data;
200+
return MP_OBJ_FROM_PTR(o);
201+
}
202+
#endif
203+
173204
STATIC mp_obj_t load_obj(mp_reader_t *reader) {
174205
byte obj_type = read_byte(reader);
175206
#if MICROPY_EMIT_MACHINE_CODE
@@ -190,13 +221,15 @@ STATIC mp_obj_t load_obj(mp_reader_t *reader) {
190221
if (len == 0 && obj_type == MP_PERSISTENT_OBJ_BYTES) {
191222
read_byte(reader); // skip null terminator
192223
return mp_const_empty_bytes;
193-
} else if (obj_type == MP_PERSISTENT_OBJ_TUPLE) {
194-
mp_obj_tuple_t *tuple = MP_OBJ_TO_PTR(mp_obj_new_tuple(len, NULL));
195-
for (size_t i = 0; i < len; ++i) {
196-
tuple->items[i] = load_obj(reader);
197-
}
198-
return MP_OBJ_FROM_PTR(tuple);
199224
}
225+
226+
#if MICROPY_VFS_BLOB
227+
if (reader_is_map(reader)) {
228+
const uint8_t *data = map_read_bytes(reader, len + 1);
229+
return mp_obj_new_str_static(obj_type == MP_PERSISTENT_OBJ_STR ? &mp_type_str : &mp_type_bytes, data, len);
230+
}
231+
#endif
232+
200233
vstr_t vstr;
201234
vstr_init_len(&vstr, len);
202235
read_bytes(reader, (byte *)vstr.buf, len);
@@ -225,7 +258,7 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co
225258
}
226259
#endif
227260

228-
uint8_t *fun_data = NULL;
261+
const uint8_t *fun_data = NULL;
229262
#if MICROPY_EMIT_MACHINE_CODE
230263
size_t prelude_offset = 0;
231264
mp_uint_t native_scope_flags = 0;
@@ -234,11 +267,18 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co
234267
#endif
235268

236269
if (kind == MP_CODE_BYTECODE) {
237-
// Allocate memory for the bytecode
238-
fun_data = m_new(uint8_t, fun_data_len);
239-
// Load bytecode
240-
read_bytes(reader, fun_data, fun_data_len);
241-
270+
#if MICROPY_VFS_BLOB
271+
if (reader_is_map(reader)) {
272+
fun_data = map_read_bytes(reader, fun_data_len);
273+
} else
274+
#endif
275+
{
276+
// Allocate memory for the bytecode
277+
uint8_t *data = m_new(uint8_t, fun_data_len);
278+
// Load bytecode
279+
read_bytes(reader, data, fun_data_len);
280+
fun_data = data;
281+
}
242282
#if MICROPY_EMIT_MACHINE_CODE
243283
} else {
244284
// Allocate memory for native data and load it

‎py/qstr.c

Lines changed: 49 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,7 @@ qstr qstr_from_str(const char *str) {
201201
return qstr_from_strn(str, strlen(str));
202202
}
203203

204-
qstr qstr_from_strn(const char *str, size_t len) {
204+
STATIC qstr qstr_from_strn_helper(const char *str, size_t len, bool is_static) {
205205
QSTR_ENTER();
206206
qstr q = qstr_find_strn(str, len);
207207
if (q == 0) {
@@ -213,56 +213,69 @@ qstr qstr_from_strn(const char *str, size_t len) {
213213
mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("name too long"));
214214
}
215215

216-
// compute number of bytes needed to intern this string
217-
size_t n_bytes = len + 1;
218-
219-
if (MP_STATE_VM(qstr_last_chunk) != NULL && MP_STATE_VM(qstr_last_used) + n_bytes > MP_STATE_VM(qstr_last_alloc)) {
220-
// not enough room at end of previously interned string so try to grow
221-
char *new_p = m_renew_maybe(char, MP_STATE_VM(qstr_last_chunk), MP_STATE_VM(qstr_last_alloc), MP_STATE_VM(qstr_last_alloc) + n_bytes, false);
222-
if (new_p == NULL) {
223-
// could not grow existing memory; shrink it to fit previous
224-
(void)m_renew_maybe(char, MP_STATE_VM(qstr_last_chunk), MP_STATE_VM(qstr_last_alloc), MP_STATE_VM(qstr_last_used), false);
225-
MP_STATE_VM(qstr_last_chunk) = NULL;
226-
} else {
227-
// could grow existing memory
228-
MP_STATE_VM(qstr_last_alloc) += n_bytes;
216+
if (is_static) {
217+
assert(str[len] == '\0');
218+
} else {
219+
// compute number of bytes needed to intern this string
220+
size_t n_bytes = len + 1;
221+
222+
if (MP_STATE_VM(qstr_last_chunk) != NULL && MP_STATE_VM(qstr_last_used) + n_bytes > MP_STATE_VM(qstr_last_alloc)) {
223+
// not enough room at end of previously interned string so try to grow
224+
char *new_p = m_renew_maybe(char, MP_STATE_VM(qstr_last_chunk), MP_STATE_VM(qstr_last_alloc), MP_STATE_VM(qstr_last_alloc) + n_bytes, false);
225+
if (new_p == NULL) {
226+
// could not grow existing memory; shrink it to fit previous
227+
(void)m_renew_maybe(char, MP_STATE_VM(qstr_last_chunk), MP_STATE_VM(qstr_last_alloc), MP_STATE_VM(qstr_last_used), false);
228+
MP_STATE_VM(qstr_last_chunk) = NULL;
229+
} else {
230+
// could grow existing memory
231+
MP_STATE_VM(qstr_last_alloc) += n_bytes;
232+
}
229233
}
230-
}
231234

232-
if (MP_STATE_VM(qstr_last_chunk) == NULL) {
233-
// no existing memory for the interned string so allocate a new chunk
234-
size_t al = n_bytes;
235-
if (al < MICROPY_ALLOC_QSTR_CHUNK_INIT) {
236-
al = MICROPY_ALLOC_QSTR_CHUNK_INIT;
237-
}
238-
MP_STATE_VM(qstr_last_chunk) = m_new_maybe(char, al);
239235
if (MP_STATE_VM(qstr_last_chunk) == NULL) {
240-
// failed to allocate a large chunk so try with exact size
241-
MP_STATE_VM(qstr_last_chunk) = m_new_maybe(char, n_bytes);
236+
// no existing memory for the interned string so allocate a new chunk
237+
size_t al = n_bytes;
238+
if (al < MICROPY_ALLOC_QSTR_CHUNK_INIT) {
239+
al = MICROPY_ALLOC_QSTR_CHUNK_INIT;
240+
}
241+
MP_STATE_VM(qstr_last_chunk) = m_new_maybe(char, al);
242242
if (MP_STATE_VM(qstr_last_chunk) == NULL) {
243-
QSTR_EXIT();
244-
m_malloc_fail(n_bytes);
243+
// failed to allocate a large chunk so try with exact size
244+
MP_STATE_VM(qstr_last_chunk) = m_new_maybe(char, n_bytes);
245+
if (MP_STATE_VM(qstr_last_chunk) == NULL) {
246+
QSTR_EXIT();
247+
m_malloc_fail(n_bytes);
248+
}
249+
al = n_bytes;
245250
}
246-
al = n_bytes;
251+
MP_STATE_VM(qstr_last_alloc) = al;
252+
MP_STATE_VM(qstr_last_used) = 0;
247253
}
248-
MP_STATE_VM(qstr_last_alloc) = al;
249-
MP_STATE_VM(qstr_last_used) = 0;
250-
}
251254

252-
// allocate memory from the chunk for this new interned string's data
253-
char *q_ptr = MP_STATE_VM(qstr_last_chunk) + MP_STATE_VM(qstr_last_used);
254-
MP_STATE_VM(qstr_last_used) += n_bytes;
255+
// allocate memory from the chunk for this new interned string's data
256+
char *q_ptr = MP_STATE_VM(qstr_last_chunk) + MP_STATE_VM(qstr_last_used);
257+
MP_STATE_VM(qstr_last_used) += n_bytes;
258+
259+
memcpy(q_ptr, str, len);
260+
q_ptr[len] = '\0';
261+
str = q_ptr;
262+
}
255263

256264
// store the interned strings' data
257265
mp_uint_t hash = qstr_compute_hash((const byte *)str, len);
258-
memcpy(q_ptr, str, len);
259-
q_ptr[len] = '\0';
260-
q = qstr_add(hash, len, q_ptr);
266+
q = qstr_add(hash, len, str);
261267
}
262268
QSTR_EXIT();
263269
return q;
264270
}
265271

272+
qstr qstr_from_strn(const char *str, size_t len) {
273+
return qstr_from_strn_helper(str, len, false);
274+
}
275+
qstr qstr_from_strn_static(const char *str, size_t len) {
276+
return qstr_from_strn_helper(str, len, true);
277+
}
278+
266279
mp_uint_t qstr_hash(qstr q) {
267280
const qstr_pool_t *pool = find_qstr(&q);
268281
return pool->hashes[q];

‎py/qstr.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ qstr qstr_find_strn(const char *str, size_t str_len); // returns MP_QSTRnull if
8383

8484
qstr qstr_from_str(const char *str);
8585
qstr qstr_from_strn(const char *str, size_t len);
86+
qstr qstr_from_strn_static(const char *str, size_t len);
8687

8788
mp_uint_t qstr_hash(qstr q);
8889
const char *qstr_str(qstr q);

‎py/reader.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
typedef struct _mp_reader_t {
3737
void *data;
3838
mp_uint_t (*readbyte)(void *data);
39+
const uint8_t *(*readchunk)(void *data, size_t len);
3940
void (*close)(void *data);
4041
} mp_reader_t;
4142

0 commit comments

Comments
 (0)
0