8000 py: Only search frozen modules when '.frozen' is found in sys.path. · micropython/micropython@e0bf461 · GitHub
[go: up one dir, main page]
Skip to content
8000

Commit e0bf461

Browse files
jimmodpgeorge
jimmo
authored and
dpgeorge
committed
py: Only search frozen modules when '.frozen' is found in sys.path.
This changes makemanifest.py & mpy-tool.py to merge string and mpy names into the same list (now mp_frozen_names). The various paths for loading a frozen module (mp_find_frozen_module) and checking existence of a frozen module (mp_frozen_stat) use a common function that searches this list. In addition, the frozen lookup will now only take place if the path starts with ".frozen", which needs to be added to sys.path. This fixes issues #1804, #2322, #3509, #6419. Signed-off-by: Jim Mussared <jim.mussared@gmail.com>
1 parent f853e3e commit e0bf461

File tree

8 files changed

+143
-151
lines changed

8 files changed

+143
-151
lines changed

‎py/builtinhelp.c

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -67,10 +67,10 @@ STATIC void mp_help_add_from_map(mp_obj_t list, const mp_map_t *map) {
6767
#if MICROPY_MODULE_FROZEN
6868
STATIC void mp_help_add_from_names(mp_obj_t list, const char *name) {
6969
while (*name) {
70-
size_t l = strlen(name);
70+
size_t len = strlen(name);
7171
// name should end in '.py' and we strip it off
72-
mp_obj_list_append(list, mp_obj_new_str(name, l - 3));
73-
name += l + 1;
72+
mp_obj_list_append(list, mp_obj_new_str(name, len - 3));
73+
name += len + 1;
7474
}
7575
}
7676
#endif
@@ -80,14 +80,9 @@ STATIC void mp_help_print_modules(void) {
8080

8181
mp_help_add_from_map(list, &mp_builtin_module_map);
8282

83-
#if MICROPY_MODULE_FROZEN_STR
84-
extern const char mp_frozen_str_names[];
85-
mp_help_add_from_names(list, mp_frozen_str_names);
86-
#endif
87-
88-
#if MICROPY_MODULE_FROZEN_MPY
89-
extern const char mp_frozen_mpy_names[];
90-
mp_help_add_from_names(list, mp_frozen_mpy_names);
83+
#if MICROPY_MODULE_FROZEN
84+
extern const char mp_frozen_names[];
85+
mp_help_add_from_names(list, mp_frozen_names);
9186
#endif
9287

9388
// sort the list so it's printed in alphabetical order

‎py/builtinimport.c

Lines changed: 29 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,9 @@
5050
// Must be a string of one byte.
5151
#define PATH_SEP_CHAR "/"
5252

53+
// Virtual sys.path entry that maps to the frozen modules.
54+
#define MP_FROZEN_PATH_PREFIX ".frozen/"
55+
5356
bool mp_obj_is_package(mp_obj_t module) {
5457
mp_obj_t dest[2];
5558
mp_load_method_maybe(module, MP_QSTR___path__, dest);
@@ -62,9 +65,10 @@ bool mp_obj_is_package(mp_obj_t module) {
6265
// will return whether the path is a file, directory, or doesn't exist.
6366
STATIC mp_import_stat_t stat_path_or_frozen(const char *path) {
6467
#if MICROPY_MODULE_FROZEN
65-
mp_import_stat_t st = mp_frozen_stat(path);
66-
if (st != MP_IMPORT_STAT_NO_EXIST) {
67-
return st;
68+
// Only try and load as a frozen module if it starts with .frozen/.
69+
const int frozen_path_prefix_len = strlen(MP_FROZEN_PATH_PREFIX);
70+
if (strncmp(path, MP_FROZEN_PATH_PREFIX, frozen_path_prefix_len) == 0) {
71+
return mp_find_frozen_module(path + frozen_path_prefix_len, NULL, NULL);
6872
}
6973
#endif
7074
return mp_import_stat(path);
@@ -193,32 +197,36 @@ STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code, co
193197

194198
STATIC void do_load(mp_obj_t module_obj, vstr_t *file) {
195199
#if MICROPY_MODULE_FROZEN || MICROPY_ENABLE_COMPILER || (MICROPY_PERSISTENT_CODE_LOAD && MICROPY_HAS_FILE_READER)
196-
char *file_str = vstr_null_terminated_str(file);
200+
const char *file_str = vstr_null_terminated_str(file);
197201
#endif
198202

199203
// If we support frozen modules (either as str or mpy) then try to find the
200204
// requested filename in the list of frozen module filenames.
201205
#if MICROPY_MODULE_FROZEN
202206
void *modref;
203-
int frozen_type = mp_find_frozen_module(file_str, file->len, &modref);
204-
205-
// If we support frozen str modules and the compiler is enabled, and we
206-
// found the filename in the list of frozen files, then load and execute it.
207-
#if MICROPY_MODULE_FROZEN_STR
208-
if (frozen_type == MP_FROZEN_STR) {
209-
do_load_from_lexer(module_obj, modref);
210-
return;
211-
}
212-
#endif
207+
int frozen_type;
208+
const int frozen_path_prefix_len = strlen(MP_FROZEN_PATH_PREFIX);
209+
if (strncmp(file_str, MP_FROZEN_PATH_PREFIX, frozen_path_prefix_len) == 0) {
210+
mp_find_frozen_module(file_str + frozen_path_prefix_len, &frozen_type, &modref);
211+
212+
// If we support frozen str modules and the compiler is enabled, and we
213+
// found the filename in the list of frozen files, then load and execute it.
214+
#if MICROPY_MODULE_FROZEN_STR
215+
if (frozen_type == MP_FROZEN_STR) {
216+
do_load_from_lexer(module_obj, modref);
217+
return;
218+
}
219+
#endif
213220

214-
// If we support frozen mpy modules and we found a corresponding file (and
215-
// its data) in the list of frozen files, execute it.
216-
#if MICROPY_MODULE_FROZEN_MPY
217-
if (frozen_type == MP_FROZEN_MPY) {
218-
do_execute_raw_code(module_obj, modref, file_str);
219-
return;
221+
// If we support frozen mpy modules and we found a corresponding file (and
222+
// its data) in the list of frozen files, execute it.
223+
#if MICROPY_MODULE_FROZEN_MPY
224+
if (frozen_type == MP_FROZEN_MPY) {
225+
do_execute_raw_code(module_obj, modref, file_str + frozen_path_prefix_len);
226+
return;
227+
}
228+
#endif
220229
}
221-
#endif
222230

223231
#endif // MICROPY_MODULE_FROZEN
224232

‎py/frozenmod.c

Lines changed: 71 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
*
66
* Copyright (c) 2015 Paul Sokolovsky
77
* Copyright (c) 2016 Damien P. George
8+
* Copyright (c) 2021 Jim Mussared
89
*
910
* Permission is hereby granted, free of charge, to any person obtaining a copy
1011
* of this software and associated documentation files (the "Software"), to deal
@@ -31,6 +32,13 @@
3132
#include "py/lexer.h"
3233
#include "py/frozenmod.h"
3334

35+
#if MICROPY_MODULE_FROZEN
36+
37+
// Null-separated frozen file names. All string-type entries are listed first,
38+
// followed by mpy-type entries. Use mp_frozen_str_sizes to determine how
39+
// many string entries.
40+
extern const char mp_frozen_names[];
41+
3442
#if MICROPY_MODULE_FROZEN_STR
3543

3644
#ifndef MICROPY_MODULE_FROZEN_LEXER
@@ -39,118 +47,89 @@
3947
mp_lexer_t *MICROPY_MODULE_FROZEN_LEXER(qstr src_name, const char *str, mp_uint_t len, mp_uint_t free_len);
4048
#endif
4149

42-
extern const char mp_frozen_str_names[];
50+
// Size in bytes of each string entry, followed by a zero (terminator).
4351
extern const uint32_t mp_frozen_str_sizes[];
52+
// Null-separated string content.
4453
extern const char mp_frozen_str_content[];
45-
46-
// On input, *len contains size of name, on output - size of content
47-
const char *mp_find_frozen_str(const char *str, size_t *len) {
48-
const char *name = mp_frozen_str_names;
49-
50-
size_t offset = 0;
51-
for (int i = 0; *name != 0; i++) {
52-
size_t l = strlen(name);
53-
if (l == *len && !memcmp(str, name, l)) {
54-
*len = mp_frozen_str_sizes[i];
55-
return mp_frozen_str_content + offset;
56-
}
57-
name += l + 1;
58-
offset += mp_frozen_str_sizes[i] + 1;
59-
}
60-
return NULL;
61-
}
62-
63-
STATIC mp_lexer_t *mp_lexer_frozen_str(const char *str, size_t len) {
64-
size_t name_len = len;
65-
const char *content = mp_find_frozen_str(str, &len);
66-
67-
if (content == NULL) {
68-
return NULL;
69-
}
70-
71-
qstr source = qstr_from_strn(str, name_len);
72-
mp_lexer_t *lex = MICROPY_MODULE_FROZEN_LEXER(source, content, len, 0);
73-
return lex;
74-
}
75-
76-
#endif
54+
#endif // MICROPY_MODULE_FROZEN_STR
7755

7856
#if MICROPY_MODULE_FROZEN_MPY
7957

8058
#include "py/emitglue.h"
8159

82-
extern const 6D4E char mp_frozen_mpy_names[];
8360
extern const mp_raw_code_t *const mp_frozen_mpy_content[];
8461

85-
STATIC const mp_raw_code_t *mp_find_frozen_mpy(const char *str, size_t len) {
86-
const char *name = mp_frozen_mpy_names;
87-
for (size_t i = 0; *name != 0; i++) {
88-
size_t l = strlen(name);
89-
if (l == len && !memcmp(str, name, l)) {
90-
return mp_frozen_mpy_content[i];
91-
}
92-
name += l + 1;
93-
}
94-
return NULL;
95-
}
62+
#endif // MICROPY_MODULE_FROZEN_MPY
9663

97-
#endif
64+
// Search for "str" as a frozen entry, returning the stat result
65+
// (no-exist/file/dir), as well as the type (none/str/mpy) and data.
66+
// frozen_type can be NULL if its value isn't needed (and then data is assumed to be NULL).
67+
mp_import_stat_t mp_find_frozen_module(const char *str, int *frozen_type, void **data) {
68+
size_t len = strlen(str);
69+
const char *name = mp_frozen_names;
9870

99-
#if MICROPY_MODULE_FROZEN
71+
if (frozen_type != NULL) {
72+
*frozen_type = MP_FROZEN_NONE;
73+
}
10074

101-
STATIC mp_import_stat_t mp_frozen_stat_helper(const char *name, const char *str) {
102-
size_t len = strlen(str);
75+
// Count the number of str lengths we have to find how many str entries.
76+
size_t num_str = 0;
77+
#if MICROPY_MODULE_FROZEN_STR && MICROPY_MODULE_FROZEN_MPY
78+
for (const uint32_t *s = mp_frozen_str_sizes; *s != 0; ++s) {
79+
++num_str;
80+
}
81+
#endif
82+
83+
for (size_t i = 0; *name != 0; i++) {
84+
size_t entry_len = strlen(name);
85+
if (entry_len >= len && memcmp(str, name, len) == 0) {
86+
// Query is a prefix of the current entry.
87+
if (entry_len == len) {
88+
// Exact match --> file.
89+
90+
if (frozen_type != NULL) {
91+
#if MICROPY_MODULE_FROZEN_STR
92+
if (i < num_str) {
93+
*frozen_type = MP_FROZEN_STR;
94+
// Use the size table to figure out where this index starts.
95+
size_t offset = 0;
96+
for (size_t j = 0; j < i; ++j) {
97+
offset += mp_frozen_str_sizes[j] + 1;
98+
}
99+
size_t content_len = mp_frozen_str_sizes[i];
100+
const char *content = &mp_frozen_str_content[offset];
101+
102+
// Note: str & len have been updated by find_frozen_entry to strip
103+
// the ".frozen/" prefix (to avoid this being a distinct qstr to
104+
// the original path QSTR in frozen_content.c).
105+
qstr source = qstr_from_strn(str, len);
106+
mp_lexer_t *lex = MICROPY_MODULE_FROZEN_LEXER(source, content, content_len, 0);
107+
*data = lex;
108+
}
109+
#endif
110+
111+
#if MICROPY_MODULE_FROZEN_MPY
112+
if (i >= num_str) {
113+
*frozen_type = MP_FROZEN_MPY;
114+
// Load the corresponding index as a raw_code, taking
115+
// into account any string entries to offset by.
116+
*data = (void *)mp_frozen_mpy_content[i - num_str];
117+
}
118+
#endif
119+
}
103120

104-
for (int i = 0; *name != 0; i++) {
105-
size_t l = strlen(name);
106-
if (l >= len && !memcmp(str, name, len)) {
107-
if (name[len] == 0) {
108121
return MP_IMPORT_STAT_FILE;
109122
} else if (name[len] == '/') {
123+
// Matches up to directory separator, this is a valid
124+
// directory path.
110125
return MP_IMPORT_STAT_DIR;
111126
}
112127
}
113-
name += l + 1;
114-
}
115-
return MP_IMPORT_STAT_NO_EXIST;
116-
}
117-
118-
mp_import_stat_t mp_frozen_stat(const char *str) {
119-
mp_import_stat_t stat;
120-
121-
#if MICROPY_MODULE_FROZEN_STR
122-
stat = mp_frozen_stat_helper(mp_frozen_str_names, str);
123-
if (stat != MP_IMPORT_STAT_NO_EXIST) {
124-
return stat;
128+
// Skip null separator.
129+
name += entry_len + 1;
125130
}
126-
#endif
127-
128-
#if MICROPY_MODULE_FROZEN_MPY
129-
stat = mp_frozen_stat_helper(mp_frozen_mpy_names, str);
130-
if (stat != MP_IMPORT_STAT_NO_EXIST) {
131-
return stat;
132-
}
133-
#endif
134131

135132
return MP_IMPORT_STAT_NO_EXIST;
136133
}
137134

138-
int mp_find_frozen_module(const char *str, size_t len, void **data) {
139-
#if MICROPY_MODULE_FROZEN_STR
140-
mp_lexer_t *lex = mp_lexer_frozen_str(str, len);
141-
if (lex != NULL) {
142-
*data = lex;
143-
return MP_FROZEN_STR;
144-
}
145-
#endif
146-
#if MICROPY_MODULE_FROZEN_MPY
147-
const mp_raw_code_t *rc = mp_find_frozen_mpy(str, len);
148-
if (rc != NULL) {
149-
*data = (void *)rc;
150-
return MP_FROZEN_MPY;
151-
}
152-
#endif
153-
return MP_FROZEN_NONE;
154-
}
155-
156-
#endif
135+
#endif // MICROPY_MODULE_FROZEN

‎py/frozenmod.h

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,6 @@ enum {
3535
MP_FROZEN_MPY,
3636
};
3737

38-
int mp_find_frozen_module(const char *str, size_t len, void **data);
39-
const char *mp_find_frozen_str(const char *str, size_t *len);
40-
mp_import_stat_t mp_frozen_stat(const char *str);
38+
mp_import_stat_t mp_find_frozen_module(const char *str, int *frozen_type, void **data);
4139

4240
#endif // MICROPY_INCLUDED_PY_FROZENMOD_H

‎py/qstrdefs.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,10 @@ Q(<string>)
6060
Q(<stdin>)
6161
Q(utf-8)
6262

63+
#if MICROPY_MODULE_FROZEN
64+
Q(.frozen)
65+
#endif
66+
6367
#if MICROPY_ENABLE_PYSTACK
6468
Q(pystack exhausted)
6569
#endif

‎shared/runtime/pyexec.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -674,7 +674,7 @@ int pyexec_file(const char *filename) {
674674

675675
int pyexec_file_if_exists(const char *filename) {
676676
#if MICROPY_MODULE_FROZEN
677-
if (mp_frozen_stat(filename) == MP_IMPORT_STAT_FILE) {
677+
if (mp_find_frozen_module(filename, NULL, NULL) == MP_IMPORT_STAT_FILE) {
678678
return pyexec_frozen_module(filename);
679679
}
680680
#endif
@@ -687,7 +687,8 @@ int pyexec_file_if_exists(const char *filename) {
687687
#if MICROPY_MODULE_FROZEN
688688
int pyexec_frozen_module(const char *name) {
689689
void *frozen_data;
690-
int frozen_type = mp_find_frozen_module(name, strlen(name), &frozen_data);
690+
int frozen_type;
691+
mp_find_frozen_module(name, &frozen_type, &frozen_data);
691692

692693
switch (frozen_type) {
693694
#if MICROPY_MODULE_FROZEN_STR

0 commit comments

Comments
 (0)
0