diff --git a/mpy-cross/mpy_cross/__init__.py b/mpy-cross/mpy_cross/__init__.py new file mode 100644 index 0000000000000..d4c0930bb9198 --- /dev/null +++ b/mpy-cross/mpy_cross/__init__.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# +# This file is part of the MicroPython project, http://micropython.org/ +# +# The MIT License (MIT) +# +# Copyright (c) 2022 Andrew Leech +# Copyright (c) 2022 Jim Mussared +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +from __future__ import print_function +import os +import stat +import subprocess + +NATIVE_ARCH_X86 = "x86" +NATIVE_ARCH_X64 = "x64" +NATIVE_ARCH_ARMV6 = "armv6" +NATIVE_ARCH_ARMV6M = "armv6m" +NATIVE_ARCH_ARMV7M = "armv7m" +NATIVE_ARCH_ARMV7EM = "armv7em" +NATIVE_ARCH_ARMV7EMSP = "armv7emsp" +NATIVE_ARCH_ARMV7EMDP = "armv7emdp" +NATIVE_ARCH_XTENSA = "xtensa" +NATIVE_ARCH_XTENSAWIN = "xtensawin" + +NATIVE_ARCHS = [ + NATIVE_ARCH_X86, + NATIVE_ARCH_X64, + NATIVE_ARCH_ARMV6, + NATIVE_ARCH_ARMV6M, + NATIVE_ARCH_ARMV7M, + NATIVE_ARCH_ARMV7EM, + NATIVE_ARCH_ARMV7EMSP, + NATIVE_ARCH_ARMV7EMDP, + NATIVE_ARCH_XTENSA, + NATIVE_ARCH_XTENSAWIN, +] + +__all__ = ["compile", "run", "CrossCompileError"] + + +class CrossCompileError(Exception): + pass + + +def find_mpy_cross_binary(mpy_cross): + if mpy_cross: + return mpy_cross + return os.path.abspath(os.path.join(os.path.dirname(__file__), "../mpy-cross")) + + +def compile(src, dest=None, src_path=None, opt=None, march=None, mpy_cross=None, extra_args=None): + if not src: + raise ValueError("src is required") + if not os.path.exists(src): + raise CrossCompileError("Input .py file not found: {}.".format(src_py)) + + args = [] + + if src_path: + args += ["-s", src_path] + + if dest: + args += ["-o", dest] + + if march: + args += ["-march", march] + + if opt is not None: + args += ["-O{}".format(opt)] + + if extra_args: + args += extra_args + + args += [src] + + run(args, mpy_cross) + + +def run(args, mpy_cross=None): + mpy_cross = find_mpy_cross_binary(mpy_cross) + + if not os.path.exists(mpy_cross): + raise CrossCompileError("mpy-cross binary not found at {}.".format(mpy_cross)) + + try: + st = os.stat(mpy_cross) + os.chmod(mpy_cross, st.st_mode | stat.S_IEXEC) + except OSError: + pass + + try: + subprocess.check_output([mpy_cross] + args, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as er: + raise CrossCompileError(er.output) diff --git a/mpy-cross/mpy_cross/__main__.py b/mpy-cross/mpy_cross/__main__.py new file mode 100644 index 0000000000000..9d957bca025d6 --- /dev/null +++ b/mpy-cross/mpy_cross/__main__.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +# +# This file is part of the MicroPython project, http://micropython.org/ +# +# The MIT License (MIT) +# +# Copyright (c) 2022 Andrew Leech +# Copyright (c) 2022 Jim Mussared +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +from __future__ import print_function +import argparse +import sys + +from . import run, CrossCompileError + +try: + run(sys.argv[1:]) +except CrossCompileError as er: + print(er.args[0], file=sys.stderr) + raise SystemExit(1) diff --git a/ports/esp8266/boards/GENERIC/manifest.py b/ports/esp8266/boards/GENERIC/manifest.py index 9ce3ffe3aae15..ef25040ef8a76 100644 --- a/ports/esp8266/boards/GENERIC/manifest.py +++ b/ports/esp8266/boards/GENERIC/manifest.py @@ -7,15 +7,13 @@ # drivers freeze("$(MPY_DIR)/drivers/display", "ssd1306.py") -# Libraries from micropython-lib, include only if the library directory exists -if os.path.isdir(convert_path("$(MPY_LIB_DIR)")): - # file utilities - freeze("$(MPY_LIB_DIR)/micropython/upysh", "upysh.py") +# micropython-lib: file utilities +freeze("$(MPY_LIB_DIR)/micropython/upysh", "upysh.py") - # requests - freeze("$(MPY_LIB_DIR)/python-ecosys/urequests", "urequests.py") - freeze("$(MPY_LIB_DIR)/micropython/urllib.urequest", "urllib/urequest.py") +# micropython-lib: requests +freeze("$(MPY_LIB_DIR)/python-ecosys/urequests", "urequests.py") +freeze("$(MPY_LIB_DIR)/micropython/urllib.urequest", "urllib/urequest.py") - # umqtt - freeze("$(MPY_LIB_DIR)/micropython/umqtt.simple", "umqtt/simple.py") - freeze("$(MPY_LIB_DIR)/micropython/umqtt.robust", "umqtt/robust.py") +# micropython-lib: umqtt +freeze("$(MPY_LIB_DIR)/micropython/umqtt.simple", "umqtt/simple.py") +freeze("$(MPY_LIB_DIR)/micropython/umqtt.robust", "umqtt/robust.py") diff --git a/ports/rp2/boards/ARDUINO_NANO_RP2040_CONNECT/manifest.py b/ports/rp2/boards/ARDUINO_NANO_RP2040_CONNECT/manifest.py index 0448aa89a11e4..7a2f1c9e2738c 100644 --- a/ports/rp2/boards/ARDUINO_NANO_RP2040_CONNECT/manifest.py +++ b/ports/rp2/boards/ARDUINO_NANO_RP2040_CONNECT/manifest.py @@ -8,5 +8,4 @@ l2cap=True, security=True, ) -if os.path.isdir(convert_path("$(MPY_LIB_DIR)")): - freeze("$(MPY_LIB_DIR)/python-ecosys/urequests", "urequests.py") +freeze("$(MPY_LIB_DIR)/python-ecosys/urequests", "urequests.py") diff --git a/ports/rp2/boards/PICO_W/manifest.py b/ports/rp2/boards/PICO_W/manifest.py index eb748da485c98..d2c76602affe8 100644 --- a/ports/rp2/boards/PICO_W/manifest.py +++ b/ports/rp2/boards/PICO_W/manifest.py @@ -4,5 +4,4 @@ freeze("$(MPY_DIR)/tools", "upip_utarfile.py") freeze("$(MPY_DIR)/extmod", "ntptime.py") -if os.path.isdir(convert_path("$(MPY_LIB_DIR)")): - freeze("$(MPY_LIB_DIR)/python-ecosys/urequests", "urequests.py") +freeze("$(MPY_LIB_DIR)/python-ecosys/urequests", "urequests.py") diff --git a/tools/makemanifest.py b/tools/makemanifest.py index e69698d3f2340..d059d4a2660fe 100644 --- a/tools/makemanifest.py +++ b/tools/makemanifest.py @@ -29,127 +29,13 @@ import os import subprocess +sys.path.append(os.path.join(os.path.dirname(__file__), "../mpy-cross")) +import mpy_cross -########################################################################### -# Public functions to be used in the manifest - - -def include(manifest, **kwargs): - """Include another manifest. - - The manifest argument can be a string (filename) or an iterable of - strings. - - Relative paths are resolved with respect to the current manifest file. - - Optional kwargs can be provided which will be available to the - included script via the `options` variable. - - e.g. include("path.py", extra_features=True) - - in path.py: - options.defaults(standard_features=True) - - # freeze minimal modules. - if options.standard_features: - # freeze standard modules. - if options.extra_features: - # freeze extra modules. - """ - - if not isinstance(manifest, str): - for m in manifest: - include(m) - else: - manifest = convert_path(manifest) - with open(manifest) as f: - # Make paths relative to this manifest file while processing it. - # Applies to includes and input files. - prev_cwd = os.getcwd() - os.chdir(os.path.dirname(manifest)) - exec(f.read(), globals(), {"options": IncludeOptions(**kwargs)}) - os.chdir(prev_cwd) - - -def freeze(path, script=None, opt=0): - """Freeze the input, automatically determining its type. A .py script - will be compiled to a .mpy first then frozen, and a .mpy file will be - frozen directly. - - `path` must be a directory, which is the base directory to search for - files from. When importing the resulting frozen modules, the name of - the module will start after `path`, ie `path` is excluded from the - module name. - - If `path` is relative, it is resolved to the current manifest.py. - Use $(MPY_DIR), $(MPY_LIB_DIR), $(PORT_DIR), $(BOARD_DIR) if you need - to access specific paths. - - If `script` is None all files in `path` will be frozen. - - If `script` is an iterable then freeze() is called on all items of the - iterable (with the same `path` and `opt` passed through). - - If `script` is a string then it specifies the file or directory to - freeze, and can include extra directories before the file or last - directory. The file or directory will be searched for in `path`. If - `script` is a directory then all files in that directory will be frozen. - - `opt` is the optimisation level to pass to mpy-cross when compiling .py - to .mpy. - """ - - freeze_internal(KIND_AUTO, path, script, opt) - - -def freeze_as_str(path): - """Freeze the given `path` and all .py scripts within it as a string, - which will be compiled upon import. - """ - - freeze_internal(KIND_AS_STR, path, None, 0) - - -def freeze_as_mpy(path, script=None, opt=0): - """Freeze the input (see above) by first compiling the .py scripts to - .mpy files, then freezing the resulting .mpy files. - """ - - freeze_internal(KIND_AS_MPY, path, script, opt) - - -def freeze_mpy(path, script=None, opt=0): - """Freeze the input (see above), which must be .mpy files that are - frozen directly. - """ - - freeze_internal(KIND_MPY, path, script, opt) - - -########################################################################### -# Internal implementation - -KIND_AUTO = 0 -KIND_AS_STR = 1 -KIND_AS_MPY = 2 -KIND_MPY = 3 +import manifestfile VARS = {} -manifest_list = [] - - -class IncludeOptions: - def __init__(self, **kwargs): - self._kwargs = kwargs - self._defaults = {} - - def defaults(self, **kwargs): - self._defaults = kwargs - - def __getattr__(self, name): - return self._kwargs.get(name, self._defaults.get(name, None)) - class FreezeError(Exception): pass @@ -163,15 +49,6 @@ def system(cmd): return -1, er.output -def convert_path(path): - # Perform variable substituion. - for name, value in VARS.items(): - path = path.replace("$({})".format(name), value) - # Convert to absolute path (so that future operations don't rely on - # still being chdir'ed). - return os.path.abspath(path) - - def get_timestamp(path, default=None): try: stat = os.stat(path) @@ -182,119 +59,64 @@ def get_timestamp(path, default=None): return default -def get_timestamp_newest(path): - ts_newest = 0 - for dirpath, dirnames, filenames in os.walk(path, followlinks=True): - for f in filenames: - ts_newest = max(ts_newest, get_timestamp(os.path.join(dirpath, f))) - return ts_newest - - def mkdir(filename): path = os.path.dirname(filename) if not os.path.isdir(path): os.makedirs(path) -def freeze_internal(kind, path, script, opt): - path = convert_path(path) - if not os.path.isdir(path): - raise FreezeError("freeze path must be a directory: {}".format(path)) - if script is None and kind == KIND_AS_STR: - manifest_list.append((KIND_AS_STR, path, script, opt)) - elif script is None or isinstance(script, str) and script.find(".") == -1: - # Recursively search `path` for files to freeze, optionally restricted - # to a subdirectory specified by `script` - if script is None: - subdir = "" - else: - subdir = "/" + script - for dirpath, dirnames, filenames in os.walk(path + subdir, followlinks=True): - for f in filenames: - freeze_internal(kind, path, (dirpath + "/" + f)[len(path) + 1 :], opt) - elif not isinstance(script, str): - # `script` is an iterable of items to freeze - for s in script: - freeze_internal(kind, path, s, opt) - else: - # `script` should specify an individual file to be frozen - extension_kind = {KIND_AS_MPY: ".py", KIND_MPY: ".mpy"} - if kind == KIND_AUTO: - for k, ext in extension_kind.items(): - if script.endswith(ext): - kind = k - break - else: - print("warn: unsupported file type, skipped freeze: {}".format(script)) - return - wanted_extension = extension_kind[kind] - if not script.endswith(wanted_extension): - raise FreezeError("expecting a {} file, got {}".format(wanted_extension, script)) - manifest_list.append((kind, path, script, opt)) - - # Formerly make-frozen.py. # This generates: # - MP_FROZEN_STR_NAMES macro # - mp_frozen_str_sizes # - mp_frozen_str_content -def generate_frozen_str_content(paths): - def module_name(f): - return f - - modules = [] - output = [b"#include \n"] - - for path in paths: - root = path.rstrip("/") - root_len = len(root) - - for dirpath, dirnames, filenames in os.walk(root): - for f in filenames: - fullpath = dirpath + "/" + f - st = os.stat(fullpath) - modules.append((path, fullpath[root_len + 1 :], st)) - - output.append(b"#define MP_FROZEN_STR_NAMES \\\n") - for _path, f, st in modules: - m = module_name(f) - output.append(b'"%s\\0" \\\n' % m.encode()) +def generate_frozen_str_content(modules): + output = [ + b"#include \n", + b"#define MP_FROZEN_STR_NAMES \\\n", + ] + + for _, target_path in modules: + print("STR", target_path) + output.append(b'"%s\\0" \\\n' % target_path.encode()) output.append(b"\n") output.append(b"const uint32_t mp_frozen_str_sizes[] = { ") - for _path, f, st in modules: + for full_path, _ in modules: + st = os.stat(full_path) output.append(b"%d, " % st.st_size) output.append(b"0 };\n") output.append(b"const char mp_frozen_str_content[] = {\n") - for path, f, st in modules: - data = open(path + "/" + f, "rb").read() - - # We need to properly escape the script data to create a C string. - # When C parses hex characters of the form \x00 it keeps parsing the hex - # data until it encounters a non-hex character. Thus one must create - # strings of the form "data\x01" "abc" to properly encode this kind of - # data. We could just encode all characters as hex digits but it's nice - # to be able to read the resulting C code as ASCII when possible. - - data = bytearray(data) # so Python2 extracts each byte as an integer - esc_dict = {ord("\n"): b"\\n", ord("\r"): b"\\r", ord('"'): b'\\"', ord("\\"): b"\\\\"} - output.append(b'"') - break_str = False - for c in data: - try: - output.append(esc_dict[c]) - except KeyError: - if 32 <= c <= 126: - if break_str: - output.append(b'" "') - break_str = False - output.append(chr(c).encode()) - else: - output.append(b"\\x%02x" % c) - break_str = True - output.append(b'\\0"\n') + for full_path, _ in modules: + with open(full_path, "rb") as f: + data = f.read() + + # We need to properly escape the script data to create a C string. + # When C parses hex characters of the form \x00 it keeps parsing the hex + # data until it encounters a non-hex character. Thus one must create + # strings of the form "data\x01" "abc" to properly encode this kind of + # data. We could just encode all characters as hex digits but it's nice + # to be able to read the resulting C code as ASCII when possible. + + data = bytearray(data) # so Python2 extracts each byte as an integer + esc_dict = {ord("\n"): b"\\n", ord("\r"): b"\\r", ord('"'): b'\\"', ord("\\"): b"\\\\"} + output.append(b'"') + break_str = False + for c in data: + try: + output.append(esc_dict[c]) + except KeyError: + if 32 <= c <= 126: + if break_str: + output.append(b'" "') + break_str = False + output.append(chr(c).encode()) + else: + output.append(b"\\x%02x" % c) + break_str = True + output.append(b'\\0"\n') output.append(b'"\\0"\n};\n\n') return b"".join(output) @@ -340,14 +162,13 @@ def main(): print("mpy-cross not found at {}, please build it first".format(MPY_CROSS)) sys.exit(1) + manifest = manifestfile.ManifestFile(manifestfile.MODE_FREEZE, VARS) + # Include top-level inputs, to generate the manifest for input_manifest in args.files: try: - if input_manifest.endswith(".py"): - include(input_manifest) - else: - exec(input_manifest) - except FreezeError as er: + manifest.execute(input_manifest) + except manifestfile.ManifestFileError as er: print('freeze error executing "{}": {}'.format(input_manifest, er.args[0])) sys.exit(1) @@ -355,34 +176,42 @@ def main(): str_paths = [] mpy_files = [] ts_newest = 0 - for kind, path, script, opt in manifest_list: - if kind == KIND_AS_STR: - str_paths.append(path) - ts_outfile = get_timestamp_newest(path) - elif kind == KIND_AS_MPY: - infile = "{}/{}".format(path, script) - outfile = "{}/frozen_mpy/{}.mpy".format(args.build_dir, script[:-3]) - ts_infile = get_timestamp(infile) + for result in manifest.files(): + if result.kind == manifestfile.KIND_FREEZE_AS_STR: + str_paths.append( + ( + result.full_path, + result.target_path, + ) + ) + ts_outfile = result.timestamp + elif result.kind == manifestfile.KIND_FREEZE_AS_MPY: + outfile = "{}/frozen_mpy/{}.mpy".format(args.build_dir, result.target_path[:-3]) ts_outfile = get_timestamp(outfile, 0) - if ts_infile >= ts_outfile: - print("MPY", script) + if result.timestamp >= ts_outfile: + print("MPY", result.target_path) mkdir(outfile) - res, out = system( - [MPY_CROSS] - + args.mpy_cross_flags.split() - + ["-o", outfile, "-s", script, "-O{}".format(opt), infile] - ) - if res != 0: - print("error compiling {}:".format(infile)) - sys.stdout.buffer.write(out) - raise SystemExit(1) + # Add __version__ to the end of the file before compiling. + with manifestfile.tagged_py_file(result.full_path, result.metadata) as tagged_path: + try: + mpy_cross.compile( + tagged_path, + dest=outfile, + src_path=result.target_path, + opt=result.opt, + mpy_cross=MPY_CROSS, + extra_args=args.mpy_cross_flags.split(), + ) + except mpy_cross.CrossCompileError as ex: + print("error compiling {}:".format(target_path)) + print(ex.args[0]) + raise SystemExit(1) ts_outfile = get_timestamp(outfile) mpy_files.append(outfile) else: - assert kind == KIND_MPY - infile = "{}/{}".format(path, script) - mpy_files.append(infile) - ts_outfile = get_timestamp(infile) + assert kind == manifestfile.KIND_FREEZE_MPY + mpy_files.append(full_path) + ts_outfile = timestamp ts_newest = max(ts_newest, ts_outfile) # Check if output file needs generating diff --git a/tools/manifestfile.py b/tools/manifestfile.py new file mode 100644 index 0000000000000..84c79ed82b28f --- /dev/null +++ b/tools/manifestfile.py @@ -0,0 +1,488 @@ +#!/usr/bin/env python3 +# +# This file is part of the MicroPython project, http://micropython.org/ +# +# The MIT License (MIT) +# +# Copyright (c) 2022 Jim Mussared +# Copyright (c) 2019 Damien P. George +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +from __future__ import print_function +import contextlib +import os +import sys +import glob +import tempfile +from collections import namedtuple + +__all__ = ["ManifestFileError", "ManifestFile"] + +# Allow freeze*() etc. +MODE_FREEZE = 1 +# Only allow include/require/module/package. +MODE_COMPILE = 2 + + +# In compile mode, .py -> KIND_COMPILE_AS_MPY +# In freeze mode, .py -> KIND_FREEZE_AS_MPY, .mpy->KIND_FREEZE_MPY +KIND_AUTO = 1 +# Freeze-mode only, .py -> KIND_FREEZE_AS_MPY, .mpy->KIND_FREEZE_MPY +KIND_FREEZE_AUTO = 2 + +# Freeze-mode only, The .py file will be frozen as text. +KIND_FREEZE_AS_STR = 3 +# Freeze-mode only, The .py file will be compiled and frozen as bytecode. +KIND_FREEZE_AS_MPY = 4 +# Freeze-mode only, The .mpy file will be frozen directly. +KIND_FREEZE_MPY = 5 +# Compile mode only, the .py file should be compiled to .mpy. +KIND_COMPILE_AS_MPY = 6 + +# File on the local filesystem. +FILE_TYPE_LOCAL = 1 +# URL to file. (TODO) +FILE_TYPE_HTTP = 2 + + +class ManifestFileError(Exception): + pass + + +# The set of files that this manifest references. +ManifestOutput = namedtuple( + "ManifestOutput", + [ + "file_type", # FILE_TYPE_*. + "full_path", # The input file full path. + "target_path", # The target path on the device. + "timestamp", # Last modified date of the input file. + "kind", # KIND_*. + "metadata", # Metadata for the containing package. + "opt", # Optimisation level (or None). + ], +) + + +# Represent the metadata for a package. +class ManifestMetadata: + def __init__(self): + self.version = None + self.description = None + self.license = None + + def update(self, description=None, version=None, license=None): + if description: + self.description = description + if version: + self.version = version + if license: + self.license = version + + +# Turns a dict of options into a object with attributes used to turn the +# kwargs passed to include() and require into the "options" global in the +# included manifest. +# options = IncludeOptions(foo="bar", blah="stuff") +# options.foo # "bar" +# options.blah # "stuff" +class IncludeOptions: + def __init__(self, **kwargs): + self._kwargs = kwargs + self._defaults = {} + + def defaults(self, **kwargs): + self._defaults = kwargs + + def __getattr__(self, name): + return self._kwargs.get(name, self._defaults.get(name, None)) + + +class ManifestFile: + def __init__(self, mode, path_vars=None): + # Either MODE_FREEZE or MODE_COMPILE. + self._mode = mode + # Path substition variables. + self._path_vars = path_vars or {} + # List of files (as ManifestFileResult) references by this manifest. + self._manifest_files = [] + # Don't allow including the same file twice. + self._visited = set() + # Stack of metadata for each level. + self._metadata = [ManifestMetadata()] + + def _resolve_path(self, path): + # Convert path to an absolute path, applying variable substitutions. + for name, value in self._path_vars.items(): + if value is not None: + path = path.replace("$({})".format(name), value) + return os.path.abspath(path) + + def _manifest_globals(self, kwargs): + # This is the "API" available to a manifest file. + return { + "metadata": self.metadata, + "include": self.include, + "require": self.require, + "package": self.package, + "module": self.module, + "freeze": self.freeze, + "freeze_as_str": self.freeze_as_str, + "freeze_as_mpy": self.freeze_as_mpy, + "freeze_mpy": self.freeze_mpy, + "options": IncludeOptions(**kwargs), + } + + def files(self): + return self._manifest_files + + def execute(self, manifest_file): + if manifest_file.endswith(".py"): + # Execute file from filesystem. + self.include(manifest_file, top_level=True) + else: + # Execute manifest code snippet. + try: + exec(manifest_file, self._manifest_globals({})) + except Exception as er: + raise ManifestFileError("Error in manifest: {}".format(er)) + + def _add_file(self, full_path, target_path, kind=KIND_AUTO, opt=None): + # Check file exists and get timestamp. + try: + stat = os.stat(full_path) + timestamp = stat.st_mtime + except OSError: + raise ManifestFileError("cannot stat {}".format(full_path)) + + # Map the AUTO kinds to their actual kind based on mode and extension. + _, ext = os.path.splitext(full_path) + if self._mode == MODE_FREEZE: + if kind in ( + KIND_AUTO, + KIND_FREEZE_AUTO, + ): + if ext.lower() == ".py": + kind = KIND_FREEZE_AS_MPY + elif ext.lower() == ".mpy": + kind = KIND_FREEZE_MPY + else: + if kind != KIND_AUTO: + raise ManifestFileError("Not in freeze mode") + if ext.lower() != ".py": + raise ManifestFileError("Expected .py file") + kind = KIND_COMPILE_AS_MPY + + self._manifest_files.append( + ManifestOutput( + FILE_TYPE_LOCAL, full_path, target_path, timestamp, kind, self._metadata[-1], opt + ) + ) + + def _search(self, base_path, package_path, files, exts, kind, opt=None, strict=False): + base_path = self._resolve_path(base_path) + + if files: + # Use explicit list of files (relative to package_path). + for file in files: + if package_path: + file = os.path.join(package_path, file) + self._add_file(os.path.join(base_path, file), file, kind=kind, opt=opt) + else: + if base_path: + prev_cwd = os.getcwd() + os.chdir(self._resolve_path(base_path)) + + # Find all candidate files. + for dirpath, _, filenames in os.walk(package_path or ".", followlinks=True): + for file in filenames: + file = os.path.relpath(os.path.join(dirpath, file), ".") + _, ext = os.path.splitext(file) + if ext.lower() in exts: + self._add_file( + os.path.join(base_path, file), + file, + kind=kind, + opt=opt, + ) + elif strict: + raise ManifestFileError("Unexpected file type") + + if base_path: + os.chdir(prev_cwd) + + def metadata(self, description=None, version=None, license=None): + """ + From within a manifest file, use this to set the metadata for the + package described by current manifest. + + After executing a manifest file (via execute()), call this + to obtain the metadata for the top-level manifest file. + """ + + self._metadata[-1].update(description, version, license) + return self._metadata[-1] + + def include(self, manifest_path, top_level=False, **kwargs): + """ + Include another manifest. + + The manifest argument can be a string (filename) or an iterable of + strings. + + Relative paths are resolved with respect to the current manifest file. + + If the path is to a directory, then it implicitly includes the + manifest.py file inside that directory. + + Optional kwargs can be provided which will be available to the + included script via the `options` variable. + + e.g. include("path.py", extra_features=True) + + in path.py: + options.defaults(standard_features=True) + + # freeze minimal modules. + if options.standard_features: + # freeze standard modules. + if options.extra_features: + # freeze extra modules. + """ + if not isinstance(manifest_path, str): + for m in manifest_path: + self.include(m) + else: + manifest_path = self._resolve_path(manifest_path) + # Including a directory grabs the manifest.py inside it. + if os.path.isdir(manifest_path): + manifest_path = os.path.join(manifest_path, "manifest.py") + if manifest_path in self._visited: + return + self._visited.add(manifest_path) + if not top_level: + self._metadata.append(ManifestMetadata()) + with open(manifest_path) as f: + # Make paths relative to this manifest file while processing it. + # Applies to includes and input files. + prev_cwd = os.getcwd() + os.chdir(os.path.dirname(manifest_path)) + try: + exec(f.read(), self._manifest_globals(kwargs)) + except Exception as er: + raise ManifestFileError( + "Error in manifest file: {}: {}".format(manifest_path, er) + ) + os.chdir(prev_cwd) + if not top_level: + self._metadata.pop() + + def require(self, name, version=None, unix_ffi=False, **kwargs): + """ + Require a module by name from micropython-lib. + + Optionally specify unix_ffi=True to use a module from the unix-ffi directory. + """ + if self._path_vars["MPY_LIB_DIR"]: + lib_dirs = ["micropython", "python-stdlib", "python-ecosys"] + if unix_ffi: + # Search unix-ffi only if unix_ffi=True, and make unix-ffi modules + # take precedence. + lib_dirs = ["unix-ffi"] + lib_dirs + + for lib_dir in lib_dirs: + for manifest_path in glob.glob( + os.path.join( + self._path_vars["MPY_LIB_DIR"], lib_dir, "**", name, "manifest.py" + ), + recursive=True, + ): + self.include(manifest_path, **kwargs) + return + raise ValueError("Library not found in local micropython-lib: {}".format(name)) + else: + # TODO: HTTP request to obtain URLs from manifest.json. + raise ValueError("micropython-lib not available for require('{}').", name) + + def package(self, package_path, files=None, base_path=".", opt=None): + """ + Define a package, optionally restricting to a set of files. + + Simple case, a package in the current directory: + package("foo") + will include all .py files in foo, and will be stored as foo/bar/baz.py. + + If the package isn't in the current directory, use base_path: + package("foo", base_path="src") + + To restrict to certain files in the package use files (note: paths should be relative to the package): + package("foo", files=["bar/baz.py"]) + """ + # Include "base_path/package_path/**/*.py" --> "package_path/**/*.py" + self._search(base_path, package_path, files, exts=(".py",), kind=KIND_AUTO, opt=opt) + + def module(self, module_path, base_path=".", opt=None): + """ + Include a single Python file as a module. + + If the file is in the current directory: + module("foo.py") + + Otherwise use base_path to locate the file: + module("foo.py", "src/drivers") + """ + # Include "base_path/module_path" --> "module_path" + base_path = self._resolve_path(base_path) + _, ext = os.path.splitext(module_path) + if ext.lower() != ".py": + raise ManifestFileError("module must be .py file") + # TODO: version None + self._add_file(os.path.join(base_path, module_path), module_path, opt=opt) + + def _freeze_internal(self, path, script, exts, kind, opt): + if script is None: + self._search(path, None, None, exts=exts, kind=kind, opt=opt) + elif isinstance(script, str) and os.path.isdir(os.path.join(path, script)): + self._search(path, script, None, exts=exts, kind=kind, opt=opt) + elif not isinstance(script, str): + self._search(path, None, script, exts=exts, kind=kind, opt=opt) + else: + self._search(path, None, (script,), exts=exts, kind=kind, opt=opt) + + def freeze(self, path, script=None, opt=None): + """ + Freeze the input, automatically determining its type. A .py script + will be compiled to a .mpy first then frozen, and a .mpy file will be + frozen directly. + + `path` must be a directory, which is the base directory to _search for + files from. When importing the resulting frozen modules, the name of + the module will start after `path`, ie `path` is excluded from the + module name. + + If `path` is relative, it is resolved to the current manifest.py. + Use $(MPY_DIR), $(MPY_LIB_DIR), $(PORT_DIR), $(BOARD_DIR) if you need + to access specific paths. + + If `script` is None all files in `path` will be frozen. + + If `script` is an iterable then freeze() is called on all items of the + iterable (with the same `path` and `opt` passed through). + + If `script` is a string then it specifies the file or directory to + freeze, and can include extra directories before the file or last + directory. The file or directory will be _searched for in `path`. If + `script` is a directory then all files in that directory will be frozen. + + `opt` is the optimisation level to pass to mpy-cross when compiling .py + to .mpy. + """ + self._freeze_internal(path, script, exts=(".py", ".mpy"), kind=KIND_FREEZE_AUTO, opt=opt) + + def freeze_as_str(self, path): + """ + Freeze the given `path` and all .py scripts within it as a string, + which will be compiled upon import. + """ + self._search(path, None, None, exts=(".py"), kind=KIND_FREEZE_AS_STR) + + def freeze_as_mpy(self, path, script=None, opt=None): + """ + Freeze the input (see above) by first compiling the .py scripts to + .mpy files, then freezing the resulting .mpy files. + """ + self._freeze_internal(path, script, exts=(".py"), kind=KIND_FREEZE_AS_MPY, opt=opt) + + def freeze_mpy(self, path, script=None, opt=None): + """ + Freeze the input (see above), which must be .mpy files that are + frozen directly. + """ + self._freeze_internal(path, script, exts=(".mpy"), kind=KIND_FREEZE_MPY, opt=opt) + + +# Generate a temporary file with a line appended to the end that adds __version__. +@contextlib.contextmanager +def tagged_py_file(path, metadata): + dest_fd, dest_path = tempfile.mkstemp(suffix=".py", text=True) + try: + with os.fdopen(dest_fd, "w") as dest: + with open(path, "r") as src: + contents = src.read() + dest.write(contents) + + # Don't overwrite a version definition if the file already has one in it. + if metadata.version and "__version__ =" not in contents: + dest.write("\n\n__version__ = {}\n".format(repr(metadata.version))) + yield dest_path + finally: + os.unlink(dest_path) + + +def main(): + import argparse + + cmd_parser = argparse.ArgumentParser(description="List the files referenced by a manifest.") + cmd_parser.add_argument("--freeze", action="store_true", help="freeze mode") + cmd_parser.add_argument("--compile", action="store_true", help="compile mode") + cmd_parser.add_argument( + "--lib", + default=os.path.join(os.path.dirname(__file__), "../lib/micropython-lib"), + help="path to micropython-lib repo", + ) + cmd_parser.add_argument("--port", default=None, help="path to port dir") + cmd_parser.add_argument("--board", default=None, help="path to board dir") + cmd_parser.add_argument( + "--top", + default=os.path.join(os.path.dirname(__file__), ".."), + help="path to micropython repo", + ) + cmd_parser.add_argument("files", nargs="+", help="input manifest.py") + args = cmd_parser.parse_args() + + path_vars = { + "MPY_DIR": os.path.abspath(args.top) if args.top else None, + "BOARD_DIR": os.path.abspath(args.board) if args.board else None, + "PORT_DIR": os.path.abspath(args.port) if args.port else None, + "MPY_LIB_DIR": os.path.abspath(args.lib) if args.lib else None, + } + + mode = None + if args.freeze: + mode = MODE_FREEZE + elif args.compile: + mode = MODE_COMPILE + else: + print("Error: No mode specified.", file=sys.stderr) + exit(1) + + m = ManifestFile(mode, path_vars) + for manifest_file in args.files: + try: + m.execute(manifest_file) + except ManifestFileError as er: + print(er, file=sys.stderr) + exit(1) + for f in m.files(): + print(f) + + +if __name__ == "__main__": + main()