8000 tools/deploy.py: Add script for deploying to upip. · micropython/micropython-lib@9db0ced · GitHub
[go: up one dir, main page]
Skip to content

Commit 9db0ced

Browse files
jimmojimmo
committed
tools/deploy.py: Add script for deploying to upip.
This populates https://micropython.org/pi/v2 with compiled packages. This work was funded through GitHub Sponsors. Signed-off-by: Jim Mussared <jim.mussared@gmail.com>
1 parent 122b689 commit 9db0ced

File tree

1 file changed

+313
-0
lines changed

1 file changed

+313
-0
lines changed

‎tools/deploy.py

Lines changed: 313 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,313 @@
1+
#!/usr/bin/env python3
2+
#
3+
# This file is part of the MicroPython project, http://micropython.org/
4+
#
5+
# The MIT License (MIT)
6+
#
7+
# Copyright (c) 2022 Jim Mussared
8+
#
9+
# Permission is hereby granted, free of charge, to any person obtaining a copy
10+
# of this software and associated documentation files (the "Software"), to deal
11+
# in the Software without restriction, including without limitation the rights
12+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
13+
# copies of the Software, and to permit persons to whom the Software is
14+
# furnished to do so, subject to the following conditions:
15+
#
16+
# The above copyright notice and this permission notice shall be included in
17+
# all copies or substantial portions of the Software.
18+
#
19+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
24+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25+
# THE SOFTWARE.
26+
27+
# This script compiles all packages in this repository (excluding unix-ffi)
28+
# into a directory suitable for serving to upip with a static web server.
29+
30+
# Usage:
31+
# ./tools/deploy.py --output /tmp/lib-deploy
32+
33+
# The output directory (--output) will have the following layout
34+
# /
35+
# index.json
36+
# file/
37+
# 1d/
38+
# ddc25d
39+
# c3/
40+
# 1d7eb7
41+
# a3934b
42+
# e3/
43+
# 9dbf64
44+
# ...
45+
# package/
46+
# 6/ <-- mpy version
47+
# aioble/
48+
# latest.json
49+
# 0.1.json
50+
# ...
51+
# hmac/
52+
# latest.json
53+
# 3.4.2-3.json
54+
# ...
55+
# pyjwt/
56+
# latest.json
57+
# 0.1.json
58+
# ...
59+
# ...
60+
61+
# index.json is:
62+
# {
63+
# "updated": <utc-seconds-since-1970>,
64+
# "packages": {
65+
# {
66+
# "name": "aioble",
67+
# "version": "0.1", # Latest version of this package.
68+
# "description": "...",
69+
# "license": "MIT", # SPDX short identifier. Can be overridden by a given package.
70+
# },
71+
# ...
72+
# }
73+
# }
74+
75+
# Each file in the "file" directory is the file contents (usually .mpy), named
76+
# by the prefix of the sha256 hash of the contents. Files are never removed, and
77+
# collisions are detected and will fail the compile, and the prefix length should
78+
# be increased.
79+
# As of September 2022, there are no collisions with a hash prefix length of 4,
80+
# so the default of 8 should be sufficient for a while. Increasing the length
81+
# doesn't invalidate old packages.
82+
83+
# Each package json (either latest.json or {version}.json) is:
84+
# {
85+
# "hashes": [
86+
# ["aioble/server.mpy", "e39dbf64"],
87+
# ...
88+
# ],
89+
# "urls": [ <-- not used by micropython-lib packages
90+
# ["target/path.py", "http://url/foo/bar/path.py"],
91+
# ...
92+
# ],
93+
# "deps": [ <-- not used by micropython-lib packages
94+
# ["name", "version"],
95+
# ...
96+
# ]
97+
# "version": "0.1"
98+
# }
99+
100+
# upip (or other tools) should request /package/{mpy_version}/{package_name}/{version}.json.
101+
102+
import argparse
103+
import glob
104+
import hashlib
105+
import json
106+
import os
107+
import shutil
108+
import sys
109+
import tempfile
110+
import time
111+
112+
113+
_COLOR_ERROR_ON = "\033[1;31m"
114+
_COLOR_ERROR_OFF = "\033[0m"
115+
116+
117+
# Create all directories in the path (such that the file can be created).
118+
def ensure_path_exists(file_path):
119+
path = os.path.dirname(file_path)
120+
if not os.path.isdir(path):
121+
os.makedirs(path)
122+
123+
124+
# Returns the sha256 of the specified file object.
125+
def file_hash(f):
126+
hs256 = hashlib.sha256()
127+
hs256.update(f.read())
128+
return hs256.hexdigest()
129+
130+
131+
# Returns true if the two files contain identical contents.
132+
def identical_files(a, b):
133+
with open(a, "rb") as fa:
134+
with open(b, "rb") as fb:
135+
return fa.read() == fb.read()
136+
137+
138+
# Helper to write the object as json to the specified path, creating any
139+
# directories as required.
140+
def write_json(obj, path, minify=False):
141+
ensure_path_exists(path)
142+
with open(path, "w") as f:
143+
json.dump(
144+
obj, f, indent=(None if minify else 2), separators=((",", ":") if minify else None)
145+
)
146+
f.write("\n")
147+
148+
149+
def error_color(s):
150+
return _COLOR_ERROR_ON + s + _COLOR_ERROR_OFF
151+
152+
153+
def deploy(output_path, hash_prefix_len, mpy_cross_path):
154+
import manifestfile
155+
import mpy_cross
156+
157+
out_file_dir = os.path.join(output_path, "file")
158+
out_package_dir = os.path.join(output_path, "package")
159+
160+
path_vars = {
161+
"MPY_LIB_DIR": os.path.abspath(os.path.join(os.path.dirname(__file__), "..")),
162+
}
163+
164+
index_json = {"updated": int(time.time()), "packages": []}
165+
166+
# For now, don't process unix-ffi. In the future this can be extended to
167+
# allow a way to request unix-ffi packages via upip.
168+
lib_dirs = ["micropython", "python-stdlib", "python-ecosys"]
169+
170+
mpy_version, _mpy_sub_version = mpy_cross.mpy_version(mpy_cross=mpy_cross_path)
171+
print(f"Generating bytecode version {mpy_version}")
172+
173+
for lib_dir in lib_dirs:
174+
for manifest_path in glob.glob(os.path.join(lib_dir, "**", "manifest.py"), recursive=True):
175+
print("Processing: {}".format(os.path.dirname(manifest_path)))
176+
# .../foo/manifest.py -> foo
177+
package_name = os.path.basename(os.path.dirname(manifest_path))
178+
179+
# Compile the manifest.
180+
manifest = manifestfile.ManifestFile(manifestfile.MODE_COMPILE, path_vars)
181+
manifest.execute(manifest_path)
182+
183+
# Append this package to the index.
184+
if not manifest.metadata().version:
185+
print(error_color("Warning:"), package_name, "doesn't have a version.")
186+
187+
index_json["packages"].append(
188+
{
189+
"name": package_name,
190+
"version": manifest.metadata().version or "",
191+
"description": manifest.metadata().description or "",
192+
"license": manifest.metadata().license or "MIT",
193+
}
194+
)
195+
196+
# Create/replace latest.json with the current version and latest version of all dependencies.
197+
package_json = {"hashes": [], "version": manifest.metadata().version or ""}
198+
for result in manifest.files():
199+
# This isn't allowed in micropython-lib anyway.
200+
if result.file_type != manifestfile.FILE_TYPE_LOCAL:
201+
print("Non-local file not supported.", file=sys.stderr)
202+
sys.exit(1)
203+
204+
# Tag each file with the package metadata and compile to .mpy.
205+
with manifestfile.tagged_py_file(result.full_path, result.metadata) as tagged_path:
206+
with tempfile.NamedTemporaryFile(
207+
mode="rb", suffix=".mpy", delete=True
208+
) as mpy_file:
209+
if not result.target_path.endswith(".py"):
210+
print(
211+
"Target path isn't a .py file:",
212+
result.target_path,
213+
file=sys.stderr,
214+
)
215+
sys.exit(1)
216+
try:
217+
mpy_cross.compile(
218+
tagged_path,
219+
dest=mpy_file.name,
220+
src_path=result.target_path,
221+
opt=result.opt,
222+
mpy_cross=mpy_cross_path,
223+
)
224+
except mpy_cross.CrossCompileError as e:
225+
print(
226+
error_color("Error:"),
227+
"Unable to compile",
228+
result.target_path,
229+
"in package",
230+
package_name,
231+
file=sys.stderr,
232+
)
233+
print(e)
234+
sys.exit(1)
235+
236+
# Generate the full sha256 and the hash prefix to use as the output path.
237+
# Group files into subdirectories using the first two bytes of the hash prefix.
238+
mpy_hash = file_hash(mpy_file)
239+
short_mpy_hash = mpy_hash[:hash_prefix_len]
240+
output_file = os.path.join(short_mpy_hash[:2], short_mpy_hash[2:])
241+
output_file_path = os.path.join(out_file_dir, output_file)
242+
243+
if os.path.exists(output_file_path):
244+
# If the file exists (e.g. from a previous run of this script), then ensure
245+
# that it's actually the same file.
246+
if not identical_files(mpy_file.name, output_file_path):
247+
print(
248+
error_color("Hash collision processing:"),
249+
package_name,
250+
file=sys.stderr,
251+
)
252+
print(f" File: {result.target_path}", file=sys.stderr)
253+
print(f" Short hash: {short_mpy_hash}", file=sys.stderr)
254+
print(f" Full hash: {mpy_hash}", file=sys.stderr)
255+
with open(output_file_path, "rb") as f:
256+
print(f" Target hash: {file_hash(f)}", file=sys.stderr)
257+
print(
258+
f"Try increasing --hash-prefix (currently {hash_prefix_len})"
259+
)
260+
sys.exit(1)
261+
else:
262+
# Create new file.
263+
ensure_path_exists(output_file_path)
264+
shutil.copyfile(mpy_file.name, output_file_path)
265+
266+
# Add the file to the package json.
267+
target_path_mpy = result.target_path[:-2] + "mpy"
268+
package_json["hashes"].append((target_path_mpy, short_mpy_hash))
269+
270+
# Create/replace {package}/latest.json
271+
package_json_path = os.path.join(
272+
out_package_dir, str(mpy_version), package_name, "latest.json"
273+
)
274+
write_json(package_json, package_json_path, minify=True)
275+
276+
# Write {package}/{version}.json, but only if it doesn't already
277+
# exist. A package version is "locked" the first time it's seen
278+
# by this script, for the current mpy version.
279+
if manifest.metadata().version:
280+
versioned_package_json_path = os.path.join(
281+
out_package_dir,
282+
str(mpy_version),
283+
package_name,
284+
f"{manifest.metadata().version}.json",
285+
)
286+
if not os.path.exists(versioned_package_json_path):
287+
write_json(package_json, versioned_package_json_path, minify=True)
288+
289+
# Write package index.
290< C0F4 /code>+
write_json(index_json, os.path.join(output_path, "index.json"), minify=False)
291+
292+
293+
def main():
294+
import argparse
295+
296+
cmd_parser = argparse.ArgumentParser(
297+
description="Compile micropython-lib for serving to upip."
298+
)
299+
cmd_parser.add_argument("--output", required=True, help="output directory")
300+
cmd_parser.add_argument("--hash-prefix", default=8, type=int, help="hash prefix length")
301+
cmd_parser.add_argument("--mpy-cross", default=None, help="optional path to mpy-cross binary")
302+
cmd_parser.add_argument("--micropython", default=None, help="path to micropython repo")
303+
args = cmd_parser.parse_args()
304+
305+
if args.micropython:
306+
sys.path.append(os.path.join(args.micropython, "tools")) # for manifestfile
307+
sys.path.append(os.path.join(args.micropython, "mpy-cross")) # for mpy_cross
308+
309+
deploy(args.output, hash_prefix_len=max(4, args.hash_prefix), mpy_cross_path=args.mpy_cross)
310+
311+
312+
if __name__ == "__main__":
313+
main()

0 commit comments

Comments
 (0)
0