diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d43b404..8bb288c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,15 +15,18 @@ jobs: env: GITHUB_CONTEXT: ${{ toJson(github) }} run: echo "$GITHUB_CONTEXT" - - name: Set up Python 3.7 - uses: actions/setup-python@v1 + - name: Set up Python 3.12 + uses: actions/setup-python@v5 with: - python-version: 3.7 + python-version: "3.12" - name: Versions run: | python3 --version - name: Checkout Current Repo - uses: actions/checkout@v2 + uses: actions/checkout@v4 + with: + filter: 'blob:none' + depth: 0 - name: Install requirements run: | sudo apt-get update diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0e2aa54..8605547 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,11 +13,14 @@ jobs: upload-pypi: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 + with: + filter: 'blob:none' + depth: 0 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: - python-version: '3.7' + python-version: '3.12' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/README.md b/README.md index a8dc21d..5b85c43 100644 --- a/README.md +++ b/README.md @@ -4,40 +4,20 @@ This repo contains build scripts used to build the [Adafruit CircuitPython bundle](https://github.com/adafruit/Adafruit_CircuitPython_Bundle), [CircuitPython Community bundle](https://github.com/adafruit/CircuitPython_Community_Bundle) -and individual library release zips. Its focused on Travis CI support but will also work locally +and individual library release zips. Its focused on Github Actions support but will also work locally when a gcc compiler is present. -The pip package includes mpy-crosses that run on Travis. When building locally, the scripts will +The scripts will either fetch a pre-built mpy-cross from s3 or automatically clone the [CircuitPython repo](https://github.com/adafruit/circuitpython) and attempt -to build mpy-crosses. You'll need some version of gcc for this to work. +to build mpy-cross. You'll need some version of gcc for this to work. ## Setting up libraries -These build tools are intended for use with [Travis CI](https://travis-ci.org) -to automatically build .mpy files and zip them up for CircuitPython when a new -tagged release is created. To add support to a repo you need to: - - 1. Use the [CircuitPython cookiecutter](https://github.com/adafruit/cookiecutter-adafruit-circuitpython) to generate .travis.yml. - 2. For adafruit repositories, simply give the CircuitPythonLibrarians team - write access to the repo and Adabot will do the rest. - - Otherwise, go to travis-ci.org and find the repository (it needs to be - setup to access your github account, and your github account needs access - to write to the repo). Flip the 'ON' switch on for Travis and the repo, - see the Travis docs for more details: https://docs.travis-ci.com/user/getting-started/ - 3. Get a GitHub 'personal access token' which has at least 'public_repo' or - 'repo' scope: https://help.github.com/articles/creating-an-access-token-for-command-line-use/ - Keep this token safe and secure! Anyone with the token will be able to - access and write to your GitHub repositories. Travis will use the token - to attach the .mpy files to the release. - 4. In the Travis CI settings for the repository that was enabled find the - environment variable editing page: https://docs.travis-ci.com/user/environment-variables/#Defining-Variables-in-Repository-Settings - Add an environment variable named GITHUB_TOKEN and set it to the value - of the GitHub personal access token above. Keep 'Display value in build - log' flipped off. - 5. That's it! Tag a release and Travis should go to work to add zipped .mpy files - to the release. It takes about a 2-3 minutes for a worker to spin up, - build mpy-cross, and add the binaries to the release. +These build tools automatically build .mpy files and zip them up for +CircuitPython when a new tagged release is created. To add support to a repo +you need to use the [CircuitPython +cookiecutter](https://github.com/adafruit/cookiecutter-adafruit-circuitpython) +to generate `.github/workflows/*.yml`. The bundle build will produce one zip file for every major CircuitPython release supported containing compatible mpy files and a zip with human readable py files. @@ -71,5 +51,5 @@ circuitpython-build-bundles --filename_prefix --library_loc ## Contributing Contributions are welcome! Please read our [Code of Conduct] -(https://github.com/adafruit/Adafruit_CircuitPython_adabot/blob/master/CODE_OF_CONDUCT.md) +(https://github.com/adafruit/Adafruit\_CircuitPython\_adabot/blob/master/CODE\_OF\_CONDUCT.md) before contributing to help this project stay welcoming. diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index 0fc240f..ab28b72 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -24,10 +24,13 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. +import functools +import multiprocessing import os import os.path import platform import pathlib +import re import requests import semver import shutil @@ -35,23 +38,30 @@ import sys import subprocess import tempfile +import platformdirs + +@functools.cache +def _git_version(): + version_str = subprocess.check_output(["git", "--version"], encoding="ascii", errors="replace") + version_str = re.search("([0-9]\.*)*[0-9]", version_str).group(0) + return tuple(int(part) for part in version_str.split(".")) + +def git_filter_arg(): + clone_supports_filter = ( + False if "NO_USE_CLONE_FILTER" in os.environ else _git_version() >= (2, 36, 0) + ) + + if clone_supports_filter: + return ["--filter=blob:none"] + else: + return [] # pyproject.toml `py_modules` values that are incorrect. These should all have PRs filed! # and should be removed when the fixed version is incorporated in its respective bundle. pyproject_py_modules_blocklist = set(( - # adafruit bundle - "adafruit_colorsys", - # community bundle "at24mac_eeprom", - "circuitpython_Candlesticks", - "CircuitPython_Color_Picker", - "CircuitPython_Equalizer", - "CircuitPython_Scales", - "circuitPython_Slider", - "circuitpython_uboxplot", - "P1AM", "p1am_200_helpers", )) @@ -60,6 +70,8 @@ else: from tomli import loads as load_toml +mpy_cross_path = platformdirs.user_cache_path("circuitpython-build-tools", ensure_exists=True) + def load_pyproject_toml(lib_path: pathlib.Path): try: return load_toml((lib_path / "pyproject.toml") .read_text(encoding="utf-8")) @@ -106,9 +118,14 @@ def version_string(path=None, *, valid_semver=False): version = commitish return version -def mpy_cross(mpy_cross_filename, circuitpython_tag, quiet=False): +def mpy_cross(version, quiet=False): + circuitpython_tag = version["tag"] + name = version["name"] + ext = ".exe" * (os.name == "nt") + mpy_cross_filename = mpy_cross_path / f"mpy-cross-{name}{ext}" + if os.path.isfile(mpy_cross_filename): - return + return mpy_cross_filename # Try to pull from S3 uname = platform.uname() @@ -136,7 +153,7 @@ def mpy_cross(mpy_cross_filename, circuitpython_tag, quiet=False): os.chmod(mpy_cross_filename, os.stat(mpy_cross_filename)[0] | stat.S_IXUSR) if not quiet: print(" FOUND") - return + return mpy_cross_filename except Exception as e: if not quiet: print(f" exception fetching from S3: {e}") @@ -149,26 +166,21 @@ def mpy_cross(mpy_cross_filename, circuitpython_tag, quiet=False): print(title) print("=" * len(title)) - os.makedirs("build_deps/", exist_ok=True) - if not os.path.isdir("build_deps/circuitpython"): - clone = subprocess.run("git clone https://github.com/adafruit/circuitpython.git build_deps/circuitpython", shell=True) - if clone.returncode != 0: - sys.exit(clone.returncode) + build_dir = mpy_cross_path / f"build-circuitpython-{circuitpython_tag}" + if not os.path.isdir(build_dir): + subprocess.check_call(["git", "clone", *git_filter_arg(), "-b", circuitpython_tag, "https://github.com/adafruit/circuitpython.git", build_dir]) - current_dir = os.getcwd() - os.chdir("build_deps/circuitpython") - make = subprocess.run("git fetch && git checkout {TAG} && git submodule update".format(TAG=circuitpython_tag), shell=True) - os.chdir("tools") - make = subprocess.run("git submodule update --init .", shell=True) - os.chdir("../mpy-cross") - make = subprocess.run("make clean && make", shell=True) - os.chdir(current_dir) + subprocess.check_call(["git", "submodule", "update", "--recursive"], cwd=build_dir) + subprocess.check_call([sys.executable, "tools/ci_fetch_deps.py", "mpy-cross"], cwd=build_dir) + subprocess.check_call(["make", "clean"], cwd=build_dir / "mpy-cross") + subprocess.check_call(["make", f"-j{multiprocessing.cpu_count()}"], cwd=build_dir / "mpy-cross") - if make.returncode != 0: - print("Failed to build mpy-cross from source... bailing out") - sys.exit(make.returncode) + mpy_built = build_dir / f"mpy-cross/build/mpy-cross{ext}" + if not os.path.exists(mpy_built): + mpy_built = build_dir / f"mpy-cross/mpy-cross{ext}" - shutil.copy("build_deps/circuitpython/mpy-cross/mpy-cross", mpy_cross_filename) + shutil.copy(mpy_built, mpy_cross_filename) + return mpy_cross_filename def _munge_to_temp(original_path, temp_file, library_version): with open(original_path, "r", encoding="utf-8") as original_file: @@ -276,12 +288,6 @@ def library(library_path, output_directory, package_folder_prefix, example_files = package_info["example_files"] module_name = package_info["module_name"] - for fn in example_files: - base_dir = os.path.join(output_directory.replace("/lib", "/"), - fn.relative_to(library_path).parent) - if not os.path.isdir(base_dir): - os.makedirs(base_dir) - for fn in py_package_files: base_dir = os.path.join(output_directory, fn.relative_to(library_path).parent) @@ -311,7 +317,7 @@ def library(library_path, output_directory, package_folder_prefix, if mpy_success != 0: raise RuntimeError("mpy-cross failed on", full_path) else: - shutil.copyfile(full_path, output_file) + shutil.copyfile(temp_file_name, output_file) finally: os.remove(temp_file_name) else: @@ -338,6 +344,12 @@ def library(library_path, output_directory, package_folder_prefix, for filename in example_files: full_path = os.path.join(library_path, filename) + + relative_filename_parts = list(filename.relative_to(library_path).parts) + relative_filename_parts.insert(1, library_path.split(os.path.sep)[-1]) + final_relative_filename = os.path.join(*relative_filename_parts) output_file = os.path.join(output_directory.replace("/lib", "/"), - filename.relative_to(library_path)) + final_relative_filename) + + os.makedirs(os.path.join(*output_file.split(os.path.sep)[:-1]), exist_ok=True) shutil.copyfile(full_path, output_file) diff --git a/circuitpython_build_tools/scripts/build_bundles.py b/circuitpython_build_tools/scripts/build_bundles.py index 596b7dc..c7fad1e 100755 --- a/circuitpython_build_tools/scripts/build_bundles.py +++ b/circuitpython_build_tools/scripts/build_bundles.py @@ -26,7 +26,6 @@ import os import os.path import re -import shlex import shutil import subprocess import sys @@ -37,7 +36,10 @@ from circuitpython_build_tools import build from circuitpython_build_tools import target_versions -import pkg_resources +if sys.version_info < (3, 8): + import importlib_metadata +else: + import importlib.metadata as importlib_metadata BLINKA_LIBRARIES = [ "adafruit-blinka", @@ -115,6 +117,10 @@ def build_bundle_json(libs, bundle_version, output_filename, package_folder_pref Generate a JSON file of all the libraries in libs """ packages = {} + # TODO simplify this 2-step process + # It mostly exists so that get_bundle_requirements has a way to look up + # "pypi name to bundle name" via `package_list[pypi_name]["module_name"]` + # otherwise it's just shuffling info around for library_path in libs: package = {} package_info = build.get_package_info(library_path, package_folder_prefix) @@ -130,15 +136,16 @@ def build_bundle_json(libs, bundle_version, output_filename, package_folder_pref packages[module_name] = package library_submodules = {} - for id in packages: + for package in packages.values(): library = {} - library["package"] = packages[id]["is_folder"] - library["pypi_name"] = packages[id]["pypi_name"] - library["version"] = packages[id]["version"] - library["repo"] = packages[id]["repo"] - library["path"] = packages[id]["path"] - library["dependencies"], library["external_dependencies"] = get_bundle_requirements(packages[id]["library_path"], packages) - library_submodules[packages[id]["module_name"]] = library + library["package"] = package["is_folder"] + library["pypi_name"] = package["pypi_name"] + library["version"] = package["version"] + library["repo"] = package["repo"] + library["path"] = package["path"] + library["dependencies"], library["external_dependencies"] = get_bundle_requirements(package["library_path"], packages) + library_submodules[package["module_name"]] = library + out_file = open(output_filename, "w") json.dump(library_submodules, out_file, sort_keys=True) out_file.close() @@ -202,7 +209,7 @@ def build_bundle(libs, bundle_version, output_filename, package_folder_prefix, print() print("Zipping") - with zipfile.ZipFile(output_filename, 'w') as bundle: + with zipfile.ZipFile(output_filename, 'w', compression=zipfile.ZIP_DEFLATED) as bundle: build_metadata = {"build-tools-version": build_tools_version} bundle.comment = json.dumps(build_metadata).encode("utf-8") if multiple_libs: @@ -227,6 +234,7 @@ def _find_libraries(current_path, depth): subdirectories.extend(_find_libraries(path, depth - 1)) return subdirectories +all_modules = ["py", "mpy", "example", "json"] @click.command() @click.option('--filename_prefix', required=True, help="Filename prefix for the output zip files.") @click.option('--output_directory', default="bundles", help="Output location for the zip files.") @@ -234,8 +242,9 @@ def _find_libraries(current_path, depth): @click.option('--library_depth', default=0, help="Depth of library folders. This is useful when multiple libraries are bundled together but are initially in separate subfolders.") @click.option('--package_folder_prefix', default="adafruit_", help="Prefix string used to determine package folders to bundle.") @click.option('--remote_name', default="origin", help="Git remote name to use during building") -@click.option('--ignore', "-i", multiple=True, type=click.Choice(["py", "mpy", "example", "json"]), help="Bundles to ignore building") -def build_bundles(filename_prefix, output_directory, library_location, library_depth, package_folder_prefix, remote_name, ignore): +@click.option('--ignore', "-i", multiple=True, type=click.Choice(all_modules), help="Bundles to ignore building") +@click.option('--only', "-o", multiple=True, type=click.Choice(all_modules), help="Bundles to build building") +def build_bundles(filename_prefix, output_directory, library_location, library_depth, package_folder_prefix, remote_name, ignore, only): os.makedirs(output_directory, exist_ok=True) package_folder_prefix = package_folder_prefix.split(", ") @@ -244,10 +253,10 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d libs = _find_libraries(os.path.abspath(library_location), library_depth) - pkg = pkg_resources.get_distribution("circuitpython-build-tools") - build_tools_version = "devel" - if pkg: - build_tools_version = pkg.version + try: + build_tools_version = importlib_metadata.version("circuitpython-build-tools") + except importlib_metadata.PackageNotFoundError: + build_tools_version = "devel" build_tools_fn = "z-build_tools_version-{}.ignore".format( build_tools_version) @@ -255,6 +264,11 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d with open(build_tools_fn, "w") as f: f.write(build_tools_version) + if ignore and only: + raise SystemExit("Only specify one of --ignore / --only") + if only: + ignore = set(all_modules) - set(only) + # Build raw source .py bundle if "py" not in ignore: zip_filename = os.path.join(output_directory, @@ -265,15 +279,8 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d # Build .mpy bundle(s) if "mpy" not in ignore: - os.makedirs("build_deps", exist_ok=True) for version in target_versions.VERSIONS: - # Use prebuilt mpy-cross on Travis, otherwise build our own. - if "TRAVIS" in os.environ: - mpy_cross = pkg_resources.resource_filename( - target_versions.__name__, "data/mpy-cross-" + version["name"]) - else: - mpy_cross = "build_deps/mpy-cross-" + version["name"] + (".exe" * (os.name == "nt")) - build.mpy_cross(mpy_cross, version["tag"]) + mpy_cross = build.mpy_cross(version) zip_filename = os.path.join(output_directory, filename_prefix + '-{TAG}-mpy-{VERSION}.zip'.format( TAG=version["name"], diff --git a/circuitpython_build_tools/scripts/build_mpy_cross.py b/circuitpython_build_tools/scripts/build_mpy_cross.py index 9abe0da..4b79aca 100644 --- a/circuitpython_build_tools/scripts/build_mpy_cross.py +++ b/circuitpython_build_tools/scripts/build_mpy_cross.py @@ -28,9 +28,14 @@ import os import sys +import click + +@click.command +@click.argument("versions") +def main(versions): + print(versions) + for version in [v for v in target_versions.VERSIONS if v['name'] in versions]: + print(f"{version['name']}: {build.mpy_cross(version)}") + if __name__ == "__main__": - output_directory = sys.argv[1] - os.makedirs(output_directory, exist_ok=True) - for version in target_versions.VERSIONS: - mpy_cross = output_directory + "/mpy-cross-" + version["name"] - build.mpy_cross(mpy_cross, version["tag"]) + main() diff --git a/circuitpython_build_tools/scripts/circuitpython_mpy_cross.py b/circuitpython_build_tools/scripts/circuitpython_mpy_cross.py new file mode 100644 index 0000000..d8e5cb2 --- /dev/null +++ b/circuitpython_build_tools/scripts/circuitpython_mpy_cross.py @@ -0,0 +1,21 @@ +import subprocess + +import click + +from ..target_versions import VERSIONS +from ..build import mpy_cross + +@click.command(context_settings={"ignore_unknown_options": True}) +@click.option("--circuitpython-version", type=click.Choice([version["name"] for version in VERSIONS])) +@click.option("--quiet/--no-quiet", "quiet", type=bool, default=True) +@click.argument("mpy-cross-args", nargs=-1, required=True) +def main(circuitpython_version, quiet, mpy_cross_args): + version_info, = [v for v in VERSIONS if v["name"] == circuitpython_version] + mpy_cross_exe = str(mpy_cross(version_info, quiet)) + try: + subprocess.check_call([mpy_cross_exe, *mpy_cross_args]) + except subprocess.CalledProcessError as e: + raise SystemExit(e.returncode) + +if __name__ == '__main__': + main() diff --git a/circuitpython_build_tools/target_versions.py b/circuitpython_build_tools/target_versions.py index e6afad0..8bd2058 100644 --- a/circuitpython_build_tools/target_versions.py +++ b/circuitpython_build_tools/target_versions.py @@ -25,6 +25,6 @@ # The tag specifies which version of CircuitPython to use for mpy-cross. # The name is used when constructing the zip file names. VERSIONS = [ - {"tag": "8.2.0", "name": "8.x"}, - {"tag": "9.0.0-alpha.2", "name": "9.x"}, + {"tag": "9.2.4", "name": "9.x"}, + {"tag": "10.0.0-alpha.2", "name": "10.x"}, ] diff --git a/requirements.txt b/requirements.txt index 8a3514c..b11b4c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ requests semver wheel tomli; python_version < "3.11" +platformdirs diff --git a/setup.py b/setup.py index 9b33317..4358300 100644 --- a/setup.py +++ b/setup.py @@ -13,10 +13,11 @@ 'circuitpython_build_tools.scripts'], package_data={'circuitpython_build_tools': ['data/mpy-cross-*']}, zip_safe=False, - python_requires='>=3.7', - install_requires=['Click', 'requests', 'semver', 'tomli; python_version < "3.11"'], + python_requires='>=3.10', + install_requires=['Click', 'requests', 'semver', 'tomli; python_version < "3.11"', 'platformdirs'], entry_points=''' [console_scripts] circuitpython-build-bundles=circuitpython_build_tools.scripts.build_bundles:build_bundles + circuitpython-mpy-cross=circuitpython_build_tools.scripts.circuitpython_mpy_cross:main ''' )