diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index 0fc240f..16ff279 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -24,10 +24,13 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. +import functools +import multiprocessing import os import os.path import platform import pathlib +import re import requests import semver import shutil @@ -35,6 +38,23 @@ import sys import subprocess import tempfile +import platformdirs + +@functools.cache +def _git_version(): + version_str = subprocess.check_output(["git", "--version"], encoding="ascii", errors="replace") + version_str = re.search("([0-9]\.*)*[0-9]", version_str).group(0) + return tuple(int(part) for part in version_str.split(".")) + +def git_filter_arg(): + clone_supports_filter = ( + False if "NO_USE_CLONE_FILTER" in os.environ else _git_version() >= (2, 36, 0) + ) + + if clone_supports_filter: + return ["--filter=blob:none"] + else: + return [] # pyproject.toml `py_modules` values that are incorrect. These should all have PRs filed! # and should be removed when the fixed version is incorporated in its respective bundle. @@ -60,6 +80,8 @@ else: from tomli import loads as load_toml +mpy_cross_path = platformdirs.user_cache_path("circuitpython-build-tools", ensure_exists=True) + def load_pyproject_toml(lib_path: pathlib.Path): try: return load_toml((lib_path / "pyproject.toml") .read_text(encoding="utf-8")) @@ -106,9 +128,14 @@ def version_string(path=None, *, valid_semver=False): version = commitish return version -def mpy_cross(mpy_cross_filename, circuitpython_tag, quiet=False): +def mpy_cross(version, quiet=False): + circuitpython_tag = version["tag"] + name = version["name"] + ext = ".exe" * (os.name == "nt") + mpy_cross_filename = mpy_cross_path / f"mpy-cross-{name}{ext}" + if os.path.isfile(mpy_cross_filename): - return + return mpy_cross_filename # Try to pull from S3 uname = platform.uname() @@ -136,7 +163,7 @@ def mpy_cross(mpy_cross_filename, circuitpython_tag, quiet=False): os.chmod(mpy_cross_filename, os.stat(mpy_cross_filename)[0] | stat.S_IXUSR) if not quiet: print(" FOUND") - return + return mpy_cross_filename except Exception as e: if not quiet: print(f" exception fetching from S3: {e}") @@ -149,26 +176,21 @@ def mpy_cross(mpy_cross_filename, circuitpython_tag, quiet=False): print(title) print("=" * len(title)) - os.makedirs("build_deps/", exist_ok=True) - if not os.path.isdir("build_deps/circuitpython"): - clone = subprocess.run("git clone https://github.com/adafruit/circuitpython.git build_deps/circuitpython", shell=True) - if clone.returncode != 0: - sys.exit(clone.returncode) - - current_dir = os.getcwd() - os.chdir("build_deps/circuitpython") - make = subprocess.run("git fetch && git checkout {TAG} && git submodule update".format(TAG=circuitpython_tag), shell=True) - os.chdir("tools") - make = subprocess.run("git submodule update --init .", shell=True) - os.chdir("../mpy-cross") - make = subprocess.run("make clean && make", shell=True) - os.chdir(current_dir) - - if make.returncode != 0: - print("Failed to build mpy-cross from source... bailing out") - sys.exit(make.returncode) - - shutil.copy("build_deps/circuitpython/mpy-cross/mpy-cross", mpy_cross_filename) + build_dir = mpy_cross_path / f"build-circuitpython-{circuitpython_tag}" + if not os.path.isdir(build_dir): + subprocess.check_call(["git", "clone", *git_filter_arg(), "-b", circuitpython_tag, "https://github.com/adafruit/circuitpython.git", build_dir]) + + subprocess.check_call(["git", "submodule", "update", "--recursive"], cwd=build_dir) + subprocess.check_call([sys.executable, "tools/ci_fetch_deps.py", "mpy-cross"], cwd=build_dir) + subprocess.check_call(["make", "clean"], cwd=build_dir / "mpy-cross") + subprocess.check_call(["make", f"-j{multiprocessing.cpu_count()}"], cwd=build_dir / "mpy-cross") + + mpy_built = build_dir / f"mpy-cross/build/mpy-cross{ext}" + if not os.path.exists(mpy_built): + mpy_built = build_dir / f"mpy-cross/mpy-cross{ext}" + + shutil.copy(mpy_built, mpy_cross_filename) + return mpy_cross_filename def _munge_to_temp(original_path, temp_file, library_version): with open(original_path, "r", encoding="utf-8") as original_file: diff --git a/circuitpython_build_tools/scripts/build_bundles.py b/circuitpython_build_tools/scripts/build_bundles.py index 3c31185..c7fad1e 100755 --- a/circuitpython_build_tools/scripts/build_bundles.py +++ b/circuitpython_build_tools/scripts/build_bundles.py @@ -41,7 +41,6 @@ else: import importlib.metadata as importlib_metadata - BLINKA_LIBRARIES = [ "adafruit-blinka", "adafruit-blinka-bleio", @@ -280,10 +279,8 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d # Build .mpy bundle(s) if "mpy" not in ignore: - os.makedirs("build_deps", exist_ok=True) for version in target_versions.VERSIONS: - mpy_cross = "build_deps/mpy-cross-" + version["name"] + (".exe" * (os.name == "nt")) - build.mpy_cross(mpy_cross, version["tag"]) + mpy_cross = build.mpy_cross(version) zip_filename = os.path.join(output_directory, filename_prefix + '-{TAG}-mpy-{VERSION}.zip'.format( TAG=version["name"], diff --git a/circuitpython_build_tools/scripts/build_mpy_cross.py b/circuitpython_build_tools/scripts/build_mpy_cross.py index 9abe0da..4b79aca 100644 --- a/circuitpython_build_tools/scripts/build_mpy_cross.py +++ b/circuitpython_build_tools/scripts/build_mpy_cross.py @@ -28,9 +28,14 @@ import os import sys +import click + +@click.command +@click.argument("versions") +def main(versions): + print(versions) + for version in [v for v in target_versions.VERSIONS if v['name'] in versions]: + print(f"{version['name']}: {build.mpy_cross(version)}") + if __name__ == "__main__": - output_directory = sys.argv[1] - os.makedirs(output_directory, exist_ok=True) - for version in target_versions.VERSIONS: - mpy_cross = output_directory + "/mpy-cross-" + version["name"] - build.mpy_cross(mpy_cross, version["tag"]) + main() diff --git a/circuitpython_build_tools/scripts/circuitpython_mpy_cross.py b/circuitpython_build_tools/scripts/circuitpython_mpy_cross.py new file mode 100644 index 0000000..d8e5cb2 --- /dev/null +++ b/circuitpython_build_tools/scripts/circuitpython_mpy_cross.py @@ -0,0 +1,21 @@ +import subprocess + +import click + +from ..target_versions import VERSIONS +from ..build import mpy_cross + +@click.command(context_settings={"ignore_unknown_options": True}) +@click.option("--circuitpython-version", type=click.Choice([version["name"] for version in VERSIONS])) +@click.option("--quiet/--no-quiet", "quiet", type=bool, default=True) +@click.argument("mpy-cross-args", nargs=-1, required=True) +def main(circuitpython_version, quiet, mpy_cross_args): + version_info, = [v for v in VERSIONS if v["name"] == circuitpython_version] + mpy_cross_exe = str(mpy_cross(version_info, quiet)) + try: + subprocess.check_call([mpy_cross_exe, *mpy_cross_args]) + except subprocess.CalledProcessError as e: + raise SystemExit(e.returncode) + +if __name__ == '__main__': + main() diff --git a/requirements.txt b/requirements.txt index 8a3514c..b11b4c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ requests semver wheel tomli; python_version < "3.11" +platformdirs diff --git a/setup.py b/setup.py index 0d2bbc7..4358300 100644 --- a/setup.py +++ b/setup.py @@ -14,9 +14,10 @@ package_data={'circuitpython_build_tools': ['data/mpy-cross-*']}, zip_safe=False, python_requires='>=3.10', - install_requires=['Click', 'requests', 'semver', 'tomli; python_version < "3.11"'], + install_requires=['Click', 'requests', 'semver', 'tomli; python_version < "3.11"', 'platformdirs'], entry_points=''' [console_scripts] circuitpython-build-bundles=circuitpython_build_tools.scripts.build_bundles:build_bundles + circuitpython-mpy-cross=circuitpython_build_tools.scripts.circuitpython_mpy_cross:main ''' )