From ea6fc3d216dc3cb3a2484d4969ab31bd4807b18b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= Date: Mon, 16 May 2022 17:19:49 +0200 Subject: [PATCH 1/4] CI Use lock file for CircleCI arm64 build --- .circleci/config.yml | 9 +- build_tools/circle/build_test_arm.sh | 43 +-------- .../circle/py39_conda_forge_environment.yml | 18 ++++ .../py39_conda_forge_linux-aarch64_conda.lock | 89 +++++++++++++++++++ .../update_environments_and_lock_files.py | 64 +++++++++---- 5 files changed, 159 insertions(+), 64 deletions(-) create mode 100644 build_tools/circle/py39_conda_forge_environment.yml create mode 100644 build_tools/circle/py39_conda_forge_linux-aarch64_conda.lock rename build_tools/{azure => }/update_environments_and_lock_files.py (86%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 90098519eee0f..ea3e4ac4c808c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -120,14 +120,7 @@ jobs: - PYTHON_VERSION: '3.9' - OMP_NUM_THREADS: 2 - OPENBLAS_NUM_THREADS: 2 - - NUMPY_VERSION: 'latest' - - SCIPY_VERSION: 'latest' - - CYTHON_VERSION: 'latest' - - JOBLIB_VERSION: 'latest' - - THREADPOOLCTL_VERSION: 'latest' - - PYTEST_VERSION: 'latest' - - PYTEST_XDIST_VERSION: 'latest' - - TEST_DOCSTRINGS: 'true' + - CONDA_ENV_NAME: testenv steps: - checkout - run: ./build_tools/circle/checkout_merge_commit.sh diff --git a/build_tools/circle/build_test_arm.sh b/build_tools/circle/build_test_arm.sh index c086f3c7813a4..7376244b2203a 100755 --- a/build_tools/circle/build_test_arm.sh +++ b/build_tools/circle/build_test_arm.sh @@ -6,7 +6,6 @@ set -x UNAMESTR=`uname` N_CORES=`nproc --all` - setup_ccache() { echo "Setting up ccache" mkdir /tmp/ccache/ @@ -20,13 +19,6 @@ setup_ccache() { ccache -M 0 } -# imports get_dep -source build_tools/shared.sh - -sudo add-apt-repository --remove ppa:ubuntu-toolchain-r/test -sudo apt-get update - -# Setup conda environment MINICONDA_URL="https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-aarch64.sh" # Install Mambaforge @@ -36,40 +28,13 @@ chmod +x mambaforge.sh && ./mambaforge.sh -b -p $MINICONDA_PATH export PATH=$MINICONDA_PATH/bin:$PATH mamba init --all --verbose mamba update --yes conda +# TODO Update conda-lock version from time to time +mamba install conda-lock=1.0.5 -y +conda-lock install --name $CONDA_ENV_NAME $LOCK_FILE +source activate $CONDA_ENV_NAME -# Create environment and install dependencies -mamba create -n testenv --yes $(get_dep python $PYTHON_VERSION) -source activate testenv - -# pin pip to 22.0.4 because pip 22.1 validates build dependencies in -# pyproject.toml. oldest-supported-numpy is part of the build dependencies in -# pyproject.toml so using pip 22.1 will cause an error since -# oldest-supported-numpy is not really meant to be installed in the -# environment. See https://github.com/scikit-learn/scikit-learn/pull/23336 for -# more details. -mamba install --verbose -y ccache \ - pip==22.0.4 \ - $(get_dep numpy $NUMPY_VERSION) \ - $(get_dep scipy $SCIPY_VERSION) \ - $(get_dep cython $CYTHON_VERSION) \ - $(get_dep joblib $JOBLIB_VERSION) \ - $(get_dep threadpoolctl $THREADPOOLCTL_VERSION) \ - $(get_dep pytest $PYTEST_VERSION) \ - $(get_dep pytest-xdist $PYTEST_XDIST_VERSION) setup_ccache -if [[ "$COVERAGE" == "true" ]]; then - # XXX: coverage is temporary pinned to 6.2 because 6.3 is not fork-safe - # cf. https://github.com/nedbat/coveragepy/issues/1310 - mamba install --verbose -y codecov pytest-cov coverage=6.2 -fi - -if [[ "$TEST_DOCSTRINGS" == "true" ]]; then - # numpydoc requires sphinx - mamba install --verbose -y sphinx - mamba install --verbose -y numpydoc -fi - python --version # Set parallelism to $N_CORES + 1 to overlap IO bound tasks with CPU bound tasks on CI diff --git a/build_tools/circle/py39_conda_forge_environment.yml b/build_tools/circle/py39_conda_forge_environment.yml new file mode 100644 index 0000000000000..eb78623e03e76 --- /dev/null +++ b/build_tools/circle/py39_conda_forge_environment.yml @@ -0,0 +1,18 @@ +# DO NOT EDIT: this file is generated from the specification found in the +# following script to centralize the configuration for all Azure CI builds: +# build_tools/azure/update_environments_and_lock_files.py +channels: + - conda-forge +dependencies: + - python=3.9 + - numpy + - blas + - scipy + - cython + - joblib + - threadpoolctl + - matplotlib + - pytest=6.2.5 + - pytest-xdist + - pillow + - pip=22.0.4 diff --git a/build_tools/circle/py39_conda_forge_linux-aarch64_conda.lock b/build_tools/circle/py39_conda_forge_linux-aarch64_conda.lock new file mode 100644 index 0000000000000..138c4cdc11549 --- /dev/null +++ b/build_tools/circle/py39_conda_forge_linux-aarch64_conda.lock @@ -0,0 +1,89 @@ +# Generated by conda-lock. +# platform: linux-aarch64 +# input_hash: 5e1d4e50b6a9b4fdcb21ed2b944933a94780d4bbb9e8e8055da4a4778d048024 +@EXPLICIT +https://conda.anaconda.org/conda-forge/linux-aarch64/ca-certificates-2021.10.8-h4fd8a4c_0.tar.bz2#ad855209fcca3b45da677d409b16e021 +https://conda.anaconda.org/conda-forge/linux-aarch64/ld_impl_linux-aarch64-2.36.1-h02ad14f_2.tar.bz2#3ca1a8e406eab04ffc3bfa6e8ac0a724 +https://conda.anaconda.org/conda-forge/linux-aarch64/libgfortran5-12.1.0-h41d5c85_16.tar.bz2#f053ad62fdac14fb8e73cfed4e8d2676 +https://conda.anaconda.org/conda-forge/linux-aarch64/libstdcxx-ng-12.1.0-hd01590b_16.tar.bz2#b64391bb81cc2f914d57c0927ec8a26b +https://conda.anaconda.org/conda-forge/noarch/tzdata-2022a-h191b570_0.tar.bz2#84be5301069417a2221187d2f435e0f7 +https://conda.anaconda.org/conda-forge/linux-aarch64/libgfortran-ng-12.1.0-he9431aa_16.tar.bz2#69e5a58bbd94c934277f715160c1f0b5 +https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-2_kmp_llvm.tar.bz2#98a1185182fec3c434069fa74e6473d6 +https://conda.anaconda.org/conda-forge/linux-aarch64/libgcc-ng-12.1.0-h3242a24_16.tar.bz2#70e9f0947c17f3faf1a1974be0c110bf +https://conda.anaconda.org/conda-forge/linux-aarch64/bzip2-1.0.8-hf897c2e_4.tar.bz2#2d787570a729e273a4e75775ddf3348a +https://conda.anaconda.org/conda-forge/linux-aarch64/giflib-5.2.1-hb9de7d4_2.tar.bz2#08575a68724ff44a0cb24c6158511c0a +https://conda.anaconda.org/conda-forge/linux-aarch64/jbig-2.1-hf897c2e_2003.tar.bz2#a3d4206bcecf771e286e39155fe00b24 +https://conda.anaconda.org/conda-forge/linux-aarch64/jpeg-9e-h9cdd2b7_1.tar.bz2#584d93de10ab8d47874c5f29c29ba729 +https://conda.anaconda.org/conda-forge/linux-aarch64/lerc-3.0-h01db608_0.tar.bz2#13505433bc073878ec237f8ff8ba1ea5 +https://conda.anaconda.org/conda-forge/linux-aarch64/libbrotlicommon-1.0.9-h4e544f5_7.tar.bz2#6ee071311281942e2fec227751e7efff +https://conda.anaconda.org/conda-forge/linux-aarch64/libdeflate-1.10-hf897c2e_0.tar.bz2#cf24cc05d49de7c6a981f2d485cb6fb6 +https://conda.anaconda.org/conda-forge/linux-aarch64/libffi-3.4.2-h3557bc0_5.tar.bz2#dddd85f4d52121fab0a8b099c5e06501 +https://conda.anaconda.org/conda-forge/linux-aarch64/libnsl-2.0.0-hf897c2e_0.tar.bz2#36fdbc05c9d9145ece86f5a63c3f352e +https://conda.anaconda.org/conda-forge/linux-aarch64/libopenblas-0.3.20-pthreads_h6cb6f83_0.tar.bz2#1110034f2f90ca3c7ea35bf0d2eea15e +https://conda.anaconda.org/conda-forge/linux-aarch64/libuuid-2.32.1-hf897c2e_1000.tar.bz2#e038da5ef9095b0d79aac14a311394e7 +https://conda.anaconda.org/conda-forge/linux-aarch64/libwebp-base-1.2.2-hf897c2e_1.tar.bz2#833c405d24bf7e52022b61108e78028a +https://conda.anaconda.org/conda-forge/linux-aarch64/libzlib-1.2.11-h4e544f5_1014.tar.bz2#e829241b520bfd3187d921f82e9b4460 +https://conda.anaconda.org/conda-forge/linux-aarch64/lz4-c-1.9.3-h01db608_1.tar.bz2#25b5ec27b49b04a997a87b0f00f5e205 +https://conda.anaconda.org/conda-forge/linux-aarch64/ncurses-6.3-headf329_1.tar.bz2#486b68148e121bc8bbadc3cefae4c04f +https://conda.anaconda.org/conda-forge/linux-aarch64/openssl-3.0.3-h4e544f5_0.tar.bz2#19a1e30ba6680f2ad0795d985628663c +https://conda.anaconda.org/conda-forge/linux-aarch64/pthread-stubs-0.4-hb9de7d4_1001.tar.bz2#d0183ec6ce0b5aaa3486df25fa5f0ded +https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxau-1.0.9-h3557bc0_0.tar.bz2#e0c187f5ce240897762bbb89a8a407cc +https://conda.anaconda.org/conda-forge/linux-aarch64/xorg-libxdmcp-1.1.3-h3557bc0_0.tar.bz2#a6c9016ae1ca5c47a3603ed4cd65fedd +https://conda.anaconda.org/conda-forge/linux-aarch64/xz-5.2.5-h6dd45c4_1.tar.bz2#9da6f161d0f47f8a29fa708cda8deda2 +https://conda.anaconda.org/conda-forge/linux-aarch64/libblas-3.9.0-14_linuxaarch64_openblas.tar.bz2#e8a37033814609b27d188b35c134d007 +https://conda.anaconda.org/conda-forge/linux-aarch64/libbrotlidec-1.0.9-h4e544f5_7.tar.bz2#6839e1d20fce65ff5f0f0bcf32841993 +https://conda.anaconda.org/conda-forge/linux-aarch64/libbrotlienc-1.0.9-h4e544f5_7.tar.bz2#353cfe995ca6183994d5eb3e674ab42c +https://conda.anaconda.org/conda-forge/linux-aarch64/libxcb-1.13-h3557bc0_1004.tar.bz2#cc973f5f452272c397546eac588cddb3 +https://conda.anaconda.org/conda-forge/linux-aarch64/llvm-openmp-14.0.3-hb2805f8_0.tar.bz2#0096034fdd07049c70f40853655ad8b3 +https://conda.anaconda.org/conda-forge/linux-aarch64/openblas-0.3.20-pthreads_h2d9dd7e_0.tar.bz2#db1b9433e8d04b2591111d75be683d5e +https://conda.anaconda.org/conda-forge/linux-aarch64/readline-8.1-h1a49cc3_0.tar.bz2#ccd3c3e1bde615ec934282f3481a1ede +https://conda.anaconda.org/conda-forge/linux-aarch64/tk-8.6.12-hd8af866_0.tar.bz2#7894e82ff743bd96c76585ddebe28e2a +https://conda.anaconda.org/conda-forge/linux-aarch64/zlib-1.2.11-h4e544f5_1014.tar.bz2#5d6528db5739f248a7a5749d2dba8a27 +https://conda.anaconda.org/conda-forge/linux-aarch64/zstd-1.5.2-h41fb7a4_0.tar.bz2#03434131cc82915a59513e1ab989fdd5 +https://conda.anaconda.org/conda-forge/linux-aarch64/brotli-bin-1.0.9-h4e544f5_7.tar.bz2#4014ebf8c97d8cb219bfc0a12344ceb6 +https://conda.anaconda.org/conda-forge/linux-aarch64/libcblas-3.9.0-14_linuxaarch64_openblas.tar.bz2#8a1d66921d3e7dacacc8dd3af6d5ec5f +https://conda.anaconda.org/conda-forge/linux-aarch64/liblapack-3.9.0-14_linuxaarch64_openblas.tar.bz2#1dcb7e50e399fc73824320b9ea709477 +https://conda.anaconda.org/conda-forge/linux-aarch64/libpng-1.6.37-hbd635b3_2.tar.bz2#c95d3cee5815206d562fffa531a60a86 +https://conda.anaconda.org/conda-forge/linux-aarch64/libtiff-4.3.0-hdea21e4_3.tar.bz2#70385b4e904cf3fc9622319532c1a9eb +https://conda.anaconda.org/conda-forge/linux-aarch64/sqlite-3.38.5-hc74f5b8_0.tar.bz2#ac6e9256d71294f14a6aea6e1e537f37 +https://conda.anaconda.org/conda-forge/linux-aarch64/brotli-1.0.9-h4e544f5_7.tar.bz2#4c345a372cf4936c8e959b8d266c4396 +https://conda.anaconda.org/conda-forge/linux-aarch64/freetype-2.10.4-hdf53a3c_1.tar.bz2#c0b393690f63cdfb986b430971f09cc2 +https://conda.anaconda.org/conda-forge/linux-aarch64/lcms2-2.12-h012adcb_0.tar.bz2#d112a00b1ba7dda1f79735d2a4661ea3 +https://conda.anaconda.org/conda-forge/linux-aarch64/liblapacke-3.9.0-14_linuxaarch64_openblas.tar.bz2#e18b63ca39fc7b5d7602d88440c0d0b1 +https://conda.anaconda.org/conda-forge/linux-aarch64/libwebp-1.2.2-h21830e5_0.tar.bz2#0b0287f266b3c32803830bbf6307c48d +https://conda.anaconda.org/conda-forge/linux-aarch64/openjpeg-2.4.0-h45037e0_1.tar.bz2#c888c9aa7f6e60f2514f88c070ec6f45 +https://conda.anaconda.org/conda-forge/linux-aarch64/python-3.9.12-h5016f1d_1_cpython.tar.bz2#09cceb4a3e723338d0fbed7c7d22e2a4 +https://conda.anaconda.org/conda-forge/noarch/attrs-21.4.0-pyhd8ed1ab_0.tar.bz2#f70280205d7044c8b8358c8de3190e5d +https://conda.anaconda.org/conda-forge/linux-aarch64/blas-devel-3.9.0-14_linuxaarch64_openblas.tar.bz2#a321d5f472b8b3e9d938540ca7277606 +https://conda.anaconda.org/conda-forge/noarch/cycler-0.11.0-pyhd8ed1ab_0.tar.bz2#a50559fad0affdbb33729a68669ca1cb +https://conda.anaconda.org/conda-forge/noarch/execnet-1.9.0-pyhd8ed1ab_0.tar.bz2#0e521f7a5e60d508b121d38b04874fb2 +https://conda.anaconda.org/conda-forge/noarch/iniconfig-1.1.1-pyh9f0ad1d_0.tar.bz2#39161f81cc5e5ca45b8226fbb06c6905 +https://conda.anaconda.org/conda-forge/noarch/munkres-1.1.4-pyh9f0ad1d_0.tar.bz2#2ba8498c1018c1e9c61eb99b973dfe19 +https://conda.anaconda.org/conda-forge/noarch/py-1.11.0-pyh6c4a22f_0.tar.bz2#b4613d7e7a493916d867842a6a148054 +https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.0.9-pyhd8ed1ab_0.tar.bz2#e8fbc1b54b25f4b08281467bc13b70cc +https://conda.anaconda.org/conda-forge/linux-aarch64/python_abi-3.9-2_cp39.tar.bz2#c74e493d773fa544a312b0904abcfbfb +https://conda.anaconda.org/conda-forge/noarch/six-1.16.0-pyh6c4a22f_0.tar.bz2#e5f25f8dbc060e9a8d912e432202afc2 +https://conda.anaconda.org/conda-forge/noarch/threadpoolctl-3.1.0-pyh8a188c0_0.tar.bz2#a2995ee828f65687ac5b1e71a2ab1e0c +https://conda.anaconda.org/conda-forge/noarch/toml-0.10.2-pyhd8ed1ab_0.tar.bz2#f832c45a477c78bebd107098db465095 +https://conda.anaconda.org/conda-forge/noarch/wheel-0.37.1-pyhd8ed1ab_0.tar.bz2#1ca02aaf78d9c70d9a81a3bed5752022 +https://conda.anaconda.org/conda-forge/linux-aarch64/blas-2.114-openblas.tar.bz2#240259abe13902a22c25ad1ff8082d90 +https://conda.anaconda.org/conda-forge/linux-aarch64/certifi-2021.10.8-py39h4420490_2.tar.bz2#aebb51e6774e3d98b8412e500afa638c +https://conda.anaconda.org/conda-forge/linux-aarch64/cython-0.29.29-py39h3d8bfb9_0.tar.bz2#18507b4cf568e8d1f661d31746d0db11 +https://conda.anaconda.org/conda-forge/linux-aarch64/kiwisolver-1.4.2-py39h110580c_1.tar.bz2#e5136a6aa77ec81ab5c6d3f9ba2ebf1d +https://conda.anaconda.org/conda-forge/linux-aarch64/numpy-1.22.3-py39h451b137_2.tar.bz2#db82cd177da11c34dbbc29bee711a8d7 +https://conda.anaconda.org/conda-forge/noarch/packaging-21.3-pyhd8ed1ab_0.tar.bz2#71f1ab2de48613876becddd496371c85 +https://conda.anaconda.org/conda-forge/linux-aarch64/pillow-9.1.0-py39h2a8e185_2.tar.bz2#073167cf5063f41dd7b2918f9de492b0 +https://conda.anaconda.org/conda-forge/linux-aarch64/pluggy-1.0.0-py39ha65689a_3.tar.bz2#3ccbd600f5d9921e928da3d69a9720a9 +https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.8.2-pyhd8ed1ab_0.tar.bz2#dd999d1cc9f79e67dbb855c8924c7984 +https://conda.anaconda.org/conda-forge/linux-aarch64/setuptools-62.2.0-py39ha65689a_0.tar.bz2#09fcd847a6f83a7c0ac32892c14e069c +https://conda.anaconda.org/conda-forge/linux-aarch64/tornado-6.1-py39hb9a1dbb_3.tar.bz2#11dace3e5ebbaeb5069179cfa5b53923 +https://conda.anaconda.org/conda-forge/linux-aarch64/unicodedata2-14.0.0-py39h0fd3b05_1.tar.bz2#7182266a1f86f367d88c86a9ab560cca +https://conda.anaconda.org/conda-forge/linux-aarch64/fonttools-4.33.3-py39h0fd3b05_0.tar.bz2#0f8d88eb32e53e0756a4bceaa5d85d3e +https://conda.anaconda.org/conda-forge/noarch/joblib-1.1.0-pyhd8ed1ab_0.tar.bz2#07d1b5c8cde14d95998fd4767e1e62d2 +https://conda.anaconda.org/conda-forge/noarch/pip-22.0.4-pyhd8ed1ab_0.tar.bz2#b1239ce8ef2a1eec485c398a683c5bff +https://conda.anaconda.org/conda-forge/linux-aarch64/pytest-6.2.5-py39ha65689a_2.tar.bz2#1b663c678dd3032ef9dbdfd35165091c +https://conda.anaconda.org/conda-forge/linux-aarch64/scipy-1.8.0-py39h59f125f_1.tar.bz2#c82bb1939eae2df3a6b1fa993fb4f3a3 +https://conda.anaconda.org/conda-forge/linux-aarch64/matplotlib-base-3.5.2-py39hfed42d8_0.tar.bz2#9f90790067684c9f0ab1b07e6e82070a +https://conda.anaconda.org/conda-forge/noarch/pytest-forked-1.4.0-pyhd8ed1ab_0.tar.bz2#95286e05a617de9ebfe3246cecbfb72f +https://conda.anaconda.org/conda-forge/linux-aarch64/matplotlib-3.5.2-py39ha65689a_0.tar.bz2#e5738e0c03863f94223c19aaf0f2564d +https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-2.5.0-pyhd8ed1ab_0.tar.bz2#1fdd1f3baccf0deb647385c677a1a48e diff --git a/build_tools/azure/update_environments_and_lock_files.py b/build_tools/update_environments_and_lock_files.py similarity index 86% rename from build_tools/azure/update_environments_and_lock_files.py rename to build_tools/update_environments_and_lock_files.py index d8487f2a808c2..07f108d32616f 100644 --- a/build_tools/azure/update_environments_and_lock_files.py +++ b/build_tools/update_environments_and_lock_files.py @@ -88,6 +88,7 @@ def remove_from(alist, to_remove): conda_build_metadata_list = [ { "build_name": "pylatest_conda_forge_mkl_linux-64", + "folder": "build_tools/azure", "platform": "linux-64", "channel": "conda-forge", "conda_dependencies": common_dependencies + ["ccache"], @@ -97,6 +98,7 @@ def remove_from(alist, to_remove): }, { "build_name": "pylatest_conda_forge_mkl_osx-64", + "folder": "build_tools/azure", "platform": "osx-64", "channel": "conda-forge", "conda_dependencies": common_dependencies @@ -107,6 +109,7 @@ def remove_from(alist, to_remove): }, { "build_name": "pylatest_conda_mkl_no_openmp", + "folder": "build_tools/azure", "platform": "osx-64", "channel": "defaults", "conda_dependencies": common_dependencies + ["ccache"], @@ -116,6 +119,7 @@ def remove_from(alist, to_remove): }, { "build_name": "pylatest_conda_forge_mkl_no_coverage", + "folder": "build_tools/azure", "platform": "linux-64", "channel": "conda-forge", "conda_dependencies": common_dependencies_without_coverage + ["ccache"], @@ -125,6 +129,7 @@ def remove_from(alist, to_remove): }, { "build_name": "py38_conda_defaults_openblas", + "folder": "build_tools/azure", "platform": "linux-64", "channel": "defaults", "conda_dependencies": common_dependencies + ["ccache"], @@ -139,6 +144,7 @@ def remove_from(alist, to_remove): }, { "build_name": "py38_conda_forge_openblas_ubuntu_1804", + "folder": "build_tools/azure", "platform": "linux-64", "channel": "conda-forge", "conda_dependencies": common_dependencies_without_coverage + ["ccache"], @@ -146,6 +152,7 @@ def remove_from(alist, to_remove): }, { "build_name": "pylatest_pip_openblas_pandas", + "folder": "build_tools/azure", "platform": "linux-64", "channel": "defaults", "conda_dependencies": ["python", "ccache"], @@ -156,6 +163,7 @@ def remove_from(alist, to_remove): }, { "build_name": "pylatest_pip_scipy_dev", + "folder": "build_tools/azure", "platform": "linux-64", "channel": "defaults", "conda_dependencies": ["python", "ccache"], @@ -184,6 +192,7 @@ def remove_from(alist, to_remove): }, { "build_name": "pypy3", + "folder": "build_tools/azure", "platform": "linux-64", "channel": "conda-forge", "conda_dependencies": ["pypy"] @@ -193,12 +202,29 @@ def remove_from(alist, to_remove): + ["ccache"], "package_constraints": {"blas": "[build=openblas]"}, }, + { + "build_name": "py39_conda_forge", + "folder": "build_tools/circle", + "platform": "linux-aarch64", + "channel": "conda-forge", + "conda_dependencies": remove_from( + common_dependencies_without_coverage, ["pandas", "pyamg"] + ) + + ["pip"], + "package_constraints": { + "python": "3.9", + # TODO remove constraint when pip > 22.1 is released. See + # https://github.com/pypa/pip/issues/11116 for more details. + "pip": "22.0.4", + }, + }, ] pip_build_metadata_list = [ { "build_name": "debian_atlas_32bit", + "folder": "build_tools/azure", "pip_dependencies": ["cython", "joblib", "threadpoolctl", "pytest"], "package_constraints": { "joblib": "min", @@ -211,6 +237,7 @@ def remove_from(alist, to_remove): }, { "build_name": "ubuntu_atlas", + "folder": "build_tools/azure", "pip_dependencies": [ "cython", "joblib", @@ -301,16 +328,17 @@ def get_conda_environment_content(build_metadata): return template.render(build_metadata=build_metadata) -def write_conda_environment(build_metadata, folder_path): +def write_conda_environment(build_metadata): content = get_conda_environment_content(build_metadata) build_name = build_metadata["build_name"] + folder_path = Path(build_metadata["folder"]) output_path = folder_path / f"{build_name}_environment.yml" output_path.write_text(content) -def write_all_conda_environments(build_metadata_list, folder_path): +def write_all_conda_environments(build_metadata_list): for build_metadata in build_metadata_list: - write_conda_environment(build_metadata, folder_path) + write_conda_environment(build_metadata) def conda_lock(environment_path, lock_file_path, platform): @@ -323,8 +351,9 @@ def conda_lock(environment_path, lock_file_path, platform): execute_command(shlex.split(command)) -def create_conda_lock_file(build_metadata, folder_path): +def create_conda_lock_file(build_metadata): build_name = build_metadata["build_name"] + folder_path = Path(build_metadata["folder"]) environment_path = folder_path / f"{build_name}_environment.yml" platform = build_metadata["platform"] lock_file_basename = build_name @@ -335,10 +364,10 @@ def create_conda_lock_file(build_metadata, folder_path): conda_lock(environment_path, lock_file_path, platform) -def write_all_conda_lock_files(build_metadata_list, folder_path): +def write_all_conda_lock_files(build_metadata_list): for build_metadata in build_metadata_list: logger.info(build_metadata["build_name"]) - create_conda_lock_file(build_metadata, folder_path) + create_conda_lock_file(build_metadata) def get_pip_requirements_content(build_metadata): @@ -354,17 +383,18 @@ def get_pip_requirements_content(build_metadata): return template.render(build_metadata=build_metadata) -def write_pip_requirements(build_metadata, folder_path): +def write_pip_requirements(build_metadata): build_name = build_metadata["build_name"] content = get_pip_requirements_content(build_metadata) + folder_path = Path(build_metadata["folder"]) output_path = folder_path / f"{build_name}_requirements.txt" output_path.write_text(content) -def write_all_pip_requirements(build_metadata_list, folder_path): +def write_all_pip_requirements(build_metadata_list): for build_metadata in build_metadata_list: logger.info(build_metadata["build_name"]) - write_pip_requirements(build_metadata, folder_path) + write_pip_requirements(build_metadata) def pip_compile(pip_compile_path, requirements_path, lock_file_path): @@ -374,7 +404,7 @@ def pip_compile(pip_compile_path, requirements_path, lock_file_path): execute_command(shlex.split(command)) -def write_pip_lock_file(build_metadata, folder_path): +def write_pip_lock_file(build_metadata): build_name = build_metadata["build_name"] python_version = build_metadata["python_version"] environment_name = f"pip-tools-python{python_version}" @@ -396,24 +426,24 @@ def write_pip_lock_file(build_metadata, folder_path): environment_path = Path(environment_folder) pip_compile_path = environment_path / "bin" / "pip-compile" + folder_path = Path(build_metadata["folder"]) requirement_path = folder_path / f"{build_name}_requirements.txt" lock_file_path = folder_path / f"{build_name}_lock.txt" pip_compile(pip_compile_path, requirement_path, lock_file_path) -def write_all_pip_lock_files(build_metadata_list, folder_path): +def write_all_pip_lock_files(build_metadata_list): for build_metadata in build_metadata_list: - write_pip_lock_file(build_metadata, folder_path) + write_pip_lock_file(build_metadata) if __name__ == "__main__": - output_path = Path("build_tools/azure/") logger.info("Writing conda environments") - write_all_conda_environments(conda_build_metadata_list, output_path) + write_all_conda_environments(conda_build_metadata_list) logger.info("Writing conda lock files") - write_all_conda_lock_files(conda_build_metadata_list, output_path) + write_all_conda_lock_files(conda_build_metadata_list) logger.info("Writing pip requirements") - write_all_pip_requirements(pip_build_metadata_list, output_path) + write_all_pip_requirements(pip_build_metadata_list) logger.info("Writing pip lock files") - write_all_pip_lock_files(pip_build_metadata_list, output_path) + write_all_pip_lock_files(pip_build_metadata_list) From beae94db64bc14630bb9cc6a360032ff03f7154e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= Date: Tue, 17 May 2022 11:57:08 +0200 Subject: [PATCH 2/4] fix --- .circleci/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ea3e4ac4c808c..e34f48ec34f34 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -116,11 +116,10 @@ jobs: image: ubuntu-2004:202101-01 resource_class: arm.medium environment: - # Use the latest supported version of python - - PYTHON_VERSION: '3.9' - OMP_NUM_THREADS: 2 - OPENBLAS_NUM_THREADS: 2 - CONDA_ENV_NAME: testenv + - LOCK_FILE: build_tools/circle/py39_conda_forge_linux-aarch64_conda.lock steps: - checkout - run: ./build_tools/circle/checkout_merge_commit.sh From 51a101bcace44acd16787b367e20dd5c3ad04cd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= Date: Tue, 17 May 2022 12:02:27 +0200 Subject: [PATCH 3/4] ccache needed --- build_tools/circle/py39_conda_forge_environment.yml | 1 + build_tools/circle/py39_conda_forge_linux-aarch64_conda.lock | 4 +++- build_tools/update_environments_and_lock_files.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/build_tools/circle/py39_conda_forge_environment.yml b/build_tools/circle/py39_conda_forge_environment.yml index eb78623e03e76..6264aa1408e7b 100644 --- a/build_tools/circle/py39_conda_forge_environment.yml +++ b/build_tools/circle/py39_conda_forge_environment.yml @@ -16,3 +16,4 @@ dependencies: - pytest-xdist - pillow - pip=22.0.4 + - ccache diff --git a/build_tools/circle/py39_conda_forge_linux-aarch64_conda.lock b/build_tools/circle/py39_conda_forge_linux-aarch64_conda.lock index 138c4cdc11549..8b70af34e0e83 100644 --- a/build_tools/circle/py39_conda_forge_linux-aarch64_conda.lock +++ b/build_tools/circle/py39_conda_forge_linux-aarch64_conda.lock @@ -1,6 +1,6 @@ # Generated by conda-lock. # platform: linux-aarch64 -# input_hash: 5e1d4e50b6a9b4fdcb21ed2b944933a94780d4bbb9e8e8055da4a4778d048024 +# input_hash: e3e1ef206f1ca1cb3b6316fc18cc4c22a5dc95324159f4b0756b259d802aaf81 @EXPLICIT https://conda.anaconda.org/conda-forge/linux-aarch64/ca-certificates-2021.10.8-h4fd8a4c_0.tar.bz2#ad855209fcca3b45da677d409b16e021 https://conda.anaconda.org/conda-forge/linux-aarch64/ld_impl_linux-aarch64-2.36.1-h02ad14f_2.tar.bz2#3ca1a8e406eab04ffc3bfa6e8ac0a724 @@ -18,6 +18,7 @@ https://conda.anaconda.org/conda-forge/linux-aarch64/lerc-3.0-h01db608_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-aarch64/libbrotlicommon-1.0.9-h4e544f5_7.tar.bz2#6ee071311281942e2fec227751e7efff https://conda.anaconda.org/conda-forge/linux-aarch64/libdeflate-1.10-hf897c2e_0.tar.bz2#cf24cc05d49de7c6a981f2d485cb6fb6 https://conda.anaconda.org/conda-forge/linux-aarch64/libffi-3.4.2-h3557bc0_5.tar.bz2#dddd85f4d52121fab0a8b099c5e06501 +https://conda.anaconda.org/conda-forge/linux-aarch64/libhiredis-1.0.2-h05efe27_0.tar.bz2#a87f068744fd20334cd41489eb163bee https://conda.anaconda.org/conda-forge/linux-aarch64/libnsl-2.0.0-hf897c2e_0.tar.bz2#36fdbc05c9d9145ece86f5a63c3f352e https://conda.anaconda.org/conda-forge/linux-aarch64/libopenblas-0.3.20-pthreads_h6cb6f83_0.tar.bz2#1110034f2f90ca3c7ea35bf0d2eea15e https://conda.anaconda.org/conda-forge/linux-aarch64/libuuid-2.32.1-hf897c2e_1000.tar.bz2#e038da5ef9095b0d79aac14a311394e7 @@ -41,6 +42,7 @@ https://conda.anaconda.org/conda-forge/linux-aarch64/tk-8.6.12-hd8af866_0.tar.bz https://conda.anaconda.org/conda-forge/linux-aarch64/zlib-1.2.11-h4e544f5_1014.tar.bz2#5d6528db5739f248a7a5749d2dba8a27 https://conda.anaconda.org/conda-forge/linux-aarch64/zstd-1.5.2-h41fb7a4_0.tar.bz2#03434131cc82915a59513e1ab989fdd5 https://conda.anaconda.org/conda-forge/linux-aarch64/brotli-bin-1.0.9-h4e544f5_7.tar.bz2#4014ebf8c97d8cb219bfc0a12344ceb6 +https://conda.anaconda.org/conda-forge/linux-aarch64/ccache-4.5.1-h637f6b2_0.tar.bz2#0981c793a35b1e72d75d3a40e8dd69a4 https://conda.anaconda.org/conda-forge/linux-aarch64/libcblas-3.9.0-14_linuxaarch64_openblas.tar.bz2#8a1d66921d3e7dacacc8dd3af6d5ec5f https://conda.anaconda.org/conda-forge/linux-aarch64/liblapack-3.9.0-14_linuxaarch64_openblas.tar.bz2#1dcb7e50e399fc73824320b9ea709477 https://conda.anaconda.org/conda-forge/linux-aarch64/libpng-1.6.37-hbd635b3_2.tar.bz2#c95d3cee5815206d562fffa531a60a86 diff --git a/build_tools/update_environments_and_lock_files.py b/build_tools/update_environments_and_lock_files.py index 07f108d32616f..93af6b10385b0 100644 --- a/build_tools/update_environments_and_lock_files.py +++ b/build_tools/update_environments_and_lock_files.py @@ -210,7 +210,7 @@ def remove_from(alist, to_remove): "conda_dependencies": remove_from( common_dependencies_without_coverage, ["pandas", "pyamg"] ) - + ["pip"], + + ["pip", "ccache"], "package_constraints": { "python": "3.9", # TODO remove constraint when pip > 22.1 is released. See From 2611a86eca94aebd9a6938b02c893198ef9f3b06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= Date: Thu, 19 May 2022 08:38:41 +0200 Subject: [PATCH 4/4] Fix script path --- build_tools/azure/debian_atlas_32bit_requirements.txt | 2 +- .../azure/py38_conda_defaults_openblas_environment.yml | 2 +- .../py38_conda_forge_openblas_ubuntu_1804_environment.yml | 2 +- .../azure/pylatest_conda_forge_mkl_linux-64_environment.yml | 2 +- .../pylatest_conda_forge_mkl_no_coverage_environment.yml | 2 +- .../azure/pylatest_conda_forge_mkl_osx-64_environment.yml | 2 +- .../azure/pylatest_conda_mkl_no_openmp_environment.yml | 2 +- .../azure/pylatest_pip_openblas_pandas_environment.yml | 2 +- build_tools/azure/pylatest_pip_scipy_dev_environment.yml | 2 +- build_tools/azure/pypy3_environment.yml | 2 +- build_tools/azure/ubuntu_atlas_requirements.txt | 2 +- build_tools/circle/py39_conda_forge_environment.yml | 2 +- build_tools/update_environments_and_lock_files.py | 6 +++--- 13 files changed, 15 insertions(+), 15 deletions(-) diff --git a/build_tools/azure/debian_atlas_32bit_requirements.txt b/build_tools/azure/debian_atlas_32bit_requirements.txt index d7f36644ecec1..2708f7b8ff5e8 100644 --- a/build_tools/azure/debian_atlas_32bit_requirements.txt +++ b/build_tools/azure/debian_atlas_32bit_requirements.txt @@ -1,6 +1,6 @@ # DO NOT EDIT: this file is generated from the specification found in the # following script to centralize the configuration for all Azure CI builds: -# build_tools/azure/update_environments_and_lock_files.py +# build_tools/update_environments_and_lock_files.py cython joblib==1.0.0 # min threadpoolctl==2.2.0 diff --git a/build_tools/azure/py38_conda_defaults_openblas_environment.yml b/build_tools/azure/py38_conda_defaults_openblas_environment.yml index 549d1f7f50990..13cb49bb2af07 100644 --- a/build_tools/azure/py38_conda_defaults_openblas_environment.yml +++ b/build_tools/azure/py38_conda_defaults_openblas_environment.yml @@ -1,6 +1,6 @@ # DO NOT EDIT: this file is generated from the specification found in the # following script to centralize the configuration for all Azure CI builds: -# build_tools/azure/update_environments_and_lock_files.py +# build_tools/update_environments_and_lock_files.py channels: - defaults dependencies: diff --git a/build_tools/azure/py38_conda_forge_openblas_ubuntu_1804_environment.yml b/build_tools/azure/py38_conda_forge_openblas_ubuntu_1804_environment.yml index e4d69b9e280a4..fa82037236fe4 100644 --- a/build_tools/azure/py38_conda_forge_openblas_ubuntu_1804_environment.yml +++ b/build_tools/azure/py38_conda_forge_openblas_ubuntu_1804_environment.yml @@ -1,6 +1,6 @@ # DO NOT EDIT: this file is generated from the specification found in the # following script to centralize the configuration for all Azure CI builds: -# build_tools/azure/update_environments_and_lock_files.py +# build_tools/update_environments_and_lock_files.py channels: - conda-forge dependencies: diff --git a/build_tools/azure/pylatest_conda_forge_mkl_linux-64_environment.yml b/build_tools/azure/pylatest_conda_forge_mkl_linux-64_environment.yml index 7d7ba258422d9..318bba9517758 100644 --- a/build_tools/azure/pylatest_conda_forge_mkl_linux-64_environment.yml +++ b/build_tools/azure/pylatest_conda_forge_mkl_linux-64_environment.yml @@ -1,6 +1,6 @@ # DO NOT EDIT: this file is generated from the specification found in the # following script to centralize the configuration for all Azure CI builds: -# build_tools/azure/update_environments_and_lock_files.py +# build_tools/update_environments_and_lock_files.py channels: - conda-forge dependencies: diff --git a/build_tools/azure/pylatest_conda_forge_mkl_no_coverage_environment.yml b/build_tools/azure/pylatest_conda_forge_mkl_no_coverage_environment.yml index c51b32e65955b..1a61f4cc9395d 100644 --- a/build_tools/azure/pylatest_conda_forge_mkl_no_coverage_environment.yml +++ b/build_tools/azure/pylatest_conda_forge_mkl_no_coverage_environment.yml @@ -1,6 +1,6 @@ # DO NOT EDIT: this file is generated from the specification found in the # following script to centralize the configuration for all Azure CI builds: -# build_tools/azure/update_environments_and_lock_files.py +# build_tools/update_environments_and_lock_files.py channels: - conda-forge dependencies: diff --git a/build_tools/azure/pylatest_conda_forge_mkl_osx-64_environment.yml b/build_tools/azure/pylatest_conda_forge_mkl_osx-64_environment.yml index bf6f5caad40ef..6a619b0298772 100644 --- a/build_tools/azure/pylatest_conda_forge_mkl_osx-64_environment.yml +++ b/build_tools/azure/pylatest_conda_forge_mkl_osx-64_environment.yml @@ -1,6 +1,6 @@ # DO NOT EDIT: this file is generated from the specification found in the # following script to centralize the configuration for all Azure CI builds: -# build_tools/azure/update_environments_and_lock_files.py +# build_tools/update_environments_and_lock_files.py channels: - conda-forge dependencies: diff --git a/build_tools/azure/pylatest_conda_mkl_no_openmp_environment.yml b/build_tools/azure/pylatest_conda_mkl_no_openmp_environment.yml index 6838c1ccb78b6..90fe0e893991f 100644 --- a/build_tools/azure/pylatest_conda_mkl_no_openmp_environment.yml +++ b/build_tools/azure/pylatest_conda_mkl_no_openmp_environment.yml @@ -1,6 +1,6 @@ # DO NOT EDIT: this file is generated from the specification found in the # following script to centralize the configuration for all Azure CI builds: -# build_tools/azure/update_environments_and_lock_files.py +# build_tools/update_environments_and_lock_files.py channels: - defaults dependencies: diff --git a/build_tools/azure/pylatest_pip_openblas_pandas_environment.yml b/build_tools/azure/pylatest_pip_openblas_pandas_environment.yml index ae2503503daae..c452f7587331f 100644 --- a/build_tools/azure/pylatest_pip_openblas_pandas_environment.yml +++ b/build_tools/azure/pylatest_pip_openblas_pandas_environment.yml @@ -1,6 +1,6 @@ # DO NOT EDIT: this file is generated from the specification found in the # following script to centralize the configuration for all Azure CI builds: -# build_tools/azure/update_environments_and_lock_files.py +# build_tools/update_environments_and_lock_files.py channels: - defaults dependencies: diff --git a/build_tools/azure/pylatest_pip_scipy_dev_environment.yml b/build_tools/azure/pylatest_pip_scipy_dev_environment.yml index 1a6498fa7a511..9b8fb02d77266 100644 --- a/build_tools/azure/pylatest_pip_scipy_dev_environment.yml +++ b/build_tools/azure/pylatest_pip_scipy_dev_environment.yml @@ -1,6 +1,6 @@ # DO NOT EDIT: this file is generated from the specification found in the # following script to centralize the configuration for all Azure CI builds: -# build_tools/azure/update_environments_and_lock_files.py +# build_tools/update_environments_and_lock_files.py channels: - defaults dependencies: diff --git a/build_tools/azure/pypy3_environment.yml b/build_tools/azure/pypy3_environment.yml index 0929f6e545e7d..5dc45f334d903 100644 --- a/build_tools/azure/pypy3_environment.yml +++ b/build_tools/azure/pypy3_environment.yml @@ -1,6 +1,6 @@ # DO NOT EDIT: this file is generated from the specification found in the # following script to centralize the configuration for all Azure CI builds: -# build_tools/azure/update_environments_and_lock_files.py +# build_tools/update_environments_and_lock_files.py channels: - conda-forge dependencies: diff --git a/build_tools/azure/ubuntu_atlas_requirements.txt b/build_tools/azure/ubuntu_atlas_requirements.txt index 396b07eff14c0..320b9d8fe4a2e 100644 --- a/build_tools/azure/ubuntu_atlas_requirements.txt +++ b/build_tools/azure/ubuntu_atlas_requirements.txt @@ -1,6 +1,6 @@ # DO NOT EDIT: this file is generated from the specification found in the # following script to centralize the configuration for all Azure CI builds: -# build_tools/azure/update_environments_and_lock_files.py +# build_tools/update_environments_and_lock_files.py cython joblib==1.0.0 # min threadpoolctl==2.0.0 # min diff --git a/build_tools/circle/py39_conda_forge_environment.yml b/build_tools/circle/py39_conda_forge_environment.yml index 6264aa1408e7b..f5b6581ee2689 100644 --- a/build_tools/circle/py39_conda_forge_environment.yml +++ b/build_tools/circle/py39_conda_forge_environment.yml @@ -1,6 +1,6 @@ # DO NOT EDIT: this file is generated from the specification found in the # following script to centralize the configuration for all Azure CI builds: -# build_tools/azure/update_environments_and_lock_files.py +# build_tools/update_environments_and_lock_files.py channels: - conda-forge dependencies: diff --git a/build_tools/update_environments_and_lock_files.py b/build_tools/update_environments_and_lock_files.py index 93af6b10385b0..c441c2451911b 100644 --- a/build_tools/update_environments_and_lock_files.py +++ b/build_tools/update_environments_and_lock_files.py @@ -1,7 +1,7 @@ """Script to update CI environment files and associated lock files. To run it you need to be in the root folder of the scikit-learn repo: -python build_tools/azure/update_environments_and_lock_files.py +python build_tools/update_environments_and_lock_files.py Two scenarios where this script can be useful: - make sure that the latest versions of all the dependencies are used in the CI. @@ -310,7 +310,7 @@ def get_conda_environment_content(build_metadata): """ # DO NOT EDIT: this file is generated from the specification found in the # following script to centralize the configuration for all Azure CI builds: -# build_tools/azure/update_environments_and_lock_files.py +# build_tools/update_environments_and_lock_files.py channels: - {{ build_metadata['channel'] }} dependencies: @@ -375,7 +375,7 @@ def get_pip_requirements_content(build_metadata): """ # DO NOT EDIT: this file is generated from the specification found in the # following script to centralize the configuration for all Azure CI builds: -# build_tools/azure/update_environments_and_lock_files.py +# build_tools/update_environments_and_lock_files.py {% for pip_dep in build_metadata['pip_dependencies'] %} {{ pip_dep | get_package_with_constraint(build_metadata, uses_pip=True) }} {% endfor %}""".strip()