8000 Use ccache for faster CI builds (#16434) · NicolasHug/scikit-learn@49d26cb · GitHub
[go: up one dir, main page]
Skip to content

Commit 49d26cb

Browse files
rthrth
authored
Use ccache for faster CI builds (scikit-learn#16434)
1 parent 997a69e commit 49d26cb

File tree

2 files changed

+30
-5
lines changed

2 files changed

+30
-5
lines changed

‎build_tools/azure/install.sh

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,23 @@ make_conda() {
1111
source activate $VIRTUALENV
1212
}
1313

14+
setup_ccache() {
15+
echo "Setting up ccache"
16+
mkdir /tmp/ccache/
17+
which ccache
18+
for name in gcc g++ cc c++ x86_64-linux-gnu-gcc x86_64-linux-gnu-c++; do
19+
ln -s $(which ccache) "/tmp/ccache/${name}"
20+
done
21+
export PATH="/tmp/ccache/:${PATH}"
22+
ccache -M 256M
23+
}
24+
1425
# imports get_dep
1526
source build_tools/shared.sh
1627

1728
if [[ "$DISTRIB" == "conda" ]]; then
1829

19-
TO_INSTALL="python=$PYTHON_VERSION pip blas[build=$BLAS]"
30+
TO_INSTALL="python=$PYTHON_VERSION ccache pip blas[build=$BLAS]"
2031

2132
TO_INSTALL="$TO_INSTALL $(get_dep numpy $NUMPY_VERSION)"
2233
TO_INSTALL="$TO_INSTALL $(get_dep scipy $SCIPY_VERSION)"
@@ -38,43 +49,48 @@ if [[ "$DISTRIB" == "conda" ]]; then
3849
fi
3950
fi
4051
make_conda $TO_INSTALL
52+
setup_ccache
4153

4254
elif [[ "$DISTRIB" == "ubuntu" ]]; then
4355
sudo add-apt-repository --remove ppa:ubuntu-toolchain-r/test
4456
sudo apt-get update
45-
sudo apt-get install python3-scipy python3-matplotlib libatlas3-base libatlas-base-dev python3-virtualenv
57+
sudo apt-get install python3-scipy python3-matplotlib libatlas3-base libatlas-base-dev python3-virtualenv ccache
4658
python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV
4759
source $VIRTUALENV/bin/activate
60+
setup_ccache
4861
python -m pip install $(get_dep cython $CYTHON_VERSION) \
4962
$(get_dep joblib $JOBLIB_VERSION)
5063

5164
elif [[ "$DISTRIB" == "ubuntu-32" ]]; then
5265
apt-get update
53-
apt-get install -y python3-dev python3-scipy python3-matplotlib libatlas3-base libatlas-base-dev python3-virtualenv python3-pandas
66+
apt-get install -y python3-dev python3-scipy python3-matplotlib libatlas3-base libatlas-base-dev python3-virtualenv python3-pandas ccache
5467

5568
python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV
5669
source $VIRTUALENV/bin/activate
70+
setup_ccache
5771
python -m pip install $(get_dep cython $CYTHON_VERSION) \
5872
$(get_dep joblib $JOBLIB_VERSION)
5973

6074
elif [[ "$DISTRIB" == "conda-pip-latest" ]]; then
6175
# Since conda main channel usually lacks behind on the latest releases,
6276
# we use pypi to test against the latest releases of the dependencies.
6377
# conda is still used as a convenient way to install Python and pip.
64-
make_conda "python=$PYTHON_VERSION"
78+
make_conda "ccache python=$PYTHON_VERSION"
79+
setup_ccache
6580
python -m pip install -U pip
6681

6782
python -m pip install pandas matplotlib pyamg scikit-image
6883
# do not install dependencies for lightgbm since it requires scikit-learn
6984
# and install a version less than 3.0.0 until the issue #18316 is solved.
7085
python -m pip install "lightgbm<3.0.0" --no-deps
7186
elif [[ "$DISTRIB" == "conda-pip-scipy-dev" ]]; then
72-
make_conda "python=$PYTHON_VERSION"
87+
make_conda "ccache python=$PYTHON_VERSION"
7388
python -m pip install -U pip
7489
echo "Installing numpy and scipy master wheels"
7590
dev_anaconda_url=https://pypi.anaconda.org/scipy-wheels-nightly/simple
7691
pip install --pre --upgrade --timeout=60 --extra-index $dev_anaconda_url numpy scipy pandas
7792
pip install --pre cython
93+
setup_ccache
7894
echo "Installing joblib master"
7995
pip install https://github.com/joblib/joblib/archive/master.zip
8096
echo "Installing pillow master"
@@ -123,3 +139,5 @@ else
123139
python setup.py build_ext --inplace -j 3
124140
python setup.py develop
125141
fi
142+
143+
ccache -s

‎build_tools/azure/posix.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ jobs:
1818
OMP_NUM_THREADS: '2'
1919
OPENBLAS_NUM_THREADS: '2'
2020
SKLEARN_SKIP_NETWORK_TESTS: '1'
21+
CCACHE_DIR: $(Pipeline.Workspace)/ccache
22+
CCACHE_COMPRESS: '1'
2123
NUMPY_VERSION: 'latest'
2224
SCIPY_VERSION: 'latest'
2325
CYTHON_VERSION: 'latest'
@@ -42,6 +44,11 @@ jobs:
4244
- bash: sudo chown -R $USER $CONDA
4345
displayName: Take ownership of conda installation
4446
condition: startsWith(variables['DISTRIB'], 'conda')
47+
- task: Cache@2
48+
inputs:
49+
key: '"$(Agent.JobName)"'
50+
path: $(CCACHE_DIR)
51+
displayName: ccache
4552
- script: |
4653
build_tools/azure/install.sh
4754
displayName: 'Install'

0 commit comments

Comments
 (0)
0