8000 Update on "[caffe2] Remove IValue include from operator.h" · pytorch/pytorch@02def0c · GitHub
[go: up one dir, main page]
Skip to content

Commit 02def0c

Browse files
peterbell10peterbell10
committed
Update on "[caffe2] Remove IValue include from operator.h"
ivalue.h includes Tensor.h, so creating a compilation barrier between operator.h and ivalue.h means non-exported caffe2 ops don't need to be rebuilt when developing PyTorch. Differential Revision: [D32289812](https://our.internmc.facebook.com/intern/diff/D32289812) [ghstack-poisoned]
2 parents 5eebf69 + 4d9f432 commit 02def0c

File tree

2,693 files changed

+186567
-97883
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

2,693 files changed

+186567
-97883
lines changed

‎.bazelrc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
build --cxxopt=--std=c++14
1+
build --cxxopt=--std=c++17
22
build --copt=-I.
33
# Bazel does not support including its cc_library targets as system
44
# headers. We work around this for generated code

‎.circleci/README.md

Lines changed: 468 additions & 0 deletions
Large diffs are not rendered by default.

‎.circleci/config.yml

Lines changed: 2 additions & 25 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎.circleci/docker/build.sh

Lines changed: 30 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -75,32 +75,31 @@ elif [[ "$image" == *rocm* ]]; then
7575
DOCKERFILE="${OS}-rocm/Dockerfile"
7676
fi
7777

78-
if [[ "$image" == *bionic* ]]; then
79-
CMAKE_VERSION=3.13.5
80-
fi
78+
# CMake 3.18 is needed to support CUDA17 language variant
79+
CMAKE_VERSION=3.18.5
8180

82-
TRAVIS_DL_URL_PREFIX="https://s3.amazonaws.com/travis-python-archives/binaries/ubuntu/14.04/x86_64"
8381
_UCX_COMMIT=31e74cac7bee0ef66bef2af72e7d86d9c282e5ab
84-
_UCC_COMMIT=12944da33f911daf505d9bbc51411233d0ed85e1
82+
_UCC_COMMIT=1c7a7127186e7836f73aafbd7697bbc274a77eee
8583

8684
# It's annoying to rename jobs every time you want to rewrite a
8785
# configuration, so we hardcode everything here rather than do it
8886
# from scratch
8987
case "$image" in
90-
pytorch-linux-bionic-cuda11.3-cudnn8-py3-clang9)
91-
CUDA_VERSION=11.3.0 # Deviating from major.minor to conform to nvidia's Docker image names
88+
pytorch-linux-bionic-cuda11.6-cudnn8-py3-gcc7)
89+
CUDA_VERSION=11.6.2
9290
CUDNN_VERSION=8
93-
TENSORRT_VERSION=8.0.1.6
94-
ANACONDA_PYTHON_VERSION=3.7
95-
CLANG_VERSION=9
91+
ANACONDA_PYTHON_VERSION=3.10
92+
GCC_VERSION=7
9693
PROTOBUF=yes
9794
DB=yes
9895
VISION=yes
9996
KATEX=yes
97+
UCX_COMMIT=${_UCX_COMMIT}
98+
UCC_COMMIT=${_UCC_COMMIT}
10099
CONDA_CMAKE=yes
101100
;;
102-
pytorch-linux-bionic-cuda11.6-cudnn8-py3-gcc7)
103-
CUDA_VERSION=11.6.2
101+
pytorch-linux-bionic-cuda11.7-cudnn8-py3-gcc7)
102+
CUDA_VERSION=11.7.0
104103
CUDNN_VERSION=8
105104
ANACONDA_PYTHON_VERSION=3.10
106105
GCC_VERSION=7
@@ -112,8 +111,8 @@ case "$image" in
112111
UCC_COMMIT=${_UCC_COMMIT}
113112
CONDA_CMAKE=yes
114113
;;
115-
pytorch-linux-bionic-cuda11.7-cudnn8-py3-gcc7)
116-
CUDA_VERSION=11.7.0
114+
pytorch-linux-bionic-cuda11.8-cudnn8-py3-gcc7)
115+
CUDA_VERSION=11.8.0
117116
CUDNN_VERSION=8
118117
ANACONDA_PYTHON_VERSION=3.10
119118
GCC_VERSION=7
@@ -126,15 +125,15 @@ case "$image" in
126125
CONDA_CMAKE=yes
127126
;;
128127
pytorch-linux-focal-py3-clang7-asan)
129-
ANACONDA_PYTHON_VERSION=3.7
128+
ANACONDA_PYTHON_VERSION=3.9
130129
CLANG_VERSION=7
131130
PROTOBUF=yes
132131
DB=yes
133132
VISION=yes
134133
CONDA_CMAKE=yes
135134
;;
136135
pytorch-linux-focal-py3-clang10-onnx)
137-
ANACONDA_PYTHON_VERSION=3.7
136+
ANACONDA_PYTHON_VERSION=3.8
138137
CLANG_VERSION=10
139138
PROTOBUF=yes
140139
DB=yes
@@ -169,47 +168,28 @@ case "$image" in
169168
VISION=yes
170169
CONDA_CMAKE=yes
171170
;;
172-
pytorch-linux-bionic-cuda10.2-cudnn7-py3.7-clang9)
173-
CUDA_VERSION=10.2
174-
CUDNN_VERSION=7
175-
ANACONDA_PYTHON_VERSION=3.7
176-
CLANG_VERSION=9
177-
PROTOBUF=yes
178-
DB=yes
179-
VISION=yes
180-
CONDA_CMAKE=yes
181-
;;
182-
pytorch-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7)
183-
CUDA_VERSION=10.2
184-
CUDNN_VERSION=7
185-
ANACONDA_PYTHON_VERSION=3.9
186-
GCC_VERSION=7
187-
PROTOBUF=yes
188-
DB=yes
189-
VISION=yes
190-
CONDA_CMAKE=yes
191-
;;
192-
pytorch-linux-focal-rocm5.1-py3.8)
171+
pytorch-linux-focal-rocm5.2-py3.8)
193172
ANACONDA_PYTHON_VERSION=3.8
194173
GCC_VERSION=9
195174
PROTOBUF=yes
196175
DB=yes
197176
VISION=yes
198-
ROCM_VERSION=5.1.1
177+
ROCM_VERSION=5.2
178+
NINJA_VERSION=1.9.0
199179
CONDA_CMAKE=yes
200180
;;
201-
pytorch-linux-focal-rocm5.2-py3.8)
181+
pytorch-linux-focal-rocm5.3-py3.8)
202182
ANACONDA_PYTHON_VERSION=3.8
203183
GCC_VERSION=9
204184
PROTOBUF=yes
205185
DB=yes
206186
VISION=yes
207-
ROCM_VERSION=5.2
187+
ROCM_VERSION=5.3
188+
NINJA_VERSION=1.9.0
208189
CONDA_CMAKE=yes
209190
;;
210191
pytorch-linux-focal-py3.7-gcc7)
211192
ANACONDA_PYTHON_VERSION=3.7
212-
CMAKE_VERSION=3.16.9 # Required for precompiled header support
213193
GCC_VERSION=7
214194
PROTOBUF=yes
215195
DB=yes
@@ -235,6 +215,15 @@ case "$image" in
235215
DB=yes
236216
VISION=yes
237217
;;
218+
pytorch-linux-jammy-cuda11.8-cudnn8-py3.8-clang12)
219+
ANACONDA_PYTHON_VERSION=3.8
220+
CUDA_VERSION=11.8
221+
CUDNN_VERSION=8
222+
CLANG_VERSION=12
223+
PROTOBUF=yes
224+
DB=yes
225+
VISION=yes
226+
;;
238227
*)
239228
# Catch-all for builds that are not hardcoded.
240229
PROTOBUF=yes
@@ -273,12 +262,6 @@ case "$image" in
273262
;;
274263
esac
275264

276-
# Set Jenkins UID and GID if running Jenkins
277-
if [ -n "${JENKINS:-}" ]; then
278-
JENKINS_UID=$(id -u jenkins)
279-
JENKINS_GID=$(id -g jenkins)
280-
fi
281-
282265
tmp_tag=$(basename "$(mktemp -u)" | tr '[:upper:]' '[:lower:]')
283266

284267
#when using cudnn version 8 install it separately from cuda
@@ -295,17 +278,12 @@ fi
295278
docker build \
296279
--no-cache \
297280
--progress=plain \
298-
--build-arg "TRAVIS_DL_URL_PREFIX=${TRAVIS_DL_URL_PREFIX}" \
299281
--build-arg "BUILD_ENVIRONMENT=${image}" \
300282
--build-arg "PROTOBUF=${PROTOBUF:-}" \
301283
--build-arg "THRIFT=${THRIFT:-}" \
302284
--build-arg "LLVMDEV=${LLVMDEV:-}" \
303285
--build-arg "DB=${DB:-}" \
304286
--build-arg "VISION=${VISION:-}" \
305-
--build-arg "EC2=${EC2:-}" \
306-
--build-arg "JENKINS=${JENKINS:-}" \
307-
--build-arg "JENKINS_UID=${JENKINS_UID:-}" \
308-
--build-arg "JENKINS_GID=${JENKINS_GID:-}" \
309287
--build-arg "UBUNTU_VERSION=${UBUNTU_VERSION}" \
310288
--build-arg "CENTOS_VERSION=${CENTOS_VERSION}" \
311289
--build-arg "DEVTOOLSET_VERSION=${DEVTOOLSET_VERSION}" \

‎.circleci/docker/build_docker.sh

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ tag="${DOCKER_TAG}"
1818

1919
registry="308535385114.dkr.ecr.us-east-1.amazonaws.com"
2020
image="${registry}/pytorch/${IMAGE_NAME}"
21-
ghcr_image="ghcr.io/pytorch/ci-image"
2221

2322
login() {
2423
aws ecr get-authorization-token --region us-east-1 --output text --query 'authorizationData[].authorizationToken' |
@@ -36,9 +35,6 @@ if [[ -z "${GITHUB_ACTIONS}" ]]; then
3635
trap "docker logout ${registry}" EXIT
3736
fi
3837

39-
# export EC2=1
40-
# export JENKINS=1
41-
4238
# Try to pull the previous image (perhaps we can reuse some layers)
4339
# if [ -n "${last_tag}" ]; then
4440
# docker pull "${image}:${last_tag}" || true
@@ -55,13 +51,6 @@ if [ "${DOCKER_SKIP_PUSH:-true}" = "false" ]; then
5551
if ! docker manifest inspect "${image}:${tag}" >/dev/null 2>/dev/null; then
5652
docker push "${image}:${tag}"
5753
fi
58-
59-
if [ "${PUSH_GHCR_IMAGE:-}" = "true" ]; then
60-
# Push docker image to the ghcr.io
61-
echo $GHCR_PAT | docker login ghcr.io -u pytorch --password-stdin
62-
docker tag "${image}:${tag}" "${ghcr_image}:${IMAGE_NAME}-${tag}"
63-
docker push "${ghcr_image}:${IMAGE_NAME}-${tag}"
64-
fi
6554
fi
6655

6756
if [ -z "${DOCKER_SKIP_S3_UPLOAD:-}" ]; then

‎.circleci/docker/centos-rocm/Dockerfile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ ENV PYTORCH_ROCM_ARCH ${PYTORCH_ROCM_ARCH}
1111
# Install required packages to build Caffe2
1212

1313
# Install common dependencies (so that this step can be cached separately)
14-
ARG EC2
1514
COPY ./common/install_base.sh install_base.sh
1615
RUN bash ./install_base.sh && rm install_base.sh
1716

@@ -38,9 +37,10 @@ COPY ./common/install_user.sh install_user.sh
3837
RUN bash ./install_user.sh && rm install_user.sh
3938

4039
# Install conda and other packages (e.g., numpy, pytest)
41-
ENV PATH /opt/conda/bin:$PATH
4240
ARG ANACONDA_PYTHON_VERSION
4341
ARG CONDA_CMAKE
42+
ENV ANACONDA_PYTHON_VERSION=$ANACONDA_PYTHON_VERSION
43+
ENV PATH /opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:/opt/conda/bin:$PATH
4444
COPY requirements-ci.txt /opt/conda/requirements-ci.txt
4545
COPY ./common/install_conda.sh install_conda.sh
4646
RUN bash ./install_conda.sh && rm install_conda.sh

‎.circleci/docker/common/install_base.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ esac
157157

158158
# Install Valgrind separately since the apt-get version is too old.
159159
mkdir valgrind_build && cd valgrind_build
160-
VALGRIND_VERSION=3.16.1
160+
VALGRIND_VERSION=3.20.0
161161
wget https://ossci-linux.s3.amazonaws.com/valgrind-${VALGRIND_VERSION}.tar.bz2
162162
tar -xjf valgrind-${VALGRIND_VERSION}.tar.bz2
163163
cd valgrind-${VALGRIND_VERSION}

‎.circleci/docker/common/install_cmake.sh

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,19 @@ set -ex
55
[ -n "$CMAKE_VERSION" ]
66

77
# Remove system cmake install so it won't get used instead
8-
apt-get remove cmake -y
8+
ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
9+
case "$ID" in
10+
ubuntu)
11+
apt-get remove cmake -y
12+
;;
13+
centos)
14+
yum remove cmake -y
15+
;;
16+
*)
17+
echo "Unable to determine OS..."
18+
exit 1
19+
;;
20+
esac
921

1022
# Turn 3.6.3 into v3.6
1123
path=$(echo "${CMAKE_VERSION}" | sed -e 's/\([0-9].[0-9]\+\).*/v\1/')

‎.circleci/docker/common/install_conda.sh

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@ if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
4242

4343
pushd /tmp
4444
wget -q "${BASE_URL}/${CONDA_FILE}"
45-
chmod +x "${CONDA_FILE}"
46-
as_jenkins ./"${CONDA_FILE}" -b -f -p "/opt/conda"
45+
# NB: Manually invoke bash per https://github.com/conda/conda/issues/10431
46+
as_jenkins bash "${CONDA_FILE}" -b -f -p "/opt/conda"
4747
popd
4848

4949
# NB: Don't do this, rely on the rpath to get it right
@@ -61,21 +61,21 @@ if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
6161
# as_jenkins conda update -y -n base conda
6262

6363
# Install correct Python version
64-
as_jenkins conda install -y python="$ANACONDA_PYTHON_VERSION"
64+
as_jenkins conda create -n py_$ANACONDA_PYTHON_VERSION -y python="$ANACONDA_PYTHON_VERSION"
6565

6666
conda_install() {
6767
# Ensure that the install command don't upgrade/downgrade Python
6868
# This should be called as
6969
# conda_install pkg1 pkg2 ... [-c channel]
70-
as_jenkins conda install -q -y python="$ANACONDA_PYTHON_VERSION" $*
70+
as_jenkins conda install -q -n py_$ANACONDA_PYTHON_VERSION -y python="$ANACONDA_PYTHON_VERSION" $*
7171
}
7272

7373
pip_install() {
74-
as_jenkins pip install --progress-bar off $*
74+
as_jenkins conda run -n py_$ANACONDA_PYTHON_VERSION pip install --progress-bar off $*
7575
}
7676

7777
# Install PyTorch conda deps, as per https://github.com/pytorch/pytorch README
78-
CONDA_COMMON_DEPS="astunparse pyyaml mkl=2022.0.1 mkl-include=2022.0.1 setuptools cffi future six"
78+
CONDA_COMMON_DEPS="astunparse pyyaml mkl=2022.0.1 mkl-include=2022.0.1 setuptools six"
7979
if [ "$ANACONDA_PYTHON_VERSION" = "3.10" ]; then
8080
# Install llvm-8 as it is required to compile llvmlite-0.30.0 from source
8181
conda_install numpy=1.21.2 ${CONDA_COMMON_DEPS} llvmdev=8.0.0

‎.circleci/docker/common/install_cudnn.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,9 @@ if [[ ${CUDNN_VERSION} == 8 ]]; then
66
CUDNN_NAME="cudnn-linux-x86_64-8.3.2.44_cuda11.5-archive"
77
if [[ ${CUDA_VERSION:0:4} == "11.7" ]]; then
88
CUDNN_NAME="cudnn-linux-x86_64-8.5.0.96_cuda11-archive"
9-
curl -OLs https://ossci-linux.s3.amazonaws.com/${CUDNN_NAME}.tar.xz
9+
curl --retry 3 -OLs https://ossci-linux.s3.amazonaws.com/${CUDNN_NAME}.tar.xz
1010
else
11-
curl -OLs https://developer.download.nvidia.com/compute/redist/cudnn/v8.3.2/local_installers/11.5/${CUDNN_NAME}.tar.xz
11+
curl --retry 3 -OLs https://developer.download.nvidia.com/compute/redist/cudnn/v8.3.2/local_installers/11.5/${CUDNN_NAME}.tar.xz
1212
fi
1313

1414
tar xf ${CUDNN_NAME}.tar.xz

‎.circleci/docker/common/install_docs_reqs.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,10 @@ if [ -n "$KATEX" ]; then
77
# Ignore error if gpg-agent doesn't exist (for Ubuntu 16.04)
88
apt-get install -y gpg-agent || :
99

10-
curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash -
10+
curl --retry 3 -sL https://deb.nodesource.com/setup_12.x | sudo -E bash -
1111
sudo apt-get install -y nodejs
1212

13-
curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
13+
curl --retry 3 -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
1414
echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
1515

1616
apt-get update

‎.circleci/docker/common/install_protobuf.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ install_protobuf_317() {
1212
# g++: error: ./../lib64/crti.o: No such file or directory
1313
ln -s /usr/lib64 "$pb_dir/lib64"
1414

15-
curl -LO "https://github.com/protocolbuffers/protobuf/releases/download/v3.17.3/protobuf-all-3.17.3.tar.gz"
15+
curl -LO "https://github.com/protocolbuffers/protobuf/releases/download/v3.17.3/protobuf-all-3.17.3.tar.gz" --retry 3
1616
tar -xvz -C "$pb_dir" --strip-components 1 -f protobuf-all-3.17.3.tar.gz
1717
# -j6 to balance memory usage and speed.
1818
# naked `-j` seems to use too much memory.

‎.circleci/docker/common/install_rocm_magma.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ done
2323
# hipcc with openmp flag may cause isnan() on __device__ not to be found; depending on context, compiler may attempt to match with host definition
2424
sed -i 's/^FOPENMP/#FOPENMP/g' make.inc
2525
make -f make.gen.hipMAGMA -j $(nproc)
26-
LANG=C.UTF-8 make lib/libmagma.so -j $(nproc) MKLROOT=/opt/conda
27-
make testing/testing_dgemm -j $(nproc) MKLROOT=/opt/conda
26+
LANG=C.UTF-8 make lib/libmagma.so -j $(nproc) MKLROOT=/opt/conda/envs/py_$ANACONDA_PYTHON_VERSION
27+
make testing/testing_dgemm -j $(nproc) MKLROOT=/opt/conda/envs/py_$ANACONDA_PYTHON_VERSION
2828
popd
2929
mv magma /opt/rocm

0 commit comments

Comments
 (0)
0