8000 Update on "switch aten_headers to use strip_include_prefix instead of… · pytorch/pytorch@89dbd9f · GitHub
[go: up one dir, main page]
Skip to content

Commit 89dbd9f

Browse files
author
Michael Andreas Dagitses
committed
Update on "switch aten_headers to use strip_include_prefix instead of includes"
This is Bazel best practices and easier to maintain. Differential Revision: [D36521515](https://our.internmc.facebook.com/intern/diff/D36521515/) [ghstack-poisoned]
2 parents abf06d3 + e1aff28 commit 89dbd9f

File tree

1,565 files changed

+90188
-43436
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,565 files changed

+90188
-43436
lines changed

‎.bazelrc

Lines changed: 90 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,96 @@ build:no-tty --curses no
1313
build:no-tty --progress_report_interval 10
1414
build:no-tty --show_progress_rate_limit 10
1515

16-
# Configuration to build with GPU support
17-
build:gpu --define=cuda=true
16+
# Build with GPU support by default.
17+
build --define=cuda=true
18+
# rules_cuda configuration
19+
build --@rules_cuda//cuda:enable_cuda
20+
build --@rules_cuda//cuda:cuda_targets=sm_52
21+
build --@rules_cuda//cuda:compiler=nvcc
22+
build --repo_env=CUDA_PATH=/usr/local/cuda
23+
24+
# Configuration to build without GPU support
25+
build:cpu-only --define=cuda=false
1826
# define a separate build folder for faster switching between configs
19-
build:gpu --platform_suffix=-gpu
27+
build:cpu-only --platform_suffix=-cpu-only
2028
# See the note on the config-less build for details about why we are
21-
# doing this. We must also do it for the "-gpu" platform suffix.
22-
build --copt=-isystem --copt=bazel-out/k8-fastbuild-gpu/bin
29+
# doing this. We must also do it for the "-cpu-only" platform suffix.
30+
build --copt=-isystem --copt=bazel-out/k8-fastbuild-cpu-only/bin
2331
# rules_cuda configuration
24-
build:gpu --@rules_cuda//cuda:enable_cuda
25-
build:gpu --@rules_cuda//cuda:cuda_targets=sm_52
26-
build:gpu --@rules_cuda//cuda:compiler=nvcc
27-
build:gpu --repo_env=CUDA_PATH=/usr/local/cuda
32+
build:cpu-only --@rules_cuda//cuda:enable_cuda=False
33+
34+
# Disable all warnings for external repositories. We don't care about
35+
# their warnings.
36+
build --per_file_copt=^external/@-w
37+
38+
# Set additional warnings to error level.
39+
#
40+
# Implementation notes:
41+
# * we use file extensions to determine if we are using the C++
42+
# compiler or the cuda compiler
43+
# * we use ^// at the start of the regex to only permit matching
44+
# PyTorch files. This excludes external repos.
45+
#
46+
# Note that because this is logically a command-line flag, it is
47+
# considered the word on what warnings are enabled. This has the
48+
# unfortunate consequence of preventing us from disabling an error at
49+
# the target level because those flags will come before these flags in
50+
# the action invocation. Instead we provide per-file exceptions after
51+
# this.
52+
#
53+
# On the bright side, this means we don't have to more broadly apply
54+
# the exceptions to an entire target.
55+
#
56+
# Looking for CUDA flags? We have a cu_library macro that we can edit
57+
# directly. Look in //tools/rules:cu.bzl for details. Editing the
58+
# macro over this has the following advantages:
59+
# * making changes does not require discarding the Bazel analysis
60+
# cache
61+
# * it allows for selective overrides on individual targets since the
62+
# macro-level opts will come earlier than target level overrides
63+
64+
build --per_file_copt='^//.*\.(cpp|cc)$'@-Werror=all
65+
# The following warnings come from -Wall. We downgrade them from error
66+
# to warnings here.
67+
#
68+
# sign-compare has a tremendous amount of violations in the
69+
# codebase. It will be a lot of work to fix them, just disable it for
70+
# now.
71+
build --per_file_copt='^//.*\.(cpp|cc)$'@-Wno-sign-compare
72+
# We intentionally use #pragma unroll, which is compiler specific.
73+
build --per_file_copt='^//.*\.(cpp|cc)$'@-Wno-error=unknown-pragmas
74+
75+
build --per_file_copt='^//.*\.(cpp|cc)$'@-Werror=extra
76+
# The following warnings come from -Wextra. We downgrade them from error
77+
# to warnings here.
78+
#
79+
# unused-parameter-compare has a tremendous amount of violations in the
80+
# codebase. It will be a lot of work to fix them, just disable it for
81+
# now.
82+
build --per_file_copt='^//.*\.(cpp|cc)$'@-Wno-unused-parameter
83+
# missing-field-parameters has both a large number of violations in
84+
# the codebase, but it also is used pervasively in the Python C
85+
# API. There are a couple of catches though:
86+
# * we use multiple versions of the Python API and hence have
87+
# potentially multiple different versions of each relevant
88+
# struct. They may have different numbers of fields. It will be
89+
# unwieldy to support multiple versions in the same source file.
90+
# * Python itself for many of these structs recommends only
91+
# initializing a subset of the fields. We should respect the API
92+
# usage conventions of our dependencies.
93+
#
94+
# Hence, we just disable this warning altogether. We may want to clean
95+
# up some of the clear-cut cases that could be risky, but we still
96+
# likely want to have this disabled for the most part.
97+
build --per_file_copt='^//.*\.(cpp|cc)$'@-Wno-missing-field-initializers
98+
99+
build --per_file_copt='//:aten/src/ATen/RegisterCompositeExplicitAutograd\.cpp$'@-Wno-error=unused-function
100+
build --per_file_copt='//:aten/src/ATen/RegisterCompositeImplicitAutograd\.cpp$'@-Wno-error=unused-function
101+
build --per_file_copt='//:aten/src/ATen/RegisterMkldnnCPU\.cpp$'@-Wno-error=unused-function
102+
build --per_file_copt='//:aten/src/ATen/RegisterNestedTensorCPU\.cpp$'@-Wno-error=unused-function
103+
build --per_file_copt='//:aten/src/ATen/RegisterQuantizedCPU\.cpp$'@-Wno-error=unused-function
104+
build --per_file_copt='//:aten/src/ATen/RegisterSparseCPU\.cpp$'@-Wno-error=unused-function
105+
build --per_file_copt='//:aten/src/ATen/RegisterSparseCsrCPU\.cpp$'@-Wno-error=unused-function
106+
build --per_file_copt='//:aten/src/ATen/RegisterZeroTensor\.cpp$'@-Wno-error=unused-function
107+
build --per_file_copt='//:torch/csrc/lazy/generated/RegisterAutogradLazy\.cpp$'@-Wno-error=unused-function
108+
build --per_file_copt='//:torch/csrc/lazy/generated/RegisterLazy\.cpp$'@-Wno-error=unused-function

‎.buckconfig.oss

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
1+
[pt]
2+
is_oss=1
3+
14
[buildfile]
2-
name = BUILD.buck
5+
name = BUCK.oss
36

47
[repositories]
58
bazel_skylib = third_party/bazel-skylib/

‎.circleci/config.yml

Lines changed: 0 additions & 52 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎.circleci/docker/build.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,13 @@ case "$image" in
159159
DB=yes
160160
VISION=yes
161161
;;
162+
pytorch-linux-focal-py3-clang7-asan)
163+
ANACONDA_PYTHON_VERSION=3.7
164+
CLANG_VERSION=7
165+
PROTOBUF=yes
166+
DB=yes
167+
VISION=yes
168+
;;
162169
pytorch-linux-xenial-py3-clang7-onnx)
163170
ANACONDA_PYTHON_VERSION=3.7
164171
CLANG_VERSION=7

‎.circleci/docker/common/install_base.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,12 @@ install_ubuntu() {
2929
apt-get update
3030
# TODO: Some of these may not be necessary
3131
ccache_deps="asciidoc docbook-xml docbook-xsl xsltproc"
32+
deploy_deps="libffi-dev libbz2-dev libreadline-dev libncurses5-dev libncursesw5-dev libgdbm-dev libsqlite3-dev uuid-dev tk-dev"
3233
numpy_deps="gfortran"
3334
apt-get install -y --no-install-recommends \
3435
$ccache_deps \
3536
$numpy_deps \
37+
${deploy_deps} \
3638
${cmake3} \
3739
apt-transport-https \
3840
autoconf \

‎.circleci/docker/common/install_cache.sh

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,9 @@ fi
4646
chmod a+x /opt/cache/bin/sccache
4747

4848
function write_sccache_stub() {
49-
printf "#!/bin/sh\nif [ \$(ps -p \$PPID -o comm=) != sccache ]; then\n exec sccache $(which $1) \"\$@\"\nelse\n exec $(which $1) \"\$@\"\nfi" > "/opt/cache/bin/$1"
49+
# Unset LD_PRELOAD for ps because of asan + ps issues
50+
# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90589
51+
printf "#!/bin/sh\nif [ \$(env -u LD_PRELOAD ps -p \$PPID -o comm=) != sccache ]; then\n exec sccache $(which $1) \"\$@\"\nelse\n exec $(which $1) \"\$@\"\nfi" > "/opt/cache/bin/$1"
5052
chmod a+x "/opt/cache/bin/$1"
5153
}
5254

‎.circleci/docker/requirements-ci.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,10 +80,10 @@ librosa>=0.6.2
8080
#Pinned versions:
8181
#test that import:
8282

83-
mypy==0.812
83+
mypy==0.960
8484
# Pin MyPy version because new errors are likely to appear with each release
8585
#Description: linter
86-
#Pinned versions: 0.812
86+
#Pinned versions: 0.960
8787
#test that import: test_typing.py, test_type_hints.py
8888

8989
#networkx

‎.circleci/scripts/binary_macos_build.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ set -eux -o pipefail
44
source "${BINARY_ENV_FILE:-/Users/distiller/project/env}"
55
mkdir -p "$PYTORCH_FINAL_PACKAGE_DIR"
66

7-
if [[ -z "${IS_GHA:-}" ]]; then
7+
if [[ -z "${GITHUB_ACTIONS:-}" ]]; then
88
export PATH="${workdir:-${HOME}}/miniconda/bin:${PATH}"
99
fi
1010

‎.circleci/scripts/binary_populate_env.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ export TZ=UTC
55
tagged_version() {
66
# Grabs version from either the env variable CIRCLE_TAG
77
# or the pytorch git described version
8-
if [[ "$OSTYPE" == "msys" && -z "${IS_GHA:-}" ]]; then
8+
if [[ "$OSTYPE" == "msys" && -z "${GITHUB_ACTIONS:-}" ]]; then
99
GIT_DIR="${workdir}/p/.git"
1010
else
1111
GIT_DIR="${workdir}/pytorch/.git"
@@ -162,7 +162,7 @@ if [[ "$(uname)" != Darwin ]]; then
162162
EOL
163163
fi
164164

165-
if [[ -z "${IS_GHA:-}" ]]; then
165+
if [[ -z "${GITHUB_ACTIONS:-}" ]]; then
166166
cat >>"$envfile" <<EOL
167167
export workdir="$workdir"
168168
export MAC_PACKAGE_WORK_DIR="$workdir"

‎.circleci/scripts/setup_ci_environment.sh

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,6 @@ add_to_env_file() {
6666
esac
6767
}
6868

69-
add_to_env_file IN_CI 10000 1
7069
add_to_env_file CI_MASTER "${CI_MASTER:-}"
7170
add_to_env_file COMMIT_SOURCE "${CIRCLE_BRANCH:-}"
7271
add_to_env_file BUILD_ENVIRONMENT "${BUILD_ENVIRONMENT}"

‎.circleci/verbatim-sources/commands.yml

Lines changed: 0 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -132,43 +132,3 @@ commands:
132132
else
133133
echo "This is not a pull request, skipping..."
134134
fi
135-
136-
upload_binary_size_for_android_build:
137-
description: "Upload binary size data for Android build"
138-
parameters:
139-
build_type:
140-
type: string
141-
default: ""
142-
artifacts:
143-
type: string
144-
default: ""
145-
steps:
146-
- run:
147-
name: "Binary Size - Install Dependencies"
148-
no_output_timeout: "5m"
149-
command: |
150-
retry () {
151-
$* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
152-
}
153-
retry pip3 install requests
154-
- run:
155-
name: "Binary Size - Untar Artifacts"
156-
no_output_timeout: "5m"
157-
command: |
158-
# The artifact file is created inside docker container, which contains the result binaries.
159-
# Now unpackage it into the project folder. The subsequent script will scan project folder
160-
# to locate result binaries and report their sizes.
161-
# If artifact file is not provided it assumes that the project folder has been mounted in
162-
# the docker during build and already contains the result binaries, so this step can be skipped.
163-
export ARTIFACTS="<< parameters.artifacts >>"
164-
if [ -n "${ARTIFACTS}" ]; then
165-
tar xf "${ARTIFACTS}" -C ~/project
166-
fi
167-
- run:
168-
name: "Binary Size - Upload << parameters.build_type >>"
169-
no_output_timeout: "5m"
170-
command: |
171-
cd ~/project
172-
export ANDROID_BUILD_TYPE="<< parameters.build_type >>"
173-
export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
174-
python3 -m tools.stats.upload_binary_size_to_scuba android

0 commit comments

Comments
 (0)
0