pytorch · ezyang · Sep 11, 2022 · Sep 12, 2022 · Sep 13, 2022 · Sep 14, 2022
diff --git a/.bazelignore b/.bazelignore
@@ -0,0 +1,3 @@
+# We do not use this library in our Bazel build. It contains an
+# infinitely recursing symlink that makes Bazel very unhappy.
+third_party/ittapi/
diff --git a/.bazelrc b/.bazelrc
@@ -0,0 +1,114 @@
+build --cxxopt=--std=c++17
+build --copt=-I.
+# Bazel does not support including its cc_library targets as system
+# headers. We work around this for generated code
+# (e.g. c10/macros/cmake_macros.h) by making the generated directory a
+# system include path.
+build --copt=-isystem --copt bazel-out/k8-fastbuild/bin
+build --copt=-isystem --copt bazel-out/darwin-fastbuild/bin
+build --experimental_ui_max_stdouterr_bytes=2048576
+
+# Configuration to disable tty features for environments like CI
+build:no-tty --curses no
+build:no-tty --progress_report_interval 10
+build:no-tty --show_progress_rate_limit 10
+
+# Build with GPU support by default.
+build --define=cuda=true
+# rules_cuda configuration
+build --@rules_cuda//cuda:enable_cuda
+build --@rules_cuda//cuda:cuda_targets=sm_52
+build --@rules_cuda//cuda:compiler=nvcc
+build --repo_env=CUDA_PATH=/usr/local/cuda
+
+# Configuration to build without GPU support
+build:cpu-only --define=cuda=false
+# define a separate build folder for faster switching between configs
+build:cpu-only --platform_suffix=-cpu-only
+# See the note on the config-less build for details about why we are
+# doing this. We must also do it for the "-cpu-only" platform suffix.
+build --copt=-isystem --copt=bazel-out/k8-fastbuild-cpu-only/bin
+# rules_cuda configuration
+build:cpu-only --@rules_cuda//cuda:enable_cuda=False
+
+# Definition of --config=shell
+# interactive shell immediately before execution
+build:shell --run_under="//tools/bazel_tools:shellwrap"
+
+# Disable all warnings for external repositories. We don't care about
+# their warnings.
+build --per_file_copt=^external/@-w
+
+# Set additional warnings to error level.
+#
+# Implementation notes:
+#  * we use file extensions to determine if we are using the C++
+#    compiler or the cuda compiler
+#  * we use ^// at the start of the regex to only permit matching
+#    PyTorch files. This excludes external repos.
+#
+# Note that because this is logically a command-line flag, it is
+# considered the word on what warnings are enabled. This has the
+# unfortunate consequence of preventing us from disabling an error at
+# the target level because those flags will come before these flags in
+# the action invocation. Instead we provide per-file exceptions after
+# this.
+#
+# On the bright side, this means we don't have to more broadly apply
+# the exceptions to an entire target.
+#
+# Looking for CUDA flags? We have a cu_library macro that we can edit
+# directly. Look in //tools/rules:cu.bzl for details. Editing the
+# macro over this has the following advantages:
+#  * making changes does not require discarding the Bazel analysis
+#    cache
+#  * it allows for selective overrides on individual targets since the
+#    macro-level opts will come earlier than target level overrides
+
+build --per_file_copt='^//.*\.(cpp|cc)$'@-Werror=all
+# The following warnings come from -Wall. We downgrade them from error
+# to warnings here.
+#
+# We intentionally use #pragma unroll, which is compiler specific.
+build --per_file_copt='^//.*\.(cpp|cc)$'@-Wno-error=unknown-pragmas
+
+build --per_file_copt='^//.*\.(cpp|cc)$'@-Werror=extra
+# The following warnings come from -Wextra. We downgrade them from error
+# to warnings here.
+#
+# unused-parameter-compare has a tremendous amount of violations in the
+# codebase. It will be a lot of work to fix them, just disable it for
+# now.
+build --per_file_copt='^//.*\.(cpp|cc)$'@-Wno-unused-parameter
+# missing-field-parameters has both a large number of violations in
+# the codebase, but it also is used pervasively in the Python C
+# API. There are a couple of catches though:
+# * we use multiple versions of the Python API and hence have
+#   potentially multiple different versions of each relevant
+#   struct. They may have different numbers of fields. It will be
+#   unwieldy to support multiple versions in the same source file.
+# * Python itself for many of these structs recommends only
+#   initializing a subset of the fields. We should respect the API
+#   usage conventions of our dependencies.
+#
+# Hence, we just disable this warning altogether. We may want to clean
+# up some of the clear-cut cases that could be risky, but we still
+# likely want to have this disabled for the most part.
+build --per_file_copt='^//.*\.(cpp|cc)$'@-Wno-missing-field-initializers
+
+build --per_file_copt='^//.*\.(cpp|cc)$'@-Wno-unused-function
+build --per_file_copt='^//.*\.(cpp|cc)$'@-Wno-unused-variable
+
+build --per_file_copt='//:aten/src/ATen/RegisterCompositeExplicitAutograd\.cpp$'@-Wno-error=unused-function
+build --per_file_copt='//:aten/src/ATen/RegisterCompositeImplicitAutograd\.cpp$'@-Wno-error=unused-function
+build --per_file_copt='//:aten/src/ATen/RegisterMkldnnCPU\.cpp$'@-Wno-error=unused-function
+build --per_file_copt='//:aten/src/ATen/RegisterNestedTensorCPU\.cpp$'@-Wno-error=unused-function
+build --per_file_copt='//:aten/src/ATen/RegisterQuantizedCPU\.cpp$'@-Wno-error=unused-function
+build --per_file_copt='//:aten/src/ATen/RegisterSparseCPU\.cpp$'@-Wno-error=unused-function
+build --per_file_copt='//:aten/src/ATen/RegisterSparseCsrCPU\.cpp$'@-Wno-error=unused-function
+build --per_file_copt='//:aten/src/ATen/RegisterNestedTensorMeta\.cpp$'@-Wno-error=unused-function
+build --per_file_copt='//:aten/src/ATen/RegisterSparseMeta\.cpp$'@-Wno-error=unused-function
+build --per_file_copt='//:aten/src/ATen/RegisterQuantizedMeta\.cpp$'@-Wno-error=unused-function
+build --per_file_copt='//:aten/src/ATen/RegisterZeroTensor\.cpp$'@-Wno-error=unused-function
+build --per_file_copt='//:torch/csrc/lazy/generated/RegisterAutogradLazy\.cpp$'@-Wno-error=unused-function
+build --per_file_copt='//:torch/csrc/lazy/generated/RegisterLazy\.cpp$'@-Wno-error=unused-function
diff --git a/.bazelversion b/.bazelversion
@@ -0,0 +1 @@
+6.1.1
diff --git a/.buckconfig.oss b/.buckconfig.oss
@@ -0,0 +1,26 @@
+[pt]
+  is_oss=1
+
+[buildfile]
+  name = BUCK.oss
+  includes = //tools/build_defs/select.bzl
+
+[repositories]
+  bazel_skylib = third_party/bazel-skylib/
+  ovr_config = .
+
+[download]
+  in_build = true
+
+[cxx]
+  cxxflags = -std=c++17
+  ldflags = -Wl,--no-undefined
+  should_remap_host_platform = true
+  cpp = /usr/bin/clang
+  cc = /usr/bin/clang
+  cxx = /usr/bin/clang++
+  cxxpp = /usr/bin/clang++
+  ld = /usr/bin/clang++
+
+[project]
+  default_flavors_mode=all
@@ -0,0 +1,36 @@
+set -ex
+
+LOCAL_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+ROOT_DIR=$(cd "$LOCAL_DIR"/../.. && pwd)
+TEST_DIR="$ROOT_DIR/test"
+gtest_reports_dir="${TEST_DIR}/test-reports/cpp"
+pytest_reports_dir="${TEST_DIR}/test-reports/python"
+
+# Figure out which Python to use
+PYTHON="$(which python)"
+if [[ "${BUILD_ENVIRONMENT}" =~ py((2|3)\.?[0-9]?\.?[0-9]?) ]]; then
+  PYTHON=$(which "python${BASH_REMATCH[1]}")
+fi
+
+if [[ "${BUILD_ENVIRONMENT}" == *rocm* ]]; then
+    # HIP_PLATFORM is auto-detected by hipcc; unset to avoid build errors
+    unset HIP_PLATFORM
+    if which sccache > /dev/null; then
+        # Save sccache logs to file
+        sccache --stop-server || true
+        rm -f ~/sccache_error.log || true
+        SCCACHE_ERROR_LOG=~/sccache_error.log SCCACHE_IDLE_TIMEOUT=0 sccache --start-server
+
+        # Report sccache stats for easier debugging
+        sccache --zero-stats
+    fi
+fi
+
+# /usr/local/caffe2 is where the cpp bits are installed to in cmake-only
+# builds. In +python builds the cpp tests are copied to /usr/local/caffe2 so
+# that the test code in .ci/test.sh is the same
+INSTALL_PREFIX="/usr/local/caffe2"
+
+mkdir -p "$gtest_reports_dir" || true
+mkdir -p "$pytest_reports_dir" || true
+mkdir -p "$INSTALL_PREFIX" || true
@@ -0,0 +1,172 @@
+#!/bin/bash
+
+# shellcheck source=./common.sh
+source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
+
+if [[ ${BUILD_ENVIRONMENT} == *onnx* ]]; then
+  pip install click mock tabulate networkx==2.0
+  pip -q install --user "file:///var/lib/jenkins/workspace/third_party/onnx#egg=onnx"
+fi
+
+# Skip tests in environments where they are not built/applicable
+if [[ "${BUILD_ENVIRONMENT}" == *-android* ]]; then
+  echo 'Skipping tests'
+  exit 0
+fi
+if [[ "${BUILD_ENVIRONMENT}" == *-rocm* ]]; then
+  # temporary to locate some kernel issues on the CI nodes
+  export HSAKMT_DEBUG_LEVEL=4
+fi
+# These additional packages are needed for circleci ROCm builds.
+if [[ $BUILD_ENVIRONMENT == *rocm* ]]; then
+    # Need networkx 2.0 because bellmand_ford was moved in 2.1 . Scikit-image by
+    # defaults installs the most recent networkx version, so we install this lower
+    # version explicitly before scikit-image pulls it in as a dependency
+    pip install networkx==2.0
+    # click - onnx
+    pip install --progress-bar off click protobuf tabulate virtualenv mock typing-extensions
+fi
+
+# Find where cpp tests and Caffe2 itself are installed
+if [[ "$BUILD_ENVIRONMENT" == *cmake* ]]; then
+  # For cmake only build we install everything into /usr/local
+  cpp_test_dir="$INSTALL_PREFIX/cpp_test"
+  ld_library_path="$INSTALL_PREFIX/lib"
+else
+  # For Python builds we install into python
+  # cd to /usr first so the python import doesn't get confused by any 'caffe2'
+  # directory in cwd
+  python_installation="$(dirname $(dirname $(cd /usr && $PYTHON -c 'import os; import caffe2; print(os.path.realpath(caffe2.__file__))')))"
+  caffe2_pypath="$python_installation/caffe2"
+  cpp_test_dir="$python_installation/torch/test"
+  ld_library_path="$python_installation/torch/lib"
+fi
+
+################################################################################
+# C++ tests #
+################################################################################
+# Only run cpp tests in the first shard, don't run cpp tests a second time in the second shard
+if [[ "${SHARD_NUMBER:-1}" == "1" ]]; then
+  echo "Running C++ tests.."
+  for test in $(find "$cpp_test_dir" -executable -type f); do
+    case "$test" in
+      # skip tests we know are hanging or bad
+      */mkl_utils_test|*/aten/integer_divider_test)
+        continue
+        ;;
+      */scalar_tensor_test|*/basic|*/native_test)
+        if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
+          continue
+        else
+          LD_LIBRARY_PATH="$ld_library_path" "$test"
+        fi
+        ;;
+      */*_benchmark)
+        LD_LIBRARY_PATH="$ld_library_path" "$test" --benchmark_color=false
+        ;;
+      *)
+        # Currently, we use a mixture of gtest (caffe2) and Catch2 (ATen). While
+        # planning to migrate to gtest as the common PyTorch c++ test suite, we
+        # currently do NOT use the xml test reporter, because Catch doesn't
+        # support multiple reporters
+        # c.f. https://github.com/catchorg/Catch2/blob/master/docs/release-notes.md#223
+        # which means that enabling XML output means you lose useful stdout
+        # output for Jenkins.  It's more important to have useful console
+        # output than it is to have XML output for Jenkins.
+        # Note: in the future, if we want to use xml test reporter once we switch
+        # to all gtest, one can simply do:
+        LD_LIBRARY_PATH="$ld_library_path" \
+            "$test" --gtest_output=xml:"$gtest_reports_dir/$(basename $test).xml"
+        ;;
+    esac
+  done
+fi
+
+################################################################################
+# Python tests #
+################################################################################
+if [[ "$BUILD_ENVIRONMENT" == *cmake* ]]; then
+  exit 0
+fi
+
+# If pip is installed as root, we must use sudo.
+# CircleCI docker images could install conda as jenkins user, or use the OS's python package.
+PIP=$(which pip)
+PIP_USER=$(stat --format '%U' $PIP)
+CURRENT_USER=$(id -u -n)
+if [[ "$PIP_USER" = root && "$CURRENT_USER" != root ]]; then
+  MAYBE_SUDO=sudo
+fi
+
+# Uninstall pre-installed hypothesis and coverage to use an older version as newer
+# versions remove the timeout parameter from settings which ideep/conv_transpose_test.py uses
+$MAYBE_SUDO pip -q uninstall -y hypothesis
+$MAYBE_SUDO pip -q uninstall -y coverage
+
+# "pip install hypothesis==3.44.6" from official server is unreliable on
+# CircleCI, so we host a copy on S3 instead
+$MAYBE_SUDO pip -q install attrs==18.1.0 -f https://s3.amazonaws.com/ossci-linux/wheels/attrs-18.1.0-py2.py3-none-any.whl
+$MAYBE_SUDO pip -q install coverage==4.5.1 -f https://s3.amazonaws.com/ossci-linux/wheels/coverage-4.5.1-cp36-cp36m-macosx_10_12_x86_64.whl
+$MAYBE_SUDO pip -q install hypothesis==3.44.6 -f https://s3.amazonaws.com/ossci-linux/wheels/hypothesis-3.44.6-py3-none-any.whl
+
+# Collect additional tests to run (outside caffe2/python)
+EXTRA_TESTS=()
+
+# CUDA builds always include NCCL support
+if [[ "$BUILD_ENVIRONMENT" == *-cuda* ]] || [[ "$BUILD_ENVIRONMENT" == *-rocm* ]]; then
+  EXTRA_TESTS+=("$caffe2_pypath/contrib/nccl")
+fi
+
+rocm_ignore_test=()
+if [[ $BUILD_ENVIRONMENT == *-rocm* ]]; then
+  # Currently these tests are failing on ROCM platform:
+
+  # On ROCm, RCCL (distributed) development isn't complete.
+  # https://github.com/ROCmSoftwarePlatform/rccl
+  rocm_ignore_test+=("--ignore $caffe2_pypath/python/data_parallel_model_test.py")
+
+  # This test has been flaky in ROCm CI (but note the tests are
+  # cpu-only so should be unrelated to ROCm)
+  rocm_ignore_test+=("--ignore $caffe2_pypath/python/operator_test/blobs_queue_db_test.py")
+  # This test is skipped on Jenkins(compiled without MKL) and otherwise known flaky
+  rocm_ignore_test+=("--ignore $caffe2_pypath/python/ideep/convfusion_op_test.py")
+  # This test is skipped on Jenkins(compiled without MKL) and causing segfault on Circle
+  rocm_ignore_test+=("--ignore $caffe2_pypath/python/ideep/pool_op_test.py")
+fi
+
+echo "Running Python tests.."
+# locale setting is required by click package
+for loc in "en_US.utf8" "C.UTF-8"; do
+  if locale -a | grep "$loc" >/dev/null 2>&1; then
+    export LC_ALL="$loc"
+    export LANG="$loc"
+    break;
+  fi
+done
+
+# Some Caffe2 tests fail when run using AVX512 ISA, see https://github.com/pytorch/pytorch/issues/66111
+export DNNL_MAX_CPU_ISA=AVX2
+
+# Should still run even in the absence of SHARD_NUMBER
+if [[ "${SHARD_NUMBER:-1}" == "1" ]]; then
+  # TODO(sdym@meta.com) remove this when the linked issue resolved.
+  # py is temporary until https://github.com/Teemu/pytest-sugar/issues/241 is fixed
+  pip install --user py==1.11.0
+  pip install --user pytest-sugar
+  # NB: Warnings are disabled because they make it harder to see what
+  # the actual erroring test is
+  "$PYTHON" \
+    -m pytest \
+    -x \
+    -v \
+    --disable-warnings \
+    --junit-xml="$pytest_reports_dir/result.xml" \
+    --ignore "$caffe2_pypath/python/test/executor_test.py" \
+    --ignore "$caffe2_pypath/python/operator_test/matmul_op_test.py" \
+    --ignore "$caffe2_pypath/python/operator_test/pack_ops_test.py" \
+    --ignore "$caffe2_pypath/python/mkl/mkl_sbn_speed_test.py" \
+    --ignore "$caffe2_pypath/python/trt/test_pt_onnx_trt.py" \
+    ${rocm_ignore_test[@]} \
+    "$caffe2_pypath/python" \
+    "${EXTRA_TESTS[@]}"
+fi
@@ -0,0 +1,31 @@
+# Docker images for Jenkins
+
+This directory contains everything needed to build the Docker images
+that are used in our CI
+
+The Dockerfiles located in subdirectories are parameterized to
+conditionally run build stages depending on build arguments passed to
+`docker build`. This lets us use only a few Dockerfiles for many
+images. The different configurations are identified by a freeform
+string that we call a _build environment_. This string is persisted in
+each image as the `BUILD_ENVIRONMENT` environment variable.
+
+See `build.sh` for valid build environments (it's the giant switch).
+
+Docker builds are now defined with `.circleci/cimodel/data/simple/docker_definitions.py`
+
+## Contents
+
+* `build.sh` -- dispatch script to launch all builds
+* `common` -- scripts used to execute individual Docker build stages
+* `ubuntu-cuda` -- Dockerfile for Ubuntu image with CUDA support for nvidia-docker
+
+## Usage
+
+```bash
+# Build a specific image
+./build.sh pytorch-linux-bionic-py3.8-gcc9 -t myimage:latest
+
+# Set flags (see build.sh) and build image
+sudo bash -c 'PROTOBUF=1 ./build.sh pytorch-linux-bionic-py3.8-gcc9 -t myimage:latest
+```
@@ -0,0 +1 @@
+<manifest package="org.pytorch.deps" />