8000 Update on "[caffe2] Export operators to c10 without including ATen/Te… · pytorch/pytorch@795a88b · GitHub
[go: up one dir, main page]
Skip to content

Commit 795a88b

Browse files
peterbell10peterbell10
committed
Update on "[caffe2] Export operators to c10 without including ATen/Tensor.h"
With this change, the only caffe2 files that depend on `ATen/Tensor.h` are ones that directly use the ATen API. Specifically, ``` [ "caffe2/CMakeFiles/torch_cuda_cpp.dir/contrib/aten/aten_op_gpu.cc.o", "caffe2/CMakeFiles/torch_cpu.dir/core/tensor.cc.o", "caffe2/CMakeFiles/torch_cuda_cpp.dir/operators/layer_norm_op.cu.o", "caffe2/CMakeFiles/torch_cpu.dir/core/IValueInterface.cc.o", "caffe2/CMakeFiles/cuda_tensor_interop_test.dir/__/aten/src/ATen/test/cuda_tensor_interop_test.cpp.o", "caffe2/CMakeFiles/torch_cpu.dir/contrib/aten/aten_op.cc.o", "caffe2/CMakeFiles/caffe2_pybind11_state_gpu.dir/python/pybind_state.cc.o", "caffe2/CMakeFiles/torch_cpu.dir/operators/layer_norm_op.cc.o", "caffe2/CMakeFiles/torch_cpu.dir/core/export_c10_op_to_caffe2.cc.o", "caffe2/CMakeFiles/torch_cpu.dir/core/export_caffe2_op_to_c10.cc.o", "caffe2/CMakeFiles/torch_cpu.dir/operators/enforce_finite_op.cc.o", "caffe2/CMakeFiles/torch_cpu.dir/core/operator.cc.o", "caffe2/CMakeFiles/tensor_interop_test.dir/__/aten/src/ATen/test/tensor_interop_test.cpp.o", "caffe2/CMakeFiles/caffe2_pybind11_state.dir/python/pybind_state.cc.o" ] ``` Differential Revision: [D32289810](https://our.internmc.facebook.com/intern/diff/D32289810) [ghstack-poisoned]
2 parents 9284050 + dad41f9 commit 795a88b
  • generated-linux-xenial-py3-clang5-mobile-custom-build-static.yml
  • generated-linux-xenial-py3.7-clang7-asan.yml
  • generated-linux-xenial-py3.7-clang7-onnx.yml
  • generated-linux-xenial-py3.7-gcc5.4.yml
  • generated-linux-xenial-py3.7-gcc7-no-ops.yml
  • generated-linux-xenial-py3.7-gcc7.yml
  • generated-parallelnative-linux-xenial-py3.7-gcc5.4.yml
  • generated-periodic-libtorch-linux-bionic-cuda11.5-py3.7-gcc7.yml
  • generated-periodic-libtorch-linux-xenial-cuda11.1-py3.7-gcc7.yml
  • generated-periodic-linux-bionic-cuda11.5-py3.7-gcc7.yml
  • generated-periodic-linux-xenial-cuda10.2-py3-gcc7-slow-gradcheck.yml
  • generated-periodic-linux-xenial-cuda11.1-py3.7-gcc7-debug.yml
  • generated-pytorch-linux-xenial-py3-clang5-android-ndk-r19c-build.yml
  • generated-pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-custom-build-single-full-jit.yml
  • generated-pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-custom-build-single.yml
  • run_torchbench.yml
  • syncbranches.yml
  • trigger_nightly.yml
  • trymerge.yml
  • .jenkins/pytorch
  • BUILD.bazel
  • aten/src/ATen
  • c10
  • caffe2
  • docs/source/notes
  • modules/detectron
  • test
  • tools
  • torch

    • Some content is hidden

    Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

    170 files changed

    +9578
    -3028
    lines changed

    ‎.github/generated-ciflow-ruleset.json

    Lines changed: 11 additions & 1 deletion
    Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

    ‎.github/scripts/__init__.py

    Whitespace-only changes.

    ‎.github/scripts/generate_binary_build_matrix.py

    Lines changed: 2 additions & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -16,7 +16,7 @@
    1616
    CUDA_ARCHES = ["10.2", "11.1", "11.3", "11.5"]
    1717

    1818

    19-
    ROCM_ARCHES = ["4.2", "4.3.1"]
    19+
    ROCM_ARCHES = ["4.3.1", "4.5.2"]
    2020

    2121

    2222
    def arch_type(arch_version: str) -> str:
    @@ -65,6 +65,7 @@ def arch_type(arch_version: str) -> str:
    6565
    "3.7",
    6666
    "3.8",
    6767
    "3.9",
    68+
    "3.10"
    6869
    ]
    6970

    7071

    ‎.github/scripts/generate_ci_workflows.py

    Lines changed: 38 additions & 4 deletions
    Original file line numberDiff line numberDiff line change
    @@ -29,9 +29,22 @@
    2929
    LINUX_CPU_TEST_RUNNER = "linux.2xlarge"
    3030
    # contains 1 gpu
    3131
    LINUX_CUDA_TEST_RUNNER = "linux.4xlarge.nvidia.gpu"
    32+
    # contains 4 gpus
    33+
    LINUX_ROCM_TEST_RUNNER = "linux.rocm.gpu"
    3234
    LINUX_RUNNERS = {
    3335
    LINUX_CPU_TEST_RUNNER,
    3436
    LINUX_CUDA_TEST_RUNNER,
    37+
    LINUX_ROCM_TEST_RUNNER,
    38+
    }
    39+
    40+
    LINUX_DISTRIBUTED_GPU_RUNNERS = {
    41+
    LINUX_CUDA_TEST_RUNNER : "linux.8xlarge.nvidia.gpu",
    42+
    LINUX_ROCM_TEST_RUNNER : LINUX_ROCM_TEST_RUNNER,
    43+
    }
    44+
    45+
    LINUX_MULTIGPU_RUNNERS = {
    46+
    LINUX_CUDA_TEST_RUNNER : "linux.16xlarge.nvidia.gpu",
    47+
    LINUX_ROCM_TEST_RUNNER : LINUX_ROCM_TEST_RUNNER,
    3548
    }
    3649

    3750
    MACOS_TEST_RUNNER_10_15 = "macos-10.15"
    @@ -46,6 +59,9 @@
    4659
    WINDOWS_CUDA_TEST_RUNNER,
    4760
    LINUX_CUDA_TEST_RUNNER,
    4861
    }
    62+
    ROCM_RUNNERS = {
    63+
    LINUX_ROCM_TEST_RUNNER,
    64+
    }
    4965
    CPU_RUNNERS = {
    5066
    WINDOWS_CPU_TEST_RUNNER,
    5167
    LINUX_CPU_TEST_RUNNER,
    @@ -55,6 +71,7 @@
    5571
    LABEL_CIFLOW_BAZEL = "ciflow/bazel"
    5672
    LABEL_CIFLOW_CPU = "ciflow/cpu"
    5773
    LABEL_CIFLOW_CUDA = "ciflow/cuda"
    74+
    LABEL_CIFLOW_ROCM = "ciflow/rocm"
    5875
    LABEL_CIFLOW_DOCS = "ciflow/docs"
    5976
    LABEL_CIFLOW_DEFAULT = "ciflow/default"
    6077
    LABEL_CIFLOW_LIBTORCH = "ciflow/libtorch"
    @@ -164,6 +181,8 @@ class CIWorkflow:
    164181

    165182
    # Optional fields
    166183
    test_runner_type: str = ''
    184+
    multigpu_runner_type: str = ''
    185+
    distributed_gpu_runner_type: str = ''
    167186
    ciflow_config: CIFlowConfig = field(default_factory=CIFlowConfig)
    168187
    cuda_version: str = ''
    169188
    docker_image_base: str = ''
    @@ -205,6 +224,9 @@ def __post_init__(self) -> None:
    205224
    if self.fx2trt_test:
    206225
    self.enable_fx2trt_test = 1
    207226

    227+
    self.multigpu_runner_type = LINUX_MULTIGPU_RUNNERS.get(self.test_runner_type, "linux.16xlarge.nvidia.gpu")
    228+
    self.distributed_gpu_runner_type = LINUX_DISTRIBUTED_GPU_RUNNERS.get(self.test_runner_type, "linux.8xlarge.nvidia.gpu")
    229+
    208230
    # If num_test_shards_on_pull_request is not user-defined, default to num_test_shards unless we are
    209231
    # only running smoke tests on the pull request.
    210232
    if self.num_test_shards_on_pull_request == -1:
    @@ -235,6 +257,8 @@ def assert_valid(self) -> None:
    235257
    assert self.test_runner_type != ''
    236258
    if self.test_runner_type in CUDA_RUNNERS:
    237259
    assert LABEL_CIFLOW_CUDA in self.ciflow_config.labels
    260+
    if self.test_runner_type in ROCM_RUNNERS:
    261+
    assert LABEL_CIFLOW_ROCM in self.ciflow_config.labels
    238262
    if self.test_runner_type in CPU_RUNNERS and not self.exclude_test:
    239263
    assert LABEL_CIFLOW_CPU in self.ciflow_config.labels
    240264
    if self.is_scheduled:
    @@ -576,6 +600,16 @@ def generate_workflow_file(self, workflow_template: jinja2.Template) -> None:
    576600
    labels=set([LABEL_CIFLOW_DEFAULT, LABEL_CIFLOW_LINUX, LABEL_CIFLOW_CPU]),
    577601
    ),
    578602
    ),
    603+
    CIWorkflow(
    604+
    arch="linux",
    605+
    build_environment="linux-bionic-rocm4.5-py3.7",
    606+
    docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-rocm4.5-py3.7",
    607+
    test_runner_type=LINUX_ROCM_TEST_RUNNER,
    608+
    num_test_shards=2,
    609+
    ciflow_config=CIFlowConfig(
    610+
    labels=set([LABEL_CIFLOW_LINUX, LABEL_CIFLOW_ROCM]),
    611+
    ),
    612+
    ),
    579613
    CIWorkflow(
    580614
    arch="linux",
    581615
    build_environment="libtorch-linux-xenial-cuda11.3-py3.7-gcc7",
    @@ -836,7 +870,7 @@ def generate_workflow_file(self, workflow_template: jinja2.Template) -> None:
    836870
    package_type="manywheel",
    837871
    build_configs=generate_binary_build_matrix.generate_wheels_matrix(),
    838872
    ciflow_config=CIFlowConfig(
    839-
    labels={LABEL_CIFLOW_BINARIES, LABEL_CIFLOW_BINARIES_WHEEL},
    873+
    labels={LABEL_CIFLOW_DEFAULT, LABEL_CIFLOW_BINARIES, LABEL_CIFLOW_BINARIES_WHEEL},
    840874
    isolated_workflow=True,
    841875
    ),
    842876
    ),
    @@ -845,7 +879,7 @@ def generate_workflow_file(self, workflow_template: jinja2.Template) -> None:
    845879
    package_type="conda",
    846880
    build_configs=generate_binary_build_matrix.generate_conda_matrix(),
    847881
    ciflow_config=CIFlowConfig(
    848-
    labels={LABEL_CIFLOW_BINARIES, LABEL_CIFLOW_BINARIES_CONDA},
    882+
    labels={LABEL_CIFLOW_DEFAULT, LABEL_CIFLOW_BINARIES, LABEL_CIFLOW_BINARIES_CONDA},
    849883
    isolated_workflow=True,
    850884
    ),
    851885
    ),
    @@ -857,7 +891,7 @@ def generate_workflow_file(self, workflow_template: jinja2.Template) -> None:
    857891
    generate_binary_build_matrix.CXX11_ABI
    858892
    ),
    859893
    ciflow_config=CIFlowConfig(
    860-
    labels={LABEL_CIFLOW_BINARIES, LABEL_CIFLOW_BINARIES_LIBTORCH},
    894+
    labels={LABEL_CIFLOW_DEFAULT, LABEL_CIFLOW_BINARIES, LABEL_CIFLOW_BINARIES_LIBTORCH},
    861895
    isolated_workflow=True,
    862896
    ),
    863897
    ),
    @@ -869,7 +903,7 @@ def generate_workflow_file(self, workflow_template: jinja2.Template) -> None:
    869903
    generate_binary_build_matrix.PRE_CXX11_ABI
    870904
    ),
    871905
    ciflow_config=CIFlowConfig(
    872-
    labels={LABEL_CIFLOW_BINARIES, LABEL_CIFLOW_BINARIES_LIBTORCH},
    906+
    labels={LABEL_CIFLOW_DEFAULT, LABEL_CIFLOW_BINARIES, LABEL_CIFLOW_BINARIES_LIBTORCH},
    873907
    isolated_workflow=True,
    874908
    ),
    875909
    ),

    0 commit comments

    Comments
     (0)
    0