8000 Update on "[caffe2] Remove OperatorBase::newstyle_outputs_" · pytorch/pytorch@8d90b65 · GitHub
[go: up one dir, main page]
Skip to content

Commit 8d90b65

Browse files
peterbell10peterbell10
committed
Update on "[caffe2] Remove OperatorBase::newstyle_outputs_"
`OperatorBase` maintains `output_tensors_` and `newstyle_outputs_` which hold the same list of tensors except one is `vector<caffe2::Tensor>` and the other is `List<at::Tensor>`. This instead maintains only `output_tensors_` and handles the conversions inside of export_caffe2_op_to_c10. Differential Revision: [D32289811](https://our.internmc.facebook.com/intern/diff/D32289811) [ghstack-poisoned]
2 parents 018f0a3 + 7f7bd45 commit 8d90b65

File tree

168 files changed

+9576
-3026
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

168 files changed

+9576
-3026
lines changed

‎.github/generated-ciflow-ruleset.json

Lines changed: 11 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎.github/scripts/__init__.py

Whitespace-only changes.

‎.github/scripts/generate_binary_build_matrix.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
CUDA_ARCHES = ["10.2", "11.1", "11.3", "11.5"]
1717

1818

19-
ROCM_ARCHES = ["4.2", "4.3.1"]
19+
ROCM_ARCHES = ["4.3.1", "4.5.2"]
2020

2121

2222
def arch_type(arch_version: str) -> str:
@@ -65,6 +65,7 @@ def arch_type(arch_version: str) -> str:
6565
"3.7",
6666
"3.8",
6767
"3.9",
68+
"3.10"
6869
]
6970

7071

‎.github/scripts/generate_ci_workflows.py

Lines changed: 38 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,22 @@
2929
LINUX_CPU_TEST_RUNNER = "linux.2xlarge"
3030
# contains 1 gpu
3131
LINUX_CUDA_TEST_RUNNER = "linux.4xlarge.nvidia.gpu"
32+
# contains 4 gpus
33+
LINUX_ROCM_TEST_RUNNER = "linux.rocm.gpu"
3234
LINUX_RUNNERS = {
3335
LINUX_CPU_TEST_RUNNER,
3436
LINUX_CUDA_TEST_RUNNER,
37+
LINUX_ROCM_TEST_RUNNER,
38+
}
39+
40+
LINUX_DISTRIBUTED_GPU_RUNNERS = {
41+
LINUX_CUDA_TEST_RUNNER : "linux.8xlarge.nvidia.gpu",
42+
LINUX_ROCM_TEST_RUNNER : LINUX_ROCM_TEST_RUNNER,
43+
}
44+
45+
LINUX_MULTIGPU_RUNNERS = {
46+
LINUX_CUDA_TEST_RUNNER : "linux.16xlarge.nvidia.gpu",
47+
LINUX_ROCM_TEST_RUNNER : LINUX_ROCM_TEST_RUNNER,
3548
}
3649

3750
MACOS_TEST_RUNNER_10_15 = "macos-10.15"
@@ -46,6 +59,9 @@
4659
WINDOWS_CUDA_TEST_RUNNER,
4760
LINUX_CUDA_TEST_RUNNER,
4861
}
62+
ROCM_RUNNERS = {
63+
LINUX_ROCM_TEST_RUNNER,
64+
}
4965
CPU_RUNNERS = {
5066
WINDOWS_CPU_TEST_RUNNER,
5167
LINUX_CPU_TEST_RUNNER,
@@ -55,6 +71,7 @@
5571
LABEL_CIFLOW_BAZEL = "ciflow/bazel"
5672
LABEL_CIFLOW_CPU = "ciflow/cpu"
5773
LABEL_CIFLOW_CUDA = "ciflow/cuda"
74+
LABEL_CIFLOW_ROCM = "ciflow/rocm"
5875
LABEL_CIFLOW_DOCS = "ciflow/docs"
5976
LABEL_CIFLOW_DEFAULT = "ciflow/default"
6077
LABEL_CIFLOW_LIBTORCH = "ciflow/libtorch"
@@ -164,6 +181,8 @@ class CIWorkflow:
164181

165182
# Optional fields
166183
test_runner_type: str = ''
184+
multigpu_runner_type: str = ''
185+
distributed_gpu_runner_type: str = ''
167186
ciflow_config: CIFlowConfig = field(default_factory=CIFlowConfig)
168187
cuda_version: str = ''
169188
docker_image_base: str = ''
@@ -205,6 +224,9 @@ def __post_init__(self) -> None:
205224
if self.fx2trt_test:
206225
self.enable_fx2trt_test = 1
207226

227+
self.multigpu_runner_type = LINUX_MULTIGPU_RUNNERS.get(self.test_runner_type, "linux.16xlarge.nvidia.gpu")
228+
self.distributed_gpu_runner_type = LINUX_DISTRIBUTED_GPU_RUNNERS.get(self.test_runner_type, "linux.8xlarge.nvidia.gpu")
229+
208230
# If num_test_shards_on_pull_request is not user-defined, default to num_test_shards unless we are
209231
# only running smoke tests on the pull request.
210232
if self.num_test_shards_on_pull_request == -1:
@@ -235,6 +257,8 @@ def assert_valid(self) -> None:
235257
10000 assert self.test_runner_type != ''
236258
if self.test_runner_type in CUDA_RUNNERS:
237259
assert LABEL_CIFLOW_CUDA in self.ciflow_config.labels
260+
if self.test_runner_type in ROCM_RUNNERS:
261+
assert LABEL_CIFLOW_ROCM in self.ciflow_config.labels
238262
if self.test_runner_type in CPU_RUNNERS and not self.exclude_test:
239263
assert LABEL_CIFLOW_CPU in self.ciflow_config.labels
240264
if self.is_scheduled:
@@ -576,6 +600,16 @@ def generate_workflow_file(self, workflow_template: jinja2.Template) -> None:
576600
labels=set([LABEL_CIFLOW_DEFAULT, LABEL_CIFLOW_LINUX, LABEL_CIFLOW_CPU]),
577601
),
578602
),
603+
CIWorkflow(
604+
arch="linux",
605+
build_environment="linux-bionic-rocm4.5-py3.7",
606+
docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-rocm4.5-py3.7",
607+
test_runner_type=LINUX_ROCM_TEST_RUNNER,
608+
num_test_shards=2,
609+
ciflow_config=CIFlowConfig(
610+
labels=set([LABEL_CIFLOW_LINUX, LABEL_CIFLOW_ROCM]),
611+
),
612+
),
579613
CIWorkflow(
580614
arch="linux",
581615
build_environment="libtorch-linux-xenial-cuda11.3-py3.7-gcc7",
@@ -836,7 +870,7 @@ def generate_workflow_file(self, workflow_template: jinja2.Template) -> None:
836870
package_type="manywheel",
837871
build_configs=generate_binary_build_matrix.generate_wheels_matrix(),
838872
ciflow_config=CIFlowConfig(
839-
labels={LABEL_CIFLOW_BINARIES, LABEL_CIFLOW_BINARIES_WHEEL},
873+
labels={LABEL_CIFLOW_DEFAULT, LABEL_CIFLOW_BINARIES, LABEL_CIFLOW_BINARIES_WHEEL},
840874
isolated_workflow=True,
841875
),
842876
),
@@ -845,7 +879,7 @@ def generate_workflow_file(self, workflow_template: jinja2.Template) -> None:
845879
package_type="conda",
846880
build_configs=generate_binary_build_matrix.generate_conda_matrix(),
847881
ciflow_config=CIFlowConfig(
848-
labels={LABEL_CIFLOW_BINARIES, LABEL_CIFLOW_BINARIES_CONDA},
882+
labels={LABEL_CIFLOW_DEFAULT, LABEL_CIFLOW_BINARIES, LABEL_CIFLOW_BINARIES_CONDA},
849883
isolated_workflow=True,
850884
),
851885
),
@@ -857,7 +891,7 @@ def generate_workflow_file(self, workflow_template: jinja2.Template) -> None:
857891
generate_binary_build_matrix.CXX11_ABI
858892
),
859893
ciflow_config=CIFlowConfig(
860-
labels={LABEL_CIFLOW_BINARIES, LABEL_CIFLOW_BINARIES_LIBTORCH},
894+
labels={LABEL_CIFLOW_DEFAULT, LABEL_CIFLOW_BINARIES, LABEL_CIFLOW_BINARIES_LIBTORCH},
861895
isolated_workflow=True,
862896
),
863897
),
@@ -869,7 +903,7 @@ def generate_workflow_file(self, workflow_template: jinja2.Template) -> None:
869903
generate_binary_build_matrix.PRE_CXX11_ABI
870904
),
871905
ciflow_config=CIFlowConfig(
872-
labels={LABEL_CIFLOW_BINARIES, LABEL_CIFLOW_BINARIES_LIBTORCH},
906+
labels={LABEL_CIFLOW_DEFAULT, LABEL_CIFLOW_BINARIES, LABEL_CIFLOW_BINARIES_LIBTORCH},
873907
isolated_workflow=True,
874908
),
875909
),

0 commit comments

Comments
 (0)
0