8000 Rebase on "[caffe2] Export operators to c10 without including ATen/Te… · pytorch/pytorch@c85538f · GitHub
[go: up one dir, main page]
Skip to content

Commit c85538f

Browse files
peterbell10peterbell10
committed
Rebase on "[caffe2] Export operators to c10 without including ATen/Tensor.h"
With this change, the only caffe2 files that depend on `ATen/Tensor.h` are ones that directly use the ATen API. Specifically, ``` [ "caffe2/CMakeFiles/torch_cuda_cpp.dir/contrib/aten/aten_op_gpu.cc.o", "caffe2/CMakeFiles/torch_cpu.dir/core/tensor.cc.o", "caffe2/CMakeFiles/torch_cuda_cpp.dir/operators/layer_norm_op.cu.o", "caffe2/CMakeFiles/torch_cpu.dir/core/IValueInterface.cc.o", "caffe2/CMakeFiles/cuda_tensor_interop_test.dir/__/aten/src/ATen/test/cuda_tensor_interop_test.cpp.o", "caffe2/CMakeFiles/torch_cpu.dir/contrib/aten/aten_op.cc.o", "caffe2/CMakeFiles/caffe2_pybind11_state_gpu.dir/python/pybind_state.cc.o", "caffe2/CMakeFiles/torch_cpu.dir/operators/layer_norm_op.cc.o", "caffe2/CMakeFiles/torch_cpu.dir/core/export_c10_op_to_caffe2.cc.o", "caffe2/CMakeFiles/torch_cpu.dir/core/export_caffe2_op_to_c10.cc.o", "caffe2/CMakeFiles/torch_cpu.dir/operators/enforce_finite_op.cc.o", "caffe2/CMakeFiles/torch_cpu.dir/core/operator.cc.o", "caffe2/CMakeFiles/tensor_interop_test.dir/__/aten/src/ATen/test/tensor_interop_test.cpp.o", "caffe2/CMakeFiles/caffe2_pybind11_state.dir/python/pybind_state.cc.o" ] ``` Differential Revision: [D32289810](https://our.internmc.facebook.com/intern/diff/D32289810) [ghstack-poisoned]
2 parents 4a18522 + 787a974 commit c85538f

File tree

952 files changed

+49851
-12343
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

952 files changed

+49851
-12343
lines changed

‎.circleci/cimodel/data/dimensions.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
"102",
55
"113",
66
"115",
7+
"116",
78
]
89

910
ROCM_VERSIONS = [

‎.circleci/docker/build.sh

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,17 @@ case "$image" in
145145
VISION=yes
146146
KATEX=yes
147147
;;
148+
pytorch-linux-bionic-cuda11.6-cudnn8-py3-gcc7)
149+
CUDA_VERSION=11.6.0
150+
CUDNN_VERSION=8
151+
ANACONDA_PYTHON_VERSION=3.7
152+
CMAKE_VERSION=3.10.3
153+
GCC_VERSION=7
154+
PROTOBUF=yes
155+
DB=yes
156+
VISION=yes
157+
KATEX=yes
158+
;;
148159
pytorch-linux-xenial-py3-clang5-asan)
149160
ANACONDA_PYTHON_VERSION=3.7
150161
CLANG_VERSION=5.0
@@ -222,21 +233,21 @@ case "$image" in
222233
DB=yes
223234
VISION=yes
224235
;;
225-
pytorch-linux-bionic-rocm4.3.1-py3.7)
236+
pytorch-linux-bionic-rocm4.5-py3.7)
226237
ANACONDA_PYTHON_VERSION=3.7
227238
GCC_VERSION=9
228239
PROTOBUF=yes
229240
DB=yes
230241
VISION=yes
231-
ROCM_VERSION=4.3.1
242+
ROCM_VERSION=4.5.2
232243
;;
233-
pytorch-linux-bionic-rocm4.5-py3.7)
244+
pytorch-linux-bionic-rocm5.0-py3.7)
234245
ANACONDA_PYTHON_VERSION=3.7
235246
GCC_VERSION=9
236247
PROTOBUF=yes
237248
DB=yes
238249
VISION=yes
239-
ROCM_VERSION=4.5.2
250+
ROCM_VERSION=5.0
240251
;;
241252
*)
242253
# Catch-all for builds that are not hardcoded.

‎.circleci/docker/common/install_rocm.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ ver() {
3535
}
3636

3737
# Map ROCm version to AMDGPU version
38-
declare -A AMDGPU_VERSIONS=( ["4.5.2"]="21.40.2" )
38+
declare -A AMDGPU_VERSIONS=( ["4.5.2"]="21.40.2" ["5.0"]="21.50" )
3939

4040
install_ubuntu() {
4141
apt-get update

‎.circleci/docker/common/install_user.sh

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,9 @@
33
set -ex
44

55
# Mirror jenkins user in container
6-
echo "jenkins:x:1014:1014::/var/lib/jenkins:" >> /etc/passwd
7-
echo "jenkins:x:1014:" >> /etc/group
6+
# jenkins user as ec2-user should have the same user-id
7+
echo "jenkins:x:1000:1000::/var/lib/jenkins:" >> /etc/passwd
8+
echo "jenkins:x:1000:" >> /etc/group
89

910
# Create $HOME
1011
mkdir -p /var/lib/jenkins

‎.circleci/scripts/cpp_doc_push_script.sh

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,7 @@ cp torch/_utils_internal.py tools/shared
6565

6666
# Generate PyTorch files
6767
time python tools/setup_helpers/generate_code.py \
68-
--native-functions-path aten/src/ATen/native/native_functions.yaml \
69-
--nn-path aten/src/
68+
--native-functions-path aten/src/ATen/native/native_functions.yaml
7069

7170
# Build the docs
7271
pushd docs/cpp

‎.circleci/scripts/windows_cuda_install.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ case ${CUDA_VERSION} in
2222
cuda_installer_name="cuda_11.5.0_496.13_win10"
2323
cuda_install_packages="thrust_11.5 nvcc_11.5 cuobjdump_11.5 nvprune_11.5 nvprof_11.5 cupti_11.5 cublas_11.5 cublas_dev_11.5 cudart_11.5 cufft_11.5 cufft_dev_11.5 curand_11.5 curand_dev_11.5 cusolver_11.5 cusolver_dev_11.5 cusparse_11.5 cusparse_dev_11.5 npp_11.5 npp_dev_11.5 nvrtc_11.5 nvrtc_dev_11.5 nvml_dev_11.5"
2424
;;
25+
11.6)
26+
cuda_installer_name="cuda_11.6.0_511.23_windows"
27+
cuda_install_packages="thrust_11.6 nvcc_11.6 cuobjdump_11.6 nvprune_11.6 nvprof_11.6 cupti_11.6 cublas_11.6 cublas_dev_11.6 cudart_11.6 cufft_11.6 cufft_dev_11.6 curand_11.6 curand_dev_11.6 cusolver_11.6 cusolver_dev_11.6 cusparse_11.6 cusparse_dev_11.6 npp_11.6 npp_dev_11.6 nvrtc_11.6 nvrtc_dev_11.6 nvml_dev_11.6"
28+
;;
2529
*)
2630
echo "CUDA_VERSION $CUDA_VERSION is not supported yet"
2731
exit 1

‎.circleci/scripts/windows_cudnn_install.sh

Lines changed: 4 additions & 0 deletions
< 97AE /div>
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ case ${CUDA_VERSION} in
2222
# Since cudnn 8.3 the filename have changed
2323
cudnn_file_name="cudnn-windows-x86_64-8.3.2.44_cuda${CUDA_VERSION}-archive"
2424
;;
25+
11.6)
26+
# Use cudnn8.3 with hard-coded cuda11.5 version
27+
cudnn_file_name="cudnn-windows-x86_64-8.3.2.44_cuda11.5-archive"
28+
;;
2529
*)
2630
echo "CUDA_VERSION: ${CUDA_VERSION} not supported yet"
2731
exit 1

‎.github/actions/download-build-artifacts/action.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ runs:
1515
steps:
1616
- name: Download PyTorch Build Artifacts from S3
1717
if: ${{ !inputs.use-gha }}
18-
uses: seemethere/download-artifact-s3@0504774707cbc8603d7dca922e8026eb8bf3b47b
18+
uses: seemethere/download-artifact-s3@v3
1919
with:
2020
name: ${{ inputs.name }}
2121

‎.github/actions/get-workflow-job-id/action.yml

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: Get workflow job id
2+
3+
description: Get the ID of the workflow job that is currently running.
4+
5+
inputs:
6+
github-token:
7+
description: GITHUB_TOKEN
8+
required: true
9+
10+
outputs:
11+
job-id:
12+
description: The retrieved workflow job id
13+
value: ${{ steps.get-job-id.outputs.job-id }}
14+
15+
runs:
16+
using: composite
17+
steps:
18+
- uses: nick-fields/retry@71062288b76e2b6214ebde0e673ce0de1755740a
19+
id: get-job-id
20+
env:
21+
GITHUB_TOKEN: ${{ inputs.github-token }}
22+
with:
23+
shell: bash
24+
timeout_minutes: 10
25+
max_attempts: 3
26+
command: |
27+
set -x
28+
python3 -m pip install requests==2.26.0
29+
GHA_WORKFLOW_JOB_ID=$(python3 .github/scripts/get_workflow_job_id.py "${GITHUB_RUN_ID}" "${RUNNER_NAME}")
30+
echo "::set-output name=job-id::${GHA_WORKFLOW_JOB_ID}"

‎.github/actions/setup-win/action.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,11 @@ runs:
3131
run: |
3232
Set-ItemProperty -Path "HKLM:\\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1
3333
34+
- name: Disables Windows Defender scheduled and real-time scanning for files in pytorch directory.
35+
shell: powershell
36+
run: |
37+
Set-MpPreference -ExclusionPath $(Get-Location).tostring()
38+
3439
- name: Install Visual Studio 2019 toolchain
3540
shell: powershell
3641
env:

‎.github/actions/upload-test-artifacts/action.yml

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ runs:
1515
steps:
1616
# Mac/Linux zip
1717
- name: Zip JSONs for upload
18-
if: runner.os != 'Windows'
18+
if: runner.os != 'Windows' && !inputs.use-gha
1919
shell: bash
2020
env:
2121
FILE_SUFFIX: ${{ inputs.file-suffix }}
@@ -25,7 +25,7 @@ runs:
2525
zip -r "test-jsons-${FILE_SUFFIX}.zip" test -i '*.json'
2626
2727
- name: Zip test reports for upload
28-
if: runner.os != 'Windows'
28+
if: runner.os != 'Windows' && !inputs.use-gha
2929
shell: bash
3030
env:
3131
FILE_SUFFIX: ${{ inputs.file-suffix }}
@@ -36,7 +36,7 @@ runs:
3636
3737
# Windows zip
3838
- name: Zip JSONs for upload
39-
if: runner.os == 'Windows'
39+
if: runner.os == 'Windows' && !inputs.use-gha
4040
shell: powershell
4141
env:
4242
FILE_SUFFIX: ${{ inputs.file-suffix }}
@@ -45,7 +45,7 @@ runs:
4545
7z a "test-jsons-$Env:FILE_SUFFIX.zip" -ir'!test\*.json'
4646
4747
- name: Zip test reports for upload
48-
if: runner.os == 'Windows'
48+
if: runner.os == 'Windows' && !inputs.use-gha
4949
shell: powershell
5050
env:
5151
FILE_SUFFIX: ${{ inputs.file-suffix }}
@@ -55,15 +55,15 @@ runs:
5555
5656
# S3 upload
5757
- name: Store Test Downloaded JSONs on S3
58-
uses: seemethere/upload-artifact-s3@v3
58+
uses: seemethere/upload-artifact-s3@v4
5959
if: ${{ !inputs.use-gha }}
6060
with:
6161
retention-days: 14
6262
if-no-files-found: warn
6363
path: test-jsons-*.zip
6464

6565
- name: Store Test Reports on S3
66-
uses: seemethere/upload-artifact-s3@v3
66+
uses: seemethere/upload-artifact-s3@v4
6767
if: ${{ !inputs.use-gha }}
6868
with:
6969
retention-days: 14
@@ -75,15 +75,16 @@ runs:
7575
uses: actions/upload-artifact@v2
7676
if: inputs.use-gha
77< 10000 code>77
with:
78+
name: test-jsons-${{ inputs.file-suffix }}.zip
7879
retention-days: 14
7980
if-no-files-found: warn
80-
path: test-jsons-*.zip
81+
path: test/**/*.json
8182

8283
- name: Store Test Reports on Github
8384
uses: actions/upload-artifact@v2
8485
if: inputs.use-gha
8586
with:
86-
name: test-reports
87+
name: test-reports-${{ inputs.file-suffix }}.zip
8788
retention-days: 14
8889
if-no-files-found: error
89-
path: test-reports-*.zip
90+
path: test/**/*.xml

‎.github/merge_rules.json

Lines changed: 5 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
{
2727
"name": "OSS CI",
2828
"patterns": [".github/**", ".circleci/**", ".jenkins/**", "scripts/**", "tools/**"],
29-
"approved_by": ["janeyx99", "ezyang", "atalman"],
29+
"approved_by": ["ezyang", "pytorch/pytorch-dev-infra"],
3030
"mandatory_checks_name": ["Facebook CLA Check", "Lint"]
3131
},
3232
{
@@ -36,92 +36,15 @@
3636
"mandatory_checks_name": ["Facebook CLA Check", "Lint"]
3737
},
3838
{
39-
"name": "Android",
40-
"patterns": ["android/**", "test/mobile/**"],
41-
"approved_by": ["linbinyu", "kit1980", "IvanKobzarev"],
42-
"mandatory_checks_name": ["Facebook CLA Check", "Lint"]
43-
},
44-
{
45-
"name": "iOS",
46-
"patterns": ["ios/**", "test/mobile/**"],
47-
"approved_by": ["linbinyu", "kit1980", "xta0", "hanton"],
39+
"name": "Mobile",
40+
"patterns": ["ios/**", "android/**", "test/mobile/**"],
41+
"approved_by": ["linbinyu", "kit1980", "IvanKobzarev", "dreiss"],
4842
"mandatory_checks_name": ["Facebook CLA Check", "Lint"]
4943
},
5044
{
5145
"name": "superuser",
5246
"patterns": ["*"],
53-
"approved_by": [
54-
"aaronenyeshi",
55-
"alanwaketan",
56-
"albanD",
57-
"angelayi",
58-
"anijain2305",
59-
"anjali411",
60-
"ansley",
61-
"awgu",
62-
"bertmaher",
63-
"bdhirsh",
64-
"cbalioglu",
65-
"Chillee",
66-
"cpuhrsch",
67-
"dagitses",
68-
"davidberard98",
69-
"desertfire",
70-
"eellison",
71-
"ejguan",
72-
"ezyang",
73-
"fduwjj",
74-
"Gamrix",
75-
"gchanan",
76-
"george-qi",
77-
"gmagogsfm",
78-
"hlu1",
79-
"huiguoo",
80-
"H-Huang",
81-
"jaceyca",
82-
"jamesr66a",
83-
"jansel",
84-
"jbschlosser",
85-
"jiayisuse",
86-
"Krovatkin",
87-
"kumpera",
88-
"kwen2501",
89-
"malfet",
90-
"mikaylagawarecki",
91-
"mikeiovine",
92-
"mingzhe09088",
93-
"mrshenli",
94-
"mruberry",
95-
"navahgar",
96-
"ngimel",
97-
"nikithamalgifb",
98-
"NivekT",
99-
"osalpekar",
100-
"PaliC",
101-
"pbelevich",
102-
"pritamdamania87",
103-
"qihqi",
104-
"robieta",
105-
"rohan-varma",
106-
"saketh-are",
107-
"samdow",
108-
"seemethere",
109-
"SherlockNoMad",
110-
"shunting314",
111-
"soulitzer",
112-
"suo",
113-
"tugsbayasgalan",
114-
"VitalyFedyunin",
115-
"wanchaol",
116-
"wconstab",
117-
"wenleix",
118-
"yaroslavvb",
119-
"zdevito",
120-
"zhaojuanmao",
121-
"zhxchen17",
122-
"ZolotukhinM",
123-
"zou3519"
124-
],
47+
"approved_by": ["pytorch/metamates"],
12548
"mandatory_checks_name": ["Facebook CLA Check", "Lint"]
12649
}
12750
]

‎.github/scale-config.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ runner_types:
3030
linux.2xlar 5DC3 ge:
3131
instance_type: c5.2xlarge
3232
os: linux
33-
max_available: 500
33+
max_available: 750
3434
disk_size: 150
3535
is_ephemeral: false
3636
linux.4xlarge: # for binary-builds

0 commit comments

Comments
 (0)
0