8000 Update on "move aten_core_headers to common build structure" · pytorch/pytorch@2a79c6d · GitHub
[go: up one dir, main page]
Skip to content

Commit 2a79c6d

Browse files
author
Michael Andreas Dagitses
committed
Update on "move aten_core_headers to common build structure"
Differential Revision: [D36597720](https://our.internmc.facebook.com/intern/diff/D36597720/) **NOTE FOR REVIEWERS**: This PR has internal Facebook specific changes or comments, please review them on [Phabricator](https://our.internmc.facebook.com/intern/diff/D36597720/)! [ghstack-poisoned]
2 parents d283a01 + 9988a6b commit 2a79c6d

File tree

274 files changed

+34781
-3456
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

274 files changed

+34781
-3456
lines changed

‎.circleci/docker/build_docker.sh

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ tag="${DOCKER_TAG}"
1818

1919
registry="308535385114.dkr.ecr.us-east-1.amazonaws.com"
2020
image="${registry}/pytorch/${IMAGE_NAME}"
21+
ghcr_image="ghcr.io/pytorch/ci-image"
2122

2223
login() {
2324
aws ecr get-authorization-token --region us-east-1 --output text --query 'authorizationData[].authorizationToken' |
@@ -54,6 +55,13 @@ if [ "${DOCKER_SKIP_PUSH:-true}" = "false" ]; then
5455
if ! docker manifest inspect "${image}:${tag}" >/dev/null 2>/dev/null; then
5556
docker push "${image}:${tag}"
5657
fi
58+
59+
if [ "${PUSH_GHCR_IMAGE:-}" = "true" ]; then
60+
# Push docker image to the ghcr.io
61+
echo $GHCR_PAT | docker login ghcr.io -u pytorch --password-stdin
62+
docker tag "${image}:${tag}" "${ghcr_image}:${IMAGE_NAME}-${tag}"
63+
docker push "${ghcr_image}:${IMAGE_NAME}-${tag}"
64+
fi
5765
fi
5866

5967
if [ -z "${DOCKER_SKIP_S3_UPLOAD:-}" ]; then

‎.circleci/docker/common/install_base.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,8 @@ install_ubuntu() {
6666
software-properties-common \
6767
wget \
6868
sudo \
69-
vim
69+
vim \
70+
jq
7071

7172
# Should resolve issues related to various apt package repository cert issues
7273
# see: https://github.com/pytorch/pytorch/issues/65931

‎.github/actions/calculate-docker-image/action.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,9 @@ inputs:
2323
force_push:
2424
description: If set to any value, always run the push
2525
required: false
26+
push-ghcr-image:
27+
description: If set to any value, push docker image to the ghcr.io.
28+
required: false
2629

2730
outputs:
2831
docker-image:
@@ -102,6 +105,8 @@ runs:
102105
# Skip push if we don't need it, or if specified in the inputs
103106
DOCKER_SKIP_PUSH: ${{ steps.check.outputs.skip_push || inputs.skip_push }}
104107
DOCKER_TAG: ${{ steps.calculate-tag.outputs.docker-tag }}
108+
PUSH_GHCR_IMAGE: ${{ inputs.push-ghcr-image }}
109+
GHCR_PAT: ${{ env.GHCR_PAT }}
105110
working-directory: .circleci/docker
106111
shell: bash
107112
run: |

‎.github/ci_commit_pins/torchdynamo.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
ad21e440dc9e5cdd77785fe1ea9979a53262114e
1+
5f2f374d9bbc6374fe725a17182e7d4c270c6833

‎.github/scripts/trymerge.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -990,7 +990,7 @@ def fetch_check_run_conclusions(repo: GitRepo, commit: str) -> Dict[str, Tuple[s
990990
[owner, name] = repo.gh_owner_and_name()
991991
checks = fetch_json_dict(f'https://api.github.com/repos/{owner}/{name}/commits/{commit}/check-runs')
992992
check_run_conclusions = {}
993-
if len(checks) == 0:
993+
if len(checks['check_runs']) == 0:
994994
raise MandatoryChecksMissingError("Refusing to merge as land check(s) are not yet run")
995995
for check_run in checks['check_runs']:
996996
check_run_conclusions[check_run['name']] = (check_run['conclusion'],

‎.github/workflows/_android-build-test.yml

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,6 @@ jobs:
7373
# 1) Not shareable: it's custom selective build, which is different from default libtorch mobile build;
7474
# 2) Not parallelizable by architecture: it only builds libtorch for one architecture;
7575
76-
echo "DOCKER_IMAGE: ${DOCKER_IMAGE}"
77-
time docker pull "${DOCKER_IMAGE}" >/dev/null
78-
7976
export BUILD_LITE_INTERPRETER
8077
BUILD_LITE_INTERPRETER="1"
8178
if [[ "${BUILD_ENVIRONMENT}" == *"full-jit" ]]; then

‎.github/workflows/docker-builds.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,11 +62,14 @@ jobs:
6262
- name: Build docker image
6363
id: build-docker-image
6464
uses: ./.github/actions/calculate-docker-image
65+
env:
66+
GHCR_PAT: ${{ secrets.GHCR_PAT }}
6567
with:
6668
docker-image-name: ${{ matrix.docker-image-name }}
6769
always-rebuild: true
6870
skip_push: false
6971
force_push: true
72+
push-ghcr-image: ${{ github.event_name == 'push' }}
7073

7174
- name: Pull docker image
7275
uses: ./.github/actions/pull-docker-image

‎.github/workflows/trunk.yml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,6 @@ jobs:
223223
]}
224224
225225
linux-bionic-rocm5_1-py3_7-build:
226-
if: false
227226
name: linux-bionic-rocm5.1-py3.7
228227
uses: ./.github/workflows/_linux-build.yml
229228
with:

‎.jenkins/pytorch/common_utils.sh

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -144,8 +144,7 @@ function print_sccache_stats() {
144144
sccache --show-stats
145145

146146
if [[ -n "${OUR_GITHUB_JOB_ID}" ]]; then
147-
sccache --show-stats \
148-
| python -m tools.stats.sccache_stats_to_json \
147+
sccache --show-stats --stats-format json | jq .stats \
149148
> "sccache-stats-${BUILD_ENVIRONMENT}-${OUR_GITHUB_JOB_ID}.json"
150149
else
151150
echo "env var OUR_GITHUB_JOB_ID not set, will not write sccache stats to json"

‎.jenkins/pytorch/test.sh

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,33 @@ test_python() {
178178
assert_git_not_dirty
179179
}
180180

181+
182+
test_dynamo_shard() {
183+
if [[ -z "$NUM_TEST_SHARDS" ]]; then
184+
echo "NUM_TEST_SHARDS must be defined to run a Python test shard"
185+
exit 1
186+
fi
187+
time python test/run_test.py \
188+
--exclude-jit-executor \
189+
--exclude-distributed-tests \
190+
--exclude \
191+
test_autograd \
192+
test_proxy_tensor \
193+
test_quantization \
194+
test_public_bindings \
195+
test_dataloader \
196+
test_reductions \
197+
test_namedtensor \
198+
test_namedtuple_return_api \
199+
test_profiler \
200+
test_profiler_tree \
201+
test_overrides \
202+
test_python_dispatch \
203+
--shard "$1" "$NUM_TEST_SHARDS" \
204+
--verbose
205+
assert_git_not_dirty
206+
}
207+
181208
test_python_gloo_with_tls() {
182209
source "$(dirname "${BASH_SOURCE[0]}")/run_glootls_test.sh"
183210
assert_git_not_dirty
@@ -599,6 +626,17 @@ elif [[ "$TEST_CONFIG" == distributed ]]; then
599626
if [[ "${SHARD_NUMBER}" == 1 ]]; then
600627
test_rpc
601628
fi
629+
elif [[ "${TEST_CONFIG}" == *dynamo* && "${SHARD_NUMBER}" == 1 && $NUM_TEST_SHARDS -gt 1 ]]; then
630+
test_without_numpy
631+
install_torchvision
632+
install_torchdynamo
633+
test_dynamo_shard 1
634+
test_aten
635+
elif [[ "${TEST_CONFIG}" == *dynamo* && "${SHARD_NUMBER}" == 2 && $NUM_TEST_SHARDS -gt 1 ]]; then
636+
install_torchvision
637+
checkout_install_torchdynamo
638+
test_dynamo_shard 2
639+
test_dynamo
602640
elif [[ "${SHARD_NUMBER}" == 1 && $NUM_TEST_SHARDS -gt 1 ]]; then
603641
test_without_numpy
604642
install_torchvision
@@ -614,7 +652,6 @@ elif [[ "${SHARD_NUMBER}" == 2 && $NUM_TEST_SHARDS -gt 1 ]]; then
614652
test_custom_script_ops
615653
test_custom_backend
616654
test_torch_function_benchmark
617-
test_dynamo
618655
elif [[ "${SHARD_NUMBER}" -gt 2 ]]; then
619656
# Handle arbitrary number of shards
620657
install_torchdynamo

0 commit comments

Comments
 (0)
0