8000 Update on "Fix evaluate_expr to include suppress_guards_tls in cache … · pytorch/pytorch@decb05e · GitHub
[go: up one dir, main page]
Skip to content

Commit decb05e

Browse files
aorensteaorenste
committed
Update on "Fix evaluate_expr to include suppress_guards_tls in cache key"
ShapeEnv.evaluate_expr() behaves differently based on the (tls) global "suppress_guards" - so its cache key needs to include that value. This came up because #152662 triggered it in the test `test/dynamo/test_exc.py::ExcTests::test_trigger_bisect_on_error` - fixing this caused that test to work again. cc ezyang SherlockNoMad EikanWang jgong5 wenzhe-nrv [ghstack-poisoned]
2 parents 21eb212 + 0c39ce2 commit decb05e

File tree

206 files changed

+3389
-2964
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

206 files changed

+3389
-2964
lines changed

‎.ci/docker/build.sh

+7-5
Original file line numberDiff line numberDiff line change
@@ -95,10 +95,12 @@ if [[ "$image" == *rocm* ]]; then
9595
_UCC_COMMIT=0c0fc21559835044ab107199e334f7157d6a0d3d
9696
fi
9797

98+
tag=$(echo $image | awk -F':' '{print $2}')
99+
98100
# It's annoying to rename jobs every time you want to rewrite a
99101
# configuration, so we hardcode everything here rather than do it
100102
# from scratch
101-
case "$image" in
103+
case "$tag" in
102104
pytorch-linux-focal-cuda12.6-cudnn9-py3-gcc11)
103105
CUDA_VERSION=12.6.3
104106
CUDNN_VERSION=9
@@ -241,11 +243,11 @@ case "$image" in
241243
CONDA_CMAKE=yes
242244
TRITON=yes
243245
;;
244-
pytorch-linux-jammy-rocm-n-1-py3)
246+
pytorch-linux-focal-rocm-n-1-py3)
245247
ANACONDA_PYTHON_VERSION=3.10
246248
GCC_VERSION=11
247249
VISION=yes
248-
ROCM_VERSION=6.3
250+
ROCM_VERSION=6.2.4
249251
NINJA_VERSION=1.9.0
250252
CONDA_CMAKE=yes
251253
TRITON=yes
@@ -254,11 +256,11 @@ case "$image" in
254256
UCC_COMMIT=${_UCC_COMMIT}
255257
INDUCTOR_BENCHMARKS=yes
256258
;;
257-
pytorch-linux-jammy-rocm-n-py3)
259+
pytorch-linux-focal-rocm-n-py3)
258260
ANACONDA_PYTHON_VERSION=3.10
259261
GCC_VERSION=11
260262
VISION=yes
261-
ROCM_VERSION=6.4
263+
ROCM_VERSION=6.3
262264
NINJA_VERSION=1.9.0
263265
CONDA_CMAKE=yes
264266
TRITON=yes

‎.ci/docker/common/install_conda.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
8080
# following builds that we know should use conda. Specifically, Ubuntu bionic
8181
# and focal cannot find conda mkl with stock cmake, so we need a cmake from conda
8282
if [ -n "${CONDA_CMAKE}" ]; then
83-
conda_install cmake
83+
pip_install cmake
8484
fi
8585

8686
# Magma package names are concatenation of CUDA major and minor ignoring revision

‎.ci/docker/common/install_halide.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ if [ -n "${UBUNTU_VERSION}" ];then
1717
libopenblas-dev libeigen3-dev libatlas-base-dev libzstd-dev
1818
fi
1919

20-
conda_install numpy scipy imageio cmake ninja
20+
pip_install numpy scipy imageio cmake ninja
2121

2222
git clone --depth 1 --branch release/16.x --recursive https://github.com/llvm/llvm-project.git
2323
cmake -DCMAKE_BUILD_TYPE=Release \

‎.ci/docker/common/install_rocm.sh

+4-12
Original file line numberDiff line numberDiff line change
@@ -66,25 +66,17 @@ EOF
6666
done
6767

6868
# ROCm 6.3 had a regression where initializing static code objects had significant overhead
69-
# ROCm 6.4 did not yet fix the regression, also HIP branch names are different
70-
if [[ $(ver $ROCM_VERSION) -eq $(ver 6.3) ]] || [[ $(ver $ROCM_VERSION) -eq $(ver 6.4) ]]; then
71-
if [[ $(ver $ROCM_VERSION) -eq $(ver 6.3) ]]; then
72-
HIP_BRANCH=rocm-6.3.x
73-
VER_STR=6.3
74-
elif [[ $(ver $ROCM_VERSION) -eq $(ver 6.4) ]]; then
75-
HIP_BRANCH=release/rocm-rel-6.4
76-
VER_STR=6.4
77-
fi
69+
if [[ $(ver $ROCM_VERSION) -eq $(ver 6.3) ]]; then
7870
# clr build needs CppHeaderParser but can only find it using conda's python
7971
/opt/conda/bin/python -m pip install CppHeaderParser
80-
git clone https://github.com/ROCm/HIP -b $HIP_BRANCH
72+
git clone https://github.com/ROCm/HIP -b rocm-6.3.x
8173
HIP_COMMON_DIR=$(readlink -f HIP)
82-
git clone https://github.com/jeffdaily/clr -b release/rocm-rel-${VER_STR}-statco-hotfix
74+
git clone https://github.com/jeffdaily/clr -b release/rocm-rel-6.3-statco-hotfix
8375
mkdir -p clr/build
8476
pushd clr/build
8577
cmake .. -DCLR_BUILD_HIP=ON -DHIP_COMMON_DIR=$HIP_COMMON_DIR
8678
make -j
87-
cp hipamd/lib/libamdhip64.so.${VER_STR}.* /opt/rocm/lib/libamdhip64.so.${VER_STR}.*
79+
cp hipamd/lib/libamdhip64.so.6.3.* /opt/rocm/lib/libamdhip64.so.6.3.*
8880
popd
8981
rm -rf HIP clr
9082
fi

‎.ci/docker/common/install_triton.sh

+10-10
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,8 @@ fi
1010

1111
source "$(dirname "${BASH_SOURCE[0]}")/common_utils.sh"
1212

13-
get_conda_version() {
14-
as_jenkins conda list -n py_$ANACONDA_PYTHON_VERSION | grep -w $* | head -n 1 | awk '{print $2}'
15-
}
16-
17-
conda_reinstall() {
18-
as_jenkins conda install -q -n py_$ANACONDA_PYTHON_VERSION -y --force-reinstall $*
13+
get_pip_version() {
14+
conda_run pip list | grep -w $* | head -n 1 | awk '{print $2}'
1915
}
2016

2117
if [ -n "${XPU_VERSION}" ]; then
@@ -39,8 +35,8 @@ fi
3935

4036
if [ -n "${CONDA_CMAKE}" ]; then
4137
# Keep the current cmake and numpy version here, so we can reinstall them later
42-
CMAKE_VERSION=$(get_conda_version cmake)
43-
NUMPY_VERSION=$(get_conda_version numpy)
38+
CMAKE_VERSION=$(get_pip_version cmake)
39+
NUMPY_VERSION=$(get_pip_version numpy)
4440
fi
4541

4642
if [ -z "${MAX_JOBS}" ]; then
@@ -93,7 +89,11 @@ if [ -n "${CONDA_CMAKE}" ]; then
9389
# causes inconsistent environment. Without this, conda will attempt to install the
9490
# latest numpy version, which fails ASAN tests with the following import error: Numba
9591
# needs NumPy 1.20 or less.
96-
conda_reinstall cmake="${CMAKE_VERSION}"
9792
# Note that we install numpy with pip as conda might not have the version we want
98-
pip_install --force-reinstall numpy=="${NUMPY_VERSION}"
93+
if [ -n "${CMAKE_VERSION}" ]; then
94+
pip_install "cmake==${CMAKE_VERSION}"
95+
fi
96+
if [ -n "${NUMPY_VERSION}" ]; then
97+
pip_install "numpy==${NUMPY_VERSION}"
98+
fi
9999
fi

‎.ci/pytorch/windows/build_pytorch.bat

+14-23
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
@echo off
22

3-
:: This script parses args, installs required libraries (miniconda, MKL,
4-
:: Magma), and then delegates to cpu.bat, cuda80.bat, etc.
3+
:: This script parses args, installs required libraries (MKL, Magma, libuv)
4+
:: and then delegates to cpu.bat, cuda80.bat, etc.
55

66
if not "%CUDA_VERSION%" == "" if not "%PYTORCH_BUILD_VERSION%" == "" if not "%PYTORCH_BUILD_NUMBER%" == "" goto env_end
77
if "%~1"=="" goto arg_error
@@ -36,28 +36,18 @@ set DESIRED_PYTHON_PREFIX=py%DESIRED_PYTHON_PREFIX:;=;py%
3636
set SRC_DIR=%~dp0
3737
pushd %SRC_DIR%
3838

39-
:: Install Miniconda3
40-
set "CONDA_HOME=%CD%\conda"
41-
set "tmp_conda=%CONDA_HOME%"
42-
set "miniconda_exe=%CD%\miniconda.exe"
43-
rmdir /s /q conda
44-
del miniconda.exe
45-
curl --retry 3 -k https://repo.anaconda.com/miniconda/Miniconda3-py311_23.9.0-0-Windows-x86_64.exe -o "%miniconda_exe%"
46-
start /wait "" "%miniconda_exe%" /S /InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /D=%tmp_conda%
47-
if ERRORLEVEL 1 exit /b 1
4839
set "ORIG_PATH=%PATH%"
49-
set "PATH=%CONDA_HOME%;%CONDA_HOME%\scripts;%CONDA_HOME%\Library\bin;%PATH%"
5040

51-
:: create a new conda environment and install packages
41+
:: setup build environment
5242
:try
5343
SET /A tries=3
5444
:loop
5545
IF %tries% LEQ 0 GOTO :exception
56-
call condaenv.bat
46+
call setup_build.bat
5747
IF %ERRORLEVEL% EQU 0 GOTO :done
5848
SET /A "tries=%tries%-1"
5949
:exception
60-
echo "Failed to create conda env"
50+
echo "Failed to setup build environment"
6151
exit /B 1
6252
:done
6353

@@ -73,7 +63,7 @@ if "%DEBUG%" == "1" (
7363
if not "%CUDA_VERSION%" == "cpu" if not "%CUDA_VERSION%" == "xpu" (
7464
rmdir /s /q magma_%CUDA_PREFIX%_%BUILD_TYPE%
7565
del magma_%CUDA_PREFIX%_%BUILD_TYPE%.7z
76-
curl -k https://s3.amazonaws.com/ossci-windows/magma_%MAGMA_VERSION%_%CUDA_PREFIX%_%BUILD_TYPE%.7z -o magma_%CUDA_PREFIX%_%BUILD_TYPE%.7z
66+
curl -k https://s3.amazonaws.com/ossci-windows/magma_%MAGMA_VERSION%_%CUDA_PREFIX%_%BUILD_TYPE%.7z -o magma_%CUDA_PREFIX%_%BUILD_TYPE%.7z %= @lint-ignore =%
7767
7z x -aoa magma_%CUDA_PREFIX%_%BUILD_TYPE%.7z -omagma_%CUDA_PREFIX%_%BUILD_TYPE%
7868
)
7969

@@ -107,19 +97,20 @@ set TH_BINARY_BUILD=1
10797
set INSTALL_TEST=0
10898

10999
for %%v in (%DESIRED_PYTHON_PREFIX%) do (
110-
:: Activate Python Environment
111-
set PYTHON_PREFIX=%%v
112-
set "CONDA_LIB_PATH=%CONDA_HOME%\envs\%%v\Library\bin"
100+
101+
:: Set Environment vars for the build
102+
set "CMAKE_PREFIX_PATH=%CD%\Python\Library\;%PATH%"
103+
set "PYTHON_LIB_PATH=%CD%\Python\Library\bin"
104+
113105
if not "%ADDITIONAL_PATH%" == "" (
 10000
114-
set "PATH=%ADDITIONAL_PATH%;%CONDA_HOME%\envs\%%v;%CONDA_HOME%\envs\%%v\scripts;%CONDA_HOME%\envs\%%v\Library\bin;%ORIG_PATH%"
115-
) else (
116-
set "PATH=%CONDA_HOME%\envs\%%v;%CONDA_HOME%\envs\%%v\scripts;%CONDA_HOME%\envs\%%v\Library\bin;%ORIG_PATH%"
106+
set "PATH=%ADDITIONAL_PATH%;%PATH%"
117107
)
108+
118109
pip install ninja
119110
@setlocal
120111
:: Set Flags
121112
if not "%CUDA_VERSION%"=="cpu" if not "%CUDA_VERSION%" == "xpu" (
122-
set MAGMA_HOME=%cd%\magma_%CUDA_PREFIX%_%BUILD_TYPE%
113+
set "MAGMA_HOME=%cd%\magma_%CUDA_PREFIX%_%BUILD_TYPE%"
123114
)
124115
echo "Calling arch build script"
125116
call %CUDA_PREFIX%.bat

‎.ci/pytorch/windows/condaenv.bat

-27
This file was deleted.

‎.ci/pytorch/windows/internal/copy.bat

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ copy "%CUDA_PATH%\bin\nvrtc*64_*.dll*" pytorch\torch\lib
1010
copy "%CUDA_PATH%\extras\CUPTI\lib64\cupti64_*.dll*" pytorch\torch\lib
1111

1212
copy "C:\Program Files\NVIDIA Corporation\NvToolsExt\bin\x64\nvToolsExt64_1.dll*" pytorch\torch\lib
13-
copy "%CONDA_LIB_PATH%\libiomp*5md.dll" pytorch\torch\lib
13+
copy "%PYTHON_LIB_PATH%\libiomp*5md.dll" pytorch\torch\lib
1414

1515
:: Should be set in build_pytorch.bat
1616
copy "%libuv_ROOT%\bin\uv.dll" pytorch\torch\lib

‎.ci/pytorch/windows/internal/copy_cpu.bat

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
copy "%CONDA_LIB_PATH%\libiomp*5md.dll" pytorch\torch\lib
1+
copy "%PYTHON_LIB_PATH%\libiomp*5md.dll" pytorch\torch\lib
22
:: Should be set in build_pytorch.bat
3-
copy "%libuv_ROOT%\bin\uv.dll" pytorch\torch\lib
3+
copy "%libuv_ROOT%\bin\uv.dll" pytorch\torch\lib

‎.ci/pytorch/windows/internal/env_fix.bat

-38
This file was deleted.

‎.ci/pytorch/windows/internal/install_python.bat

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
set ADDITIONAL_OPTIONS=""
2+
set PYTHON_EXEC="python"
3+
if "%DESIRED_PYTHON%" == "3.13t" (
4+
echo Python version is set to 3.13t
5+
set "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.13.0/python-3.13.0-amd64.exe"
6+
set ADDITIONAL_OPTIONS="Include_freethreaded=1"
7+
set PYTHON_EXEC="python3.13t"
8+
) else (
9+
echo DESIRED_PYTHON not defined, Python version is set to %DESIRED_PYTHON%
10+
set "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/%DESIRED_PYTHON%.0/python-%DESIRED_PYTHON%.0-amd64.exe" %= @lint-ignore =%
11+
)
12+
13+
del python-amd64.exe
14+
curl --retry 3 -kL "%PYTHON_INSTALLER_URL%" --output python-amd64.exe
15+
if errorlevel 1 exit /b 1
16+
17+
start /wait "" python-amd64.exe /quiet InstallAllUsers=1 PrependPath=0 Include_test=0 %ADDITIONAL_OPTIONS% TargetDir=%CD%\Python
18+
if errorlevel 1 exit /b 1
19+
20+
set "PATH=%CD%\Python\Scripts;%CD%\Python;%PATH%"

‎.ci/pytorch/windows/internal/setup.bat

+2-2
Original file line numberDiff line numberDiff line change
@@ -51,7 +51 10000 ,7 @@ mkdir libtorch\test
5151

5252
mkdir build
5353
pushd build
54-
python ../tools/build_libtorch.py
54+
%PYTHON_EXEC% ../tools/build_libtorch.py
5555
popd
5656

5757
IF ERRORLEVEL 1 exit /b 1
@@ -86,7 +86,7 @@ copy /Y "%LIBTORCH_PREFIX%-%PYTORCH_BUILD_VERSION%.zip" "%PYTORCH_FINAL_PACKAGE_
8686
goto build_end
8787

8888
:pytorch
89-
python setup.py bdist_wheel -d "%PYTORCH_FINAL_PACKAGE_DIR%"
89+
%PYTHON_EXEC% setup.py bdist_wheel -d "%PYTORCH_FINAL_PACKAGE_DIR%"
9090

9191
:build_end
9292
IF ERRORLEVEL 1 exit /b 1

‎.ci/pytorch/windows/internal/smoke_test.bat

+2-30
Original file line numberDiff line numberDiff line change
@@ -35,36 +35,8 @@ exit /b 1
3535
:wheel
3636
echo "install wheel package"
3737

38-
set PYTHON_INSTALLER_URL=
39-
if "%DESIRED_PYTHON%" == "3.13t" set "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.13.0/python-3.13.0-amd64.exe"
40-
if "%DESIRED_PYTHON%" == "3.13" set "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.13.0/python-3.13.0-amd64.exe"
41-
if "%DESIRED_PYTHON%" == "3.12" set "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.12.0/python-3.12.0-amd64.exe"
42-
if "%DESIRED_PYTHON%" == "3.11" set "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.11.0/python-3.11.0-amd64.exe"
43-
if "%DESIRED_PYTHON%" == "3.10" set "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.10.0/python-3.10.0-amd64.exe"
44-
if "%DESIRED_PYTHON%" == "3.9" set "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.9.0/python-3.9.0-amd64.exe"
45-
if "%PYTHON_INSTALLER_URL%" == "" (
46-
echo Python %DESIRED_PYTHON% not supported yet
47-
)
48-
49-
set ADDITIONAL_OPTIONS=""
50-
set PYTHON_EXEC="python"
51-
if "%DESIRED_PYTHON%" == "3.13t" (
52-
set ADDITIONAL_OPTIONS="Include_freethreaded=1"
53-
set PYTHON_EXEC="python3.13t"
54-
)
55-
56-
del python-amd64.exe
57-
curl --retry 3 -kL "%PYTHON_INSTALLER_URL%" --output python-amd64.exe
58-
if errorlevel 1 exit /b 1
59-
60-
:: According to https://docs.python.org/3/using/windows.html, setting PrependPath to 1 will prepend
61-
:: the installed Python to PATH system-wide. Even calling set PATH=%ORIG_PATH% later on won't make
62-
:: a change. As the builder directory will be removed after the smoke test, all subsequent non-binary
63-
:: jobs will fail to find any Python executable there
64-
start /wait "" python-amd64.exe /quiet InstallAllUsers=1 PrependPath=0 Include_test=0 %ADDITIONAL_OPTIONS% TargetDir=%CD%\Python
65-
if errorlevel 1 exit /b 1
38+
call "internal\install_python.bat"
6639

67-
set "PATH=%CD%\Python%PYTHON_VERSION%\Scripts;%CD%\Python;%PATH%"
6840
if "%DESIRED_PYTHON%" == "3.13t" %PYTHON_EXEC% -m pip install --pre numpy==2.2.1 protobuf
6941
if "%DESIRED_PYTHON%" == "3.13" %PYTHON_EXEC% -m pip install --pre numpy==2.1.2 protobuf
7042
if "%DESIRED_PYTHON%" == "3.12" %PYTHON_EXEC% -m pip install --pre numpy==2.0.2 protobuf
@@ -81,7 +53,7 @@ if "%PYTORCH_BUILD_VERSION:dev=%" NEQ "%PYTORCH_BUILD_VERSION%" (
8153
)
8254

8355
set "EXTRA_INDEX= "
84-
if "%CUDA_VERSION%" == "xpu" set "EXTRA_INDEX=--index-url https://download.pytorch.org/whl/%CHANNEL%/xpu"
56+
if "%CUDA_VERSION%" == "xpu" set "EXTRA_INDEX=--index-url https://download.pytorch.org/whl/%CHANNEL%/xpu" %= @lint-ignore =%
8557

8658
for /F "delims=" %%i in ('where /R "%PYTORCH_FINAL_PACKAGE_DIR:/=\%" *.whl') do %PYTHON_EXEC% -m pip install "%%i" %EXTRA_INDEX%
8759
if errorlevel 1 exit /b 1

0 commit comments

Comments
 (0)
0