8000 [BE] - Remove conda test and upload scripts and env variables from Workflows Part 1 by atalman · Pull Request #144870 · pytorch/pytorch · GitHub
[go: up one dir, main page]

Skip to content

[BE] - Remove conda test and upload scripts and env variables from Workflows Part 1 #144870

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


8000
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .ci/pytorch/check_binary.sh
Original file line number Diff line number Diff line change
Expand Up @@ -387,7 +387,7 @@ fi
###############################################################################
# Check for C++ ABI compatibility between gcc7 and gcc9 compiled binaries
###############################################################################
if [[ "$(uname)" == 'Linux' && ("$PACKAGE_TYPE" == 'conda' || "$PACKAGE_TYPE" == 'manywheel')]]; then
if [[ "$(uname)" == 'Linux' && "$PACKAGE_TYPE" == 'manywheel' ]]; then
pushd /tmp
python -c "import torch; exit(0 if torch.compiled_with_cxx11_abi() else (0 if torch._C._PYBIND11_BUILD_ABI == '_cxxabi1011' else 1))"
popd
Expand Down
2 changes: 1 addition & 1 deletion .ci/pytorch/run_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ retry () {
if [[ "$#" != 3 ]]; then
if [[ -z "${DESIRED_PYTHON:-}" || -z "${DESIRED_CUDA:-}" || -z "${PACKAGE_TYPE:-}" ]]; then
echo "USAGE: run_tests.sh PACKAGE_TYPE DESIRED_PYTHON DESIRED_CUDA"
echo "The env variable PACKAGE_TYPE must be set to 'conda' or 'manywheel' or 'libtorch'"
echo "The env variable PACKAGE_TYPE must be set to 'manywheel' or 'libtorch'"
echo "The env variable DESIRED_PYTHON must be set like '2.7mu' or '3.6m' etc"
echo "The env variable DESIRED_CUDA must be set like 'cpu' or 'cu80' etc"
exit 1
Expand Down
55 changes: 0 additions & 55 deletions .ci/pytorch/windows/internal/smoke_test.bat
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ for /F "delims=" %%i in ('wmic path win32_VideoController get name') do (
endlocal & set NVIDIA_GPU_EXISTS=%NVIDIA_GPU_EXISTS%

if "%PACKAGE_TYPE%" == "wheel" goto wheel
if "%PACKAGE_TYPE%" == "conda" goto conda
if "%PACKAGE_TYPE%" == "libtorch" goto libtorch

echo "unknown package type"
Expand Down Expand Up @@ -74,60 +73,6 @@ if errorlevel 1 exit /b 1

goto smoke_test

:conda
echo "install conda package"

:: Install Miniconda3
set "CONDA_HOME=%CD%\conda"
set "tmp_conda=%CONDA_HOME%"
set "miniconda_exe=%CD%\miniconda.exe"
set "CONDA_EXTRA_ARGS=cpuonly -c pytorch-nightly"
if "%CUDA_VERSION%" == "118" (
set "CONDA_EXTRA_ARGS=pytorch-cuda=11.8 -c nvidia -c pytorch-nightly"
)
if "%CUDA_VERSION%" == "121" (
set "CONDA_EXTRA_ARGS=pytorch-cuda=12.1 -c nvidia -c pytorch-nightly"
)
if "%CUDA_VERSION%" == "124" (
set "CONDA_EXTRA_ARGS=pytorch-cuda=12.4 -c nvidia -c pytorch-nightly"
)
if "%CUDA_VERSION%" == "126" (
set "CONDA_EXTRA_ARGS=pytorch-cuda=12.6 -c nvidia -c pytorch-nightly"
)

rmdir /s /q conda
del miniconda.exe
curl -k https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -o "%miniconda_exe%"
start /wait "" "%miniconda_exe%" /S /InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /D=%tmp_conda%
if ERRORLEVEL 1 exit /b 1

set "PATH=%CONDA_HOME%;%CONDA_HOME%\scripts;%CONDA_HOME%\Library\bin;%PATH%"

conda create -qyn testenv python=%DESIRED_PYTHON%
if errorlevel 1 exit /b 1
call conda install -yq conda-build
if errorlevel 1 exit /b 1
call %CONDA_HOME%\condabin\activate.bat testenv
if errorlevel 1 exit /b 1
set "NO_ARCH_PATH=%PYTORCH_FINAL_PACKAGE_DIR:/=\%\noarch"
mkdir %NO_ARCH_PATH%
for /F "delims=" %%i in ('where /R "%PYTORCH_FINAL_PACKAGE_DIR:/=\%" *') do xcopy "%%i" %NO_ARCH_PATH% /Y
if ERRORLEVEL 1 exit /b 1
call conda index %PYTORCH_FINAL_PACKAGE_DIR%
if errorlevel 1 exit /b 1
call conda install -yq -c "file:///%PYTORCH_FINAL_PACKAGE_DIR%" pytorch==%PYTORCH_BUILD_VERSION% -c pytorch -c numba/label/dev -c nvidia
if ERRORLEVEL 1 exit /b 1
call conda install -yq numpy
if ERRORLEVEL 1 exit /b 1

set /a CUDA_VER=%CUDA_VERSION%
set CUDA_VER_MAJOR=%CUDA_VERSION:~0,-1%
set CUDA_VER_MINOR=%CUDA_VERSION:~-1,1%
set CUDA_VERSION_STR=%CUDA_VER_MAJOR%.%CUDA_VER_MINOR%

:: Install package we just build


:smoke_test
python -c "import torch"
if ERRORLEVEL 1 exit /b 1
Expand Down
8 changes: 2 additions & 6 deletions .circleci/scripts/binary_populate_env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,7 @@ fi
# Pick docker image
export DOCKER_IMAGE=${DOCKER_IMAGE:-}
if [[ -z "$DOCKER_IMAGE" ]]; then
if [[ "$PACKAGE_TYPE" == conda ]]; then
export DOCKER_IMAGE="pytorch/conda-cuda"
elif [[ "$DESIRED_CUDA" == cpu ]]; then
if [[ "$DESIRED_CUDA" == cpu ]]; then
export DOCKER_IMAGE="pytorch/manylinux:cpu"
else
export DOCKER_IMAGE="pytorch/manylinux-builder:${DESIRED_CUDA:2}"
Expand Down Expand Up @@ -63,7 +61,7 @@ if tagged_version >/dev/null; then
# Turns tag v1.6.0-rc1 -> v1.6.0
BASE_BUILD_VERSION="$(tagged_version | sed -e 's/^v//' -e 's/-.*$//')"
fi
if [[ "$(uname)" == 'Darwin' ]] || [[ "$PACKAGE_TYPE" == conda ]]; then
if [[ "$(uname)" == 'Darwin' ]]; then
export PYTORCH_BUILD_VERSION="${BASE_BUILD_VERSION}"
else
export PYTORCH_BUILD_VERSION="${BASE_BUILD_VERSION}+$DESIRED_CUDA"
Expand Down Expand Up @@ -149,8 +147,6 @@ export PYTORCH_EXTRA_INSTALL_REQUIREMENTS="${PYTORCH_EXTRA_INSTALL_REQUIREMENTS:

# TODO: We don't need this anymore IIUC
export TORCH_PACKAGE_NAME='torch'
export TORCH_CONDA_BUILD_FOLDER='pytorch-nightly'
export ANACONDA_USER='pytorch'

export USE_FBGEMM=1
export PIP_UPLOAD_FOLDER="$PIP_UPLOAD_FOLDER"
Expand Down
39 changes: 2 additions & 37 deletions .circleci/scripts/binary_upload.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

set -euo pipefail

PACKAGE_TYPE=${PACKAGE_TYPE:-conda}
PACKAGE_TYPE=${PACKAGE_TYPE:-wheel}

PKG_DIR=${PKG_DIR:-/tmp/workspace/final_pkgs}

Expand All @@ -18,10 +18,8 @@ BUILD_NAME=${BUILD_NAME:-}

DRY_RUN=${DRY_RUN:-enabled}
# Don't actually do work unless explicit
ANACONDA="true anaconda"
AWS_S3_CP="aws s3 cp --dryrun"
if [[ "${DRY_RUN}" = "disabled" ]]; then
ANACONDA="anaconda"
AWS_S3_CP="aws s3 cp"
fi

Expand All @@ -34,10 +32,6 @@ if [[ ${BUILD_NAME} == *-full* ]]; then
UPLOAD_SUBFOLDER="${UPLOAD_SUBFOLDER}_full"
fi

# Sleep 2 minutes between retries for conda upload
retry () {
"$@" || (sleep 5m && "$@") || (sleep 5m && "$@") || (sleep 5m && "$@") || (sleep 5m && "$@")
}

do_backup() {
local backup_dir
Expand All @@ -49,20 +43,6 @@ do_backup() {
)
}

conda_upload() {
(
set -x
retry \
${ANACONDA} \
upload \
${PKG_DIR}/*.tar.bz2 \
-u "pytorch-${UPLOAD_CHANNEL}" \
--label main \
--no-progress \
--force
)
}

s3_upload() {
local extension
local pkg_type
Expand All @@ -85,24 +65,9 @@ s3_upload() {
}

# Install dependencies (should be a no-op if previously installed)
conda install -yq anaconda-client
pip install -q awscli
pip install -q awscli uv

case "${PACKAGE_TYPE}" in
conda)
conda_upload
for conda_archive in ${PKG_DIR}/*.tar.bz2; do
# Fetch platform (eg. win-64, linux-64, etc.) from index file because
# there's no actual conda command to read this
subdir=$(\
tar -xOf "${conda_archive}" info/index.json \
| grep subdir \
| cut -d ':' -f2 \
| sed -e 's/[[:space:]]//' -e 's/"//g' -e 's/,//' \
)
BACKUP_DIR="conda/${subdir}"
done
;;
libtorch)
s3_upload "zip" "libtorch"
BACKUP_DIR="libtorch/${UPLOAD_CHANNEL}/${UPLOAD_SUBFOLDER}"
Expand Down
1 change: 0 additions & 1 deletion .github/templates/linux_binary_build_workflow.yml.j2
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ env:
{%- else %}
ALPINE_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/alpine"
{%- endif %}
ANACONDA_USER: pytorch
AWS_DEFAULT_REGION: us-east-1
BINARY_ENV_FILE: /tmp/env
BUILD_ENVIRONMENT: !{{ build_environment }}
Expand Down
2 changes: 0 additions & 2 deletions .github/templates/macos_binary_build_workflow.yml.j2
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,7 @@ on:
workflow_dispatch:

env:
# Needed for conda builds
ALPINE_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/alpine"
ANACONDA_USER: pytorch
AWS_DEFAULT_REGION: us-east-1
BUILD_ENVIRONMENT: !{{ build_environment }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
Expand Down
2 changes: 0 additions & 2 deletions .github/templates/upload.yml.j2
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,5 @@
{%- endif %}
secrets:
github-token: ${{ secrets.GITHUB_TOKEN }}
conda-pytorchbot-token: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }}
conda-pytorchbot-token-test: ${{ secrets.CONDA_PYTORCHBOT_TOKEN_TEST }}
uses: ./.github/workflows/_binary-upload.yml
{%- endmacro %}
1 change: 0 additions & 1 deletion .github/templates/windows_binary_build_workflow.yml.j2
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ on:
env:
# Needed for conda builds
ALPINE_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/alpine"
ANACONDA_USER: pytorch
AWS_DEFAULT_REGION: us-east-1
BUILD_ENVIRONMENT: !{{ build_environment }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
Expand Down
3 changes: 0 additions & 3 deletions .github/workflows/_binary-build-linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -107,9 +107,7 @@ jobs:
DESIRED_DEVTOOLSET: ${{ inputs.DESIRED_DEVTOOLSET }}
DESIRED_PYTHON: ${{ inputs.DESIRED_PYTHON }}
PYTORCH_EXTRA_INSTALL_REQUIREMENTS: ${{ inputs.PYTORCH_EXTRA_INSTALL_REQUIREMENTS }}
# Needed for conda builds
ALPINE_IMAGE: ${{ inputs.ALPINE_IMAGE }}
ANACONDA_USER: pytorch
AWS_DEFAULT_REGION: us-east-1
BINARY_ENV_FILE: /tmp/env
BUILD_ENVIRONMENT: ${{ inputs.build_environment }}
Expand All @@ -136,7 +134,6 @@ jobs:
echo "DESIRED_PYTHON=${{ env.DESIRED_PYTHON }}"
echo "PYTORCH_EXTRA_INSTALL_REQUIREMENTS=${{ env.PYTORCH_EXTRA_INSTALL_REQUIREMENTS }}"
echo "ALPINE_IMAGE=${{ env.ALPINE_IMAGE }}"
echo "ANACONDA_USER=${{ env.ANACONDA_USER }}"
echo "AWS_DEFAULT_REGION=${{ env.AWS_DEFAULT_REGION }}"
echo "BINARY_ENV_FILE=${{ env.BINARY_ENV_FILE }}"
echo "BUILD_ENVIRONMENT=${{ env.BUILD_ENVIRONMENT }}"
Expand Down
3 changes: 0 additions & 3 deletions .github/workflows/_binary-test-linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -94,9 +94,7 @@ jobs:
LIBTORCH_VARIANT: ${{ inputs.LIBTORCH_VARIANT }}
DESIRED_DEVTOOLSET: ${{ inputs.DESIRED_DEVTOOLSET }}
DESIRED_PYTHON: ${{ inputs.DESIRED_PYTHON }}
# Needed for conda builds
ALPINE_IMAGE: ${{ inputs.ALPINE_IMAGE }}
ANACONDA_USER: pytorch
AWS_DEFAULT_REGION: us-east-1
BINARY_ENV_FILE: /tmp/env
BUILD_ENVIRONMENT: ${{ inputs.build_environment }}
Expand Down Expand Up @@ -124,7 +122,6 @@ jobs:
echo "DESIRED_PYTHON=${{ env.DESIRED_PYTHON }}"

echo "ALPINE_IMAGE=${{ env.ALPINE_IMAGE }}"
echo "ANACONDA_USER=${{ env.ANACONDA_USER }}"
echo "AWS_DEFAULT_REGION=${{ env.AWS_DEFAULT_REGION }}"
echo "BINARY_ENV_FILE=${{ env.BINARY_ENV_FILE }}"
echo "BUILD_ENVIRONMENT=${{ env.BUILD_ENVIRONMENT }}"
Expand Down
15 changes: 0 additions & 15 deletions .github/workflows/_binary-upload.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,12 +62,6 @@ on:
github-token:
required: true
description: Github Token
conda-pytorchbot-token:
required: true
description: Conda PyTorchBot token
conda-pytorchbot-token-test:
required: true
description: Conda PyTorchBot token

jobs:
upload:
Expand All @@ -89,7 +83,6 @@ jobs:
LIBTORCH_VARIANT: ${{ inputs.LIBTORCH_VARIANT }}
DESIRED_DEVTOOLSET: ${{ inputs.DESIRED_DEVTOOLSET }}
DESIRED_PYTHON: ${{ inputs.DESIRED_PYTHON }}
ANACONDA_USER: pytorch
BINARY_ENV_FILE: /tmp/env
GITHUB_TOKEN: ${{ secrets.github-token }}
PR_NUMBER: ${{ github.event.pull_request.number }}
Expand Down Expand Up @@ -146,15 +139,7 @@ jobs:
env:
PKG_DIR: "${{ runner.temp }}/artifacts"
UPLOAD_SUBFOLDER: "${{ env.DESIRED_CUDA }}"
# When running these on pull_request events these should be blank
CONDA_PYTORCHBOT_TOKEN: ${{ secrets.conda-pytorchbot-token }}
CONDA_PYTORCHBOT_TOKEN_TEST: ${{ secrets.conda-pytorchbot-token-test }}
BUILD_NAME: ${{ inputs.build_name }}
run: |
set -ex
if [[ "${GITHUB_REF_NAME}" = *-rc[0-9]* ]]; then
export ANACONDA_API_TOKEN="${CONDA_PYTORCHBOT_TOKEN_TEST}"
else
export ANACONDA_API_TOKEN="${CONDA_PYTORCHBOT_TOKEN}"
fi
bash .circleci/scripts/binary_upload.sh
Loading
Loading
0