From 6895e823d3d2a3226f541e8c03af8f2fe4cf44a9 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 8 Apr 2025 16:20:51 +0100 Subject: [PATCH 1/4] Add new ssh key argument and don't check magic number in old versions --- release.py | 9 +++++++++ run_release.py | 33 ++++++++++++++++++++++++--------- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/release.py b/release.py index 24498c61..90412c62 100755 --- a/release.py +++ b/release.py @@ -60,6 +60,9 @@ def get(self, key: Literal["auth_info"], default: str | None = None) -> str: ... @overload def get(self, key: Literal["ssh_user"], default: str | None = None) -> str: ... + @overload + def get(self, key: Literal["ssh_key"], default: str | None = None) -> str | None: ... + @overload def get(self, key: Literal["sign_gpg"], default: bool | None = None) -> bool: ... @@ -84,6 +87,9 @@ def __getitem__(self, key: Literal["auth_info"]) -> str: ... @overload def __getitem__(self, key: Literal["ssh_user"]) -> str: ... + @overload + def __getitem__(self, key: Literal["ssh_key"]) -> str | None: ... + @overload def __getitem__(self, key: Literal["sign_gpg"]) -> bool: ... @@ -110,6 +116,9 @@ def __setitem__(self, key: Literal["auth_info"], value: str) -> None: ... @overload def __setitem__(self, key: Literal["ssh_user"], value: str) -> None: ... + @overload + def __setitem__(self, key: Literal["ssh_key"], value: str | None) -> None: ... + @overload def __setitem__(self, key: Literal["sign_gpg"], value: bool) -> None: ... diff --git a/run_release.py b/run_release.py index 8f2f6738..1581276d 100755 --- a/run_release.py +++ b/run_release.py @@ -193,6 +193,7 @@ def __init__( api_key: str, ssh_user: str, sign_gpg: bool, + ssh_key: str | None = None, first_state: Task | None = None, ) -> None: self.tasks = tasks @@ -215,6 +216,8 @@ def __init__( self.db["auth_info"] = api_key if not self.db.get("ssh_user"): self.db["ssh_user"] = ssh_user + if not self.db.get("ssh_key"): + self.db["ssh_key"] = ssh_key if not self.db.get("sign_gpg"): self.db["sign_gpg"] = sign_gpg @@ -227,6 +230,7 @@ def __init__( print(f"- Normalized release tag: {release_tag.normalized()}") print(f"- Git repo: {self.db['git_repo']}") print(f"- SSH username: {self.db['ssh_user']}") + print(f"- SSH key: {self.db['ssh_key'] or 'Default'}") print(f"- python.org API key: {self.db['auth_info']}") print(f"- Sign with GPG: {self.db['sign_gpg']}") print() @@ -313,9 +317,9 @@ def check_ssh_connection(db: ReleaseShelf) -> None: client = paramiko.SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.WarningPolicy) - client.connect(DOWNLOADS_SERVER, port=22, username=db["ssh_user"]) + client.connect(DOWNLOADS_SERVER, port=22, username=db["ssh_user"], key_filename=db["ssh_key"]) client.exec_command("pwd") - client.connect(DOCS_SERVER, port=22, username=db["ssh_user"]) + client.connect(DOCS_SERVER, port=22, username=db["ssh_user"], key_filename=db["ssh_key"]) client.exec_command("pwd") @@ -323,7 +327,7 @@ def check_sigstore_client(db: ReleaseShelf) -> None: client = paramiko.SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.WarningPolicy) - client.connect(DOWNLOADS_SERVER, port=22, username=db["ssh_user"]) + client.connect(DOWNLOADS_SERVER, port=22, username=db["ssh_user"], key_filename=db["ssh_key"]) _, stdout, _ = client.exec_command("python3 -m sigstore --version") sigstore_version = stdout.read(1000).decode() sigstore_vermatch = re.match("^sigstore ([0-9.]+)", sigstore_version) @@ -398,6 +402,9 @@ def check_cpython_repo_is_clean(db: ReleaseShelf) -> None: def check_magic_number(db: ReleaseShelf) -> None: release_tag = db["release"] + if release_tag.major == 3 and release_tag.minor <= 13: + return + if release_tag.is_final or release_tag.is_release_candidate: def out(msg: str) -> None: @@ -623,7 +630,7 @@ def sign_source_artifacts(db: ReleaseShelf) -> None: subprocess.check_call( [ - "python3", + sys.executable, "-m", "sigstore", "sign", @@ -692,7 +699,7 @@ def upload_files_to_server(db: ReleaseShelf, server: str) -> None: client = paramiko.SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.WarningPolicy) - client.connect(server, port=22, username=db["ssh_user"]) + client.connect(server, port=22, username=db["ssh_user"], key_filename=db["ssh_key"]) transport = client.get_transport() assert transport is not None, f"SSH transport to {server} is None" @@ -737,7 +744,7 @@ def place_files_in_download_folder(db: ReleaseShelf) -> None: client = paramiko.SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.WarningPolicy) - client.connect(DOWNLOADS_SERVER, port=22, username=db["ssh_user"]) + client.connect(DOWNLOADS_SERVER, port=22, username=db["ssh_user"], key_filename=db["ssh_key"]) transport = client.get_transport() assert transport is not None, f"SSH transport to {DOWNLOADS_SERVER} is None" @@ -788,7 +795,7 @@ def unpack_docs_in_the_docs_server(db: ReleaseShelf) -> None: client = paramiko.SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.WarningPolicy) - client.connect(DOCS_SERVER, port=22, username=db["ssh_user"]) + client.connect(DOCS_SERVER, port=22, username=db["ssh_user"], key_filename=db["ssh_key"]) transport = client.get_transport() assert transport is not None, f"SSH transport to {DOCS_SERVER} is None" @@ -905,7 +912,7 @@ def wait_until_all_files_are_in_folder(db: ReleaseShelf) -> None: client = paramiko.SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.WarningPolicy) - client.connect(DOWNLOADS_SERVER, port=22, username=db["ssh_user"]) + client.connect(DOWNLOADS_SERVER, port=22, username=db["ssh_user"], key_filename=db["ssh_key"]) ftp_client = client.open_sftp() destination = f"/srv/www.python.org/ftp/python/{db['release'].normalized()}" @@ -943,7 +950,7 @@ def run_add_to_python_dot_org(db: ReleaseShelf) -> None: client = paramiko.SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.WarningPolicy) - client.connect(DOWNLOADS_SERVER, port=22, username=db["ssh_user"]) + client.connect(DOWNLOADS_SERVER, port=22, username=db["ssh_user"], key_filename=db["ssh_key"]) transport = client.get_transport() assert transport is not None, f"SSH transport to {DOWNLOADS_SERVER} is None" @@ -1259,6 +1266,13 @@ def _api_key(api_key: str) -> str: help="Username to be used when authenticating via ssh", type=str, ) + parser.add_argument( + "--ssh-key", + dest="ssh_key", + default=None, + help="Path to the SSH key file to use for authentication", + type=str, + ) args = parser.parse_args() auth_key = args.auth_key or os.getenv("AUTH_INFO") @@ -1353,6 +1367,7 @@ def _api_key(api_key: str) -> str: api_key=auth_key, ssh_user=args.ssh_user, sign_gpg=not no_gpg, + ssh_key=args.ssh_key, tasks=tasks, ) automata.run() From cb2101608706e4fedb123bebccf0f23134ac0708 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 8 Apr 2025 16:24:16 +0100 Subject: [PATCH 2/4] Add a new --security-release flag --- release.py | 9 +++++++++ run_release.py | 42 ++++++++++++++++++++++++++++++++++-------- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/release.py b/release.py index 90412c62..a2539569 100755 --- a/release.py +++ b/release.py @@ -66,6 +66,9 @@ def get(self, key: Literal["ssh_key"], default: str | None = None) -> str | None @overload def get(self, key: Literal["sign_gpg"], default: bool | None = None) -> bool: ... + @overload + def get(self, key: Literal["security_release"], default: bool | None = None) -> bool: ... + @overload def get(self, key: Literal["release"], default: Tag | None = None) -> Tag: ... @@ -93,6 +96,9 @@ def __getitem__(self, key: Literal["ssh_key"]) -> str | None: ... @overload def __getitem__(self, key: Literal["sign_gpg"]) -> bool: ... + @overload + def __getitem__(self, key: Literal["security_release"]) -> bool: ... + @overload def __getitem__(self, key: Literal["release"]) -> Tag: ... @@ -122,6 +128,9 @@ def __setitem__(self, key: Literal["ssh_key"], value: str | None) -> None: ... @overload def __setitem__(self, key: Literal["sign_gpg"], value: bool) -> None: ... + @overload + def __setitem__(self, key: Literal["security_release"], value: bool) -> None: ... + @overload def __setitem__(self, key: Literal["release"], value: Tag) -> None: ... diff --git a/run_release.py b/run_release.py index 1581276d..d88f389b 100755 --- a/run_release.py +++ b/run_release.py @@ -194,6 +194,7 @@ def __init__( ssh_user: str, sign_gpg: bool, ssh_key: str | None = None, + security_release: bool = False, first_state: Task | None = None, ) -> None: self.tasks = tasks @@ -220,6 +221,8 @@ def __init__( self.db["ssh_key"] = ssh_key if not self.db.get("sign_gpg"): self.db["sign_gpg"] = sign_gpg + if not self.db.get("security_release"): + self.db["security_release"] = security_release if not self.db.get("release"): self.db["release"] = release_tag @@ -233,6 +236,7 @@ def __init__( print(f"- SSH key: {self.db['ssh_key'] or 'Default'}") print(f"- python.org API key: {self.db['auth_info']}") print(f"- Sign with GPG: {self.db['sign_gpg']}") + print(f"- Security release: {self.db['security_release']}") print() def checkpoint(self) -> None: @@ -930,18 +934,32 @@ def wait_until_all_files_are_in_folder(db: ReleaseShelf) -> None: are_windows_files_there = f"python-{release}.exe" in all_files are_macos_files_there = f"python-{release}-macos11.pkg" in all_files are_linux_files_there = f"Python-{release}.tgz" in all_files - are_all_files_there = ( - are_linux_files_there and are_windows_files_there and are_macos_files_there - ) + + if db["security_release"]: + # For security releases, only check Linux files + are_all_files_there = are_linux_files_there + else: + # For regular releases, check all platforms + are_all_files_there = ( + are_linux_files_there and are_windows_files_there and are_macos_files_there + ) + if not are_all_files_there: linux_tick = "✅" if are_linux_files_there else "❌" windows_tick = "✅" if are_windows_files_there else "❌" macos_tick = "✅" if are_macos_files_there else "❌" - print( - f"\rWaiting for files: Linux {linux_tick} Windows {windows_tick} Mac {macos_tick} ", - flush=True, - end="", - ) + if db["security_release"]: + print( + f"\rWaiting for files: Linux {linux_tick} (security release mode - only checking Linux) ", + flush=True, + end="", + ) + else: + print( + f"\rWaiting for files: Linux {linux_tick} Windows {windows_tick} Mac {macos_tick} ", + flush=True, + end="", + ) time.sleep(1) print() @@ -1273,6 +1291,13 @@ def _api_key(api_key: str) -> str: help="Path to the SSH key file to use for authentication", type=str, ) + parser.add_argument( + "--security-release", + dest="security_release", + action="store_true", + default=False, + help="Indicate this is a security release (only checks for Linux files)", + ) args = parser.parse_args() auth_key = args.auth_key or os.getenv("AUTH_INFO") @@ -1368,6 +1393,7 @@ def _api_key(api_key: str) -> str: ssh_user=args.ssh_user, sign_gpg=not no_gpg, ssh_key=args.ssh_key, + security_release=args.security_release, tasks=tasks, ) automata.run() From 9ff1c99d80b2d0cb10c3821febfe825997377ad7 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 8 Apr 2025 23:28:37 +0300 Subject: [PATCH 3/4] Format with Black to fix lint --- release.py | 8 ++++++-- run_release.py | 36 ++++++++++++++++++++++++++---------- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/release.py b/release.py index a2539569..8d3f660e 100755 --- a/release.py +++ b/release.py @@ -61,13 +61,17 @@ def get(self, key: Literal["auth_info"], default: str | None = None) -> str: ... def get(self, key: Literal["ssh_user"], default: str | None = None) -> str: ... @overload - def get(self, key: Literal["ssh_key"], default: str | None = None) -> str | None: ... + def get( + self, key: Literal["ssh_key"], default: str | None = None + ) -> str | None: ... @overload def get(self, key: Literal["sign_gpg"], default: bool | None = None) -> bool: ... @overload - def get(self, key: Literal["security_release"], default: bool | None = None) -> bool: ... + def get( + self, key: Literal["security_release"], default: bool | None = None + ) -> bool: ... @overload def get(self, key: Literal["release"], default: Tag | None = None) -> Tag: ... diff --git a/run_release.py b/run_release.py index d88f389b..1cc3de26 100755 --- a/run_release.py +++ b/run_release.py @@ -321,9 +321,13 @@ def check_ssh_connection(db: ReleaseShelf) -> None: client = paramiko.SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.WarningPolicy) - client.connect(DOWNLOADS_SERVER, port=22, username=db["ssh_user"], key_filename=db["ssh_key"]) + client.connect( + DOWNLOADS_SERVER, port=22, username=db["ssh_user"], key_filename=db["ssh_key"] + ) client.exec_command("pwd") - client.connect(DOCS_SERVER, port=22, username=db["ssh_user"], key_filename=db["ssh_key"]) + client.connect( + DOCS_SERVER, port=22, username=db["ssh_user"], key_filename=db["ssh_key"] + ) client.exec_command("pwd") @@ -331,7 +335,9 @@ def check_sigstore_client(db: ReleaseShelf) -> None: client = paramiko.SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.WarningPolicy) - client.connect(DOWNLOADS_SERVER, port=22, username=db["ssh_user"], key_filename=db["ssh_key"]) + client.connect( + DOWNLOADS_SERVER, port=22, username=db["ssh_user"], key_filename=db["ssh_key"] + ) _, stdout, _ = client.exec_command("python3 -m sigstore --version") sigstore_version = stdout.read(1000).decode() sigstore_vermatch = re.match("^sigstore ([0-9.]+)", sigstore_version) @@ -748,7 +754,9 @@ def place_files_in_download_folder(db: ReleaseShelf) -> None: client = paramiko.SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.WarningPolicy) - client.connect(DOWNLOADS_SERVER, port=22, username=db["ssh_user"], key_filename=db["ssh_key"]) + client.connect( + DOWNLOADS_SERVER, port=22, username=db["ssh_user"], key_filename=db["ssh_key"] + ) transport = client.get_transport() assert transport is not None, f"SSH transport to {DOWNLOADS_SERVER} is None" @@ -799,7 +807,9 @@ def unpack_docs_in_the_docs_server(db: ReleaseShelf) -> None: client = paramiko.SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.WarningPolicy) - client.connect(DOCS_SERVER, port=22, username=db["ssh_user"], key_filename=db["ssh_key"]) + client.connect( + DOCS_SERVER, port=22, username=db["ssh_user"], key_filename=db["ssh_key"] + ) transport = client.get_transport() assert transport is not None, f"SSH transport to {DOCS_SERVER} is None" @@ -916,7 +926,9 @@ def wait_until_all_files_are_in_folder(db: ReleaseShelf) -> None: client = paramiko.SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.WarningPolicy) - client.connect(DOWNLOADS_SERVER, port=22, username=db["ssh_user"], key_filename=db["ssh_key"]) + client.connect( + DOWNLOADS_SERVER, port=22, username=db["ssh_user"], key_filename=db["ssh_key"] + ) ftp_client = client.open_sftp() destination = f"/srv/www.python.org/ftp/python/{db['release'].normalized()}" @@ -934,16 +946,18 @@ def wait_until_all_files_are_in_folder(db: ReleaseShelf) -> None: are_windows_files_there = f"python-{release}.exe" in all_files are_macos_files_there = f"python-{release}-macos11.pkg" in all_files are_linux_files_there = f"Python-{release}.tgz" in all_files - + if db["security_release"]: # For security releases, only check Linux files are_all_files_there = are_linux_files_there else: # For regular releases, check all platforms are_all_files_there = ( - are_linux_files_there and are_windows_files_there and are_macos_files_there + are_linux_files_there + and are_windows_files_there + and are_macos_files_there ) - + if not are_all_files_there: linux_tick = "✅" if are_linux_files_there else "❌" windows_tick = "✅" if are_windows_files_there else "❌" @@ -968,7 +982,9 @@ def run_add_to_python_dot_org(db: ReleaseShelf) -> None: client = paramiko.SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.WarningPolicy) - client.connect(DOWNLOADS_SERVER, port=22, username=db["ssh_user"], key_filename=db["ssh_key"]) + client.connect( + DOWNLOADS_SERVER, port=22, username=db["ssh_user"], key_filename=db["ssh_key"] + ) transport = client.get_transport() assert transport is not None, f"SSH transport to {DOWNLOADS_SERVER} is None" From f7e1b8ddedcb7a23a2f70b001899621f241d4814 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 8 Apr 2025 23:29:25 +0300 Subject: [PATCH 4/4] Fix test --- tests/test_run_release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_run_release.py b/tests/test_run_release.py index 5c40d37c..e07d609f 100644 --- a/tests/test_run_release.py +++ b/tests/test_run_release.py @@ -34,7 +34,7 @@ def test_invalid_extract_github_owner() -> None: def test_check_magic_number() -> None: db = { - "release": Tag("3.13.0rc1"), + "release": Tag("3.14.0rc1"), "git_repo": str(Path(__file__).parent / "magicdata"), } with pytest.raises(