8000 Merge remote-tracking branch 'upstream/main' into log-elapsed-time · python/docsbuild-scripts@d32c793 · GitHub
[go: up one dir, main page]
Skip to content

Commit d32c793

Browse files
hugovkhugovk
committed
Merge remote-tracking branch 'upstream/main' into log-elapsed-time
2 parents f2d2ffb + e1b1138 commit d32c793

File tree

2 files changed

+95
-36
lines changed

2 files changed

+95
-36
lines changed

‎build_docs.py

Lines changed: 94 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,10 @@
4545
from typing import Iterable
4646
from urllib.parse import urljoin
4747

48-
import zc.lockfile
4948
import jinja2
50-
import requests
5149
import tomlkit
52-
50+
import urllib3
51+
import zc.lockfile
5352

5453
try:
5554
from os import EX_OK, EX_SOFTWARE as EX_FAILURE
@@ -433,7 +432,8 @@ def build_robots_txt(
433432
www_root: Path,
434433
group,
435434
skip_cache_invalidation,
436-
):
435+
http: urllib3.PoolManager,
436+
) -> None:
437437
"""Disallow crawl of EOL versions in robots.txt."""
438438
if not www_root.exists():
439439
logging.info("Skipping robots.txt generation (www root does not even exist).")
@@ -448,7 +448,7 @@ def build_robots_txt(
448448
robots_file.chmod(0o775)
449449
run(["chgrp", group, robots_file])
450450
if not skip_cache_invalidation:
451-
purge("robots.txt")
451+
purge(http, "robots.txt")
452452

453453

454454
def build_sitemap(
@@ -641,7 +641,7 @@ def full_build(self):
641641
"""
642642
return not self.quick and not self.language.html_only
643643

644-
def run(self) -> bool:
644+
def run(self, http: urllib3.PoolManager) -> bool:
645645
"""Build and publish a Python doc, for a language, and a version."""
646646
start_time = perf_counter()
647647
logging.info("Running.")
@@ -652,7 +652,7 @@ def run(self) -> bool:
652652
if self.should_rebuild():
653653
self.build_venv()
654654
self.build()
655-
self.copy_build_to_webroot()
655+
self.copy_build_to_webroot(http)
656656
self.save_state(build_duration=perf_counter() - start_time)
657657
except Exception as err:
658658
logging.exception("Badly handled exception, human, please help.")
@@ -798,7 +798,7 @@ def build_venv(self):
798798
run([venv_path / "bin" / "python", "-m", "pip", "freeze", "--all"])
799799
self.venv = venv_path
800800

801-
def copy_build_to_webroot(self):
801+
def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None:
802802
"""Copy a given build to the appropriate webroot with appropriate rights."""
803803
logging.info("Publishing start.")
804804
start_time = perf_counter()
@@ -911,9 +911,9 @@ def copy_build_to_webroot(self):
911911
prefixes = run(["find", "-L", targets_dir, "-samefile", target]).stdout
912912
prefixes = prefixes.replace(targets_dir + "/", "")
913913
prefixes = [prefix + "/" for prefix in prefixes.split("\n") if prefix]
914-
purge(*prefixes)
914+
purge(http, *prefixes)
915915
for prefix in prefixes:
916-
purge(*[prefix + p for p in changed])
916+
purge(http, *[prefix + p for p in changed])
917917
logging.info(
918918
"Publishing done (%s).", format_seconds(perf_counter() - start_time)
 6D40
919919
)
@@ -981,7 +981,15 @@ def save_state(self, build_duration: float):
981981
state_file.write_text(tomlkit.dumps(states), encoding="UTF-8")
982982

983983

984-
def symlink(www_root: Path, language: Language, directory: str, name: str, group: str, skip_cache_invalidation: bool):
984+
def symlink(
985+
www_root: Path,
986+
language: Language,
987+
directory: str,
988+
name: str,
989+
group: str,
990+
skip_cache_invalidation: bool,
991+
http: urllib3.PoolManager,
992+
) -> None:
985993
"""Used by major_symlinks and dev_symlink to maintain symlinks."""
986994
if language.tag == "en": # English is rooted on /, no /en/
987995
path = www_root
@@ -998,12 +1006,17 @@ def symlink(www_root: Path, language: Language, directory: str, name: str, group
9981006
link.symlink_to(directory)
9991007
run(["chown", "-h", ":" + group, str(link)])
10001008
if not skip_cache_invalidation:
1001-
purge_path(www_root, link)
1009+
purge_path(http, www_root, link)
10021010

10031011

10041012
def major_symlinks(
1005-
www_root: Path, group, versions: Iterable[Version], languages: Iterable[Language], skip_cache_invalidation: bool
1006-
):
1013+
www_root: Path,
1014+
group: str,
1015+
versions: Iterable[Version],
1016+
languages: Iterable[Language],
1017+
skip_cache_invalidation: bool,
1018+
http: urllib3.PoolManager,
1019+
) -> None:
10071020
"""Maintains the /2/ and /3/ symlinks for each language.
10081021
10091022
Like:
@@ -1013,11 +1026,26 @@ def major_symlinks(
10131026
"""
10141027
current_stable = Version.current_stable(versions).name
10151028
for language in languages:
1016-
symlink(www_root, language, current_stable, "3", group, skip_cache_invalidation)
1017-
symlink(www_root, language, "2.7", "2", group, skip_cache_invalidation)
1029+
symlink(
1030+
www_root,
1031+
language,
1032+
current_stable,
1033+
"3",
1034+
group,
1035+
skip_cache_invalidation,
1036+
http,
1037+
)
1038+
symlink(www_root, language, "2.7", "2", group, skip_cache_invalidation, http)
10181039

10191040

1020-
def dev_symlink(www_root: Path, group, versions, languages, skip_cache_invalidation: bool):
1041+
def dev_symlink(
1042+
www_root: Path,
1043+
group,
1044+
versions,
1045+
languages,
1046+
skip_cache_invalidation: bool,
1047+
http: urllib3.PoolManager,
1048+
) -> None:
10211049
"""Maintains the /dev/ symlinks for each language.
10221050
10231051
Like:
@@ -1027,10 +1055,18 @@ def dev_symlink(www_root: Path, group, versions, languages, skip_cache_invalidat
10271055
"""
10281056
current_dev = Version.current_dev(versions).name
10291057
for language in languages:
1030-
symlink(www_root, language, current_dev, "dev", group, skip_cache_invalidation)
1058+
symlink(
1059+
www_root,
1060+
language,
1061+
current_dev,
1062+
"dev",
1063+
group,
1064+
skip_cache_invalidation,
1065+
http,
1066+
)
10311067

10321068

1033-
def purge(*paths):
1069+
def purge(http: urllib3.PoolManager, *paths: Path | str) -> None:
10341070
"""Remove one or many paths from docs.python.org's CDN.
10351071
10361072
To be used when a file changes, so the CDN fetches the new one.
@@ -1039,20 +1075,22 @@ def purge(*paths):
10391075
for path in paths:
10401076
url = urljoin(base, str(path))
10411077
logging.debug("Purging %s from CDN", url)
1042-
requests.request("PURGE", url, timeout=30)
1078+
http.request("PURGE", url, timeout=30)
10431079

10441080

1045-
def purge_path(www_root: Path, path: Path):
1081+
def purge_path(http: urllib3.PoolManager, www_root: Path, path: Path) -> None:
10461082
"""Recursively remove a path from docs.python.org's CDN.
10471083
10481084
To be used when a directory changes, so the CDN fetches the new one.
10491085
"""
1050-
purge(*[file.relative_to(www_root) for file in path.glob("**/*")])
1051-
purge(path.relative_to(www_root))
1052-
purge(str(path.relative_to(www_root)) + "/")
1086+
purge(http, *[file.relative_to(www_root) for file in path.glob("**/*")])
1087+
purge(http, path.relative_to(www_root))
1088+
purge(http, str(path.relative_to(www_root)) + "/")
10531089

10541090

1055-
def proofread_canonicals(www_root: Path, skip_cache_invalidation: bool) -> None:
1091+
def proofread_canonicals(
1092+
www_root: Path, skip_cache_invalidation: bool, http: urllib3.PoolManager
1093+
) -> None:
10561094
"""In www_root we check that all canonical links point to existing contents.
10571095
10581096
It can happen that a canonical is "broken":
@@ -1074,11 +1112,12 @@ def proofread_canonicals(www_root: Path, skip_cache_invalidation: bool) -> None:
10741112
html = html.replace(canonical.group(0), "")
10751113
file.write_text(html, encoding="UTF-8", errors="surrogateescape")
10761114
if not skip_cache_invalidation:
1077-
purge(str(file).replace("/srv/docs.python.org/", ""))
1115+
purge(http, str(file).replace("/srv/docs.python.org/", ""))
10781116

10791117

1080-
def parse_versions_from_devguide():
1081-
releases = requests.get(
1118+
def parse_versions_from_devguide(http: urllib3.PoolManager) -> list[Version]:
1119+
releases = http.request(
1120+
"GET",
10821121
"https://raw.githubusercontent.com/"
10831122
"python/devguide/main/include/release-cycle.json",
10841123
timeout=30,
@@ -1124,7 +1163,8 @@ def build_docs(args) -> bool:
11241163
"""Build all docs (each language and each version)."""
11251164
logging.info("Full build start.")
11261165
start_time = perf_counter()
1127-
versions = parse_versions_from_devguide()
1166+
http = urllib3.PoolManager()
1167+
versions = parse_versions_from_devguide(http)
11281168
languages = parse_languages_from_config()
11291169
todo = [
11301170
(version, language)
@@ -1137,7 +1177,6 @@ def build_docs(args) -> bool:
11371177
cpython_repo = Repository(
11381178
"https://github.com/python/cpython.git", args.build_root / "cpython"
11391179
)
1140-
cpython_repo.update()
11411180
while todo:
11421181
version, language = todo.pop()
11431182
logging.root.handlers[0].setFormatter(
@@ -1149,22 +1188,42 @@ def build_docs(args) -> bool:
11491188
scope = sentry_sdk.get_isolation_scope()
11501189
scope.set_tag("version", version.name)
11511190
scope.set_tag("language", language.tag)
1191+
cpython_repo.update()
11521192
builder = DocBuilder(
11531193
version, versions, language, languages, cpython_repo, **vars(args)
11541194
)
1155-
all_built_successfully &= builder.run()
1195+
all_built_successfully &= builder.run(http)
11561196
logging.root.handlers[0].setFormatter(
11571197
logging.Formatter("%(asctime)s %(levelname)s: %(message)s")
11581198
)
11591199

11601200
build_sitemap(versions, languages, args.www_root, args.group)
11611201
build_404(args.www_root, args.group)
11621202
build_robots_txt(
1163-
versions, languages, args.www_root, args.group, args.skip_cache_invalidation
1203+
versions,
1204+
languages,
1205+
args.www_root,
1206+
args.group,
1207+
args.skip_cache_invalidation,
1208+
http,
1209+
)
1210+
major_symlinks(
1211+
args.www_root,
1212+
args.group,
1213+
versions,
1214+
languages,
1215+
args.skip_cache_invalidation,
1216+
http,
1217+
)
1218+
dev_symlink(
1219+
args.www_root,
1220+
args.group,
1221+
versions,
1222+
languages,
1223+
args.skip_cache_invalidation,
1224+
http,
11641225
)
1165-
major_symlinks(args.www_root, args.group, versions, languages, args.skip_cache_invalidation)
1166-
dev_symlink(args.www_root, args.group, versions, languages, args.skip_cache_invalidation)
1167-
proofread_canonicals(args.www_root, args.skip_cache_invalidation)
1226+
proofread_canonicals(args.www_root, args.skip_cache_invalidation, http)
11681227

11691228
logging.info("Full build done (%s).", format_seconds(perf_counter() - start_time))
11701229

‎requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
jinja2
2-
requests
32
sentry-sdk>=2
43
tomlkit
4+
urllib3>=2
55
zc.lockfile

0 commit comments

Comments
 (0)
0