8000 Replace requests with urllib3 · python/docsbuild-scripts@4d05bb0 · GitHub
[go: up one dir, main page]
Skip to content

Commit 4d05bb0

Browse files
hugovkhugovk
committed
Replace requests with urllib3
1 parent 12f7735 commit 4d05bb0

File tree

2 files changed

+35
-35
lines changed

2 files changed

+35
-35
lines changed

‎build_docs.py

Lines changed: 34 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -46,11 +46,10 @@
4646
from typing import Iterable
4747
from urllib.parse import urljoin
4848

49-
import zc.lockfile
5049
import jinja2
51-
import requests
5250
import tomlkit
53-
51+
import urllib3
52+
import zc.lockfile
5453

5554
try:
5655
from os import EX_OK, EX_SOFTWARE as EX_FAILURE
@@ -434,7 +433,7 @@ def build_robots_txt(
434433
www_root: Path,
435434
group,
436435
skip_cache_invalidation,
437-
session: requests.Session,
436+
http: urllib3.PoolManager,
438437
) -> None:
439438
"""Disallow crawl of EOL versions in robots.txt."""
440439
if not www_root.exists():
@@ -450,7 +449,7 @@ def build_robots_txt(
450449
robots_file.chmod(0o775)
451450
run(["chgrp", group, robots_file])
452451
if not skip_cache_invalidation:
453-
purge(session, "robots.txt")
452+
purge(http, "robots.txt")
454453

455454

456455
def build_sitemap(
@@ -643,7 +642,7 @@ def full_build(self):
643642
"""
644643
return not self.quick and not self.language.html_only
645644

646-
def run(self, session: requests.Session) -> bool:
645+
def run(self, http: urllib3.PoolManager) -> bool:
647646
"""Build and publish a Python doc, for a language, and a version."""
648647
start_time = perf_counter()
649648
logging.info("Running.")
@@ -654,7 +653,7 @@ def run(self, session: requests.Session) -> bool:
654653
if self.should_rebuild():
655654
self.build_venv()
656655
self.build()
657-
self.copy_build_to_webroot(session)
656+
self.copy_build_to_webroot(http)
658657
self.save_state(build_duration=perf_counter() - start_time)
659658
except Exception as err:
660659
logging.exception("Badly handled exception, human, please help.")
@@ -798,7 +797,7 @@ def build_venv(self):
798797
run([venv_path / "bin" / "python", "-m", "pip", "freeze", "--all"])
799798
self.venv = venv_path
800799

801-
def copy_build_to_webroot(self, session: requests.Session) -> None:
800+
def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None:
802801
"""Copy a given build to the appropriate webroot with appropriate rights."""
803802
logging.info("Publishing start.")
804803
self.www_root.mkdir(parents=True, exist_ok=True)
@@ -910,9 +909,9 @@ def copy_build_to_webroot(self, session: requests.Session) -> None:
910909
prefixes = run(["find", "-L", targets_dir, "-samefile", target]).stdout
911910
prefixes = prefixes.replace(targets_dir + "/", "")
912911
prefixes = [prefix + "/" for prefix in prefixes.split("\n") if prefix]
913-
purge(session, *prefixes)
912+
purge(http, *prefixes)
914913
for prefix in prefixes:
915-
purge(session, *[prefix + p for p in changed])
914+
purge(http, *[prefix + p for p in changed])
916915
logging.info("Publishing done")
917916

918917
def should_rebuild(self):
@@ -985,7 +984,7 @@ def symlink(
985984
name: str,
986985
group: str,
987986
skip_cache_invalidation: bool,
988-
session: requests.Session,
987+
http: urllib3.PoolManager,
989988
) -> None:
990989
"""Used by major_symlinks and dev_symlink to maintain symlinks."""
991990
if language.tag == "en": # english is rooted on /, no /en/
@@ -1003,7 +1002,7 @@ def symlink(
10031002
link.symlink_to(directory)
10041003
run(["chown", "-h", ":" + group, str(link)])
10051004
if not skip_cache_invalidation:
1006-
purge_path(session, www_root, link)
1005+
purge_path(http, www_root, link)
10071006

10081007

10091008
def major_symlinks(
@@ -1012,7 +1011,7 @@ def major_symlinks(
10121011
versions: Iterable[Version],
10131012
languages: Iterable[Language],
10141013
skip_cache_invalidation: bool,
1015-
session: requests.Session,
1014+
http: urllib3.PoolManager,
10161015
) -> None:
10171016
"""Maintains the /2/ and /3/ symlinks for each languages.
10181017
@@ -1030,9 +1029,9 @@ def major_symlinks(
10301029
"3",
10311030
group,
10321031
skip_cache_invalidation,
1033-
session,
1032+
http,
10341033
)
1035-
symlink(www_root, language, "2.7", "2", group, skip_cache_invalidation, session)
1034+
symlink(www_root, language, "2.7", "2", group, skip_cache_invalidation, http)
10361035

10371036

10381037
def dev_symlink(
@@ -1041,7 +1040,7 @@ def dev_symlink(
10411040
versions,
10421041
languages,
10431042
skip_cache_invalidation: bool,
1044-
session: requests.Session,
1043+
http: urllib3.PoolManager,
10451044
) -> None:
10461045
"""Maintains the /dev/ symlinks for each languages.
10471046
@@ -1059,11 +1058,11 @@ def dev_symlink(
10591058
"dev",
10601059
group,
10611060
skip_cache_invalidation,
1062-
session,
1061+
http,
10631062
)
10641063

10651064

1066-
def purge(session: requests.Session, *paths: Path | str) -> None:
1065+
def purge(http: urllib3.PoolManager, *paths: Path | str) -> None:
10671066
"""Remove one or many paths from docs.python.org's CDN.
10681067
10691068
To be used when a file change, so the CDN fetch the new one.
@@ -1072,21 +1071,21 @@ def purge(session: requests.Session, *paths: Path | str) -> None:
10721071
for path in paths:
10731072
url = urljoin(base, str(path))
10741073
logging.debug("Purging %s from CDN", url)
1075-
session.request("PURGE", url, timeout=30)
1074+
http.request("PURGE", url, timeout=30)
10761075

10771076

1078-
def purge_path(session: requests.Session, www_root: Path, path: Path) -> None:
1077+
def purge_path(http: urllib3.PoolManager, www_root: Path, path: Path) -> None:
10791078
"""Recursively remove a path from docs.python.org's CDN.
10801079
10811080
To be used when a directory change, so the CDN fetch the new one.
10821081
"""
1083-
purge(session, *[file.relative_to(www_root) for file in path.glob("**/*")])
1084-
purge(session, path.relative_to(www_root))
1085-
purge(session, str(path.relative_to(www_root)) + "/")
1082+
purge(http, *[file.relative_to(www_root) for file in path.glob("**/*")])
1083+
purge(http, path.relative_to(www_root))
1084+
purge(http, str(path.relative_to(www_root)) + "/")
10861085

10871086

10881087
def proofread_canonicals(
1089-
www_root: Path, skip_cache_invalidation: bool, session: requests.Session
1088+
www_root: Path, skip_cache_invalidation: bool, http: urllib3.PoolManager
10901089
) -> None:
10911090
"""In www_root we check that all canonical links point to existing contents.
10921091
@@ -1109,11 +1108,12 @@ def proofread_canonicals(
11091108
html = html.replace(canonical.group(0), "")
11101109
file.write_text(html, encoding="UTF-8", errors="surrogateescape")
11111110
if not skip_cache_invalidation:
1112-
purge(session, str(file).replace("/srv/docs.python.org/", ""))
1111+
purge(http, str(file).replace("/srv/docs.python.org/", ""))
11131112

11141113

1115-
def parse_versions_from_devguide(session: requests.Session) -> list[Version]:
1116-
releases = session.get(
1114+
def parse_versions_from_devguide(http: urllib3.PoolManager) -> list[Version]:
1115+
releases = http.request(
1116+
"GET",
11171117
"https://raw.githubusercontent.com/"
11181118
"python/devguide/main/include/release-cycle.json",
11191119
timeout=30,
@@ -1141,8 +1141,8 @@ def parse_languages_from_config():
11411141

11421142
def build_docs(args) -> bool:
11431143
"""Build all docs (each language and each version)."""
1144-
session = requests.Session()
1145-
versions = parse_versions_from_devguide(session)
1144+
http = urllib3.PoolManager()
1145+
versions = parse_versions_from_devguide(http)
11461146
languages = parse_languages_from_config()
11471147
todo = [
11481148
(version, language)
@@ -1170,7 +1170,7 @@ def build_docs(args) -> bool:
11701170
builder = DocBuilder(
11711171
version, versions, language, languages, cpython_repo, **vars(args)
11721172
)
1173-
all_built_successfully &= builder.run(session)
1173+
all_built_successfully &= builder.run(http)
11741174
logging.root.handlers[0].setFormatter(
11751175
logging.Formatter("%(asctime)s %(levelname)s: %(message)s")
11761176
)
@@ -1183,25 +1183,25 @@ def build_docs(args) -> bool:
11831183
args.www_root,
11841184
args.group,
11851185
args.skip_cache_invalidation,
1186-
session,
1186+
http,
11871187
)
11881188
major_symlinks(
11891189
args.www_root,
11901190
args.group,
11911191
versions,
11921192
languages,
11931193
args.skip_cache_invalidation,
1194-
session,
1194+
http,
11951195
)
11961196
dev_symlink(
11971197
args.www_root,
11981198
args.group,
11991199
versions,
12001200< B2B2 div class="diff-text-inner"> languages,
12011201
args.skip_cache_invalidation,
1202-
session,
1202+
http,
12031203
)
1204-
proofread_canonicals(args.www_root, args.skip_cache_invalidation, session)
1204+
proofread_canonicals(args.www_root, args.skip_cache_invalidation, http)
12051205

12061206
return all_built_successfully
12071207

‎requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
jinja2
2-
requests
32
sentry-sdk
43
tomlkit
4+
urllib3
55
zc.lockfile

0 commit comments

Comments
 (0)
0