8000 Implement a simple github webook server to build the doc. (#101) · python/docsbuild-scripts@66b922e · GitHub
[go: up one dir, main page]
Skip to content

Commit 66b922e

Browse files
JulienPalardJulienPalard
authored
Implement a simple github webook server to build the doc. (#101)
* Use a file lock to avoid running the same build in parallel. * Implement a simple github webook server to build the doc. * Explicitly warn where a build is needed for an unknown branch.
1 parent d48a271 commit 66b922e

File tree

5 files changed

+317
-42
lines changed

5 files changed

+317
-42
lines changed

‎README.md

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,3 +50,15 @@ of Sphinx we're using where::
5050
3.10 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1
5151
3.11 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1
5252
======== ======= ===== ======= ===== ===== ===== ======= ===== =====
53+
54+
55+
## The github hook server
56+
57+
`build_docs_server.py` is a simple HTTP server handling Github Webhooks
58+
requests to build the doc when needed. It only needs `push` events.
59+
60+
Its logging can be configured by giving a yaml file path to the
61+
`--logging-config` argument.
62+
63+
By default the loglevel is `DEBUG` on `stderr`, the default config can
64+
be found in the code so one can bootstrap a different config from it.

‎build_docs.py

Lines changed: 82 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
from string import Template
4141
from textwrap import indent
4242

43+
import zc.lockfile
4344
import jinja2
4445

4546
HERE = Path(__file__).resolve().parent
@@ -110,9 +111,7 @@ def title(self):
110111
Version("3.7", "3.7", "security-fixes", sphinx_version="2.3.1"),
111112
Version("3.8", "3.8", "security-fixes", sphinx_version="2.4.4"),
112113
Version("3.9", "3.9", "stable", sphinx_version="2.4.4"),
113-
Version(
114-
"3.10", "3.10", "pre-release", sphinx_version="3.2.1", sphinxopts=["-j4"]
115-
),
114+
Version("3.10", "3.10", "pre-release", sphinx_version="3.2.1", sphinxopts=["-j4"]),
116115
Version(
117116
"3.11", "main", "in development", sphinx_version="3.2.1", sphinxopts=["-j4"]
118117
),
@@ -174,6 +173,7 @@ def run(cmd) -> subprocess.CompletedProcess:
174173
stdout=subprocess.PIPE,
175174
encoding="utf-8",
176175
errors="backslashreplace",
176+
check=False,
177177
)
178178
if result.returncode:
179179
# Log last 20 lines, those are likely the interesting ones.
@@ -372,13 +372,13 @@ def setup_switchers(html_root):
372372
script = """ <script type="text/javascript" src="{}_static/switchers.js"></script>\n""".format(
373373
"../" * depth
374374
)
375-
with edit(file) as (i, o):
376-
for line in i:
375+
with edit(file) as (ifile, ofile):
376+
for line in ifile:
377377
if line == script:
378378
continue
379379
if line == " </body>\n":
380-
o.write(script)
381-
o.write(line)
380+
ofile.write(script)
381+
ofile.write(line)
382382

383383

384384
def build_one(
@@ -750,12 +750,75 @@ def setup_logging(log_directory):
750750
logging.getLogger().setLevel(logging.DEBUG)
751751

752752

753+
def build_and_publish(
754+
build_root,
755+
www_root,
756+
version,
757+
language,
758+
quick,
759+
group,
760+
log_directory,
761+
skip_cache_invalidation,
762+
theme,
763+
):
764+
"""Build and publish a Python doc, for a language, and a version.
765+
766+
Also ensures that a single process is doing it by using a `.lock`
767+
file per language / version pair.
768+
"""
769+
try:
770+
lock = zc.lockfile.LockFile(
771+
os.path.join(
772+
HERE,
773+
"{version}-{lang}.lock".format(version=version.name, lang=language.tag),
774+
)
775+
)
776+
777+
try:
778+
venv = build_venv(build_root, version, theme)
779+
build_one(
780+
version,
781+
quick,
782+
venv,
783+
build_root,
784+
group,
785+
log_directory,
786+
language,
787+
)
788+
copy_build_to_webroot(
789+
build_root,
790+
version,
791+
language,
792+
group,
793+
quick,
794+
skip_cache_invalidation,
795+
www_root,
796+
)
797+
except Exception as err:
798+
logging.exception(
799+
"Exception while building %s version %s",
800+
language.tag,
801+
version.name,
802+
)
803+
if sentry_sdk:
804+
sentry_sdk.capture_exception(err)
805+
806+
except zc.lockfile.LockError:
807+
logging.info(
808+
"Skipping build of %s/%s (build already running)",
809+
language.tag,
810+
version.name,
811+
)
812+
else:
813+
lock.close()
814+
815+
753816
def main():
754817
args = parse_args()
755818
languages_dict = {language.tag: language for language in LANGUAGES}
756819
if args.version:
757820
version_info()
758-
exit(0)
821+
sys.exit(0)
759822
if args.log_directory:
760823
args.log_directory = os.path.abspath(args.log_directory)
761824
if args.build_root:
@@ -782,34 +845,17 @@ def main():
782845
scope.set_tag("version", version.name)
783846
scope.set_tag("language", language_tag)
784847
language = languages_dict[language_tag]
785-
try:
786-
venv = build_venv(args.build_root, version, args.theme)
787-
build_one(
788-
version,
789-
args.quick,
790-
venv,
791-
args.build_root,
792-
args.group,
793-
args.log_directory,
794-
language,
795-
)
796-
copy_build_to_webroot(
797-
args.build_root,
798-
version,
799-
language,
800-
args.group,
801-
args.quick,
802-
args.skip_cache_invalidation,
803-
args.www_root,
804-
)
805-
except Exception as err:
806-
logging.exception(
807-
"Exception while building %s version %s",
808-
language_tag,
809-
version.name,
810-
)
811-
if sentry_sdk:
812-
sentry_sdk.capture_exception(err)
848+
build_and_publish(
849+
args.build_root,
850+
args.www_root,
851+
version,
852+
language,
853+
args.quick,
854+
args.group,
855+
args.log_directory,
856+
args.skip_cache_invalidation,
857+
args.theme,
858+
)
813859
build_sitemap(args.www_root)
814860
build_robots_txt(args.www_root, args.group, args.skip_cache_invalidation)
815861

‎build_docs_server.py

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
"""Github hook server.
2+
3+
This is a simple HTTP server handling Github Webhooks requests to
4+
build the doc when needed.
5+
6+
It needs a GH_SECRET environment variable to be able to receive hooks
7+
on `/hook/github`.
8+
9+
Its logging can be configured by giving a yaml file path to the
10+
`--logging-config` argument.
11+
12+
By default the loglevel is `DEBUG` on `stderr`, the default config can
13+
be found in the code so one can bootstrap a different config from it.
14+
"""
15+
16+
from pathlib import Path
17+
import argparse
18+
import asyncio
19+
import logging.config
20+
import os
21+
22+
from aiohttp import web
23+
from gidgethub import sansio
24+
import yaml
25+
26+
from build_docs import VERSIONS
27+
28+
29+
__version__ = "0.0.1"
30+
31+
DEFAULT_LOGGING_CONFIG = """
32+
---
33+
34+
version: 1
35+
disable_existing_loggers: false
36+
formatters:
37+
normal:
38+
format: '%(asctime)s - %(levelname)s - %(message)s'
39+
handlers:
40+
stderr:
41+
class: logging.StreamHandler
42+
stream: ext://sys.stderr
43+
level: DEBUG
44+
formatter: normal
45+
loggers:
46+
build_docs_server:
47+
level: DEBUG
48+
handlers: [stderr]
49+
aiohttp.access:
50+
level: DEBUG
51+
handlers: [stderr]
52+
aiohttp.client:
53+
level: DEBUG
54+
handlers: [stderr]
55+
aiohttp.internal:
56+
level: DEBUG
57+
handlers: [stderr]
58+
aiohttp.server:
59+
level: DEBUG
60+
handlers: [stderr]
61+
aiohttp.web:
62+
level: DEBUG
63+
handlers: [stderr]
64+
aiohttp.websocket:
65+
level: DEBUG
66+
handlers: [stderr]
67+
"""
68+
69+
logger = logging.getLogger("build_docs_server")
70+
71+
72+
async def version(request):
73+
return web.json_response(
74+
{
75+
"name": "docs.python.org Github handler",
76+
"version": __version__,
77+
"source": "https://github.com/python/docsbuild-scripts",
78+
}
79+
)
80+
81+
82+
async def child_waiter(app):
83+
while True:
84+
try:
85+
status = os.waitid(os.P_ALL, 0, os.WNOHANG | os.WEXITED)
86+
logger.debug("Child completed with status %s", str(status))
87+
except ChildProcessError:
88+
await asyncio.sleep(600)
89+
90+
91+
async def start_child_waiter(app):
92+
app["child_waiter"] = asyncio.ensure_future(child_waiter(app))
93+
94+
95+
async def stop_child_waiter(app):
96+
app["child_waiter"].cancel()
97+
98+
99+
async def hook(request):
100+
body = await request.read()
101+
event = sansio.Event.from_http(
102+
request.headers, body, secret=os.environ.get("GH_SECRET")
103+
)
104+
if event.event != "push":
105+
logger.debug("Received a %s event, nothing to do.", event.event)
106+
return web.Response()
107+
touched_files = (
108+
set(event.data["head_commit"]["added"])
109+
| set(event.data["head_commit"]["modified"])
110+
| set(event.data["head_commit"]["removed"])
111+
)
112+
if not any("Doc" in touched_file for touched_file in touched_files):
113+
logger.debug("No documentation file modified, ignoring.")
114+
return web.Response() # Nothing to do
115+
branch = event.data["ref"].split("/")[-1]
116+
known_branches = {version.branch for version in VERSION}
117+
if branch not in known_branches:
118+
logger.warning("Ignoring a change in branch %s (unknown branch)", branch)
119+
return web.Response() # Nothing to do
120+
logger.debug("Forking a build for branch %s", branch)
121+
pid = os.fork()
122+
if pid == 0:
123+
os.execl(
124+
"/usr/bin/env",
125+
"/usr/bin/env",
126+
"python",
127+
"build_docs.py",
128+
"--branch",
129+
branch,
130+
)
131+
else:
132+
return web.Response()
133+
134+
135+
def parse_args():
136+
parser = argparse.ArgumentParser(description=__doc__)
137+
parser.add_argument("--path", help="Unix socket to listen for connections.")
138+
parser.add_argument("--port", help="Local port to listen for connections.")
139+
parser.add_argument(
140+
"--logging-config",
141+
help="yml file containing a Python logging dictconfig, see README.md",
142+
)
143+
return parser.parse_args()
144+
145+
146+
def main():
147+
args = parse_args()
148+
logging.config.dictConfig(
149+
yaml.load(
150+
Path(args.logging_config).read_text()
151+
if args.logging_config
152+
else DEFAULT_LOGGING_CONFIG,
153+
Loader=yaml.SafeLoader,
154+
)
155+
)
156+
app = web.Application()
157+
app.on_startup.append(start_child_waiter)
158+
app.on_cleanup.append(stop_child_waiter)
159+
app.add_routes(
160+
[
161+
web.get("/", version),
162+
web.post("/hooks/github", hook),
163+
]
164+
)
165+
web.run_app(app, path=args.path, port=args.port)
166+
167+
168+
if __name__ == "__main__":
169+
main()

‎requirements.in

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,6 @@
1-
sentry-sdk
1+
aiohttp
2+
gidgethub
23
jinja2
4+
pyyaml
5+
sentry-sdk
6+
zc.lockfile

0 commit comments

Comments
 (0)
0