8000 Merge pull request #7 from JulienPalard/portability · python/docsbuild-scripts@54724b2 · GitHub
[go: up one dir, main page]
Skip to content

Commit 54724b2

Browse files
ewdurbinewdurbin
authored
Merge pull request #7 from JulienPalard/portability
Testability / portability (and fastly invalidation on devguide)
2 parents 6ed4f59 + 98d4454 commit 54724b2

File tree

1 file changed

+72
-28
lines changed

1 file changed

+72
-28
lines changed

‎build_docs.py

Lines changed: 72 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -63,51 +63,63 @@ def shell_out(cmd):
6363
raise
6464

6565

66-
def build_one(version, isdev, quick, sphinxbuild, build_root, www_root):
66+
def changed_files(directory, other):
67+
logging.info("Computing changed files")
68+
changed = []
69+
if directory[-1] != '/':
70+
directory += '/'
71+
for dirpath, dirnames, filenames in os.walk(directory):
72+
dir_rel = dirpath[len(directory):]
73+
for fn in filenames:
74+
local_path = os.path.join(dirpath, fn)
75+
rel_path = os.path.join(dir_rel, fn)
76+
target_path = os.path.join(other, rel_path)
77+
if (os.path.exists(target_path) and
78+
not _file_unchanged(target_path, local_path)):
79+
changed.append(rel_path)
80+
return changed
81+
82+
83+
def build_one(version, isdev, quick, sphinxbuild, build_root, www_root,
84+
skip_cache_invalidation=False, group='docs', git=False,
85+
log_directory='/var/log/docsbuild/'):
6786
checkout = build_root + "/python" + str(version).replace('.', '')
6887
target = www_root + "/" + str(version)
6988
logging.info("Doc autobuild started in %s", checkout)
7089
os.chdir(checkout)
7190

7291
logging.info("Updating checkout")
73-
shell_out("hg pull -u")
92+
if git:
93+
shell_out("git reset --hard HEAD")
94+
shell_out("git pull --ff-only")
95+
else:
96+
shell_out("hg pull -u")
7497

7598
maketarget = "autobuild-" + ("dev" if isdev else "stable") + ("-html" if quick else "")
7699
logging.info("Running make %s", maketarget)
77100
logname = os.path.basename(checkout) + ".log"
78-
shell_out("cd Doc; make SPHINXBUILD=%s %s >> /var/log/docsbuild/%s 2>&1" %
79-
(sphinxbuild, maketarget, logname))
80-
81-
logging.info("Computing changed files")
82-
changed = []
83-
for dirpath, dirnames, filenames in os.walk("Doc/build/html/"):
84-
dir_rel = dirpath[len("Doc/build/html/"):]
85-
for fn in filenames:
86-
local_path = os.path.join(dirpath, fn)
87-
rel_path = os.path.join(dir_rel, fn)
88-
target_path = os.path.join(target, rel_path)
89-
if (os.path.exists(target_path) and
90-
not _file_unchanged(target_path, local_path)):
91-
changed.append(rel_path)
101+
shell_out("cd Doc; make SPHINXBUILD=%s %s >> %s 2>&1" %
102+
(sphinxbuild, maketarget, os.path.join(log_directory, logname)))
92103

104+
changed = changed_files(os.path.join(checkout, "Doc/build/html"), target)
93105
logging.info("Copying HTML files to %s", target)
94-
shell_out("chown -R :docs Doc/build/html/")
106+
shell_out("chown -R :{} Doc/build/html/".format(group))
95107
shell_out("chmod -R o+r Doc/build/html/")
96108
shell_out("find Doc/build/html/ -type d -exec chmod o+x {} ';'")
97109
shell_out("cp -a Doc/build/html/* %s" % target)
98110
if not quick:
99111
logging.debug("Copying dist files")
100-
shell_out("chown -R :docs Doc/dist/")
112+
shell_out("chown -R :{} Doc/dist/".format(group))
101113
shell_out("chmod -R o+r Doc/dist/")
102114
shell_out("mkdir -m o+rx -p %s/archives" % target)
103-
shell_out("chown :docs %s/archives" % target)
115+
shell_out("chown :{} {}/archives".format(group, target))
104116
shell_out("cp -a Doc/dist/* %s/archives" % target)
105117
changed.append("archives/")
106118
for fn in os.listdir(os.path.join(target, "archives")):
107119
changed.append("archives/" + fn)
108120

109121
logging.info("%s files changed", len(changed))
110-
if changed:
122+
if changed and not skip_cache_invalidation:
111123
target_ino = os.stat(target).st_ino
112124
targets_dir = os.path.dirname(target)
113125
prefixes = []
@@ -123,12 +135,22 @@ def build_one(version, isdev, quick, sphinxbuild, build_root, www_root):
123135
logging.info("Finished %s", checkout)
124136

125137

126-
def build_devguide(devguide_checkout, devguide_target, sphinxbuild):
138+
def build_devguide(devguide_checkout, devguide_target, sphinxbuild,
139+
skip_cache_invalidation=False):
140+
build_directory = os.path.join(devguide_checkout, "build/html")
127141
logging.info("Building devguide")
128142
shell_out("git -C %s pull" % (devguide_checkout,))
129-
shell_out("%s %s %s" % (sphinxbuild, devguide_checkout, devguide_target))
143+
shell_out("%s %s %s" % (sphinxbuild, devguide_checkout, build_directory))
144+
changed = changed_files(build_directory, devguide_target)
145+
shell_out("mkdir -p {}".format(devguide_target))
146+
shell_out("cp -a {}/* {}".format(build_directory, devguide_target))
130147
shell_out("chmod -R o+r %s" % (devguide_target,))
131-
# TODO Do Fastly invalidation.
148+
if changed and not skip_cache_invalidation:
149+
prefix = os.path.basename(devguide_target)
150+
to_purge = [prefix]
151+
to_purge.extend(prefix + "/" + p for p in changed)
152+
logging.info("Running CDN purge")
153+
shell_out("curl -X PURGE \"https://docs.python.org/{%s}\"" % ",".join(to_purge))
132154

133155

134156
def parse_args():
@@ -164,27 +186,49 @@ def parse_args():
164186
"--devguide-target",
165187
help="Path where the generated devguide should be copied.",
166188
default="/srv/docs.python.org/devguide")
189+
parser.add_argument(
190+
"--skip-cache-invalidation",
191+
help="Skip fastly cache invalidation.",
192+
action="store_true")
193+
parser.add_argument(
194+
"--group",
195+
help="Group files on targets and www-root file should get.",
196+
default="docs")
197+
parser.add_argument(
198+
"--git",
199+
help="Use git instead of mercurial.",
200+
action="store_true")
201+
parser.add_argument(
202+
"--log-directory",
203+
help="Directory used to store logs.",
204+
default="/var/log/docsbuild/")
167205
return parser.parse_args()
168206

169207

170208
if __name__ == '__main__':
209+
args = parse_args()
171210
if sys.stderr.isatty():
172211
logging.basicConfig(format="%(levelname)s:%(message)s",
173212
stream=sys.stderr)
174213
else:
175214
logging.basicConfig(format="%(levelname)s:%(asctime)s:%(message)s",
176-
filename="/var/log/docsbuild/docsbuild.log")
215+
filename=os.path.join(args.log_directory,
216+
"docsbuild.log"))
177217
logging.root.setLevel(logging.DEBUG)
178-
args = parse_args()
179218
sphinxbuild = os.path.join(args.build_root, "environment/bin/sphinx-build")
180219
try:
181220
if args.branch:
182221
build_one(args.branch, args.devel, args.quick, sphinxbuild,
183-
args.build_root, args.www_root)
222+
args.build_root, args.www_root,
223+
args.skip_cache_invalidation,
224+
args.group, args.git, args.log_directory)
184225
else:
185226
for version, devel in BRANCHES:
186227
build_one(version, devel, args.quick, sphinxbuild,
187-
args.build_root, args.www_root)
188-
build_devguide(args.devguide_checkout, args.devguide_target, sphinxbuild)
228+
args.build_root, args.www_root,
229+
args.skip_cache_invalidation, args.group, args.git,
230+
args.log_directory)
231+
build_devguide(args.devguide_checkout, args.devguide_target,
232+
sphinxbuild, args.skip_cache_invalidation)
189233
except Exception:
190234
logging.exception("docs build raised exception")

0 commit comments

Comments
 (0)
0