8000 DOC: Doc build for a single doc made much faster, and clean up by datapythonista · Pull Request #24428 · pandas-dev/pandas · GitHub
[go: up one dir, main page]
Skip to content

DOC: Doc build for a single doc made much faster, and clean up #24428

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Dec 30, 2018
Merged

DOC: Doc build for a single doc made much faster, and clean up #24428

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
2571572
WIP: Major refactoring and clean up, still some problems on building …
datapythonista Dec 14, 2018
83c3b45
building only api pages when needed, based on SPHINX_PATTERN, and not…
datapythonista Dec 14, 2018
a5e7ad2
Merge from master
datapythonista Dec 23, 2018
0fda07e
Merge remote-tracking branch 'upstream/master' into single_doc
datapythonista Dec 24, 2018
e851cae
WIP simplifying doc build (pending not generate rst files from exclud…
datapythonista Dec 25, 2018
fc9c359
Not generating all the api pages or the intersphinx db when a single …
datapythonista Dec 25, 2018
8e1e9e4
Merge remote-tracking branch 'upstream/master' into single_doc
datapythonista Dec 25, 2018
ade46e6
Restoring python path env
datapythonista Dec 25, 2018
648da45
Fixing comment with wrong info
datapythonista Dec 26, 2018
c489aa6
Removing not needed raw_input
datapythonista Dec 26, 2018
2be1d61
Merging from master
datapythonista Dec 28, 2018
55e47a2
Adding value to --single error message
datapythonista Dec 29, 2018
4406988
Changing num_jobs default value to 0 everywhere, to be consistent
datapythonista Dec 29, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
WIP simplifying doc build (pending not generate rst files from exclud… 
…ed api.rst)
  • Loading branch information
@datapythonista
datapythonista committed Dec 25, 2018
commit e851cae2d157f147ec45bca7d41df0b3fe9bcad8
52 changes: 24 additions & 28 deletions doc/make.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,8 @@
import sys
import os
import shutil
# import subprocess
import subprocess
import argparse
from contextlib import contextmanager
import webbrowser


Expand Down Expand Up @@ -47,12 +46,11 @@ def __init__(self, num_jobs=1, include_api=True, single_doc=None,
elif not include_api:
os.environ['SPHINX_PATTERN'] = '-api'

# if self.single_doc_type == 'docstring':
# self._run_os('sphinx-autogen', os.path.join(SOURCE_PATH, 'index.rst'))

if single_doc:
if single_doc and single_doc.endswith('.rst'):
self.single_doc_html = os.path.splitext(single_doc)[0] + '.html'

elif single_doc:
self.single_doc_html = 'generated/pandas.{}.html'.format(
single_doc)

def _process_single_doc(self, single_doc):
"""
Expand All @@ -78,7 +76,7 @@ def _process_single_doc(self, single_doc):
except AttributeError:
raise ImportError('Could not import {}'.format(single_doc))
else:
return os.path.join('generated', '{}.rst'.format(single_doc))
return single_doc[len('pandas.'):]
else:
raise ValueError('--single value should be a valid path to a '
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you print out single_doc here, so the user sees what they passed?

'.rst or .ipynb file, or a valid pandas object '
Expand All @@ -98,14 +96,7 @@ def _run_os(*args):
--------
>>> DocBuilder()._run_os('python', '--version')
"""
# TODO check_call should be more safe, but it fails with
# exclude patterns, needs investigation
# subprocess.check_call(args, stderr=subprocess.STDOUT)
exit_status = os.system(' '.join(args))
if exit_status:
msg = 'Command "{}" finished with exit code {}'
raise RuntimeError(msg.format(' '.join(args), exit_status))
#subprocess.check_call(args, stderr=subprocess.STDOUT)
subprocess.check_call(args, stdout=sys.stdout, stderr=sys.stderr)

def _sphinx_build(self, kind):
"""
Expand All @@ -125,21 +116,26 @@ def _sphinx_build(self, kind):
raise ValueError('kind must be html or latex, '
'not {}'.format(kind))

self._run_os('sphinx-build',
'-j{}'.format(self.num_jobs),
'-b{}'.format(kind),
'-W' if self.warnings_are_errors else '',
'-{}'.format(
'v' * self.verbosity) if self.verbosity else '',
'-d"{}"'.format(os.path.join(BUILD_PATH, 'doctrees')),
'"{}"'.format(SOURCE_PATH),
'"{}"'.format(os.path.join(BUILD_PATH, kind)))
self.clean()

cmd = ['sphinx-build', '-b', kind]
if self.num_jobs:
cmd += ['-j', str(self.num_jobs)]
if self.warnings_are_errors:
cmd.append('-W')
if self.verbosity:
cmd.append('-{}'.format('v' * self.verbosity))
cmd += ['-d', os.path.join(BUILD_PATH, 'doctrees'),
SOURCE_PATH, os.path.join(BUILD_PATH, kind)]
cmd = ['sphinx-build', SOURCE_PATH, os.path.join(BUILD_PATH, kind)]
self._run_os(*cmd)

def _open_browser(self, single_doc_html):
"""
Open a browser tab showing single
"""
url = os.path.join('file://', DOC_PATH, 'build', 'html', single_doc_html)
url = os.path.join('file://', DOC_PATH, 'build', 'html',
single_doc_html)
webbrowser.open(url, new=2)

def html(self):
Expand Down Expand Up @@ -217,7 +213,7 @@ def main():
help='command to run: {}'.format(', '.join(cmds)))
argparser.add_argument('--num-jobs',
type=int,
default=1,
default=0,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the DocBuilder init, the default is 1. Make them both the same?

What does a value of 0 do?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point. with 0 I don't pass a -j value to sphinx-build, while with 1, I pass -j 1, which is actually the same (unless sphinx changes its default).

It's a bit weird, because the -j parameter is supposed to make sphinx-build run with multiple cores, but it actually doesn't work, and the process takes exactly the same with -j 1 and -j 4. It could make sense to remove this option, but I guess at some point sphinx will be fixed at some point, so I guess it's worth leaving it.

I'm happy setting our default to 0 (use sphinx default), 1, or auto, just let me know if you have a preference.

I address your other comment.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm I think multiple cores speeds things up for me, at least for a full doc build.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I checked it with time in Linux, 1 vs 4 cores, it took 17m30s in both cases (with a difference of less than 5 seconds between them). I tried it in the past with a different computer (also Linux) and was the same.

help='number of jobs used by sphinx-build')
argparser.add_argument('--no-api',
default=False,
Expand Down Expand Up @@ -249,7 +245,7 @@ def main():
# external libraries (namely Sphinx) to compile this module and resolve
# the import of `python_path` correctly. The latter is used to resolve
# the import within the module, injecting it into the global namespace
os.environ['PYTHONPATH'] = args.python_path
#os.environ['PYTHONPATH'] = args.python_path
sys.path.append(args.python_path)
globals()['pandas'] = importlib.import_module('pandas')

Expand Down
8 changes: 1 addition & 7 deletions doc/source/conf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,16 +114,10 @@
t = jinja2.Template(f.read())
with open(os.path.join(source_path, 'index.rst'), 'w') as f:
f.write(t.render(include_api=pattern is None,
single_doc=(os.path.splitext(pattern)[0]
single_doc=(pattern
if pattern is not None and pattern != '-api'
else None)))

# numpydoc
# for now use old parameter listing (styling + **kwargs problem)
numpydoc_use_blockquotes = True
# use member listing for attributes
numpydoc_attributes_as_param_list = False

# matplotlib plot directive
plot_include_source = True
plot_formats = [("png", 90)]
Expand Down
12 changes: 8 additions & 4 deletions doc/source/index.rst.template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,17 +106,21 @@ Some other notes
See the package overview for more detail about what's in the library.


{% if single_doc and single_doc.startswith('generated') -%}
{% if single_doc and single_doc.endswith('.rst') -%}
.. toctree::
:maxdepth: 4

{{ single_doc[:-4] }}
{% elif single_doc %}
.. autosummary::
:toctree: generated/

{{ single_doc }}
{% else -%}
.. toctree::
:maxdepth: 4
{% endif %}

{% if single_doc -%}
{{ single_doc }}
{% endif -%}
{% if not single_doc -%}
What's New <whatsnew/v0.24.0>
install
Expand Down
0