From fdb3cd2b0d5de30d5b2c94eb4dd1c7d80fafef4a Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 25 Feb 2019 10:14:00 -0500 Subject: [PATCH 1/4] BUG: Only rebuild when necessary --- .gitignore | 3 +- README.rst | 2 +- appveyor.yml | 4 +- doc/configuration.rst | 65 +++++++- examples/local_module.py | 3 + examples/plot_function_identifier.py | 6 +- sphinx_gallery/backreferences.py | 32 +++- sphinx_gallery/binder.py | 6 +- sphinx_gallery/docs_resolv.py | 2 +- sphinx_gallery/downloads.py | 17 +- sphinx_gallery/gen_gallery.py | 42 ++--- sphinx_gallery/gen_rst.py | 31 ++-- sphinx_gallery/notebook.py | 3 +- sphinx_gallery/py_source_parser.py | 4 + sphinx_gallery/scrapers.py | 12 +- sphinx_gallery/sorting.py | 10 +- sphinx_gallery/tests/test_full.py | 235 +++++++++++++++++++++++++++ sphinx_gallery/tests/test_sorting.py | 26 ++- sphinx_gallery/utils.py | 29 +++- 19 files changed, 440 insertions(+), 92 deletions(-) diff --git a/.gitignore b/.gitignore index 81c1b2ca8..7e754f8aa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] +.coverage* # C extensions *.so @@ -73,4 +74,4 @@ target/ .history # Jupyter notebooks -.ipynb_checkpoints \ No newline at end of file +.ipynb_checkpoints diff --git a/README.rst b/README.rst index 5191c6eec..ce90cb856 100644 --- a/README.rst +++ b/README.rst @@ -55,7 +55,7 @@ Sphinx-Gallery will not manage its dependencies when installing, thus you are required to install them manually. Our minimal dependencies are: -* Sphinx +* Sphinx >= 1.5 * Matplotlib * Pillow diff --git a/appveyor.yml b/appveyor.yml index 4c6b9ade6..38b018ef0 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,12 +7,12 @@ environment: global: PYTHON: "C:\\conda" MINICONDA_VERSION: "latest" - CONDA_DEPENDENCIES: "numpy seaborn matplotlib sphinx pillow pytest pytest-cov sphinx_rtd_theme memory_profiler" + CONDA_DEPENDENCIES: "numpy seaborn matplotlib sphinx pillow pytest pytest-cov sphinx_rtd_theme vtk pyface traits traitsui" matrix: - PYTHON_VERSION: "2.7" PYTHON_ARCH: "64" - - PYTHON_VERSION: "3.5" + - PYTHON_VERSION: "3.6" PYTHON_ARCH: "64" platform: diff --git a/doc/configuration.rst b/doc/configuration.rst index e9057cd05..96bea9949 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -38,6 +38,7 @@ file: - ``binder`` (:ref:`binder_links`) - ``first_notebook_cell`` (:ref:`first_notebook_cell`) - ``junit`` (:ref:`junit_xml`) +- ``log_level`` (:ref:`log_level`) Some options can also be set or overridden on a file-by-file basis: @@ -75,7 +76,8 @@ Keep in mind that both lists have to be of the same length. .. note:: If your examples take a long time to run, consider looking at the :ref:`execution times ` - file that is generated for each gallery dir. + file that is generated for each gallery dir (as long as any examples + were actually executed in that directory during the build). .. _build_pattern: @@ -122,6 +124,13 @@ you would do:: Here, one should escape the dot ``r'\.'`` as otherwise python `regular expressions`_ matches any character. Nevertheless, as one is targeting a specific file, it would match the dot in the filename even without this escape character. +.. note:: + Sphinx-gallery only re-runs examples that have changed (according to their + md5 hash). You can delete the associated MD5 files (e.g., + ``./auto_examples/plot_awesome_example.py.md5``) to force a rebuild if + you have not changed the example itself between subsequent ``sphinx`` + calls. + Similarly, to build only examples in a specific directory, you can do:: sphinx_gallery_conf = { @@ -182,6 +191,22 @@ the converse does not hold. If you so desire you can implement your own sorting key. It will be provided the relative paths to `conf.py` of each sub gallery folder. +.. warning:: If you create your own class for ``'subsection_order'``, ensure + that the ``__str__`` of your class is stable across runs. + Sphinx determines if the build environment has changed + (and thus if *all* documents should be rewritten) + by examining the config values using + ``md5(str(obj).encode()).hexdigest()`` in + ``sphinx/builders/html.py``. Default class instances + in Python have their memory address in their ``__repr__`` which + will in general change for each build. For ``ExplicitOrder`` + for example, this is fixed via:: + + def __repr__(self): + return '<%s: %s>' % (self.__class__.__name__, self.ordered_list) + + Thus the files are only all rebuilt if the specified ordered list + is changed. .. _within_gallery_order: @@ -429,9 +454,12 @@ displays a comment along-side each the code shown above. Which is achieved by the following configuration:: - 'first_notebook_cell': ("# This cell is added by sphinx-gallery\n" - "# It can be customized to whatever you like\n" - "%matplotlib inline") + sphinx_gallery_conf = { + ... + 'first_notebook_cell': ("# This cell is added by sphinx-gallery\n" + "# It can be customized to whatever you like\n" + "%matplotlib inline") + } If the value of ``first_notebook_cell`` is set to ``None``, then no extra first cell will be added to the notebook. @@ -470,6 +498,28 @@ For more information on CircleCI integration, peruse the related `CircleCI doc `__ and `blog post `__. + +.. _log_level: + +Setting log level +================= + +Sphinx-Gallery logs output at several stages. Warnings can be generated for +code that requires case sensitivity (e.g., ``plt.subplot`` and ``plt.Subplot``) +when building docs on a filesystem that does not support case sensitive +naming (e.g., Windows). In this case, by default a ``logger.warning`` is +emitted, which will lead to a build failure when buidling with ``-W``. +The log level can be set with:: + + sphinx_gallery_conf = { + ... + 'log_level': {'backreference_missing': 'warning'}, + } + +The only valid key currently is ``backreference_missing``. +The valid values are ``'debug'``, ``'info'``, ``'warning'``, and ``'error'``. + + .. _disable_all_scripts_download: Disabling download button of all scripts @@ -549,7 +599,7 @@ If a Sphinx-Gallery configuration for Binder is discovered, the following extra 1. The dependency files specified in ``dependencies`` will be copied to a ``binder/`` folder in your built documentation. 2. The built Jupyter Notebooks from the documentation will be copied to a folder called ```` at the root of your built documentation (they will follow the same folder hierarchy within the notebooks directory folder. -2. The rST output of each Sphinx-Gallery example will now have a ``launch binder`` button in it. +3. The rST output of each Sphinx-Gallery example will now have a ``launch binder`` button in it. 4. That button will point to a binder link with the following structure:: /v2/gh///?filepath=//path/to/notebook.ipynb @@ -646,8 +696,6 @@ figures. This behavior is equivalent to the default of:: Built-in support is also provided for finding :mod:`Mayavi ` figures. Enable this feature with the following configuration:: -.. code-block:: python - sphinx_gallery_conf = { ... 'image_scrapers': ('matplotlib', 'mayavi'), @@ -771,6 +819,9 @@ Here you list the examples you allow to fail during the build process, keep in mind to specify the full relative path from your `conf.py` to the example script. +.. note:: If an example is expected to fail, sphinx-gallery will error if + the example runs without error. + .. _setting_thumbnail_size: diff --git a/examples/local_module.py b/examples/local_module.py index 392a5f3d3..2c3535e83 100644 --- a/examples/local_module.py +++ b/examples/local_module.py @@ -1,4 +1,7 @@ """ +Local module +============ + A trivial local module to provide a value for plot_exp.py. """ diff --git a/examples/plot_function_identifier.py b/examples/plot_function_identifier.py index b40996907..de069e2c7 100644 --- a/examples/plot_function_identifier.py +++ b/examples/plot_function_identifier.py @@ -12,10 +12,10 @@ import os # noqa, analysis:ignore import matplotlib.pyplot as plt -import sphinx_gallery.backreferences as spback +from sphinx_gallery.backreferences import identify_names -filename = spback.__file__.replace('.pyc', '.py') -names = spback.identify_names(filename) +filename = os.__file__.replace('.pyc', '.py') +names = identify_names(filename) figheight = len(names) + .5 fontsize = 20 diff --git a/sphinx_gallery/backreferences.py b/sphinx_gallery/backreferences.py index ecd02db7e..80e500ef5 100644 --- a/sphinx_gallery/backreferences.py +++ b/sphinx_gallery/backreferences.py @@ -11,9 +11,13 @@ import ast import codecs +import collections import os import re +from . import sphinx_compatibility +from .utils import _replace_md5 + # Try Python 2 first, otherwise load from Python 3 try: import cPickle as pickle @@ -129,7 +133,7 @@ def identify_names(filename): names = list(finder.get_mapping()) names += extract_object_names_from_docs(filename) - example_code_obj = {} + example_code_obj = collections.OrderedDict() for name, full_name in names: if name in example_code_obj: continue # if someone puts it in the docstring and code @@ -154,9 +158,10 @@ def scan_used_functions(example_file, gallery_conf): """save variables so we can later add links to the documentation""" example_code_obj = identify_names(example_file) if example_code_obj: - codeobj_fname = example_file[:-3] + '_codeobj.pickle' + codeobj_fname = example_file[:-3] + '_codeobj.pickle.new' with open(codeobj_fname, 'wb') as fid: pickle.dump(example_code_obj, fid, pickle.HIGHEST_PROTOCOL) + _replace_md5(codeobj_fname) backrefs = set('{module_short}.{name}'.format(**entry) for entry in example_code_obj.values() @@ -216,7 +221,7 @@ def write_backreferences(seen_backrefs, gallery_conf, for backref in backrefs: include_path = os.path.join(gallery_conf['src_dir'], gallery_conf['backreferences_dir'], - '%s.examples' % backref) + '%s.examples.new' % backref) seen = backref in seen_backrefs with codecs.open(include_path, 'a' if seen else 'w', encoding='utf-8') as ex_file: @@ -227,3 +232,24 @@ def write_backreferences(seen_backrefs, gallery_conf, ex_file.write(_thumbnail_div(build_target_dir, fname, snippet, is_backref=True)) seen_backrefs.add(backref) + + +def finalize_backreferences(seen_backrefs, gallery_conf): + """Replace backref files only if necessary.""" + logger = sphinx_compatibility.getLogger('sphinx-gallery') + if gallery_conf['backreferences_dir'] is None: + return + + for backref in seen_backrefs: + path = os.path.join(gallery_conf['src_dir'], + gallery_conf['backreferences_dir'], + '%s.examples.new' % backref) + if os.path.isfile(path): + _replace_md5(path) + else: + level = gallery_conf['log_level'].get('backreference_missing', + 'warning') + func = getattr(logger, level) + func('Could not find backreferences file: %s' % (path,)) + func('The backreferences are likely to be erroneous ' + 'due to file system case insensitivity.') diff --git a/sphinx_gallery/binder.py b/sphinx_gallery/binder.py index 64056b81d..e86cabb88 100644 --- a/sphinx_gallery/binder.py +++ b/sphinx_gallery/binder.py @@ -38,7 +38,7 @@ def gen_binder_url(fpath, binder_conf, gallery_conf): ---------- fpath: str The path to the `.py` file for which a Binder badge will be generated. - binder_conf: dict | None + binder_conf: dict or None The Binder configuration dictionary. See `gen_binder_rst` for details. Returns @@ -85,7 +85,7 @@ def gen_binder_rst(fpath, binder_conf, gallery_conf): ---------- fpath: str The path to the `.py` file for which a Binder badge will be generated. - binder_conf: dict | None + binder_conf: dict or None If a dictionary it must have the following keys: 'binderhub_url': The URL of the BinderHub instance that's running a Binder @@ -204,7 +204,7 @@ def check_binder_conf(binder_conf): raise ValueError('`binder_conf` must be a dictionary or None.') if len(binder_conf) == 0: return binder_conf - + if binder_conf.get('url') and not binder_conf.get('binderhub_url'): logger.warning( 'Found old BinderHub URL keyword ("url"). Please update your ' diff --git a/sphinx_gallery/docs_resolv.py b/sphinx_gallery/docs_resolv.py index 12061690e..43090335e 100644 --- a/sphinx_gallery/docs_resolv.py +++ b/sphinx_gallery/docs_resolv.py @@ -240,7 +240,7 @@ def resolve(self, cobj, this_url): Returns ------- - link : str | None + link : str or None The link (URL) to the documentation. """ full_name = cobj['module_short'] + '.' + cobj['name'] diff --git a/sphinx_gallery/downloads.py b/sphinx_gallery/downloads.py index 69bd32e17..a80a8e458 100644 --- a/sphinx_gallery/downloads.py +++ b/sphinx_gallery/downloads.py @@ -12,6 +12,8 @@ import os import zipfile +from .utils import _replace_md5 + CODE_DOWNLOAD = """ .. _sphx_glr_download_{3}: @@ -49,7 +51,7 @@ def python_zip(file_list, gallery_path, extension='.py'): Parameters ---------- - file_list : list of strings + file_list : list Holds all the file names to be included in zip file gallery_path : str path to where the zipfile is stored @@ -67,13 +69,12 @@ def python_zip(file_list, gallery_path, extension='.py'): zipname = os.path.basename(os.path.normpath(gallery_path)) zipname += '_python' if extension == '.py' else '_jupyter' zipname = os.path.join(gallery_path, zipname + '.zip') - - zipf = zipfile.ZipFile(zipname, mode='w') - for fname in file_list: - file_src = os.path.splitext(fname)[0] + extension - zipf.write(file_src, os.path.relpath(file_src, gallery_path)) - zipf.close() - + zipname_new = zipname + '.new' + with zipfile.ZipFile(zipname_new, mode='w') as zipf: + for fname in file_list: + file_src = os.path.splitext(fname)[0] + extension + zipf.write(file_src, os.path.relpath(file_src, gallery_path)) + _replace_md5(zipname_new) return zipname diff --git a/sphinx_gallery/gen_gallery.py b/sphinx_gallery/gen_gallery.py index 8e5e6f7ef..e1b7d7891 100644 --- a/sphinx_gallery/gen_gallery.py +++ b/sphinx_gallery/gen_gallery.py @@ -20,6 +20,8 @@ from sphinx.util.console import red from . import sphinx_compatibility, glr_path_static, __version__ as _sg_version +from .utils import _replace_md5 +from .backreferences import finalize_backreferences from .gen_rst import (generate_dir_rst, SPHX_GLR_SIG, _get_memory_base, extract_intro_and_title, get_docstring_and_rest) from .scrapers import _scraper_dict, _reset_dict @@ -70,36 +72,12 @@ 'first_notebook_cell': '%matplotlib inline', 'show_memory': False, 'junit': '', + 'log_level': {'backreference_missing': 'warning'}, } logger = sphinx_compatibility.getLogger('sphinx-gallery') -def clean_gallery_out(build_dir): - """Deletes images under the sphx_glr namespace in the build directory""" - # Sphinx hack: sphinx copies generated images to the build directory - # each time the docs are made. If the desired image name already - # exists, it appends a digit to prevent overwrites. The problem is, - # the directory is never cleared. This means that each time you build - # the docs, the number of images in the directory grows. - # - # This question has been asked on the sphinx development list, but there - # was no response: https://git.net/ml/sphinx-dev/2011-02/msg00123.html - # - # The following is a hack that prevents this behavior by clearing the - # image build directory from gallery images each time the docs are built. - # If sphinx changes their layout between versions, this will not - # work (though it should probably not cause a crash). - # Tested successfully on Sphinx 1.0.7 - - build_image_dir = os.path.join(build_dir, '_images') - if os.path.exists(build_image_dir): - filelist = os.listdir(build_image_dir) - for filename in filelist: - if filename.startswith('sphx_glr') and filename.endswith('png'): - os.remove(os.path.join(build_image_dir, filename)) - - def parse_config(app): """Process the Sphinx Gallery configuration""" try: @@ -199,7 +177,7 @@ def _complete_gallery_conf(sphinx_gallery_conf, src_dir, plot_gallery, def get_subsections(srcdir, examples_dir, sortkey): - """Returns the list of subsections of a gallery + """Return the list of subsections of a gallery Parameters ---------- @@ -257,8 +235,6 @@ def generate_gallery_rst(app): logger.info('generating gallery...', color='white') gallery_conf = parse_config(app) - clean_gallery_out(app.builder.outdir) - seen_backrefs = set() computation_times = [] @@ -292,8 +268,8 @@ def generate_gallery_rst(app): this_computation_times) # we create an index.rst with all examples - with codecs.open(os.path.join(gallery_dir, 'index.rst'), 'w', - encoding='utf-8') as fhindex: + index_rst_new = os.path.join(gallery_dir, 'index.rst.new') + with codecs.open(index_rst_new, 'w', encoding='utf-8') as fhindex: # :orphan: to suppress "not included in TOCTREE" sphinx warnings fhindex.write(":orphan:\n\n" + this_fhindex) @@ -315,6 +291,8 @@ def generate_gallery_rst(app): fhindex.write(download_fhindex) fhindex.write(SPHX_GLR_SIG) + _replace_md5(index_rst_new) + finalize_backreferences(seen_backrefs, gallery_conf) if gallery_conf['plot_gallery']: logger.info("computation time summary:", color='white') @@ -354,7 +332,7 @@ def _sec_to_readable(t): def write_computation_times(gallery_conf, target_dir, computation_times): - if not gallery_conf['plot_gallery']: + if all(time[0] == 0 for time in computation_times): return target_dir_clean = os.path.relpath( target_dir, gallery_conf['src_dir']).replace(os.path.sep, '_') @@ -586,11 +564,11 @@ def setup(app): app.connect('autodoc-process-docstring', touch_empty_backreferences) app.connect('builder-inited', generate_gallery_rst) - app.connect('build-finished', copy_binder_files) app.connect('build-finished', summarize_failing_examples) app.connect('build-finished', embed_code_links) metadata = {'parallel_read_safe': True, + 'parallel_write_safe': False, 'version': _sg_version} return metadata diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py index 6f932d72e..c9518bb69 100644 --- a/sphinx_gallery/gen_rst.py +++ b/sphinx_gallery/gen_rst.py @@ -17,11 +17,9 @@ import copy import ast import codecs -import hashlib import gc import os import re -import shutil import subprocess import sys import traceback @@ -30,7 +28,7 @@ from distutils.version import LooseVersion from .scrapers import save_figures, ImagePathIterator, clean_modules -from .utils import replace_py_ipynb, scale_image +from .utils import replace_py_ipynb, scale_image, get_md5sum, _replace_md5 # Try Python 2 first, otherwise load from Python 3 try: @@ -224,16 +222,6 @@ def extract_intro_and_title(filename, docstring): return intro, title -def get_md5sum(src_file): - """Returns md5sum of file""" - - with open(src_file, 'rb') as src_data: - src_content = src_data.read() - - src_md5 = hashlib.md5(src_content).hexdigest() - return src_md5 - - def md5sum_is_current(src_file): """Checks whether src_file has the same md5 hash as the one on disk""" @@ -614,7 +602,7 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf): """ src_file = os.path.normpath(os.path.join(src_dir, fname)) target_file = os.path.join(target_dir, fname) - shutil.copyfile(src_file, target_file) + _replace_md5(src_file, target_file, 'copy') intro, _ = extract_intro_and_title(fname, get_docstring_and_rest(src_file)[0]) @@ -650,13 +638,17 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf): example_rst = rst_blocks(script_blocks, output_blocks, file_conf, gallery_conf) memory_used = gallery_conf['memory_base'] + script_vars['memory_delta'] + if not executable: + time_elapsed = memory_used = 0. # don't let the output change save_rst_example(example_rst, target_file, time_elapsed, memory_used, gallery_conf) save_thumbnail(image_path_template, src_file, file_conf, gallery_conf) example_nb = jupyter_notebook(script_blocks, gallery_conf) - save_notebook(example_nb, replace_py_ipynb(target_file)) + ipy_fname = replace_py_ipynb(target_file) + '.new' + save_notebook(example_nb, ipy_fname) + _replace_md5(ipy_fname) return intro, time_elapsed @@ -714,7 +706,7 @@ def rst_blocks(script_blocks, output_blocks, file_conf, gallery_conf): def save_rst_example(example_rst, example_file, time_elapsed, memory_used, gallery_conf): - """Saves the rst notebook to example_file including necessary header & footer + """Saves the rst notebook to example_file including header & footer Parameters ---------- @@ -765,6 +757,9 @@ def save_rst_example(example_rst, example_file, time_elapsed, ref_fname) example_rst += SPHX_GLR_SIG - write_file = re.sub(r'\.py$', '.rst', example_file) - with codecs.open(write_file, 'w', encoding="utf-8") as f: + write_file_new = re.sub(r'\.py$', '.rst.new', example_file) + with codecs.open(write_file_new, 'w', encoding="utf-8") as f: f.write(example_rst) + # in case it wasn't in our pattern, only replace the file if it's + # still stale. + _replace_md5(write_file_new) diff --git a/sphinx_gallery/notebook.py b/sphinx_gallery/notebook.py index 947109d1f..036da5adf 100644 --- a/sphinx_gallery/notebook.py +++ b/sphinx_gallery/notebook.py @@ -160,7 +160,8 @@ def fill_notebook(work_notebook, script_blocks): Parameters ---------- - script_blocks : list of tuples + script_blocks : list + Each list element should be a tuple of (label, content, lineno). """ for blabel, bcontent, lineno in script_blocks: diff --git a/sphinx_gallery/py_source_parser.py b/sphinx_gallery/py_source_parser.py index 0c13bbb54..66536f13b 100644 --- a/sphinx_gallery/py_source_parser.py +++ b/sphinx_gallery/py_source_parser.py @@ -15,6 +15,10 @@ import tokenize from textwrap import dedent +from .sphinx_compatibility import getLogger + +logger = getLogger('sphinx-gallery') + SYNTAX_ERROR_DOCSTRING = """ SyntaxError =========== diff --git a/sphinx_gallery/scrapers.py b/sphinx_gallery/scrapers.py index ad418ebb9..1b4a035b2 100644 --- a/sphinx_gallery/scrapers.py +++ b/sphinx_gallery/scrapers.py @@ -72,8 +72,7 @@ def matplotlib_scraper(block, block_vars, gallery_conf): ------- rst : str The ReSTructuredText that will be rendered to HTML containing - the images. This is often produced by - :func:`sphinx_gallery.gen_rst.figure_rst`. + the images. This is often produced by :func:`figure_rst`. """ matplotlib, plt = _import_matplotlib() image_path_iterator = block_vars['image_path_iterator'] @@ -111,8 +110,7 @@ def mayavi_scraper(block, block_vars, gallery_conf): ------- rst : str The ReSTructuredText that will be rendered to HTML containing - the images. This is often produced by - :func:`sphinx_gallery.gen_rst.figure_rst`. + the images. This is often produced by :func:`figure_rst`. """ from mayavi import mlab image_path_iterator = block_vars['image_path_iterator'] @@ -232,8 +230,8 @@ def figure_rst(figure_list, sources_dir): Parameters ---------- - figure_list : list of str - Strings are the figures' absolute paths + figure_list : list + List of strings of the figures' absolute paths. sources_dir : str absolute path of Sphinx documentation sources @@ -315,7 +313,7 @@ def clean_modules(gallery_conf, fname): ---------- gallery_conf : dict The gallery configuration. - fname : str | None + fname : str or None The example being run. Will be None when this is called entering a directory of examples to be built. """ diff --git a/sphinx_gallery/sorting.py b/sphinx_gallery/sorting.py index 007bf08bf..2d9be7da1 100644 --- a/sphinx_gallery/sorting.py +++ b/sphinx_gallery/sorting.py @@ -24,7 +24,7 @@ class ExplicitOrder(object): Parameters ---------- - ordered_list : list, tuple, types.GeneratorType + ordered_list : list, tuple, or generator Hold the paths of each galleries' subsections. Raises @@ -50,6 +50,9 @@ def __call__(self, item): 'must specify all folders. Explicit order not ' 'found for {}'.format(item)) + def __repr__(self): + return '<%s : %s>' % (self.__class__.__name__, self.ordered_list) + class _SortKey(object): """Base class for section order key classes.""" @@ -57,6 +60,9 @@ class _SortKey(object): def __init__(self, src_dir): self.src_dir = src_dir + def __repr__(self): + return '<%s>' % (self.__class__.__name__,) + class NumberOfCodeLinesSortKey(_SortKey): """Sort examples in src_dir by the number of code lines. @@ -87,7 +93,7 @@ class FileSizeSortKey(_SortKey): def __call__(self, filename): src_file = os.path.normpath(os.path.join(self.src_dir, filename)) - return os.stat(src_file).st_size + return int(os.stat(src_file).st_size) class FileNameSortKey(_SortKey): diff --git a/sphinx_gallery/tests/test_full.py b/sphinx_gallery/tests/test_full.py index 308acd0f9..5e3f7873e 100644 --- a/sphinx_gallery/tests/test_full.py +++ b/sphinx_gallery/tests/test_full.py @@ -10,9 +10,15 @@ from distutils.version import LooseVersion import os import os.path as op +import re import shutil import sys +import time +import numpy as np +from numpy.testing import assert_allclose + +import sphinx from sphinx.application import Sphinx from sphinx.util.docutils import docutils_namespace from sphinx_gallery.gen_rst import MixedEncodingStringIO @@ -175,3 +181,232 @@ def test_backreferences(sphinx_app): lines = fid.read() assert 'identify_names' in lines # in API doc assert 'plot_future_imports.html' in lines # backref via doc block + + +def _assert_mtimes(list_orig, list_new, different=(), ignore=()): + assert ([op.basename(x) for x in list_orig] == + [op.basename(x) for x in list_new]) + for orig, new in zip(list_orig, list_new): + if op.basename(orig) in different: + assert np.abs(op.getmtime(orig) - op.getmtime(new)) > 0.1 + elif op.basename(orig) not in ignore: + assert_allclose(op.getmtime(orig), op.getmtime(new), + atol=1e-3, rtol=1e-20, err_msg=op.basename(orig)) + + +def test_rebuild(tmpdir_factory, sphinx_app): + # Make sure that examples that haven't been changed aren't run twice. + + # + # First run completes in the fixture. + # + status = sphinx_app._status.getvalue() + assert re.match('.*16 added, 0 changed, 0 removed$.*', + status, re.MULTILINE | re.DOTALL) + assert re.match('.*targets for 1 source files that are out of date$.*', + status, re.MULTILINE | re.DOTALL) + assert re.match('.*executed 3 out of 4.*' + 'after excluding 0 files.*based on MD5.*', + status, re.MULTILINE | re.DOTALL) + old_src_dir = (tmpdir_factory.getbasetemp() / 'root_old').strpath + shutil.copytree(sphinx_app.srcdir, old_src_dir) + generated_modules_0 = sorted( + op.join(old_src_dir, 'gen_modules', f) + for f in os.listdir(op.join(old_src_dir, 'gen_modules')) + if op.isfile(op.join(old_src_dir, 'gen_modules', f))) + generated_backrefs_0 = sorted( + op.join(old_src_dir, 'gen_modules', 'backreferences', f) + for f in os.listdir(op.join(old_src_dir, 'gen_modules', + 'backreferences'))) + generated_rst_0 = sorted( + op.join(old_src_dir, 'auto_examples', f) + for f in os.listdir(op.join(old_src_dir, 'auto_examples')) + if f.endswith('.rst')) + generated_pickle_0 = sorted( + op.join(old_src_dir, 'auto_examples', f) + for f in os.listdir(op.join(old_src_dir, 'auto_examples')) + if f.endswith('.pickle')) + copied_py_0 = sorted( + op.join(old_src_dir, 'auto_examples', f) + for f in os.listdir(op.join(old_src_dir, 'auto_examples')) + if f.endswith('.py')) + copied_ipy_0 = sorted( + op.join(old_src_dir, 'auto_examples', f) + for f in os.listdir(op.join(old_src_dir, 'auto_examples')) + if f.endswith('.ipynb')) + assert len(generated_modules_0) > 0 + assert len(generated_backrefs_0) > 0 + assert len(generated_rst_0) > 0 + assert len(generated_pickle_0) > 0 + assert len(copied_py_0) > 0 + assert len(copied_ipy_0) > 0 + assert len(sphinx_app.config.sphinx_gallery_conf['stale_examples']) == 0 + assert op.isfile(op.join(sphinx_app.outdir, '_images', + 'sphx_glr_plot_numpy_matplotlib_001.png')) + + # + # run a second time, no files should be updated + # + + src_dir = sphinx_app.srcdir + del sphinx_app # don't accidentally use it below + conf_dir = src_dir + out_dir = op.join(src_dir, '_build', 'html') + toctrees_dir = op.join(src_dir, '_build', 'toctrees') + time.sleep(0.1) + with docutils_namespace(): + new_app = Sphinx(src_dir, conf_dir, out_dir, toctrees_dir, + buildername='html', status=MixedEncodingStringIO()) + new_app.build(False, []) + status = new_app._status.getvalue() + lines = [line for line in status.split('\n') if '0 removed' in line] + assert re.match('.*0 added, [3|6] changed, 0 removed$.*', + status, re.MULTILINE | re.DOTALL), lines + assert re.match('.*executed 0 out of 1.*' + 'after excluding 3 files.*based on MD5.*', + status, re.MULTILINE | re.DOTALL) + assert len(new_app.config.sphinx_gallery_conf['stale_examples']) == 3 + assert op.isfile(op.join(new_app.outdir, '_images', + 'sphx_glr_plot_numpy_matplotlib_001.png')) + + generated_modules_1 = sorted( + op.join(new_app.srcdir, 'gen_modules', f) + for f in os.listdir(op.join(new_app.srcdir, 'gen_modules')) + if op.isfile(op.join(new_app.srcdir, 'gen_modules', f))) + generated_backrefs_1 = sorted( + op.join(new_app.srcdir, 'gen_modules', 'backreferences', f) + for f in os.listdir(op.join(new_app.srcdir, 'gen_modules', + 'backreferences'))) + generated_rst_1 = sorted( + op.join(new_app.srcdir, 'auto_examples', f) + for f in os.listdir(op.join(new_app.srcdir, 'auto_examples')) + if f.endswith('.rst')) + generated_pickle_1 = sorted( + op.join(new_app.srcdir, 'auto_examples', f) + for f in os.listdir(op.join(new_app.srcdir, 'auto_examples')) + if f.endswith('.pickle')) + copied_py_1 = sorted( + op.join(new_app.srcdir, 'auto_examples', f) + for f in os.listdir(op.join(new_app.srcdir, 'auto_examples')) + if f.endswith('.py')) + copied_ipy_1 = sorted( + op.join(new_app.srcdir, 'auto_examples', f) + for f in os.listdir(op.join(new_app.srcdir, 'auto_examples')) + if f.endswith('.ipynb')) + + # mtimes for modules + _assert_mtimes(generated_modules_0, generated_modules_1) + + # mtimes for backrefs (gh-394) + _assert_mtimes(generated_backrefs_0, generated_backrefs_1) + + # generated RST files + different = ( + # this one should get rewritten as we retried it + 'plot_future_imports_broken.rst', + ) + ignore = ( + # this one should almost always be different, but in case we + # get extremely unlucky and have identical run times + # on the one script above that changes... + 'sg_execution_times.rst', + ) + _assert_mtimes(generated_rst_0, generated_rst_1, different, ignore) + + # mtimes for pickles + _assert_mtimes(generated_pickle_0, generated_pickle_1) + + # mtimes for .py files (gh-395) + _assert_mtimes(copied_py_0, copied_py_1) + + # mtimes for .ipynb files + _assert_mtimes(copied_ipy_0, copied_ipy_1) + + # + # run a third time, changing one file + # + + time.sleep(0.1) + fname = op.join(src_dir, 'examples', 'plot_numpy_matplotlib.py') + with open(fname, 'r') as fid: + lines = fid.readlines() + with open(fname, 'w') as fid: + for line in lines: + if line.startswith('FYI this'): + line = 'A ' + line + fid.write(line) + with docutils_namespace(): + new_app = Sphinx(src_dir, conf_dir, out_dir, toctrees_dir, + buildername='html', status=MixedEncodingStringIO()) + new_app.build(False, []) + status = new_app._status.getvalue() + if LooseVersion(sphinx.__version__) <= LooseVersion('1.6'): + n = 16 + else: + n = '[2|3]' + lines = [line for line in status.split('\n') if 'source files tha' in line] + assert re.match('.*targets for %s source files that are out of date$.*' + % n, status, re.MULTILINE | re.DOTALL), lines + assert re.match('.*executed 1 out of 2.*' + 'after excluding 2 files.*based on MD5.*', + status, re.MULTILINE | re.DOTALL) + assert len(new_app.config.sphinx_gallery_conf['stale_examples']) == 2 + assert op.isfile(op.join(new_app.outdir, '_images', + 'sphx_glr_plot_numpy_matplotlib_001.png')) + + generated_modules_1 = sorted( + op.join(new_app.srcdir, 'gen_modules', f) + for f in os.listdir(op.join(new_app.srcdir, 'gen_modules')) + if op.isfile(op.join(new_app.srcdir, 'gen_modules', f))) + generated_backrefs_1 = sorted( + op.join(new_app.srcdir, 'gen_modules', 'backreferences', f) + for f in os.listdir(op.join(new_app.srcdir, 'gen_modules', + 'backreferences'))) + generated_rst_1 = sorted( + op.join(new_app.srcdir, 'auto_examples', f) + for f in os.listdir(op.join(new_app.srcdir, 'auto_examples')) + if f.endswith('.rst')) + generated_pickle_1 = sorted( + op.join(new_app.srcdir, 'auto_examples', f) + for f in os.listdir(op.join(new_app.srcdir, 'auto_examples')) + if f.endswith('.pickle')) + copied_py_1 = sorted( + op.join(new_app.srcdir, 'auto_examples', f) + for f in os.listdir(op.join(new_app.srcdir, 'auto_examples')) + if f.endswith('.py')) + copied_ipy_1 = sorted( + op.join(new_app.srcdir, 'auto_examples', f) + for f in os.listdir(op.join(new_app.srcdir, 'auto_examples')) + if f.endswith('.ipynb')) + + # mtimes for modules + _assert_mtimes(generated_modules_0, generated_modules_1) + + # mtimes for backrefs (gh-394) + _assert_mtimes(generated_backrefs_0, generated_backrefs_1) + + # generated RST files + different = ( + # this one should get rewritten as we retried it + 'plot_future_imports_broken.rst', + 'plot_numpy_matplotlib.rst', + ) + ignore = ( + # this one should almost always be different, but in case we + # get extremely unlucky and have identical run times + # on the one script above that changes... + 'sg_execution_times.rst', + ) + _assert_mtimes(generated_rst_0, generated_rst_1, different, ignore) + + # mtimes for pickles + _assert_mtimes(generated_pickle_0, generated_pickle_1, + different=('plot_numpy_matplotlib.codeobj.pickle')) + + # mtimes for .py files (gh-395) + _assert_mtimes(copied_py_0, copied_py_1, + different=('plot_numpy_matplotlib.py')) + + # mtimes for .ipynb files + _assert_mtimes(copied_ipy_0, copied_ipy_1, + different=('plot_numpy_matplotlib.ipynb')) diff --git a/sphinx_gallery/tests/test_sorting.py b/sphinx_gallery/tests/test_sorting.py index d6a866d3a..8d0aeb874 100644 --- a/sphinx_gallery/tests/test_sorting.py +++ b/sphinx_gallery/tests/test_sorting.py @@ -8,12 +8,22 @@ # License: 3-clause BSD from __future__ import division, absolute_import, print_function +import os.path as op import pytest +from sphinx_gallery.sorting import (ExplicitOrder, NumberOfCodeLinesSortKey, + FileNameSortKey, FileSizeSortKey, + ExampleTitleSortKey) + + +try: + basestring +except NameError: + basestring = str + def test_ExplicitOrder_sorting_key(): """Test ExplicitOrder""" - from sphinx_gallery.sorting import ExplicitOrder all_folders = ['e', 'f', 'd', 'c', '01b', 'a'] explicit_folders = ['f', 'd'] @@ -30,3 +40,17 @@ def test_ExplicitOrder_sorting_key(): with pytest.raises(ValueError) as excinfo: sorted_folders = sorted(all_folders, key=key) excinfo.match('If you use an explicit folder ordering') + + # str(obj) stability for sphinx non-rebuilds + assert str(key).startswith('' % (klass.__name__,) + out = sorter(op.basename(__file__)) + assert isinstance(out, type_), type(out) diff --git a/sphinx_gallery/utils.py b/sphinx_gallery/utils.py index 3e7b698d7..e88e54644 100644 --- a/sphinx_gallery/utils.py +++ b/sphinx_gallery/utils.py @@ -10,9 +10,10 @@ from __future__ import division, absolute_import, print_function -import tempfile -from shutil import rmtree +import hashlib import os +from shutil import rmtree, move, copyfile +import tempfile class _TempDir(str): @@ -90,3 +91,27 @@ def replace_py_ipynb(fname): % (allowed_extension, extension)) new_extension = '.ipynb' return '{}{}'.format(fname_prefix, new_extension) + + +def get_md5sum(src_file): + """Returns md5sum of file""" + with open(src_file, 'rb') as src_data: + src_content = src_data.read() + return hashlib.md5(src_content).hexdigest() + + +def _replace_md5(fname_new, fname_old=None, method='move'): + assert method in ('move', 'copy') + if fname_old is None: + assert fname_new.endswith('.new') + fname_old = fname_new[:-4] + if os.path.isfile(fname_old) and (get_md5sum(fname_old) == + get_md5sum(fname_new)): + if method == 'move': + os.remove(fname_new) + else: + if method == 'move': + move(fname_new, fname_old) + else: + copyfile(fname_new, fname_old) + assert os.path.isfile(fname_old) From 12a8374b670bf44cb2e30bf6c4b89966a06b5d9c Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Sat, 2 Mar 2019 22:16:56 -0500 Subject: [PATCH 2/4] FIX: More tolerant --- sphinx_gallery/tests/test_full.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/sphinx_gallery/tests/test_full.py b/sphinx_gallery/tests/test_full.py index 5e3f7873e..2ddd18ae1 100644 --- a/sphinx_gallery/tests/test_full.py +++ b/sphinx_gallery/tests/test_full.py @@ -202,12 +202,12 @@ def test_rebuild(tmpdir_factory, sphinx_app): # status = sphinx_app._status.getvalue() assert re.match('.*16 added, 0 changed, 0 removed$.*', - status, re.MULTILINE | re.DOTALL) + status, re.MULTILINE | re.DOTALL) is not None assert re.match('.*targets for 1 source files that are out of date$.*', - status, re.MULTILINE | re.DOTALL) + status, re.MULTILINE | re.DOTALL) is not None assert re.match('.*executed 3 out of 4.*' 'after excluding 0 files.*based on MD5.*', - status, re.MULTILINE | re.DOTALL) + status, re.MULTILINE | re.DOTALL) is not None old_src_dir = (tmpdir_factory.getbasetemp() / 'root_old').strpath shutil.copytree(sphinx_app.srcdir, old_src_dir) generated_modules_0 = sorted( @@ -260,11 +260,11 @@ def test_rebuild(tmpdir_factory, sphinx_app): new_app.build(False, []) status = new_app._status.getvalue() lines = [line for line in status.split('\n') if '0 removed' in line] - assert re.match('.*0 added, [3|6] changed, 0 removed$.*', - status, re.MULTILINE | re.DOTALL), lines + assert re.match('.*0 added, [2|3|6|7] changed, 0 removed$.*', + status, re.MULTILINE | re.DOTALL) is not None, lines assert re.match('.*executed 0 out of 1.*' 'after excluding 3 files.*based on MD5.*', - status, re.MULTILINE | re.DOTALL) + status, re.MULTILINE | re.DOTALL) is not None assert len(new_app.config.sphinx_gallery_conf['stale_examples']) == 3 assert op.isfile(op.join(new_app.outdir, '_images', 'sphx_glr_plot_numpy_matplotlib_001.png')) @@ -346,10 +346,10 @@ def test_rebuild(tmpdir_factory, sphinx_app): n = '[2|3]' lines = [line for line in status.split('\n') if 'source files tha' in line] assert re.match('.*targets for %s source files that are out of date$.*' - % n, status, re.MULTILINE | re.DOTALL), lines + % n, status, re.MULTILINE | re.DOTALL) is not None, lines assert re.match('.*executed 1 out of 2.*' 'after excluding 2 files.*based on MD5.*', - status, re.MULTILINE | re.DOTALL) + status, re.MULTILINE | re.DOTALL) is not None assert len(new_app.config.sphinx_gallery_conf['stale_examples']) == 2 assert op.isfile(op.join(new_app.outdir, '_images', 'sphx_glr_plot_numpy_matplotlib_001.png')) From be65cde2745e72d3d5fcac4a767e099e65c1c2bd Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Sun, 3 Mar 2019 23:21:22 -0500 Subject: [PATCH 3/4] STY: Splitext --- setup.cfg | 2 +- sphinx_gallery/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index b280b07bd..bad2a05f5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [tool:pytest] -addopts = --cov-report= --cov=sphinx_gallery --durations=5 -ra +addopts = --cov-report= --cov=sphinx_gallery --durations=5 -ra --showlocals python_files = tests/*.py norecursedirs = build _build auto_examples gen_modules sphinx_gallery/tests/tinybuild diff --git a/sphinx_gallery/utils.py b/sphinx_gallery/utils.py index e88e54644..60f0ccf85 100644 --- a/sphinx_gallery/utils.py +++ b/sphinx_gallery/utils.py @@ -104,7 +104,7 @@ def _replace_md5(fname_new, fname_old=None, method='move'): assert method in ('move', 'copy') if fname_old is None: assert fname_new.endswith('.new') - fname_old = fname_new[:-4] + fname_old = os.path.splitext(fname_new)[0] if os.path.isfile(fname_old) and (get_md5sum(fname_old) == get_md5sum(fname_new)): if method == 'move': From dd70a47b96076f2bb836ab648611bfbde8e54d56 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 4 Mar 2019 07:23:42 -0500 Subject: [PATCH 4/4] FIX: Heisenbug --- sphinx_gallery/tests/test_full.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/sphinx_gallery/tests/test_full.py b/sphinx_gallery/tests/test_full.py index 2ddd18ae1..e142b7639 100644 --- a/sphinx_gallery/tests/test_full.py +++ b/sphinx_gallery/tests/test_full.py @@ -301,17 +301,14 @@ def test_rebuild(tmpdir_factory, sphinx_app): _assert_mtimes(generated_backrefs_0, generated_backrefs_1) # generated RST files - different = ( - # this one should get rewritten as we retried it - 'plot_future_imports_broken.rst', - ) ignore = ( - # this one should almost always be different, but in case we + # these two should almost always be different, but in case we # get extremely unlucky and have identical run times - # on the one script above that changes... + # on the one script that gets re-run (because it's a fail)... 'sg_execution_times.rst', + 'plot_future_imports_broken.rst', ) - _assert_mtimes(generated_rst_0, generated_rst_1, different, ignore) + _assert_mtimes(generated_rst_0, generated_rst_1, ignore=ignore) # mtimes for pickles _assert_mtimes(generated_pickle_0, generated_pickle_1)