diff --git a/.github/workflows/update-changelog.yaml b/.github/workflows/update-changelog.yaml new file mode 100644 index 00000000..4f08d5af --- /dev/null +++ b/.github/workflows/update-changelog.yaml @@ -0,0 +1,34 @@ +# This workflow takes the GitHub release notes an updates the changelog on the +# master branch with the body of the release notes, thereby keeping a log in +# the git repo of the changes. + +name: "Update Changelog" + +on: + workflow_dispatch: + release: + types: [released] + +jobs: + update: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + ref: master + + - name: Update Changelog + uses: stefanzweifel/changelog-updater-action@v1 + with: + release-notes: ${{ github.event.release.body }} + latest-version: ${{ github.event.release.name }} + path-to-changelog: CHANGES.md + + - name: Commit updated CHANGELOG + uses: stefanzweifel/git-auto-commit-action@v4 + with: + branch: master + commit_message: Update CHANGELOG + file_pattern: CHANGES.md diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f043720c..97cd8689 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -75,7 +75,7 @@ repos: - id: isort args: ["--sp", "setup.cfg"] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v4.2.0 hooks: - id: check-ast - id: check-case-conflict diff --git a/CHANGES.md b/CHANGES.md index 3b6a6b39..470ace47 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,11 @@ +0.15 (unreleased) +----------------- + +- An updated hash library will be saved to the results directory when + generating a HTML summary page or when the `--mpl-results-always` flag is + set. A button to download this file is included in the HTML summary. + Various bugfixes, test improvements and documentation updates. [#138] + 0.14 (2022-02-09) ----------------- diff --git a/README.rst b/README.rst index ebdee082..3dd9b742 100644 --- a/README.rst +++ b/README.rst @@ -16,7 +16,7 @@ section below. Installing ---------- -This plugin is compatible with Python 2.7, and 3.6 and later, and +This plugin is compatible with Python 3.6 and later, and requires `pytest `__ and `matplotlib `__ to be installed. @@ -86,6 +86,10 @@ can either be specified via the ``--mpl-hash-library=`` command line argument, or via the ``hash_library=`` keyword argument to the ``@pytest.mark.mpl_image_compare`` decorator. +When generating a hash library, the tests will also be run as usual against the +existing hash library specified by ``--mpl-hash-library`` or the keyword argument. +However, generating baseline images will always result in the tests being skipped. + Hybrid Mode: Hashes and Images ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -225,6 +229,12 @@ test (based on the hash library) also shown in the generated summary. This option is applied automatically when generating a HTML summary. +When the ``--mpl-results-always`` option is active, and some hash +comparison tests are performed, a hash library containing all the +result hashes will also be saved to the root of the results directory. +The filename will be extracted from ``--mpl-generate-hash-library``, +``--mpl-hash-library`` or ``hash_library=`` in that order. + Base style ^^^^^^^^^^ diff --git a/RELEASING.md b/RELEASING.md index f3618c72..d91e4bce 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -2,11 +2,10 @@ To make a new release of pytest-mpl follow the following steps: -* Update the chanelog with the date of the release, and ensure that all relevant PRs have changelog entries. -* Push the chanelog to master (via a PR) -* Ensure the sdist and wheel GitHub Actions jobs succeeded on master after the merge of the changelog. +* Ensure the sdist and wheel GitHub Actions jobs succeeded on master after the last merge. * Also ensure that the tarball built has an autogenerated version number from setuptools_scm. -* Tag the new release, using the format `vX.Y.X`. -* Push the tag with `git push upstream master --follow-tags` -* Watch as GitHub actions builds the sdist and universal wheel and pushes them to PyPI for you. +* Write the release notes in the GitHub releases UI, use the autogenerated + notes and tidy up a little. +* Publish the new release, using the format `vX.Y.X`. +* Watch as GitHub actions builds the sdist and universal wheel and pushes them to PyPI for you, and updates CHANGES.md on the master branch. * Enjoy the beverage of your choosing 🍻. diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index d147e57c..b23f50a4 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -34,6 +34,7 @@ import shutil import hashlib import inspect +import logging import tempfile import warnings import contextlib @@ -54,27 +55,6 @@ {actual_path}""" -def _download_file(baseline, filename): - # Note that baseline can be a comma-separated list of URLs that we can - # then treat as mirrors - for base_url in baseline.split(','): - try: - u = urlopen(base_url + filename) - content = u.read() - except Exception as e: - warnings.warn('Downloading {0} failed: {1}'.format(base_url + filename, e)) - else: - break - else: - raise Exception("Could not download baseline image from any of the " - "available URLs") - result_dir = Path(tempfile.mkdtemp()) - filename = result_dir / 'downloaded' - with open(str(filename), 'wb') as tmpfile: - tmpfile.write(content) - return Path(filename) - - def _hash_file(in_stream): """ Hashes an already opened file. @@ -181,8 +161,6 @@ def pytest_configure(config): if generate_dir is not None: if baseline_dir is not None: warnings.warn("Ignoring --mpl-baseline-path since --mpl-generate-path is set") - if results_dir is not None and generate_dir is not None: - warnings.warn("Ignoring --mpl-result-path since --mpl-generate-path is set") if baseline_dir is not None and not baseline_dir.startswith(("https", "http")): baseline_dir = os.path.abspath(baseline_dir) @@ -283,11 +261,23 @@ def __init__(self, self.results_dir = Path(tempfile.mkdtemp(dir=self.results_dir)) self.results_dir.mkdir(parents=True, exist_ok=True) + # Decide what to call the downloadable results hash library + if self.hash_library is not None: + self.results_hash_library_name = self.hash_library.name + else: # Use the first filename encountered in a `hash_library=` kwarg + self.results_hash_library_name = None + # We need global state to store all the hashes generated over the run self._generated_hash_library = {} self._test_results = {} self._test_stats = None + # https://stackoverflow.com/questions/51737378/how-should-i-log-in-my-pytest-plugin + # turn debug prints on only if "-vv" or more passed + level = logging.DEBUG if config.option.verbose > 1 else logging.INFO + logging.basicConfig(level=level) + self.logger = logging.getLogger('pytest-mpl') + def get_compare(self, item): """ Return the mpl_image_compare marker for the given item. @@ -360,6 +350,26 @@ def get_baseline_directory(self, item): return baseline_dir + def _download_file(self, baseline, filename): + # Note that baseline can be a comma-separated list of URLs that we can + # then treat as mirrors + for base_url in baseline.split(','): + try: + u = urlopen(base_url + filename) + content = u.read() + except Exception as e: + self.logger.info(f'Downloading {base_url + filename} failed: {repr(e)}') + else: + break + else: + raise Exception("Could not download baseline image from any of the " + "available URLs") + result_dir = Path(tempfile.mkdtemp()) + filename = result_dir / 'downloaded' + with open(str(filename), 'wb') as tmpfile: + tmpfile.write(content) + return Path(filename) + def obtain_baseline_image(self, item, target_dir): """ Copy the baseline image to our working directory. @@ -374,7 +384,7 @@ def obtain_baseline_image(self, item, target_dir): if baseline_remote: # baseline_dir can be a list of URLs when remote, so we have to # pass base and filename to download - baseline_image = _download_file(baseline_dir, filename) + baseline_image = self._download_file(baseline_dir, filename) else: baseline_image = (baseline_dir / filename).absolute() @@ -390,11 +400,14 @@ def generate_baseline_image(self, item, fig): if not os.path.exists(self.generate_dir): os.makedirs(self.generate_dir) - fig.savefig(str((self.generate_dir / self.generate_filename(item)).absolute()), - **savefig_kwargs) + baseline_filename = self.generate_filename(item) + baseline_path = (self.generate_dir / baseline_filename).absolute() + fig.savefig(str(baseline_path), **savefig_kwargs) close_mpl_figure(fig) + return baseline_path + def generate_image_hash(self, item, fig): """ For a `matplotlib.figure.Figure`, returns the SHA256 hash as a hexadecimal @@ -435,6 +448,7 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): if not os.path.exists(baseline_image_ref): summary['status'] = 'failed' + summary['image_status'] = 'missing' error_message = ("Image file not found for comparison test in: \n\t" f"{self.get_baseline_directory(item)}\n" "(This is expected for new tests.)\n" @@ -456,6 +470,7 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): actual_shape = imread(str(test_image)).shape[:2] if expected_shape != actual_shape: summary['status'] = 'failed' + summary['image_status'] = 'diff' error_message = SHAPE_MISMATCH_ERROR.format(expected_path=baseline_image, expected_shape=expected_shape, actual_path=test_image, @@ -467,10 +482,12 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): summary['tolerance'] = tolerance if results is None: summary['status'] = 'passed' + summary['image_status'] = 'match' summary['status_msg'] = 'Image comparison passed.' return None else: summary['status'] = 'failed' + summary['image_status'] = 'diff' summary['rms'] = results['rms'] diff_image = (result_dir / 'result-failed-diff.png').absolute() summary['diff_image'] = diff_image.relative_to(self.results_dir).as_posix() @@ -496,6 +513,10 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): compare = self.get_compare(item) savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) + if not self.results_hash_library_name: + # Use hash library name of current test as results hash library name + self.results_hash_library_name = Path(compare.kwargs.get("hash_library", "")).name + hash_library_filename = self.hash_library or compare.kwargs.get('hash_library', None) hash_library_filename = (Path(item.fspath).parent / hash_library_filename).absolute() @@ -512,14 +533,17 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): if baseline_hash is None: # hash-missing summary['status'] = 'failed' + summary['hash_status'] = 'missing' summary['status_msg'] = (f"Hash for test '{hash_name}' not found in {hash_library_filename}. " f"Generated hash is {test_hash}.") elif test_hash == baseline_hash: # hash-match hash_comparison_pass = True summary['status'] = 'passed' + summary['hash_status'] = 'match' summary['status_msg'] = 'Test hash matches baseline hash.' else: # hash-diff summary['status'] = 'failed' + summary['hash_status'] = 'diff' summary['status_msg'] = (f"Hash {test_hash} doesn't match hash " f"{baseline_hash} in library " f"{hash_library_filename} for test {hash_name}.") @@ -544,7 +568,8 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): except Exception as baseline_error: # Append to test error later baseline_comparison = str(baseline_error) else: # Update main summary - for k in ['baseline_image', 'diff_image', 'rms', 'tolerance', 'result_image']: + for k in ['image_status', 'baseline_image', 'diff_image', + 'rms', 'tolerance', 'result_image']: summary[k] = summary[k] or baseline_summary.get(k) # Append the log from image comparison @@ -597,9 +622,12 @@ def item_function_wrapper(*args, **kwargs): remove_ticks_and_titles(fig) test_name = self.generate_test_name(item) + result_dir = self.make_test_results_dir(item) summary = { 'status': None, + 'image_status': None, + 'hash_status': None, 'status_msg': None, 'baseline_image': None, 'diff_image': None, @@ -614,21 +642,23 @@ def item_function_wrapper(*args, **kwargs): # reference images or simply running the test. if self.generate_dir is not None: summary['status'] = 'skipped' + summary['image_status'] = 'generated' summary['status_msg'] = 'Skipped test, since generating image.' - self.generate_baseline_image(item, fig) - if self.generate_hash_library is None: - self._test_results[str(pathify(test_name))] = summary - pytest.skip("Skipping test, since generating image.") + generate_image = self.generate_baseline_image(item, fig) + if self.results_always: # Make baseline image available in HTML + result_image = (result_dir / "baseline.png").absolute() + shutil.copy(generate_image, result_image) + summary['baseline_image'] = \ + result_image.relative_to(self.results_dir).as_posix() if self.generate_hash_library is not None: + summary['hash_status'] = 'generated' image_hash = self.generate_image_hash(item, fig) self._generated_hash_library[test_name] = image_hash - summary['result_hash'] = image_hash + summary['baseline_hash'] = image_hash # Only test figures if not generating images if self.generate_dir is None: - result_dir = self.make_test_results_dir(item) - # Compare to hash library if self.hash_library or compare.kwargs.get('hash_library', None): msg = self.compare_image_to_hash_library(item, fig, result_dir, summary=summary) @@ -645,12 +675,15 @@ def item_function_wrapper(*args, **kwargs): for image_type in ['baseline_image', 'diff_image', 'result_image']: summary[image_type] = None # image no longer exists else: - self._test_results[str(pathify(test_name))] = summary + self._test_results[test_name] = summary pytest.fail(msg, pytrace=False) close_mpl_figure(fig) - self._test_results[str(pathify(test_name))] = summary + self._test_results[test_name] = summary + + if summary['status'] == 'skipped': + pytest.skip(summary['status_msg']) if item.cls is not None: setattr(item.cls, item.function.__name__, item_function_wrapper) @@ -667,21 +700,36 @@ def pytest_unconfigure(self, config): """ Save out the hash library at the end of the run. """ + result_hash_library = self.results_dir / (self.results_hash_library_name or "temp.json") if self.generate_hash_library is not None: hash_library_path = Path(config.rootdir) / self.generate_hash_library hash_library_path.parent.mkdir(parents=True, exist_ok=True) with open(hash_library_path, "w") as fp: json.dump(self._generated_hash_library, fp, indent=2) + if self.results_always: # Make accessible in results directory + # Use same name as generated + result_hash_library = self.results_dir / hash_library_path.name + shutil.copy(hash_library_path, result_hash_library) + elif self.results_always and self.results_hash_library_name: + result_hashes = {k: v['result_hash'] for k, v in self._test_results.items() + if v['result_hash']} + if len(result_hashes) > 0: # At least one hash comparison test + with open(result_hash_library, "w") as fp: + json.dump(result_hashes, fp, indent=2) if self.generate_summary: + kwargs = {} if 'json' in self.generate_summary: summary = self.generate_summary_json() print(f"A JSON report can be found at: {summary}") + if result_hash_library.exists(): # link to it in the HTML + kwargs["hash_library"] = result_hash_library.name if 'html' in self.generate_summary: - summary = generate_summary_html(self._test_results, self.results_dir) + summary = generate_summary_html(self._test_results, self.results_dir, **kwargs) print(f"A summary of the failed tests can be found at: {summary}") if 'basic-html' in self.generate_summary: - summary = generate_summary_basic_html(self._test_results, self.results_dir) + summary = generate_summary_basic_html(self._test_results, self.results_dir, + **kwargs) print(f"A summary of the failed tests can be found at: {summary}") diff --git a/pytest_mpl/summary/html.py b/pytest_mpl/summary/html.py index 858cfe21..78fbcc19 100644 --- a/pytest_mpl/summary/html.py +++ b/pytest_mpl/summary/html.py @@ -26,26 +26,12 @@ class Results: def __init__(self, results, title="Image comparison"): self.title = title # HTML - # If any baseline images or baseline hashes are present, - # assume all results should have them - self.warn_missing = {'baseline_image': False, 'baseline_hash': False} - for key in self.warn_missing.keys(): - for result in results.values(): - if result[key] is not None: - self.warn_missing[key] = True - break - - # Additional <body> classes - self.classes = [] - if self.warn_missing['baseline_hash'] is False: - self.classes += ['no-hash-test'] - # Generate sorted list of results self.cards = [] pad = len(str(len(results.items()))) # maximum length of a result index for collect_n, (name, item) in enumerate(results.items()): card_id = str(collect_n).zfill(pad) # zero pad for alphanumerical sorting - self.cards += [Result(name, item, card_id, self.warn_missing)] + self.cards += [Result(name, item, card_id)] self.cards = sorted(self.cards, key=lambda i: i.indexes['status'], reverse=True) @cached_property @@ -66,6 +52,22 @@ def statistics(self): stats['skipped'] += 1 return stats + @cached_property + def image_comparison(self): + """Whether at least one image comparison test or generation was performed.""" + for result in self.cards: + if result.image_status: + return True + return False + + @cached_property + def hash_comparison(self): + """Whether at least one hash comparison test or generation was performed.""" + for result in self.cards: + if result.hash_status: + return True + return False + class Result: """ @@ -81,20 +83,14 @@ class Result: id : str The test number in order collected. Numbers must be zero padded due to alphanumerical sorting. - warn_missing : dict - Whether to include relevant status badges for images and/or hashes. - Must have keys ``baseline_image`` and ``baseline_hash``. """ - def __init__(self, name, item, id, warn_missing): + def __init__(self, name, item, id): # Make the summary dictionary available as attributes self.__dict__ = item # Sort index for collection order self.id = id - # Whether to show image and/or hash status badges - self.warn_missing = warn_missing - # Name of test with module and test function together and separate self.full_name = name self.name = name.split('.')[-1] @@ -107,30 +103,6 @@ def __init__(self, name, item, id, warn_missing): ('hash', self.hash_status), ]] - @cached_property - def image_status(self): - """Status of the image comparison test.""" - if self.rms is None and self.tolerance is not None: - return 'match' - elif self.rms is not None: - return 'diff' - elif self.baseline_image is None: - return 'missing' - else: - raise ValueError('Unknown image result.') - - @cached_property - def hash_status(self): - """Status of the hash comparison test.""" - if self.baseline_hash is not None or self.result_hash is not None: - if self.baseline_hash is None: - return 'missing' - elif self.baseline_hash == self.result_hash: - return 'match' - else: - return 'diff' - return None - @cached_property def indexes(self): """Dictionary with strings optimized for sorting.""" @@ -178,8 +150,6 @@ def badges(self): """Additional badges to show beside overall status badge.""" for test_type, status_getter in [('image', image_status_msg), ('hash', hash_status_msg)]: status = getattr(self, f'{test_type}_status') - if not self.warn_missing[f'baseline_{test_type}']: - continue # not expected to exist if ( (status == 'missing') or (self.status == 'failed' and status == 'match') or @@ -198,6 +168,7 @@ def status_class(status): 'match': 'success', 'diff': 'danger', 'missing': 'warning', + 'generated': 'warning', } return classes[status] @@ -208,6 +179,7 @@ def image_status_msg(status): 'match': 'Baseline image matches', 'diff': 'Baseline image differs', 'missing': 'Baseline image not found', + 'generated': 'Baseline image was generated', } return messages[status] @@ -218,11 +190,12 @@ def hash_status_msg(status): 'match': 'Baseline hash matches', 'diff': 'Baseline hash differs', 'missing': 'Baseline hash not found', + 'generated': 'Baseline hash was generated', } return messages[status] -def generate_summary_html(results, results_dir): +def generate_summary_html(results, results_dir, hash_library=None): """Generate the HTML summary. Parameters @@ -231,6 +204,9 @@ def generate_summary_html(results, results_dir): The `pytest_mpl.plugin.ImageComparison._test_results` object. results_dir : Path Path to the output directory. + hash_library : str, optional, default=None + Filename of the generated hash library at the root of `results_dir`. + Will be linked to in HTML if not None. """ # Initialize Jinja @@ -246,7 +222,7 @@ def generate_summary_html(results, results_dir): # Render HTML starting from the base template template = env.get_template("base.html") - html = template.render(results=Results(results)) + html = template.render(results=Results(results), hash_library=hash_library) # Write files for file in ['styles.css', 'extra.js', 'hash.svg', 'image.svg']: @@ -259,7 +235,7 @@ def generate_summary_html(results, results_dir): return html_file -def generate_summary_basic_html(results, results_dir): +def generate_summary_basic_html(results, results_dir, hash_library=None): """Generate the basic HTML summary. Parameters @@ -268,6 +244,9 @@ def generate_summary_basic_html(results, results_dir): The `pytest_mpl.plugin.ImageComparison._test_results` object. results_dir : Path Path to the output directory. + hash_library : str, optional, default=None + Filename of the generated hash library at the root of `results_dir`. + Will be linked to in HTML if not None. """ # Initialize Jinja @@ -278,7 +257,7 @@ def generate_summary_basic_html(results, results_dir): # Render HTML starting from the base template template = env.get_template("basic.html") - html = template.render(results=Results(results)) + html = template.render(results=Results(results), hash_library=hash_library) # Write files html_file = results_dir / 'fig_comparison_basic.html' diff --git a/pytest_mpl/summary/templates/base.html b/pytest_mpl/summary/templates/base.html index b34464b9..4ac3c66e 100644 --- a/pytest_mpl/summary/templates/base.html +++ b/pytest_mpl/summary/templates/base.html @@ -8,7 +8,7 @@ integrity="sha384-1BmE4kWBq78iYhFldvKuhfTAU6auU8tT94WrHftjDbrCEXSU1oBoqyl2QvZ6jIW3" crossorigin="anonymous"> <title>{{ results.title }} - + {% include 'navbar.html' %}
diff --git a/pytest_mpl/summary/templates/basic.html b/pytest_mpl/summary/templates/basic.html index 3feb1095..27e6417c 100644 --- a/pytest_mpl/summary/templates/basic.html +++ b/pytest_mpl/summary/templates/basic.html @@ -48,6 +48,11 @@

Image test comparison

of those have a matching baseline image {%- endif %}

+{% if hash_library -%} +

+ Download generated hash library +

+{%- endif %} diff --git a/pytest_mpl/summary/templates/extra.js b/pytest_mpl/summary/templates/extra.js index cb80eefe..d5c3c341 100644 --- a/pytest_mpl/summary/templates/extra.js +++ b/pytest_mpl/summary/templates/extra.js @@ -1,10 +1,3 @@ -// Remove all elements of class mpl-hash if hash test not run -if (document.body.classList[0] == 'no-hash-test') { - document.querySelectorAll('.mpl-hash').forEach(function (elem) { - elem.parentElement.removeChild(elem); - }); -} - // Enable tooltips var tooltipTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="tooltip"]')) var tooltipList = tooltipTriggerList.map(function (tooltipTriggerEl) { diff --git a/pytest_mpl/summary/templates/filter.html b/pytest_mpl/summary/templates/filter.html index 3cd8432f..81e282cc 100644 --- a/pytest_mpl/summary/templates/filter.html +++ b/pytest_mpl/summary/templates/filter.html @@ -16,7 +16,7 @@
Sort tests by...
{{ sort_option('status-sort', 'status', 'desc', default=true) }} {{ sort_option('collected-sort', 'collected', 'asc') }} {{ sort_option('test-name', 'name') }} - {% if results.warn_missing['baseline_image'] -%} + {% if results.image_comparison -%} {{ sort_option('rms-sort', 'RMS', 'desc') }} {%- endif %} @@ -34,15 +34,15 @@
Show tests which have...
{{ filter_option('overall-failed', 'failed') }} {{ filter_option('overall-skipped', 'skipped') }} - {% if results.warn_missing['baseline_image'] -%} + {% if results.image_comparison -%}
{{ filter_option('image-match', 'matching images') }} {{ filter_option('image-diff', 'differing images') }} {{ filter_option('image-missing', 'no baseline image') }}
{%- endif %} - {% if results.warn_missing['baseline_hash'] -%} -
+ {% if results.hash_comparison -%} +
{{ filter_option('hash-match', 'matching hashes') }} {{ filter_option('hash-diff', 'differing hashes') }} {{ filter_option('hash-missing', 'no baseline hash') }} diff --git a/pytest_mpl/summary/templates/navbar.html b/pytest_mpl/summary/templates/navbar.html index 3ba762ac..17fa7bfd 100644 --- a/pytest_mpl/summary/templates/navbar.html +++ b/pytest_mpl/summary/templates/navbar.html @@ -9,6 +9,17 @@ + {% if hash_library -%} + + + + + + {%- endif %}
- {%- else -%} + {%- elif r.result_image -%} result image + {%- elif r.baseline_image -%} + baseline image {%- endif %} {% filter indent(width=8) -%} diff --git a/pytest_mpl/summary/templates/result_images.html b/pytest_mpl/summary/templates/result_images.html index 822bbd46..36ab2943 100644 --- a/pytest_mpl/summary/templates/result_images.html +++ b/pytest_mpl/summary/templates/result_images.html @@ -6,14 +6,14 @@
{{ r.module }}
{{ r.name }}
- {% if results.warn_missing['baseline_image'] -%} + {% if r.image_status and r.image_status != 'generated' -%}
{% macro image_card(file, name) -%}
{{ name }}
{% if file -%} - + {{ name }} {%- endif %}
@@ -41,21 +41,25 @@
{{ r.name }}
{%- endfilter %} {%- endmacro -%} - {% if results.warn_missing['baseline_image'] -%} -
+ {% if r.image_status -%} +
{{ r.image_status | image_status_msg }}
+ {% if r.image_status == 'match' or r.image_status == 'diff' -%}
{{ pre_data('rms', r.rms_str, 'RMS') }} {{ pre_data('tolerance', r.tolerance, 'Tolerance') }}
+ {%- endif %}
{%- endif %} - {% if results.warn_missing['baseline_hash'] -%} -
+ {% if r.hash_status -%} +
{{ r.hash_status | hash_status_msg }}
{{ pre_data('baseline_hash', r.baseline_hash, 'Baseline') }} + {% if r.hash_status != 'generated' -%} {{ pre_data('result_hash', r.result_hash, 'Result') }} + {%- endif %}
{%- endif %} diff --git a/pytest_mpl/summary/templates/styles.css b/pytest_mpl/summary/templates/styles.css index 3effb1df..c5870b09 100644 --- a/pytest_mpl/summary/templates/styles.css +++ b/pytest_mpl/summary/templates/styles.css @@ -1,6 +1,3 @@ -body.no-hash-test .mpl-hash { - display: none; -} .navbar-brand span.repo { font-weight: bold; } @@ -14,6 +11,9 @@ body.no-hash-test .mpl-hash { .nav-filtertools .nav-searchbar { flex: 10; } +.nav-filtertools svg { + vertical-align: text-bottom; +} #filterForm .spacer { flex: 1; } @@ -29,12 +29,12 @@ body.no-hash-test .mpl-hash { max-width: 400px; } } -.image-missing .mpl-image .card-body { - display: none; -} pre { white-space: pre-line; } +.tooltip-inner pre { + margin: 0; +} div.result div.status-badge button img { vertical-align: sub; } diff --git a/setup.cfg b/setup.cfg index cd193ab4..60478bd3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,7 +11,6 @@ classifiers = Topic :: Software Development :: Testing Topic :: Scientific/Engineering :: Visualization Programming Language :: Python - Programming Language :: Python :: 2 Programming Language :: Python :: 3 Operating System :: OS Independent License :: OSI Approved :: BSD License diff --git a/tests/subtests/helpers.py b/tests/subtests/helpers.py index 0c688843..0f345f03 100644 --- a/tests/subtests/helpers.py +++ b/tests/subtests/helpers.py @@ -9,7 +9,8 @@ class MatchError(Exception): pass -def diff_summary(baseline, result, baseline_hash_library=None, result_hash_library=None): +def diff_summary(baseline, result, baseline_hash_library=None, result_hash_library=None, + generating_hashes=False): """Diff a pytest-mpl summary dictionary. Parameters @@ -26,6 +27,9 @@ def diff_summary(baseline, result, baseline_hash_library=None, result_hash_libra Path to the "baseline" image hash library. Result hashes in the baseline summary are updated to these values to handle different Matplotlib versions. + generating_hashes : bool, optional, default=False + Whether `--mpl-generate-hash-library` was specified and + both of `--mpl-hash-library` and `hash_library=` were not. """ if baseline_hash_library and baseline_hash_library.exists(): # Load "correct" baseline hashes @@ -57,10 +61,13 @@ def diff_summary(baseline, result, baseline_hash_library=None, result_hash_libra # Swap the baseline and result hashes in the summary # for the corresponding hashes in each hash library - if baseline_hash_library and test in baseline_hash_library: + if baseline_hash_library and test in baseline_hash_library and not generating_hashes: baseline_summary = replace_hash(baseline_summary, 'baseline_hash', baseline_hash_library[test]) if result_hash_library: + if generating_hashes: # Newly generate result will appear as baseline_hash + baseline_summary = replace_hash(baseline_summary, 'baseline_hash', + result_hash_library[test]) baseline_summary = replace_hash(baseline_summary, 'result_hash', result_hash_library[test]) diff --git a/tests/subtests/summaries/test_default.json b/tests/subtests/summaries/test_default.json index a4acadc6..b4b51444 100644 --- a/tests/subtests/summaries/test_default.json +++ b/tests/subtests/summaries/test_default.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -34,6 +40,8 @@ }, "subtests.subtest.test_hdiff_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -45,6 +53,8 @@ }, "subtests.subtest.test_hdiff_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", @@ -56,6 +66,8 @@ }, "subtests.subtest.test_hdiff_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -67,6 +79,8 @@ }, "subtests.subtest.test_hmissing_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -78,6 +92,8 @@ }, "subtests.subtest.test_hmissing_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", @@ -89,6 +105,8 @@ }, "subtests.subtest.test_hmissing_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -100,6 +118,8 @@ }, "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -111,6 +131,8 @@ }, "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", @@ -122,6 +144,8 @@ }, "subtests.subtest.test_hdiff_imatch_savefig": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -133,6 +157,8 @@ }, "subtests.subtest.test_hdiff_imatch_style": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -144,6 +170,8 @@ }, "subtests.subtest.test_hdiff_imatch_removetext": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, diff --git a/tests/subtests/summaries/test_generate.json b/tests/subtests/summaries/test_generate.json new file mode 100644 index 00000000..6f9f78e2 --- /dev/null +++ b/tests/subtests/summaries/test_generate.json @@ -0,0 +1,184 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03", + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa", + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856", + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "8864803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e3c8de36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e4c06cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a", + "result_hash": null + } +} diff --git a/tests/subtests/summaries/test_generate_hashes_only.json b/tests/subtests/summaries/test_generate_hashes_only.json new file mode 100644 index 00000000..8dee25fa --- /dev/null +++ b/tests/subtests/summaries/test_generate_hashes_only.json @@ -0,0 +1,184 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03", + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", + "rms": 11.100353848213828, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmatch_idiff/result.png", + "baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa", + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "failed", + "image_status": "missing", + "hash_status": "generated", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmatch_imissing/result.png", + "baseline_hash": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", + "rms": 11.182677079602481, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_idiff/result.png", + "baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "failed", + "image_status": "missing", + "hash_status": "generated", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hdiff_imissing/result.png", + "baseline_hash": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856", + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", + "rms": 12.12938597648977, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmissing_idiff/result.png", + "baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "failed", + "image_status": "missing", + "hash_status": "generated", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmissing_imissing/result.png", + "baseline_hash": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 200, + "result_image": null, + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", + "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", + "rms": 29.260332173249314, + "tolerance": 3, + "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "8864803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "e3c8de36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "e4c06cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a", + "result_hash": null + } +} diff --git a/tests/subtests/summaries/test_generate_images_only.json b/tests/subtests/summaries/test_generate_images_only.json new file mode 100644 index 00000000..7720807d --- /dev/null +++ b/tests/subtests/summaries/test_generate_images_only.json @@ -0,0 +1,184 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + } +} diff --git a/tests/subtests/summaries/test_hash.json b/tests/subtests/summaries/test_hash.json index bec04c5a..4304bf03 100644 --- a/tests/subtests/summaries/test_hash.json +++ b/tests/subtests/summaries/test_hash.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -35,6 +41,8 @@ "subtests.subtest.test_hdiff_imatch": { "status": "failed", "status_msg": "REGEX:Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -46,6 +54,8 @@ "subtests.subtest.test_hdiff_idiff": { "status": "failed", "status_msg": "REGEX:Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -57,6 +67,8 @@ "subtests.subtest.test_hdiff_imissing": { "status": "failed", "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -68,6 +80,8 @@ "subtests.subtest.test_hmissing_imatch": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -79,6 +93,8 @@ "subtests.subtest.test_hmissing_idiff": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -90,6 +106,8 @@ "subtests.subtest.test_hmissing_imissing": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -101,6 +119,8 @@ "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -112,6 +132,8 @@ "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -123,6 +145,8 @@ "subtests.subtest.test_hdiff_imatch_savefig": { "status": "failed", "status_msg": "REGEX:Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -134,6 +158,8 @@ "subtests.subtest.test_hdiff_imatch_style": { "status": "failed", "status_msg": "REGEX:Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -145,6 +171,8 @@ "subtests.subtest.test_hdiff_imatch_removetext": { "status": "failed", "status_msg": "REGEX:Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, diff --git a/tests/subtests/summaries/test_html_generate.json b/tests/subtests/summaries/test_html_generate.json new file mode 100644 index 00000000..84a8e8b9 --- /dev/null +++ b/tests/subtests/summaries/test_html_generate.json @@ -0,0 +1,184 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03", + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa", + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856", + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "8864803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e3c8de36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e4c06cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a", + "result_hash": null + } +} diff --git a/tests/subtests/summaries/test_html_generate_hashes_only.json b/tests/subtests/summaries/test_html_generate_hashes_only.json new file mode 100644 index 00000000..0a813734 --- /dev/null +++ b/tests/subtests/summaries/test_html_generate_hashes_only.json @@ -0,0 +1,184 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmatch_imatch/result.png", + "baseline_hash": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03", + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", + "rms": 11.100353848213828, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmatch_idiff/result.png", + "baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa", + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "failed", + "image_status": "missing", + "hash_status": "generated", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmatch_imissing/result.png", + "baseline_hash": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_imatch/result.png", + "baseline_hash": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", + "rms": 11.182677079602481, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_idiff/result.png", + "baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "failed", + "image_status": "missing", + "hash_status": "generated", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hdiff_imissing/result.png", + "baseline_hash": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmissing_imatch/result.png", + "baseline_hash": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856", + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", + "rms": 12.12938597648977, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmissing_idiff/result.png", + "baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "failed", + "image_status": "missing", + "hash_status": "generated", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmissing_imissing/result.png", + "baseline_hash": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 200, + "result_image": "subtests.subtest.test_hdiff_imatch_tolerance/result.png", + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", + "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", + "rms": 29.260332173249314, + "tolerance": 3, + "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_imatch_savefig/result.png", + "baseline_hash": "8864803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_imatch_style/result.png", + "baseline_hash": "e3c8de36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_imatch_removetext/result.png", + "baseline_hash": "e4c06cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a", + "result_hash": null + } +} diff --git a/tests/subtests/summaries/test_html_generate_images_only.json b/tests/subtests/summaries/test_html_generate_images_only.json new file mode 100644 index 00000000..bed75fae --- /dev/null +++ b/tests/subtests/summaries/test_html_generate_images_only.json @@ -0,0 +1,184 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + } +} diff --git a/tests/subtests/summaries/test_html_hashes_only.json b/tests/subtests/summaries/test_html_hashes_only.json index 448ce9c2..a55666ac 100644 --- a/tests/subtests/summaries/test_html_hashes_only.json +++ b/tests/subtests/summaries/test_html_hashes_only.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -35,6 +41,8 @@ "subtests.subtest.test_hdiff_imatch": { "status": "failed", "status_msg": "REGEX:Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -46,6 +54,8 @@ "subtests.subtest.test_hdiff_idiff": { "status": "failed", "status_msg": "REGEX:Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -57,6 +67,8 @@ "subtests.subtest.test_hdiff_imissing": { "status": "failed", "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -68,6 +80,8 @@ "subtests.subtest.test_hmissing_imatch": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -79,6 +93,8 @@ "subtests.subtest.test_hmissing_idiff": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -90,6 +106,8 @@ "subtests.subtest.test_hmissing_imissing": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -101,6 +119,8 @@ "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -112,6 +132,8 @@ "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -123,6 +145,8 @@ "subtests.subtest.test_hdiff_imatch_savefig": { "status": "failed", "status_msg": "REGEX:Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -134,6 +158,8 @@ "subtests.subtest.test_hdiff_imatch_style": { "status": "failed", "status_msg": "REGEX:Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -145,6 +171,8 @@ "subtests.subtest.test_hdiff_imatch_removetext": { "status": "failed", "status_msg": "REGEX:Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, diff --git a/tests/subtests/summaries/test_html_images_only.json b/tests/subtests/summaries/test_html_images_only.json index c8a5058c..0b33035e 100644 --- a/tests/subtests/summaries/test_html_images_only.json +++ b/tests/subtests/summaries/test_html_images_only.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -34,6 +40,8 @@ }, "subtests.subtest.test_hdiff_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", "diff_image": null, @@ -45,6 +53,8 @@ }, "subtests.subtest.test_hdiff_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", @@ -56,6 +66,8 @@ }, "subtests.subtest.test_hdiff_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -67,6 +79,8 @@ }, "subtests.subtest.test_hmissing_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", "diff_image": null, @@ -78,6 +92,8 @@ }, "subtests.subtest.test_hmissing_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", @@ -89,6 +105,8 @@ }, "subtests.subtest.test_hmissing_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -100,6 +118,8 @@ }, "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", "diff_image": null, @@ -111,6 +131,8 @@ }, "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", @@ -122,6 +144,8 @@ }, "subtests.subtest.test_hdiff_imatch_savefig": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", "diff_image": null, @@ -133,6 +157,8 @@ }, "subtests.subtest.test_hdiff_imatch_style": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", "diff_image": null, @@ -144,6 +170,8 @@ }, "subtests.subtest.test_hdiff_imatch_removetext": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", "diff_image": null, diff --git a/tests/subtests/summaries/test_hybrid.json b/tests/subtests/summaries/test_hybrid.json index 2f2930b0..851e67f1 100644 --- a/tests/subtests/summaries/test_hybrid.json +++ b/tests/subtests/summaries/test_hybrid.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -35,6 +41,8 @@ "subtests.subtest.test_hdiff_imatch": { "status": "failed", "status_msg": "REGEX:Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", "diff_image": null, "rms": null, @@ -46,6 +54,8 @@ "subtests.subtest.test_hdiff_idiff": { "status": "failed", "status_msg": "REGEX:Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "image_status": "diff", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", "rms": 11.182677079602481, @@ -57,6 +67,8 @@ "subtests.subtest.test_hdiff_imissing": { "status": "failed", "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "image_status": "missing", + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -68,6 +80,8 @@ "subtests.subtest.test_hmissing_imatch": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "missing", "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", "diff_image": null, "rms": null, @@ -79,6 +93,8 @@ "subtests.subtest.test_hmissing_idiff": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "image_status": "diff", + "hash_status": "missing", "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", "rms": 12.12938597648977, @@ -90,6 +106,8 @@ "subtests.subtest.test_hmissing_imissing": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "image_status": "missing", + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -101,6 +119,8 @@ "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", "diff_image": null, "rms": null, @@ -112,6 +132,8 @@ "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", + "image_status": "diff", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", "rms": 29.260332173249314, @@ -123,6 +145,8 @@ "subtests.subtest.test_hdiff_imatch_savefig": { "status": "failed", "status_msg": "REGEX:Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", "diff_image": null, "rms": null, @@ -134,6 +158,8 @@ "subtests.subtest.test_hdiff_imatch_style": { "status": "failed", "status_msg": "REGEX:Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", "diff_image": null, "rms": null, @@ -145,6 +171,8 @@ "subtests.subtest.test_hdiff_imatch_removetext": { "status": "failed", "status_msg": "REGEX:Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", "diff_image": null, "rms": null, diff --git a/tests/subtests/summaries/test_results_always.json b/tests/subtests/summaries/test_results_always.json index c42fee29..ea7af1d2 100644 --- a/tests/subtests/summaries/test_results_always.json +++ b/tests/subtests/summaries/test_results_always.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": "match", + "hash_status": "match", "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "passed", + "image_status": "diff", + "hash_status": "match", "status_msg": "REGEX:Test hash matches baseline hash\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "passed", + "image_status": "missing", + "hash_status": "match", "status_msg": "REGEX:Test hash matches baseline hash\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -35,6 +41,8 @@ "subtests.subtest.test_hdiff_imatch": { "status": "failed", "status_msg": "REGEX:Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", "diff_image": null, "rms": null, @@ -46,6 +54,8 @@ "subtests.subtest.test_hdiff_idiff": { "status": "failed", "status_msg": "REGEX:Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "image_status": "diff", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", "rms": 11.182677079602481, @@ -57,6 +67,8 @@ "subtests.subtest.test_hdiff_imissing": { "status": "failed", "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "image_status": "missing", + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -68,6 +80,8 @@ "subtests.subtest.test_hmissing_imatch": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "missing", "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", "diff_image": null, "rms": null, @@ -79,6 +93,8 @@ "subtests.subtest.test_hmissing_idiff": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "image_status": "diff", + "hash_status": "missing", "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", "rms": 12.12938597648977, @@ -90,6 +106,8 @@ "subtests.subtest.test_hmissing_imissing": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "image_status": "missing", + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -101,6 +119,8 @@ "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", "diff_image": null, "rms": null, @@ -112,6 +132,8 @@ "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", + "image_status": "diff", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", "rms": 29.260332173249314, @@ -123,6 +145,8 @@ "subtests.subtest.test_hdiff_imatch_savefig": { "status": "failed", "status_msg": "REGEX:Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", "diff_image": null, "rms": null, @@ -134,6 +158,8 @@ "subtests.subtest.test_hdiff_imatch_style": { "status": "failed", "status_msg": "REGEX:Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", "diff_image": null, "rms": null, @@ -145,6 +171,8 @@ "subtests.subtest.test_hdiff_imatch_removetext": { "status": "failed", "status_msg": "REGEX:Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", "diff_image": null, "rms": null, diff --git a/tests/subtests/test_subtest.py b/tests/subtests/test_subtest.py index 48a2f461..eb85aa5c 100644 --- a/tests/subtests/test_subtest.py +++ b/tests/subtests/test_subtest.py @@ -34,6 +34,7 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=True, + has_result_hashes=False, generating_hashes=False, update_baseline=UPDATE_BASELINE, update_summary=UPDATE_SUMMARY): """ Run pytest (within pytest) and check JSON summary report. @@ -49,6 +50,12 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=Tru Summaries to generate in addition to `json`. xfail : bool, optional, default=True Whether the overall pytest run should fail. + has_result_hashes : bool or str, optional, default=False + Whether a hash library is expected to exist in the results directory. + If a string, this is the name of the expected results file. + generating_hashes : bool, optional, default=False + Whether `--mpl-generate-hash-library` was specified and + both of `--mpl-hash-library` and `hash_library=` were not. """ # Parse arguments if summaries is None: @@ -105,11 +112,30 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=Tru # Compare summaries diff_summary(baseline_summary, result_summary, - baseline_hash_library=HASH_LIBRARY, result_hash_library=RESULT_LIBRARY) + baseline_hash_library=HASH_LIBRARY, result_hash_library=RESULT_LIBRARY, + generating_hashes=generating_hashes) # Ensure reported images exist assert_existence(result_summary, path=results_path) + # Get expected name for the hash library saved to the results directory + if isinstance(has_result_hashes, str): + result_hash_file = tmp_path / 'results' / has_result_hashes + has_result_hashes = True # convert to bool after processing str + else: + result_hash_file = tmp_path / 'results' / HASH_LIBRARY.name + + # Compare the generated hash library to the expected hash library + if has_result_hashes: + assert result_hash_file.exists() + with open(RESULT_LIBRARY, "r") as f: + baseline = json.load(f) + with open(result_hash_file, "r") as f: + result = json.load(f) + diff_summary({'a': baseline}, {'a': result}) + else: + assert not result_hash_file.exists() + def test_default(tmp_path): run_subtest('test_default', tmp_path, []) @@ -128,13 +154,15 @@ def test_hybrid(tmp_path): @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") def test_results_always(tmp_path): run_subtest('test_results_always', tmp_path, - [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS, '--mpl-results-always']) + [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS, '--mpl-results-always'], + has_result_hashes=True) @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") def test_html(tmp_path): run_subtest('test_results_always', tmp_path, - [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS], summaries=['html']) + [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS], summaries=['html'], + has_result_hashes=True) assert (tmp_path / 'results' / 'fig_comparison.html').exists() assert (tmp_path / 'results' / 'extra.js').exists() assert (tmp_path / 'results' / 'styles.css').exists() @@ -142,7 +170,8 @@ def test_html(tmp_path): @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") def test_html_hashes_only(tmp_path): - run_subtest('test_html_hashes_only', tmp_path, [HASH_LIBRARY_FLAG], summaries=['html']) + run_subtest('test_html_hashes_only', tmp_path, [HASH_LIBRARY_FLAG], summaries=['html'], + has_result_hashes=True) assert (tmp_path / 'results' / 'fig_comparison.html').exists() assert (tmp_path / 'results' / 'extra.js').exists() assert (tmp_path / 'results' / 'styles.css').exists() @@ -158,5 +187,67 @@ def test_html_images_only(tmp_path): @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") def test_basic_html(tmp_path): run_subtest('test_results_always', tmp_path, - [HASH_LIBRARY_FLAG, *BASELINE_IMAGES_FLAG_REL], summaries=['basic-html']) + [HASH_LIBRARY_FLAG, *BASELINE_IMAGES_FLAG_REL], summaries=['basic-html'], + has_result_hashes=True) assert (tmp_path / 'results' / 'fig_comparison_basic.html').exists() + + +@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") +def test_generate(tmp_path): + # generating hashes and images; no testing + run_subtest('test_generate', tmp_path, + [rf'--mpl-generate-path={tmp_path}', + rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}'], + xfail=False, generating_hashes=True) + + +def test_generate_images_only(tmp_path): + # generating images; no testing + run_subtest('test_generate_images_only', tmp_path, + [rf'--mpl-generate-path={tmp_path}'], xfail=False) + + +@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") +def test_generate_hashes_only(tmp_path): + # generating hashes; testing images + run_subtest('test_generate_hashes_only', tmp_path, + [rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}'], + generating_hashes=True) + + +@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") +def test_html_generate(tmp_path): + # generating hashes and images; no testing + run_subtest('test_html_generate', tmp_path, + [rf'--mpl-generate-path={tmp_path}', + rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}'], + summaries=['html'], xfail=False, has_result_hashes="test_hashes.json", + generating_hashes=True) + assert (tmp_path / 'results' / 'fig_comparison.html').exists() + + +def test_html_generate_images_only(tmp_path): + # generating images; no testing + run_subtest('test_html_generate_images_only', tmp_path, + [rf'--mpl-generate-path={tmp_path}'], + summaries=['html'], xfail=False) + assert (tmp_path / 'results' / 'fig_comparison.html').exists() + + +@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") +def test_html_generate_hashes_only(tmp_path): + # generating hashes; testing images + run_subtest('test_html_generate_hashes_only', tmp_path, + [rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}'], + summaries=['html'], has_result_hashes="test_hashes.json", generating_hashes=True) + assert (tmp_path / 'results' / 'fig_comparison.html').exists() + + +@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") +def test_html_run_generate_hashes_only(tmp_path): + # generating hashes; testing hashes + run_subtest('test_html_hashes_only', tmp_path, + [rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}', + HASH_LIBRARY_FLAG], + summaries=['html'], has_result_hashes="test_hashes.json") + assert (tmp_path / 'results' / 'fig_comparison.html').exists()