From 503f85c583b57f17fe99e07f26b33fdb3928035a Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 9 Feb 2022 15:33:58 +0000 Subject: [PATCH 01/19] post release --- CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 3b6a6b39..8e6c55e8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +0.15 (unreleased) +----------------- + + 0.14 (2022-02-09) ----------------- From 0f46553f7eb165f1721f0fc75c997e2e8dbc648e Mon Sep 17 00:00:00 2001 From: Ryan May Date: Tue, 15 Feb 2022 14:26:09 -0700 Subject: [PATCH 02/19] MNT: Remove Python 2 from package classifiers --- setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index cd193ab4..60478bd3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,7 +11,6 @@ classifiers = Topic :: Software Development :: Testing Topic :: Scientific/Engineering :: Visualization Programming Language :: Python - Programming Language :: Python :: 2 Programming Language :: Python :: 3 Operating System :: OS Independent License :: OSI Approved :: BSD License From e81013dc63403a40f6de3d0b414fcab99f671e75 Mon Sep 17 00:00:00 2001 From: Ryan May Date: Tue, 15 Feb 2022 15:26:54 -0700 Subject: [PATCH 03/19] MNT: Remove Python 2.7 from readme --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index ebdee082..4a80aec3 100644 --- a/README.rst +++ b/README.rst @@ -16,7 +16,7 @@ section below. Installing ---------- -This plugin is compatible with Python 2.7, and 3.6 and later, and +This plugin is compatible with Python 3.6 and later, and requires `pytest `__ and `matplotlib `__ to be installed. From 7027739c4dc1b2cbe76244dd638ef90833e10c8f Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Mon, 14 Feb 2022 20:40:42 +0000 Subject: [PATCH 04/19] Show baseline image in HTML when generating --- pytest_mpl/plugin.py | 29 ++++++++++++------- pytest_mpl/summary/html.py | 7 ++++- pytest_mpl/summary/templates/result.html | 4 ++- .../summary/templates/result_images.html | 2 +- 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index d147e57c..4cd7bda8 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -390,11 +390,14 @@ def generate_baseline_image(self, item, fig): if not os.path.exists(self.generate_dir): os.makedirs(self.generate_dir) - fig.savefig(str((self.generate_dir / self.generate_filename(item)).absolute()), - **savefig_kwargs) + baseline_filename = self.generate_filename(item) + baseline_path = (self.generate_dir / baseline_filename).absolute() + fig.savefig(str(baseline_path), **savefig_kwargs) close_mpl_figure(fig) + return baseline_path + def generate_image_hash(self, item, fig): """ For a `matplotlib.figure.Figure`, returns the SHA256 hash as a hexadecimal @@ -597,6 +600,7 @@ def item_function_wrapper(*args, **kwargs): remove_ticks_and_titles(fig) test_name = self.generate_test_name(item) + result_dir = self.make_test_results_dir(item) summary = { 'status': None, @@ -615,20 +619,20 @@ def item_function_wrapper(*args, **kwargs): if self.generate_dir is not None: summary['status'] = 'skipped' summary['status_msg'] = 'Skipped test, since generating image.' - self.generate_baseline_image(item, fig) - if self.generate_hash_library is None: - self._test_results[str(pathify(test_name))] = summary - pytest.skip("Skipping test, since generating image.") + generate_image = self.generate_baseline_image(item, fig) + if self.results_always: # Make baseline image available in HTML + result_image = (result_dir / "baseline.png").absolute() + shutil.copy(generate_image, result_image) + summary['baseline_image'] = \ + result_image.relative_to(self.results_dir).as_posix() if self.generate_hash_library is not None: image_hash = self.generate_image_hash(item, fig) self._generated_hash_library[test_name] = image_hash - summary['result_hash'] = image_hash + summary['baseline_hash'] = image_hash # Only test figures if not generating images if self.generate_dir is None: - result_dir = self.make_test_results_dir(item) - # Compare to hash library if self.hash_library or compare.kwargs.get('hash_library', None): msg = self.compare_image_to_hash_library(item, fig, result_dir, summary=summary) @@ -645,12 +649,15 @@ def item_function_wrapper(*args, **kwargs): for image_type in ['baseline_image', 'diff_image', 'result_image']: summary[image_type] = None # image no longer exists else: - self._test_results[str(pathify(test_name))] = summary + self._test_results[test_name] = summary pytest.fail(msg, pytrace=False) close_mpl_figure(fig) - self._test_results[str(pathify(test_name))] = summary + self._test_results[test_name] = summary + + if summary['status'] == 'skipped': + pytest.skip(summary['status_msg']) if item.cls is not None: setattr(item.cls, item.function.__name__, item_function_wrapper) diff --git a/pytest_mpl/summary/html.py b/pytest_mpl/summary/html.py index 858cfe21..aab93d27 100644 --- a/pytest_mpl/summary/html.py +++ b/pytest_mpl/summary/html.py @@ -117,7 +117,7 @@ def image_status(self): elif self.baseline_image is None: return 'missing' else: - raise ValueError('Unknown image result.') + return 'generated' @cached_property def hash_status(self): @@ -125,6 +125,8 @@ def hash_status(self): if self.baseline_hash is not None or self.result_hash is not None: if self.baseline_hash is None: return 'missing' + elif self.result_hash is None: + return 'generated' elif self.baseline_hash == self.result_hash: return 'match' else: @@ -198,6 +200,7 @@ def status_class(status): 'match': 'success', 'diff': 'danger', 'missing': 'warning', + 'generated': 'warning', } return classes[status] @@ -208,6 +211,7 @@ def image_status_msg(status): 'match': 'Baseline image matches', 'diff': 'Baseline image differs', 'missing': 'Baseline image not found', + 'generated': 'Baseline image was generated', } return messages[status] @@ -218,6 +222,7 @@ def hash_status_msg(status): 'match': 'Baseline hash matches', 'diff': 'Baseline hash differs', 'missing': 'Baseline hash not found', + 'generated': 'Baseline hash was generated', } return messages[status] diff --git a/pytest_mpl/summary/templates/result.html b/pytest_mpl/summary/templates/result.html index 5cdfac55..06d34973 100644 --- a/pytest_mpl/summary/templates/result.html +++ b/pytest_mpl/summary/templates/result.html @@ -17,8 +17,10 @@ result image - {%- else -%} + {%- elif r.result_image -%} result image + {%- elif r.baseline_image -%} + baseline image {%- endif %} {% filter indent(width=8) -%} diff --git a/pytest_mpl/summary/templates/result_images.html b/pytest_mpl/summary/templates/result_images.html index 822bbd46..60797105 100644 --- a/pytest_mpl/summary/templates/result_images.html +++ b/pytest_mpl/summary/templates/result_images.html @@ -6,7 +6,7 @@
{{ r.module }}
{{ r.name }}
- {% if results.warn_missing['baseline_image'] -%} + {% if results.warn_missing['baseline_image'] and r.image_status != 'generated' -%}
{% macro image_card(file, name) -%}
From d4746bc720941b19879637a582e48bf849df47af Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Mon, 14 Feb 2022 20:42:09 +0000 Subject: [PATCH 05/19] Clarify what happens when generating baseline hashes --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index 4a80aec3..d0c4eead 100644 --- a/README.rst +++ b/README.rst @@ -86,6 +86,10 @@ can either be specified via the ``--mpl-hash-library=`` command line argument, or via the ``hash_library=`` keyword argument to the ``@pytest.mark.mpl_image_compare`` decorator. +When generating a hash library, the tests will also be run as usual against the +existing hash library specified by ``--mpl-hash-library`` or the keyword argument. +However, generating baseline images will always result in the tests being skipped. + Hybrid Mode: Hashes and Images ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From d19a995deb155e3876bb055aa7dcf32050f16638 Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Thu, 17 Feb 2022 18:15:01 +0000 Subject: [PATCH 06/19] Save a result hash library to results dir --- README.rst | 6 ++++++ pytest_mpl/plugin.py | 20 ++++++++++++++++++++ tests/subtests/test_subtest.py | 34 ++++++++++++++++++++++++++++++---- 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index d0c4eead..3dd9b742 100644 --- a/README.rst +++ b/README.rst @@ -229,6 +229,12 @@ test (based on the hash library) also shown in the generated summary. This option is applied automatically when generating a HTML summary. +When the ``--mpl-results-always`` option is active, and some hash +comparison tests are performed, a hash library containing all the +result hashes will also be saved to the root of the results directory. +The filename will be extracted from ``--mpl-generate-hash-library``, +``--mpl-hash-library`` or ``hash_library=`` in that order. + Base style ^^^^^^^^^^ diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 4cd7bda8..0ae4af57 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -283,6 +283,12 @@ def __init__(self, self.results_dir = Path(tempfile.mkdtemp(dir=self.results_dir)) self.results_dir.mkdir(parents=True, exist_ok=True) + # Decide what to call the downloadable results hash library + if self.hash_library is not None: + self.results_hash_library_name = self.hash_library.name + else: # Use the first filename encountered in a `hash_library=` kwarg + self.results_hash_library_name = None + # We need global state to store all the hashes generated over the run self._generated_hash_library = {} self._test_results = {} @@ -499,6 +505,10 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): compare = self.get_compare(item) savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) + if not self.results_hash_library_name: + # Use hash library name of current test as results hash library name + self.results_hash_library_name = Path(compare.kwargs.get("hash_library", "")).name + hash_library_filename = self.hash_library or compare.kwargs.get('hash_library', None) hash_library_filename = (Path(item.fspath).parent / hash_library_filename).absolute() @@ -674,11 +684,21 @@ def pytest_unconfigure(self, config): """ Save out the hash library at the end of the run. """ + result_hash_library = self.results_dir / (self.results_hash_library_name or "temp.json") if self.generate_hash_library is not None: hash_library_path = Path(config.rootdir) / self.generate_hash_library hash_library_path.parent.mkdir(parents=True, exist_ok=True) with open(hash_library_path, "w") as fp: json.dump(self._generated_hash_library, fp, indent=2) + if self.results_always: # Make accessible in results directory + result_hash_library.name = hash_library_path.name # use same name as generated + shutil.copy(hash_library_path, result_hash_library) + elif self.results_always and self.results_hash_library_name: + result_hashes = {k: v['result_hash'] for k, v in self._test_results.items() + if v['result_hash']} + if len(result_hashes) > 0: # At least one hash comparison test + with open(result_hash_library, "w") as fp: + json.dump(result_hashes, fp, indent=2) if self.generate_summary: if 'json' in self.generate_summary: diff --git a/tests/subtests/test_subtest.py b/tests/subtests/test_subtest.py index 48a2f461..0a31f7ff 100644 --- a/tests/subtests/test_subtest.py +++ b/tests/subtests/test_subtest.py @@ -34,6 +34,7 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=True, + has_result_hashes=False, update_baseline=UPDATE_BASELINE, update_summary=UPDATE_SUMMARY): """ Run pytest (within pytest) and check JSON summary report. @@ -49,6 +50,9 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=Tru Summaries to generate in addition to `json`. xfail : bool, optional, default=True Whether the overall pytest run should fail. + has_result_hashes : bool or str, optional, default=False + Whether a hash library is expected to exist in the results directory. + If a string, this is the name of the expected results file. """ # Parse arguments if summaries is None: @@ -110,6 +114,24 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=Tru # Ensure reported images exist assert_existence(result_summary, path=results_path) + # Get expected name for the hash library saved to the results directory + if isinstance(has_result_hashes, str): + result_hash_file = tmp_path / 'results' / has_result_hashes + has_result_hashes = True # convert to bool after processing str + else: + result_hash_file = tmp_path / 'results' / HASH_LIBRARY.name + + # Compare the generated hash library to the expected hash library + if has_result_hashes: + assert result_hash_file.exists() + with open(RESULT_LIBRARY, "r") as f: + baseline = json.load(f) + with open(result_hash_file, "r") as f: + result = json.load(f) + diff_summary({'a': baseline}, {'a': result}) + else: + assert not result_hash_file.exists() + def test_default(tmp_path): run_subtest('test_default', tmp_path, []) @@ -128,13 +150,15 @@ def test_hybrid(tmp_path): @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") def test_results_always(tmp_path): run_subtest('test_results_always', tmp_path, - [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS, '--mpl-results-always']) + [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS, '--mpl-results-always'], + has_result_hashes=True) @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") def test_html(tmp_path): run_subtest('test_results_always', tmp_path, - [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS], summaries=['html']) + [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS], summaries=['html'], + has_result_hashes=True) assert (tmp_path / 'results' / 'fig_comparison.html').exists() assert (tmp_path / 'results' / 'extra.js').exists() assert (tmp_path / 'results' / 'styles.css').exists() @@ -142,7 +166,8 @@ def test_html(tmp_path): @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") def test_html_hashes_only(tmp_path): - run_subtest('test_html_hashes_only', tmp_path, [HASH_LIBRARY_FLAG], summaries=['html']) + run_subtest('test_html_hashes_only', tmp_path, [HASH_LIBRARY_FLAG], summaries=['html'], + has_result_hashes=True) assert (tmp_path / 'results' / 'fig_comparison.html').exists() assert (tmp_path / 'results' / 'extra.js').exists() assert (tmp_path / 'results' / 'styles.css').exists() @@ -158,5 +183,6 @@ def test_html_images_only(tmp_path): @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") def test_basic_html(tmp_path): run_subtest('test_results_always', tmp_path, - [HASH_LIBRARY_FLAG, *BASELINE_IMAGES_FLAG_REL], summaries=['basic-html']) + [HASH_LIBRARY_FLAG, *BASELINE_IMAGES_FLAG_REL], summaries=['basic-html'], + has_result_hashes=True) assert (tmp_path / 'results' / 'fig_comparison_basic.html').exists() From 7e9af62ed08a6b90fa5f0412720d5629b22904e0 Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Thu, 17 Feb 2022 18:44:47 +0000 Subject: [PATCH 07/19] Remove incorrect warning A HTML summary can be produced when generating --- pytest_mpl/plugin.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 0ae4af57..a0b31997 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -181,8 +181,6 @@ def pytest_configure(config): if generate_dir is not None: if baseline_dir is not None: warnings.warn("Ignoring --mpl-baseline-path since --mpl-generate-path is set") - if results_dir is not None and generate_dir is not None: - warnings.warn("Ignoring --mpl-result-path since --mpl-generate-path is set") if baseline_dir is not None and not baseline_dir.startswith(("https", "http")): baseline_dir = os.path.abspath(baseline_dir) From f8b473c958d8d682e4d7cd68a5c75d9f288e866a Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Thu, 17 Feb 2022 20:08:51 +0000 Subject: [PATCH 08/19] Test options for generating --- pytest_mpl/plugin.py | 3 +- tests/subtests/helpers.py | 11 +- tests/subtests/summaries/test_generate.json | 156 ++++++++++++++++++ .../summaries/test_generate_hashes_only.json | 156 ++++++++++++++++++ .../summaries/test_generate_images_only.json | 156 ++++++++++++++++++ .../summaries/test_html_generate.json | 156 ++++++++++++++++++ .../test_html_generate_hashes_only.json | 156 ++++++++++++++++++ .../test_html_generate_images_only.json | 156 ++++++++++++++++++ tests/subtests/test_subtest.py | 69 +++++++- 9 files changed, 1014 insertions(+), 5 deletions(-) create mode 100644 tests/subtests/summaries/test_generate.json create mode 100644 tests/subtests/summaries/test_generate_hashes_only.json create mode 100644 tests/subtests/summaries/test_generate_images_only.json create mode 100644 tests/subtests/summaries/test_html_generate.json create mode 100644 tests/subtests/summaries/test_html_generate_hashes_only.json create mode 100644 tests/subtests/summaries/test_html_generate_images_only.json diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index a0b31997..e97669f0 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -689,7 +689,8 @@ def pytest_unconfigure(self, config): with open(hash_library_path, "w") as fp: json.dump(self._generated_hash_library, fp, indent=2) if self.results_always: # Make accessible in results directory - result_hash_library.name = hash_library_path.name # use same name as generated + # Use same name as generated + result_hash_library = self.results_dir / hash_library_path.name shutil.copy(hash_library_path, result_hash_library) elif self.results_always and self.results_hash_library_name: result_hashes = {k: v['result_hash'] for k, v in self._test_results.items() diff --git a/tests/subtests/helpers.py b/tests/subtests/helpers.py index 0c688843..0f345f03 100644 --- a/tests/subtests/helpers.py +++ b/tests/subtests/helpers.py @@ -9,7 +9,8 @@ class MatchError(Exception): pass -def diff_summary(baseline, result, baseline_hash_library=None, result_hash_library=None): +def diff_summary(baseline, result, baseline_hash_library=None, result_hash_library=None, + generating_hashes=False): """Diff a pytest-mpl summary dictionary. Parameters @@ -26,6 +27,9 @@ def diff_summary(baseline, result, baseline_hash_library=None, result_hash_libra Path to the "baseline" image hash library. Result hashes in the baseline summary are updated to these values to handle different Matplotlib versions. + generating_hashes : bool, optional, default=False + Whether `--mpl-generate-hash-library` was specified and + both of `--mpl-hash-library` and `hash_library=` were not. """ if baseline_hash_library and baseline_hash_library.exists(): # Load "correct" baseline hashes @@ -57,10 +61,13 @@ def diff_summary(baseline, result, baseline_hash_library=None, result_hash_libra # Swap the baseline and result hashes in the summary # for the corresponding hashes in each hash library - if baseline_hash_library and test in baseline_hash_library: + if baseline_hash_library and test in baseline_hash_library and not generating_hashes: baseline_summary = replace_hash(baseline_summary, 'baseline_hash', baseline_hash_library[test]) if result_hash_library: + if generating_hashes: # Newly generate result will appear as baseline_hash + baseline_summary = replace_hash(baseline_summary, 'baseline_hash', + result_hash_library[test]) baseline_summary = replace_hash(baseline_summary, 'result_hash', result_hash_library[test]) diff --git a/tests/subtests/summaries/test_generate.json b/tests/subtests/summaries/test_generate.json new file mode 100644 index 00000000..109b09ea --- /dev/null +++ b/tests/subtests/summaries/test_generate.json @@ -0,0 +1,156 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03", + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa", + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856", + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "8864803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e3c8de36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e4c06cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a", + "result_hash": null + } +} diff --git a/tests/subtests/summaries/test_generate_hashes_only.json b/tests/subtests/summaries/test_generate_hashes_only.json new file mode 100644 index 00000000..0da9359f --- /dev/null +++ b/tests/subtests/summaries/test_generate_hashes_only.json @@ -0,0 +1,156 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "passed", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03", + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "failed", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", + "rms": 11.100353848213828, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmatch_idiff/result.png", + "baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa", + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "failed", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmatch_imissing/result.png", + "baseline_hash": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "passed", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "failed", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", + "rms": 11.182677079602481, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_idiff/result.png", + "baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "failed", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hdiff_imissing/result.png", + "baseline_hash": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "passed", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856", + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "failed", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", + "rms": 12.12938597648977, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmissing_idiff/result.png", + "baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "failed", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmissing_imissing/result.png", + "baseline_hash": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "passed", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 200, + "result_image": null, + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "failed", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", + "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", + "rms": 29.260332173249314, + "tolerance": 3, + "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "passed", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "8864803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "passed", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "e3c8de36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "passed", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "e4c06cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a", + "result_hash": null + } +} diff --git a/tests/subtests/summaries/test_generate_images_only.json b/tests/subtests/summaries/test_generate_images_only.json new file mode 100644 index 00000000..7b59ed18 --- /dev/null +++ b/tests/subtests/summaries/test_generate_images_only.json @@ -0,0 +1,156 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + } +} diff --git a/tests/subtests/summaries/test_html_generate.json b/tests/subtests/summaries/test_html_generate.json new file mode 100644 index 00000000..f5de8f30 --- /dev/null +++ b/tests/subtests/summaries/test_html_generate.json @@ -0,0 +1,156 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03", + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa", + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856", + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "8864803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e3c8de36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e4c06cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a", + "result_hash": null + } +} diff --git a/tests/subtests/summaries/test_html_generate_hashes_only.json b/tests/subtests/summaries/test_html_generate_hashes_only.json new file mode 100644 index 00000000..02749b03 --- /dev/null +++ b/tests/subtests/summaries/test_html_generate_hashes_only.json @@ -0,0 +1,156 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "passed", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmatch_imatch/result.png", + "baseline_hash": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03", + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "failed", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", + "rms": 11.100353848213828, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmatch_idiff/result.png", + "baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa", + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "failed", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmatch_imissing/result.png", + "baseline_hash": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "passed", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_imatch/result.png", + "baseline_hash": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "failed", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", + "rms": 11.182677079602481, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_idiff/result.png", + "baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "failed", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hdiff_imissing/result.png", + "baseline_hash": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "passed", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmissing_imatch/result.png", + "baseline_hash": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856", + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "failed", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", + "rms": 12.12938597648977, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmissing_idiff/result.png", + "baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "failed", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmissing_imissing/result.png", + "baseline_hash": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "passed", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 200, + "result_image": "subtests.subtest.test_hdiff_imatch_tolerance/result.png", + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "failed", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", + "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", + "rms": 29.260332173249314, + "tolerance": 3, + "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "passed", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_imatch_savefig/result.png", + "baseline_hash": "8864803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "passed", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_imatch_style/result.png", + "baseline_hash": "e3c8de36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "passed", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_imatch_removetext/result.png", + "baseline_hash": "e4c06cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a", + "result_hash": null + } +} diff --git a/tests/subtests/summaries/test_html_generate_images_only.json b/tests/subtests/summaries/test_html_generate_images_only.json new file mode 100644 index 00000000..9107bb94 --- /dev/null +++ b/tests/subtests/summaries/test_html_generate_images_only.json @@ -0,0 +1,156 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "skipped", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + } +} diff --git a/tests/subtests/test_subtest.py b/tests/subtests/test_subtest.py index 0a31f7ff..eb85aa5c 100644 --- a/tests/subtests/test_subtest.py +++ b/tests/subtests/test_subtest.py @@ -34,7 +34,7 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=True, - has_result_hashes=False, + has_result_hashes=False, generating_hashes=False, update_baseline=UPDATE_BASELINE, update_summary=UPDATE_SUMMARY): """ Run pytest (within pytest) and check JSON summary report. @@ -53,6 +53,9 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=Tru has_result_hashes : bool or str, optional, default=False Whether a hash library is expected to exist in the results directory. If a string, this is the name of the expected results file. + generating_hashes : bool, optional, default=False + Whether `--mpl-generate-hash-library` was specified and + both of `--mpl-hash-library` and `hash_library=` were not. """ # Parse arguments if summaries is None: @@ -109,7 +112,8 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=Tru # Compare summaries diff_summary(baseline_summary, result_summary, - baseline_hash_library=HASH_LIBRARY, result_hash_library=RESULT_LIBRARY) + baseline_hash_library=HASH_LIBRARY, result_hash_library=RESULT_LIBRARY, + generating_hashes=generating_hashes) # Ensure reported images exist assert_existence(result_summary, path=results_path) @@ -186,3 +190,64 @@ def test_basic_html(tmp_path): [HASH_LIBRARY_FLAG, *BASELINE_IMAGES_FLAG_REL], summaries=['basic-html'], has_result_hashes=True) assert (tmp_path / 'results' / 'fig_comparison_basic.html').exists() + + +@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") +def test_generate(tmp_path): + # generating hashes and images; no testing + run_subtest('test_generate', tmp_path, + [rf'--mpl-generate-path={tmp_path}', + rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}'], + xfail=False, generating_hashes=True) + + +def test_generate_images_only(tmp_path): + # generating images; no testing + run_subtest('test_generate_images_only', tmp_path, + [rf'--mpl-generate-path={tmp_path}'], xfail=False) + + +@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") +def test_generate_hashes_only(tmp_path): + # generating hashes; testing images + run_subtest('test_generate_hashes_only', tmp_path, + [rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}'], + generating_hashes=True) + + +@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") +def test_html_generate(tmp_path): + # generating hashes and images; no testing + run_subtest('test_html_generate', tmp_path, + [rf'--mpl-generate-path={tmp_path}', + rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}'], + summaries=['html'], xfail=False, has_result_hashes="test_hashes.json", + generating_hashes=True) + assert (tmp_path / 'results' / 'fig_comparison.html').exists() + + +def test_html_generate_images_only(tmp_path): + # generating images; no testing + run_subtest('test_html_generate_images_only', tmp_path, + [rf'--mpl-generate-path={tmp_path}'], + summaries=['html'], xfail=False) + assert (tmp_path / 'results' / 'fig_comparison.html').exists() + + +@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") +def test_html_generate_hashes_only(tmp_path): + # generating hashes; testing images + run_subtest('test_html_generate_hashes_only', tmp_path, + [rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}'], + summaries=['html'], has_result_hashes="test_hashes.json", generating_hashes=True) + assert (tmp_path / 'results' / 'fig_comparison.html').exists() + + +@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") +def test_html_run_generate_hashes_only(tmp_path): + # generating hashes; testing hashes + run_subtest('test_html_hashes_only', tmp_path, + [rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}', + HASH_LIBRARY_FLAG], + summaries=['html'], has_result_hashes="test_hashes.json") + assert (tmp_path / 'results' / 'fig_comparison.html').exists() From 6388f6f36728c97104a0f67d1476e1ae30f79b8f Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Thu, 17 Feb 2022 21:30:41 +0000 Subject: [PATCH 09/19] Download hash library in HTML --- pytest_mpl/plugin.py | 8 ++++++-- pytest_mpl/summary/html.py | 14 ++++++++++---- pytest_mpl/summary/templates/basic.html | 5 +++++ pytest_mpl/summary/templates/filter.html | 7 +++++++ pytest_mpl/summary/templates/styles.css | 3 +++ 5 files changed, 31 insertions(+), 6 deletions(-) diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index e97669f0..7a0de40f 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -700,14 +700,18 @@ def pytest_unconfigure(self, config): json.dump(result_hashes, fp, indent=2) if self.generate_summary: + kwargs = {} if 'json' in self.generate_summary: summary = self.generate_summary_json() print(f"A JSON report can be found at: {summary}") + if result_hash_library.exists(): # link to it in the HTML + kwargs["hash_library"] = result_hash_library.name if 'html' in self.generate_summary: - summary = generate_summary_html(self._test_results, self.results_dir) + summary = generate_summary_html(self._test_results, self.results_dir, **kwargs) print(f"A summary of the failed tests can be found at: {summary}") if 'basic-html' in self.generate_summary: - summary = generate_summary_basic_html(self._test_results, self.results_dir) + summary = generate_summary_basic_html(self._test_results, self.results_dir, + **kwargs) print(f"A summary of the failed tests can be found at: {summary}") diff --git a/pytest_mpl/summary/html.py b/pytest_mpl/summary/html.py index aab93d27..082eb22b 100644 --- a/pytest_mpl/summary/html.py +++ b/pytest_mpl/summary/html.py @@ -227,7 +227,7 @@ def hash_status_msg(status): return messages[status] -def generate_summary_html(results, results_dir): +def generate_summary_html(results, results_dir, hash_library=None): """Generate the HTML summary. Parameters @@ -236,6 +236,9 @@ def generate_summary_html(results, results_dir): The `pytest_mpl.plugin.ImageComparison._test_results` object. results_dir : Path Path to the output directory. + hash_library : str, optional, default=None + Filename of the generated hash library at the root of `results_dir`. + Will be linked to in HTML if not None. """ # Initialize Jinja @@ -251,7 +254,7 @@ def generate_summary_html(results, results_dir): # Render HTML starting from the base template template = env.get_template("base.html") - html = template.render(results=Results(results)) + html = template.render(results=Results(results), hash_library=hash_library) # Write files for file in ['styles.css', 'extra.js', 'hash.svg', 'image.svg']: @@ -264,7 +267,7 @@ def generate_summary_html(results, results_dir): return html_file -def generate_summary_basic_html(results, results_dir): +def generate_summary_basic_html(results, results_dir, hash_library=None): """Generate the basic HTML summary. Parameters @@ -273,6 +276,9 @@ def generate_summary_basic_html(results, results_dir): The `pytest_mpl.plugin.ImageComparison._test_results` object. results_dir : Path Path to the output directory. + hash_library : str, optional, default=None + Filename of the generated hash library at the root of `results_dir`. + Will be linked to in HTML if not None. """ # Initialize Jinja @@ -283,7 +289,7 @@ def generate_summary_basic_html(results, results_dir): # Render HTML starting from the base template template = env.get_template("basic.html") - html = template.render(results=Results(results)) + html = template.render(results=Results(results), hash_library=hash_library) # Write files html_file = results_dir / 'fig_comparison_basic.html' diff --git a/pytest_mpl/summary/templates/basic.html b/pytest_mpl/summary/templates/basic.html index 3feb1095..27e6417c 100644 --- a/pytest_mpl/summary/templates/basic.html +++ b/pytest_mpl/summary/templates/basic.html @@ -48,6 +48,11 @@

Image test comparison

of those have a matching baseline image {%- endif %}

+{% if hash_library -%} +

+ Download generated hash library +

+{%- endif %} diff --git a/pytest_mpl/summary/templates/filter.html b/pytest_mpl/summary/templates/filter.html index 3cd8432f..a9e8edbe 100644 --- a/pytest_mpl/summary/templates/filter.html +++ b/pytest_mpl/summary/templates/filter.html @@ -62,5 +62,12 @@
Show tests which have...
+ {% if hash_library -%} +
+ Download generated hash library + {%- endif %} diff --git a/pytest_mpl/summary/templates/styles.css b/pytest_mpl/summary/templates/styles.css index 3effb1df..48a80186 100644 --- a/pytest_mpl/summary/templates/styles.css +++ b/pytest_mpl/summary/templates/styles.css @@ -35,6 +35,9 @@ body.no-hash-test .mpl-hash { pre { white-space: pre-line; } +.tooltip-inner pre { + margin: 0; +} div.result div.status-badge button img { vertical-align: sub; } From 113acba96cd220cea3723baae10fc0e516a822f3 Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Sat, 19 Feb 2022 22:20:40 +0000 Subject: [PATCH 10/19] Refactor image_status and hash_status to plugin.py Include specific results from the image and hash test in the JSON summaries instead of infering them from RMS etc. --- pytest_mpl/plugin.py | 14 +++++++++- pytest_mpl/summary/html.py | 26 ----------------- tests/subtests/summaries/test_default.json | 28 +++++++++++++++++++ tests/subtests/summaries/test_generate.json | 28 +++++++++++++++++++ .../summaries/test_generate_hashes_only.json | 28 +++++++++++++++++++ .../summaries/test_generate_images_only.json | 28 +++++++++++++++++++ tests/subtests/summaries/test_hash.json | 28 +++++++++++++++++++ .../summaries/test_html_generate.json | 28 +++++++++++++++++++ .../test_html_generate_hashes_only.json | 28 +++++++++++++++++++ .../test_html_generate_images_only.json | 28 +++++++++++++++++++ .../summaries/test_html_hashes_only.json | 28 +++++++++++++++++++ .../summaries/test_html_images_only.json | 28 +++++++++++++++++++ tests/subtests/summaries/test_hybrid.json | 28 +++++++++++++++++++ .../summaries/test_results_always.json | 28 +++++++++++++++++++ 14 files changed, 349 insertions(+), 27 deletions(-) diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 7a0de40f..3891dbc1 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -442,6 +442,7 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): if not os.path.exists(baseline_image_ref): summary['status'] = 'failed' + summary['image_status'] = 'missing' error_message = ("Image file not found for comparison test in: \n\t" f"{self.get_baseline_directory(item)}\n" "(This is expected for new tests.)\n" @@ -463,6 +464,7 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): actual_shape = imread(str(test_image)).shape[:2] if expected_shape != actual_shape: summary['status'] = 'failed' + summary['image_status'] = 'diff' error_message = SHAPE_MISMATCH_ERROR.format(expected_path=baseline_image, expected_shape=expected_shape, actual_path=test_image, @@ -474,10 +476,12 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): summary['tolerance'] = tolerance if results is None: summary['status'] = 'passed' + summary['image_status'] = 'match' summary['status_msg'] = 'Image comparison passed.' return None else: summary['status'] = 'failed' + summary['image_status'] = 'diff' summary['rms'] = results['rms'] diff_image = (result_dir / 'result-failed-diff.png').absolute() summary['diff_image'] = diff_image.relative_to(self.results_dir).as_posix() @@ -523,14 +527,17 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): if baseline_hash is None: # hash-missing summary['status'] = 'failed' + summary['hash_status'] = 'missing' summary['status_msg'] = (f"Hash for test '{hash_name}' not found in {hash_library_filename}. " f"Generated hash is {test_hash}.") elif test_hash == baseline_hash: # hash-match hash_comparison_pass = True summary['status'] = 'passed' + summary['hash_status'] = 'match' summary['status_msg'] = 'Test hash matches baseline hash.' else: # hash-diff summary['status'] = 'failed' + summary['hash_status'] = 'diff' summary['status_msg'] = (f"Hash {test_hash} doesn't match hash " f"{baseline_hash} in library " f"{hash_library_filename} for test {hash_name}.") @@ -555,7 +562,8 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): except Exception as baseline_error: # Append to test error later baseline_comparison = str(baseline_error) else: # Update main summary - for k in ['baseline_image', 'diff_image', 'rms', 'tolerance', 'result_image']: + for k in ['image_status', 'baseline_image', 'diff_image', + 'rms', 'tolerance', 'result_image']: summary[k] = summary[k] or baseline_summary.get(k) # Append the log from image comparison @@ -612,6 +620,8 @@ def item_function_wrapper(*args, **kwargs): summary = { 'status': None, + 'image_status': None, + 'hash_status': None, 'status_msg': None, 'baseline_image': None, 'diff_image': None, @@ -626,6 +636,7 @@ def item_function_wrapper(*args, **kwargs): # reference images or simply running the test. if self.generate_dir is not None: summary['status'] = 'skipped' + summary['image_status'] = 'generated' summary['status_msg'] = 'Skipped test, since generating image.' generate_image = self.generate_baseline_image(item, fig) if self.results_always: # Make baseline image available in HTML @@ -635,6 +646,7 @@ def item_function_wrapper(*args, **kwargs): result_image.relative_to(self.results_dir).as_posix() if self.generate_hash_library is not None: + summary['hash_status'] = 'generated' image_hash = self.generate_image_hash(item, fig) self._generated_hash_library[test_name] = image_hash summary['baseline_hash'] = image_hash diff --git a/pytest_mpl/summary/html.py b/pytest_mpl/summary/html.py index 082eb22b..ae680d88 100644 --- a/pytest_mpl/summary/html.py +++ b/pytest_mpl/summary/html.py @@ -107,32 +107,6 @@ def __init__(self, name, item, id, warn_missing): ('hash', self.hash_status), ]] - @cached_property - def image_status(self): - """Status of the image comparison test.""" - if self.rms is None and self.tolerance is not None: - return 'match' - elif self.rms is not None: - return 'diff' - elif self.baseline_image is None: - return 'missing' - else: - return 'generated' - - @cached_property - def hash_status(self): - """Status of the hash comparison test.""" - if self.baseline_hash is not None or self.result_hash is not None: - if self.baseline_hash is None: - return 'missing' - elif self.result_hash is None: - return 'generated' - elif self.baseline_hash == self.result_hash: - return 'match' - else: - return 'diff' - return None - @cached_property def indexes(self): """Dictionary with strings optimized for sorting.""" diff --git a/tests/subtests/summaries/test_default.json b/tests/subtests/summaries/test_default.json index a4acadc6..b4b51444 100644 --- a/tests/subtests/summaries/test_default.json +++ b/tests/subtests/summaries/test_default.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -34,6 +40,8 @@ }, "subtests.subtest.test_hdiff_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -45,6 +53,8 @@ }, "subtests.subtest.test_hdiff_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", @@ -56,6 +66,8 @@ }, "subtests.subtest.test_hdiff_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -67,6 +79,8 @@ }, "subtests.subtest.test_hmissing_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -78,6 +92,8 @@ }, "subtests.subtest.test_hmissing_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", @@ -89,6 +105,8 @@ }, "subtests.subtest.test_hmissing_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -100,6 +118,8 @@ }, "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -111,6 +131,8 @@ }, "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", @@ -122,6 +144,8 @@ }, "subtests.subtest.test_hdiff_imatch_savefig": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -133,6 +157,8 @@ }, "subtests.subtest.test_hdiff_imatch_style": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -144,6 +170,8 @@ }, "subtests.subtest.test_hdiff_imatch_removetext": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, diff --git a/tests/subtests/summaries/test_generate.json b/tests/subtests/summaries/test_generate.json index 109b09ea..6f9f78e2 100644 --- a/tests/subtests/summaries/test_generate.json +++ b/tests/subtests/summaries/test_generate.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -34,6 +40,8 @@ }, "subtests.subtest.test_hdiff_imatch": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -45,6 +53,8 @@ }, "subtests.subtest.test_hdiff_idiff": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -56,6 +66,8 @@ }, "subtests.subtest.test_hdiff_imissing": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -67,6 +79,8 @@ }, "subtests.subtest.test_hmissing_imatch": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -78,6 +92,8 @@ }, "subtests.subtest.test_hmissing_idiff": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -89,6 +105,8 @@ }, "subtests.subtest.test_hmissing_imissing": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -100,6 +118,8 @@ }, "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -111,6 +131,8 @@ }, "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -122,6 +144,8 @@ }, "subtests.subtest.test_hdiff_imatch_savefig": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -133,6 +157,8 @@ }, "subtests.subtest.test_hdiff_imatch_style": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -144,6 +170,8 @@ }, "subtests.subtest.test_hdiff_imatch_removetext": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, diff --git a/tests/subtests/summaries/test_generate_hashes_only.json b/tests/subtests/summaries/test_generate_hashes_only.json index 0da9359f..8dee25fa 100644 --- a/tests/subtests/summaries/test_generate_hashes_only.json +++ b/tests/subtests/summaries/test_generate_hashes_only.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": "match", + "hash_status": "generated", "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": "generated", "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -34,6 +40,8 @@ }, "subtests.subtest.test_hdiff_imatch": { "status": "passed", + "image_status": "match", + "hash_status": "generated", "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -45,6 +53,8 @@ }, "subtests.subtest.test_hdiff_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", @@ -56,6 +66,8 @@ }, "subtests.subtest.test_hdiff_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": "generated", "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -67,6 +79,8 @@ }, "subtests.subtest.test_hmissing_imatch": { "status": "passed", + "image_status": "match", + "hash_status": "generated", "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -78,6 +92,8 @@ }, "subtests.subtest.test_hmissing_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", @@ -89,6 +105,8 @@ }, "subtests.subtest.test_hmissing_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": "generated", "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -100,6 +118,8 @@ }, "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "passed", + "image_status": "match", + "hash_status": "generated", "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -111,6 +131,8 @@ }, "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", + "image_status": "diff", + "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", @@ -122,6 +144,8 @@ }, "subtests.subtest.test_hdiff_imatch_savefig": { "status": "passed", + "image_status": "match", + "hash_status": "generated", "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -133,6 +157,8 @@ }, "subtests.subtest.test_hdiff_imatch_style": { "status": "passed", + "image_status": "match", + "hash_status": "generated", "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -144,6 +170,8 @@ }, "subtests.subtest.test_hdiff_imatch_removetext": { "status": "passed", + "image_status": "match", + "hash_status": "generated", "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, diff --git a/tests/subtests/summaries/test_generate_images_only.json b/tests/subtests/summaries/test_generate_images_only.json index 7b59ed18..7720807d 100644 --- a/tests/subtests/summaries/test_generate_images_only.json +++ b/tests/subtests/summaries/test_generate_images_only.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -34,6 +40,8 @@ }, "subtests.subtest.test_hdiff_imatch": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -45,6 +53,8 @@ }, "subtests.subtest.test_hdiff_idiff": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -56,6 +66,8 @@ }, "subtests.subtest.test_hdiff_imissing": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -67,6 +79,8 @@ }, "subtests.subtest.test_hmissing_imatch": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -78,6 +92,8 @@ }, "subtests.subtest.test_hmissing_idiff": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -89,6 +105,8 @@ }, "subtests.subtest.test_hmissing_imissing": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -100,6 +118,8 @@ }, "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -111,6 +131,8 @@ }, "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -122,6 +144,8 @@ }, "subtests.subtest.test_hdiff_imatch_savefig": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -133,6 +157,8 @@ }, "subtests.subtest.test_hdiff_imatch_style": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, @@ -144,6 +170,8 @@ }, "subtests.subtest.test_hdiff_imatch_removetext": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": null, "diff_image": null, diff --git a/tests/subtests/summaries/test_hash.json b/tests/subtests/summaries/test_hash.json index bec04c5a..4304bf03 100644 --- a/tests/subtests/summaries/test_hash.json +++ b/tests/subtests/summaries/test_hash.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -35,6 +41,8 @@ "subtests.subtest.test_hdiff_imatch": { "status": "failed", "status_msg": "REGEX:Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -46,6 +54,8 @@ "subtests.subtest.test_hdiff_idiff": { "status": "failed", "status_msg": "REGEX:Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -57,6 +67,8 @@ "subtests.subtest.test_hdiff_imissing": { "status": "failed", "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -68,6 +80,8 @@ "subtests.subtest.test_hmissing_imatch": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -79,6 +93,8 @@ "subtests.subtest.test_hmissing_idiff": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -90,6 +106,8 @@ "subtests.subtest.test_hmissing_imissing": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -101,6 +119,8 @@ "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -112,6 +132,8 @@ "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -123,6 +145,8 @@ "subtests.subtest.test_hdiff_imatch_savefig": { "status": "failed", "status_msg": "REGEX:Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -134,6 +158,8 @@ "subtests.subtest.test_hdiff_imatch_style": { "status": "failed", "status_msg": "REGEX:Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -145,6 +171,8 @@ "subtests.subtest.test_hdiff_imatch_removetext": { "status": "failed", "status_msg": "REGEX:Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, diff --git a/tests/subtests/summaries/test_html_generate.json b/tests/subtests/summaries/test_html_generate.json index f5de8f30..84a8e8b9 100644 --- a/tests/subtests/summaries/test_html_generate.json +++ b/tests/subtests/summaries/test_html_generate.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", "diff_image": null, @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hmatch_imissing/baseline.png", "diff_image": null, @@ -34,6 +40,8 @@ }, "subtests.subtest.test_hdiff_imatch": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", "diff_image": null, @@ -45,6 +53,8 @@ }, "subtests.subtest.test_hdiff_idiff": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", "diff_image": null, @@ -56,6 +66,8 @@ }, "subtests.subtest.test_hdiff_imissing": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hdiff_imissing/baseline.png", "diff_image": null, @@ -67,6 +79,8 @@ }, "subtests.subtest.test_hmissing_imatch": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", "diff_image": null, @@ -78,6 +92,8 @@ }, "subtests.subtest.test_hmissing_idiff": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", "diff_image": null, @@ -89,6 +105,8 @@ }, "subtests.subtest.test_hmissing_imissing": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hmissing_imissing/baseline.png", "diff_image": null, @@ -100,6 +118,8 @@ }, "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", "diff_image": null, @@ -111,6 +131,8 @@ }, "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", "diff_image": null, @@ -122,6 +144,8 @@ }, "subtests.subtest.test_hdiff_imatch_savefig": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", "diff_image": null, @@ -133,6 +157,8 @@ }, "subtests.subtest.test_hdiff_imatch_style": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", "diff_image": null, @@ -144,6 +170,8 @@ }, "subtests.subtest.test_hdiff_imatch_removetext": { "status": "skipped", + "image_status": "generated", + "hash_status": "generated", "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", "diff_image": null, diff --git a/tests/subtests/summaries/test_html_generate_hashes_only.json b/tests/subtests/summaries/test_html_generate_hashes_only.json index 02749b03..0a813734 100644 --- a/tests/subtests/summaries/test_html_generate_hashes_only.json +++ b/tests/subtests/summaries/test_html_generate_hashes_only.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": "match", + "hash_status": "generated", "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": "generated", "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -34,6 +40,8 @@ }, "subtests.subtest.test_hdiff_imatch": { "status": "passed", + "image_status": "match", + "hash_status": "generated", "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", "diff_image": null, @@ -45,6 +53,8 @@ }, "subtests.subtest.test_hdiff_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", @@ -56,6 +66,8 @@ }, "subtests.subtest.test_hdiff_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": "generated", "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -67,6 +79,8 @@ }, "subtests.subtest.test_hmissing_imatch": { "status": "passed", + "image_status": "match", + "hash_status": "generated", "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", "diff_image": null, @@ -78,6 +92,8 @@ }, "subtests.subtest.test_hmissing_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", @@ -89,6 +105,8 @@ }, "subtests.subtest.test_hmissing_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": "generated", "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -100,6 +118,8 @@ }, "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "passed", + "image_status": "match", + "hash_status": "generated", "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", "diff_image": null, @@ -111,6 +131,8 @@ }, "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", + "image_status": "diff", + "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", @@ -122,6 +144,8 @@ }, "subtests.subtest.test_hdiff_imatch_savefig": { "status": "passed", + "image_status": "match", + "hash_status": "generated", "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", "diff_image": null, @@ -133,6 +157,8 @@ }, "subtests.subtest.test_hdiff_imatch_style": { "status": "passed", + "image_status": "match", + "hash_status": "generated", "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", "diff_image": null, @@ -144,6 +170,8 @@ }, "subtests.subtest.test_hdiff_imatch_removetext": { "status": "passed", + "image_status": "match", + "hash_status": "generated", "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", "diff_image": null, diff --git a/tests/subtests/summaries/test_html_generate_images_only.json b/tests/subtests/summaries/test_html_generate_images_only.json index 9107bb94..bed75fae 100644 --- a/tests/subtests/summaries/test_html_generate_images_only.json +++ b/tests/subtests/summaries/test_html_generate_images_only.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", "diff_image": null, @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hmatch_imissing/baseline.png", "diff_image": null, @@ -34,6 +40,8 @@ }, "subtests.subtest.test_hdiff_imatch": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", "diff_image": null, @@ -45,6 +53,8 @@ }, "subtests.subtest.test_hdiff_idiff": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", "diff_image": null, @@ -56,6 +66,8 @@ }, "subtests.subtest.test_hdiff_imissing": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hdiff_imissing/baseline.png", "diff_image": null, @@ -67,6 +79,8 @@ }, "subtests.subtest.test_hmissing_imatch": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", "diff_image": null, @@ -78,6 +92,8 @@ }, "subtests.subtest.test_hmissing_idiff": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", "diff_image": null, @@ -89,6 +105,8 @@ }, "subtests.subtest.test_hmissing_imissing": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hmissing_imissing/baseline.png", "diff_image": null, @@ -100,6 +118,8 @@ }, "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", "diff_image": null, @@ -111,6 +131,8 @@ }, "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", "diff_image": null, @@ -122,6 +144,8 @@ }, "subtests.subtest.test_hdiff_imatch_savefig": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", "diff_image": null, @@ -133,6 +157,8 @@ }, "subtests.subtest.test_hdiff_imatch_style": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", "diff_image": null, @@ -144,6 +170,8 @@ }, "subtests.subtest.test_hdiff_imatch_removetext": { "status": "skipped", + "image_status": "generated", + "hash_status": null, "status_msg": "Skipped test, since generating image.", "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", "diff_image": null, diff --git a/tests/subtests/summaries/test_html_hashes_only.json b/tests/subtests/summaries/test_html_hashes_only.json index 448ce9c2..a55666ac 100644 --- a/tests/subtests/summaries/test_html_hashes_only.json +++ b/tests/subtests/summaries/test_html_hashes_only.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -35,6 +41,8 @@ "subtests.subtest.test_hdiff_imatch": { "status": "failed", "status_msg": "REGEX:Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -46,6 +54,8 @@ "subtests.subtest.test_hdiff_idiff": { "status": "failed", "status_msg": "REGEX:Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -57,6 +67,8 @@ "subtests.subtest.test_hdiff_imissing": { "status": "failed", "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -68,6 +80,8 @@ "subtests.subtest.test_hmissing_imatch": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -79,6 +93,8 @@ "subtests.subtest.test_hmissing_idiff": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -90,6 +106,8 @@ "subtests.subtest.test_hmissing_imissing": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -101,6 +119,8 @@ "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -112,6 +132,8 @@ "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -123,6 +145,8 @@ "subtests.subtest.test_hdiff_imatch_savefig": { "status": "failed", "status_msg": "REGEX:Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -134,6 +158,8 @@ "subtests.subtest.test_hdiff_imatch_style": { "status": "failed", "status_msg": "REGEX:Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -145,6 +171,8 @@ "subtests.subtest.test_hdiff_imatch_removetext": { "status": "failed", "status_msg": "REGEX:Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, diff --git a/tests/subtests/summaries/test_html_images_only.json b/tests/subtests/summaries/test_html_images_only.json index c8a5058c..0b33035e 100644 --- a/tests/subtests/summaries/test_html_images_only.json +++ b/tests/subtests/summaries/test_html_images_only.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -34,6 +40,8 @@ }, "subtests.subtest.test_hdiff_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", "diff_image": null, @@ -45,6 +53,8 @@ }, "subtests.subtest.test_hdiff_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", @@ -56,6 +66,8 @@ }, "subtests.subtest.test_hdiff_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -67,6 +79,8 @@ }, "subtests.subtest.test_hmissing_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", "diff_image": null, @@ -78,6 +92,8 @@ }, "subtests.subtest.test_hmissing_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", @@ -89,6 +105,8 @@ }, "subtests.subtest.test_hmissing_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -100,6 +118,8 @@ }, "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", "diff_image": null, @@ -111,6 +131,8 @@ }, "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", @@ -122,6 +144,8 @@ }, "subtests.subtest.test_hdiff_imatch_savefig": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", "diff_image": null, @@ -133,6 +157,8 @@ }, "subtests.subtest.test_hdiff_imatch_style": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", "diff_image": null, @@ -144,6 +170,8 @@ }, "subtests.subtest.test_hdiff_imatch_removetext": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", "diff_image": null, diff --git a/tests/subtests/summaries/test_hybrid.json b/tests/subtests/summaries/test_hybrid.json index 2f2930b0..851e67f1 100644 --- a/tests/subtests/summaries/test_hybrid.json +++ b/tests/subtests/summaries/test_hybrid.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -35,6 +41,8 @@ "subtests.subtest.test_hdiff_imatch": { "status": "failed", "status_msg": "REGEX:Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", "diff_image": null, "rms": null, @@ -46,6 +54,8 @@ "subtests.subtest.test_hdiff_idiff": { "status": "failed", "status_msg": "REGEX:Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "image_status": "diff", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", "rms": 11.182677079602481, @@ -57,6 +67,8 @@ "subtests.subtest.test_hdiff_imissing": { "status": "failed", "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "image_status": "missing", + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -68,6 +80,8 @@ "subtests.subtest.test_hmissing_imatch": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "missing", "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", "diff_image": null, "rms": null, @@ -79,6 +93,8 @@ "subtests.subtest.test_hmissing_idiff": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "image_status": "diff", + "hash_status": "missing", "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", "rms": 12.12938597648977, @@ -90,6 +106,8 @@ "subtests.subtest.test_hmissing_imissing": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "image_status": "missing", + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -101,6 +119,8 @@ "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", "diff_image": null, "rms": null, @@ -112,6 +132,8 @@ "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", + "image_status": "diff", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", "rms": 29.260332173249314, @@ -123,6 +145,8 @@ "subtests.subtest.test_hdiff_imatch_savefig": { "status": "failed", "status_msg": "REGEX:Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", "diff_image": null, "rms": null, @@ -134,6 +158,8 @@ "subtests.subtest.test_hdiff_imatch_style": { "status": "failed", "status_msg": "REGEX:Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", "diff_image": null, "rms": null, @@ -145,6 +171,8 @@ "subtests.subtest.test_hdiff_imatch_removetext": { "status": "failed", "status_msg": "REGEX:Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", "diff_image": null, "rms": null, diff --git a/tests/subtests/summaries/test_results_always.json b/tests/subtests/summaries/test_results_always.json index c42fee29..ea7af1d2 100644 --- a/tests/subtests/summaries/test_results_always.json +++ b/tests/subtests/summaries/test_results_always.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": "match", + "hash_status": "match", "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "passed", + "image_status": "diff", + "hash_status": "match", "status_msg": "REGEX:Test hash matches baseline hash\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "passed", + "image_status": "missing", + "hash_status": "match", "status_msg": "REGEX:Test hash matches baseline hash\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -35,6 +41,8 @@ "subtests.subtest.test_hdiff_imatch": { "status": "failed", "status_msg": "REGEX:Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", "diff_image": null, "rms": null, @@ -46,6 +54,8 @@ "subtests.subtest.test_hdiff_idiff": { "status": "failed", "status_msg": "REGEX:Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "image_status": "diff", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", "rms": 11.182677079602481, @@ -57,6 +67,8 @@ "subtests.subtest.test_hdiff_imissing": { "status": "failed", "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "image_status": "missing", + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -68,6 +80,8 @@ "subtests.subtest.test_hmissing_imatch": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "missing", "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", "diff_image": null, "rms": null, @@ -79,6 +93,8 @@ "subtests.subtest.test_hmissing_idiff": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "image_status": "diff", + "hash_status": "missing", "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", "rms": 12.12938597648977, @@ -90,6 +106,8 @@ "subtests.subtest.test_hmissing_imissing": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "image_status": "missing", + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -101,6 +119,8 @@ "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", "diff_image": null, "rms": null, @@ -112,6 +132,8 @@ "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", + "image_status": "diff", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", "rms": 29.260332173249314, @@ -123,6 +145,8 @@ "subtests.subtest.test_hdiff_imatch_savefig": { "status": "failed", "status_msg": "REGEX:Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", "diff_image": null, "rms": null, @@ -134,6 +158,8 @@ "subtests.subtest.test_hdiff_imatch_style": { "status": "failed", "status_msg": "REGEX:Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", "diff_image": null, "rms": null, @@ -145,6 +171,8 @@ "subtests.subtest.test_hdiff_imatch_removetext": { "status": "failed", "status_msg": "REGEX:Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", "diff_image": null, "rms": null, From 39b8d83810624410f97d862d50e2c28d254976e0 Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Sat, 19 Feb 2022 23:00:08 +0000 Subject: [PATCH 11/19] Improve hiding of irrelevant info --- pytest_mpl/summary/html.py | 42 ++++++++----------- pytest_mpl/summary/templates/base.html | 2 +- pytest_mpl/summary/templates/extra.js | 7 ---- pytest_mpl/summary/templates/filter.html | 8 ++-- .../summary/templates/result_images.html | 16 ++++--- pytest_mpl/summary/templates/styles.css | 6 --- 6 files changed, 33 insertions(+), 48 deletions(-) diff --git a/pytest_mpl/summary/html.py b/pytest_mpl/summary/html.py index ae680d88..78fbcc19 100644 --- a/pytest_mpl/summary/html.py +++ b/pytest_mpl/summary/html.py @@ -26,26 +26,12 @@ class Results: def __init__(self, results, title="Image comparison"): self.title = title # HTML - # If any baseline images or baseline hashes are present, - # assume all results should have them - self.warn_missing = {'baseline_image': False, 'baseline_hash': False} - for key in self.warn_missing.keys(): - for result in results.values(): - if result[key] is not None: - self.warn_missing[key] = True - break - - # Additional <body> classes - self.classes = [] - if self.warn_missing['baseline_hash'] is False: - self.classes += ['no-hash-test'] - # Generate sorted list of results self.cards = [] pad = len(str(len(results.items()))) # maximum length of a result index for collect_n, (name, item) in enumerate(results.items()): card_id = str(collect_n).zfill(pad) # zero pad for alphanumerical sorting - self.cards += [Result(name, item, card_id, self.warn_missing)] + self.cards += [Result(name, item, card_id)] self.cards = sorted(self.cards, key=lambda i: i.indexes['status'], reverse=True) @cached_property @@ -66,6 +52,22 @@ def statistics(self): stats['skipped'] += 1 return stats + @cached_property + def image_comparison(self): + """Whether at least one image comparison test or generation was performed.""" + for result in self.cards: + if result.image_status: + return True + return False + + @cached_property + def hash_comparison(self): + """Whether at least one hash comparison test or generation was performed.""" + for result in self.cards: + if result.hash_status: + return True + return False + class Result: """ @@ -81,20 +83,14 @@ class Result: id : str The test number in order collected. Numbers must be zero padded due to alphanumerical sorting. - warn_missing : dict - Whether to include relevant status badges for images and/or hashes. - Must have keys ``baseline_image`` and ``baseline_hash``. """ - def __init__(self, name, item, id, warn_missing): + def __init__(self, name, item, id): # Make the summary dictionary available as attributes self.__dict__ = item # Sort index for collection order self.id = id - # Whether to show image and/or hash status badges - self.warn_missing = warn_missing - # Name of test with module and test function together and separate self.full_name = name self.name = name.split('.')[-1] @@ -154,8 +150,6 @@ def badges(self): """Additional badges to show beside overall status badge.""" for test_type, status_getter in [('image', image_status_msg), ('hash', hash_status_msg)]: status = getattr(self, f'{test_type}_status') - if not self.warn_missing[f'baseline_{test_type}']: - continue # not expected to exist if ( (status == 'missing') or (self.status == 'failed' and status == 'match') or diff --git a/pytest_mpl/summary/templates/base.html b/pytest_mpl/summary/templates/base.html index b34464b9..4ac3c66e 100644 --- a/pytest_mpl/summary/templates/base.html +++ b/pytest_mpl/summary/templates/base.html @@ -8,7 +8,7 @@ integrity="sha384-1BmE4kWBq78iYhFldvKuhfTAU6auU8tT94WrHftjDbrCEXSU1oBoqyl2QvZ6jIW3" crossorigin="anonymous"> <title>{{ results.title }} - + {% include 'navbar.html' %}
diff --git a/pytest_mpl/summary/templates/extra.js b/pytest_mpl/summary/templates/extra.js index cb80eefe..d5c3c341 100644 --- a/pytest_mpl/summary/templates/extra.js +++ b/pytest_mpl/summary/templates/extra.js @@ -1,10 +1,3 @@ -// Remove all elements of class mpl-hash if hash test not run -if (document.body.classList[0] == 'no-hash-test') { - document.querySelectorAll('.mpl-hash').forEach(function (elem) { - elem.parentElement.removeChild(elem); - }); -} - // Enable tooltips var tooltipTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="tooltip"]')) var tooltipList = tooltipTriggerList.map(function (tooltipTriggerEl) { diff --git a/pytest_mpl/summary/templates/filter.html b/pytest_mpl/summary/templates/filter.html index a9e8edbe..ad2d9803 100644 --- a/pytest_mpl/summary/templates/filter.html +++ b/pytest_mpl/summary/templates/filter.html @@ -16,7 +16,7 @@
Sort tests by...
{{ sort_option('status-sort', 'status', 'desc', default=true) }} {{ sort_option('collected-sort', 'collected', 'asc') }} {{ sort_option('test-name', 'name') }} - {% if results.warn_missing['baseline_image'] -%} + {% if results.image_comparison -%} {{ sort_option('rms-sort', 'RMS', 'desc') }} {%- endif %}
@@ -34,15 +34,15 @@
Show tests which have...
{{ filter_option('overall-failed', 'failed') }} {{ filter_option('overall-skipped', 'skipped') }}
- {% if results.warn_missing['baseline_image'] -%} + {% if results.image_comparison -%}
{{ filter_option('image-match', 'matching images') }} {{ filter_option('image-diff', 'differing images') }} {{ filter_option('image-missing', 'no baseline image') }}
{%- endif %} - {% if results.warn_missing['baseline_hash'] -%} -
+ {% if results.hash_comparison -%} +
{{ filter_option('hash-match', 'matching hashes') }} {{ filter_option('hash-diff', 'differing hashes') }} {{ filter_option('hash-missing', 'no baseline hash') }} diff --git a/pytest_mpl/summary/templates/result_images.html b/pytest_mpl/summary/templates/result_images.html index 60797105..36ab2943 100644 --- a/pytest_mpl/summary/templates/result_images.html +++ b/pytest_mpl/summary/templates/result_images.html @@ -6,14 +6,14 @@
{{ r.module }}
{{ r.name }}
- {% if results.warn_missing['baseline_image'] and r.image_status != 'generated' -%} + {% if r.image_status and r.image_status != 'generated' -%}
{% macro image_card(file, name) -%}
{{ name }}
{% if file -%} - + {{ name }} {%- endif %}
@@ -41,21 +41,25 @@
{{ r.name }}
{%- endfilter %} {%- endmacro -%} - {% if results.warn_missing['baseline_image'] -%} -
+ {% if r.image_status -%} +
{{ r.image_status | image_status_msg }}
+ {% if r.image_status == 'match' or r.image_status == 'diff' -%}
{{ pre_data('rms', r.rms_str, 'RMS') }} {{ pre_data('tolerance', r.tolerance, 'Tolerance') }}
+ {%- endif %}
{%- endif %} - {% if results.warn_missing['baseline_hash'] -%} -
+ {% if r.hash_status -%} +
{{ r.hash_status | hash_status_msg }}
{{ pre_data('baseline_hash', r.baseline_hash, 'Baseline') }} + {% if r.hash_status != 'generated' -%} {{ pre_data('result_hash', r.result_hash, 'Result') }} + {%- endif %}
{%- endif %} diff --git a/pytest_mpl/summary/templates/styles.css b/pytest_mpl/summary/templates/styles.css index 48a80186..6564238a 100644 --- a/pytest_mpl/summary/templates/styles.css +++ b/pytest_mpl/summary/templates/styles.css @@ -1,6 +1,3 @@ -body.no-hash-test .mpl-hash { - display: none; -} .navbar-brand span.repo { font-weight: bold; } @@ -29,9 +26,6 @@ body.no-hash-test .mpl-hash { max-width: 400px; } } -.image-missing .mpl-image .card-body { - display: none; -} pre { white-space: pre-line; } From 0b631141be56499eb9baebd35bf96fcab69b2684 Mon Sep 17 00:00:00 2001 From: Conor MacBride Date: Mon, 21 Feb 2022 12:12:54 +0000 Subject: [PATCH 12/19] Add hashes download to navbar --- CHANGES.md | 4 ++++ pytest_mpl/summary/templates/filter.html | 7 ------- pytest_mpl/summary/templates/navbar.html | 11 +++++++++++ pytest_mpl/summary/templates/styles.css | 3 +++ 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8e6c55e8..470ace47 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,10 @@ 0.15 (unreleased) ----------------- +- An updated hash library will be saved to the results directory when + generating a HTML summary page or when the `--mpl-results-always` flag is + set. A button to download this file is included in the HTML summary. + Various bugfixes, test improvements and documentation updates. [#138] 0.14 (2022-02-09) ----------------- diff --git a/pytest_mpl/summary/templates/filter.html b/pytest_mpl/summary/templates/filter.html index ad2d9803..81e282cc 100644 --- a/pytest_mpl/summary/templates/filter.html +++ b/pytest_mpl/summary/templates/filter.html @@ -62,12 +62,5 @@
Show tests which have...
- {% if hash_library -%} -
- Download generated hash library - {%- endif %}
diff --git a/pytest_mpl/summary/templates/navbar.html b/pytest_mpl/summary/templates/navbar.html index 3ba762ac..17fa7bfd 100644 --- a/pytest_mpl/summary/templates/navbar.html +++ b/pytest_mpl/summary/templates/navbar.html @@ -9,6 +9,17 @@ + {% if hash_library -%} + + + + + + {%- endif %}