8000 Save a result hash library to results dir · matplotlib/pytest-mpl@d19a995 · GitHub
[go: up one dir, main page]
Skip to content

Commit d19a995

Browse files
ConorMacBrideConorMacBride
committed
Save a result hash library to results dir
1 parent d4746bc commit d19a995

File tree

3 files changed

+56
-4
lines changed

3 files changed

+56
-4
lines changed

‎README.rst

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,12 @@ test (based on the hash library) also shown in the generated
229229
summary. This option is applied automatically when generating
230230
a HTML summary.
231231

232+
When the ``--mpl-results-always`` option is active, and some hash
233+
comparison tests are performed, a hash library containing all the
234+
result hashes will also be saved to the root of the results directory.
235+
The filename will be extracted from ``--mpl-generate-hash-library``,
236+
``--mpl-hash-library`` or ``hash_library=`` in that order.
237+
232238
Base style
233239
^^^^^^^^^^
234240

‎pytest_mpl/plugin.py

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -283,6 +283,12 @@ def __init__(self,
283283
self.results_dir = Path(tempfile.mkdtemp(dir=self.results_dir))
284284
self.results_dir.mkdir(parents=True, exist_ok=True)
285285

286+
# Decide what to call the downloadable results hash library
287+
if self.hash_library is not None:
288+
self.results_hash_library_name = self.hash_library.name
289+
else: # Use the first filename encountered in a `hash_library=` kwarg
290+
self.results_hash_library_name = None
291+
286292
# We need global state to store all the hashes generated over the run
287293
self._generated_hash_library = {}
288294
self._test_results = {}
@@ -499,6 +505,10 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None):
499505
compare = self.get_compare(item)
500506
savefig_kwargs = compare.kwargs.get('savefig_kwargs', {})
501507

508+
if not self.results_hash_library_name:
509+
# Use hash library name of current test as results hash library name
510+
self.results_hash_library_name = Path(compare.kwargs.get("hash_library", "")).name
511+
502512
hash_library_filename = self.hash_library or compare.kwargs.get('hash_library', None)
503513
hash_library_filename = (Path(item.fspath).parent / hash_library_filename).absolute()
504514

@@ -674,11 +684,21 @@ def pytest_unconfigure(self, config):
674684
"""
675685
Save out the hash library at the end of the run.
676686
"""
687+
result_hash_library = self.results_dir / (self.results_hash_library_name or "temp.json")
677688
if self.generate_hash_library is not None:
678689
hash_library_path = Path(config.rootdir) / self.generate_hash_library
679690
hash_library_path.parent.mkdir(parents=True, exist_ok=True)
680691
with open(hash_library_path, "w") as fp:
681692
json.dump(self._generated_hash_library, fp, indent=2)
693+
if self.results_always: # Make accessible in results directory
694+
result_hash_library.name = hash_library_path.name # use same name as generated
695+
shutil.copy(hash_library_path, result_hash_library)
696+
elif self.results_always and self.results_hash_library_name:
697+
result_hashes = {k: v['result_hash'] for k, v in self._test_results.items()
698+
if v['result_hash']}
699+
if len(result_hashes) > 0: # At least one hash comparison test
700+
with open(result_hash_library, "w") as fp:
701+
json.dump(result_hashes, fp, indent=2)
682702

683703
if self.generate_summary:
684704
if 'json' in self.generate_summary:

‎tests/subtests/test_subtest.py

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434

3535

3636
def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=True,
37+
has_result_hashes=False,
3738
update_baseline=UPDATE_BASELINE, update_summary=UPDATE_SUMMARY):
3839
""" Run pytest (within pytest) and check JSON summary report.
3940
@@ -49,6 +50,9 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=Tru
4950
Summaries to generate in addition to `json`.
5051
xfail : bool, optional, default=True
5152
Whether the overall pytest run should fail.
53+
has_result_hashes : bool or str, optional, default=False
54+
Whether a hash library is expected to exist in the results directory.
55+
If a string, this is the name of the expected results file.
5256
"""
5357
# Parse arguments
5458
if summaries is None:
@@ -110,6 +114,24 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=Tru
110114
# Ensure reported images exist
111115
assert_existence(result_summary, path=results_path)
112116

117+
# Get expected name for the hash library saved to the results directory
118+
if 6D40 isinstance(has_result_hashes, str):
119+
result_hash_file = tmp_path / 'results' / has_result_hashes
120+
has_result_hashes = True # convert to bool after processing str
121+
else:
122+
result_hash_file = tmp_path / 'results' / HASH_LIBRARY.name
123+
124+
# Compare the generated hash library to the expected hash library
125+
if has_result_hashes:
126+
assert result_hash_file.exists()
127+
with open(RESULT_LIBRARY, "r") as f:
128+
baseline = json.load(f)
129+
with open(result_hash_file, "r") as f:
130+
result = json.load(f)
131+
diff_summary({'a': baseline}, {'a': result})
132+
else:
133+
assert not result_hash_file.exists()
134+
113135

114136
def test_default(tmp_path):
115137
run_subtest('test_default', tmp_path, [])
@@ -128,21 +150,24 @@ def test_hybrid(tmp_path):
128150
@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version")
129151
def test_results_always(tmp_path):
130152
run_subtest('test_results_always', tmp_path,
131-
[HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS, '--mpl-results-always'])
153+
[HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS, '--mpl-results-always'],
154+
has_result_hashes=True)
132155

133156

134157
@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version")
135158
def test_html(tmp_path):
136159
run_subtest('test_results_always', tmp_path,
137-
[HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS], summaries=['html'])
160+
[HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS], summaries=['html'],
161+
has_result_hashes=True)
138162
assert (tmp_path / 'results' / 'fig_comparison.html').exists()
139163
assert (tmp_path / 'results' / 'extra.js').exists()
140164
assert (tmp_path / 'results' / 'styles.css').exists()
141165

142166

143167
@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version")
144168
def test_html_hashes_only(tmp_path):
145-
run_subtest('test_html_hashes_only', tmp_path, [HASH_LIBRARY_FLAG], summaries=['html'])
169+
run_subtest('test_html_hashes_only', tmp_path, [HASH_LIBRARY_FLAG], summaries=['html'],
170+
has_result_hashes=True)
146171
assert (tmp_path / 'results' / 'fig_comparison.html').exists()
147172
assert (tmp_path / 'results' / 'extra.js').exists()
148173
assert (tmp_path / 'results' / 'styles.css').exists()
@@ -158,5 +183,6 @@ def test_html_images_only(tmp_path):
158183
@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version")
159184
def test_basic_html(tmp_path):
160185
run_subtest('test_results_always', tmp_path,
161-
[HASH_LIBRARY_FLAG, *BASELINE_IMAGES_FLAG_REL], summaries=['basic-html'])
186+
[HASH_LIBRARY_FLAG, *BASELINE_IMAGES_FLAG_REL], summaries=['basic-html'],
187+
has_result_hashes=True)
162188
assert (tmp_path / 'results' / 'fig_comparison_basic.html').exists()

0 commit comments

Comments
 (0)
0