Support for version specific patches

matplotlib · Cadair · Feb 9, 2022 · Feb 3, 2022 · Feb 3, 2022 · Feb 4, 2022
commit 9f7deb0d3b0d194d3d16f07308aded9bb0d1c8eb
diff --git a/tests/subtests/__init__.py b/tests/subtests/__init__.py
diff --git a/tests/subtests/helpers.py b/tests/subtests/helpers.py
@@ -1,15 +1,21 @@
 import re
+import json
 from pathlib import Path
 
-__all__ = ['diff_summary', 'assert_existence']
+__all__ = ['diff_summary', 'assert_existence', 'replace_baseline_hash', 'patch_summary']
 
 
 class MatchError(Exception):
     pass
 
 
-def diff_summary(baseline, result):
+def diff_summary(baseline, result, hash_library=None):
     """Diff a pytest-mpl summary dictionary."""
+    if hash_library and hash_library.exists():
+        # Load "correct" baseline hashes
+        with open(hash_library, 'r') as f:
+            hash_library = json.load(f)
+
     # Get test names
     baseline_tests = set(baseline.keys())
     result_tests = set(result.keys())
@@ -23,6 +29,10 @@ def diff_summary(baseline, result):
         baseline_summary = baseline[test]
         result_summary = result[test]
 
+        # Swap the baseline hashes in the summary for the baseline hashes in the hash library
+        if hash_library:
+            baseline_summary = replace_baseline_hash(baseline_summary, hash_library[test])
+
         # Get keys of recorded items
         baseline_keys = set(baseline_summary.keys())
         result_keys = set(result_summary.keys())
@@ -78,6 +88,46 @@ def diff_dict_item(baseline, result, error=''):
     raise MatchError(error)
 
 
+def patch_summary(summary, patch_file):
+    """Replace in `summary` any items defined in `patch_file`."""
+    # By only applying patches, changes between MPL versions are more obvious.
+    with open(patch_file, 'r') as f:
+        patch = json.load(f)
+    for test, test_summary in patch.items():
+        for k, v in test_summary.items():
+            summary[test][k] = v
+    return summary
+
+
+def replace_baseline_hash(summary, new_baseline):
+    """Replace a baseline hash in a pytest-mpl summary with a different baseline.
+
+    Result hashes which match the existing baseline are also updated.
+
+    Parameters
+    ----------
+    summary : dict
+        A single test from a pytest-mpl summary.
+    new_baseline : str
+        The new baseline.
+    """
+    assert isinstance(new_baseline, str)
+    old_baseline = summary['baseline_hash']
+    if not isinstance(old_baseline, str) or old_baseline == new_baseline:
+        return summary  # Either already correct or missing
+
+    # If the old result hash matches the old baseline hash, also update the result hash
+    old_result = summary['result_hash']
+    if isinstance(old_result, str) and old_result == old_baseline:
+        summary['result_hash'] = new_baseline
+
+    # Update the baseline hash
+    summary['baseline_hash'] = new_baseline
+    summary['status_msg'] = summary['status_msg'].replace(old_baseline, new_baseline)
+
+    return summary
+
+
 def assert_existence(summary, items=('baseline_image', 'diff_image', 'result_image'), path=''):
     """Assert that images included in a pytest-mpl summary exist.
 

diff --git a/tests/subtests/test_subtest.py b/tests/subtests/test_subtest.py
@@ -4,20 +4,36 @@
 import subprocess
 from pathlib import Path
 
-from helpers import assert_existence, diff_summary
+import matplotlib
+import matplotlib.ft2font
+from packaging.version import Version
+
+from .helpers import assert_existence, diff_summary, patch_summary
+
+# Handle Matplotlib and FreeType versions
+MPL_VERSION = Version(matplotlib.__version__)
+FTV = matplotlib.ft2font.__freetype_version__.replace('.', '')
+VERSION_ID = f"mpl{MPL_VERSION.major}{MPL_VERSION.minor}_ft{FTV}"
+HASH_LIBRARY = Path(__file__).parent / 'hashes' / (VERSION_ID + ".json")
+HASH_LIBRARY_FLAG = rf'--mpl-hash-library={HASH_LIBRARY}'
 
 TEST_FILE = Path(__file__).parent / 'subtest.py'
 
-# Set to True to replace existing baseline summaries with current result
-UPDATE_BASELINE_SUMMARIES = True
+# Global settings to update baselines when running pytest
+# Note: when updating baseline make sure you don't commit "fixes"
+# for tests that are expected to fail
+# (See also `run_subtest` argument `update_baseline` and `update_summary`.)
+UPDATE_BASELINE = False  # baseline images and hashes
+UPDATE_SUMMARY = False  # baseline summaries
 
 
-def run_subtest(baseline_summary, tmp_path, args, summaries=None, xfail=True):
+def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=True,
+                update_baseline=UPDATE_BASELINE, update_summary=UPDATE_SUMMARY):
     """ Run pytest (within pytest) and check JSON summary report.
 
     Parameters
     ----------
-    baseline_summary : str
+    baseline_summary_name : str
         String of the filename without extension for the baseline summary.
     tmp_path : pathlib.Path
         Path of a temporary directory to store results.
@@ -42,10 +58,19 @@ def run_subtest(baseline_summary, tmp_path, args, summaries=None, xfail=True):
     pytest_args = [sys.executable, '-m', 'pytest', str(TEST_FILE)]
     mpl_args = ['--mpl', rf'--mpl-results-path={results_path.as_posix()}',
                 f'--mpl-generate-summary={summaries}']
+    if update_baseline:
+        mpl_args += ['--mpl-generate-path=baseline']
+        if HASH_LIBRARY.exists():
+            mpl_args += [rf'--mpl-generate-hash-library={HASH_LIBRARY}']
 
     # Run the test and record exit status
     status = subprocess.call(pytest_args + mpl_args + args)
 
+    # If updating baseline, don't check summaries
+    if update_baseline:
+        assert status == 0
+        return
+
     # Ensure exit status is as expected
     if xfail:
         assert status != 0
@@ -54,18 +79,23 @@ def run_subtest(baseline_summary, tmp_path, args, summaries=None, xfail=True):
 
     # Load summaries
     baseline_path = Path(__file__).parent / 'summaries'
-    baseline_file = baseline_path / (baseline_summary + '.json')
+    baseline_file = baseline_path / (baseline_summary_name + '.json')
+    results_file = results_path / 'results.json'
+    if update_summary:
+        shutil.copy(results_file, baseline_file)
     with open(baseline_file, 'r') as f:
         baseline_summary = json.load(f)
-    results_file = results_path / 'results.json'
     with open(results_file, 'r') as f:
         result_summary = json.load(f)
 
-    if UPDATE_BASELINE_SUMMARIES:
-        shutil.copy(results_file, baseline_file)
+    # Apply version specific patches
+    patch = baseline_path / (baseline_summary_name + f'_{VERSION_ID}.patch.json')
+    if patch.exists():
+        baseline_summary = patch_summary(baseline_summary, patch)
+    # Note: version specific hashes should be handled by diff_summary instead
 
     # Compare summaries
-    diff_summary(baseline_summary, result_summary)
+    diff_summary(baseline_summary, result_summary, hash_library=HASH_LIBRARY)
 
     # Ensure reported images exist
     assert_existence(result_summary, path=results_path)