Autogenerate baseline data and reduce number of tests

matplotlib · Cadair · Jun 14, 2022 · May 27, 2022 · May 27, 2022 · May 27, 2022
commit c248740f67fe32b9b48d877c95c5bb440f02d880
diff --git a/setup.cfg b/setup.cfg
@@ -30,6 +30,7 @@ install_requires =
     importlib_resources;python_version<'3.8'
     packaging
     Jinja2
+    Pillow
 
 [options.entry_points]
 pytest11 =

diff --git a/tests/subtests/baseline/test_hdiff_idiff.png b/tests/subtests/baseline/test_hdiff_idiff.png
diff --git a/tests/subtests/baseline/test_hdiff_idiff_tolerance.png b/tests/subtests/baseline/test_hdiff_idiff_tolerance.png
diff --git a/tests/subtests/baseline/test_hdiff_idiffshape.png b/tests/subtests/baseline/test_hdiff_idiffshape.png
diff --git a/tests/subtests/baseline/test_hdiff_imatch.png b/tests/subtests/baseline/test_hdiff_imatch.png
diff --git a/tests/subtests/baseline/test_hdiff_imatch_removetext.png b/tests/subtests/baseline/test_hdiff_imatch_removetext.png
diff --git a/tests/subtests/baseline/test_hdiff_imatch_savefig.png b/tests/subtests/baseline/test_hdiff_imatch_savefig.png
diff --git a/tests/subtests/baseline/test_hdiff_imatch_style.png b/tests/subtests/baseline/test_hdiff_imatch_style.png
diff --git a/tests/subtests/baseline/test_hdiff_imatch_tolerance.png b/tests/subtests/baseline/test_hdiff_imatch_tolerance.png
diff --git a/tests/subtests/baseline/test_hmatch_idiff.png b/tests/subtests/baseline/test_hmatch_idiff.png
diff --git a/tests/subtests/baseline/test_hmatch_idiffshape.png b/tests/subtests/baseline/test_hmatch_idiffshape.png
diff --git a/tests/subtests/baseline/test_hmatch_imatch.png b/tests/subtests/baseline/test_hmatch_imatch.png
diff --git a/tests/subtests/baseline/test_hmissing_idiff.png b/tests/subtests/baseline/test_hmissing_idiff.png
diff --git a/tests/subtests/baseline/test_hmissing_idiffshape.png b/tests/subtests/baseline/test_hmissing_idiffshape.png
diff --git a/tests/subtests/baseline/test_hmissing_imatch.png b/tests/subtests/baseline/test_hmissing_imatch.png
diff --git a/tests/subtests/hashes/mpl35_ft261.json b/tests/subtests/hashes/mpl35_ft261.json
@@ -1,15 +1,15 @@
 {
-  "subtests.subtest.test_hmatch_imatch": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03",
-  "subtests.subtest.test_hmatch_idiff": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa",
-  "subtests.subtest.test_hmatch_idiffshape": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e",
-  "subtests.subtest.test_hmatch_imissing": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae",
-  "subtests.subtest.test_hdiff_imatch": "d1ff8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f",
-  "subtests.subtest.test_hdiff_idiff": "d1ff21206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde",
-  "subtests.subtest.test_hdiff_idiffshape": "d1ff745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580",
-  "subtests.subtest.test_hdiff_imissing": "d1ff11cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0",
-  "subtests.subtest.test_hdiff_imatch_tolerance": "d1ff3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6",
-  "subtests.subtest.test_hdiff_idiff_tolerance": "d1ff3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6",
-  "subtests.subtest.test_hdiff_imatch_savefig": "d1ff803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc",
-  "subtests.subtest.test_hdiff_imatch_style": "d1ffde36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13",
-  "subtests.subtest.test_hdiff_imatch_removetext": "d1ff6cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a"
+  "subtests.subtest.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc",
+  "subtests.subtest.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe",
+  "subtests.subtest.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8",
+  "subtests.subtest.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c",
+  "subtests.subtest.test_hdiff_imatch": "d1ff5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282",
+  "subtests.subtest.test_hdiff_idiff": "d1ff014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95",
+  "subtests.subtest.test_hdiff_idiffshape": "d1ff3bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219",
+  "subtests.subtest.test_hdiff_imissing": "d1ffd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff",
+  "subtests.subtest.test_hdiff_imatch_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1",
+  "subtests.subtest.test_hdiff_idiff_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1",
+  "subtests.subtest.test_hdiff_imatch_savefig": "d1ff14c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423",
+  "subtests.subtest.test_hdiff_imatch_style": "d1ffd00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b",
+  "subtests.subtest.test_hdiff_imatch_removetext": "d1ffd7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a"
 }
diff --git a/tests/subtests/helpers.py b/tests/subtests/helpers.py
@@ -1,8 +1,12 @@
+import os
 import re
 import json
 from pathlib import Path
 
-__all__ = ['diff_summary', 'assert_existence', 'patch_summary', 'apply_regex']
+from PIL import Image, ImageDraw
+
+__all__ = ['diff_summary', 'assert_existence', 'patch_summary', 'apply_regex',
+           'remove_specific_hashes', 'transform_hashes', 'transform_images']
 
 
 class MatchError(Exception):
@@ -288,3 +292,53 @@ def remove_specific_hashes(summary_file):
 
     with open(summary_file, "w") as f:
         json.dump(summary, f, indent=2)
+
+
+def transform_hashes(hash_file):
+    """Make hash comparison tests fail correctly.
+
+    Makes hashes of tests *hdiff* in hash_file fail hash comparison
+    and remove *hmissing* hashes that should be missing.
+    """
+
+    with open(hash_file, "r") as f:
+        hashes = json.load(f)
+
+    for test in list(hashes.keys()):
+        h = hashes[test]
+        if "hdiff" in test and h is not None:
+            # Replace first four letters with d1ff to force mismatch
+            hashes[test] = "d1ff" + h[4:]
+        if "hmissing" in test and h is not None:
+            # Remove hashes that should be missing
+            del hashes[test]
+
+    with open(hash_file, "w") as f:
+        json.dump(hashes, f, indent=2)
+
+
+def transform_images(baseline_path):
+    """Make image comparison tests fail correctly.
+
+    Makes images of tests *idiff* under baseline_path fail image comparison
+    and deletes images for *imissing* tests.
+    """
+
+    # Delete imissing files
+    for file in baseline_path.glob("**/*imissing*.png"):
+        file.unlink()
+
+    # Add red cross to idiff files
+    for file in baseline_path.glob("**/*idiff*.png"):
+        with Image.open(file) as im:
+            draw = ImageDraw.Draw(im)
+            draw.line((0, 0) + im.size, "#f00", 3)
+            draw.line((0, im.size[1], im.size[0], 0), "#f00", 3)
+            im.save(file)
+
+    # Resize idiffshape files
+    for file in baseline_path.glob("**/*idiffshape*.png"):
+        with Image.open(file) as im:
+            (width, height) = (im.width // 2, im.height // 2)
+            im_resized = im.resize((width, height))
+            im_resized.save(file)
diff --git a/tests/subtests/result_hashes/mpl35_ft261.json b/tests/subtests/result_hashes/mpl35_ft261.json
@@ -1,19 +1,19 @@
 {
-  "subtests.subtest.test_hmatch_imatch": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03",
-  "subtests.subtest.test_hmatch_idiff": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa",
-  "subtests.subtest.test_hmatch_idiffshape": "e3fed4ad2d22aff2cd771c5503dcb30c6161b21d154430ededa5faa1ec54366e",
-  "subtests.subtest.test_hmatch_imissing": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae",
-  "subtests.subtest.test_hdiff_imatch": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f",
-  "subtests.subtest.test_hdiff_idiff": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde",
-  "subtests.subtest.test_hdiff_idiffshape": "5d37745bbdf2aac6743dbd830afb1877c2ac25a5f926d4f6483c1e24d19a0580",
-  "subtests.subtest.test_hdiff_imissing": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0",
-  "subtests.subtest.test_hmissing_imatch": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856",
-  "subtests.subtest.test_hmissing_idiff": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd",
-  "subtests.subtest.test_hmissing_idiffshape": "204d43b643538cafafa2d0fb7dafac9cf7009dbb421d4b32c71da2e400ceb59d",
-  "subtests.subtest.test_hmissing_imissing": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e",
-  "subtests.subtest.test_hdiff_imatch_tolerance": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6",
-  "subtests.subtest.test_hdiff_idiff_tolerance": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6",
-  "subtests.subtest.test_hdiff_imatch_savefig": "8864803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc",
-  "subtests.subtest.test_hdiff_imatch_style": "e3c8de36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13",
-  "subtests.subtest.test_hdiff_imatch_removetext": "e4c06cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a"
+  "subtests.subtest.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc",
+  "subtests.subtest.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe",
+  "subtests.subtest.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8",
+  "subtests.subtest.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c",
+  "subtests.subtest.test_hdiff_imatch": "b92c5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282",
+  "subtests.subtest.test_hdiff_idiff": "567f014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95",
+  "subtests.subtest.test_hdiff_idiffshape": "b6673bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219",
+  "subtests.subtest.test_hdiff_imissing": "e37bd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff",
+  "subtests.subtest.test_hmissing_imatch": "592d12cc2d5749a6607bbf98d715b95c06a2af6572a7e298bcae349648b9997e",
+  "subtests.subtest.test_hmissing_idiff": "9e98dbd2027525c776212daa061180b4fc40ad12dfc2cdfe4b86694ede14e0c3",
+  "subtests.subtest.test_hmissing_idiffshape": "5534324f9da5c1c104c3ef3435dc6fa9792c0d0d8b762fad5d7f81abd91fbb89",
+  "subtests.subtest.test_hmissing_imissing": "2d15274f0e9b44f1c16e6b237710cd36a3de5c5b7596ef0e65e7a33ce4624cf4",
+  "subtests.subtest.test_hdiff_imatch_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1",
+  "subtests.subtest.test_hdiff_idiff_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1",
+  "subtests.subtest.test_hdiff_imatch_savefig": "0a0514c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423",
+  "subtests.subtest.test_hdiff_imatch_style": "588ad00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b",
+  "subtests.subtest.test_hdiff_imatch_removetext": "a78ad7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a"
 }
diff --git a/tests/subtests/subtest.py b/tests/subtests/subtest.py
@@ -15,86 +15,99 @@ def plot(line, **kwargs):
 
 # hash match
 
-@pytest.mark.mpl_image_compare()
+@pytest.mark.image
+@pytest.mark.hash
+@pytest.mark.mpl_image_compare(savefig_kwargs={'metadata': {'Software': None}})
 def test_hmatch_imatch():
     return plot([1, 2, 3, 4])
 
 
-@pytest.mark.mpl_image_compare()
+@pytest.mark.image
+@pytest.mark.mpl_image_compare(savefig_kwargs={'metadata': {'Software': None}})
 def test_hmatch_idiff():
     return plot([1, 3, 2, 4])
 
 
-@pytest.mark.mpl_image_compare()
+@pytest.mark.image
+@pytest.mark.mpl_image_compare(savefig_kwargs={'metadata': {'Software': None}})
 def test_hmatch_idiffshape():
     return plot([4, 2, 3, 1, 2])
 
 
-@pytest.mark.mpl_image_compare()
+@pytest.mark.image
+@pytest.mark.mpl_image_compare(savefig_kwargs={'metadata': {'Software': None}})
 def test_hmatch_imissing():
     return plot([4, 3, 2, 1])
 
 
 # hash diff
 
-@pytest.mark.mpl_image_compare()
+@pytest.mark.hash
+@pytest.mark.mpl_image_compare(savefig_kwargs={'metadata': {'Software': None}})
 def test_hdiff_imatch():
     return plot([1, 4, 2, 3])
 
 
-@pytest.mark.mpl_image_compare()
+@pytest.mark.mpl_image_compare(savefig_kwargs={'metadata': {'Software': None}})
 def test_hdiff_idiff():
     return plot([1, 2, 4, 3])
 
 
-@pytest.mark.mpl_image_compare()
+@pytest.mark.mpl_image_compare(savefig_kwargs={'metadata': {'Software': None}})
 def test_hdiff_idiffshape():
     return plot([4, 2, 3, 1, 3])
 
 
-@pytest.mark.mpl_image_compare()
+@pytest.mark.mpl_image_compare(savefig_kwargs={'metadata': {'Software': None}})
 def test_hdiff_imissing():
     return plot([3, 2, 4, 1])
 
 
 # hash missing
 
-@pytest.mark.mpl_image_compare()
+@pytest.mark.hash
+@pytest.mark.mpl_image_compare(savefig_kwargs={'metadata': {'Software': None}})
 def test_hmissing_imatch():
     return plot([1, 3, 4, 2])
 
 
-@pytest.mark.mpl_image_compare()
+@pytest.mark.mpl_image_compare(savefig_kwargs={'metadata': {'Software': None}})
 def test_hmissing_idiff():
     return plot([1, 4, 3, 2])
 
 
-@pytest.mark.mpl_image_compare()
+@pytest.mark.mpl_image_compare(savefig_kwargs={'metadata': {'Software': None}})
 def test_hmissing_idiffshape():
     return plot([4, 2, 3, 1, 4])
 
 
-@pytest.mark.mpl_image_compare()
+@pytest.mark.mpl_image_compare(savefig_kwargs={'metadata': {'Software': None}})
 def test_hmissing_imissing():
     return plot([2, 4, 3, 1])
 
 
 # ### Specialized tests
 
 # Tolerance: high to force image match
-@pytest.mark.mpl_image_compare(tolerance=200)
+@pytest.mark.image
+@pytest.mark.hash
+@pytest.mark.mpl_image_compare(tolerance=200, savefig_kwargs={'metadata': {'Software': None}})
 def test_hdiff_imatch_tolerance():
     return plot([1, 2, 3, 4], linestyle='--')
 
 
 # Tolerance: non-default to verify option recorded in JSON
-@pytest.mark.mpl_image_compare(tolerance=3)
+@pytest.mark.image
+@pytest.mark.hash
+@pytest.mark.mpl_image_compare(tolerance=3, savefig_kwargs={'metadata': {'Software': None}})
 def test_hdiff_idiff_tolerance():
     return plot([1, 2, 3, 4], linestyle='--')
 
 
 # Savefig kwargs
-@pytest.mark.mpl_image_compare(savefig_kwargs={'facecolor': 'r'})
+@pytest.mark.image
+@pytest.mark.hash
+@pytest.mark.mpl_image_compare(savefig_kwargs={'facecolor': 'r', 'metadata': {'Software': None}})
 def test_hdiff_imatch_savefig():
     return plot([1, 2, 3, 4])
 
@@ -120,12 +133,16 @@ def test_hdiff_imatch_savefig():
 
 
 # Different style
-@pytest.mark.mpl_image_compare(style='fivethirtyeight')
+@pytest.mark.image
+@pytest.mark.hash
+@pytest.mark.mpl_image_compare(style='fivethirtyeight', savefig_kwargs={'metadata': {'Software': None}})
 def test_hdiff_imatch_style():
     return plot([4, 2, 1, 4])
 
 
 # Remove text
-@pytest.mark.mpl_image_compare(remove_text=True)
+@pytest.mark.image
+@pytest.mark.hash
+@pytest.mark.mpl_image_compare(remove_text=True, savefig_kwargs={'metadata': {'Software': None}})
 def test_hdiff_imatch_removetext():
     return plot([4, 2, 1, 4])
diff --git a/tests/subtests/summaries/test_default.json b/tests/subtests/summaries/test_default.json
@@ -16,10 +16,10 @@
     "status": "failed",
     "image_status": "diff",
     "hash_status": null,
-    "status_msg": "REGEX:Error: Image files did not match\\.\n  RMS Value: 11\\.1[0-9]*\n  Expected:  \n    .*baseline\\.png\n  Actual:    \n    .*result\\.png\n  Difference:\n    .*result-failed-diff\\.png\n  Tolerance: \n    2",
+    "status_msg": "REGEX:Error: Image files did not match\\.\n  RMS Value: 24\\.6[0-9]*\n  Expected:  \n    .*baseline\\.png\n  Actual:    \n    .*result\\.png\n  Difference:\n    .*result-failed-diff\\.png\n  Tolerance: \n    2",
     "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png",
     "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png",
-    "rms": 11.100353848213828,
+    "rms": 24.618234716477044,
     "tolerance": 2,
     "result_image": "subtests.subtest.test_hmatch_idiff/result.png",
     "baseline_hash": null,
@@ -29,7 +29,7 @@
     "status": "failed",
     "image_status": "diff",
     "hash_status": null,
-    "status_msg": "REGEX:Error: Image dimensions did not match\\.\n  Expected shape: \\(400, 400\\)\n    .*baseline\\.png\n  Actual shape: \\(600, 800\\)\n    .*result\\.png",
+    "status_msg": "REGEX:Error: Image dimensions did not match\\.\n  Expected shape: \\(300, 400\\)\n    .*baseline\\.png\n  Actual shape: \\(600, 800\\)\n    .*result\\.png",
     "baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png",
     "diff_image": null,
     "rms": null,
@@ -68,10 +68,10 @@
     "status": "failed",
     "image_status": "diff",
     "hash_status": null,
-    "status_msg": "REGEX:Error: Image files did not match\\.\n  RMS Value: 11\\.1[0-9]*\n  Expected:  \n    .*baseline\\.png\n  Actual:    \n    .*result\\.png\n  Difference:\n    .*result-failed-diff\\.png\n  Tolerance: \n    2",
+    "status_msg": "REGEX:Error: Image files did not match\\.\n  RMS Value: 24\\.6[0-9]*\n  Expected:  \n    .*baseline\\.png\n  Actual:    \n    .*result\\.png\n  Difference:\n    .*result-failed-diff\\.png\n  Tolerance: \n    2",
     "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png",
     "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png",
-    "rms": 11.182677079602481,
+    "rms": 24.620185259705547,
     "tolerance": 2,
     "result_image": "subtests.subtest.test_hdiff_idiff/result.png",
     "baseline_hash": null,
@@ -81,7 +81,7 @@
     "status": "failed",
     "image_status": "diff",
     "hash_status": null,
-    "status_msg": "REGEX:Error: Image dimensions did not match\\.\n  Expected shape: \\(400, 400\\)\n    .*baseline\\.png\n  Actual shape: \\(600, 800\\)\n    .*result\\.png",
+    "status_msg": "REGEX:Error: Image dimensions did not match\\.\n  Expected shape: \\(300, 400\\)\n    .*baseline\\.png\n  Actual shape: \\(600, 800\\)\n    .*result\\.png",
     "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png",
     "diff_image": null,
     "rms": null,
@@ -120,10 +120,10 @@
     "status": "failed",
     "image_status": "diff",
     "hash_status": null,
-    "status_msg": "REGEX:Error: Image files did not match\\.\n  RMS Value: 12\\.1[0-9]*\n  Expected:  \n    .*baseline\\.png\n  Actual:    \n    .*result\\.png\n  Difference:\n    .*result-failed-diff\\.png\n  Tolerance: \n    2",
+    "status_msg": "REGEX:Error: Image files did not match\\.\n  RMS Value: 24\\.6[0-9]*\n  Expected:  \n    .*baseline\\.png\n  Actual:    \n    .*result\\.png\n  Difference:\n    .*result-failed-diff\\.png\n  Tolerance: \n    2",
     "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png",
     "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png",
-    "rms": 12.12938597648977,
+    "rms": 24.6217358806762,
     "tolerance": 2,
     "result_image": "subtests.subtest.test_hmissing_idiff/result.png",
     "baseline_hash": null,
@@ -133,7 +133,7 @@
     "status": "failed",
     "image_status": "diff",
     "hash_status": null,
-    "status_msg": "REGEX:Error: Image dimensions did not match\\.\n  Expected shape: \\(400, 400\\)\n    .*baseline\\.png\n  Actual shape: \\(600, 800\\)\n    .*result\\.png",
+    "status_msg": "REGEX:Error: Image dimensions did not match\\.\n  Expected shape: \\(300, 400\\)\n    .*baseline\\.png\n  Actual shape: \\(600, 800\\)\n    .*result\\.png",
     "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png",
     "diff_image": null,
     "rms": null,
@@ -172,10 +172,10 @@
     "status": "failed",
     "image_status": "diff",
     "hash_status": null,
-    "status_msg": "REGEX:Error: Image files did not match\\.\n  RMS Value: 29\\.2[0-9]*\n  Expected:  \n    .*baseline\\.png\n  Actual:    \n    .*result\\.png\n  Difference:\n    .*result-failed-diff\\.png\n  Tolerance: \n    3",
+    "status_msg": "REGEX:Error: Image files did not match\\.\n  RMS Value: 24\\.5[0-9]*\n  Expected:  \n    .*baseline\\.png\n  Actual:    \n    .*result\\.png\n  Difference:\n    .*result-failed-diff\\.png\n  Tolerance: \n    3",
     "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png",
     "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png",
-    "rms": 29.260332173249314,
+    "rms": 24.576566752362574,
     "tolerance": 3,
     "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png",
     "baseline_hash": null,