diff --git a/.github/release.yml b/.github/release.yml index f38daab2..b0797b87 100644 --- a/.github/release.yml +++ b/.github/release.yml @@ -5,7 +5,7 @@ changelog: labels: - ignore-for-release-notes authors: - - pre-commit-ci[bot] + - pre-commit-ci categories: - title: Fixes labels: @@ -14,6 +14,9 @@ changelog: labels: - enhancement - title: Other Changes + exclude: + labels: + - infrastructure labels: - "*" - title: Infrastructure Changes diff --git a/.github/workflows/test_and_publish.yml b/.github/workflows/test_and_publish.yml index 8781e615..e2519a71 100644 --- a/.github/workflows/test_and_publish.yml +++ b/.github/workflows/test_and_publish.yml @@ -13,30 +13,42 @@ on: # Allow manual runs through the web UI workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: test: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 with: + libraries: | + apt: + - ghostscript + - inkscape envs: | - # Test the oldest and newest configuration on Mac and Windows + # Test oldest NEP 29 configurations + - linux: py39-test-mpl33 + - macos: py39-test-mpl33 + - windows: py39-test-mpl33 + # Test oldest non-NEP 29 configurations + - linux: py36-test-mpl20 + runs-on: ubuntu-20.04 - macos: py36-test-mpl20 - - macos: py310-test-mpl35 - windows: py36-test-mpl20 - - windows: py310-test-mpl35 - # Test all configurations on Linux - - linux: py36-test-mpl20 - - linux: py36-test-mpl21 - - linux: py36-test-mpl22 - - linux: py37-test-mpl30 - - linux: py37-test-mpl31 - - linux: py37-test-mpl32 - - linux: py38-test-mpl33 - - linux: py39-test-mpl34 + # Test newest configurations + - linux: py312-test-mpl38 + - macos: py312-test-mpl38 + - windows: py312-test-mpl38 + # Test intermediate NEP 29 configurations on Linux + - linux: py39-test-mpl38 + - linux: py310-test-mpl38 - linux: py310-test-mpl35 + - linux: py311-test-mpl36 + - linux: py311-test-mpl37 # Test different versions of pytest - - linux: py310-test-mpl35-pytestdev - - linux: py310-test-mpl35-pytest62 - - linux: py38-test-mpl35-pytest54 + - linux: py312-test-mpldev-pytestdev + - linux: py39-test-mpl33-pytest62 + - linux: py38-test-mpl31-pytest54 coverage: 'codecov' publish: @@ -45,5 +57,7 @@ jobs: uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish_pure_python.yml@v1 with: test_command: pytest $GITHUB_WORKSPACE/tests; pytest --mpl $GITHUB_WORKSPACE/tests + # Remove python-version when python-dateutil >2.8.2 + python-version: "3.11" secrets: pypi_token: ${{ secrets.pypi_password }} diff --git a/.gitignore b/.gitignore index 627f8278..6b2c3631 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ __pycache__ build dist .tmp +.DS_Store +/docs/sample +/docs/_build diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 55f8e7ae..e2140b32 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -60,7 +60,7 @@ repos: # W605 - Fix invalid escape sequence 'x'. # W690 - Fix various deprecated code (via lib2to3). - repo: https://github.com/PyCQA/flake8 - rev: 4.0.1 + rev: 7.0.0 hooks: - id: flake8 args: @@ -70,12 +70,13 @@ repos: "E101,E11,E111,E112,E113,E121,E122,E123,E124,E125,E126,E127,E128,E129,E131,E133,E20,E211,E231,E241,E242,E251,E252,E26,E265,E266,E27,E301,E302,E303,E304,E305,E306,E401,E402,E502,E701,E711,E712,E713,E714,E722,E731,E901,E902,F822,F823,W191,W291,W292,W293,W391,W601,W602,W603,W604,W605,W690", ] - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.13.2 hooks: - id: isort args: ["--sp", "setup.cfg"] + exclude: ".*(docs/conf.py)$" - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.5.0 hooks: - id: check-ast - id: check-case-conflict diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..02fe0e98 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,16 @@ +version: 2 + +build: + os: "ubuntu-22.04" + tools: + python: "3.11" + +python: + install: + - method: pip + path: . + extra_requirements: + - docs + +# Don't build any extra formats +formats: [] diff --git a/CHANGES.md b/CHANGES.md index d8eb9188..2f431f29 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,12 +1,27 @@ +## v0.16.1 - 2022-07-23 + + +### What's Changed + +#### Fixes + +- Fix tests which exit before returning a figure or use `unittest.TestCase` by @ConorMacBride in https://github.com/matplotlib/pytest-mpl/pull/171 + +#### Other Changes + +- Rename default branch to `main` by @ConorMacBride in https://github.com/matplotlib/pytest-mpl/pull/169 + +**Full Changelog**: https://github.com/matplotlib/pytest-mpl/compare/v0.16.0...v0.16.1 + ## v0.16.0 - 2022-06-14 ### Fixes - Make summary log message about test results in general instead of failures by @neutrinoceros in https://github.com/matplotlib/pytest-mpl/pull/148 - Add support for classes with pytest 7 by @ConorMacBride in https://github.com/matplotlib/pytest-mpl/pull/164 - Note that this change necessitated a minor breaking change for figure tests within classes only, and the following will need to be done: - - Hash library test names will need to be regenerated/updated to include the class name. - - If the undocumented `mpl-use-full-test-name` ini option is enabled, the the baseline images will need to be regenerated, or have their filename updated to include the class name. +- Note that this change necessitated a minor breaking change for figure tests within classes only, and the following will need to be done: +- - Hash library test names will need to be regenerated/updated to include the class name. +- - If the undocumented `mpl-use-full-test-name` ini option is enabled, the the baseline images will need to be regenerated, or have their filename updated to include the class name. ### Other Changes diff --git a/README.rst b/README.rst index 3dd9b742..409f4b73 100644 --- a/README.rst +++ b/README.rst @@ -1,344 +1,66 @@ -About ------ - -This is a plugin to facilitate image comparison for -`Matplotlib `__ figures in pytest. - -For each figure to test, the reference image is subtracted from the -generated image, and the RMS of the residual is compared to a -user-specified tolerance. If the residual is too large, the test will -fail (this is implemented using helper functions from -``matplotlib.testing``). - -For more information on how to write tests to do this, see the **Using** -section below. - -Installing ----------- +``pytest-mpl`` +============== -This plugin is compatible with Python 3.6 and later, and -requires `pytest `__ and -`matplotlib `__ to be installed. +``pytest-mpl`` is a `pytest `__ plugin to facilitate image comparison for `Matplotlib `__ figures. -To install, you can do:: +For each figure to test, an image is generated and then subtracted from an existing reference image. +If the RMS of the residual is larger than a user-specified tolerance, the test will fail. +Alternatively, the generated image can be hashed and compared to an expected value. - pip install pytest-mpl +For more information, see the `pytest-mpl documentation `__. -You can check that the plugin is registered with pytest by doing:: +Installation +------------ +.. code-block:: bash - pytest --version + pip install pytest-mpl -which will show a list of plugins: +For detailed instructions, see the `installation guide `__ in the ``pytest-mpl`` docs. -:: - - This is pytest version 2.7.1, imported from ... - setuptools registered plugins: - pytest-mpl-0.1 at ... - -Using +Usage ----- +First, write test functions that create a figure. +These image comparison tests are decorated with ``@pytest.mark.mpl_image_compare`` and return the figure for testing: -With Baseline Images -^^^^^^^^^^^^^^^^^^^^ - -To use, you simply need to mark the function where you want to compare -images using ``@pytest.mark.mpl_image_compare``, and make sure that the -function returns a Matplotlib figure (or any figure object that has a -``savefig`` method): - -.. code:: python - - import pytest - import matplotlib.pyplot as plt - - @pytest.mark.mpl_image_compare - def test_succeeds(): - fig = plt.figure() - ax = fig.add_subplot(1,1,1) - ax.plot([1,2,3]) - return fig - -To generate the baseline images, run the tests with the -``--mpl-generate-path`` option with the name of the directory where the -generated images should be placed:: - - pytest --mpl-generate-path=baseline - -If the directory does not exist, it will be created. The directory will -be interpreted as being relative to where you are running ``pytest``. -Once you are happy with the generated images, you should move them to a -sub-directory called ``baseline`` relative to the test files (this name -is configurable, see below). You can also generate the baseline image -directly in the right directory. - -With a Hash Library -^^^^^^^^^^^^^^^^^^^ - -Instead of comparing to baseline images, you can instead compare against a JSON -library of SHA-256 hashes. This has the advantage of not having to check baseline -images into the repository with the tests, or download them from a remote -source. - -The hash library can be generated with -``--mpl-generate-hash-library=path_to_file.json``. The hash library to be used -can either be specified via the ``--mpl-hash-library=`` command line argument, -or via the ``hash_library=`` keyword argument to the -``@pytest.mark.mpl_image_compare`` decorator. +.. code-block:: python -When generating a hash library, the tests will also be run as usual against the -existing hash library specified by ``--mpl-hash-library`` or the keyword argument. -However, generating baseline images will always result in the tests being skipped. + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare + def test_plot(): + fig, ax = plt.subplots() + ax.plot([1, 2]) + return fig -Hybrid Mode: Hashes and Images -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Then, generate reference images by running the test suite with the ``--mpl-generate-path`` option: -It is possible to configure both hashes and baseline images. In this scenario -only the hash comparison can determine the test result. If the hash comparison -fails, the test will fail, however a comparison to the baseline image will be -carried out so the actual difference can be seen. If the hash comparison passes, -the comparison to the baseline image is skipped (unless **results always** is -configured). +.. code-block:: bash -This is especially useful if the baseline images are external to the repository -containing the tests, and are accessed via HTTP. In this situation, if the hashes -match, the baseline images won't be retrieved, saving time and bandwidth. Also, it -allows the tests to be modified and the hashes updated to reflect the changes -without having to modify the external images. + pytest --mpl-generate-path=baseline +Then, run the test suite as usual, but pass ``--mpl`` to compare the returned figures to the reference images: -Running Tests -^^^^^^^^^^^^^ +.. code-block:: bash -Once tests are written with baseline images, a hash library, or both to compare -against, the tests can be run with:: + pytest --mpl - pytest --mpl - -and the tests will pass if the images are the same. If you omit the -``--mpl`` option, the tests will run but will only check that the code -runs, without checking the output images. - - -Generating a Test Summary -^^^^^^^^^^^^^^^^^^^^^^^^^ - -By specifying the ``--mpl-generate-summary=html`` CLI argument, a HTML summary -page will be generated showing the test result, log entry and generated result -image. When in the (default) image comparison mode, the baseline image, diff -image and RMS (if any), and tolerance of each test will also be shown. -When in the hash comparison mode, the baseline hash and result hash will -also be shown. When in hybrid mode, all of these are included. - -When generating a HTML summary, the ``--mpl-results-always`` option is -automatically applied (see section below). Therefore images for passing -tests will also be shown. +By also passing ``--mpl-generate-summary=html``, a summary of the image comparison results will be generated in HTML format: +---------------+---------------+---------------+ | |html all| | |html filter| | |html result| | +---------------+---------------+---------------+ -As well as ``html``, ``basic-html`` can be specified for an alternative HTML -summary which does not rely on JavaScript or external resources. A ``json`` -summary can also be saved. Multiple options can be specified comma-separated. - -Options -------- - -Tolerance -^^^^^^^^^ - -The RMS tolerance for the image comparison (which defaults to 2) can be -specified in the ``mpl_image_compare`` decorator with the ``tolerance`` -argument: - -.. code:: python - - @pytest.mark.mpl_image_compare(tolerance=20) - def test_image(): - ... - -Savefig options -^^^^^^^^^^^^^^^ - -You can pass keyword arguments to ``savefig`` by using -``savefig_kwargs`` in the ``mpl_image_compare`` decorator: - -.. code:: python - - @pytest.mark.mpl_image_compare(savefig_kwargs={'dpi':300}) - def test_image(): - ... - -Baseline images -^^^^^^^^^^^^^^^ - -The baseline directory (which defaults to ``baseline`` ) and the -filename of the plot (which defaults to the name of the test with a -``.png`` suffix) can be customized with the ``baseline_dir`` and -``filename`` arguments in the ``mpl_image_compare`` decorator: - -.. code:: python - - @pytest.mark.mpl_image_compare(baseline_dir='baseline_images', - filename='other_name.png') - def test_image(): - ... - -The baseline directory in the decorator above will be interpreted as -being relative to the test file. Note that the baseline directory can -also be a URL (which should start with ``http://`` or ``https://`` and -end in a slash). If you want to specify mirrors, set ``baseline_dir`` to -a comma-separated list of URLs (real commas in the URL should be encoded -as ``%2C``). - -Finally, you can also set a custom baseline directory globally when -running tests by running ``pytest`` with:: - - pytest --mpl --mpl-baseline-path=baseline_images - -This directory will be interpreted as being relative to where pytest -is run. However, if the ``--mpl-baseline-relative`` option is also -included, this directory will be interpreted as being relative to -the current test directory. -In addition, if both this option and the ``baseline_dir`` -option in the ``mpl_image_compare`` decorator are used, the one in the -decorator takes precedence. - -Results always -^^^^^^^^^^^^^^ - -By default, result images are only saved for tests that fail. -Passing ``--mpl-results-always`` to pytest will force result images -to be saved for all tests, even for tests that pass. - -When in **hybrid mode**, even if a test passes hash comparison, -a comparison to the baseline image will also be carried out, -with the baseline image and diff image (if image comparison fails) -saved for all tests. This secondary comparison will not affect -the success status of the test. - -This option is useful for always *comparing* the result images against -the baseline images, while only *assessing* the tests against the -hash library. -If you only update your baseline images after merging a PR, this -option means that the generated summary will always show how the -PR affects the baseline images, with the success status of each -test (based on the hash library) also shown in the generated -summary. This option is applied automatically when generating -a HTML summary. - -When the ``--mpl-results-always`` option is active, and some hash -comparison tests are performed, a hash library containing all the -result hashes will also be saved to the root of the results directory. -The filename will be extracted from ``--mpl-generate-hash-library``, -``--mpl-hash-library`` or ``hash_library=`` in that order. - -Base style -^^^^^^^^^^ - -By default, tests will be run using the Matplotlib 'classic' style -(ignoring any locally defined RC parameters). This can be overridden by -using the ``style`` argument: - -.. code:: python - - @pytest.mark.mpl_image_compare(style='fivethirtyeight') - def test_image(): - ... - -Package version dependencies -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Different versions of Matplotlib and FreeType may result in slightly -different images. When testing on multiple platforms or as part of a -pipeline, it is important to ensure that the versions of these -packages match the versions used to generate the images used for -comparison. It can be useful to pin versions of Matplotlib and FreeType -so as to avoid automatic updates that fail tests. - -Removing text -^^^^^^^^^^^^^ - -If you are running a test for which you are not interested in comparing -the text labels, you can use the ``remove_text`` argument to the -decorator: - -.. code:: python - - @pytest.mark.mpl_image_compare(remove_text=True) - def test_image(): - ... - -This will make the test insensitive to changes in e.g. the freetype -library. - -Test failure example --------------------- - -If the images produced by the tests are correct, then the test will -pass, but if they are not, the test will fail with a message similar to -the following:: - - E Exception: Error: Image files did not match. - E RMS Value: 142.2287807767823 - E Expected: - E /var/folders/zy/t1l3sx310d3d6p0kyxqzlrnr0000gr/T/tmp4h4oxr7y/baseline-coords_overlay_auto_coord_meta.png - E Actual: - E /var/folders/zy/t1l3sx310d3d6p0kyxqzlrnr0000gr/T/tmp4h4oxr7y/coords_overlay_auto_coord_meta.png - E Difference: - E /var/folders/zy/t1l3sx310d3d6p0kyxqzlrnr0000gr/T/tmp4h4oxr7y/coords_overlay_auto_coord_meta-failed-diff.png - E Tolerance: - E 10 - -The image paths included in the exception are then available for -inspection: - -+----------------+----------------+-------------+ -| Expected | Actual | Difference | -+================+================+=============+ -| |expected| | |actual| | |diff| | -+----------------+----------------+-------------+ - -In this case, the differences are very clear, while in some cases it may -be necessary to use the difference image, or blink the expected and -actual images, in order to see what changed. - -The default tolerance is 2, which is very strict. In some cases, you may -want to relax this to account for differences in fonts across different -systems. - -By default, the expected, actual and difference files are written to a -temporary directory with a non-deterministic path. If you want to instead -write them to a specific directory, you can use:: - - pytest --mpl --mpl-results-path=results - -The ``results`` directory will then contain one sub-directory per test, and each -sub-directory will contain the three files mentioned above. If you are using a -continuous integration service, you can then use the option to upload artifacts -to upload these results to somewhere where you can view them. For more -information, see: - -* `Uploading artifacts on Travis-CI `_ -* `Build Artifacts (CircleCI) `_ -* `Packaging Artifacts (AppVeyor) `_ - -Running the tests for pytest-mpl --------------------------------- - -If you are contributing some changes and want to run the tests, first -install the latest version of the plugin then do:: +For more information on how to configure and use ``pytest-mpl``, see the `pytest-mpl documentation `__. - cd tests - pytest --mpl +Contributing +------------ +``pytest-mpl`` is a community project maintained for and by its users. +There are many ways you can help! -The reason for having to install the plugin first is to ensure that the -plugin is correctly loaded as part of the test suite. +- Report a bug or request a feature `on GitHub `__ +- Improve the documentation or code -.. |html all| image:: images/html_all.png -.. |html filter| image:: images/html_filter.png -.. |html result| image:: images/html_result.png -.. |expected| image:: images/baseline-coords_overlay_auto_coord_meta.png -.. |actual| image:: images/coords_overlay_auto_coord_meta.png -.. |diff| image:: images/coords_overlay_auto_coord_meta-failed-diff.png +.. |html all| image:: docs/images/html_all.png +.. |html filter| image:: docs/images/html_filter.png +.. |html result| image:: docs/images/html_result.png diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..d4bb2cbb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_templates/mpl_third_party_sidebar.html b/docs/_templates/mpl_third_party_sidebar.html new file mode 100644 index 00000000..43068887 --- /dev/null +++ b/docs/_templates/mpl_third_party_sidebar.html @@ -0,0 +1,8 @@ + diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..f372bbf8 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,78 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +import datetime +from packaging.version import Version + +from pytest_mpl import __version__ + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'pytest-mpl' +author = 'Thomas Robitaille' +copyright = '{}, {}'.format(datetime.datetime.now().year, author) + +release = __version__ +pytest_mpl_version = Version(__version__) +is_release = not (pytest_mpl_version.is_prerelease or pytest_mpl_version.is_devrelease) + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sample_summaries', + 'sphinx.ext.intersphinx', + 'sphinx_design', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +intersphinx_mapping = { + "matplotlib": ("https://matplotlib.org/stable", None), +} + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "mpl_sphinx_theme" +html_theme_options = { + "navbar_links": "absolute", + "show_prev_next": False, + "logo": {"link": "https://matplotlib.org/stable/", + "image_light": "images/logo_light.svg", + "image_dark": "images/logo_dark.svg"}, + "collapse_navigation": False, +} +html_sidebars = { + "**": ["mpl_third_party_sidebar.html", "sidebar-nav-bs.html"] +} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] diff --git a/docs/configuration.rst b/docs/configuration.rst new file mode 100644 index 00000000..1afce3af --- /dev/null +++ b/docs/configuration.rst @@ -0,0 +1,436 @@ +.. title:: Configuration + +############# +Configuration +############# + +This section defines all the ``pytest-mpl`` configuration options. + +There are three ways to configure the plugin: + +1. Passing kwargs to the ``pytest.mark.mpl_image_compare`` decorator (kwarg) +2. Passing options to pytest from the command line (CLI) +3. Setting INI options in pytest configuration files (INI) + +The CLI and INI options are global, and will apply to all tests. +The kwarg options are local, and will only apply to the test function that they are specified in. + +If set, the kwarg options will override the equivalent CLI and INI options. +Furthermore, CLI options will override equivalent INI options. + +See the `pytest documentation `__ for more information on how to set INI options. + +Enabling the plugin +=================== + +Enable testing +-------------- +| **kwarg**: --- +| **CLI**: ``--mpl`` +| **INI**: --- +| Default: ``False`` + +To enable image comparison testing, pass ``--mpl`` when running pytest. + +.. code:: bash + + pytest --mpl + +By default, this option will enable :doc:`baseline image comparison `. +:doc:`Baseline hash comparison ` can be enabled by configuring the :ref:`hash library configuration option `. + +Without this option, the tests will still run. +However, the returned figures will be closed without being compared to a baseline image or hash. + +Enable baseline image generation +-------------------------------- +| **kwarg**: --- +| **CLI**: ``--mpl-generate-path=`` +| **INI**: --- +| Default: ``None`` + +Baseline images will be generated and saved to the specified directory path, relative to where pytest was run. + +.. code:: bash + + pytest --mpl-generate-path=baseline + +The baseline directory specified by the :ref:`baseline directory configuration option ` will be ignored. +However, the filename of the baseline image will still be determined by the :ref:`filename configuration option ` and the :ref:`use full test name configuration option `. +This option overrides the ``--mpl`` option. + +Enable baseline hash generation +------------------------------- +| **kwarg**: --- +| **CLI**: ``--mpl-generate-hash-library=`` +| **INI**: --- +| Default: ``None`` + +Baseline hashes will be generated and saved to the specified JSON file path, relative to where pytest was run. + +.. code:: bash + + pytest --mpl-generate-hash-library=hashes.json + +Enabling this option will also set the ``--mpl`` option, as it is important to visually inspect the figures before generating baseline hashes. +The hash library specified by the :ref:`hash library configuration option ` will be ignored. + +Locating baseline images +======================== + +.. _baseline-dir: + +Directory containing baseline images +------------------------------------ +| **kwarg**: ``baseline_dir=`` +| **CLI**: ``--mpl-baseline-path=`` +| **INI**: ``mpl-baseline-path`` +| Default: ``baseline/`` *(relative to the test file)* + +The directory containing the baseline images that will be compared to the test figures. +The kwarg option (``baseline_dir``) is relative to the test file, while the CLI option (``--mpl-baseline-path``) and INI option (``mpl-baseline-path``) are relative to where pytest was run. +Absolute paths can also be used. +If the directory does not exist, it will be created along with any missing parent directories. + +.. code:: bash + + pytest --mpl --mpl-baseline-path=baseline_images + +The baseline directory can also be a URL, which should start with ``http://`` or ``https://`` and end in a slash. +Alternative URLs, or mirrors, can be configured by specifying a comma-separated list of URLs. +Baseline images will be searched for in the order that the URLs are specified, and the first successful download will be used. +Real commas in URLs should be encoded as ``%2C``. + +.. code:: bash + + pytest --mpl --mpl-baseline-path=https://example.com/baseline/,https://mirror.example.com/baseline/ + +.. code:: python + + @pytest.mark.mpl_image_compare(baseline_dir="https://example.com/baseline/", + filename="other_name.png") + def test_plot(): + ... + +Whether ``--mpl-baseline-path`` should also be relative to the test file +------------------------------------------------------------------------ +| **kwarg**: --- +| **CLI**: ``--mpl-baseline-relative`` +| **INI**: --- +| Default: ``False`` + +If this option is set, the baseline directory specified by ``--mpl-baseline-path`` will be interpreted as being relative to the test file. +This option is only relevant if ``--mpl-baseline-path`` refers to a directory and not a URL. + +.. code:: bash + + pytest --mpl --mpl-baseline-path=baseline_images --mpl-baseline-relative + +.. _filename: + +Filename of the baseline image +------------------------------ +| **kwarg**: ``filename=`` +| **CLI**: --- +| **INI**: --- +| Default: *name of the test with a file extension suffix* + +The filename of the baseline image that will be compared to the test figure. +The default file extension is ``png``, unless overridden by :ref:`savefig_kwargs["format"] `. +This option has no effect if the :ref:`use full test name configuration option ` is enabled. + +.. code:: python + + @pytest.mark.mpl_image_compare(baseline_dir="baseline_images", + filename="other_name.png") + def test_plot(): + ... + +If you specify a filename that has an extension other than ``png``, you must also specify it in :ref:`savefig_kwargs["format"] `. + +.. code:: python + + @pytest.mark.mpl_image_compare(filename="plot.pdf", + savefig_kwargs={"format": "pdf"}) + def test_plot(): + ... + +.. _full-test-name: + +Whether to include the module name in the filename +-------------------------------------------------- +| **kwarg**: --- +| **CLI**: ``--mpl-use-full-test-name`` +| **INI**: ``mpl-use-full-test-name`` +| Default: ``False`` + +Whether to include the module name (and class name) in the baseline image filename. + +This option is useful if you have multiple tests with the same name in different modules. +Or have multiple tests with the same name in the same module, but in different classes. +If this option is set, the baseline image filename will be ``[.]..``. +The file extension is the default extension as documented in the :ref:`filename option documentation `. + +Enabling this should ensure baseline image filenames are unique. +The :ref:`filename configuration option ` can also be used to fix the filename of the baseline image. + +.. note:: + + Filename collisions are permitted. + This is useful if, for example, you want to verify that two tests produce the same figure. + However, unexpected collisions should become apparent when the tests are run and failures are reported. + +This option overrides the :ref:`filename configuration option `. + +Locating baseline hashes +======================== + +.. _hash-library: + +File containing baseline hashes +------------------------------- +| **kwarg**: ``hash_library=`` +| **CLI**: ``--mpl-hash-library=`` +| **INI**: ``mpl-hash-library = `` +| Default: *no hash comparison* + +The file containing the baseline hashes that will be compared to the test figures. +The kwarg option (``hash_library``) is relative to the test file, while the INI option (``mpl-hash-library``) is relative to where pytest was run. +The file must be a JSON file in the same format as one generated by ``--mpl-generate-hash-library``. +If its directory does not exist, it will be created along with any missing parent directories. + +.. attention:: + + For backwards compatibility, the CLI option (``--mpl-hash-library``) is relative to the test file. + Also, the CLI option takes precedence over the kwarg option, but the kwarg option takes precedence over the INI option as usual. + +Configuring this option disables baseline image comparison. +If you want to enable both hash and baseline image comparison, which we call :doc:`"hybrid mode" `, you must explicitly set the :ref:`baseline directory configuration option `. + +.. _controlling-sensitivity: + +Controlling the sensitivity of the comparison +============================================= + +.. rubric:: Package version dependencies + +Different versions of Matplotlib and FreeType may result in slightly different images. +When testing on multiple platforms or as part of a pipeline, it is important to ensure that the versions of these packages match the versions used to generate the images and/or hashes used for comparison. +It can be useful to pin versions of Matplotlib and FreeType so as to avoid automatic updates that fail tests. + +The ``pytest-mpl`` configuration options in this section allow you to control the sensitivity of the comparison. +Adjusting these options *may* allow tests to pass across a range of Matplotlib and FreeType versions. + +.. _tolerance: + +RMS tolerance +------------- +| **kwarg**: ``tolerance=`` +| **CLI**: ``--mpl-default-tolerance=`` +| **INI**: ``mpl-default-tolerance = `` +| Default: ``2`` + +The maximum RMS difference between the result image and the baseline image before the test fails. +The specified tolerance value can be a float or an integer between 0 and 255. + +.. code:: python + + @pytest.mark.mpl_image_compare(tolerance=20) + def test_plot(): + ... + +.. rubric:: How the RMS difference is calculated + +Result images and baseline images are *always* converted to PNG files before comparison. +Each are read as an array of RGBA pixels (or just RGB if fully opaque) with values between 0 and 255. +If the result image and the baseline image have different aspect ratios, the test will always fail. +The RMS difference is calculated as the square root of the mean of the squared differences between the result image and the baseline image. +If the RMS difference is greater than the tolerance, the test will fail. + +Whether to make metadata deterministic +-------------------------------------- +| **kwarg**: ``deterministic=`` +| **CLI**: ``--mpl-deterministic`` or ``--mpl-no-deterministic`` +| **INI**: ``mpl-deterministic = `` +| Default: ``True`` (PNG: ``False``) + +Whether to make the image file metadata deterministic. + +By default, Matplotlib does not produce deterministic output that will have a consistent hash every time it is run, or over different Matplotlib versions. +Depending on the file format, enabling this option does a number of things such as, e.g., setting the creation date in the metadata to be constant, and avoids hard-coding the Matplotlib version in the file. +Supported formats for deterministic metadata are ``"eps"``, ``"pdf"``, ``"png"``, and ``"svg"``. + +.. code:: python + + @pytest.mark.mpl_image_compare(deterministic=True) + def test_plot(): + ... + +By default, ``pytest-mpl`` will save and compare figures in PNG format. +However, it is possible to set the format to use by setting, e.g., ``savefig_kwargs={"format": "pdf"}`` when configuring the :ref:`savefig_kwargs configuration option `. +Note that Ghostscript is required to be installed for comparing PDF and EPS figures, while Inkscape is required for SVG comparison. + +.. note:: + + A future major release of ``pytest-mpl`` will generate deterministic PNG files by default. + It is recommended to explicitly set this configuration option to avoid hashes changing. + +Whether to remove titles and axis tick labels +--------------------------------------------- +| **kwargs**: ``remove_text=`` +| **CLI**: --- +| **INI**: --- +| Default: ``False`` + +Enabling this option will remove titles and axis tick labels from the figure before saving and comparing. +This will make the test less sensitive to changes in the FreeType library version. +This feature, provided by :func:`matplotlib.testing.decorators.remove_ticks_and_titles`, will not remove any other text such as axis labels and annotations. + +.. code:: python + + @pytest.mark.mpl_image_compare(remove_text=True) + def test_plot(): + ... + +Modifying the figure before saving +================================== + +.. _savefig-kwargs: + +Matplotlib savefig kwargs +------------------------- +| **kwarg**: ``savefig_kwargs=`` +| **CLI**: --- +| **INI**: --- +| Default: ``{}`` + +A dictionary of keyword arguments to pass to :func:`matplotlib.pyplot.savefig`. + +.. code:: python + + @pytest.mark.mpl_image_compare(savefig_kwargs={"dpi": 300}) + def test_plot(): + ... + +Matplotlib style +---------------- +| **kwarg**: ``style=`` +| **CLI**: ``--mpl-default-style=`` +| **INI**: ``mpl-default-style = `` +| Default: ``"classic"`` + +The Matplotlib style to use when saving the figure. +See the :func:`matplotlib.style.context` ``style`` documentation for the options available. +``pytest-mpl`` will ignore any locally defined :class:`~matplotlib.RcParams`. + +.. code:: python + + @pytest.mark.mpl_image_compare(style="fivethirtyeight") + def test_plot(): + ... + +.. note:: + + It is recommended to use the ``"default"`` style for new code. + + .. code:: python + + @pytest.mark.mpl_image_compare(style="default") + def test_plot(): + ... + + The ``"classic"`` style (which ``pytest-mpl`` currently uses by default) was the default style for Matplotlib versions prior to 2.0. + A future major release of ``pytest-mpl`` *may* change the default style to ``"default"``. + +Matplotlib backend +------------------ +| **kwarg**: ``backend=`` +| **CLI**: ``--mpl-default-backend=`` +| **INI**: ``mpl-default-backend = `` +| Default: ``"agg"`` + +The Matplotlib backend to use when saving the figure. +See the :ref:`Matplotlib backend documentation ` for the options available. +``pytest-mpl`` will ignore any locally defined :class:`~matplotlib.RcParams`. + +Recording test results +====================== + +.. _results-path: + +Directory to write testing artifacts to +--------------------------------------- +| **kwarg**: --- +| **CLI**: ``--mpl-results-path=`` +| **INI**: ``mpl-results-path = `` +| Default: *temporary directory* + +The directory to write result images and test summary reports to. +The path is relative to where pytest was run. +Absolute paths are also supported. +If the directory does not exist, it will be created along with any missing parent directories. + +.. _results-always: + +Whether to save result images for passing tests +----------------------------------------------- +| **kwarg**: --- +| **CLI**: ``--mpl-results-always`` +| **INI**: ``mpl-results-always = `` +| Default: ``False`` (``True`` if generating a HTML summary) + +By default, result images are only saved for tests that fail. +Enabling this option will force result images to be saved for all tests, even for tests that pass. + +When this option is enabled, and some hash comparison tests are performed, a hash library containing all the result hashes will also be saved to the root of the results directory. +The filename will be extracted from ``--mpl-generate-hash-library``, ``--mpl-hash-library``, or ``hash_library=`` in that order. + +This option is applied automatically when generating a HTML summary. + +.. rubric:: Relevance to "hybrid mode" + +When in :doc:`"hybrid mode" `, a baseline image comparison is only performed if the test fails hash comparison. +However, enabling this option will force a comparison to the baseline image even if the test passes hash comparison. +This option is useful for always *comparing* the result images against the baseline images, while only *assessing* the tests against the hash library. +This secondary comparison will **not** affect the success status of the test, but any failures (including diff images) will be included in generated summary reports. + +Some projects store their baseline images in a separate repository, and only keep the baseline hash library in the main repository. +This means that they cannot update the baseline images until after the PR is merged. +Enabling this option allows them to ensure the hashes are correct before merging the PR, but also see how the PR affects the baseline images, as the diff images will always be shown in the HTML summary. + +.. _generate-summary: + +Generate test summaries +----------------------- +| **kwarg**: --- +| **CLI**: ``--mpl-generate-summary={html,json,basic-html}`` +| **INI**: ``mpl-generate-summary = {html,json,basic-html}`` +| Default: ``None`` + +This option specifies the format of the test summary report to generate, if any. +Multiple options can be specified comma-separated. +The available options are: + +``html`` + Generate a HTML summary report showing the test result, log entry and generated result image. + Results can be searched and filtered. + When in the (default) image comparison mode, the baseline image, diff image and RMS difference (if any), and RMS tolerance of each test will also be shown. + When in the hash comparison mode, the baseline hash and result hash will also be shown. + When in hybrid mode, all of these are included. +``json`` + Generate a JSON summary report. + This format includes the same information as the HTML summary, but is more suitable for automated processing. +``basic-html`` + Generate a HTML summary report with a simplified layout. + This format does not include any JavaScript or need internet access to load web resources. + +Summary reports can also be produced when generating baseline images and hash libraries. +The summaries will be written to the :ref:`results directory `. +When generating a HTML summary, the ``--mpl-results-always`` option is automatically applied. +Therefore images for passing tests will also be shown. + +For examples of how the summary reports look in different operating modes, see: + +* :doc:`image_mode` +* :doc:`hash_mode` +* :doc:`hybrid_mode` diff --git a/docs/hash_mode.rst b/docs/hash_mode.rst new file mode 100644 index 00000000..13fa7de8 --- /dev/null +++ b/docs/hash_mode.rst @@ -0,0 +1,83 @@ +.. title:: Hash comparison mode + +##################### +Hash Comparison Mode +##################### + +This how-to guide will show you how to use the hash comparison mode of ``pytest-mpl``. + +In this mode, the hash of the image is compared to the hash of the baseline image. +Only the hash value of the baseline image, rather than the full image, needs to be stored in the repository. +This means that the repository size is reduced, and the images can be regenerated if necessary. +This approach does however make it more difficult to visually inspect any changes to the images. + +If your goal is to not commit any images to the code repository, then you should consider using :doc:`hybrid mode ` instead. +In this mode, the hashes can be stored in the code repository, while the baseline images are stored in a separate repository and accessed through a URL when testing. + +Generating baseline hashes +========================== + +Once a suite of image comparison tests have been written, baseline hashes should be generated by setting ``--mpl-generate-hash-library``: + +.. code-block:: bash + + pytest --mpl-generate-hash-library=your_project/tests/hashes.json + +It is important to visually inspect the figures before generating baseline hashes. +So, as well as generating baseline hashes, this command runs baseline image comparison tests. +If no baseline images exist in the default directory, this command will fail. + +A better option is to generate baseline images along with the baseline hashes to ensure that the images are as expected, even if you do not wish to use them for comparison: + +.. code-block:: bash + + pytest \ + --mpl-generate-hash-library=your_project/tests/mpl35_ft261.json \ + --mpl-generate-path=baseline + +To assist with inspecting the generated images (and hashes), a HTML summary report can be generated by setting ``--mpl-generate-summary``: + +.. code-block:: bash + + pytest \ + --mpl-generate-hash-library=test_hashes.json \ + --mpl-generate-path=baseline \ + --mpl-results-path=results \ + --mpl-generate-summary=html,json + +:summary:`test_html_generate` + +You should choose a directory within you repository to store the baseline hashes. +It's usually a good idea to encode the Matplotlib version and the FreeType version in the filename, e.g. ``mpl35_ft261.json``. +The hash library file should then be committed to the repository. + +Running hash comparison tests +============================= + +When running the tests, the ``--mpl`` flag should be used along with a :ref:`configured hash library path ` to enable baseline hash comparison testing: + +.. code-block:: bash + + pytest --mpl \ + --mpl-hash-library=your_project/tests/mpl35_ft261.json + +Optionally, a HTML summary report can be generated by setting ``--mpl-generate-summary``: + +.. code-block:: bash + + pytest --mpl \ + --mpl-hash-library=your_project/tests/mpl35_ft261.json \ + --mpl-results-path=results \ + --mpl-generate-summary=html,json + +:summary:`test_html_hashes_only` + +The ``--mpl-results-path`` flag can be used to set the directory where the generated HTML summary will be stored. +If this is not set, the images will be stored in a temporary directory. + +Continue reading +================ + +``pytest-mpl`` has many configuration options that can be used to customize the behavior of the hash comparison mode. +Only a few of the most commonly used options are covered in this guide. +See the :doc:`configuration options documentation ` for full details. diff --git a/docs/hybrid_mode.rst b/docs/hybrid_mode.rst new file mode 100644 index 00000000..f5b1ebfb --- /dev/null +++ b/docs/hybrid_mode.rst @@ -0,0 +1,90 @@ +.. title:: Hybrid mode + +############################## +Hybrid Mode: Hashes and Images +############################## + +This how-to guide will show you how to use the hybrid mode of ``pytest-mpl``. + +For a full description of the hybrid mode, see the :ref:`hybrid mode section of the get started guide `. +In summary, hybrid mode uses both baseline images and hashes. +First, the hash of the image is compared to the hash of the baseline image. +If the hashes match, the test passes. +If the hashes do not match, the test fails. + +The difference with hybrid mode is that a baseline image comparison will also be carried out if the hashes do not match, or always :ref:`if this has been configured `. +The purpose of the additional image comparison (which does not affect the test result) is to allow the user to visually inspect the difference between the baseline image and the image generated by the test. + +In order to keep the code repository size small, it is recommended to store the baseline hashes in the code repository, and the baseline images in a separate repository. +The baseline hashes should be updated where appropriate in PRs to the code repository. +However, the baseline images are not updated in these PRs. +Instead, they should be updated once the PR has been merged, preferably by a CI job. + +Another benefit of only updating the baseline images once the PR has been merged is that the PR tests will show the difference between the remote baseline images and the images generated by the PR. +Even though the tests will pass when the baseline hash matches, the images will still be compared and the difference will be shown in the HTML test summary report, which is useful when reviewing the PR. + +Generating baseline hashes and images +===================================== + +Once a suite of image comparison tests have been written, baseline hashes and images should be generated by setting ``--mpl-generate-path`` and ``--mpl-generate-hash-library``: + +.. code-block:: bash + + pytest \ + --mpl-generate-hash-library=your_project/tests/test_hashes.json \ + --mpl-generate-path=baseline \ + --mpl-results-path=results \ + --mpl-generate-summary=html,json + +:summary:`test_html_generate` + +Open the HTML summary file and inspect the figures to ensure that the baseline images are correct. +If they are, the baseline hashes can be committed to the code repository. +It's usually a good idea to encode the Matplotlib version and the FreeType version in the filename, e.g. ``mpl35_ft261.json``. +The baseline images should be copied to a separate repository; preferably within a version specific directory, e.g. ``mpl35_ft261/``. + +Running hash comparison tests +============================= + +When running the tests, the ``--mpl`` flag should be used along with a configured :ref:`hash library path ` and :ref:`baseline image path ` to enable hybrid mode testing: + +.. code-block:: bash + + pytest --mpl \ + --mpl-hash-library=your_project/tests/mpl35_ft261.json \ + --mpl-baseline-path=https://raw.githubusercontent.com/your-org/your-project-figure-tests/mpl35_ft261/ \ + --mpl-results-path=results \ + --mpl-generate-summary=html,json + +:summary:`test_html` + +The ``--mpl-results-path`` flag can be used to set the directory where the generated HTML summary will be stored. +If this is not set, the images will be stored in a temporary directory. + +Notice that the baseline image path is set to a URL, which is the location of the baseline images in the separate repository. +When the baseline image comparison is carried out, the baseline images will be downloaded from this URL. + +It is recommended to create a CI job that updates the baseline images in the separate repository once the PR has been merged. +The CI job should run when new commits are pushed to the default branch. +The baseline images should only be regenerated and updated if the tests pass in the hash comparison mode. + +.. rubric:: Aside: basic HTML summary + +This is what the basic HTML summary looks like for the same test as above: + +.. code-block:: bash + + pytest --mpl \ + --mpl-hash-library=your_project/tests/mpl35_ft261.json \ + --mpl-baseline-path=https://raw.githubusercontent.com/your-org/your-project-figure-tests/mpl35_ft261/ \ + --mpl-results-path=results \ + --mpl-generate-summary=basic-html,json + +:summary:`test_basic_html` + +Continue reading +================ + +``pytest-mpl`` has many configuration options that can be used to customize the behavior of the hybrid mode. +Only a few of the most commonly used options are covered in this guide. +See the :doc:`configuration options documentation ` for full details. diff --git a/docs/image_mode.rst b/docs/image_mode.rst new file mode 100644 index 00000000..13a907f5 --- /dev/null +++ b/docs/image_mode.rst @@ -0,0 +1,70 @@ +.. title:: Image comparison mode + +##################### +Image Comparison Mode +##################### + +This how-to guide will show you how to use the image comparison mode of ``pytest-mpl``. +This is the default mode when ``pytest-mpl`` is invoked with ``--mpl``. + +In this mode, ``pytest-mpl`` will compare the images generated by the test with a baseline image. +If the images are different, :ref:`to a specified RMS tolerance `, the test will fail. +The test will also fail if the baseline image is not found or if the test does not generate a figure. +If the figure has a different aspect ratio to the baseline image, the test will also fail. + +Generating baseline images +========================== + +Once a suite of image comparison tests have been written, baseline images should be generated by setting ``--mpl-generate-path``: + +.. code-block:: bash + + pytest --mpl-generate-path=your_project/tests/baseline + +You should choose a directory within you repository to store the baseline images. +The generated images should be checked to ensure they are as expected. +The images should then be committed to the repository. + +To assist with inspecting the generated images, a HTML summary report can be generated by setting ``--mpl-generate-summary``: + +.. code-block:: bash + + pytest \ + --mpl-generate-path=your_project/tests/baseline \ + --mpl-generate-summary=html,json + +:summary:`test_html_generate_images_only` + +Running image comparison tests +============================== + +When running the tests, the ``--mpl`` flag should be used to enable baseline image comparison testing: + +.. code-block:: bash + + pytest --mpl \ + --mpl-baseline-path=your_project/tests/baseline + +The :ref:`baseline path ` should be set to the directory containing the baseline images. +If the baseline images are not found, the test will fail. + +Optionally, a HTML summary report can be generated by setting ``--mpl-generate-summary``: + +.. code-block:: bash + + pytest --mpl \ + --mpl-baseline-path=your_project/tests/baseline \ + --mpl-results-path=results \ + --mpl-generate-summary=html,json + +:summary:`test_html_images_only` + +The ``--mpl-results-path`` flag can be used to set the directory where the generated HTML summary will be stored. +If this is not set, the images will be stored in a temporary directory. + +Continue reading +================ + +``pytest-mpl`` has many configuration options that can be used to customize the behavior of the image comparison mode. +Only a few of the most commonly used options are covered in this guide. +See the :doc:`configuration options documentation ` for full details. diff --git a/images/baseline-coords_overlay_auto_coord_meta.png b/docs/images/baseline-coords_overlay_auto_coord_meta.png similarity index 100% rename from images/baseline-coords_overlay_auto_coord_meta.png rename to docs/images/baseline-coords_overlay_auto_coord_meta.png diff --git a/images/coords_overlay_auto_coord_meta-failed-diff.png b/docs/images/coords_overlay_auto_coord_meta-failed-diff.png similarity index 100% rename from images/coords_overlay_auto_coord_meta-failed-diff.png rename to docs/images/coords_overlay_auto_coord_meta-failed-diff.png diff --git a/images/coords_overlay_auto_coord_meta.png b/docs/images/coords_overlay_auto_coord_meta.png similarity index 100% rename from images/coords_overlay_auto_coord_meta.png rename to docs/images/coords_overlay_auto_coord_meta.png diff --git a/images/html_all.png b/docs/images/html_all.png similarity index 100% rename from images/html_all.png rename to docs/images/html_all.png diff --git a/images/html_filter.png b/docs/images/html_filter.png similarity index 100% rename from images/html_filter.png rename to docs/images/html_filter.png diff --git a/images/html_result.png b/docs/images/html_result.png similarity index 100% rename from images/html_result.png rename to docs/images/html_result.png diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..f48b5448 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,98 @@ +.. title:: pytest-mpl documentation + +.. module:: pytest-mpl + +.. toctree:: + :hidden: + + installing + usage + image_mode + hash_mode + hybrid_mode + configuration + +################################## +pytest-mpl |release| documentation +################################## + +``pytest-mpl`` is a `pytest `__ plugin to facilitate image comparison for `Matplotlib `__ figures. + +For each figure to test, an image is generated and then subtracted from an existing reference image. +If the RMS of the residual is larger than a user-specified tolerance, the test will fail. +Alternatively, the generated image can be hashed and compared to an expected value. + +************ +Installation +************ + +.. grid:: 1 1 2 2 + + .. grid-item:: + + Install using `pip `__: + + .. code-block:: bash + + pip install pytest-mpl + + .. grid-item:: + + Install from `conda-forge `__ using `conda `__: + + .. code-block:: bash + + conda install pytest-mpl + +Further details are available in the :doc:`Installation Guide `. + + +****************** +Learning resources +****************** + +.. grid:: 1 1 2 2 + + .. grid-item-card:: + :padding: 2 + + Tutorials + ^^^ + + - :doc:`Get started ` + + .. grid-item-card:: + :padding: 2 + + How-tos + ^^^ + + - :doc:`Image comparison mode ` + - :doc:`Hash comparison mode ` + - :doc:`Hybrid mode ` + + .. grid-item-card:: + :padding: 2 + + Understand how pytest-mpl works + ^^^ + + Explanatory information is included where relevant throughout the documentation. + + .. grid-item-card:: + :padding: 2 + + Reference + ^^^ + + - :doc:`Configuration ` + +************ +Contributing +************ + +``pytest-mpl`` is a community project maintained for and by its users. +There are many ways you can help! + +- Report a bug or request a feature `on GitHub `__ +- Improve the documentation or code diff --git a/docs/installing.rst b/docs/installing.rst new file mode 100644 index 00000000..9bd0279c --- /dev/null +++ b/docs/installing.rst @@ -0,0 +1,86 @@ +.. title:: Installation Guide + +################## +Installation Guide +################## + +This plugin is compatible with Python 3.6 and later, and +requires `pytest `__ and +`matplotlib `__ to be installed. + +Using pip +========= + +``pytest-mpl`` can be installed with ``pip``: + +.. code-block:: bash + + pip install pytest-mpl + + +Using conda +=========== + +Installing ``pytest-mpl`` from the ``conda-forge`` channel can be achieved by adding ``conda-forge`` to your channels with: + +.. code-block:: bash + + conda config --add channels conda-forge + conda config --set channel_priority strict + +Once the ``conda-forge`` channel has been enabled, ``pytest-mpl`` can be installed with ``conda``: + +.. code-block:: bash + + conda install pytest-mpl + +or with ``mamba``: + +.. code-block:: bash + + mamba install pytest-mpl + +It is possible to list all of the versions of ``pytest-mpl`` available on your platform with ``conda``: + +.. code-block:: bash + + conda search pytest-mpl --channel conda-forge + +or with ``mamba``: + +.. code-block:: bash + + mamba search pytest-mpl --channel conda-forge + +Alternatively, ``mamba repoquery`` may provide more information: + +.. code-block:: bash + + # Search all versions available on your platform: + mamba repoquery search pytest-mpl --channel conda-forge + + # List packages depending on pytest-mpl: + mamba repoquery whoneeds pytest-mpl --channel conda-forge + + # List dependencies of pytest-mpl: + mamba repoquery depends pytest-mpl --channel conda-forge + +Installing the development version +================================== + +Clone the `pytest-mpl GitHub repository `__, or your own fork of it. +Then install ``pytest-mpl`` using ``pip`` from the root directory of the repo: + +.. code-block:: bash + + pip install -e ".[test,docs]" + + +Troubleshooting +=============== + +To check that ``pytest-mpl`` has been installed correctly and is recognised by ``pytest``, run: + +.. code-block:: bash + + pytest --trace-config diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 00000000..32bb2452 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/sample_summaries.py b/docs/sample_summaries.py new file mode 100644 index 00000000..c1114fb0 --- /dev/null +++ b/docs/sample_summaries.py @@ -0,0 +1,112 @@ +import os +import shutil +import pathlib +import tempfile +import subprocess + +from docutils import nodes +from sphinx.util.docutils import SphinxRole +from sphinx.util.osutil import canon_path + +REPO_ROOT = pathlib.Path(__file__).parent.parent +TEST_FILE = REPO_ROOT / "tests" / "subtests" / "test_subtest.py" +SAMPLE_DIR = "sample" + + +def run_pytest(test_name): + + # Create generated samples directory + sample_dir_abs = pathlib.Path(__file__).parent / SAMPLE_DIR + if not sample_dir_abs.exists(): + os.mkdir(sample_dir_abs) + + # Form path to current sample + dest = sample_dir_abs / test_name + if dest.exists(): + return dest # skip if already generated + + # Generate the current sample + tmp_dir = tempfile.mkdtemp() + command = f"python -m pytest {TEST_FILE}::{test_name} -v --mpl --basetemp={tmp_dir}" + subprocess.run(command, shell=True, check=True) + + # Find the name of the directory the sample is within + # (directory name is sometimes truncated) + src = next(filter( + lambda x: x.name[:-1] in test_name, + pathlib.Path(tmp_dir).glob("*0") + )) / "results" + shutil.copytree(src, dest) + + return dest + + +class SummaryButtons(nodes.General, nodes.Inline, nodes.TextElement): + pass + + +class SummaryRole(SphinxRole): + def run(self): + node = SummaryButtons(name=self.text) + return [node], [] + + +def move_summaries(app, *args, **kwargs): + gen_sample_dir = pathlib.Path(__file__).parent / SAMPLE_DIR + out_sample_dir = pathlib.Path(app.outdir) / SAMPLE_DIR + if out_sample_dir.exists(): + shutil.rmtree(out_sample_dir) + shutil.copytree(gen_sample_dir, out_sample_dir) + + +def html_visit_summary(self, node): + + test_name = str(node["name"]) + out = run_pytest(test_name) + + classes = ( + "sd-sphinx-override sd-btn sd-text-wrap sd-btn-{importance} " + "sd-shadow-sm sd-me-2 reference internal" + ) + button = ( + '{label}' + ) + + summary_types = { + "HTML": "fig_comparison.html", + "Basic HTML": "fig_comparison_basic.html", + "JSON": "results.json", + } + + current_filename = self.builder.current_docname + self.builder.out_suffix + current_dir = pathlib.PurePath(current_filename).parent + first_button = True + for label, file in summary_types.items(): + if (out / file).exists(): + importance = "primary" if first_button else "secondary" + self.body.append(button.format( + classes=classes.format(importance=importance), + href=canon_path((current_dir / SAMPLE_DIR / test_name / file).as_posix()), + label=label, + )) + first_button = False + + raise nodes.SkipNode + + +def skip(self, node): + raise nodes.SkipNode + + +def setup(app): + app.connect("build-finished", move_summaries) + app.add_node( + SummaryButtons, + html=(html_visit_summary, None), + latex=(skip, None), + text=(skip, None), + man=(skip, None), + texinfo=(skip, None), + ) + app.add_role("summary", SummaryRole()) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs/usage.rst b/docs/usage.rst new file mode 100644 index 00000000..0c944171 --- /dev/null +++ b/docs/usage.rst @@ -0,0 +1,247 @@ +.. title:: Get Started + +########### +Get Started +########### + +This section describes how to configure simple image comparison testing with ``pytest-mpl``. +It assumes that you already have a `pytest `__ test suite set up for your project. + +Install ``pytest-mpl`` +^^^^^^^^^^^^^^^^^^^^^^ + +First, install ``pytest-mpl`` and also include it in your project's "test" dependencies: + +.. code-block:: python + + pip install pytest-mpl + +Write image comparison tests +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Then, alongside your existing tests, write test functions that create a figure. +These image comparison tests must be decorated with ``@pytest.mark.mpl_image_compare`` and return the figure for testing: + +.. code-block:: python + + import matplotlib.pyplot as plt + import pytest + + @pytest.mark.mpl_image_compare + def test_plot(): + fig, ax = plt.subplots() + ax.plot([1, 2]) + return fig + +Generate baseline images +^^^^^^^^^^^^^^^^^^^^^^^^ + +Then, generate reference images by running the test suite with the ``--mpl-generate-path`` option: + +.. code-block:: bash + + pytest --mpl-generate-path=baseline + +If, for example, you are using a directory structure like this: + +.. code-block:: + + . + └── tests + ├── plotting + │ └── test_plotting.py + └── utils + └── test_utils.py + +Then, the generated images will be placed in a new directory called ``baseline`` located where you ran pytest: + +.. code-block:: + + . + ├── baseline + │ ├── test_plot.png + │ └── test_util.png + └── tests + ├── plotting + │ └── test_plotting.py + └── utils + └── test_utils.py + +Take a look at the generated images inside the new ``baseline`` directory. +If they are correct, move each baseline image to a sub-directory called ``baseline`` relative to the test files: + +.. code-block:: + + . + └── tests + ├── plotting + │ ├── baseline + │ │ └── test_plot.png + │ └── test_plotting.py + └── utils + ├── baseline + │ └── test_util.png + └── test_utils.py + +Then, commit these baseline images to your repository. + +Run the image comparison tests +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You now have a set of baseline images that can be used to verify the figures generated by your code. +You can now run the test suite as usual. +To enable image comparison testing, pass ``--mpl``: + +.. code-block:: bash + + pytest --mpl + +You should observe that the tests pass. + +Try modifying the test function and run the test suite again. +The test should now fail, because the generated image does not match the reference image. +Try running the test suite without ``--mpl``. +Even through the figure has changed, the test will pass, because image comparison testing is disabled. + +.. rubric:: Running pytest without pytest-mpl installed + +If ``pytest-mpl`` is not installed, the image comparison tests will cause pytest to show a warning, ``PytestReturnNotNoneWarning``. +Installing pytest-mpl will solve this issue. +When ``pytest-mpl`` is installed but not enabled, it will intercept the returned figure and close it without doing any comparison. + +Alternatively, the image comparison tests can be deselected by running pytest with ``-m "not mpl_image_compare"``. +Or the following can be included in your test functions to skip if ``pytest-mpl`` is not installed: + +.. code-block:: python + + @pytest.mark.mpl_image_compare + def test_plot(): + pytest.importorskip("pytest_mpl") + ... + +.. rubric:: Tests can fail when Matplotlib and FreeType versions change + +If the Matplotlib version changes, or if the FreeType version changes, the generated images may change. +This is mostly because the text rendering in Matplotlib is dependent on the FreeType version. +It is recommended to pin the Matplotlib and FreeType versions in your testing environments to avoid this issue. +There are also a number of :ref:`configuration options for controlling the sensitivity of the comparison `. + +Image comparison mode +^^^^^^^^^^^^^^^^^^^^^ + +The above example uses the image comparison mode, which is the default when just ``--mpl`` is set. +Pros and cons of this mode are: + +- :octicon:`diff-added;1em;sd-text-success` Easy to configure +- :octicon:`diff-added;1em;sd-text-success` Easy to run the tests and see the results +- :octicon:`diff-removed;1em;sd-text-danger` Baseline images usually need to be checked into the repository with the tests (larger repo) + +For a more detailed example of image comparison testing, see the :doc:`image comparison mode how-to guide `. +Also see the :doc:`configuration guide ` for more information on configuring image comparison testing. + +Hash comparison mode +^^^^^^^^^^^^^^^^^^^^ + +Instead of comparing to baseline images, you can instead compare against a JSON library of SHA-256 hashes of the baseline image files. +Pros and cons of this mode are: + +- :octicon:`diff-added;1em;sd-text-success` Easy to configure +- :octicon:`diff-removed;1em;sd-text-danger` Difficult to *see* how a failing test differs from the baseline +- :octicon:`diff-added;1em;sd-text-success` Baseline images do not need to be checked into the repository with the tests (smaller repo) + +See the :doc:`hash comparison mode how-to guide ` for more information on how to use this mode. +Also see the :doc:`configuration guide ` for more information on configuring hash comparison testing. + +.. _hybrid-usage: + +Hybrid mode: hash, then image, comparison +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can also use a "hybrid mode", which uses both baseline images and hashes. +A figure is first *assessed* against the baseline hash. +If the hash does not match, the figure will then be *compared* to the baseline image. +This mode can also be configured to :ref:`always compare the figure to the baseline image `, even if the hash matches. + +In this mode, only the hash assessment will affect the success status of the test. +If the hash assessment fails, the test will fail, even if the image comparison passes. + +This mode is intended for projects which have a large number of tests, and where it is impractical to store all of the baseline images in the repository. +These projects can use this mode to store the baseline images in a separate repository, and only store hashes in the main repository. +In PRs, contributors only need to update the hashes, and the CI tests will pass if the hashes match. +For the baseline image comparison, ``pytest-mpl`` will download the baseline image from a URL and compare it to the generated image. + +Pros and cons of this mode are: + +- :octicon:`diff-removed;1em;sd-text-danger` Usually more complex to configure (managing a separate baseline image repository) +- :octicon:`diff-added;1em;sd-text-success` Easy to run the tests and see the results +- :octicon:`diff-added;1em;sd-text-success` Baseline images can be stored in a separate repository (smaller main repo) + +See the :doc:`hybrid mode how-to guide ` for more information on how to use this mode. +Also see the :doc:`configuration guide ` for more information on configuring hybrid comparison testing. + +Test results +^^^^^^^^^^^^ + +By default, the expected, actual, and difference files are written to a temporary directory with a non-deterministic path. +You can :ref:`configure the results directory ` to save to a specific location:: + + pytest --mpl --mpl-results-path=results + +The ``results`` directory will then contain one sub-directory per test, and each sub-directory will contain the files mentioned above. +If you are using a continuous integration (CI) service, you can upload this directory as an artifact. + +HTML summary reports +-------------------- + +``pytest-mpl`` can also generate HTML reports of the image comparison results, allowing you to see the results of the image comparison tests in a web browser. +See the :ref:`configuration documentation ` for more information on how to generate the HTML report. +Some CI services, such as CircleCI, can host the HTML summary on a web server so it can be viewed directly in the browser. +On other CI services, such as GitHub Actions, you can download the artifact and open the local HTML file in a web browser. + ++---------------+---------------+---------------+ +| |html all| | |html filter| | |html result| | ++---------------+---------------+---------------+ + +Test failure example +-------------------- + +If the images produced by the tests are correct, then the test will pass. +If the images are not correct, the test will fail and a message similar to the following will be shown in the pytest logs:: + + E Exception: Error: Image files did not match. + E RMS Value: 142.2287807767823 + E Expected: + E /var/folders/zy/t1l3sx310d3d6p0kyxqzlrnr0000gr/T/tmp4h4oxr7y/baseline-coords_overlay_auto_coord_meta.png + E Actual: + E /var/folders/zy/t1l3sx310d3d6p0kyxqzlrnr0000gr/T/tmp4h4oxr7y/coords_overlay_auto_coord_meta.png + E Difference: + E /var/folders/zy/t1l3sx310d3d6p0kyxqzlrnr0000gr/T/tmp4h4oxr7y/coords_overlay_auto_coord_meta-failed-diff.png + E Tolerance: + E 10 + +The image paths included in the exception are then available for inspection: + ++----------------+----------------+-------------+ +| Expected | Actual | Difference | ++================+================+=============+ +| |expected| | |actual| | |diff| | ++----------------+----------------+-------------+ + +In this case, the differences are very clear, while in some cases it may be necessary to use the difference image, or blink the expected and actual images, in order to see what changed. + +Continue reading +^^^^^^^^^^^^^^^^ + +See the :doc:`configuration guide ` for more information on configuring ``pytest-mpl``. +For examples of how to configure the different operating modes, see the following how-to guides: + +* :doc:`image_mode` +* :doc:`hash_mode` +* :doc:`hybrid_mode` + +.. |html all| image:: images/html_all.png +.. |html filter| image:: images/html_filter.png +.. |html result| image:: images/html_result.png +.. |expected| image:: images/baseline-coords_overlay_auto_coord_meta.png +.. |actual| image:: images/coords_overlay_auto_coord_meta.png +.. |diff| image:: images/coords_overlay_auto_coord_meta-failed-diff.png diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 3ddc24b4..6232db20 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -41,10 +41,15 @@ from urllib.request import urlopen import pytest +from packaging.version import Version from pytest_mpl.summary.html import generate_summary_basic_html, generate_summary_html -SUPPORTED_FORMATS = {'html', 'json', 'basic-html'} +DEFAULT_STYLE = "classic" +DEFAULT_TOLERANCE = 2 +DEFAULT_BACKEND = "agg" + +SUPPORTED_FORMATS = {"html", "json", "basic-html"} SHAPE_MISMATCH_ERROR = """Error: Image dimensions did not match. Expected shape: {expected_shape} @@ -52,6 +57,19 @@ Actual shape: {actual_shape} {actual_path}""" +PYTEST_LT_7 = Version(pytest.__version__) < Version("7.0.0") + +# The following are the subsets of formats supported by the Matplotlib image +# comparison machinery +RASTER_IMAGE_FORMATS = ['png'] +VECTOR_IMAGE_FORMATS = ['eps', 'pdf', 'svg'] +ALL_IMAGE_FORMATS = RASTER_IMAGE_FORMATS + VECTOR_IMAGE_FORMATS + + +def _get_item_dir(item): + path = Path(item.fspath) if PYTEST_LT_7 else item.path + return path.parent + def _hash_file(in_stream): """ @@ -70,8 +88,8 @@ def pathify(path): """ path = Path(path) ext = '' - if path.suffixes[-1] == '.png': - ext = '.png' + if path.suffixes[-1][1:] in ALL_IMAGE_FORMATS: + ext = path.suffixes[-1] path = str(path).split(ext)[0] path = str(path) path = path.replace('[', '_').replace(']', '_') @@ -110,7 +128,7 @@ def wrapper(*args, **kwargs): item.obj = figure_interceptor(plugin, item.obj) -def pytest_report_header(config, startdir): +def pytest_report_header(): import matplotlib import matplotlib.ft2font return ["Matplotlib: {0}".format(matplotlib.__version__), @@ -119,71 +137,141 @@ def pytest_report_header(config, startdir): def pytest_addoption(parser): group = parser.getgroup("matplotlib image comparison") - group.addoption('--mpl', action='store_true', - help="Enable comparison of matplotlib figures to reference files") - group.addoption('--mpl-generate-path', - help="directory to generate reference images in, relative " - "to location where py.test is run", action='store') - group.addoption('--mpl-generate-hash-library', - help="filepath to save a generated hash library, relative " - "to location where py.test is run", action='store') - group.addoption('--mpl-baseline-path', - help="directory containing baseline images, relative to " - "location where py.test is run unless --mpl-baseline-relative is given. " - "This can also be a URL or a set of comma-separated URLs (in case " - "mirrors are specified)", action='store') - group.addoption("--mpl-baseline-relative", help="interpret the baseline directory as " - "relative to the test location.", action="store_true") - group.addoption('--mpl-hash-library', - help="json library of image hashes, relative to " - "location where py.test is run", action='store') - group.addoption('--mpl-generate-summary', action='store', - help="Generate a summary report of any failed tests" - ", in --mpl-results-path. The type of the report should be " - "specified. Supported types are `html`, `json` and `basic-html`. " - "Multiple types can be specified separated by commas.") - - results_path_help = "directory for test results, relative to location where py.test is run" - group.addoption('--mpl-results-path', help=results_path_help, action='store') - parser.addini('mpl-results-path', help=results_path_help) - - results_always_help = ("Always compare to baseline images and save result images, even for passing tests. " - "This option is automatically applied when generating a HTML summary.") - group.addoption('--mpl-results-always', action='store_true', - help=results_always_help) - parser.addini('mpl-results-always', help=results_always_help) - - parser.addini('mpl-use-full-test-name', help="use fully qualified test name as the filename.", - type='bool') + + msg = "Enable comparison of matplotlib figures to reference files" + group.addoption("--mpl", help=msg, action="store_true") + + msg = "directory to generate reference images in, relative to location where py.test is run" + group.addoption("--mpl-generate-path", help=msg, action="store") + + msg = "filepath to save a generated hash library, relative to location where py.test is run" + group.addoption("--mpl-generate-hash-library", help=msg, action="store") + + msg = ( + "directory containing baseline images, relative to " + "location where py.test is run unless --mpl-baseline-relative is given. " + "This can also be a URL or a set of comma-separated URLs (in case " + "mirrors are specified)" + ) + option = "mpl-baseline-path" + group.addoption(f"--{option}", help=msg, action="store") + parser.addini(option, help=msg) + + msg = "interpret the baseline directory as relative to the test location." + group.addoption("--mpl-baseline-relative", help=msg, action="store_true") + + msg = "json library of image hashes, relative to location where py.test is run" + option = "mpl-hash-library" + group.addoption(f"--{option}", help=msg, action="store") + parser.addini(option, help=msg) + + msg = ( + "Generate a summary report of any failed tests" + ", in --mpl-results-path. The type of the report should be " + "specified. Supported types are `html`, `json` and `basic-html`. " + "Multiple types can be specified separated by commas." + ) + option = "mpl-generate-summary" + group.addoption(f"--{option}", help=msg, action="store") + parser.addini(option, help=msg) + + msg = "directory for test results, relative to location where py.test is run" + option = "mpl-results-path" + group.addoption(f"--{option}", help=msg, action="store") + parser.addini(option, help=msg) + + msg = ( + "Always compare to baseline images and save result images, even for passing tests. " + "This option is automatically applied when generating a HTML summary." + ) + option = "mpl-results-always" + group.addoption(f"--{option}", help=msg, action="store_true") + parser.addini(option, help=msg) + + msg = "use fully qualified test name as the filename." + option = "mpl-use-full-test-name" + group.addoption(f"--{option}", help=msg, action="store_true") + parser.addini(option, help=msg, type="bool") + + msg = "default style to use for tests, unless specified in the mpl_image_compare decorator" + option = "mpl-default-style" + group.addoption(f"--{option}", help=msg, action="store") + parser.addini(option, help=msg) + + msg = "default tolerance to use for tests, unless specified in the mpl_image_compare decorator" + option = "mpl-default-tolerance" + group.addoption(f"--{option}", help=msg, action="store") + parser.addini(option, help=msg) + + msg = "whether to make the image file metadata deterministic" + option_true = "mpl-deterministic" + option_false = "mpl-no-deterministic" + group.addoption(f"--{option_true}", help=msg, action="store_true") + group.addoption(f"--{option_false}", help=msg, action="store_true") + parser.addini(option_true, help=msg, type="bool", default=None) + + msg = "default backend to use for tests, unless specified in the mpl_image_compare decorator" + option = "mpl-default-backend" + group.addoption(f"--{option}", help=msg, action="store") + parser.addini(option, help=msg) def pytest_configure(config): - config.addinivalue_line('markers', - "mpl_image_compare: Compares matplotlib figures " - "against a baseline image") + config.addinivalue_line( + "markers", + "mpl_image_compare: Compares matplotlib figures against a baseline image", + ) - if (config.getoption("--mpl") or - config.getoption("--mpl-generate-path") is not None or - config.getoption("--mpl-generate-hash-library") is not None): + if ( + config.getoption("--mpl") + or config.getoption("--mpl-generate-path") is not None + or config.getoption("--mpl-generate-hash-library") is not None + ): + + def get_cli_or_ini(name, default=None): + return config.getoption(f"--{name}") or config.getini(name) or default - baseline_dir = config.getoption("--mpl-baseline-path") generate_dir = config.getoption("--mpl-generate-path") generate_hash_lib = config.getoption("--mpl-generate-hash-library") - results_dir = config.getoption("--mpl-results-path") or config.getini("mpl-results-path") - hash_library = config.getoption("--mpl-hash-library") - generate_summary = config.getoption("--mpl-generate-summary") - results_always = (config.getoption("--mpl-results-always") or - config.getini("mpl-results-always")) + baseline_dir = get_cli_or_ini("mpl-baseline-path") if config.getoption("--mpl-baseline-relative"): baseline_relative_dir = config.getoption("--mpl-baseline-path") else: baseline_relative_dir = None + use_full_test_name = get_cli_or_ini("mpl-use-full-test-name") + + hash_library = get_cli_or_ini("mpl-hash-library") + _hash_library_from_cli = bool(config.getoption("--mpl-hash-library")) # for backwards compatibility + + default_tolerance = get_cli_or_ini("mpl-default-tolerance", DEFAULT_TOLERANCE) + if isinstance(default_tolerance, str): + if default_tolerance.isdigit(): # prefer int if possible + default_tolerance = int(default_tolerance) + else: + default_tolerance = float(default_tolerance) + + deterministic_ini = config.getini("mpl-deterministic") + deterministic_flag_true = config.getoption("--mpl-deterministic") + deterministic_flag_false = config.getoption("--mpl-no-deterministic") + if deterministic_flag_true and deterministic_flag_false: + raise ValueError("Only one of `--mpl-deterministic` and `--mpl-no-deterministic` can be set.") + if deterministic_flag_true: + deterministic = True + elif deterministic_flag_false: + deterministic = False + elif isinstance(deterministic_ini, bool): + deterministic = deterministic_ini + else: + deterministic = None + + default_style = get_cli_or_ini("mpl-default-style", DEFAULT_STYLE) + default_backend = get_cli_or_ini("mpl-default-backend", DEFAULT_BACKEND) - # Note that results_dir is an empty string if not specified - if not results_dir: - results_dir = None + results_dir = get_cli_or_ini("mpl-results-path") + results_always = get_cli_or_ini("mpl-results-always") + generate_summary = get_cli_or_ini("mpl-generate-summary") if generate_dir is not None: if baseline_dir is not None: @@ -195,19 +283,31 @@ def pytest_configure(config): baseline_dir = os.path.abspath(generate_dir) if results_dir is not None: results_dir = os.path.abspath(results_dir) - - config.pluginmanager.register(ImageComparison(config, - baseline_dir=baseline_dir, - baseline_relative_dir=baseline_relative_dir, - generate_dir=generate_dir, - results_dir=results_dir, - hash_library=hash_library, - generate_hash_library=generate_hash_lib, - generate_summary=generate_summary, - results_always=results_always)) + if hash_library is not None: + # For backwards compatibility, don't make absolute if set via CLI option + if not _hash_library_from_cli: + hash_library = os.path.abspath(hash_library) + + plugin = ImageComparison( + config, + baseline_dir=baseline_dir, + baseline_relative_dir=baseline_relative_dir, + generate_dir=generate_dir, + results_dir=results_dir, + hash_library=hash_library, + generate_hash_library=generate_hash_lib, + generate_summary=generate_summary, + results_always=results_always, + use_full_test_name=use_full_test_name, + default_style=default_style, + default_tolerance=default_tolerance, + deterministic=deterministic, + default_backend=default_backend, + _hash_library_from_cli=_hash_library_from_cli, + ) + config.pluginmanager.register(plugin) else: - config.pluginmanager.register(FigureCloser(config)) @@ -250,24 +350,31 @@ def path_is_not_none(apath): class ImageComparison: - - def __init__(self, - config, - baseline_dir=None, - baseline_relative_dir=None, - generate_dir=None, - results_dir=None, - hash_library=None, - generate_hash_library=None, - generate_summary=None, - results_always=False - ): + def __init__( + self, + config, + baseline_dir=None, + baseline_relative_dir=None, + generate_dir=None, + results_dir=None, + hash_library=None, + generate_hash_library=None, + generate_summary=None, + results_always=False, + use_full_test_name=False, + default_style=DEFAULT_STYLE, + default_tolerance=DEFAULT_TOLERANCE, + deterministic=None, + default_backend=DEFAULT_BACKEND, + _hash_library_from_cli=False, # for backwards compatibility + ): self.config = config self.baseline_dir = baseline_dir self.baseline_relative_dir = path_is_not_none(baseline_relative_dir) self.generate_dir = path_is_not_none(generate_dir) self.results_dir = path_is_not_none(results_dir) self.hash_library = path_is_not_none(hash_library) + self._hash_library_from_cli = _hash_library_from_cli # for backwards compatibility self.generate_hash_library = path_is_not_none(generate_hash_library) if generate_summary: generate_summary = {i.lower() for i in generate_summary.split(',')} @@ -280,6 +387,12 @@ def __init__(self, results_always = True self.generate_summary = generate_summary self.results_always = results_always + self.use_full_test_name = use_full_test_name + + self.default_style = default_style + self.default_tolerance = default_tolerance + self.deterministic = deterministic + self.default_backend = default_backend # Generate the containing dir for all test results if not self.results_dir: @@ -298,24 +411,41 @@ def __init__(self, self._test_stats = None self.return_value = {} - # https://stackoverflow.com/questions/51737378/how-should-i-log-in-my-pytest-plugin - # turn debug prints on only if "-vv" or more passed + # configure a separate logger for this pluggin which is independent + # of the options that are configured for pytest or for the code that + # is tested; turn debug prints on only if "-vv" or more passed level = logging.DEBUG if config.option.verbose > 1 else logging.INFO - logging.basicConfig(level=level) + if config.option.log_cli_format is not None: + fmt = config.option.log_cli_format + else: + # use pytest's default fmt + fmt = "%(levelname)-8s %(name)s:%(filename)s:%(lineno)d %(message)s" + formatter = logging.Formatter(fmt) + handler = logging.StreamHandler() + handler.setFormatter(formatter) self.logger = logging.getLogger('pytest-mpl') + self.logger.propagate = False + self.logger.setLevel(level) + self.logger.addHandler(handler) + + def _file_extension(self, item): + compare = get_compare(item) + savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) + return savefig_kwargs.get('format', 'png') def generate_filename(self, item): """ Given a pytest item, generate the figure filename. """ - if self.config.getini('mpl-use-full-test-name'): - filename = generate_test_name(item) + '.png' + ext = self._file_extension(item) + if self.use_full_test_name: + filename = generate_test_name(item) + f'.{ext}' else: compare = get_compare(item) # Find test name to use as plot name filename = compare.kwargs.get('filename', None) if filename is None: - filename = item.name + '.png' + filename = item.name + f'.{ext}' filename = str(pathify(filename)) return filename @@ -348,11 +478,11 @@ def get_baseline_directory(self, item): baseline_dir = compare.kwargs.get('baseline_dir', None) if baseline_dir is None: if self.baseline_dir is None: - baseline_dir = Path(item.fspath).parent / 'baseline' + baseline_dir = _get_item_dir(item) / 'baseline' else: if self.baseline_relative_dir: # baseline dir is relative to the current test - baseline_dir = Path(item.fspath).parent / self.baseline_relative_dir + baseline_dir = _get_item_dir(item) / self.baseline_relative_dir else: # baseline dir is relative to where pytest was run baseline_dir = self.baseline_dir @@ -360,7 +490,7 @@ def get_baseline_directory(self, item): baseline_remote = (isinstance(baseline_dir, str) and # noqa baseline_dir.startswith(('http://', 'https://'))) if not baseline_remote: - return Path(item.fspath).parent / baseline_dir + return _get_item_dir(item) / baseline_dir return baseline_dir @@ -375,16 +505,15 @@ def _download_file(self, baseline, filename): self.logger.info(f'Downloading {base_url + filename} failed: {repr(e)}') else: break - else: - raise Exception("Could not download baseline image from any of the " - "available URLs") + else: # Could not download baseline image from any of the available URLs + return result_dir = Path(tempfile.mkdtemp()) filename = result_dir / 'downloaded' with open(str(filename), 'wb') as tmpfile: tmpfile.write(content) return Path(filename) - def obtain_baseline_image(self, item, target_dir): + def obtain_baseline_image(self, item): """ Copy the baseline image to our working directory. @@ -408,16 +537,13 @@ def generate_baseline_image(self, item, fig): """ Generate reference figures. """ - compare = get_compare(item) - savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) if not os.path.exists(self.generate_dir): os.makedirs(self.generate_dir) baseline_filename = self.generate_filename(item) baseline_path = (self.generate_dir / baseline_filename).absolute() - fig.savefig(str(baseline_path), **savefig_kwargs) - + self.save_figure(item, fig, baseline_path) close_mpl_figure(fig) return baseline_path @@ -427,13 +553,9 @@ def generate_image_hash(self, item, fig): For a `matplotlib.figure.Figure`, returns the SHA256 hash as a hexadecimal string. """ - compare = get_compare(item) - savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) imgdata = io.BytesIO() - - fig.savefig(imgdata, **savefig_kwargs) - + self.save_figure(item, fig, imgdata) out = _hash_file(imgdata) imgdata.close() @@ -451,20 +573,32 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): summary = {} compare = get_compare(item) - tolerance = compare.kwargs.get('tolerance', 2) - savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) + tolerance = compare.kwargs.get('tolerance', self.default_tolerance) - baseline_image_ref = self.obtain_baseline_image(item, result_dir) + ext = self._file_extension(item) - test_image = (result_dir / "result.png").absolute() - fig.savefig(str(test_image), **savefig_kwargs) - summary['result_image'] = test_image.relative_to(self.results_dir).as_posix() + test_image = (result_dir / f"result.{ext}").absolute() + self.save_figure(item, fig, test_image) + + if ext in ['png', 'svg']: # Use original file + summary['result_image'] = test_image.relative_to(self.results_dir).as_posix() + else: + summary['result_image'] = (result_dir / f"result_{ext}.png").relative_to(self.results_dir).as_posix() + + baseline_image_ref = self.obtain_baseline_image(item) - if not os.path.exists(baseline_image_ref): + baseline_missing = None + if baseline_image_ref is None: + baseline_missing = ("Could not download the baseline image from " + "any of the available URLs.\n") + elif not os.path.exists(baseline_image_ref): + baseline_missing = ("Image file not found for comparison test in: \n\t" + f"{self.get_baseline_directory(item)}\n") + + if baseline_missing: summary['status'] = 'failed' summary['image_status'] = 'missing' - error_message = ("Image file not found for comparison test in: \n\t" - f"{self.get_baseline_directory(item)}\n" + error_message = (baseline_missing + "(This is expected for new tests.)\n" "Generated Image: \n\t" f"{test_image}") @@ -473,26 +607,33 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): # setuptools may put the baseline images in non-accessible places, # copy to our tmpdir to be sure to keep them in case of failure - baseline_image = (result_dir / "baseline.png").absolute() + baseline_image = (result_dir / f"baseline.{ext}").absolute() shutil.copyfile(baseline_image_ref, baseline_image) - summary['baseline_image'] = baseline_image.relative_to(self.results_dir).as_posix() + + if ext in ['png', 'svg']: # Use original file + summary['baseline_image'] = baseline_image.relative_to(self.results_dir).as_posix() + else: + summary['baseline_image'] = (result_dir / f"baseline_{ext}.png").relative_to(self.results_dir).as_posix() # Compare image size ourselves since the Matplotlib # exception is a bit cryptic in this case and doesn't show - # the filenames - expected_shape = imread(str(baseline_image)).shape[:2] - actual_shape = imread(str(test_image)).shape[:2] - if expected_shape != actual_shape: - summary['status'] = 'failed' - summary['image_status'] = 'diff' - error_message = SHAPE_MISMATCH_ERROR.format(expected_path=baseline_image, - expected_shape=expected_shape, - actual_path=test_image, - actual_shape=actual_shape) - summary['status_msg'] = error_message - return error_message + # the filenames. However imread won't work for vector graphics so we + # only do this for raster files. + if ext in RASTER_IMAGE_FORMATS: + expected_shape = imread(str(baseline_image)).shape[:2] + actual_shape = imread(str(test_image)).shape[:2] + if expected_shape != actual_shape: + summary['status'] = 'failed' + summary['image_status'] = 'diff' + error_message = SHAPE_MISMATCH_ERROR.format(expected_path=baseline_image, + expected_shape=expected_shape, + actual_path=test_image, + actual_shape=actual_shape) + summary['status_msg'] = error_message + return error_message results = compare_images(str(baseline_image), str(test_image), tol=tolerance, in_decorator=True) + summary['tolerance'] = tolerance if results is None: summary['status'] = 'passed' @@ -503,8 +644,7 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): summary['status'] = 'failed' summary['image_status'] = 'diff' summary['rms'] = results['rms'] - diff_image = (result_dir / 'result-failed-diff.png').absolute() - summary['diff_image'] = diff_image.relative_to(self.results_dir).as_posix() + summary['diff_image'] = Path(results['diff']).relative_to(self.results_dir).as_posix() template = ['Error: Image files did not match.', 'RMS Value: {rms}', 'Expected: \n {expected}', @@ -519,20 +659,98 @@ def load_hash_library(self, library_path): with open(str(library_path)) as fp: return json.load(fp) + def save_figure(self, item, fig, filename): + if isinstance(filename, Path): + filename = str(filename) + compare = get_compare(item) + savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) + deterministic = compare.kwargs.get('deterministic', self.deterministic) + + original_source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH', None) + + extra_rcparams = {} + + ext = self._file_extension(item) + + if deterministic is None: + + # The deterministic option should only matter for hash-based tests, + # so we first check if a hash library is being used + + if self.hash_library or compare.kwargs.get('hash_library', None): + + if ext == 'png': + if 'metadata' not in savefig_kwargs or 'Software' not in savefig_kwargs['metadata']: + warnings.warn("deterministic option not set (currently defaulting to False), " + "in future this will default to True to give consistent " + "hashes across Matplotlib versions. To suppress this warning, " + "set deterministic to True if you are happy with the future " + "behavior or to False if you want to preserve the old behavior.", + FutureWarning) + else: + # Set to False but in practice because Software is set to a constant value + # by the caller, the output will be deterministic (we don't want to change + # Software to None if the caller set it to e.g. 'test') + deterministic = False + else: + deterministic = True + + else: + + # We can just default to True since it shouldn't matter and in + # case generated images are somehow used in future to compute + # hashes + + deterministic = True + + if deterministic: + + # Make sure we don't modify the original dictionary in case is a common + # object used by different tests + savefig_kwargs = savefig_kwargs.copy() + + if 'metadata' not in savefig_kwargs: + savefig_kwargs['metadata'] = {} + + if ext == 'png': + extra_metadata = {"Software": None} + elif ext == 'pdf': + extra_metadata = {"Creator": None, "Producer": None, "CreationDate": None} + elif ext == 'eps': + extra_metadata = {"Creator": "test"} + os.environ['SOURCE_DATE_EPOCH'] = '1680254601' + elif ext == 'svg': + extra_metadata = {"Date": None} + extra_rcparams["svg.hashsalt"] = "test" + + savefig_kwargs['metadata'].update(extra_metadata) + + import matplotlib.pyplot as plt + + with plt.rc_context(rc=extra_rcparams): + fig.savefig(filename, **savefig_kwargs) + + if original_source_date_epoch is not None: + os.environ['SOURCE_DATE_EPOCH'] = original_source_date_epoch + def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): hash_comparison_pass = False if summary is None: summary = {} compare = get_compare(item) - savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) + + ext = self._file_extension(item) if not self.results_hash_library_name: # Use hash library name of current test as results hash library name self.results_hash_library_name = Path(compare.kwargs.get("hash_library", "")).name - hash_library_filename = self.hash_library or compare.kwargs.get('hash_library', None) - hash_library_filename = (Path(item.fspath).parent / hash_library_filename).absolute() + # Order of precedence for hash library: CLI, kwargs, INI (for backwards compatibility) + hash_library_filename = compare.kwargs.get("hash_library", None) or self.hash_library + if self._hash_library_from_cli: # for backwards compatibility + hash_library_filename = self.hash_library + hash_library_filename = _get_item_dir(item) / hash_library_filename if not Path(hash_library_filename).exists(): pytest.fail(f"Can't find hash library at path {hash_library_filename}") @@ -563,8 +781,8 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): f"{hash_library_filename} for test {hash_name}.") # Save the figure for later summary (will be removed later if not needed) - test_image = (result_dir / "result.png").absolute() - fig.savefig(str(test_image), **savefig_kwargs) + test_image = (result_dir / f"result.{ext}").absolute() + self.save_figure(item, fig, test_image) summary['result_image'] = test_image.relative_to(self.results_dir).as_posix() # Hybrid mode (hash and image comparison) @@ -580,6 +798,7 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): baseline_comparison = self.compare_image_to_baseline(item, fig, result_dir, summary=baseline_summary) except Exception as baseline_error: # Append to test error later + summary['image_status'] = 'diff' # (not necessarily diff, but makes user aware) baseline_comparison = str(baseline_error) else: # Update main summary for k in ['image_status', 'baseline_image', 'diff_image', @@ -612,31 +831,22 @@ def pytest_runtest_call(self, item): # noqa from matplotlib.testing.decorators import ImageComparisonTest as MplImageComparisonTest remove_ticks_and_titles = MplImageComparisonTest.remove_text - style = compare.kwargs.get('style', 'classic') + style = compare.kwargs.get('style', self.default_style) remove_text = compare.kwargs.get('remove_text', False) - backend = compare.kwargs.get('backend', 'agg') + backend = compare.kwargs.get('backend', self.default_backend) + + ext = self._file_extension(item) with plt.style.context(style, after_reset=True), switch_backend(backend): - # Run test and get figure object - wrap_figure_interceptor(self, item) - yield test_name = generate_test_name(item) - if test_name not in self.return_value: - # Test function did not complete successfully - return - fig = self.return_value[test_name] - - if remove_text: - remove_ticks_and_titles(fig) - - result_dir = self.make_test_results_dir(item) + # Store fallback summary in case of exceptions summary = { - 'status': None, + 'status': 'failed', 'image_status': None, 'hash_status': None, - 'status_msg': None, + 'status_msg': 'An exception was raised while testing the figure.', 'baseline_image': None, 'diff_image': None, 'rms': None, @@ -645,53 +855,80 @@ def pytest_runtest_call(self, item): # noqa 'baseline_hash': None, 'result_hash': None, } + self._test_results[test_name] = summary - # What we do now depends on whether we are generating the - # reference images or simply running the test. - if self.generate_dir is not None: - summary['status'] = 'skipped' - summary['image_status'] = 'generated' - summary['status_msg'] = 'Skipped test, since generating image.' - generate_image = self.generate_baseline_image(item, fig) - if self.results_always: # Make baseline image available in HTML - result_image = (result_dir / "baseline.png").absolute() - shutil.copy(generate_image, result_image) - summary['baseline_image'] = \ - result_image.relative_to(self.results_dir).as_posix() - - if self.generate_hash_library is not None: - summary['hash_status'] = 'generated' - image_hash = self.generate_image_hash(item, fig) - self._generated_hash_library[test_name] = image_hash - summary['baseline_hash'] = image_hash - - # Only test figures if not generating images - if self.generate_dir is None: - # Compare to hash library - if self.hash_library or compare.kwargs.get('hash_library', None): - msg = self.compare_image_to_hash_library(item, fig, result_dir, summary=summary) - - # Compare against a baseline if specified - else: - msg = self.compare_image_to_baseline(item, fig, result_dir, summary=summary) - - close_mpl_figure(fig) + # Run test and get figure object + wrap_figure_interceptor(self, item) - if msg is None: - if not self.results_always: - shutil.rmtree(result_dir) - for image_type in ['baseline_image', 'diff_image', 'result_image']: - summary[image_type] = None # image no longer exists - else: + # See https://github.com/pytest-dev/pytest/issues/11714 + result = yield + try: + if test_name not in self.return_value: + # Test function did not complete successfully + summary['status'] = 'failed' + summary['status_msg'] = ('Test function raised an exception ' + 'before returning a figure.') self._test_results[test_name] = summary - pytest.fail(msg, pytrace=False) + return + fig = self.return_value[test_name] + + if remove_text: + remove_ticks_and_titles(fig) + + result_dir = self.make_test_results_dir(item) + + # What we do now depends on whether we are generating the + # reference images or simply running the test. + if self.generate_dir is not None: + summary['status'] = 'skipped' + summary['image_status'] = 'generated' + summary['status_msg'] = 'Skipped test, since generating image.' + generate_image = self.generate_baseline_image(item, fig) + if self.results_always: # Make baseline image available in HTML + result_image = (result_dir / f"baseline.{ext}").absolute() + shutil.copy(generate_image, result_image) + summary['baseline_image'] = \ + result_image.relative_to(self.results_dir).as_posix() + + if self.generate_hash_library is not None: + summary['hash_status'] = 'generated' + image_hash = self.generate_image_hash(item, fig) + self._generated_hash_library[test_name] = image_hash + summary['baseline_hash'] = image_hash + + # Only test figures if not generating images + if self.generate_dir is None: + # Compare to hash library + if self.hash_library or compare.kwargs.get('hash_library', None): + msg = self.compare_image_to_hash_library(item, fig, result_dir, summary=summary) + + # Compare against a baseline if specified + else: + msg = self.compare_image_to_baseline(item, fig, result_dir, summary=summary) + + close_mpl_figure(fig) + + if msg is None: + if not self.results_always: + shutil.rmtree(result_dir) + for image_type in ['baseline_image', 'diff_image', 'result_image']: + summary[image_type] = None # image no longer exists + else: + self._test_results[test_name] = summary + pytest.fail(msg, pytrace=False) - close_mpl_figure(fig) + close_mpl_figure(fig) - self._test_results[test_name] = summary + self._test_results[test_name] = summary - if summary['status'] == 'skipped': - pytest.skip(summary['status_msg']) + if summary['status'] == 'skipped': + pytest.skip(summary['status_msg']) + except BaseException as e: + if hasattr(result, "force_exception"): # pluggy>=1.2.0 + result.force_exception(e) + else: + result._result = None + result._excinfo = (type(e), e, e.__traceback__) def generate_summary_json(self): json_file = self.results_dir / 'results.json' diff --git a/setup.cfg b/setup.cfg index d2fb5e0a..a874af32 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,23 +39,27 @@ pytest11 = [options.extras_require] test = pytest-cov +docs = + sphinx + mpl_sphinx_theme>=3.6.0.dev0 + sphinx_design + matplotlib==3.6 [tool:pytest] -testpaths = "tests" -markers = - image: run test during image comparison only mode. - hash: run test during hash comparison only mode. +testpaths = tests +norecursedirs = tests/subtests/subtest filterwarnings = error ignore:distutils Version classes are deprecated ignore:the imp module is deprecated in favour of importlib + ignore:The NumPy module was reloaded [flake8] -max-line-length = 100 -ignore = W504 +max-line-length = 120 +ignore = W503,W504 [pycodestyle] -max_line_length = 100 +max_line_length = 120 [isort] balanced_wrapping = True diff --git a/tests/baseline/2.0.x/test_format_eps.eps b/tests/baseline/2.0.x/test_format_eps.eps new file mode 100644 index 00000000..d8438114 --- /dev/null +++ b/tests/baseline/2.0.x/test_format_eps.eps @@ -0,0 +1,780 @@ +%!PS-Adobe-3.0 EPSF-3.0 +%%Title: test_format_eps.eps +%%Creator: Matplotlib v3.6.3, https://matplotlib.org/ +%%CreationDate: Thu Mar 30 10:58:11 2023 +%%Orientation: portrait +%%BoundingBox: 18 180 594 612 +%%HiResBoundingBox: 18.000000 180.000000 594.000000 612.000000 +%%EndComments +%%BeginProlog +/mpldict 11 dict def +mpldict begin +/_d { bind def } bind def +/m { moveto } _d +/l { lineto } _d +/r { rlineto } _d +/c { curveto } _d +/cl { closepath } _d +/ce { closepath eofill } _d +/box { + m + 1 index 0 r + 0 exch r + neg 0 r + cl + } _d +/clipbox { + box + clip + newpath + } _d +/sc { setcachedevice } _d +%!PS-Adobe-3.0 Resource-Font +%%Creator: Converted from TrueType to Type 3 by Matplotlib. +10 dict begin +/FontName /DejaVuSans def +/PaintType 0 def +/FontMatrix [0.00048828125 0 0 0.00048828125 0 0] def +/FontBBox [-2090 -948 3673 2524] def +/FontType 3 def +/Encoding [/period /zero /one /two /three /five] def +/CharStrings 7 dict dup begin +/.notdef 0 def +/period{651 0 219 0 430 254 sc +219 254 m +430 254 l +430 0 l +219 0 l +219 254 l + +ce} _d +/zero{1303 0 135 -29 1167 1520 sc +651 1360 m +547 1360 469 1309 416 1206 c +364 1104 338 950 338 745 c +338 540 364 387 416 284 c +469 182 547 131 651 131 c +756 131 834 182 886 284 c +939 387 965 540 965 745 c +965 950 939 1104 886 1206 c +834 1309 756 1360 651 1360 c + +651 1520 m +818 1520 946 1454 1034 1321 c +1123 1189 1167 997 1167 745 c +1167 494 1123 302 1034 169 c +946 37 818 -29 651 -29 c +484 -29 356 37 267 169 c +179 302 135 494 135 745 c +135 997 179 1189 267 1321 c +356 1454 484 1520 651 1520 c + +ce} _d +/one{1303 0 225 0 1114 1493 sc +254 170 m +584 170 l +584 1309 l +225 1237 l +225 1421 l +582 1493 l +784 1493 l +784 170 l +1114 170 l +1114 0 l +254 0 l +254 170 l + +ce} _d +/two{1303 0 150 0 1098 1520 sc +393 170 m +1098 170 l +1098 0 l +150 0 l +150 170 l +227 249 331 356 463 489 c +596 623 679 709 713 748 c +778 821 823 882 848 932 c +874 983 887 1032 887 1081 c +887 1160 859 1225 803 1275 c +748 1325 675 1350 586 1350 c +523 1350 456 1339 385 1317 c +315 1295 240 1262 160 1217 c +160 1421 l +241 1454 317 1478 388 1495 c +459 1512 523 1520 582 1520 c +737 1520 860 1481 952 1404 c +1044 1327 1090 1223 1090 1094 c +1090 1033 1078 974 1055 919 c +1032 864 991 800 930 725 c +913 706 860 650 771 557 c +682 465 556 336 393 170 c + +ce} _d +/three{1303 0 156 -29 1139 1520 sc +831 805 m +928 784 1003 741 1057 676 c +1112 611 1139 530 1139 434 c +1139 287 1088 173 987 92 c +886 11 742 -29 555 -29 c +492 -29 428 -23 361 -10 c +295 2 227 20 156 45 c +156 240 l +212 207 273 183 340 166 c +407 149 476 141 549 141 c +676 141 772 166 838 216 c +905 266 938 339 938 434 c +938 522 907 591 845 640 c +784 690 698 715 588 715 c +414 715 l +414 881 l +596 881 l +695 881 771 901 824 940 c +877 980 903 1037 903 1112 c +903 1189 876 1247 821 1288 c +767 1329 689 1350 588 1350 c +533 1350 473 1344 410 1332 c +347 1320 277 1301 201 1276 c +201 1456 l +278 1477 349 1493 416 1504 c +483 1515 547 1520 606 1520 c +759 1520 881 1485 970 1415 c +1059 1346 1104 1252 1104 1133 c +1104 1050 1080 980 1033 923 c +986 866 918 827 831 805 c + +ce} _d +/five{1303 0 158 -29 1124 1493 sc +221 1493 m +1014 1493 l +1014 1323 l +406 1323 l +406 957 l +435 967 465 974 494 979 c +523 984 553 987 582 987 c +749 987 881 941 978 850 c +1075 759 1124 635 1124 479 c +1124 318 1074 193 974 104 c +874 15 733 -29 551 -29 c +488 -29 424 -24 359 -13 c +294 -2 227 14 158 35 c +158 238 l +218 205 280 181 344 165 c +408 149 476 141 547 141 c +662 141 754 171 821 232 c +888 293 922 375 922 479 c +922 583 888 665 821 726 c +754 787 662 817 547 817 c +493 817 439 811 385 799 c +332 787 277 768 221 743 c +221 1493 l + +ce} _d +end readonly def + +/BuildGlyph { + exch begin + CharStrings exch + 2 copy known not {pop /.notdef} if + true 3 1 roll get exec + end +} _d + +/BuildChar { + 1 index /Encoding get exch get + 1 index /BuildGlyph get exec +} _d + +FontName currentdict end definefont pop +end +%%EndProlog +mpldict begin +18 180 translate +576 432 0 0 clipbox +gsave +0 0 m +576 0 l +576 432 l +0 432 l +cl +1.000 setgray +fill +grestore +gsave +72 43.2 m +518.4 43.2 l +518.4 388.8 l +72 388.8 l +cl +1.000 setgray +fill +grestore +1.000 setlinewidth +1 setlinejoin +2 setlinecap +[] 0 setdash +0.000 0.000 1.000 setrgbcolor +gsave +446.4 345.6 72 43.2 clipbox +72 43.2 m +295.2 216 l +518.4 388.8 l +stroke +grestore +0 setlinejoin +0.000 setgray +gsave +72 43.2 m +72 388.8 l +stroke +grestore +gsave +518.4 43.2 m +518.4 388.8 l +stroke +grestore +gsave +72 43.2 m +518.4 43.2 l +stroke +grestore +gsave +72 388.8 m +518.4 388.8 l +stroke +grestore +0.500 setlinewidth +1 setlinejoin +0 setlinecap +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +0 4 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +72 43.2 o +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +0 -4 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +72 388.8 o +grestore +/DejaVuSans 12.000 selectfont +gsave + +62.4531 30.075 translate +0 rotate +0 0 m /zero glyphshow +7.63477 0 m /period glyphshow +11.4492 0 m /zero glyphshow +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +0 4 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +183.6 43.2 o +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +0 -4 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +183.6 388.8 o +grestore +/DejaVuSans 12.000 selectfont +gsave + +174.053 30.075 translate +0 rotate +0 0 m /zero glyphshow +7.63477 0 m /period glyphshow +11.4492 0 m /five glyphshow +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +0 4 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +295.2 43.2 o +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +0 -4 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +295.2 388.8 o +grestore +/DejaVuSans 12.000 selectfont +gsave + +285.653 30.075 translate +0 rotate +0 0 m /one glyphshow +7.63477 0 m /period glyphshow +11.4492 0 m /zero glyphshow +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +0 4 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +406.8 43.2 o +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +0 -4 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +406.8 388.8 o +grestore +/DejaVuSans 12.000 selectfont +gsave + +397.253 30.075 translate +0 rotate +0 0 m /one glyphshow +7.63477 0 m /period glyphshow +11.4492 0 m /five glyphshow +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +0 4 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +518.4 43.2 o +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +0 -4 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +518.4 388.8 o +grestore +/DejaVuSans 12.000 selectfont +gsave + +508.853 30.075 translate +0 rotate +0 0 m /two glyphshow +7.63477 0 m /period glyphshow +11.4492 0 m /zero glyphshow +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +4 0 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +72 43.2 o +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +-0 0 m +-4 0 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +518.4 43.2 o +grestore +/DejaVuSans 12.000 selectfont +gsave + +48.9062 39.8875 translate +0 rotate +0 0 m /one glyphshow +7.63477 0 m /period glyphshow +11.4492 0 m /zero glyphshow +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +4 0 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +72 129.6 o +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +-0 0 m +-4 0 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +518.4 129.6 o +grestore +/DejaVuSans 12.000 selectfont +gsave + +48.9062 126.288 translate +0 rotate +0 0 m /one glyphshow +7.63477 0 m /period glyphshow +11.4492 0 m /five glyphshow +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +4 0 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +72 216 o +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +-0 0 m +-4 0 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +518.4 216 o +grestore +/DejaVuSans 12.000 selectfont +gsave + +48.9062 212.688 translate +0 rotate +0 0 m /two glyphshow +7.63477 0 m /period glyphshow +11.4492 0 m /zero glyphshow +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +4 0 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +72 302.4 o +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +-0 0 m +-4 0 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +518.4 302.4 o +grestore +/DejaVuSans 12.000 selectfont +gsave + +48.9062 299.087 translate +0 rotate +0 0 m /two glyphshow +7.63477 0 m /period glyphshow +11.4492 0 m /five glyphshow +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +0 0 m +4 0 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +72 388.8 o +grestore +gsave +/o { +gsave +newpath +translate +0.5 setlinewidth +1 setlinejoin + +0 setlinecap + +-0 0 m +-4 0 l + +gsave +0.000 setgray +fill +grestore +stroke +grestore +} bind def +518.4 388.8 o +grestore +/DejaVuSans 12.000 selectfont +gsave + +48.9062 385.488 translate +0 rotate +0 0 m /three glyphshow +7.63477 0 m /period glyphshow +11.4492 0 m /zero glyphshow +grestore + +end +showpage diff --git a/tests/baseline/2.0.x/test_format_pdf.pdf b/tests/baseline/2.0.x/test_format_pdf.pdf new file mode 100644 index 00000000..04d2a9f1 Binary files /dev/null and b/tests/baseline/2.0.x/test_format_pdf.pdf differ diff --git a/tests/baseline/2.0.x/test_format_png.png b/tests/baseline/2.0.x/test_format_png.png new file mode 100644 index 00000000..68ac04da Binary files /dev/null and b/tests/baseline/2.0.x/test_format_png.png differ diff --git a/tests/baseline/2.0.x/test_format_svg.svg b/tests/baseline/2.0.x/test_format_svg.svg new file mode 100644 index 00000000..4ea32f65 --- /dev/null +++ b/tests/baseline/2.0.x/test_format_svg.svg @@ -0,0 +1,430 @@ + + + + + + + + 2023-03-30T10:58:11.674243 + image/svg+xml + + + Matplotlib v3.6.3, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/baseline/hashes/mpl20_ft261.json b/tests/baseline/hashes/mpl20_ft261.json index 76322063..47c4eacf 100644 --- a/tests/baseline/hashes/mpl20_ft261.json +++ b/tests/baseline/hashes/mpl20_ft261.json @@ -1,16 +1,7 @@ { - "test_pytest_mpl.test_succeeds": "cd01b7a39330033b18d54b507635236214cae5e24f9e09d84b56462a9ac81052", - "test_pytest_mpl.test_succeeds_remote": "cd01b7a39330033b18d54b507635236214cae5e24f9e09d84b56462a9ac81052", - "test_pytest_mpl.test_succeeds_faulty_mirror": "cd01b7a39330033b18d54b507635236214cae5e24f9e09d84b56462a9ac81052", - "test_pytest_mpl.test_dpi": "cc21ba84cc41dfcf77f46f5543affed721df2384958fa0adc4e5e60470852781", - "test_pytest_mpl.test_tolerance": "ab176e655d3dca536ce74f7447d4eb8b7f70649a81a4055e5ce77c1f53c3a46f", - "test_pytest_mpl.test_base_style": "92497fd88fc46bf7a99b1afd32e7fb73fa06fa1ceaee7bcc443324e45c6b4d91", - "test_pytest_mpl.test_remove_text": "9c284d7bcbbb1d6c1362b417859e4ce842b573a2fe32c7ceaafcf328a1eb7057", - "test_pytest_mpl.test_parametrized[5]": "04c998af2d7932ca4a851d610e8a020d94a2f623d1301dbe9b59fe6efd28a5f7", - "test_pytest_mpl.test_parametrized[50]": "937d986ab6b209e7d48eb30cc30e9db62c93bbc4c86768e276a5b454e63bca93", - "test_pytest_mpl.test_parametrized[500]": "e39ed724b0762b8736879801e32dc0c1525afd03c0567a43b119435aaa608498", "test_pytest_mpl.test_hash_succeeds": "480062c2239ed9d70e361d1a5b578dc2aa756971161ac6e7287b492ae6118c59", "test.test_modified": "54f6cf83d5b06fa2ecb7fa23d6e87898679178ef5d0dfdd2551a139f1932127b", "test.test_new": "54f6cf83d5b06fa2ecb7fa23d6e87898679178ef5d0dfdd2551a139f1932127b", - "test.test_unmodified": "54f6cf83d5b06fa2ecb7fa23d6e87898679178ef5d0dfdd2551a139f1932127b" + "test.test_unmodified": "54f6cf83d5b06fa2ecb7fa23d6e87898679178ef5d0dfdd2551a139f1932127b", + "test_formats.test_format_png": "480062c2239ed9d70e361d1a5b578dc2aa756971161ac6e7287b492ae6118c59" } diff --git a/tests/baseline/hashes/mpl21_ft261.json b/tests/baseline/hashes/mpl21_ft261.json index 8b2beb5a..73286d52 100644 --- a/tests/baseline/hashes/mpl21_ft261.json +++ b/tests/baseline/hashes/mpl21_ft261.json @@ -1,16 +1,9 @@ { - "test_pytest_mpl.test_succeeds": "cd01b7a39330033b18d54b507635236214cae5e24f9e09d84b56462a9ac81052", - "test_pytest_mpl.test_succeeds_remote": "cd01b7a39330033b18d54b507635236214cae5e24f9e09d84b56462a9ac81052", - "test_pytest_mpl.test_succeeds_faulty_mirror": "cd01b7a39330033b18d54b507635236214cae5e24f9e09d84b56462a9ac81052", - "test_pytest_mpl.test_dpi": "cc21ba84cc41dfcf77f46f5543affed721df2384958fa0adc4e5e60470852781", - "test_pytest_mpl.test_tolerance": "ab176e655d3dca536ce74f7447d4eb8b7f70649a81a4055e5ce77c1f53c3a46f", - "test_pytest_mpl.test_base_style": "92497fd88fc46bf7a99b1afd32e7fb73fa06fa1ceaee7bcc443324e45c6b4d91", - "test_pytest_mpl.test_remove_text": "9c284d7bcbbb1d6c1362b417859e4ce842b573a2fe32c7ceaafcf328a1eb7057", - "test_pytest_mpl.test_parametrized[5]": "04c998af2d7932ca4a851d610e8a020d94a2f623d1301dbe9b59fe6efd28a5f7", - "test_pytest_mpl.test_parametrized[50]": "937d986ab6b209e7d48eb30cc30e9db62c93bbc4c86768e276a5b454e63bca93", - "test_pytest_mpl.test_parametrized[500]": "e39ed724b0762b8736879801e32dc0c1525afd03c0567a43b119435aaa608498", "test_pytest_mpl.test_hash_succeeds": "17b65dd0247b0dfd8c1b4b079352414ae0fe03c0a3e79d63c8b8670d84d4098f", "test.test_modified": "14d326881467bc613e6504b87bd7d556a5e58668ff16b896fa3c15745cfb6336", "test.test_new": "14d326881467bc613e6504b87bd7d556a5e58668ff16b896fa3c15745cfb6336", - "test.test_unmodified": "14d326881467bc613e6504b87bd7d556a5e58668ff16b896fa3c15745cfb6336" + "test.test_unmodified": "14d326881467bc613e6504b87bd7d556a5e58668ff16b896fa3c15745cfb6336", + "test_formats.test_format_eps": "f8a0fbb33dcd473ef5cfdd69317db6eb58d64a7f5f3b5072e0de69aa9e201224", + "test_formats.test_format_pdf": "82b2b58fc3a74591c85cdf2f06b2f72dfc154820fca98e7cfd5cb6904ed60b23", + "test_formats.test_format_png": "d577a3c8c7550413d8d50bc26a68f3e8d9c35d4763c52cbcc15df4f61c8406b2" } diff --git a/tests/baseline/hashes/mpl22_ft261.json b/tests/baseline/hashes/mpl22_ft261.json index 08c26f4d..fbde7569 100644 --- a/tests/baseline/hashes/mpl22_ft261.json +++ b/tests/baseline/hashes/mpl22_ft261.json @@ -1,16 +1,9 @@ { - "test_pytest_mpl.test_succeeds": "cd01b7a39330033b18d54b507635236214cae5e24f9e09d84b56462a9ac81052", - "test_pytest_mpl.test_succeeds_remote": "cd01b7a39330033b18d54b507635236214cae5e24f9e09d84b56462a9ac81052", - "test_pytest_mpl.test_succeeds_faulty_mirror": "cd01b7a39330033b18d54b507635236214cae5e24f9e09d84b56462a9ac81052", - "test_pytest_mpl.test_dpi": "cc21ba84cc41dfcf77f46f5543affed721df2384958fa0adc4e5e60470852781", - "test_pytest_mpl.test_tolerance": "ab176e655d3dca536ce74f7447d4eb8b7f70649a81a4055e5ce77c1f53c3a46f", - "test_pytest_mpl.test_base_style": "92497fd88fc46bf7a99b1afd32e7fb73fa06fa1ceaee7bcc443324e45c6b4d91", - "test_pytest_mpl.test_remove_text": "9c284d7bcbbb1d6c1362b417859e4ce842b573a2fe32c7ceaafcf328a1eb7057", - "test_pytest_mpl.test_parametrized[5]": "04c998af2d7932ca4a851d610e8a020d94a2f623d1301dbe9b59fe6efd28a5f7", - "test_pytest_mpl.test_parametrized[50]": "937d986ab6b209e7d48eb30cc30e9db62c93bbc4c86768e276a5b454e63bca93", - "test_pytest_mpl.test_parametrized[500]": "e39ed724b0762b8736879801e32dc0c1525afd03c0567a43b119435aaa608498", "test_pytest_mpl.test_hash_succeeds": "e80557c8784fb920fb79b03b26dc072649a98811f00a8c212df8761e4351acde", "test.test_modified": "80e0ee6df7cf7d9d9407395a25af30beb8763e98820a7be972764899246d2cd7", "test.test_new": "80e0ee6df7cf7d9d9407395a25af30beb8763e98820a7be972764899246d2cd7", - "test.test_unmodified": "80e0ee6df7cf7d9d9407395a25af30beb8763e98820a7be972764899246d2cd7" + "test.test_unmodified": "80e0ee6df7cf7d9d9407395a25af30beb8763e98820a7be972764899246d2cd7", + "test_formats.test_format_eps": "f8a0fbb33dcd473ef5cfdd69317db6eb58d64a7f5f3b5072e0de69aa9e201224", + "test_formats.test_format_pdf": "8963ba9209080091c0961553bdf195cdcd0f2ba29081a122f9aad8e94c444aff", + "test_formats.test_format_png": "d577a3c8c7550413d8d50bc26a68f3e8d9c35d4763c52cbcc15df4f61c8406b2" } diff --git a/tests/baseline/hashes/mpl30_ft261.json b/tests/baseline/hashes/mpl30_ft261.json index 313078ea..b928fbbb 100644 --- a/tests/baseline/hashes/mpl30_ft261.json +++ b/tests/baseline/hashes/mpl30_ft261.json @@ -1,16 +1,9 @@ { - "test_pytest_mpl.test_succeeds": "cd01b7a39330033b18d54b507635236214cae5e24f9e09d84b56462a9ac81052", - "test_pytest_mpl.test_succeeds_remote": "cd01b7a39330033b18d54b507635236214cae5e24f9e09d84b56462a9ac81052", - "test_pytest_mpl.test_succeeds_faulty_mirror": "cd01b7a39330033b18d54b507635236214cae5e24f9e09d84b56462a9ac81052", - "test_pytest_mpl.test_dpi": "cc21ba84cc41dfcf77f46f5543affed721df2384958fa0adc4e5e60470852781", - "test_pytest_mpl.test_tolerance": "ab176e655d3dca536ce74f7447d4eb8b7f70649a81a4055e5ce77c1f53c3a46f", - "test_pytest_mpl.test_base_style": "92497fd88fc46bf7a99b1afd32e7fb73fa06fa1ceaee7bcc443324e45c6b4d91", - "test_pytest_mpl.test_remove_text": "9c284d7bcbbb1d6c1362b417859e4ce842b573a2fe32c7ceaafcf328a1eb7057", - "test_pytest_mpl.test_parametrized[5]": "04c998af2d7932ca4a851d610e8a020d94a2f623d1301dbe9b59fe6efd28a5f7", - "test_pytest_mpl.test_parametrized[50]": "937d986ab6b209e7d48eb30cc30e9db62c93bbc4c86768e276a5b454e63bca93", - "test_pytest_mpl.test_parametrized[500]": "e39ed724b0762b8736879801e32dc0c1525afd03c0567a43b119435aaa608498", - "test_pytest_mpl.test_hash_succeeds": "4e1157a93733cdb327f1741afdb0525f4d0e3f12e60b54f72c93db9f9c9ae27f", - "test.test_modified": "6e2e4ba7b77caf62df24f6b92d6fc51ab1b837bf98039750334f65c0a6c5d898", - "test.test_new": "6e2e4ba7b77caf62df24f6b92d6fc51ab1b837bf98039750334f65c0a6c5d898", - "test.test_unmodified": "6e2e4ba7b77caf62df24f6b92d6fc51ab1b837bf98039750334f65c0a6c5d898" + "test_pytest_mpl.test_hash_succeeds": "ffbe386e7bbd8c720a44d2775dfe207b6dcca9304d845b557071a72b8c31ce04", + "test.test_modified": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test.test_new": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test.test_unmodified": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test_formats.test_format_eps": "4a605a2cd24101b9292151f5ab6d6846ba1b9c856cfda2bee6a142380e257b04", + "test_formats.test_format_pdf": "34a9eb10372b35c0bd26472e8571a91031c055ab47cc3682ebc0c5e47c2b6cbd", + "test_formats.test_format_png": "e73d228183ddfdced366191399cdecef9685d1248b852162f179750fc7b8b904" } diff --git a/tests/baseline/hashes/mpl31_ft261.json b/tests/baseline/hashes/mpl31_ft261.json index dbde4d54..421a2ab6 100644 --- a/tests/baseline/hashes/mpl31_ft261.json +++ b/tests/baseline/hashes/mpl31_ft261.json @@ -1,16 +1,9 @@ { - "test_pytest_mpl.test_succeeds": "2a4da3a36b384df539f3f47d476f67a918f5eee1df360dbab9469b96260df78f", - "test_pytest_mpl.test_succeeds_remote": "2a4da3a36b384df539f3f47d476f67a918f5eee1df360dbab9469b96260df78f", - "test_pytest_mpl.test_succeeds_faulty_mirror": "2a4da3a36b384df539f3f47d476f67a918f5eee1df360dbab9469b96260df78f", - "test_pytest_mpl.test_dpi": "2fd74751a8654eb0b32ff0663fd8ec137fba20aec0eb6871e312e9a8ff1fe296", - "test_pytest_mpl.test_tolerance": "3675e5a48388e8cc341580e9b41115d3cf63d2465cf11eeed3faa23e84030fc2", - "test_pytest_mpl.test_base_style": "781074a09615b6efb9aae8ea7630c4c59dce4f7c02b7fc4ce594ccf681955586", - "test_pytest_mpl.test_remove_text": "41c85cd546792245d6b2b2ebf5742d3053e679a5e088407738f4785517d8dbac", - "test_pytest_mpl.test_parametrized[5]": "be7dc9de64a5d6fd458c1f930d4aa56cf8196ddb0e8b5b07ab79a1f0ea9eb820", - "test_pytest_mpl.test_parametrized[50]": "a8ae2427337803dc864784d88c4428a6af5a3e47d2bfc84c98b68b25fde75704", - "test_pytest_mpl.test_parametrized[500]": "590ef42388378173e293bd37e95ff22d8e753d53327d1fb5d6bdf2bac4f84d01", - "test_pytest_mpl.test_hash_succeeds": "2a4da3a36b384df539f3f47d476f67a918f5eee1df360dbab9469b96260df78f", - "test.test_modified": "3675e5a48388e8cc341580e9b41115d3cf63d2465cf11eeed3faa23e84030fc2", - "test.test_new": "3675e5a48388e8cc341580e9b41115d3cf63d2465cf11eeed3faa23e84030fc2", - "test.test_unmodified": "3675e5a48388e8cc341580e9b41115d3cf63d2465cf11eeed3faa23e84030fc2" + "test_pytest_mpl.test_hash_succeeds": "d577a3c8c7550413d8d50bc26a68f3e8d9c35d4763c52cbcc15df4f61c8406b2", + "test.test_modified": "b7a4b4a01e8cf66c805ab456bf3f807066acf86a11af669b557b2aa98586f2c0", + "test.test_new": "b7a4b4a01e8cf66c805ab456bf3f807066acf86a11af669b557b2aa98586f2c0", + "test.test_unmodified": "b7a4b4a01e8cf66c805ab456bf3f807066acf86a11af669b557b2aa98586f2c0", + "test_formats.test_format_eps": "108a341ce450cb5adef9f41e27175da1809fcdeb64f17b13f58d3eb0efc08006", + "test_formats.test_format_pdf": "8963ba9209080091c0961553bdf195cdcd0f2ba29081a122f9aad8e94c444aff", + "test_formats.test_format_png": "d577a3c8c7550413d8d50bc26a68f3e8d9c35d4763c52cbcc15df4f61c8406b2" } diff --git a/tests/baseline/hashes/mpl32_ft261.json b/tests/baseline/hashes/mpl32_ft261.json index c39964b0..b928fbbb 100644 --- a/tests/baseline/hashes/mpl32_ft261.json +++ b/tests/baseline/hashes/mpl32_ft261.json @@ -1,16 +1,9 @@ { - "test_pytest_mpl.test_succeeds": "8b8ff9ce044bc9075876278781667a708414460209bba25a39d8262ed73d0f04", - "test_pytest_mpl.test_succeeds_remote": "8b8ff9ce044bc9075876278781667a708414460209bba25a39d8262ed73d0f04", - "test_pytest_mpl.test_succeeds_faulty_mirror": "8b8ff9ce044bc9075876278781667a708414460209bba25a39d8262ed73d0f04", - "test_pytest_mpl.test_dpi": "812d841fdc4b84bb7d80e08e484932fd2775dd83e347c50516ed5941c8ad933c", - "test_pytest_mpl.test_tolerance": "3b7db65812fd59403d17a2fba3ebe1fd0abdfde8633df06636e4e1daea259da0", - "test_pytest_mpl.test_base_style": "23c0c0cd74c3daa99a60d9d0500fb2ec4a6f87ddcaa46309ebd608b03b04a0a9", - "test_pytest_mpl.test_remove_text": "cbc0daabceb9b46eaff0031a045bf659ce694b0962443ef26d19aa54425aaa39", - "test_pytest_mpl.test_parametrized[5]": "9b2b5b1df784c8f9a5fc624840138fe7b4dbdd42cf592fe5733c9c825e5dda91", - "test_pytest_mpl.test_parametrized[50]": "fcf0566ef5514674e2b4bf1e9b4c7f52451c6f98abdc75dc876f43c97a23bc32", - "test_pytest_mpl.test_parametrized[500]": "38dccccfc980b44359bc1b325bef48471bc084db37ed622af00a553792a8b093", - "test_pytest_mpl.test_hash_succeeds": "8b8ff9ce044bc9075876278781667a708414460209bba25a39d8262ed73d0f04", - "test.test_modified": "3b7db65812fd59403d17a2fba3ebe1fd0abdfde8633df06636e4e1daea259da0", - "test.test_new": "3b7db65812fd59403d17a2fba3ebe1fd0abdfde8633df06636e4e1daea259da0", - "test.test_unmodified": "3b7db65812fd59403d17a2fba3ebe1fd0abdfde8633df06636e4e1daea259da0" + "test_pytest_mpl.test_hash_succeeds": "ffbe386e7bbd8c720a44d2775dfe207b6dcca9304d845b557071a72b8c31ce04", + "test.test_modified": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test.test_new": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test.test_unmodified": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test_formats.test_format_eps": "4a605a2cd24101b9292151f5ab6d6846ba1b9c856cfda2bee6a142380e257b04", + "test_formats.test_format_pdf": "34a9eb10372b35c0bd26472e8571a91031c055ab47cc3682ebc0c5e47c2b6cbd", + "test_formats.test_format_png": "e73d228183ddfdced366191399cdecef9685d1248b852162f179750fc7b8b904" } diff --git a/tests/baseline/hashes/mpl33_ft261.json b/tests/baseline/hashes/mpl33_ft261.json index 77c5f03f..3fdf0240 100644 --- a/tests/baseline/hashes/mpl33_ft261.json +++ b/tests/baseline/hashes/mpl33_ft261.json @@ -1,16 +1,10 @@ { - "test_pytest_mpl.test_succeeds": "cd01b7a39330033b18d54b507635236214cae5e24f9e09d84b56462a9ac81052", - "test_pytest_mpl.test_succeeds_remote": "cd01b7a39330033b18d54b507635236214cae5e24f9e09d84b56462a9ac81052", - "test_pytest_mpl.test_succeeds_faulty_mirror": "cd01b7a39330033b18d54b507635236214cae5e24f9e09d84b56462a9ac81052", - "test_pytest_mpl.test_dpi": "cc21ba84cc41dfcf77f46f5543affed721df2384958fa0adc4e5e60470852781", - "test_pytest_mpl.test_tolerance": "ab176e655d3dca536ce74f7447d4eb8b7f70649a81a4055e5ce77c1f53c3a46f", - "test_pytest_mpl.test_base_style": "92497fd88fc46bf7a99b1afd32e7fb73fa06fa1ceaee7bcc443324e45c6b4d91", - "test_pytest_mpl.test_remove_text": "9c284d7bcbbb1d6c1362b417859e4ce842b573a2fe32c7ceaafcf328a1eb7057", - "test_pytest_mpl.test_parametrized[5]": "04c998af2d7932ca4a851d610e8a020d94a2f623d1301dbe9b59fe6efd28a5f7", - "test_pytest_mpl.test_parametrized[50]": "937d986ab6b209e7d48eb30cc30e9db62c93bbc4c86768e276a5b454e63bca93", - "test_pytest_mpl.test_parametrized[500]": "e39ed724b0762b8736879801e32dc0c1525afd03c0567a43b119435aaa608498", - "test_pytest_mpl.test_hash_succeeds": "55ad74a44c99606f1df1e79f67a59a4986bddc2b48ea2b2d7ea8365db91dc7ca", - "test.test_modified": "ce07de6b726c3b01afb03aa7c9e939d584bc71a54b9737d69853a0d915cd6181", - "test.test_new": "ce07de6b726c3b01afb03aa7c9e939d584bc71a54b9737d69853a0d915cd6181", - "test.test_unmodified": "ce07de6b726c3b01afb03aa7c9e939d584bc71a54b9737d69853a0d915cd6181" + "test_pytest_mpl.test_hash_succeeds": "e73d228183ddfdced366191399cdecef9685d1248b852162f179750fc7b8b904", + "test.test_modified": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test.test_new": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test.test_unmodified": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test_formats.test_format_eps": "4a605a2cd24101b9292151f5ab6d6846ba1b9c856cfda2bee6a142380e257b04", + "test_formats.test_format_pdf": "34a9eb10372b35c0bd26472e8571a91031c055ab47cc3682ebc0c5e47c2b6cbd", + "test_formats.test_format_png": "e73d228183ddfdced366191399cdecef9685d1248b852162f179750fc7b8b904", + "test_formats.test_format_svg": "b7f85f4b44e0c5871f5cc230b5a9042f2e73aa70384ab584d6cd8cde29344cd2" } diff --git a/tests/baseline/hashes/mpl34_ft261.json b/tests/baseline/hashes/mpl34_ft261.json new file mode 100644 index 00000000..3fdf0240 --- /dev/null +++ b/tests/baseline/hashes/mpl34_ft261.json @@ -0,0 +1,10 @@ +{ + "test_pytest_mpl.test_hash_succeeds": "e73d228183ddfdced366191399cdecef9685d1248b852162f179750fc7b8b904", + "test.test_modified": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test.test_new": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test.test_unmodified": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test_formats.test_format_eps": "4a605a2cd24101b9292151f5ab6d6846ba1b9c856cfda2bee6a142380e257b04", + "test_formats.test_format_pdf": "34a9eb10372b35c0bd26472e8571a91031c055ab47cc3682ebc0c5e47c2b6cbd", + "test_formats.test_format_png": "e73d228183ddfdced366191399cdecef9685d1248b852162f179750fc7b8b904", + "test_formats.test_format_svg": "b7f85f4b44e0c5871f5cc230b5a9042f2e73aa70384ab584d6cd8cde29344cd2" +} diff --git a/tests/baseline/hashes/mpl35_ft261.json b/tests/baseline/hashes/mpl35_ft261.json new file mode 100644 index 00000000..5abbe0d1 --- /dev/null +++ b/tests/baseline/hashes/mpl35_ft261.json @@ -0,0 +1,10 @@ +{ + "test_pytest_mpl.test_hash_succeeds": "e73d228183ddfdced366191399cdecef9685d1248b852162f179750fc7b8b904", + "test.test_modified": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test.test_new": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test.test_unmodified": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test_formats.test_format_eps": "eafdced089e99355680744ac0b03d0beb3908c356c15f42e3fb299fa96cce46b", + "test_formats.test_format_pdf": "43ff9cc711b1d5e94bb904fc0ba872ea1c0a3fc778ed6eb705c5fc9305928c6b", + "test_formats.test_format_png": "e73d228183ddfdced366191399cdecef9685d1248b852162f179750fc7b8b904", + "test_formats.test_format_svg": "43a859a3642b9ce15578d25c6fd7d13edfee3a18457949a1ed4e841a83aa2942" +} diff --git a/tests/baseline/hashes/mpl36_ft261.json b/tests/baseline/hashes/mpl36_ft261.json new file mode 100644 index 00000000..ef249c31 --- /dev/null +++ b/tests/baseline/hashes/mpl36_ft261.json @@ -0,0 +1,10 @@ +{ + "test_pytest_mpl.test_hash_succeeds": "e73d228183ddfdced366191399cdecef9685d1248b852162f179750fc7b8b904", + "test.test_modified": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test.test_new": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test.test_unmodified": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test_formats.test_format_eps": "f7a03867e26b8ae32ed6f22972dfdc571a974df80881dc24bb8f840681f2ac5f", + "test_formats.test_format_pdf": "492152532625c1c4ea404854b75923c5dae0c253873b3715edba29d93b326d07", + "test_formats.test_format_png": "e73d228183ddfdced366191399cdecef9685d1248b852162f179750fc7b8b904", + "test_formats.test_format_svg": "4d1390489561a93e5413071632cfdab79ca1efbebf55337af2ab462cf1d4b9db" +} diff --git a/tests/baseline/hashes/mpl37_ft261.json b/tests/baseline/hashes/mpl37_ft261.json new file mode 100644 index 00000000..d02bd2e4 --- /dev/null +++ b/tests/baseline/hashes/mpl37_ft261.json @@ -0,0 +1,10 @@ +{ + "test_pytest_mpl.test_hash_succeeds": "e73d228183ddfdced366191399cdecef9685d1248b852162f179750fc7b8b904", + "test.test_modified": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test.test_new": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test.test_unmodified": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test_formats.test_format_eps": "f7a03867e26b8ae32ed6f22972dfdc571a974df80881dc24bb8f840681f2ac5f", + "test_formats.test_format_pdf": "492152532625c1c4ea404854b75923c5dae0c253873b3715edba29d93b326d07", + "test_formats.test_format_png": "e73d228183ddfdced366191399cdecef9685d1248b852162f179750fc7b8b904", + "test_formats.test_format_svg": "cb648df098fa1dd1ba0ed81e6498d31a152a43c47882e2503d093f83306ff403" +} diff --git a/tests/baseline/hashes/mpl38_ft261.json b/tests/baseline/hashes/mpl38_ft261.json new file mode 100644 index 00000000..7104aecc --- /dev/null +++ b/tests/baseline/hashes/mpl38_ft261.json @@ -0,0 +1,10 @@ +{ + "test_pytest_mpl.test_hash_succeeds": "e73d228183ddfdced366191399cdecef9685d1248b852162f179750fc7b8b904", + "test.test_modified": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test.test_new": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test.test_unmodified": "025d344bc9667222abc27f2d746618bce9da8ae44fa9afa1220ac83f230b4e4e", + "test_formats.test_format_eps": "f730e67270aeeb2a0bedee168d7c421b952cbf486ea371987cc4cea503a7d7ff", + "test_formats.test_format_pdf": "492152532625c1c4ea404854b75923c5dae0c253873b3715edba29d93b326d07", + "test_formats.test_format_png": "e73d228183ddfdced366191399cdecef9685d1248b852162f179750fc7b8b904", + "test_formats.test_format_svg": "6309fd2c3f328ca3c5614c58c118780c94035eb3690fc4b95d1923699b28ff8e" +} diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 00000000..ce412355 --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,36 @@ +import sys +from pathlib import Path + +import matplotlib +import pytest +from matplotlib.testing.compare import converter +from packaging.version import Version + +MPL_VERSION = Version(matplotlib.__version__) + + +def pytester_path(pytester): + if hasattr(pytester, "path"): + return pytester.path + return Path(pytester.tmpdir) # pytest v5 + + +def skip_if_format_unsupported(file_format, using_hashes=False): + if file_format == 'svg' and MPL_VERSION < Version('3.3'): + pytest.skip('SVG comparison is only supported in Matplotlib 3.3 and above') + + if using_hashes: + + if file_format == 'pdf' and MPL_VERSION < Version('2.1'): + pytest.skip('PDF hashes are only deterministic in Matplotlib 2.1 and above') + elif file_format == 'eps' and MPL_VERSION < Version('2.1'): + pytest.skip('EPS hashes are only deterministic in Matplotlib 2.1 and above') + + if using_hashes and not sys.platform.startswith('linux'): + pytest.skip('Hashes for vector graphics are only provided in the hash library for Linux') + + if file_format != 'png' and file_format not in converter: + if file_format == 'svg': + pytest.skip('Comparing SVG files requires inkscape to be installed') + else: + pytest.skip('Comparing EPS and PDF files requires ghostscript to be installed') diff --git a/tests/subtests/hashes/mpl33_ft261.json b/tests/subtests/hashes/mpl33_ft261.json deleted file mode 100644 index 5bec9574..00000000 --- a/tests/subtests/hashes/mpl33_ft261.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "subtests.subtest.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", - "subtests.subtest.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", - "subtests.subtest.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", - "subtests.subtest.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", - "subtests.subtest.test_hdiff_imatch": "d1ff5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", - "subtests.subtest.test_hdiff_idiff": "d1ff014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", - "subtests.subtest.test_hdiff_idiffshape": "d1ff3bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", - "subtests.subtest.test_hdiff_imissing": "d1ffd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", - "subtests.subtest.test_hdiff_imatch_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_idiff_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_imatch_savefig": "d1ff14c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", - "subtests.subtest.test_hdiff_imatch_style": "d1ffd00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", - "subtests.subtest.test_hdiff_imatch_removetext": "d1ffd7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a" -} diff --git a/tests/subtests/hashes/mpl34_ft261.json b/tests/subtests/hashes/mpl34_ft261.json deleted file mode 100644 index 5bec9574..00000000 --- a/tests/subtests/hashes/mpl34_ft261.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "subtests.subtest.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", - "subtests.subtest.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", - "subtests.subtest.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", - "subtests.subtest.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", - "subtests.subtest.test_hdiff_imatch": "d1ff5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", - "subtests.subtest.test_hdiff_idiff": "d1ff014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", - "subtests.subtest.test_hdiff_idiffshape": "d1ff3bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", - "subtests.subtest.test_hdiff_imissing": "d1ffd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", - "subtests.subtest.test_hdiff_imatch_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_idiff_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_imatch_savefig": "d1ff14c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", - "subtests.subtest.test_hdiff_imatch_style": "d1ffd00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", - "subtests.subtest.test_hdiff_imatch_removetext": "d1ffd7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a" -} diff --git a/tests/subtests/hashes/mpl35_ft261.json b/tests/subtests/hashes/mpl35_ft261.json deleted file mode 100644 index 5bec9574..00000000 --- a/tests/subtests/hashes/mpl35_ft261.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "subtests.subtest.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", - "subtests.subtest.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", - "subtests.subtest.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", - "subtests.subtest.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", - "subtests.subtest.test_hdiff_imatch": "d1ff5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", - "subtests.subtest.test_hdiff_idiff": "d1ff014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", - "subtests.subtest.test_hdiff_idiffshape": "d1ff3bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", - "subtests.subtest.test_hdiff_imissing": "d1ffd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", - "subtests.subtest.test_hdiff_imatch_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_idiff_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_imatch_savefig": "d1ff14c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", - "subtests.subtest.test_hdiff_imatch_style": "d1ffd00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", - "subtests.subtest.test_hdiff_imatch_removetext": "d1ffd7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a" -} diff --git a/tests/subtests/result_hashes/mpl33_ft261.json b/tests/subtests/result_hashes/mpl33_ft261.json index 6079e0fe..95904f0e 100644 --- a/tests/subtests/result_hashes/mpl33_ft261.json +++ b/tests/subtests/result_hashes/mpl33_ft261.json @@ -1,19 +1,34 @@ { - "subtests.subtest.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", - "subtests.subtest.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", - "subtests.subtest.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", - "subtests.subtest.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", - "subtests.subtest.test_hdiff_imatch": "b92c5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", - "subtests.subtest.test_hdiff_idiff": "567f014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", - "subtests.subtest.test_hdiff_idiffshape": "b6673bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", - "subtests.subtest.test_hdiff_imissing": "e37bd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", - "subtests.subtest.test_hmissing_imatch": "592d12cc2d5749a6607bbf98d715b95c06a2af6572a7e298bcae349648b9997e", - "subtests.subtest.test_hmissing_idiff": "9e98dbd2027525c776212daa061180b4fc40ad12dfc2cdfe4b86694ede14e0c3", - "subtests.subtest.test_hmissing_idiffshape": "5534324f9da5c1c104c3ef3435dc6fa9792c0d0d8b762fad5d7f81abd91fbb89", - "subtests.subtest.test_hmissing_imissing": "2d15274f0e9b44f1c16e6b237710cd36a3de5c5b7596ef0e65e7a33ce4624cf4", - "subtests.subtest.test_hdiff_imatch_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_idiff_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_imatch_savefig": "0a0514c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", - "subtests.subtest.test_hdiff_imatch_style": "588ad00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", - "subtests.subtest.test_hdiff_imatch_removetext": "a78ad7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a" + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": "2bb430367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": "ee0c290d66d17e4e40991c31eee0c55bd724922a3ecb66f7300b2ce8abf3d204", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": "3f8d2c1facdf6e34e43b4343abd888bd5ee9d2d451ce285e62bf0c65d6cdcdce", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": "9a6b008431f0b47a8866d87a9f1e9352fd73407a3ee2dd756a8ccbfa5a835a08", + "subtests.subtest.test_functions.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", + "subtests.subtest.test_functions.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", + "subtests.subtest.test_functions.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", + "subtests.subtest.test_functions.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", + "subtests.subtest.test_functions.test_hdiff_imatch": "b92c5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", + "subtests.subtest.test_functions.test_hdiff_idiff": "567f014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", + "subtests.subtest.test_functions.test_hdiff_idiffshape": "b6673bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", + "subtests.subtest.test_functions.test_hdiff_imissing": "e37bd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", + "subtests.subtest.test_functions.test_hmissing_imatch": "592d12cc2d5749a6607bbf98d715b95c06a2af6572a7e298bcae349648b9997e", + "subtests.subtest.test_functions.test_hmissing_idiff": "9e98dbd2027525c776212daa061180b4fc40ad12dfc2cdfe4b86694ede14e0c3", + "subtests.subtest.test_functions.test_hmissing_idiffshape": "5534324f9da5c1c104c3ef3435dc6fa9792c0d0d8b762fad5d7f81abd91fbb89", + "subtests.subtest.test_functions.test_hmissing_imissing": "2d15274f0e9b44f1c16e6b237710cd36a3de5c5b7596ef0e65e7a33ce4624cf4", + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_imatch_savefig": "0a0514c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", + "subtests.subtest.test_special.test_hdiff_imatch_style": "588ad00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", + "subtests.subtest.test_special.test_hdiff_imatch_removetext": "a78ad7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a", + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": "2bb430367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602" } diff --git a/tests/subtests/result_hashes/mpl34_ft261.json b/tests/subtests/result_hashes/mpl34_ft261.json index 6079e0fe..95904f0e 100644 --- a/tests/subtests/result_hashes/mpl34_ft261.json +++ b/tests/subtests/result_hashes/mpl34_ft261.json @@ -1,19 +1,34 @@ { - "subtests.subtest.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", - "subtests.subtest.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", - "subtests.subtest.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", - "subtests.subtest.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", - "subtests.subtest.test_hdiff_imatch": "b92c5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", - "subtests.subtest.test_hdiff_idiff": "567f014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", - "subtests.subtest.test_hdiff_idiffshape": "b6673bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", - "subtests.subtest.test_hdiff_imissing": "e37bd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", - "subtests.subtest.test_hmissing_imatch": "592d12cc2d5749a6607bbf98d715b95c06a2af6572a7e298bcae349648b9997e", - "subtests.subtest.test_hmissing_idiff": "9e98dbd2027525c776212daa061180b4fc40ad12dfc2cdfe4b86694ede14e0c3", - "subtests.subtest.test_hmissing_idiffshape": "5534324f9da5c1c104c3ef3435dc6fa9792c0d0d8b762fad5d7f81abd91fbb89", - "subtests.subtest.test_hmissing_imissing": "2d15274f0e9b44f1c16e6b237710cd36a3de5c5b7596ef0e65e7a33ce4624cf4", - "subtests.subtest.test_hdiff_imatch_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_idiff_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_imatch_savefig": "0a0514c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", - "subtests.subtest.test_hdiff_imatch_style": "588ad00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", - "subtests.subtest.test_hdiff_imatch_removetext": "a78ad7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a" + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": "2bb430367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": "ee0c290d66d17e4e40991c31eee0c55bd724922a3ecb66f7300b2ce8abf3d204", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": "3f8d2c1facdf6e34e43b4343abd888bd5ee9d2d451ce285e62bf0c65d6cdcdce", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": "9a6b008431f0b47a8866d87a9f1e9352fd73407a3ee2dd756a8ccbfa5a835a08", + "subtests.subtest.test_functions.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", + "subtests.subtest.test_functions.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", + "subtests.subtest.test_functions.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", + "subtests.subtest.test_functions.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", + "subtests.subtest.test_functions.test_hdiff_imatch": "b92c5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", + "subtests.subtest.test_functions.test_hdiff_idiff": "567f014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", + "subtests.subtest.test_functions.test_hdiff_idiffshape": "b6673bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", + "subtests.subtest.test_functions.test_hdiff_imissing": "e37bd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", + "subtests.subtest.test_functions.test_hmissing_imatch": "592d12cc2d5749a6607bbf98d715b95c06a2af6572a7e298bcae349648b9997e", + "subtests.subtest.test_functions.test_hmissing_idiff": "9e98dbd2027525c776212daa061180b4fc40ad12dfc2cdfe4b86694ede14e0c3", + "subtests.subtest.test_functions.test_hmissing_idiffshape": "5534324f9da5c1c104c3ef3435dc6fa9792c0d0d8b762fad5d7f81abd91fbb89", + "subtests.subtest.test_functions.test_hmissing_imissing": "2d15274f0e9b44f1c16e6b237710cd36a3de5c5b7596ef0e65e7a33ce4624cf4", + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_imatch_savefig": "0a0514c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", + "subtests.subtest.test_special.test_hdiff_imatch_style": "588ad00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", + "subtests.subtest.test_special.test_hdiff_imatch_removetext": "a78ad7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a", + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": "2bb430367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602" } diff --git a/tests/subtests/result_hashes/mpl35_ft261.json b/tests/subtests/result_hashes/mpl35_ft261.json index 6079e0fe..95904f0e 100644 --- a/tests/subtests/result_hashes/mpl35_ft261.json +++ b/tests/subtests/result_hashes/mpl35_ft261.json @@ -1,19 +1,34 @@ { - "subtests.subtest.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", - "subtests.subtest.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", - "subtests.subtest.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", - "subtests.subtest.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", - "subtests.subtest.test_hdiff_imatch": "b92c5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", - "subtests.subtest.test_hdiff_idiff": "567f014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", - "subtests.subtest.test_hdiff_idiffshape": "b6673bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", - "subtests.subtest.test_hdiff_imissing": "e37bd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", - "subtests.subtest.test_hmissing_imatch": "592d12cc2d5749a6607bbf98d715b95c06a2af6572a7e298bcae349648b9997e", - "subtests.subtest.test_hmissing_idiff": "9e98dbd2027525c776212daa061180b4fc40ad12dfc2cdfe4b86694ede14e0c3", - "subtests.subtest.test_hmissing_idiffshape": "5534324f9da5c1c104c3ef3435dc6fa9792c0d0d8b762fad5d7f81abd91fbb89", - "subtests.subtest.test_hmissing_imissing": "2d15274f0e9b44f1c16e6b237710cd36a3de5c5b7596ef0e65e7a33ce4624cf4", - "subtests.subtest.test_hdiff_imatch_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_idiff_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_imatch_savefig": "0a0514c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", - "subtests.subtest.test_hdiff_imatch_style": "588ad00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", - "subtests.subtest.test_hdiff_imatch_removetext": "a78ad7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a" + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": "2bb430367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": "ee0c290d66d17e4e40991c31eee0c55bd724922a3ecb66f7300b2ce8abf3d204", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": "3f8d2c1facdf6e34e43b4343abd888bd5ee9d2d451ce285e62bf0c65d6cdcdce", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": "9a6b008431f0b47a8866d87a9f1e9352fd73407a3ee2dd756a8ccbfa5a835a08", + "subtests.subtest.test_functions.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", + "subtests.subtest.test_functions.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", + "subtests.subtest.test_functions.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", + "subtests.subtest.test_functions.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", + "subtests.subtest.test_functions.test_hdiff_imatch": "b92c5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", + "subtests.subtest.test_functions.test_hdiff_idiff": "567f014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", + "subtests.subtest.test_functions.test_hdiff_idiffshape": "b6673bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", + "subtests.subtest.test_functions.test_hdiff_imissing": "e37bd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", + "subtests.subtest.test_functions.test_hmissing_imatch": "592d12cc2d5749a6607bbf98d715b95c06a2af6572a7e298bcae349648b9997e", + "subtests.subtest.test_functions.test_hmissing_idiff": "9e98dbd2027525c776212daa061180b4fc40ad12dfc2cdfe4b86694ede14e0c3", + "subtests.subtest.test_functions.test_hmissing_idiffshape": "5534324f9da5c1c104c3ef3435dc6fa9792c0d0d8b762fad5d7f81abd91fbb89", + "subtests.subtest.test_functions.test_hmissing_imissing": "2d15274f0e9b44f1c16e6b237710cd36a3de5c5b7596ef0e65e7a33ce4624cf4", + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_imatch_savefig": "0a0514c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", + "subtests.subtest.test_special.test_hdiff_imatch_style": "588ad00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", + "subtests.subtest.test_special.test_hdiff_imatch_removetext": "a78ad7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a", + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": "2bb430367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602" } diff --git a/tests/subtests/result_hashes/mpl36_ft261.json b/tests/subtests/result_hashes/mpl36_ft261.json new file mode 100644 index 00000000..95904f0e --- /dev/null +++ b/tests/subtests/result_hashes/mpl36_ft261.json @@ -0,0 +1,34 @@ +{ + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": "2bb430367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": "ee0c290d66d17e4e40991c31eee0c55bd724922a3ecb66f7300b2ce8abf3d204", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": "3f8d2c1facdf6e34e43b4343abd888bd5ee9d2d451ce285e62bf0c65d6cdcdce", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": "9a6b008431f0b47a8866d87a9f1e9352fd73407a3ee2dd756a8ccbfa5a835a08", + "subtests.subtest.test_functions.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", + "subtests.subtest.test_functions.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", + "subtests.subtest.test_functions.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", + "subtests.subtest.test_functions.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", + "subtests.subtest.test_functions.test_hdiff_imatch": "b92c5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", + "subtests.subtest.test_functions.test_hdiff_idiff": "567f014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", + "subtests.subtest.test_functions.test_hdiff_idiffshape": "b6673bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", + "subtests.subtest.test_functions.test_hdiff_imissing": "e37bd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", + "subtests.subtest.test_functions.test_hmissing_imatch": "592d12cc2d5749a6607bbf98d715b95c06a2af6572a7e298bcae349648b9997e", + "subtests.subtest.test_functions.test_hmissing_idiff": "9e98dbd2027525c776212daa061180b4fc40ad12dfc2cdfe4b86694ede14e0c3", + "subtests.subtest.test_functions.test_hmissing_idiffshape": "5534324f9da5c1c104c3ef3435dc6fa9792c0d0d8b762fad5d7f81abd91fbb89", + "subtests.subtest.test_functions.test_hmissing_imissing": "2d15274f0e9b44f1c16e6b237710cd36a3de5c5b7596ef0e65e7a33ce4624cf4", + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_imatch_savefig": "0a0514c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", + "subtests.subtest.test_special.test_hdiff_imatch_style": "588ad00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", + "subtests.subtest.test_special.test_hdiff_imatch_removetext": "a78ad7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a", + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": "2bb430367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602" +} diff --git a/tests/subtests/result_hashes/mpl37_ft261.json b/tests/subtests/result_hashes/mpl37_ft261.json new file mode 100644 index 00000000..95904f0e --- /dev/null +++ b/tests/subtests/result_hashes/mpl37_ft261.json @@ -0,0 +1,34 @@ +{ + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": "2bb430367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": "ee0c290d66d17e4e40991c31eee0c55bd724922a3ecb66f7300b2ce8abf3d204", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": "3f8d2c1facdf6e34e43b4343abd888bd5ee9d2d451ce285e62bf0c65d6cdcdce", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": "9a6b008431f0b47a8866d87a9f1e9352fd73407a3ee2dd756a8ccbfa5a835a08", + "subtests.subtest.test_functions.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", + "subtests.subtest.test_functions.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", + "subtests.subtest.test_functions.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", + "subtests.subtest.test_functions.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", + "subtests.subtest.test_functions.test_hdiff_imatch": "b92c5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", + "subtests.subtest.test_functions.test_hdiff_idiff": "567f014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", + "subtests.subtest.test_functions.test_hdiff_idiffshape": "b6673bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", + "subtests.subtest.test_functions.test_hdiff_imissing": "e37bd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", + "subtests.subtest.test_functions.test_hmissing_imatch": "592d12cc2d5749a6607bbf98d715b95c06a2af6572a7e298bcae349648b9997e", + "subtests.subtest.test_functions.test_hmissing_idiff": "9e98dbd2027525c776212daa061180b4fc40ad12dfc2cdfe4b86694ede14e0c3", + "subtests.subtest.test_functions.test_hmissing_idiffshape": "5534324f9da5c1c104c3ef3435dc6fa9792c0d0d8b762fad5d7f81abd91fbb89", + "subtests.subtest.test_functions.test_hmissing_imissing": "2d15274f0e9b44f1c16e6b237710cd36a3de5c5b7596ef0e65e7a33ce4624cf4", + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_imatch_savefig": "0a0514c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", + "subtests.subtest.test_special.test_hdiff_imatch_style": "588ad00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", + "subtests.subtest.test_special.test_hdiff_imatch_removetext": "a78ad7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a", + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": "2bb430367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602" +} diff --git a/tests/subtests/result_hashes/mpl38_ft261.json b/tests/subtests/result_hashes/mpl38_ft261.json new file mode 100644 index 00000000..95904f0e --- /dev/null +++ b/tests/subtests/result_hashes/mpl38_ft261.json @@ -0,0 +1,34 @@ +{ + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": "2bb430367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": "ee0c290d66d17e4e40991c31eee0c55bd724922a3ecb66f7300b2ce8abf3d204", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": "3f8d2c1facdf6e34e43b4343abd888bd5ee9d2d451ce285e62bf0c65d6cdcdce", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": "9a6b008431f0b47a8866d87a9f1e9352fd73407a3ee2dd756a8ccbfa5a835a08", + "subtests.subtest.test_functions.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", + "subtests.subtest.test_functions.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", + "subtests.subtest.test_functions.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", + "subtests.subtest.test_functions.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", + "subtests.subtest.test_functions.test_hdiff_imatch": "b92c5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", + "subtests.subtest.test_functions.test_hdiff_idiff": "567f014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", + "subtests.subtest.test_functions.test_hdiff_idiffshape": "b6673bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", + "subtests.subtest.test_functions.test_hdiff_imissing": "e37bd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", + "subtests.subtest.test_functions.test_hmissing_imatch": "592d12cc2d5749a6607bbf98d715b95c06a2af6572a7e298bcae349648b9997e", + "subtests.subtest.test_functions.test_hmissing_idiff": "9e98dbd2027525c776212daa061180b4fc40ad12dfc2cdfe4b86694ede14e0c3", + "subtests.subtest.test_functions.test_hmissing_idiffshape": "5534324f9da5c1c104c3ef3435dc6fa9792c0d0d8b762fad5d7f81abd91fbb89", + "subtests.subtest.test_functions.test_hmissing_imissing": "2d15274f0e9b44f1c16e6b237710cd36a3de5c5b7596ef0e65e7a33ce4624cf4", + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_imatch_savefig": "0a0514c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", + "subtests.subtest.test_special.test_hdiff_imatch_style": "588ad00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", + "subtests.subtest.test_special.test_hdiff_imatch_removetext": "a78ad7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a", + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": "2bb430367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602" +} diff --git a/tests/subtests/subtest/__init__.py b/tests/subtests/subtest/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/subtests/baseline/test_hdiff_idiff.png b/tests/subtests/subtest/baseline/test_hdiff_idiff.png similarity index 100% rename from tests/subtests/baseline/test_hdiff_idiff.png rename to tests/subtests/subtest/baseline/test_hdiff_idiff.png diff --git a/tests/subtests/subtest/baseline/test_hdiff_idiff_testcasewithsetup.png b/tests/subtests/subtest/baseline/test_hdiff_idiff_testcasewithsetup.png new file mode 100644 index 00000000..a1962b97 Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hdiff_idiff_testcasewithsetup.png differ diff --git a/tests/subtests/subtest/baseline/test_hdiff_idiff_testcasewithsetupclass.png b/tests/subtests/subtest/baseline/test_hdiff_idiff_testcasewithsetupclass.png new file mode 100644 index 00000000..349e4c5d Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hdiff_idiff_testcasewithsetupclass.png differ diff --git a/tests/subtests/subtest/baseline/test_hdiff_idiff_testclass.png b/tests/subtests/subtest/baseline/test_hdiff_idiff_testclass.png new file mode 100644 index 00000000..e5346158 Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hdiff_idiff_testclass.png differ diff --git a/tests/subtests/subtest/baseline/test_hdiff_idiff_testclasswithsetupclass.png b/tests/subtests/subtest/baseline/test_hdiff_idiff_testclasswithsetupclass.png new file mode 100644 index 00000000..c2cb95a4 Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hdiff_idiff_testclasswithsetupclass.png differ diff --git a/tests/subtests/subtest/baseline/test_hdiff_idiff_testclasswithsetupmethod.png b/tests/subtests/subtest/baseline/test_hdiff_idiff_testclasswithsetupmethod.png new file mode 100644 index 00000000..a1962b97 Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hdiff_idiff_testclasswithsetupmethod.png differ diff --git a/tests/subtests/baseline/test_hdiff_idiff_tolerance.png b/tests/subtests/subtest/baseline/test_hdiff_idiff_tolerance.png similarity index 100% rename from tests/subtests/baseline/test_hdiff_idiff_tolerance.png rename to tests/subtests/subtest/baseline/test_hdiff_idiff_tolerance.png diff --git a/tests/subtests/baseline/test_hdiff_idiffshape.png b/tests/subtests/subtest/baseline/test_hdiff_idiffshape.png similarity index 100% rename from tests/subtests/baseline/test_hdiff_idiffshape.png rename to tests/subtests/subtest/baseline/test_hdiff_idiffshape.png diff --git a/tests/subtests/baseline/test_hdiff_imatch.png b/tests/subtests/subtest/baseline/test_hdiff_imatch.png similarity index 100% rename from tests/subtests/baseline/test_hdiff_imatch.png rename to tests/subtests/subtest/baseline/test_hdiff_imatch.png diff --git a/tests/subtests/baseline/test_hdiff_imatch_removetext.png b/tests/subtests/subtest/baseline/test_hdiff_imatch_removetext.png similarity index 100% rename from tests/subtests/baseline/test_hdiff_imatch_removetext.png rename to tests/subtests/subtest/baseline/test_hdiff_imatch_removetext.png diff --git a/tests/subtests/baseline/test_hdiff_imatch_savefig.png b/tests/subtests/subtest/baseline/test_hdiff_imatch_savefig.png similarity index 100% rename from tests/subtests/baseline/test_hdiff_imatch_savefig.png rename to tests/subtests/subtest/baseline/test_hdiff_imatch_savefig.png diff --git a/tests/subtests/baseline/test_hdiff_imatch_style.png b/tests/subtests/subtest/baseline/test_hdiff_imatch_style.png similarity index 100% rename from tests/subtests/baseline/test_hdiff_imatch_style.png rename to tests/subtests/subtest/baseline/test_hdiff_imatch_style.png diff --git a/tests/subtests/baseline/test_hdiff_imatch_tolerance.png b/tests/subtests/subtest/baseline/test_hdiff_imatch_tolerance.png similarity index 100% rename from tests/subtests/baseline/test_hdiff_imatch_tolerance.png rename to tests/subtests/subtest/baseline/test_hdiff_imatch_tolerance.png diff --git a/tests/subtests/baseline/test_hmatch_idiff.png b/tests/subtests/subtest/baseline/test_hmatch_idiff.png similarity index 100% rename from tests/subtests/baseline/test_hmatch_idiff.png rename to tests/subtests/subtest/baseline/test_hmatch_idiff.png diff --git a/tests/subtests/baseline/test_hmatch_idiffshape.png b/tests/subtests/subtest/baseline/test_hmatch_idiffshape.png similarity index 100% rename from tests/subtests/baseline/test_hmatch_idiffshape.png rename to tests/subtests/subtest/baseline/test_hmatch_idiffshape.png diff --git a/tests/subtests/baseline/test_hmatch_imatch.png b/tests/subtests/subtest/baseline/test_hmatch_imatch.png similarity index 100% rename from tests/subtests/baseline/test_hmatch_imatch.png rename to tests/subtests/subtest/baseline/test_hmatch_imatch.png diff --git a/tests/subtests/subtest/baseline/test_hmatch_imatch_multiplefigures_first.png b/tests/subtests/subtest/baseline/test_hmatch_imatch_multiplefigures_first.png new file mode 100644 index 00000000..bdce3923 Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hmatch_imatch_multiplefigures_first.png differ diff --git a/tests/subtests/subtest/baseline/test_hmatch_imatch_multiplefigures_second.png b/tests/subtests/subtest/baseline/test_hmatch_imatch_multiplefigures_second.png new file mode 100644 index 00000000..df6c3983 Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hmatch_imatch_multiplefigures_second.png differ diff --git a/tests/subtests/subtest/baseline/test_hmatch_imatch_testcasewithsetup.png b/tests/subtests/subtest/baseline/test_hmatch_imatch_testcasewithsetup.png new file mode 100644 index 00000000..460a6d39 Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hmatch_imatch_testcasewithsetup.png differ diff --git a/tests/subtests/subtest/baseline/test_hmatch_imatch_testcasewithsetupclass.png b/tests/subtests/subtest/baseline/test_hmatch_imatch_testcasewithsetupclass.png new file mode 100644 index 00000000..83437d00 Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hmatch_imatch_testcasewithsetupclass.png differ diff --git a/tests/subtests/subtest/baseline/test_hmatch_imatch_testclass.png b/tests/subtests/subtest/baseline/test_hmatch_imatch_testclass.png new file mode 100644 index 00000000..c41a213b Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hmatch_imatch_testclass.png differ diff --git a/tests/subtests/subtest/baseline/test_hmatch_imatch_testclasswithfixture.png b/tests/subtests/subtest/baseline/test_hmatch_imatch_testclasswithfixture.png new file mode 100644 index 00000000..9c45afe5 Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hmatch_imatch_testclasswithfixture.png differ diff --git a/tests/subtests/subtest/baseline/test_hmatch_imatch_testclasswithsetupclass.png b/tests/subtests/subtest/baseline/test_hmatch_imatch_testclasswithsetupclass.png new file mode 100644 index 00000000..1a185036 Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hmatch_imatch_testclasswithsetupclass.png differ diff --git a/tests/subtests/subtest/baseline/test_hmatch_imatch_testclasswithsetupmethod.png b/tests/subtests/subtest/baseline/test_hmatch_imatch_testclasswithsetupmethod.png new file mode 100644 index 00000000..460a6d39 Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hmatch_imatch_testclasswithsetupmethod.png differ diff --git a/tests/subtests/baseline/test_hmissing_idiff.png b/tests/subtests/subtest/baseline/test_hmissing_idiff.png similarity index 100% rename from tests/subtests/baseline/test_hmissing_idiff.png rename to tests/subtests/subtest/baseline/test_hmissing_idiff.png diff --git a/tests/subtests/baseline/test_hmissing_idiffshape.png b/tests/subtests/subtest/baseline/test_hmissing_idiffshape.png similarity index 100% rename from tests/subtests/baseline/test_hmissing_idiffshape.png rename to tests/subtests/subtest/baseline/test_hmissing_idiffshape.png diff --git a/tests/subtests/baseline/test_hmissing_imatch.png b/tests/subtests/subtest/baseline/test_hmissing_imatch.png similarity index 100% rename from tests/subtests/baseline/test_hmissing_imatch.png rename to tests/subtests/subtest/baseline/test_hmissing_imatch.png diff --git a/tests/subtests/subtest/hashes/mpl33_ft261.json b/tests/subtests/subtest/hashes/mpl33_ft261.json new file mode 100644 index 00000000..ed5bb0e0 --- /dev/null +++ b/tests/subtests/subtest/hashes/mpl33_ft261.json @@ -0,0 +1,30 @@ +{ + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": "d1ff30367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": "d1ff0363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": "d1ffa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": "ee0c290d66d17e4e40991c31eee0c55bd724922a3ecb66f7300b2ce8abf3d204", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": "3f8d2c1facdf6e34e43b4343abd888bd5ee9d2d451ce285e62bf0c65d6cdcdce", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": "9a6b008431f0b47a8866d87a9f1e9352fd73407a3ee2dd756a8ccbfa5a835a08", + "subtests.subtest.test_functions.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", + "subtests.subtest.test_functions.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", + "subtests.subtest.test_functions.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", + "subtests.subtest.test_functions.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", + "subtests.subtest.test_functions.test_hdiff_imatch": "d1ff5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", + "subtests.subtest.test_functions.test_hdiff_idiff": "d1ff014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", + "subtests.subtest.test_functions.test_hdiff_idiffshape": "d1ff3bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", + "subtests.subtest.test_functions.test_hdiff_imissing": "d1ffd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_imatch_savefig": "d1ff14c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", + "subtests.subtest.test_special.test_hdiff_imatch_style": "d1ffd00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", + "subtests.subtest.test_special.test_hdiff_imatch_removetext": "d1ffd7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a", + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": "d1ff30367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": "d1ff0363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": "d1ff6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602" +} diff --git a/tests/subtests/subtest/hashes/mpl34_ft261.json b/tests/subtests/subtest/hashes/mpl34_ft261.json new file mode 100644 index 00000000..ed5bb0e0 --- /dev/null +++ b/tests/subtests/subtest/hashes/mpl34_ft261.json @@ -0,0 +1,30 @@ +{ + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": "d1ff30367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": "d1ff0363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": "d1ffa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": "ee0c290d66d17e4e40991c31eee0c55bd724922a3ecb66f7300b2ce8abf3d204", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": "3f8d2c1facdf6e34e43b4343abd888bd5ee9d2d451ce285e62bf0c65d6cdcdce", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": "9a6b008431f0b47a8866d87a9f1e9352fd73407a3ee2dd756a8ccbfa5a835a08", + "subtests.subtest.test_functions.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", + "subtests.subtest.test_functions.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", + "subtests.subtest.test_functions.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", + "subtests.subtest.test_functions.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", + "subtests.subtest.test_functions.test_hdiff_imatch": "d1ff5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", + "subtests.subtest.test_functions.test_hdiff_idiff": "d1ff014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", + "subtests.subtest.test_functions.test_hdiff_idiffshape": "d1ff3bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", + "subtests.subtest.test_functions.test_hdiff_imissing": "d1ffd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_imatch_savefig": "d1ff14c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", + "subtests.subtest.test_special.test_hdiff_imatch_style": "d1ffd00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", + "subtests.subtest.test_special.test_hdiff_imatch_removetext": "d1ffd7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a", + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": "d1ff30367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": "d1ff0363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": "d1ff6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602" +} diff --git a/tests/subtests/subtest/hashes/mpl35_ft261.json b/tests/subtests/subtest/hashes/mpl35_ft261.json new file mode 100644 index 00000000..ed5bb0e0 --- /dev/null +++ b/tests/subtests/subtest/hashes/mpl35_ft261.json @@ -0,0 +1,30 @@ +{ + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": "d1ff30367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": "d1ff0363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": "d1ffa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": "ee0c290d66d17e4e40991c31eee0c55bd724922a3ecb66f7300b2ce8abf3d204", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": "3f8d2c1facdf6e34e43b4343abd888bd5ee9d2d451ce285e62bf0c65d6cdcdce", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": "9a6b008431f0b47a8866d87a9f1e9352fd73407a3ee2dd756a8ccbfa5a835a08", + "subtests.subtest.test_functions.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", + "subtests.subtest.test_functions.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", + "subtests.subtest.test_functions.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", + "subtests.subtest.test_functions.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", + "subtests.subtest.test_functions.test_hdiff_imatch": "d1ff5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", + "subtests.subtest.test_functions.test_hdiff_idiff": "d1ff014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", + "subtests.subtest.test_functions.test_hdiff_idiffshape": "d1ff3bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", + "subtests.subtest.test_functions.test_hdiff_imissing": "d1ffd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_imatch_savefig": "d1ff14c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", + "subtests.subtest.test_special.test_hdiff_imatch_style": "d1ffd00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", + "subtests.subtest.test_special.test_hdiff_imatch_removetext": "d1ffd7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a", + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": "d1ff30367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": "d1ff0363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": "d1ff6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602" +} diff --git a/tests/subtests/subtest/hashes/mpl36_ft261.json b/tests/subtests/subtest/hashes/mpl36_ft261.json new file mode 100644 index 00000000..ed5bb0e0 --- /dev/null +++ b/tests/subtests/subtest/hashes/mpl36_ft261.json @@ -0,0 +1,30 @@ +{ + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": "d1ff30367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": "d1ff0363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": "d1ffa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": "ee0c290d66d17e4e40991c31eee0c55bd724922a3ecb66f7300b2ce8abf3d204", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": "3f8d2c1facdf6e34e43b4343abd888bd5ee9d2d451ce285e62bf0c65d6cdcdce", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": "9a6b008431f0b47a8866d87a9f1e9352fd73407a3ee2dd756a8ccbfa5a835a08", + "subtests.subtest.test_functions.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", + "subtests.subtest.test_functions.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", + "subtests.subtest.test_functions.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", + "subtests.subtest.test_functions.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", + "subtests.subtest.test_functions.test_hdiff_imatch": "d1ff5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", + "subtests.subtest.test_functions.test_hdiff_idiff": "d1ff014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", + "subtests.subtest.test_functions.test_hdiff_idiffshape": "d1ff3bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", + "subtests.subtest.test_functions.test_hdiff_imissing": "d1ffd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_imatch_savefig": "d1ff14c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", + "subtests.subtest.test_special.test_hdiff_imatch_style": "d1ffd00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", + "subtests.subtest.test_special.test_hdiff_imatch_removetext": "d1ffd7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a", + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": "d1ff30367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": "d1ff0363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": "d1ff6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602" +} diff --git a/tests/subtests/subtest/hashes/mpl37_ft261.json b/tests/subtests/subtest/hashes/mpl37_ft261.json new file mode 100644 index 00000000..ed5bb0e0 --- /dev/null +++ b/tests/subtests/subtest/hashes/mpl37_ft261.json @@ -0,0 +1,30 @@ +{ + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": "d1ff30367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": "d1ff0363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": "d1ffa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": "ee0c290d66d17e4e40991c31eee0c55bd724922a3ecb66f7300b2ce8abf3d204", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": "3f8d2c1facdf6e34e43b4343abd888bd5ee9d2d451ce285e62bf0c65d6cdcdce", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": "9a6b008431f0b47a8866d87a9f1e9352fd73407a3ee2dd756a8ccbfa5a835a08", + "subtests.subtest.test_functions.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", + "subtests.subtest.test_functions.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", + "subtests.subtest.test_functions.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", + "subtests.subtest.test_functions.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", + "subtests.subtest.test_functions.test_hdiff_imatch": "d1ff5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", + "subtests.subtest.test_functions.test_hdiff_idiff": "d1ff014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", + "subtests.subtest.test_functions.test_hdiff_idiffshape": "d1ff3bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", + "subtests.subtest.test_functions.test_hdiff_imissing": "d1ffd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_imatch_savefig": "d1ff14c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", + "subtests.subtest.test_special.test_hdiff_imatch_style": "d1ffd00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", + "subtests.subtest.test_special.test_hdiff_imatch_removetext": "d1ffd7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a", + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": "d1ff30367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": "d1ff0363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": "d1ff6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602" +} diff --git a/tests/subtests/subtest/hashes/mpl38_ft261.json b/tests/subtests/subtest/hashes/mpl38_ft261.json new file mode 100644 index 00000000..ed5bb0e0 --- /dev/null +++ b/tests/subtests/subtest/hashes/mpl38_ft261.json @@ -0,0 +1,30 @@ +{ + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": "d1ff30367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": "d1ff0363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": "d1ffa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": "ee0c290d66d17e4e40991c31eee0c55bd724922a3ecb66f7300b2ce8abf3d204", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": "3f8d2c1facdf6e34e43b4343abd888bd5ee9d2d451ce285e62bf0c65d6cdcdce", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": "9a6b008431f0b47a8866d87a9f1e9352fd73407a3ee2dd756a8ccbfa5a835a08", + "subtests.subtest.test_functions.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", + "subtests.subtest.test_functions.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", + "subtests.subtest.test_functions.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", + "subtests.subtest.test_functions.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", + "subtests.subtest.test_functions.test_hdiff_imatch": "d1ff5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", + "subtests.subtest.test_functions.test_hdiff_idiff": "d1ff014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", + "subtests.subtest.test_functions.test_hdiff_idiffshape": "d1ff3bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", + "subtests.subtest.test_functions.test_hdiff_imissing": "d1ffd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_imatch_savefig": "d1ff14c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", + "subtests.subtest.test_special.test_hdiff_imatch_style": "d1ffd00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", + "subtests.subtest.test_special.test_hdiff_imatch_removetext": "d1ffd7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a", + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": "d1ff30367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": "d1ff0363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": "d1ff6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602" +} diff --git a/tests/subtests/subtest/helpers.py b/tests/subtests/subtest/helpers.py new file mode 100644 index 00000000..91b00ed7 --- /dev/null +++ b/tests/subtests/subtest/helpers.py @@ -0,0 +1,30 @@ +from functools import wraps + +import matplotlib.pyplot as plt +import numpy as np +import pytest + + +def plot(line, **kwargs): + fig = plt.figure() + ax = fig.add_subplot(1, 1, 1) + ax.plot(line, **kwargs) + return fig + + +def bar(height, **kwargs): + fig = plt.figure() + ax = fig.add_subplot(1, 1, 1) + x = np.arange(len(height)) + ax.bar(x, height, align="center", **kwargs) + return fig + + +def figure_test(test_function, **kwargs): + @pytest.mark.image + @pytest.mark.hash + @pytest.mark.mpl_image_compare(savefig_kwargs={'metadata': {'Software': None}}, **kwargs) + @wraps(test_function) + def test_wrapper(*args, **kwargs): + return test_function(*args, **kwargs) + return test_wrapper diff --git a/tests/subtests/subtest/pytest.ini b/tests/subtests/subtest/pytest.ini new file mode 100644 index 00000000..faaaaad6 --- /dev/null +++ b/tests/subtests/subtest/pytest.ini @@ -0,0 +1,8 @@ +[pytest] +markers = + image: run test during image comparison only mode. + hash: run test during hash comparison only mode. +filterwarnings = + error + ignore:distutils Version classes are deprecated + ignore:the imp module is deprecated in favour of importlib diff --git a/tests/subtests/subtest/test_classes.py b/tests/subtests/subtest/test_classes.py new file mode 100644 index 00000000..6e361276 --- /dev/null +++ b/tests/subtests/subtest/test_classes.py @@ -0,0 +1,93 @@ +import matplotlib.pyplot as plt +import pytest + +from pytest_mpl.plugin import switch_backend + +from .helpers import bar, figure_test + + +class TestClass: + + @figure_test + def test_hmatch_imatch_testclass(self): + return bar([1, 2, 3, 4]) + + @figure_test + def test_hdiff_idiff_testclass(self): + return bar([1, 3, 2, 4]) + + +class TestClassWithSetupMethod: + + def setup_method(self, method): + self.x = [4, 2, 3, 1] + + @figure_test + def test_hmatch_imatch_testclasswithsetupmethod(self): + return bar(self.x) + + @figure_test + def test_hdiff_idiff_testclasswithsetupmethod(self): + return bar(self.x) + + +class TestClassWithSetupClass: + + @classmethod + def setup_class(cls): + cls.x = [1, 0, 3, 2] + + @figure_test + def test_hmatch_imatch_testclasswithsetupclass(self): + return bar(self.x) + + @figure_test + def test_hdiff_idiff_testclasswithsetupclass(self): + return bar(self.x) + + +@pytest.fixture() +def figure_axes(): + with plt.style.context("classic", after_reset=True), switch_backend("agg"): + fig = bar([3, 0, 4, 1]) + yield fig.gca() + # Should not appear in test result + fig.gca().plot([3, 0, 4, 1], c="yellow") + + +class TestClassWithFixture: + @figure_test + def test_hmatch_imatch_testclasswithfixture(self, figure_axes): + figure_axes.plot([4, 1, 5, 2], c="red") + return figure_axes.get_figure() + + +def generate_two_figures(): + for num, line in ((1, [1, 0, 1, 0]), (2, [0, 1, 0, 1])): + plt.close(num) + fig = plt.figure(num) + ax = fig.add_subplot(1, 1, 1) + ax.plot(line) + + +class TestMultipleFigures: + """ + Test figures are accessible, and can be passed to individual functions. + + See https://github.com/matplotlib/pytest-mpl/issues/133 + """ + @classmethod + def setup_class(cls): + # setting style and backend inside decorator has no effect + # because figures are generated outside the test function + with plt.style.context("classic", after_reset=True), switch_backend("agg"): + generate_two_figures() + cls.figs = [plt.figure(i) for i in (1, 2)] + + @figure_test + def test_hmatch_imatch_multiplefigures_first(self): + return self.figs[0] + + @figure_test + def test_hmatch_imatch_multiplefigures_second(self): + return self.figs[1] diff --git a/tests/subtests/subtest.py b/tests/subtests/subtest/test_functions.py similarity index 50% rename from tests/subtests/subtest.py rename to tests/subtests/subtest/test_functions.py index c02abea6..045a1dfc 100644 --- a/tests/subtests/subtest.py +++ b/tests/subtests/subtest/test_functions.py @@ -1,18 +1,12 @@ -import matplotlib.pyplot as plt import pytest - -def plot(line, **kwargs): - fig = plt.figure() - ax = fig.add_subplot(1, 1, 1) - ax.plot(line, **kwargs) - return fig - +from .helpers import plot # ### Test all permutations of: # baseline hash: match, diff, or missing # baseline image: match, diff, or missing + # hash match @pytest.mark.image @@ -84,65 +78,3 @@ def test_hmissing_idiffshape(): @pytest.mark.mpl_image_compare(savefig_kwargs={'metadata': {'Software': None}}) def test_hmissing_imissing(): return plot([2, 4, 3, 1]) - - -# ### Specialized tests - -# Tolerance: high to force image match -@pytest.mark.image -@pytest.mark.hash -@pytest.mark.mpl_image_compare(tolerance=200, savefig_kwargs={'metadata': {'Software': None}}) -def test_hdiff_imatch_tolerance(): - return plot([1, 2, 3, 4], linestyle='--') - - -# Tolerance: non-default to verify option recorded in JSON -@pytest.mark.image -@pytest.mark.hash -@pytest.mark.mpl_image_compare(tolerance=3, savefig_kwargs={'metadata': {'Software': None}}) -def test_hdiff_idiff_tolerance(): - return plot([1, 2, 3, 4], linestyle='--') - - -# Savefig kwargs -@pytest.mark.image -@pytest.mark.hash -@pytest.mark.mpl_image_compare(savefig_kwargs={'facecolor': 'r', 'metadata': {'Software': None}}) -def test_hdiff_imatch_savefig(): - return plot([1, 2, 3, 4]) - - -# TODO: Implement these path altering tests later -# # Different baseline directory -# # TODO: Test with a remote `baseline_dir` -# @pytest.mark.mpl_image_compare(baseline_dir='baseline/other') -# def test_hdiff_imatch_baselinedir(): -# return plot([4, 2, 1, 4]) -# -# -# # Different filename -# @pytest.mark.mpl_image_compare(filename='test_hdiff_imatch_filename_other.png') -# def test_hdiff_imatch_filename(): -# return plot([4, 2, 1, 4]) -# -# -# # Different hash library -# @pytest.mark.mpl_image_compare(hash_library='hashes/other/other.json') -# def test_hdiff_imatch_hashlibrary(): -# return plot([4, 2, 1, 4]) - - -# Different style -@pytest.mark.image -@pytest.mark.hash -@pytest.mark.mpl_image_compare(style='fivethirtyeight', savefig_kwargs={'metadata': {'Software': None}}) -def test_hdiff_imatch_style(): - return plot([4, 2, 1, 4]) - - -# Remove text -@pytest.mark.image -@pytest.mark.hash -@pytest.mark.mpl_image_compare(remove_text=True, savefig_kwargs={'metadata': {'Software': None}}) -def test_hdiff_imatch_removetext(): - return plot([4, 2, 1, 4]) diff --git a/tests/subtests/subtest/test_special.py b/tests/subtests/subtest/test_special.py new file mode 100644 index 00000000..3659c26b --- /dev/null +++ b/tests/subtests/subtest/test_special.py @@ -0,0 +1,65 @@ +import pytest + +from .helpers import plot + +# ### Specialized tests + + +# Tolerance: high to force image match +@pytest.mark.image +@pytest.mark.hash +@pytest.mark.mpl_image_compare(tolerance=200, savefig_kwargs={'metadata': {'Software': None}}) +def test_hdiff_imatch_tolerance(): + return plot([1, 2, 3, 4], linestyle='--') + + +# Tolerance: non-default to verify option recorded in JSON +@pytest.mark.image +@pytest.mark.hash +@pytest.mark.mpl_image_compare(tolerance=3, savefig_kwargs={'metadata': {'Software': None}}) +def test_hdiff_idiff_tolerance(): + return plot([1, 2, 3, 4], linestyle='--') + + +# Savefig kwargs +@pytest.mark.image +@pytest.mark.hash +@pytest.mark.mpl_image_compare(savefig_kwargs={'facecolor': 'r', 'metadata': {'Software': None}}) +def test_hdiff_imatch_savefig(): + return plot([1, 2, 3, 4]) + + +# TODO: Implement these path altering tests later +# # Different baseline directory +# # TODO: Test with a remote `baseline_dir` +# @pytest.mark.mpl_image_compare(baseline_dir='baseline/other') +# def test_hdiff_imatch_baselinedir(): +# return plot([4, 2, 1, 4]) +# +# +# # Different filename +# @pytest.mark.mpl_image_compare(filename='test_hdiff_imatch_filename_other.png') +# def test_hdiff_imatch_filename(): +# return plot([4, 2, 1, 4]) +# +# +# # Different hash library +# @pytest.mark.mpl_image_compare(hash_library='hashes/other/other.json') +# def test_hdiff_imatch_hashlibrary(): +# return plot([4, 2, 1, 4]) + + +# Different style +@pytest.mark.image +@pytest.mark.hash +@pytest.mark.mpl_image_compare(style='fivethirtyeight', savefig_kwargs={'metadata': {'Software': None}}) +def test_hdiff_imatch_style(): + return plot([4, 2, 1, 4]) + + +# Remove text +@pytest.mark.image +@pytest.mark.hash +@pytest.mark.mpl_image_compare(remove_text=True, savefig_kwargs={'metadata': {'Software': None}}) +def test_hdiff_imatch_removetext(): + return plot([4, 2, 1, 4]) diff --git a/tests/subtests/subtest/test_unittest.py b/tests/subtests/subtest/test_unittest.py new file mode 100644 index 00000000..6c66e00c --- /dev/null +++ b/tests/subtests/subtest/test_unittest.py @@ -0,0 +1,43 @@ +import unittest + +from .helpers import bar, figure_test + + +class TestCase(unittest.TestCase): + + @figure_test + def test_hmatch_imatch_testclass(self): + return bar([1, 2, 3, 4]) + + @figure_test + def test_hdiff_idiff_testclass(self): + return bar([1, 3, 2, 4]) + + +class TestCaseWithSetUp(unittest.TestCase): + + def setUp(self): + self.x = [4, 2, 3, 1] + + @figure_test + def test_hmatch_imatch_testcasewithsetup(self): + return bar(self.x) + + @figure_test + def test_hdiff_idiff_testcasewithsetup(self): + return bar(self.x) + + +class TestCaseWithSetUpClass(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.x = [4, 3, 2, 4] + + @figure_test + def test_hmatch_imatch_testcasewithsetupclass(self): + return bar(self.x) + + @figure_test + def test_hdiff_idiff_testcasewithsetupclass(self): + return bar(self.x) diff --git a/tests/subtests/summaries/test_default.json b/tests/subtests/summaries/test_default.json index 61efc2dd..cbea11ff 100644 --- a/tests/subtests/summaries/test_default.json +++ b/tests/subtests/summaries/test_default.json @@ -1,5 +1,5 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "passed", "image_status": "match", "hash_status": null, @@ -12,33 +12,150 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_idiff": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result-failed-diff.png", + "rms": 24.563779228775037, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmatch_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmatch_idiff": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hmatch_idiff/result-failed-diff.png", "rms": 24.618234716477044, "tolerance": 2, - "result_image": "subtests.subtest.test_hmatch_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_idiff/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_idiffshape": { + "subtests.subtest.test_functions.test_hmatch_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmatch_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_imissing": { + "subtests.subtest.test_functions.test_hmatch_imissing": { "status": "failed", "image_status": "missing", "hash_status": null, @@ -47,11 +164,11 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmatch_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_imissing/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_functions.test_hdiff_imatch": { "status": "passed", "image_status": "match", "hash_status": null, @@ -64,33 +181,33 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiff": { + "subtests.subtest.test_functions.test_hdiff_idiff": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hdiff_idiff/result-failed-diff.png", "rms": 24.620185259705547, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiff/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiffshape": { + "subtests.subtest.test_functions.test_hdiff_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imissing": { + "subtests.subtest.test_functions.test_hdiff_imissing": { "status": "failed", "image_status": "missing", "hash_status": null, @@ -99,11 +216,11 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_imissing/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_functions.test_hmissing_imatch": { "status": "passed", "image_status": "match", "hash_status": null, @@ -116,33 +233,33 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_idiff": { + "subtests.subtest.test_functions.test_hmissing_idiff": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hmissing_idiff/result-failed-diff.png", "rms": 24.6217358806762, "tolerance": 2, - "result_image": "subtests.subtest.test_hmissing_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiff/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_idiffshape": { + "subtests.subtest.test_functions.test_hmissing_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_imissing": { + "subtests.subtest.test_functions.test_hmissing_imissing": { "status": "failed", "image_status": "missing", "hash_status": null, @@ -151,11 +268,11 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imissing/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { "status": "passed", "image_status": "match", "hash_status": null, @@ -168,20 +285,33 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", - "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result-failed-diff.png", "rms": 24.576566752362574, "tolerance": 3, - "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_special.test_hdiff_imatch_style": { "status": "passed", "image_status": "match", "hash_status": null, @@ -194,7 +324,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { "status": "passed", "image_status": "match", "hash_status": null, @@ -207,7 +337,72 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result-failed-diff.png", + "rms": 24.520257314106637, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { "status": "passed", "image_status": "match", "hash_status": null, diff --git a/tests/subtests/summaries/test_generate.json b/tests/subtests/summaries/test_generate.json index 04a1d288..f5f2cf7f 100644 --- a/tests/subtests/summaries/test_generate.json +++ b/tests/subtests/summaries/test_generate.json @@ -1,5 +1,5 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -12,7 +12,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_idiff": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -25,7 +25,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_idiffshape": { + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -38,7 +38,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_imissing": { + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -51,7 +51,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -64,7 +64,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiff": { + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -77,7 +77,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiffshape": { + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -90,7 +90,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imissing": { + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -103,7 +103,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -116,7 +116,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_idiff": { + "subtests.subtest.test_functions.test_hmatch_imatch": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -129,7 +129,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_idiffshape": { + "subtests.subtest.test_functions.test_hmatch_idiff": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -142,7 +142,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_imissing": { + "subtests.subtest.test_functions.test_hmatch_idiffshape": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -155,7 +155,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_functions.test_hmatch_imissing": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -168,7 +168,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_functions.test_hdiff_imatch": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -181,7 +181,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_functions.test_hdiff_idiff": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -194,7 +194,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_functions.test_hdiff_idiffshape": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -207,7 +207,202 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_functions.test_hdiff_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_idiffshape": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_style": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { "status": "skipped", "image_status": "generated", "hash_status": "generated", diff --git a/tests/subtests/summaries/test_generate_hashes_only.json b/tests/subtests/summaries/test_generate_hashes_only.json index b8b686a0..aaba44b9 100644 --- a/tests/subtests/summaries/test_generate_hashes_only.json +++ b/tests/subtests/summaries/test_generate_hashes_only.json @@ -1,5 +1,5 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "passed", "image_status": "match", "hash_status": "generated", @@ -12,33 +12,150 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_idiff": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result-failed-diff.png", + "rms": 24.563779228775037, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmatch_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmatch_idiff": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hmatch_idiff/result-failed-diff.png", "rms": 24.618234716477044, "tolerance": 2, - "result_image": "subtests.subtest.test_hmatch_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_idiff/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_idiffshape": { + "subtests.subtest.test_functions.test_hmatch_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmatch_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_imissing": { + "subtests.subtest.test_functions.test_hmatch_imissing": { "status": "failed", "image_status": "missing", "hash_status": "generated", @@ -47,11 +164,11 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmatch_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_imissing/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_functions.test_hdiff_imatch": { "status": "passed", "image_status": "match", "hash_status": "generated", @@ -64,33 +181,33 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiff": { + "subtests.subtest.test_functions.test_hdiff_idiff": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hdiff_idiff/result-failed-diff.png", "rms": 24.620185259705547, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiff/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiffshape": { + "subtests.subtest.test_functions.test_hdiff_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imissing": { + "subtests.subtest.test_functions.test_hdiff_imissing": { "status": "failed", "image_status": "missing", "hash_status": "generated", @@ -99,11 +216,11 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_imissing/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_functions.test_hmissing_imatch": { "status": "passed", "image_status": "match", "hash_status": "generated", @@ -116,33 +233,33 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_idiff": { + "subtests.subtest.test_functions.test_hmissing_idiff": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hmissing_idiff/result-failed-diff.png", "rms": 24.6217358806762, "tolerance": 2, - "result_image": "subtests.subtest.test_hmissing_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiff/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_idiffshape": { + "subtests.subtest.test_functions.test_hmissing_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_imissing": { + "subtests.subtest.test_functions.test_hmissing_imissing": { "status": "failed", "image_status": "missing", "hash_status": "generated", @@ -151,11 +268,11 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imissing/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { "status": "passed", "image_status": "match", "hash_status": "generated", @@ -168,20 +285,33 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", - "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result-failed-diff.png", "rms": 24.576566752362574, "tolerance": 3, - "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_special.test_hdiff_imatch_style": { "status": "passed", "image_status": "match", "hash_status": "generated", @@ -194,7 +324,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { "status": "passed", "image_status": "match", "hash_status": "generated", @@ -207,7 +337,72 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result-failed-diff.png", + "rms": 24.520257314106637, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { "status": "passed", "image_status": "match", "hash_status": "generated", diff --git a/tests/subtests/summaries/test_generate_images_only.json b/tests/subtests/summaries/test_generate_images_only.json index 553b5589..9e701649 100644 --- a/tests/subtests/summaries/test_generate_images_only.json +++ b/tests/subtests/summaries/test_generate_images_only.json @@ -1,5 +1,5 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -12,7 +12,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_idiff": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -25,7 +25,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_idiffshape": { + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -38,7 +38,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_imissing": { + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -51,7 +51,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -64,7 +64,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiff": { + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -77,7 +77,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiffshape": { + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -90,7 +90,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imissing": { + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -103,7 +103,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -116,7 +116,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_idiff": { + "subtests.subtest.test_functions.test_hmatch_imatch": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -129,7 +129,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_idiffshape": { + "subtests.subtest.test_functions.test_hmatch_idiff": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -142,7 +142,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_imissing": { + "subtests.subtest.test_functions.test_hmatch_idiffshape": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -155,7 +155,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_functions.test_hmatch_imissing": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -168,7 +168,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_functions.test_hdiff_imatch": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -181,7 +181,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_functions.test_hdiff_idiff": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -194,7 +194,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_functions.test_hdiff_idiffshape": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -207,7 +207,202 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_functions.test_hdiff_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_idiffshape": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_style": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { "status": "skipped", "image_status": "generated", "hash_status": null, diff --git a/tests/subtests/summaries/test_hash.json b/tests/subtests/summaries/test_hash.json index cdd18c90..f463019d 100644 --- a/tests/subtests/summaries/test_hash.json +++ b/tests/subtests/summaries/test_hash.json @@ -1,5 +1,5 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "passed", "image_status": null, "hash_status": "match", @@ -12,94 +12,289 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClass\\.test_hdiff_idiff_testclass\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imatch/result.png", + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupMethod\\.test_hdiff_idiff_testclasswithsetupmethod\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupClass\\.test_hdiff_idiff_testclasswithsetupclass\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_functions.test_hmatch_imatch": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_functions.test_hdiff_imatch": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_functions\\.test_hdiff_imatch\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_functions.test_hdiff_imatch/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_functions.test_hmissing_imatch": { "status": "failed", "image_status": null, "hash_status": "missing", - "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_functions\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_imatch/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imatch/result.png", "baseline_hash": null, "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_tolerance\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_idiff_tolerance\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_savefig\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_special.test_hdiff_imatch_style": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_style\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imatch_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_style/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_removetext\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCase\\.test_hdiff_idiff_testclass\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imatch_savefig/result.png", + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUp\\.test_hdiff_idiff_testcasewithsetup\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imatch_style/result.png", + "result_image": null, "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUpClass\\.test_hdiff_idiff_testcasewithsetupclass\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imatch_removetext/result.png", + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" } diff --git a/tests/subtests/summaries/test_html_generate.json b/tests/subtests/summaries/test_html_generate.json index d4366938..5d25715f 100644 --- a/tests/subtests/summaries/test_html_generate.json +++ b/tests/subtests/summaries/test_html_generate.json @@ -1,10 +1,10 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -12,12 +12,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_idiff": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -25,12 +25,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_idiffshape": { + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -38,12 +38,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_imissing": { + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmatch_imissing/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -51,12 +51,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -64,12 +64,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiff": { + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -77,12 +77,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiffshape": { + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -90,12 +90,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imissing": { + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imissing/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -103,12 +103,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -116,12 +116,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_idiff": { + "subtests.subtest.test_functions.test_hmatch_imatch": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -129,12 +129,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_idiffshape": { + "subtests.subtest.test_functions.test_hmatch_idiff": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiff/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -142,12 +142,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_imissing": { + "subtests.subtest.test_functions.test_hmatch_idiffshape": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmissing_imissing/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -155,12 +155,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_functions.test_hmatch_imissing": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_imissing/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -168,12 +168,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_functions.test_hdiff_imatch": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -181,12 +181,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_functions.test_hdiff_idiff": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiff/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -194,12 +194,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_functions.test_hdiff_idiffshape": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -207,12 +207,207 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_functions.test_hdiff_imissing": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_idiffshape": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_style": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_style/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/baseline.png", "diff_image": null, "rms": null, "tolerance": null, diff --git a/tests/subtests/summaries/test_html_generate_hashes_only.json b/tests/subtests/summaries/test_html_generate_hashes_only.json index eb455d56..3036ca89 100644 --- a/tests/subtests/summaries/test_html_generate_hashes_only.json +++ b/tests/subtests/summaries/test_html_generate_hashes_only.json @@ -1,44 +1,161 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "passed", "image_status": "match", "hash_status": "generated", "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hmatch_imatch/result.png", + "result_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_idiff": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result-failed-diff.png", + "rms": 24.563779228775037, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmatch_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_functions.test_hmatch_imatch/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmatch_idiff": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hmatch_idiff/result-failed-diff.png", "rms": 24.618234716477044, "tolerance": 2, - "result_image": "subtests.subtest.test_hmatch_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_idiff/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_idiffshape": { + "subtests.subtest.test_functions.test_hmatch_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmatch_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_imissing": { + "subtests.subtest.test_functions.test_hmatch_imissing": { "status": "failed", "image_status": "missing", "hash_status": "generated", @@ -47,50 +164,50 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmatch_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_imissing/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_functions.test_hdiff_imatch": { "status": "passed", "image_status": "match", "hash_status": "generated", "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_imatch/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiff": { + "subtests.subtest.test_functions.test_hdiff_idiff": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hdiff_idiff/result-failed-diff.png", "rms": 24.620185259705547, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiff/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiffshape": { + "subtests.subtest.test_functions.test_hdiff_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imissing": { + "subtests.subtest.test_functions.test_hdiff_imissing": { "status": "failed", "image_status": "missing", "hash_status": "generated", @@ -99,50 +216,50 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_imissing/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_functions.test_hmissing_imatch": { "status": "passed", "image_status": "match", "hash_status": "generated", "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hmissing_imatch/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imatch/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_idiff": { + "subtests.subtest.test_functions.test_hmissing_idiff": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hmissing_idiff/result-failed-diff.png", "rms": 24.6217358806762, "tolerance": 2, - "result_image": "subtests.subtest.test_hmissing_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiff/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_idiffshape": { + "subtests.subtest.test_functions.test_hmissing_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_imissing": { + "subtests.subtest.test_functions.test_hmissing_imissing": { "status": "failed", "image_status": "missing", "hash_status": "generated", @@ -151,72 +268,150 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imissing/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { "status": "passed", "image_status": "match", "hash_status": "generated", "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/baseline.png", "diff_image": null, "rms": null, "tolerance": 200, - "result_image": "subtests.subtest.test_hdiff_imatch_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", - "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result-failed-diff.png", "rms": 24.576566752362574, "tolerance": 3, - "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_style": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_style/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_style/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { "status": "passed", "image_status": "match", "hash_status": "generated", "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_savefig/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { "status": "passed", "image_status": "match", "hash_status": "generated", "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_style/result.png", + "result_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result-failed-diff.png", + "rms": 24.520257314106637, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { "status": "passed", "image_status": "match", "hash_status": "generated", "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_removetext/result.png", + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null } diff --git a/tests/subtests/summaries/test_html_generate_images_only.json b/tests/subtests/summaries/test_html_generate_images_only.json index 9d1947eb..0b1c3eb8 100644 --- a/tests/subtests/summaries/test_html_generate_images_only.json +++ b/tests/subtests/summaries/test_html_generate_images_only.json @@ -1,10 +1,10 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -12,12 +12,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_idiff": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -25,12 +25,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_idiffshape": { + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -38,12 +38,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_imissing": { + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmatch_imissing/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -51,12 +51,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -64,12 +64,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiff": { + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -77,12 +77,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiffshape": { + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -90,12 +90,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imissing": { + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imissing/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -103,12 +103,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -116,12 +116,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_idiff": { + "subtests.subtest.test_functions.test_hmatch_imatch": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -129,12 +129,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_idiffshape": { + "subtests.subtest.test_functions.test_hmatch_idiff": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiff/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -142,12 +142,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_imissing": { + "subtests.subtest.test_functions.test_hmatch_idiffshape": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmissing_imissing/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -155,12 +155,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_functions.test_hmatch_imissing": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_imissing/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -168,12 +168,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_functions.test_hdiff_imatch": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -181,12 +181,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_functions.test_hdiff_idiff": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiff/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -194,12 +194,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_functions.test_hdiff_idiffshape": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -207,12 +207,207 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_functions.test_hdiff_imissing": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_idiffshape": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_style": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_style/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/baseline.png", "diff_image": null, "rms": null, "tolerance": null, diff --git a/tests/subtests/summaries/test_html_hashes_only.json b/tests/subtests/summaries/test_html_hashes_only.json index 789795c0..c613126c 100644 --- a/tests/subtests/summaries/test_html_hashes_only.json +++ b/tests/subtests/summaries/test_html_hashes_only.json @@ -1,5 +1,5 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "passed", "image_status": null, "hash_status": "match", @@ -8,98 +8,293 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmatch_imatch/result.png", + "result_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClass\\.test_hdiff_idiff_testclass\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imatch/result.png", + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupMethod\\.test_hdiff_idiff_testclasswithsetupmethod\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupClass\\.test_hdiff_idiff_testclasswithsetupclass\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_functions.test_hmatch_imatch": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_functions.test_hmatch_imatch/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_functions.test_hdiff_imatch": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_functions\\.test_hdiff_imatch\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_functions.test_hdiff_imatch/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_functions.test_hmissing_imatch": { "status": "failed", "image_status": null, "hash_status": "missing", - "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_functions\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_imatch/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imatch/result.png", "baseline_hash": null, "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_tolerance\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_idiff_tolerance\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imatch_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_savefig\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_special.test_hdiff_imatch_style": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_style\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imatch_savefig/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_style/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_removetext\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imatch_style/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCase\\.test_hdiff_idiff_testclass\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUp\\.test_hdiff_idiff_testcasewithsetup\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUpClass\\.test_hdiff_idiff_testcasewithsetupclass\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imatch_removetext/result.png", + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" } diff --git a/tests/subtests/summaries/test_html_images_only.json b/tests/subtests/summaries/test_html_images_only.json index fa10a4fb..04534f58 100644 --- a/tests/subtests/summaries/test_html_images_only.json +++ b/tests/subtests/summaries/test_html_images_only.json @@ -1,44 +1,161 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "passed", "image_status": "match", "hash_status": null, "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hmatch_imatch/result.png", + "result_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_idiff": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result-failed-diff.png", + "rms": 24.563779228775037, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmatch_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_functions.test_hmatch_imatch/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmatch_idiff": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hmatch_idiff/result-failed-diff.png", "rms": 24.618234716477044, "tolerance": 2, - "result_image": "subtests.subtest.test_hmatch_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_idiff/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_idiffshape": { + "subtests.subtest.test_functions.test_hmatch_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmatch_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_imissing": { + "subtests.subtest.test_functions.test_hmatch_imissing": { "status": "failed", "image_status": "missing", "hash_status": null, @@ -47,50 +164,50 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmatch_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_imissing/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_functions.test_hdiff_imatch": { "status": "passed", "image_status": "match", "hash_status": null, "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_imatch/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiff": { + "subtests.subtest.test_functions.test_hdiff_idiff": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hdiff_idiff/result-failed-diff.png", "rms": 24.620185259705547, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiff/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiffshape": { + "subtests.subtest.test_functions.test_hdiff_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imissing": { + "subtests.subtest.test_functions.test_hdiff_imissing": { "status": "failed", "image_status": "missing", "hash_status": null, @@ -99,50 +216,50 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_imissing/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_functions.test_hmissing_imatch": { "status": "passed", "image_status": "match", "hash_status": null, "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hmissing_imatch/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imatch/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_idiff": { + "subtests.subtest.test_functions.test_hmissing_idiff": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hmissing_idiff/result-failed-diff.png", "rms": 24.6217358806762, "tolerance": 2, - "result_image": "subtests.subtest.test_hmissing_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiff/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_idiffshape": { + "subtests.subtest.test_functions.test_hmissing_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_imissing": { + "subtests.subtest.test_functions.test_hmissing_imissing": { "status": "failed", "image_status": "missing", "hash_status": null, @@ -151,72 +268,150 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imissing/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { "status": "passed", "image_status": "match", "hash_status": null, "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/baseline.png", "diff_image": null, "rms": null, "tolerance": 200, - "result_image": "subtests.subtest.test_hdiff_imatch_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", - "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result-failed-diff.png", "rms": 24.576566752362574, "tolerance": 3, - "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_style": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_style/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_style/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { "status": "passed", "image_status": "match", "hash_status": null, "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_savefig/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { "status": "passed", "image_status": "match", "hash_status": null, "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_style/result.png", + "result_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result-failed-diff.png", + "rms": 24.520257314106637, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { "status": "passed", "image_status": "match", "hash_status": null, "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_removetext/result.png", + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/result.png", "baseline_hash": null, "result_hash": null } diff --git a/tests/subtests/summaries/test_hybrid.json b/tests/subtests/summaries/test_hybrid.json index 24bfd2fa..62aa6269 100644 --- a/tests/subtests/summaries/test_hybrid.json +++ b/tests/subtests/summaries/test_hybrid.json @@ -1,5 +1,5 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "passed", "image_status": null, "hash_status": "match", @@ -12,7 +12,111 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmatch_idiff": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClass\\.test_hdiff_idiff_testclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupMethod\\.test_hdiff_idiff_testclasswithsetupmethod\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupClass\\.test_hdiff_idiff_testclasswithsetupclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result-failed-diff.png", + "rms": 24.563779228775037, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_functions.test_hmatch_imatch": { "status": "passed", "image_status": null, "hash_status": "match", @@ -25,7 +129,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmatch_idiffshape": { + "subtests.subtest.test_functions.test_hmatch_idiff": { "status": "passed", "image_status": null, "hash_status": "match", @@ -38,7 +142,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmatch_imissing": { + "subtests.subtest.test_functions.test_hmatch_idiffshape": { "status": "passed", "image_status": null, "hash_status": "match", @@ -51,172 +155,263 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_functions.test_hmatch_imissing": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_functions.test_hdiff_imatch": { "status": "failed", "image_status": "match", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_functions\\.test_hdiff_imatch\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_imatch/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_idiff": { + "subtests.subtest.test_functions.test_hdiff_idiff": { "status": "failed", "image_status": "diff", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_functions\\.test_hdiff_idiff\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hdiff_idiff/result-failed-diff.png", "rms": 24.620185259705547, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiff/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_idiffshape": { + "subtests.subtest.test_functions.test_hdiff_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiffshape\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_functions\\.test_hdiff_idiffshape\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imissing": { + "subtests.subtest.test_functions.test_hdiff_imissing": { "status": "failed", "image_status": "missing", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_functions\\.test_hdiff_imissing\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_imissing/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_functions.test_hmissing_imatch": { "status": "failed", "image_status": "match", "hash_status": "missing", - "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_functions\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hmissing_imatch/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imatch/result.png", "baseline_hash": null, "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmissing_idiff": { + "subtests.subtest.test_functions.test_hmissing_idiff": { "status": "failed", "image_status": "diff", "hash_status": "missing", - "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_functions\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hmissing_idiff/result-failed-diff.png", "rms": 24.6217358806762, "tolerance": 2, - "result_image": "subtests.subtest.test_hmissing_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiff/result.png", "baseline_hash": null, "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmissing_idiffshape": { + "subtests.subtest.test_functions.test_hmissing_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": "missing", - "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiffshape' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_functions\\.test_hmissing_idiffshape' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/result.png", "baseline_hash": null, "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmissing_imissing": { + "subtests.subtest.test_functions.test_hmissing_imissing": { "status": "failed", "image_status": "missing", "hash_status": "missing", - "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_functions\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imissing/result.png", "baseline_hash": null, "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { "status": "failed", "image_status": "match", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_tolerance\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/baseline.png", "diff_image": null, "rms": null, "tolerance": 200, - "result_image": "subtests.subtest.test_hdiff_imatch_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { "status": "failed", "image_status": "diff", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", - "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_idiff_tolerance\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", + "baseline_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result-failed-diff.png", "rms": 24.576566752362574, "tolerance": 3, - "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { "status": "failed", "image_status": "match", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_savefig\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_savefig/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_special.test_hdiff_imatch_style": { "status": "failed", "image_status": "match", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_style\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_style/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_style/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_style/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { "status": "failed", "image_status": "match", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_removetext\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_removetext/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCase\\.test_hdiff_idiff_testclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUp\\.test_hdiff_idiff_testcasewithsetup\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUpClass\\.test_hdiff_idiff_testcasewithsetupclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result-failed-diff.png", + "rms": 24.520257314106637, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" } diff --git a/tests/subtests/summaries/test_results_always.json b/tests/subtests/summaries/test_results_always.json index 4e8f4250..e59d399d 100644 --- a/tests/subtests/summaries/test_results_always.json +++ b/tests/subtests/summaries/test_results_always.json @@ -1,44 +1,161 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "passed", "image_status": "match", "hash_status": "match", "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", - "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hmatch_imatch/result.png", + "result_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmatch_idiff": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClass\\.test_hdiff_idiff_testclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupMethod\\.test_hdiff_idiff_testclasswithsetupmethod\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupClass\\.test_hdiff_idiff_testclasswithsetupclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result-failed-diff.png", + "rms": 24.563779228775037, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_functions.test_hmatch_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_functions.test_hmatch_imatch/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_functions.test_hmatch_idiff": { "status": "passed", "image_status": "diff", "hash_status": "match", "status_msg": "REGEX:Test hash matches baseline hash\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hmatch_idiff/result-failed-diff.png", "rms": 24.618234716477044, "tolerance": 2, - "result_image": "subtests.subtest.test_hmatch_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_idiff/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmatch_idiffshape": { + "subtests.subtest.test_functions.test_hmatch_idiffshape": { "status": "passed", "image_status": "diff", "hash_status": "match", "status_msg": "REGEX:Test hash matches baseline hash\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmatch_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmatch_imissing": { + "subtests.subtest.test_functions.test_hmatch_imissing": { "status": "passed", "image_status": "missing", "hash_status": "match", @@ -47,176 +164,254 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmatch_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_imissing/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_functions.test_hdiff_imatch": { "status": "failed", "image_status": "match", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_functions\\.test_hdiff_imatch\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_imatch/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_idiff": { + "subtests.subtest.test_functions.test_hdiff_idiff": { "status": "failed", "image_status": "diff", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_functions\\.test_hdiff_idiff\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hdiff_idiff/result-failed-diff.png", "rms": 24.620185259705547, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiff/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_idiffshape": { + "subtests.subtest.test_functions.test_hdiff_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiffshape\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_functions\\.test_hdiff_idiffshape\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imissing": { + "subtests.subtest.test_functions.test_hdiff_imissing": { "status": "failed", "image_status": "missing", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_functions\\.test_hdiff_imissing\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_imissing/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_functions.test_hmissing_imatch": { "status": "failed", "image_status": "match", "hash_status": "missing", - "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_functions\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hmissing_imatch/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imatch/result.png", "baseline_hash": null, "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmissing_idiff": { + "subtests.subtest.test_functions.test_hmissing_idiff": { "status": "failed", "image_status": "diff", "hash_status": "missing", - "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_functions\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hmissing_idiff/result-failed-diff.png", "rms": 24.6217358806762, "tolerance": 2, - "result_image": "subtests.subtest.test_hmissing_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiff/result.png", "baseline_hash": null, "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmissing_idiffshape": { + "subtests.subtest.test_functions.test_hmissing_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": "missing", - "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiffshape' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_functions\\.test_hmissing_idiffshape' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/result.png", "baseline_hash": null, "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmissing_imissing": { + "subtests.subtest.test_functions.test_hmissing_imissing": { "status": "failed", "image_status": "missing", "hash_status": "missing", - "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_functions\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imissing/result.png", "baseline_hash": null, "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { "status": "failed", "image_status": "match", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_tolerance\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/baseline.png", "diff_image": null, "rms": null, "tolerance": 200, - "result_image": "subtests.subtest.test_hdiff_imatch_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { "status": "failed", "image_status": "diff", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", - "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_idiff_tolerance\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", + "baseline_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result-failed-diff.png", "rms": 24.576566752362574, "tolerance": 3, - "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { "status": "failed", "image_status": "match", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_savefig\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_savefig/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_special.test_hdiff_imatch_style": { "status": "failed", "image_status": "match", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_style\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_style/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_style/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { + "status": "failed", + "image_status": "match", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_removetext\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCase\\.test_hdiff_idiff_testclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_style/result.png", + "result_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUp\\.test_hdiff_idiff_testcasewithsetup\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "failed", + "image_status": "diff", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUpClass\\.test_hdiff_idiff_testcasewithsetupclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result-failed-diff.png", + "rms": 24.520257314106637, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_removetext/result.png", + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" } diff --git a/tests/subtests/test_subtest.py b/tests/subtests/test_subtest.py index 4993fa9f..6bf4c8d1 100644 --- a/tests/subtests/test_subtest.py +++ b/tests/subtests/test_subtest.py @@ -6,7 +6,6 @@ import subprocess from pathlib import Path -import matplotlib import matplotlib.ft2font import pytest from packaging.version import Version @@ -18,10 +17,10 @@ MPL_VERSION = Version(matplotlib.__version__) FTV = matplotlib.ft2font.__freetype_version__.replace('.', '') VERSION_ID = f"mpl{MPL_VERSION.major}{MPL_VERSION.minor}_ft{FTV}" -HASH_LIBRARY = Path(__file__).parent / 'hashes' / (VERSION_ID + ".json") +HASH_LIBRARY = Path(__file__).parent / 'subtest' / 'hashes' / (VERSION_ID + ".json") RESULT_LIBRARY = Path(__file__).parent / 'result_hashes' / (VERSION_ID + ".json") HASH_LIBRARY_FLAG = rf'--mpl-hash-library={HASH_LIBRARY}' -FULL_BASELINE_PATH = Path(__file__).parent / 'baseline' +FULL_BASELINE_PATH = Path(__file__).parent / 'subtest' / 'baseline' BASELINE_IMAGES_FLAG_REL = ['--mpl-baseline-path=baseline', '--mpl-baseline-relative'] BASELINE_IMAGES_FLAG_ABS = rf'--mpl-baseline-path={FULL_BASELINE_PATH}' @@ -30,7 +29,7 @@ HASH_COMPARISON_MODE = ["-k", "hash"] # HYBRID_MODE = [] -TEST_FILE = Path(__file__).parent / 'subtest.py' +TEST_FILE = Path(__file__).parent / 'subtest' # Global settings to update baselines when running pytest # Note: when updating baseline make sure you don't commit "fixes" diff --git a/tests/test_baseline_path.py b/tests/test_baseline_path.py new file mode 100644 index 00000000..745006a5 --- /dev/null +++ b/tests/test_baseline_path.py @@ -0,0 +1,50 @@ +import shutil +from pathlib import Path + +import pytest +from helpers import pytester_path + + +@pytest.mark.parametrize( + "ini, cli, kwarg, expected_baseline_path, success_expected", + [ + ("dir1", None, None, "dir1", True), + ("dir1", "dir2", None, "dir2", True), + ("dir1", "dir2", "dir3", "dir3", True), + ("dir1", "dir2", "dir3", "dir2", False), + (None, None, "dir3", "dir3", True), + ], +) +def test_config(pytester, ini, cli, kwarg, expected_baseline_path, success_expected): + path = pytester_path(pytester) + (path / expected_baseline_path).mkdir() + shutil.copyfile( # Test will only pass if baseline is at expected path + Path(__file__).parent / "baseline" / "2.0.x" / "test_base_style.png", + path / expected_baseline_path / "test_mpl.png", + ) + ini = f"mpl-baseline-path = {path / ini}" if ini is not None else "" + pytester.makeini( + f""" + [pytest] + mpl-default-style = fivethirtyeight + {ini} + """ + ) + kwarg = f"baseline_dir=r'{path / kwarg}'" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = f"--mpl-baseline-path={path / cli}" if cli else "" + result = pytester.runpytest("--mpl", cli) + if success_expected: + result.assert_outcomes(passed=1) + else: + result.assert_outcomes(failed=1) diff --git a/tests/test_default_backend.py b/tests/test_default_backend.py new file mode 100644 index 00000000..192d3cdd --- /dev/null +++ b/tests/test_default_backend.py @@ -0,0 +1,35 @@ +import pytest + + +@pytest.mark.parametrize( + "ini, cli, kwarg, expected", + [ + ("backend1", None, None, "backend1"), + ("backend1", "backend2", None, "backend2"), + ("backend1", "backend2", "backend3", "backend3"), + ], +) +def test_config(pytester, ini, cli, kwarg, expected): + ini = f"mpl-default-backend = {ini}" if ini else "" + pytester.makeini( + f""" + [pytest] + {ini} + """ + ) + kwarg = f"backend='{kwarg}'" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = f"--mpl-default-backend={cli}" if cli else "" + result = pytester.runpytest("--mpl", cli) + result.assert_outcomes(failed=1) + result.stdout.re_match_lines([f".*(ModuleNotFound|Value)Error: .*{expected}.*"]) diff --git a/tests/test_default_style.py b/tests/test_default_style.py new file mode 100644 index 00000000..90880085 --- /dev/null +++ b/tests/test_default_style.py @@ -0,0 +1,38 @@ +import pytest +from helpers import pytester_path + + +@pytest.mark.parametrize( + "ini, cli, kwarg, expected", + [ + ("sty1", None, None, "sty1"), + ("sty1", "sty2", None, "sty2"), + ("sty1", "sty2", "sty3", "sty3"), + ], +) +def test_config(pytester, ini, cli, kwarg, expected): + ini = "mpl-default-style = " + ini if ini else "" + pytester.makeini( + f""" + [pytest] + mpl-baseline-path = {pytester_path(pytester)} + {ini} + """ + ) + kwarg = f"style='{kwarg}'" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + + cli = "--mpl-default-style=" + cli if cli else "" + result = pytester.runpytest("--mpl", cli) + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines([f"*OSError: *'{expected}'*"]) diff --git a/tests/test_default_tolerance.py b/tests/test_default_tolerance.py new file mode 100644 index 00000000..e9b154e5 --- /dev/null +++ b/tests/test_default_tolerance.py @@ -0,0 +1,57 @@ +from pathlib import Path + +import pytest +from PIL import Image, ImageDraw + +TEST_NAME = "test_base_style" + + +@pytest.fixture(scope="module") +def baseline_image(tmp_path_factory): + path = Path(__file__).parent / "baseline" / "2.0.x" / f"{TEST_NAME}.png" + with Image.open(path) as image: + draw = ImageDraw.Draw(image) + draw.rectangle(((0, 0), (100, 100)), fill="red") + output = tmp_path_factory.mktemp("data") / f"{TEST_NAME}.png" + image.save(output) + return output + + +@pytest.mark.parametrize( + "ini, cli, kwarg, success_expected", + [ + (40, None, None, True), + (30, 40, None, True), + (30, 30, 40, True), + (30, 40, 30, False), + (40, 30, 30, False), + ], +) +def test_config(pytester, baseline_image, ini, cli, kwarg, success_expected): + ini = f"mpl-default-tolerance = {ini}" if ini else "" + pytester.makeini( + f""" + [pytest] + mpl-default-style = fivethirtyeight + mpl-baseline-path = {baseline_image.parent} + {ini} + """ + ) + kwarg = f"tolerance={kwarg}" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def {TEST_NAME}(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = f"--mpl-default-tolerance={cli}" if cli else "" + result = pytester.runpytest("--mpl", cli) + if success_expected: + result.assert_outcomes(passed=1) + else: + result.assert_outcomes(failed=1) diff --git a/tests/test_deterministic.py b/tests/test_deterministic.py new file mode 100644 index 00000000..edd26b83 --- /dev/null +++ b/tests/test_deterministic.py @@ -0,0 +1,129 @@ +import matplotlib +import matplotlib.pyplot as plt +import pytest +from helpers import pytester_path, skip_if_format_unsupported +from packaging.version import Version +from PIL import Image + +MPL_VERSION = Version(matplotlib.__version__) + +METADATA = { + "png": {"Software": None}, + "pdf": {"Creator": None, "Producer": None, "CreationDate": None}, + "eps": {"Creator": "test"}, + "svg": {"Date": None}, +} + + +def test_multiple_cli_flags(pytester): + result = pytester.runpytest("--mpl", "--mpl-deterministic", "--mpl-no-deterministic") + result.stderr.fnmatch_lines( + ["*ValueError: Only one of `--mpl-deterministic` and `--mpl-no-deterministic` can be set.*"] + ) + + +def test_warning(pytester): + path = pytester_path(pytester) + hash_library = path / "hash_library.json" + kwarg = f"hash_library=r'{hash_library}'" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 3, 2]) + return fig + """ + ) + result = pytester.runpytest(f"--mpl-generate-hash-library={hash_library}") + result.stdout.fnmatch_lines(["*FutureWarning: deterministic option not set*"]) + result.assert_outcomes(failed=1) + + +@pytest.mark.parametrize("file_format", ["eps", "pdf", "png", "svg"]) +@pytest.mark.parametrize( + "ini, cli, kwarg, success_expected", + [ + ("true", "", None, True), + ("false", "--mpl-deterministic", None, True), + ("true", "--mpl-no-deterministic", None, False), + ("", "--mpl-no-deterministic", True, True), + ("true", "", False, False), + ], +) +@pytest.mark.skipif(MPL_VERSION < Version("3.3.0"), reason="Test unsupported: Default metadata is different in MPL<3.3") +def test_config(pytester, file_format, ini, cli, kwarg, success_expected): + skip_if_format_unsupported(file_format, using_hashes=True) + + path = pytester_path(pytester) + baseline_dir = path / "baseline" + hash_library = path / "hash_library.json" + + ini = f"mpl-deterministic = {ini}" if ini else "" + pytester.makeini( + f""" + [pytest] + mpl-hash-library = {hash_library} + {ini} + """ + ) + + kwarg = f", deterministic={kwarg}" if isinstance(kwarg, bool) else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare(savefig_kwargs={{'format': '{file_format}'}}{kwarg}) + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + + # Generate baseline hashes + assert not hash_library.exists() + pytester.runpytest( + f"--mpl-generate-path={baseline_dir}", + f"--mpl-generate-hash-library={hash_library}", + cli, + ) + assert hash_library.exists() + baseline_image = baseline_dir / f"test_mpl.{file_format}" + assert baseline_image.exists() + deterministic_metadata = METADATA[file_format] + + if file_format == "svg": # The only format that is reliably non-deterministic between runs + result = pytester.runpytest("--mpl", f"--mpl-baseline-path={baseline_dir}", cli) + if success_expected: + result.assert_outcomes(passed=1) + else: + result.assert_outcomes(failed=1) + + elif file_format == "pdf": + with open(baseline_image, "rb") as fp: + file = str(fp.read()) + for metadata_key in deterministic_metadata.keys(): + key_in_file = fr"/{metadata_key}" in file + if success_expected: # metadata keys should not be in the file + assert not key_in_file + else: + assert key_in_file + + else: # "eps" or "png" + with Image.open(str(baseline_image)) as image: + actual_metadata = image.info + for k, expected in deterministic_metadata.items(): + actual = actual_metadata.get(k, None) + if success_expected: # metadata keys should not be in the file + if expected is None: + assert actual is None + else: + assert actual == expected + else: # metadata keys should still be in the file + if expected is None: + assert actual is not None + else: + assert actual != expected diff --git a/tests/test_generate.py b/tests/test_generate.py new file mode 100644 index 00000000..61b14471 --- /dev/null +++ b/tests/test_generate.py @@ -0,0 +1,48 @@ +from helpers import pytester_path + +PYFILE = ( + """ + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ +) + + +def test_generate_baseline_images(pytester): + pytester.makepyfile(PYFILE) + baseline_dir = pytester_path(pytester) / "alternative_baseline" + result = pytester.runpytest(f"--mpl-generate-path={baseline_dir}") + result.assert_outcomes(skipped=1) + assert (baseline_dir / "test_mpl.png").exists() + + +def test_generate_baseline_hashes(pytester): + pytester.makepyfile(PYFILE) + path = pytester_path(pytester) + hash_library = path / "alternative_baseline" / "hash_library_1.json" + result = pytester.runpytest( + f"--mpl-generate-hash-library={hash_library}", + f"--mpl-results-path={path}", + ) + result.assert_outcomes(failed=1) # this option enables --mpl + assert hash_library.exists() + assert (path / "test_generate_baseline_hashes.test_mpl" / "result.png").exists() + + +def test_generate_baseline_images_and_hashes(pytester): + pytester.makepyfile(PYFILE) + path = pytester_path(pytester) + baseline_dir = path / "alternative_baseline" + hash_library = path / "alternative_baseline" / "hash_library_1.json" + result = pytester.runpytest( + f"--mpl-generate-path={baseline_dir}", + f"--mpl-generate-hash-library={hash_library}", + ) + result.assert_outcomes(skipped=1) + assert (baseline_dir / "test_mpl.png").exists() + assert hash_library.exists() diff --git a/tests/test_generate_summary.py b/tests/test_generate_summary.py new file mode 100644 index 00000000..6350975f --- /dev/null +++ b/tests/test_generate_summary.py @@ -0,0 +1,65 @@ +import json + +import pytest +from helpers import pytester_path + + +@pytest.mark.parametrize( + "ini, cli, expected", + [ + ("json", None, {"json"}), + ("json", "html", {"html"}), + ("basic-html", "json", {"json"}), + (None, "json,basic-html,html", {"json", "basic-html", "html"}), + ], +) +def test_config(pytester, ini, cli, expected): + path = pytester_path(pytester) + ini = f"mpl-generate-summary = {ini}" if ini else "" + pytester.makeini( + f""" + [pytest] + mpl-results-path = {path} + {ini} + """ + ) + pytester.makepyfile( + """ + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = f"--mpl-generate-summary={cli}" if cli else "" + result = pytester.runpytest("--mpl", cli) + result.assert_outcomes(failed=1) + + json_summary = path / "results.json" + if "json" in expected: + with open(json_summary) as fp: + results = json.load(fp) + assert "test_config.test_mpl" in results + else: + assert not json_summary.exists() + + html_summary = path / "fig_comparison.html" + if "html" in expected: + with open(html_summary) as fp: + raw = fp.read() + assert "bootstrap" in raw + assert "test_config.test_mpl" in raw + else: + assert not html_summary.exists() + + basic_html_summary = path / "fig_comparison_basic.html" + if "basic-html" in expected: + with open(basic_html_summary) as fp: + raw = fp.read() + assert "bootstrap" not in raw + assert "test_config.test_mpl" in raw + else: + assert not basic_html_summary.exists() diff --git a/tests/test_hash_library.py b/tests/test_hash_library.py new file mode 100644 index 00000000..0afa0544 --- /dev/null +++ b/tests/test_hash_library.py @@ -0,0 +1,51 @@ +import json + +import pytest +from helpers import pytester_path + + +@pytest.mark.parametrize( + "ini, cli, kwarg, success_expected", + [ + ("bad", None, None, False), + ("good", None, None, True), + ("bad", "good", None, True), + ("bad", "bad", "good", False), # Note: CLI overrides kwarg + ("bad", "good", "bad", True), + ], +) +def test_config(pytester, ini, cli, kwarg, success_expected): + path = pytester_path(pytester) + hash_libraries = { + "good": path / "good_hash_library.json", + "bad": path / "bad_hash_library.json", + } + ini = f"mpl-hash-library = {hash_libraries[ini]}" if ini else "" + pytester.makeini( + f""" + [pytest] + mpl-deterministic: true + {ini} + """ + ) + kwarg = f"hash_library=r'{hash_libraries[kwarg]}'" if kwarg else "" + pytester.makepyfile( + f""" + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare({kwarg}) + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 3, 2]) + return fig + """ + ) + pytester.runpytest(f"--mpl-generate-hash-library={hash_libraries['good']}") + with open(hash_libraries["bad"], "w") as fp: + json.dump({"test_config.test_mpl": "bad-value"}, fp) + cli = f"--mpl-hash-library={hash_libraries[cli]}" if cli else "" + result = pytester.runpytest("--mpl", cli) + if success_expected: + result.assert_outcomes(passed=1) + else: + result.assert_outcomes(failed=1) diff --git a/tests/test_pytest_mpl.py b/tests/test_pytest_mpl.py index a9d2cba1..e04cec93 100644 --- a/tests/test_pytest_mpl.py +++ b/tests/test_pytest_mpl.py @@ -9,6 +9,7 @@ import matplotlib.ft2font import matplotlib.pyplot as plt import pytest +from helpers import skip_if_format_unsupported from packaging.version import Version MPL_VERSION = Version(matplotlib.__version__) @@ -119,11 +120,10 @@ def test_fail(): """ -def test_fails(tmpdir): - - test_file = tmpdir.join('test.py').strpath - with open(test_file, 'w') as f: - f.write(TEST_FAILING) +def test_fails(tmp_path): + test_file = tmp_path / "test.py" + test_file.write_text(TEST_FAILING) + test_file = str(test_file) # If we use --mpl, it should detect that the figure is wrong code = call_pytest(['--mpl', test_file]) @@ -146,16 +146,15 @@ def test_output_dir(): """ -def test_output_dir(tmpdir): - test_file = tmpdir.join('test.py').strpath - with open(test_file, 'w') as f: - f.write(TEST_OUTPUT_DIR) +def test_output_dir(tmp_path): + test_file = tmp_path / "test.py" + test_file.write_text(TEST_OUTPUT_DIR) - output_dir = tmpdir.join('test_output_dir') + output_dir = tmp_path / "test_output_dir" # When we run the test, we should get output images where we specify code = call_pytest([f'--mpl-results-path={output_dir}', - '--mpl', test_file]) + '--mpl', str(test_file)]) assert code != 0 assert output_dir.exists() @@ -174,13 +173,14 @@ def test_gen(): """ -def test_generate(tmpdir): +def test_generate(tmp_path): - test_file = tmpdir.join('test.py').strpath - with open(test_file, 'w') as f: - f.write(TEST_GENERATE) + test_file = tmp_path / "test.py" + test_file.write_text(TEST_GENERATE) + test_file = str(test_file) - gen_dir = tmpdir.mkdir('spam').mkdir('egg').strpath + gen_dir = tmp_path / "spam" / "egg" + gen_dir.mkdir(parents=True) # If we don't generate, the test will fail assert_pytest_fails_with(['--mpl', test_file], 'Image file not found for comparison test') @@ -280,7 +280,7 @@ def test_succeeds(self): # hashlib @pytest.mark.skipif(not hash_library.exists(), reason="No hash library for this mpl version") -@pytest.mark.mpl_image_compare(hash_library=hash_library) +@pytest.mark.mpl_image_compare(hash_library=hash_library, deterministic=True) def test_hash_succeeds(): fig = plt.figure() ax = fig.add_subplot(1, 1, 1) @@ -291,7 +291,7 @@ def test_hash_succeeds(): TEST_FAILING_HASH = rf""" import pytest import matplotlib.pyplot as plt -@pytest.mark.mpl_image_compare(hash_library=r"{fail_hash_library}") +@pytest.mark.mpl_image_compare(hash_library=r"{fail_hash_library}", deterministic=True) def test_hash_fails(): fig = plt.figure() ax = fig.add_subplot(1,1,1) @@ -300,11 +300,10 @@ def test_hash_fails(): """ -def test_hash_fails(tmpdir): - - test_file = tmpdir.join('test.py').strpath - with open(test_file, 'w', encoding='ascii') as f: - f.write(TEST_FAILING_HASH) +def test_hash_fails(tmp_path): + test_file = tmp_path / "test.py" + test_file.write_text(TEST_FAILING_HASH, encoding="ascii") + test_file = str(test_file) # If we use --mpl, it should detect that the figure is wrong output = assert_pytest_fails_with(['--mpl', test_file], "doesn't match hash FAIL in library") @@ -329,7 +328,7 @@ def test_hash_fails(tmpdir): import pytest import matplotlib.pyplot as plt @pytest.mark.mpl_image_compare(hash_library=r"{fail_hash_library}", - tolerance=2) + tolerance=2, deterministic=True) def test_hash_fail_hybrid(): fig = plt.figure() ax = fig.add_subplot(1,1,1) @@ -339,11 +338,11 @@ def test_hash_fail_hybrid(): @pytest.mark.skipif(ftv != '261', reason="Incorrect freetype version for hash check") -def test_hash_fail_hybrid(tmpdir): +def test_hash_fail_hybrid(tmp_path): - test_file = tmpdir.join('test.py').strpath - with open(test_file, 'w', encoding='ascii') as f: - f.write(TEST_FAILING_HYBRID) + test_file = tmp_path / "test.py" + test_file.write_text(TEST_FAILING_HYBRID, encoding="ascii") + test_file = str(test_file) # Assert that image comparison runs and fails output = assert_pytest_fails_with(['--mpl', test_file, @@ -381,18 +380,18 @@ def test_hash_fails(): @pytest.mark.skipif(ftv != '261', reason="Incorrect freetype version for hash check") -def test_hash_fail_new_hashes(tmpdir): +def test_hash_fail_new_hashes(tmp_path): # Check that the hash comparison fails even if a new hash file is requested - test_file = tmpdir.join('test.py').strpath - with open(test_file, 'w', encoding='ascii') as f: - f.write(TEST_FAILING_NEW_HASH) + test_file = tmp_path / "test.py" + test_file.write_text(TEST_FAILING_NEW_HASH, encoding="ascii") + test_file = str(test_file) # Assert that image comparison runs and fails assert_pytest_fails_with(['--mpl', test_file, f'--mpl-hash-library={fail_hash_library}'], "doesn't match hash FAIL in library") - hash_file = tmpdir.join('new_hashes.json').strpath + hash_file = tmp_path / "new_hashes.json" # Assert that image comparison runs and fails assert_pytest_fails_with(['--mpl', test_file, f'--mpl-hash-library={fail_hash_library}', @@ -412,11 +411,10 @@ def test_hash_missing(): """ -def test_hash_missing(tmpdir): - - test_file = tmpdir.join('test.py').strpath - with open(test_file, 'w') as f: - f.write(TEST_MISSING_HASH) +def test_hash_missing(tmp_path): + test_file = tmp_path / "test.py" + test_file.write_text(TEST_MISSING_HASH) + test_file = str(test_file) # Assert fails if hash library missing assert_pytest_fails_with(['--mpl', test_file, '--mpl-hash-library=/not/a/path'], @@ -439,41 +437,36 @@ def plot(): ax = fig.add_subplot(1,1,1) ax.plot([1,2,2]) return fig -@pytest.mark.mpl_image_compare +@pytest.mark.mpl_image_compare(deterministic=True) def test_modified(): return plot() -@pytest.mark.mpl_image_compare +@pytest.mark.mpl_image_compare(deterministic=True) def test_new(): return plot() -@pytest.mark.mpl_image_compare +@pytest.mark.mpl_image_compare(deterministic=True) def test_unmodified(): return plot() """ @pytest.mark.skipif(not hash_library.exists(), reason="No hash library for this mpl version") -def test_results_always(tmpdir): - - test_file = tmpdir.join('test.py').strpath - with open(test_file, 'w') as f: - f.write(TEST_RESULTS_ALWAYS) - results_path = tmpdir.mkdir('results') +def test_results_always(tmp_path): + test_file = tmp_path / "test.py" + test_file.write_text(TEST_RESULTS_ALWAYS) + results_path = tmp_path / "results" + results_path.mkdir() - code = call_pytest(['--mpl', test_file, '--mpl-results-always', + code = call_pytest(['--mpl', str(test_file), '--mpl-results-always', rf'--mpl-hash-library={hash_library}', rf'--mpl-baseline-path={baseline_dir_abs}', '--mpl-generate-summary=html,json,basic-html', - rf'--mpl-results-path={results_path.strpath}']) + rf'--mpl-results-path={results_path}']) assert code == 0 # hashes correct, so all should pass # assert files for interactive HTML exist - assert results_path.join('fig_comparison.html').exists() - assert results_path.join('styles.css').exists() - assert results_path.join('extra.js').exists() + assert (results_path / "fig_comparison.html").exists() + assert (results_path / "styles.css").exists() + assert (results_path / "extra.js").exists() - comparison_file = results_path.join('fig_comparison_basic.html') - with open(comparison_file, 'r') as f: - html = f.read() - - json_file = results_path.join('results.json') - with open(json_file, 'r') as f: + html = (results_path / "fig_comparison_basic.html").read_text() + with (results_path / "results.json").open("r") as f: json_results = json.load(f) # each test, and which images should exist @@ -494,7 +487,7 @@ def test_results_always(tmpdir): for image_type in ['baseline', 'result-failed-diff', 'result']: image = f'{test_name}/{image_type}.png' - image_exists = results_path.join(*image.split('/')).exists() + image_exists = (results_path / image).exists() json_image_key = f"{image_type.split('-')[-1]}_image" if image_type in exists: # assert image so pytest prints it on error assert image and image_exists @@ -553,11 +546,11 @@ def test_fails(self): TEST_FAILING_CLASS_SETUP_METHOD, TEST_FAILING_UNITTEST_TESTCASE, ]) -def test_class_fail(code, tmpdir): +def test_class_fail(code, tmp_path): - test_file = tmpdir.join('test.py').strpath - with open(test_file, 'w') as f: - f.write(code) + test_file = tmp_path / "test.py" + test_file.write_text(code) + test_file = str(test_file) # Assert fails if hash library missing assert_pytest_fails_with(['--mpl', test_file, '--mpl-hash-library=/not/a/path'], @@ -670,3 +663,39 @@ def test_raises(): result = pytester.runpytest(*runpytest_args) result.assert_outcomes(failed=1) result.stdout.fnmatch_lines("FAILED*ValueError*User code*") + + +@pytest.mark.parametrize('use_hash_library', (False, True)) +@pytest.mark.parametrize('passes', (False, True)) +@pytest.mark.parametrize("file_format", ['eps', 'pdf', 'png', 'svg']) +def test_formats(pytester, use_hash_library, passes, file_format): + """ + Note that we don't test all possible formats as some do not compress well + and would bloat the baseline directory. + """ + skip_if_format_unsupported(file_format, using_hashes=use_hash_library) + if use_hash_library and not hash_library.exists(): + pytest.skip("No hash library for this mpl version") + + pytester.makepyfile( + f""" + import os + import pytest + import matplotlib.pyplot as plt + @pytest.mark.mpl_image_compare(baseline_dir=r"{baseline_dir_abs}", + {f'hash_library=r"{hash_library}",' if use_hash_library else ''} + tolerance={DEFAULT_TOLERANCE}, + deterministic=True, + savefig_kwargs={{'format': '{file_format}'}}) + def test_format_{file_format}(): + fig = plt.figure() + ax = fig.add_subplot(1, 1, 1) + ax.plot([{1 if passes else 3}, 2, 3]) + return fig + """ + ) + result = pytester.runpytest('--mpl', '-rs') + if passes: + result.assert_outcomes(passed=1) + else: + result.assert_outcomes(failed=1) diff --git a/tests/test_results_always.py b/tests/test_results_always.py new file mode 100644 index 00000000..ff2cc84d --- /dev/null +++ b/tests/test_results_always.py @@ -0,0 +1,43 @@ +from pathlib import Path + +import pytest +from helpers import pytester_path + + +@pytest.mark.parametrize( + "ini, cli, enabled_expected", + [ + (None, None, False), + (True, None, True), + (False, None, False), + (False, True, True), + (True, True, True), + ], +) +def test_config(pytester, ini, cli, enabled_expected): + path = pytester_path(pytester) + ini = f"mpl-results-always = {ini}" if ini else "" + pytester.makeini( + f""" + [pytest] + mpl-default-style = fivethirtyeight + mpl-baseline-path = {Path(__file__).parent / "baseline" / "2.0.x"} + mpl-results-path = {path} + {ini} + """ + ) + pytester.makepyfile( + """ + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare + def test_base_style(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = "--mpl-results-always" if cli else "" + result = pytester.runpytest("--mpl", cli) + result.assert_outcomes(passed=1) + assert (path / "test_config.test_base_style" / "result.png").exists() == enabled_expected diff --git a/tests/test_results_path.py b/tests/test_results_path.py new file mode 100644 index 00000000..a727c2a3 --- /dev/null +++ b/tests/test_results_path.py @@ -0,0 +1,35 @@ +import pytest +from helpers import pytester_path + + +@pytest.mark.parametrize( + "ini, cli, expected", + [ + ("dir1", None, "dir1"), + ("dir1", "dir2", "dir2"), + (None, "dir2", "dir2"), + ], +) +def test_config(pytester, ini, cli, expected): + ini = f"mpl-results-path = {ini}" if ini else "" + pytester.makeini( + f""" + [pytest] + {ini} + """ + ) + pytester.makepyfile( + """ + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare + def test_mpl(): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = f"--mpl-results-path={cli}" if cli else "" + result = pytester.runpytest("--mpl", cli) + result.assert_outcomes(failed=1) + assert (pytester_path(pytester) / expected / "test_config.test_mpl" / "result.png").exists() diff --git a/tests/test_use_full_test_name.py b/tests/test_use_full_test_name.py new file mode 100644 index 00000000..56aaca55 --- /dev/null +++ b/tests/test_use_full_test_name.py @@ -0,0 +1,54 @@ +import shutil +from pathlib import Path + +import pytest +from helpers import pytester_path + +FULL_TEST_NAME = "test_config.TestClass.test_mpl" +SHORT_TEST_NAME = "test_mpl" + + +@pytest.mark.parametrize( + "ini, cli, expected_baseline_name, success_expected", + [ + (None, None, SHORT_TEST_NAME, True), + (False, None, SHORT_TEST_NAME, True), + (True, None, FULL_TEST_NAME, True), + (False, True, FULL_TEST_NAME, True), + (None, True, FULL_TEST_NAME, True), + (True, True, "bad_name", False), + ], +) +def test_config(pytester, ini, cli, expected_baseline_name, success_expected): + path = pytester_path(pytester) + shutil.copyfile( # Test will only pass if baseline is at expected path + Path(__file__).parent / "baseline" / "2.0.x" / "test_base_style.png", + path / f"{expected_baseline_name}.png", + ) + ini = f"mpl-use-full-test-name = {ini}" if ini is not None else "" + pytester.makeini( + f""" + [pytest] + mpl-default-style = fivethirtyeight + mpl-baseline-path = {path} + {ini} + """ + ) + pytester.makepyfile( + """ + import matplotlib.pyplot as plt + import pytest + class TestClass: + @pytest.mark.mpl_image_compare + def test_mpl(self): + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + """ + ) + cli = "--mpl-use-full-test-name" if cli else "" + result = pytester.runpytest("--mpl", cli) + if success_expected: + result.assert_outcomes(passed=1) + else: + result.assert_outcomes(failed=1) diff --git a/tox.ini b/tox.ini index c4bdf468..450f49de 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,7 @@ [tox] envlist = - py{36,310}-test-mpldev - py{36,37,38,39,310}-test-mpl{34,35} - py37-test-mpl{30,31,32,33} - py36-test-mpl{20,21,22,30} + py{39,310,311,312}-test-mpl{33,34,35,36,37,38} + py312-test-mpldev-pytestdev codestyle requires = setuptools >= 30.3.0 @@ -11,12 +9,17 @@ requires = isolated_build = true [testenv] -passenv = DISPLAY WINDIR MPL_UPDATE_* +passenv = + DISPLAY + WINDIR + MPL_UPDATE_* setenv = MPLBACKEND = Agg + mpldev: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/scientific-python-nightly-wheels/simple changedir = .tmp/{envname} description = run tests deps = + py312: git+https://github.com/dateutil/dateutil.git#egg=python-dateutil mpl20: matplotlib==2.0.* mpl20: nose mpl21: matplotlib==2.1.* @@ -27,13 +30,19 @@ deps = mpl33: matplotlib==3.3.* mpl34: matplotlib==3.4.* mpl35: matplotlib==3.5.* - mpldev: git+https://github.com/matplotlib/matplotlib.git#egg=matplotlib + mpl36: matplotlib==3.6.* + mpl37: matplotlib==3.7.* + mpl38: matplotlib==3.8.* + mpldev: matplotlib>=0.0.dev0 pytest54: pytest==5.4.* pytest60: pytest==6.0.* pytest61: pytest==6.1.* pytest62: pytest==6.2.* pytest70: pytest==7.0.* pytest71: pytest==7.1.* + pytest72: pytest==7.2.* + pytest73: pytest==7.3.* + pytest74: pytest==7.4.* pytestdev: git+https://github.com/pytest-dev/pytest.git#egg=pytest extras = test