diff --git a/lib/matplotlib/tests/test_arrow_patches.py b/lib/matplotlib/tests/test_arrow_patches.py
index 254b86cb54d6..431d1eb6eaf6 100644
--- a/lib/matplotlib/tests/test_arrow_patches.py
+++ b/lib/matplotlib/tests/test_arrow_patches.py
@@ -12,7 +12,7 @@ def draw_arrow(ax, t, r):
 
 
 @image_comparison(['fancyarrow_test_image'],
-                  tol=0.012 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.012)
 def test_fancyarrow():
     # Added 0 to test division by zero error described in issue 3930
     r = [0.4, 0.3, 0.2, 0.1, 0]
@@ -149,7 +149,7 @@ def test_arrow_styles():
 
 
 @image_comparison(['connection_styles.png'], style='mpl20', remove_text=True,
-                  tol=0.013 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.013)
 def test_connection_styles():
     styles = mpatches.ConnectionStyle.get_styles()
 
diff --git a/lib/matplotlib/tests/test_axes.py b/lib/matplotlib/tests/test_axes.py
index 435a1462d2ee..cd5cd08fbf74 100644
--- a/lib/matplotlib/tests/test_axes.py
+++ b/lib/matplotlib/tests/test_axes.py
@@ -243,7 +243,7 @@ def test_matshow(fig_test, fig_ref):
                    'formatter_ticker_004',
                    'formatter_ticker_005',
                    ],
-                  tol=0.031 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.031)
 def test_formatter_ticker():
     import matplotlib.testing.jpl_units as units
     units.register()
@@ -444,7 +444,7 @@ def test_twin_logscale(fig_test, fig_ref, twin):
 
 
 @image_comparison(['twin_autoscale.png'],
-                  tol=0.009 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.009)
 def test_twinx_axis_scales():
     x = np.array([0, 0.5, 1])
     y = 0.5 * x
@@ -1218,9 +1218,8 @@ def test_imshow():
     ax.imshow("r", data=data)
 
 
-@image_comparison(
-    ['imshow_clip'], style='mpl20',
-    tol=1.24 if platform.machine() in ('aarch64', 'arm64', 'ppc64le', 's390x') else 0)
+@image_comparison(['imshow_clip'], style='mpl20',
+                  tol=0 if platform.machine() == 'x86_64' else 1.24)
 def test_imshow_clip():
     # As originally reported by Gellule Xg <gellule.xg@free.fr>
     # use former defaults to match existing baseline image
@@ -1299,7 +1298,7 @@ def test_fill_betweenx_input(y, x1, x2):
 
 
 @image_comparison(['fill_between_interpolate'], remove_text=True,
-                  tol=0.012 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.012)
 def test_fill_between_interpolate():
     x = np.arange(0.0, 2, 0.02)
     y1 = np.sin(2*np.pi*x)
@@ -1727,7 +1726,8 @@ def test_pcolorauto(fig_test, fig_ref, snap):
     ax.pcolormesh(x2, y2, Z, snap=snap)
 
 
-@image_comparison(['canonical'], tol=0.02 if platform.machine() == 'arm64' else 0)
+@image_comparison(['canonical'],
+                  tol=0 if platform.machine() == 'x86_64' else 0.02)
 def test_canonical():
     fig, ax = plt.subplots()
     ax.plot([1, 2, 3])
@@ -2652,9 +2652,8 @@ def test_contour_hatching():
                 extend='both', alpha=0.5)
 
 
-@image_comparison(
-    ['contour_colorbar'], style='mpl20',
-    tol=0.54 if platform.machine() in ('aarch64', 'arm64', 'ppc64le', 's390x') else 0)
+@image_comparison(['contour_colorbar'], style='mpl20',
+                  tol=0 if platform.machine() == 'x86_64' else 0.54)
 def test_contour_colorbar():
     x, y, z = contour_dat()
 
@@ -3232,7 +3231,7 @@ def test_log_scales_invalid():
 
 
 @image_comparison(['stackplot_test_image', 'stackplot_test_image'],
-                  tol=0.031 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.031)
 def test_stackplot():
     fig = plt.figure()
     x = np.linspace(0, 10, 10)
@@ -5126,7 +5125,7 @@ def test_marker_styles():
 
 
 @image_comparison(['rc_markerfill.png'],
-                  tol=0.037 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.037)
 def test_markers_fillstyle_rcparams():
     fig, ax = plt.subplots()
     x = np.arange(7)
@@ -5709,7 +5708,7 @@ def test_twin_remove(fig_test, fig_ref):
 
 
 @image_comparison(['twin_spines.png'], remove_text=True,
-                  tol=0.022 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.022)
 def test_twin_spines():
 
     def make_patch_spines_invisible(ax):
@@ -6306,7 +6305,7 @@ def test_pie_hatch_multi(fig_test, fig_ref):
 
 
 @image_comparison(['set_get_ticklabels.png'],
-                  tol=0.025 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.025)
 def test_set_get_ticklabels():
     # test issue 2246
     fig, ax = plt.subplots(2)
@@ -6899,7 +6898,7 @@ def test_loglog():
 
 
 @image_comparison(["test_loglog_nonpos.png"], remove_text=True, style='mpl20',
-                  tol=0.029 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.029)
 def test_loglog_nonpos():
     fig, axs = plt.subplots(3, 3)
     x = np.arange(1, 11)
@@ -7839,7 +7838,7 @@ def test_scatter_empty_data():
 
 
 @image_comparison(['annotate_across_transforms.png'], style='mpl20', remove_text=True,
-                  tol=0.025 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.025)
 def test_annotate_across_transforms():
     x = np.linspace(0, 10, 200)
     y = np.exp(-x) * np.sin(x)
@@ -7870,7 +7869,7 @@ def inverted(self):
 
 
 @image_comparison(['secondary_xy.png'], style='mpl20',
-                  tol=0.027 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.027)
 def test_secondary_xy():
     fig, axs = plt.subplots(1, 2, figsize=(10, 5), constrained_layout=True)
 
@@ -9134,7 +9133,7 @@ def test_zorder_and_explicit_rasterization():
 
 
 @image_comparison(["preset_clip_paths.png"], remove_text=True, style="mpl20",
-                  tol=0.027 if platform.machine() in ("arm64", "ppc64le") else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.027)
 def test_preset_clip_paths():
     fig, ax = plt.subplots()
 
@@ -9467,7 +9466,7 @@ def test_boxplot_orientation(fig_test, fig_ref):
 
 
 @image_comparison(["use_colorizer_keyword.png"],
-                   tol=0.05 if platform.machine() == 'arm64' else 0)
+                   tol=0 if platform.machine() == 'x86_64' else 0.05)
 def test_use_colorizer_keyword():
     # test using the colorizer keyword
     np.random.seed(0)
diff --git a/lib/matplotlib/tests/test_bbox_tight.py b/lib/matplotlib/tests/test_bbox_tight.py
index 5f2b397b650d..7a2809931757 100644
--- a/lib/matplotlib/tests/test_bbox_tight.py
+++ b/lib/matplotlib/tests/test_bbox_tight.py
@@ -45,7 +45,7 @@ def test_bbox_inches_tight():
 
 @image_comparison(['bbox_inches_tight_suptile_legend'],
                   savefig_kwarg={'bbox_inches': 'tight'},
-                  tol=0.02 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.02)
 def test_bbox_inches_tight_suptile_legend():
     plt.plot(np.arange(10), label='a straight line')
     plt.legend(bbox_to_anchor=(0.9, 1), loc='upper left')
diff --git a/lib/matplotlib/tests/test_collections.py b/lib/matplotlib/tests/test_collections.py
index 11934cfca2c3..db9c4be81a44 100644
--- a/lib/matplotlib/tests/test_collections.py
+++ b/lib/matplotlib/tests/test_collections.py
@@ -391,7 +391,7 @@ def test_barb_limits():
 
 
 @image_comparison(['EllipseCollection_test_image.png'], remove_text=True,
-                  tol=0.021 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.021)
 def test_EllipseCollection():
     # Test basic functionality
     fig, ax = plt.subplots()
diff --git a/lib/matplotlib/tests/test_constrainedlayout.py b/lib/matplotlib/tests/test_constrainedlayout.py
index e42e2ee9bfd8..77f9f34bc2d8 100644
--- a/lib/matplotlib/tests/test_constrainedlayout.py
+++ b/lib/matplotlib/tests/test_constrainedlayout.py
@@ -198,7 +198,7 @@ def test_constrained_layout9():
 
 
 @image_comparison(['constrained_layout10.png'],
-                  tol=0.032 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.032)
 def test_constrained_layout10():
     """Test for handling legend outside axis"""
     fig, axs = plt.subplots(2, 2, layout="constrained")
diff --git a/lib/matplotlib/tests/test_contour.py b/lib/matplotlib/tests/test_contour.py
index e0ea82973af7..543cff18c697 100644
--- a/lib/matplotlib/tests/test_contour.py
+++ b/lib/matplotlib/tests/test_contour.py
@@ -140,7 +140,7 @@ def test_contour_label_with_disconnected_segments():
 
 
 @image_comparison(['contour_manual_colors_and_levels.png'], remove_text=True,
-                  tol=0.018 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.018)
 def test_given_colors_levels_and_extends():
     # Remove this line when this test image is regenerated.
     plt.rcParams['pcolormesh.snap'] = False
@@ -416,10 +416,8 @@ def test_contourf_log_extension():
     cb = plt.colorbar(c3, ax=ax3)
 
 
-@image_comparison(
-    ['contour_addlines.png'], remove_text=True, style='mpl20',
-    tol=0.15 if platform.machine() in ('aarch64', 'arm64', 'ppc64le', 's390x')
-        else 0.03)
+@image_comparison(['contour_addlines.png'], remove_text=True, style='mpl20',
+                  tol=0.03 if platform.machine() == 'x86_64' else 0.15)
 # tolerance is because image changed minutely when tick finding on
 # colorbars was cleaned up...
 def test_contour_addlines():
diff --git a/lib/matplotlib/tests/test_figure.py b/lib/matplotlib/tests/test_figure.py
index edf5ea05f119..c80f53413181 100644
--- a/lib/matplotlib/tests/test_figure.py
+++ b/lib/matplotlib/tests/test_figure.py
@@ -210,7 +210,7 @@ def test_clf_keyword():
 
 
 @image_comparison(['figure_today'],
-                  tol=0.015 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.015)
 def test_figure():
     # named figure support
     fig = plt.figure('today')
diff --git a/lib/matplotlib/tests/test_legend.py b/lib/matplotlib/tests/test_legend.py
index 2d92db4008e4..8baa7bb0f7d9 100644
--- a/lib/matplotlib/tests/test_legend.py
+++ b/lib/matplotlib/tests/test_legend.py
@@ -140,7 +140,7 @@ def test_various_labels():
 
 
 @image_comparison(['legend_labels_first.png'], remove_text=True,
-                  tol=0.013 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.013)
 def test_labels_first():
     # test labels to left of markers
     fig, ax = plt.subplots()
@@ -151,7 +151,7 @@ def test_labels_first():
 
 
 @image_comparison(['legend_multiple_keys.png'], remove_text=True,
-                  tol=0.013 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.013)
 def test_multiple_keys():
     # test legend entries with multiple keys
     fig, ax = plt.subplots()
@@ -514,7 +514,7 @@ def test_figure_legend_outside():
 
 
 @image_comparison(['legend_stackplot.png'],
-                  tol=0.031 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.031)
 def test_legend_stackplot():
     """Test legend for PolyCollection using stackplot."""
     # related to #1341, #1943, and PR #3303
@@ -650,7 +650,7 @@ def test_empty_bar_chart_with_legend():
 
 
 @image_comparison(['shadow_argument_types.png'], remove_text=True, style='mpl20',
-                  tol=0.028 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.028)
 def test_shadow_argument_types():
     # Test that different arguments for shadow work as expected
     fig, ax = plt.subplots()
diff --git a/lib/matplotlib/tests/test_lines.py b/lib/matplotlib/tests/test_lines.py
index ee8b5b4aaa9e..bbaea2510215 100644
--- a/lib/matplotlib/tests/test_lines.py
+++ b/lib/matplotlib/tests/test_lines.py
@@ -140,7 +140,7 @@ def test_valid_linestyles():
 
 
 @image_comparison(['drawstyle_variants.png'], remove_text=True,
-                  tol=0.03 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.03)
 def test_drawstyle_variants():
     fig, axs = plt.subplots(6)
     dss = ["default", "steps-mid", "steps-pre", "steps-post", "steps", None]
@@ -183,9 +183,8 @@ def test_set_drawstyle():
     assert len(line.get_path().vertices) == len(x)
 
 
-@image_comparison(
-    ['line_collection_dashes'], remove_text=True, style='mpl20',
-    tol=0 if platform.machine() == 'x86_64' else 0.65)
+@image_comparison(['line_collection_dashes'], remove_text=True, style='mpl20',
+                  tol=0 if platform.machine() == 'x86_64' else 0.65)
 def test_set_line_coll_dash_image():
     fig, ax = plt.subplots()
     np.random.seed(0)
diff --git a/lib/matplotlib/tests/test_patches.py b/lib/matplotlib/tests/test_patches.py
index 9cb67616a96b..2ea0df2540eb 100644
--- a/lib/matplotlib/tests/test_patches.py
+++ b/lib/matplotlib/tests/test_patches.py
@@ -438,7 +438,7 @@ def test_wedge_movement():
 
 
 @image_comparison(['wedge_range'], remove_text=True,
-                  tol=0.009 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.009)
 def test_wedge_range():
     ax = plt.axes()
 
@@ -564,7 +564,7 @@ def test_units_rectangle():
 
 
 @image_comparison(['connection_patch.png'], style='mpl20', remove_text=True,
-                  tol=0.024 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.024)
 def test_connection_patch():
     fig, (ax1, ax2) = plt.subplots(1, 2)
 
diff --git a/lib/matplotlib/tests/test_path.py b/lib/matplotlib/tests/test_path.py
index 2c4df6ea3b39..5424160dad93 100644
--- a/lib/matplotlib/tests/test_path.py
+++ b/lib/matplotlib/tests/test_path.py
@@ -151,7 +151,7 @@ def test_nonlinear_containment():
 
 
 @image_comparison(['arrow_contains_point.png'], remove_text=True, style='mpl20',
-                  tol=0.027 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.027)
 def test_arrow_contains_point():
     # fix bug (#8384)
     fig, ax = plt.subplots()
@@ -283,7 +283,7 @@ def test_marker_paths_pdf():
 
 @image_comparison(['nan_path'], style='default', remove_text=True,
                   extensions=['pdf', 'svg', 'eps', 'png'],
-                  tol=0.009 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.009)
 def test_nan_isolated_points():
 
     y0 = [0, np.nan, 2, np.nan, 4, 5, 6]
diff --git a/lib/matplotlib/tests/test_patheffects.py b/lib/matplotlib/tests/test_patheffects.py
index bf067b2abbfd..6c89d1c531e0 100644
--- a/lib/matplotlib/tests/test_patheffects.py
+++ b/lib/matplotlib/tests/test_patheffects.py
@@ -30,7 +30,7 @@ def test_patheffect1():
 
 
 @image_comparison(['patheffect2'], remove_text=True, style='mpl20',
-                  tol=0.06 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.06)
 def test_patheffect2():
 
     ax2 = plt.subplot()
@@ -45,7 +45,8 @@ def test_patheffect2():
                                                    foreground="w")])
 
 
-@image_comparison(['patheffect3'], tol=0.019 if platform.machine() == 'arm64' else 0)
+@image_comparison(['patheffect3'],
+                  tol=0 if platform.machine() == 'x86_64' else 0.019)
 def test_patheffect3():
     p1, = plt.plot([1, 3, 5, 4, 3], 'o-b', lw=4)
     p1.set_path_effects([path_effects.SimpleLineShadow(),
diff --git a/lib/matplotlib/tests/test_simplification.py b/lib/matplotlib/tests/test_simplification.py
index a052c24cb655..58b4d7a9b24f 100644
--- a/lib/matplotlib/tests/test_simplification.py
+++ b/lib/matplotlib/tests/test_simplification.py
@@ -29,7 +29,7 @@ def test_clipping():
 
 
 @image_comparison(['overflow'], remove_text=True,
-                  tol=0.007 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.007)
 def test_overflow():
     x = np.array([1.0, 2.0, 3.0, 2.0e5])
     y = np.arange(len(x))
diff --git a/lib/matplotlib/tests/test_skew.py b/lib/matplotlib/tests/test_skew.py
index fd7e7cebfacb..20ec6bf32b06 100644
--- a/lib/matplotlib/tests/test_skew.py
+++ b/lib/matplotlib/tests/test_skew.py
@@ -146,7 +146,7 @@ def test_set_line_coll_dash_image():
 
 
 @image_comparison(['skew_rects'], remove_text=True,
-                  tol=0.009 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.009)
 def test_skew_rectangle():
 
     fix, axes = plt.subplots(5, 5, sharex=True, sharey=True, figsize=(8, 8))
diff --git a/lib/matplotlib/tests/test_subplots.py b/lib/matplotlib/tests/test_subplots.py
index 70215c9531ba..9fd887831106 100644
--- a/lib/matplotlib/tests/test_subplots.py
+++ b/lib/matplotlib/tests/test_subplots.py
@@ -175,7 +175,7 @@ def test_exceptions():
 
 
 @image_comparison(['subplots_offset_text'],
-                  tol=0.028 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.028)
 def test_subplots_offsettext():
     x = np.arange(0, 1e10, 1e9)
     y = np.arange(0, 100, 10)+1e4
diff --git a/lib/mpl_toolkits/axes_grid1/tests/test_axes_grid1.py b/lib/mpl_toolkits/axes_grid1/tests/test_axes_grid1.py
index 778bd9ca04d0..b045e8dc8756 100644
--- a/lib/mpl_toolkits/axes_grid1/tests/test_axes_grid1.py
+++ b/lib/mpl_toolkits/axes_grid1/tests/test_axes_grid1.py
@@ -346,7 +346,7 @@ def test_fill_facecolor():
 # Update style when regenerating the test image
 @image_comparison(['zoomed_axes.png', 'inverted_zoomed_axes.png'],
                   style=('classic', '_classic_test_patch'),
-                  tol=0.02 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.02)
 def test_zooming_with_inverted_axes():
     fig, ax = plt.subplots()
     ax.plot([1, 2, 3], [1, 2, 3])
diff --git a/lib/mpl_toolkits/mplot3d/tests/test_axes3d.py b/lib/mpl_toolkits/mplot3d/tests/test_axes3d.py
index ad952e4395af..b30c2556b105 100644
--- a/lib/mpl_toolkits/mplot3d/tests/test_axes3d.py
+++ b/lib/mpl_toolkits/mplot3d/tests/test_axes3d.py
@@ -221,9 +221,8 @@ def test_bar3d_lightsource():
     np.testing.assert_array_max_ulp(color, collection._facecolor3d[1::6], 4)
 
 
-@mpl3d_image_comparison(
-    ['contour3d.png'], style='mpl20',
-    tol=0.002 if platform.machine() in ('aarch64', 'arm64', 'ppc64le', 's390x') else 0)
+@mpl3d_image_comparison(['contour3d.png'], style='mpl20',
+                        tol=0 if platform.machine() == 'x86_64' else 0.002)
 def test_contour3d():
     plt.rcParams['axes3d.automargin'] = True  # Remove when image is regenerated
     fig = plt.figure()
@@ -1713,7 +1712,7 @@ def test_errorbar3d_errorevery():
 
 
 @mpl3d_image_comparison(['errorbar3d.png'], style='mpl20',
-                        tol=0.02 if platform.machine() == 'arm64' else 0)
+                        tol=0 if platform.machine() == 'x86_64' else 0.02)
 def test_errorbar3d():
     """Tests limits, color styling, and legend for 3D errorbars."""
     fig = plt.figure()
@@ -1729,7 +1728,7 @@ def test_errorbar3d():
     ax.legend()
 
 
-@image_comparison(['stem3d.png'], style='mpl20', tol=0.008)
+@image_comparison(['stem3d.png'], style='mpl20', tol=0.009)
 def test_stem3d():
     plt.rcParams['axes3d.automargin'] = True  # Remove when image is regenerated
     fig, axs = plt.subplots(2, 3, figsize=(8, 6),
diff --git a/lib/mpl_toolkits/mplot3d/tests/test_legend3d.py b/lib/mpl_toolkits/mplot3d/tests/test_legend3d.py
index 0935bbe7f6b0..7fd676df1e31 100644
--- a/lib/mpl_toolkits/mplot3d/tests/test_legend3d.py
+++ b/lib/mpl_toolkits/mplot3d/tests/test_legend3d.py
@@ -28,7 +28,7 @@ def test_legend_bar():
 
 
 @image_comparison(['fancy.png'], remove_text=True, style='mpl20',
-                  tol=0.011 if platform.machine() == 'arm64' else 0)
+                  tol=0 if platform.machine() == 'x86_64' else 0.011)
 def test_fancy():
     fig, ax = plt.subplots(subplot_kw=dict(projection='3d'))
     ax.plot(np.arange(10), np.full(10, 5), np.full(10, 5), 'o--', label='line')