diff --git a/lib/matplotlib/backends/backend_agg.py b/lib/matplotlib/backends/backend_agg.py
index a740336f80c7..1741e140ae0e 100644
--- a/lib/matplotlib/backends/backend_agg.py
+++ b/lib/matplotlib/backends/backend_agg.py
@@ -222,7 +222,7 @@ def get_text_width_height_descent(self, s, prop, ismath):
         to the baseline), in display coords, of the string *s* with
         :class:`~matplotlib.font_manager.FontProperties` *prop*
         """
-        if rcParams['text.usetex']:
+        if ismath in ["TeX", "TeX!"]:
             # todo: handle props
             size = prop.get_size_in_points()
             texmanager = self.get_texmanager()
diff --git a/lib/matplotlib/tests/test_text.py b/lib/matplotlib/tests/test_text.py
index 32d3b468119d..7bc878accc95 100644
--- a/lib/matplotlib/tests/test_text.py
+++ b/lib/matplotlib/tests/test_text.py
@@ -14,6 +14,11 @@
 from matplotlib.testing.decorators import image_comparison
 
 
+needs_usetex = pytest.mark.xfail(
+    not matplotlib.checkdep_usetex(True),
+    reason="This test needs a TeX installation")
+
+
 @image_comparison(baseline_images=['font_styles'])
 def test_font_styles():
     from matplotlib import _get_data_path
@@ -459,3 +464,13 @@ def test_hinting_factor_backends():
     # Backends should apply hinting_factor consistently (within 10%).
     np.testing.assert_allclose(t.get_window_extent().intervalx, expected,
                                rtol=0.1)
+
+
+@needs_usetex
+def test_single_artist_usetex():
+    # Check that a single artist marked with usetex does not get passed through
+    # the mathtext parser at all (for the Agg backend) (the mathtext parser
+    # currently fails to parse \frac12, requiring \frac{1}{2} instead).
+    fig, ax = plt.subplots()
+    ax.text(.5, .5, r"$\frac12$", usetex=True)
+    fig.canvas.draw()