diff --git a/lib/matplotlib/backends/backend_pgf.py b/lib/matplotlib/backends/backend_pgf.py
index 6cc27928733a..e59290338ac5 100644
--- a/lib/matplotlib/backends/backend_pgf.py
+++ b/lib/matplotlib/backends/backend_pgf.py
@@ -192,6 +192,12 @@ def __init__(self, message, latex_output=""):
         super().__init__(message)
         self.latex_output = latex_output
 
+    def __str__(self):
+        s, = self.args
+        if self.latex_output:
+            s += "\n" + self.latex_output
+        return s
+
 
 class LatexManager:
     """
@@ -286,7 +292,7 @@ def __init__(self):
         stdout, stderr = latex.communicate(test_input)
         if latex.returncode != 0:
             raise LatexError("LaTeX returned an error, probably missing font "
-                             "or error in preamble:\n%s" % stdout)
+                             "or error in preamble.", stdout)
 
         self.latex = None  # Will be set up on first use.
         self.str_cache = {}  # cache for strings already processed