diff --git a/doc/api/api_changes/2020-04-26-merged-rasterizations.rst b/doc/api/api_changes/2020-04-26-merged-rasterizations.rst
new file mode 100644
index 000000000000..5aed41e9f352
--- /dev/null
+++ b/doc/api/api_changes/2020-04-26-merged-rasterizations.rst
@@ -0,0 +1,15 @@
+Consecutive rasterized draws now merged
+---------------------------------------
+
+Tracking of depth of raster draws has moved from
+`.backend_mixed.MixedModeRenderer.start_rasterizing` and
+`.backend_mixed.MixedModeRenderer.stop_rasterizing` into
+`.artist.allow_rasterization`. This means the start and stop functions are
+only called when the rasterization actually needs to be started and stopped.
+
+The output of vector backends will change in the case that rasterized
+elements are merged. This should not change the appearance of outputs.
+
+The renders in 3rd party backends are now expected to have
+``self._raster_depth`` and ``self._rasterizing`` initialized to ``0`` and
+``False`` respectively.
diff --git a/doc/users/next_whats_new/2020-04-26-merged-rasterizations.rst b/doc/users/next_whats_new/2020-04-26-merged-rasterizations.rst
new file mode 100644
index 000000000000..d6ad7c4a4d0e
--- /dev/null
+++ b/doc/users/next_whats_new/2020-04-26-merged-rasterizations.rst
@@ -0,0 +1,13 @@
+Consecutive rasterized draws now merged
+---------------------------------------
+
+Elements of a vector output can be individually set to rasterized, using
+the ``rasterized`` keyword, or `~.artist.Artist.set_rasterized()`. This can
+be useful to reduce file sizes. For figures with multiple raster elements
+they are now automatically merged into a smaller number of bitmaps where
+this will not effect the visual output. For cases with many elements this
+can result in significantly smaller file sizes.
+
+To ensure this happens do not place vector elements between raster ones.
+
+To inhibit this merging set ``Figure.suppressComposite`` to True.
diff --git a/lib/matplotlib/artist.py b/lib/matplotlib/artist.py
index 41e890352fbc..4265db5948e5 100644
--- a/lib/matplotlib/artist.py
+++ b/lib/matplotlib/artist.py
@@ -34,7 +34,17 @@ def allow_rasterization(draw):
     def draw_wrapper(artist, renderer, *args, **kwargs):
         try:
             if artist.get_rasterized():
-                renderer.start_rasterizing()
+                if renderer._raster_depth == 0 and not renderer._rasterizing:
+                    renderer.start_rasterizing()
+                    renderer._rasterizing = True
+                renderer._raster_depth += 1
+            else:
+                if renderer._raster_depth == 0 and renderer._rasterizing:
+                    # Only stop when we are not in a rasterized parent
+                    # and something has be rasterized since last stop
+                    renderer.stop_rasterizing()
+                    renderer._rasterizing = False
+
             if artist.get_agg_filter() is not None:
                 renderer.start_filter()
 
@@ -43,12 +53,32 @@ def draw_wrapper(artist, renderer, *args, **kwargs):
             if artist.get_agg_filter() is not None:
                 renderer.stop_filter(artist.get_agg_filter())
             if artist.get_rasterized():
+                renderer._raster_depth -= 1
+            if (renderer._rasterizing and artist.figure and
+                    artist.figure.suppressComposite):
+                # restart rasterizing to prevent merging
                 renderer.stop_rasterizing()
+                renderer.start_rasterizing()
 
     draw_wrapper._supports_rasterization = True
     return draw_wrapper
 
 
+def _finalize_rasterization(draw):
+    """
+    Decorator for Artist.draw method. Needed on the outermost artist, i.e.
+    Figure, to finish up if the render is still in rasterized mode.
+    """
+    @wraps(draw)
+    def draw_wrapper(artist, renderer, *args, **kwargs):
+        result = draw(artist, renderer, *args, **kwargs)
+        if renderer._rasterizing:
+            renderer.stop_rasterizing()
+            renderer._rasterizing = False
+        return result
+    return draw_wrapper
+
+
 def _stale_axes_callback(self, val):
     if self.axes:
         self.axes.stale = val
diff --git a/lib/matplotlib/backend_bases.py b/lib/matplotlib/backend_bases.py
index cf9453959853..5dceb2e3736c 100644
--- a/lib/matplotlib/backend_bases.py
+++ b/lib/matplotlib/backend_bases.py
@@ -142,6 +142,8 @@ def __init__(self):
         super().__init__()
         self._texmanager = None
         self._text2path = textpath.TextToPath()
+        self._raster_depth = 0
+        self._rasterizing = False
 
     def open_group(self, s, gid=None):
         """
diff --git a/lib/matplotlib/backends/backend_mixed.py b/lib/matplotlib/backends/backend_mixed.py
index 4bd77e8105e1..9a588ece8f9d 100644
--- a/lib/matplotlib/backends/backend_mixed.py
+++ b/lib/matplotlib/backends/backend_mixed.py
@@ -52,7 +52,6 @@ def __init__(self, figure, width, height, dpi, vector_renderer,
         self._vector_renderer = vector_renderer
 
         self._raster_renderer = None
-        self._rasterizing = 0
 
         # A reference to the figure is needed as we need to change
         # the figure dpi before and after the rasterization. Although
@@ -84,47 +83,40 @@ def start_rasterizing(self):
             r = process_figure_for_rasterizing(self.figure,
                                                self._bbox_inches_restore)
             self._bbox_inches_restore = r
-        if self._rasterizing == 0:
-            self._raster_renderer = self._raster_renderer_class(
-                self._width*self.dpi, self._height*self.dpi, self.dpi)
-            self._renderer = self._raster_renderer
-        self._rasterizing += 1
+
+        self._raster_renderer = self._raster_renderer_class(
+            self._width*self.dpi, self._height*self.dpi, self.dpi)
+        self._renderer = self._raster_renderer
 
     def stop_rasterizing(self):
         """
         Exit "raster" mode.  All of the drawing that was done since
         the last `start_rasterizing` call will be copied to the
         vector backend by calling draw_image.
-
-        If `start_rasterizing` has been called multiple times,
-        `stop_rasterizing` must be called the same number of times before
-        "raster" mode is exited.
         """
-        self._rasterizing -= 1
-        if self._rasterizing == 0:
-            self._renderer = self._vector_renderer
-
-            height = self._height * self.dpi
-            buffer, bounds = self._raster_renderer.tostring_rgba_minimized()
-            l, b, w, h = bounds
-            if w > 0 and h > 0:
-                image = np.frombuffer(buffer, dtype=np.uint8)
-                image = image.reshape((h, w, 4))
-                image = image[::-1]
-                gc = self._renderer.new_gc()
-                # TODO: If the mixedmode resolution differs from the figure's
-                #       dpi, the image must be scaled (dpi->_figdpi). Not all
-                #       backends support this.
-                self._renderer.draw_image(
-                    gc,
-                    l * self._figdpi / self.dpi,
-                    (height-b-h) * self._figdpi / self.dpi,
-                    image)
-            self._raster_renderer = None
-            self._rasterizing = False
-
-            # restore the figure dpi.
-            self.figure.set_dpi(self._figdpi)
+
+        self._renderer = self._vector_renderer
+
+        height = self._height * self.dpi
+        buffer, bounds = self._raster_renderer.tostring_rgba_minimized()
+        l, b, w, h = bounds
+        if w > 0 and h > 0:
+            image = np.frombuffer(buffer, dtype=np.uint8)
+            image = image.reshape((h, w, 4))
+            image = image[::-1]
+            gc = self._renderer.new_gc()
+            # TODO: If the mixedmode resolution differs from the figure's
+            #       dpi, the image must be scaled (dpi->_figdpi). Not all
+            #       backends support this.
+            self._renderer.draw_image(
+                gc,
+                l * self._figdpi / self.dpi,
+                (height-b-h) * self._figdpi / self.dpi,
+                image)
+        self._raster_renderer = None
+
+        # restore the figure dpi.
+        self.figure.set_dpi(self._figdpi)
 
         if self._bbox_inches_restore:  # when tight bbox is used
             r = process_figure_for_rasterizing(self.figure,
diff --git a/lib/matplotlib/figure.py b/lib/matplotlib/figure.py
index b7dc60859525..a619b58d2505 100644
--- a/lib/matplotlib/figure.py
+++ b/lib/matplotlib/figure.py
@@ -18,7 +18,8 @@
 from matplotlib import __version__ as _mpl_version
 
 import matplotlib.artist as martist
-from matplotlib.artist import Artist, allow_rasterization
+from matplotlib.artist import (
+    Artist, allow_rasterization, _finalize_rasterization)
 from matplotlib.backend_bases import (
     FigureCanvasBase, NonGuiException, MouseButton)
 import matplotlib.cbook as cbook
@@ -1689,6 +1690,7 @@ def clear(self, keep_observers=False):
         """Clear the figure -- synonym for `clf`."""
         self.clf(keep_observers=keep_observers)
 
+    @_finalize_rasterization
     @allow_rasterization
     def draw(self, renderer):
         # docstring inherited
diff --git a/lib/matplotlib/tests/test_backend_svg.py b/lib/matplotlib/tests/test_backend_svg.py
index 7bdc3ca33b0a..c64ffd8e4e69 100644
--- a/lib/matplotlib/tests/test_backend_svg.py
+++ b/lib/matplotlib/tests/test_backend_svg.py
@@ -10,7 +10,7 @@
 from matplotlib import dviread
 from matplotlib.figure import Figure
 import matplotlib.pyplot as plt
-from matplotlib.testing.decorators import image_comparison
+from matplotlib.testing.decorators import image_comparison, check_figures_equal
 
 
 needs_usetex = pytest.mark.skipif(
@@ -94,6 +94,104 @@ def test_bold_font_output_with_none_fonttype():
     ax.set_title('bold-title', fontweight='bold')
 
 
+@check_figures_equal(tol=20)
+def test_rasterized(fig_test, fig_ref):
+    t = np.arange(0, 100) * (2.3)
+    x = np.cos(t)
+    y = np.sin(t)
+
+    ax_ref = fig_ref.subplots()
+    ax_ref.plot(x, y, "-", c="r", lw=10)
+    ax_ref.plot(x+1, y, "-", c="b", lw=10)
+
+    ax_test = fig_test.subplots()
+    ax_test.plot(x, y, "-", c="r", lw=10, rasterized=True)
+    ax_test.plot(x+1, y, "-", c="b", lw=10, rasterized=True)
+
+
+@check_figures_equal()
+def test_rasterized_ordering(fig_test, fig_ref):
+    t = np.arange(0, 100) * (2.3)
+    x = np.cos(t)
+    y = np.sin(t)
+
+    ax_ref = fig_ref.subplots()
+    ax_ref.set_xlim(0, 3)
+    ax_ref.set_ylim(-1.1, 1.1)
+    ax_ref.plot(x, y, "-", c="r", lw=10, rasterized=True)
+    ax_ref.plot(x+1, y, "-", c="b", lw=10, rasterized=False)
+    ax_ref.plot(x+2, y, "-", c="g", lw=10, rasterized=True)
+    ax_ref.plot(x+3, y, "-", c="m", lw=10, rasterized=True)
+
+    ax_test = fig_test.subplots()
+    ax_test.set_xlim(0, 3)
+    ax_test.set_ylim(-1.1, 1.1)
+    ax_test.plot(x, y, "-", c="r", lw=10, rasterized=True, zorder=1.1)
+    ax_test.plot(x+2, y, "-", c="g", lw=10, rasterized=True, zorder=1.3)
+    ax_test.plot(x+3, y, "-", c="m", lw=10, rasterized=True, zorder=1.4)
+    ax_test.plot(x+1, y, "-", c="b", lw=10, rasterized=False, zorder=1.2)
+
+
+def test_count_bitmaps():
+    def count_tag(fig, tag):
+        fd = BytesIO()
+        fig.savefig(fd, format='svg')
+        fd.seek(0)
+        buf = fd.read().decode()
+        fd.close()
+        open("test.svg", "w").write(buf)
+        return buf.count("<%s" % tag)
+
+    # No rasterized elements
+    fig1 = plt.figure()
+    ax1 = fig1.add_subplot(1, 1, 1)
+    ax1.set_axis_off()
+    for n in range(5):
+        ax1.plot([0, 20], [0, n], "b-", rasterized=False)
+    assert count_tag(fig1, "image") == 0
+    assert count_tag(fig1, "path") == 6  # axis patch plus lines
+
+    # rasterized can be merged
+    fig2 = plt.figure()
+    ax2 = fig2.add_subplot(1, 1, 1)
+    ax2.set_axis_off()
+    for n in range(5):
+        ax2.plot([0, 20], [0, n], "b-", rasterized=True)
+    assert count_tag(fig2, "image") == 1
+    assert count_tag(fig2, "path") == 1  # axis patch
+
+    # rasterized can't be merged without effecting draw order
+    fig3 = plt.figure()
+    ax3 = fig3.add_subplot(1, 1, 1)
+    ax3.set_axis_off()
+    for n in range(5):
+        ax3.plot([0, 20], [n, 0], "b-", rasterized=False)
+        ax3.plot([0, 20], [0, n], "b-", rasterized=True)
+    assert count_tag(fig3, "image") == 5
+    assert count_tag(fig3, "path") == 6
+
+    # rasterized whole axes
+    fig4 = plt.figure()
+    ax4 = fig4.add_subplot(1, 1, 1)
+    ax4.set_axis_off()
+    ax4.set_rasterized(True)
+    for n in range(5):
+        ax4.plot([0, 20], [n, 0], "b-", rasterized=False)
+        ax4.plot([0, 20], [0, n], "b-", rasterized=True)
+    assert count_tag(fig4, "image") == 1
+    assert count_tag(fig4, "path") == 1
+
+    # rasterized can be merged, but inhibited by suppressComposite
+    fig5 = plt.figure()
+    fig5.suppressComposite = True
+    ax5 = fig5.add_subplot(1, 1, 1)
+    ax5.set_axis_off()
+    for n in range(5):
+        ax5.plot([0, 20], [0, n], "b-", rasterized=True)
+    assert count_tag(fig5, "image") == 5
+    assert count_tag(fig5, "path") == 1  # axis patch
+
+
 @needs_usetex
 def test_missing_psfont(monkeypatch):
     """An error is raised if a TeX font lacks a Type-1 equivalent"""