diff --git a/doc/api/api_changes/2017-03-11_TAC.rst b/doc/api/api_changes/2017-03-11_TAC.rst
new file mode 100644
index 000000000000..16ef9875f849
--- /dev/null
+++ b/doc/api/api_changes/2017-03-11_TAC.rst
@@ -0,0 +1,12 @@
+Round instead of truncating pixel number when using Agg
+```````````````````````````````````````````````````````
+
+When rasterizing a figure the allowed figure sizes are effectively
+discretized by the finite dpi.  Previously, we simply truncated when
+computing the number of pixels to use, now we use `numpy.round` which
+will avoid floating point issues when setting the size of the figure
+in inches to get a specific size of in pixels.
+
+Additionally, the Agg based backends will adjust the size of the
+figure to match the actual size (given by the number of pixels and the
+dpi) during the draw process.
diff --git a/lib/matplotlib/backends/backend_agg.py b/lib/matplotlib/backends/backend_agg.py
index 0fe0fc40c00b..9aa84ddc60fa 100644
--- a/lib/matplotlib/backends/backend_agg.py
+++ b/lib/matplotlib/backends/backend_agg.py
@@ -404,7 +404,14 @@ def draw(self):
 
     @_api.delete_parameter("3.6", "cleared", alternative="renderer.clear()")
     def get_renderer(self, cleared=False):
-        w, h = self.figure.bbox.size
+        w, h = np.round(self.figure.bbox.size).astype(int)
+        dpi = self.figure.dpi
+
+        # we know we are using Agg, thus are tied to discrete sizes
+        # set by the dpi.  Feed this back so that the transforms are
+        # mapped to the available pixels
+        self.figure.set_size_inches(w / dpi, h / dpi)
+
         key = w, h, self.figure.dpi
         reuse_renderer = (self._lastKey == key)
         if not reuse_renderer:
diff --git a/lib/matplotlib/figure.py b/lib/matplotlib/figure.py
index 6c18ba1a643e..2ac513137a53 100644
--- a/lib/matplotlib/figure.py
+++ b/lib/matplotlib/figure.py
@@ -2954,10 +2954,12 @@ def set_size_inches(self, w, h=None, forward=True):
         if not np.isfinite(size).all() or (size < 0).any():
             raise ValueError(f'figure size must be positive finite not {size}')
         self.bbox_inches.p1 = size
+        if tuple(size) == (w, h) and not forward:
+            return
         if forward:
             manager = self.canvas.manager
             if manager is not None:
-                manager.resize(*(size * self.dpi).astype(int))
+                manager.resize(*np.round((size * self.dpi)).astype(int))
         self.stale = True
 
     def get_size_inches(self):
diff --git a/lib/matplotlib/tests/baseline_images/test_arrow_patches/boxarrow_test_image.png b/lib/matplotlib/tests/baseline_images/test_arrow_patches/boxarrow_test_image.png
index 9e613ff40ca9..de144dd4d00d 100644
Binary files a/lib/matplotlib/tests/baseline_images/test_arrow_patches/boxarrow_test_image.png and b/lib/matplotlib/tests/baseline_images/test_arrow_patches/boxarrow_test_image.png differ
diff --git a/lib/matplotlib/tests/baseline_images/test_bbox_tight/bbox_inches_tight_suptile_legend.png b/lib/matplotlib/tests/baseline_images/test_bbox_tight/bbox_inches_tight_suptile_legend.png
index 4b781c866dd9..5acd63cd3fff 100644
Binary files a/lib/matplotlib/tests/baseline_images/test_bbox_tight/bbox_inches_tight_suptile_legend.png and b/lib/matplotlib/tests/baseline_images/test_bbox_tight/bbox_inches_tight_suptile_legend.png differ
diff --git a/lib/mpl_toolkits/axes_grid1/tests/baseline_images/test_axes_grid1/image_grid.png b/lib/mpl_toolkits/axes_grid1/tests/baseline_images/test_axes_grid1/image_grid.png
index a696787a0248..9a83ce914034 100644
Binary files a/lib/mpl_toolkits/axes_grid1/tests/baseline_images/test_axes_grid1/image_grid.png and b/lib/mpl_toolkits/axes_grid1/tests/baseline_images/test_axes_grid1/image_grid.png differ
diff --git a/lib/mpl_toolkits/axes_grid1/tests/baseline_images/test_axes_grid1/image_grid_each_left_label_mode_all.png b/lib/mpl_toolkits/axes_grid1/tests/baseline_images/test_axes_grid1/image_grid_each_left_label_mode_all.png
index 23abe8b9649d..28274e9a61c3 100644
Binary files a/lib/mpl_toolkits/axes_grid1/tests/baseline_images/test_axes_grid1/image_grid_each_left_label_mode_all.png and b/lib/mpl_toolkits/axes_grid1/tests/baseline_images/test_axes_grid1/image_grid_each_left_label_mode_all.png differ
diff --git a/lib/mpl_toolkits/axes_grid1/tests/baseline_images/test_axes_grid1/image_grid_single_bottom_label_mode_1.png b/lib/mpl_toolkits/axes_grid1/tests/baseline_images/test_axes_grid1/image_grid_single_bottom_label_mode_1.png
index 1a0f4cd1fc9a..d7c6f78099c6 100644
Binary files a/lib/mpl_toolkits/axes_grid1/tests/baseline_images/test_axes_grid1/image_grid_single_bottom_label_mode_1.png and b/lib/mpl_toolkits/axes_grid1/tests/baseline_images/test_axes_grid1/image_grid_single_bottom_label_mode_1.png differ