diff --git a/lib/matplotlib/collections.py b/lib/matplotlib/collections.py
index 181e46b4584f..88ef027b64bb 100644
--- a/lib/matplotlib/collections.py
+++ b/lib/matplotlib/collections.py
@@ -211,8 +211,6 @@ def get_datalim(self, transData):
             # we may have transform.contains_branch(transData) but not
             # transforms.get_affine().contains_branch(transData).  But later,
             # be careful to only apply the affine part that remains.
-        if not transOffset.is_affine:
-            offsets = transOffset.transform_non_affine(offsets)
 
         if isinstance(offsets, np.ma.MaskedArray):
             offsets = offsets.filled(np.nan)
@@ -226,7 +224,8 @@ def get_datalim(self, transData):
                 # also use this algorithm (like streamplot).
                 result = mpath.get_path_collection_extents(
                     transform.get_affine(), paths, self.get_transforms(),
-                    offsets, transOffset.get_affine().frozen())
+                    transOffset.transform_non_affine(offsets),
+                    transOffset.get_affine().frozen())
                 return result.transformed(transData.inverted())
             if not self._offsetsNone:
                 # this is for collections that have their paths (shapes)
@@ -234,9 +233,9 @@ def get_datalim(self, transData):
                 # (i.e. like scatter). We can't uniquely set limits based on
                 # those shapes, so we just set the limits based on their
                 # location.
-                # Finish the transform:
-                offsets = (transOffset.get_affine() +
-                           transData.inverted()).transform(offsets)
+
+                offsets = (transOffset - transData).transform(offsets)
+                # note A-B means A B^{-1}
                 offsets = np.ma.masked_invalid(offsets)
                 if not offsets.mask.all():
                     points = np.row_stack((offsets.min(axis=0),
diff --git a/lib/matplotlib/tests/test_collections.py b/lib/matplotlib/tests/test_collections.py
index 84eaf25fd4e9..2eaf703474e0 100644
--- a/lib/matplotlib/tests/test_collections.py
+++ b/lib/matplotlib/tests/test_collections.py
@@ -630,3 +630,10 @@ def test_blended_collection_autolim():
     ax.add_collection(LineCollection(line_segs, transform=trans))
     ax.autoscale_view(scalex=True, scaley=False)
     np.testing.assert_allclose(ax.get_xlim(), [1., 4.])
+
+
+def test_singleton_autolim():
+    fig, ax = plt.subplots()
+    ax.scatter(0, 0)
+    np.testing.assert_allclose(ax.get_ylim(), [-0.06, 0.06])
+    np.testing.assert_allclose(ax.get_xlim(), [-0.06, 0.06])