diff --git a/zarr/core.py b/zarr/core.py
index 80d1830c07..33734aa304 100644
--- a/zarr/core.py
+++ b/zarr/core.py
@@ -1476,6 +1476,17 @@ def _set_basic_selection_zd(self, selection, value, fields=None):
         else:
             chunk[selection] = value
 
+        # clear chunk if it only contains the fill value
+        if np.all(np.equal(chunk, self._fill_value)):
+            try:
+                del self.chunk_store[ckey]
+                return
+            except KeyError:
+                return
+            except Exception:
+                # deleting failed, fallback to overwriting
+                pass
+
         # encode and store
         cdata = self._encode_chunk(chunk)
         self.chunk_store[ckey] = cdata
@@ -1716,10 +1727,19 @@ def _chunk_setitem_nosync(self, chunk_coords, chunk_selection, value, fields=Non
             else:
                 chunk[chunk_selection] = value
 
-        # encode chunk
-        cdata = self._encode_chunk(chunk)
+        # clear chunk if it only contains the fill value
+        if np.all(np.equal(chunk, self._fill_value)):
+            try:
+                del self.chunk_store[ckey]
+                return
+            except KeyError:
+                return
+            except Exception:
+                # deleting failed, fallback to overwriting
+                pass
 
-        # store
+        # encode and store
+        cdata = self._encode_chunk(chunk)
         self.chunk_store[ckey] = cdata
 
     def _chunk_key(self, chunk_coords):