diff --git a/lib/matplotlib/offsetbox.py b/lib/matplotlib/offsetbox.py
index 49f0946f1ee9..6fd999f75fff 100644
--- a/lib/matplotlib/offsetbox.py
+++ b/lib/matplotlib/offsetbox.py
@@ -1488,6 +1488,8 @@ def on_motion(self, evt):
 
     def on_pick(self, evt):
         if self._check_still_parented():
+            if evt.name == "scroll_event":
+                return
             if evt.artist == self.ref_artist:
                 self.mouse_x = evt.mouseevent.x
                 self.mouse_y = evt.mouseevent.y