diff --git a/lib/matplotlib/backend_managers.py b/lib/matplotlib/backend_managers.py
index e21e12f982e3..53dc39ee0241 100644
--- a/lib/matplotlib/backend_managers.py
+++ b/lib/matplotlib/backend_managers.py
@@ -56,14 +56,12 @@ class ToolManager(object):
         `LockDraw` object to know if the message is available to write
     """
 
-    def __init__(self, canvas):
+    def __init__(self, figure=None):
         warnings.warn('Treat the new Tool classes introduced in v1.5 as ' +
                        'experimental for now, the API will likely change in ' +
                        'version 2.1 and perhaps the rcParam as well')
-        self.canvas = canvas
 
-        self._key_press_handler_id = self.canvas.mpl_connect(
-            'key_press_event', self._key_press)
+        self._key_press_handler_id = None
 
         self._tools = {}
         self._keys = {}
@@ -74,6 +72,45 @@ def __init__(self, canvas):
         self.keypresslock = widgets.LockDraw()
         self.messagelock = widgets.LockDraw()
 
+        self._figure = None
+        self.set_figure(figure)
+
+    @property
+    def canvas(self):
+        """Canvas managed by FigureManager"""
+        if not self._figure:
+            return None
+        return self._figure.canvas
+
+    @property
+    def figure(self):
+        """Figure that holds the canvas"""
+        return self._figure
+
+    @figure.setter
+    def figure(self, figure):
+        self.set_figure(figure)
+
+    def set_figure(self, figure, update_tools=True):
+        """
+        Sets the figure to interact with the tools
+
+        Parameters
+        ==========
+        figure: `Figure`
+        update_tools: bool
+            Force tools to update figure
+        """
+        if self._key_press_handler_id:
+            self.canvas.mpl_disconnect(self._key_press_handler_id)
+        self._figure = figure
+        if figure:
+            self._key_press_handler_id = self.canvas.mpl_connect(
+                'key_press_event', self._key_press)
+        if update_tools:
+            for tool in self._tools.values():
+                tool.figure = figure
+
     def toolmanager_connect(self, s, func):
         """
         Connect event with string *s* to *func*.
@@ -245,6 +282,7 @@ def add_tool(self, name, tool, *args, **kwargs):
             else:
                 self._toggled.setdefault(tool_obj.radio_group, None)
 
+        tool_obj.set_figure(self.figure)
         self._tool_added_event(tool_obj)
         return tool_obj
 
diff --git a/lib/matplotlib/backend_tools.py b/lib/matplotlib/backend_tools.py
index b86c966142af..ec2ccaf0fd22 100644
--- a/lib/matplotlib/backend_tools.py
+++ b/lib/matplotlib/backend_tools.py
@@ -77,14 +77,37 @@ def __init__(self, toolmanager, name):
                       'experimental for now, the API will likely change in ' +
                       'version 2.1, and some tools might change name')
         self._name = name
+        self._toolmanager = toolmanager
         self._figure = None
-        self.toolmanager = toolmanager
-        self.figure = toolmanager.canvas.figure
 
     @property
     def figure(self):
         return self._figure
 
+    @figure.setter
+    def figure(self, figure):
+        self.set_figure(figure)
+
+    @property
+    def canvas(self):
+        if not self._figure:
+            return None
+        return self._figure.canvas
+
+    @property
+    def toolmanager(self):
+        return self._toolmanager
+
+    def set_figure(self, figure):
+        """
+        Assign a figure to the tool
+
+        Parameters
+        ----------
+        figure: `Figure`
+        """
+        self._figure = figure
+
     def trigger(self, sender, event, data=None):
         """
         Called when this tool gets used
@@ -104,20 +127,6 @@ def trigger(self, sender, event, data=None):
 
         pass
 
-    @figure.setter
-    def figure(self, figure):
-        """
-        Set the figure
-
-        Set the figure to be affected by this tool
-
-        Parameters
-        ----------
-        figure: `Figure`
-        """
-
-        self._figure = figure
-
     @property
     def name(self):
         """Tool Id"""
@@ -193,6 +202,14 @@ def toggled(self):
 
         return self._toggled
 
+    def set_figure(self, figure):
+        toggled = self.toggled
+        if toggled:
+            self.trigger(self, None)
+        ToolBase.set_figure(self, figure)
+        if figure and toggled:
+            self.trigger(self, None)
+
 
 class SetCursorBase(ToolBase):
     """
@@ -203,8 +220,7 @@ class SetCursorBase(ToolBase):
     """
     def __init__(self, *args, **kwargs):
         ToolBase.__init__(self, *args, **kwargs)
-        self._idDrag = self.figure.canvas.mpl_connect(
-            'motion_notify_event', self._set_cursor_cbk)
+        self._idDrag = None
         self._cursor = None
         self._default_cursor = cursors.POINTER
         self._last_cursor = self._default_cursor
@@ -215,6 +231,14 @@ def __init__(self, *args, **kwargs):
         for tool in self.toolmanager.tools.values():
             self._add_tool(tool)
 
+    def set_figure(self, figure):
+        if self._idDrag:
+            self.canvas.mpl_disconnect(self._idDrag)
+        ToolBase.set_figure(self, figure)
+        if figure:
+            self._idDrag = self.canvas.mpl_connect(
+                'motion_notify_event', self._set_cursor_cbk)
+
     def _tool_trigger_cbk(self, event):
         if event.tool.toggled:
             self._cursor = event.tool.cursor
@@ -266,9 +290,16 @@ class ToolCursorPosition(ToolBase):
     This tool runs in the background reporting the position of the cursor
     """
     def __init__(self, *args, **kwargs):
+        self._idDrag = None
         ToolBase.__init__(self, *args, **kwargs)
-        self._idDrag = self.figure.canvas.mpl_connect(
-            'motion_notify_event', self.send_message)
+
+    def set_figure(self, figure):
+        if self._idDrag:
+            self.canvas.mpl_disconnect(self._idDrag)
+        ToolBase.set_figure(self, figure)
+        if figure:
+            self._idDrag = self.canvas.mpl_connect(
+                'motion_notify_event', self.send_message)
 
     def send_message(self, event):
         """Call `matplotlib.backend_managers.ToolManager.message_event`"""
@@ -458,16 +489,17 @@ def __init__(self, *args, **kwargs):
         self.home_views = WeakKeyDictionary()
         ToolBase.__init__(self, *args, **kwargs)
 
-    def add_figure(self):
+    def add_figure(self, figure):
         """Add the current figure to the stack of views and positions"""
-        if self.figure not in self.views:
-            self.views[self.figure] = cbook.Stack()
-            self.positions[self.figure] = cbook.Stack()
-            self.home_views[self.figure] = WeakKeyDictionary()
+
+        if figure not in self.views:
+            self.views[figure] = cbook.Stack()
+            self.positions[figure] = cbook.Stack()
+            self.home_views[figure] = WeakKeyDictionary()
             # Define Home
-            self.push_current()
+            self.push_current(figure)
             # Make sure we add a home view for new axes as they're added
-            self.figure.add_axobserver(lambda fig: self.update_home_views())
+            figure.add_axobserver(lambda fig: self.update_home_views(fig))
 
     def clear(self, figure):
         """Reset the axes stack"""
@@ -508,18 +540,19 @@ def update_view(self):
 
         self.figure.canvas.draw_idle()
 
-    def push_current(self):
+    def push_current(self, figure=None):
         """
         Push the current view limits and position onto their respective stacks
         """
-
+        if not figure:
+            figure = self.figure
         views = WeakKeyDictionary()
         pos = WeakKeyDictionary()
-        for a in self.figure.get_axes():
+        for a in figure.get_axes():
             views[a] = a._get_view()
             pos[a] = self._axes_pos(a)
-        self.views[self.figure].push(views)
-        self.positions[self.figure].push(pos)
+        self.views[figure].push(views)
+        self.positions[figure].push(pos)
 
     def _axes_pos(self, ax):
         """
@@ -539,15 +572,17 @@ def _axes_pos(self, ax):
         return (ax.get_position(True).frozen(),
                 ax.get_position().frozen())
 
-    def update_home_views(self):
+    def update_home_views(self, figure=None):
         """
         Make sure that self.home_views has an entry for all axes present in the
         figure
         """
 
-        for a in self.figure.get_axes():
-            if a not in self.home_views[self.figure]:
-                self.home_views[self.figure][a] = a._get_view()
+        if not figure:
+            figure = self.figure
+        for a in figure.get_axes():
+            if a not in self.home_views[figure]:
+                self.home_views[figure][a] = a._get_view()
 
     def refresh_locators(self):
         """Redraw the canvases, update the locators"""
@@ -592,7 +627,7 @@ class ViewsPositionsBase(ToolBase):
     _on_trigger = None
 
     def trigger(self, sender, event, data=None):
-        self.toolmanager.get_tool(_views_positions).add_figure()
+        self.toolmanager.get_tool(_views_positions).add_figure(self.figure)
         getattr(self.toolmanager.get_tool(_views_positions),
                 self._on_trigger)()
         self.toolmanager.get_tool(_views_positions).update_view()
@@ -672,7 +707,7 @@ def disable(self, event):
         self.figure.canvas.mpl_disconnect(self._idScroll)
 
     def trigger(self, sender, event, data=None):
-        self.toolmanager.get_tool(_views_positions).add_figure()
+        self.toolmanager.get_tool(_views_positions).add_figure(self.figure)
         ToolToggleBase.trigger(self, sender, event, data)
 
     def scroll_zoom(self, event):
diff --git a/lib/matplotlib/backends/backend_gtk3.py b/lib/matplotlib/backends/backend_gtk3.py
index 9df30ff3dbd3..6a3bfa49ffcf 100644
--- a/lib/matplotlib/backends/backend_gtk3.py
+++ b/lib/matplotlib/backends/backend_gtk3.py
@@ -482,7 +482,6 @@ def full_screen_toggle (self):
             self.window.unfullscreen()
     _full_screen_flag = False
 
-
     def _get_toolbar(self):
         # must be inited after the window, drawingArea and figure
         # attrs are set
@@ -497,7 +496,7 @@ def _get_toolbar(self):
     def _get_toolmanager(self):
         # must be initialised after toolbar has been setted
         if rcParams['toolbar'] != 'toolbar2':
-            toolmanager = ToolManager(self.canvas)
+            toolmanager = ToolManager(self.canvas.figure)
         else:
             toolmanager = None
         return toolmanager
diff --git a/lib/matplotlib/backends/backend_tkagg.py b/lib/matplotlib/backends/backend_tkagg.py
index 45dca4be3804..eb9707c89c7b 100644
--- a/lib/matplotlib/backends/backend_tkagg.py
+++ b/lib/matplotlib/backends/backend_tkagg.py
@@ -570,7 +570,7 @@ def _get_toolbar(self):
 
     def _get_toolmanager(self):
         if rcParams['toolbar'] != 'toolbar2':
-            toolmanager = ToolManager(self.canvas)
+            toolmanager = ToolManager(self.canvas.figure)
         else:
             toolmanager = None
         return toolmanager