diff --git a/glue/core/roi.py b/glue/core/roi.py
index f5f0114bc..23629d206 100644
--- a/glue/core/roi.py
+++ b/glue/core/roi.py
@@ -1147,7 +1147,6 @@ def __init__(self, axes, data_space=True):
         self._patch.set_visible(False)
         if not self._data_space:
             self._patch.set_transform(self._axes.transAxes)
-        self._axes.add_patch(self._patch)
 
     def start_selection(self, event):
         if event.inaxes != self._axes:
@@ -1229,7 +1228,10 @@ def _sync_patch(self):
             self._patch.set_height(height)
             self._patch.set(**self.plot_opts)
             self._patch.set_visible(True)
+            self._axes.add_patch(self._patch)
         else:
+            if self._patch in self._axes.patches:
+                self._patch._remove_method(self._patch)
             self._patch.set_visible(False)
 
     def __str__(self):
@@ -1264,7 +1266,6 @@ def __init__(self, axes, data_space=True):
             trans = self._axes.transAxes
         self._patch = Rectangle((0., 0.), 1., 1., transform=trans, zorder=100)
         self._patch.set_visible(False)
-        self._axes.add_patch(self._patch)
 
     def start_selection(self, event):
 
@@ -1336,7 +1337,10 @@ def _sync_patch(self):
             self._patch.set_height(1)
             self._patch.set(**self.plot_opts)
             self._patch.set_visible(True)
+            self._axes.add_patch(self._patch)
         else:
+            if self._patch in self._axes.patches:
+                self._patch._remove_method(self._patch)
             self._patch.set_visible(False)
 
 
@@ -1368,7 +1372,6 @@ def __init__(self, axes, data_space=True):
             trans = self._axes.transAxes
         self._patch = Rectangle((0., 0.), 1., 1., transform=trans, zorder=100)
         self._patch.set_visible(False)
-        self._axes.add_patch(self._patch)
 
     def start_selection(self, event):
 
@@ -1442,7 +1445,10 @@ def _sync_patch(self):
             self._patch.set_width(1)
             self._patch.set(**self.plot_opts)
             self._patch.set_visible(True)
+            self._axes.add_patch(self._patch)
         else:
+            if self._patch in self._axes.patches:
+                self._patch._remove_method(self._patch)
             self._patch.set_visible(False)
 
 
@@ -1476,7 +1482,6 @@ def __init__(self, axes, data_space=True):
         self._patch = Ellipse((0., 0.), transform=IdentityTransform(),
                               width=0., height=0., zorder=100)
         self._patch.set_visible(False)
-        self._axes.add_patch(self._patch)
 
     def _sync_patch(self):
         if self._roi.defined():
@@ -1487,7 +1492,10 @@ def _sync_patch(self):
             self._patch.height = 2. * r
             self._patch.set(**self.plot_opts)
             self._patch.set_visible(True)
+            self._axes.add_patch(self._patch)
         else:
+            if self._patch in self._axes.patches:
+                self._patch._remove_method(self._patch)
             self._patch.set_visible(False)
 
     def start_selection(self, event):
@@ -1616,16 +1624,17 @@ def __init__(self, axes, roi=None, data_space=True):
         self._patch.set_visible(False)
         if not self._data_space:
             self._patch.set_transform(self._axes.transAxes)
-        self._axes.add_patch(self._patch)
 
     def _sync_patch(self):
         if self._roi.defined():
             x, y = self._roi.to_polygon()
-            self._patch.set_xy(list(zip(x + [x[0]],
-                                        y + [y[0]])))
+            self._patch.set_xy(list(zip(x + [x[0]], y + [y[0]])))
             self._patch.set_visible(True)
             self._patch.set(**self.plot_opts)
+            self._axes.add_patch(self._patch)
         else:
+            if self._patch in self._axes.patches:
+                self._patch._remove_method(self._patch)
             self._patch.set_visible(False)
 
     def start_selection(self, event, scrubbing=False):