diff --git a/src/resources.rs b/src/resources.rs
index 5968e4c6e54cb0..19fd5ee7810e75 100644
--- a/src/resources.rs
+++ b/src/resources.rs
@@ -200,9 +200,7 @@ impl Resource {
     let r = table.remove(&self.rid);
     assert!(r.is_some());
     // If TcpListener, we must kill all pending accepts!
-    if let Repr::TcpListener(l, m) = r.unwrap() {
-      // Drop first
-      std::mem::drop(l);
+    if let Repr::TcpListener(_, m) = r.unwrap() {
       // Call notify on each task, so that they would error out
       for (_, t) in m {
         t.notify();
diff --git a/src/tokio_util.rs b/src/tokio_util.rs
index 930ad2d0dbd8bf..0bfc24ed785227 100644
--- a/src/tokio_util.rs
+++ b/src/tokio_util.rs
@@ -85,7 +85,10 @@ impl Future for Accept {
           r.track_task(self.task_id);
           return Ok(futures::prelude::Async::NotReady);
         }
-        Err(e) => return Err(From::from(e)),
+        Err(e) => {
+          r.untrack_task(self.task_id);
+          return Err(From::from(e));
+        }
       },
       AcceptState::Empty => panic!("poll Accept after it's done"),
     };