diff --git a/modules/idlemove.py b/modules/idlemove.py
index 0ac6172..dc5739d 100644
--- a/modules/idlemove.py
+++ b/modules/idlemove.py
@@ -121,7 +121,7 @@ def UpdateUserAutoAway(self, server, user):
         try:
             index = self.affectedusers[sid]
         except KeyError:
-            self.affectedusers[sid] = set()
+            self.affectedusers[sid] = {}
             index = self.affectedusers[sid]
 
         # Check if the user is whitelisted
@@ -165,7 +165,7 @@ def UpdateUserAutoAway(self, server, user):
                     update = True
 
                 if update:
-                    index.add(user.session)
+                    index[user.session] = user.channel
                     log.info(
                         '%ds > %ds: State transition for user %s (%d/%d) from mute %s -> %s / deaf %s -> %s | channel %d -> %d on server %d',
                         user.idlesecs, threshold, user.name, user.session, user.userid, user.mute, mute, user.deaf,
@@ -176,8 +176,10 @@ def UpdateUserAutoAway(self, server, user):
         if not over_threshold and user.session in self.affectedusers[sid]:
             deafen = False
             mute = False
-            channel = user.channel
-            index.remove(user.session)
+            try:
+                channel = index.pop(user.session)
+            except KeyError:
+                log.warning("Missing previous channel information on restore")
             log.info("Restore user %s (%d/%d) on server %d", user.name, user.session, user.userid, server.id())
             update = True
 
@@ -222,10 +224,10 @@ def channelStateChanged(self, server, state, context=None):
 
     def started(self, server, context=None):
         sid = server.id()
-        self.affectedusers[sid] = set()
+        self.affectedusers[sid] = {}
         self.log().debug('Handling server %d', sid)
 
     def stopped(self, server, context=None):
         sid = server.id()
-        self.affectedusers[sid] = set()
+        self.affectedusers[sid] = {}
         self.log().debug('Server %d gone', sid)