diff --git a/sessionpeermanager/sessionpeermanager.go b/sessionpeermanager/sessionpeermanager.go
index 471e982e..b6fafe09 100644
--- a/sessionpeermanager/sessionpeermanager.go
+++ b/sessionpeermanager/sessionpeermanager.go
@@ -328,7 +328,12 @@ type peerTimeoutMessage struct {
 
 func (ptm *peerTimeoutMessage) handle(spm *SessionPeerManager) {
 	data, ok := spm.activePeers[ptm.p]
-	if !ok || !data.lt.WasCancelled(ptm.k) {
+	// If the request was cancelled, make sure we clean up the request tracker
+	if ok && data.lt.WasCancelled(ptm.k) {
+		data.lt.RemoveRequest(ptm.k)
+	} else {
+		// If the request was not cancelled, record the latency. Note that we
+		// do this even if we didn't previously know about this peer.
 		spm.recordResponse(ptm.p, ptm.k)
 	}
 }
diff --git a/sessionpeermanager/sessionpeermanager_test.go b/sessionpeermanager/sessionpeermanager_test.go
index c0d6512b..c743cfb7 100644
--- a/sessionpeermanager/sessionpeermanager_test.go
+++ b/sessionpeermanager/sessionpeermanager_test.go
@@ -342,12 +342,18 @@ func TestTimeoutsAndCancels(t *testing.T) {
 	sessionPeerManager.RecordCancel(c4[0])
 	time.Sleep(2 * time.Millisecond)
 	sessionPeerManager.RecordPeerResponse(peer2, c4[0])
+	time.Sleep(2 * time.Millisecond)
 
 	// call again
 	fourthSessionPeers := sessionPeerManager.GetOptimizedPeers()
 	if thirdSessionPeers[1].OptimizationRating >= fourthSessionPeers[1].OptimizationRating {
 		t.Fatal("Timeout should have affected optimization rating but did not")
 	}
+
+	// ensure all peer latency tracking has been cleaned up
+	if len(sessionPeerManager.activePeers[peer2].lt.requests) > 0 {
+		t.Fatal("Latency request tracking should have been cleaned up but was not")
+	}
 }
 
 func TestUntaggingPeers(t *testing.T) {