diff --git a/quinn-proto/src/connection/mod.rs b/quinn-proto/src/connection/mod.rs
index 228ac839..7043abb5 100644
--- a/quinn-proto/src/connection/mod.rs
+++ b/quinn-proto/src/connection/mod.rs
@@ -4279,18 +4279,27 @@ impl Connection {
                         .get_mut(&path_id)
                         .expect("payload is processed only after the path becomes known");
 
+                    use PathTimer::*;
                     use paths::OnPathResponseReceived::*;
                     match path.data.on_path_response_received(now, response.0, remote) {
                         OnPath { was_open } => {
-                            // TODO(@divma): reset timers using the remaining off-path challenges
-                            use PathTimer::*;
                             let qlog = self.qlog.with_time(now);
 
                             self.timers
                                 .stop(Timer::PerPath(path_id, PathValidation), qlog.clone());
                             self.timers
-                                .stop(Timer::PerPath(path_id, PathChallengeLost), qlog.clone());
-                            self.timers.stop(Timer::PerPath(path_id, PathOpen), qlog);
+                                .stop(Timer::PerPath(path_id, PathOpen), qlog.clone());
+
+                            let next_challenge = path
+                                .data
+                                .earliest_expiring_challenge()
+                                .map(|time| time + self.ack_frequency.max_ack_delay_for_pto());
+                            self.timers.set_or_stop(
+                                Timer::PerPath(path_id, PathChallengeLost),
+                                next_challenge,
+                                qlog,
+                            );
+
                             if !was_open {
                                 self.events
                                     .push_back(Event::Path(PathEvent::Opened { id: path_id }));
@@ -4309,6 +4318,15 @@ impl Connection {
                         }
                         OffPath => {
                             debug!("Response to off-path PathChallenge!");
+                            let next_challenge = path
+                                .data
+                                .earliest_expiring_challenge()
+                                .map(|time| time + self.ack_frequency.max_ack_delay_for_pto());
+                            self.timers.set_or_stop(
+                                Timer::PerPath(path_id, PathChallengeLost),
+                                next_challenge,
+                                self.qlog.with_time(now),
+                            );
                         }
                         Invalid { expected } => {
                             debug!(%response, from=%remote, %expected, "ignoring invalid PATH_RESPONSE")
diff --git a/quinn-proto/src/connection/paths.rs b/quinn-proto/src/connection/paths.rs
index e787f9fe..76bdaa47 100644
--- a/quinn-proto/src/connection/paths.rs
+++ b/quinn-proto/src/connection/paths.rs
@@ -385,6 +385,19 @@ impl PathData {
         }
     }
 
+    /// The earliest time at which a sent challenge is considered lost.
+    pub(super) fn earliest_expiring_challenge(&self) -> Option<Instant> {
+        if self.challenges_sent.is_empty() {
+            return None;
+        }
+        let pto = self.rtt.pto_base();
+        self.challenges_sent
+            .values()
+            .map(|info| info.sent_instant)
+            .min()
+            .map(|sent_instant| sent_instant + pto)
+    }
+
     /// Handle receiving a PATH_RESPONSE.
     pub(super) fn on_path_response_received(
         &mut self,
diff --git a/quinn-proto/src/connection/timer.rs b/quinn-proto/src/connection/timer.rs
index dad32088..349185e4 100644
--- a/quinn-proto/src/connection/timer.rs
+++ b/quinn-proto/src/connection/timer.rs
@@ -269,6 +269,18 @@ impl TimerTable {
         qlog.emit_timer_set(timer, time);
     }
 
+    pub(super) fn set_or_stop(
+        &mut self,
+        timer: Timer,
+        time: Option<Instant>,
+        qlog: QlogSinkWithTime<'_>,
+    ) {
+        match time {
+            Some(time) => self.set(timer, time, qlog),
+            None => self.stop(timer, qlog),
+        }
+    }
+
     pub(super) fn stop(&mut self, timer: Timer, qlog: QlogSinkWithTime<'_>) {
         match timer {
             Timer::Conn(timer) => {