diff --git a/protocols/kad/src/handler_priv.rs b/protocols/kad/src/handler_priv.rs
index 7901b7e10a2b..99bd1485b176 100644
--- a/protocols/kad/src/handler_priv.rs
+++ b/protocols/kad/src/handler_priv.rs
@@ -759,7 +759,13 @@ where
 
         if self.outbound_substreams.is_empty() && self.inbound_substreams.is_empty() {
             // We destroyed all substreams in this function.
-            self.keep_alive = KeepAlive::Until(Instant::now() + self.config.idle_timeout);
+            self.keep_alive = if let KeepAlive::Until(time) = self.keep_alive {
+                // Preserve the existing idle timeout
+                KeepAlive::Until(time)
+            } else {
+                // Setup idle timeout
+                KeepAlive::Until(Instant::now() + self.config.idle_timeout)
+            };
         } else {
             self.keep_alive = KeepAlive::Yes;
         }