diff --git a/src/Swarm.cs b/src/Swarm.cs
index ccc0ccc..3fe5a29 100644
--- a/src/Swarm.cs
+++ b/src/Swarm.cs
@@ -889,10 +889,6 @@ public async Task StopListeningAsync(MultiAddress address)
                 if (!listener.IsCancellationRequested)
                 {
                     listener.Cancel(false);
-
-                    // Give some time away, so that cancel can run.
-                    // TODO: Would be nice to make this deterministic.
-                    await Task.Delay(TimeSpan.FromMilliseconds(100));
                 }
 
                 // Remove any local peer address that depend on the cancellation token.
@@ -910,6 +906,10 @@ public async Task StopListeningAsync(MultiAddress address)
                 {
                     listeners.TryRemove(other, out CancellationTokenSource _);
                 }
+
+                // Give some time away, so that cancel can run
+                // TODO: Would be nice to make this deterministic.
+                await Task.Delay(TimeSpan.FromMilliseconds(100)).ConfigureAwait(false);
             }
             catch (Exception e)
             {