diff --git a/lib/media/segment_prefetch.js b/lib/media/segment_prefetch.js
index 0ef35de308..b4f0b5b8c2 100644
--- a/lib/media/segment_prefetch.js
+++ b/lib/media/segment_prefetch.js
@@ -233,6 +233,15 @@ shaka.media.SegmentPrefetch = class {
     this.prefetchPosTime_ = 0;
   }
 
+  /**
+   * Remove a reference of prefetched segment if already exists.
+   * @param {!shaka.media.SegmentReference} reference
+   * @public
+   */
+  removeReference(reference) {
+    this.abortPrefetchedSegment_(reference);
+  }
+
   /**
    * @param {number} time
    * @param {boolean=} clearInitSegments
diff --git a/lib/media/streaming_engine.js b/lib/media/streaming_engine.js
index b1bac5e0f2..df474789ed 100644
--- a/lib/media/streaming_engine.js
+++ b/lib/media/streaming_engine.js
@@ -1702,6 +1702,11 @@ shaka.media.StreamingEngine = class {
             error.code);
         mediaState.hasError = true;
 
+        if (error.category == shaka.util.Error.Category.NETWORK &&
+            mediaState.segmentPrefetch) {
+          mediaState.segmentPrefetch.removeReference(reference);
+        }
+
         error.severity = shaka.util.Error.Severity.CRITICAL;
         await this.handleStreamingError_(mediaState, error);
       }