diff --git a/lib/private/Collaboration/Reference/ReferenceManager.php b/lib/private/Collaboration/Reference/ReferenceManager.php
index 1db87a5649461..06f7ccc66bb5c 100644
--- a/lib/private/Collaboration/Reference/ReferenceManager.php
+++ b/lib/private/Collaboration/Reference/ReferenceManager.php
@@ -117,6 +117,11 @@ public function resolveReference(string $referenceId): ?IReference {
 
 		$reference = $matchedProvider->resolveReference($referenceId);
 		if ($reference) {
+			$cachePrefix = $matchedProvider->getCachePrefix($referenceId);
+			if ($cachePrefix !== '') {
+				// If a prefix is used we set an additional key to know when we need to delete by prefix during invalidateCache()
+				$this->cache->set('hasPrefix-' . md5($cachePrefix), true, self::CACHE_TTL);
+			}
 			$this->cache->set($cacheKey, Reference::toCache($reference), self::CACHE_TTL);
 			return $reference;
 		}
@@ -161,11 +166,15 @@ private function getFullCacheKey(IReferenceProvider $provider, string $reference
 	 */
 	public function invalidateCache(string $cachePrefix, ?string $cacheKey = null): void {
 		if ($cacheKey === null) {
-			$this->cache->clear(md5($cachePrefix));
+			// clear might be a heavy operation, so we only do it if there have actually been keys set
+			if ($this->cache->remove('hasPrefix-' . md5($cachePrefix))) {
+				$this->cache->clear(md5($cachePrefix));
+			}
+
 			return;
 		}
 
-		$this->cache->remove(md5($cachePrefix) . '-' . md5($cacheKey));
+		$this->cache->remove(md5($cachePrefix) . '-' . md5($cacheKey ?? ''));
 	}
 
 	/**