Performance(diff): faster rename detection

src/dvc_data/index/diff.py

@@ -1,6 +1,7 @@
-from collections import deque
+import itertools
+from collections import defaultdict, deque
 from collections.abc import Iterable
-from typing import TYPE_CHECKING, Any, Callable, Optional
+from typing import TYPE_CHECKING, Any, Callable, Optional, cast
 
 from attrs import define
 from fsspec.callbacks import DEFAULT_CALLBACK, Callback
@@ -246,44 +247,56 @@ def _detect_renames(changes: Iterable[Change]):
 
     for change in changes:
         if change.typ == ADD:
+            assert change.new
             added.append(change)
         elif change.typ == DELETE:
+            assert change.old
             deleted.append(change)
         else:
             yield change
 
     def _get_key(change):
         return change.key
 
-    added[:] = sorted(added, key=_get_key)
-    deleted[:] = sorted(deleted, key=_get_key)
+    # Sort the lists to maintain the same order
+    # as older implementation.
+    added.sort(key=_get_key)
+    deleted.sort(key=_get_key)
+
+    # Create a dictionary for fast lookup of deletions by hash_info
+    deleted_dict: dict[Optional[HashInfo], deque[Change]] = defaultdict(deque)
+    for change in deleted:
+        # We checked change.old for all deleted above, so cast
+        change_hash = cast(DataIndexEntry, change.old).hash_info
+        # appendleft to get queue behaviour (we pop off right)
+        deleted_dict[change_hash].appendleft(change)
 
     for change in added:
-        new_entry = change.new
-        assert new_entry
+        # We checked change.new for all new above, so cast
+        new_entry = cast(DataIndexEntry, change.new)
 
         if not new_entry.hash_info:
             yield change
             continue
 
-        index, old_entry = None, None
-        for idx, ch in enumerate(deleted):
-            assert ch.old
-            if ch.old.hash_info == new_entry.hash_info:
-                index, old_entry = idx, ch.old
-                break
+        # If the new entry is the same as a deleted change,
+        # it is in fact a rename.
+        # Note: get instead of __getitem__, to avoid creating
+        # unnecessary entries.
+        if deleted_dict.get(new_entry.hash_info):
+            deletion = deleted_dict[new_entry.hash_info].pop()
 
-        if index is not None:
-            del deleted[index]
             yield Change(
                 RENAME,
-                old_entry,
+                deletion.old,
                 new_entry,
             )
         else:
             yield change
 
-    yield from deleted
+    # Yield the remaining unmatched deletions
+    if deleted_dict:
+        yield from itertools.chain.from_iterable(deleted_dict.values())
 
 
 def diff(  # noqa: PLR0913