Skip to content

Commit

Permalink
debug
Browse files Browse the repository at this point in the history
  • Loading branch information
efiop committed Jan 25, 2024
1 parent c472553 commit 1301d92
Show file tree
Hide file tree
Showing 2 changed files with 27 additions and 29 deletions.
7 changes: 3 additions & 4 deletions src/dvc_data/index/fetch.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,8 @@ def _filter_changed(index):
meta = Meta.from_info(info)
old = getattr(entry.meta, data_fs.PARAM_CHECKSUM, None) if entry.meta else None
new = getattr(meta, data_fs.PARAM_CHECKSUM, None)
if old and new:
if old == new:
ret.add(entry)
if old and new and old == new:
ret.add(entry)

return ret

Expand Down Expand Up @@ -132,7 +131,7 @@ def fetch(
fetched += len(result.transferred)
failed += len(result.failed)
elif isinstance(cache, ObjectStorage):
updated = md5(fs_index, check_meta=True)
updated = md5(fs_index)

def _on_error(failed, oid, exc):
if isinstance(exc, FileNotFoundError):
Expand Down
49 changes: 24 additions & 25 deletions src/dvc_data/index/save.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,29 @@
from .index import BaseDataIndex, DataIndex, DataIndexKey


def _meta_matches(fs, path, old_meta):
try:
info = fs.info(path)
except FileNotFoundError:
return False

if getattr(fs, "immutable", False):
return True

new_meta = Meta.from_info(info, fs.protocol)
old = getattr(old_meta, fs.PARAM_CHECKSUM, None) if old_meta else None
new = getattr(new_meta, fs.PARAM_CHECKSUM, None)
if not old or not new:
return None

return old == new


def md5(
index: "BaseDataIndex",
state: Optional["StateBase"] = None,
storage: str = "data",
name: str = DEFAULT_ALGORITHM,
check_meta: bool = True,
) -> "DataIndex":
from .index import DataIndex, DataIndexEntry

Expand All @@ -37,37 +54,19 @@ def md5(
if entry.hash_info and entry.hash_info.name in ("md5", "md5-dos2unix"):
hash_info = entry.hash_info

if hash_info and not check_meta:
ret.add(entry)
continue

try:
fs, path = index.storage_map.get_storage(entry, storage)
except ValueError:
continue

info = None
if check_meta:
try:
info = fs.info(path)
except FileNotFoundError:
continue

if getattr(fs, "immutable", False):
ret.add(entry)
else:
meta = Meta.from_info(info, fs.protocol)
old = (
getattr(entry.meta, fs.PARAM_CHECKSUM, None) if entry.meta else None
)
new = getattr(meta, fs.PARAM_CHECKSUM, None)
if old and new:
if old == new:
ret.add(entry)
continue
matches = _meta_matches(fs, path, entry.meta)
if matches:
ret.add(entry)
elif matches is not None:
continue

try:
_, hi = hash_file(path, fs, name, state=state, info=info)
_, hi = hash_file(path, fs, name, state=state)
except FileNotFoundError:
continue

Expand Down

0 comments on commit 1301d92

Please sign in to comment.