From 9f21bb88765e8c55b512fa96985c987ca08a8eab Mon Sep 17 00:00:00 2001 From: angrybayblade Date: Fri, 6 Oct 2023 10:24:42 +0530 Subject: [PATCH 1/5] feat: download dependencies and populate the third party hashes when running sync --- aea/package_manager/base.py | 6 ++++-- aea/package_manager/v1.py | 8 ++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/aea/package_manager/base.py b/aea/package_manager/base.py index 594adbd784..925e63be07 100644 --- a/aea/package_manager/base.py +++ b/aea/package_manager/base.py @@ -147,7 +147,7 @@ def _sync( sync_needed = True self._logger.info(f"{package_id} not found locally, downloading...") package_id_with_hash = package_id.with_hash(packages[package_id]) - self.add_package(package_id=package_id_with_hash) + self.add_package(package_id=package_id_with_hash, with_dependencies=True) return sync_needed, hash_updates, package_updates @@ -315,6 +315,7 @@ def add_package( if not actual_package_id: # no package on fs, download one + self._logger.info(f"Adding {package_id.without_hash()}") self._fetch_package(package_id) elif not is_update_needed: # actual version already, nothing to do @@ -324,6 +325,7 @@ def add_package( f"Required package and package in the registry does not match: {package_id} vs {actual_package_id}" ) else: + self._logger.info(f"Updating {package_id.without_hash()}") self._update_package(package_id) if with_dependencies: @@ -357,12 +359,12 @@ def _fetch_package(self, package_id: PackageId) -> None: (package_type_collection / "__init__.py").touch() download_path = package_type_collection / package_id.name - fetch_ipfs( str(package_id.package_type), package_id.public_id, dest=str(download_path), ) + self._logger.debug(f"Downloaded {package_id.without_hash()}") def add_dependencies_for_package( self, package_id: PackageId, allow_update: bool = False diff --git a/aea/package_manager/v1.py b/aea/package_manager/v1.py index ad22fa33a7..3e5c7d808e 100644 --- a/aea/package_manager/v1.py +++ b/aea/package_manager/v1.py @@ -199,7 +199,9 @@ def _update_hashes_from_sources(self, sources: List[str]) -> None: self.dump(file=self._packages_file) def register( - self, package_path: Path, package_type: Optional[PackageType] = None + self, + package_path: Path, + package_type: Optional[PackageType] = None, ) -> "PackageManagerV1": """Add package to the index.""" package_type = package_type or PackageType(package_path.parent.name[:-1]) @@ -241,8 +243,8 @@ def sync( update_hashes=update_hashes, update_packages=update_packages, ) - sync_needed = sync_needed or _sync_needed + sync_needed = sync_needed or _sync_needed if update_hashes and hash_updates_third_party: third_party_package_id = "\n\t- ".join( map(str, hash_updates_third_party) @@ -281,6 +283,8 @@ def sync( if sync_needed: self._logger.info("Sync complete") + self._logger.info("Updating hashes") + self.dump() else: self._logger.info("No package was updated.") From d301cb793e10a60992f6ff6270f58ab834f6e46a Mon Sep 17 00:00:00 2001 From: angrybayblade Date: Fri, 6 Oct 2023 10:31:39 +0530 Subject: [PATCH 2/5] fix: mock call asserts in the package manager tests --- tests/test_package_manager/test_v1.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/test_package_manager/test_v1.py b/tests/test_package_manager/test_v1.py index b5cc5b6cb7..bc004b81d3 100644 --- a/tests/test_package_manager/test_v1.py +++ b/tests/test_package_manager/test_v1.py @@ -142,7 +142,12 @@ def test_sync( with mock.patch.object(pm, "add_package") as update_patch: pm.sync(dev=True, third_party=False) - update_patch.assert_called_with(package_id=DUMMY_PACKAGE_ID) + update_patch.assert_called_with( + package_id=DUMMY_PACKAGE_ID.with_hash( + "bafybei0000000000000000000000000000000000000000000000000000" + ), + with_dependencies=True, + ) # test package already exists. with mock.patch.object(pm, "add_package") as update_patch, mock.patch( @@ -204,7 +209,12 @@ def test_sync( with mock.patch.object(pm, "add_package") as update_patch: pm.sync(dev=False, third_party=True) - update_patch.assert_called_with(package_id=DUMMY_PACKAGE_ID) + update_patch.assert_called_with( + package_id=DUMMY_PACKAGE_ID.with_hash( + "bafybei0000000000000000000000000000000000000000000000000000" + ), + with_dependencies=True, + ) # 3d part hashes pm = PackageManagerV1( @@ -214,7 +224,12 @@ def test_sync( with mock.patch.object(pm, "add_package") as update_patch: pm.sync(dev=False, third_party=True, update_hashes=True) - update_patch.assert_called_with(package_id=DUMMY_PACKAGE_ID) + update_patch.assert_called_with( + package_id=DUMMY_PACKAGE_ID.with_hash( + "bafybei0000000000000000000000000000000000000000000000000000" + ), + with_dependencies=True, + ) # 3d part packages with mock.patch.object(pm, "update_package") as update_patch, mock.patch.object( From 8ce0f465b28a5db80c64b0a2dcdb9a7ccfa3640e Mon Sep 17 00:00:00 2001 From: angrybayblade Date: Fri, 6 Oct 2023 11:39:57 +0530 Subject: [PATCH 3/5] fix: package manager tests --- tests/test_package_manager/test_base.py | 2 ++ tests/test_package_manager/test_v0.py | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_package_manager/test_base.py b/tests/test_package_manager/test_base.py index 66f346cd92..d64f472bbe 100644 --- a/tests/test_package_manager/test_base.py +++ b/tests/test_package_manager/test_base.py @@ -20,6 +20,7 @@ """Test package manager base.""" +import logging import re from collections import OrderedDict from pathlib import Path @@ -93,6 +94,7 @@ def __init__( self.path = path self.packages = packages self.config_loader = config_loader + self._logger = logging.getLogger() @classmethod def from_dir( diff --git a/tests/test_package_manager/test_v0.py b/tests/test_package_manager/test_v0.py index ced9fa3b72..a28330be1d 100644 --- a/tests/test_package_manager/test_v0.py +++ b/tests/test_package_manager/test_v0.py @@ -101,7 +101,12 @@ def test_sync( with mock.patch.object(pm, "add_package") as update_patch: pm.sync() - update_patch.assert_called_with(package_id=DUMMY_PACKAGE_ID) + update_patch.assert_called_with( + package_id=DUMMY_PACKAGE_ID.with_hash( + "bafybei0000000000000000000000000000000000000000000000000000" + ), + with_dependencies=True, + ) with pytest.raises( ValueError, From 99f445b3f47c49bc1439a71c1641d793288f5482 Mon Sep 17 00:00:00 2001 From: angrybayblade Date: Fri, 6 Oct 2023 12:42:29 +0530 Subject: [PATCH 4/5] fix: sync tests --- tests/test_cli/test_package_manager/test_sync.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_cli/test_package_manager/test_sync.py b/tests/test_cli/test_package_manager/test_sync.py index ef495c95d7..99d57f61f6 100644 --- a/tests/test_cli/test_package_manager/test_sync.py +++ b/tests/test_cli/test_package_manager/test_sync.py @@ -66,7 +66,9 @@ def test_sync_with_missing_dev_packages(self, ipfs_mock, hash_mock, caplog) -> N with mock.patch.object( PackageManagerV1, "dev_packages", new=packages - ), caplog.at_level(logging.INFO): + ), caplog.at_level(logging.INFO), mock.patch.object( + PackageManagerV1, "add_package" + ): result = self.run_cli_command("packages", "sync", "--dev") assert result.exit_code == 0 assert ( @@ -87,7 +89,9 @@ def test_sync_with_missing_third_party_packages( with mock.patch.object( PackageManagerV1, "third_party_packages", new=packages - ), caplog.at_level(logging.INFO): + ), caplog.at_level(logging.INFO), mock.patch.object( + PackageManagerV1, "add_package" + ): result = self.run_cli_command("packages", "sync") assert result.exit_code == 0 assert ( From 4c6b0e78e82afa01b253f0254aa4fa34f2bfc7bf Mon Sep 17 00:00:00 2001 From: angrybayblade Date: Fri, 6 Oct 2023 12:43:04 +0530 Subject: [PATCH 5/5] fix: copyright --- tests/test_cli/test_package_manager/test_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cli/test_package_manager/test_sync.py b/tests/test_cli/test_package_manager/test_sync.py index 99d57f61f6..0c35ded4c8 100644 --- a/tests/test_cli/test_package_manager/test_sync.py +++ b/tests/test_cli/test_package_manager/test_sync.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2022 Valory AG +# Copyright 2022-2023 Valory AG # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.