From ce8e0d1ddc9e8968958313e82481d46498d51f25 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Sun, 16 Jun 2024 17:31:59 +0530 Subject: [PATCH] Handle duplicate aliases in yarn v1 Signed-off-by: Ayan Sinha Mahapatra --- src/packagedcode/npm.py | 16 +- .../data/npm/yarn-lock/v1-complex3/yarn.lock | 48 +++ .../yarn-lock/v1-complex3/yarn.lock-expected | 399 ++++++++++++++++++ tests/packagedcode/test_npm.py | 15 +- 4 files changed, 469 insertions(+), 9 deletions(-) create mode 100644 tests/packagedcode/data/npm/yarn-lock/v1-complex3/yarn.lock create mode 100644 tests/packagedcode/data/npm/yarn-lock/v1-complex3/yarn.lock-expected diff --git a/src/packagedcode/npm.py b/src/packagedcode/npm.py index 5d35634d628..6085c67fb55 100644 --- a/src/packagedcode/npm.py +++ b/src/packagedcode/npm.py @@ -881,7 +881,7 @@ def parse(cls, location, package_only=False): with io.open(location, encoding='utf-8') as yl: yl_dependencies = yl.read().split('\n\n') - dependencies = [] + dependencies_by_purl = {} for yl_dependency in yl_dependencies: lines = yl_dependency.splitlines(False) if all(l.startswith('#') or not l.strip() for l in lines): @@ -986,15 +986,15 @@ def parse(cls, location, package_only=False): resolved_package_data.dependencies.append(subdep) # we create a purl with a version, since we are resolved - dep_purl = PackageURL( + dep_purl = str(PackageURL( type=cls.default_package_type, namespace=ns, name=name, version=version, - ) + )) dep = models.DependentPackage( - purl=str(dep_purl), + purl=dep_purl, extracted_requirement=extracted_requirement, is_resolved=True, # FIXME: these are NOT correct @@ -1004,8 +1004,14 @@ def parse(cls, location, package_only=False): is_direct=False, resolved_package=resolved_package_data.to_dict(), ) - dependencies.append(dep.to_dict()) + if not dep_purl in dependencies_by_purl: + dependencies_by_purl[dep_purl] = dep.to_dict() + else: + # We have duplicate dependencies because of aliases + pass + + dependencies = list(dependencies_by_purl.values()) update_dependencies_as_resolved(dependencies=dependencies) package_data = dict( datasource_id=cls.datasource_id, diff --git a/tests/packagedcode/data/npm/yarn-lock/v1-complex3/yarn.lock b/tests/packagedcode/data/npm/yarn-lock/v1-complex3/yarn.lock new file mode 100644 index 00000000000..3b025cd4893 --- /dev/null +++ b/tests/packagedcode/data/npm/yarn-lock/v1-complex3/yarn.lock @@ -0,0 +1,48 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + +wrap-ansi@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-5.1.0.tgz#1fd1f67235d5b6d0fee781056001bfb694c03b09" + integrity sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q== + dependencies: + ansi-styles "^3.2.0" + string-width "^3.0.0" + strip-ansi "^5.0.0" + +wrap-ansi@^6.2.0: + version "6.2.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-6.2.0.tgz#e9393ba07102e6c91a3b221478f0257cd2856e53" + integrity sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + +wrap-ansi@^8.1.0: + version "8.1.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" + integrity sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ== + dependencies: + ansi-styles "^6.1.0" + string-width "^5.0.1" + strip-ansi "^7.0.1" \ No newline at end of file diff --git a/tests/packagedcode/data/npm/yarn-lock/v1-complex3/yarn.lock-expected b/tests/packagedcode/data/npm/yarn-lock/v1-complex3/yarn.lock-expected new file mode 100644 index 00000000000..b9d26811937 --- /dev/null +++ b/tests/packagedcode/data/npm/yarn-lock/v1-complex3/yarn.lock-expected @@ -0,0 +1,399 @@ +[ + { + "type": "npm", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:npm/wrap-ansi@7.0.0", + "extracted_requirement": "^7.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": false, + "resolved_package": { + "type": "npm", + "namespace": "", + "name": "wrap-ansi", + "version": "7.0.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "size": null, + "sha1": "67e145cff510a6a6984bdf1152911d69d2eb9e43", + "md5": null, + "sha256": null, + "sha512": "6151888f691a98b493c70e8db198e80717d2c2c9f4c9c75eb26738a7e436d5ce733ee675a65f8d7f155dc4fb5d1ef98d54e43a5d2606e0052dcadfc58bb0f5e9", + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": true, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:npm/ansi-styles", + "extracted_requirement": "^4.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/string-width", + "extracted_requirement": "^4.1.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/strip-ansi", + "extracted_requirement": "^6.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://www.npmjs.com/package/wrap-ansi", + "repository_download_url": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "api_data_url": "https://registry.npmjs.org/wrap-ansi/7.0.0", + "datasource_id": "yarn_lock_v1", + "purl": "pkg:npm/wrap-ansi@7.0.0" + }, + "extra_data": {} + }, + { + "purl": "pkg:npm/wrap-ansi@5.1.0", + "extracted_requirement": "^5.1.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": false, + "resolved_package": { + "type": "npm", + "namespace": "", + "name": "wrap-ansi", + "version": "5.1.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-5.1.0.tgz", + "size": null, + "sha1": "1fd1f67235d5b6d0fee781056001bfb694c03b09", + "md5": null, + "sha256": null, + "sha512": "402d7f88dff6fd13d52798d82bc046b6d8f9cfcdcb9922a6bdbbeb5cf3422d94846f7d8a2950c90e5fcc3add8dd35a94d87fc593311af4f2ada3506a0e3b5ded", + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": true, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:npm/ansi-styles", + "extracted_requirement": "^3.2.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/string-width", + "extracted_requirement": "^3.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/strip-ansi", + "extracted_requirement": "^5.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://www.npmjs.com/package/wrap-ansi", + "repository_download_url": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz", + "api_data_url": "https://registry.npmjs.org/wrap-ansi/5.1.0", + "datasource_id": "yarn_lock_v1", + "purl": "pkg:npm/wrap-ansi@5.1.0" + }, + "extra_data": {} + }, + { + "purl": "pkg:npm/wrap-ansi@6.2.0", + "extracted_requirement": "^6.2.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": false, + "resolved_package": { + "type": "npm", + "namespace": "", + "name": "wrap-ansi", + "version": "6.2.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "size": null, + "sha1": "e9393ba07102e6c91a3b221478f0257cd2856e53", + "md5": null, + "sha256": null, + "sha512": "afa94f7011b1657948732984bbb227c43321756d0a0f1a4b82814b720b9ab3109a27f48e219c0835ab4af4a63fb5ff99ae5cb038a5345038f70135d405fc495c", + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": true, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:npm/ansi-styles", + "extracted_requirement": "^4.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/string-width", + "extracted_requirement": "^4.1.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/strip-ansi", + "extracted_requirement": "^6.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://www.npmjs.com/package/wrap-ansi", + "repository_download_url": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "api_data_url": "https://registry.npmjs.org/wrap-ansi/6.2.0", + "datasource_id": "yarn_lock_v1", + "purl": "pkg:npm/wrap-ansi@6.2.0" + }, + "extra_data": {} + }, + { + "purl": "pkg:npm/wrap-ansi@8.1.0", + "extracted_requirement": "^8.1.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": false, + "resolved_package": { + "type": "npm", + "namespace": "", + "name": "wrap-ansi", + "version": "8.1.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "size": null, + "sha1": "56dc22368ee570face1b49819975d9b9a5ead214", + "md5": null, + "sha256": null, + "sha512": "b22ed0588eb350cab9e9b11216f6a0b66ccc7463ada317d1f927b3d753286df73bb66f9591472493d6d6d9479f7d319551b3a4b31992c34000da0b3c83bd4d09", + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": true, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:npm/ansi-styles", + "extracted_requirement": "^6.1.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/string-width", + "extracted_requirement": "^5.0.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/strip-ansi", + "extracted_requirement": "^7.0.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://www.npmjs.com/package/wrap-ansi", + "repository_download_url": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "api_data_url": "https://registry.npmjs.org/wrap-ansi/8.1.0", + "datasource_id": "yarn_lock_v1", + "purl": "pkg:npm/wrap-ansi@8.1.0" + }, + "extra_data": {} + } + ], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "yarn_lock_v1", + "purl": null + } +] \ No newline at end of file diff --git a/tests/packagedcode/test_npm.py b/tests/packagedcode/test_npm.py index 70dd5dc71a9..05d7ed244f3 100644 --- a/tests/packagedcode/test_npm.py +++ b/tests/packagedcode/test_npm.py @@ -304,17 +304,24 @@ def test_parse_yarn_lock_v1(self): packages = npm.YarnLockV1Handler.parse(test_file) self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES) - def test_parse_complex_yarn_lock_v1(self): + def test_parse_yarn_lock_v1_complex_with_aliases(self): + test_file = self.get_test_loc('npm/yarn-lock/v1-complex/yarn.lock') + expected_loc = self.get_test_loc( + 'npm/yarn-lock/v1-complex/yarn.lock-expected') + packages = npm.YarnLockV1Handler.parse(test_file) + self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES) + + def test_parse_yarn_lock_v1_complex_with_multiple_requirements(self): test_file = self.get_test_loc('npm/yarn-lock/v1-complex2/yarn.lock') expected_loc = self.get_test_loc( 'npm/yarn-lock/v1-complex2/yarn.lock-expected') packages = npm.YarnLockV1Handler.parse(test_file) self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES) - def test_parse_yarn_lock_v1_complex(self): - test_file = self.get_test_loc('npm/yarn-lock/v1-complex/yarn.lock') + def test_parse_yarn_lock_v1_complex_with_duplicate_aliases(self): + test_file = self.get_test_loc('npm/yarn-lock/v1-complex3/yarn.lock') expected_loc = self.get_test_loc( - 'npm/yarn-lock/v1-complex/yarn.lock-expected') + 'npm/yarn-lock/v1-complex3/yarn.lock-expected') packages = npm.YarnLockV1Handler.parse(test_file) self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES)