diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 81d71149..69805e82 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,7 +4,7 @@ Changelog Next Release ---------------- -- Improve web template for API web page https://github.com/nexB/purldb/issues/132 +- Improve web template for API web page https://github.com/aboutcode-org/purldb/issues/132 - The API endpoints for ``approximate_directory_content_index``, ``approximate_directory_structure_index``, ``exact_file_index``, ``exact_package_archive_index``, ``cditems``, ``on_demand_queue`` have been @@ -12,13 +12,13 @@ Next Release - The `/api/collect/` and `/api/collect/index_packages/` API endpoints have been updated such that Package scan and processing requests made with purls with versions are processed ahead of those made with versionless purls. - https://github.com/nexB/purldb/issues/502 + https://github.com/aboutcode-org/purldb/issues/502 - The `/api/scan_queue/` endpoint has been updated. `/api/scan_queue/get_next_download_url/` now returns a `webhook_url`, where the purldb scan worker will submit results. This is a URL to the `/api/scan_queue/index_package_scan/` endpoint. `/api/scan_queue/update_status/` is now an action on a ScannableURI. - https://github.com/nexB/purldb/issues/504 + https://github.com/aboutcode-org/purldb/issues/504 v5.0.0 @@ -27,11 +27,11 @@ v5.0.0 - Add `/api/docs` Swagger API documentation for API endpoints. - Add `/api/from_purl/purl2git` endpoint to get a git repo for a purl. - Add `/api/to_purl/go` endpoint to get a purl from a golang import string or a package string in go.mod. -- Support indexing of PURLs listed in https://github.com/nexB/purldb/issues/326, - https://github.com/nexB/purldb/issues/327, https://github.com/nexB/purldb/issues/328, - https://github.com/nexB/purldb/issues/329 and https://github.com/nexB/purldb/issues/356. -- Support ``addon_pipelines`` for symbol and string collection in ``/api/collect`` endpoint. https://github.com/nexB/purldb/pull/393 -- Store ``source_symbols`` and ``source_strings`` in ``extra_data`` field. https://github.com/nexB/purldb/pull/351 +- Support indexing of PURLs listed in https://github.com/aboutcode-org/purldb/issues/326, + https://github.com/aboutcode-org/purldb/issues/327, https://github.com/aboutcode-org/purldb/issues/328, + https://github.com/aboutcode-org/purldb/issues/329 and https://github.com/aboutcode-org/purldb/issues/356. +- Support ``addon_pipelines`` for symbol and string collection in ``/api/collect`` endpoint. https://github.com/aboutcode-org/purldb/pull/393 +- Store ``source_symbols`` and ``source_strings`` in ``extra_data`` field. https://github.com/aboutcode-org/purldb/pull/351 v3.0.0 diff --git a/Dockerfile b/Dockerfile index 5d91e321..8cb9bcd9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM --platform=linux/amd64 python:3.12-slim -LABEL org.opencontainers.image.source="https://github.com/nexB/purldb" +LABEL org.opencontainers.image.source="https://github.com/aboutcode-org/purldb" LABEL org.opencontainers.image.description="PurlDB" LABEL org.opencontainers.image.licenses="Apache-2.0" diff --git a/Makefile b/Makefile index 8aa2aec5..bc8eeaea 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/NOTICE b/NOTICE index 480e2970..95fbda27 100644 --- a/NOTICE +++ b/NOTICE @@ -7,6 +7,6 @@ # See https://www.apache.org/licenses/LICENSE-2.0 for the license text. # See https://creativecommons.org/licenses/by-sa/4.0/legalcode for the license text. # -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/clearcode/management/commands/clearload.py b/clearcode/management/commands/clearload.py index 7f872dbb..c56c2f48 100644 --- a/clearcode/management/commands/clearload.py +++ b/clearcode/management/commands/clearload.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/clearcode/management/commands/clearsync.py b/clearcode/management/commands/clearsync.py index 5481d8f5..ab7cf94a 100644 --- a/clearcode/management/commands/clearsync.py +++ b/clearcode/management/commands/clearsync.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/clearcode/management/commands/store_scans.py b/clearcode/management/commands/store_scans.py index e5fc5b60..47212f1d 100644 --- a/clearcode/management/commands/store_scans.py +++ b/clearcode/management/commands/store_scans.py @@ -3,13 +3,14 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # from clearcode.store_scans import store_scancode_scans_from_cd_items from minecode.management.commands import VerboseCommand + class Command(VerboseCommand): help = 'Store scancode scans in git repositories' @@ -19,4 +20,5 @@ def add_arguments(self, parser): parser.add_argument('--count', type=int, default=0) def handle(self, *args, **options): - store_scancode_scans_from_cd_items(work_dir=options['work_dir'], github_org=options['github_org'], count=options['count']) \ No newline at end of file + store_scancode_scans_from_cd_items( + work_dir=options['work_dir'], github_org=options['github_org'], count=options['count']) diff --git a/clearindex/harvest.py b/clearindex/harvest.py index 7de66321..2cf4ef42 100644 --- a/clearindex/harvest.py +++ b/clearindex/harvest.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -85,7 +85,8 @@ def create_from_harvest(package_scan={}, files_data=[], cditem_path=''): 'source_packages', ) - package_data = {field_name: package_scan.get(field_name) for field_name in fields} + package_data = {field_name: package_scan.get( + field_name) for field_name in fields} stringify_null_purl_fields(package_data) @@ -98,7 +99,8 @@ def create_from_harvest(package_scan={}, files_data=[], cditem_path=''): download_url = package_data.get('download_url') if not download_url: - logger.error('Null `download_url` value for `package_data`: {}'.format(package_data)) + logger.error( + 'Null `download_url` value for `package_data`: {}'.format(package_data)) return # This ugly block is needed until https://github.com/nexB/packagedb/issues/14 @@ -119,9 +121,11 @@ def create_from_harvest(package_scan={}, files_data=[], cditem_path=''): new_package_data=package_data, replace=False ) - package.append_to_history('Updated package from CDitem harvest: {}'.format(cditem_path)) + package.append_to_history( + 'Updated package from CDitem harvest: {}'.format(cditem_path)) - logger.info('Merged package data from scancode harvest: {}'.format(package)) + logger.info( + 'Merged package data from scancode harvest: {}'.format(package)) except Package.DoesNotExist: try: @@ -132,15 +136,19 @@ def create_from_harvest(package_scan={}, files_data=[], cditem_path=''): new_package_data=package_data, replace=False ) - package.append_to_history('Updated package from CDitem harvest: {}'.format(cditem_path)) + package.append_to_history( + 'Updated package from CDitem harvest: {}'.format(cditem_path)) - logger.info('Merged package data from scancode harvest: {}'.format(package)) + logger.info( + 'Merged package data from scancode harvest: {}'.format(package)) except Package.DoesNotExist: package = Package.objects.create(**package_data) - package.append_to_history('Created package from CDitem harvest: {}'.format(cditem_path)) + package.append_to_history( + 'Created package from CDitem harvest: {}'.format(cditem_path)) - logger.info('Created package from scancode harvest: {}'.format(package)) + logger.info( + 'Created package from scancode harvest: {}'.format(package)) # Now, add resources to the Package. for f in files_data: @@ -179,7 +187,8 @@ def map_scancode_harvest(cditem): try: harvest_data = cditem.data except ValueError: - err_msg = 'CDitemError: empty content field for CDitem: {}'.format(cditem.path) + err_msg = 'CDitemError: empty content field for CDitem: {}'.format( + cditem.path) logger.error(err_msg) cditem.map_error = err_msg @@ -198,7 +207,8 @@ def map_scancode_harvest(cditem): download_url = package_scan.get('download_url') if not download_url: purl = package_scan.get('purl') - err_msg = 'CDitemError: empty download_url for package: {}'.format(purl) + err_msg = 'CDitemError: empty download_url for package: {}'.format( + purl) logger.error(err_msg) cditem.map_error = err_msg diff --git a/clearindex/management/commands/run_clearindex.py b/clearindex/management/commands/run_clearindex.py index b760e9d8..9af70702 100644 --- a/clearindex/management/commands/run_clearindex.py +++ b/clearindex/management/commands/run_clearindex.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -118,8 +118,10 @@ def handle(self, *args, **options): logger.info('Graceful exit of the map loop.') break - mappable_definitions = CDitem.objects.mappable_definitions()[:MAP_BATCH_SIZE] - mappable_scancode_harvests = CDitem.objects.mappable_scancode_harvests()[:MAP_BATCH_SIZE] + mappable_definitions = CDitem.objects.mappable_definitions()[ + :MAP_BATCH_SIZE] + mappable_scancode_harvests = CDitem.objects.mappable_scancode_harvests()[ + :MAP_BATCH_SIZE] try: if not mappable_definitions and not mappable_scancode_harvests: @@ -187,7 +189,7 @@ def get_coords_des_and_lic_from_def(definition): return definition.get('coordinates', {}), definition.get('described', {}), definition.get('licensed', {}) -#CD_TYPES_WITH_SOURCE = ('debsrc', 'npm', 'sourcearchive',) +# CD_TYPES_WITH_SOURCE = ('debsrc', 'npm', 'sourcearchive',) def get_or_create_package_from_cditem_definition(cditem): @@ -197,7 +199,8 @@ def get_or_create_package_from_cditem_definition(cditem): definition = cditem.data if not definition: raise Exception('No data available for this definition') - coordinates, described, licensed = get_coords_des_and_lic_from_def(definition) + coordinates, described, licensed = get_coords_des_and_lic_from_def( + definition) download_url = described.get('urls', {}).get('download', '') if not download_url: @@ -219,7 +222,8 @@ def get_or_create_package_from_cditem_definition(cditem): name = coordinates.get('name') version = coordinates.get('revision') package_type = coordinates.get('type') - converted_package_type = PACKAGE_TYPES_BY_CD_TYPE.get(package_type) or package_type + converted_package_type = PACKAGE_TYPES_BY_CD_TYPE.get( + package_type) or package_type # TODO: Source packages need to be updated for clearlydefined, link source packages to binary packages hashes = described.get('hashes', {}) sha1 = hashes.get('sha1') @@ -227,7 +231,8 @@ def get_or_create_package_from_cditem_definition(cditem): homepage_url = described.get('projectWebsite') release_date = described.get('releaseDate') declared_license = licensed.get('declared') - normalized_license_expression = licensing.get_normalized_expression(declared_license) + normalized_license_expression = licensing.get_normalized_expression( + declared_license) copyrights = get_parties_from_licensed(licensed) copyrights = '\n'.join(copyrights) definition_mining_level = 0 @@ -257,7 +262,8 @@ def get_or_create_package_from_cditem_definition(cditem): ) # log history if package was created if created: - package.append_to_history('Created package from CDitem definition: {}'.format(cditem.path)) + package.append_to_history( + 'Created package from CDitem definition: {}'.format(cditem.path)) else: # TODO: This is temporary until we fold clearindex into minecode mapping @@ -283,7 +289,8 @@ def get_or_create_package_from_cditem_definition(cditem): replace=True ) package = existing_package - package.append_to_history('Updated package from CDitem definition: {}'.format(cditem.path)) + package.append_to_history( + 'Updated package from CDitem definition: {}'.format(cditem.path)) return package @@ -303,7 +310,8 @@ def create_download_url_from_coords(coord): package_type = PACKAGE_TYPES_BY_CD_TYPE.get(ptype) if not package_type: - raise Exception('Unsupported ClearlyDefined package type: {}'.format(ptype)) + raise Exception( + 'Unsupported ClearlyDefined package type: {}'.format(ptype)) get_urls = PACKAGE_TYPES_WITH_GET_URLS.get(package_type) if get_urls: diff --git a/clearindex/utils.py b/clearindex/utils.py index ebe0a5f2..dcf19ba4 100644 --- a/clearindex/utils.py +++ b/clearindex/utils.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index 18e3e574..eec7bcf7 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -17,9 +17,9 @@ Do Your Homework ---------------- Before adding a contribution or create a new issue, take a look at the project’s -`README `_, read through our +`README `_, read through our `documentation `_, -and browse existing `issues `_, +and browse existing `issues `_, to develop some understanding of the project and confirm whether a given issue/feature has previously been discussed. @@ -34,7 +34,7 @@ First Timers You are here to help, but you're a new contributor! No worries, we always welcome newcomer contributors. We maintain some -`good first issues `_ +`good first issues `_ and encourage new contributors to work on those issues for a smooth start. .. warning:: @@ -55,15 +55,15 @@ and encourage new contributors to work on those issues for a smooth start. Report Issues ^^^^^^^^^^^^^ -- Report a new `bug `_; just remember to be +- Report a new `bug `_; just remember to be as specific as possible. -- Create a `new issue `_ to request a +- Create a `new issue `_ to request a feature, submit a feedback, or ask a question. -- Look into existing `bugs `_, +- Look into existing `bugs `_, try to reproduce the issue on your side, and discuss solutions in the comments. .. note:: - Make sure to check existing `issues `_, + Make sure to check existing `issues `_, and :ref:`faq` to confirm whether a given issue or a question has previously been discussed. @@ -72,7 +72,7 @@ Code Contributions Code is contributed to the codebase using **pull requests**. A pull request should always be attached to an existing issue. -When there is no existing issues, start by `creating one `_ +When there is no existing issues, start by `creating one `_ to discuss potential solutions and implementation details before writing any code. We use several conventions to ensure code quality regarding format, testing, and @@ -102,7 +102,7 @@ See also :ref:`testing` for module-specific testing commands. 3. **Commit messages and Developer Certificate of Origin** Follow the instructions at `Writing good Commit Messages `_ - and `check some examples `_. + and `check some examples `_. **You must include a "Signed-off-by" to your commit messages**:: diff --git a/docs/source/how-to-guides/deploy_to_devel.rst b/docs/source/how-to-guides/deploy_to_devel.rst index 805dddb2..36a58be7 100644 --- a/docs/source/how-to-guides/deploy_to_devel.rst +++ b/docs/source/how-to-guides/deploy_to_devel.rst @@ -130,7 +130,7 @@ Install PurlDB and matchcode.io from a clone of the PurlDB git repository Then run these commands:: - git clone https://github.com/nexB/purldb + git clone https://github.com/aboutcode-org/purldb cd purldb make dev make envfile diff --git a/docs/source/how-to-guides/installation.rst b/docs/source/how-to-guides/installation.rst index 27cb6b13..f9847504 100644 --- a/docs/source/how-to-guides/installation.rst +++ b/docs/source/how-to-guides/installation.rst @@ -178,9 +178,9 @@ latest/getting-started/install.html#prerequisites>`_ for more details. Clone and Configure ^^^^^^^^^^^^^^^^^^^ - * Clone the `PurlDB GitHub repository `_:: + * Clone the `PurlDB GitHub repository `_:: - git clone https://github.com/nexB/purldb.git && cd purldb + git clone https://github.com/aboutcode-org/purldb.git && cd purldb * Inside the :guilabel:`purldb/` directory, install the required dependencies:: diff --git a/docs/source/license.rst b/docs/source/license.rst index 62dd2645..deff99be 100644 --- a/docs/source/license.rst +++ b/docs/source/license.rst @@ -15,6 +15,6 @@ See https://www.apache.org/licenses/LICENSE-2.0 for the license text. See https://creativecommons.org/licenses/by-sa/4.0/legalcode for the license text. -See https://github.com/nexB/purldb for support or download. +See https://github.com/aboutcode-org/purldb for support or download. See https://aboutcode.org for more information about nexB OSS projects. diff --git a/docs/source/purldb/rest_api.rst b/docs/source/purldb/rest_api.rst index 564a2bf3..db7ff7a9 100644 --- a/docs/source/purldb/rest_api.rst +++ b/docs/source/purldb/rest_api.rst @@ -1283,7 +1283,7 @@ matching Given a ScanCode.io JSON output ``upload_file``, match directory and resources of the codebase in ``upload_file`` to Packages indexed in the PurlDB. -This endpoint runs the ``matching`` pipeline at https://github.com/nexB/purldb/blob/main/matchcode_pipeline/pipelines/matching.py +This endpoint runs the ``matching`` pipeline at https://github.com/aboutcode-org/purldb/blob/main/matchcode_pipeline/pipelines/matching.py Using cURL to upload a scan for matching: diff --git a/manage_matchcode.py b/manage_matchcode.py index b19f6acb..872f8398 100755 --- a/manage_matchcode.py +++ b/manage_matchcode.py @@ -4,7 +4,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -15,5 +15,6 @@ if __name__ == '__main__': from django.core.management import execute_from_command_line - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'matchcode_project.settings') + os.environ.setdefault('DJANGO_SETTINGS_MODULE', + 'matchcode_project.settings') execute_from_command_line(sys.argv) diff --git a/manage_purldb.py b/manage_purldb.py index d69b6c0f..2dbe57a9 100755 --- a/manage_purldb.py +++ b/manage_purldb.py @@ -4,7 +4,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/matchcode-toolkit/README.rst b/matchcode-toolkit/README.rst index b9c28b7d..2500befd 100644 --- a/matchcode-toolkit/README.rst +++ b/matchcode-toolkit/README.rst @@ -54,4 +54,4 @@ added step of computing fingerprints for directories. .. note:: MatchCode toolkit has moved to its own repo at https://github.com/nexB/matchcode-toolkit - from its previous location https://github.com/nexB/purldb/tree/main/matchcode-toolkit + from its previous location https://github.com/aboutcode-org/purldb/tree/main/matchcode-toolkit diff --git a/matchcode/api.py b/matchcode/api.py index 9810645a..98762547 100644 --- a/matchcode/api.py +++ b/matchcode/api.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # from django.db.models import Q @@ -66,6 +66,7 @@ class BaseDirectoryIndexSerializer(ModelSerializer): read_only=True ) + class ApproximateDirectoryContentIndexSerializer(BaseDirectoryIndexSerializer): class Meta: model = ApproximateDirectoryContentIndex @@ -100,6 +101,7 @@ class CharMultipleWidget(widgets.TextInput): Enables the support for `MultiValueDict` `?field=a&field=b` reusing the `SelectMultiple.value_from_datadict()` but render as a `TextInput`. """ + def value_from_datadict(self, data, files, name): value = widgets.SelectMultiple().value_from_datadict(data, files, name) if not value or value == ['']: @@ -158,6 +160,7 @@ class MultipleSHA1Filter(MultipleCharFilter): Overrides `MultipleCharFilter.filter()` to convert the SHA1 into a bytearray so it can be queried """ + def filter(self, qs, value): if not value: return qs @@ -180,6 +183,7 @@ class MultipleFingerprintFilter(MultipleCharFilter): must process the fingerprint into the correct parts so we can use those parts to query the different fields. """ + def filter(self, qs, value): if not value: return qs diff --git a/matchcode/match.py b/matchcode/match.py index 97da516f..46936bc2 100644 --- a/matchcode/match.py +++ b/matchcode/match.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -115,7 +115,8 @@ def approximate_directory_content_match(codebase): continue match_count += directory_matches.count() - tag_matched_resources(resource, codebase, directory_matches, match_type) + tag_matched_resources(resource, codebase, + directory_matches, match_type) return match_count @@ -135,7 +136,8 @@ def approximate_directory_structure_match(codebase): continue match_count += directory_matches.count() - tag_matched_resources(resource, codebase, directory_matches, match_type) + tag_matched_resources(resource, codebase, + directory_matches, match_type) return match_count @@ -182,7 +184,8 @@ def get_directory_content_match(resource): """ Match a directory to a Package using its contents """ - directory_content_fingerprint = resource.extra_data.get('directory_content', '') + directory_content_fingerprint = resource.extra_data.get( + 'directory_content', '') matches = ApproximateDirectoryContentIndex.objects.none() match_type = '' if directory_content_fingerprint: @@ -200,7 +203,8 @@ def get_directory_structure_match(resource): """ Match a directory to a Package using its structure """ - directory_structure_fingerprint = resource.extra_data.get('directory_structure', '') + directory_structure_fingerprint = resource.extra_data.get( + 'directory_structure', '') matches = ApproximateDirectoryStructureIndex.objects.none() match_type = '' if directory_structure_fingerprint: @@ -276,7 +280,8 @@ def tag_matched_resources(resource, codebase, matches, match_type): # by or), then querying the matched packages resources to see if any of # those suffixes match a package child resource path for child in resource.walk(codebase): - query = reduce(or_, (Q(path=suffix) for suffix in path_suffixes(child.path)), Q()) + query = reduce(or_, (Q(path=suffix) + for suffix in path_suffixes(child.path)), Q()) matched_child_resources = match.package.resources.filter(query) if len(matched_child_resources) > 0: tag_matched_resource(child, codebase, purl) diff --git a/matchcode/models.py b/matchcode/models.py index 079e4198..83aa59fa 100644 --- a/matchcode/models.py +++ b/matchcode/models.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -189,7 +189,8 @@ class ApproximateMatchingHashMixin(models.Model): class Meta: abstract = True - unique_together = ['chunk1', 'chunk2', 'chunk3', 'chunk4', 'package', 'path'] + unique_together = ['chunk1', 'chunk2', + 'chunk3', 'chunk4', 'package', 'path'] def __str__(self): return self.fingerprint() @@ -206,7 +207,8 @@ def index(cls, fingerprint, resource_path, package): """ try: indexed_elements_count, fp = split_fingerprint(fingerprint) - fp_chunk1, fp_chunk2, fp_chunk3, fp_chunk4 = create_halohash_chunks(fp) + fp_chunk1, fp_chunk2, fp_chunk3, fp_chunk4 = create_halohash_chunks( + fp) bdi, created = cls.objects.get_or_create( indexed_elements_count=indexed_elements_count, chunk1=fp_chunk1, @@ -307,7 +309,8 @@ def match(cls, fingerprint, resource=None, exact_match=False): # TODO: try other thresholds if this is too restrictive if hd < 8: # Save match to `matches_by_hamming_distance` by adding the matched object to the queryset - matches_by_hamming_distance[hd] |= cls.objects.filter(pk=match.pk) + matches_by_hamming_distance[hd] |= cls.objects.filter( + pk=match.pk) if TRACE: logger_debug(list(matches_by_hamming_distance.items())) @@ -359,15 +362,18 @@ def match(cls, fingerprint, resource=None, exact_match=False): # Compute size and name difference if matched_resource.is_file: - size_difference = abs(resource_size - matched_resource.size) + size_difference = abs( + resource_size - matched_resource.size) else: # TODO: index number of files in a directory so we can use # that for size comparison. For now, we are going to # disregard size as a factor. size_difference = 0 - name_sequence_matcher = SequenceMatcher(a=resource.name, b=matched_resource.name) + name_sequence_matcher = SequenceMatcher( + a=resource.name, b=matched_resource.name) name_difference = 1 - name_sequence_matcher.ratio() - rank_attributes = (hamming_distance, size_difference, name_difference) + rank_attributes = ( + hamming_distance, size_difference, name_difference) matches_by_rank_attributes[rank_attributes].append(match) if TRACE: @@ -394,7 +400,8 @@ def match(cls, fingerprint, resource=None, exact_match=False): dct ) - matches = cls.objects.filter(pk__in=[match.pk for match in ranked_matches]) + matches = cls.objects.filter( + pk__in=[match.pk for match in ranked_matches]) return matches def get_chunks(self): @@ -407,7 +414,8 @@ def get_chunks(self): def fingerprint(self): indexed_element_count_as_hex_bytes = b'%08x' % self.indexed_elements_count chunk1, chunk2, chunk3, chunk4 = self.get_chunks() - fingerprint = indexed_element_count_as_hex_bytes + chunk1 + chunk2 + chunk3 + chunk4 + fingerprint = indexed_element_count_as_hex_bytes + \ + chunk1 + chunk2 + chunk3 + chunk4 return fingerprint.decode('utf-8') diff --git a/matchcode/tests/__init__.py b/matchcode/tests/__init__.py index 55c388e9..d4b312b7 100644 --- a/matchcode/tests/__init__.py +++ b/matchcode/tests/__init__.py @@ -3,11 +3,11 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -import os +import os FIXTURES_REGEN = os.environ.get("MATCHCODE_TEST_FIXTURES_REGEN", False) diff --git a/matchcode/tests/test_match.py b/matchcode/tests/test_match.py index af68620c..fa9df079 100644 --- a/matchcode/tests/test_match.py +++ b/matchcode/tests/test_match.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -85,9 +85,11 @@ def setUp(self): # Populate ExactPackageArchiveIndexFingerprint table index_packages_sha1() - load_resources_from_scan(self.get_test_loc('models/match-test.json'), self.test_package4) + load_resources_from_scan(self.get_test_loc( + 'models/match-test.json'), self.test_package4) index_package_directories(self.test_package4) - index_package_files_sha1(self.test_package4, self.get_test_loc('models/match-test.json')) + index_package_files_sha1( + self.test_package4, self.get_test_loc('models/match-test.json')) # Add approximate file resource self.test_package5, _ = Package.objects.get_or_create( @@ -113,31 +115,39 @@ def setUp(self): def test_do_match_package_archive_match(self): input_file = self.get_test_loc('models/match-test.json') vc = run_do_match_from_scan(input_file, EXACT_PACKAGE_ARCHIVE_MATCH) - expected = self.get_test_loc('models/match-test-exact-package-results.json') + expected = self.get_test_loc( + 'models/match-test-exact-package-results.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) def test_do_match_approximate_directory_structure_match(self): input_file = self.get_test_loc('models/match-test.json') - vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH) - expected = self.get_test_loc('models/match-test-approximate-directory-structure-results.json') + vc = run_do_match_from_scan( + input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH) + expected = self.get_test_loc( + 'models/match-test-approximate-directory-structure-results.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) def test_do_match_approximate_directory_content_match(self): input_file = self.get_test_loc('models/match-test.json') - vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_CONTENT_MATCH) - expected = self.get_test_loc('models/match-test-approximate-directory-content-results.json') + vc = run_do_match_from_scan( + input_file, APPROXIMATE_DIRECTORY_CONTENT_MATCH) + expected = self.get_test_loc( + 'models/match-test-approximate-directory-content-results.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) def test_do_match_package_file_match(self): input_file = self.get_test_loc('models/match-test.json') vc = run_do_match_from_scan(input_file, EXACT_FILE_MATCH) - expected = self.get_test_loc('models/match-test-exact-file-results.json') + expected = self.get_test_loc( + 'models/match-test-exact-file-results.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) def test_do_match_approximate_package_file_match(self): - input_file = self.get_test_loc('match/approximate-file-matching/approximate-match-test.json') + input_file = self.get_test_loc( + 'match/approximate-file-matching/approximate-match-test.json') vc = run_do_match_from_scan(input_file, APPROXIMATE_FILE_MATCH) - expected = self.get_test_loc('match/approximate-file-matching/approximate-match-test-results.json') + expected = self.get_test_loc( + 'match/approximate-file-matching/approximate-match-test-results.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) @@ -159,7 +169,8 @@ def setUp(self): download_url='https://registry.npmjs.org/@umijs/plugin-request/-/plugin-request-2.4.1.tgz', type='npm', ) - load_resources_from_scan(self.get_test_loc('match/nested/plugin-request-2.4.1-ip.json'), self.test_package1) + load_resources_from_scan(self.get_test_loc( + 'match/nested/plugin-request-2.4.1-ip.json'), self.test_package1) index_package_directories(self.test_package1) self.test_package2, _ = Package.objects.get_or_create( @@ -171,19 +182,24 @@ def setUp(self): download_url='https://registry.npmjs.org/@types/underscore/-/underscore-1.10.9.tgz', type='npm', ) - load_resources_from_scan(self.get_test_loc('match/nested/underscore-1.10.9-ip.json'), self.test_package2) + load_resources_from_scan(self.get_test_loc( + 'match/nested/underscore-1.10.9-ip.json'), self.test_package2) index_package_directories(self.test_package2) def test_do_match_approximate_directory_structure_match(self): input_file = self.get_test_loc('match/nested/nested.json') - vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH) - expected = self.get_test_loc('match/nested/nested-directory-structure-match-expected.json') + vc = run_do_match_from_scan( + input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH) + expected = self.get_test_loc( + 'match/nested/nested-directory-structure-match-expected.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) def test_do_match_approximate_directory_content_match(self): input_file = self.get_test_loc('match/nested/nested.json') - vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_CONTENT_MATCH) - expected = self.get_test_loc('match/nested/nested-directory-content-match-expected.json') + vc = run_do_match_from_scan( + input_file, APPROXIMATE_DIRECTORY_CONTENT_MATCH) + expected = self.get_test_loc( + 'match/nested/nested-directory-content-match-expected.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) @@ -209,7 +225,8 @@ def setUp(self): type='npm', download_url='https://registry.npmjs.org/abbrev/-/abbrev-1.0.3.tgz', ) - load_resources_from_scan(self.get_test_loc('match/directory-matching/abbrev-1.0.3-i.json'), self.test_package1) + load_resources_from_scan(self.get_test_loc( + 'match/directory-matching/abbrev-1.0.3-i.json'), self.test_package1) index_package_directories(self.test_package1) self.test_package2, _ = Package.objects.get_or_create( @@ -220,7 +237,8 @@ def setUp(self): type='npm', download_url='https://registry.npmjs.org/abbrev/-/abbrev-1.0.4.tgz', ) - load_resources_from_scan(self.get_test_loc('match/directory-matching/abbrev-1.0.4-i.json'), self.test_package2) + load_resources_from_scan(self.get_test_loc( + 'match/directory-matching/abbrev-1.0.4-i.json'), self.test_package2) index_package_directories(self.test_package2) self.test_package3, _ = Package.objects.get_or_create( @@ -231,7 +249,8 @@ def setUp(self): type='npm', download_url='https://registry.npmjs.org/abbrev/-/abbrev-1.0.5.tgz', ) - load_resources_from_scan(self.get_test_loc('match/directory-matching/abbrev-1.0.5-i.json'), self.test_package3) + load_resources_from_scan(self.get_test_loc( + 'match/directory-matching/abbrev-1.0.5-i.json'), self.test_package3) index_package_directories(self.test_package3) self.test_package4, _ = Package.objects.get_or_create( @@ -242,7 +261,8 @@ def setUp(self): type='npm', download_url='https://registry.npmjs.org/abbrev/-/abbrev-1.0.6.tgz', ) - load_resources_from_scan(self.get_test_loc('match/directory-matching/abbrev-1.0.6-i.json'), self.test_package4) + load_resources_from_scan(self.get_test_loc( + 'match/directory-matching/abbrev-1.0.6-i.json'), self.test_package4) index_package_directories(self.test_package4) self.test_package5, _ = Package.objects.get_or_create( @@ -253,7 +273,8 @@ def setUp(self): type='npm', download_url='https://registry.npmjs.org/abbrev/-/abbrev-1.0.7.tgz', ) - load_resources_from_scan(self.get_test_loc('match/directory-matching/abbrev-1.0.7-i.json'), self.test_package5) + load_resources_from_scan(self.get_test_loc( + 'match/directory-matching/abbrev-1.0.7-i.json'), self.test_package5) index_package_directories(self.test_package5) self.test_package6, _ = Package.objects.get_or_create( @@ -264,7 +285,8 @@ def setUp(self): type='npm', download_url='https://registry.npmjs.org/abbrev/-/abbrev-1.0.9.tgz', ) - load_resources_from_scan(self.get_test_loc('match/directory-matching/abbrev-1.0.9-i.json'), self.test_package6) + load_resources_from_scan(self.get_test_loc( + 'match/directory-matching/abbrev-1.0.9-i.json'), self.test_package6) index_package_directories(self.test_package6) self.test_package7, _ = Package.objects.get_or_create( @@ -275,7 +297,8 @@ def setUp(self): type='npm', download_url='https://registry.npmjs.org/abbrev/-/abbrev-1.1.0.tgz', ) - load_resources_from_scan(self.get_test_loc('match/directory-matching/abbrev-1.1.0-i.json'), self.test_package7) + load_resources_from_scan(self.get_test_loc( + 'match/directory-matching/abbrev-1.1.0-i.json'), self.test_package7) index_package_directories(self.test_package7) self.test_package8, _ = Package.objects.get_or_create( @@ -286,113 +309,168 @@ def setUp(self): type='npm', download_url='https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz', ) - load_resources_from_scan(self.get_test_loc('match/directory-matching/abbrev-1.1.1-i.json'), self.test_package8) + load_resources_from_scan(self.get_test_loc( + 'match/directory-matching/abbrev-1.1.1-i.json'), self.test_package8) index_package_directories(self.test_package8) def test_match_ApproximateDirectoryStructureIndex_abbrev_1_0_3(self): - input_file = self.get_test_loc('match/directory-matching/abbrev-1.0.3-i.json') - vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH) - expected = self.get_test_loc('match/directory-matching/abbrev-1.0.3-i-expected.json') + input_file = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.3-i.json') + vc = run_do_match_from_scan( + input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH) + expected = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.3-i-expected.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) def test_match_ApproximateDirectoryStructureIndex_abbrev_1_0_4(self): - input_file = self.get_test_loc('match/directory-matching/abbrev-1.0.4-i.json') - vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH) - expected = self.get_test_loc('match/directory-matching/abbrev-1.0.4-i-expected.json') + input_file = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.4-i.json') + vc = run_do_match_from_scan( + input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH) + expected = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.4-i-expected.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) def test_match_ApproximateDirectoryStructureIndex_abbrev_1_0_5(self): - input_file = self.get_test_loc('match/directory-matching/abbrev-1.0.5-i.json') - vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH) - expected = self.get_test_loc('match/directory-matching/abbrev-1.0.5-i-expected.json') + input_file = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.5-i.json') + vc = run_do_match_from_scan( + input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH) + expected = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.5-i-expected.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) def test_match_ApproximateDirectoryStructureIndex_abbrev_1_0_6(self): - input_file = self.get_test_loc('match/directory-matching/abbrev-1.0.6-i.json') - vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH) - expected = self.get_test_loc('match/directory-matching/abbrev-1.0.6-i-expected.json') + input_file = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.6-i.json') + vc = run_do_match_from_scan( + input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH) + expected = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.6-i-expected.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) def test_match_ApproximateDirectoryStructureIndex_abbrev_1_0_7(self): - input_file = self.get_test_loc('match/directory-matching/abbrev-1.0.7-i.json') - vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH) - expected = self.get_test_loc('match/directory-matching/abbrev-1.0.7-i-expected.json') + input_file = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.7-i.json') + vc = run_do_match_from_scan( + input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH) + expected = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.7-i-expected.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) def test_match_ApproximateDirectoryStructureIndex_abbrev_1_0_9(self): - input_file = self.get_test_loc('match/directory-matching/abbrev-1.0.9-i.json') - vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH) - expected = self.get_test_loc('match/directory-matching/abbrev-1.0.9-i-expected.json') + input_file = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.9-i.json') + vc = run_do_match_from_scan( + input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH) + expected = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.9-i-expected.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) def test_match_ApproximateDirectoryStructureIndex_abbrev_1_1_0(self): - input_file = self.get_test_loc('match/directory-matching/abbrev-1.1.0-i.json') - vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH) - expected = self.get_test_loc('match/directory-matching/abbrev-1.1.0-i-expected.json') + input_file = self.get_test_loc( + 'match/directory-matching/abbrev-1.1.0-i.json') + vc = run_do_match_from_scan( + input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH) + expected = self.get_test_loc( + 'match/directory-matching/abbrev-1.1.0-i-expected.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) def test_match_ApproximateDirectoryStructureIndex_abbrev_1_1_1(self): - input_file = self.get_test_loc('match/directory-matching/abbrev-1.1.1-i.json') - vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH) - expected = self.get_test_loc('match/directory-matching/abbrev-1.1.1-i-expected.json') + input_file = self.get_test_loc( + 'match/directory-matching/abbrev-1.1.1-i.json') + vc = run_do_match_from_scan( + input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH) + expected = self.get_test_loc( + 'match/directory-matching/abbrev-1.1.1-i-expected.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) def test_match_ApproximateDirectoryStructureIndex_get_stdin_3_0_2(self): - input_file = self.get_test_loc('match/directory-matching/get-stdin-3.0.2-i.json') - vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH) - expected = self.get_test_loc('match/directory-matching/get-stdin-3.0.2-i-expected.json') + input_file = self.get_test_loc( + 'match/directory-matching/get-stdin-3.0.2-i.json') + vc = run_do_match_from_scan( + input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH) + expected = self.get_test_loc( + 'match/directory-matching/get-stdin-3.0.2-i-expected.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) def test_match_ApproximateDirectoryContentIndex_abbrev_1_0_3(self): - input_file = self.get_test_loc('match/directory-matching/abbrev-1.0.3-i.json') - vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_CONTENT_MATCH) - expected = self.get_test_loc('match/directory-matching/abbrev-1.0.3-i-expected.json') + input_file = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.3-i.json') + vc = run_do_match_from_scan( + input_file, APPROXIMATE_DIRECTORY_CONTENT_MATCH) + expected = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.3-i-expected.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) def test_match_ApproximateDirectoryContentIndex_abbrev_1_0_4(self): - input_file = self.get_test_loc('match/directory-matching/abbrev-1.0.4-i.json') - vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_CONTENT_MATCH) - expected = self.get_test_loc('match/directory-matching/abbrev-1.0.4-i-expected.json') + input_file = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.4-i.json') + vc = run_do_match_from_scan( + input_file, APPROXIMATE_DIRECTORY_CONTENT_MATCH) + expected = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.4-i-expected.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) def test_match_ApproximateDirectoryContentIndex_abbrev_1_0_5(self): - input_file = self.get_test_loc('match/directory-matching/abbrev-1.0.5-i.json') - vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_CONTENT_MATCH) - expected = self.get_test_loc('match/directory-matching/abbrev-1.0.5-i-expected.json') + input_file = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.5-i.json') + vc = run_do_match_from_scan( + input_file, APPROXIMATE_DIRECTORY_CONTENT_MATCH) + expected = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.5-i-expected.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) def test_match_ApproximateDirectoryContentIndex_abbrev_1_0_6(self): - input_file = self.get_test_loc('match/directory-matching/abbrev-1.0.6-i.json') - vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_CONTENT_MATCH) - expected = self.get_test_loc('match/directory-matching/abbrev-1.0.6-i-expected.json') + input_file = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.6-i.json') + vc = run_do_match_from_scan( + input_file, APPROXIMATE_DIRECTORY_CONTENT_MATCH) + expected = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.6-i-expected.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) def test_match_ApproximateDirectoryContentIndex_abbrev_1_0_7(self): - input_file = self.get_test_loc('match/directory-matching/abbrev-1.0.7-i.json') - vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_CONTENT_MATCH) - expected = self.get_test_loc('match/directory-matching/abbrev-1.0.7-i-expected.json') + input_file = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.7-i.json') + vc = run_do_match_from_scan( + input_file, APPROXIMATE_DIRECTORY_CONTENT_MATCH) + expected = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.7-i-expected.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) def test_match_ApproximateDirectoryContentIndex_abbrev_1_0_9(self): - input_file = self.get_test_loc('match/directory-matching/abbrev-1.0.9-i.json') - vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_CONTENT_MATCH) - expected = self.get_test_loc('match/directory-matching/abbrev-1.0.9-i-expected.json') + input_file = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.9-i.json') + vc = run_do_match_from_scan( + input_file, APPROXIMATE_DIRECTORY_CONTENT_MATCH) + expected = self.get_test_loc( + 'match/directory-matching/abbrev-1.0.9-i-expected.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) def test_match_ApproximateDirectoryContentIndex_abbrev_1_1_0(self): - input_file = self.get_test_loc('match/directory-matching/abbrev-1.1.0-i.json') - vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_CONTENT_MATCH) - expected = self.get_test_loc('match/directory-matching/abbrev-1.1.0-i-expected.json') + input_file = self.get_test_loc( + 'match/directory-matching/abbrev-1.1.0-i.json') + vc = run_do_match_from_scan( + input_file, APPROXIMATE_DIRECTORY_CONTENT_MATCH) + expected = self.get_test_loc( + 'match/directory-matching/abbrev-1.1.0-i-expected.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) def test_match_ApproximateDirectoryContentIndex_abbrev_1_1_1(self): - input_file = self.get_test_loc('match/directory-matching/abbrev-1.1.1-i.json') - vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_CONTENT_MATCH) - expected = self.get_test_loc('match/directory-matching/abbrev-1.1.1-i-expected.json') + input_file = self.get_test_loc( + 'match/directory-matching/abbrev-1.1.1-i.json') + vc = run_do_match_from_scan( + input_file, APPROXIMATE_DIRECTORY_CONTENT_MATCH) + expected = self.get_test_loc( + 'match/directory-matching/abbrev-1.1.1-i-expected.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) def test_match_ApproximateDirectoryContentIndex_get_stdin_3_0_2(self): - input_file = self.get_test_loc('match/directory-matching/get-stdin-3.0.2-i.json') - vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_CONTENT_MATCH) - expected = self.get_test_loc('match/directory-matching/get-stdin-3.0.2-i-expected.json') + input_file = self.get_test_loc( + 'match/directory-matching/get-stdin-3.0.2-i.json') + vc = run_do_match_from_scan( + input_file, APPROXIMATE_DIRECTORY_CONTENT_MATCH) + expected = self.get_test_loc( + 'match/directory-matching/get-stdin-3.0.2-i-expected.json') self.check_codebase(vc, expected, regen=FIXTURES_REGEN) diff --git a/matchcode/tests/test_models.py b/matchcode/tests/test_models.py index e22b0c7c..d3a84c6e 100644 --- a/matchcode/tests/test_models.py +++ b/matchcode/tests/test_models.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -94,21 +94,25 @@ def setUp(self): index_packages_sha1() # Populate ExactFileIndexFingerprint table - load_resources_from_scan(self.get_test_loc('models/match-test.json'), self.test_package4) + load_resources_from_scan(self.get_test_loc( + 'models/match-test.json'), self.test_package4) index_package_directories(self.test_package4) - index_package_files_sha1(self.test_package4, self.get_test_loc('models/match-test.json')) + index_package_files_sha1( + self.test_package4, self.get_test_loc('models/match-test.json')) class ExactPackageArchiveIndexModelTestCase(BaseModelTest): def test_ExactPackageArchiveIndex_index(self): # Test index sha1 = 'b6bbe0b067469d719708ca38de5c237cb526c3d2' - epai, created = ExactPackageArchiveIndex.index(sha1, self.test_package1) + epai, created = ExactPackageArchiveIndex.index( + sha1, self.test_package1) self.assertTrue(created) self.assertEqual(sha1, epai.fingerprint()) # Test index of existing sha1 - epai, created = ExactPackageArchiveIndex.index(sha1, self.test_package1) + epai, created = ExactPackageArchiveIndex.index( + sha1, self.test_package1) self.assertFalse(created) self.assertEqual(sha1, epai.fingerprint()) @@ -120,7 +124,8 @@ def test_ExactPackageArchiveIndex_index(self): ) def test_ExactPackageArchiveIndex_single_sha1_single_match(self): - result = ExactPackageArchiveIndex.match('51d28a27d919ce8690a40f4f335b9d591ceb16e9') + result = ExactPackageArchiveIndex.match( + '51d28a27d919ce8690a40f4f335b9d591ceb16e9') result = [r.package.to_dict() for r in result] expected = [self.test_package1_metadata] self.assertEqual(expected, result) @@ -168,7 +173,8 @@ def test_ExactFileIndex_match(self): resource.matched_to.append(p['purl']) resource.save(codebase) - expected = self.get_test_loc('models/exact-file-matching-standalone-test-results.json') + expected = self.get_test_loc( + 'models/exact-file-matching-standalone-test-results.json') self.check_codebase(codebase, expected, regen=FIXTURES_REGEN) @@ -188,7 +194,8 @@ def setUp(self): type='npm', ) self.test_package1_metadata = self.test_package1.to_dict() - load_resources_from_scan(self.get_test_loc('models/directory-matching/async-0.2.10.tgz-i.json'), self.test_package1) + load_resources_from_scan(self.get_test_loc( + 'models/directory-matching/async-0.2.10.tgz-i.json'), self.test_package1) index_package_directories(self.test_package1) self.test_package2, _ = Package.objects.get_or_create( @@ -202,7 +209,8 @@ def setUp(self): type='npm', ) self.test_package2_metadata = self.test_package2.to_dict() - load_resources_from_scan(self.get_test_loc('models/directory-matching/async-0.2.9-i.json'), self.test_package2) + load_resources_from_scan(self.get_test_loc( + 'models/directory-matching/async-0.2.9-i.json'), self.test_package2) index_package_directories(self.test_package2) def test_ApproximateDirectoryStructureIndex_index(self): @@ -238,10 +246,12 @@ def test_ApproximateDirectoryStructureIndex_index(self): ) def test_ApproximateDirectoryStructureIndex_match_subdir(self): - scan_location = self.get_test_loc('models/directory-matching/async-0.2.9-i.json') + scan_location = self.get_test_loc( + 'models/directory-matching/async-0.2.9-i.json') vc = VirtualCodebase( location=scan_location, - resource_attributes=dict(packages=attr.ib(default=attr.Factory(list))) + resource_attributes=dict( + packages=attr.ib(default=attr.Factory(list))) ) codebase = compute_codebase_directory_fingerprints(vc) @@ -260,7 +270,8 @@ def test_ApproximateDirectoryStructureIndex_match_subdir(self): resource.packages.append(p) resource.save(codebase) - expected = self.get_test_loc('models/directory-matching/async-0.2.9-i-expected-structure.json') + expected = self.get_test_loc( + 'models/directory-matching/async-0.2.9-i-expected-structure.json') self.check_codebase(codebase, expected, regen=FIXTURES_REGEN) def test_ApproximateDirectoryContentIndex_index(self): @@ -296,10 +307,12 @@ def test_ApproximateDirectoryContentIndex_index(self): ) def test_ApproximateDirectoryContentIndex_match_subdir(self): - scan_location = self.get_test_loc('models/directory-matching/async-0.2.9-i.json') + scan_location = self.get_test_loc( + 'models/directory-matching/async-0.2.9-i.json') vc = VirtualCodebase( location=scan_location, - resource_attributes=dict(packages=attr.ib(default=attr.Factory(list))) + resource_attributes=dict( + packages=attr.ib(default=attr.Factory(list))) ) codebase = compute_codebase_directory_fingerprints(vc) @@ -318,7 +331,8 @@ def test_ApproximateDirectoryContentIndex_match_subdir(self): resource.packages.append(p) resource.save(codebase) - expected = self.get_test_loc('models/directory-matching/async-0.2.9-i-expected-content.json') + expected = self.get_test_loc( + 'models/directory-matching/async-0.2.9-i-expected-content.json') self.check_codebase(codebase, expected, regen=FIXTURES_REGEN) @@ -365,7 +379,8 @@ def setUp(self): extension='js', package=self.test_package1 ) - test_resource1_loc = self.get_test_loc('match/approximate-file-matching/index.js') + test_resource1_loc = self.get_test_loc( + 'match/approximate-file-matching/index.js') fingerprints = get_file_fingerprint_hashes(test_resource1_loc) self.test_resource1_fingerprint = fingerprints['halo1'] ApproximateResourceContentIndex.index( @@ -407,10 +422,12 @@ def test_ApproximateResourceContentIndex_index(self): ) def test_ApproximateResourceContentIndex_match(self): - scan_location = self.get_test_loc('match/approximate-file-matching/approximate-match-test.json') + scan_location = self.get_test_loc( + 'match/approximate-file-matching/approximate-match-test.json') codebase = VirtualCodebase( location=scan_location, - resource_attributes=dict(packages=attr.ib(default=attr.Factory(list))) + resource_attributes=dict( + packages=attr.ib(default=attr.Factory(list))) ) # populate codebase with match results @@ -427,17 +444,21 @@ def test_ApproximateResourceContentIndex_match(self): resource.packages.append(p) resource.save(codebase) - expected = self.get_test_loc('match/approximate-file-matching/approximate-match-model-test-results.json') + expected = self.get_test_loc( + 'match/approximate-file-matching/approximate-match-model-test-results.json') self.check_codebase(codebase, expected, regen=FIXTURES_REGEN) def test_ApproximateResourceContentIndex_match_deep_equals(self): - test_file_loc = self.get_test_loc('match/approximate-file-matching/index-modified.js') + test_file_loc = self.get_test_loc( + 'match/approximate-file-matching/index-modified.js') fingerprints = get_file_fingerprint_hashes(test_file_loc) fp = fingerprints['halo1'] matches = ApproximateResourceContentIndex.match(fp) results = [match.package.to_dict() for match in matches] - expected_results_loc = self.get_test_loc('match/approximate-file-matching/index-modified.js-expected.json') - self.check_expected_results(results, expected_results_loc, regen=FIXTURES_REGEN) + expected_results_loc = self.get_test_loc( + 'match/approximate-file-matching/index-modified.js-expected.json') + self.check_expected_results( + results, expected_results_loc, regen=FIXTURES_REGEN) class MatchcodeModelUtilsTestCase(MatchcodeTestCase): diff --git a/matchcode/utils.py b/matchcode/utils.py index 06dfba9c..7168847a 100644 --- a/matchcode/utils.py +++ b/matchcode/utils.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -76,14 +76,16 @@ def serializer(r): # Normalize package_uid package_uid = package_data.get('package_uid') if package_uid: - package_data['package_uid'] = purl_with_fake_uuid(package_uid) + package_data['package_uid'] = purl_with_fake_uuid( + package_uid) return rd results = list(map(serializer, codebase.walk(topdown=True))) if regen: with open(expected_codebase_json_loc, 'w') as reg: - json.dump(dict(files=results), reg, indent=2, separators=(',', ': ')) + json.dump(dict(files=results), reg, + indent=2, separators=(',', ': ')) expected_vc = VirtualCodebase(location=expected_codebase_json_loc) expected = list(map(serializer, expected_vc.walk(topdown=True))) @@ -224,8 +226,10 @@ def index_resource_fingerprints(codebase, package): indexed_adsi = 0 indexed_arci = 0 for resource in codebase.walk(topdown=False): - directory_content_fingerprint = resource.extra_data.get('directory_content', '') - directory_structure_fingerprint = resource.extra_data.get('directory_structure', '') + directory_content_fingerprint = resource.extra_data.get( + 'directory_content', '') + directory_structure_fingerprint = resource.extra_data.get( + 'directory_structure', '') resource_content_fingerprint = resource.extra_data.get('halo1', '') if directory_content_fingerprint: diff --git a/matchcode_pipeline/api.py b/matchcode_pipeline/api.py index 94893b40..0945bd0e 100644 --- a/matchcode_pipeline/api.py +++ b/matchcode_pipeline/api.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # from uuid import uuid4 @@ -155,7 +155,8 @@ def create(self, validated_data, matching_pipeline_name='matching'): downloads, errors = fetch_urls(input_urls) if errors: - raise serializers.ValidationError("Could not fetch: " + "\n".join(errors)) + raise serializers.ValidationError( + "Could not fetch: " + "\n".join(errors)) project = super().create(validated_data) @@ -264,7 +265,8 @@ def create(self, validated_data, matching_pipeline_name='d2d'): errors = check_urls_availability(input_urls) if errors: - raise serializers.ValidationError("Could not fetch: " + "\n".join(errors)) + raise serializers.ValidationError( + "Could not fetch: " + "\n".join(errors)) project = super().create(validated_data) @@ -285,7 +287,8 @@ def create(self, validated_data, matching_pipeline_name='d2d'): for url in urls: project.add_input_source(download_url=url) - project.add_pipeline(matching_pipeline_name, selected_groups=["Java", "Javascript", "Elf", "Go"], execute_now=execute_now) + project.add_pipeline(matching_pipeline_name, selected_groups=[ + "Java", "Javascript", "Elf", "Go"], execute_now=execute_now) return project @@ -300,7 +303,7 @@ class MatchingViewSet( """ Take a ScanCode.io JSON of a codebase `upload_file` or from a list of `input_urls` and run the ``matching`` pipeline - (https://github.com/nexB/purldb/blob/main/matchcode_pipeline/pipelines/matching.py) + (https://github.com/aboutcode-org/purldb/blob/main/matchcode_pipeline/pipelines/matching.py) on it. The ``matching`` pipeline matches directory and resources of the codebase in diff --git a/matchcode_pipeline/tests/test_api.py b/matchcode_pipeline/tests/test_api.py index 2762d7df..8783cdd6 100644 --- a/matchcode_pipeline/tests/test_api.py +++ b/matchcode_pipeline/tests/test_api.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -35,7 +35,8 @@ def setUp(self): project=self.project1, path='daglib-0.3.2.tar.gz-extract/daglib-0.3.2/PKG-INFO', ) - self.discovered_package1 = self.resource1.create_and_add_package(package_data1) + self.discovered_package1 = self.resource1.create_and_add_package( + package_data1) self.discovered_dependency1 = DiscoveredDependency.create_from_data( self.project1, dependency_data1 ) @@ -47,9 +48,11 @@ def setUp(self): ) self.matching_list_url = reverse('matching-list') - self.project1_detail_url = reverse('matching-detail', args=[self.project1.uuid]) + self.project1_detail_url = reverse( + 'matching-detail', args=[self.project1.uuid]) - self.user = User.objects.create_user('username', 'e@mail.com', 'secret') + self.user = User.objects.create_user( + 'username', 'e@mail.com', 'secret') self.auth = f'Token {self.user.auth_token.key}' self.csrf_client = APIClient(enforce_csrf_checks=True) @@ -86,7 +89,8 @@ def test_matchcode_pipeline_api_matching_detail(self): 'with_missing_resources': 0, 'with_modified_resources': 0, } - self.assertEqual(expected, response.data['discovered_packages_summary']) + self.assertEqual( + expected, response.data['discovered_packages_summary']) expected = { 'total': 1, @@ -94,13 +98,16 @@ def test_matchcode_pipeline_api_matching_detail(self): 'is_optional': 0, 'is_resolved': 0, } - self.assertEqual(expected, response.data['discovered_dependencies_summary']) + self.assertEqual( + expected, response.data['discovered_dependencies_summary']) expected = {'java_to_class': 1} self.assertEqual(expected, response.data['codebase_relations_summary']) - input1 = self.project1.add_input_source(filename='file1', is_uploaded=True) - input2 = self.project1.add_input_source(filename='file2', download_url='https://download.url') + input1 = self.project1.add_input_source( + filename='file1', is_uploaded=True) + input2 = self.project1.add_input_source( + filename='file2', download_url='https://download.url') self.project1.save() response = self.csrf_client.get(self.project1_detail_url) expected = [ @@ -145,7 +152,8 @@ def test_matching_pipeline_api_matching_create(self, mock_execute_pipeline_task) # Check that the file was uploaded response = self.csrf_client.get(created_matching_project_detail_url) - self.assertEqual('test-out.json', response.data['input_sources'][0]['filename']) + self.assertEqual( + 'test-out.json', response.data['input_sources'][0]['filename']) @mock.patch('scanpipe.models.Run.execute_task_async') def test_matching_pipeline_api_matching_create_multiple_input_urls(self, mock_execute_pipeline_task): @@ -197,9 +205,11 @@ class D2DPipelineAPITest(TransactionTestCase): def setUp(self): self.project1 = Project.objects.create(name='Analysis') self.d2d_list_url = reverse('d2d-list') - self.project1_detail_url = reverse('d2d-detail', args=[self.project1.uuid]) + self.project1_detail_url = reverse( + 'd2d-detail', args=[self.project1.uuid]) - self.user = User.objects.create_user('username', 'a@mail.com', 'secret') + self.user = User.objects.create_user( + 'username', 'a@mail.com', 'secret') self.auth = f'Token {self.user.auth_token.key}' self.csrf_client = APIClient(enforce_csrf_checks=True) self.csrf_client.credentials(HTTP_AUTHORIZATION=self.auth) @@ -240,7 +250,8 @@ def test_d2d_pipeline_api_run_detail(self): self.assertEqual(str(run1.uuid), response.data['uuid']) self.assertIn(project1_detail_url, response.data['project']) self.assertEqual('d2d', response.data['pipeline_name']) - self.assertEqual('Establish relationships between two code trees: deployment and development.', response.data['description']) + self.assertEqual( + 'Establish relationships between two code trees: deployment and development.', response.data['description']) self.assertEqual('', response.data['scancodeio_version']) self.assertIsNone(response.data['task_id']) self.assertIsNone(response.data['task_start_date']) diff --git a/matchcode_project/dbrouter.py b/matchcode_project/dbrouter.py index 330543b9..582f77d7 100644 --- a/matchcode_project/dbrouter.py +++ b/matchcode_project/dbrouter.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/matchcode_project/settings.py b/matchcode_project/settings.py index 77a4c7f5..3e4b2b19 100644 --- a/matchcode_project/settings.py +++ b/matchcode_project/settings.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/matchcode_project/urls.py b/matchcode_project/urls.py index 13019d45..94812be2 100644 --- a/matchcode_project/urls.py +++ b/matchcode_project/urls.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -12,7 +12,7 @@ from django.views.generic import RedirectView from rest_framework import routers -from matchcode_pipeline.api import D2DViewSet +from matchcode_pipeline.api import D2DViewSet from matchcode_pipeline.api import MatchingViewSet from matchcode_pipeline.api import RunViewSet diff --git a/matchcode_project/wsgi.py b/matchcode_project/wsgi.py index 83964488..ae478fac 100644 --- a/matchcode_project/wsgi.py +++ b/matchcode_project/wsgi.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/__init__.py b/minecode/__init__.py index 542d97cb..f31542f8 100644 --- a/minecode/__init__.py +++ b/minecode/__init__.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/api.py b/minecode/api.py index 0fff2332..13146acb 100644 --- a/minecode/api.py +++ b/minecode/api.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -105,7 +105,7 @@ class Meta: class ScannableURIViewSet(viewsets.ModelViewSet): queryset = ScannableURI.objects.all() serializer_class = ScannableURISerializer - permission_classes = [IsScanQueueWorkerAPIUser|IsAdminUser] + permission_classes = [IsScanQueueWorkerAPIUser | IsAdminUser] lookup_field = 'uuid' @action(detail=False, methods=['get']) @@ -153,7 +153,8 @@ def update_status(self, request, *args, **kwargs): scannable_uri = self.get_object() scannable_uri_uuid = scannable_uri.uuid - scannable_uri_status = ScannableURI.SCAN_STATUSES_BY_CODE.get(scannable_uri.scan_status) + scannable_uri_status = ScannableURI.SCAN_STATUSES_BY_CODE.get( + scannable_uri.scan_status) if scannable_uri.scan_status in [ ScannableURI.SCAN_INDEXED, @@ -239,6 +240,6 @@ def index_package_scan(request, key): ) msg = { 'status': f'scan results for scannable_uri {scannable_uri.uuid} ' - 'have been queued for indexing' + 'have been queued for indexing' } return Response(msg) diff --git a/minecode/apps.py b/minecode/apps.py index 478d1bee..148452e7 100644 --- a/minecode/apps.py +++ b/minecode/apps.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/collectors/__init__.py b/minecode/collectors/__init__.py index 2eb8f9f0..e1521118 100644 --- a/minecode/collectors/__init__.py +++ b/minecode/collectors/__init__.py @@ -3,6 +3,6 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/command.py b/minecode/command.py index bf924e44..b46f29ae 100644 --- a/minecode/command.py +++ b/minecode/command.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/debutils.py b/minecode/debutils.py index 8493f2d4..5c33f531 100644 --- a/minecode/debutils.py +++ b/minecode/debutils.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/filter.py b/minecode/filter.py index 0d45f5f7..38d89c49 100644 --- a/minecode/filter.py +++ b/minecode/filter.py @@ -5,7 +5,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/ls.py b/minecode/ls.py index 69d72366..3d636b8b 100644 --- a/minecode/ls.py +++ b/minecode/ls.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -174,22 +174,26 @@ def parse_directory_listing(dir_listing, from_find=False): try: file_stat = parser.parse_line(line) if TRACE: - logger.debug('parse_directory_listing:file_stat: ' + repr(file_stat)) + logger.debug( + 'parse_directory_listing:file_stat: ' + repr(file_stat)) dt = datetime.utcfromtimestamp(file_stat.st_mtime) dt = datetime.isoformat(dt) - logger.debug('parse_directory_listing:file_stat:date: ' + repr(dt)) + logger.debug( + 'parse_directory_listing:file_stat:date: ' + repr(dt)) except ParserError as pe: # this is likely a directory line from an ls -LR listing. Strip # trailing colon and keep track of the base directory if not line.endswith(':'): - raise Exception('Unknown directory listing line format: #%(ln)d: %(line)r' % locals()) + raise Exception( + 'Unknown directory listing line format: #%(ln)d: %(line)r' % locals()) base_dir = line.strip(':') continue if file_stat._st_name in ('.', '..'): continue - entry = Entry.from_stat(file_stat, base_dir=base_dir, use_utc_time=False) + entry = Entry.from_stat( + file_stat, base_dir=base_dir, use_utc_time=False) if entry: yield entry diff --git a/minecode/management/__init__.py b/minecode/management/__init__.py index 2eb8f9f0..e1521118 100644 --- a/minecode/management/__init__.py +++ b/minecode/management/__init__.py @@ -3,6 +3,6 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/management/commands/__init__.py b/minecode/management/commands/__init__.py index 0bb58fc7..27a8178c 100644 --- a/minecode/management/commands/__init__.py +++ b/minecode/management/commands/__init__.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/management/commands/check_licenses.py b/minecode/management/commands/check_licenses.py index 23272cd5..5716c626 100644 --- a/minecode/management/commands/check_licenses.py +++ b/minecode/management/commands/check_licenses.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -73,7 +73,8 @@ def handle(self, *args, **options): self.stdout.write('Visited {} packages'.format(visited_counter)) self.stdout.write('Found {} possible packages'.format(found_counter)) if found_counter > 0: - self.stdout.write('Found packages dumped to: {}'.format(file_location)) + self.stdout.write( + 'Found packages dumped to: {}'.format(file_location)) def find_ambiguous_packages(types=('maven',), keywords=('unknown', 'proprietary', 'commercial',)): @@ -82,9 +83,11 @@ def find_ambiguous_packages(types=('maven',), keywords=('unknown', 'proprietary' contain "unknown", "proprietary" and "commercial" words. """ # filter to detect declared_license field - filter_expression = [Q(declared_license_expression__icontains=word) for word in keywords] + filter_expression = [ + Q(declared_license_expression__icontains=word) for word in keywords] # filter to detect license_expression field, add or relationship between these two fields - filter_expression.extend([Q(other_license_expression__icontains=word) for word in keywords]) + filter_expression.extend( + [Q(other_license_expression__icontains=word) for word in keywords]) license_filter = reduce(operator.or_, filter_expression) for package in Package.objects.filter(type__in=types).filter(license_filter): diff --git a/minecode/management/commands/check_uri.py b/minecode/management/commands/check_uri.py index 664fa3f7..f3a67355 100644 --- a/minecode/management/commands/check_uri.py +++ b/minecode/management/commands/check_uri.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -61,7 +61,8 @@ def handle(self, *args, **options): show_data = options.get('show_data') # get the last 10 uris - uris = ResourceURI.objects.filter(uri__startswith=uri_prefix).order_by("-id")[:limit] + uris = ResourceURI.objects.filter( + uri__startswith=uri_prefix).order_by("-id")[:limit] # TODO: add if the uri be resolved by visit and/or map router for uri in uris: diff --git a/minecode/management/commands/create-scan-queue-worker-user.py b/minecode/management/commands/create-scan-queue-worker-user.py index 76e7b928..7708f963 100644 --- a/minecode/management/commands/create-scan-queue-worker-user.py +++ b/minecode/management/commands/create-scan-queue-worker-user.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -24,7 +24,8 @@ def handle(self, *args, **options): verbosity=verbosity ) # Add user to `scan_queue_workers` group - scan_queue_workers_group, _ = Group.objects.get_or_create(name='scan_queue_workers') + scan_queue_workers_group, _ = Group.objects.get_or_create( + name='scan_queue_workers') scan_queue_workers_group.user_set.add(user) msg = f'User {username} added to `scan_queue_workers` group' self.stdout.write(msg, self.style.SUCCESS) diff --git a/minecode/management/commands/create-user.py b/minecode/management/commands/create-user.py index 64b8b238..47087f27 100644 --- a/minecode/management/commands/create-user.py +++ b/minecode/management/commands/create-user.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/management/commands/dump_purls.py b/minecode/management/commands/dump_purls.py index 485e9ae1..a81d8aba 100644 --- a/minecode/management/commands/dump_purls.py +++ b/minecode/management/commands/dump_purls.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -22,7 +22,8 @@ def dump_purls(package_type, output): for i, package in enumerate(Package.objects.filter(type=package_type).all()): if not output: out = open(f"{output}-{i}.json", "w") - purl = dict(purl=package.package_url, download_url=package.download_url) + purl = dict(purl=package.package_url, + download_url=package.download_url) if not i % 500: print(f"#{i} purl: {package.package_url}") out.write(json.dumps(purl, separators=compact_separators)) @@ -37,4 +38,3 @@ def dump_purls(package_type, output): package_type = args[0] output = args[1] dump_purls(package_type=package_type, output=output) - diff --git a/minecode/management/commands/get_maven_release_dates.py b/minecode/management/commands/get_maven_release_dates.py index c120b67e..5dc1e928 100644 --- a/minecode/management/commands/get_maven_release_dates.py +++ b/minecode/management/commands/get_maven_release_dates.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -47,22 +47,27 @@ def handle(self, *args, **options): for index, package in enumerate(iterator, start=1): download_url = package.download_url package_url = package.package_url - logger.info(f'Updating release_date for package {package_url} ({download_url})') + logger.info( + f'Updating release_date for package {package_url} ({download_url})') package_version_page_url = dirname(download_url) filename = download_url.rsplit('/')[-1] response = requests.get(package_version_page_url) if response: - timestamps_by_links = collect_links_from_text(response.text, filter=filter_for_artifacts) + timestamps_by_links = collect_links_from_text( + response.text, filter=filter_for_artifacts) timestamp = timestamps_by_links.get(filename) if not timestamp: - logger.info(f'\tCould not get release_date for package {package_url} ({download_url})') + logger.info( + f'\tCould not get release_date for package {package_url} ({download_url})') continue timestamp = dateutil_parse(timestamp) package.release_date = timestamp unsaved_objects.append(package) - logger.info(f'\t{package_url} ({download_url}) release_date has been updated to {timestamp}') + logger.info( + f'\t{package_url} ({download_url}) release_date has been updated to {timestamp}') else: - logger.info(f'\t{package_url} not updated: error encountered when visiting {package_version_page_url}') + logger.info( + f'\t{package_url} not updated: error encountered when visiting {package_version_page_url}') if not (index % chunk_size) and unsaved_objects: logger.info(f'{index:,} / {object_count:,} Packages processed') diff --git a/minecode/management/commands/get_status.py b/minecode/management/commands/get_status.py index 9f24d12f..144d2850 100644 --- a/minecode/management/commands/get_status.py +++ b/minecode/management/commands/get_status.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -34,14 +34,17 @@ def handle(self, *args, **options): ('visitables', ResourceURI.objects.get_visitables().count()), ('visited', ResourceURI.objects.visited().count()), ('successfully_visited', ResourceURI.objects.successfully_visited().count()), - ('unsuccessfully_visited', ResourceURI.objects.unsuccessfully_visited().count()), + ('unsuccessfully_visited', + ResourceURI.objects.unsuccessfully_visited().count()), ('never_visited', ResourceURI.objects.never_visited().count()), - ('visit_in_progress', ResourceURI.objects.filter(wip_date__isnull=False, last_visit_date__isnull=True).count()), + ('visit_in_progress', ResourceURI.objects.filter( + wip_date__isnull=False, last_visit_date__isnull=True).count()), ('mappables', ResourceURI.objects.get_mappables().count()), ('mapped', ResourceURI.objects.mapped().count()), ('successfully_mapped', ResourceURI.objects.successfully_mapped().count()), - ('unsuccessfully_mapped', ResourceURI.objects.unsuccessfully_mapped().count()), + ('unsuccessfully_mapped', + ResourceURI.objects.unsuccessfully_mapped().count()), ('never_mapped', ResourceURI.objects.never_mapped().count()), ]) diff --git a/minecode/management/commands/import_queue.py b/minecode/management/commands/import_queue.py index 9003de5f..52c0c355 100644 --- a/minecode/management/commands/import_queue.py +++ b/minecode/management/commands/import_queue.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -129,7 +129,8 @@ def process_request(importable_uri): else: namespace, name, _ = determine_namespace_name_version_from_url(uri) - timestamps_by_directory_links = collect_links_from_text(data, filter_only_directories) + timestamps_by_directory_links = collect_links_from_text( + data, filter_only_directories) # Go into each version directory for directory_link in timestamps_by_directory_links.keys(): version = directory_link.rstrip('/') @@ -137,7 +138,8 @@ def process_request(importable_uri): timestamps_by_artifact_links = get_artifact_links(version_page_url) for artifact_link, timestamp in timestamps_by_artifact_links.items(): sha1 = get_artifact_sha1(artifact_link) - classifier = get_classifier_from_artifact_url(artifact_link, version_page_url, name, version) + classifier = get_classifier_from_artifact_url( + artifact_link, version_page_url, name, version) qualifiers = None if classifier: qualifiers = f'classifier={classifier}' diff --git a/minecode/management/commands/load_priority_queue.py b/minecode/management/commands/load_priority_queue.py index efc83645..8e4fbfde 100644 --- a/minecode/management/commands/load_priority_queue.py +++ b/minecode/management/commands/load_priority_queue.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -42,7 +42,8 @@ def handle(self, *args, **options): maven_api_search_url = f'https://search.maven.org/solrsearch/select?q=1:{resource.sha1}' response = requests.get(maven_api_search_url) if not response.ok: - logger.error(f"API query failed for: {maven_api_search_url}") + logger.error( + f"API query failed for: {maven_api_search_url}") continue contents = response.json() resp = contents.get('response', {}) @@ -53,5 +54,6 @@ def handle(self, *args, **options): version = matched_package.get('v', '') if namespace and name and version: purl = f'pkg:maven/{namespace}/{name}@{version}' - PriorityResourceURI.objects.create(uri=purl, package_url=purl, sha1=resource.sha1) + PriorityResourceURI.objects.create( + uri=purl, package_url=purl, sha1=resource.sha1) logger.info(f'Added {purl} to priority queue') diff --git a/minecode/management/commands/maven_crawler.py b/minecode/management/commands/maven_crawler.py index 51c2be96..647a94ae 100644 --- a/minecode/management/commands/maven_crawler.py +++ b/minecode/management/commands/maven_crawler.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/management/commands/priority_queue.py b/minecode/management/commands/priority_queue.py index 372b972e..94fc4edd 100644 --- a/minecode/management/commands/priority_queue.py +++ b/minecode/management/commands/priority_queue.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -124,7 +124,8 @@ def process_request(priority_resource_uri, _priority_router=priority_router): errors = _priority_router.process(purl_to_visit, **kwargs) if TRACE: new_uris_to_visit = list(new_uris_to_visit or []) - logger.debug('visit_uri: new_uris_to_visit: {}'.format(new_uris_to_visit)) + logger.debug( + 'visit_uri: new_uris_to_visit: {}'.format(new_uris_to_visit)) return errors diff --git a/minecode/management/commands/remap.py b/minecode/management/commands/remap.py index 97a08080..303655b1 100644 --- a/minecode/management/commands/remap.py +++ b/minecode/management/commands/remap.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -35,7 +35,10 @@ def handle(self, *args, **options): uri.wip_date = None uri.save() - ResourceURI.objects.successfully_mapped().filter(uri__contains='maven').update(last_map_date=None) - ResourceURI.objects.successfully_mapped().filter(uri__contains='npm').update(last_map_date=None) + ResourceURI.objects.successfully_mapped().filter( + uri__contains='maven').update(last_map_date=None) + ResourceURI.objects.successfully_mapped().filter( + uri__contains='npm').update(last_map_date=None) - ResourceURI.objects.successfully_mapped().exclude(uri__startswith='http://repo1').exclude(uri__startswith='maven-index://').exclude(uri__startswith='https://replicate').exclude(uri__startswith='https://registry.npmjs.org').update(is_mappable=False) + ResourceURI.objects.successfully_mapped().exclude(uri__startswith='http://repo1').exclude(uri__startswith='maven-index://').exclude( + uri__startswith='https://replicate').exclude(uri__startswith='https://registry.npmjs.org').update(is_mappable=False) diff --git a/minecode/management/commands/run_map.py b/minecode/management/commands/run_map.py index 0ac88126..0ae56574 100644 --- a/minecode/management/commands/run_map.py +++ b/minecode/management/commands/run_map.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -154,7 +154,8 @@ def map_uri(resource_uri, _map_router=map_router): for scanned_package in mapped_scanned_packages: visit_level = resource_uri.mining_level - package, package_created, _, m_err = merge_or_create_package(scanned_package, visit_level) + package, package_created, _, m_err = merge_or_create_package( + scanned_package, visit_level) map_error += m_err if package_created: # Add this Package to the scan queue @@ -164,7 +165,8 @@ def map_uri(resource_uri, _map_router=map_router): package=package, ) if scannable_uri_created: - logger.debug(' + Inserted ScannableURI\t: {}'.format(package_uri)) + logger.debug( + ' + Inserted ScannableURI\t: {}'.format(package_uri)) except Exception as e: msg = 'Error: Failed to map while processing ResourceURI: {}\n'.format( diff --git a/minecode/management/commands/run_visit.py b/minecode/management/commands/run_visit.py index b441ae1d..0bae2cca 100644 --- a/minecode/management/commands/run_visit.py +++ b/minecode/management/commands/run_visit.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -172,7 +172,8 @@ def visit_uris(ignore_robots=False, ignore_throttle=False, if not resource_uri: if exit_on_empty: - logger.info('exit-on-empty requested: No more visitable resource, exiting...') + logger.info( + 'exit-on-empty requested: No more visitable resource, exiting...') break # Only log a single message when we go to sleep @@ -197,7 +198,8 @@ def visit_uris(ignore_robots=False, ignore_throttle=False, if not ignore_throttle: sleep_time = get_sleep_time(resource_uri) if sleep_time: - logger.debug('Respecting revisit delay: wait for {} for {}'.format(sleep_time, resource_uri.uri)) + logger.debug('Respecting revisit delay: wait for {} for {}'.format( + sleep_time, resource_uri.uri)) time.sleep(sleep_time) # Set new value in cache 'visit_delay_by_hostname' right before making the request # TODO: The cache logic should move closer to the requests calls @@ -213,7 +215,8 @@ def visit_uris(ignore_robots=False, ignore_throttle=False, uri_counter_by_visitor=uri_counter_by_visitor) if max_loops and int(visited_counter) > int(max_loops): - logger.info('Stopping visits after max_loops: {} visit loops.'.format(max_loops)) + logger.info( + 'Stopping visits after max_loops: {} visit loops.'.format(max_loops)) break return visited_counter, inserted_counter @@ -252,10 +255,12 @@ def visit_uri(resource_uri, max_uris=0, uri_counter_by_visitor=None, _visit_rout logger.debug('visit_uri: uri: {}'.format(uri_to_visit)) # TODO: Consider pass a full visitors.URI plain object rather than a plain string - new_uris_to_visit, visited_data, visit_error = _visit_router.process(uri_to_visit) + new_uris_to_visit, visited_data, visit_error = _visit_router.process( + uri_to_visit) if TRACE: new_uris_to_visit = list(new_uris_to_visit or []) - logger.debug('visit_uri: new_uris_to_visit: {}'.format(new_uris_to_visit)) + logger.debug( + 'visit_uri: new_uris_to_visit: {}'.format(new_uris_to_visit)) except NoRouteAvailable: logger.error('No route available.') @@ -304,7 +309,8 @@ def visit_uri(resource_uri, max_uris=0, uri_counter_by_visitor=None, _visit_rout visited_uri['last_modified_date'] = last_modified_date if vuri_count % 1000 == 0: - logger.debug(' * Processed: {} visited URIs'.format(vuri_count)) + logger.debug( + ' * Processed: {} visited URIs'.format(vuri_count)) try: # insert new if pre-visited @@ -314,7 +320,8 @@ def visit_uri(resource_uri, max_uris=0, uri_counter_by_visitor=None, _visit_rout visited_uri['last_visit_date'] = timezone.now() new_uri = ResourceURI(**visited_uri) new_uri.save() - logger.debug(' + Inserted pre-visited:\t{}'.format(uri_str)) + logger.debug( + ' + Inserted pre-visited:\t{}'.format(uri_str)) inserted_count += 1 if max_uris: uri_counter_by_visitor[visitor_key] += 1 @@ -333,7 +340,8 @@ def visit_uri(resource_uri, max_uris=0, uri_counter_by_visitor=None, _visit_rout except Exception as e: # FIXME: is catching all expections here correct? - msg = 'ERROR while processing URI from a visit through: {}'.format(uri_str) + msg = 'ERROR while processing URI from a visit through: {}'.format( + uri_str) msg += '\n' msg += repr(visited_uri) msg += '\n' @@ -341,11 +349,13 @@ def visit_uri(resource_uri, max_uris=0, uri_counter_by_visitor=None, _visit_rout visit_errors.append(msg) logger.error(msg) if len(visit_errors) > 10: - logger.error(' ! Breaking after processing over 10 vuris errors for: {}'.format(uri_str)) + logger.error( + ' ! Breaking after processing over 10 vuris errors for: {}'.format(uri_str)) break if max_uris and int(uri_counter_by_visitor[visitor_key]) > int(max_uris): - logger.info(' ! Breaking after processing max-uris: {} URIs.'.format(max_uris)) + logger.info( + ' ! Breaking after processing max-uris: {} URIs.'.format(max_uris)) break except Exception as e: diff --git a/minecode/management/commands/seed.py b/minecode/management/commands/seed.py index 48bfbb73..9ee8fd9f 100644 --- a/minecode/management/commands/seed.py +++ b/minecode/management/commands/seed.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/management/commands/update_maven_package_data.py b/minecode/management/commands/update_maven_package_data.py index 0106364d..c62a0be4 100644 --- a/minecode/management/commands/update_maven_package_data.py +++ b/minecode/management/commands/update_maven_package_data.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # from dateutil.parser import parse as dateutil_parse @@ -120,7 +120,7 @@ def process_packages( updated = True if unsaved_existing_packages_lowercased: - fields_to_update=[ + fields_to_update = [ 'namespace', 'name', 'version', @@ -133,7 +133,8 @@ def process_packages( 'last_modified_date', 'history', ] - upc = update_packages(unsaved_existing_packages_lowercased, fields_to_update) + upc = update_packages( + unsaved_existing_packages_lowercased, fields_to_update) updated_packages_count += upc unsaved_existing_packages_lowercased = [] if upc > 0: @@ -154,7 +155,8 @@ def process_packages( packages_to_delete = [] deleted_packages_count += dpc if dpc > 0: - logger.info(f'Deleted {deleted_packages_count:,} Duplicate Maven Packages') + logger.info( + f'Deleted {deleted_packages_count:,} Duplicate Maven Packages') return unsaved_existing_packages, unsaved_existing_packages_lowercased, unsaved_new_packages, packages_to_delete @@ -196,7 +198,8 @@ def update_maven_packages(maven_package, fields_to_update, lowercased_purl_field namespace = maven_package.namespace name = maven_package.name version = maven_package.version - normalized_qualifiers = normalize_qualifiers(maven_package.qualifiers, encode=True) + normalized_qualifiers = normalize_qualifiers( + maven_package.qualifiers, encode=True) if lowercased_purl_fields: namespace = namespace.lower() @@ -215,7 +218,8 @@ def update_maven_packages(maven_package, fields_to_update, lowercased_purl_field duplicate_packages = [] for existing_package in existing_packages: if existing_package.download_url != maven_package.download_url: - logger.debug(f'Deleted duplicate Package with incorrect download URL {existing_package.package_uid}') + logger.debug( + f'Deleted duplicate Package with incorrect download URL {existing_package.package_uid}') duplicate_packages.append(existing_package) duplicate_packages_pks = [p.pk for p in duplicate_packages] @@ -316,12 +320,14 @@ def handle(self, *args, **options): lowercased_purl_fields=True ) if existing_package_lowercased: - unsaved_existing_packages_lowercased.append(existing_package_lowercased) + unsaved_existing_packages_lowercased.append( + existing_package_lowercased) packages_to_delete.extend(duplicate_packages) continue if Package.objects.filter(download_url=maven_package.download_url).exists(): - logger.debug(f'Skipping creation of {maven_package.purl} - already exists') + logger.debug( + f'Skipping creation of {maven_package.purl} - already exists') continue if create_package: diff --git a/minecode/management/user_creation.py b/minecode/management/user_creation.py index 3a3fe6f2..ee8cba8a 100644 --- a/minecode/management/user_creation.py +++ b/minecode/management/user_creation.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -30,7 +30,8 @@ def __init__(self, *args, **kwargs): ) def add_arguments(self, parser): - parser.add_argument('username', help='Specifies the username for the user.') + parser.add_argument( + 'username', help='Specifies the username for the user.') parser.add_argument( '--no-input', action='store_false', @@ -57,7 +58,8 @@ def create_user(self, username, interactive, verbosity): if interactive: password = self.get_password_from_stdin(username) - user = self.UserModel._default_manager.create_user(username, password=password) + user = self.UserModel._default_manager.create_user( + username, password=password) token, _ = Token._default_manager.get_or_create(user=user) if verbosity >= 1: diff --git a/minecode/mappers/__init__.py b/minecode/mappers/__init__.py index 192bd767..f8ccc9fe 100644 --- a/minecode/mappers/__init__.py +++ b/minecode/mappers/__init__.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -17,6 +17,7 @@ class Mapper(object): get_packages() method and use a routing decorator for the URIs they can handle. """ + def __call__(self, uri, resource_uri): # Note: we let exceptions bubble up and they will be caught and # processed by the worker loop diff --git a/minecode/mappers/apache.py b/minecode/mappers/apache.py index a62f7f85..25b68aec 100644 --- a/minecode/mappers/apache.py +++ b/minecode/mappers/apache.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -93,7 +93,8 @@ def build_packages_from_projects(metadata, uri=None): for maintainer in maintainers: mailbox = maintainer.get('mbox', '').replace('mailto:', '') name = maintainer.get('name') - party = scan_models.Party(type=scan_models.party_person, name=name, role='maintainer', email=mailbox) + party = scan_models.Party( + type=scan_models.party_person, name=name, role='maintainer', email=mailbox) parties = common_data.get('parties') if not parties: common_data['parties'] = [] @@ -117,7 +118,8 @@ def build_packages_from_projects(metadata, uri=None): keywords.append(kw) common_data['keywords'] = keywords - common_data['primary_language'] = project_meta.get('programming-language') + common_data['primary_language'] = project_meta.get( + 'programming-language') # FIXME: these cannot be related to actual packages with a download URL releases = project_meta.get('release') @@ -128,7 +130,8 @@ def build_packages_from_projects(metadata, uri=None): if release.get('created') and len(release.get('created')) == 10: rdata['release_date'] = parse_date(release.get('created')) else: - logger.warn('Unexpected date format for release date: {}'.format(release.get('created'))) + logger.warn('Unexpected date format for release date: {}'.format( + release.get('created'))) package = scan_models.Package.from_package_data( package_data=rdata, datafile_path=uri, @@ -136,9 +139,9 @@ def build_packages_from_projects(metadata, uri=None): yield package else: package = scan_models.Package.from_package_data( - package_data=common_data, - datafile_path=uri, - ) + package_data=common_data, + datafile_path=uri, + ) yield package @@ -186,12 +189,14 @@ def get_packages(self, uri, resource_uri): # 1. create a regular package from the URL stripped from its checksum extension archive_uri, _, checksum_type = uri.rpartition('.') - pack = build_package_from_download(archive_uri, resource_uri.package_url) + pack = build_package_from_download( + archive_uri, resource_uri.package_url) # 2. collect the checksum inside the file # and attach it to the package checksum_value = resource_uri.data.strip() if checksum_value: - checksum_field_name = 'download_{checksum_type}'.format(**locals()) + checksum_field_name = 'download_{checksum_type}'.format( + **locals()) setattr(pack, checksum_field_name, checksum_value) yield pack else: diff --git a/minecode/mappers/bitbucket.py b/minecode/mappers/bitbucket.py index 6fed6c2f..5764cb58 100644 --- a/minecode/mappers/bitbucket.py +++ b/minecode/mappers/bitbucket.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -36,7 +36,8 @@ def get_packages(self, uri, resource_uri): """ Yield Package built from resource_uri record for a single package version. """ - downloads_data = json.loads(resource_uri.data, object_pairs_hook=OrderedDict) + downloads_data = json.loads( + resource_uri.data, object_pairs_hook=OrderedDict) for download_data in downloads_data.get('values', []): for package in build_bitbucket_download_packages( download_data, resource_uri.package_url): @@ -75,6 +76,7 @@ class BitbucketIndexMapper(Mapper): """ Build a Package for a repo. """ + def get_packages(self, uri, resource_uri): repo = json.loads(resource_uri.data, object_pairs_hook=OrderedDict) if not repo: @@ -88,6 +90,7 @@ class BitbucketRepoMapper(Mapper): """ Build a Package for a repo. """ + def get_packages(self, uri, resource_uri): repo = json.loads(resource_uri.data, object_pairs_hook=OrderedDict) if not repo: @@ -104,7 +107,8 @@ def build_bitbucket_repo_package(repo_data, purl): scm_protocol = repo_data.get('scm') if not scm_protocol: scm_protocol = 'git' - bb_url = '{protocol}+https://bitbucket.org/{namespace}/{name}'.format(protocol=scm_protocol, **purl.to_dict()) + bb_url = '{protocol}+https://bitbucket.org/{namespace}/{name}'.format( + protocol=scm_protocol, **purl.to_dict()) owner = repo_data.get('owner') owner_party = scan_models.Party( diff --git a/minecode/mappers/bower.py b/minecode/mappers/bower.py index ffc1a8fa..c27f6fc3 100644 --- a/minecode/mappers/bower.py +++ b/minecode/mappers/bower.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -55,7 +55,8 @@ def build_packages_from_jsonfile(metadata, uri=None, purl=None): if devdependencies: for key, value in devdependencies.items(): dev_dependencies.append( - DependentPackage(purl=key, extracted_requirement=value, scope='devdependency').to_dict() + DependentPackage( + purl=key, extracted_requirement=value, scope='devdependency').to_dict() ) dependencies = content.get('dependencies') @@ -63,7 +64,8 @@ def build_packages_from_jsonfile(metadata, uri=None, purl=None): if dependencies: for key, value in dependencies.items(): dependencies_build.append( - DependentPackage(purl=key, extracted_requirement=value, scope='runtime').to_dict() + DependentPackage( + purl=key, extracted_requirement=value, scope='runtime').to_dict() ) if name: @@ -85,14 +87,16 @@ def build_packages_from_jsonfile(metadata, uri=None, purl=None): ) if extracted_license_statement: - common_data['extracted_license_statement'] = list(extracted_license_statement) + common_data['extracted_license_statement'] = list( + extracted_license_statement) author_content = content.get('author') if author_content: parties = common_data.get('parties') if not parties: common_data['parties'] = [] - common_data['parties'].append(scan_models.Party(name=author_content, role='author',).to_dict()) + common_data['parties'].append(scan_models.Party( + name=author_content, role='author',).to_dict()) else: parties = common_data.get('parties') if not parties: @@ -101,7 +105,8 @@ def build_packages_from_jsonfile(metadata, uri=None, purl=None): for author in author_content: author_split = author.split(':') if len(author_split) > 1: - common_data['parties'].append(scan_models.Party(name=author_split[1].strip(), role='author',).to_dict()) + common_data['parties'].append(scan_models.Party( + name=author_split[1].strip(), role='author',).to_dict()) dependencies = [] if dependencies_build: diff --git a/minecode/mappers/cpan.py b/minecode/mappers/cpan.py index 1f57231f..4ea7fce4 100644 --- a/minecode/mappers/cpan.py +++ b/minecode/mappers/cpan.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -27,7 +27,8 @@ def get_packages(self, uri, resource_uri): Yield packages by parsing the json returned from release search request. """ metadata = resource_uri.data - build_packages_from_release_json(metadata, resource_uri.uri, resource_uri.package_url) + build_packages_from_release_json( + metadata, resource_uri.uri, resource_uri.package_url) def build_packages_from_release_json(metadata, uri=None): @@ -47,7 +48,8 @@ def build_packages_from_release_json(metadata, uri=None): if not name: continue - extracted_license_statement = [l for l in release.get('license', []) if l and l.strip()] + extracted_license_statement = [ + l for l in release.get('license', []) if l and l.strip()] common_data = dict( datasource_id="cpan_release_json", @@ -84,7 +86,8 @@ def build_packages_from_release_json(metadata, uri=None): # like perl_5. The license here under resources section is the # url of license for example: http://dev.perl.org/licenses/ So # it's useful to collect both information... - license_url = [l for l in resources.get('license', []) if l and l.strip()] + license_url = [l for l in resources.get( + 'license', []) if l and l.strip()] if license_url: common_data['extracted_license_statement'].extend(license_url) @@ -109,7 +112,8 @@ def build_packages_from_release_json(metadata, uri=None): package_data=common_data, datafile_path=uri, ) - package_url = PackageURL(type='cpan', name=release.get('name'), version=release.get('version')) + package_url = PackageURL(type='cpan', name=release.get( + 'name'), version=release.get('version')) package.set_purl(package_url.to_string()) yield package @@ -140,7 +144,8 @@ def get_packages(self, uri, resource_uri): Yield as many Package as there are download URLs. """ metadata = resource_uri.data - build_packages_from_metafile(metadata, resource_uri.uri, resource_uri.package_url) + build_packages_from_metafile( + metadata, resource_uri.uri, resource_uri.package_url) def build_packages_from_metafile(metadata, uri=None, purl=None): @@ -245,7 +250,8 @@ def get_packages(self, uri, resource_uri): Yield as many Package as there are download URLs. """ metadata = resource_uri.data - build_packages_from_metafile(metadata, resource_uri.uri, resource_uri.package_url) + build_packages_from_metafile( + metadata, resource_uri.uri, resource_uri.package_url) def build_packages_from_readmefile(metadata, uri=None, purl=None): @@ -280,7 +286,8 @@ def build_packages_from_readmefile(metadata, uri=None, purl=None): for author_content in authors: author_split = author_content.split('<') if len(author_split) > 1: - party = scan_models.Party(type=scan_models.party_person, name=author_split[0].rstrip(), role='author', email=author_split[1].replace('>', '')) + party = scan_models.Party(type=scan_models.party_person, name=author_split[0].rstrip( + ), role='author', email=author_split[1].replace('>', '')) parties = common_data.get('parties') if not parties: common_data['parties'] = [] diff --git a/minecode/mappers/cran.py b/minecode/mappers/cran.py index 6b7dc359..d63b98e9 100644 --- a/minecode/mappers/cran.py +++ b/minecode/mappers/cran.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -29,7 +29,8 @@ def get_packages(self, uri, resource_uri): Yield as many Package as there are download URLs. """ metadata = resource_uri.data - build_packages_from_html(metadata, resource_uri.uri, resource_uri.package_url) + build_packages_from_html( + metadata, resource_uri.uri, resource_uri.package_url) def get_download_url(url): @@ -93,7 +94,8 @@ def build_packages_from_html(metadata, uri=None, purl=None): cols = row.find_all('td') for ele in cols: if ele.find_all('a'): - col_values.append([a['href'].strip() for a in ele.find_all('a')]) + col_values.append([a['href'].strip() + for a in ele.find_all('a')]) col_values.append(ele.text.strip()) if len(cols) >= 2: key = col_values[0] @@ -115,7 +117,8 @@ def build_packages_from_html(metadata, uri=None, purl=None): parties = common_data.get('parties') if not parties: common_data['parties'] = [] - party = scan_models.Party(type=scan_models.party_person, name=value, role='author') + party = scan_models.Party( + type=scan_models.party_person, name=value, role='author') common_data['parties'].append(party.to_dict()) elif key == 'Maintainer:': maintainer_split = value.split('<') @@ -123,7 +126,8 @@ def build_packages_from_html(metadata, uri=None, purl=None): parties = common_data.get('parties') if not parties: common_data['parties'] = [] - party = scan_models.Party(type=scan_models.party_person, name=maintainer_split[0].rstrip(), role='maintainer', email=maintainer_split[1].replace('>', '').replace(' at ', '@')) + party = scan_models.Party(type=scan_models.party_person, name=maintainer_split[0].rstrip( + ), role='maintainer', email=maintainer_split[1].replace('>', '').replace(' at ', '@')) common_data['parties'].append(party.to_dict()) elif 'source' in key or 'binaries' in key: if type(value) == list: diff --git a/minecode/mappers/debian.py b/minecode/mappers/debian.py index 2a6b5ca3..a2f6e01a 100644 --- a/minecode/mappers/debian.py +++ b/minecode/mappers/debian.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -57,7 +57,8 @@ def get_dependencies(data): # FIXME:!!! for name in dependencies: purl = PackageURL(type='deb', namespace='debian', name=name) - dep = scan_models.DependentPackage(purl=purl.to_string(), score=scope, **flags) + dep = scan_models.DependentPackage( + purl=purl.to_string(), score=scope, **flags) dep_pkgs.append(dep) return dep_pkgs @@ -76,7 +77,8 @@ def get_vcs_repo(description): repos.append((vcs_tool, vcs_repo)) if len(repos) > 1: - raise TypeError('Debian description with more than one Vcs repos: %(repos)r' % locals()) + raise TypeError( + 'Debian description with more than one Vcs repos: %(repos)r' % locals()) if repos: vcs_tool, vcs_repo = repos[0] @@ -170,9 +172,12 @@ def collect_files(existing_files, field_value, checksum_name): # TODO: what do we do with files? # FIXME: we should store them in the package record files = defaultdict(File) - collect_files(existing_files=files, field_value=metadata.get('Files'), checksum_name='md5') - collect_files(existing_files=files, field_value=metadata.get('Checksums-Sha1'), checksum_name='sha1') - collect_files(existing_files=files, field_value=metadata.get('Checksums-Sha256'), checksum_name='sha256') + collect_files(existing_files=files, field_value=metadata.get( + 'Files'), checksum_name='md5') + collect_files(existing_files=files, field_value=metadata.get( + 'Checksums-Sha1'), checksum_name='sha1') + collect_files(existing_files=files, field_value=metadata.get( + 'Checksums-Sha256'), checksum_name='sha256') # FIXME: craft a download_url download_url = None @@ -208,7 +213,8 @@ def build_source_file_packages(metadata, purl=None): package_name = source.get('Package') parties = [] - maintainer_names = debutils.comma_separated(source.get('Maintainer', '')) + maintainer_names = debutils.comma_separated( + source.get('Maintainer', '')) if maintainer_names: for maintainer in maintainer_names: name, email = debutils.parse_email(maintainer) @@ -216,7 +222,8 @@ def build_source_file_packages(metadata, purl=None): party = scan_models.Party( name=name, role='maintainer', email=email) parties.append(party) - contributor_names = debutils.comma_separated(source.get('Uploaders', '')) + contributor_names = debutils.comma_separated( + source.get('Uploaders', '')) if contributor_names: for contributor in contributor_names: name, email = debutils.parse_email(contributor) @@ -318,7 +325,8 @@ def parse_packages(metadata, purl=None): filename = pack.get('Filename'), if filename: - data['download_url'] = 'http://ftp.debian.org/debian/{}'.format(filename) + data['download_url'] = 'http://ftp.debian.org/debian/{}'.format( + filename) maintainers = pack.get('Maintainer') if maintainers: @@ -425,9 +433,9 @@ def get_resourceuri_by_uri(uri): else: # yield package without a download_url value package = scan_models.Package.from_package_data( - package_data=common_data, - datafile_path=uri, - ) + package_data=common_data, + datafile_path=uri, + ) # FIXME: this is NOT RIGHT: purl is not defined package.set_purl(package.purl) yield package diff --git a/minecode/mappers/dockerhub.py b/minecode/mappers/dockerhub.py index aeacd3c0..92b8697f 100644 --- a/minecode/mappers/dockerhub.py +++ b/minecode/mappers/dockerhub.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -25,7 +25,8 @@ def get_packages(self, uri, resource_uri): Yield as many Package as there are download URLs. """ metadata = resource_uri.data - build_packages_from_jsonfile(metadata, resource_uri.uri, resource_uri.package_url) + build_packages_from_jsonfile( + metadata, resource_uri.uri, resource_uri.package_url) def build_packages_from_jsonfile(metadata, uri=None, purl=None): @@ -47,7 +48,8 @@ def build_packages_from_jsonfile(metadata, uri=None, purl=None): type='docker', name=name, description=description, - homepage_url=dockhub_library_htmlpage_template.format(project=name), + homepage_url=dockhub_library_htmlpage_template.format( + project=name), ) package = scan_models.Package(**common_data) package.set_purl(purl) diff --git a/minecode/mappers/eclipse.py b/minecode/mappers/eclipse.py index 2358384b..9edd9615 100644 --- a/minecode/mappers/eclipse.py +++ b/minecode/mappers/eclipse.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -61,10 +61,12 @@ def build_packages_with_json(metadata, purl=None, uri=None): bug_tracking_urls = project_metadata.get('bugzilla') if bug_tracking_urls and len(bug_tracking_urls) > 0: - common_data['bug_tracking_url'] = bug_tracking_urls[0].get('query_url') + common_data['bug_tracking_url'] = bug_tracking_urls[0].get( + 'query_url') if project_metadata.get('licenses'): - common_data['extracted_license_statement'] = [l.get('name') for l in project_metadata.get('licenses', [])] + common_data['extracted_license_statement'] = [ + l.get('name') for l in project_metadata.get('licenses', [])] common_data['license_detections'] = [] # FIXME: this is a download page and NOT a download URL!!!!! diff --git a/minecode/mappers/fdroid.py b/minecode/mappers/fdroid.py index 4e2e2c3a..e50e59ca 100644 --- a/minecode/mappers/fdroid.py +++ b/minecode/mappers/fdroid.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -99,7 +99,7 @@ def build_packages(purl, data): extracted_license_statement=extracted_license_statement, vcs_url=vcs_url, homepage_url=homepage_url, - repository_homepage_url = f'https://f-droid.org/en/packages/{base_purl.name}', + repository_homepage_url=f'https://f-droid.org/en/packages/{base_purl.name}', description=description, parties=parties, ) @@ -114,7 +114,8 @@ def build_packages(purl, data): for _sha256_of_apk, version_data in versions.items(): # TODO: collect versionName version_code = str(version_data['manifest']['versionCode']) - logger.debug(f'build_packages: base_purl: {base_purl} version: {version_code}') + logger.debug( + f'build_packages: base_purl: {base_purl} version: {version_code}') logger.debug(f'build_packages: data: {version_data}') # TODO: add release_date from "added": 1655164800000, @@ -212,4 +213,3 @@ def build_description(metadata, language='en-US'): non_empty_parts = [p for p in [name, summary, description] if p] return '\n'.join(non_empty_parts) - diff --git a/minecode/mappers/freebsd.py b/minecode/mappers/freebsd.py index cd3d89e9..fb0a760b 100644 --- a/minecode/mappers/freebsd.py +++ b/minecode/mappers/freebsd.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/mappers/freedesktop.py b/minecode/mappers/freedesktop.py index 185a4e26..297b48c1 100644 --- a/minecode/mappers/freedesktop.py +++ b/minecode/mappers/freedesktop.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/mappers/github.py b/minecode/mappers/github.py index aced9044..6ef72f00 100644 --- a/minecode/mappers/github.py +++ b/minecode/mappers/github.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -98,7 +98,8 @@ def build_github_packages(visited_data, uri, purl=None): for tag in tags: package = attr.evolve(package) package.version = tag.get('name') - package_url = PackageURL(type='github', name=package.name, namespace=namespace, version=tag.get('name')).to_string() + package_url = PackageURL(type='github', name=package.name, + namespace=namespace, version=tag.get('name')).to_string() package.sha1 = tag.get('sha1') if tag.get('tarball_url'): package.download_url = tag.get('tarball_url') diff --git a/minecode/mappers/gitlab.py b/minecode/mappers/gitlab.py index fb200186..8a6d7c6c 100644 --- a/minecode/mappers/gitlab.py +++ b/minecode/mappers/gitlab.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/mappers/golang.py b/minecode/mappers/golang.py index a431934e..cd750da1 100644 --- a/minecode/mappers/golang.py +++ b/minecode/mappers/golang.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -31,7 +31,8 @@ def build_golang_package(package_data, purl): """ package_url = PackageURL.from_string(purl) vcs_url = package_url.qualifiers.get('vcs_repository') - homepage_url = '/'.join(['https:/', package_url.namespace, package_url.name]) + homepage_url = '/'.join(['https:/', + package_url.namespace, package_url.name]) vcs_tool = 'git' if 'github.com' in package_url.namespace else None if vcs_tool: vcs_url = form_vcs_url(vcs_tool, vcs_url) diff --git a/minecode/mappers/googlecode.py b/minecode/mappers/googlecode.py index 03e09312..b06bc538 100644 --- a/minecode/mappers/googlecode.py +++ b/minecode/mappers/googlecode.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/mappers/gstreamer.py b/minecode/mappers/gstreamer.py index 7758cb2e..8d953302 100644 --- a/minecode/mappers/gstreamer.py +++ b/minecode/mappers/gstreamer.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/mappers/haxe.py b/minecode/mappers/haxe.py index 7fc21de6..a8b6e594 100644 --- a/minecode/mappers/haxe.py +++ b/minecode/mappers/haxe.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/mappers/maven.py b/minecode/mappers/maven.py index 781cf218..a3ec9855 100644 --- a/minecode/mappers/maven.py +++ b/minecode/mappers/maven.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -68,7 +68,8 @@ def get_mini_package(data, uri, purl): qualifiers = None if purl and purl.qualifiers: - qualifiers = packageurl.normalize_qualifiers(purl.qualifiers, encode=False) + qualifiers = packageurl.normalize_qualifiers( + purl.qualifiers, encode=False) if qualifiers: assert isinstance(qualifiers, dict) logger.debug('get_mini_package: qualifiers: {}'.format(qualifiers)) @@ -85,8 +86,10 @@ def get_mini_package(data, uri, purl): size=artifact.size, sha1=artifact.sha1 or None, ) - logger.debug('get_mini_package: package.qualifiers: {}'.format(package.qualifiers)) - logger.debug('get_mini_package for uri: {}, package: {}'.format(uri, package)) + logger.debug('get_mini_package: package.qualifiers: {}'.format( + package.qualifiers)) + logger.debug( + 'get_mini_package for uri: {}, package: {}'.format(uri, package)) return package @@ -96,6 +99,7 @@ class MavenPomMapper(Mapper): """ Map a proper full POM visited as XML. """ + def get_packages(self, uri, resource_uri): logger.debug('MavenPomMapper.get_packages: uri: {}, resource_uri: {}, purl:' diff --git a/minecode/mappers/npm.py b/minecode/mappers/npm.py index 34d8f718..0e13d5d5 100644 --- a/minecode/mappers/npm.py +++ b/minecode/mappers/npm.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/mappers/nuget.py b/minecode/mappers/nuget.py index d939f980..7b4b2c0d 100644 --- a/minecode/mappers/nuget.py +++ b/minecode/mappers/nuget.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -69,7 +69,8 @@ def build_packages_with_json(metadata, purl=None): parties=authors, keywords=keywords, ) - package = scan_models.PackageData.from_data(package_data=package_mapping) + package = scan_models.PackageData.from_data( + package_data=package_mapping) package.set_purl(purl) yield package @@ -159,7 +160,8 @@ def build_packages_from_html(metadata, uri, purl=None): if not version or not version.strip(): continue version = version.strip() - download_url = download_url_format.format(name=name, version=version) + download_url = download_url_format.format( + name=name, version=version) package_mapping = dict( datasource_id="nuget_metadata_json", name=name, diff --git a/minecode/mappers/openssl.py b/minecode/mappers/openssl.py index fd76ac51..c082cd52 100644 --- a/minecode/mappers/openssl.py +++ b/minecode/mappers/openssl.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/mappers/openwrt.py b/minecode/mappers/openwrt.py index 4145e484..7c1b9ec1 100644 --- a/minecode/mappers/openwrt.py +++ b/minecode/mappers/openwrt.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -67,7 +67,8 @@ def build_packages(metadata, purl=None, uri=None): parties = common_data.get('parties') if not parties: common_data['parties'] = [] - party = scan_models.Party(name=name, role='maintainer', email=email) + party = scan_models.Party( + name=name, role='maintainer', email=email) common_data['parties'].append(party) lic = metadata.get('License') diff --git a/minecode/mappers/packagist.py b/minecode/mappers/packagist.py index fe70fe7d..dc05dd30 100644 --- a/minecode/mappers/packagist.py +++ b/minecode/mappers/packagist.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -69,20 +69,23 @@ def build_packages_with_json(metadata, purl=None, uri=None): if not parties: common['parties'] = [] common['parties'].append( - scan_models.Party(name=author.get('name'), role='author', url=author.get('homepage'), email=author.get('email')).to_dict() + scan_models.Party(name=author.get('name'), role='author', url=author.get( + 'homepage'), email=author.get('email')).to_dict() ) extracted_license_statement = set([]) for lic in version_content.get('license'): extracted_license_statement.add(lic) if extracted_license_statement: - common['extracted_license_statement'] = list(extracted_license_statement) + common['extracted_license_statement'] = list( + extracted_license_statement) common['license_detections'] = [] dependencies = [] for name, version in version_content.get('require', {}).items(): dependencies.append( - DependentPackage(purl=name, extracted_requirement=version, scope='runtime').to_dict() + DependentPackage( + purl=name, extracted_requirement=version, scope='runtime').to_dict() ) if dependencies: common['dependencies'] = dependencies diff --git a/minecode/mappers/pypi.py b/minecode/mappers/pypi.py index c97e84f3..69110414 100644 --- a/minecode/mappers/pypi.py +++ b/minecode/mappers/pypi.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -83,7 +83,8 @@ def build_packages(metadata, purl=None): classifiers = info.get('classifiers') if classifiers and not extracted_license_statement: - licenses = [lic for lic in classifiers if lic.lower().startswith('license')] + licenses = [ + lic for lic in classifiers if lic.lower().startswith('license')] for lic in licenses: extracted_license_statement.append(lic) @@ -91,7 +92,8 @@ def build_packages(metadata, purl=None): kw = info.get('keywords') if kw: - common_data['keywords'] = [k.strip() for k in kw.split(',') if k.strip()] + common_data['keywords'] = [k.strip() + for k in kw.split(',') if k.strip()] # FIXME: we should either support "extra" data in a ScannedPackage or just ignore this kind of FIXME comments for now diff --git a/minecode/mappers/repomd.py b/minecode/mappers/repomd.py index 6876d0ac..10b8bf66 100644 --- a/minecode/mappers/repomd.py +++ b/minecode/mappers/repomd.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/mappers/rubygems.py b/minecode/mappers/rubygems.py index aee34e85..19f0e0c1 100644 --- a/minecode/mappers/rubygems.py +++ b/minecode/mappers/rubygems.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -69,7 +69,8 @@ def build_rubygem_packages_from_api_data(metadata, name, purl=None): if version_details.get('sha'): package['sha256'] = version_details.get('sha') - package['release_date'] = parse_date(version_details.get('created_at') or '') or None + package['release_date'] = parse_date( + version_details.get('created_at') or '') or None author = version_details.get('authors') if author: diff --git a/minecode/mappers/sourceforge.py b/minecode/mappers/sourceforge.py index 8b2b40b0..887e6748 100644 --- a/minecode/mappers/sourceforge.py +++ b/minecode/mappers/sourceforge.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -50,7 +50,8 @@ def build_packages_from_metafile(metadata, purl=None, uri=None): type='sourceforge', name=metadata.get('shortname', metadata.get('name')), description=description, - homepage_url=metadata.get('external_homepage', metadata.get('url')), + homepage_url=metadata.get( + 'external_homepage', metadata.get('url')), license_detections=[], ) @@ -61,7 +62,8 @@ def build_packages_from_metafile(metadata, purl=None, uri=None): common_data['parties'] = [] if dev.get('name'): common_data['parties'].append( - scan_models.Party(name=dev.get('name'), role='contributor', url=dev.get('url')).to_dict() + scan_models.Party(name=dev.get( + 'name'), role='contributor', url=dev.get('url')).to_dict() ) categories = metadata.get('categories', {}) diff --git a/minecode/mappings/__init__.py b/minecode/mappings/__init__.py index 2eb8f9f0..e1521118 100644 --- a/minecode/mappings/__init__.py +++ b/minecode/mappings/__init__.py @@ -3,6 +3,6 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/mappings/gcode_keywords.py b/minecode/mappings/gcode_keywords.py index 7660aa0b..57fe1648 100644 --- a/minecode/mappings/gcode_keywords.py +++ b/minecode/mappings/gcode_keywords.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/mappings/gcode_licenses.py b/minecode/mappings/gcode_licenses.py index 40d2de59..ca5f6483 100644 --- a/minecode/mappings/gcode_licenses.py +++ b/minecode/mappings/gcode_licenses.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/mappings/gcode_programming_languages.py b/minecode/mappings/gcode_programming_languages.py index 08a1b626..e4ef8608 100644 --- a/minecode/mappings/gcode_programming_languages.py +++ b/minecode/mappings/gcode_programming_languages.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/mappings/pypi_trove.py b/minecode/mappings/pypi_trove.py index 9b7b6802..4c860c2d 100644 --- a/minecode/mappings/pypi_trove.py +++ b/minecode/mappings/pypi_trove.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/mappings/sfnet_licenses.py b/minecode/mappings/sfnet_licenses.py index 93f8eadb..dd8810dd 100644 --- a/minecode/mappings/sfnet_licenses.py +++ b/minecode/mappings/sfnet_licenses.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/mappings/sfnet_programming_languages.py b/minecode/mappings/sfnet_programming_languages.py index ee811398..3a646004 100644 --- a/minecode/mappings/sfnet_programming_languages.py +++ b/minecode/mappings/sfnet_programming_languages.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/models.py b/minecode/models.py index 774b58c2..1e64d99c 100644 --- a/minecode/models.py +++ b/minecode/models.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -214,7 +214,8 @@ def needs_revisit(self, uri, hours): if existing: return False - revisitable = self.get_revisitables(hours=hours).filter(uri=uri).exists() + revisitable = self.get_revisitables( + hours=hours).filter(uri=uri).exists() if revisitable: return True else: @@ -596,10 +597,10 @@ def get_processables(self): database for update. """ qs = self.filter(scan_status__in=[ - ScannableURI.SCAN_SUBMITTED, - ScannableURI.SCAN_IN_PROGRESS, - ScannableURI.SCAN_COMPLETED - ], + ScannableURI.SCAN_SUBMITTED, + ScannableURI.SCAN_IN_PROGRESS, + ScannableURI.SCAN_COMPLETED + ], wip_date=None, scan_error=None, ) # NOTE: this matches an index for efficient ordering @@ -621,7 +622,8 @@ def statistics(self): """ Return a statistics mapping with summary counts of ScannableURI grouped by status. """ - statuses = list(self.values('scan_status').annotate(count=models.Count('scan_status')).order_by('scan_status'),) + statuses = list(self.values('scan_status').annotate( + count=models.Count('scan_status')).order_by('scan_status'),) for stat in statuses: stat['scan_status'] = ScannableURI.SCAN_STATUSES_BY_CODE[stat['scan_status']] stats = { @@ -632,12 +634,18 @@ def statistics(self): } most_recent = dict( - most_recent_submitted=self._recent(scan_status=ScannableURI.SCAN_SUBMITTED), - most_recent_indexed=self._recent(scan_status=ScannableURI.SCAN_INDEXED), - most_recent_failed=self._recent(scan_status=ScannableURI.SCAN_FAILED, extra_value="scan_error",), - most_recent_in_progress=self._recent(scan_status=ScannableURI.SCAN_IN_PROGRESS), - most_recent_completed=self._recent(scan_status=ScannableURI.SCAN_COMPLETED), - most_recent_index_errors=self._recent(scan_status=ScannableURI.SCAN_INDEX_FAILED, extra_value="index_error",), + most_recent_submitted=self._recent( + scan_status=ScannableURI.SCAN_SUBMITTED), + most_recent_indexed=self._recent( + scan_status=ScannableURI.SCAN_INDEXED), + most_recent_failed=self._recent( + scan_status=ScannableURI.SCAN_FAILED, extra_value="scan_error",), + most_recent_in_progress=self._recent( + scan_status=ScannableURI.SCAN_IN_PROGRESS), + most_recent_completed=self._recent( + scan_status=ScannableURI.SCAN_COMPLETED), + most_recent_index_errors=self._recent( + scan_status=ScannableURI.SCAN_INDEX_FAILED, extra_value="index_error",), ) stats.update(most_recent) return stats @@ -648,7 +656,8 @@ def _recent(self, scan_status, extra_value=None, most_recent=10): ``scan_status``. Include an optional ``extra value`` field name. """ - recent_uris = self.filter(scan_status=scan_status).order_by('-scan_date')[:most_recent] + recent_uris = self.filter(scan_status=scan_status).order_by( + '-scan_date')[:most_recent] for scauri in recent_uris: recent = dict( # this is NOT a field requiring this loop @@ -898,7 +907,8 @@ def get_next_request(self): NOTE: this method can only be called from within a transaction.atomic block. """ - priority_resource_uri = self.get_requests().select_for_update(skip_locked=True).first() + priority_resource_uri = self.get_requests( + ).select_for_update(skip_locked=True).first() if not priority_resource_uri: return priority_resource_uri.wip_date = timezone.now() diff --git a/minecode/route.py b/minecode/route.py index 607828b5..59a0cd97 100644 --- a/minecode/route.py +++ b/minecode/route.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -44,6 +44,7 @@ class Rule(object): (typically a URI) for the rule to be considered, i.e. for the endpoint to be resolved and eventually invoked for a given string (typically a URI). """ + def __init__(self, pattern, endpoint): # To ensure the pattern will match entirely, we wrap the pattern # with start of line ^ and end of line $. @@ -101,6 +102,7 @@ class Router(object): Multiple routers can co-exist as needed, such as a router to collect, another to fetch, etc. """ + def __init__(self, route_map=None): """ 'route_map' is an ordered mapping of pattern -> Rule. diff --git a/minecode/rsync.py b/minecode/rsync.py index 1e0971db..164b4e5d 100644 --- a/minecode/rsync.py +++ b/minecode/rsync.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -133,7 +133,8 @@ def fetch_directory(uri, recurse=True): Return the location of a tempfile containing an rsync dir listing for uri. Recursive if recurse is True. Raise an Exception with error details. """ - temp_file = get_temp_file(file_name='minecode-rsync-dir-', extension='.rsync') + temp_file = get_temp_file( + file_name='minecode-rsync-dir-', extension='.rsync') with open(temp_file, 'w') as tmp: file_name = tmp.name ends = not uri.endswith('/') and '/' or '' diff --git a/minecode/saneyaml.py b/minecode/saneyaml.py index 70f91aff..499c1eee 100644 --- a/minecode/saneyaml.py +++ b/minecode/saneyaml.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -72,6 +72,7 @@ class SaneLoader(SafeLoader): """ A safe loader configured with many sane defaults. """ + def ignore_aliases(self, data): return True @@ -182,8 +183,10 @@ def string_dumper(dumper, value, _tag=u'tag:yaml.org,2002:str'): SaneDumper.add_representer(str, string_dumper) # treat number as strings, not as numbers -SaneDumper.add_representer(int, partial(string_dumper, _tag=u'tag:yaml.org,2002:int')) -SaneDumper.add_representer(float, partial(string_dumper, _tag=u'tag:yaml.org,2002:float')) +SaneDumper.add_representer(int, partial( + string_dumper, _tag=u'tag:yaml.org,2002:int')) +SaneDumper.add_representer(float, partial( + string_dumper, _tag=u'tag:yaml.org,2002:float')) def boolean_dumper(dumper, value): diff --git a/minecode/seed.py b/minecode/seed.py index 8baccc2a..61892ef3 100644 --- a/minecode/seed.py +++ b/minecode/seed.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/tasks.py b/minecode/tasks.py index 836192b4..072ab06f 100644 --- a/minecode/tasks.py +++ b/minecode/tasks.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/tests/__init__.py b/minecode/tests/__init__.py index e27909e9..8598b637 100644 --- a/minecode/tests/__init__.py +++ b/minecode/tests/__init__.py @@ -3,11 +3,11 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -import os +import os FIXTURES_REGEN = os.environ.get("MINECODE_TEST_FIXTURES_REGEN", False) diff --git a/minecode/tests/test_apache.py b/minecode/tests/test_apache.py index 918a120c..306c08bb 100644 --- a/minecode/tests/test_apache.py +++ b/minecode/tests/test_apache.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -33,7 +33,8 @@ def test_ApacheDistIndexVisitor(self): mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _, _ = apache.ApacheDistIndexVisitor(uri) - expected_loc = self.get_test_loc('apache/find-ls.gz_uris-expected.json') + expected_loc = self.get_test_loc( + 'apache/find-ls.gz_uris-expected.json') self.check_expected_uris(uris, expected_loc, regen=FIXTURES_REGEN) def test_ApacheChecksumVisitor(self): @@ -136,43 +137,53 @@ def test_build_package_from_download(self): 'http://archive.apache.org/dist/groovy/2.4.6/sources/apache-groovy-src-2.4.6.zip', 'pkg:apache/groovy@2.4.6') expected_loc = self.get_test_loc('apache/map-groovy_expected.json') - self.check_expected_results(package.to_dict(), expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + package.to_dict(), expected_loc, regen=FIXTURES_REGEN) def test_build_package_from_download2(self): package = mappers.apache.build_package_from_download( 'http://archive.apache.org/dist/turbine/maven/turbine-webapp-2.3.3-1.0.0-source-release.zip', 'pkg:apache/turbine-webapp@2.3.3-1.0.0-source-release') - expected_loc = self.get_test_loc('apache/map-turbine-webapp_expected.json') - self.check_expected_results(package.to_dict(), expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'apache/map-turbine-webapp_expected.json') + self.check_expected_results( + package.to_dict(), expected_loc, regen=FIXTURES_REGEN) # TODO: add tests for checksums def test_build_packages_from_projects_json(self): with open(self.get_test_loc('apache/projects.json')) as projectsjson_meta: - metadata = json.load(projectsjson_meta, object_pairs_hook=OrderedDict) + metadata = json.load( + projectsjson_meta, object_pairs_hook=OrderedDict) packages = mappers.apache.build_packages_from_projects(metadata) packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('apache/projects_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_packages_from_one_podling_json(self): with open(self.get_test_loc('apache/podling_amaterasu.json')) as podlings_meta: metadata = json.load(podlings_meta, object_pairs_hook=OrderedDict) - packages = mappers.apache.build_packages_from_podlings(metadata, purl='pkg:apache-podlings/amaterasu') + packages = mappers.apache.build_packages_from_podlings( + metadata, purl='pkg:apache-podlings/amaterasu') packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('apache/podling_amaterasu_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'apache/podling_amaterasu_expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) # TODO: add real mapper class tests c def test_regex_1(self): regex = re.compile(r'^https?://(archive\.)?apache\.org/dist/.*$') - result = re.match(regex, 'http://archive.apache.org/dist/groovy/2.4.6/sources/apache-groovy-src-2.4.6.zip') + result = re.match( + regex, 'http://archive.apache.org/dist/groovy/2.4.6/sources/apache-groovy-src-2.4.6.zip') self.assertTrue(result) def test_regex_2(self): regex = re.compile(r'^https?://(archive\.)?apache\.org/dist/.*$') - result = re.match(regex, 'https://apache.org/dist/chemistry/opencmis/1.1.0/chemistry-opencmis-dist-1.1.0-server-webapps.zip') + result = re.match( + regex, 'https://apache.org/dist/chemistry/opencmis/1.1.0/chemistry-opencmis-dist-1.1.0-server-webapps.zip') self.assertTrue(result) diff --git a/minecode/tests/test_api.py b/minecode/tests/test_api.py index 9330b6fe..f3507a12 100644 --- a/minecode/tests/test_api.py +++ b/minecode/tests/test_api.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -31,12 +31,15 @@ def setUp(self): email="e@mail.com", password="secret" ) - scan_queue_workers_group, _ = Group.objects.get_or_create(name='scan_queue_workers') + scan_queue_workers_group, _ = Group.objects.get_or_create( + name='scan_queue_workers') scan_queue_workers_group.user_set.add(self.scan_queue_worker_user) self.scan_queue_worker_auth = f"Token {self.scan_queue_worker_user.auth_token.key}" self.scan_queue_worker_client = APIClient(enforce_csrf_checks=True) - self.scan_queue_worker_client.credentials(HTTP_AUTHORIZATION=self.scan_queue_worker_auth) - self.scan_queue_worker_user_id_str = str(self.scan_queue_worker_user.id) + self.scan_queue_worker_client.credentials( + HTTP_AUTHORIZATION=self.scan_queue_worker_auth) + self.scan_queue_worker_user_id_str = str( + self.scan_queue_worker_user.id) # create a staff user self.staff_user = User.objects.create_user( @@ -98,7 +101,8 @@ def test_api_scannable_uri_permissions(self): response = self.anonymous_client.get('/api/scan_queue/') self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) - response = self.anonymous_client.get('/api/scan_queue/get_next_download_url/') + response = self.anonymous_client.get( + '/api/scan_queue/get_next_download_url/') self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) response = self.anonymous_client.post('/api/scan_queue/update_status/') @@ -107,7 +111,8 @@ def test_api_scannable_uri_permissions(self): response = self.regular_client.get('/api/scan_queue/') self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) - response = self.regular_client.get('/api/scan_queue/get_next_download_url/') + response = self.regular_client.get( + '/api/scan_queue/get_next_download_url/') self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) response = self.regular_client.post('/api/scan_queue/update_status/') @@ -127,33 +132,45 @@ def check_webhook_url(self, webhook_url): webhook_url = response.data.get('webhook_url') key = webhook_url.rstrip('/').split('/')[-1] self.assertIn('/api/scan_queue/index_package_scan/', webhook_url) - self.assertEqual(signing.loads(key), str(self.scan_queue_worker_user.id)) + self.assertEqual(signing.loads(key), str( + self.scan_queue_worker_user.id)) - response = self.scan_queue_worker_client.get('/api/scan_queue/get_next_download_url/') + response = self.scan_queue_worker_client.get( + '/api/scan_queue/get_next_download_url/') self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.data.get('scannable_uri_uuid'), self.scannable_uri1.uuid) - self.assertEqual(response.data.get('download_url'), self.scannable_uri1.uri) + self.assertEqual(response.data.get( + 'scannable_uri_uuid'), self.scannable_uri1.uuid) + self.assertEqual(response.data.get( + 'download_url'), self.scannable_uri1.uri) check_webhook_url(self, response.data.get('webhook_url')) - response = self.scan_queue_worker_client.get('/api/scan_queue/get_next_download_url/') + response = self.scan_queue_worker_client.get( + '/api/scan_queue/get_next_download_url/') self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.data.get('scannable_uri_uuid'), self.scannable_uri2.uuid) - self.assertEqual(response.data.get('download_url'), self.scannable_uri2.uri) + self.assertEqual(response.data.get( + 'scannable_uri_uuid'), self.scannable_uri2.uuid) + self.assertEqual(response.data.get( + 'download_url'), self.scannable_uri2.uri) check_webhook_url(self, response.data.get('webhook_url')) - response = self.scan_queue_worker_client.get('/api/scan_queue/get_next_download_url/') + response = self.scan_queue_worker_client.get( + '/api/scan_queue/get_next_download_url/') self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.data.get('scannable_uri_uuid'), self.scannable_uri3.uuid) - self.assertEqual(response.data.get('download_url'), self.scannable_uri3.uri) + self.assertEqual(response.data.get( + 'scannable_uri_uuid'), self.scannable_uri3.uuid) + self.assertEqual(response.data.get( + 'download_url'), self.scannable_uri3.uri) check_webhook_url(self, response.data.get('webhook_url')) - response = self.scan_queue_worker_client.get('/api/scan_queue/get_next_download_url/') + response = self.scan_queue_worker_client.get( + '/api/scan_queue/get_next_download_url/') self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data.get('scannable_uri_uuid'), '') self.assertEqual(response.data.get('download_url'), '') self.assertEqual(response.data.get('webhook_url'), '') - response = self.staff_client.get('/api/scan_queue/get_next_download_url/') + response = self.staff_client.get( + '/api/scan_queue/get_next_download_url/') self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data.get('scannable_uri_uuid'), '') self.assertEqual(response.data.get('download_url'), '') @@ -165,22 +182,26 @@ def test_api_scannable_uri_update_status(self): scannable_uri1_update_status_url = f'/api/scan_queue/{scannable_uri1_uuid}/update_status/' scannable_uri2_update_status_url = f'/api/scan_queue/{scannable_uri2_uuid}/update_status/' - self.assertEqual(ScannableURI.SCAN_NEW, self.scannable_uri1.scan_status) + self.assertEqual(ScannableURI.SCAN_NEW, + self.scannable_uri1.scan_status) data = { "scannable_uri_uuid": scannable_uri1_uuid, "scan_status": 'failed', 'scan_log': 'scan_log', } - response = self.scan_queue_worker_client.post(scannable_uri1_update_status_url, data=data) + response = self.scan_queue_worker_client.post( + scannable_uri1_update_status_url, data=data) self.assertEqual(response.status_code, status.HTTP_200_OK) self.scannable_uri1.refresh_from_db() - self.assertEqual(ScannableURI.SCAN_FAILED, self.scannable_uri1.scan_status) + self.assertEqual(ScannableURI.SCAN_FAILED, + self.scannable_uri1.scan_status) self.assertEqual('scan_log', self.scannable_uri1.scan_error) data = { 'scan_status': '' } - response = self.scan_queue_worker_client.post(scannable_uri2_update_status_url, data=data) + response = self.scan_queue_worker_client.post( + scannable_uri2_update_status_url, data=data) expected_response = {'error': 'missing scan_status'} self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertEqual(expected_response, response.data) @@ -188,14 +209,17 @@ def test_api_scannable_uri_update_status(self): data = { 'scan_status': 'invalid' } - response = self.scan_queue_worker_client.post(scannable_uri2_update_status_url, data=data) + response = self.scan_queue_worker_client.post( + scannable_uri2_update_status_url, data=data) expected_response = {'error': 'invalid scan_status: invalid'} self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertEqual(expected_response, response.data) data = {} - response = self.scan_queue_worker_client.post('/api/scan_queue/asdf/', data=data) - self.assertEqual(response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED) + response = self.scan_queue_worker_client.post( + '/api/scan_queue/asdf/', data=data) + self.assertEqual(response.status_code, + status.HTTP_405_METHOD_NOT_ALLOWED) def test_api_scannable_uri_update_status_update_finished_scannable_uri(self): scannable_uri_uuid = self.scannable_uri3.uuid @@ -216,8 +240,8 @@ def test_api_scannable_uri_update_status_update_finished_scannable_uri(self): ) expected_response = { 'error': 'cannot update status for scannable_uri ' - f'{self.scannable_uri3.uuid}: scannable_uri has finished ' - f'with status "{ScannableURI.SCAN_STATUSES_BY_CODE[scan_status]}"' + f'{self.scannable_uri3.uuid}: scannable_uri has finished ' + f'with status "{ScannableURI.SCAN_STATUSES_BY_CODE[scan_status]}"' } self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertEqual(expected_response, response.data) @@ -232,7 +256,8 @@ def test_api_scannable_uri_index_package_scan(self): self.assertFalse(self.package2.copyright) self.assertEqual(0, Resource.objects.all().count()) scan_file_location = self.get_test_loc('scancodeio/get_scan_data.json') - summary_file_location = self.get_test_loc('scancodeio/scan_summary_response.json') + summary_file_location = self.get_test_loc( + 'scancodeio/scan_summary_response.json') project_extra_data = { 'scannable_uri_uuid': self.scannable_uri2.uuid, 'md5': 'md5', @@ -255,19 +280,24 @@ def test_api_scannable_uri_index_package_scan(self): 'summary': summary, } - webhook_url = get_webhook_url('index_package_scan', self.scan_queue_worker_user.id) + webhook_url = get_webhook_url( + 'index_package_scan', self.scan_queue_worker_user.id) - response = self.scan_queue_worker_client.post(webhook_url, data=data, format='json') + response = self.scan_queue_worker_client.post( + webhook_url, data=data, format='json') self.assertEqual(response.status_code, status.HTTP_200_OK) self.scannable_uri2.refresh_from_db() - self.assertEqual(ScannableURI.SCAN_INDEXED, self.scannable_uri2.scan_status) + self.assertEqual(ScannableURI.SCAN_INDEXED, + self.scannable_uri2.scan_status) self.package2.refresh_from_db() self.assertEqual('md5', self.package2.md5) self.assertEqual('sha1', self.package2.sha1) self.assertEqual('sha256', self.package2.sha256) self.assertEqual('sha512', self.package2.sha512) self.assertEqual(100, self.package2.size) - self.assertEqual('apache-2.0', self.package2.declared_license_expression) - self.assertEqual('Copyright (c) Apache Software Foundation', self.package2.copyright) + self.assertEqual( + 'apache-2.0', self.package2.declared_license_expression) + self.assertEqual( + 'Copyright (c) Apache Software Foundation', self.package2.copyright) self.assertFalse(self.scannable_uri2.scan_error) self.assertEqual(64, Resource.objects.all().count()) diff --git a/minecode/tests/test_bitbucket.py b/minecode/tests/test_bitbucket.py index f99eb6ac..1284c4ac 100644 --- a/minecode/tests/test_bitbucket.py +++ b/minecode/tests/test_bitbucket.py @@ -4,7 +4,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -39,11 +39,14 @@ def test_BitbucketIndexVisitor(self): mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, data, _ = BitbucketIndexVisitor(uri) - expected_uri_loc = self.get_test_loc('bitbucket/visit/index-repositories_expected_uris.json') + expected_uri_loc = self.get_test_loc( + 'bitbucket/visit/index-repositories_expected_uris.json') self.check_expected_uris(uris, expected_uri_loc, regen=FIXTURES_REGEN) - expected_data_loc = self.get_test_loc('bitbucket/visit/index-repositories_expected_data.json') - self.check_expected_results(data, expected_data_loc, regen=FIXTURES_REGEN) + expected_data_loc = self.get_test_loc( + 'bitbucket/visit/index-repositories_expected_data.json') + self.check_expected_results( + data, expected_data_loc, regen=FIXTURES_REGEN) def test_BitbucketSingleRepoVisitor(self): uri = 'https://api.bitbucket.org/2.0/repositories/bastiand/mercurialeclipse/' @@ -53,10 +56,13 @@ def test_BitbucketSingleRepoVisitor(self): mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, data, _ = BitbucketSingleRepoVisitor(uri) - expected_data_loc = self.get_test_loc('bitbucket/visit/singlerepo_expected_data.json') - self.check_expected_results(data, expected_data_loc, regen=FIXTURES_REGEN) + expected_data_loc = self.get_test_loc( + 'bitbucket/visit/singlerepo_expected_data.json') + self.check_expected_results( + data, expected_data_loc, regen=FIXTURES_REGEN) - expected_uris_loc = self.get_test_loc('bitbucket/visit/singlerepo_expected_uris.json') + expected_uris_loc = self.get_test_loc( + 'bitbucket/visit/singlerepo_expected_uris.json') self.check_expected_uris(uris, expected_uris_loc, regen=FIXTURES_REGEN) def test_BitbucketDetailsVisitorPaginated(self): @@ -67,10 +73,13 @@ def test_BitbucketDetailsVisitorPaginated(self): mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, data, _ = BitbucketDetailsVisitorPaginated(uri) - expected_data_loc = self.get_test_loc('bitbucket/visit/paginated_tags_expected_data.json') - self.check_expected_results(data, expected_data_loc, regen=FIXTURES_REGEN) + expected_data_loc = self.get_test_loc( + 'bitbucket/visit/paginated_tags_expected_data.json') + self.check_expected_results( + data, expected_data_loc, regen=FIXTURES_REGEN) - expected_uris_loc = self.get_test_loc('bitbucket/visit/paginated_tags_expected_uris.json') + expected_uris_loc = self.get_test_loc( + 'bitbucket/visit/paginated_tags_expected_uris.json') self.check_expected_uris(uris, expected_uris_loc, regen=FIXTURES_REGEN) @@ -100,16 +109,20 @@ def test_build_bitbucket_repo_package(self): repo_data = json.load(pck, object_pairs_hook=OrderedDict) purl = 'pkg:bitbucket/bastiand/mercurialeclipse' package = build_bitbucket_repo_package(repo_data, purl) - expected_loc = self.get_test_loc('bitbucket/map/repository_expected.json') - self.check_expected_results(package.to_dict(), expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'bitbucket/map/repository_expected.json') + self.check_expected_results( + package.to_dict(), expected_loc, regen=FIXTURES_REGEN) def test_build_bitbucket_repo_package_with_issues(self): with open(self.get_test_loc('bitbucket/map/tweakmsg.json')) as pck: repo_data = json.load(pck, object_pairs_hook=OrderedDict) purl = 'pkg:bitbucket/phlogistonjohn/tweakmsg' package = build_bitbucket_repo_package(repo_data, purl) - expected_loc = self.get_test_loc('bitbucket/map/tweakmsg_expected.json') - self.check_expected_results(package.to_dict(), expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'bitbucket/map/tweakmsg_expected.json') + self.check_expected_results( + package.to_dict(), expected_loc, regen=FIXTURES_REGEN) def test_build_bitbucket_download_packages_single(self): with open(self.get_test_loc('bitbucket/map/downloads.json')) as pck: @@ -117,8 +130,10 @@ def test_build_bitbucket_download_packages_single(self): purl = 'pkg:bitbucket/bastiand/mercurialeclipse' packages = build_bitbucket_download_packages(dnl_data, purl) packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('bitbucket/map/downloads_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'bitbucket/map/downloads_expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_bitbucket_download_packages_many(self): with open(self.get_test_loc('bitbucket/map/downloads_many.json')) as pck: @@ -126,5 +141,7 @@ def test_build_bitbucket_download_packages_many(self): purl = 'pkg:bitbucket/pypa/setuptools' packages = build_bitbucket_download_packages(dnl_data, purl) packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('bitbucket/map/downloads_many_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'bitbucket/map/downloads_many_expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) diff --git a/minecode/tests/test_bower.py b/minecode/tests/test_bower.py index 811454db..69aa33c5 100644 --- a/minecode/tests/test_bower.py +++ b/minecode/tests/test_bower.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -63,5 +63,6 @@ def test_build_packages_metafile_from_bowerjson2(self): result = mappers.bower.build_packages_from_jsonfile( metadata, 'https://coding.net/u/QiaoButang/p/jquery.easing-qbt/git/raw/master/bower.json', 'pkg:bower/1140-grid') result = [p.to_dict() for p in result] - expected_loc = self.get_test_loc('bower/expected_mapper_example1_bower.json') + expected_loc = self.get_test_loc( + 'bower/expected_mapper_example1_bower.json') self.check_expected_results(result, expected_loc, regen=FIXTURES_REGEN) diff --git a/minecode/tests/test_command.py b/minecode/tests/test_command.py index a21f8d9e..d620ac83 100644 --- a/minecode/tests/test_command.py +++ b/minecode/tests/test_command.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/tests/test_conan.py b/minecode/tests/test_conan.py index c4558707..61faa8e7 100644 --- a/minecode/tests/test_conan.py +++ b/minecode/tests/test_conan.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -26,8 +26,10 @@ class ConanPriorityQueueTests(JsonBasedTesting, TestCase): def setUp(self): super(ConanPriorityQueueTests, self).setUp() self.package_url1 = PackageURL.from_string("pkg:conan/zlib@1.3.1") - zlib_conanfile_loc = self.get_test_loc("conan/zlib/manifest/conanfile.py") - zlib_conandata_loc = self.get_test_loc("conan/zlib/manifest/conandata.yml") + zlib_conanfile_loc = self.get_test_loc( + "conan/zlib/manifest/conanfile.py") + zlib_conandata_loc = self.get_test_loc( + "conan/zlib/manifest/conandata.yml") zlib_config_loc = self.get_test_loc("conan/zlib/manifest/config.yml") with open(zlib_conanfile_loc) as f: @@ -39,7 +41,8 @@ def setUp(self): with open(zlib_conandata_loc) as f: self.zlib_conandata_contents = f.read() - self.zlib_conandata_contents_dict = saneyaml.load(self.zlib_conandata_contents) + self.zlib_conandata_contents_dict = saneyaml.load( + self.zlib_conandata_contents) @patch("requests.get") def test_get_conan_recipe(self, mock_get): diff --git a/minecode/tests/test_cpan.py b/minecode/tests/test_cpan.py index 845a7944..4ebeb281 100644 --- a/minecode/tests/test_cpan.py +++ b/minecode/tests/test_cpan.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -40,12 +40,14 @@ def test_release_search_from_author_visitors(self): with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) _, data, _ = cpan.CpanModulesVisitors(uri) - expected_loc = self.get_test_loc('cpan/expected_release_from_author_ABERNDT.json') + expected_loc = self.get_test_loc( + 'cpan/expected_release_from_author_ABERNDT.json') self.check_expected_results(data, expected_loc, regen=FIXTURES_REGEN) def test_visit_html_modules(self): uri = 'http://www.cpan.org/modules/01modules.index.html' - test_loc = self.get_test_loc('cpan/Modules on CPAN alphabetically.html') + test_loc = self.get_test_loc( + 'cpan/Modules on CPAN alphabetically.html') with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _, _ = cpan.CpanModulesVisitors(uri) @@ -63,13 +65,15 @@ def test_visit_html_files(self): def test_visit_readme_file(self): uri = 'http://www.cpan.org/authors/id/A/AM/AMIRITE/Mojolicious-Plugin-Nour-Config-0.09.readme' - test_loc = self.get_test_loc('cpan/Mojolicious-Plugin-Nour-Config-0.09.readme') + test_loc = self.get_test_loc( + 'cpan/Mojolicious-Plugin-Nour-Config-0.09.readme') with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) _, data, _ = cpan.CpanReadmeVisitors(uri) result = json.loads(data, object_pairs_hook=OrderedDict) expected_file = self.get_test_loc('cpan/expected_readme.json') - self.check_expected_results(result, expected_file, regen=FIXTURES_REGEN) + self.check_expected_results( + result, expected_file, regen=FIXTURES_REGEN) class CpanMapperTest(JsonBasedTesting): @@ -82,7 +86,8 @@ def test_build_from_release_search_json(self): metadata, 'https://fastapi.metacpan.org/release/_search?q=author:ABERNDT&size=5000') packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('cpan/expected_release_search.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_from_release_search_json2(self): with open(self.get_test_loc('cpan/MIYAGAWA_author_release_search.json')) as cpan_metadata: @@ -90,8 +95,10 @@ def test_build_from_release_search_json2(self): packages = mappers.cpan.build_packages_from_release_json( metadata, 'https://fastapi.metacpan.org/release/_search?q=author:MIYAGAWA&size=5000') packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('cpan/expected_release_search_author_MIYAGAWA.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'cpan/expected_release_search_author_MIYAGAWA.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_packages_metafile_from_yaml(self): with open(self.get_test_loc('cpan/variable-2009110702.meta')) as cpan_metadata: @@ -100,7 +107,8 @@ def test_build_packages_metafile_from_yaml(self): metadata, 'http://www.cpan.org/authors/id/A/AB/ABIGAIL/variable-2009110702.metadata', 'pkg:cpan/variable@2009110702') packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('cpan/expected_yaml_cpanmapper.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_packages_metafile_from_json(self): with open(self.get_test_loc('cpan/Regexp-Common-2016010701.meta')) as cpan_metadata: @@ -109,23 +117,28 @@ def test_build_packages_metafile_from_json(self): metadata, 'http://www.cpan.org/authors/id/A/AB/ABIGAIL/Regexp-Common-2016010701.metadata', 'pkg:cpan/Regexp-Common@2016010701') packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('cpan/expected_json_cpanmapper.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_packages_readme_from_json(self): uri = 'http://www.cpan.org/authors/id/A/AM/AMIRITE/Mojolicious-Plugin-Nour-Config-0.09.readme' - test_loc = self.get_test_loc('cpan/Mojolicious-Plugin-Nour-Config-0.09.readme') + test_loc = self.get_test_loc( + 'cpan/Mojolicious-Plugin-Nour-Config-0.09.readme') with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) _, data, _ = cpan.CpanReadmeVisitors(uri) packages = mappers.cpan.build_packages_from_readmefile( data, 'http://www.cpan.org/authors/id/A/AM/AMIRITE/Mojolicious-Plugin-Nour-Config-0.09.readme', 'pkg:cpan/Mojolicious-Plugin-Nour-Config@0.09') packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('cpan/expected_json_readmefile_cpanmapper.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'cpan/expected_json_readmefile_cpanmapper.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_packages_readme_from_json2(self): uri = 'http://www.cpan.org/authors/id/A/AB/ABIGAIL/Algorithm-Graphs-TransitiveClosure-2009110901.readme' - test_loc = self.get_test_loc('cpan/Algorithm-Graphs-TransitiveClosure-2009110901.readme', 'pkg:cpan/Algorithm-Graphs-TransitiveClosure@2009110901') + test_loc = self.get_test_loc('cpan/Algorithm-Graphs-TransitiveClosure-2009110901.readme', + 'pkg:cpan/Algorithm-Graphs-TransitiveClosure@2009110901') with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) _, data, _ = cpan.CpanReadmeVisitors(uri) @@ -133,5 +146,7 @@ def test_build_packages_readme_from_json2(self): data, 'http://www.cpan.org/authors/id/A/AB/ABIGAIL/Algorithm-Graphs-TransitiveClosure-2009110901.readme') packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('cpan/expected_json_readmefile_cpanmapper2.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'cpan/expected_json_readmefile_cpanmapper2.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) diff --git a/minecode/tests/test_cran.py b/minecode/tests/test_cran.py index 9592182d..1f5e3ef8 100644 --- a/minecode/tests/test_cran.py +++ b/minecode/tests/test_cran.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -40,22 +40,28 @@ class CranMapperTest(JsonBasedTesting, DjangoTestCase): test_data_dir = os.path.join(os.path.dirname(__file__), 'testfiles') def test_build_packages_from_directory_listing(self): - ResourceURI.objects.create(uri='https://cloud.r-project.org/web/packages/ANN2/index.html') + ResourceURI.objects.create( + uri='https://cloud.r-project.org/web/packages/ANN2/index.html') with open(self.get_test_loc('cran/CRAN_Package_ANN2.html')) as html_metadata: metadata = html_metadata.read() - packages = mappers.cran.build_packages_from_html(metadata, 'https://cloud.r-project.org/web/packages/ANN2/index.html', 'pkg:cran/ANN2') + packages = mappers.cran.build_packages_from_html( + metadata, 'https://cloud.r-project.org/web/packages/ANN2/index.html', 'pkg:cran/ANN2') packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('cran/mapper_ANN2_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_packages_from_directory_listing2(self): - ResourceURI.objects.create(uri='https://cloud.r-project.org/web/packages/abe/index.html') + ResourceURI.objects.create( + uri='https://cloud.r-project.org/web/packages/abe/index.html') with open(self.get_test_loc('cran/CRAN_Package_abe.html')) as html_metadata: metadata = html_metadata.read() - packages = mappers.cran.build_packages_from_html(metadata, 'https://cloud.r-project.org/web/packages/abe/index.htm', 'pkg:cran/abe') + packages = mappers.cran.build_packages_from_html( + metadata, 'https://cloud.r-project.org/web/packages/abe/index.htm', 'pkg:cran/abe') packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('cran/mapper_abe_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_replace_downloadurl(self): url = "../../../src/contrib/Archive/ANN2" diff --git a/minecode/tests/test_debian.py b/minecode/tests/test_debian.py index 881d7b16..996af85c 100644 --- a/minecode/tests/test_debian.py +++ b/minecode/tests/test_debian.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -113,7 +113,8 @@ def test_debcon_get_paragraph_data_from_file_with_non_existing_path(self): def test_parse_deb822_dsc(self): dsc_file = self.get_test_loc('debian/debutils/3dldf_2.0.3+dfsg-2.dsc') result = debcon.get_paragraph_data_from_file(dsc_file) - expected_loc = self.get_test_loc('debian/debutils/3dldf_2.0.3+dfsg-2.dsc-expected') + expected_loc = self.get_test_loc( + 'debian/debutils/3dldf_2.0.3+dfsg-2.dsc-expected') self.check_expected_deb822(result, expected_loc, regen=FIXTURES_REGEN) ################################################################# @@ -141,7 +142,8 @@ def test_parse_email_3(self): def test_comma_separated(self): tags = 'implemented-in::perl, role::program, use::converting, works-with::pim' result = list(debutils.comma_separated(tags)) - self.assertEqual([u'implemented-in::perl', u'role::program', u'use::converting', u'works-with::pim'], result) + self.assertEqual([u'implemented-in::perl', u'role::program', + u'use::converting', u'works-with::pim'], result) class DebianReleaseTest(BaseDebianTest): @@ -155,7 +157,8 @@ def test_parse_release(self): def test_parse_release_with_md5(self): release_file = self.get_test_loc('debian/release/Release_with_md5') result = list(debian_visitor.parse_release(release_file)) - expected_loc = self.get_test_loc('debian/release/Release_with_md5_expected') + expected_loc = self.get_test_loc( + 'debian/release/Release_with_md5_expected') self.check_expected_deb822(result, expected_loc) @expectedFailure @@ -167,7 +170,8 @@ def test_visit_debian_release(self): _, data, _ = debian_visitor.DebianReleaseVisitor(uri) result = json.loads(data) - release_file = self.get_test_loc('debian/release/visited_Release-expected.json') + release_file = self.get_test_loc( + 'debian/release/visited_Release-expected.json') self.check_expected_deb822(result, release_file) @@ -177,28 +181,35 @@ class DebianCopyrightTest(BaseDebianTest): @expectedFailure def test_parse_copyright_only_basic(self): copyright_file = self.get_test_loc('debian/copyright/basic_copyright') - copyrights = [info for info in debian_visitor.parse_copyright_only(copyright_file)] - self.assertTrue('Copyright 1998 John Doe ' in copyrights) - self.assertTrue('Copyright 1998 Jane Doe ' in copyrights) + copyrights = [ + info for info in debian_visitor.parse_copyright_only(copyright_file)] + self.assertTrue( + 'Copyright 1998 John Doe ' in copyrights) + self.assertTrue( + 'Copyright 1998 Jane Doe ' in copyrights) @expectedFailure def test_parse_copyright_only_with_incorrect_file(self): - copyright_file = self.get_test_loc('debian/copyright/invalid_copyright') + copyright_file = self.get_test_loc( + 'debian/copyright/invalid_copyright') with self.assertRaises(Exception) as context: - [info for info in debian_visitor.parse_copyright_only(copyright_file)] + [info for info in debian_visitor.parse_copyright_only( + copyright_file)] self.assertTrue('no paragraphs in input' in context.exception) @expectedFailure def test_parse_copyright_only_with_incorrect_path(self): copyright_file = 'path_invalid' with self.assertRaises(Exception) as context: - [info for info in debian_visitor.parse_copyright_only(copyright_file)] + [info for info in debian_visitor.parse_copyright_only( + copyright_file)] self.assertTrue('No such file or directory' in context.exception) @expectedFailure def test_parse_copyright_allinfo_basic(self): copyright_file = self.get_test_loc('debian/copyright/basic_copyright') - copyright_data = [info for info in debian_visitor.parse_copyright_allinfo(copyright_file)] + copyright_data = [ + info for info in debian_visitor.parse_copyright_allinfo(copyright_file)] expected = [ {'files': (u'*',), 'license': u'GPL-2+', @@ -213,16 +224,19 @@ def test_parse_copyright_allinfo_basic(self): @expectedFailure def test_parse_copyright_allinfo_with_invalid_file(self): - copyright_file = self.get_test_loc('debian/copyright/invalid_copyright') + copyright_file = self.get_test_loc( + 'debian/copyright/invalid_copyright') with self.assertRaises(Exception) as context: - [info for info in debian_visitor.parse_copyright_allinfo(copyright_file)] + [info for info in debian_visitor.parse_copyright_allinfo( + copyright_file)] self.assertTrue('no paragraphs in input' in context.exception) @expectedFailure def test_parse_copyright_allinfo_with_incorrect_path(self): copyright_file = 'path_invalid' with self.assertRaises(Exception) as context: - [info for info in debian_visitor.parse_copyright_allinfo(copyright_file)] + [info for info in debian_visitor.parse_copyright_allinfo( + copyright_file)] self.assertTrue('No such file or directory' in context.exception) @expectedFailure @@ -253,7 +267,8 @@ def test_parse_license_basic(self): @expectedFailure def test_parse_license_with_invalid_file(self): - copyright_file = self.get_test_loc('debian/copyright/invalid_copyright') + copyright_file = self.get_test_loc( + 'debian/copyright/invalid_copyright') with self.assertRaises(Exception) as context: debian_visitor.parse_license(copyright_file) self.assertTrue('no paragraphs in input' in context.exception) @@ -270,15 +285,21 @@ class DebianSourcesTest(BaseDebianTest): def test_collect_source_packages(self): index_file = self.get_test_loc('debian/sources/debian_Sources') - source_info = [info for info in debian_visitor.collect_source_packages(index_file)] - expected_loc = self.get_test_loc('debian/sources/debian_Sources_visit_expected') - self.check_objects_expected(source_info, expected_loc, regen=FIXTURES_REGEN) + source_info = [ + info for info in debian_visitor.collect_source_packages(index_file)] + expected_loc = self.get_test_loc( + 'debian/sources/debian_Sources_visit_expected') + self.check_objects_expected( + source_info, expected_loc, regen=FIXTURES_REGEN) def test_collect_source_packages_ubuntu(self): index_file = self.get_test_loc('debian/sources/ubuntu_Sources') - source_info = [info for info in debian_visitor.collect_source_packages(index_file)] - expected_loc = self.get_test_loc('debian/sources/ubuntu_Sources_visit_expected') - self.check_objects_expected(source_info, expected_loc, regen=FIXTURES_REGEN) + source_info = [ + info for info in debian_visitor.collect_source_packages(index_file)] + expected_loc = self.get_test_loc( + 'debian/sources/ubuntu_Sources_visit_expected') + self.check_objects_expected( + source_info, expected_loc, regen=FIXTURES_REGEN) @expectedFailure def test_DebianSourcesVisitor(self): @@ -287,7 +308,8 @@ def test_DebianSourcesVisitor(self): with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _, _ = debian_visitor.DebianSourcesVisitor(uri) - expected_loc = self.get_test_loc('debian/sources/Sources.gz-expected.json') + expected_loc = self.get_test_loc( + 'debian/sources/Sources.gz-expected.json') self.check_expected_uris(list(uris), expected_loc) @expectedFailure @@ -304,7 +326,8 @@ def test_build_source_file_packages(self): with open(self.get_test_loc('debian/sources/debian_Sources')) as packs: packages = debian_mapper.build_source_file_packages(packs.read()) packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('debian/sources/debian_Sources_mapped-expected-packages.json') + expected_loc = self.get_test_loc( + 'debian/sources/debian_Sources_mapped-expected-packages.json') self.check_expected_results(packages, expected_loc) @@ -312,33 +335,42 @@ class DebianPackagesTest(BaseDebianTest): def test_parse_packages_index(self): index_file = self.get_test_loc('debian/packages/debian_Packages') - package_info = [info for info in debian_visitor.parse_packages_index(index_file)] - expected_loc = self.get_test_loc('debian/packages/debian_Packages-visit-expected.json') - self.check_objects_expected(package_info, expected_loc, regen=FIXTURES_REGEN) + package_info = [ + info for info in debian_visitor.parse_packages_index(index_file)] + expected_loc = self.get_test_loc( + 'debian/packages/debian_Packages-visit-expected.json') + self.check_objects_expected( + package_info, expected_loc, regen=FIXTURES_REGEN) @expectedFailure def test_parse_packages_from_debian_Packages(self): with open(self.get_test_loc('debian/packages/debian_Packages')) as packs: packages = debian_mapper.parse_packages(packs.read()) packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('debian/packages/debian_Packages-expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'debian/packages/debian_Packages-expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) @expectedFailure def test_parse_packages_from_ubuntu_Packages(self): with open(self.get_test_loc('debian/packages/ubuntu_Packages')) as packs: packages = debian_mapper.parse_packages(packs.read()) packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('debian/packages/ubuntu_Packages-expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'debian/packages/ubuntu_Packages-expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) @expectedFailure def test_parse_packages_from_installed_status(self): with open(self.get_test_loc('debian/status/simple_status')) as packs: packages = debian_mapper.parse_packages(packs.read()) packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('debian/packages/ubuntu_Packages-expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'debian/packages/ubuntu_Packages-expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) class DebianLSLRTest(BaseDebianTest): @@ -348,9 +380,11 @@ def test_DebianDirectoryIndexVisitor_from_debian(self): test_loc = self.get_test_loc('debian/lslr/ls-lR_debian') temp_gz_location = self.get_tmp_gz_file(test_loc) with patch('requests.get') as mock_http_get: - mock_http_get.return_value = mocked_requests_get(uri, temp_gz_location) + mock_http_get.return_value = mocked_requests_get( + uri, temp_gz_location) uris, _, _ = debian_visitor.DebianDirectoryIndexVisitor(uri) - expected_loc = self.get_test_loc('debian/lslr/ls-lR_debian.gz-expected.json') + expected_loc = self.get_test_loc( + 'debian/lslr/ls-lR_debian.gz-expected.json') self.check_expected_uris(list(uris), expected_loc) def test_DebianDirectoryIndexVisitor_from_ubuntu(self): @@ -358,7 +392,8 @@ def test_DebianDirectoryIndexVisitor_from_ubuntu(self): test_loc = self.get_test_loc('debian/lslr/ls-lR_ubuntu') temp_gz_location = self.get_tmp_gz_file(test_loc) with patch('requests.get') as mock_http_get: - mock_http_get.return_value = mocked_requests_get(uri, temp_gz_location) + mock_http_get.return_value = mocked_requests_get( + uri, temp_gz_location) uris, _, _ = debian_visitor.DebianDirectoryIndexVisitor(uri) expected_loc = self.get_test_loc( 'debian/lslr/ls-lR_ubuntu.gz-expected.json') @@ -384,7 +419,8 @@ def test_parse_description(self): metadata = json.load(debian_description_meta) packages = debian_mapper.parse_description(metadata) packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('debian/dsc/description-expected.json') + expected_loc = self.get_test_loc( + 'debian/dsc/description-expected.json') self.check_expected_results(packages, expected_loc) @@ -406,6 +442,7 @@ def test_get_dependencies(self): self.assertEqual(None, result[1].requirement) def test_get_programming_language(self): - tags = ['role::program', 'implemented-in::perl', 'use::converting', 'works-with::pim'] + tags = ['role::program', 'implemented-in::perl', + 'use::converting', 'works-with::pim'] result = debian_mapper.get_programming_language(tags) self.assertEqual('perl', result) diff --git a/minecode/tests/test_dockerhub.py b/minecode/tests/test_dockerhub.py index 0dbde735..66e48ec0 100644 --- a/minecode/tests/test_dockerhub.py +++ b/minecode/tests/test_dockerhub.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -33,7 +33,8 @@ class DockerHubVistorTest(DockerHubTest): def test_searching_condition(self): combinations = dockerhub.get_search_conditions() expected_file = self.get_test_loc('dockerhub/conditions_expected') - self.check_expected_results(combinations, expected_file, regen=FIXTURES_REGEN) + self.check_expected_results( + combinations, expected_file, regen=FIXTURES_REGEN) def test_seeds(self): seed = dockerhub.DockerHubSeed() @@ -61,7 +62,8 @@ def test_visit_dockerhub_project(self): result = json.loads(data, object_pairs_hook=OrderedDict) expected_file = self.get_test_loc( 'dockerhub/visitor_library_elixir_expected') - self.check_expected_results(result, expected_file, regen=FIXTURES_REGEN) + self.check_expected_results( + result, expected_file, regen=FIXTURES_REGEN) def test_visit_dockerhub_search_api(self): uri = 'https://index.docker.io/v1/search?q=1a&n=100&page=2' @@ -83,4 +85,5 @@ def test_build_packages_fromjson(self): packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc( 'dockerhub/expected_dockerhubmapper.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) diff --git a/minecode/tests/test_eclipse.py b/minecode/tests/test_eclipse.py index 58c4e947..db744de9 100644 --- a/minecode/tests/test_eclipse.py +++ b/minecode/tests/test_eclipse.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -38,7 +38,8 @@ def test_visit_eclipse_projects(self): def test_visit_eclipse_project(self): uri = 'https://projects.eclipse.org/projects/modeling.m2t.acceleo' - test_loc = self.get_test_loc('eclipse/Acceleo_projects.eclipse.org.html') + test_loc = self.get_test_loc( + 'eclipse/Acceleo_projects.eclipse.org.html') with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) _, data, _ = eclipse.EclipseSingleProjectVisitor(uri) @@ -108,12 +109,15 @@ def test_build_packages(self): packages = mappers.eclipse.build_packages_with_json(metadata) packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('eclipse/eclipse_birt_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_eclipse_html_packages(self): with open(self.get_test_loc('eclipse/Acceleo_projects.eclipse.org.html')) as eclipse_metadata: metadata = eclipse_metadata.read() packages = mappers.eclipse.build_packages(metadata) packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('eclipse/Acceleo_projects_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'eclipse/Acceleo_projects_expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) diff --git a/minecode/tests/test_fdroid.py b/minecode/tests/test_fdroid.py index 2405ad97..021b5f53 100644 --- a/minecode/tests/test_fdroid.py +++ b/minecode/tests/test_fdroid.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -33,7 +33,8 @@ def test_FdroidPackageRepoVisitor(self): # this is a non-persistent visitor, lets make sure we dont return any data assert not data - expected_loc = self.get_test_loc('fdroid/index-v2-expected-visit.json',) + expected_loc = self.get_test_loc( + 'fdroid/index-v2-expected-visit.json',) self.check_expected_uris(uris, expected_loc, regen=FIXTURES_REGEN) @@ -52,5 +53,7 @@ def test_build_packages(self): packages.extend(pkgs) packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('fdroid/index-v2-visited-expected-mapped.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'fdroid/index-v2-visited-expected-mapped.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) diff --git a/minecode/tests/test_filter.py b/minecode/tests/test_filter.py index 0c38874a..03cb7697 100644 --- a/minecode/tests/test_filter.py +++ b/minecode/tests/test_filter.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/tests/test_freebsd.py b/minecode/tests/test_freebsd.py index 7632d356..9f3aaa00 100644 --- a/minecode/tests/test_freebsd.py +++ b/minecode/tests/test_freebsd.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -40,7 +40,8 @@ def test_visit_freebsd_subHTML(self): with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _, _ = freebsd.FreeBSDSubHTMLVisitors(uri) - expected_loc = self.get_test_loc('freebsd/FreeBSD-10-i386_release_0_.html_expected') + expected_loc = self.get_test_loc( + 'freebsd/FreeBSD-10-i386_release_0_.html_expected') self.check_expected_uris(uris, expected_loc, regen=FIXTURES_REGEN) def test_visit_freebsd_indexvisitor(self): @@ -63,4 +64,5 @@ def test_map_index_file(self): packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc( 'freebsd/indexfile_expected_mapper.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) diff --git a/minecode/tests/test_freedesktop.py b/minecode/tests/test_freedesktop.py index 896ee593..278c3da3 100644 --- a/minecode/tests/test_freedesktop.py +++ b/minecode/tests/test_freedesktop.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -33,7 +33,8 @@ def test_visit_software_html_page(self): with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _, _ = freedesktop.FreedesktopHTMLVisitor(uri) - expected_loc = self.get_test_loc('freedesktop/freedesktop_software_expected') + expected_loc = self.get_test_loc( + 'freedesktop/freedesktop_software_expected') self.check_expected_uris(uris, expected_loc) @@ -47,8 +48,10 @@ def test_map_software_html_page_hal(self): 'https://www.freedesktop.org/wiki/Software/hal', purl='pkg:freedesktop/hal') packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('freedesktop/hal_project_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'freedesktop/hal_project_expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_map_software_html_page_libinput(self): with open(self.get_test_loc('freedesktop/libinput.html')) as freedesktop_metadata: @@ -58,5 +61,7 @@ def test_map_software_html_page_libinput(self): 'https://www.freedesktop.org/wiki/Software/libinput/', purl='pkg:freedesktop/libinput') packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('freedesktop/libinput_project_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'freedesktop/libinput_project_expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) diff --git a/minecode/tests/test_generic.py b/minecode/tests/test_generic.py index fef28ece..69300c86 100644 --- a/minecode/tests/test_generic.py +++ b/minecode/tests/test_generic.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -32,7 +32,8 @@ def test_process_request(self): package = Package.objects.first() self.assertEqual('test', package.name) self.assertEqual('1.0.0', package.version) - self.assertEqual('http://example.com/test.tar.gz', package.download_url) + self.assertEqual('http://example.com/test.tar.gz', + package.download_url) def test_process_request_no_download_url(self): package_count = Package.objects.all().count() @@ -58,14 +59,16 @@ def test_map_generic_package(self): package = Package.objects.first() self.assertEqual('test', package.name) self.assertEqual('1.0.0', package.version) - self.assertEqual('http://example.com/test.tar.gz', package.download_url) + self.assertEqual('http://example.com/test.tar.gz', + package.download_url) def test_map_fetchcode_supported_package(self): package_count = Package.objects.all().count() self.assertEqual(0, package_count) purl = PackageURL.from_string("pkg:generic/udhcp@0.9.1") - error_msg = generic.map_fetchcode_supported_package(purl, ('test_pipeline')) + error_msg = generic.map_fetchcode_supported_package( + purl, ('test_pipeline')) self.assertEqual('', error_msg) package_count = Package.objects.all().count() @@ -78,7 +81,7 @@ def test_map_fetchcode_supported_package(self): "https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source//udhcp-0.9.1.tar.gz", package.download_url, ) - + def test_process_request_fetchcode_generic(self): package_count = Package.objects.all().count() self.assertEqual(0, package_count) diff --git a/minecode/tests/test_github.py b/minecode/tests/test_github.py index dbf9ebb9..b673a408 100644 --- a/minecode/tests/test_github.py +++ b/minecode/tests/test_github.py @@ -4,7 +4,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -127,7 +127,8 @@ def test_GithubRepoVisitor_without_tag_without_download(self, mock_get_repo): mock_get_repo.return_value = repository uri = 'https://api.github.com/repos/collectiveidea/calendar_builder' _, data, _ = github.GithubSingleRepoVisitor(uri) - expected_loc = self.get_test_loc('github/calendar_builder-expected.json') + expected_loc = self.get_test_loc( + 'github/calendar_builder-expected.json') self.check_expected_results(data, expected_loc, regen=FIXTURES_REGEN) def test_GithubReposVisitor(self): @@ -146,15 +147,21 @@ class GithubMapperTest(JsonBasedTesting): def test_github_repo_mapper1(self): with open(self.get_test_loc('github/calendar_builder.json')) as json_metadata: metadata = json_metadata.read() - packages = mappers.github.build_github_packages(metadata, 'https://api.github.com/repos/collectiveidea/calendar_builder') + packages = mappers.github.build_github_packages( + metadata, 'https://api.github.com/repos/collectiveidea/calendar_builder') packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('github/mapper_calendar_builder_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'github/mapper_calendar_builder_expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_github_repo_mapper2(self): with open(self.get_test_loc('github/mojombo_grit_from_visitor_4mapper_input.json')) as json_metadata: metadata = json_metadata.read() - packages = mappers.github.build_github_packages(metadata, 'https://api.github.com/repos/mojombo/grit') + packages = mappers.github.build_github_packages( + metadata, 'https://api.github.com/repos/mojombo/grit') packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('github/mojombo_grit_result_mapper_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'github/mojombo_grit_result_mapper_expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) diff --git a/minecode/tests/test_gitlab.py b/minecode/tests/test_gitlab.py index 368c7192..5450f0a6 100644 --- a/minecode/tests/test_gitlab.py +++ b/minecode/tests/test_gitlab.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -56,4 +56,5 @@ def test_map_software_html_page_hal(self): packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc( 'gitlab/microservice-express-mongo_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) diff --git a/minecode/tests/test_gnu.py b/minecode/tests/test_gnu.py index ec336f59..db9d7249 100644 --- a/minecode/tests/test_gnu.py +++ b/minecode/tests/test_gnu.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/tests/test_golang.py b/minecode/tests/test_golang.py index 02113875..d63be206 100644 --- a/minecode/tests/test_golang.py +++ b/minecode/tests/test_golang.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -43,7 +43,8 @@ def test_GodocSearchVisitor(self): with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _, _ = GodocSearchVisitor(uri) - expected_loc = self.get_test_loc('golang/godoc_search_expected_uris.json') + expected_loc = self.get_test_loc( + 'golang/godoc_search_expected_uris.json') self.check_expected_uris(uris, expected_loc, regen=FIXTURES_REGEN) def test_GodocSearchVisitor_with_non_github_urls(self): @@ -52,7 +53,8 @@ def test_GodocSearchVisitor_with_non_github_urls(self): with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _, _ = GodocSearchVisitor(uri) - expected_loc = self.get_test_loc('golang/godoc_search_off_github_expected_uris.json') + expected_loc = self.get_test_loc( + 'golang/godoc_search_off_github_expected_uris.json') self.check_expected_uris(uris, expected_loc, regen=FIXTURES_REGEN) def test_parse_package_path(self): @@ -74,7 +76,8 @@ def test_build_golang_package(self): package = build_golang_package(package_data, purl) package = package.to_dict() expected_loc = self.get_test_loc('golang/glog_expected.json') - self.check_expected_results(package, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + package, expected_loc, regen=FIXTURES_REGEN) def test_build_golang_package_bitbucket(self): purl = 'pkg:bitbucket/bitbucket.org/zombiezen/yaml?vcs_repository=https://bitbucket.org/zombiezen/yaml' @@ -83,7 +86,8 @@ def test_build_golang_package_bitbucket(self): package = build_golang_package(package_data, purl) package = package.to_dict() expected_loc = self.get_test_loc('golang/math3_expected.json') - self.check_expected_results(package, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + package, expected_loc, regen=FIXTURES_REGEN) def test_build_golang_package_non_well_known(self): purl = 'pkg:golang/winterdrache.de/bindings/sdl' @@ -92,4 +96,5 @@ def test_build_golang_package_non_well_known(self): package = build_golang_package(package_data, purl) package = package.to_dict() expected_loc = self.get_test_loc('golang/winter_expected.json') - self.check_expected_results(package, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + package, expected_loc, regen=FIXTURES_REGEN) diff --git a/minecode/tests/test_googlecode.py b/minecode/tests/test_googlecode.py index 9eb704e7..ae0f974b 100644 --- a/minecode/tests/test_googlecode.py +++ b/minecode/tests/test_googlecode.py @@ -4,7 +4,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -32,25 +32,30 @@ def test_visit_google_download_zip_visitor(self): with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _, _ = googlecode.GooglecodeArchiveVisitor(uri) - expected_loc = self.get_test_loc('googlecode/expected_google-code-archive.txt.zip.json') + expected_loc = self.get_test_loc( + 'googlecode/expected_google-code-archive.txt.zip.json') self.check_expected_uris(uris, expected_loc) def test_visit_google_projectpages(self): uri = 'https://code.google.com/archive/search?q=domain:code.google.com' - test_loc = self.get_test_loc('googlecode/v2_api/GoogleCodeProjectHosting.htm') + test_loc = self.get_test_loc( + 'googlecode/v2_api/GoogleCodeProjectHosting.htm') with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _, _ = googlecode.GoogleDownloadsPageJsonVisitor(uri) - expected_loc = self.get_test_loc('googlecode/v2_api/expected_googleprojects.json') + expected_loc = self.get_test_loc( + 'googlecode/v2_api/expected_googleprojects.json') self.check_expected_uris(uris, expected_loc) def test_visit_google_projectpage2(self): uri = 'https://code.google.com/archive/search?q=domain:code.google.com&page=2' - test_loc = self.get_test_loc('googlecode/v2_api/GoogleCodeProjectHosting_page2.htm') + test_loc = self.get_test_loc( + 'googlecode/v2_api/GoogleCodeProjectHosting_page2.htm') with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _, _ = googlecode.GoogleDownloadsPageJsonVisitor(uri) - expected_loc = self.get_test_loc('googlecode/v2_api/expected_googleproject_page2.json') + expected_loc = self.get_test_loc( + 'googlecode/v2_api/expected_googleproject_page2.json') self.check_expected_uris(uris, expected_loc) def test_visit_google_download_json(self): @@ -59,7 +64,8 @@ def test_visit_google_download_json(self): with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _, _ = googlecode.GoogleProjectJsonVisitor(uri) - self.assertEqual([URI(uri=u'https://storage.googleapis.com/google-code-archive/v2/code.google.com/hg4j/downloads-page-1.json')], list(uris)) + self.assertEqual( + [URI(uri=u'https://storage.googleapis.com/google-code-archive/v2/code.google.com/hg4j/downloads-page-1.json')], list(uris)) def test_visit_google_json(self): uri = 'https://storage.googleapis.com/google-code-archive/v2/code.google.com/hg4j/downloads-page-1.json' @@ -67,16 +73,19 @@ def test_visit_google_json(self): with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _, _ = googlecode.GoogleDownloadsPageJsonVisitor(uri) - expected_loc = self.get_test_loc('googlecode/v2_api/hg4j_download_expected.json') + expected_loc = self.get_test_loc( + 'googlecode/v2_api/hg4j_download_expected.json') self.check_expected_uris(uris, expected_loc, regen=FIXTURES_REGEN) def test_visit_googleapi_project_json(self): uri = 'https://www.googleapis.com/storage/v1/b/google-code-archive/o/v2%2Fapache-extras.org%2F124799961-qian%2Fproject.json?alt=media' - test_loc = self.get_test_loc('googlecode/v2_apache-extras.org_124799961-qian_project.json') + test_loc = self.get_test_loc( + 'googlecode/v2_apache-extras.org_124799961-qian_project.json') with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) _, data, _ = googlecode.GoogleDownloadsPageJsonVisitor(uri) - expected_loc = self.get_test_loc('googlecode/expected_v2_apache-extras.org_124799961-qian_project2.json') + expected_loc = self.get_test_loc( + 'googlecode/expected_v2_apache-extras.org_124799961-qian_project2.json') self.check_expected_results(data, expected_loc) @@ -86,15 +95,21 @@ class GoogleNewAPIMappersTest(JsonBasedTesting): def test_build_packages_from_v2_projects_json(self): with open(self.get_test_loc('googlecode/v2_api/project.json')) as projectsjson_meta: metadata = json.load(projectsjson_meta) - packages = mappers.googlecode.build_packages_from_projectsjson_v2(metadata) + packages = mappers.googlecode.build_packages_from_projectsjson_v2( + metadata) packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('googlecode/v2_api/package_expected_project.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'googlecode/v2_api/package_expected_project.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_packages_from_v1_projects_json(self): with open(self.get_test_loc('googlecode/v2_apache-extras.org_124799961-qian_project.json')) as projectsjson_meta: metadata = json.load(projectsjson_meta) - packages = mappers.googlecode.build_packages_from_projectsjson_v1(metadata) + packages = mappers.googlecode.build_packages_from_projectsjson_v1( + metadata) packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('googlecode/mapper_expected_v2_apache-extras.org_124799961-qian_project.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'googlecode/mapper_expected_v2_apache-extras.org_124799961-qian_project.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) diff --git a/minecode/tests/test_gstreamer.py b/minecode/tests/test_gstreamer.py index bd8ab006..4c1fcd48 100644 --- a/minecode/tests/test_gstreamer.py +++ b/minecode/tests/test_gstreamer.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -39,7 +39,8 @@ def test_visit_Gstreamer_subpath_contains_file_resources(self): with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _, _ = gstreamer.GstreamerHTMLVisitor(uri) - expected_loc = self.get_test_loc('gstreamer/src_gst-openmax_pre.html-expected') + expected_loc = self.get_test_loc( + 'gstreamer/src_gst-openmax_pre.html-expected') self.check_expected_uris(uris, expected_loc) @@ -47,12 +48,17 @@ class GstreamerMappersTest(JsonBasedTesting): test_data_dir = os.path.join(os.path.dirname(__file__), 'testfiles') def test_build_package_mapper_regex(self): - regex = re.compile(r'^https://gstreamer.freedesktop.org/src/([\w\-\.]+/)*[\w\-\.]+[.tar\.bz2|\.sha1sum|\.md5|\.gz|\.tar\.xz|\.asc]$') - result = re.match(regex, 'https://gstreamer.freedesktop.org/src/gst-openmax/pre/gst-openmax-0.10.0.2.tar.bz2') + regex = re.compile( + r'^https://gstreamer.freedesktop.org/src/([\w\-\.]+/)*[\w\-\.]+[.tar\.bz2|\.sha1sum|\.md5|\.gz|\.tar\.xz|\.asc]$') + result = re.match( + regex, 'https://gstreamer.freedesktop.org/src/gst-openmax/pre/gst-openmax-0.10.0.2.tar.bz2') self.assertTrue(result) def test_build_package_from_url(self): - packages = mappers.gstreamer.build_package_from_url('https://gstreamer.freedesktop.org/src/gst-openmax/pre/gst-openmax-0.10.0.2.tar.bz2') + packages = mappers.gstreamer.build_package_from_url( + 'https://gstreamer.freedesktop.org/src/gst-openmax/pre/gst-openmax-0.10.0.2.tar.bz2') packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('gstreamer/mapper_build_from_url-expected') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'gstreamer/mapper_build_from_url-expected') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) diff --git a/minecode/tests/test_haxe.py b/minecode/tests/test_haxe.py index 116b666e..0c27d4c0 100644 --- a/minecode/tests/test_haxe.py +++ b/minecode/tests/test_haxe.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -39,7 +39,8 @@ def test_visit_haxe_versions(self): with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _, _ = haxe.HaxeVersionsVisitor(uri) - expected_loc = self.get_test_loc('haxe/all_versions_openfl.html-expected') + expected_loc = self.get_test_loc( + 'haxe/all_versions_openfl.html-expected') self.check_expected_uris(uris, expected_loc, regen=FIXTURES_REGEN) def test_visit_haxe_package_json(self): @@ -48,7 +49,8 @@ def test_visit_haxe_package_json(self): with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) _, data, _ = haxe.HaxePackageJsonVisitor(uri) - expected_loc = self.get_test_loc('haxe/openfl-8.5.1-package.json-expected') + expected_loc = self.get_test_loc( + 'haxe/openfl-8.5.1-package.json-expected') self.check_expected_results(data, expected_loc, regen=FIXTURES_REGEN) @@ -61,4 +63,5 @@ def test_build_project_package_json(self): packages = mappers.haxe.build_packages_with_json(metadata) packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('haxe/project_package.json-expected') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) diff --git a/minecode/tests/test_housekeeping.py b/minecode/tests/test_housekeeping.py index 62b87077..6d9ec6d2 100644 --- a/minecode/tests/test_housekeeping.py +++ b/minecode/tests/test_housekeeping.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -41,8 +41,10 @@ def test_find_ambiguous_packages_declared_license(self): type='maven' ) packages = [p.to_dict() for p in find_ambiguous_packages()] - expected_loc = self.get_test_loc('housekeeping/declared_license_search_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'housekeeping/declared_license_search_expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_find_ambiguous_packages_license_expression(self): packagedb.models.Package.objects.create( @@ -53,8 +55,10 @@ def test_find_ambiguous_packages_license_expression(self): ) packages = [p.to_dict() for p in find_ambiguous_packages()] - expected_loc = self.get_test_loc('housekeeping/license_expression_search_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'housekeeping/license_expression_search_expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_find_ambiguous_packages_license_expression_ignore_uppercase(self): packagedb.models.Package.objects.create( @@ -65,9 +69,11 @@ def test_find_ambiguous_packages_license_expression_ignore_uppercase(self): ) packages = [p.to_dict() for p in find_ambiguous_packages()] - expected_loc = self.get_test_loc('housekeeping/ignore_upper_case_search_expected.json') + expected_loc = self.get_test_loc( + 'housekeeping/ignore_upper_case_search_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_run_check_licenses_command(self): packagedb.models.Package.objects.create( @@ -80,8 +86,10 @@ def test_run_check_licenses_command(self): expected_loc = self.get_test_loc('housekeeping/example_expected.json') output = StringIO() - management.call_command('check_licenses', '-o', results_loc, stdout=output) - self.assertTrue('Visited 1 packages\nFound 1 possible packages\nFound packages dumped to:' in output.getvalue()) + management.call_command('check_licenses', '-o', + results_loc, stdout=output) + self.assertTrue( + 'Visited 1 packages\nFound 1 possible packages\nFound packages dumped to:' in output.getvalue()) with open(results_loc) as results: res = json.load(results) @@ -90,12 +98,15 @@ def test_run_check_licenses_command(self): def test_run_check_licenses_command_with_empty_package(self): output = StringIO() results_loc = self.get_temp_file() - management.call_command('check_licenses', '-o', results_loc, stdout=output) - self.assertTrue('Visited 0 packages\nFound 0 possible packages' in output.getvalue()) + management.call_command('check_licenses', '-o', + results_loc, stdout=output) + self.assertTrue( + 'Visited 0 packages\nFound 0 possible packages' in output.getvalue()) def test_visit_and_map_using_pom(self): uri = 'http://repo1.maven.org/maven2/org/bytesoft/bytejta-supports/0.5.0-ALPHA4/bytejta-supports-0.5.0-ALPHA4.pom' - test_loc = self.get_test_loc('housekeeping/bytejta-supports-0.5.0-ALPHA4.pom') + test_loc = self.get_test_loc( + 'housekeeping/bytejta-supports-0.5.0-ALPHA4.pom') resource_uri = ResourceURI.objects.insert(uri=uri) @@ -105,5 +116,7 @@ def test_visit_and_map_using_pom(self): visit_uri(resource_uri) map_uri(resource_uri) packages = [p.to_dict() for p in find_ambiguous_packages()] - expected_loc = self.get_test_loc('housekeeping/bytejta-supports-0.5.0-ALPHA4.pom_search_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'housekeeping/bytejta-supports-0.5.0-ALPHA4.pom_search_expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) diff --git a/minecode/tests/test_indexing.py b/minecode/tests/test_indexing.py index 6988e0c0..d55c420d 100644 --- a/minecode/tests/test_indexing.py +++ b/minecode/tests/test_indexing.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -52,30 +52,37 @@ def test_indexing_index_package_files(self): self.assertEqual(0, ExactFileIndex.objects.count()) self.assertEqual(0, Resource.objects.count()) - scan_data_loc = self.get_test_loc('indexing/scancodeio_wagon-api-20040705.181715.json') + scan_data_loc = self.get_test_loc( + 'indexing/scancodeio_wagon-api-20040705.181715.json') with open(scan_data_loc, 'rb') as f: scan_data = json.loads(f.read()) - indexing_errors = indexing.index_package_files(self.package1, scan_data) + indexing_errors = indexing.index_package_files( + self.package1, scan_data) self.assertEqual(0, len(indexing_errors)) self.assertEqual(11, ApproximateDirectoryContentIndex.objects.count()) - self.assertEqual(11, ApproximateDirectoryStructureIndex.objects.count()) + self.assertEqual( + 11, ApproximateDirectoryStructureIndex.objects.count()) self.assertEqual(2, ApproximateResourceContentIndex.objects.count()) self.assertEqual(45, ExactFileIndex.objects.count()) resources = Resource.objects.filter(package=self.package1) self.assertEqual(64, len(resources)) resource_data = [r.to_dict() for r in resources] - expected_resources_loc = self.get_test_loc('indexing/scancodeio_wagon-api-20040705.181715-expected.json') - self.check_expected_results(resource_data, expected_resources_loc, regen=FIXTURES_REGEN) + expected_resources_loc = self.get_test_loc( + 'indexing/scancodeio_wagon-api-20040705.181715-expected.json') + self.check_expected_results( + resource_data, expected_resources_loc, regen=FIXTURES_REGEN) def test_indexing_index_package(self): - scan_data_loc = self.get_test_loc('indexing/scancodeio_wagon-api-20040705.181715.json') + scan_data_loc = self.get_test_loc( + 'indexing/scancodeio_wagon-api-20040705.181715.json') with open(scan_data_loc, 'rb') as f: scan_data = json.load(f) - scan_summary_loc = self.get_test_loc('indexing/scancodeio_wagon-api-20040705.181715-summary.json') + scan_summary_loc = self.get_test_loc( + 'indexing/scancodeio_wagon-api-20040705.181715-summary.json') with open(scan_summary_loc, 'rb') as f: scan_summary = json.load(f) @@ -118,8 +125,10 @@ def test_indexing_index_package(self): ) # Make sure that Package data is updated - self.assertEqual('apache-2.0', self.package1.declared_license_expression) - self.assertEqual('Copyright (c) Apache Software Foundation', self.package1.copyright) + self.assertEqual( + 'apache-2.0', self.package1.declared_license_expression) + self.assertEqual( + 'Copyright (c) Apache Software Foundation', self.package1.copyright) self.assertEqual('md5', self.package1.md5) self.assertEqual('sha1', self.package1.sha1) self.assertEqual('sha256', self.package1.sha256) @@ -142,7 +151,8 @@ def test_indexing_index_package_dwarf(self): with open(scan_data_loc, 'rb') as f: scan_data = json.load(f) - scan_summary_loc = self.get_test_loc('indexing/scan_summary_dwarf.json') + scan_summary_loc = self.get_test_loc( + 'indexing/scan_summary_dwarf.json') with open(scan_summary_loc, 'rb') as f: scan_summary = json.load(f) diff --git a/minecode/tests/test_ls.py b/minecode/tests/test_ls.py index ed142f9c..f8c2c8cc 100644 --- a/minecode/tests/test_ls.py +++ b/minecode/tests/test_ls.py @@ -5,7 +5,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -34,7 +34,8 @@ def test_remove_inode_works_even_with_space_at_line_start(self): def check_listing(self, test_file, expected_file, from_find=True, regen=FIXTURES_REGEN): test_file = self.get_test_loc(test_file) test_text = open(test_file).read() - results = list(ls.parse_directory_listing(test_text, from_find=from_find)) + results = list(ls.parse_directory_listing( + test_text, from_find=from_find)) for r in results: if r.date: # we remove the year in YYYY-MM-DD to avoid date-sensitive test @@ -48,20 +49,24 @@ def check_listing(self, test_file, expected_file, from_find=True, regen=FIXTURES def test_parse_listing_from_findls(self): test_file = 'directories/find-ls' expected_file = 'directories/find-ls-expected.json' - self.check_listing(test_file, expected_file, from_find=True, regen=FIXTURES_REGEN) + self.check_listing(test_file, expected_file, + from_find=True, regen=FIXTURES_REGEN) def test_parse_listing_from_findls_from_apache_does_not_fail_on_first_line(self): test_file = 'directories/find-ls-apache-start' expected_file = 'directories/find-ls-apache-start-expected.json' - self.check_listing(test_file, expected_file, from_find=True, regen=FIXTURES_REGEN) + self.check_listing(test_file, expected_file, + from_find=True, regen=FIXTURES_REGEN) def test_parse_listing_from_lslr(self): test_file = 'directories/ls-lr' expected_file = 'directories/ls-lr-expected.json' - self.check_listing(test_file, expected_file, from_find=False, regen=FIXTURES_REGEN) + self.check_listing(test_file, expected_file, + from_find=False, regen=FIXTURES_REGEN) def test_parse_listing_from_lslr_at_ubuntu(self): test_file = 'directories/ls-lr-ubuntu' expected_file = 'directories/ls-lr-ubuntu-expected.json' self.maxDiff = None - self.check_listing(test_file, expected_file, from_find=False, regen=FIXTURES_REGEN) + self.check_listing(test_file, expected_file, + from_find=False, regen=FIXTURES_REGEN) diff --git a/minecode/tests/test_maven.py b/minecode/tests/test_maven.py index 32325adb..8518b4bd 100644 --- a/minecode/tests/test_maven.py +++ b/minecode/tests/test_maven.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -50,31 +50,40 @@ class MavenMiscTest(JsonBasedTesting, DjangoTestCase): test_data_dir = os.path.join(os.path.dirname(__file__), 'testfiles') def test_get_entries(self): - index = self.get_test_loc('maven/index/nexus-maven-repository-index.gz') - fields = list(maven_visitor.ENTRY_FIELDS.keys()) + list(maven_visitor.ENTRY_FIELDS_OTHER.keys()) + list(maven_visitor.ENTRY_FIELDS_IGNORED.keys()) + index = self.get_test_loc( + 'maven/index/nexus-maven-repository-index.gz') + fields = list(maven_visitor.ENTRY_FIELDS.keys( + )) + list(maven_visitor.ENTRY_FIELDS_OTHER.keys()) + list(maven_visitor.ENTRY_FIELDS_IGNORED.keys()) fields = set(fields) result = list(maven_visitor.get_entries(index, fields=fields)) expected_loc = self.get_test_loc('maven/index/expected_entries.json') self.check_expected_results(result, expected_loc, regen=FIXTURES_REGEN) def test_get_entries_increment(self): - index = self.get_test_loc('maven/index/increment/nexus-maven-repository-index.445.gz') - fields = list(maven_visitor.ENTRY_FIELDS.keys()) + list(maven_visitor.ENTRY_FIELDS_OTHER.keys()) + list(maven_visitor.ENTRY_FIELDS_IGNORED.keys()) + index = self.get_test_loc( + 'maven/index/increment/nexus-maven-repository-index.445.gz') + fields = list(maven_visitor.ENTRY_FIELDS.keys( + )) + list(maven_visitor.ENTRY_FIELDS_OTHER.keys()) + list(maven_visitor.ENTRY_FIELDS_IGNORED.keys()) fields = set(fields) result = list(maven_visitor.get_entries(index, fields=fields)) - expected_loc = self.get_test_loc('maven/index/increment/expected_entries.json') + expected_loc = self.get_test_loc( + 'maven/index/increment/expected_entries.json') self.check_expected_results(result, expected_loc, regen=FIXTURES_REGEN) def test_get_entries_buggy(self): - index = self.get_test_loc('maven/index/buggy/nexus-maven-repository-index.gz') - fields = list(maven_visitor.ENTRY_FIELDS.keys()) + list(maven_visitor.ENTRY_FIELDS_OTHER.keys()) + list(maven_visitor.ENTRY_FIELDS_IGNORED.keys()) + index = self.get_test_loc( + 'maven/index/buggy/nexus-maven-repository-index.gz') + fields = list(maven_visitor.ENTRY_FIELDS.keys( + )) + list(maven_visitor.ENTRY_FIELDS_OTHER.keys()) + list(maven_visitor.ENTRY_FIELDS_IGNORED.keys()) fields = set(fields) result = list(maven_visitor.get_entries(index, fields=fields)) - expected_loc = self.get_test_loc('maven/index/buggy/expected_entries.json') + expected_loc = self.get_test_loc( + 'maven/index/buggy/expected_entries.json') self.check_expected_results(result, expected_loc, regen=FIXTURES_REGEN) def test_get_artifacts_full(self): - index = self.get_test_loc('maven/index/nexus-maven-repository-index.gz') + index = self.get_test_loc( + 'maven/index/nexus-maven-repository-index.gz') fields = ( list(maven_visitor.ENTRY_FIELDS) + @@ -83,52 +92,70 @@ def test_get_artifacts_full(self): ) fields = set(fields) - result = [a.to_dict() for a in maven_visitor.get_artifacts(index, fields, include_all=True)] + result = [a.to_dict() for a in maven_visitor.get_artifacts( + index, fields, include_all=True)] expected_loc = self.get_test_loc('maven/index/expected_artifacts.json') self.check_expected_results(result, expected_loc, regen=FIXTURES_REGEN) def test_get_artifacts_increment(self): - index = self.get_test_loc('maven/index/increment/nexus-maven-repository-index.445.gz') - fields = list(maven_visitor.ENTRY_FIELDS.keys()) + list(maven_visitor.ENTRY_FIELDS_OTHER.keys()) + list(maven_visitor.ENTRY_FIELDS_IGNORED.keys()) + index = self.get_test_loc( + 'maven/index/increment/nexus-maven-repository-index.445.gz') + fields = list(maven_visitor.ENTRY_FIELDS.keys( + )) + list(maven_visitor.ENTRY_FIELDS_OTHER.keys()) + list(maven_visitor.ENTRY_FIELDS_IGNORED.keys()) fields = set(fields) - result = [a.to_dict() for a in maven_visitor.get_artifacts(index, fields, include_all=True)] - expected_loc = self.get_test_loc('maven/index/increment/expected_artifacts.json') + result = [a.to_dict() for a in maven_visitor.get_artifacts( + index, fields, include_all=True)] + expected_loc = self.get_test_loc( + 'maven/index/increment/expected_artifacts.json') self.check_expected_results(result, expected_loc, regen=FIXTURES_REGEN) def test_get_artifacts_buggy(self): - index = self.get_test_loc('maven/index/buggy/nexus-maven-repository-index.gz') - fields = list(maven_visitor.ENTRY_FIELDS.keys()) + list(maven_visitor.ENTRY_FIELDS_OTHER.keys()) + list(maven_visitor.ENTRY_FIELDS_IGNORED.keys()) + index = self.get_test_loc( + 'maven/index/buggy/nexus-maven-repository-index.gz') + fields = list(maven_visitor.ENTRY_FIELDS.keys( + )) + list(maven_visitor.ENTRY_FIELDS_OTHER.keys()) + list(maven_visitor.ENTRY_FIELDS_IGNORED.keys()) fields = set(fields) - result = [a.to_dict() for a in maven_visitor.get_artifacts(index, fields, include_all=True)] - expected_loc = self.get_test_loc('maven/index/buggy/expected_artifacts.json') + result = [a.to_dict() for a in maven_visitor.get_artifacts( + index, fields, include_all=True)] + expected_loc = self.get_test_loc( + 'maven/index/buggy/expected_artifacts.json') self.check_expected_results(result, expected_loc, regen=FIXTURES_REGEN) def test_get_artifacts_defaults(self): - index = self.get_test_loc('maven/index/nexus-maven-repository-index.gz') + index = self.get_test_loc( + 'maven/index/nexus-maven-repository-index.gz') result = [a.to_dict() for a in maven_visitor.get_artifacts(index)] - expected_loc = self.get_test_loc('maven/index/expected_artifacts-defaults.json') + expected_loc = self.get_test_loc( + 'maven/index/expected_artifacts-defaults.json') self.check_expected_results(result, expected_loc) def test_get_artifacts_no_worthyness(self): - index = self.get_test_loc('maven/index/nexus-maven-repository-index.gz') + index = self.get_test_loc( + 'maven/index/nexus-maven-repository-index.gz') def worth(a): return True - result = [a.to_dict() for a in maven_visitor.get_artifacts(index, worthyness=worth)] - expected_loc = self.get_test_loc('maven/index/expected_artifacts-all-worthy.json') + result = [a.to_dict() + for a in maven_visitor.get_artifacts(index, worthyness=worth)] + expected_loc = self.get_test_loc( + 'maven/index/expected_artifacts-all-worthy.json') self.check_expected_results(result, expected_loc) def test_get_artifacts_defaults_increment(self): - index = self.get_test_loc('maven/index/increment/nexus-maven-repository-index.445.gz') + index = self.get_test_loc( + 'maven/index/increment/nexus-maven-repository-index.445.gz') result = [a.to_dict() for a in maven_visitor.get_artifacts(index)] - expected_loc = self.get_test_loc('maven/index/increment/expected_artifacts-defaults.json') + expected_loc = self.get_test_loc( + 'maven/index/increment/expected_artifacts-defaults.json') self.check_expected_results(result, expected_loc) def test_get_artifacts_defaults_buggy(self): - index = self.get_test_loc('maven/index/buggy/nexus-maven-repository-index.gz') + index = self.get_test_loc( + 'maven/index/buggy/nexus-maven-repository-index.gz') result = [a.to_dict() for a in maven_visitor.get_artifacts(index)] - expected_loc = self.get_test_loc('maven/index/buggy/expected_artifacts-defaults.json') + expected_loc = self.get_test_loc( + 'maven/index/buggy/expected_artifacts-defaults.json') self.check_expected_results(result, expected_loc) def test_build_artifact(self): @@ -165,22 +192,29 @@ def test_build_url_and_filename_1(self): test = {'group_id': 'de.alpharogroup', 'artifact_id': 'address-book-domain', 'version': '3.12.0', 'classifier': None, 'extension': 'jar'} expected = 'https://repo1.maven.org/maven2/de/alpharogroup/address-book-domain/3.12.0/address-book-domain-3.12.0.jar', 'address-book-domain-3.12.0.jar' - self.assertEqual(expected, maven_visitor.build_url_and_filename(**test)) + self.assertEqual( + expected, maven_visitor.build_url_and_filename(**test)) def test_build_url_and_filename_2(self): - test = {'group_id': 'de.alpharogroup', 'artifact_id': 'address-book-data', 'version': '3.12.0', 'classifier': None, 'extension': 'pom'} + test = {'group_id': 'de.alpharogroup', 'artifact_id': 'address-book-data', + 'version': '3.12.0', 'classifier': None, 'extension': 'pom'} expected = 'https://repo1.maven.org/maven2/de/alpharogroup/address-book-data/3.12.0/address-book-data-3.12.0.pom', 'address-book-data-3.12.0.pom' - self.assertEqual(expected, maven_visitor.build_url_and_filename(**test)) + self.assertEqual( + expected, maven_visitor.build_url_and_filename(**test)) def test_build_url_and_filename_3(self): - test = {'group_id': 'de.alpharogroup', 'artifact_id': 'address-book-rest-web', 'version': '3.12.0', 'classifier': None, 'extension': 'war'} + test = {'group_id': 'de.alpharogroup', 'artifact_id': 'address-book-rest-web', + 'version': '3.12.0', 'classifier': None, 'extension': 'war'} expected = 'https://repo1.maven.org/maven2/de/alpharogroup/address-book-rest-web/3.12.0/address-book-rest-web-3.12.0.war', 'address-book-rest-web-3.12.0.war' - self.assertEqual(expected, maven_visitor.build_url_and_filename(**test)) + self.assertEqual( + expected, maven_visitor.build_url_and_filename(**test)) def test_build_url_and_filename_4(self): - test = {'group_id': 'uk.com.robust-it', 'artifact_id': 'cloning', 'version': '1.9.5', 'classifier': 'sources', 'extension': 'jar'} + test = {'group_id': 'uk.com.robust-it', 'artifact_id': 'cloning', + 'version': '1.9.5', 'classifier': 'sources', 'extension': 'jar'} expected = 'https://repo1.maven.org/maven2/uk/com/robust-it/cloning/1.9.5/cloning-1.9.5-sources.jar', 'cloning-1.9.5-sources.jar' - self.assertEqual(expected, maven_visitor.build_url_and_filename(**test)) + self.assertEqual( + expected, maven_visitor.build_url_and_filename(**test)) def test_build_url_and_filename_with_alternate_base(self): test = { @@ -188,10 +222,12 @@ def test_build_url_and_filename_with_alternate_base(self): 'version': '1.9.5', 'classifier': 'sources', 'extension': 'jar', 'base_repo_url': 'maven-index://'} expected = 'maven-index:///uk/com/robust-it/cloning/1.9.5/cloning-1.9.5-sources.jar', 'cloning-1.9.5-sources.jar' - self.assertEqual(expected, maven_visitor.build_url_and_filename(**test)) + self.assertEqual( + expected, maven_visitor.build_url_and_filename(**test)) def test_build_maven_xml_url(self): - test = {'group_id': 'de.alpharogroup', 'artifact_id': 'address-book-domain'} + test = {'group_id': 'de.alpharogroup', + 'artifact_id': 'address-book-domain'} expected = 'https://repo1.maven.org/maven2/de/alpharogroup/address-book-domain/maven-metadata.xml' self.assertEqual(expected, maven_visitor.build_maven_xml_url(**test)) @@ -201,46 +237,59 @@ class MavenVisitorTest(JsonBasedTesting, DjangoTestCase): def test_MavenNexusIndexVisitor_uris(self): uri = 'https://repo1.maven.org/maven2/.index/nexus-maven-repository-index.gz' - test_loc = self.get_test_loc('maven/index/nexus-maven-repository-index.gz') + test_loc = self.get_test_loc( + 'maven/index/nexus-maven-repository-index.gz') with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _data, _errors = maven_visitor.MavenNexusIndexVisitor(uri) expected_loc = self.get_test_loc('maven/index/expected_uris.json') - self.check_expected_uris(uris, expected_loc, data_is_json=True, regen=FIXTURES_REGEN) + self.check_expected_uris( + uris, expected_loc, data_is_json=True, regen=FIXTURES_REGEN) def test_MavenNexusIndexPropertiesVisitor(self): uri = 'https://repo1.maven.org/maven2/.index/nexus-maven-repository-index.properties' - test_loc = self.get_test_loc('maven/index/increment/nexus-maven-repository-index.properties') + test_loc = self.get_test_loc( + 'maven/index/increment/nexus-maven-repository-index.properties') with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) - uris, _data, _errors = maven_visitor.MavenNexusPropertiesVisitor(uri) - expected_loc = self.get_test_loc('maven/index/increment/expected_properties_uris.json') - self.check_expected_uris(uris, expected_loc, data_is_json=True, regen=FIXTURES_REGEN) + uris, _data, _errors = maven_visitor.MavenNexusPropertiesVisitor( + uri) + expected_loc = self.get_test_loc( + 'maven/index/increment/expected_properties_uris.json') + self.check_expected_uris( + uris, expected_loc, data_is_json=True, regen=FIXTURES_REGEN) def test_MavenNexusIndexVisitor_uris_increment(self): uri = 'https://repo1.maven.org/maven2/.index/nexus-maven-repository-index.445.gz' - test_loc = self.get_test_loc('maven/index/increment/nexus-maven-repository-index.445.gz') + test_loc = self.get_test_loc( + 'maven/index/increment/nexus-maven-repository-index.445.gz') with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _data, _errors = maven_visitor.MavenNexusIndexVisitor(uri) - expected_loc = self.get_test_loc('maven/index/increment/expected_uris.json') - self.check_expected_uris(uris, expected_loc, data_is_json=True, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/index/increment/expected_uris.json') + self.check_expected_uris( + uris, expected_loc, data_is_json=True, regen=FIXTURES_REGEN) def test_MavenNexusIndexVisitor_uris_buggy(self): uri = 'https://repo1.maven.org/maven2/.index/nexus-maven-repository-index.gz' - test_loc = self.get_test_loc('maven/index/buggy/nexus-maven-repository-index.gz') + test_loc = self.get_test_loc( + 'maven/index/buggy/nexus-maven-repository-index.gz') with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _data, _errors = maven_visitor.MavenNexusIndexVisitor(uri) - expected_loc = self.get_test_loc('maven/index/buggy/expected_uris.json') - self.check_expected_uris(uris, expected_loc, data_is_json=True, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/index/buggy/expected_uris.json') + self.check_expected_uris( + uris, expected_loc, data_is_json=True, regen=FIXTURES_REGEN) def test_visit_uri_does_not_fail_on_incorrect_sha1(self): uri = 'https://repo1.maven.org/maven2/.index/nexus-maven-repository-index.gz' resource_uri = ResourceURI.objects.insert(uri=uri) before = [p.id for p in ResourceURI.objects.all()] - test_loc = self.get_test_loc('maven/index/buggy/nexus-maven-repository-index.gz') + test_loc = self.get_test_loc( + 'maven/index/buggy/nexus-maven-repository-index.gz') with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) visit_uri(resource_uri) @@ -250,10 +299,13 @@ def test_visit_uri_does_not_fail_on_incorrect_sha1(self): else: visited = ResourceURI.objects.all() - results = [model_to_dict(rec, fields=['uri', 'sha1']) for rec in visited] + results = [model_to_dict(rec, fields=['uri', 'sha1']) + for rec in visited] results = sorted(results, key=itemgetter('uri')) - expected_loc = self.get_test_loc('maven/index/buggy/expected_visited_uris.json') - self.check_expected_results(results, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/index/buggy/expected_visited_uris.json') + self.check_expected_results( + results, expected_loc, regen=FIXTURES_REGEN) visited.delete() def test_MavenPOMVisitor_data(self): @@ -277,7 +329,8 @@ def test_MavenNexusIndexVisitor_with_run_visit_then_map_end2end(self): uri = 'https://repo1.maven.org/maven2/.index/nexus-maven-repository-index.445.gz' resource_uri = ResourceURI.objects.insert(uri=uri) - test_index = self.get_test_loc('maven/index/nexus-maven-repository-index.gz') + test_index = self.get_test_loc( + 'maven/index/nexus-maven-repository-index.gz') with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_index) visit_uri(resource_uri) @@ -289,8 +342,10 @@ def test_MavenNexusIndexVisitor_with_run_visit_then_map_end2end(self): results = list(model_to_dict(rec, exclude=['id']) for rec in visited) results = sorted(results, key=itemgetter('uri')) - expected_loc = self.get_test_loc('maven/end2end/expected_visited_uris.json') - self.check_expected_results(results, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/end2end/expected_visited_uris.json') + self.check_expected_results( + results, expected_loc, regen=FIXTURES_REGEN) pre_visited_uris = ResourceURI.objects.filter( uri__contains='maven-index://').exclude(id__in=before) @@ -299,7 +354,8 @@ def test_MavenNexusIndexVisitor_with_run_visit_then_map_end2end(self): all(ru.last_visit_date and not ru.last_map_date for ru in pre_visited_uris)) - package_ids_before = sorted(p.id for p in packagedb.models.Package.objects.all()) + package_ids_before = sorted( + p.id for p in packagedb.models.Package.objects.all()) # now onto mapping the previsited URIs # setup @@ -313,16 +369,20 @@ def test_MavenNexusIndexVisitor_with_run_visit_then_map_end2end(self): self.assertEqual(19, newly_mapped.count()) package_results = list(pac.to_dict() for pac in newly_mapped) - expected_loc = self.get_test_loc('maven/end2end/expected_mapped_packages.json') - self.check_expected_results(package_results, expected_loc, fields_to_remove=['package_sets'], regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/end2end/expected_mapped_packages.json') + self.check_expected_results(package_results, expected_loc, fields_to_remove=[ + 'package_sets'], regen=FIXTURES_REGEN) # check that the map status has been updated correctly - visited_then_mapped = ResourceURI.objects.filter(uri__contains='maven-index://') + visited_then_mapped = ResourceURI.objects.filter( + uri__contains='maven-index://') self.assertTrue(all(ru.last_map_date for ru in visited_then_mapped)) def test_visit_and_map_using_pom_with_unicode(self): uri = 'https://repo1.maven.org/maven2/edu/psu/swe/commons/commons-jaxrs/1.22/commons-jaxrs-1.22.pom' - test_loc = self.get_test_loc('maven/end2end_unicode/commons-jaxrs-1.22.pom') + test_loc = self.get_test_loc( + 'maven/end2end_unicode/commons-jaxrs-1.22.pom') before_uri = [p.id for p in ResourceURI.objects.all()] before_pkg = [p.id for p in packagedb.models.Package.objects.all()] @@ -340,18 +400,24 @@ def test_visit_and_map_using_pom_with_unicode(self): else: visited = ResourceURI.objects.all() - uri_results = sorted(model_to_dict(rec, exclude=['id']) for rec in visited) - expected_loc = self.get_test_loc('maven/end2end_unicode/expected_visited_commons-jaxrs-1.22.json') - self.check_expected_results(uri_results, expected_loc, regen=FIXTURES_REGEN) + uri_results = sorted(model_to_dict( + rec, exclude=['id']) for rec in visited) + expected_loc = self.get_test_loc( + 'maven/end2end_unicode/expected_visited_commons-jaxrs-1.22.json') + self.check_expected_results( + uri_results, expected_loc, regen=FIXTURES_REGEN) if before_pkg: - mapped = packagedb.models.Package.objects.exclude(id__in=before_pkg) + mapped = packagedb.models.Package.objects.exclude( + id__in=before_pkg) else: mapped = packagedb.models.Package.objects.all() package_results = sorted(pac.to_dict() for pac in mapped) - expected_loc = self.get_test_loc('maven/end2end_unicode/expected_mapped_commons-jaxrs-1.22.json') - self.check_expected_results(package_results, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/end2end_unicode/expected_mapped_commons-jaxrs-1.22.json') + self.check_expected_results( + package_results, expected_loc, regen=FIXTURES_REGEN) def test_visit_and_map_using_pom_with_unicode_multisteps(self): # this test deals with a single POM and the results from @@ -361,41 +427,52 @@ def test_visit_and_map_using_pom_with_unicode_multisteps(self): before_pkg = [p.id for p in packagedb.models.Package.objects.all()] # this is a pre-visited as from the Maven index URI - index_uri_test_loc = self.get_test_loc('maven/end2end_multisteps/commons-jaxrs-1.21-index-data.json') + index_uri_test_loc = self.get_test_loc( + 'maven/end2end_multisteps/commons-jaxrs-1.21-index-data.json') index_uri = json.load(open(index_uri_test_loc, 'rb')) idx_resource_uri = ResourceURI.objects.insert(**index_uri) map_uri(idx_resource_uri) if before_pkg: - mapped = packagedb.models.Package.objects.exclude(id__in=before_pkg) + mapped = packagedb.models.Package.objects.exclude( + id__in=before_pkg) else: mapped = packagedb.models.Package.objects.all() - package_results = sorted((pac.to_dict() for pac in mapped), key=lambda d: list(d.keys())) - expected_loc = self.get_test_loc('maven/end2end_multisteps/expected_mapped_commons-jaxrs-1.21-from-index.json') - self.check_expected_results(package_results, expected_loc, fields_to_remove=['package_sets'], regen=FIXTURES_REGEN) + package_results = sorted((pac.to_dict() + for pac in mapped), key=lambda d: list(d.keys())) + expected_loc = self.get_test_loc( + 'maven/end2end_multisteps/expected_mapped_commons-jaxrs-1.21-from-index.json') + self.check_expected_results(package_results, expected_loc, fields_to_remove=[ + 'package_sets'], regen=FIXTURES_REGEN) # Step 2: map a POM # this is a pre-visited URI as from a POM - pom_uri_test_loc = self.get_test_loc('maven/end2end_multisteps/commons-jaxrs-1.21-pom-data.json') + pom_uri_test_loc = self.get_test_loc( + 'maven/end2end_multisteps/commons-jaxrs-1.21-pom-data.json') pom_uri = json.load(open(pom_uri_test_loc, 'rb')) pom_resource_uri = ResourceURI.objects.insert(**pom_uri) map_uri(pom_resource_uri) if before_pkg: - mapped = packagedb.models.Package.objects.exclude(id__in=before_pkg) + mapped = packagedb.models.Package.objects.exclude( + id__in=before_pkg) else: mapped = packagedb.models.Package.objects.all() - package_results = sorted((pac.to_dict() for pac in mapped), key=lambda d: list(d.keys())) - expected_loc = self.get_test_loc('maven/end2end_multisteps/expected_mapped_commons-jaxrs-1.21-from-pom.json') - self.check_expected_results(package_results, expected_loc, fields_to_remove=['package_sets'], regen=FIXTURES_REGEN) + package_results = sorted((pac.to_dict() + for pac in mapped), key=lambda d: list(d.keys())) + expected_loc = self.get_test_loc( + 'maven/end2end_multisteps/expected_mapped_commons-jaxrs-1.21-from-pom.json') + self.check_expected_results(package_results, expected_loc, fields_to_remove=[ + 'package_sets'], regen=FIXTURES_REGEN) def test_visit_and_map_with_index(self): uri = 'https://repo1.maven.org/maven2/.index/nexus-maven-repository-index.properties' - test_loc = self.get_test_loc('maven/end2end_index/nexus-maven-repository-index.properties') + test_loc = self.get_test_loc( + 'maven/end2end_index/nexus-maven-repository-index.properties') before_uri = [p.id for p in ResourceURI.objects.all()] before_pkg = [p.id for p in packagedb.models.Package.objects.all()] @@ -408,17 +485,22 @@ def test_visit_and_map_with_index(self): visit_uri(resource_uri) if before_uri: - visited = ResourceURI.objects.exclude(id__in=before_uri).order_by('uri') + visited = ResourceURI.objects.exclude( + id__in=before_uri).order_by('uri') else: visited = ResourceURI.objects.all().order_by('uri') - uri_results = list(model_to_dict(rec, exclude=['id']) for rec in visited) - expected_loc = self.get_test_loc('maven/end2end_index/expected_visited_index.json') - self.check_expected_results(uri_results, expected_loc, regen=FIXTURES_REGEN) + uri_results = list(model_to_dict( + rec, exclude=['id']) for rec in visited) + expected_loc = self.get_test_loc( + 'maven/end2end_index/expected_visited_index.json') + self.check_expected_results( + uri_results, expected_loc, regen=FIXTURES_REGEN) uri = 'https://repo1.maven.org/maven2/.index/nexus-maven-repository-index.543.gz' # Use a small index file for test cases - test_loc = self.get_test_loc('maven/end2end_index/nexus-maven-repository-index.163.gz') + test_loc = self.get_test_loc( + 'maven/end2end_index/nexus-maven-repository-index.163.gz') resource_uri = ResourceURI.objects.get(uri=uri) with patch('requests.get') as mock_http_get: @@ -427,13 +509,17 @@ def test_visit_and_map_with_index(self): visit_uri(resource_uri) if before_uri: - visited = ResourceURI.objects.exclude(id__in=before_uri).order_by('uri') + visited = ResourceURI.objects.exclude( + id__in=before_uri).order_by('uri') else: visited = ResourceURI.objects.all().order_by('uri') - uri_results = list(model_to_dict(rec, exclude=['id']) for rec in visited) - expected_loc = self.get_test_loc('maven/end2end_index/expected_visited_increment_index.json') - self.check_expected_results(uri_results, expected_loc, regen=FIXTURES_REGEN) + uri_results = list(model_to_dict( + rec, exclude=['id']) for rec in visited) + expected_loc = self.get_test_loc( + 'maven/end2end_index/expected_visited_increment_index.json') + self.check_expected_results( + uri_results, expected_loc, regen=FIXTURES_REGEN) class MavenXmlMetadataVisitorTest(JsonBasedTesting): @@ -445,7 +531,8 @@ def test_visit_maven_medatata_xml_file(self): with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _, _ = maven_visitor.MavenMetaDataVisitor(uri) - expected_loc = self.get_test_loc('maven/maven-metadata/expected_maven_xml.json') + expected_loc = self.get_test_loc( + 'maven/maven-metadata/expected_maven_xml.json') self.check_expected_uris(uris, expected_loc) @@ -458,7 +545,8 @@ def test_visit_maven_medatata_html_index_jcenter_1(self): with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _, _ = maven_visitor.MavenMetaDataVisitor(uri) - expected_loc = self.get_test_loc('maven/html/visitor_expected_jcenter.bintray.com2.html.json') + expected_loc = self.get_test_loc( + 'maven/html/visitor_expected_jcenter.bintray.com2.html.json') self.check_expected_uris(uris, expected_loc) def test_visit_maven_medatata_html_index_jcenter_2(self): @@ -467,7 +555,8 @@ def test_visit_maven_medatata_html_index_jcenter_2(self): with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _, _ = maven_visitor.MavenMetaDataVisitor(uri) - expected_loc = self.get_test_loc('maven/html/visitor_expected_app.html.json') + expected_loc = self.get_test_loc( + 'maven/html/visitor_expected_app.html.json') self.check_expected_uris(uris, expected_loc) def test_visit_maven_medatata_html_index_jcenter_3(self): @@ -476,7 +565,8 @@ def test_visit_maven_medatata_html_index_jcenter_3(self): with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _, _ = maven_visitor.MavenMetaDataVisitor(uri) - expected_loc = self.get_test_loc('maven/html/visitor_expected_stateframework-compiler.html.json') + expected_loc = self.get_test_loc( + 'maven/html/visitor_expected_stateframework-compiler.html.json') self.check_expected_uris(uris, expected_loc) @@ -491,8 +581,10 @@ def test_visit_and_build_package_from_pom_axis(self): mock_http_get.return_value = mocked_requests_get(uri, test_loc) _, data, _ = maven_visitor.MavenPOMVisitor(uri) package = maven_mapper.get_package(data).to_dict() - expected_loc = self.get_test_loc('maven/mapper/axis-1.4.pom.package.json') - self.check_expected_results(package, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/mapper/axis-1.4.pom.package.json') + self.check_expected_results( + package, expected_loc, regen=FIXTURES_REGEN) def test_visit_and_build_package_from_pom_commons_pool(self): uri = 'https://repo1.maven.org/maven2/commons-pool/commons-pool/1.5.7/commons-pool-1.5.7.pom' @@ -501,8 +593,10 @@ def test_visit_and_build_package_from_pom_commons_pool(self): mock_http_get.return_value = mocked_requests_get(uri, test_loc) _, data, _ = maven_visitor.MavenPOMVisitor(uri) package = maven_mapper.get_package(data).to_dict() - expected_loc = self.get_test_loc('maven/mapper/commons-pool-1.5.7.pom.package.json') - self.check_expected_results(package, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/mapper/commons-pool-1.5.7.pom.package.json') + self.check_expected_results( + package, expected_loc, regen=FIXTURES_REGEN) def test_visit_and_build_package_from_pom_struts(self): uri = 'https://repo1.maven.org/maven2/struts-menu/struts-menu/2.4.2/struts-menu-2.4.2.pom' @@ -511,18 +605,23 @@ def test_visit_and_build_package_from_pom_struts(self): mock_http_get.return_value = mocked_requests_get(uri, test_loc) _, data, _ = maven_visitor.MavenPOMVisitor(uri) package = maven_mapper.get_package(data).to_dict() - expected_loc = self.get_test_loc('maven/mapper/struts-menu-2.4.2.pom.package.json') - self.check_expected_results(package, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/mapper/struts-menu-2.4.2.pom.package.json') + self.check_expected_results( + package, expected_loc, regen=FIXTURES_REGEN) def test_visit_and_build_package_from_pom_mysql(self): uri = 'https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.27/mysql-connector-java-5.1.27.pom' - test_loc = self.get_test_loc('maven/mapper/mysql-connector-java-5.1.27.pom') + test_loc = self.get_test_loc( + 'maven/mapper/mysql-connector-java-5.1.27.pom') with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) _, data, _ = maven_visitor.MavenPOMVisitor(uri) package = maven_mapper.get_package(data).to_dict() - expected_loc = self.get_test_loc('maven/mapper/mysql-connector-java-5.1.27.pom.package.json') - self.check_expected_results(package, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/mapper/mysql-connector-java-5.1.27.pom.package.json') + self.check_expected_results( + package, expected_loc, regen=FIXTURES_REGEN) def test_visit_and_build_package_from_pom_xbean(self): uri = 'https://repo1.maven.org/maven2/xbean/xbean-jmx/2.0/xbean-jmx-2.0.pom' @@ -531,8 +630,10 @@ def test_visit_and_build_package_from_pom_xbean(self): mock_http_get.return_value = mocked_requests_get(uri, test_loc) _, data, _ = maven_visitor.MavenPOMVisitor(uri) package = maven_mapper.get_package(data).to_dict() - expected_loc = self.get_test_loc('maven/mapper/xbean-jmx-2.0.pom.package.json') - self.check_expected_results(package, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/mapper/xbean-jmx-2.0.pom.package.json') + self.check_expected_results( + package, expected_loc, regen=FIXTURES_REGEN) def test_visit_and_build_package_from_pom_maven_all(self): uri = 'https://repo1.maven.org/maven2/date/yetao/maven/maven-all/1.0-RELEASE/maven-all-1.0-RELEASE.pom' @@ -541,8 +642,10 @@ def test_visit_and_build_package_from_pom_maven_all(self): mock_http_get.return_value = mocked_requests_get(uri, test_loc) _, data, _ = maven_visitor.MavenPOMVisitor(uri) package = maven_mapper.get_package(data).to_dict() - expected_loc = self.get_test_loc('maven/mapper/maven-all-1.0-RELEASE.pom.package.json') - self.check_expected_results(package, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/mapper/maven-all-1.0-RELEASE.pom.package.json') + self.check_expected_results( + package, expected_loc, regen=FIXTURES_REGEN) def test_visit_and_build_package_from_pom_with_unicode(self): uri = 'https://repo1.maven.org/maven2/edu/psu/swe/commons/commons-jaxrs/1.21/commons-jaxrs-1.21.pom' @@ -551,8 +654,10 @@ def test_visit_and_build_package_from_pom_with_unicode(self): mock_http_get.return_value = mocked_requests_get(uri, test_loc) _, data, _ = maven_visitor.MavenPOMVisitor(uri) package = maven_mapper.get_package(data).to_dict() - expected_loc = self.get_test_loc('maven/mapper/commons-jaxrs-1.21.pom.package.json') - self.check_expected_results(package, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/mapper/commons-jaxrs-1.21.pom.package.json') + self.check_expected_results( + package, expected_loc, regen=FIXTURES_REGEN) class MavenMapperGetPackageTest(JsonBasedTesting, DjangoTestCase): @@ -562,111 +667,150 @@ def test_get_package_from_pom_1(self): test_loc = self.get_test_loc('maven/parsing/parse/jds-3.0.1.pom') data = open(test_loc).read() package = maven_mapper.get_package(data).to_dict() - expected_loc = self.get_test_loc('maven/parsing/parse/jds-3.0.1.pom.package.json') - self.check_expected_results(package, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/parsing/parse/jds-3.0.1.pom.package.json') + self.check_expected_results( + package, expected_loc, regen=FIXTURES_REGEN) def test_get_package_from_pom_2(self): - test_loc = self.get_test_loc('maven/parsing/parse/springmvc-rest-docs-maven-plugin-1.0-RC1.pom') + test_loc = self.get_test_loc( + 'maven/parsing/parse/springmvc-rest-docs-maven-plugin-1.0-RC1.pom') data = open(test_loc).read() package = maven_mapper.get_package(data).to_dict() - expected_loc = self.get_test_loc('maven/parsing/parse/springmvc-rest-docs-maven-plugin-1.0-RC1.pom.package.json') - self.check_expected_results(package, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/parsing/parse/springmvc-rest-docs-maven-plugin-1.0-RC1.pom.package.json') + self.check_expected_results( + package, expected_loc, regen=FIXTURES_REGEN) def test_get_package_from_pom_3(self): test_loc = self.get_test_loc('maven/parsing/parse/jds-2.17.0718b.pom') data = open(test_loc).read() package = maven_mapper.get_package(data).to_dict() - expected_loc = self.get_test_loc('maven/parsing/parse/jds-2.17.0718b.pom.package.json') - self.check_expected_results(package, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/parsing/parse/jds-2.17.0718b.pom.package.json') + self.check_expected_results( + package, expected_loc, regen=FIXTURES_REGEN) def test_get_package_from_pom_4(self): - test_loc = self.get_test_loc('maven/parsing/parse/maven-javanet-plugin-1.7.pom') + test_loc = self.get_test_loc( + 'maven/parsing/parse/maven-javanet-plugin-1.7.pom') data = open(test_loc).read() package = maven_mapper.get_package(data).to_dict() - expected_loc = self.get_test_loc('maven/parsing/parse/maven-javanet-plugin-1.7.pom.package.json') - self.check_expected_results(package, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/parsing/parse/maven-javanet-plugin-1.7.pom.package.json') + self.check_expected_results( + package, expected_loc, regen=FIXTURES_REGEN) def test_get_package_from_pom_5(self): test_loc = self.get_test_loc('maven/parsing/loop/coreplugin-1.0.0.pom') data = open(test_loc).read() package = maven_mapper.get_package(data).to_dict() - expected_loc = self.get_test_loc('maven/parsing/loop/coreplugin-1.0.0.pom.package.json') - self.check_expected_results(package, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/parsing/loop/coreplugin-1.0.0.pom.package.json') + self.check_expected_results( + package, expected_loc, regen=FIXTURES_REGEN) def test_get_package_from_pom_6(self): - test_loc = self.get_test_loc('maven/parsing/loop/argus-webservices-2.7.0.pom') + test_loc = self.get_test_loc( + 'maven/parsing/loop/argus-webservices-2.7.0.pom') data = open(test_loc).read() package = maven_mapper.get_package(data).to_dict() - expected_loc = self.get_test_loc('maven/parsing/loop/argus-webservices-2.7.0.pom.package.json') - self.check_expected_results(package, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/parsing/loop/argus-webservices-2.7.0.pom.package.json') + self.check_expected_results( + package, expected_loc, regen=FIXTURES_REGEN) def test_get_package_from_pom_7(self): test_loc = self.get_test_loc('maven/parsing/loop/pkg-2.0.13.1005.pom') data = open(test_loc).read() package = maven_mapper.get_package(data).to_dict() - expected_loc = self.get_test_loc('maven/parsing/loop/pkg-2.0.13.1005.pom.package.json') - self.check_expected_results(package, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/parsing/loop/pkg-2.0.13.1005.pom.package.json') + self.check_expected_results( + package, expected_loc, regen=FIXTURES_REGEN) def test_get_package_from_pom_8(self): - test_loc = self.get_test_loc('maven/parsing/loop/ojcms-beans-0.1-beta.pom') + test_loc = self.get_test_loc( + 'maven/parsing/loop/ojcms-beans-0.1-beta.pom') data = open(test_loc).read() package = maven_mapper.get_package(data).to_dict() - expected_loc = self.get_test_loc('maven/parsing/loop/ojcms-beans-0.1-beta.pom.package.json') - self.check_expected_results(package, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/parsing/loop/ojcms-beans-0.1-beta.pom.package.json') + self.check_expected_results( + package, expected_loc, regen=FIXTURES_REGEN) def test_get_package_from_pom_9(self): - test_loc = self.get_test_loc('maven/parsing/loop/jacuzzi-annotations-0.2.1.pom') + test_loc = self.get_test_loc( + 'maven/parsing/loop/jacuzzi-annotations-0.2.1.pom') data = open(test_loc).read() package = maven_mapper.get_package(data).to_dict() - expected_loc = self.get_test_loc('maven/parsing/loop/jacuzzi-annotations-0.2.1.pom.package.json') - self.check_expected_results(package, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/parsing/loop/jacuzzi-annotations-0.2.1.pom.package.json') + self.check_expected_results( + package, expected_loc, regen=FIXTURES_REGEN) def test_get_package_from_pom_10(self): - test_loc = self.get_test_loc('maven/parsing/loop/argus-webservices-2.8.0.pom') + test_loc = self.get_test_loc( + 'maven/parsing/loop/argus-webservices-2.8.0.pom') data = open(test_loc).read() package = maven_mapper.get_package(data).to_dict() - expected_loc = self.get_test_loc('maven/parsing/loop/argus-webservices-2.8.0.pom.package.json') - self.check_expected_results(package, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/parsing/loop/argus-webservices-2.8.0.pom.package.json') + self.check_expected_results( + package, expected_loc, regen=FIXTURES_REGEN) def test_get_package_from_pom_11(self): - test_loc = self.get_test_loc('maven/parsing/loop/jacuzzi-database-0.2.1.pom') + test_loc = self.get_test_loc( + 'maven/parsing/loop/jacuzzi-database-0.2.1.pom') data = open(test_loc).read() package = maven_mapper.get_package(data).to_dict() - expected_loc = self.get_test_loc('maven/parsing/loop/jacuzzi-database-0.2.1.pom.package.json') - self.check_expected_results(package, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/parsing/loop/jacuzzi-database-0.2.1.pom.package.json') + self.check_expected_results( + package, expected_loc, regen=FIXTURES_REGEN) def test_get_package_from_pom_12(self): - test_loc = self.get_test_loc('maven/parsing/empty/common-object-1.0.2.pom') + test_loc = self.get_test_loc( + 'maven/parsing/empty/common-object-1.0.2.pom') data = open(test_loc).read() package = maven_mapper.get_package(data).to_dict() - expected_loc = self.get_test_loc('maven/parsing/empty/common-object-1.0.2.pom.package.json') - self.check_expected_results(package, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/parsing/empty/common-object-1.0.2.pom.package.json') + self.check_expected_results( + package, expected_loc, regen=FIXTURES_REGEN) def test_get_package_from_pom_13(self): test_loc = self.get_test_loc('maven/parsing/empty/osgl-http-1.1.2.pom') data = open(test_loc).read() package = maven_mapper.get_package(data).to_dict() - expected_loc = self.get_test_loc('maven/parsing/empty/osgl-http-1.1.2.pom.package.json') - self.check_expected_results(package, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/parsing/empty/osgl-http-1.1.2.pom.package.json') + self.check_expected_results( + package, expected_loc, regen=FIXTURES_REGEN) def test_regex_maven_pom_mapper_1(self): regex = re.compile(r'^https?://repo1.maven.org/maven2/.*\.pom$') - result = re.match(regex, 'https://repo1.maven.org/maven2/com/google/appengine/appengine-api-1.0-sdk/1.2.0/appengine-api-1.0-sdk-1.2.0.pom') + result = re.match( + regex, 'https://repo1.maven.org/maven2/com/google/appengine/appengine-api-1.0-sdk/1.2.0/appengine-api-1.0-sdk-1.2.0.pom') self.assertTrue(result) def test_MavenNexusIndexVisitor_uris_increment_contain_correct_purl(self): uri = 'https://repo1.maven.org/maven2/.index/nexus-maven-repository-index.457.gz' - test_loc = self.get_test_loc('maven/index/increment2/nexus-maven-repository-index.457.gz') + test_loc = self.get_test_loc( + 'maven/index/increment2/nexus-maven-repository-index.457.gz') with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _data, _errors = maven_visitor.MavenNexusIndexVisitor(uri) uris = [u for i, u in enumerate(uris) if i % 500 == 0] - expected_loc = self.get_test_loc('maven/index/increment2/expected_uris.json') - self.check_expected_uris(uris, expected_loc, data_is_json=True, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/index/increment2/expected_uris.json') + self.check_expected_uris( + uris, expected_loc, data_is_json=True, regen=FIXTURES_REGEN) def test_MavenNexusIndexVisitor_then_get_mini_package_from_index_data(self): uri = 'https://repo1.maven.org/maven2/.index/nexus-maven-repository-index.457.gz' - test_loc = self.get_test_loc('maven/index/increment2/nexus-maven-repository-index.457.gz') + test_loc = self.get_test_loc( + 'maven/index/increment2/nexus-maven-repository-index.457.gz') with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _data, _errors = maven_visitor.MavenNexusIndexVisitor(uri) @@ -674,10 +818,13 @@ def test_MavenNexusIndexVisitor_then_get_mini_package_from_index_data(self): for i, u in enumerate(uris): # only get a few records if i % 500 == 0: - minip = maven_mapper.get_mini_package(u.data, u.uri, u.package_url) + minip = maven_mapper.get_mini_package( + u.data, u.uri, u.package_url) results.append(minip and minip.to_dict() or minip) - expected_loc = self.get_test_loc('maven/index/increment2/expected_mini_package.json') - self.check_expected_results(results, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'maven/index/increment2/expected_mini_package.json') + self.check_expected_results( + results, expected_loc, regen=FIXTURES_REGEN) def test_get_package_from_pom_does_create_a_correct_qualifier(self): 'https://repo1.maven.org/maven2/org/hspconsortium/reference/hspc-reference-auth-server-webapp/1.9.1/hspc-reference-auth-server-webapp-1.9.1.pom' @@ -689,7 +836,8 @@ class MavenPriorityQueueTests(JsonBasedTesting, DjangoTestCase): def setUp(self): super(MavenPriorityQueueTests, self).setUp() - self.expected_pom_loc = self.get_test_loc('maven/pom/classworlds-1.1.pom') + self.expected_pom_loc = self.get_test_loc( + 'maven/pom/classworlds-1.1.pom') with open(self.expected_pom_loc) as f: self.expected_pom_contents = f.read() @@ -727,7 +875,8 @@ def test_map_maven_package(self): package_count = packagedb.models.Package.objects.all().count() self.assertEqual(0, package_count) package_url = PackageURL.from_string(self.scan_package.purl) - maven_visitor.map_maven_package(package_url, packagedb.models.PackageContentType.BINARY, ('test_pipeline')) + maven_visitor.map_maven_package( + package_url, packagedb.models.PackageContentType.BINARY, ('test_pipeline')) package_count = packagedb.models.Package.objects.all().count() self.assertEqual(1, package_count) package = packagedb.models.Package.objects.all().first() @@ -739,14 +888,14 @@ def test_map_maven_package_custom_repo_url(self): self.assertEqual(0, package_count) custom_repo_purl = "pkg:maven/org.eclipse.core/runtime@20070801?repository_url=https://packages.atlassian.com/mvn/maven-atlassian-external/" package_url = PackageURL.from_string(custom_repo_purl) - maven_visitor.map_maven_package(package_url, packagedb.models.PackageContentType.BINARY, ('test_pipeline')) + maven_visitor.map_maven_package( + package_url, packagedb.models.PackageContentType.BINARY, ('test_pipeline')) package_count = packagedb.models.Package.objects.all().count() self.assertEqual(1, package_count) package = packagedb.models.Package.objects.all().first() expected_repo_url = 'https://packages.atlassian.com/mvn/maven-atlassian-external//org/eclipse/core/runtime/20070801/runtime-20070801.jar' self.assertEqual(expected_repo_url, package.download_url) - def test_process_request(self): purl_str = 'pkg:maven/org.apache.twill/twill-core@0.12.0' download_url = 'https://repo1.maven.org/maven2/org/apache/twill/twill-core/0.12.0/twill-core-0.12.0.jar' @@ -809,8 +958,10 @@ def test_merge_parent(self, regen=FIXTURES_REGEN): 'Java', text=pom_text ) - expected_before_loc = self.get_test_loc('maven/pom/ant-antlr-1.10.1-package_before.json') - self.check_expected_results(package.to_dict(), expected_before_loc, regen=regen) + expected_before_loc = self.get_test_loc( + 'maven/pom/ant-antlr-1.10.1-package_before.json') + self.check_expected_results( + package.to_dict(), expected_before_loc, regen=regen) parent_pom_loc = self.get_test_loc('maven/pom/ant-parent-1.10.1.pom') with open(parent_pom_loc) as f: @@ -822,8 +973,10 @@ def test_merge_parent(self, regen=FIXTURES_REGEN): text=parent_pom_text ) package = maven_visitor.merge_parent(package, parent_package) - expected_after_loc = self.get_test_loc('maven/pom/ant-antlr-1.10.1-package_after.json') - self.check_expected_results(package.to_dict(), expected_after_loc, regen=regen) + expected_after_loc = self.get_test_loc( + 'maven/pom/ant-antlr-1.10.1-package_after.json') + self.check_expected_results( + package.to_dict(), expected_after_loc, regen=regen) def test_merge_ancestors(self, regen=FIXTURES_REGEN): pom_loc = self.get_test_loc('maven/pom/pulsar-client-1x-2.5.1.pom') @@ -835,8 +988,10 @@ def test_merge_ancestors(self, regen=FIXTURES_REGEN): 'Java', text=pom_text ) - expected_before_loc = self.get_test_loc('maven/pom/pulsar-client-1x-2.5.1-package_before.json') - self.check_expected_results(package.to_dict(), expected_before_loc, regen=regen) + expected_before_loc = self.get_test_loc( + 'maven/pom/pulsar-client-1x-2.5.1-package_before.json') + self.check_expected_results( + package.to_dict(), expected_before_loc, regen=regen) ancestor_pom_texts = [] for loc in [ @@ -849,8 +1004,10 @@ def test_merge_ancestors(self, regen=FIXTURES_REGEN): ancestor_pom_texts.append(pom_text) maven_visitor.merge_ancestors(ancestor_pom_texts, package) - expected_after_loc = self.get_test_loc('maven/pom/pulsar-client-1x-2.5.1-package_after.json') - self.check_expected_results(package.to_dict(), expected_after_loc, regen=regen) + expected_after_loc = self.get_test_loc( + 'maven/pom/pulsar-client-1x-2.5.1-package_after.json') + self.check_expected_results( + package.to_dict(), expected_after_loc, regen=regen) @mock.patch("minecode.visitors.maven.get_pom_text") def test_get_merged_ancestor_package_from_maven_package(self, get_pom_text_mock, regen=FIXTURES_REGEN): @@ -873,9 +1030,12 @@ def test_get_merged_ancestor_package_from_maven_package(self, get_pom_text_mock, type="maven", download_url="https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-client/2.5.1/pulsar-client-2.5.1.jar", ) - merged_package = maven_visitor.get_merged_ancestor_package_from_maven_package(package=db_package) - expected_loc = self.get_test_loc('maven/pom/pulsar-client-merged-ancestor-package.json') - self.check_expected_results(merged_package.to_dict(), expected_loc, regen=regen) + merged_package = maven_visitor.get_merged_ancestor_package_from_maven_package( + package=db_package) + expected_loc = self.get_test_loc( + 'maven/pom/pulsar-client-merged-ancestor-package.json') + self.check_expected_results( + merged_package.to_dict(), expected_loc, regen=regen) class MavenCrawlerFunctionsTest(JsonBasedTesting, DjangoTestCase): @@ -925,19 +1085,22 @@ def test_check_on_page(self, mock_request_get): checker = maven_visitor.check_if_page_has_pom_files mock_request_get.return_value.ok = True mock_request_get.return_value.text = 'parent-7.11.0.pom' - self.assertTrue(maven_visitor.check_on_page('https://repo1.maven.org/maven2/net/shibboleth/parent/7.11.0/', checker)) + self.assertTrue(maven_visitor.check_on_page( + 'https://repo1.maven.org/maven2/net/shibboleth/parent/7.11.0/', checker)) @mock.patch('requests.get') def test_is_maven_root(self, mock_request_get): mock_request_get.return_value.ok = True mock_request_get.return_value.text = 'archetype-catalog.xml' - self.assertTrue(maven_visitor.is_maven_root('https://repo1.maven.org/maven2/')) + self.assertTrue(maven_visitor.is_maven_root( + 'https://repo1.maven.org/maven2/')) @mock.patch('requests.get') def test_is_package_page(self, mock_request_get): mock_request_get.return_value.ok = True mock_request_get.return_value.text = 'maven-metadata.xml' - self.assertTrue(maven_visitor.is_package_page('https://repo1.maven.org/maven2/xml-apis/xml-apis/')) + self.assertTrue(maven_visitor.is_package_page( + 'https://repo1.maven.org/maven2/xml-apis/xml-apis/')) @mock.patch('requests.get') def test_is_package_version_page(self, mock_request_get): @@ -946,7 +1109,8 @@ def test_is_package_version_page(self, mock_request_get): ../ parent-7.11.0.pom ''' - self.assertTrue(maven_visitor.is_package_version_page('https://repo1.maven.org/maven2/xml-apis/xml-apis/1.0.b2/')) + self.assertTrue(maven_visitor.is_package_version_page( + 'https://repo1.maven.org/maven2/xml-apis/xml-apis/1.0.b2/')) def test_url_parts(self): url = 'https://example.com/foo/bar/baz.jar' @@ -971,7 +1135,8 @@ def test_get_maven_root(self, mock_request_get): mock_request_get.return_value.text = 'archetype-catalog.xml' self.assertEqual( 'https://repo1.maven.org/maven2', - maven_visitor.get_maven_root('https://repo1.maven.org/maven2/net/shibboleth/parent/7.11.0/') + maven_visitor.get_maven_root( + 'https://repo1.maven.org/maven2/net/shibboleth/parent/7.11.0/') ) @mock.patch('requests.get') @@ -992,7 +1157,8 @@ def test_determine_namespace_name_version_from_url(self, mock_request_get): xml-apis-1.0.b2.pom 2005-09-20 05:53 2249 ''' - package_version_page = mock.Mock(ok=True, text=package_version_page_text) + package_version_page = mock.Mock( + ok=True, text=package_version_page_text) mock_request_get.side_effect = [ mock.Mock(ok=True, text=''), mock.Mock(ok=True, text=''), @@ -1001,7 +1167,8 @@ def test_determine_namespace_name_version_from_url(self, mock_request_get): package_version_page ] - namespace, package_name, package_version = maven_visitor.determine_namespace_name_version_from_url(url, root_url) + namespace, package_name, package_version = maven_visitor.determine_namespace_name_version_from_url( + url, root_url) self.assertEqual('xml-apis', namespace) self.assertEqual('xml-apis', package_name) self.assertEqual('1.0.b2', package_version) @@ -1026,7 +1193,8 @@ def test_add_to_import_queue(self, mock_request_get): xml-apis-1.0.b2.pom 2005-09-20 05:53 2249 ''' - package_version_page = mock.Mock(ok=True, text=package_version_page_text) + package_version_page = mock.Mock( + ok=True, text=package_version_page_text) mock_request_get.side_effect = [ package_page, mock.Mock(ok=True, text=''), @@ -1037,10 +1205,11 @@ def test_add_to_import_queue(self, mock_request_get): ] self.assertEqual(0, ImportableURI.objects.all().count()) - maven_visitor.add_to_import_queue(url, root_url ) + maven_visitor.add_to_import_queue(url, root_url) self.assertEqual(1, ImportableURI.objects.all().count()) importable_uri = ImportableURI.objects.get(uri=url) - self.assertEqual('pkg:maven/xml-apis/xml-apis', importable_uri.package_url) + self.assertEqual('pkg:maven/xml-apis/xml-apis', + importable_uri.package_url) def test_filter_only_directories(self): timestamps_by_links = { @@ -1091,7 +1260,8 @@ def test_filter_for_artifacts(self): 'foo.xar': '2023-09-28', 'foo.zip': '2023-09-28', } - self.assertEqual(expected, maven_visitor.filter_for_artifacts(timestamps_by_links)) + self.assertEqual( + expected, maven_visitor.filter_for_artifacts(timestamps_by_links)) def test_collect_links_from_text(self): filter = maven_visitor.filter_only_directories @@ -1127,7 +1297,8 @@ def test_create_absolute_urls_for_links(self): } self.assertEqual( expected, - maven_visitor.create_absolute_urls_for_links(text, url, filter=filter) + maven_visitor.create_absolute_urls_for_links( + text, url, filter=filter) ) @mock.patch('requests.get') @@ -1174,7 +1345,8 @@ def test_get_artifact_sha1(self, mock_request_get): sha1 = '3136ca936f64c9d68529f048c2618bd356bf85c9' mock_request_get.return_value.ok = True mock_request_get.return_value.text = sha1 - self.assertEqual(sha1, maven_visitor.get_artifact_sha1('https://repo1.maven.org/maven2/xml-apis/xml-apis/1.0.b2/xml-apis-1.0.b2.jar.sha1')) + self.assertEqual(sha1, maven_visitor.get_artifact_sha1( + 'https://repo1.maven.org/maven2/xml-apis/xml-apis/1.0.b2/xml-apis-1.0.b2.jar.sha1')) def test_get_classifier_from_artifact_url(self): artifact_url = 'https://repo1.maven.org/maven2/net/alchim31/livereload-jvm/0.2.0/livereload-jvm-0.2.0-onejar.jar' diff --git a/minecode/tests/test_model_utils.py b/minecode/tests/test_model_utils.py index 7aaf74f6..bf64067b 100644 --- a/minecode/tests/test_model_utils.py +++ b/minecode/tests/test_model_utils.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -26,7 +26,8 @@ class ModelUtilsTestCase(MiningTestCase, JsonBasedTesting): def setUp(self): pom_loc = self.get_test_loc('maven/pom/pulsar-2.5.1.pom') - self.scanned_package = _parse('maven_pom', 'maven', 'Java', location=pom_loc) + self.scanned_package = _parse( + 'maven_pom', 'maven', 'Java', location=pom_loc) self.scanned_package.download_url = 'https://repo1.maven.org/maven2/org/apache/pulsar/pulsar/2.5.1/pulsar-2.5.1.jar' def test_merge_or_create_package_create_package(self): @@ -92,12 +93,15 @@ def test_merge_or_create_package_merge_package(self): 'Package field values have been updated.', message, ) - last_modified_date_formatted = package.last_modified_date.strftime("%Y-%m-%d-%H:%M:%S") + last_modified_date_formatted = package.last_modified_date.strftime( + "%Y-%m-%d-%H:%M:%S") self.assertEqual(timestamp, last_modified_date_formatted) data = entry['data'] updated_fields = data['updated_fields'] - expected_updated_fields_loc = self.get_test_loc('model_utils/expected_updated_fields.json') - self.check_expected_results(updated_fields, expected_updated_fields_loc, regen=FIXTURES_REGEN) + expected_updated_fields_loc = self.get_test_loc( + 'model_utils/expected_updated_fields.json') + self.check_expected_results( + updated_fields, expected_updated_fields_loc, regen=FIXTURES_REGEN) class UpdateORCreateResourceTest(TransactionTestCase): diff --git a/minecode/tests/test_models.py b/minecode/tests/test_models.py index f3ab4ef5..7b0864cd 100644 --- a/minecode/tests/test_models.py +++ b/minecode/tests/test_models.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -114,28 +114,33 @@ def test_needs_revisit_force_revisit_at_0_hours(self): self.resource.last_visit_date = timezone.now() self.resource.save() - self.assertTrue(ResourceURI.objects.needs_revisit(uri=self.uri, hours=0)) + self.assertTrue(ResourceURI.objects.needs_revisit( + uri=self.uri, hours=0)) def test_needs_revisit_very_old_visit(self): self.resource.last_visit_date = timezone.now() - timedelta(days=20) self.resource.save() - self.assertTrue(ResourceURI.objects.needs_revisit(uri=self.uri, hours=240)) + self.assertTrue(ResourceURI.objects.needs_revisit( + uri=self.uri, hours=240)) def test_needs_revisit_near_visit(self): self.resource.last_visit_date = timezone.now() - timedelta(hours=3) self.resource.save() - self.assertTrue(ResourceURI.objects.needs_revisit(uri=self.uri, hours=2)) + self.assertTrue(ResourceURI.objects.needs_revisit( + uri=self.uri, hours=2)) def test_needs_revisit_recent_visit(self): self.resource.last_visit_date = timezone.now() self.resource.save() - self.assertFalse(ResourceURI.objects.needs_revisit(uri=self.uri, hours=2)) + self.assertFalse( + ResourceURI.objects.needs_revisit(uri=self.uri, hours=2)) def test_needs_revisit_never_been_visited(self): - self.assertFalse(ResourceURI.objects.needs_revisit(uri=self.uri, hours=200)) + self.assertFalse(ResourceURI.objects.needs_revisit( + uri=self.uri, hours=200)) class ResourceURIManagerGetRevisitablesUnmappableURITestCase(TestCase): @@ -148,18 +153,25 @@ def test_get_revisitables_last_visit_date_now(self): self.resource.last_visit_date = timezone.now() self.resource.save() - self.assertEqual(1, ResourceURI.objects.get_revisitables(hours=0).count()) - self.assertEqual(0, ResourceURI.objects.get_revisitables(hours=1).count()) - self.assertEqual(0, ResourceURI.objects.get_revisitables(hours=240).count()) + self.assertEqual( + 1, ResourceURI.objects.get_revisitables(hours=0).count()) + self.assertEqual( + 0, ResourceURI.objects.get_revisitables(hours=1).count()) + self.assertEqual( + 0, ResourceURI.objects.get_revisitables(hours=240).count()) def test_get_revisitables_last_visit_date_10_days_ago(self): self.resource.last_visit_date = timezone.now() - timedelta(hours=240) self.resource.save() - self.assertEqual(1, ResourceURI.objects.get_revisitables(hours=0).count()) - self.assertEqual(1, ResourceURI.objects.get_revisitables(hours=1).count()) - self.assertEqual(1, ResourceURI.objects.get_revisitables(hours=240).count()) - self.assertEqual(0, ResourceURI.objects.get_revisitables(hours=241).count()) + self.assertEqual( + 1, ResourceURI.objects.get_revisitables(hours=0).count()) + self.assertEqual( + 1, ResourceURI.objects.get_revisitables(hours=1).count()) + self.assertEqual( + 1, ResourceURI.objects.get_revisitables(hours=240).count()) + self.assertEqual( + 0, ResourceURI.objects.get_revisitables(hours=241).count()) class ResourceURIManagerGetRevisitablesMappableURITestCase(TestCase): @@ -173,50 +185,66 @@ def test_get_revisitables_unmapped_last_visit_date_now(self): self.resource.last_visit_date = timezone.now() self.resource.save() - self.assertEqual(0, ResourceURI.objects.get_revisitables(hours=0).count()) - self.assertEqual(0, ResourceURI.objects.get_revisitables(hours=1).count()) - self.assertEqual(0, ResourceURI.objects.get_revisitables(hours=240).count()) + self.assertEqual( + 0, ResourceURI.objects.get_revisitables(hours=0).count()) + self.assertEqual( + 0, ResourceURI.objects.get_revisitables(hours=1).count()) + self.assertEqual( + 0, ResourceURI.objects.get_revisitables(hours=240).count()) def test_get_revisitables_unmapped_last_visit_date_less_than_threshold(self): self.resource.last_visit_date = timezone.now() self.resource.save() - self.assertEqual(0, ResourceURI.objects.get_revisitables(hours=1).count()) + self.assertEqual( + 0, ResourceURI.objects.get_revisitables(hours=1).count()) def test_get_revisitables_unmapped_last_visit_date_10_days_ago(self): self.resource.last_visit_date = timezone.now() - timedelta(hours=240) self.resource.save() - self.assertEqual(0, ResourceURI.objects.get_revisitables(hours=0).count()) - self.assertEqual(0, ResourceURI.objects.get_revisitables(hours=1).count()) - self.assertEqual(0, ResourceURI.objects.get_revisitables(hours=240).count()) - self.assertEqual(0, ResourceURI.objects.get_revisitables(hours=241).count()) + self.assertEqual( + 0, ResourceURI.objects.get_revisitables(hours=0).count()) + self.assertEqual( + 0, ResourceURI.objects.get_revisitables(hours=1).count()) + self.assertEqual( + 0, ResourceURI.objects.get_revisitables(hours=240).count()) + self.assertEqual( + 0, ResourceURI.objects.get_revisitables(hours=241).count()) def test_get_revisitables_mapped_last_visit_date_now(self): self.resource.last_visit_date = timezone.now() self.resource.last_map_date = timezone.now() self.resource.save() - self.assertEqual(1, ResourceURI.objects.get_revisitables(hours=0).count()) - self.assertEqual(0, ResourceURI.objects.get_revisitables(hours=1).count()) - self.assertEqual(0, ResourceURI.objects.get_revisitables(hours=240).count()) + self.assertEqual( + 1, ResourceURI.objects.get_revisitables(hours=0).count()) + self.assertEqual( + 0, ResourceURI.objects.get_revisitables(hours=1).count()) + self.assertEqual( + 0, ResourceURI.objects.get_revisitables(hours=240).count()) def test_get_revisitables_mapped_last_visit_date_less_than_threshold(self): self.resource.last_visit_date = timezone.now() self.resource.last_map_date = timezone.now() self.resource.save() - self.assertEqual(0, ResourceURI.objects.get_revisitables(hours=1).count()) + self.assertEqual( + 0, ResourceURI.objects.get_revisitables(hours=1).count()) def test_get_revisitables_mapped_last_visit_date_10_days_ago(self): self.resource.last_visit_date = timezone.now() - timedelta(hours=240) self.resource.last_map_date = timezone.now() self.resource.save() - self.assertEqual(1, ResourceURI.objects.get_revisitables(hours=0).count()) - self.assertEqual(1, ResourceURI.objects.get_revisitables(hours=1).count()) - self.assertEqual(1, ResourceURI.objects.get_revisitables(hours=240).count()) - self.assertEqual(0, ResourceURI.objects.get_revisitables(hours=241).count()) + self.assertEqual( + 1, ResourceURI.objects.get_revisitables(hours=0).count()) + self.assertEqual( + 1, ResourceURI.objects.get_revisitables(hours=1).count()) + self.assertEqual( + 1, ResourceURI.objects.get_revisitables(hours=240).count()) + self.assertEqual( + 0, ResourceURI.objects.get_revisitables(hours=241).count()) class ResourceURIManagerGetNextVisitableUnmappableURITestCase(TestCase): @@ -228,8 +256,10 @@ def setUp(self): self.resource1 = ResourceURI.objects.insert(uri=self.uri1, priority=2) def test_get_next_visitable_unvisited(self): - self.assertEqual(self.resource1, ResourceURI.objects.get_next_visitable()) - self.assertEqual(self.resource0, ResourceURI.objects.get_next_visitable()) + self.assertEqual( + self.resource1, ResourceURI.objects.get_next_visitable()) + self.assertEqual( + self.resource0, ResourceURI.objects.get_next_visitable()) self.assertIsNone(ResourceURI.objects.get_next_visitable()) def test_get_next_visitable_none_when_both_visited_less_than_10_days_ago(self): @@ -246,8 +276,10 @@ def test_get_next_visitable_when_both_visited_10_days_ago(self): self.resource0.save() self.resource1.save() - self.assertEqual(self.resource1, ResourceURI.objects.get_next_visitable()) - self.assertEqual(self.resource0, ResourceURI.objects.get_next_visitable()) + self.assertEqual( + self.resource1, ResourceURI.objects.get_next_visitable()) + self.assertEqual( + self.resource0, ResourceURI.objects.get_next_visitable()) self.assertIsNone(ResourceURI.objects.get_next_visitable()) def test_get_next_visitable_when_one_unvisited_and_one_visited_less_than_10_days_ago(self): @@ -256,7 +288,8 @@ def test_get_next_visitable_when_one_unvisited_and_one_visited_less_than_10_days self.resource0.save() self.resource1.save() - self.assertEqual(self.resource0, ResourceURI.objects.get_next_visitable()) + self.assertEqual( + self.resource0, ResourceURI.objects.get_next_visitable()) self.assertIsNone(ResourceURI.objects.get_next_visitable()) self.resource0.last_visit_date = timezone.now() - timedelta(hours=24) @@ -264,7 +297,8 @@ def test_get_next_visitable_when_one_unvisited_and_one_visited_less_than_10_days self.resource0.save() self.resource1.save() - self.assertEqual(self.resource1, ResourceURI.objects.get_next_visitable()) + self.assertEqual( + self.resource1, ResourceURI.objects.get_next_visitable()) self.assertIsNone(ResourceURI.objects.get_next_visitable()) def test_get_next_visitable_when_one_visited_more_and_one_visited_less_than_10_days_ago(self): @@ -273,7 +307,8 @@ def test_get_next_visitable_when_one_visited_more_and_one_visited_less_than_10_d self.resource0.save() self.resource1.save() - self.assertEqual(self.resource0, ResourceURI.objects.get_next_visitable()) + self.assertEqual( + self.resource0, ResourceURI.objects.get_next_visitable()) self.assertIsNone(ResourceURI.objects.get_next_visitable()) self.resource0.last_visit_date = timezone.now() - timedelta(hours=24) @@ -281,7 +316,8 @@ def test_get_next_visitable_when_one_visited_more_and_one_visited_less_than_10_d self.resource0.save() self.resource1.save() - self.assertEqual(self.resource1, ResourceURI.objects.get_next_visitable()) + self.assertEqual( + self.resource1, ResourceURI.objects.get_next_visitable()) self.assertIsNone(ResourceURI.objects.get_next_visitable()) @@ -291,12 +327,16 @@ def setUp(self): # this is a mappable ResourceURI self.uri0 = 'http://repo1.maven.org/maven2/org/ye/mav/mav-all/1.0/mav-all-1.0.pom' self.uri1 = 'http://repo1.maven.org/maven2/org/ye/mav/mav-all/1.1/mav-all-1.1.pom' - self.resource0 = ResourceURI.objects.insert(uri=self.uri0, priority=100) - self.resource1 = ResourceURI.objects.insert(uri=self.uri1, priority=100) + self.resource0 = ResourceURI.objects.insert( + uri=self.uri0, priority=100) + self.resource1 = ResourceURI.objects.insert( + uri=self.uri1, priority=100) def test_get_next_visitable_unvisited(self): - self.assertEqual(self.resource1, ResourceURI.objects.get_next_visitable()) - self.assertEqual(self.resource0, ResourceURI.objects.get_next_visitable()) + self.assertEqual( + self.resource1, ResourceURI.objects.get_next_visitable()) + self.assertEqual( + self.resource0, ResourceURI.objects.get_next_visitable()) self.assertIsNone(ResourceURI.objects.get_next_visitable()) def test_get_next_visitable_visited_unmapped(self): @@ -315,8 +355,10 @@ def test_get_next_visitable_visited_10_days_ago_mapped(self): self.resource0.save() self.resource1.save() - self.assertEqual(self.resource1, ResourceURI.objects.get_next_visitable()) - self.assertEqual(self.resource0, ResourceURI.objects.get_next_visitable()) + self.assertEqual( + self.resource1, ResourceURI.objects.get_next_visitable()) + self.assertEqual( + self.resource0, ResourceURI.objects.get_next_visitable()) self.assertIsNone(ResourceURI.objects.get_next_visitable()) def test_get_next_visitable_visited_10_days_ago_one_unmapped(self): @@ -326,7 +368,8 @@ def test_get_next_visitable_visited_10_days_ago_one_unmapped(self): self.resource0.save() self.resource1.save() - self.assertEqual(self.resource0, ResourceURI.objects.get_next_visitable()) + self.assertEqual( + self.resource0, ResourceURI.objects.get_next_visitable()) self.assertIsNone(ResourceURI.objects.get_next_visitable()) self.resource0.last_map_date = None @@ -334,7 +377,8 @@ def test_get_next_visitable_visited_10_days_ago_one_unmapped(self): self.resource0.save() self.resource1.save() - self.assertEqual(self.resource1, ResourceURI.objects.get_next_visitable()) + self.assertEqual( + self.resource1, ResourceURI.objects.get_next_visitable()) self.assertIsNone(ResourceURI.objects.get_next_visitable()) def test_get_next_visitable_recently_visited_mapped(self): @@ -353,8 +397,10 @@ class ResourceURIManagerGetMappablesTestCase(TestCase): def setUp(self): self.uri1 = 'maven-index://repo1.maven.org/o/a/this.jar' self.uri2 = 'maven-index://repo1.maven.org/o/a/thisother.jar' - self.resource1 = ResourceURI.objects.create(uri=self.uri1, priority=1, last_visit_date=timezone.now()) - self.resource2 = ResourceURI.objects.create(uri=self.uri2, priority=2, last_visit_date=timezone.now()) + self.resource1 = ResourceURI.objects.create( + uri=self.uri1, priority=1, last_visit_date=timezone.now()) + self.resource2 = ResourceURI.objects.create( + uri=self.uri2, priority=2, last_visit_date=timezone.now()) def test_get_mappables(self): assert self.resource1.is_mappable @@ -363,7 +409,8 @@ def test_get_mappables(self): self.resource1.last_map_date = timezone.now() self.resource1.save() resource1 = ResourceURI.objects.get(id=self.resource1.id) - self.assertEqual([self.resource2], list(ResourceURI.objects.get_mappables())) + self.assertEqual([self.resource2], list( + ResourceURI.objects.get_mappables())) def test_get_mappables__map_error_must_make_a_resourceuri_non_mappable(self): assert self.resource1.is_mappable @@ -379,19 +426,23 @@ def test_get_mappables__map_error_must_make_a_resourceuri_non_mappable(self): class ScannableURIManagerTestCase(TestCase): def setUp(self): self.test_uri1 = 'http://example.com' - self.test_package1 = Package.objects.create(download_url=self.test_uri1, name='Foo', version='12.34') + self.test_package1 = Package.objects.create( + download_url=self.test_uri1, name='Foo', version='12.34') self.scannable_uri1 = ScannableURI.objects.create(uri=self.test_uri1, package=self.test_package1, scan_status=ScannableURI.SCAN_NEW) self.test_uri2 = 'http://elpmaxe.com' - self.test_package2 = Package.objects.create(download_url=self.test_uri2, name='Bar', version='11.75') + self.test_package2 = Package.objects.create( + download_url=self.test_uri2, name='Bar', version='11.75') self.scannable_uri2 = ScannableURI.objects.create(uri=self.test_uri2, package=self.test_package2, scan_status=ScannableURI.SCAN_SUBMITTED) self.test_uri3 = 'http://nexb.com' - self.test_package3 = Package.objects.create(download_url=self.test_uri3, name='Baz', version='5') + self.test_package3 = Package.objects.create( + download_url=self.test_uri3, name='Baz', version='5') self.scannable_uri3 = ScannableURI.objects.create(uri=self.test_uri3, package=self.test_package3, scan_status=ScannableURI.SCAN_IN_PROGRESS) self.test_uri4 = 'http://realsite.com' - self.test_package4 = Package.objects.create(download_url=self.test_uri4, name='Qux', version='87') + self.test_package4 = Package.objects.create( + download_url=self.test_uri4, name='Qux', version='87') self.scannable_uri4 = ScannableURI.objects.create(uri=self.test_uri4, package=self.test_package4, scan_status=ScannableURI.SCAN_COMPLETED) @@ -423,17 +474,20 @@ def test_ScannableURI_get_next_processable(self): class ScannableURIModelTestCase(TestCase): def setUp(self): self.test_uri = 'http://example.com' - self.test_package = Package.objects.create(download_url=self.test_uri, name='Foo', version='12.34') + self.test_package = Package.objects.create( + download_url=self.test_uri, name='Foo', version='12.34') def test_ScannableURI_create_basic_record(self): - scannable_uri = ScannableURI.objects.create(uri=self.test_uri, package=self.test_package) + scannable_uri = ScannableURI.objects.create( + uri=self.test_uri, package=self.test_package) result = ScannableURI.objects.get(uri=self.test_uri) self.assertEqual(self.test_uri, result.uri) self.assertEqual(self.test_package, result.package) def test_ScannableURI_save(self): test_error_message = 'error' - scannable_uri = ScannableURI.objects.create(uri=self.test_uri, package=self.test_package) + scannable_uri = ScannableURI.objects.create( + uri=self.test_uri, package=self.test_package) self.assertFalse(scannable_uri.scan_error) scannable_uri.scan_error = test_error_message scannable_uri.save() @@ -441,7 +495,8 @@ def test_ScannableURI_save(self): self.assertEqual(test_error_message, result.scan_error) def test_ScannableURI_save_set_canonical_uri(self): - scannable_uri = ScannableURI(uri=self.test_uri, package=self.test_package) + scannable_uri = ScannableURI( + uri=self.test_uri, package=self.test_package) self.assertFalse(scannable_uri.canonical) scannable_uri.save() result = ScannableURI.objects.get(uri=self.test_uri) diff --git a/minecode/tests/test_npm.py b/minecode/tests/test_npm.py index 1887480e..5271308a 100644 --- a/minecode/tests/test_npm.py +++ b/minecode/tests/test_npm.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -72,7 +72,8 @@ def test_build_packages(self): packages = mappers.npm.build_packages(metadata) packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('npm/0flux_npm_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_package2(self): with open(self.get_test_loc('npm/2112.json')) as npm_metadata: @@ -80,7 +81,8 @@ def test_build_package2(self): packages = mappers.npm.build_packages(metadata) packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('npm/npm_2112_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_package3(self): with open(self.get_test_loc('npm/microdata.json')) as npm_metadata: @@ -88,7 +90,8 @@ def test_build_package3(self): packages = mappers.npm.build_packages(metadata) packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('npm/microdata-node_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_package_with_visitor_data(self): uri = 'https://replicate.npmjs.com/registry/_changes?include_docs=true&limit=1000&since=77777' @@ -103,14 +106,16 @@ def test_build_package_with_visitor_data(self): packages = mappers.npm.build_packages(json.loads(metadata)) packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('npm/29_record_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) # Randomly pick a record from 0-1000 metadata = uris_list[554].data packages = mappers.npm.build_packages(json.loads(metadata)) packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('npm/554_record_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_package_with_ticket_439(self): uri = 'https://replicate.npmjs.com/registry/_changes?include_docs=true&limit=10&since=7333426' @@ -126,7 +131,8 @@ def test_build_package_with_ticket_439(self): packages = mappers.npm.build_packages(json.loads(metadata)) packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('npm/expected_ticket_439.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_package_verify_ticket_440(self): uri = 'https://replicate.npmjs.com/registry/_changes?include_docs=true&limit=10&since=7632607' @@ -142,7 +148,8 @@ def test_build_package_verify_ticket_440(self): packages = mappers.npm.build_packages(json.loads(metadata)) packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('npm/expected_ticket_440.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_npm_mapper(self): test_uri = 'https://registry.npmjs.org/angular-compare-validator' @@ -156,7 +163,8 @@ def test_npm_mapper(self): packages = mappers.npm.NpmPackageMapper(test_uri, test_res_uri) packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('npm/mapper/index.expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_package_for_jsonp_filter(self): with open(self.get_test_loc('npm/jsonp-filter.json')) as npm_metadata: @@ -164,19 +172,23 @@ def test_build_package_for_jsonp_filter(self): packages = mappers.npm.build_packages(metadata) packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('npm/jsonp-filter-expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_regex_npm_mapper(self): regex = re.compile(r'^https://registry.npmjs.org/[^\/]+$') - result = re.match(regex, 'https://registry.npmjs.org/react-mobile-navigation-modal') + result = re.match( + regex, 'https://registry.npmjs.org/react-mobile-navigation-modal') self.assertTrue(result) + class NpmPriorityQueueTests(JsonBasedTesting, DjangoTestCase): test_data_dir = os.path.join(os.path.dirname(__file__), 'testfiles') def setUp(self): super(NpmPriorityQueueTests, self).setUp() - self.expected_json_loc = self.get_test_loc('npm/lodash_package-expected.json') + self.expected_json_loc = self.get_test_loc( + 'npm/lodash_package-expected.json') with open(self.expected_json_loc) as f: self.expected_json_contents = json.load(f) diff --git a/minecode/tests/test_nuget.py b/minecode/tests/test_nuget.py index 4977e50c..875a086f 100644 --- a/minecode/tests/test_nuget.py +++ b/minecode/tests/test_nuget.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -50,7 +50,8 @@ def test_NugetAPIJsonVisitor(self): with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _data, _errors = nuget.NugetAPIJsonVisitor(uri) - expected_loc = self.get_test_loc('nuget/nuget_downlloadvisitor_json_expected') + expected_loc = self.get_test_loc( + 'nuget/nuget_downlloadvisitor_json_expected') self.check_expected_uris(uris, expected_loc, regen=FIXTURES_REGEN) def test_NugetHTMLPageVisitor(self): @@ -82,16 +83,19 @@ def test_build_packages(self): packages = mappers.nuget.build_packages_with_json(metadata) packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('nuget/nuget_mapper_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_regex_1(self): regex = re.compile(r'^https://api.nuget.org/packages/.*\.nupkg$') - result = re.match(regex, 'https://api.nuget.org/packages/entityframework.4.3.1.nupkg') + result = re.match( + regex, 'https://api.nuget.org/packages/entityframework.4.3.1.nupkg') self.assertTrue(result) def test_regex_2(self): regex = re.compile(r'^https://api.nuget.org/v3/catalog.+\.json$') - result = re.match(regex, 'https://api.nuget.org/v3/catalog0/data/2015.02.07.22.31.06/entityframework.4.3.1.json') + result = re.match( + regex, 'https://api.nuget.org/v3/catalog0/data/2015.02.07.22.31.06/entityframework.4.3.1.json') self.assertTrue(result) def test_build_packages_from_html(self): @@ -102,5 +106,7 @@ def test_build_packages_from_html(self): _, data, _errors = nuget.NugetHTMLPackageVisitor(uri) packages = mappers.nuget.build_packages_from_html(data, uri,) packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('nuget/nuget_mapper_log4net_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'nuget/nuget_mapper_log4net_expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) diff --git a/minecode/tests/test_openssl.py b/minecode/tests/test_openssl.py index e088b3d1..7230557e 100644 --- a/minecode/tests/test_openssl.py +++ b/minecode/tests/test_openssl.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -64,4 +64,5 @@ def test_OpenSSL_mapper(self): packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc( 'openssl/openssl_mapper_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) diff --git a/minecode/tests/test_openwrt.py b/minecode/tests/test_openwrt.py index 23737714..06d957cc 100644 --- a/minecode/tests/test_openwrt.py +++ b/minecode/tests/test_openwrt.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -26,7 +26,8 @@ class OpenWRTVistorTest(JsonBasedTesting): def test_visit_openwrt_download_pages(self): uri = 'https://downloads.openwrt.org/chaos_calmer/15.05/' - test_loc = self.get_test_loc('openwrt/Index_of_chaos_calmer_15.05_.html') + test_loc = self.get_test_loc( + 'openwrt/Index_of_chaos_calmer_15.05_.html') with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _, _ = openwrt.OpenWrtDownloadPagesVisitor(uri) @@ -35,11 +36,13 @@ def test_visit_openwrt_download_pages(self): def test_visitor_openwrt_download_pages2(self): uri = 'https://downloads.openwrt.org/chaos_calmer/15.05/adm5120/rb1xx/packages/base/' - test_loc = self.get_test_loc('openwrt/Index_of_chaos_calmer_15.05_adm5120_rb1xx_packages_base_.html') + test_loc = self.get_test_loc( + 'openwrt/Index_of_chaos_calmer_15.05_adm5120_rb1xx_packages_base_.html') with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _, _ = openwrt.OpenWrtDownloadPagesVisitor(uri) - expected_loc = self.get_test_loc('openwrt/chaos_calmer_15.05_expected_2') + expected_loc = self.get_test_loc( + 'openwrt/chaos_calmer_15.05_expected_2') self.check_expected_uris(uris, expected_loc) @expectedFailure @@ -74,7 +77,8 @@ def test_visitor_openwrt_ipk2(self): _, data, _ = openwrt.OpenWrtPackageIndexVisitor(uri) result = json.loads(data) - json_file = self.get_test_loc('openwrt/wpa-cli_0.5.7-1_mipsel.ipk_expected') + json_file = self.get_test_loc( + 'openwrt/wpa-cli_0.5.7-1_mipsel.ipk_expected') self.check_expected_results(result, json_file) @@ -87,7 +91,8 @@ def test_build_packages_1(self): metadata = json.load(openwrt_ipk_meta) packages = mappers.openwrt.build_packages(metadata) packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('openwrt/6to4_12-2_all_ipk_expected_mapper.json') + expected_loc = self.get_test_loc( + 'openwrt/6to4_12-2_all_ipk_expected_mapper.json') self.check_expected_results(packages, expected_loc) @expectedFailure @@ -96,5 +101,6 @@ def test_build_packages_2(self): metadata = json.load(openwrt_ipk_meta) packages = mappers.openwrt.build_packages(metadata) packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('openwrt/wpa-cli_0.5.7-1_mipsel.ipk_expected_mapper.json') + expected_loc = self.get_test_loc( + 'openwrt/wpa-cli_0.5.7-1_mipsel.ipk_expected_mapper.json') self.check_expected_results(packages, expected_loc) diff --git a/minecode/tests/test_packagist.py b/minecode/tests/test_packagist.py index 138fa4ce..4303e171 100644 --- a/minecode/tests/test_packagist.py +++ b/minecode/tests/test_packagist.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -42,5 +42,7 @@ def test_build_packages(self): metadata = json.load(packagist_package) packages = mappers.packagist.build_packages_with_json(metadata) packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('packagist/packaglist_00f100_cakephp-opauth_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'packagist/packaglist_00f100_cakephp-opauth_expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) diff --git a/minecode/tests/test_priority_queue.py b/minecode/tests/test_priority_queue.py index a81260b4..2c94f02d 100644 --- a/minecode/tests/test_priority_queue.py +++ b/minecode/tests/test_priority_queue.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -14,6 +14,7 @@ from minecode.management.commands import priority_queue from packagedb.models import Package + class PriorityQueueTests(JsonBasedTesting, DjangoTestCase): def test_process_request(self): package_count = Package.objects.all().count() diff --git a/minecode/tests/test_pypi.py b/minecode/tests/test_pypi.py index e007130e..0ed802b4 100644 --- a/minecode/tests/test_pypi.py +++ b/minecode/tests/test_pypi.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -124,10 +124,12 @@ def test_PypiPackageReleaseVisitor_boolean(self): mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, data, _error = visitors.pypi.PypiPackageReleaseVisitor(uri) - expected_loc = self.get_test_loc('pypi/expected_uris-boolean.py-2.0.dev3.json') + expected_loc = self.get_test_loc( + 'pypi/expected_uris-boolean.py-2.0.dev3.json') self.check_expected_uris(uris, expected_loc) - expected_loc = self.get_test_loc('pypi/expected_data-boolean.py-2.0.dev3.json') + expected_loc = self.get_test_loc( + 'pypi/expected_data-boolean.py-2.0.dev3.json') self.check_expected_results(data, expected_loc) @@ -149,15 +151,18 @@ def test_build_packages_lxml(self): packages = mappers.pypi.build_packages(metadata) packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('pypi/expected-lxml-3.2.0.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_packages_boolean(self): with open(self.get_test_loc('pypi/boolean.py-2.0.dev3.json')) as pypi_meta: metadata = json.load(pypi_meta) packages = mappers.pypi.build_packages(metadata) packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('pypi/expected-boolean.py-2.0.dev3.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'pypi/expected-boolean.py-2.0.dev3.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_packages_cage13(self): with open(self.get_test_loc('pypi/cage_1.1.3.json')) as pypi_meta: @@ -165,7 +170,8 @@ def test_build_packages_cage13(self): packages = mappers.pypi.build_packages(metadata) packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('pypi/expected-CAGE-1.1.3.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_packages_cage12(self): with open(self.get_test_loc('pypi/cage_1.1.2.json')) as pypi_meta: @@ -173,7 +179,8 @@ def test_build_packages_cage12(self): packages = mappers.pypi.build_packages(metadata) packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('pypi/expected-CAGE-1.1.2.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_PypiPackageMapper_cage(self): data = open(self.get_test_loc('pypi/cage_1.1.2.json')).read() @@ -182,7 +189,8 @@ def test_PypiPackageMapper_cage(self): packages = mappers.pypi.PypiPackageMapper(uri, resuri) packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('pypi/expected-CAGE-1.1.2.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_PypiPackageMapper_lxml(self): data = open(self.get_test_loc('pypi/lxml-3.2.0.json')).read() @@ -191,7 +199,8 @@ def test_PypiPackageMapper_lxml(self): packages = mappers.pypi.PypiPackageMapper(uri, resuri) packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('pypi/expected-lxml-3.2.0.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_pypi_map(self): # setup: add a mappable URI @@ -203,17 +212,21 @@ def test_pypi_map(self): packages = mappers.pypi.PypiPackageMapper(resuri.uri, resuri) packages = [p.to_dict() for p in packages] expected_loc = self.get_test_loc('pypi/map/expected-3to2-1.1.1.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) # build a mock router router = Router() - router.append('https://pypi.python.org/pypi/3to2/1.1.1/json', mappers.pypi.PypiPackageMapper) + router.append('https://pypi.python.org/pypi/3to2/1.1.1/json', + mappers.pypi.PypiPackageMapper) # sanity check expected_mapped_package_uri = 'https://pypi.python.org/packages/8f/ab/58a363eca982c40e9ee5a7ca439e8ffc5243dde2ae660ba1ffdd4868026b/3to2-1.1.1.zip' - self.assertEqual(0, Package.objects.filter(download_url=expected_mapped_package_uri).count()) + self.assertEqual(0, Package.objects.filter( + download_url=expected_mapped_package_uri).count()) # test proper map_uri(resuri, _map_router=router) - mapped = Package.objects.filter(download_url=expected_mapped_package_uri) + mapped = Package.objects.filter( + download_url=expected_mapped_package_uri) self.assertEqual(1, mapped.count()) diff --git a/minecode/tests/test_repodata.py b/minecode/tests/test_repodata.py index 9a1ace27..e91aa009 100644 --- a/minecode/tests/test_repodata.py +++ b/minecode/tests/test_repodata.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -59,9 +59,9 @@ def test_get_pkg_infos(self): u'url': 'https://github.com/openstack/python-ceilometerclient', u'checksum': '36547e200627ea25c4e3fb6f9735d58e682f8e35cd815dceed796c83628e60d5', u'directories': [], - u'summary':'Python API and CLI for OpenStack Ceilometer', - u'start_header_range':'880', - u'required_rpms':[ + u'summary': 'Python API and CLI for OpenStack Ceilometer', + u'start_header_range': '880', + u'required_rpms': [ { u'name': 'python-d2to1' }, diff --git a/minecode/tests/test_repodata_rpms.py b/minecode/tests/test_repodata_rpms.py index f798cb00..8ea5f8d8 100644 --- a/minecode/tests/test_repodata_rpms.py +++ b/minecode/tests/test_repodata_rpms.py @@ -5,7 +5,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -23,8 +23,10 @@ class RepodataRPMVisitorsTest(MiningTestCase): BASE_DIR = os.path.join(os.path.dirname(__file__), 'testfiles') def test_collect_rsync_urls(self): - directory_listing_loc = self.get_test_loc('repodata_rpms/centos_dir_listing') + directory_listing_loc = self.get_test_loc( + 'repodata_rpms/centos_dir_listing') base_url = 'http://mirrors.kernel.org/centos/' - uris = repodata_rpms.collect_rsync_urls(directory_listing_loc, base_url, file_names=('repomd.xml',)) + uris = repodata_rpms.collect_rsync_urls( + directory_listing_loc, base_url, file_names=('repomd.xml',)) uris = list(uris) self.assertEqual(1, len(uris)) diff --git a/minecode/tests/test_repomd_parser.py b/minecode/tests/test_repomd_parser.py index 03715519..0e42d574 100644 --- a/minecode/tests/test_repomd_parser.py +++ b/minecode/tests/test_repomd_parser.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -36,49 +36,63 @@ def test_combine_list_of_dicts(self): self.assertEqual(expected, output) def test_generate_rpm_objects(self): - packages = [{'name': 'python-ceilometerclient', 'arch': 'src', 'ver': '1.5.0', 'rel': '1.el7', 'href': '/python-ceilometerclient-1.5.0-1.el7.src.rpm'}] + packages = [{'name': 'python-ceilometerclient', 'arch': 'src', 'ver': '1.5.0', + 'rel': '1.el7', 'href': '/python-ceilometerclient-1.5.0-1.el7.src.rpm'}] repomdxml_url = 'http://vault.centos.org/7.1.1503/cloud/Source/openstack-liberty' rpms = list(generate_rpm_objects(packages, repomdxml_url)) self.assertEqual(1, len(rpms)) rpm = rpms[0] self.assertEqual('python-ceilometerclient', rpm.name) - self.assertEqual(EVR(version='1.5.0', release='1.el7').to_string(), rpm.version) + self.assertEqual( + EVR(version='1.5.0', release='1.el7').to_string(), rpm.version) def test_collect_rpm_packages_from_repomd_cloudera(self): uri2loc = { 'http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.3.2/repodata/repomd.xml': - self.get_test_loc('repodata_rpms/repomd_parser/cloudera/repomd.xml'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/cloudera/repomd.xml'), 'http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.3.2/repodata/filelists.xml.gz': - self.get_test_loc('repodata_rpms/repomd_parser/cloudera/filelists.xml.gz'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/cloudera/filelists.xml.gz'), 'http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.3.2/repodata/other.xml.gz': - self.get_test_loc('repodata_rpms/repomd_parser/cloudera/other.xml.gz'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/cloudera/other.xml.gz'), 'http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.3.2/repodata/primary.xml.gz': - self.get_test_loc('repodata_rpms/repomd_parser/cloudera/primary.xml.gz'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/cloudera/primary.xml.gz'), } uri = 'http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.3.2/repodata/repomd.xml' with patch('requests.get') as mock_http_get: - mock_http_get.side_effect = lambda * args, **kwargs: mocked_requests_get_for_uris(uri2loc, *args, **kwargs) + mock_http_get.side_effect = lambda * args, **kwargs: mocked_requests_get_for_uris( + uri2loc, *args, **kwargs) _uris, packages, _error = collect_rpm_packages_from_repomd(uri) - expected_loc = self.get_test_loc('repodata_rpms/repomd_parser/cloudera/expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'repodata_rpms/repomd_parser/cloudera/expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_collect_rpm_packages_from_repomd_centos(self): uri2loc = { 'http://vault.centos.org/3.8/updates/x86_64/repodata/repomd.xml': - self.get_test_loc('repodata_rpms/repomd_parser/centos/repomd.xml'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/centos/repomd.xml'), 'http://vault.centos.org/3.8/updates/x86_64/repodata/filelists.xml.gz': - self.get_test_loc('repodata_rpms/repomd_parser/centos/filelists.xml.gz'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/centos/filelists.xml.gz'), 'http://vault.centos.org/3.8/updates/x86_64/repodata/other.xml.gz': - self.get_test_loc('repodata_rpms/repomd_parser/centos/other.xml.gz'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/centos/other.xml.gz'), 'http://vault.centos.org/3.8/updates/x86_64/repodata/primary.xml.gz': - self.get_test_loc('repodata_rpms/repomd_parser/centos/primary.xml.gz'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/centos/primary.xml.gz'), } uri = 'http://vault.centos.org/3.8/updates/x86_64/repodata/repomd.xml' with patch('requests.get') as mock_http_get: - mock_http_get.side_effect = lambda * args, **kwargs: mocked_requests_get_for_uris(uri2loc, *args, **kwargs) + mock_http_get.side_effect = lambda * args, **kwargs: mocked_requests_get_for_uris( + uri2loc, *args, **kwargs) uris, packages, _error = collect_rpm_packages_from_repomd(uri) expected_uris = [ @@ -88,44 +102,58 @@ def test_collect_rpm_packages_from_repomd_centos(self): ] self.assertEqual(expected_uris, uris) - expected_loc = self.get_test_loc('repodata_rpms/repomd_parser/centos/expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'repodata_rpms/repomd_parser/centos/expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_collect_rpm_packages_from_repomd_cloudera_2(self): uri2loc = { 'http://archive.cloudera.com/cm5/redhat/5/x86_64/cm/5.2.0/repodata/repomd.xml': - self.get_test_loc('repodata_rpms/repomd_parser/cloudera2/repomd.xml'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/cloudera2/repomd.xml'), 'http://archive.cloudera.com/cm5/redhat/5/x86_64/cm/5.2.0/repodata/filelists.xml.gz': - self.get_test_loc('repodata_rpms/repomd_parser/cloudera2/filelists.xml.gz'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/cloudera2/filelists.xml.gz'), 'http://archive.cloudera.com/cm5/redhat/5/x86_64/cm/5.2.0/repodata/primary.xml.gz': - self.get_test_loc('repodata_rpms/repomd_parser/cloudera2/primary.xml.gz'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/cloudera2/primary.xml.gz'), 'http://archive.cloudera.com/cm5/redhat/5/x86_64/cm/5.2.0/repodata/other.xml.gz': - self.get_test_loc('repodata_rpms/repomd_parser/cloudera2/other.xml.gz'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/cloudera2/other.xml.gz'), } uri = 'http://archive.cloudera.com/cm5/redhat/5/x86_64/cm/5.2.0/repodata/repomd.xml' with patch('requests.get') as mock_http_get: - mock_http_get.side_effect = lambda * args, **kwargs: mocked_requests_get_for_uris(uri2loc, *args, **kwargs) + mock_http_get.side_effect = lambda * args, **kwargs: mocked_requests_get_for_uris( + uri2loc, *args, **kwargs) _uris, packages, _error = collect_rpm_packages_from_repomd(uri) - expected_loc = self.get_test_loc('repodata_rpms/repomd_parser/cloudera2/expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'repodata_rpms/repomd_parser/cloudera2/expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_collect_rpm_packages_from_repomd_postgresql(self): uri2loc = { 'http://yum.postgresql.org/9.2/redhat/rhel-6-x86_64/repodata/repomd.xml': - self.get_test_loc('repodata_rpms/repomd_parser/postgresql/repomd.xml'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/postgresql/repomd.xml'), 'http://yum.postgresql.org/9.2/redhat/rhel-6-x86_64/repodata/d5b4a2d13632cceb2a13a42fdb2887a22c1e262e6eeeb7270a80beec453392cd-filelists.xml.gz': - self.get_test_loc('repodata_rpms/repomd_parser/postgresql/d5b4a2d13632cceb2a13a42fdb2887a22c1e262e6eeeb7270a80beec453392cd-filelists.xml.gz'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/postgresql/d5b4a2d13632cceb2a13a42fdb2887a22c1e262e6eeeb7270a80beec453392cd-filelists.xml.gz'), 'http://yum.postgresql.org/9.2/redhat/rhel-6-x86_64/repodata/fc8c4fa6295d68abddcf5bba71435ecf585c439b86d7e75e0ba9bf3951f914b5-other.xml.gz': - self.get_test_loc('repodata_rpms/repomd_parser/postgresql/fc8c4fa6295d68abddcf5bba71435ecf585c439b86d7e75e0ba9bf3951f914b5-other.xml.gz'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/postgresql/fc8c4fa6295d68abddcf5bba71435ecf585c439b86d7e75e0ba9bf3951f914b5-other.xml.gz'), 'http://yum.postgresql.org/9.2/redhat/rhel-6-x86_64/repodata/d5cb2a54df0aa000ac2a007b1d9b0d1f2e6a924d2d97584acbe654e59aa993e8-primary.xml.gz': - self.get_test_loc('repodata_rpms/repomd_parser/postgresql/d5cb2a54df0aa000ac2a007b1d9b0d1f2e6a924d2d97584acbe654e59aa993e8-primary.xml.gz'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/postgresql/d5cb2a54df0aa000ac2a007b1d9b0d1f2e6a924d2d97584acbe654e59aa993e8-primary.xml.gz'), } uri = 'http://yum.postgresql.org/9.2/redhat/rhel-6-x86_64/repodata/repomd.xml' with patch('requests.get') as mock_http_get: - mock_http_get.side_effect = lambda * args, **kwargs: mocked_requests_get_for_uris(uri2loc, *args, **kwargs) + mock_http_get.side_effect = lambda * args, **kwargs: mocked_requests_get_for_uris( + uri2loc, *args, **kwargs) uris, packages, error = collect_rpm_packages_from_repomd(uri) self.assertEqual(None, error) expected_uris = [ @@ -135,52 +163,69 @@ def test_collect_rpm_packages_from_repomd_postgresql(self): ] self.assertEqual(expected_uris, uris) - expected_loc = self.get_test_loc('repodata_rpms/repomd_parser/postgresql/expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'repodata_rpms/repomd_parser/postgresql/expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_collect_rpm_packages_from_repomd_opensuse(self): uri2loc = { 'http://download.opensuse.org/distribution/12.3/repo/oss/suse/repodata/repomd.xml': - self.get_test_loc('repodata_rpms/repomd_parser/opensuse/repomd.xml'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/opensuse/repomd.xml'), 'http://download.opensuse.org/distribution/12.3/repo/oss/suse/repodata/09ed18eaa761fe64c863137db5c51fdb4e60fbb29d6c9b0c424e3119ba4875cd-filelists.xml.gz': - self.get_test_loc('repodata_rpms/repomd_parser/opensuse/09ed18eaa761fe64c863137db5c51fdb4e60fbb29d6c9b0c424e3119ba4875cd-filelists.xml.gz'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/opensuse/09ed18eaa761fe64c863137db5c51fdb4e60fbb29d6c9b0c424e3119ba4875cd-filelists.xml.gz'), 'http://download.opensuse.org/distribution/12.3/repo/oss/suse/repodata/9c100bbff252834349ca677813f333881ce9d2ca9db8091ce387156ba7a22859-other.xml.gz': - self.get_test_loc('repodata_rpms/repomd_parser/opensuse/9c100bbff252834349ca677813f333881ce9d2ca9db8091ce387156ba7a22859-other.xml.gz'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/opensuse/9c100bbff252834349ca677813f333881ce9d2ca9db8091ce387156ba7a22859-other.xml.gz'), 'http://download.opensuse.org/distribution/12.3/repo/oss/suse/repodata/314da4321afcff987bd3e28672e60f1a2324f2698480b84812f7ec0a1aef4041-primary.xml.gz': - self.get_test_loc('repodata_rpms/repomd_parser/opensuse/314da4321afcff987bd3e28672e60f1a2324f2698480b84812f7ec0a1aef4041-primary.xml.gz'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/opensuse/314da4321afcff987bd3e28672e60f1a2324f2698480b84812f7ec0a1aef4041-primary.xml.gz'), } uri = 'http://download.opensuse.org/distribution/12.3/repo/oss/suse/repodata/repomd.xml' with patch('requests.get') as mock_http_get: - mock_http_get.side_effect = lambda * args, **kwargs: mocked_requests_get_for_uris(uri2loc, *args, **kwargs) + mock_http_get.side_effect = lambda * args, **kwargs: mocked_requests_get_for_uris( + uri2loc, *args, **kwargs) _uris, packages, _error = collect_rpm_packages_from_repomd(uri) - expected_loc = self.get_test_loc('repodata_rpms/repomd_parser/opensuse/expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'repodata_rpms/repomd_parser/opensuse/expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_collect_rpm_packages_from_repomd_pgpool(self): uri2loc = { 'http://pgpool.net/yum/rpms/3.4/redhat/rhel-6-x86_64/repodata/repomd.xml': - self.get_test_loc('repodata_rpms/repomd_parser/pgpool/repomd.xml'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/pgpool/repomd.xml'), 'http://pgpool.net/yum/rpms/3.4/redhat/rhel-6-x86_64/repodata/filelists.xml.gz': - self.get_test_loc('repodata_rpms/repomd_parser/pgpool/filelists.xml.gz'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/pgpool/filelists.xml.gz'), 'http://pgpool.net/yum/rpms/3.4/redhat/rhel-6-x86_64/repodata/other.xml.gz': - self.get_test_loc('repodata_rpms/repomd_parser/pgpool/other.xml.gz'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/pgpool/other.xml.gz'), 'http://pgpool.net/yum/rpms/3.4/redhat/rhel-6-x86_64/repodata/primary.xml.gz': - self.get_test_loc('repodata_rpms/repomd_parser/pgpool/primary.xml.gz'), + self.get_test_loc( + 'repodata_rpms/repomd_parser/pgpool/primary.xml.gz'), } uri = 'http://pgpool.net/yum/rpms/3.4/redhat/rhel-6-x86_64/repodata/repomd.xml' with patch('requests.get') as mock_http_get: - mock_http_get.side_effect = lambda * args, **kwargs: mocked_requests_get_for_uris(uri2loc, *args, **kwargs) + mock_http_get.side_effect = lambda * args, **kwargs: mocked_requests_get_for_uris( + uri2loc, *args, **kwargs) _uris, packages, _error = collect_rpm_packages_from_repomd(uri) - expected_loc = self.get_test_loc('repodata_rpms/repomd_parser/pgpool/expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'repodata_rpms/repomd_parser/pgpool/expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_combine_dicts_using_pkgid(self): all_dicts = [ - {'pkgid': '36547e200627ea25c4e3fb6f9735d58e682f8e35cd815dceed796c83628e60d5', 'name': 'python-ceilometerclient'}, + {'pkgid': '36547e200627ea25c4e3fb6f9735d58e682f8e35cd815dceed796c83628e60d5', + 'name': 'python-ceilometerclient'}, {'pkgid': '36547e200627ea25c4e3fb6f9735d58e682f8e35cd815dceed796c83628e60d5', 'ver': '1.5.0'}, {'pkgid': '36547e200627ea25c4e3fb6f9735d58e682f8e35cd815dceed796c83628e60d5', 'rel': '1.el7'} ] diff --git a/minecode/tests/test_route.py b/minecode/tests/test_route.py index 0ff2660b..2a20ce7c 100644 --- a/minecode/tests/test_route.py +++ b/minecode/tests/test_route.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/tests/test_rsync.py b/minecode/tests/test_rsync.py index 4a4f5f3e..7a3901e4 100644 --- a/minecode/tests/test_rsync.py +++ b/minecode/tests/test_rsync.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -37,12 +37,18 @@ def test_entry_rsync_31(self): '-rw-rw-r-- 0 2015/07/23 17:36:47 bar/that/baz', ] expected = [ - rsync.Entry('d', 'rwxrwxr-x', 4096, '2015-07-23T17:36:47+00:00', '.')._asdict(), - rsync.Entry('-', 'rw-rw-r--', 0, '2015-07-23T17:36:47+00:00', 'foo')._asdict(), - rsync.Entry('d', 'rwxrwxr-x', 4096, '2015-07-23T17:36:47+00:00', 'bar')._asdict(), - rsync.Entry('-', 'rw-rw-r--', 0, '2015-07-23T17:36:47+00:00', 'bar/this')._asdict(), - rsync.Entry('d', 'rwxrwxr-x', 4096, '2015-07-23T17:36:47+00:00', 'bar/that')._asdict(), - rsync.Entry('-', 'rw-rw-r--', 0, '2015-07-23T17:36:47+00:00', 'bar/that/baz')._asdict(), + rsync.Entry('d', 'rwxrwxr-x', 4096, + '2015-07-23T17:36:47+00:00', '.')._asdict(), + rsync.Entry('-', 'rw-rw-r--', 0, + '2015-07-23T17:36:47+00:00', 'foo')._asdict(), + rsync.Entry('d', 'rwxrwxr-x', 4096, + '2015-07-23T17:36:47+00:00', 'bar')._asdict(), + rsync.Entry('-', 'rw-rw-r--', 0, + '2015-07-23T17:36:47+00:00', 'bar/this')._asdict(), + rsync.Entry('d', 'rwxrwxr-x', 4096, + '2015-07-23T17:36:47+00:00', 'bar/that')._asdict(), + rsync.Entry('-', 'rw-rw-r--', 0, + '2015-07-23T17:36:47+00:00', 'bar/that/baz')._asdict(), ] for test, exp in zip(lines, expected): @@ -59,9 +65,12 @@ def test_entry(self): ] expected = [ - rsync.Entry('-', 'rw-r--r--', 4399746, '2008-11-23T16:03:57+00:00', 'zz/ZZUL P/ZUL.gz')._asdict(), - rsync.Entry('d', 'rwxrwxr-x', 4096, '2004-08-09T00:47:02+00:00', 'pub/sou/a/a7')._asdict(), - rsync.Entry('-', 'rwxrwxr-x', 4096, '2004-08-09T00:47:02+00:00', 'pub/\xe5sou/a/a7')._asdict(), + rsync.Entry('-', 'rw-r--r--', 4399746, + '2008-11-23T16:03:57+00:00', 'zz/ZZUL P/ZUL.gz')._asdict(), + rsync.Entry('d', 'rwxrwxr-x', 4096, + '2004-08-09T00:47:02+00:00', 'pub/sou/a/a7')._asdict(), + rsync.Entry('-', 'rwxrwxr-x', 4096, + '2004-08-09T00:47:02+00:00', 'pub/\xe5sou/a/a7')._asdict(), None, None, ] @@ -75,27 +84,48 @@ def test_directory(self): output = list(rsync.directory_entries(test_dir)) expected = [ - rsync.Entry(type='d', perm='rwxrwxr-x', size=4096, date='2014-03-18T19:02:46+00:00', path='.'), - rsync.Entry(type='-', perm='rw-rw-r--', size=5, date='2014-03-18T19:02:46+00:00', path='.revision'), - rsync.Entry(type='d', perm='rwxrwxr-x', size=4096, date='2014-02-05T09:34:20+00:00', path='1.4.23'), - rsync.Entry(type='-', perm='rw-rw-r--', size=95314, date='2014-02-05T09:23:44+00:00', path='1.4.23/CHANGELOG-1.4'), - rsync.Entry(type='-', perm='rw-rw-r--', size=3712820, date='2014-02-05T09:23:44+00:00', path='1.4.23/apache-wicket-1.4.23-source.tgz'), - rsync.Entry(type='d', perm='rwxrwxr-x', size=4096, date='2014-02-05T09:34:20+00:00', path='1.4.23/binaries'), - rsync.Entry(type='-', perm='rw-rw-r--', size=23622515, date='2014-02-05T09:23:44+00:00', path='1.4.23/binaries/apache-wicket-1.4.23.tar.gz'), - rsync.Entry(type='-', perm='rw-rw-r--', size=32524295, date='2014-02-05T09:23:44+00:00', path='1.4.23/binaries/apache-wicket-1.4.23.zip'), - rsync.Entry(type='d', perm='rwxrwxr-x', size=4096, date='2014-01-27T09:09:40+00:00', path='1.5.11'), - rsync.Entry(type='-', perm='rw-rw-r--', size=115587, date='2014-01-20T16:53:10+00:00', path='1.5.11/CHANGELOG-1.5'), - rsync.Entry(type='-', perm='rw-rw-r--', size=4116809, date='2014-01-20T16:53:10+00:00', path='1.5.11/apache-wicket-1.5.11-source.tgz'), - rsync.Entry(type='d', perm='rwxrwxr-x', size=4096, date='2014-01-27T09:09:39+00:00', path='1.5.11/binaries'), - rsync.Entry(type='-', perm='rw-rw-r--', size=26048500, date='2014-01-20T16:53:10+00:00', path='1.5.11/binaries/apache-wicket-1.5.11.tar.gz'), - rsync.Entry(type='-', perm='rw-rw-r--', size=36156260, date='2014-01-20T16:53:10+00:00', path='1.5.11/binaries/apache-wicket-1.5.11.zip'), - rsync.Entry(type='d', perm='rwxrwxr-x', size=4096, date='2014-02-19T08:36:07+00:00', path='6.14.0'), - rsync.Entry(type='-', perm='rw-rw-r--', size=78058, date='2014-02-14T15:51:23+00:00', path='6.14.0/CHANGELOG-6.x'), - rsync.Entry(type='-', perm='rw-rw-r--', size=4792619, date='2014-02-14T15:51:23+00:00', path='6.14.0/apache-wicket-6.14.0.tar.gz'), - rsync.Entry(type='-', perm='rw-rw-r--', size=9038442, date='2014-02-14T15:51:23+00:00', path='6.14.0/apache-wicket-6.14.0.zip'), - rsync.Entry(type='d', perm='rwxrwxr-x', size=4096, date='2014-02-19T08:36:05+00:00', path='6.14.0/binaries'), - rsync.Entry(type='-', perm='rw-rw-r--', size=29851252, date='2014-02-14T15:51:23+00:00', path='6.14.0/binaries/apache-wicket-6.14.0-bin.tar.gz'), - rsync.Entry(type='-', perm='rw-rw-r--', size=29890658, date='2014-02-14T15:51:23+00:00', path='6.14.0/binaries/apache-wicket-6.14.0-bin.zip') + rsync.Entry(type='d', perm='rwxrwxr-x', size=4096, + date='2014-03-18T19:02:46+00:00', path='.'), + rsync.Entry(type='-', perm='rw-rw-r--', size=5, + date='2014-03-18T19:02:46+00:00', path='.revision'), + rsync.Entry(type='d', perm='rwxrwxr-x', size=4096, + date='2014-02-05T09:34:20+00:00', path='1.4.23'), + rsync.Entry(type='-', perm='rw-rw-r--', size=95314, + date='2014-02-05T09:23:44+00:00', path='1.4.23/CHANGELOG-1.4'), + rsync.Entry(type='-', perm='rw-rw-r--', size=3712820, + date='2014-02-05T09:23:44+00:00', path='1.4.23/apache-wicket-1.4.23-source.tgz'), + rsync.Entry(type='d', perm='rwxrwxr-x', size=4096, + date='2014-02-05T09:34:20+00:00', path='1.4.23/binaries'), + rsync.Entry(type='-', perm='rw-rw-r--', size=23622515, date='2014-02-05T09:23:44+00:00', + path='1.4.23/binaries/apache-wicket-1.4.23.tar.gz'), + rsync.Entry(type='-', perm='rw-rw-r--', size=32524295, + date='2014-02-05T09:23:44+00:00', path='1.4.23/binaries/apache-wicket-1.4.23.zip'), + rsync.Entry(type='d', perm='rwxrwxr-x', size=4096, + date='2014-01-27T09:09:40+00:00', path='1.5.11'), + rsync.Entry(type='-', perm='rw-rw-r--', size=115587, + date='2014-01-20T16:53:10+00:00', path='1.5.11/CHANGELOG-1.5'), + rsync.Entry(type='-', perm='rw-rw-r--', size=4116809, + date='2014-01-20T16:53:10+00:00', path='1.5.11/apache-wicket-1.5.11-source.tgz'), + rsync.Entry(type='d', perm='rwxrwxr-x', size=4096, + date='2014-01-27T09:09:39+00:00', path='1.5.11/binaries'), + rsync.Entry(type='-', perm='rw-rw-r--', size=26048500, date='2014-01-20T16:53:10+00:00', + path='1.5.11/binaries/apache-wicket-1.5.11.tar.gz'), + rsync.Entry(type='-', perm='rw-rw-r--', size=36156260, + date='2014-01-20T16:53:10+00:00', path='1.5.11/binaries/apache-wicket-1.5.11.zip'), + rsync.Entry(type='d', perm='rwxrwxr-x', size=4096, + date='2014-02-19T08:36:07+00:00', path='6.14.0'), + rsync.Entry(type='-', perm='rw-rw-r--', size=78058, + date='2014-02-14T15:51:23+00:00', path='6.14.0/CHANGELOG-6.x'), + rsync.Entry(type='-', perm='rw-rw-r--', size=4792619, + date='2014-02-14T15:51:23+00:00', path='6.14.0/apache-wicket-6.14.0.tar.gz'), + rsync.Entry(type='-', perm='rw-rw-r--', size=9038442, + date='2014-02-14T15:51:23+00:00', path='6.14.0/apache-wicket-6.14.0.zip'), + rsync.Entry(type='d', perm='rwxrwxr-x', size=4096, + date='2014-02-19T08:36:05+00:00', path='6.14.0/binaries'), + rsync.Entry(type='-', perm='rw-rw-r--', size=29851252, date='2014-02-14T15:51:23+00:00', + path='6.14.0/binaries/apache-wicket-6.14.0-bin.tar.gz'), + rsync.Entry(type='-', perm='rw-rw-r--', size=29890658, date='2014-02-14T15:51:23+00:00', + path='6.14.0/binaries/apache-wicket-6.14.0-bin.zip') ] expected = [dict(x._asdict()) for x in expected] self.assertEqual(expected, output) diff --git a/minecode/tests/test_rubygems.py b/minecode/tests/test_rubygems.py index 2deb3ea1..2126d096 100644 --- a/minecode/tests/test_rubygems.py +++ b/minecode/tests/test_rubygems.py @@ -4,12 +4,11 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # - import codecs import json import os @@ -65,10 +64,12 @@ def test_RubyGemsIndexVisitor_latest(self): with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _, _ = RubyGemsIndexVisitor(uri) - expected_loc = self.get_test_loc('rubygems/index/latest_specs.4.8.gz.expected.json') + expected_loc = self.get_test_loc( + 'rubygems/index/latest_specs.4.8.gz.expected.json') uris_list = list(uris) self.assertTrue(len(uris_list) > 1000) - self.check_expected_uris(uris_list[0:1000], expected_loc, regen=FIXTURES_REGEN) + self.check_expected_uris( + uris_list[0:1000], expected_loc, regen=FIXTURES_REGEN) def test_RubyGemsApiVersionVisitor(self): uri = 'https://rubygems.org/api/v1/versions/0xffffff.json' @@ -76,7 +77,8 @@ def test_RubyGemsApiVersionVisitor(self): with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) _, data, _ = RubyGemsApiManyVersionsVisitor(uri) - expected_loc = self.get_test_loc('rubygems/apiv1/expected_0xffffff.api.json') + expected_loc = self.get_test_loc( + 'rubygems/apiv1/expected_0xffffff.api.json') self.check_expected_results(data, expected_loc, regen=FIXTURES_REGEN) def test_RubyGemsApiVersionVisitor2(self): @@ -85,7 +87,8 @@ def test_RubyGemsApiVersionVisitor2(self): with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) _, data, _ = RubyGemsApiManyVersionsVisitor(uri) - expected_loc = self.get_test_loc('rubygems/apiv1/expected_a1630ty_a1630ty.api.json') + expected_loc = self.get_test_loc( + 'rubygems/apiv1/expected_a1630ty_a1630ty.api.json') self.check_expected_results(data, expected_loc, regen=FIXTURES_REGEN) def test_RubyGemsApiVersionVisitor3(self): @@ -94,7 +97,8 @@ def test_RubyGemsApiVersionVisitor3(self): with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) _, data, _ = RubyGemsApiManyVersionsVisitor(uri) - expected_loc = self.get_test_loc('rubygems/apiv1/expected_zuck.api.json') + expected_loc = self.get_test_loc( + 'rubygems/apiv1/expected_zuck.api.json') self.check_expected_results(data, expected_loc, regen=FIXTURES_REGEN) def test_RubyGemsPackageArchiveMetadataVisitor(self): @@ -116,32 +120,42 @@ def test_build_rubygem_packages_from_api_data_1(self): apidata = json.load(api) packages = build_rubygem_packages_from_api_data(apidata, '0xffffff') packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('rubygems/apiv1/0xffffff.api.package.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'rubygems/apiv1/0xffffff.api.package.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_rubygem_packages_from_api_data_2(self): with open(self.get_test_loc('rubygems/apiv1/zuck.api.json')) as api: apidata = json.load(api) packages = build_rubygem_packages_from_api_data(apidata, 'zuck') packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('rubygems/apiv1/zuck.api.package.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'rubygems/apiv1/zuck.api.package.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_rubygem_packages_from_api_data_3(self): with open(self.get_test_loc('rubygems/apiv1/a1630ty_a1630ty.api.json')) as api: apidata = json.load(api) - packages = mappers.rubygems.build_rubygem_packages_from_api_data(apidata, 'a1630ty_a1630ty') + packages = mappers.rubygems.build_rubygem_packages_from_api_data( + apidata, 'a1630ty_a1630ty') packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('rubygems/apiv1/a1630ty_a1630ty.api.package.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'rubygems/apiv1/a1630ty_a1630ty.api.package.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_rubygem_packages_from_api_data_with_deps(self): with open(self.get_test_loc('rubygems/apiv1/action_tracker.api.json')) as api: apidata = json.load(api) - packages = mappers.rubygems.build_rubygem_packages_from_api_data(apidata, 'action_tracker') + packages = mappers.rubygems.build_rubygem_packages_from_api_data( + apidata, 'action_tracker') packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('rubygems/apiv1/action_tracker.api.package.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'rubygems/apiv1/action_tracker.api.package.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_RubyGemsApiVersionsJsonMapper(self): test_uri = 'https://rubygems.org/api/v1/versions/a1630ty_a1630ty.json' @@ -154,8 +168,10 @@ def test_RubyGemsApiVersionsJsonMapper(self): test_res_uri = ResourceURI(uri=test_uri, data=test_data) packages = RubyGemsApiVersionsJsonMapper(test_uri, test_res_uri) packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('rubygems/apiv1/a1630ty_a1630ty.api.mapped.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'rubygems/apiv1/a1630ty_a1630ty.api.mapped.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) class RubyGemsArchiveMapperTest(JsonBasedTesting): @@ -165,15 +181,18 @@ def test_test_RubyGemsPackageArchiveMetadataMapper(self): test_uri = 'https://rubygems.org/downloads/mysmallidea-address_standardization-0.4.1.gem' router = route.Router() router.append(test_uri, RubyGemsPackageArchiveMetadataMapper) - test_loc = self.get_test_loc('rubygems/mysmallidea-address_standardization-0.4.1.gem.metadata') + test_loc = self.get_test_loc( + 'rubygems/mysmallidea-address_standardization-0.4.1.gem.metadata') with codecs.open(test_loc, encoding='utf-8') as test_file: test_data = test_file.read() test_res_uri = ResourceURI(uri=test_uri, data=test_data) packages = RubyGemsPackageArchiveMetadataMapper(test_uri, test_res_uri) packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('rubygems/mysmallidea-address_standardization-0.4.1.gem.mapped.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'rubygems/mysmallidea-address_standardization-0.4.1.gem.mapped.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def check_mapped_packages(self, test_loc, expected_loc, extract=True, regen=FIXTURES_REGEN): @@ -185,7 +204,8 @@ def check_mapped_packages(self, test_loc, expected_loc, extract=True, regen=FIXT with open(test_loc) as tl: metadata = tl.read() - download_url = 'https://rubygems.org/downloads/{}'.format(file_name(test_loc).replace('.metadata', '')) + download_url = 'https://rubygems.org/downloads/{}'.format( + file_name(test_loc).replace('.metadata', '')) results = build_rubygem_packages_from_metadata(metadata, download_url) results = [p.to_dict() for p in results] @@ -271,7 +291,8 @@ def test_visit_and_map_end2end(self): import packagedb uri = 'https://rubygems.org/downloads/sprockets-vendor_gems-0.1.3.gem' - test_loc = self.get_test_loc('rubygems/sprockets-vendor_gems-0.1.3.gem', copy=True) + test_loc = self.get_test_loc( + 'rubygems/sprockets-vendor_gems-0.1.3.gem', copy=True) before_uri = [p.id for p in ResourceURI.objects.all()] before_pkg = [p.id for p in packagedb.models.Package.objects.all()] @@ -290,14 +311,19 @@ def test_visit_and_map_end2end(self): visited = ResourceURI.objects.all() uri_results = [model_to_dict(rec, exclude=['id']) for rec in visited] - expected_loc = self.get_test_loc('rubygems/sprockets-vendor_gems-0.1.3.gem.visited.json') - self.check_expected_results(uri_results, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'rubygems/sprockets-vendor_gems-0.1.3.gem.visited.json') + self.check_expected_results( + uri_results, expected_loc, regen=FIXTURES_REGEN) if before_pkg: - mapped = packagedb.models.Package.objects.exclude(id__in=before_pkg) + mapped = packagedb.models.Package.objects.exclude( + id__in=before_pkg) else: mapped = packagedb.models.Package.objects.all() package_results = [pac.to_dict() for pac in mapped] - expected_loc = self.get_test_loc('rubygems/sprockets-vendor_gems-0.1.3.gem.mapped.json') - self.check_expected_results(package_results, expected_loc, fields_to_remove=['package_sets'], regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'rubygems/sprockets-vendor_gems-0.1.3.gem.mapped.json') + self.check_expected_results(package_results, expected_loc, fields_to_remove=[ + 'package_sets'], regen=FIXTURES_REGEN) diff --git a/minecode/tests/test_run_map.py b/minecode/tests/test_run_map.py index e269bbbe..7bca14f1 100644 --- a/minecode/tests/test_run_map.py +++ b/minecode/tests/test_run_map.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -26,7 +26,6 @@ import packagedb - class RunMapTest(JsonBasedTesting, MiningTestCase): BASE_DIR = os.path.join(os.path.dirname(__file__), 'testfiles') maxDiff = None @@ -73,7 +72,8 @@ def mock_mapper(uri, resource_uri): download_url='http://testdomap.com') self.assertEqual(1, mapped.count()) mapped_package = mapped.first() - self.assertEqual('pkg:maven/org.apache.spark/spark-streaming_2.10@1.2.0?extension=pom', mapped_package.package_url) + self.assertEqual( + 'pkg:maven/org.apache.spark/spark-streaming_2.10@1.2.0?extension=pom', mapped_package.package_url) # test history history = mapped_package.get_history() @@ -81,7 +81,8 @@ def mock_mapper(uri, resource_uri): self.assertEqual(1, len(history)) entry = history[0] message = entry.get('message') - self.assertEqual('New Package created from URI: {}'.format(uri), message) + self.assertEqual( + 'New Package created from URI: {}'.format(uri), message) # check that the ResourceURI status has been updated correctly resource_uri = ResourceURI.objects.get(uri=uri) @@ -170,7 +171,8 @@ def mock_mapper(uri, resource_uri): resource_uri = ResourceURI.objects.get(uri=uri) self.assertEqual(None, resource_uri.wip_date) self.assertFalse(resource_uri.last_map_date is None) - self.assertTrue('Not a ScanCode PackageData type' in resource_uri.map_error) + self.assertTrue( + 'Not a ScanCode PackageData type' in resource_uri.map_error) # check that a ScannableURI has not been created scannable = ScannableURI.objects.filter(uri='http://testdomap.com') @@ -217,7 +219,8 @@ def mock_mapper(uri, resource_uri): resource_uri = ResourceURI.objects.get(uri=uri) self.assertEqual(None, resource_uri.wip_date) self.assertFalse(resource_uri.last_map_date is None) - self.assertTrue('No download_url for package' in resource_uri.map_error) + self.assertTrue( + 'No download_url for package' in resource_uri.map_error) # check that a ScannableURI has not been created scannable = ScannableURI.objects.filter(uri='http://testdomap.com') @@ -355,7 +358,8 @@ def mock_mapper(uri, resource_uri): # test proper map_uri(resource_uri, _map_router=router) - mapped = packagedb.models.Package.objects.filter(download_url=download_url) + mapped = packagedb.models.Package.objects.filter( + download_url=download_url) self.assertEqual(1, mapped.count()) mapped_package = mapped.first() @@ -371,7 +375,8 @@ def mock_mapper(uri, resource_uri): expected_updated_fields_loc = self.get_test_loc( 'run_map/test_map_uri_does_update_with_same_mining_level_expected_updated_fields.json' ) - self.check_expected_results(updated_fields, expected_updated_fields_loc, regen=FIXTURES_REGEN) + self.check_expected_results( + updated_fields, expected_updated_fields_loc, regen=FIXTURES_REGEN) # check that the ResourceURI status has been updated correctly resource_uri = ResourceURI.objects.get(uri=uri) @@ -379,7 +384,8 @@ def mock_mapper(uri, resource_uri): self.assertFalse(resource_uri.last_map_date is None) # check that the Package has been updated correctly - expected_loc = self.get_test_loc('run_map/test_map_uri_does_update_with_same_mining_level-expected.json') + expected_loc = self.get_test_loc( + 'run_map/test_map_uri_does_update_with_same_mining_level-expected.json') result = mapped_package.to_dict() self.check_expected_results(result, expected_loc, regen=FIXTURES_REGEN) @@ -437,7 +443,8 @@ def mock_mapper(uri, resource_uri): # test proper map_uri(resource_uri, _map_router=router) - mapped = packagedb.models.Package.objects.filter(download_url=download_url) + mapped = packagedb.models.Package.objects.filter( + download_url=download_url) self.assertEqual(1, mapped.count()) mapped_package = mapped.first() @@ -470,7 +477,8 @@ def mock_mapper(uri, resource_uri): self.assertFalse(resource_uri.last_map_date is None) # check that the Package has been updated correctly - expected_loc = self.get_test_loc('run_map/test_map_uri_update_only_empties_with_lesser_new_mining_level-expected.json') + expected_loc = self.get_test_loc( + 'run_map/test_map_uri_update_only_empties_with_lesser_new_mining_level-expected.json') result = mapped[0].to_dict() self.check_expected_results(result, expected_loc, regen=FIXTURES_REGEN) @@ -527,7 +535,8 @@ def mock_mapper(uri, resource_uri): # test proper map_uri(resource_uri, _map_router=router) - mapped = packagedb.models.Package.objects.filter(download_url=download_url) + mapped = packagedb.models.Package.objects.filter( + download_url=download_url) self.assertEqual(1, mapped.count()) mapped_package = mapped.first() @@ -543,8 +552,8 @@ def mock_mapper(uri, resource_uri): expected_updated_fields_loc = self.get_test_loc( 'run_map/test_map_uri_replace_with_new_with_higher_new_mining_level_expected_updated_fields.json' ) - self.check_expected_results(updated_fields, expected_updated_fields_loc, regen=FIXTURES_REGEN) - + self.check_expected_results( + updated_fields, expected_updated_fields_loc, regen=FIXTURES_REGEN) # check that the ResourceURI status has been updated correctly resource_uri = ResourceURI.objects.get(uri=uri) @@ -552,7 +561,8 @@ def mock_mapper(uri, resource_uri): self.assertFalse(resource_uri.last_map_date is None) # check that the Package has been updated correctly - expected_loc = self.get_test_loc('run_map/test_map_uri_replace_with_new_with_higher_new_mining_level-expected.json') + expected_loc = self.get_test_loc( + 'run_map/test_map_uri_replace_with_new_with_higher_new_mining_level-expected.json') result = mapped[0].to_dict() self.check_expected_results(result, expected_loc, regen=FIXTURES_REGEN) @@ -580,7 +590,8 @@ def test_merge_packages_no_replace(self): download_url=download_url ).to_dict() merge_packages(existing_package, new_package_data, replace=False) - expected_loc = self.get_test_loc('run_map/test_merge_packages_no_replace-expected.json') + expected_loc = self.get_test_loc( + 'run_map/test_merge_packages_no_replace-expected.json') result = existing_package.to_dict() self.check_expected_results(result, expected_loc, regen=FIXTURES_REGEN) @@ -602,7 +613,8 @@ def test_merge_packages_with_replace(self): download_url=download_url, ).to_dict() merge_packages(existing_package, new_package_data, replace=True) - expected_loc = self.get_test_loc('run_map/test_merge_packages_with_replace-expected.json') + expected_loc = self.get_test_loc( + 'run_map/test_merge_packages_with_replace-expected.json') result = existing_package.to_dict() self.check_expected_results(result, expected_loc, regen=FIXTURES_REGEN) diff --git a/minecode/tests/test_run_visit.py b/minecode/tests/test_run_visit.py index 73e200c5..08f0e480 100644 --- a/minecode/tests/test_run_visit.py +++ b/minecode/tests/test_run_visit.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -31,10 +31,14 @@ def test_visit_uri_with_counter_0_max_uris_3_multi_uri(self): def mock_visitor(uri): return [URI(uri='http://test-counter-0-max-uris-3-multi-uri1.com', package_url='pkg:npm/foobar@12.3.1'), - URI(uri='http://test-counter-0-max-uris-3-multi-uri2.com', package_url='pkg:npm/foobar@12.3.2'), - URI(uri='http://test-counter-0-max-uris-3-multi-uri3.com', package_url='pkg:npm/foobar@12.3.3'), - URI(uri='http://test-counter-0-max-uris-3-multi-uri4.com', package_url='pkg:npm/foobar@12.3.4'), - URI(uri='http://test-counter-0-max-uris-3-multi-uri5.com', package_url='pkg:npm/foobar@12.3.5'), + URI(uri='http://test-counter-0-max-uris-3-multi-uri2.com', + package_url='pkg:npm/foobar@12.3.2'), + URI(uri='http://test-counter-0-max-uris-3-multi-uri3.com', + package_url='pkg:npm/foobar@12.3.3'), + URI(uri='http://test-counter-0-max-uris-3-multi-uri4.com', + package_url='pkg:npm/foobar@12.3.4'), + URI(uri='http://test-counter-0-max-uris-3-multi-uri5.com', + package_url='pkg:npm/foobar@12.3.5'), URI(uri='http://test-counter-0-max-uris-3-multi-uri6.com', package_url='pkg:npm/foobar@12.3.5')], None, None router = Router() @@ -459,7 +463,8 @@ def test_visit_uri(self): self.assertEqual(1, visited.count()) def test_visit_uri_with_no_route_defined_does_not_visit(self): - resource_uri = ResourceURI.objects.create(uri='http://undefined-route.com') + resource_uri = ResourceURI.objects.create( + uri='http://undefined-route.com') resource_uri.is_visitable = True resource_uri.save() @@ -479,7 +484,8 @@ def test_run_visit_command(self): def test_visit_uri_always_inserts_new_uri(self): # test proper visit_uri(self.resource_uri, _visit_router=self.router2) - visited = ResourceURI.objects.filter(uri='http://test.com').order_by('-package_url') + visited = ResourceURI.objects.filter( + uri='http://test.com').order_by('-package_url') expected = [ URI(uri=u'http://test.com', data=u"{'some': 'data'}"), URI(uri=u'http://test.com', package_url='pkg:npm/foobar@12.3.1'), diff --git a/minecode/tests/test_seed.py b/minecode/tests/test_seed.py index ee17dee2..017cb477 100644 --- a/minecode/tests/test_seed.py +++ b/minecode/tests/test_seed.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -41,7 +41,8 @@ def get_seeds(self): def test_insert_seed_uris_revisit_before_10_days_custom_revisit_after(self): # we consume generators to insert seed URI - list(insert_seed_uris(pattern='.*python.org/pypi/.*', seeders=[self.SampleSeed1])) + list(insert_seed_uris(pattern='.*python.org/pypi/.*', + seeders=[self.SampleSeed1])) seeded = ResourceURI.objects.all() self.assertEqual(1, len(seeded)) @@ -50,13 +51,15 @@ def test_insert_seed_uris_revisit_before_10_days_custom_revisit_after(self): s.last_visit_date = timezone.now() - timedelta(minutes=10) s.save() - list(insert_seed_uris(pattern='.*python.org/pypi/.*', seeders=[self.SampleSeed1])) + list(insert_seed_uris(pattern='.*python.org/pypi/.*', + seeders=[self.SampleSeed1])) seeded = ResourceURI.objects.all() self.assertEqual(1, len(seeded)) def test_insert_seed_uris_revisit_after_10_days_custom_revisit_after(self): # we consume generators to insert seed URI - list(insert_seed_uris(pattern='.*python.org/pypi/.*', seeders=[self.SampleSeed1])) + list(insert_seed_uris(pattern='.*python.org/pypi/.*', + seeders=[self.SampleSeed1])) seeded = ResourceURI.objects.all() self.assertEqual(1, len(seeded)) @@ -65,13 +68,15 @@ def test_insert_seed_uris_revisit_after_10_days_custom_revisit_after(self): s.last_visit_date = timezone.now() - timedelta(days=10) s.save() - list(insert_seed_uris(pattern='.*python.org/pypi/.*', seeders=[self.SampleSeed1])) + list(insert_seed_uris(pattern='.*python.org/pypi/.*', + seeders=[self.SampleSeed1])) seeded = ResourceURI.objects.all() self.assertEqual(2, len(seeded)) def test_insert_seed_uris_revisit_before_10_days_default_revisit_after(self): # we consume generators to insert seed URI - list(insert_seed_uris(pattern='.*python.org/pypi/.*', seeders=[self.SampleSeed0])) + list(insert_seed_uris(pattern='.*python.org/pypi/.*', + seeders=[self.SampleSeed0])) seeded = ResourceURI.objects.all() self.assertEqual(1, len(seeded)) @@ -80,13 +85,15 @@ def test_insert_seed_uris_revisit_before_10_days_default_revisit_after(self): s.last_visit_date = timezone.now() - timedelta(days=9) s.save() - list(insert_seed_uris(pattern='.*python.org/pypi/.*', seeders=[self.SampleSeed0])) + list(insert_seed_uris(pattern='.*python.org/pypi/.*', + seeders=[self.SampleSeed0])) seeded = ResourceURI.objects.all() self.assertEqual(1, len(seeded)) def test_insert_seed_uris_revisit_after_10_days_default_revisit_after(self): # we consume generators to insert seed URI - list(insert_seed_uris(pattern='.*python.org/pypi/.*', seeders=[self.SampleSeed0])) + list(insert_seed_uris(pattern='.*python.org/pypi/.*', + seeders=[self.SampleSeed0])) seeded = ResourceURI.objects.all() self.assertEqual(1, len(seeded)) @@ -95,7 +102,8 @@ def test_insert_seed_uris_revisit_after_10_days_default_revisit_after(self): s.last_visit_date = timezone.now() - timedelta(days=10) s.save() - list(insert_seed_uris(pattern='.*python.org/pypi/.*', seeders=[self.SampleSeed0])) + list(insert_seed_uris(pattern='.*python.org/pypi/.*', + seeders=[self.SampleSeed0])) seeded = ResourceURI.objects.all() self.assertEqual(2, len(seeded)) @@ -167,7 +175,8 @@ def test_insert_seed_uris_inserts_uris_for_active_seeders_with_pattern(self, moc mock_get_active_seeders.return_value = [self.SampleSeed1] before = list(ResourceURI.objects.all().values_list('id')) seeders = seed.get_active_seeders() - results = sorted(insert_seed_uris(pattern='.*python.*igloo.json', seeders=seeders)) + results = sorted(insert_seed_uris( + pattern='.*python.*igloo.json', seeders=seeders)) if before: seeded = ResourceURI.objects.exclude(uri__in=before) else: diff --git a/minecode/tests/test_sourceforge.py b/minecode/tests/test_sourceforge.py index ee19a7dd..0397b96f 100644 --- a/minecode/tests/test_sourceforge.py +++ b/minecode/tests/test_sourceforge.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -29,9 +29,11 @@ def test_visit_sf_sitemap_index_new(self): test_loc = self.get_test_loc('sourceforge/sitemap.xml') with patch('requests.get') as mock_http_get: mock_http_get.return_value = mocked_requests_get(uri, test_loc) - uris, _data, error = sourceforge.SourceforgeSitemapIndexVisitor(uri) + uris, _data, error = sourceforge.SourceforgeSitemapIndexVisitor( + uri) - expected_loc = self.get_test_loc('sourceforge/expected_sf_sitemap_new.json') + expected_loc = self.get_test_loc( + 'sourceforge/expected_sf_sitemap_new.json') self.check_expected_uris(uris, expected_loc) self.assertIsNone(error) @@ -42,7 +44,8 @@ def test_visit_sf_sitemap_page_new(self): mock_http_get.return_value = mocked_requests_get(uri, test_loc) uris, _, error = sourceforge.SourceforgeSitemapPageVisitor(uri) - expected_loc = self.get_test_loc('sourceforge/expected_sf_sitemap_page_new.json') + expected_loc = self.get_test_loc( + 'sourceforge/expected_sf_sitemap_page_new.json') self.check_expected_uris(uris, expected_loc) self.assertIsNone(error) @@ -77,29 +80,37 @@ def test_build_packages(self): metadata = json.load(sourceforge_metadata) packages = mappers.sourceforge.build_packages_from_metafile(metadata) packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('sourceforge/mapper_odanur_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'sourceforge/mapper_odanur_expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_packages2(self): with open(self.get_test_loc('sourceforge/openstunts.json')) as sourceforge_metadata: metadata = json.load(sourceforge_metadata) packages = mappers.sourceforge.build_packages_from_metafile(metadata) packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('sourceforge/mapper_openstunts_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'sourceforge/mapper_openstunts_expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_packages3(self): with open(self.get_test_loc('sourceforge/monoql.json')) as sourceforge_metadata: metadata = json.load(sourceforge_metadata) packages = mappers.sourceforge.build_packages_from_metafile(metadata) packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('sourceforge/mapper_omonoql_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'sourceforge/mapper_omonoql_expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) def test_build_packages4(self): with open(self.get_test_loc('sourceforge/niftyphp.json')) as sourceforge_metadata: metadata = json.load(sourceforge_metadata) packages = mappers.sourceforge.build_packages_from_metafile(metadata) packages = [p.to_dict() for p in packages] - expected_loc = self.get_test_loc('sourceforge/mapper_niftyphp_expected.json') - self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN) + expected_loc = self.get_test_loc( + 'sourceforge/mapper_niftyphp_expected.json') + self.check_expected_results( + packages, expected_loc, regen=FIXTURES_REGEN) diff --git a/minecode/tests/test_tasks.py b/minecode/tests/test_tasks.py index bac8c0c3..c020d9ef 100644 --- a/minecode/tests/test_tasks.py +++ b/minecode/tests/test_tasks.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -54,7 +54,8 @@ def test_minecode_tasks_process_scan_results(self, mock_delete): self.assertFalse(self.package1.copyright) self.assertEqual(0, self.package1.resources.count()) scan_file_location = self.get_test_loc('scancodeio/get_scan_data.json') - summary_file_location = self.get_test_loc('scancodeio/scan_summary_response.json') + summary_file_location = self.get_test_loc( + 'scancodeio/scan_summary_response.json') tasks.process_scan_results( self.scannable_uri1.uuid, scan_results_location=scan_file_location, @@ -67,15 +68,18 @@ def test_minecode_tasks_process_scan_results(self, mock_delete): self.assertEqual('sha256', self.package1.sha256) self.assertEqual('sha512', self.package1.sha512) self.assertEqual(100, self.package1.size) - self.assertEqual('apache-2.0', self.package1.declared_license_expression) - self.assertEqual('Copyright (c) Apache Software Foundation', self.package1.copyright) + self.assertEqual( + 'apache-2.0', self.package1.declared_license_expression) + self.assertEqual( + 'Copyright (c) Apache Software Foundation', self.package1.copyright) self.assertFalse(self.scannable_uri1.scan_error) self.assertEqual(64, self.package1.resources.count()) def test_minecode_tasks_process_scan_results_scannableuri_does_not_exist(self): nonexisting_uuid = '420db78a-625f-4622-b1a0-93d1ea853194' scan_file_location = self.get_test_loc('scancodeio/get_scan_data.json') - summary_file_location = self.get_test_loc('scancodeio/scan_summary_response.json') + summary_file_location = self.get_test_loc( + 'scancodeio/scan_summary_response.json') project_extra_data = json.dumps(self.project_extra_data1) with self.assertRaises(Exception) as context: diff --git a/minecode/tests/test_utils.py b/minecode/tests/test_utils.py index 144e259e..4f23d11f 100644 --- a/minecode/tests/test_utils.py +++ b/minecode/tests/test_utils.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/tests/test_version.py b/minecode/tests/test_version.py index 3e66fef6..8f278e59 100644 --- a/minecode/tests/test_version.py +++ b/minecode/tests/test_version.py @@ -5,7 +5,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/minecode/utils.py b/minecode/utils.py index 7935bfa9..cbcd4d3e 100644 --- a/minecode/utils.py +++ b/minecode/utils.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -41,7 +41,8 @@ def stringify_null_purl_fields(data): Modify `data` in place by ensuring `purl` fields are not None. This is useful for cleaning data before saving to db. """ - purl_fields = ('type', 'namespace', 'name', 'version', 'qualifiers', 'subpath') + purl_fields = ('type', 'namespace', 'name', + 'version', 'qualifiers', 'subpath') for field in purl_fields: try: if not data[field]: @@ -73,6 +74,7 @@ class DataObject(object): A data object, using attributes for storage and a to_dict method to get a dict back. """ + def __init__(self, defaults=None, **kw): if defaults: for k, v in defaults.items(): @@ -226,12 +228,14 @@ def get_http_response(uri, timeout=10): requests_args['timeout'] = timeout if not uri.lower().startswith('http'): - raise Exception('get_http_response: Not an HTTP URI: %(uri)r' % locals()) + raise Exception( + 'get_http_response: Not an HTTP URI: %(uri)r' % locals()) try: response = requests.get(uri, **requests_args) except (ConnectionError, InvalidSchema) as e: - logger.error('get_http_response: Download failed for %(uri)r' % locals()) + logger.error( + 'get_http_response: Download failed for %(uri)r' % locals()) raise status = response.status_code @@ -271,7 +275,8 @@ def fetch_and_write_file_from_url(url): metadata_content = response.text filename = url.split("/")[-1] file_name, _, extension = filename.rpartition(".") - temp_metadata_file = get_temp_file(file_name=file_name, extension=extension) + temp_metadata_file = get_temp_file( + file_name=file_name, extension=extension) with open(temp_metadata_file, 'a') as metadata_file: metadata_file.write(metadata_content) @@ -381,7 +386,8 @@ def form_vcs_url(vcs_tool, vcs_url, revision_tag_or_branch=None, sub_path=None): if vcs_tool: vcs_url = '+'.join(str(v) for v in [vcs_tool, vcs_url]) if revision_tag_or_branch: - vcs_url = '@'.join(str(v) for v in [vcs_url, revision_tag_or_branch]) + vcs_url = '@'.join(str(v) + for v in [vcs_url, revision_tag_or_branch]) if sub_path: vcs_url = '#'.join(str(v) for v in [vcs_url, sub_path]) return vcs_url @@ -397,17 +403,18 @@ def validate_uuid(uuid_string): # This is from https://stackoverflow.com/questions/4856882/limiting-memory-use-in-a-large-django-queryset/5188179#5188179 class MemorySavingQuerysetIterator(object): - def __init__(self,queryset,max_obj_num=1000): + def __init__(self, queryset, max_obj_num=1000): self._base_queryset = queryset self._generator = self._setup() self.max_obj_num = max_obj_num def _setup(self): - for i in range(0,self._base_queryset.count(),self.max_obj_num): + for i in range(0, self._base_queryset.count(), self.max_obj_num): # By making a copy of of the queryset and using that to actually access # the objects we ensure that there are only `max_obj_num` objects in # memory at any given time - smaller_queryset = copy.deepcopy(self._base_queryset)[i:i+self.max_obj_num] + smaller_queryset = copy.deepcopy(self._base_queryset)[ + i:i+self.max_obj_num] logger.debug('Grabbing next %s objects from DB' % self.max_obj_num) for obj in smaller_queryset.iterator(): yield obj diff --git a/minecode/utils_test.py b/minecode/utils_test.py index 03c73ed7..ca2ea0bd 100644 --- a/minecode/utils_test.py +++ b/minecode/utils_test.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -222,7 +222,8 @@ def _normalize_results(self, data, fields_to_remove=[]): ): value = purl_with_fake_uuid(value) if key == "for_packages": - value = [purl_with_fake_uuid(package_uid) for package_uid in value] + value = [purl_with_fake_uuid(package_uid) + for package_uid in value] if key in fields_to_remove: continue normalized_data[key] = value @@ -287,7 +288,8 @@ def _normalize_results(self, data, fields_to_remove=[]): ): value = purl_with_fake_uuid(value) if key == "for_packages": - value = [purl_with_fake_uuid(package_uid) for package_uid in value] + value = [purl_with_fake_uuid(package_uid) + for package_uid in value] if key in fields_to_remove: continue normalized_data[key] = value @@ -345,7 +347,8 @@ def check_expected_uris(self, uris, expected_loc, data_is_json=False, regen=FIXT # FIXME: we should ONLY have strings there!!! uri_dict['date'] = str(uri_dict.get('date')) results.append(uri_dict) - self.check_expected_results(results=results, expected_loc=expected_loc, regen=regen) + self.check_expected_results( + results=results, expected_loc=expected_loc, regen=regen) def model_to_dict(instance, fields=None, exclude=None): diff --git a/minecode/version.py b/minecode/version.py index b33ab02d..220bc087 100644 --- a/minecode/version.py +++ b/minecode/version.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -78,7 +78,8 @@ def version_hint(path, ignore_pre_releases=False, remove_v_prefix=False): NON_VERSION_TAGS = ('win32', 'am64', 'x86_64', 'i386', 'i586', 'i586', 'x86', 'macosx',) -NON_VT_RES = [re.compile(re.escape(t), re.IGNORECASE) for t in NON_VERSION_TAGS] +NON_VT_RES = [re.compile(re.escape(t), re.IGNORECASE) + for t in NON_VERSION_TAGS] def strip_version_tags(path): diff --git a/minecode/visitors/__init__.py b/minecode/visitors/__init__.py index 587ded60..0120c82d 100644 --- a/minecode/visitors/__init__.py +++ b/minecode/visitors/__init__.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -189,6 +189,7 @@ class HttpVisitor(Visitor): Abstract base class for HTTP-based visitors. Subclasses must implement the get_uris() method and use a routing decorator for the URIs they can handle. """ + def fetch(self, uri, timeout=10): """ Fetch and return the content found at a remote uri. @@ -216,7 +217,8 @@ def fetch(self, uri, timeout=10): `timeout` is a default timeout. """ - content = super(NonPersistentHttpVisitor, self).fetch(uri, timeout=timeout) + content = super(NonPersistentHttpVisitor, + self).fetch(uri, timeout=timeout) temp_file = get_temp_file('NonPersistentHttpVisitor') with open(temp_file, 'wb') as tmp: tmp.write(content) diff --git a/minecode/visitors/conan.py b/minecode/visitors/conan.py index 5ae41987..35c1af94 100644 --- a/minecode/visitors/conan.py +++ b/minecode/visitors/conan.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -62,7 +62,8 @@ def get_conan_recipe(name, version): folder = recipe_location.get("folder") if not folder: - logger.error(f"No folder found for version {version} of package {name}") + logger.error( + f"No folder found for version {version} of package {name}") return None, None conanfile_py_url = f"{base_index_url}/{name}/{folder}/conanfile.py" diff --git a/minecode/visitors/debian.py b/minecode/visitors/debian.py index 7bb9323f..1e1ff956 100644 --- a/minecode/visitors/debian.py +++ b/minecode/visitors/debian.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -125,11 +125,13 @@ def get_uris(self, content): elif is_ubuntu_url(self.uri): namespace = 'ubuntu' else: - logger.error('Unknown Debian URI namespace: {}'.format(self.uri)) + logger.error( + 'Unknown Debian URI namespace: {}'.format(self.uri)) continue if file_name.endswith(('.deb', '.udeb', '.tar.gz', '.tar.xz', '.tar.bz2', '.tar.lzma')): - name, version, arch = debian_inspector.package.get_nva(file_name) + name, version, arch = debian_inspector.package.get_nva( + file_name) package_url = PackageURL( type='deb', namespace=namespace, @@ -399,7 +401,8 @@ def map_debian_package(debian_package, package_content, pipelines, priority=0): error += error_metadata return db_package, error - package_copyright, error_copyright = get_debian_package_copyright(debian_package) + package_copyright, error_copyright = get_debian_package_copyright( + debian_package) package.update_purl_fields(package_data=purl_package, replace=True) if package_copyright: update_license_copyright_fields( @@ -482,7 +485,8 @@ def get_debian_package_copyright(debian_package): logger.error(msg) return None, error - packages = StandaloneDebianCopyrightFileHandler.parse(location=temp_metadata_file) + packages = StandaloneDebianCopyrightFileHandler.parse( + location=temp_metadata_file) package = list(packages).pop() package.qualifiers = debian_package.package_url.qualifiers @@ -615,9 +619,9 @@ def binary_archive_url(self): purl_version = self.package_archive_version arch = self.package_url.qualifiers.get("arch") if arch: - archive_name =f"{self.package_url.name}_{purl_version}_{arch}.deb" + archive_name = f"{self.package_url.name}_{purl_version}_{arch}.deb" else: - archive_name =f"{self.package_url.name}_{purl_version}.deb" + archive_name = f"{self.package_url.name}_{purl_version}.deb" binary_package_url = self.archive_directory_url + f"{archive_name}" return binary_package_url @@ -641,9 +645,11 @@ def source_archive_url(self): for archive_format in debian_source_archive_formats: if ".orig" in archive_format: base_version_source = source_version.split('-')[0] - archive_name = f"{source_package_name}_{base_version_source}" + archive_format + archive_name = f"{source_package_name}_{base_version_source}" + \ + archive_format else: - archive_name = f"{source_package_name}_{source_version}" + archive_format + archive_name = f"{source_package_name}_{source_version}" + \ + archive_format source_package_url = self.archive_directory_url + archive_name response = requests.get(source_package_url) if response.ok: @@ -665,10 +671,12 @@ def package_metadata_url(self): metadata_version = self.source_package_url.version base_version_metadata = metadata_version.split('+')[0] - metadata_dsc_package_url = self.archive_directory_url + f"{metadata_package_name}_{base_version_metadata}.dsc" + metadata_dsc_package_url = self.archive_directory_url + \ + f"{metadata_package_name}_{base_version_metadata}.dsc" response = requests.get(metadata_dsc_package_url) if not response.ok: - metadata_dsc_package_url = self.archive_directory_url + f"{metadata_package_name}_{metadata_version}.dsc" + metadata_dsc_package_url = self.archive_directory_url + \ + f"{metadata_package_name}_{metadata_version}.dsc" return metadata_dsc_package_url @@ -692,11 +700,13 @@ def package_copyright_url(self): if self.source_package_url.version: metadata_version = self.source_package_url.version - copyright_package_url = self.metadata_directory_url + f"{metadata_package_name}_{metadata_version}{copyright_file_string}" + copyright_package_url = self.metadata_directory_url + \ + f"{metadata_package_name}_{metadata_version}{copyright_file_string}" response = requests.get(copyright_package_url) if not response.ok: base_version_metadata = metadata_version.split('+')[0] - copyright_package_url = self.metadata_directory_url + f"{metadata_package_name}_{base_version_metadata}{copyright_file_string}" + copyright_package_url = self.metadata_directory_url + \ + f"{metadata_package_name}_{base_version_metadata}{copyright_file_string}" return copyright_package_url diff --git a/minecode/visitors/fdroid.py b/minecode/visitors/fdroid.py index 4d171d8c..81384d80 100644 --- a/minecode/visitors/fdroid.py +++ b/minecode/visitors/fdroid.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -87,7 +87,8 @@ def get_uris(self, content): uri=purl, package_url=purl, source_uri=self.uri, - data=json.dumps(package_data, separators=(',', ':'), ensure_ascii=False), + data=json.dumps(package_data, separators=( + ',', ':'), ensure_ascii=False), # note: visited is True since there nothing more to visit visited=True ) diff --git a/minecode/visitors/freebsd.py b/minecode/visitors/freebsd.py index fe9e9edf..241db9ad 100644 --- a/minecode/visitors/freebsd.py +++ b/minecode/visitors/freebsd.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -56,6 +56,7 @@ class FreeBSDSubHTMLVisitors(HttpVisitor): """ Visit the sub repo URL and yield all uris in the page and in its children page """ + def get_uris(self, content): page = BeautifulSoup(content, 'lxml') base_url = self.uri + '{path}' @@ -73,6 +74,7 @@ class FreeBSDIndexVisitors(NonPersistentHttpVisitor): """ Extract packagesite.txz index file, get the data of packagesite.yaml file. """ + def dumps(self, content): """ Extract the file packagesite.yaml and read the content of the file and return. @@ -83,4 +85,5 @@ def dumps(self, content): with open(manifest_file) as file_handler: return file_handler.read() else: - logger.warn('The packagesite.yaml is not existing in index file:' + content) + logger.warn( + 'The packagesite.yaml is not existing in index file:' + content) diff --git a/minecode/visitors/generic.py b/minecode/visitors/generic.py index c6a327e3..79340af4 100644 --- a/minecode/visitors/generic.py +++ b/minecode/visitors/generic.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -193,7 +193,8 @@ def process_request_fetchcode_generic(purl_str, **kwargs): error = f"error occurred when parsing {purl_str}: {e}" return error - error_msg = map_fetchcode_supported_package(package_url, pipelines, priority) + error_msg = map_fetchcode_supported_package( + package_url, pipelines, priority) if error_msg: return error_msg diff --git a/minecode/visitors/gnu.py b/minecode/visitors/gnu.py index 0f269703..c6761ac6 100644 --- a/minecode/visitors/gnu.py +++ b/minecode/visitors/gnu.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -41,7 +41,8 @@ def process_request(purl_str, **kwargs): if not package_url.version: return - error_msg = map_fetchcode_supported_package(package_url, pipelines, priority) + error_msg = map_fetchcode_supported_package( + package_url, pipelines, priority) if error_msg: return error_msg diff --git a/minecode/visitors/maven.py b/minecode/visitors/maven.py index 1fabe5a0..d08ffb21 100644 --- a/minecode/visitors/maven.py +++ b/minecode/visitors/maven.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -283,7 +283,8 @@ def map_maven_package(package_url, package_content, pipelines, priority=0, reind base_url=base_url, ) ancestor_pom_texts = get_ancestry(pom_text=pom_text, base_url=base_url) - package = merge_ancestors(ancestor_pom_texts=ancestor_pom_texts, package=package) + package = merge_ancestors( + ancestor_pom_texts=ancestor_pom_texts, package=package) urls = get_urls( namespace=package_url.namespace, @@ -312,7 +313,8 @@ def map_maven_package(package_url, package_content, pipelines, priority=0, reind if sha1: package.sha1 = sha1 override = reindex_metadata - db_package, _, _, _ = merge_or_create_package(package, visit_level=50, override=override) + db_package, _, _, _ = merge_or_create_package( + package, visit_level=50, override=override) else: msg = f'Failed to retrieve JAR: {package_url}' error += msg + '\n' @@ -451,7 +453,7 @@ def process_request(purl_str, **kwargs): has_version = bool(package_url.version) if has_version: - reindex_metadata=kwargs.get("reindex_metadata", False) + reindex_metadata = kwargs.get("reindex_metadata", False) error = map_maven_binary_and_source( package_url, pipelines, @@ -629,7 +631,8 @@ def add_to_import_queue(url, root_url): response = requests.get(url) if response: data = response.text - namespace, name, _ = determine_namespace_name_version_from_url(url, root_url) + namespace, name, _ = determine_namespace_name_version_from_url( + url, root_url) purl = PackageURL( type='maven', namespace=namespace, @@ -852,7 +855,8 @@ def get_uris(self, content): """ index_location = content - artifacts = get_artifacts(index_location, worthyness=is_worthy_artifact) + artifacts = get_artifacts( + index_location, worthyness=is_worthy_artifact) for artifact in artifacts: # we cannot do much without these @@ -957,7 +961,8 @@ def get_uris(self, content): url = a.get('href') if not url: continue - if url.startswith(':'): # Remove : symbol since it's a special char for bintray repo. + # Remove : symbol since it's a special char for bintray repo. + if url.startswith(':'): url = url[1:] filename = None # default is folder, the filename is None. if not url.endswith('/'): @@ -1247,7 +1252,8 @@ def to_dict(self): Artifact = namedtuple('Artifact', _artifact_base_fields) Artifact.to_dict = to_dict -ArtifactExtended = namedtuple('ArtifactExtended', _artifact_base_fields + _artifact_extended_fields) +ArtifactExtended = namedtuple( + 'ArtifactExtended', _artifact_base_fields + _artifact_extended_fields) ArtifactExtended.to_dict = to_dict @@ -1404,8 +1410,10 @@ def get_entries(location, fields=frozenset(ENTRY_FIELDS)): except EOFError: if TRACE_DEEP: - print('Index version: %(_index_version)r last_modified: %(_last_modified)r' % locals()) - print('Processed %(entries_count)d docs. Last entry: %(entry)r' % locals()) + print( + 'Index version: %(_index_version)r last_modified: %(_last_modified)r' % locals()) + print( + 'Processed %(entries_count)d docs. Last entry: %(entry)r' % locals()) print('Unique keys:') for k in sorted(keys): print(k) diff --git a/minecode/visitors/npm.py b/minecode/visitors/npm.py index bde949df..118a10aa 100644 --- a/minecode/visitors/npm.py +++ b/minecode/visitors/npm.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -47,6 +47,7 @@ class NpmRegistryVisitor(NonPersistentHttpVisitor): each NPM package (that contains all the versions for this package) as previsited for mapping. """ + def get_uris(self, content): """ Yield a URI for the next index sequence to visit and one URI for each @@ -63,7 +64,8 @@ def get_uris(self, content): last_seq = content['last_seq'] except KeyError: # provide a more meaningful message in case the JSON is incorrect - raise Exception('NpmRegistryVisitor: Missing "last_seq" field: Aborting.') + raise Exception( + 'NpmRegistryVisitor: Missing "last_seq" field: Aborting.') # Always yield an index URI, even if there is no results to avoid stopping the index visits yield URI(uri=next_visitable_index_url_template.format(last_seq=last_seq), source_uri=self.uri) @@ -102,7 +104,8 @@ def get_uris(self, content): uri=package_api_url, package_url=package_url, source_uri=self.uri, - data=json.dumps(doc, separators=(',', ':'), ensure_ascii=False), + data=json.dumps(doc, separators=( + ',', ':'), ensure_ascii=False), # note: visited is True since there nothing more to visit visited=True) diff --git a/minecode/visitors/pypi.py b/minecode/visitors/pypi.py index f83b9c1d..ba9425a6 100644 --- a/minecode/visitors/pypi.py +++ b/minecode/visitors/pypi.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -50,6 +50,7 @@ class PypiIndexVisitor(Visitor): """ Collect package metadata URIs from the top level pypi index for each package. """ + def fetch(self, uri, timeout=None): """ Specialized fetching using XML RPCs. @@ -85,6 +86,7 @@ class PypiPackageVisitor(HttpJsonVisitor): The url will contain only the package name, for example: https://pypi.org/pypi/vmock/json By parsing the content, the goal is to form the json with version/release: https://pypi.org/pypi/vmock/0.1/json """ + def get_uris(self, content): url_template = 'https://pypi.python.org/pypi/{name}/{release}/json' @@ -92,7 +94,8 @@ def get_uris(self, content): name = info.get('name') if name: for release in content['releases']: - package_url = PackageURL(type='pypi', name=name, version=release).to_string() + package_url = PackageURL( + type='pypi', name=name, version=release).to_string() yield URI(uri=url_template.format(name=name, release=release), package_url=package_url, source_uri=self.uri) @@ -102,6 +105,7 @@ class PypiPackageReleaseVisitor(HttpJsonVisitor): Collect package download URIs for all packages archives of one Pypi package release. The example is: https://pypi.org/pypi/vmock/0.1/json """ + def get_uris(self, content): # TODO: this is likely best ignored entirely??? # A download_url may be provided for an off-Pypi-download @@ -111,7 +115,8 @@ def get_uris(self, content): download_url = info.get('download_url') if download_url and download_url != 'UNKNOWN': version = info.get('version') - package_url = PackageURL(type='pypi', name=name, version=version).to_string() + package_url = PackageURL( + type='pypi', name=name, version=version).to_string() yield URI(uri=download_url, package_url=package_url, source_uri=self.uri) # Common on-Pypi-download URLs are in the urls block @@ -119,7 +124,8 @@ def get_uris(self, content): url = download.get('url') if not url: continue - package_url = PackageURL(type='pypi', name=name, version=version).to_string() + package_url = PackageURL( + type='pypi', name=name, version=version).to_string() yield URI(url, package_url=package_url, file_name=download.get('filename'), size=download.get('size'), date=download.get('upload_time'), md5=download.get('md5_digest'), source_uri=self.uri) diff --git a/minecode/visitors/rubygems.py b/minecode/visitors/rubygems.py index 999d21ea..2bac0c10 100644 --- a/minecode/visitors/rubygems.py +++ b/minecode/visitors/rubygems.py @@ -4,7 +4,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -81,7 +81,8 @@ def get_uris(self, content): download_url += '.gem' download_url = download_url.format(**locals()) - package_url = PackageURL(type='gem', name=name, version=version).to_string() + package_url = PackageURL( + type='gem', name=name, version=version).to_string() yield URI(uri=download_url, package_url=package_url, source_uri=self.uri) @@ -104,8 +105,10 @@ def get_uris(self, content): self.uri.index('/versions/') + len('/versions/'):-len('.json')] version = version_details.get('number') gem_name = '%(name)s-%(version)s' % locals() - package_url = PackageURL(type='gem', name=name, version=version).to_string() - download_url = 'https://rubygems.org/downloads/%(gem_name)s.gem' % locals() + package_url = PackageURL( + type='gem', name=name, version=version).to_string() + download_url = 'https://rubygems.org/downloads/%(gem_name)s.gem' % locals( + ) yield URI(uri=download_url, source_uri=self.uri, package_url=package_url, data=json.dumps(version_details)) @@ -136,6 +139,7 @@ def get_gem_metadata(location): # Extract the embedded metadata gz file extract_parent_location = extract_file(metadata_gz) # Get the first file in the etracted folder which is the meta file location - meta_extracted_file = os.path.join(extract_parent_location, os.listdir(extract_parent_location)[0]) + meta_extracted_file = os.path.join( + extract_parent_location, os.listdir(extract_parent_location)[0]) with open(meta_extracted_file) as meta_file: return meta_file.read() diff --git a/minecode/visitors/sourceforge.py b/minecode/visitors/sourceforge.py index 8ab0d306..7b2d7a7a 100644 --- a/minecode/visitors/sourceforge.py +++ b/minecode/visitors/sourceforge.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -73,7 +73,8 @@ def get_uris(self, content): project_name = loc.text.partition( 'https://sourceforge.net/projects/')[-1].strip('/') project_json_url = project_json_baseurl.format(project_name) - package_url = PackageURL(type='sourceforge', name=project_name).to_string() + package_url = PackageURL( + type='sourceforge', name=project_name).to_string() # The priority in the xml has different view with the priority in visitor, so skip it. yield URI(uri=project_json_url, package_url=package_url, source_uri=self.uri) diff --git a/packagedb/__init__.py b/packagedb/__init__.py index 2eb8f9f0..e1521118 100644 --- a/packagedb/__init__.py +++ b/packagedb/__init__.py @@ -3,6 +3,6 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/packagedb/api.py b/packagedb/api.py index 305e1539..f0e128ea 100644 --- a/packagedb/api.py +++ b/packagedb/api.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -268,7 +268,8 @@ def filter_by_checksums(self, request, *args, **kwargs): qs = Resource.objects.filter(lookups) paginated_qs = self.paginate_queryset(qs) - serializer = ResourceAPISerializer(paginated_qs, many=True, context={'request': request}) + serializer = ResourceAPISerializer( + paginated_qs, many=True, context={'request': request}) return self.get_paginated_response(serializer.data) @@ -332,16 +333,16 @@ class PackageFilterSet(FilterSet): ) sort = OrderingFilter(fields=[ - 'type', - 'namespace', - 'name', - 'version', - 'qualifiers', - 'subpath', - 'download_url', - 'filename', - 'size', - 'release_date' + 'type', + 'namespace', + 'name', + 'version', + 'qualifiers', + 'subpath', + 'download_url', + 'filename', + 'size', + 'release_date' ]) class Meta: @@ -382,7 +383,8 @@ def latest_version(self, request, *args, **kwargs): latest_version = package.get_latest_version() if latest_version: return Response( - PackageAPISerializer(latest_version, context={'request': request}).data + PackageAPISerializer(latest_version, context={ + 'request': request}).data ) return Response({}) @@ -405,7 +407,8 @@ def resources(self, request, *args, **kwargs): qs = Resource.objects.filter(package=package) paginated_qs = self.paginate_queryset(qs) - serializer = ResourceAPISerializer(paginated_qs, many=True, context={'request': request}) + serializer = ResourceAPISerializer( + paginated_qs, many=True, context={'request': request}) return self.get_paginated_response(serializer.data) @action(detail=True) @@ -457,7 +460,8 @@ def filter_by_checksums(self, request, *args, **kwargs): data = dict(request.data) unsupported_fields = [] - supported_fields = ['md5', 'sha1', 'sha256', 'sha512', 'enhance_package_data'] + supported_fields = ['md5', 'sha1', 'sha256', + 'sha512', 'enhance_package_data'] for field, value in data.items(): if field not in supported_fields: unsupported_fields.append(field) @@ -495,9 +499,11 @@ def filter_by_checksums(self, request, *args, **kwargs): qs = Package.objects.filter(lookups) paginated_qs = self.paginate_queryset(qs) if enhance_package_data: - serialized_package_data = [get_enhanced_package(package=package) for package in paginated_qs] + serialized_package_data = [get_enhanced_package( + package=package) for package in paginated_qs] else: - serializer = PackageAPISerializer(paginated_qs, many=True, context={'request': request}) + serializer = PackageAPISerializer( + paginated_qs, many=True, context={'request': request}) serialized_package_data = serializer.data return self.get_paginated_response(serialized_package_data) @@ -506,7 +512,6 @@ class PackageViewSet(PackagePublicViewSet): @action(detail=True) def reindex_package(self, request, *args, **kwargs): - """ Reindex this package instance """ @@ -693,13 +698,15 @@ def _get_enhanced_package(package, packages): package_data = package.to_dict() # always default to PackageContentType.BINARY as we can have None/NULL in the model for now - # Reference: https://github.com/nexB/purldb/issues/490 - package_content = (package and package.package_content) or PackageContentType.BINARY + # Reference: https://github.com/aboutcode-org/purldb/issues/490 + package_content = ( + package and package.package_content) or PackageContentType.BINARY for peer in packages: # always default to PackageContentType.BINARY as we can have None/NULL in the model for now - # Reference: https://github.com/nexB/purldb/issues/490 - peer_content = (peer and peer.package_content) or PackageContentType.BINARY + # Reference: https://github.com/aboutcode-org/purldb/issues/490 + peer_content = ( + peer and peer.package_content) or PackageContentType.BINARY if peer_content >= package_content: # We do not want to mix data with peers of the same package content @@ -713,7 +720,8 @@ def _get_enhanced_package(package, packages): if field == 'parties': peer_value = PartySerializer(peer_value, many=True).data if field == 'dependencies': - peer_value = DependentPackageSerializer(peer_value, many=True).data + peer_value = DependentPackageSerializer( + peer_value, many=True).data package_data[field] = peer_value enhanced = True if enhanced: @@ -793,18 +801,20 @@ class CollectViewSet(viewsets.ViewSet): serializer_class = CollectPackageSerializer @extend_schema( - parameters=[ - OpenApiParameter('purl', str, 'query', description='PackageURL', required=True), - OpenApiParameter('source_purl', str, 'query', description='Source PackageURL'), - - # There is no OpenApiTypes.LIST https://github.com/tfranzel/drf-spectacular/issues/341 - OpenApiParameter( - 'addon_pipelines', - build_array_type(build_basic_type(OpenApiTypes.STR)), - 'query', description='Addon pipelines', - ), - ], - responses={200:PackageAPISerializer()}, + parameters=[ + OpenApiParameter('purl', str, 'query', + description='PackageURL', required=True), + OpenApiParameter('source_purl', str, 'query', + description='Source PackageURL'), + + # There is no OpenApiTypes.LIST https://github.com/tfranzel/drf-spectacular/issues/341 + OpenApiParameter( + 'addon_pipelines', + build_array_type(build_basic_type(OpenApiTypes.STR)), + 'query', description='Addon pipelines', + ), + ], + responses={200: PackageAPISerializer()}, ) def list(self, request, format=None): serializer = self.serializer_class(data=request.query_params) @@ -851,7 +861,8 @@ def list(self, request, format=None): for package in packages: get_source_package_and_add_to_package_set(package) - serializer = PackageAPISerializer(packages, many=True, context={'request': request}) + serializer = PackageAPISerializer( + packages, many=True, context={'request': request}) return Response(serializer.data) @extend_schema( @@ -974,16 +985,19 @@ def _reindex_package(package, reindexed_packages, **kwargs): reindexed_packages = [] requeued_packages = [] - supported_ecosystems = ['maven', 'npm', 'deb', 'generic', 'gnu', 'openssl', 'github', 'conan'] + supported_ecosystems = ['maven', 'npm', 'deb', + 'generic', 'gnu', 'openssl', 'github', 'conan'] - unique_packages, unsupported_packages, unsupported_vers = get_resolved_packages(packages, supported_ecosystems) + unique_packages, unsupported_packages, unsupported_vers = get_resolved_packages( + packages, supported_ecosystems) if reindex: for package in unique_packages: purl = package['purl'] kwargs = dict() if addon_pipelines := package.get('addon_pipelines'): - kwargs["addon_pipelines"] = [pipe for pipe in addon_pipelines if is_supported_addon_pipeline(pipe)] + kwargs["addon_pipelines"] = [ + pipe for pipe in addon_pipelines if is_supported_addon_pipeline(pipe)] lookups = purl_to_lookups(purl) packages = Package.objects.filter(**lookups) if packages.count() > 0: @@ -993,10 +1007,12 @@ def _reindex_package(package, reindexed_packages, **kwargs): if reindex_set: for package_set in package.package_sets.all(): for p in package_set.packages.all(): - _reindex_package(p, reindexed_packages, **kwargs) + _reindex_package( + p, reindexed_packages, **kwargs) else: nonexistent_packages.append(package) - requeued_packages.extend([p.package_url for p in reindexed_packages]) + requeued_packages.extend( + [p.package_url for p in reindexed_packages]) if not reindex or nonexistent_packages: interesting_packages = nonexistent_packages if nonexistent_packages else unique_packages @@ -1011,10 +1027,12 @@ def _reindex_package(package, reindexed_packages, **kwargs): if source_purl := package.get('source_purl'): extra_fields['source_uri'] = source_purl if addon_pipelines := package.get('addon_pipelines'): - extra_fields['addon_pipelines'] = [pipe for pipe in addon_pipelines if is_supported_addon_pipeline(pipe)] + extra_fields['addon_pipelines'] = [ + pipe for pipe in addon_pipelines if is_supported_addon_pipeline(pipe)] if priority := package.get('priority'): extra_fields['priority'] = priority - priority_resource_uri = PriorityResourceURI.objects.insert(purl, **extra_fields) + priority_resource_uri = PriorityResourceURI.objects.insert( + purl, **extra_fields) if priority_resource_uri: queued_packages.append(purl) else: @@ -1033,14 +1051,16 @@ def _reindex_package(package, reindexed_packages, **kwargs): 'unsupported_vers': unsupported_vers, } - serializer = IndexPackagesResponseSerializer(response_data, context={'request': request}) + serializer = IndexPackagesResponseSerializer( + response_data, context={'request': request}) return Response(serializer.data) @extend_schema( parameters=[ - OpenApiParameter('purl', str, 'query', description='PackageURL', required=True), + OpenApiParameter('purl', str, 'query', + description='PackageURL', required=True), ], - responses={200:PackageAPISerializer()}, + responses={200: PackageAPISerializer()}, ) @action(detail=False, methods=['get'], serializer_class=CollectPackageSerializer) def reindex_metadata(self, request, *args, **kwargs): @@ -1099,11 +1119,11 @@ def reindex_metadata(self, request, *args, **kwargs): } return Response(message, status=status.HTTP_400_BAD_REQUEST) - serializer = PackageAPISerializer(packages, many=True, context={'request': request}) + serializer = PackageAPISerializer( + packages, many=True, context={'request': request}) return Response(serializer.data) - class PurlValidateViewSet(viewsets.ViewSet): """ Take a `purl` and check whether it's valid PackageURL or not. @@ -1132,7 +1152,8 @@ def get_view_name(self): @extend_schema( parameters=[ OpenApiParameter('purl', str, 'query', description='PackageURL'), - OpenApiParameter('check_existence', bool, 'query', description='Check existence', default=False), + OpenApiParameter('check_existence', bool, 'query', + description='Check existence', default=False), ], responses={200: PurlValidateResponseSerializer()}, ) @@ -1168,7 +1189,8 @@ def list(self, request): try: package_url = PackageURL.from_string(purl) except ValueError: - serializer = PurlValidateResponseSerializer(response, context={'request': request}) + serializer = PurlValidateResponseSerializer( + response, context={'request': request}) return Response(serializer.data, status=status.HTTP_400_BAD_REQUEST) response['valid'] = True @@ -1208,7 +1230,8 @@ def list(self, request): else: response["message"] = message_valid_but_does_not_exist - serializer = PurlValidateResponseSerializer(response, context={'request': request}) + serializer = PurlValidateResponseSerializer( + response, context={'request': request}) return Response(serializer.data) @@ -1312,7 +1335,8 @@ def resolve_versions(parsed_purl, vers): ) result.append(str(package_url)) except InvalidConstraintsError: - logger.warning(f"Invalid constraints sequence in '{vers}' for '{parsed_purl}'") + logger.warning( + f"Invalid constraints sequence in '{vers}' for '{parsed_purl}'") return return result @@ -1335,7 +1359,7 @@ def get_all_versions_plain(purl: PackageURL): return all_versions = versionAPI().fetch(package_name) or [] - return [ version.value for version in all_versions ] + return [version.value for version in all_versions] def get_all_versions(purl): @@ -1357,4 +1381,5 @@ def get_all_versions(purl): return result -VERSION_CLASS_BY_PACKAGE_TYPE = {pkg_type: range_class.version_class for pkg_type, range_class in RANGE_CLASS_BY_SCHEMES.items()} +VERSION_CLASS_BY_PACKAGE_TYPE = { + pkg_type: range_class.version_class for pkg_type, range_class in RANGE_CLASS_BY_SCHEMES.items()} diff --git a/packagedb/api_custom.py b/packagedb/api_custom.py index 37b3a18f..e793fe8a 100644 --- a/packagedb/api_custom.py +++ b/packagedb/api_custom.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/packagedb/filters.py b/packagedb/filters.py index 9c38a0f9..2ea6f168 100644 --- a/packagedb/filters.py +++ b/packagedb/filters.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -49,7 +49,8 @@ def parse_query_string_to_lookups(query_string, default_lookup_expr, default_fie search_value = term field_name = default_field - lookups &= Q(**{f"{field_name}__{lookup_expr}": search_value}, _negated=negated) + lookups &= Q( + **{f"{field_name}__{lookup_expr}": search_value}, _negated=negated) return lookups @@ -84,7 +85,8 @@ def filter(self, qs, value): if "://" not in value and ":" in value: return super().filter(qs, value) - search_fields = ["type", "namespace", "name", "version", "download_url"] + search_fields = ["type", "namespace", + "name", "version", "download_url"] lookups = Q() for field_names in search_fields: lookups |= Q(**{f"{field_names}__{self.lookup_expr}": value}) diff --git a/packagedb/from_purl.py b/packagedb/from_purl.py index 0706ac8a..b6102a9c 100644 --- a/packagedb/from_purl.py +++ b/packagedb/from_purl.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -22,7 +22,8 @@ @extend_schema( parameters=[ - OpenApiParameter("package_url", str, "query", description="package url"), + OpenApiParameter("package_url", str, "query", + description="package url"), ], responses={200: PurltoGitRepoResponseSerializer()}, ) diff --git a/packagedb/management/commands/create_source_repo_packages.py b/packagedb/management/commands/create_source_repo_packages.py index ba15c4bd..1d7d461c 100644 --- a/packagedb/management/commands/create_source_repo_packages.py +++ b/packagedb/management/commands/create_source_repo_packages.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/packagedb/management/commands/fix_purl_values.py b/packagedb/management/commands/fix_purl_values.py index 440a1ea9..61c6587b 100644 --- a/packagedb/management/commands/fix_purl_values.py +++ b/packagedb/management/commands/fix_purl_values.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -44,7 +44,8 @@ def get_timestamps_by_links(package_version_page_url): timestamps_by_links = {} response = requests.get(package_version_page_url) if response: - timestamps_by_links = collect_links_from_text(response.text, filter=filter_for_artifacts) + timestamps_by_links = collect_links_from_text( + response.text, filter=filter_for_artifacts) timestamps_by_links = { link: dateutil_parse(timestamp) for link, timestamp in timestamps_by_links.items() } @@ -83,7 +84,8 @@ def __init__(self, namespace, name, version, qualifiers='', ec=[]): extension=qualifiers_mapping.get('type') or 'jar', classifier=qualifiers_mapping.get('classifier'), ) - timestamps_by_links = get_timestamps_by_links(self.repository_homepage_url) + timestamps_by_links = get_timestamps_by_links( + self.repository_homepage_url) self.release_date = timestamps_by_links.get(filename) self.related_artifacts = list( self._populate_related_artifacts( @@ -148,9 +150,11 @@ class Command(VerboseCommand): help = 'Update maven Package download_url values' def handle(self, *args, **options): - maven_packages = Package.objects.filter(type='maven', sha1__is_null=False) + maven_packages = Package.objects.filter( + type='maven', sha1__is_null=False) maven_packages_count = maven_packages.count() - logger.info(f'Checking {maven_packages_count:,} Maven Package PackageURL values') + logger.info( + f'Checking {maven_packages_count:,} Maven Package PackageURL values') packages_to_delete = [] for package in MemorySavingQuerysetIterator(maven_packages): diff --git a/packagedb/management/commands/get_source_download_url.py b/packagedb/management/commands/get_source_download_url.py index d13b9a84..87c72365 100644 --- a/packagedb/management/commands/get_source_download_url.py +++ b/packagedb/management/commands/get_source_download_url.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/packagedb/management/commands/run_scheduler.py b/packagedb/management/commands/run_scheduler.py index abc0850f..b70f6e94 100644 --- a/packagedb/management/commands/run_scheduler.py +++ b/packagedb/management/commands/run_scheduler.py @@ -3,7 +3,7 @@ # PurlDB is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/packagedb/management/commands/watch_packages.py b/packagedb/management/commands/watch_packages.py index c6f3d01d..f6297f4e 100644 --- a/packagedb/management/commands/watch_packages.py +++ b/packagedb/management/commands/watch_packages.py @@ -3,7 +3,7 @@ # PurlDB is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -36,7 +36,8 @@ def handle(self, *args, **options): purl_value = options.get("purl") packages_qs = ( - Package.objects.filter(type__in=PRIORITY_QUEUE_SUPPORTED_ECOSYSTEMS) + Package.objects.filter( + type__in=PRIORITY_QUEUE_SUPPORTED_ECOSYSTEMS) .filter(type__in=SUPPORTED_ECOSYSTEMS) .distinct("type", "namespace", "name") .order_by("type", "namespace", "name") diff --git a/packagedb/models.py b/packagedb/models.py index dd0ad36a..4ed64760 100644 --- a/packagedb/models.py +++ b/packagedb/models.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -56,7 +56,8 @@ def insert(self, download_url, **extra_fields): Create and return a new Package. Return None if the insertion failed when an identical entry already exist. """ - package, created = self.get_or_create(download_url=download_url, defaults=extra_fields) + package, created = self.get_or_create( + download_url=download_url, defaults=extra_fields) if created: return package @@ -353,7 +354,8 @@ class AbstractPackage(models.Model): copyright = models.TextField( blank=True, null=True, - help_text=_("Copyright statements for this package. Typically one per line."), + help_text=_( + "Copyright statements for this package. Typically one per line."), ) holder = models.TextField( blank=True, @@ -527,13 +529,15 @@ class Package( package_sets = models.ManyToManyField( 'PackageSet', related_name='packages', - help_text=_('A set representing the Package sets this Package is a member of.'), + help_text=_( + 'A set representing the Package sets this Package is a member of.'), ) package_content = models.IntegerField( null=True, choices=PackageContentType.choices, help_text=_( - 'Content of this Package as one of: {}'.format(', '.join(PackageContentType.labels)) + 'Content of this Package as one of: {}'.format( + ', '.join(PackageContentType.labels)) ), ) summary = models.JSONField( @@ -632,7 +636,8 @@ def reindex(self, **kwargs): addon_pipelines = kwargs.get('addon_pipelines', []) pipelines = DEFAULT_PIPELINES + tuple(addon_pipelines) - add_package_to_scan_queue(self, pipelines=pipelines, reindex_uri=True, priority=100) + add_package_to_scan_queue( + self, pipelines=pipelines, reindex_uri=True, priority=100) def update_fields(self, save=False, **values_by_fields): """ @@ -656,7 +661,8 @@ def update_fields(self, save=False, **values_by_fields): if not hasattr(self, field): # Raise exception when we we are given a keyword argument that # doesn't correspond to a Package field - raise AttributeError(f"'{class_name}' has no attribute '{field}'") + raise AttributeError( + f"'{class_name}' has no attribute '{field}'") related_model_fields = [ 'dependencies', @@ -671,7 +677,8 @@ def update_fields(self, save=False, **values_by_fields): dep = DependentPackage( package=self, purl=dep_data.get('purl'), - extracted_requirement=dep_data.get('extracted_requirement'), + extracted_requirement=dep_data.get( + 'extracted_requirement'), scope=dep_data.get('scope'), is_runtime=dep_data.get('is_runtime'), is_optional=dep_data.get('is_optional'), @@ -719,7 +726,8 @@ def update_fields(self, save=False, **values_by_fields): sha256=resource_data.get('sha256'), mime_type=resource_data.get('mime_type'), file_type=resource_data.get('file_type'), - programming_language=resource_data.get('programming_language'), + programming_language=resource_data.get( + 'programming_language'), is_binary=resource_data.get('is_binary'), is_text=resource_data.get('is_text'), is_archive=resource_data.get('is_archive'), @@ -742,7 +750,8 @@ def update_fields(self, save=False, **values_by_fields): model_count = self.dependencies.all().count() with transaction.atomic(): self.dependencies.all().delete() - DependentPackage.objects.bulk_create(unsaved_models) + DependentPackage.objects.bulk_create( + unsaved_models) if field == 'parties': model_count = self.parties.all().count() @@ -901,7 +910,8 @@ class DependentPackage(models.Model): max_length=200, blank=True, null=True, - help_text=_('A string defining version(s)requirements. Package-type specific.') + help_text=_( + 'A string defining version(s)requirements. Package-type specific.') ) scope = models.CharField( @@ -942,13 +952,15 @@ class AbstractResource(models.Model): path = models.CharField( max_length=2000, - help_text=_('The full path value of a resource (file or directory) in the archive it is from.'), + help_text=_( + 'The full path value of a resource (file or directory) in the archive it is from.'), ) name = models.CharField( max_length=255, blank=True, - help_text=_("File or directory name of this resource with its extension."), + help_text=_( + "File or directory name of this resource with its extension."), ) extension = models.CharField( @@ -986,7 +998,8 @@ class AbstractResource(models.Model): max_length=50, blank=True, null=True, - help_text=_("Programming language of this resource if this is a code file."), + help_text=_( + "Programming language of this resource if this is a code file."), ) is_binary = models.BooleanField(default=False) @@ -997,7 +1010,8 @@ class AbstractResource(models.Model): is_file = models.BooleanField( default=False, - help_text=_('True if this Resource is a file, False if it is a Directory') + help_text=_( + 'True if this Resource is a file, False if it is a Directory') ) @property @@ -1065,7 +1079,8 @@ class ScanFieldsModelMixin(models.Model): authors = models.JSONField( blank=True, default=list, - help_text=_("List of detected authors (and related detection details)."), + help_text=_( + "List of detected authors (and related detection details)."), ) package_data = models.JSONField( default=list, @@ -1286,13 +1301,15 @@ class PackageWatch(models.Model): depth = models.PositiveSmallIntegerField( choices=DEPTH_CHOICES, default=3, - help_text=_("Depth of data collection from listing versions up to a full scan."), + help_text=_( + "Depth of data collection from listing versions up to a full scan."), ) watch_interval = models.PositiveSmallIntegerField( validators=[ MinValueValidator(1, message="Interval must be at least 1 day."), - MaxValueValidator(365, message="Interval must be at most 365 days."), + MaxValueValidator( + 365, message="Interval must be at most 365 days."), ], default=7, help_text=_("Number of days to wait between watches of this package."), @@ -1460,7 +1477,8 @@ def _validate_username(self, email): except models.ObjectDoesNotExist: pass else: - raise exceptions.ValidationError(f"Error: This email already exists: {email}") + raise exceptions.ValidationError( + f"Error: This email already exists: {email}") @receiver(models.signals.post_save, sender=settings.AUTH_USER_MODEL) diff --git a/packagedb/schedules.py b/packagedb/schedules.py index 3ba28815..65b44eb8 100644 --- a/packagedb/schedules.py +++ b/packagedb/schedules.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -25,7 +25,8 @@ def get_next_execution(watch_interval_days, last_watch_date): """ current_date_time = datetime.datetime.now(tz=datetime.timezone.utc) if last_watch_date: - next_execution = last_watch_date + datetime.timedelta(days=watch_interval_days) + next_execution = last_watch_date + \ + datetime.timedelta(days=watch_interval_days) if next_execution > current_date_time: return next_execution @@ -40,7 +41,7 @@ def schedule_watch(watch): watch_interval = watch.watch_interval last_watch_date = watch.last_watch_date - first_execution = get_next_execution(watch_interval,last_watch_date) + first_execution = get_next_execution(watch_interval, last_watch_date) interval_in_seconds = watch_interval * 24 * 60 * 60 job = scheduler.schedule( @@ -75,7 +76,7 @@ def clear_zombie_watch_schedules(logger=log): """ from packagedb.models import PackageWatch schedule_ids = PackageWatch.objects.all().values_list("schedule_work_id", flat=True) - + for job in scheduler.get_jobs(): if job._id not in schedule_ids: logger.info(f"Deleting scheduled job {job}") diff --git a/packagedb/serializers.py b/packagedb/serializers.py index 1d5cf6f3..90b6322c 100644 --- a/packagedb/serializers.py +++ b/packagedb/serializers.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -31,7 +31,8 @@ class ResourceAPISerializer(HyperlinkedModelSerializer): - package = HyperlinkedRelatedField(view_name='api:package-detail', lookup_field='uuid', read_only=True) + package = HyperlinkedRelatedField( + view_name='api:package-detail', lookup_field='uuid', read_only=True) purl = CharField(source='package.package_url') class Meta: @@ -150,13 +151,15 @@ class Meta: ) def to_representation(self, instance): - reverse_uri = reverse_lazy('api:package-detail', kwargs={'uuid': instance.uuid}) + reverse_uri = reverse_lazy( + 'api:package-detail', kwargs={'uuid': instance.uuid}) request = self.context['request'] return request.build_absolute_uri(reverse_uri) class PackageSetAPISerializer(ModelSerializer): packages = PackageInPackageSetAPISerializer(many=True) + class Meta: model = PackageSet fields = ( @@ -168,9 +171,12 @@ class Meta: class PackageAPISerializer(HyperlinkedModelSerializer): dependencies = DependentPackageSerializer(many=True) parties = PartySerializer(many=True) - resources = HyperlinkedIdentityField(view_name='api:package-resources', lookup_field='uuid') - history = HyperlinkedIdentityField(view_name='api:package-history', lookup_field='uuid') - url = HyperlinkedIdentityField(view_name='api:package-detail', lookup_field='uuid') + resources = HyperlinkedIdentityField( + view_name='api:package-resources', lookup_field='uuid') + history = HyperlinkedIdentityField( + view_name='api:package-history', lookup_field='uuid') + url = HyperlinkedIdentityField( + view_name='api:package-detail', lookup_field='uuid') package_sets = PackageSetAPISerializer(many=True) package_content = SerializerMethodField() declared_license_expression_spdx = CharField() @@ -251,6 +257,7 @@ def to_representation(self, instance): class PackageSetMetadataSerializer(ModelSerializer): packages = PackageInPackageSetMetadataSerializer(many=True) + class Meta: model = PackageSet fields = ( @@ -342,6 +349,7 @@ class PackageWatchAPISerializer(HyperlinkedModelSerializer): view_name='api:packagewatch-detail', lookup_field='package_url' ) + class Meta: model = PackageWatch fields = [ @@ -362,7 +370,8 @@ class Meta: model = PackageWatch fields = ["package_url", "depth", "watch_interval", "is_active"] extra_kwargs = { - field: {"initial": PackageWatch._meta.get_field(field).get_default()} + field: {"initial": PackageWatch._meta.get_field( + field).get_default()} for field in ["depth", "watch_interval", "is_active"] } @@ -376,24 +385,24 @@ class Meta: class CollectPackageSerializer(Serializer): purl = CharField(help_text="PackageURL strings in canonical form.") source_purl = CharField( - required=False, + required=False, help_text="Source PackageURL.", - ) + ) addon_pipelines = ListField( - child = CharField(), + child=CharField(), required=False, allow_empty=True, help_text="Addon pipelines to run on the package.", - ) - + ) + def validate_purl(self, value): try: PackageURL.from_string(value) except ValueError as e: raise ValidationError(f'purl validation error: {e}') return value - + def validate_source_purl(self, value): if value: try: @@ -403,9 +412,11 @@ def validate_source_purl(self, value): return value def validate_addon_pipelines(self, value): - invalid_pipelines = [pipe for pipe in value if not is_supported_addon_pipeline(pipe)] + invalid_pipelines = [ + pipe for pipe in value if not is_supported_addon_pipeline(pipe)] if invalid_pipelines: - raise ValidationError(f'Error unsupported addon pipelines: {",".join(invalid_pipelines)}') + raise ValidationError( + f'Error unsupported addon pipelines: {",".join(invalid_pipelines)}') return value @@ -419,7 +430,7 @@ class PackageVersSerializer(Serializer): required=False, allow_empty=True, help_text="Addon pipelines to run on the package.", - ) + ) class PackageUpdateSerializer(Serializer): @@ -444,27 +455,32 @@ class PurlUpdateResponseSerializer(Serializer): class IndexPackagesResponseSerializer(Serializer): - queued_packages_count = IntegerField(help_text="Number of package urls placed on the index queue.") + queued_packages_count = IntegerField( + help_text="Number of package urls placed on the index queue.") queued_packages = ListField( child=CharField(), help_text="List of package urls that were placed on the index queue." ) - requeued_packages_count = IntegerField(help_text="Number of existing package urls placed on the rescan queue.") + requeued_packages_count = IntegerField( + help_text="Number of existing package urls placed on the rescan queue.") requeued_packages = ListField( child=CharField(), help_text="List of existing package urls that were placed on the rescan queue." ) - unqueued_packages_count = IntegerField(help_text="Number of package urls not placed on the index queue.") + unqueued_packages_count = IntegerField( + help_text="Number of package urls not placed on the index queue.") unqueued_packages = ListField( child=CharField(), help_text="List of package urls that were not placed on the index queue." ) - unsupported_packages_count = IntegerField(help_text="Number of package urls that are not processable by the index queue.") + unsupported_packages_count = IntegerField( + help_text="Number of package urls that are not processable by the index queue.") unsupported_packages = ListField( child=CharField(), help_text="List of package urls that are not processable by the index queue." ) - unsupported_vers_count = IntegerField(help_text="Number of vers range that are not supported by the univers or package_manager.") + unsupported_vers_count = IntegerField( + help_text="Number of vers range that are not supported by the univers or package_manager.") unsupported_vers = ListField( child=CharField(), help_text="List of vers range that are not supported by the univers or package_manager." diff --git a/packagedb/tasks.py b/packagedb/tasks.py index c4a9099b..2c390f61 100644 --- a/packagedb/tasks.py +++ b/packagedb/tasks.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -67,7 +67,8 @@ def get_and_index_new_purls(package_url): try: local_versions = [version_class(version) for version in local_versions] - all_versions = [version_class(version.value) for version in all_versions] + all_versions = [version_class(version.value) + for version in all_versions] except InvalidVersion as e: return f"InvalidVersion exception: {e}" @@ -100,7 +101,8 @@ def is_supported_watch_ecosystem(watch): watch.watch_error = ( f"`{watch.type}` ecosystem is not supported by {error_message}" ) - watch.last_watch_date = datetime.datetime.now(tz=datetime.timezone.utc) + watch.last_watch_date = datetime.datetime.now( + tz=datetime.timezone.utc) watch.save(update_fields=["last_watch_date"]) return False diff --git a/packagedb/tests/__init__.py b/packagedb/tests/__init__.py index 2eb8f9f0..e1521118 100644 --- a/packagedb/tests/__init__.py +++ b/packagedb/tests/__init__.py @@ -3,6 +3,6 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/packagedb/tests/test_api.py b/packagedb/tests/test_api.py index 184c4268..c451925c 100644 --- a/packagedb/tests/test_api.py +++ b/packagedb/tests/test_api.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -83,10 +83,12 @@ def test_api_resource_list_endpoint(self): self.assertEqual(2, response.data.get('count')) def test_api_resource_retrieve_endpoint(self): - response = self.client.get('/api/resources/{}/'.format(self.resource1.sha1)) + response = self.client.get( + '/api/resources/{}/'.format(self.resource1.sha1)) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.data.get('package'), self.test_url.format(str(self.package1.uuid))) + self.assertEqual(response.data.get('package'), + self.test_url.format(str(self.package1.uuid))) self.assertEqual(response.data.get('purl'), self.package1.package_url) self.assertEqual(response.data.get('path'), self.resource1.path) self.assertEqual(response.data.get('size'), self.resource1.size) @@ -94,17 +96,21 @@ def test_api_resource_retrieve_endpoint(self): self.assertEqual(response.data.get('md5'), self.resource1.md5) self.assertEqual(response.data.get('sha256'), self.resource1.sha256) self.assertEqual(response.data.get('sha512'), self.resource1.sha512) - self.assertEqual(response.data.get('git_sha1'), self.resource1.git_sha1) - self.assertEqual(response.data.get('extra_data'), self.resource1.extra_data) + self.assertEqual(response.data.get( + 'git_sha1'), self.resource1.git_sha1) + self.assertEqual(response.data.get('extra_data'), + self.resource1.extra_data) self.assertEqual(response.data.get('type'), self.resource1.type) def test_api_resource_list_endpoint_returns_none_when_filtering_by_non_uuid_value(self): - response = self.client.get('/api/resources/?package={}'.format('not-a-uuid')) + response = self.client.get( + '/api/resources/?package={}'.format('not-a-uuid')) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(0, response.data.get('count')) def test_api_resource_list_endpoint_returns_none_when_filtering_by_wrong_uuid(self): - response = self.client.get('/api/resources/?package={}'.format('4eb22e66-3e1c-4818-9b5e-858008a7c2b5')) + response = self.client.get( + '/api/resources/?package={}'.format('4eb22e66-3e1c-4818-9b5e-858008a7c2b5')) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(0, response.data.get('count')) @@ -114,13 +120,15 @@ def test_api_resource_list_endpoint_returns_none_when_filtering_by_blank_uuid(se self.assertEqual(2, response.data.get('count')) def test_api_resource_list_endpoint_filters_by_package1_uuid(self): - response = self.client.get('/api/resources/?package={}'.format(self.package1.uuid)) + response = self.client.get( + '/api/resources/?package={}'.format(self.package1.uuid)) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(1, response.data.get('count')) test_resource = response.data.get('results')[0] - self.assertEqual(test_resource.get('package'), self.test_url.format(str(self.package1.uuid))) + self.assertEqual(test_resource.get('package'), + self.test_url.format(str(self.package1.uuid))) self.assertEqual(test_resource.get('purl'), self.package1.package_url) self.assertEqual(test_resource.get('path'), self.resource1.path) self.assertEqual(test_resource.get('size'), self.resource1.size) @@ -128,18 +136,22 @@ def test_api_resource_list_endpoint_filters_by_package1_uuid(self): self.assertEqual(test_resource.get('md5'), self.resource1.md5) self.assertEqual(test_resource.get('sha256'), self.resource1.sha256) self.assertEqual(test_resource.get('sha512'), self.resource1.sha512) - self.assertEqual(test_resource.get('git_sha1'), self.resource1.git_sha1) - self.assertEqual(test_resource.get('extra_data'), self.resource1.extra_data) + self.assertEqual(test_resource.get( + 'git_sha1'), self.resource1.git_sha1) + self.assertEqual(test_resource.get('extra_data'), + self.resource1.extra_data) self.assertEqual(test_resource.get('type'), self.resource1.type) def test_api_resource_list_endpoint_filters_by_package2_uuid(self): - response = self.client.get('/api/resources/?package={}'.format(self.package2.uuid)) + response = self.client.get( + '/api/resources/?package={}'.format(self.package2.uuid)) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(1, response.data.get('count')) test_resource = response.data.get('results')[0] - self.assertEqual(test_resource.get('package'), self.test_url.format(str(self.package2.uuid))) + self.assertEqual(test_resource.get('package'), + self.test_url.format(str(self.package2.uuid))) self.assertEqual(test_resource.get('purl'), self.package2.package_url) self.assertEqual(test_resource.get('path'), self.resource2.path) self.assertEqual(test_resource.get('size'), self.resource2.size) @@ -147,12 +159,15 @@ def test_api_resource_list_endpoint_filters_by_package2_uuid(self): self.assertEqual(test_resource.get('md5'), self.resource2.md5) self.assertEqual(test_resource.get('sha256'), self.resource2.sha256) self.assertEqual(test_resource.get('sha512'), self.resource2.sha512) - self.assertEqual(test_resource.get('git_sha1'), self.resource2.git_sha1) - self.assertEqual(test_resource.get('extra_data'), self.resource2.extra_data) + self.assertEqual(test_resource.get( + 'git_sha1'), self.resource2.git_sha1) + self.assertEqual(test_resource.get('extra_data'), + self.resource2.extra_data) self.assertEqual(test_resource.get('type'), self.resource2.type) def test_api_resource_list_endpoint_returns_none_when_filtering_by_wrong_purl(self): - response = self.client.get('/api/resources/?purl={}'.format('pkg:npm/test@1.0.0')) + response = self.client.get( + '/api/resources/?purl={}'.format('pkg:npm/test@1.0.0')) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(0, response.data.get('count')) @@ -162,13 +177,15 @@ def test_api_resource_list_endpoint_returns_none_when_filtering_by_blank_uuid(se self.assertEqual(2, response.data.get('count')) def test_api_resource_list_endpoint_filters_by_package1_purl(self): - response = self.client.get('/api/resources/?purl={}'.format(self.package1.package_url)) + response = self.client.get( + '/api/resources/?purl={}'.format(self.package1.package_url)) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(1, response.data.get('count')) test_resource = response.data.get('results')[0] - self.assertEqual(test_resource.get('package'), self.test_url.format(str(self.package1.uuid))) + self.assertEqual(test_resource.get('package'), + self.test_url.format(str(self.package1.uuid))) self.assertEqual(test_resource.get('purl'), self.package1.package_url) self.assertEqual(test_resource.get('path'), self.resource1.path) self.assertEqual(test_resource.get('size'), self.resource1.size) @@ -176,18 +193,22 @@ def test_api_resource_list_endpoint_filters_by_package1_purl(self): self.assertEqual(test_resource.get('md5'), self.resource1.md5) self.assertEqual(test_resource.get('sha256'), self.resource1.sha256) self.assertEqual(test_resource.get('sha512'), self.resource1.sha512) - self.assertEqual(test_resource.get('git_sha1'), self.resource1.git_sha1) - self.assertEqual(test_resource.get('extra_data'), self.resource1.extra_data) + self.assertEqual(test_resource.get( + 'git_sha1'), self.resource1.git_sha1) + self.assertEqual(test_resource.get('extra_data'), + self.resource1.extra_data) self.assertEqual(test_resource.get('type'), self.resource1.type) def test_api_resource_list_endpoint_filters_by_package2_purl(self): - response = self.client.get('/api/resources/?purl={}'.format(self.package2.package_url)) + response = self.client.get( + '/api/resources/?purl={}'.format(self.package2.package_url)) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(1, response.data.get('count')) test_resource = response.data.get('results')[0] - self.assertEqual(test_resource.get('package'), self.test_url.format(str(self.package2.uuid))) + self.assertEqual(test_resource.get('package'), + self.test_url.format(str(self.package2.uuid))) self.assertEqual(test_resource.get('purl'), self.package2.package_url) self.assertEqual(test_resource.get('path'), self.resource2.path) self.assertEqual(test_resource.get('size'), self.resource2.size) @@ -195,8 +216,10 @@ def test_api_resource_list_endpoint_filters_by_package2_purl(self): self.assertEqual(test_resource.get('md5'), self.resource2.md5) self.assertEqual(test_resource.get('sha256'), self.resource2.sha256) self.assertEqual(test_resource.get('sha512'), self.resource2.sha512) - self.assertEqual(test_resource.get('git_sha1'), self.resource2.git_sha1) - self.assertEqual(test_resource.get('extra_data'), self.resource2.extra_data) + self.assertEqual(test_resource.get( + 'git_sha1'), self.resource2.git_sha1) + self.assertEqual(test_resource.get('extra_data'), + self.resource2.extra_data) self.assertEqual(test_resource.get('type'), self.resource2.type) def test_api_resource_filter_by_checksums(self): @@ -207,25 +230,31 @@ def test_api_resource_filter_by_checksums(self): data = { 'sha1': sha1s } - response = self.client.post('/api/resources/filter_by_checksums/', data=data) + response = self.client.post( + '/api/resources/filter_by_checksums/', data=data) self.assertEqual(2, response.data['count']) - expected = self.get_test_loc('api/resource-filter_by_checksums-expected.json') - self.check_expected_results(response.data['results'], expected, fields_to_remove=["url", "uuid", "package"], regen=FIXTURES_REGEN) + expected = self.get_test_loc( + 'api/resource-filter_by_checksums-expected.json') + self.check_expected_results(response.data['results'], expected, fields_to_remove=[ + "url", "uuid", "package"], regen=FIXTURES_REGEN) data = { 'does-not-exist': 'dne' } - response = self.client.post('/api/resources/filter_by_checksums/', data=data) + response = self.client.post( + '/api/resources/filter_by_checksums/', data=data) self.assertEqual(status.HTTP_400_BAD_REQUEST, response.status_code) expected_status = 'Unsupported field(s) given: does-not-exist' self.assertEqual(expected_status, response.data['status']) data = {} - response = self.client.post('/api/resources/filter_by_checksums/', data=data) + response = self.client.post( + '/api/resources/filter_by_checksums/', data=data) self.assertEqual(status.HTTP_400_BAD_REQUEST, response.status_code) expected_status = 'No values provided' self.assertEqual(expected_status, response.data['status']) + class PackageApiTestCase(JsonBasedTesting, TestCase): test_data_dir = os.path.join(os.path.dirname(__file__), 'testfiles') @@ -372,7 +401,8 @@ def test_package_api_list_endpoint(self): def test_package_api_list_endpoint_filter(self): for key, value in self.package_data.items(): - response = self.client.get('/api/packages/?{}={}'.format(key, value)) + response = self.client.get( + '/api/packages/?{}={}'.format(key, value)) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(1, response.data.get('count')) @@ -382,11 +412,13 @@ def test_package_api_list_endpoint_filter_by_purl_fields_ignores_case(self): if key not in ['type', 'namespace', 'name']: continue - response = self.client.get('/api/packages/?{}={}'.format(key, value.lower())) + response = self.client.get( + '/api/packages/?{}={}'.format(key, value.lower())) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(1, response.data.get('count')) - response = self.client.get('/api/packages/?{}={}'.format(key, value.upper())) + response = self.client.get( + '/api/packages/?{}={}'.format(key, value.upper())) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(1, response.data.get('count')) @@ -400,7 +432,8 @@ def test_package_api_list_endpoint_search(self): download_url='https://dummy.com/dummy' ) - response = self.client.get('/api/packages/?search={}'.format('generic')) + response = self.client.get( + '/api/packages/?search={}'.format('generic')) assert response.data.get('count') == 2 response = self.client.get('/api/packages/?search={}'.format('dummy')) assert response.data.get('count') == 1 @@ -408,17 +441,20 @@ def test_package_api_list_endpoint_search(self): assert response.data.get('count') == 1 response = self.client.get('/api/packages/?search={}'.format('12.35')) assert response.data.get('count') == 1 - response = self.client.get('/api/packages/?search={}'.format('https://dummy.com/dummy')) + response = self.client.get( + '/api/packages/?search={}'.format('https://dummy.com/dummy')) assert response.data.get('count') == 1 def test_package_api_retrieve_endpoint(self): - response = self.client.get('/api/packages/{}/'.format(self.package.uuid)) + response = self.client.get( + '/api/packages/{}/'.format(self.package.uuid)) self.assertEqual(response.status_code, status.HTTP_200_OK) for key, value in response.data.items(): # Handle the API-only `url` key if key == 'url': - self.assertEqual(value, self.test_url.format(str(self.package.uuid))) + self.assertEqual(value, self.test_url.format( + str(self.package.uuid))) continue if key in ['type', 'namespace', 'name', 'version', 'qualifiers', 'subpath']: @@ -434,25 +470,33 @@ def test_package_api_retrieve_endpoint(self): self.assertEqual(value, getattr(self.package, key)) def test_api_package_latest_version_action(self): - p1 = Package.objects.create(download_url='http://a.a', type='generic', name='name', version='1.0') - p2 = Package.objects.create(download_url='http://b.b', type='generic', name='name', version='2.0') - p3 = Package.objects.create(download_url='http://c.c', type='generic', name='name', version='3.0') + p1 = Package.objects.create( + download_url='http://a.a', type='generic', name='name', version='1.0') + p2 = Package.objects.create( + download_url='http://b.b', type='generic', name='name', version='2.0') + p3 = Package.objects.create( + download_url='http://c.c', type='generic', name='name', version='3.0') - response = self.client.get(reverse('api:package-latest-version', args=[p1.uuid])) + response = self.client.get( + reverse('api:package-latest-version', args=[p1.uuid])) self.assertEqual('3.0', response.data['version']) - response = self.client.get(reverse('api:package-latest-version', args=[p2.uuid])) + response = self.client.get( + reverse('api:package-latest-version', args=[p2.uuid])) self.assertEqual('3.0', response.data['version']) - response = self.client.get(reverse('api:package-latest-version', args=[p3.uuid])) + response = self.client.get( + reverse('api:package-latest-version', args=[p3.uuid])) self.assertEqual('3.0', response.data['version']) def test_api_package_resources_action(self): # create 10 resources for i in range(0, 10): - Resource.objects.create(package=self.package, path='path{}/'.format(i)) + Resource.objects.create( + package=self.package, path='path{}/'.format(i)) - response = self.client.get(reverse('api:package-resources', args=[self.package.uuid])) + response = self.client.get( + reverse('api:package-resources', args=[self.package.uuid])) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(10, response.data['count']) @@ -488,30 +532,39 @@ def test_package_api_filter_by_checksums(self): data = { 'sha1': sha1s, } - response = self.client.post('/api/packages/filter_by_checksums/', data=data) + response = self.client.post( + '/api/packages/filter_by_checksums/', data=data) self.assertEqual(5, response.data['count']) - expected = self.get_test_loc('api/package-filter_by_checksums-expected.json') - self.check_expected_results(response.data['results'], expected, fields_to_remove=["url", "uuid", "resources", "package_sets", "history"], regen=FIXTURES_REGEN) + expected = self.get_test_loc( + 'api/package-filter_by_checksums-expected.json') + self.check_expected_results(response.data['results'], expected, fields_to_remove=[ + "url", "uuid", "resources", "package_sets", "history"], regen=FIXTURES_REGEN) data["enhance_package_data"] = True - enhanced_response = self.client.post('/api/packages/filter_by_checksums/', data=data) + enhanced_response = self.client.post( + '/api/packages/filter_by_checksums/', data=data) self.assertEqual(5, len(enhanced_response.data['results'])) - expected = self.get_test_loc('api/package-filter_by_checksums-enhanced-package-data-expected.json') - self.check_expected_results(enhanced_response.data['results'], expected, fields_to_remove=["url", "uuid", "resources", "package_sets", "history"], regen=FIXTURES_REGEN) + expected = self.get_test_loc( + 'api/package-filter_by_checksums-enhanced-package-data-expected.json') + self.check_expected_results(enhanced_response.data['results'], expected, fields_to_remove=[ + "url", "uuid", "resources", "package_sets", "history"], regen=FIXTURES_REGEN) data = { 'does-not-exist': 'dne' } - response = self.client.post('/api/packages/filter_by_checksums/', data=data) + response = self.client.post( + '/api/packages/filter_by_checksums/', data=data) self.assertEqual(status.HTTP_400_BAD_REQUEST, response.status_code) expected_status = 'Unsupported field(s) given: does-not-exist' self.assertEqual(expected_status, response.data['status']) data = {} - response = self.client.post('/api/packages/filter_by_checksums/', data=data) + response = self.client.post( + '/api/packages/filter_by_checksums/', data=data) self.assertEqual(status.HTTP_400_BAD_REQUEST, response.status_code) expected_status = 'No values provided' self.assertEqual(expected_status, response.data['status']) + class PackageApiReindexingTestCase(JsonBasedTesting, TestCase): test_data_dir = os.path.join(os.path.dirname(__file__), 'testfiles') @@ -546,10 +599,13 @@ def setUp(self): def test_reindex_package(self): self.assertEqual(1, ScannableURI.objects.all().count()) - response = self.client.get(f'/api/packages/{self.package.uuid}/reindex_package/') - self.assertEqual('pkg:maven/sample/Baz@90.12 has been queued for reindexing', response.data['status']) + response = self.client.get( + f'/api/packages/{self.package.uuid}/reindex_package/') + self.assertEqual( + 'pkg:maven/sample/Baz@90.12 has been queued for reindexing', response.data['status']) self.assertEqual(2, ScannableURI.objects.all().count()) - new_scannable_uri = ScannableURI.objects.exclude(pk=self.scannableuri.pk).first() + new_scannable_uri = ScannableURI.objects.exclude( + pk=self.scannableuri.pk).first() self.assertEqual(self.package, new_scannable_uri.package) self.assertEqual(True, new_scannable_uri.reindex_uri) self.assertEqual(100, new_scannable_uri.priority) @@ -592,7 +648,7 @@ def setUp(self): 'namespace': '', 'name': 'test', 'version': '1.0.0', - 'qualifiers':'', + 'qualifiers': '', 'package_content': PackageContentType.BINARY, 'download_url': 'https://example.com/test-1.0.0.jar', } @@ -602,7 +658,7 @@ def setUp(self): 'namespace': '', 'name': 'test', 'version': '1.0.0', - 'qualifiers':'classifier=sources', + 'qualifiers': 'classifier=sources', 'declared_license_expression': 'apache-2.0', 'copyright': 'Copyright (c) example corp.', 'holder': 'example corp.', @@ -644,12 +700,14 @@ def test_package_api_purl_filter_by_query_param_no_value(self): self.assertEqual(4, response.data.get('count')) def test_package_api_purl_filter_by_query_param_non_existant_purl(self): - response = self.client.get('/api/packages/?purl={}'.format(self.missing_purl)) + response = self.client.get( + '/api/packages/?purl={}'.format(self.missing_purl)) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(0, response.data.get('count')) def test_package_api_purl_filter_by_query_param_no_version(self): - response = self.client.get('/api/packages/?purl={}'.format('pkg:maven/org.apache.commons/io')) + response = self.client.get( + '/api/packages/?purl={}'.format('pkg:maven/org.apache.commons/io')) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(2, response.data.get('count')) @@ -659,12 +717,18 @@ def test_package_api_purl_filter_by_query_param1(self): self.assertEqual(1, response.data.get('count')) test_package = response.data.get('results')[0] - self.assertEqual(test_package.get('type'), self.package_data1.get('type')) - self.assertEqual(test_package.get('namespace'), self.package_data1.get('namespace')) - self.assertEqual(test_package.get('name'), self.package_data1.get('name')) - self.assertEqual(test_package.get('version'), self.package_data1.get('version')) - self.assertEqual(test_package.get('download_url'), self.package_data1.get('download_url')) - self.assertEqual(test_package.get('extra_data'), self.package_data1.get('extra_data')) + self.assertEqual(test_package.get('type'), + self.package_data1.get('type')) + self.assertEqual(test_package.get('namespace'), + self.package_data1.get('namespace')) + self.assertEqual(test_package.get('name'), + self.package_data1.get('name')) + self.assertEqual(test_package.get('version'), + self.package_data1.get('version')) + self.assertEqual(test_package.get('download_url'), + self.package_data1.get('download_url')) + self.assertEqual(test_package.get('extra_data'), + self.package_data1.get('extra_data')) def test_package_api_purl_filter_by_query_param2(self): response = self.client.get('/api/packages/?purl={}'.format(self.purl2)) @@ -672,33 +736,52 @@ def test_package_api_purl_filter_by_query_param2(self): self.assertEqual(1, response.data.get('count')) test_package = response.data.get('results')[0] - self.assertEqual(test_package.get('type'), self.package_data2.get('type')) - self.assertEqual(test_package.get('namespace'), self.package_data2.get('namespace')) - self.assertEqual(test_package.get('name'), self.package_data2.get('name')) - self.assertEqual(test_package.get('version'), self.package_data2.get('version')) - self.assertEqual(test_package.get('download_url'), self.package_data2.get('download_url')) - self.assertEqual(test_package.get('extra_data'), self.package_data2.get('extra_data')) + self.assertEqual(test_package.get('type'), + self.package_data2.get('type')) + self.assertEqual(test_package.get('namespace'), + self.package_data2.get('namespace')) + self.assertEqual(test_package.get('name'), + self.package_data2.get('name')) + self.assertEqual(test_package.get('version'), + self.package_data2.get('version')) + self.assertEqual(test_package.get('download_url'), + self.package_data2.get('download_url')) + self.assertEqual(test_package.get('extra_data'), + self.package_data2.get('extra_data')) def test_package_api_purl_filter_by_both_query_params(self): - response = self.client.get('/api/packages/?purl={}&purl={}'.format(self.purl1, self.purl2)) + response = self.client.get( + '/api/packages/?purl={}&purl={}'.format(self.purl1, self.purl2)) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(2, response.data.get('count')) test_package = response.data.get('results')[0] - self.assertEqual(test_package.get('type'), self.package_data1.get('type')) - self.assertEqual(test_package.get('namespace'), self.package_data1.get('namespace')) - self.assertEqual(test_package.get('name'), self.package_data1.get('name')) - self.assertEqual(test_package.get('version'), self.package_data1.get('version')) - self.assertEqual(test_package.get('download_url'), self.package_data1.get('download_url')) - self.assertEqual(test_package.get('extra_data'), self.package_data1.get('extra_data')) + self.assertEqual(test_package.get('type'), + self.package_data1.get('type')) + self.assertEqual(test_package.get('namespace'), + self.package_data1.get('namespace')) + self.assertEqual(test_package.get('name'), + self.package_data1.get('name')) + self.assertEqual(test_package.get('version'), + self.package_data1.get('version')) + self.assertEqual(test_package.get('download_url'), + self.package_data1.get('download_url')) + self.assertEqual(test_package.get('extra_data'), + self.package_data1.get('extra_data')) test_package = response.data.get('results')[1] - self.assertEqual(test_package.get('type'), self.package_data2.get('type')) - self.assertEqual(test_package.get('namespace'), self.package_data2.get('namespace')) - self.assertEqual(test_package.get('name'), self.package_data2.get('name')) - self.assertEqual(test_package.get('version'), self.package_data2.get('version')) - self.assertEqual(test_package.get('download_url'), self.package_data2.get('download_url')) - self.assertEqual(test_package.get('extra_data'), self.package_data2.get('extra_data')) + self.assertEqual(test_package.get('type'), + self.package_data2.get('type')) + self.assertEqual(test_package.get('namespace'), + self.package_data2.get('namespace')) + self.assertEqual(test_package.get('name'), + self.package_data2.get('name')) + self.assertEqual(test_package.get('version'), + self.package_data2.get('version')) + self.assertEqual(test_package.get('download_url'), + self.package_data2.get('download_url')) + self.assertEqual(test_package.get('extra_data'), + self.package_data2.get('extra_data')) def test_package_api_purl_filter_by_two_purl_values_on_multiple_packages(self): extra_test_package = Package.objects.create( @@ -708,49 +791,72 @@ def test_package_api_purl_filter_by_two_purl_values_on_multiple_packages(self): version='2.2.2' ) - response = self.client.get('/api/packages/?purl={}&purl={}'.format(self.purl1, self.purl2)) + response = self.client.get( + '/api/packages/?purl={}&purl={}'.format(self.purl1, self.purl2)) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(2, response.data.get('count')) test_package = response.data.get('results')[0] - self.assertEqual(test_package.get('type'), self.package_data1.get('type')) - self.assertEqual(test_package.get('namespace'), self.package_data1.get('namespace')) - self.assertEqual(test_package.get('name'), self.package_data1.get('name')) - self.assertEqual(test_package.get('version'), self.package_data1.get('version')) - self.assertEqual(test_package.get('download_url'), self.package_data1.get('download_url')) - self.assertEqual(test_package.get('extra_data'), self.package_data1.get('extra_data')) + self.assertEqual(test_package.get('type'), + self.package_data1.get('type')) + self.assertEqual(test_package.get('namespace'), + self.package_data1.get('namespace')) + self.assertEqual(test_package.get('name'), + self.package_data1.get('name')) + self.assertEqual(test_package.get('version'), + self.package_data1.get('version')) + self.assertEqual(test_package.get('download_url'), + self.package_data1.get('download_url')) + self.assertEqual(test_package.get('extra_data'), + self.package_data1.get('extra_data')) test_package = response.data.get('results')[1] - self.assertEqual(test_package.get('type'), self.package_data2.get('type')) - self.assertEqual(test_package.get('namespace'), self.package_data2.get('namespace')) - self.assertEqual(test_package.get('name'), self.package_data2.get('name')) - self.assertEqual(test_package.get('version'), self.package_data2.get('version')) - self.assertEqual(test_package.get('download_url'), self.package_data2.get('download_url')) - self.assertEqual(test_package.get('extra_data'), self.package_data2.get('extra_data')) + self.assertEqual(test_package.get('type'), + self.package_data2.get('type')) + self.assertEqual(test_package.get('namespace'), + self.package_data2.get('namespace')) + self.assertEqual(test_package.get('name'), + self.package_data2.get('name')) + self.assertEqual(test_package.get('version'), + self.package_data2.get('version')) + self.assertEqual(test_package.get('download_url'), + self.package_data2.get('download_url')) + self.assertEqual(test_package.get('extra_data'), + self.package_data2.get('extra_data')) def test_package_api_purl_filter_by_one_purl_multiple_params(self): - response = self.client.get('/api/packages/?purl={}&purl={}'.format(self.purl1, self.missing_purl)) + response = self.client.get( + '/api/packages/?purl={}&purl={}'.format(self.purl1, self.missing_purl)) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(1, response.data.get('count')) test_package = response.data.get('results')[0] - self.assertEqual(test_package.get('type'), self.package_data1.get('type')) - self.assertEqual(test_package.get('namespace'), self.package_data1.get('namespace')) - self.assertEqual(test_package.get('name'), self.package_data1.get('name')) - self.assertEqual(test_package.get('version'), self.package_data1.get('version')) - self.assertEqual(test_package.get('download_url'), self.package_data1.get('download_url')) - self.assertEqual(test_package.get('extra_data'), self.package_data1.get('extra_data')) + self.assertEqual(test_package.get('type'), + self.package_data1.get('type')) + self.assertEqual(test_package.get('namespace'), + self.package_data1.get('namespace')) + self.assertEqual(test_package.get('name'), + self.package_data1.get('name')) + self.assertEqual(test_package.get('version'), + self.package_data1.get('version')) + self.assertEqual(test_package.get('download_url'), + self.package_data1.get('download_url')) + self.assertEqual(test_package.get('extra_data'), + self.package_data1.get('extra_data')) def test_package_api_purl_filter_by_multiple_blank_purl(self): - response = self.client.get('/api/packages/?purl={}&purl={}'.format('', '')) + response = self.client.get( + '/api/packages/?purl={}&purl={}'.format('', '')) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(4, response.data.get('count')) def test_package_api_get_enhanced_package(self): - response = self.client.get(reverse('api:package-get-enhanced-package-data', args=[self.package3.uuid])) + response = self.client.get( + reverse('api:package-get-enhanced-package-data', args=[self.package3.uuid])) result = response.data expected = self.get_test_loc('api/enhanced_package.json') - self.check_expected_results(result, expected, fields_to_remove=['package_sets'], regen=FIXTURES_REGEN) + self.check_expected_results(result, expected, fields_to_remove=[ + 'package_sets'], regen=FIXTURES_REGEN) class CollectApiTestCase(JsonBasedTesting, TestCase): @@ -819,11 +925,15 @@ def test_package_live(self): purl_sources_str = f'{purl_str}?classifier=sources' sources_download_url = 'https://repo1.maven.org/maven2/org/apache/twill/twill-core/0.12.0/twill-core-0.12.0-sources.jar' - self.assertEqual(0, Package.objects.filter(download_url=download_url).count()) - self.assertEqual(0, Package.objects.filter(download_url=sources_download_url).count()) + self.assertEqual(0, Package.objects.filter( + download_url=download_url).count()) + self.assertEqual(0, Package.objects.filter( + download_url=sources_download_url).count()) response = self.client.get(f'/api/collect/?purl={purl_str}') - self.assertEqual(1, Package.objects.filter(download_url=download_url).count()) - self.assertEqual(1, Package.objects.filter(download_url=sources_download_url).count()) + self.assertEqual(1, Package.objects.filter( + download_url=download_url).count()) + self.assertEqual(1, Package.objects.filter( + download_url=sources_download_url).count()) expected = self.get_test_loc('api/twill-core-0.12.0.json') self.assertEqual(2, len(response.data)) @@ -838,13 +948,15 @@ def test_package_live(self): 'history' ] - self.check_expected_results(result, expected, fields_to_remove=fields_to_remove, regen=FIXTURES_REGEN) + self.check_expected_results( + result, expected, fields_to_remove=fields_to_remove, regen=FIXTURES_REGEN) # Ensure that the created ScannableURI objects have a priority of 100 package = Package.objects.get(download_url=download_url) source_package = Package.objects.get(download_url=sources_download_url) package_scannable_uri = ScannableURI.objects.get(package=package) - source_package_scannable_uri = ScannableURI.objects.get(package=source_package) + source_package_scannable_uri = ScannableURI.objects.get( + package=source_package) self.assertEqual(100, package_scannable_uri.priority) self.assertEqual(100, source_package_scannable_uri.priority) @@ -854,12 +966,17 @@ def test_package_live_works_with_purl2vcs(self): purl_sources_str = f'{purl}?classifier=sources' sources_download_url = 'https://repo1.maven.org/maven2/org/elasticsearch/plugin/elasticsearch-scripting-painless-spi/6.8.15/elasticsearch-scripting-painless-spi-6.8.15-sources.jar' - self.assertEqual(0, Package.objects.filter(download_url=download_url).count()) - self.assertEqual(0, Package.objects.filter(download_url=sources_download_url).count()) + self.assertEqual(0, Package.objects.filter( + download_url=download_url).count()) + self.assertEqual(0, Package.objects.filter( + download_url=sources_download_url).count()) response = self.client.get(f'/api/collect/?purl={purl}') - self.assertEqual(1, Package.objects.filter(download_url=download_url).count()) - self.assertEqual(1, Package.objects.filter(download_url=sources_download_url).count()) - expected = self.get_test_loc('api/elasticsearch-scripting-painless-spi-6.8.15.json') + self.assertEqual(1, Package.objects.filter( + download_url=download_url).count()) + self.assertEqual(1, Package.objects.filter( + download_url=sources_download_url).count()) + expected = self.get_test_loc( + 'api/elasticsearch-scripting-painless-spi-6.8.15.json') self.assertEqual(2, len(response.data)) result = response.data[0] @@ -873,20 +990,22 @@ def test_package_live_works_with_purl2vcs(self): 'history' ] - self.check_expected_results(result, expected, fields_to_remove=fields_to_remove, regen=FIXTURES_REGEN) + self.check_expected_results( + result, expected, fields_to_remove=fields_to_remove, regen=FIXTURES_REGEN) def test_package_api_index_packages_endpoint(self): priority_resource_uris_count = PriorityResourceURI.objects.all().count() self.assertEqual(0, priority_resource_uris_count) packages = [ - {'purl':'pkg:maven/ch.qos.reload4j/reload4j@1.2.24'}, - {'purl':'pkg:maven/com.esotericsoftware.kryo/kryo@2.24.0'}, - {'purl':'pkg:bitbucket/example/example@1.0.0'}, + {'purl': 'pkg:maven/ch.qos.reload4j/reload4j@1.2.24'}, + {'purl': 'pkg:maven/com.esotericsoftware.kryo/kryo@2.24.0'}, + {'purl': 'pkg:bitbucket/example/example@1.0.0'}, ] data = { 'packages': packages } - response = self.client.post('/api/collect/index_packages/', data=data, content_type="application/json") + response = self.client.post( + '/api/collect/index_packages/', data=data, content_type="application/json") self.assertEqual(2, response.data['queued_packages_count']) expected_queued_packages = [ 'pkg:maven/ch.qos.reload4j/reload4j@1.2.24', @@ -902,21 +1021,23 @@ def test_package_api_index_packages_endpoint(self): expected_unsupported_packages = [ 'pkg:bitbucket/example/example@1.0.0' ] - self.assertEqual(expected_unsupported_packages, response.data['unsupported_packages']) + self.assertEqual(expected_unsupported_packages, + response.data['unsupported_packages']) priority_resource_uris_count = PriorityResourceURI.objects.all().count() self.assertEqual(2, priority_resource_uris_count) # Ensure that we don't add the same packages to the queue if they have # not yet been processed purls = [ - {'purl':'pkg:maven/ch.qos.reload4j/reload4j@1.2.24'}, - {'purl':'pkg:maven/com.esotericsoftware.kryo/kryo@2.24.0'}, - {'purl':'pkg:bitbucket/example/example@1.0.0'}, + {'purl': 'pkg:maven/ch.qos.reload4j/reload4j@1.2.24'}, + {'purl': 'pkg:maven/com.esotericsoftware.kryo/kryo@2.24.0'}, + {'purl': 'pkg:bitbucket/example/example@1.0.0'}, ] data = { 'packages': purls } - response = self.client.post('/api/collect/index_packages/', data=data, content_type="application/json") + response = self.client.post( + '/api/collect/index_packages/', data=data, content_type="application/json") self.assertEqual(0, response.data['queued_packages_count']) self.assertEqual([], response.data['queued_packages']) self.assertEqual(0, response.data['requeued_packages_count']) @@ -934,10 +1055,12 @@ def test_package_api_index_packages_endpoint(self): expected_unsupported_packages = [ 'pkg:bitbucket/example/example@1.0.0' ] - self.assertEqual(expected_unsupported_packages, response.data['unsupported_packages']) + self.assertEqual(expected_unsupported_packages, + response.data['unsupported_packages']) bad_data = {'does-not-exist': 'dne'} - response = self.client.post('/api/collect/index_packages/', data=bad_data, content_type="application/json") + response = self.client.post( + '/api/collect/index_packages/', data=bad_data, content_type="application/json") expected_errors = {'packages': ['This field is required.']} self.assertEqual(status.HTTP_400_BAD_REQUEST, response.status_code) self.assertEqual(expected_errors, response.data['errors']) @@ -987,7 +1110,8 @@ def test_package_api_index_packages_endpoint_with_vers(self, mock_get_all_versio "pkg:maven/ch.qos.reload4j/reload4j@1.2.23", ] self.assertEqual( - sorted(expected_queued_packages), sorted(response.data["queued_packages"]) + sorted(expected_queued_packages), sorted( + response.data["queued_packages"]) ) self.assertEqual(0, response.data['requeued_packages_count']) self.assertEqual([], response.data['requeued_packages']) @@ -1045,7 +1169,8 @@ def test_package_api_index_packages_endpoint_all_version_index(self, mock_get_al "pkg:maven/ch.qos.reload4j/reload4j@1.2.25", ] self.assertEqual( - sorted(expected_queued_packages), sorted(response.data["queued_packages"]) + sorted(expected_queued_packages), sorted( + response.data["queued_packages"]) ) self.assertEqual(0, response.data['requeued_packages_count']) self.assertEqual([], response.data['requeued_packages']) @@ -1064,16 +1189,20 @@ def test_reindex_packages_bulk(self): self.assertEqual(self.scan_uuid, self.scannableuri.scan_uuid) self.assertEqual('error', self.scannableuri.scan_error) self.assertEqual('error', self.scannableuri.index_error) - self.assertEqual(self.scan_request_date, self.scannableuri.scan_request_date) - self.assertEqual(ScannableURI.SCAN_INDEX_FAILED, self.scannableuri.scan_status) + self.assertEqual(self.scan_request_date, + self.scannableuri.scan_request_date) + self.assertEqual(ScannableURI.SCAN_INDEX_FAILED, + self.scannableuri.scan_status) self.assertEqual(False, self.scannableuri2.reindex_uri) self.assertEqual(0, self.scannableuri2.priority) self.assertEqual(self.scan_uuid2, self.scannableuri2.scan_uuid) self.assertEqual('error', self.scannableuri2.scan_error) self.assertEqual('error', self.scannableuri2.index_error) - self.assertEqual(self.scan_request_date2, self.scannableuri2.scan_request_date) - self.assertEqual(ScannableURI.SCAN_INDEX_FAILED, self.scannableuri2.scan_status) + self.assertEqual(self.scan_request_date2, + self.scannableuri2.scan_request_date) + self.assertEqual(ScannableURI.SCAN_INDEX_FAILED, + self.scannableuri2.scan_status) packages = [ # Existing package @@ -1085,10 +1214,10 @@ def test_reindex_packages_bulk(self): }, # NOt in DB and unsupported { - "purl":'pkg:pypi/does/not-exist@1', + "purl": 'pkg:pypi/does/not-exist@1', }, ] - data = {"packages": packages, "reindex":True} + data = {"packages": packages, "reindex": True} existing_purls = [ 'pkg:maven/sample/Baz@90.12', @@ -1099,13 +1228,16 @@ def test_reindex_packages_bulk(self): 'pkg:pypi/does/not-exist@1', ] - response = self.client.post(f'/api/collect/index_packages/', data=data, content_type="application/json") + response = self.client.post( + f'/api/collect/index_packages/', data=data, content_type="application/json") self.assertEqual(2, response.data['requeued_packages_count']) - self.assertListEqual(sorted(existing_purls), sorted(response.data['requeued_packages'])) + self.assertListEqual(sorted(existing_purls), sorted( + response.data['requeued_packages'])) self.assertEqual(1, response.data['unsupported_packages_count']) - self.assertListEqual(unsupported_purls, response.data['unsupported_packages']) + self.assertListEqual( + unsupported_purls, response.data['unsupported_packages']) self.assertEqual(0, response.data['queued_packages_count']) self.assertEqual([], response.data['queued_packages']) @@ -1113,7 +1245,8 @@ def test_reindex_packages_bulk(self): self.assertEqual([], response.data["unqueued_packages"]) self.assertEqual(4, ScannableURI.objects.all().count()) - new_scannable_uris = ScannableURI.objects.exclude(pk__in=[self.scannableuri.pk, self.scannableuri2.pk]) + new_scannable_uris = ScannableURI.objects.exclude( + pk__in=[self.scannableuri.pk, self.scannableuri2.pk]) self.assertEqual(2, new_scannable_uris.count()) for scannable_uri in new_scannable_uris: @@ -1128,13 +1261,14 @@ def test_package_api_index_packages_priority(self): priority_resource_uris_count = PriorityResourceURI.objects.all().count() self.assertEqual(0, priority_resource_uris_count) packages = [ - {'purl':'pkg:maven/ch.qos.reload4j/reload4j@1.2.24'}, - {'purl':'pkg:maven/com.esotericsoftware.kryo/kryo'}, + {'purl': 'pkg:maven/ch.qos.reload4j/reload4j@1.2.24'}, + {'purl': 'pkg:maven/com.esotericsoftware.kryo/kryo'}, ] data = { 'packages': packages } - response = self.client.post('/api/collect/index_packages/', data=data, content_type="application/json") + response = self.client.post( + '/api/collect/index_packages/', data=data, content_type="application/json") self.assertEqual(14, response.data['queued_packages_count']) expected_kryo_packages = [ 'pkg:maven/com.esotericsoftware.kryo/kryo@2.10', @@ -1151,24 +1285,28 @@ def test_package_api_index_packages_priority(self): 'pkg:maven/com.esotericsoftware.kryo/kryo@2.23.1', 'pkg:maven/com.esotericsoftware.kryo/kryo@2.24.0', ] - expected_queued_packages = expected_kryo_packages + ['pkg:maven/ch.qos.reload4j/reload4j@1.2.24'] + expected_queued_packages = expected_kryo_packages + \ + ['pkg:maven/ch.qos.reload4j/reload4j@1.2.24'] self.assertEqual( sorted(expected_queued_packages), sorted(response.data['queued_packages']) ) - priority_resource_uri = PriorityResourceURI.objects.get(package_url='pkg:maven/ch.qos.reload4j/reload4j@1.2.24') + priority_resource_uri = PriorityResourceURI.objects.get( + package_url='pkg:maven/ch.qos.reload4j/reload4j@1.2.24') self.assertEqual(100, priority_resource_uri.priority) for purl in expected_kryo_packages: - priority_resource_uri = PriorityResourceURI.objects.get(package_url=purl) + priority_resource_uri = PriorityResourceURI.objects.get( + package_url=purl) self.assertEqual(0, priority_resource_uri.priority) def test_collect_errors(self): invalid_purl = 'pkg:asdf1' response = self.client.get(f'/api/collect/?purl={invalid_purl}') self.assertEqual(status.HTTP_400_BAD_REQUEST, response.status_code) - expected_status = {'purl': ["purl validation error: purl is missing the required type component: 'pkg:asdf1'."]} + expected_status = {'purl': [ + "purl validation error: purl is missing the required type component: 'pkg:asdf1'."]} self.assertEqual(expected_status, response.data['errors']) unhandled_purl = 'pkg:does-not-exist/does-not-exist@1.0' @@ -1228,7 +1366,8 @@ def test_api_resource_checksum_filter(self): filters = f'?md5={self.resource1.md5}&md5={self.resource2.md5}' response = self.client.get(f'/api/resources/{filters}') self.assertEqual(2, response.data['count']) - names = sorted([result.get('name') for result in response.data['results']]) + names = sorted([result.get('name') + for result in response.data['results']]) expected_names = sorted([ self.resource1.name, self.resource2.name, @@ -1238,7 +1377,8 @@ def test_api_resource_checksum_filter(self): filters = f'?sha1={self.resource1.sha1}&sha1={self.resource2.sha1}' response = self.client.get(f'/api/resources/{filters}') self.assertEqual(2, response.data["count"]) - names = sorted([result.get('name') for result in response.data['results']]) + names = sorted([result.get('name') + for result in response.data['results']]) expected_names = sorted([ self.resource1.name, self.resource2.name, @@ -1270,62 +1410,70 @@ def setUp(self): def test_api_purl_updation(self): data = { - "purls": [ - {"purl": "pkg:npm/hologram@1.1.0", "content_type": "CURATION"}] - , - "uuid": str(self.new_package_set_uuid) + "purls": [ + {"purl": "pkg:npm/hologram@1.1.0", "content_type": "CURATION"}], + "uuid": str(self.new_package_set_uuid) } - response = self.client.post(f"/api/update_packages/", data=data, content_type="application/json") + response = self.client.post( + f"/api/update_packages/", data=data, content_type="application/json") - expected = [{"purl": "pkg:npm/hologram@1.1.0", "update_status": "Updated"}] + expected = [{"purl": "pkg:npm/hologram@1.1.0", + "update_status": "Updated"}] self.assertEqual(expected, response.data) def test_api_purl_updation_existing_package(self): data = { - "purls": [ - {"purl": "pkg:npm/foobar@1.1.0", "content_type": "PATCH"} - ], - "uuid": str(self.new_package_set_uuid) + "purls": [ + {"purl": "pkg:npm/foobar@1.1.0", "content_type": "PATCH"} + ], + "uuid": str(self.new_package_set_uuid) } - expected = [{"purl": "pkg:npm/foobar@1.1.0", "update_status": "Already Exists"}] + expected = [{"purl": "pkg:npm/foobar@1.1.0", + "update_status": "Already Exists"}] - response = self.client.post(f"/api/update_packages/", data=data, content_type="application/json") + response = self.client.post( + f"/api/update_packages/", data=data, content_type="application/json") self.assertEqual(expected, response.data) def test_api_purl_updation_non_existing_uuid(self): data = { - "purls": [ - {"purl": "pkg:npm/foobar@1.1.0", "content_type": "SOURCE_REPO"} - ], - "uuid": "ac9c36f4-a1ed-4824-8448-c6ed8f1da71d" + "purls": [ + {"purl": "pkg:npm/foobar@1.1.0", "content_type": "SOURCE_REPO"} + ], + "uuid": "ac9c36f4-a1ed-4824-8448-c6ed8f1da71d" } - expected = {"update_status": "No Package Set found for ac9c36f4-a1ed-4824-8448-c6ed8f1da71d"} + expected = { + "update_status": "No Package Set found for ac9c36f4-a1ed-4824-8448-c6ed8f1da71d"} - response = self.client.post(f"/api/update_packages/", data=data, content_type="application/json") + response = self.client.post( + f"/api/update_packages/", data=data, content_type="application/json") self.assertEqual(expected, response.data) def test_api_purl_updation_without_uuid(self): data = { - "purls": [ - {"purl": "pkg:npm/jammy@1.1.9", "content_type": "BINARY"} - ] + "purls": [ + {"purl": "pkg:npm/jammy@1.1.9", "content_type": "BINARY"} + ] } - expected = [{"purl": "pkg:npm/jammy@1.1.9", "update_status": "Updated"}] + expected = [{"purl": "pkg:npm/jammy@1.1.9", + "update_status": "Updated"}] - response = self.client.post(f"/api/update_packages/", data=data, content_type="application/json") + response = self.client.post( + f"/api/update_packages/", data=data, content_type="application/json") self.assertEqual(expected, response.data) def test_api_purl_validation_empty_request(self): data = {} - response = self.client.post(f"/api/update_packages/", data=data, content_type="application/json") + response = self.client.post( + f"/api/update_packages/", data=data, content_type="application/json") expected = { "errors": { @@ -1384,7 +1532,6 @@ def test_api_purl_validation(self): ) self.assertEqual(status.HTTP_400_BAD_REQUEST, response2.status_code) - def test_api_purl_validation_unsupported_package_type(self): data1 = { "purl": "pkg:random/foobar@1.1.0", @@ -1394,7 +1541,8 @@ def test_api_purl_validation_unsupported_package_type(self): self.assertEqual(True, response1.data["valid"]) self.assertEqual( - "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", response1.data["message"] + "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", response1.data[ + "message"] ) self.assertEqual(None, response1.data["exists"]) @@ -1517,7 +1665,8 @@ def test_api_package_watch_put_not_allowed(self): "/api/watch/pkg:npm/foobar/", data=data, content_type="application/json" ) - self.assertEqual(status.HTTP_405_METHOD_NOT_ALLOWED, response1.status_code) + self.assertEqual(status.HTTP_405_METHOD_NOT_ALLOWED, + response1.status_code) class ToGolangPurlTestCase(TestCase): diff --git a/packagedb/tests/test_filters.py b/packagedb/tests/test_filters.py index 358ead31..c4d5fb65 100644 --- a/packagedb/tests/test_filters.py +++ b/packagedb/tests/test_filters.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -80,5 +80,6 @@ def test_packagedb_filters_parse_query_string_to_lookups(self): } for query_string, expected in inputs.items(): - lookups = parse_query_string_to_lookups(query_string, "icontains", "name") + lookups = parse_query_string_to_lookups( + query_string, "icontains", "name") self.assertEqual(expected, str(lookups)) diff --git a/packagedb/tests/test_models.py b/packagedb/tests/test_models.py index 4353450f..aa9f3334 100644 --- a/packagedb/tests/test_models.py +++ b/packagedb/tests/test_models.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -34,7 +34,8 @@ def tearDown(self): Resource.objects.all().delete() def test_resource_is_created_on_a_package(self): - Resource.objects.create(package=self.package, path=self.resource_paths[0]) + Resource.objects.create(package=self.package, + path=self.resource_paths[0]) self.assertEqual(1, Resource.objects.all().count()) @@ -48,7 +49,8 @@ def test_duplicate_resources_are_not_created(self): for path in self.resource_paths: Resource.objects.create(package=self.package, path=path) for path in self.resource_paths: - self.assertRaises(IntegrityError, Resource.objects.create, package=self.package, path=path) + self.assertRaises( + IntegrityError, Resource.objects.create, package=self.package, path=path) self.assertEqual(2, Resource.objects.all().count()) @@ -56,7 +58,7 @@ def test_duplicate_resources_are_not_created(self): class PackageModelHistoryFieldTestCase(TransactionTestCase): def setUp(self): self.test_package = Package.objects.create( - download_url='https://test.com', + download_url='https://test.com', ) self.message0 = 'test-message0' self.message1 = 'test-message1' @@ -111,12 +113,16 @@ def setUp(self): 'version': '12.34', } - self.created_package = Package.objects.create(**self.created_package_data) - self.inserted_package = Package.objects.insert(**self.inserted_package_data) + self.created_package = Package.objects.create( + **self.created_package_data) + self.inserted_package = Package.objects.insert( + **self.inserted_package_data) def test_package_download_url_is_unique(self): - self.assertIsNone(Package.objects.insert(download_url=self.created_package_download_url)) - self.assertIsNone(Package.objects.insert(download_url=self.inserted_package_download_url)) + self.assertIsNone(Package.objects.insert( + download_url=self.created_package_download_url)) + self.assertIsNone(Package.objects.insert( + download_url=self.inserted_package_download_url)) def test_packagedb_package_model_history_field(self): self.created_package.append_to_history('test-message') @@ -125,9 +131,12 @@ def test_packagedb_package_model_history_field(self): self.assertEqual('test-message', entry.get('message')) def test_packagedb_package_model_get_all_versions(self): - p1 = Package.objects.create(download_url='http://a.a', type='generic', name='name', version='1.0') - p2 = Package.objects.create(download_url='http://b.b', type='generic', name='name', version='2.0') - p3 = Package.objects.create(download_url='http://c.c', type='generic', name='name', version='3.0') + p1 = Package.objects.create( + download_url='http://a.a', type='generic', name='name', version='1.0') + p2 = Package.objects.create( + download_url='http://b.b', type='generic', name='name', version='2.0') + p3 = Package.objects.create( + download_url='http://c.c', type='generic', name='name', version='3.0') p4 = Package.objects.create(download_url='http://d.d', type='generic', namespace='space', name='name', version='4.0') @@ -137,9 +146,12 @@ def test_packagedb_package_model_get_all_versions(self): self.assertEqual([p4], list(p4.get_all_versions())) def test_packagedb_package_model_get_latest_version(self): - p1 = Package.objects.create(download_url='http://a.a', name='name', version='1.0') - p2 = Package.objects.create(download_url='http://b.b', name='name', version='2.0') - p3 = Package.objects.create(download_url='http://c.c', name='name', version='3.0') + p1 = Package.objects.create( + download_url='http://a.a', name='name', version='1.0') + p2 = Package.objects.create( + download_url='http://b.b', name='name', version='2.0') + p3 = Package.objects.create( + download_url='http://c.c', name='name', version='3.0') p4 = Package.objects.create(download_url='http://d.d', namespace='space', name='name', version='4.0') @@ -149,11 +161,13 @@ def test_packagedb_package_model_get_latest_version(self): self.assertEqual(p4, p4.get_latest_version()) def test_packagedb_package_model_update_fields(self): - p1 = Package.objects.create(download_url='http://a.a', name='name', version='1.0') + p1 = Package.objects.create( + download_url='http://a.a', name='name', version='1.0') self.assertFalse(p1.history) self.assertEqual('', p1.namespace) self.assertEqual(None, p1.homepage_url) - package, updated_fields = p1.update_fields(namespace='test', homepage_url='https://example.com') + package, updated_fields = p1.update_fields( + namespace='test', homepage_url='https://example.com') self.assertEqual( sorted(updated_fields), sorted(['homepage_url', 'history', 'namespace']) @@ -184,7 +198,8 @@ def test_packagedb_package_model_update_fields(self): self.assertEqual(expected_history_entry, history_entry) def test_packagedb_package_model_update_fields_special_cases(self): - p1 = Package.objects.create(download_url='http://a.a', name='name', version='1.0') + p1 = Package.objects.create( + download_url='http://a.a', name='name', version='1.0') # Test dates date_fields = [ 'created_date', @@ -213,7 +228,7 @@ def test_packagedb_package_model_update_fields_special_cases(self): 'classifier': 'sources', 'type': 'war', } - string_qualifiers1='classifier=sources&type=war' + string_qualifiers1 = 'classifier=sources&type=war' package, updated_fields = p1.update_fields(qualifiers=dict_qualifiers1) self.assertEqual( sorted(['qualifiers', 'history']), @@ -223,8 +238,9 @@ def test_packagedb_package_model_update_fields_special_cases(self): string_qualifiers1, p1.qualifiers ) - string_qualifiers2='classifier=somethingelse' - package, updated_fields = p1.update_fields(qualifiers=string_qualifiers2) + string_qualifiers2 = 'classifier=somethingelse' + package, updated_fields = p1.update_fields( + qualifiers=string_qualifiers2) self.assertEqual( sorted(['qualifiers', 'history']), sorted(updated_fields), @@ -287,7 +303,8 @@ def test_packagedb_package_model_update_fields_special_cases(self): self.assertEqual(expected_history, history) def test_packagedb_package_model_update_fields_related_models(self): - p1 = Package.objects.create(download_url='http://a.a', name='name', version='1.0') + p1 = Package.objects.create( + download_url='http://a.a', name='name', version='1.0') path = 'asdf' resources = [Resource(package=p1, path=path)] _, updated_fields = p1.update_fields(resources=resources) @@ -300,7 +317,8 @@ def test_packagedb_package_model_update_fields_related_models(self): history_message = p1.history[0]['message'] self.assertEqual(expected_message, history_message) - p2 = Package.objects.create(download_url='http://b.b', name='example', version='1.0') + p2 = Package.objects.create( + download_url='http://b.b', name='example', version='1.0') resources = [ { "path": "example.jar", @@ -350,15 +368,16 @@ def test_packagedb_package_model_update_fields_related_models(self): history_message = p2.history[0]['message'] self.assertEqual(expected_message, history_message) - p3 = Package.objects.create(download_url='http://foo', name='foo', version='1.0') + p3 = Package.objects.create( + download_url='http://foo', name='foo', version='1.0') parties = [ - dict( + dict( type='admin', role='admin', name='foo', email='foo@foo.com', url='foo.com', - ) + ) ] _, updated_fields = p3.update_fields(parties=parties) self.assertEqual( @@ -370,16 +389,17 @@ def test_packagedb_package_model_update_fields_related_models(self): history_message = p3.history[0]['message'] self.assertEqual(expected_message, history_message) - p4 = Package.objects.create(download_url='http://bar', name='bar', version='1.0') + p4 = Package.objects.create( + download_url='http://bar', name='bar', version='1.0') parties = [ - Party( + Party( package=p4, type='admin', role='admin', name='bar', email='bar@bar.com', url='foo.com', - ) + ) ] _, updated_fields = p4.update_fields(parties=parties) self.assertEqual( @@ -391,7 +411,8 @@ def test_packagedb_package_model_update_fields_related_models(self): history_message = p4.history[0]['message'] self.assertEqual(expected_message, history_message) - p5 = Package.objects.create(download_url='http://baz', name='baz', version='1.0') + p5 = Package.objects.create( + download_url='http://baz', name='baz', version='1.0') dependencies = [ dict( purl='pkg:baz_dep@1.0', @@ -412,7 +433,8 @@ def test_packagedb_package_model_update_fields_related_models(self): history_message = p5.history[0]['message'] self.assertEqual(expected_message, history_message) - p6 = Package.objects.create(download_url='http://qux', name='qux', version='1.0') + p6 = Package.objects.create( + download_url='http://qux', name='qux', version='1.0') dependencies = [ DependentPackage( package=p6, @@ -435,7 +457,8 @@ def test_packagedb_package_model_update_fields_related_models(self): self.assertEqual(expected_message, history_message) def test_packagedb_package_model_update_fields_exceptions(self): - p1 = Package.objects.create(download_url='http://a.a', name='name', version='1.0') + p1 = Package.objects.create( + download_url='http://a.a', name='name', version='1.0') with self.assertRaises(AttributeError): p1.update_fields(asdf=123) @@ -501,16 +524,16 @@ def test_package_watch_reschedule_on_modification(self, mock_create_new_job): self.package_watch1.watch_interval = 1 self.package_watch1.save() - self.assertEqual("reschedule_id_new_interval", self.package_watch1.schedule_work_id) + self.assertEqual("reschedule_id_new_interval", + self.package_watch1.schedule_work_id) self.package_watch1.is_active = False self.package_watch1.save() self.assertEqual(None, self.package_watch1.schedule_work_id) - - def test_get_or_none(self): - Package.objects.create(download_url='http://a.ab', name='name', version='1.0', type="foo") + Package.objects.create(download_url='http://a.ab', + name='name', version='1.0', type="foo") package = Package.objects.filter( download_url="http://a.ab" ).get_or_none() diff --git a/packagedb/tests/test_schedules.py b/packagedb/tests/test_schedules.py index 36c23756..c2c5ae02 100644 --- a/packagedb/tests/test_schedules.py +++ b/packagedb/tests/test_schedules.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -39,5 +39,7 @@ def test_get_next_execution(): with patch("datetime.datetime", wraps=datetime.datetime) as dt: dt.now.return_value = time_now - assert expected1 == get_next_execution(watch_interval_days1, last_watch_date1) - assert expected2 == get_next_execution(watch_interval_days2, last_watch_date2) + assert expected1 == get_next_execution( + watch_interval_days1, last_watch_date1) + assert expected2 == get_next_execution( + watch_interval_days2, last_watch_date2) diff --git a/packagedb/tests/test_tasks.py b/packagedb/tests/test_tasks.py index e38a10fb..851a36e6 100644 --- a/packagedb/tests/test_tasks.py +++ b/packagedb/tests/test_tasks.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/packagedb/tests/test_throttling.py b/packagedb/tests/test_throttling.py index 96f16c2c..f21a26d8 100644 --- a/packagedb/tests/test_throttling.py +++ b/packagedb/tests/test_throttling.py @@ -3,7 +3,7 @@ # PurlDB is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -13,6 +13,7 @@ from django.contrib.auth.models import User + @patch('rest_framework.throttling.UserRateThrottle.get_rate', lambda x: '20/day') @patch('rest_framework.throttling.AnonRateThrottle.get_rate', lambda x: '10/day') class ThrottleApiTests(APITestCase): diff --git a/packagedb/tests/test_views.py b/packagedb/tests/test_views.py index 3276f30c..227e3885 100644 --- a/packagedb/tests/test_views.py +++ b/packagedb/tests/test_views.py @@ -3,7 +3,7 @@ # PurlDB is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/packagedb/throttling.py b/packagedb/throttling.py index 80c634d5..6f310043 100644 --- a/packagedb/throttling.py +++ b/packagedb/throttling.py @@ -3,7 +3,7 @@ # PurlDB is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # from rest_framework.exceptions import Throttled diff --git a/packagedb/to_purl.py b/packagedb/to_purl.py index 736aa4a6..0fd03afa 100644 --- a/packagedb/to_purl.py +++ b/packagedb/to_purl.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -21,7 +21,8 @@ @extend_schema( parameters=[ - OpenApiParameter("go_package", str, "query", description="go import package"), + OpenApiParameter("go_package", str, "query", + description="go import package"), ], responses={200: GoLangPurlResponseSerializer()}, ) @@ -29,7 +30,7 @@ class GolangPurlViewSet(viewsets.ViewSet): """ Return a ``golang_purl`` PackageURL from a standard go import string or a go.mod string ``go_package``. - + For example: >>> get_golang_purl("github.com/gorilla/mux v1.8.1").to_string() diff --git a/purl2vcs/CHANGELOG.rst b/purl2vcs/CHANGELOG.rst index 56dc7c65..0cd4cb54 100644 --- a/purl2vcs/CHANGELOG.rst +++ b/purl2vcs/CHANGELOG.rst @@ -5,7 +5,7 @@ v1.0.2 ------ Fix bug when collecting versions. -Reference: https://github.com/nexB/purldb/issues/486 +Reference: https://github.com/aboutcode-org/purldb/issues/486 v1.0.1 diff --git a/purl2vcs/NOTICE b/purl2vcs/NOTICE index 480e2970..95fbda27 100644 --- a/purl2vcs/NOTICE +++ b/purl2vcs/NOTICE @@ -7,6 +7,6 @@ # See https://www.apache.org/licenses/LICENSE-2.0 for the license text. # See https://creativecommons.org/licenses/by-sa/4.0/legalcode for the license text. # -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/purl2vcs/README.rst b/purl2vcs/README.rst index 0e05411a..9ccfad81 100644 --- a/purl2vcs/README.rst +++ b/purl2vcs/README.rst @@ -67,6 +67,6 @@ See https://www.apache.org/licenses/LICENSE-2.0 for the license text. See https://creativecommons.org/licenses/by-sa/4.0/legalcode for the license text. -See https://github.com/nexB/purldb for support or download. +See https://github.com/aboutcode-org/purldb for support or download. See https://aboutcode.org for more information about nexB OSS projects. diff --git a/purl2vcs/pyproject.toml b/purl2vcs/pyproject.toml index f7e4516f..f4655085 100644 --- a/purl2vcs/pyproject.toml +++ b/purl2vcs/pyproject.toml @@ -53,7 +53,7 @@ dependencies = [ ] [project.urls] -Homepage = "https://github.com/nexB/purldb" +Homepage = "https://github.com/aboutcode-org/purldb" [project.optional-dependencies] @@ -77,7 +77,7 @@ allow_dirty = true files = [ { filename = "pyproject.toml" }, - + ] diff --git a/purl2vcs/src/purl2vcs/find_source_repo.py b/purl2vcs/src/purl2vcs/find_source_repo.py index 34db4861..a3bd708c 100644 --- a/purl2vcs/src/purl2vcs/find_source_repo.py +++ b/purl2vcs/src/purl2vcs/find_source_repo.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -165,10 +165,13 @@ def get_source_package_and_add_to_package_set(package): package_content=PackageContentType.SOURCE_REPO, ) add_package_to_scan_queue(source_package) - logger.info(f"Created source repo package {source_purl} for {package.purl}") - package_set_uuids = [item["uuid"] for item in package.package_sets.all().values("uuid")] + logger.info( + f"Created source repo package {source_purl} for {package.purl}") + package_set_uuids = [item["uuid"] + for item in package.package_sets.all().values("uuid")] package_set_ids = set(package_set_uuids) - source_package_set_ids = set(source_package.package_sets.all().values_list("uuid")) + source_package_set_ids = set( + source_package.package_sets.all().values_list("uuid")) # If the package exists and already in the set then there is nothing left to do if package_set_ids.intersection(source_package_set_ids): @@ -215,12 +218,13 @@ def get_repo_urls(package: Package) -> Generator[str, None, None]: """ Return the URL of the source repository of a package """ - source_urls = get_source_urls_from_package_data_and_resources(package=package) + source_urls = get_source_urls_from_package_data_and_resources( + package=package) if source_urls: yield from source_urls # TODO: Use univers to sort versions - # TODO: Also consider using dates https://github.com/nexB/purldb/issues/136 + # TODO: Also consider using dates https://github.com/aboutcode-org/purldb/issues/136 for version_package in package.get_all_versions().order_by("-version"): source_urls = get_source_urls_from_package_data_and_resources( package=version_package @@ -230,7 +234,8 @@ def get_repo_urls(package: Package) -> Generator[str, None, None]: if package.type == "maven": yield from get_source_urls_from_package_data_and_resources( - package=get_merged_ancestor_package_from_maven_package(package=package) + package=get_merged_ancestor_package_from_maven_package( + package=package) ) @@ -277,7 +282,8 @@ def convert_repo_url_to_purls(source_url): source_url = source_url.replace("https+//", "https://") if ( - source_url.startswith("git+https://") or source_url.startswith("git://") + source_url.startswith( + "git+https://") or source_url.startswith("git://") ) and "@" in source_url: # remove the commit from the end of the URL source_url, _, _ = source_url.rpartition("@") @@ -344,7 +350,8 @@ def get_urls_from_package_data(package) -> Generator[str, None, None]: homepage_text = homepage_response and homepage_response.text found_urls.extend(get_urls_from_text(text=homepage_text)) - repository_homepage_response = fetch_response(url=package.repository_homepage_url) + repository_homepage_response = fetch_response( + url=package.repository_homepage_url) repository_homepage_text = ( repository_homepage_response and repository_homepage_response.text ) diff --git a/purl2vcs/tests/test_find_source_repo.py b/purl2vcs/tests/test_find_source_repo.py index 38bce2a2..a7e7c526 100644 --- a/purl2vcs/tests/test_find_source_repo.py +++ b/purl2vcs/tests/test_find_source_repo.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -29,7 +29,8 @@ ) BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "testfiles", "find_source_repo", "tags_commits.txt") +TEST_DATA = os.path.join(BASE_DIR, "testfiles", + "find_source_repo", "tags_commits.txt") TAGS_COMMITS_FILE = os.path.join( BASE_DIR, "testfiles", "find_source_repo", "tags_commits_list.txt" ) @@ -134,7 +135,8 @@ def test_get_source_purl_from_package_data(self, mock_text, mock_response): ] mock_text.side_effect = [["https://bitbucket/ab/cd"], [], [], []] source_urls = list( - get_urls_from_package_data(self.package_with_resources_and_package_data) + get_urls_from_package_data( + self.package_with_resources_and_package_data) ) assert source_urls == [ "https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions/src/master/", @@ -187,14 +189,16 @@ def test_get_repo_urls(self, mock1, mock2): mock1.return_value = None mock2.return_value = [] source_urls = list( - get_repo_urls(package=self.package_without_resources_and_package_data) + get_repo_urls( + package=self.package_without_resources_and_package_data) ) assert source_urls == [ "https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions", "https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions", "https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions/tree/master/oauth-oidc-sdk/src/main/resources/META-INF/MANIFEST.MF", ] - source_urls = list(get_repo_urls(package=self.package_without_versions)) + source_urls = list(get_repo_urls( + package=self.package_without_versions)) assert source_urls == [] source_urls = list( get_repo_urls(package=self.package_with_resources_and_package_data) @@ -312,7 +316,8 @@ def test_get_source_repo(self): namespace="connect2id", name="oauth-2.0-sdk-with-openid-connect-extensions", version="9.36", - qualifiers={"commit": "e86fb3431972d302fcb615aca0baed4d8ab89791"}, + qualifiers={ + "commit": "e86fb3431972d302fcb615aca0baed4d8ab89791"}, subpath=None, ) @@ -326,7 +331,8 @@ def test_fetch_response(self, mock_get): def test_from_purl_to_git(self): response = self.client.get( "/api/from_purl/purl2git", - data={"package_url": str(self.package_without_resources_and_package_data)}, + data={"package_url": str( + self.package_without_resources_and_package_data)}, follow=True, ) expected = "pkg:bitbucket/connect2id/oauth-2.0-sdk-with-openid-connect-extensions@9.36?commit=e86fb3431972d302fcb615aca0baed4d8ab89791" diff --git a/purldb-toolkit/NOTICE b/purldb-toolkit/NOTICE index 480e2970..95fbda27 100644 --- a/purldb-toolkit/NOTICE +++ b/purldb-toolkit/NOTICE @@ -7,6 +7,6 @@ # See https://www.apache.org/licenses/LICENSE-2.0 for the license text. # See https://creativecommons.org/licenses/by-sa/4.0/legalcode for the license text. # -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/purldb-toolkit/README.rst b/purldb-toolkit/README.rst index 7c85e360..cbed4b74 100644 --- a/purldb-toolkit/README.rst +++ b/purldb-toolkit/README.rst @@ -29,12 +29,12 @@ The purlcli command exposes multiple subcommands. Run this to command to get bas purlcli --help Usage: purlcli [OPTIONS] COMMAND [ARGS]... - + Return information for a PURL or list of PURLs. - + Options: --help Show this message and exit. - + Commands: d2d Run deploy-to-devel "back2source" analysis between packages. metadata Fetch package metadata for a PURL. @@ -555,20 +555,20 @@ Examples **Run a d2d analysis between two Java JARs (source and binary)** You first need to install and run matchcode locally so you have the endpoint accessible. Starting -from a https://github.com/nexB/purldb/ clone:: +from a https://github.com/aboutcode-org/purldb/ clone:: - git clone https://github.com/nexB/purldb + git clone https://github.com/aboutcode-org/purldb cd purldb make dev make envfile - SECRET_KEY="1" make postgres_matchcodeio - SECRET_KEY="1" make run_matchcodeio + SECRET_KEY="1" make postgres_matchcodeio + SECRET_KEY="1" make run_matchcodeio Then in another shell:: cd purldb source venv/bin/activate - + Finally run the command: .. code-block:: console @@ -676,6 +676,6 @@ See https://www.apache.org/licenses/LICENSE-2.0 for the license text. See https://creativecommons.org/licenses/by-sa/4.0/legalcode for the license text. -See https://github.com/nexB/purldb for support or download. +See https://github.com/aboutcode-org/purldb for support or download. See https://aboutcode.org for more information about nexB OSS projects. diff --git a/purldb-toolkit/pyproject.toml b/purldb-toolkit/pyproject.toml index 18009d9a..8a25c390 100644 --- a/purldb-toolkit/pyproject.toml +++ b/purldb-toolkit/pyproject.toml @@ -43,7 +43,7 @@ dependencies = [ "requests >= 2.7.0", ] -urls = { Homepage = "https://github.com/nexB/purldb/tree/main/purldb-toolkit" } +urls = { Homepage = "https://github.com/aboutcode-org/purldb/tree/main/purldb-toolkit" } [project.optional-dependencies] diff --git a/purldb-toolkit/src/purldb_toolkit/purlcli.py b/purldb-toolkit/src/purldb_toolkit/purlcli.py index 431c9164..d2e5cd1d 100644 --- a/purldb-toolkit/src/purldb_toolkit/purlcli.py +++ b/purldb-toolkit/src/purldb_toolkit/purlcli.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -83,7 +83,8 @@ def get_metadata(purls, output, file, unique): context = click.get_current_context() command_name = context.command.name - metadata_info = get_metadata_details(purls, output, file, unique, command_name) + metadata_info = get_metadata_details( + purls, output, file, unique, command_name) json.dump(metadata_info, output, indent=4) @@ -352,7 +353,8 @@ def get_urls(purls, output, file, unique, head): context = click.get_current_context() command_name = context.command.name - urls_info = get_urls_details(purls, output, file, unique, head, command_name) + urls_info = get_urls_details( + purls, output, file, unique, head, command_name) json.dump(urls_info, output, indent=4) @@ -403,7 +405,8 @@ def get_urls_details(purls, output, file, unique, head, command_name): {"url": inferred} for inferred in purl2url.get_inferred_urls(purl) ] - url_detail["repo_download_url"] = {"url": purl2url.get_repo_download_url(purl)} + url_detail["repo_download_url"] = { + "url": purl2url.get_repo_download_url(purl)} url_detail["repo_download_url_by_package_type"] = { "url": purl2url.get_repo_download_url_by_package_type( @@ -596,7 +599,8 @@ def validate(purls, output, file, unique): context = click.get_current_context() command_name = context.command.name - validated_purls = get_validate_details(purls, output, file, unique, command_name) + validated_purls = get_validate_details( + purls, output, file, unique, command_name) json.dump(validated_purls, output, indent=4) @@ -702,7 +706,8 @@ def validate_purl(purl): except json.decoder.JSONDecodeError as e: - print(f"validate_purl(): json.decoder.JSONDecodeError for '{purl}': {e}") + print( + f"validate_purl(): json.decoder.JSONDecodeError for '{purl}': {e}") logging.basicConfig( filename=LOG_FILE_LOCATION, @@ -711,14 +716,16 @@ def validate_purl(purl): filemode="w", ) - logger.error(f"validate_purl(): json.decoder.JSONDecodeError for '{purl}': {e}") + logger.error( + f"validate_purl(): json.decoder.JSONDecodeError for '{purl}': {e}") except Exception as e: print(f"'validate' endpoint error for '{purl}': {e}") else: if response is None: - print(f"'{purl}' -- response.status_code for None = {response.status_code}") + print( + f"'{purl}' -- response.status_code for None = {response.status_code}") return response @@ -765,7 +772,8 @@ def get_versions(purls, output, file, unique): context = click.get_current_context() command_name = context.command.name - purl_versions = get_versions_details(purls, output, file, unique, command_name) + purl_versions = get_versions_details( + purls, output, file, unique, command_name) json.dump(purl_versions, output, indent=4) @@ -991,9 +999,12 @@ def get_package_pairs_for_d2d(packages): for content, content_packages in groupby(packages, key=lambda p: p.package_content): packages_by_content[content] = list(content_packages) - source_repo_packages = packages_by_content.get(PackageContentType.SOURCE_REPO.name.lower(), []) - source_archive_packages = packages_by_content.get(PackageContentType.SOURCE_ARCHIVE.name.lower(), []) - binary_packages = packages_by_content.get(PackageContentType.BINARY.name.lower(), []) + source_repo_packages = packages_by_content.get( + PackageContentType.SOURCE_REPO.name.lower(), []) + source_archive_packages = packages_by_content.get( + PackageContentType.SOURCE_ARCHIVE.name.lower(), []) + binary_packages = packages_by_content.get( + PackageContentType.BINARY.name.lower(), []) yield from generate_d2d_package_pairs(from_packages=source_repo_packages, to_packages=binary_packages) yield from generate_d2d_package_pairs(from_packages=source_archive_packages, to_packages=binary_packages) @@ -1007,16 +1018,19 @@ def validate_purls_for_d2d(ctx, param, value): purls = value len_purls = len(purls) if len_purls > 2: - raise click.BadParameter("Invalid number of --purl options. Only one or two options are allowed.") + raise click.BadParameter( + "Invalid number of --purl options. Only one or two options are allowed.") if len_purls == 1: if not purls[0].startswith("pkg:"): - raise click.BadParameter(f"Invalid PURL: {purls[0]!r}. Must start with `pkg:`") + raise click.BadParameter( + f"Invalid PURL: {purls[0]!r}. Must start with `pkg:`") else: return value elif len_purls != 2: - raise click.BadParameter(f"Invalid number of --purl options. There should be exactly two --purl options.") + raise click.BadParameter( + f"Invalid number of --purl options. There should be exactly two --purl options.") elif not (all_purls(purls) or all_urls(purls)): purls = '\n'.join(purls) @@ -1177,7 +1191,8 @@ def run_d2d_purl_set(purl, purldb_api_url, matchcode_api_url): for d2d_packages in get_packages_by_set(purl, purldb_api_url): package_pairs = get_package_pairs_for_d2d(d2d_packages) for package_pair in package_pairs: - click.echo(f"Running D2D for: {package_pair.from_package.purl} -> {package_pair.to_package.purl}", err=True) + click.echo( + f"Running D2D for: {package_pair.from_package.purl} -> {package_pair.to_package.purl}", err=True) run_id, project_url = map_deploy_to_devel( from_purl=package_pair.from_package.purl, to_purl=package_pair.to_package.purl, @@ -1199,10 +1214,12 @@ def run_d2d_purl_set(purl, purldb_api_url, matchcode_api_url): continue # TODO: Use a better progress indicator. click.echo(".", err=True) - data = get_run_data(matchcode_api_url=matchcode_api_url, run_id=run_id) + data = get_run_data( + matchcode_api_url=matchcode_api_url, run_id=run_id) if data.get("status") != "running": project.done = True - project.result = get_project_results(project_url=project.project_url) + project.result = get_project_results( + project_url=project.project_url) time.sleep(1) time.sleep(POLLING_INTERVAL) if all(project.done for project in projects): @@ -1235,11 +1252,13 @@ def map_deploy_to_devel(from_purl, to_purl, purldb_api_url, matchcode_api_url): """ from_url = get_download_url(purl=from_purl, purldb_api_url=purldb_api_url) if not from_url: - raise Exception(f"Could not find download URL for the `from` PURL: {from_purl}.") + raise Exception( + f"Could not find download URL for the `from` PURL: {from_purl}.") to_url = get_download_url(purl=to_purl, purldb_api_url=purldb_api_url) if not to_url: - raise Exception(f"Could not find download URL for the `to` PURL: {to_url}.") + raise Exception( + f"Could not find download URL for the `to` PURL: {to_url}.") return map_deploy_to_devel_urls( from_url=from_url, @@ -1254,11 +1273,13 @@ def map_deploy_to_devel_urls(from_url, to_url, matchcode_api_url): """ input_urls = (f"{from_url}#from", f"{to_url}#to",) - d2d_launch = launch_d2d(input_urls=input_urls, matchcode_api_url=matchcode_api_url) + d2d_launch = launch_d2d(input_urls=input_urls, + matchcode_api_url=matchcode_api_url) project_url = d2d_launch.get("url") or None run_url = d2d_launch.get("runs") if not run_url: - raise Exception(f"Could not find a run URL for the input URLs {input_urls!r}.") + raise Exception( + f"Could not find a run URL for the input URLs {input_urls!r}.") return run_url[0], project_url @@ -1270,7 +1291,7 @@ def launch_d2d(matchcode_api_url, input_urls): """ url = urljoin(matchcode_api_url, "d2d/") headers = {'Content-Type': 'application/json'} - payload = json.dumps({"input_urls":input_urls, "runs":[]}) + payload = json.dumps({"input_urls": input_urls, "runs": []}) d2d_results = requests.post( url=url, data=payload, diff --git a/purldb-toolkit/tests/test_purlcli.py b/purldb-toolkit/tests/test_purlcli.py index 91e364a7..dd817464 100644 --- a/purldb-toolkit/tests/test_purlcli.py +++ b/purldb-toolkit/tests/test_purlcli.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -39,7 +39,8 @@ def test_metadata_cli_duplicate_input_sources(self): "-", ] runner = CliRunner() - result = runner.invoke(purlcli.get_metadata, options, catch_exceptions=False) + result = runner.invoke(purlcli.get_metadata, + options, catch_exceptions=False) assert "Use either purls or file but not both." in result.output assert result.exit_code == 2 @@ -52,7 +53,8 @@ def test_metadata_cli_no_input_sources(self): "-", ] runner = CliRunner() - result = runner.invoke(purlcli.get_metadata, options, catch_exceptions=False) + result = runner.invoke(purlcli.get_metadata, + options, catch_exceptions=False) assert "Error: Use either purls" in result.output assert result.exit_code == 2 @@ -608,7 +610,8 @@ def test_normalize_purls( self, test_input, expected_input_purls, expected_normalized_purls ): unique = True - input_purls, normalized_purls = purlcli.normalize_purls(test_input[0], unique) + input_purls, normalized_purls = purlcli.normalize_purls( + test_input[0], unique) assert input_purls == expected_input_purls assert normalized_purls == expected_normalized_purls @@ -758,7 +761,8 @@ def test_urls_cli_head(self, mock_make_head_request): expected_result_file = test_env.get_test_loc( "purlcli/expected_urls_output_head_mock.json" ) - actual_result_file = test_env.get_temp_file("actual_urls_output_head_mock.json") + actual_result_file = test_env.get_temp_file( + "actual_urls_output_head_mock.json") options = [ "--purl", "pkg:pypi/fetchcode", @@ -767,7 +771,8 @@ def test_urls_cli_head(self, mock_make_head_request): actual_result_file, ] runner = CliRunner() - result = runner.invoke(purlcli.get_urls, options, catch_exceptions=False) + result = runner.invoke(purlcli.get_urls, options, + catch_exceptions=False) assert result.exit_code == 0 with open(actual_result_file) as f_output: @@ -783,7 +788,8 @@ def test_urls_cli_head(self, mock_make_head_request): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), + (output_data["headers"][0]["purls"], + expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -827,7 +833,8 @@ def test_urls_cli_duplicate_input_sources(self): "-", ] runner = CliRunner() - result = runner.invoke(purlcli.get_urls, options, catch_exceptions=False) + result = runner.invoke(purlcli.get_urls, options, + catch_exceptions=False) assert "Use either purls or file but not both." in result.output assert result.exit_code == 2 @@ -840,7 +847,8 @@ def test_urls_cli_no_input_sources(self): "-", ] runner = CliRunner() - result = runner.invoke(purlcli.get_urls, options, catch_exceptions=False) + result = runner.invoke(purlcli.get_urls, options, + catch_exceptions=False) assert "Use either purls or file." in result.output assert result.exit_code == 2 diff --git a/purldb-toolkit/tests/test_purlcli_live.py b/purldb-toolkit/tests/test_purlcli_live.py index 929bc1af..024e7ad1 100644 --- a/purldb-toolkit/tests/test_purlcli_live.py +++ b/purldb-toolkit/tests/test_purlcli_live.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -34,7 +34,8 @@ def test_metadata_cli(self): expected_result_file = test_env.get_test_loc( "purlcli/expected_metadata_output.json" ) - actual_result_file = test_env.get_temp_file("actual_metadata_output.json") + actual_result_file = test_env.get_temp_file( + "actual_metadata_output.json") options = [ "--purl", "pkg:pypi/fetchcode", @@ -58,7 +59,8 @@ def test_metadata_cli(self): actual_result_file, ] runner = CliRunner() - result = runner.invoke(purlcli.get_metadata, options, catch_exceptions=False) + result = runner.invoke(purlcli.get_metadata, + options, catch_exceptions=False) assert result.exit_code == 0 with open(actual_result_file) as f_output: @@ -76,7 +78,8 @@ def test_metadata_cli(self): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), + (output_data["headers"][0]["purls"], + expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -116,7 +119,8 @@ def test_metadata_cli_unique(self): expected_result_file = test_env.get_test_loc( "purlcli/expected_metadata_output_unique.json" ) - actual_result_file = test_env.get_temp_file("actual_metadata_output.json") + actual_result_file = test_env.get_temp_file( + "actual_metadata_output.json") options = [ "--purl", "pkg:pypi/fetchcode", @@ -141,7 +145,8 @@ def test_metadata_cli_unique(self): "--unique", ] runner = CliRunner() - result = runner.invoke(purlcli.get_metadata, options, catch_exceptions=False) + result = runner.invoke(purlcli.get_metadata, + options, catch_exceptions=False) assert result.exit_code == 0 with open(actual_result_file) as f_output: @@ -159,7 +164,8 @@ def test_metadata_cli_unique(self): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), + (output_data["headers"][0]["purls"], + expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -204,7 +210,8 @@ def test_metadata_cli_duplicate_input_sources(self): "-", ] runner = CliRunner() - result = runner.invoke(purlcli.get_metadata, options, catch_exceptions=False) + result = runner.invoke(purlcli.get_metadata, + options, catch_exceptions=False) assert "Use either purls or file but not both." in result.output assert result.exit_code == 2 @@ -217,7 +224,8 @@ def test_metadata_cli_no_input_sources(self): "-", ] runner = CliRunner() - result = runner.invoke(purlcli.get_metadata, options, catch_exceptions=False) + result = runner.invoke(purlcli.get_metadata, + options, catch_exceptions=False) assert "Use either purls or file." in result.output assert result.exit_code == 2 @@ -777,7 +785,8 @@ def test_normalize_purls( self, test_input, expected_input_purls, expected_normalized_purls ): unique = True - input_purls, normalized_purls = purlcli.normalize_purls(test_input[0], unique) + input_purls, normalized_purls = purlcli.normalize_purls( + test_input[0], unique) assert input_purls == expected_input_purls assert normalized_purls == expected_normalized_purls @@ -957,7 +966,8 @@ def test_urls_cli(self): actual_result_file, ] runner = CliRunner() - result = runner.invoke(purlcli.get_urls, options, catch_exceptions=False) + result = runner.invoke(purlcli.get_urls, options, + catch_exceptions=False) assert result.exit_code == 0 with open(actual_result_file) as f_output: @@ -973,7 +983,8 @@ def test_urls_cli(self): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), + (output_data["headers"][0]["purls"], + expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -1007,7 +1018,8 @@ def test_urls_cli_unique(self): expected_result_file = test_env.get_test_loc( "purlcli/expected_urls_output_unique.json" ) - actual_result_file = test_env.get_temp_file("actual_urls_output_unique.json") + actual_result_file = test_env.get_temp_file( + "actual_urls_output_unique.json") options = [ "--purl", "pkg:pypi/fetchcode", @@ -1054,7 +1066,8 @@ def test_urls_cli_unique(self): "--unique", ] runner = CliRunner() - result = runner.invoke(purlcli.get_urls, options, catch_exceptions=False) + result = runner.invoke(purlcli.get_urls, options, + catch_exceptions=False) assert result.exit_code == 0 with open(actual_result_file) as f_output: @@ -1070,7 +1083,8 @@ def test_urls_cli_unique(self): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), + (output_data["headers"][0]["purls"], + expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -1104,7 +1118,8 @@ def test_urls_cli_head(self): expected_result_file = test_env.get_test_loc( "purlcli/expected_urls_output_head.json" ) - actual_result_file = test_env.get_temp_file("actual_urls_output_head.json") + actual_result_file = test_env.get_temp_file( + "actual_urls_output_head.json") options = [ "--purl", "pkg:pypi/fetchcode", @@ -1151,7 +1166,8 @@ def test_urls_cli_head(self): actual_result_file, ] runner = CliRunner() - result = runner.invoke(purlcli.get_urls, options, catch_exceptions=False) + result = runner.invoke(purlcli.get_urls, options, + catch_exceptions=False) assert result.exit_code == 0 with open(actual_result_file) as f_output: @@ -1167,7 +1183,8 @@ def test_urls_cli_head(self): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), + (output_data["headers"][0]["purls"], + expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -1211,7 +1228,8 @@ def test_urls_cli_duplicate_input_sources(self): "-", ] runner = CliRunner() - result = runner.invoke(purlcli.get_urls, options, catch_exceptions=False) + result = runner.invoke(purlcli.get_urls, options, + catch_exceptions=False) assert "Use either purls or file but not both." in result.output assert result.exit_code == 2 @@ -1224,7 +1242,8 @@ def test_urls_cli_no_input_sources(self): "-", ] runner = CliRunner() - result = runner.invoke(purlcli.get_urls, options, catch_exceptions=False) + result = runner.invoke(purlcli.get_urls, options, + catch_exceptions=False) assert "Use either purls or file." in result.output assert result.exit_code == 2 @@ -1574,7 +1593,8 @@ def test_validate_cli(self): expected_result_file = test_env.get_test_loc( "purlcli/expected_validate_output.json" ) - actual_result_file = test_env.get_temp_file("actual_validate_output.json") + actual_result_file = test_env.get_temp_file( + "actual_validate_output.json") options = [ "--purl", "pkg:pypi/fetchcode", @@ -1598,7 +1618,8 @@ def test_validate_cli(self): actual_result_file, ] runner = CliRunner() - result = runner.invoke(purlcli.validate, options, catch_exceptions=False) + result = runner.invoke(purlcli.validate, options, + catch_exceptions=False) assert result.exit_code == 0 with open(actual_result_file) as f_output: @@ -1612,7 +1633,8 @@ def test_validate_cli(self): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), + (output_data["headers"][0]["purls"], + expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -1647,7 +1669,8 @@ def test_validate_cli_unique(self): expected_result_file = test_env.get_test_loc( "purlcli/expected_validate_output_unique.json" ) - actual_result_file = test_env.get_temp_file("actual_validate_output.json") + actual_result_file = test_env.get_temp_file( + "actual_validate_output.json") options = [ "--purl", "pkg:pypi/fetchcode", @@ -1672,7 +1695,8 @@ def test_validate_cli_unique(self): "--unique", ] runner = CliRunner() - result = runner.invoke(purlcli.validate, options, catch_exceptions=False) + result = runner.invoke(purlcli.validate, options, + catch_exceptions=False) assert result.exit_code == 0 with open(actual_result_file) as f_output: @@ -1686,7 +1710,8 @@ def test_validate_cli_unique(self): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), + (output_data["headers"][0]["purls"], + expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -1876,7 +1901,8 @@ def test_versions_cli(self): expected_result_file = test_env.get_test_loc( "purlcli/expected_versions_output.json" ) - actual_result_file = test_env.get_temp_file("actual_versions_output.json") + actual_result_file = test_env.get_temp_file( + "actual_versions_output.json") options = [ "--purl", "pkg:pypi/fetchcode", @@ -1898,7 +1924,8 @@ def test_versions_cli(self): actual_result_file, ] runner = CliRunner() - result = runner.invoke(purlcli.get_versions, options, catch_exceptions=False) + result = runner.invoke(purlcli.get_versions, + options, catch_exceptions=False) assert result.exit_code == 0 with open(actual_result_file) as f_output: @@ -1914,7 +1941,8 @@ def test_versions_cli(self): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), + (output_data["headers"][0]["purls"], + expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -1950,7 +1978,8 @@ def test_versions_cli_unique(self): expected_result_file = test_env.get_test_loc( "purlcli/expected_versions_output_unique.json" ) - actual_result_file = test_env.get_temp_file("actual_versions_output.json") + actual_result_file = test_env.get_temp_file( + "actual_versions_output.json") options = [ "--purl", "pkg:pypi/fetchcode", @@ -1973,7 +2002,8 @@ def test_versions_cli_unique(self): "--unique", ] runner = CliRunner() - result = runner.invoke(purlcli.get_versions, options, catch_exceptions=False) + result = runner.invoke(purlcli.get_versions, + options, catch_exceptions=False) assert result.exit_code == 0 with open(actual_result_file) as f_output: @@ -1989,7 +2019,8 @@ def test_versions_cli_unique(self): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), + (output_data["headers"][0]["purls"], + expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], diff --git a/purldb_project/__init__.py b/purldb_project/__init__.py index 7399cfd5..7b44e1e6 100644 --- a/purldb_project/__init__.py +++ b/purldb_project/__init__.py @@ -3,7 +3,7 @@ # PurlDB is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/purldb_project/settings.py b/purldb_project/settings.py index a9592932..1138a317 100644 --- a/purldb_project/settings.py +++ b/purldb_project/settings.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -32,7 +32,8 @@ SECRET_KEY = env.str("SECRET_KEY") -ALLOWED_HOSTS = env.list("ALLOWED_HOSTS", default=[".localhost", "127.0.0.1", "[::1]"]) +ALLOWED_HOSTS = env.list("ALLOWED_HOSTS", default=[ + ".localhost", "127.0.0.1", "[::1]"]) # SECURITY WARNING: do not run with debug turned on in production DEBUG = env.bool("PURLDB_DEBUG", default=False) @@ -240,7 +241,8 @@ # Django restframework -REST_FRAMEWORK_DEFAULT_THROTTLE_RATES = {'anon': '3600/hour', 'user': '10800/hour'} +REST_FRAMEWORK_DEFAULT_THROTTLE_RATES = { + 'anon': '3600/hour', 'user': '10800/hour'} REST_FRAMEWORK = { 'DEFAULT_AUTHENTICATION_CLASSES': ('rest_framework.authentication.TokenAuthentication',), diff --git a/purldb_project/urls.py b/purldb_project/urls.py index 5fd3c993..36e17371 100644 --- a/purldb_project/urls.py +++ b/purldb_project/urls.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -36,23 +36,27 @@ api_router.register('resources', ResourceViewSet) api_router.register('validate', PurlValidateViewSet, 'validate') api_router.register('collect', CollectViewSet, 'collect') -api_router.register('watch',PackageWatchViewSet) +api_router.register('watch', PackageWatchViewSet) api_router.register('scan_queue', ScannableURIViewSet) -api_router.register('approximate_directory_content_index', ApproximateDirectoryContentIndexViewSet) -api_router.register('approximate_directory_structure_index', ApproximateDirectoryStructureIndexViewSet) +api_router.register('approximate_directory_content_index', + ApproximateDirectoryContentIndexViewSet) +api_router.register('approximate_directory_structure_index', + ApproximateDirectoryStructureIndexViewSet) urlpatterns = [ path( 'robots.txt', - TemplateView.as_view(template_name='robots.txt', content_type='text/plain'), + TemplateView.as_view(template_name='robots.txt', + content_type='text/plain'), ), path('api/', include((api_router.urls, 'api'))), path('api/to_purl/', include((api_to_purl_router.urls, 'api_to'))), path('api/from_purl/', include((api_from_purl_router.urls, 'api_from'))), path("", RedirectView.as_view(url="api/")), path('api/schema/', SpectacularAPIView.as_view(), name='schema'), - path('api/docs/', SpectacularSwaggerView.as_view(url_name='schema'), name='swagger-ui'), + path('api/docs/', SpectacularSwaggerView.as_view(url_name='schema'), + name='swagger-ui'), path( 'api/scan_queue/index_package_scan//', index_package_scan, name='index_package_scan' ), diff --git a/purldb_project/wsgi.py b/purldb_project/wsgi.py index 60836189..485a66e1 100644 --- a/purldb_project/wsgi.py +++ b/purldb_project/wsgi.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/purldb_public_project/__init__.py b/purldb_public_project/__init__.py index ba4a78f1..adb81bf9 100644 --- a/purldb_public_project/__init__.py +++ b/purldb_public_project/__init__.py @@ -3,7 +3,7 @@ # PurlDB is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -15,5 +15,6 @@ def command_line(): '''Command line entry point.''' from django.core.management import execute_from_command_line - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'purldb_public_project.settings') + os.environ.setdefault('DJANGO_SETTINGS_MODULE', + 'purldb_public_project.settings') execute_from_command_line(sys.argv) diff --git a/purldb_public_project/settings.py b/purldb_public_project/settings.py index 18961615..5e6d5bee 100644 --- a/purldb_public_project/settings.py +++ b/purldb_public_project/settings.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/purldb_public_project/urls.py b/purldb_public_project/urls.py index 31f4fc20..14e5546a 100644 --- a/purldb_public_project/urls.py +++ b/purldb_public_project/urls.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -30,10 +30,12 @@ urlpatterns = [ path( 'robots.txt', - TemplateView.as_view(template_name='robots.txt', content_type='text/plain'), + TemplateView.as_view(template_name='robots.txt', + content_type='text/plain'), ), path('api/', include((api_router.urls, 'api'))), path('', RedirectView.as_view(url='api/')), path('api/schema/', SpectacularAPIView.as_view(), name='schema'), - path('api/docs/', SpectacularSwaggerView.as_view(url_name='schema'), name='swagger-ui'), + path('api/docs/', SpectacularSwaggerView.as_view(url_name='schema'), + name='swagger-ui'), ] diff --git a/purldb_public_project/wsgi.py b/purldb_public_project/wsgi.py index 75add9c5..f1448671 100644 --- a/purldb_public_project/wsgi.py +++ b/purldb_public_project/wsgi.py @@ -3,7 +3,7 @@ # purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. +# See https://github.com/aboutcode-org/purldb for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -18,6 +18,7 @@ """ -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'purldb_public_project.settings') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', + 'purldb_public_project.settings') application = get_wsgi_application() diff --git a/setup.cfg b/setup.cfg index 11742517..cb476757 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,7 +13,7 @@ license = Apache-2.0 AND CC-BY-SA-4.0 description = A Django-based utility to collect Package data and Resources for collection and matching. long_description = file:README.rst long_description_content_type = text/x-rst -url = https://github.com/nexB/purldb +url = https://github.com/aboutcode-org/purldb classifiers = Intended Audience :: Developers