diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d85945392..284b04dae 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,6 +17,14 @@ v34.6.0 (unreleased) Project error message. https://github.com/nexB/scancode.io/issues/1249 +- Rename DiscoveredDependency ``resolved_to`` to ``resolved_to_package``, and + ``resolved_dependencies`` to ``resolved_from_dependencies`` for clarity and + consistency. + Add ``children_packages`` and ``parent_packages`` ManyToMany field on the + DiscoveredPackage model. + Add full dependency tree in the CycloneDX output. + https://github.com/nexB/scancode.io/issues/1066 + v34.5.0 (2024-05-22) -------------------- diff --git a/scanpipe/api/serializers.py b/scanpipe/api/serializers.py index f689d8aa3..b6fdf0de3 100644 --- a/scanpipe/api/serializers.py +++ b/scanpipe/api/serializers.py @@ -392,7 +392,7 @@ class Meta: class DiscoveredDependencySerializer(serializers.ModelSerializer): purl = serializers.ReadOnlyField() for_package_uid = serializers.ReadOnlyField() - resolved_to_uid = serializers.ReadOnlyField() + resolved_to_package_uid = serializers.ReadOnlyField() datafile_path = serializers.ReadOnlyField() package_type = serializers.ReadOnlyField(source="type") @@ -407,7 +407,7 @@ class Meta: "is_resolved", "dependency_uid", "for_package_uid", - "resolved_to_uid", + "resolved_to_package_uid", "datafile_path", "datasource_id", "package_type", diff --git a/scanpipe/filters.py b/scanpipe/filters.py index a7ca46601..4c32b13f7 100644 --- a/scanpipe/filters.py +++ b/scanpipe/filters.py @@ -748,7 +748,7 @@ class DependencyFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): "is_optional", "is_resolved", "for_package", - "resolved_to", + "resolved_to_package", "datafile_resource", "datasource_id", ], diff --git a/scanpipe/migrations/0060_discovereddependency_renames.py b/scanpipe/migrations/0060_discovereddependency_renames.py new file mode 100644 index 000000000..dbdfd267b --- /dev/null +++ b/scanpipe/migrations/0060_discovereddependency_renames.py @@ -0,0 +1,41 @@ +# Generated by Django 5.0.6 on 2024-06-03 11:32 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("scanpipe", "0059_alter_codebaseresource_status"), + ] + + operations = [ + migrations.RenameField( + model_name="discovereddependency", + old_name="resolved_to", + new_name="resolved_to_package", + ), + migrations.AlterField( + model_name="discovereddependency", + name="resolved_to_package", + field=models.ForeignKey( + blank=True, + editable=False, + help_text="The resolved package for this dependency. If empty, it indicates the dependency is unresolved.", + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="resolved_from_dependencies", + to="scanpipe.discoveredpackage", + ), + ), + migrations.AddField( + model_name="discoveredpackage", + name="children_packages", + field=models.ManyToManyField( + related_name="parent_packages", + through="scanpipe.DiscoveredDependency", + to="scanpipe.discoveredpackage", + ), + ), + ] diff --git a/scanpipe/models.py b/scanpipe/models.py index cf1c71191..3fa0b4bcb 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -3034,6 +3034,13 @@ class DiscoveredPackage( codebase_resources = models.ManyToManyField( "CodebaseResource", related_name="discovered_packages" ) + children_packages = models.ManyToManyField( + "self", + through="DiscoveredDependency", + symmetrical=False, + related_name="parent_packages", + through_fields=("for_package", "resolved_to_package"), + ) missing_resources = models.JSONField(default=list, blank=True) modified_resources = models.JSONField(default=list, blank=True) package_uid = models.CharField( @@ -3232,6 +3239,15 @@ def as_spdx(self): external_refs=external_refs, ) + @property + def cyclonedx_bom_ref(self): + """ + Use the package_uid when available to ensure having unique bom_ref + in the SBOM when several instances of the same DiscoveredPackage + (i.e. same purl) are present in the project. + """ + return self.package_uid or str(self.get_package_url()) + def as_cyclonedx(self): """Return this DiscoveredPackage as an CycloneDX Component entry.""" licenses = [] @@ -3298,17 +3314,12 @@ def as_cyclonedx(self): ], ) - package_url = self.get_package_url() - # Use the package_uid when available to ensure having unique bom_ref - # in the SBOM when several instances of the same DiscoveredPackage - # (i.e. same purl) are present in the project. - bom_ref = self.package_uid or str(package_url) - return cyclonedx_component.Component( name=self.name, version=self.version, - bom_ref=bom_ref, - purl=package_url, # Warning: Use the real purl and not package_uid here. + bom_ref=self.cyclonedx_bom_ref, + # Warning: Use the real purl and not package_uid here. + purl=self.get_package_url(), licenses=licenses, copyright=self.copyright, description=self.description, @@ -3332,6 +3343,10 @@ def prefetch_for_serializer(self): Prefetch( "for_package", queryset=DiscoveredPackage.objects.only("package_uid") ), + Prefetch( + "resolved_to_package", + queryset=DiscoveredPackage.objects.only("package_uid"), + ), Prefetch( "datafile_resource", queryset=CodebaseResource.objects.only("path") ), @@ -3373,9 +3388,9 @@ class DiscoveredDependency( blank=True, null=True, ) - resolved_to = models.ForeignKey( + resolved_to_package = models.ForeignKey( DiscoveredPackage, - related_name="resolved_dependencies", + related_name="resolved_from_dependencies", help_text=_( "The resolved package for this dependency. " "If empty, it indicates the dependency is unresolved." @@ -3468,9 +3483,9 @@ def for_package_uid(self): return self.for_package.package_uid @cached_property - def resolved_to_uid(self): - if self.resolved_to: - return self.resolved_to.package_uid + def resolved_to_package_uid(self): + if self.resolved_to_package: + return self.resolved_to_package.package_uid @cached_property def datafile_path(self): diff --git a/scanpipe/pipes/output.py b/scanpipe/pipes/output.py index df093bdad..e02cc7427 100644 --- a/scanpipe/pipes/output.py +++ b/scanpipe/pipes/output.py @@ -675,23 +675,36 @@ def get_cyclonedx_bom(project): ], ) - components = [] vulnerabilities = [] - for package in get_queryset(project, "discoveredpackage"): + dependencies = {} + + package_qs = get_queryset(project, "discoveredpackage") + package_qs = package_qs.prefetch_related("children_packages") + + for package in package_qs: component = package.as_cyclonedx() - components.append(component) + bom.components.add(component) + bom.register_dependency(project_as_root_component, [component]) + + # Store the component dependencies to be added later since all components need + # to be added on the BOM first. + dependencies[component] = [ + package.cyclonedx_bom_ref for package in package.children_packages.all() + ] for vulnerability_data in package.affected_by_vulnerabilities: vulnerabilities.append( - vulnerability_as_cyclonedx( - vulnerability_data=vulnerability_data, - component_bom_ref=component.bom_ref, - ) + vulnerability_as_cyclonedx(vulnerability_data, component.bom_ref) ) - for component in components: - bom.components.add(component) - bom.register_dependency(project_as_root_component, [component]) + for component, depends_on_bom_refs in dependencies.items(): + if not depends_on_bom_refs: + continue + # Craft disposable Component instances for registering dependencies + dependencies = [ + cdx_component.Component(name="", bom_ref=ref) for ref in depends_on_bom_refs + ] + bom.register_dependency(component, dependencies) bom.vulnerabilities = vulnerabilities diff --git a/scanpipe/templates/scanpipe/dependency_list.html b/scanpipe/templates/scanpipe/dependency_list.html index 7147365c1..e26ea9c24 100644 --- a/scanpipe/templates/scanpipe/dependency_list.html +++ b/scanpipe/templates/scanpipe/dependency_list.html @@ -55,9 +55,9 @@ {% endif %} - {% if dependency.resolved_to %} + {% if dependency.resolved_to_package %} {# CAUTION: Avoid relying on get_absolute_url to prevent unnecessary query triggers #} - {{ dependency.resolved_to.purl }} + {{ dependency.resolved_to_package.purl }} {% endif %} diff --git a/scanpipe/tests/__init__.py b/scanpipe/tests/__init__.py index fbd745be3..b18c7ed4e 100644 --- a/scanpipe/tests/__init__.py +++ b/scanpipe/tests/__init__.py @@ -27,6 +27,8 @@ from django.apps import apps from scanpipe.models import CodebaseResource +from scanpipe.models import DiscoveredDependency +from scanpipe.models import DiscoveredPackage from scanpipe.tests.pipelines.do_nothing import DoNothing from scanpipe.tests.pipelines.profile_step import ProfileStep from scanpipe.tests.pipelines.raise_exception import RaiseException @@ -65,6 +67,17 @@ def make_resource_directory(project, path, **extra): ) +def make_package(project, package_url, **extra): + package = DiscoveredPackage(project=project, **extra) + package.set_package_url(package_url) + package.save() + return package + + +def make_dependency(project, **extra): + return DiscoveredDependency.objects.create(project=project, **extra) + + resource_data1 = { "path": "notice.NOTICE", "type": "file", diff --git a/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json b/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json index af85a6df2..037d59811 100644 --- a/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json +++ b/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json @@ -277,7 +277,7 @@ "is_resolved": false, "dependency_uid": "pkg:pypi/pytest?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fixed-uid-done-for-testing-5642512d1758", - "resolved_to_uid": null, + "resolved_to_package_uid": null, "datafile_path": "asgiref-3.3.0-py3-none-any.whl", "datasource_id": "pypi_wheel", "package_type": "pypi", @@ -292,7 +292,7 @@ "is_resolved": false, "dependency_uid": "pkg:pypi/pytest?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fixed-uid-done-for-testing-5642512d1758", - "resolved_to_uid": null, + "resolved_to_package_uid": null, "datafile_path": "asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA", "datasource_id": "pypi_wheel_metadata", "package_type": "pypi", @@ -307,7 +307,7 @@ "is_resolved": false, "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fixed-uid-done-for-testing-5642512d1758", - "resolved_to_uid": null, + "resolved_to_package_uid": null, "datafile_path": "asgiref-3.3.0-py3-none-any.whl", "datasource_id": "pypi_wheel", "package_type": "pypi", @@ -322,7 +322,7 @@ "is_resolved": false, "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fixed-uid-done-for-testing-5642512d1758", - "resolved_to_uid": null, + "resolved_to_package_uid": null, "datafile_path": "asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA", "datasource_id": "pypi_wheel_metadata", "package_type": "pypi", diff --git a/scanpipe/tests/data/daglib-0.6.0-py3-none-any.whl_scan_codebase.json b/scanpipe/tests/data/daglib-0.6.0-py3-none-any.whl_scan_codebase.json index c7467c5d4..69c7f52f8 100644 --- a/scanpipe/tests/data/daglib-0.6.0-py3-none-any.whl_scan_codebase.json +++ b/scanpipe/tests/data/daglib-0.6.0-py3-none-any.whl_scan_codebase.json @@ -257,7 +257,7 @@ "is_resolved": false, "dependency_uid": "pkg:pypi/dask?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758", - "resolved_to_uid": null, + "resolved_to_package_uid": null, "datafile_path": "daglib-0.6.0-py3-none-any.whl", "datasource_id": "pypi_wheel", "package_type": "pypi", @@ -272,7 +272,7 @@ "is_resolved": false, "dependency_uid": "pkg:pypi/dask?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758", - "resolved_to_uid": null, + "resolved_to_package_uid": null, "datafile_path": "daglib-0.6.0-py3-none-any.whl-extract/daglib-0.6.0.dist-info/METADATA", "datasource_id": "pypi_wheel_metadata", "package_type": "pypi", @@ -287,7 +287,7 @@ "is_resolved": false, "dependency_uid": "pkg:pypi/graphviz?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758", - "resolved_to_uid": null, + "resolved_to_package_uid": null, "datafile_path": "daglib-0.6.0-py3-none-any.whl", "datasource_id": "pypi_wheel", "package_type": "pypi", @@ -302,7 +302,7 @@ "is_resolved": false, "dependency_uid": "pkg:pypi/graphviz?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758", - "resolved_to_uid": null, + "resolved_to_package_uid": null, "datafile_path": "daglib-0.6.0-py3-none-any.whl-extract/daglib-0.6.0.dist-info/METADATA", "datasource_id": "pypi_wheel_metadata", "package_type": "pypi", @@ -317,7 +317,7 @@ "is_resolved": false, "dependency_uid": "pkg:pypi/ipycytoscape?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758", - "resolved_to_uid": null, + "resolved_to_package_uid": null, "datafile_path": "daglib-0.6.0-py3-none-any.whl", "datasource_id": "pypi_wheel", "package_type": "pypi", @@ -332,7 +332,7 @@ "is_resolved": false, "dependency_uid": "pkg:pypi/ipycytoscape?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758", - "resolved_to_uid": null, + "resolved_to_package_uid": null, "datafile_path": "daglib-0.6.0-py3-none-any.whl-extract/daglib-0.6.0.dist-info/METADATA", "datasource_id": "pypi_wheel_metadata", "package_type": "pypi", @@ -347,7 +347,7 @@ "is_resolved": false, "dependency_uid": "pkg:pypi/networkx?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758", - "resolved_to_uid": null, + "resolved_to_package_uid": null, "datafile_path": "daglib-0.6.0-py3-none-any.whl", "datasource_id": "pypi_wheel", "package_type": "pypi", @@ -362,7 +362,7 @@ "is_resolved": false, "dependency_uid": "pkg:pypi/networkx?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758", - "resolved_to_uid": null, + "resolved_to_package_uid": null, "datafile_path": "daglib-0.6.0-py3-none-any.whl-extract/daglib-0.6.0.dist-info/METADATA", "datasource_id": "pypi_wheel_metadata", "package_type": "pypi", diff --git a/scanpipe/tests/data/is-npm-1.0.0_scan_codebase.json b/scanpipe/tests/data/is-npm-1.0.0_scan_codebase.json index a8d6a6e24..985603dfb 100644 --- a/scanpipe/tests/data/is-npm-1.0.0_scan_codebase.json +++ b/scanpipe/tests/data/is-npm-1.0.0_scan_codebase.json @@ -128,7 +128,7 @@ "is_resolved": false, "dependency_uid": "pkg:npm/ava?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:npm/is-npm@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", - "resolved_to_uid": null, + "resolved_to_package_uid": null, "datafile_path": "is-npm-1.0.0.tgz-extract/package/package.json", "datasource_id": "npm_package_json", "package_type": "npm", diff --git a/scanpipe/tests/pipes/test_output.py b/scanpipe/tests/pipes/test_output.py index f634a8e9a..471d56c1b 100644 --- a/scanpipe/tests/pipes/test_output.py +++ b/scanpipe/tests/pipes/test_output.py @@ -44,6 +44,8 @@ from scanpipe.models import ProjectMessage from scanpipe.pipes import output from scanpipe.tests import FIXTURES_REGEN +from scanpipe.tests import make_dependency +from scanpipe.tests import make_package from scanpipe.tests import mocked_now from scanpipe.tests import package_data1 @@ -281,6 +283,32 @@ def test_scanpipe_pipes_outputs_to_cyclonedx(self, regen=FIXTURES_REGEN): ) self.assertEqual("1.5", results_json["specVersion"]) + def test_scanpipe_pipes_outputs_get_cyclonedx_bom_dependency_tree(self): + project = Project.objects.create(name="project") + + a = make_package(project, "pkg:type/a") + b = make_package(project, "pkg:type/b") + c = make_package(project, "pkg:type/c") + + # A -> B -> C + make_dependency(project, for_package=a, resolved_to_package=b) + make_dependency(project, for_package=b, resolved_to_package=c) + + with self.assertNumQueries(2): + output_file = output.to_cyclonedx(project=project) + results_json = json.loads(output_file.read_text()) + + expected = [ + { + "dependsOn": ["pkg:type/a", "pkg:type/b", "pkg:type/c"], + "ref": str(project.uuid), + }, + {"dependsOn": ["pkg:type/b"], "ref": "pkg:type/a"}, + {"dependsOn": ["pkg:type/c"], "ref": "pkg:type/b"}, + {"ref": "pkg:type/c"}, + ] + self.assertEqual(expected, results_json["dependencies"]) + def test_scanpipe_pipes_outputs_to_spdx(self): fixtures = self.data_path / "asgiref-3.3.0_fixtures.json" call_command("loaddata", fixtures, **{"verbosity": 0}) @@ -428,7 +456,7 @@ def test_scanpipe_pipes_outputs_to_attribution(self): package_data["notice_text"] = "Notice text" pipes.update_or_create_package(project, package_data) - with self.assertNumQueries(1): + with self.assertNumQueries(2): output_file = output.to_attribution(project=project) expected_file = self.data_path / "outputs" / "expected_attribution.html" diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index 87302c555..e152cd542 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -70,6 +70,8 @@ from scanpipe.tests import dependency_data1 from scanpipe.tests import dependency_data2 from scanpipe.tests import license_policies_index +from scanpipe.tests import make_dependency +from scanpipe.tests import make_package from scanpipe.tests import make_resource_directory from scanpipe.tests import make_resource_file from scanpipe.tests import mocked_now @@ -2112,6 +2114,32 @@ def test_scanpipe_discovered_dependency_model_update_from_data(self): self.assertEqual(["scope"], updated_fields) self.assertEqual(new_data["scope"], dependency.scope) + def test_scanpipe_discovered_dependency_model_many_to_many(self): + project = Project.objects.create(name="project") + + a = make_package(project, "pkg:type/a") + b = make_package(project, "pkg:type/b") + c = make_package(project, "pkg:type/c") + # A -> B -> C + a_b = make_dependency(project, for_package=a, resolved_to_package=b) + b_c = make_dependency(project, for_package=b, resolved_to_package=c) + + # *_packages fields return DiscoveredPackage QuerySet + self.assertEqual([b], list(a.children_packages.all())) + self.assertEqual([], list(a.parent_packages.all())) + self.assertEqual([c], list(b.children_packages.all())) + self.assertEqual([a], list(b.parent_packages.all())) + self.assertEqual([], list(c.children_packages.all())) + self.assertEqual([b], list(c.parent_packages.all())) + + # *_dependencies fields return DiscoveredDependency QuerySet + self.assertEqual([a_b], list(a.declared_dependencies.all())) + self.assertEqual([], list(a.resolved_from_dependencies.all())) + self.assertEqual([b_c], list(b.declared_dependencies.all())) + self.assertEqual([a_b], list(b.resolved_from_dependencies.all())) + self.assertEqual([], list(c.declared_dependencies.all())) + self.assertEqual([b_c], list(c.resolved_from_dependencies.all())) + def test_scanpipe_discovered_dependency_model_is_vulnerable_property(self): package = DiscoveredPackage.create_from_data(self.project1, package_data1) self.assertFalse(package.is_vulnerable) @@ -2136,7 +2164,9 @@ def test_scanpipe_package_model_integrity_with_toolkit_package_model(self): "compliance_alert", "tag", "declared_dependencies", - "resolved_dependencies", + "resolved_from_dependencies", + "parent_packages", + "children_packages", ] package_data_only_field = ["datasource_id", "dependencies"] diff --git a/scanpipe/views.py b/scanpipe/views.py index 0d1a33e65..880de02ed 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -1509,7 +1509,8 @@ class DiscoveredDependencyListView( "for_package", queryset=DiscoveredPackage.objects.only("uuid", *PURL_FIELDS) ), Prefetch( - "resolved_to", queryset=DiscoveredPackage.objects.only("uuid", *PURL_FIELDS) + "resolved_to_package", + queryset=DiscoveredPackage.objects.only("uuid", *PURL_FIELDS), ), Prefetch( "datafile_resource", queryset=CodebaseResource.objects.only("path", "name") @@ -1543,7 +1544,7 @@ class DiscoveredDependencyListView( "filter_fieldname": "is_resolved", }, "for_package", - "resolved_to", + "resolved_to_package", "datafile_resource", { "field_name": "datasource_id", @@ -2006,7 +2007,7 @@ class DiscoveredDependencyDetailsView( ), ), Prefetch( - "resolved_to", + "resolved_to_package", queryset=DiscoveredPackage.objects.only( "uuid", *PURL_FIELDS, "package_uid", "project_id" ), @@ -2025,7 +2026,7 @@ class DiscoveredDependencyDetailsView( "template": "scanpipe/tabset/field_related_package.html", }, { - "field_name": "resolved_to", + "field_name": "resolved_to_package", "template": "scanpipe/tabset/field_related_package.html", }, { @@ -2043,7 +2044,7 @@ class DiscoveredDependencyDetailsView( "fields": [ "dependency_uid", "for_package_uid", - "resolved_to_uid", + "resolved_to_package_uid", "is_runtime", "is_optional", "is_resolved",