Add new resolved_to field on DiscoveredDependency #1066 (#1240)

Signed-off-by: tdruez <tdruez@nexb.com>
aboutcode-org · May 17, 2024 · dcb4e5e · dcb4e5e
1 parent 44f778d
commit dcb4e5e
Show file tree

Hide file tree

Showing 13 changed files with 128 additions and 12 deletions.
diff --git a/scanpipe/api/serializers.py b/scanpipe/api/serializers.py
@@ -392,6 +392,7 @@ class Meta:
 class DiscoveredDependencySerializer(serializers.ModelSerializer):
     purl = serializers.ReadOnlyField()
     for_package_uid = serializers.ReadOnlyField()
+    resolved_to_uid = serializers.ReadOnlyField()
     datafile_path = serializers.ReadOnlyField()
     package_type = serializers.ReadOnlyField(source="type")
 
@@ -406,6 +407,7 @@ class Meta:
             "is_resolved",
             "dependency_uid",
             "for_package_uid",
+            "resolved_to_uid",
             "datafile_path",
             "datasource_id",
             "package_type",

diff --git a/scanpipe/filters.py b/scanpipe/filters.py
@@ -740,6 +740,7 @@ class DependencyFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
             "is_optional",
             "is_resolved",
             "for_package",
+            "resolved_to",
             "datafile_resource",
             "datasource_id",
         ],

diff --git a/scanpipe/migrations/0058_discovereddependency_resolved_to_and_more.py b/scanpipe/migrations/0058_discovereddependency_resolved_to_and_more.py
@@ -0,0 +1,53 @@
+# Generated by Django 5.0.4 on 2024-05-17 07:11
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scanpipe", "0057_rename_symbol_collection_pipelines"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="discovereddependency",
+            name="resolved_to",
+            field=models.ForeignKey(
+                blank=True,
+                editable=False,
+                help_text="The resolved package for this dependency. If empty, it indicates the dependency is unresolved.",
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name="resolved_dependencies",
+                to="scanpipe.discoveredpackage",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="discovereddependency",
+            name="datafile_resource",
+            field=models.ForeignKey(
+                blank=True,
+                editable=False,
+                help_text="The codebase resource (e.g., manifest or lockfile) that declares this dependency.",
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="declared_dependencies",
+                to="scanpipe.codebaseresource",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="discovereddependency",
+            name="for_package",
+            field=models.ForeignKey(
+                blank=True,
+                editable=False,
+                help_text="The package that declares this dependency.",
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="declared_dependencies",
+                to="scanpipe.discoveredpackage",
+            ),
+        ),
+    ]
diff --git a/scanpipe/models.py b/scanpipe/models.py
@@ -3290,6 +3290,8 @@ class DiscoveredDependency(
     """
     A project's Discovered Dependencies are records of the dependencies used by
     system and application packages discovered in the code under analysis.
+    Dependencies are usually collected from parsed package data such as a package
+    manifest or lockfile.
     """
 
     # Overrides the `project` field from `ProjectRelatedModel` to set the proper
@@ -3306,15 +3308,32 @@ class DiscoveredDependency(
     )
     for_package = models.ForeignKey(
         DiscoveredPackage,
-        related_name="dependencies",
+        related_name="declared_dependencies",
+        help_text=_("The package that declares this dependency."),
         on_delete=models.CASCADE,
         editable=False,
         blank=True,
         null=True,
     )
+    resolved_to = models.ForeignKey(
+        DiscoveredPackage,
+        related_name="resolved_dependencies",
+        help_text=_(
+            "The resolved package for this dependency. "
+            "If empty, it indicates the dependency is unresolved."
+        ),
+        on_delete=models.SET_NULL,
+        editable=False,
+        blank=True,
+        null=True,
+    )
     datafile_resource = models.ForeignKey(
         CodebaseResource,
-        related_name="dependencies",
+        related_name="declared_dependencies",
+        help_text=_(
+            "The codebase resource (e.g., manifest or lockfile) that declares this "
+            "dependency."
+        ),
         on_delete=models.CASCADE,
         editable=False,
         blank=True,
@@ -3390,6 +3409,11 @@ def for_package_uid(self):
         if self.for_package:
             return self.for_package.package_uid
 
+    @cached_property
+    def resolved_to_uid(self):
+        if self.resolved_to:
+            return self.resolved_to.package_uid
+
     @cached_property
     def datafile_path(self):
         if self.datafile_resource:

diff --git a/scanpipe/templates/scanpipe/dependency_list.html b/scanpipe/templates/scanpipe/dependency_list.html
@@ -33,12 +33,12 @@
             <td>
               <a href="?type={{ dependency.type }}" class="is-black-link">{{ dependency.type }}</a>
             </td>
-            <td>
-              {{ dependency.extracted_requirement }}
-            </td>
             <td class="break-normal">
               <a href="?scope={{ dependency.scope }}" class="is-black-link">{{ dependency.scope }}</a>
             </td>
+            <td>
+              {{ dependency.extracted_requirement }}
+            </td>
             <td>
               <a href="?is_runtime={{ dependency.is_runtime }}" class="is-black-link">{{ dependency.is_runtime }}</a>
             </td>
@@ -54,6 +54,12 @@
                 <a href="{% url 'package_detail' project.slug dependency.for_package.uuid %}" title="{{ dependency.for_package.purl }}">{{ dependency.for_package.purl }}</a>
               {% endif %}
             </td>
+            <td>
+              {% if dependency.resolved_to %}
+                {# CAUTION: Avoid relying on get_absolute_url to prevent unnecessary query triggers #}
+                <a href="{% url 'package_detail' project.slug dependency.for_package.uuid %}" title="{{ dependency.resolved_to.purl }}">{{ dependency.resolved_to.purl }}</a>
+              {% endif %}
+            </td>
             <td>
               {% if dependency.datafile_resource %}
                 {# CAUTION: Avoid relying on get_absolute_url to prevent unnecessary query triggers #}

diff --git a/...es/scanpipe/tabset/field_for_package.html → ...canpipe/tabset/field_related_package.html b/...es/scanpipe/tabset/field_for_package.html → ...canpipe/tabset/field_related_package.html
diff --git a/scanpipe/templates/scanpipe/tabset/tab_dependencies.html b/scanpipe/templates/scanpipe/tabset/tab_dependencies.html
@@ -9,7 +9,7 @@
       </tr>
     </thead>
     <tbody>
-      {% for dependency in tab_data.fields.dependencies.value %}
+      {% for dependency in tab_data.fields.declared_dependencies.value %}
         <tr>
           <td title="{{ dependency.dependency_uid }}">
             <a href="{{ dependency.get_absolute_url }}">{{ dependency.purl }}</a>

diff --git a/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json b/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json
@@ -277,6 +277,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/pytest?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "resolved_to_uid": null,
       "datafile_path": "asgiref-3.3.0-py3-none-any.whl",
       "datasource_id": "pypi_wheel",
       "package_type": "pypi",
@@ -291,6 +292,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/pytest?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "resolved_to_uid": null,
       "datafile_path": "asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA",
       "datasource_id": "pypi_wheel_metadata",
       "package_type": "pypi",
@@ -305,6 +307,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "resolved_to_uid": null,
       "datafile_path": "asgiref-3.3.0-py3-none-any.whl",
       "datasource_id": "pypi_wheel",
       "package_type": "pypi",
@@ -319,6 +322,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "resolved_to_uid": null,
       "datafile_path": "asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA",
       "datasource_id": "pypi_wheel_metadata",
       "package_type": "pypi",

diff --git a/scanpipe/tests/data/daglib-0.6.0-py3-none-any.whl_scan_codebase.json b/scanpipe/tests/data/daglib-0.6.0-py3-none-any.whl_scan_codebase.json
@@ -257,6 +257,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/dask?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "resolved_to_uid": null,
       "datafile_path": "daglib-0.6.0-py3-none-any.whl",
       "datasource_id": "pypi_wheel",
       "package_type": "pypi",
@@ -271,6 +272,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/dask?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "resolved_to_uid": null,
       "datafile_path": "daglib-0.6.0-py3-none-any.whl-extract/daglib-0.6.0.dist-info/METADATA",
       "datasource_id": "pypi_wheel_metadata",
       "package_type": "pypi",
@@ -285,6 +287,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/graphviz?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "resolved_to_uid": null,
       "datafile_path": "daglib-0.6.0-py3-none-any.whl",
       "datasource_id": "pypi_wheel",
       "package_type": "pypi",
@@ -299,6 +302,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/graphviz?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "resolved_to_uid": null,
       "datafile_path": "daglib-0.6.0-py3-none-any.whl-extract/daglib-0.6.0.dist-info/METADATA",
       "datasource_id": "pypi_wheel_metadata",
       "package_type": "pypi",
@@ -313,6 +317,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/ipycytoscape?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "resolved_to_uid": null,
       "datafile_path": "daglib-0.6.0-py3-none-any.whl",
       "datasource_id": "pypi_wheel",
       "package_type": "pypi",
@@ -327,6 +332,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/ipycytoscape?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "resolved_to_uid": null,
       "datafile_path": "daglib-0.6.0-py3-none-any.whl-extract/daglib-0.6.0.dist-info/METADATA",
       "datasource_id": "pypi_wheel_metadata",
       "package_type": "pypi",
@@ -341,6 +347,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/networkx?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "resolved_to_uid": null,
       "datafile_path": "daglib-0.6.0-py3-none-any.whl",
       "datasource_id": "pypi_wheel",
       "package_type": "pypi",
@@ -355,6 +362,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:pypi/networkx?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "resolved_to_uid": null,
       "datafile_path": "daglib-0.6.0-py3-none-any.whl-extract/daglib-0.6.0.dist-info/METADATA",
       "datasource_id": "pypi_wheel_metadata",
       "package_type": "pypi",

diff --git a/scanpipe/tests/data/is-npm-1.0.0_scan_codebase.json b/scanpipe/tests/data/is-npm-1.0.0_scan_codebase.json
@@ -128,6 +128,7 @@
       "is_resolved": false,
       "dependency_uid": "pkg:npm/ava?uuid=fixed-uid-done-for-testing-5642512d1758",
       "for_package_uid": "pkg:npm/is-npm@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "resolved_to_uid": null,
       "datafile_path": "is-npm-1.0.0.tgz-extract/package/package.json",
       "datasource_id": "npm_package_json",
       "package_type": "npm",

diff --git a/scanpipe/tests/test_api.py b/scanpipe/tests/test_api.py
@@ -998,7 +998,7 @@ def test_scanpipe_api_serializer_get_model_serializer(self):
 
     def test_scanpipe_api_serializer_get_serializer_fields(self):
         self.assertEqual(46, len(get_serializer_fields(DiscoveredPackage)))
-        self.assertEqual(12, len(get_serializer_fields(DiscoveredDependency)))
+        self.assertEqual(13, len(get_serializer_fields(DiscoveredDependency)))
         self.assertEqual(33, len(get_serializer_fields(CodebaseResource)))
         self.assertEqual(5, len(get_serializer_fields(CodebaseRelation)))
         self.assertEqual(7, len(get_serializer_fields(ProjectMessage)))

diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py
@@ -2069,9 +2069,11 @@ def test_scanpipe_package_model_integrity_with_toolkit_package_model(self):
             "affected_by_vulnerabilities",
             "compliance_alert",
             "tag",
+            "declared_dependencies",
+            "resolved_dependencies",
         ]
 
-        package_data_only_field = ["datasource_id"]
+        package_data_only_field = ["datasource_id", "dependencies"]
 
         discovered_package_fields = [
             field.name

diff --git a/scanpipe/views.py b/scanpipe/views.py
@@ -1505,6 +1505,9 @@ class DiscoveredDependencyListView(
         Prefetch(
             "for_package", queryset=DiscoveredPackage.objects.only("uuid", *PURL_FIELDS)
         ),
+        Prefetch(
+            "resolved_to", queryset=DiscoveredPackage.objects.only("uuid", *PURL_FIELDS)
+        ),
         Prefetch(
             "datafile_resource", queryset=CodebaseResource.objects.only("path", "name")
         ),
@@ -1519,11 +1522,11 @@ class DiscoveredDependencyListView(
             "label": "Package type",
             "filter_fieldname": "type",
         },
-        "extracted_requirement",
         {
             "field_name": "scope",
             "filter_fieldname": "scope",
         },
+        "extracted_requirement",
         {
             "field_name": "is_runtime",
             "filter_fieldname": "is_runtime",
@@ -1537,6 +1540,7 @@ class DiscoveredDependencyListView(
             "filter_fieldname": "is_resolved",
         },
         "for_package",
+        "resolved_to",
         "datafile_resource",
         {
             "field_name": "datasource_id",
@@ -1835,7 +1839,7 @@ class DiscoveredPackageDetailsView(
                 "project_id",
             ),
         ),
-        "dependencies__project",
+        "declared_dependencies__project",
     ]
     tabset = {
         "essentials": {
@@ -1910,7 +1914,7 @@ class DiscoveredPackageDetailsView(
             "template": "scanpipe/tabset/tab_resources.html",
         },
         "dependencies": {
-            "fields": ["dependencies"],
+            "fields": ["declared_dependencies"],
             "icon_class": "fa-solid fa-layer-group",
             "template": "scanpipe/tabset/tab_dependencies.html",
         },
@@ -1998,6 +2002,12 @@ class DiscoveredDependencyDetailsView(
                 "uuid", *PURL_FIELDS, "package_uid", "project_id"
             ),
         ),
+        Prefetch(
+            "resolved_to",
+            queryset=DiscoveredPackage.objects.only(
+                "uuid", *PURL_FIELDS, "package_uid", "project_id"
+            ),
+        ),
         Prefetch(
             "datafile_resource",
             queryset=CodebaseResource.objects.only("path", "name", "project_id"),
@@ -2009,7 +2019,11 @@ class DiscoveredDependencyDetailsView(
                 "package_url",
                 {
                     "field_name": "for_package",
-                    "template": "scanpipe/tabset/field_for_package.html",
+                    "template": "scanpipe/tabset/field_related_package.html",
+                },
+                {
+                    "field_name": "resolved_to",
+                    "template": "scanpipe/tabset/field_related_package.html",
                 },
                 {
                     "field_name": "datafile_resource",
@@ -2026,6 +2040,7 @@ class DiscoveredDependencyDetailsView(
             "fields": [
                 "dependency_uid",
                 "for_package_uid",
+                "resolved_to_uid",
                 "is_runtime",
                 "is_optional",
                 "is_resolved",