Add global Vulnerability list #95 (#171)

Signed-off-by: tdruez <tdruez@nexb.com>

CHANGELOG.rst

@@ -93,6 +93,13 @@ Release notes
   - Pull ScanCode.io Project data
   https://github.com/aboutcode-org/dejacode/issues/94
 
+- Add a new Vulnerabilities list available from the "Tools" menu when
+  ``enable_vulnerablecodedb_access`` is enabled on a Dataspace.
+  This implementation focuses on ranking/sorting: Vulnerabilities can be sorted and
+  filtered by severity score.
+  It's also possible to sort by the count of affected packages to help prioritize.
+  https://github.com/aboutcode-org/dejacode/issues/94
+
 - Display warning when a "download_url" could not be determined from a PURL in
   "Add Package".
   https://github.com/aboutcode-org/dejacode/issues/163

component_catalog/filters.py

@@ -8,6 +8,7 @@
 
 from django import forms
 from django.contrib.admin.options import IncorrectLookupParameters
+from django.db.models import F
 from django.utils.functional import cached_property
 from django.utils.translation import gettext_lazy as _
 
@@ -16,12 +17,14 @@
 from component_catalog.models import Component
 from component_catalog.models import ComponentKeyword
 from component_catalog.models import Package
+from component_catalog.models import Vulnerability
 from component_catalog.programming_languages import PROGRAMMING_LANGUAGES
 from dje.filters import DataspacedFilterSet
 from dje.filters import DefaultOrderingFilter
 from dje.filters import HasRelationFilter
 from dje.filters import MatchOrderedSearchFilter
 from dje.filters import RelatedLookupListFilter
+from dje.filters import SearchFilter
 from dje.widgets import BootstrapSelectMultipleWidget
 from dje.widgets import DropDownRightWidget
 from dje.widgets import SortDropDownWidget
@@ -104,6 +107,10 @@ class ComponentFilterSet(DataspacedFilterSet):
         field_name="affected_by_vulnerabilities",
         widget=DropDownRightWidget(link_content='<i class="fas fa-bug"></i>'),
     )
+    affected_by = django_filters.CharFilter(
+        field_name="affected_by_vulnerabilities__vulnerability_id",
+        label=_("Affected by"),
+    )
 
     class Meta:
         model = Component
@@ -242,6 +249,10 @@ class PackageFilterSet(DataspacedFilterSet):
         field_name="affected_by_vulnerabilities",
         widget=DropDownRightWidget(link_content='<i class="fas fa-bug"></i>'),
     )
+    affected_by = django_filters.CharFilter(
+        field_name="affected_by_vulnerabilities__vulnerability_id",
+        label=_("Affected by"),
+    )
 
     class Meta:
         model = Package
@@ -272,3 +283,82 @@ def show_created_date(self):
     @cached_property
     def show_last_modified_date(self):
         return not self.sort_value or self.has_sort_by("last_modified_date")
+
+
+class NullsLastOrderingFilter(django_filters.OrderingFilter):
+    """
+    A custom ordering filter that ensures null values are sorted last.
+
+    When sorting by fields with potential null values, this filter modifies the
+    ordering to use Django's `nulls_last` clause for better handling of null values,
+    whether in ascending or descending order.
+    """
+
+    def filter(self, qs, value):
+        if not value:
+            return qs
+
+        ordering = []
+        for field in value:
+            if field.startswith("-"):
+                field_name = field[1:]
+                ordering.append(F(field_name).desc(nulls_last=True))
+            else:
+                ordering.append(F(field).asc(nulls_last=True))
+
+        return qs.order_by(*ordering)
+
+
+vulnerability_score_ranges = {
+    "low": (0.1, 3),
+    "medium": (4.0, 6.9),
+    "high": (7.0, 8.9),
+    "critical": (9.0, 10.0),
+}
+
+SCORE_CHOICES = [
+    (key, f"{key.capitalize()} ({value[0]} - {value[1]})")
+    for key, value in vulnerability_score_ranges.items()
+]
+
+
+class VulnerabilityFilterSet(DataspacedFilterSet):
+    q = SearchFilter(
+        label=_("Search"),
+        search_fields=["vulnerability_id", "aliases"],
+    )
+    sort = NullsLastOrderingFilter(
+        label=_("Sort"),
+        fields=[
+            "max_score",
+            "min_score",
+            "affected_products_count",
+            "affected_packages_count",
+            "fixed_packages_count",
+            "created_date",
+            "last_modified_date",
+        ],
+        widget=SortDropDownWidget,
+    )
+    max_score = django_filters.ChoiceFilter(
+        choices=SCORE_CHOICES,
+        method="filter_by_score_range",
+        label="Score Range",
+        help_text="Select a score range to filter.",
+    )
+
+    class Meta:
+        model = Vulnerability
+        fields = [
+            "q",
+        ]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.filters["max_score"].extra["widget"] = DropDownRightWidget()
+
+    def filter_by_score_range(self, queryset, name, value):
+        if value in vulnerability_score_ranges:
+            low, high = vulnerability_score_ranges[value]
+            return queryset.filter(max_score__gte=low, max_score__lte=high)
+        return queryset

component_catalog/migrations/0007_vulnerability_fixed_packages_count_and_more.py

@@ -0,0 +1,28 @@
+# Generated by Django 5.0.6 on 2024-08-27 14:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('component_catalog', '0006_vulnerability_model_and_missing_indexes'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='vulnerability',
+            name='fixed_packages_count',
+            field=models.GeneratedField(db_persist=True, expression=models.Func(models.F('fixed_packages'), function='jsonb_array_length'), output_field=models.IntegerField()),
+        ),
+        migrations.AddField(
+            model_name='vulnerability',
+            name='max_score',
+            field=models.FloatField(blank=True, help_text='The maximum score of the range.', null=True),
+        ),
+        migrations.AddField(
+            model_name='vulnerability',
+            name='min_score',
+            field=models.FloatField(blank=True, help_text='The minimum score of the range.', null=True),
+        ),
+    ]

component_catalog/models.py

@@ -19,6 +19,7 @@
 from django.core.validators import EMPTY_VALUES
 from django.db import models
 from django.db.models import CharField
+from django.db.models import Count
 from django.db.models import Exists
 from django.db.models import OuterRef
 from django.db.models.functions import Concat
@@ -134,7 +135,7 @@ class VulnerabilityQuerySetMixin:
     def with_vulnerability_count(self):
         """Annotate the QuerySet with the vulnerability_count."""
         return self.annotate(
-            vulnerability_count=models.Count("affected_by_vulnerabilities", distinct=True)
+            vulnerability_count=Count("affected_by_vulnerabilities", distinct=True)
         )
 
     def vulnerable(self):
@@ -1748,7 +1749,7 @@ def declared_dependencies_count(self, product):
         dependencies are always scoped to a Product.
         """
         return self.annotate(
-            declared_dependencies_count=models.Count(
+            declared_dependencies_count=Count(
                 "declared_dependencies",
                 filter=models.Q(declared_dependencies__product=product),
             )
@@ -2562,6 +2563,22 @@ def __str__(self):
         return f"<{self.component}>: {self.package}"
 
 
+class VulnerabilityQuerySet(DataspacedQuerySet):
+    def with_affected_products_count(self):
+        """Annotate the QuerySet with the affected_products_count."""
+        return self.annotate(
+            affected_products_count=Count(
+                "affected_packages__productpackages__product", distinct=True
+            ),
+        )
+
+    def with_affected_packages_count(self):
+        """Annotate the QuerySet with the affected_packages_count."""
+        return self.annotate(
+            affected_packages_count=Count("affected_packages", distinct=True),
+        )
+
+
 class Vulnerability(HistoryDateFieldsMixin, DataspacedModel):
     """
     A software vulnerability with a unique identifier and alternate aliases.
@@ -2574,6 +2591,7 @@ class Vulnerability(HistoryDateFieldsMixin, DataspacedModel):
     automatically on object addition or during schedule tasks.
     """
 
+    # The first set of fields are storing data as fetched from VulnerableCode
     vulnerability_id = models.CharField(
         max_length=20,
         help_text=_(
@@ -2603,6 +2621,23 @@ class Vulnerability(HistoryDateFieldsMixin, DataspacedModel):
         blank=True,
         help_text=_("A list of packages that are not affected by this vulnerability."),
     )
+    fixed_packages_count = models.GeneratedField(
+        expression=models.Func(models.F("fixed_packages"), function="jsonb_array_length"),
+        output_field=models.IntegerField(),
+        db_persist=True,
+    )
+    min_score = models.FloatField(
+        null=True,
+        blank=True,
+        help_text=_("The minimum score of the range."),
+    )
+    max_score = models.FloatField(
+        null=True,
+        blank=True,
+        help_text=_("The maximum score of the range."),
+    )
+
+    objects = DataspacedManager.from_queryset(VulnerabilityQuerySet)()
 
     class Meta:
         verbose_name_plural = "Vulnerabilities"
@@ -2640,11 +2675,57 @@ def add_affected_components(self, components):
         """Assign the ``components`` as affected to this vulnerability."""
         self.affected_components.add(*components)
 
+    @staticmethod
+    def range_to_values(self, range_str):
+        try:
+            min_score, max_score = range_str.split("-")
+            return float(min_score.strip()), float(max_score.strip())
+        except Exception:
+            return
+
     @classmethod
     def create_from_data(cls, dataspace, data, validate=False, affecting=None):
+        # Computing the min_score and max_score from the `references` as those data
+        # are not provided by the VulnerableCode API.
+        # https://github.com/aboutcode-org/vulnerablecode/issues/1573
+        # severity_range_score = data.get("severity_range_score")
+        # if severity_range_score:
+        #     min_score, max_score = self.range_to_values(severity_range_score)
+        #     data["min_score"] = min_score
+        #     data["max_score"] = max_score
+
+        severities = [
+            score for reference in data.get("references") for score in reference.get("scores", [])
+        ]
+        if scores := cls.get_severity_scores(severities):
+            data["min_score"] = min(scores)
+            data["max_score"] = max(scores)
+
         instance = super().create_from_data(user=dataspace, data=data, validate=False)
 
         if affecting:
             instance.add_affected(affecting)
 
         return instance
+
+    @staticmethod
+    def get_severity_scores(severities):
+        score_map = {
+            "low": [0.1, 3],
+            "moderate": [4.0, 6.9],
+            "medium": [4.0, 6.9],
+            "high": [7.0, 8.9],
+            "important": [7.0, 8.9],
+            "critical": [9.0, 10.0],
+        }
+
+        consolidated_scores = []
+        for severity in severities:
+            score = severity.get("value")
+            try:
+                consolidated_scores.append(float(score))
+            except ValueError:
+                if score_range := score_map.get(score.lower(), None):
+                    consolidated_scores.extend(score_range)
+
+        return consolidated_scores

component_catalog/templates/component_catalog/includes/vulnerability_aliases.html

@@ -0,0 +1,21 @@
+<ul class="list-unstyled">
+  {% for alias in aliases %}
+    <li>
+      {% if alias|slice:":3" == "CVE" %}
+        <a href="https://nvd.nist.gov/vuln/detail/{{ alias }}" target="_blank">{{ alias }}
+          <i class="fa-solid fa-up-right-from-square mini"></i>
+        </a>
+      {% elif alias|slice:":4" == "GHSA" %}
+        <a href="https://github.com/advisories/{{ alias }}" target="_blank">{{ alias }}
+          <i class="fa-solid fa-up-right-from-square mini"></i>
+        </a>
+      {% elif alias|slice:":3" == "NPM" %}
+        <a href="https://github.com/nodejs/security-wg/blob/main/vuln/npm/{{ alias|slice:"4:" }}.json" target="_blank">{{ alias }}
+          <i class="fa-solid fa-up-right-from-square mini"></i>
+        </a>
+      {% else %}
+        {{ alias }}
+      {% endif %}
+    </li>
+  {% endfor %}
+</ul>