From eb0622181e9fbe8f88659e7b09cb93e49720e98e Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 22 Aug 2024 09:08:39 +0400 Subject: [PATCH 1/7] Update references to aboutcode-org in docs and workflow #158 Signed-off-by: tdruez --- .github/workflows/publish-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index c4859ff0..06e14da0 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -59,5 +59,5 @@ jobs: push: true tags: | ${{ steps.meta.outputs.tags }} - ${{ env.REGISTRY }}/nexb/dejacode:latest + ${{ env.REGISTRY }}/aboutcode-org/dejacode:latest labels: ${{ steps.meta.outputs.labels }} From 2fdfb5c1897590d9f862f59e54f5e22f18eda60f Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Thu, 22 Aug 2024 15:53:57 +0400 Subject: [PATCH 2/7] Display warning when a "download_url" could not be determined from a PURL #163 (#170) Signed-off-by: tdruez --- CHANGELOG.rst | 4 ++++ component_catalog/views.py | 18 +++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4634d4a2..e9ba1a3f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -93,6 +93,10 @@ Release notes - Pull ScanCode.io Project data https://github.com/aboutcode-org/dejacode/issues/94 +- Display warning when a "download_url" could not be determined from a PURL in + "Add Package". + https://github.com/aboutcode-org/dejacode/issues/163 + ### Version 5.1.0 - Upgrade Python version to 3.12 and Django to 5.0.x diff --git a/component_catalog/views.py b/component_catalog/views.py index 22aff7de..572296ca 100644 --- a/component_catalog/views.py +++ b/component_catalog/views.py @@ -1492,13 +1492,13 @@ def package_create_ajax_view(request): errors.append(str(error)) redirect_url = reverse("component_catalog:package_list") - len_created = len(created) scancodeio = ScanCodeIO(dataspace) - if scancodeio.is_configured() and dataspace.enable_package_scanning: + download_urls = [package.download_url for package in created if package.download_url] + if download_urls and dataspace.enable_package_scanning and scancodeio.is_configured(): # The availability of the each `download_url` is checked in the task. tasks.scancodeio_submit_scan.delay( - uris=[package.download_url for package in created if package.download_url], + uris=download_urls, user_uuid=user.uuid, dataspace_uuid=dataspace.uuid, ) @@ -1508,6 +1508,7 @@ def package_create_ajax_view(request): for package in created: package.fetch_vulnerabilities() + len_created = len(created) if len_created == 1: redirect_url = created[0].get_absolute_url() messages.success(request, "The Package was successfully created.") @@ -1516,6 +1517,17 @@ def package_create_ajax_view(request): msg = f"The following Packages were successfully created{scan_msg}:\n{packages}" messages.success(request, format_html(msg)) + purls_wihtout_download_url = [package for package in created if not package.download_url] + if purls_wihtout_download_url: + packages = [package.get_absolute_link() for package in purls_wihtout_download_url] + msg = ( + "Unable to determine a download URL for the following packages." + " A download URL is required to fetch and scan a package from DejaCode." + "\nAlternatively, you can directly provide a download URL instead of a " + "package URL to create the packages.\n" + ) + messages.warning(request, format_html(msg + "\n".join(packages))) + if errors: messages.error(request, format_html("\n".join(errors))) if warnings: From 21a353dbd6a4c590838dd20675d94251c2a0bfee Mon Sep 17 00:00:00 2001 From: tdruez Date: Fri, 23 Aug 2024 12:16:33 +0400 Subject: [PATCH 3/7] Fix a bug related to during copy #168 Signed-off-by: tdruez --- dje/forms.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dje/forms.py b/dje/forms.py index 0694cf09..460bb957 100644 --- a/dje/forms.py +++ b/dje/forms.py @@ -435,7 +435,12 @@ def __init__(self, user, *args, **kwargs): self.model_class = ContentType.objects.get(pk=ct).model_class() - exclude_choices = self.model_class.get_exclude_choices() + # Some implicitly created models, such as some Many2Many, do not inherit the + # get_exclude_choices method from the DataspacedModel class. + exclude_choices = [] + if hasattr(self.model_class, "get_exclude_choices"): + exclude_choices = self.model_class.get_exclude_choices() + self.fields["exclude_copy"].choices = exclude_choices self.fields["exclude_update"].choices = exclude_choices From 636d63bb958cbdc1fca64db8699cdc6dee5e3e2c Mon Sep 17 00:00:00 2001 From: tdruez Date: Fri, 23 Aug 2024 12:32:46 +0400 Subject: [PATCH 4/7] Display the current active filter in the Inventory tab #112 Signed-off-by: tdruez --- .../templates/product_portfolio/tabs/tab_inventory.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/product_portfolio/templates/product_portfolio/tabs/tab_inventory.html b/product_portfolio/templates/product_portfolio/tabs/tab_inventory.html index e08c2213..39974053 100644 --- a/product_portfolio/templates/product_portfolio/tabs/tab_inventory.html +++ b/product_portfolio/templates/product_portfolio/tabs/tab_inventory.html @@ -23,6 +23,9 @@ +
+ {% include 'includes/filters_breadcrumbs.html' with filterset=filter_productcomponent fragment=tab_id only %} +
{% include 'pagination/object_list_pagination.html' %} From ada0091fa0ee4d0edd809c53fe8774b6c6dddcb5 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Tue, 27 Aug 2024 20:53:40 +0400 Subject: [PATCH 5/7] Add global Vulnerability list #95 (#171) Signed-off-by: tdruez --- CHANGELOG.rst | 7 ++ component_catalog/filters.py | 90 +++++++++++++++++++ ...erability_fixed_packages_count_and_more.py | 28 ++++++ component_catalog/models.py | 85 +++++++++++++++++- .../includes/vulnerability_aliases.html | 21 +++++ .../tables/vulnerability_list_table.html | 70 +++++++++++++++ .../tabs/tab_vulnerabilities.html | 19 +--- .../component_catalog/vulnerability_list.html | 23 +++++ component_catalog/tests/__init__.py | 14 +-- component_catalog/tests/test_filters.py | 78 +++++++++++++++- component_catalog/tests/test_models.py | 53 ++++++++--- component_catalog/tests/test_views.py | 64 +++++++++++++ component_catalog/urls.py | 11 ++- component_catalog/views.py | 57 ++++++++++++ dejacode/static/css/dejacode_bootstrap.css | 18 ++++ dje/templates/admin/base_site.html | 5 ++ dje/templates/includes/navbar_header.html | 1 + .../includes/navbar_header_tools_menu.html | 8 ++ .../templates/modals}/filterset_modal.html | 6 +- dje/tests/__init__.py | 6 ++ product_portfolio/filters.py | 4 + product_portfolio/tests/__init__.py | 32 +++++++ purldb/templates/purldb/purldb_list.html | 4 +- purldb/tests/test_views.py | 4 +- purldb/views.py | 4 +- 25 files changed, 658 insertions(+), 54 deletions(-) create mode 100644 component_catalog/migrations/0007_vulnerability_fixed_packages_count_and_more.py create mode 100644 component_catalog/templates/component_catalog/includes/vulnerability_aliases.html create mode 100644 component_catalog/templates/component_catalog/tables/vulnerability_list_table.html create mode 100644 component_catalog/templates/component_catalog/vulnerability_list.html rename {purldb/templates/purldb/includes => dje/templates/modals}/filterset_modal.html (72%) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e9ba1a3f..08df610c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -93,6 +93,13 @@ Release notes - Pull ScanCode.io Project data https://github.com/aboutcode-org/dejacode/issues/94 +- Add a new Vulnerabilities list available from the "Tools" menu when + ``enable_vulnerablecodedb_access`` is enabled on a Dataspace. + This implementation focuses on ranking/sorting: Vulnerabilities can be sorted and + filtered by severity score. + It's also possible to sort by the count of affected packages to help prioritize. + https://github.com/aboutcode-org/dejacode/issues/94 + - Display warning when a "download_url" could not be determined from a PURL in "Add Package". https://github.com/aboutcode-org/dejacode/issues/163 diff --git a/component_catalog/filters.py b/component_catalog/filters.py index 5efdefa5..421c3748 100644 --- a/component_catalog/filters.py +++ b/component_catalog/filters.py @@ -8,6 +8,7 @@ from django import forms from django.contrib.admin.options import IncorrectLookupParameters +from django.db.models import F from django.utils.functional import cached_property from django.utils.translation import gettext_lazy as _ @@ -16,12 +17,14 @@ from component_catalog.models import Component from component_catalog.models import ComponentKeyword from component_catalog.models import Package +from component_catalog.models import Vulnerability from component_catalog.programming_languages import PROGRAMMING_LANGUAGES from dje.filters import DataspacedFilterSet from dje.filters import DefaultOrderingFilter from dje.filters import HasRelationFilter from dje.filters import MatchOrderedSearchFilter from dje.filters import RelatedLookupListFilter +from dje.filters import SearchFilter from dje.widgets import BootstrapSelectMultipleWidget from dje.widgets import DropDownRightWidget from dje.widgets import SortDropDownWidget @@ -104,6 +107,10 @@ class ComponentFilterSet(DataspacedFilterSet): field_name="affected_by_vulnerabilities", widget=DropDownRightWidget(link_content=''), ) + affected_by = django_filters.CharFilter( + field_name="affected_by_vulnerabilities__vulnerability_id", + label=_("Affected by"), + ) class Meta: model = Component @@ -242,6 +249,10 @@ class PackageFilterSet(DataspacedFilterSet): field_name="affected_by_vulnerabilities", widget=DropDownRightWidget(link_content=''), ) + affected_by = django_filters.CharFilter( + field_name="affected_by_vulnerabilities__vulnerability_id", + label=_("Affected by"), + ) class Meta: model = Package @@ -272,3 +283,82 @@ def show_created_date(self): @cached_property def show_last_modified_date(self): return not self.sort_value or self.has_sort_by("last_modified_date") + + +class NullsLastOrderingFilter(django_filters.OrderingFilter): + """ + A custom ordering filter that ensures null values are sorted last. + + When sorting by fields with potential null values, this filter modifies the + ordering to use Django's `nulls_last` clause for better handling of null values, + whether in ascending or descending order. + """ + + def filter(self, qs, value): + if not value: + return qs + + ordering = [] + for field in value: + if field.startswith("-"): + field_name = field[1:] + ordering.append(F(field_name).desc(nulls_last=True)) + else: + ordering.append(F(field).asc(nulls_last=True)) + + return qs.order_by(*ordering) + + +vulnerability_score_ranges = { + "low": (0.1, 3), + "medium": (4.0, 6.9), + "high": (7.0, 8.9), + "critical": (9.0, 10.0), +} + +SCORE_CHOICES = [ + (key, f"{key.capitalize()} ({value[0]} - {value[1]})") + for key, value in vulnerability_score_ranges.items() +] + + +class VulnerabilityFilterSet(DataspacedFilterSet): + q = SearchFilter( + label=_("Search"), + search_fields=["vulnerability_id", "aliases"], + ) + sort = NullsLastOrderingFilter( + label=_("Sort"), + fields=[ + "max_score", + "min_score", + "affected_products_count", + "affected_packages_count", + "fixed_packages_count", + "created_date", + "last_modified_date", + ], + widget=SortDropDownWidget, + ) + max_score = django_filters.ChoiceFilter( + choices=SCORE_CHOICES, + method="filter_by_score_range", + label="Score Range", + help_text="Select a score range to filter.", + ) + + class Meta: + model = Vulnerability + fields = [ + "q", + ] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.filters["max_score"].extra["widget"] = DropDownRightWidget() + + def filter_by_score_range(self, queryset, name, value): + if value in vulnerability_score_ranges: + low, high = vulnerability_score_ranges[value] + return queryset.filter(max_score__gte=low, max_score__lte=high) + return queryset diff --git a/component_catalog/migrations/0007_vulnerability_fixed_packages_count_and_more.py b/component_catalog/migrations/0007_vulnerability_fixed_packages_count_and_more.py new file mode 100644 index 00000000..fdcce3a0 --- /dev/null +++ b/component_catalog/migrations/0007_vulnerability_fixed_packages_count_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 5.0.6 on 2024-08-27 14:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('component_catalog', '0006_vulnerability_model_and_missing_indexes'), + ] + + operations = [ + migrations.AddField( + model_name='vulnerability', + name='fixed_packages_count', + field=models.GeneratedField(db_persist=True, expression=models.Func(models.F('fixed_packages'), function='jsonb_array_length'), output_field=models.IntegerField()), + ), + migrations.AddField( + model_name='vulnerability', + name='max_score', + field=models.FloatField(blank=True, help_text='The maximum score of the range.', null=True), + ), + migrations.AddField( + model_name='vulnerability', + name='min_score', + field=models.FloatField(blank=True, help_text='The minimum score of the range.', null=True), + ), + ] diff --git a/component_catalog/models.py b/component_catalog/models.py index 30aa218a..a01bb891 100644 --- a/component_catalog/models.py +++ b/component_catalog/models.py @@ -19,6 +19,7 @@ from django.core.validators import EMPTY_VALUES from django.db import models from django.db.models import CharField +from django.db.models import Count from django.db.models import Exists from django.db.models import OuterRef from django.db.models.functions import Concat @@ -134,7 +135,7 @@ class VulnerabilityQuerySetMixin: def with_vulnerability_count(self): """Annotate the QuerySet with the vulnerability_count.""" return self.annotate( - vulnerability_count=models.Count("affected_by_vulnerabilities", distinct=True) + vulnerability_count=Count("affected_by_vulnerabilities", distinct=True) ) def vulnerable(self): @@ -1748,7 +1749,7 @@ def declared_dependencies_count(self, product): dependencies are always scoped to a Product. """ return self.annotate( - declared_dependencies_count=models.Count( + declared_dependencies_count=Count( "declared_dependencies", filter=models.Q(declared_dependencies__product=product), ) @@ -2562,6 +2563,22 @@ def __str__(self): return f"<{self.component}>: {self.package}" +class VulnerabilityQuerySet(DataspacedQuerySet): + def with_affected_products_count(self): + """Annotate the QuerySet with the affected_products_count.""" + return self.annotate( + affected_products_count=Count( + "affected_packages__productpackages__product", distinct=True + ), + ) + + def with_affected_packages_count(self): + """Annotate the QuerySet with the affected_packages_count.""" + return self.annotate( + affected_packages_count=Count("affected_packages", distinct=True), + ) + + class Vulnerability(HistoryDateFieldsMixin, DataspacedModel): """ A software vulnerability with a unique identifier and alternate aliases. @@ -2574,6 +2591,7 @@ class Vulnerability(HistoryDateFieldsMixin, DataspacedModel): automatically on object addition or during schedule tasks. """ + # The first set of fields are storing data as fetched from VulnerableCode vulnerability_id = models.CharField( max_length=20, help_text=_( @@ -2603,6 +2621,23 @@ class Vulnerability(HistoryDateFieldsMixin, DataspacedModel): blank=True, help_text=_("A list of packages that are not affected by this vulnerability."), ) + fixed_packages_count = models.GeneratedField( + expression=models.Func(models.F("fixed_packages"), function="jsonb_array_length"), + output_field=models.IntegerField(), + db_persist=True, + ) + min_score = models.FloatField( + null=True, + blank=True, + help_text=_("The minimum score of the range."), + ) + max_score = models.FloatField( + null=True, + blank=True, + help_text=_("The maximum score of the range."), + ) + + objects = DataspacedManager.from_queryset(VulnerabilityQuerySet)() class Meta: verbose_name_plural = "Vulnerabilities" @@ -2640,11 +2675,57 @@ def add_affected_components(self, components): """Assign the ``components`` as affected to this vulnerability.""" self.affected_components.add(*components) + @staticmethod + def range_to_values(self, range_str): + try: + min_score, max_score = range_str.split("-") + return float(min_score.strip()), float(max_score.strip()) + except Exception: + return + @classmethod def create_from_data(cls, dataspace, data, validate=False, affecting=None): + # Computing the min_score and max_score from the `references` as those data + # are not provided by the VulnerableCode API. + # https://github.com/aboutcode-org/vulnerablecode/issues/1573 + # severity_range_score = data.get("severity_range_score") + # if severity_range_score: + # min_score, max_score = self.range_to_values(severity_range_score) + # data["min_score"] = min_score + # data["max_score"] = max_score + + severities = [ + score for reference in data.get("references") for score in reference.get("scores", []) + ] + if scores := cls.get_severity_scores(severities): + data["min_score"] = min(scores) + data["max_score"] = max(scores) + instance = super().create_from_data(user=dataspace, data=data, validate=False) if affecting: instance.add_affected(affecting) return instance + + @staticmethod + def get_severity_scores(severities): + score_map = { + "low": [0.1, 3], + "moderate": [4.0, 6.9], + "medium": [4.0, 6.9], + "high": [7.0, 8.9], + "important": [7.0, 8.9], + "critical": [9.0, 10.0], + } + + consolidated_scores = [] + for severity in severities: + score = severity.get("value") + try: + consolidated_scores.append(float(score)) + except ValueError: + if score_range := score_map.get(score.lower(), None): + consolidated_scores.extend(score_range) + + return consolidated_scores diff --git a/component_catalog/templates/component_catalog/includes/vulnerability_aliases.html b/component_catalog/templates/component_catalog/includes/vulnerability_aliases.html new file mode 100644 index 00000000..e64fae7a --- /dev/null +++ b/component_catalog/templates/component_catalog/includes/vulnerability_aliases.html @@ -0,0 +1,21 @@ +
    + {% for alias in aliases %} +
  • + {% if alias|slice:":3" == "CVE" %} + {{ alias }} + + + {% elif alias|slice:":4" == "GHSA" %} + {{ alias }} + + + {% elif alias|slice:":3" == "NPM" %} + {{ alias }} + + + {% else %} + {{ alias }} + {% endif %} +
    • + {% endfor %} +
\ No newline at end of file diff --git a/component_catalog/templates/component_catalog/tables/vulnerability_list_table.html b/component_catalog/templates/component_catalog/tables/vulnerability_list_table.html new file mode 100644 index 00000000..c6da9176 --- /dev/null +++ b/component_catalog/templates/component_catalog/tables/vulnerability_list_table.html @@ -0,0 +1,70 @@ +{% load i18n %} +{% load inject_preserved_filters from dje_tags %} +{% load urlize_target_blank from dje_tags %} +{% load naturaltime_short from dje_tags %} + + + {% include 'includes/object_list_table_header.html' %} + + {% for vulnerability in object_list %} + + + + + + + + + + {% empty %} + + {% endfor %} + +
+ + + {{ vulnerability.vulnerability_id }} + + + + + {% include 'component_catalog/includes/vulnerability_aliases.html' with aliases=vulnerability.aliases only %} + + {% if vulnerability.min_score %} + {{ vulnerability.min_score }} - + {% endif %} + {% if vulnerability.max_score %} + + {{ vulnerability.max_score }} + + {% endif %} + + {% if vulnerability.summary %} + {% if vulnerability.summary|length > 120 %} +
+ {{ vulnerability.summary|slice:":120" }}... + {{ vulnerability.summary|slice:"120:" }} +
+ {% else %} + {{ vulnerability.summary }} + {% endif %} + {% endif %} + 		+ {% if vulnerability.affected_products_count %} + + {{ vulnerability.affected_products_count }} + + {% else %} + 0 + {% endif %} + + {% if vulnerability.affected_packages_count %} + + {{ vulnerability.affected_packages_count }} + + {% else %} + 0 + {% endif %} + + {{ vulnerability.fixed_packages_count }} +
No results.
\ No newline at end of file diff --git a/component_catalog/templates/component_catalog/tabs/tab_vulnerabilities.html b/component_catalog/templates/component_catalog/tabs/tab_vulnerabilities.html index 9f29dea3..dbca137d 100644 --- a/component_catalog/templates/component_catalog/tabs/tab_vulnerabilities.html +++ b/component_catalog/templates/component_catalog/tabs/tab_vulnerabilities.html @@ -37,24 +37,7 @@ {{ vulnerability.summary }} - {% for alias in vulnerability.aliases %} - {% if alias|slice:":3" == "CVE" %} - {{ alias }} - - - {% elif alias|slice:":4" == "GHSA" %} - {{ alias }} - - - {% elif alias|slice:":3" == "NPM" %} - {{ alias }} - - - {% else %} - {{ alias }} - {% endif %} -
- {% endfor %} + {% include 'component_catalog/includes/vulnerability_aliases.html' with aliases=vulnerability.aliases only %} {% if vulnerability.fixed_packages_html %} diff --git a/component_catalog/templates/component_catalog/vulnerability_list.html b/component_catalog/templates/component_catalog/vulnerability_list.html new file mode 100644 index 00000000..ec99b8a0 --- /dev/null +++ b/component_catalog/templates/component_catalog/vulnerability_list.html @@ -0,0 +1,23 @@ +{% extends 'object_list_base.html' %} +{% load i18n %} + +{% block page_title %}{% trans "Vulnerabilities" %}{% endblock %} + +{% block nav-list-head %} + {{ block.super }} +
  • +
    +
    + + +
    +
    +
    • +{% endblock %} + +{% block top-right-buttons %} + {{ block.super }} +
    + {{ filter.form.sort }} +
    +{% endblock %} \ No newline at end of file diff --git a/component_catalog/tests/__init__.py b/component_catalog/tests/__init__.py index 8b27d87e..f94e5fc5 100644 --- a/component_catalog/tests/__init__.py +++ b/component_catalog/tests/__init__.py @@ -6,22 +6,16 @@ # See https://aboutcode.org for more information about AboutCode FOSS projects. # -import random -import string - from component_catalog.models import Component from component_catalog.models import Package from component_catalog.models import Vulnerability - - -def make_string(length): - return "".join(random.choices(string.ascii_letters, k=length)) +from dje.tests import make_string def make_package(dataspace, package_url=None, is_vulnerable=False, **data): """Create a package for test purposes.""" if not package_url and "filename" not in data: - data["filename"] = make_string(10) + data["filename"] = f"package-{make_string(10)}" package = Package(dataspace=dataspace, **data) if package_url: @@ -37,7 +31,7 @@ def make_package(dataspace, package_url=None, is_vulnerable=False, **data): def make_component(dataspace, is_vulnerable=False, **data): """Create a component for test purposes.""" if "name" not in data: - data["name"] = make_string(10) + data["name"] = f"component-{make_string(10)}" component = Component.objects.create( dataspace=dataspace, @@ -53,7 +47,7 @@ def make_component(dataspace, is_vulnerable=False, **data): def make_vulnerability(dataspace, affecting=None, **data): """Create a vulnerability for test purposes.""" if "vulnerability_id" not in data: - data["vulnerability_id"] = f"VCID-0000-{random.randint(1, 9999):04}" + data["vulnerability_id"] = f"VCID-0000-{make_string(4)}" vulnerability = Vulnerability.objects.create( dataspace=dataspace, diff --git a/component_catalog/tests/test_filters.py b/component_catalog/tests/test_filters.py index 1899b7b4..61ee9b76 100644 --- a/component_catalog/tests/test_filters.py +++ b/component_catalog/tests/test_filters.py @@ -14,11 +14,13 @@ from component_catalog.filters import ComponentFilterSet from component_catalog.filters import PackageFilterSet +from component_catalog.filters import VulnerabilityFilterSet from component_catalog.models import Component from component_catalog.models import ComponentKeyword from component_catalog.models import ComponentType from component_catalog.tests import make_component from component_catalog.tests import make_package +from component_catalog.tests import make_vulnerability from dje.models import Dataspace from dje.tests import create_superuser from dje.tests import create_user @@ -283,7 +285,7 @@ def test_component_filterset_sort_keeps_default_ordering_from_model(self): ) -class PackageFilterSearchTestCase(TestCase): +class PackageFilterSetTestCase(TestCase): def sorted_results(self, qs): return sorted([str(package) for package in qs]) @@ -380,9 +382,7 @@ def test_package_filterset_search_match_order_on_purl_fields(self): self.assertEqual(sorted(expected), self.sorted_results(filterset.qs)) def test_package_filterset_is_vulnerable_filter(self): - package1 = make_package( - self.dataspace, package_url="pkg:pypi/django@5.0", is_vulnerable=True - ) + package1 = make_package(self.dataspace, is_vulnerable=True) self.assertTrue(package1.is_vulnerable) filterset = PackageFilterSet(dataspace=self.dataspace) @@ -395,3 +395,73 @@ def test_package_filterset_is_vulnerable_filter(self): data = {"is_vulnerable": "no"} filterset = PackageFilterSet(dataspace=self.dataspace, data=data) self.assertNotIn(package1, filterset.qs) + + def test_package_filterset_affected_by_filter(self): + package1 = make_package(self.dataspace) + package2 = make_package(self.dataspace) + vulnerability1 = make_vulnerability(self.dataspace, affecting=package1) + self.assertTrue(package1.is_vulnerable) + self.assertFalse(package2.is_vulnerable) + + filterset = PackageFilterSet(dataspace=self.dataspace) + self.assertIn(package1, filterset.qs) + self.assertIn(package2, filterset.qs) + + data = {"affected_by": vulnerability1.vulnerability_id} + filterset = PackageFilterSet(dataspace=self.dataspace, data=data) + self.assertQuerySetEqual(filterset.qs, [package1]) + + +class VulnerabilityFilterSetTestCase(TestCase): + def setUp(self): + self.dataspace = Dataspace.objects.create(name="Reference") + self.vulnerability1 = make_vulnerability(self.dataspace, max_score=10.0) + self.vulnerability2 = make_vulnerability( + self.dataspace, max_score=5.5, aliases=["ALIAS-V2"] + ) + self.vulnerability3 = make_vulnerability(self.dataspace, max_score=2.0) + self.vulnerability4 = make_vulnerability(self.dataspace, max_score=None) + + def test_vulnerability_filterset_search(self): + data = {"q": self.vulnerability1.vulnerability_id} + filterset = VulnerabilityFilterSet(dataspace=self.dataspace, data=data) + self.assertQuerySetEqual(filterset.qs, [self.vulnerability1]) + + data = {"q": "ALIAS-V2"} + filterset = VulnerabilityFilterSet(dataspace=self.dataspace, data=data) + self.assertQuerySetEqual(filterset.qs, [self.vulnerability2]) + + def test_vulnerability_filterset_sort_nulls_last_ordering(self): + data = {"sort": "max_score"} + filterset = VulnerabilityFilterSet(dataspace=self.dataspace, data=data) + expected = [ + self.vulnerability3, + self.vulnerability2, + self.vulnerability1, + self.vulnerability4, # The max_score=None are always last + ] + self.assertQuerySetEqual(filterset.qs, expected) + + data = {"sort": "-max_score"} + filterset = VulnerabilityFilterSet(dataspace=self.dataspace, data=data) + expected = [ + self.vulnerability1, + self.vulnerability2, + self.vulnerability3, + self.vulnerability4, # The max_score=None are always last + ] + self.assertQuerySetEqual(filterset.qs, expected) + + def test_vulnerability_filterset_max_score(self): + data = {"max_score": "critical"} + filterset = VulnerabilityFilterSet(dataspace=self.dataspace, data=data) + self.assertQuerySetEqual(filterset.qs, [self.vulnerability1]) + data = {"max_score": "high"} + filterset = VulnerabilityFilterSet(dataspace=self.dataspace, data=data) + self.assertQuerySetEqual(filterset.qs, []) + data = {"max_score": "medium"} + filterset = VulnerabilityFilterSet(dataspace=self.dataspace, data=data) + self.assertQuerySetEqual(filterset.qs, [self.vulnerability2]) + data = {"max_score": "low"} + filterset = VulnerabilityFilterSet(dataspace=self.dataspace, data=data) + self.assertQuerySetEqual(filterset.qs, [self.vulnerability3]) diff --git a/component_catalog/tests/test_models.py b/component_catalog/tests/test_models.py index 0fedc6cd..1cdb302e 100644 --- a/component_catalog/tests/test_models.py +++ b/component_catalog/tests/test_models.py @@ -58,9 +58,7 @@ from license_library.models import LicenseChoice from license_library.models import LicenseTag from organization.models import Owner -from product_portfolio.models import Product -from product_portfolio.models import ProductComponent -from product_portfolio.models import ProductPackage +from product_portfolio.tests import make_product class ComponentCatalogModelsTestCase(TestCase): @@ -2490,11 +2488,7 @@ def test_package_create_save_set_usage_policy_from_license(self): self.assertEqual(package_policy, package5.usage_policy) def test_component_model_where_used_property(self): - product1 = Product.objects.create(name="P1", dataspace=self.dataspace) - ProductComponent.objects.create( - product=product1, component=self.component1, dataspace=self.dataspace - ) - + make_product(self.dataspace, inventory=[self.component1]) basic_user = create_user("basic_user", self.dataspace) self.assertEqual("Product 0\n", self.component1.where_used(user=basic_user)) @@ -2502,9 +2496,8 @@ def test_component_model_where_used_property(self): self.assertEqual("Product 1\n", self.component1.where_used(user=self.user)) def test_package_model_where_used_property(self): - product1 = Product.objects.create(name="P1", dataspace=self.dataspace) package1 = Package.objects.create(filename="package", dataspace=self.dataspace) - ProductPackage.objects.create(product=product1, package=package1, dataspace=self.dataspace) + make_product(self.dataspace, inventory=[package1]) basic_user = create_user("basic_user", self.dataspace) self.assertEqual("Product 0\nComponent 0\n", package1.where_used(user=basic_user)) @@ -2623,6 +2616,18 @@ def test_vulnerability_model_add_affected(self): self.assertQuerySetEqual(vulnerablity2.affected_packages.all(), [package1]) self.assertQuerySetEqual(vulnerablity2.affected_components.all(), [component1]) + def test_vulnerability_model_fixed_packages_count_generated_field(self): + vulnerablity1 = make_vulnerability(dataspace=self.dataspace) + self.assertEqual(0, vulnerablity1.fixed_packages_count) + + vulnerablity1.fixed_packages = [ + {"purl": "pkg:pypi/gitpython@3.1.41", "is_vulnerable": True}, + {"purl": "pkg:pypi/gitpython@3.2", "is_vulnerable": False}, + ] + vulnerablity1.save() + vulnerablity1.refresh_from_db() + self.assertEqual(2, vulnerablity1.fixed_packages_count) + def test_vulnerability_model_create_from_data(self): package1 = make_package(self.dataspace) vulnerability_data = { @@ -2654,4 +2659,32 @@ def test_vulnerability_model_create_from_data(self): self.assertEqual(vulnerability_data["summary"], vulnerability1.summary) self.assertEqual(vulnerability_data["aliases"], vulnerability1.aliases) self.assertEqual(vulnerability_data["references"], vulnerability1.references) + self.assertEqual(7.5, vulnerability1.min_score) + self.assertEqual(7.5, vulnerability1.max_score) self.assertQuerySetEqual(vulnerability1.affected_packages.all(), [package1]) + + def test_vulnerability_model_create_from_data_computed_scores(self): + response_file = self.data / "vulnerabilities" / "idna_3.6_response.json" + json_data = json.loads(response_file.read_text()) + affected_by_vulnerabilities = json_data["results"][0]["affected_by_vulnerabilities"] + vulnerability1 = Vulnerability.create_from_data( + dataspace=self.dataspace, + data=affected_by_vulnerabilities[0], + ) + self.assertEqual(2.1, vulnerability1.min_score) + self.assertEqual(7.5, vulnerability1.max_score) + + def test_vulnerability_model_queryset_count_methods(self): + package1 = make_package(self.dataspace) + package2 = make_package(self.dataspace) + vulnerablity1 = make_vulnerability(dataspace=self.dataspace) + vulnerablity1.add_affected([package1, package2]) + make_product(self.dataspace, inventory=[package1, package2]) + + qs = ( + Vulnerability.objects.scope(self.dataspace) + .with_affected_products_count() + .with_affected_packages_count() + ) + self.assertEqual(2, qs[0].affected_packages_count) + self.assertEqual(1, qs[0].affected_products_count) diff --git a/component_catalog/tests/test_views.py b/component_catalog/tests/test_views.py index f1c040c3..78f8b79d 100644 --- a/component_catalog/tests/test_views.py +++ b/component_catalog/tests/test_views.py @@ -39,6 +39,9 @@ from component_catalog.models import ComponentType from component_catalog.models import Package from component_catalog.models import Subcomponent +from component_catalog.models import Vulnerability +from component_catalog.tests import make_component +from component_catalog.tests import make_package from component_catalog.tests import make_vulnerability from component_catalog.views import ComponentAddView from component_catalog.views import ComponentListView @@ -48,6 +51,7 @@ from dejacode_toolkit.vulnerablecode import get_plain_purls from dje.copier import copy_object from dje.models import Dataspace +from dje.models import DataspaceConfiguration from dje.models import ExternalReference from dje.models import ExternalSource from dje.models import History @@ -4811,3 +4815,63 @@ def test_anonymous_user_cannot_access_reference_data(self): self.assertEqual(404, self.client.get(self.d1c2.get_absolute_url()).status_code) self.assertEqual(200, self.client.get(self.dmc1.get_absolute_url()).status_code) self.assertEqual(200, self.client.get(self.dmc2.get_absolute_url()).status_code) + + +class VulnerabilityViewsTestCase(TestCase): + def setUp(self): + self.dataspace = Dataspace.objects.create( + name="Dataspace", + enable_vulnerablecodedb_access=True, + ) + DataspaceConfiguration.objects.create( + dataspace=self.dataspace, + vulnerablecode_url="vulnerablecode_url/", + ) + self.super_user = create_superuser("super_user", self.dataspace) + + self.component1 = make_component(self.dataspace) + self.component2 = make_component(self.dataspace) + self.package1 = make_package(self.dataspace) + self.package2 = make_package(self.dataspace) + self.vulnerability_p1 = make_vulnerability(self.dataspace, affecting=self.component1) + self.vulnerability_c1 = make_vulnerability(self.dataspace, affecting=self.package1) + self.vulnerability1 = make_vulnerability(self.dataspace) + + def test_vulnerability_list_view_num_queries(self): + self.client.login(username=self.super_user.username, password="secret") + with self.assertNumQueries(8): + response = self.client.get(reverse("component_catalog:vulnerability_list")) + + vulnerability_count = Vulnerability.objects.count() + expected = f'{vulnerability_count} results' + self.assertContains(response, expected, html=True) + + def test_vulnerability_list_view_enable_vulnerablecodedb_access(self): + self.client.login(username=self.super_user.username, password="secret") + vulnerability_list_url = reverse("component_catalog:vulnerability_list") + response = self.client.get(vulnerability_list_url) + self.assertEqual(200, response.status_code) + vulnerability_header_link = ( + f'' + ) + self.assertContains(response, vulnerability_header_link) + + self.dataspace.enable_vulnerablecodedb_access = False + self.dataspace.save() + response = self.client.get(reverse("component_catalog:vulnerability_list")) + self.assertEqual(404, response.status_code) + + response = self.client.get(reverse("component_catalog:package_list")) + self.assertNotContains(response, vulnerability_header_link) + + def test_vulnerability_list_view_vulnerability_id_link(self): + self.client.login(username=self.super_user.username, password="secret") + response = self.client.get(reverse("component_catalog:vulnerability_list")) + expected = f""" + + {self.vulnerability1.vulnerability_id} + + + """ + self.assertContains(response, expected, html=True) diff --git a/component_catalog/urls.py b/component_catalog/urls.py index 53ff217a..50e87277 100644 --- a/component_catalog/urls.py +++ b/component_catalog/urls.py @@ -21,6 +21,7 @@ from component_catalog.views import PackageTabScanView from component_catalog.views import PackageUpdateView from component_catalog.views import ScanListView +from component_catalog.views import VulnerabilityListView from component_catalog.views import component_create_ajax_view from component_catalog.views import delete_scan_view from component_catalog.views import package_create_ajax_view @@ -148,6 +149,14 @@ ), ] +vulnerabilities_patterns = [ + path( + "vulnerabilities/", + VulnerabilityListView.as_view(), + name="vulnerability_list", + ), +] + # WARNING: we moved the components/ patterns from the include to the following # since we need the packages/ and scans/ to be register on the root "/" @@ -239,4 +248,4 @@ def component_path(path_segment, view): ] -urlpatterns = packages_patterns + component_patterns + scans_patterns +urlpatterns = packages_patterns + component_patterns + scans_patterns + vulnerabilities_patterns diff --git a/component_catalog/views.py b/component_catalog/views.py index 572296ca..aa703af2 100644 --- a/component_catalog/views.py +++ b/component_catalog/views.py @@ -22,6 +22,7 @@ from django.core import signing from django.core.validators import EMPTY_VALUES from django.db.models import Count +from django.db.models import F from django.db.models import Prefetch from django.http import FileResponse from django.http import Http404 @@ -52,6 +53,7 @@ from component_catalog.filters import ComponentFilterSet from component_catalog.filters import PackageFilterSet +from component_catalog.filters import VulnerabilityFilterSet from component_catalog.forms import AddMultipleToComponentForm from component_catalog.forms import AddToComponentForm from component_catalog.forms import AddToProductAdminForm @@ -71,6 +73,7 @@ from component_catalog.models import Package from component_catalog.models import PackageAlreadyExistsWarning from component_catalog.models import Subcomponent +from component_catalog.models import Vulnerability from dejacode_toolkit.download import DataCollectionException from dejacode_toolkit.purldb import PurlDB from dejacode_toolkit.scancodeio import ScanCodeIO @@ -2476,3 +2479,57 @@ def get_tab_fields(self): tab_fields.extend(get_purldb_tab_fields(purldb_entry, user.dataspace)) return {"fields": tab_fields} + + +class VulnerabilityListView( + LoginRequiredMixin, + DataspacedFilterView, +): + model = Vulnerability + filterset_class = VulnerabilityFilterSet + template_name = "component_catalog/vulnerability_list.html" + template_list_table = "component_catalog/tables/vulnerability_list_table.html" + table_headers = ( + Header("vulnerability_id", _("Vulnerability")), + Header("aliases", _("Aliases")), + # Keep `max_score` to enable column sorting + Header("max_score", _("Score"), help_text="Severity score range", filter="max_score"), + Header("summary", _("Summary")), + Header("affected_products_count", _("Affected products"), help_text="Affected products"), + Header("affected_packages_count", _("Affected packages"), help_text="Affected packages"), + Header("fixed_packages_count", _("Fixed by"), help_text="Fixed by packages"), + ) + + def get_queryset(self): + return ( + super() + .get_queryset() + .only( + "uuid", + "vulnerability_id", + "aliases", + "summary", + "fixed_packages_count", + "max_score", + "min_score", + "created_date", + "last_modified_date", + "dataspace", + ) + .with_affected_products_count() + .with_affected_packages_count() + .order_by( + F("max_score").desc(nulls_last=True), + "-min_score", + ) + ) + + def get_context_data(self, **kwargs): + context_data = super().get_context_data(**kwargs) + + if not self.dataspace.enable_vulnerablecodedb_access: + raise Http404("VulnerableCode access is not enabled.") + + vulnerablecode = VulnerableCode(self.dataspace) + context_data["vulnerablecode_url"] = vulnerablecode.service_url + return context_data diff --git a/dejacode/static/css/dejacode_bootstrap.css b/dejacode/static/css/dejacode_bootstrap.css index 26bf7b52..c344a404 100644 --- a/dejacode/static/css/dejacode_bootstrap.css +++ b/dejacode/static/css/dejacode_bootstrap.css @@ -358,6 +358,24 @@ table.packages-table .column-primary_language { margin-right: 0.3rem; } +/* -- Vulnerability List -- */ +table.vulnerabilities-table .column-vulnerability_id { + width: 220px; +} +table.vulnerabilities-table .column-aliases { + width: 210px; +} +table.vulnerabilities-table .column-score_range { + width: 110px; +} +table.vulnerabilities-table .column-priority { + width: 110px; +} +table.vulnerabilities-table .column-summary { + width: 300px; + max-width: 300px; +} + /* -- Package Details -- */ textarea.licenseexpressionwidget { height: 62px; diff --git a/dje/templates/admin/base_site.html b/dje/templates/admin/base_site.html index 8c7ca25a..ae0db44c 100644 --- a/dje/templates/admin/base_site.html +++ b/dje/templates/admin/base_site.html @@ -42,6 +42,7 @@

    {% url 'workflow:request_list' as request_list_url %} {% url 'component_catalog:scan_list' as scan_list_url %} {% url 'purldb:purldb_list' as purldb_list_url %} + {% url 'component_catalog:vulnerability_list' as vulnerability_list_url %} {% url 'api_v2:api-root' as api_root_url %} {% if report_list_url or request_list_url or api_root_url %}
  • @@ -64,6 +65,10 @@

  • {% trans 'PurlDB' %}
    • {% endif %} {% endif %} + {% if user.dataspace.enable_vulnerablecodedb_access %} +
  • {% trans 'Vulnerabilities' %}
    • +
  • {% trans 'Vulnerabilities' %}
    • + {% endif %} {% if api_root_url %}
  • {% trans 'API' %}
  • {% trans 'API Root' %}
    • diff --git a/dje/templates/includes/navbar_header.html b/dje/templates/includes/navbar_header.html index a4c283c6..44948a75 100644 --- a/dje/templates/includes/navbar_header.html +++ b/dje/templates/includes/navbar_header.html @@ -11,6 +11,7 @@ {% url 'workflow:request_list' as request_list_url %} {% url 'component_catalog:scan_list' as scan_list_url %} {% url 'purldb:purldb_list' as purldb_list_url %} +{% url 'component_catalog:vulnerability_list' as vulnerability_list_url %} {% url 'django_registration_register' as register_url %} {% url 'api_v2:api-root' as api_root_url %} {% url 'account_profile' as account_profile_url %} diff --git a/dje/templates/includes/navbar_header_tools_menu.html b/dje/templates/includes/navbar_header_tools_menu.html index 133cb0b6..28b4f707 100644 --- a/dje/templates/includes/navbar_header_tools_menu.html +++ b/dje/templates/includes/navbar_header_tools_menu.html @@ -32,6 +32,14 @@ {% endif %} {% endif %} + {% if user.dataspace.enable_vulnerablecodedb_access %} +
    +
    {% trans 'Vulnerabilities' %}
    + + + {% trans 'Vulnerabilities' %} + + {% endif %}
    API
    diff --git a/purldb/templates/purldb/includes/filterset_modal.html b/dje/templates/modals/filterset_modal.html similarity index 72% rename from purldb/templates/purldb/includes/filterset_modal.html rename to dje/templates/modals/filterset_modal.html index 6aabba38..86f86e29 100644 --- a/purldb/templates/purldb/includes/filterset_modal.html +++ b/dje/templates/modals/filterset_modal.html @@ -1,14 +1,14 @@ {% load crispy_forms_tags %} -