Base implementation of a Vulnerability models #94

CHANGELOG.rst

@@ -58,6 +58,41 @@ Release notes
   Product.
   https://github.com/nexB/dejacode/issues/138
 
+- Add a task scheduler service to the Docker Compose stack.
+  This service runs a dedicated ``setupcron`` management command to create the
+  application's scheduled cron jobs.
+  The scheduler is configured to run the daily vulnerabilities update task.
+  https://github.com/nexB/dejacode/issues/94
+
+- Add a new Vulnerability model and all the code logic to fetch and create
+  Vulnerability records and assign those to Package/Component through ManyToMany
+  relationships.
+  A fetchvulnerabilities management command is available to fetch all the relevant
+  data from VulnerableCode for a given Dataspace.
+  The latest vulnerability data refresh date is displayed in the Admin dashboard in a
+  new "Data updates" section in the bottom right corner.
+  It is also available in the "Integration Status" page.
+  The Package/Component views that display vulnerability information (icon or tab)
+  are now using the data from the Vulnerability model in place of calling the
+  VulnerableCode API on each request. This results into much better performances as
+  we do not depend on the VulnerableCode service to render the DejaCode view anymore.
+  Also, this will make Vulnerability data available in the Reporting system.
+  The vulnerability icon is displayed next to the Package/Component identifier in the
+  Product views: "Inventory", "Hierarchy", "Dependencies" tabs.
+  The vulnerability data is available in Reporting either through the is_vulnerable
+  property on Package/Component column template or going through the full
+  affected_by_vulnerabilities m2m field.
+  This is available in both Query and ColumnTemplate.
+  The vulnerabilities are fetched each time a Package is created/modified
+  (note that a purl is required on the package for the lookup).
+  Also, all the Packages of a Product are updated with latest vulnerabilities from
+  the VulnerableCode service following importing data in Product using:
+  - Import data from Scan
+  - Load Packages from SBOMs
+  - Import Packages from manifests
+  - Pull ScanCode.io Project data
+  https://github.com/nexB/dejacode/issues/94
+
 ### Version 5.1.0
 
 - Upgrade Python version to 3.12 and Django to 5.0.x

component_catalog/filters.py

@@ -28,6 +28,21 @@
 from license_library.models import License
 
 
+class IsVulnerableFilter(HasRelationFilter):
+    def __init__(self, *args, **kwargs):
+        kwargs["lookup_expr"] = "isnull"
+        kwargs["empty_label"] = "Any"
+        kwargs.setdefault("label", _("Is Vulnerable"))
+        kwargs.setdefault(
+            "choices",
+            (
+                ("yes", _("Affected by vulnerabilities")),
+                ("no", _("No vulnerabilities found")),
+            ),
+        )
+        super().__init__(*args, **kwargs)
+
+
 class ComponentFilterSet(DataspacedFilterSet):
     related_only = [
         "licenses",
@@ -85,6 +100,10 @@ class ComponentFilterSet(DataspacedFilterSet):
             search_placeholder="Search keywords",
         ),
     )
+    is_vulnerable = IsVulnerableFilter(
+        field_name="affected_by_vulnerabilities",
+        widget=DropDownRightWidget(link_content='<i class="fas fa-bug"></i>'),
+    )
 
     class Meta:
         model = Component
@@ -219,6 +238,10 @@ class PackageFilterSet(DataspacedFilterSet):
         empty_label="Last modified (default)",
         widget=SortDropDownWidget,
     )
+    is_vulnerable = IsVulnerableFilter(
+        field_name="affected_by_vulnerabilities",
+        widget=DropDownRightWidget(link_content='<i class="fas fa-bug"></i>'),
+    )
 
     class Meta:
         model = Package

component_catalog/forms.py

@@ -446,6 +446,9 @@ def save(self, *args, **kwargs):
             )
             self.cleaned_data["scan_submitted"] = True
 
+        if self.user.dataspace.enable_vulnerablecodedb_access:
+            instance.fetch_vulnerabilities()
+
         return instance
 
 
@@ -1043,6 +1046,11 @@ def save(self, commit=True):
         self._set_purldb_uuid_on_instance()
         return super().save(commit)
 
+    def _save_m2m(self):
+        super()._save_m2m()
+        if self.dataspace.enable_vulnerablecodedb_access:
+            self.instance.fetch_vulnerabilities()
+
 
 class ComponentMassUpdateForm(
     LicenseExpressionFormMixin,

component_catalog/importers.py

@@ -30,6 +30,7 @@
 from component_catalog.models import Package
 from component_catalog.models import Subcomponent
 from component_catalog.programming_languages import PROGRAMMING_LANGUAGES
+from component_catalog.vulnerabilities import fetch_for_queryset
 from dje.fields import SmartFileField
 from dje.forms import JSONListField
 from dje.importers import BaseImporter
@@ -182,6 +183,7 @@ class Meta:
             "packages",
             "completion_level",
             "request_count",
+            "affected_by_vulnerabilities",
             # JSONField not supported
             "dependencies",
         )
@@ -261,6 +263,7 @@ class Meta:
             "file_references",
             "request_count",
             "parties",
+            "affected_by_vulnerabilities",
         ]
 
     def __init__(self, *args, **kwargs):
@@ -343,7 +346,7 @@ def prepare_data_json(self, data):
         input_as_list_of_dict = packages
 
         # Using a dict comprehension to keep the original key order and
-        # ensure we have all possibile headers.
+        # ensure we have all possible headers.
         header_row = {key: None for package in packages for key in package.keys()}
         header_row = list(header_row.keys())
 
@@ -423,6 +426,15 @@ def get_form_kwargs(self):
         kwargs["is_from_scancode"] = getattr(self, "is_from_scancode", False)
         return kwargs
 
+    def save_all(self):
+        """Fetch vulnerabilities for imported Packages."""
+        super().save_all()
+
+        if self.dataspace.enable_vulnerablecodedb_access:
+            package_pks = [package.pk for package in self.results["added"]]
+            package_qs = Package.objects.scope(dataspace=self.dataspace).filter(pk__in=package_pks)
+            fetch_for_queryset(package_qs, self.dataspace)
+
 
 class SubcomponentImportForm(
     ComponentRelatedFieldImportMixin,

component_catalog/management/commands/fetchvulnerabilities.py

@@ -0,0 +1,46 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# DejaCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: AGPL-3.0-only
+# See https://github.com/nexB/dejacode for support or download.
+# See https://aboutcode.org for more information about AboutCode FOSS projects.
+#
+
+from django.core.management.base import CommandError
+
+from component_catalog.vulnerabilities import fetch_from_vulnerablecode
+from dejacode_toolkit.vulnerablecode import VulnerableCode
+from dje.management.commands import DataspacedCommand
+
+
+class Command(DataspacedCommand):
+    help = "Fetch vulnerabilities for the provided Dataspace"
+
+    def add_arguments(self, parser):
+        super().add_arguments(parser)
+        parser.add_argument(
+            "--batch-size",
+            type=int,
+            default=50,
+            help="Specifies the number of objects per requests to the VulnerableCode service",
+        )
+        parser.add_argument(
+            "--timeout",
+            type=int,
+            default=30,
+            help="Request timeout in seconds",
+        )
+
+    def handle(self, *args, **options):
+        super().handle(*args, **options)
+        batch_size = options["batch_size"]
+        timeout = options["timeout"]
+
+        if not self.dataspace.enable_vulnerablecodedb_access:
+            raise CommandError("VulnerableCode is not enabled on this Dataspace.")
+
+        vulnerablecode = VulnerableCode(self.dataspace)
+        if not vulnerablecode.is_configured():
+            raise CommandError("VulnerableCode is not configured.")
+
+        fetch_from_vulnerablecode(self.dataspace, batch_size, timeout, log_func=self.stdout.write)

component_catalog/migrations/0006_vulnerability_model_and_missing_indexes.py

@@ -0,0 +1,129 @@
+# Generated by Django 5.0.6 on 2024-08-21 11:20
+
+import component_catalog.models
+import django.db.models.deletion
+import dje.fields
+import uuid
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('component_catalog', '0005_remove_component_concluded_license_and_more'),
+        ('dje', '0004_dataspace_vulnerabilities_updated_at'),
+        ('license_library', '0002_initial'),
+        ('policy', '0002_initial'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='package',
+            name='filename',
+            field=models.CharField(blank=True, help_text='The exact file name (typically an archive of some type) of the package. This is usually the name of the file as downloaded from a website.', max_length=255, validators=[component_catalog.models.validate_filename], verbose_name='Filename'),
+        ),
+        migrations.AlterField(
+            model_name='package',
+            name='license_expression',
+            field=models.CharField(blank=True, help_text='The License Expression assigned to a DejaCode Package or Component is an editable value equivalent to a "concluded license" as determined by a curator who has performed analysis to clarify or correct the declared license expression, which may have been assigned automatically (from a scan or an associated package definition) when the Package or Component was originally created. A license expression defines the relationship of one or more licenses to a software object. More than one applicable license can be expressed as "license-key-a AND license-key-b". A choice of applicable licenses can be expressed as "license-key-a OR license-key-b", and you can indicate the primary (preferred) license by placing it first, on the left-hand side of the OR relationship. The relationship words (OR, AND) can be combined as needed, and the use of parentheses can be applied to clarify the meaning; for example "((license-key-a AND license-key-b) OR (license-key-c))". An exception to a license can be expressed as "license-key WITH license-exception-key".', max_length=1024, verbose_name='Concluded license expression'),
+        ),
+        migrations.AlterField(
+            model_name='package',
+            name='md5',
+            field=models.CharField(blank=True, help_text='MD5 checksum hex-encoded, as in md5sum.', max_length=32, verbose_name='MD5'),
+        ),
+        migrations.AlterField(
+            model_name='package',
+            name='primary_language',
+            field=models.CharField(blank=True, help_text='The primary programming language associated with the package.', max_length=50),
+        ),
+        migrations.AlterField(
+            model_name='package',
+            name='project',
+            field=models.CharField(blank=True, help_text='Project is a free-form label that you can use to group and find packages and components that interest you; for example, you may be starting a new development project, evaluating them for use in a product or you may want to get approval to use them.', max_length=50),
+        ),
+        migrations.AlterField(
+            model_name='package',
+            name='sha1',
+            field=models.CharField(blank=True, help_text='SHA1 checksum hex-encoded, as in sha1sum.', max_length=40, verbose_name='SHA1'),
+        ),
+        migrations.AlterField(
+            model_name='package',
+            name='size',
+            field=models.BigIntegerField(blank=True, help_text='The size of the package file in bytes.', null=True),
+        ),
+        migrations.CreateModel(
+            name='Vulnerability',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, verbose_name='UUID')),
+                ('created_date', models.DateTimeField(auto_now_add=True, db_index=True, help_text='The date and time the object was created.')),
+                ('last_modified_date', models.DateTimeField(auto_now=True, db_index=True, help_text='The date and time the object was last modified.')),
+                ('vulnerability_id', models.CharField(help_text="A unique identifier for the vulnerability, prefixed with 'VCID-'. For example, 'VCID-2024-0001'.", max_length=20)),
+                ('summary', models.TextField(blank=True, help_text='A brief summary of the vulnerability, outlining its nature and impact.')),
+                ('aliases', dje.fields.JSONListField(blank=True, default=list, help_text="A list of aliases for this vulnerability, such as CVE identifiers (e.g., 'CVE-2017-1000136').")),
+                ('references', dje.fields.JSONListField(blank=True, default=list, help_text='A list of references for this vulnerability. Each reference includes a URL, an optional reference ID, scores, and the URL for further details. ')),
+                ('fixed_packages', dje.fields.JSONListField(blank=True, default=list, help_text='A list of packages that are not affected by this vulnerability.')),
+                ('dataspace', models.ForeignKey(editable=False, help_text='A Dataspace is an independent, exclusive set of DejaCode data, which can be either nexB master reference data or installation-specific data.', on_delete=django.db.models.deletion.PROTECT, to='dje.dataspace')),
+            ],
+            options={
+                'verbose_name_plural': 'Vulnerabilities',
+            },
+        ),
+        migrations.AddField(
+            model_name='component',
+            name='affected_by_vulnerabilities',
+            field=models.ManyToManyField(help_text='Vulnerabilities affecting this object.', related_name='affected_%(class)ss', to='component_catalog.vulnerability'),
+        ),
+        migrations.AddField(
+            model_name='package',
+            name='affected_by_vulnerabilities',
+            field=models.ManyToManyField(help_text='Vulnerabilities affecting this object.', related_name='affected_%(class)ss', to='component_catalog.vulnerability'),
+        ),
+        migrations.AddIndex(
+            model_name='package',
+            index=models.Index(fields=['type'], name='component_c_type_4af44a_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='package',
+            index=models.Index(fields=['namespace'], name='component_c_namespa_d10c6b_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='package',
+            index=models.Index(fields=['name'], name='component_c_name_c82010_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='package',
+            index=models.Index(fields=['version'], name='component_c_version_ad217d_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='package',
+            index=models.Index(fields=['filename'], name='component_c_filenam_b60780_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='package',
+            index=models.Index(fields=['size'], name='component_c_size_22e128_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='package',
+            index=models.Index(fields=['primary_language'], name='component_c_primary_2e8211_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='package',
+            index=models.Index(fields=['project'], name='component_c_project_74b1c0_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='package',
+            index=models.Index(fields=['license_expression'], name='component_c_license_20719b_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='vulnerability',
+            index=models.Index(fields=['vulnerability_id'], name='component_c_vulnera_e18d50_idx'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='vulnerability',
+            unique_together={('dataspace', 'uuid'), ('dataspace', 'vulnerability_id')},
+        ),
+    ]