From 024697b923b6e370d88b0decd5b045f746919262 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Fri, 29 Mar 2024 15:53:14 +0400 Subject: [PATCH] Add support to import packages from manifest #65 (#67) Signed-off-by: tdruez --- CHANGELOG.rst | 3 + dejacode_toolkit/scancodeio.py | 8 +- dje/tasks.py | 18 +-- product_portfolio/api.py | 37 ++++++ product_portfolio/forms.py | 29 ++++- product_portfolio/importers.py | 21 +++- product_portfolio/models.py | 4 +- .../import_manifests_form.html | 77 ++++++++++++ .../product_portfolio/product_details.html | 1 + .../scancodeio_project_status.html | 35 +++++- .../scancodeio_pull_data_status.html | 28 ----- product_portfolio/tests/test_api.py | 29 +++++ product_portfolio/tests/test_views.py | 43 +++++-- product_portfolio/urls.py | 2 + product_portfolio/views.py | 111 ++++++++---------- 15 files changed, 325 insertions(+), 121 deletions(-) create mode 100644 product_portfolio/templates/product_portfolio/import_manifests_form.html delete mode 100644 product_portfolio/templates/product_portfolio/scancodeio_pull_data_status.html diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6e28c6ab..c0dfefcf 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -31,6 +31,9 @@ Release notes - Refactor the "Import manifest" feature as "Load SBOMs". https://github.com/nexB/dejacode/issues/61 +- Add support to import packages from manifest. + https://github.com/nexB/dejacode/issues/65 + - Add a vulnerability link to the VulnerableCode app in the Vulnerability tab. https://github.com/nexB/dejacode/issues/4 diff --git a/dejacode_toolkit/scancodeio.py b/dejacode_toolkit/scancodeio.py index 962043e5..332217a8 100644 --- a/dejacode_toolkit/scancodeio.py +++ b/dejacode_toolkit/scancodeio.py @@ -76,10 +76,12 @@ def submit_scan(self, uri, user_uuid, dataspace_uuid): logger.debug(f'{self.label}: submit scan uri="{uri}" webhook_url="{webhook_url}"') return self.request_post(url=self.project_api_url, json=data) - def submit_load_sbom(self, project_name, file_location, user_uuid, execute_now=False): + def submit_project( + self, project_name, pipeline_name, file_location, user_uuid, execute_now=False + ): data = { "name": project_name, - "pipeline": "load_sbom", + "pipeline": pipeline_name, "execute_now": execute_now, } files = { @@ -92,7 +94,7 @@ def submit_load_sbom(self, project_name, file_location, user_uuid, execute_now=F data["webhook_url"] = webhook_url logger.debug( - f"{self.label}: submit load sbom " + f"{self.label}: submit pipeline={pipeline_name} " f'project_name="{project_name}" webhook_url="{webhook_url}"' ) return self.request_post(url=self.project_api_url, data=data, files=files) diff --git a/dje/tasks.py b/dje/tasks.py index 3f7c810d..724fd458 100644 --- a/dje/tasks.py +++ b/dje/tasks.py @@ -115,13 +115,14 @@ def scancodeio_submit_scan(uris, user_uuid, dataspace_uuid): @job -def scancodeio_submit_load_sbom(scancodeproject_uuid, user_uuid): +def scancodeio_submit_project(scancodeproject_uuid, user_uuid, pipeline_name): """Submit the provided SBOM file to ScanCode.io as an asynchronous task.""" from dje.models import DejacodeUser logger.info( - f"Entering scancodeio_submit_load_sbom task with " - f"scancodeproject_uuid={scancodeproject_uuid} user_uuid={user_uuid}" + f"Entering scancodeio_submit_project task with " + f"scancodeproject_uuid={scancodeproject_uuid} user_uuid={user_uuid} " + f"pipeline_name={pipeline_name}" ) ScanCodeProject = apps.get_model("product_portfolio", "scancodeproject") @@ -137,24 +138,25 @@ def scancodeio_submit_load_sbom(scancodeproject_uuid, user_uuid): # Create a Project instance on ScanCode.io without immediate execution of the # pipeline. This allows to get instant feedback from ScanCode.io about the Project # creation status and its related data, even in SYNC mode. - response = scancodeio.submit_load_sbom( + response = scancodeio.submit_project( project_name=scancodeproject_uuid, + pipeline_name=pipeline_name, file_location=scancode_project.input_file.path, user_uuid=user_uuid, execute_now=False, ) if not response: - logger.info("Error submitting the SBOM file to ScanCode.io server") + logger.info("Error submitting the file to ScanCode.io server") scancode_project.status = ScanCodeProject.Status.FAILURE - msg = "- Error: SBOM could not be submitted to ScanCode.io" + msg = "- Error: File could not be submitted to ScanCode.io" scancode_project.append_to_log(msg, save=True) return logger.info("Update the ScanCodeProject instance") scancode_project.status = ScanCodeProject.Status.SUBMITTED scancode_project.project_uuid = response.get("uuid") - msg = "- SBOM file submitted to ScanCode.io for inspection" + msg = "- File submitted to ScanCode.io for inspection" scancode_project.append_to_log(msg, save=True) # Delay the execution of the pipeline after the ScancodeProject instance was @@ -164,7 +166,7 @@ def scancodeio_submit_load_sbom(scancodeproject_uuid, user_uuid): transaction.on_commit(lambda: scancodeio.start_pipeline(run_url=runs[0]["url"])) -@job +@job("default", timeout=1200) def pull_project_data_from_scancodeio(scancodeproject_uuid): """ Pull Project data from ScanCode.io as an asynchronous task for the provided diff --git a/product_portfolio/api.py b/product_portfolio/api.py index c8c4a1d8..45a22304 100644 --- a/product_portfolio/api.py +++ b/product_portfolio/api.py @@ -35,6 +35,7 @@ from dje.permissions import assign_all_object_permissions from product_portfolio.filters import ComponentCompletenessAPIFilter from product_portfolio.forms import ImportFromScanForm +from product_portfolio.forms import ImportManifestsForm from product_portfolio.forms import LoadSBOMsForm from product_portfolio.forms import PullProjectDataForm from product_portfolio.models import CodebaseResource @@ -216,6 +217,25 @@ class LoadSBOMsFormSerializer(serializers.Serializer): ) +class ImportManifestsFormSerializer(serializers.Serializer): + """Serializer equivalent of ImportManifestsForm, used for API documentation.""" + + input_file = serializers.FileField( + required=True, + help_text=ImportManifestsForm.base_fields["input_file"].label, + ) + update_existing_packages = serializers.BooleanField( + required=False, + default=False, + help_text=ImportManifestsForm.base_fields["update_existing_packages"].help_text, + ) + scan_all_packages = serializers.BooleanField( + required=False, + default=False, + help_text=ImportManifestsForm.base_fields["scan_all_packages"].help_text, + ) + + class ImportFromScanSerializer(serializers.Serializer): """Serializer equivalent of ImportFromScanForm, used for API documentation.""" @@ -319,6 +339,23 @@ def load_sboms(self, request, *args, **kwargs): form.submit(product=product, user=request.user) return Response({"status": "SBOM file submitted to ScanCode.io for inspection."}) + @action(detail=True, methods=["post"], serializer_class=ImportManifestsFormSerializer) + def import_manifests(self, request, *args, **kwargs): + """ + Import Packages from Manifests. + + Multiple Manifests: You can provide multiple files by packaging them into a zip + archive. DejaCode will handle and process them accordingly. + """ + product = self.get_object() + + form = ImportManifestsForm(data=request.POST, files=request.FILES) + if not form.is_valid(): + return Response(form.errors, status=status.HTTP_400_BAD_REQUEST) + + form.submit(product=product, user=request.user) + return Response({"status": "Manifest file submitted to ScanCode.io for inspection."}) + @action(detail=True, methods=["post"], serializer_class=ImportFromScanSerializer) def import_from_scan(self, request, *args, **kwargs): """ diff --git a/product_portfolio/forms.py b/product_portfolio/forms.py index 2b901568..29d3681a 100644 --- a/product_portfolio/forms.py +++ b/product_portfolio/forms.py @@ -587,11 +587,15 @@ def save(self, product): return warnings, created_counts -class LoadSBOMsForm(forms.Form): +class BaseProductImportFormView(forms.Form): + project_type = None + input_label = "" + input_file = SmartFileField( - label=_("SBOM file or zip archive"), + label=_("file or zip archive"), required=True, ) + update_existing_packages = forms.BooleanField( label=_("Update existing packages with discovered packages data"), required=False, @@ -614,6 +618,10 @@ class LoadSBOMsForm(forms.Form): ), ) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields["input_file"].label = _(f"{self.input_label} file or zip archive") + @property def helper(self): helper = FormHelper() @@ -627,7 +635,7 @@ def submit(self, product, user): scancode_project = ScanCodeProject.objects.create( product=product, dataspace=product.dataspace, - type=ScanCodeProject.ProjectType.LOAD_SBOMS, + type=self.project_type, input_file=self.cleaned_data.get("input_file"), update_existing_packages=self.cleaned_data.get("update_existing_packages"), scan_all_packages=self.cleaned_data.get("scan_all_packages"), @@ -635,13 +643,26 @@ def submit(self, product, user): ) transaction.on_commit( - lambda: tasks.scancodeio_submit_load_sbom.delay( + lambda: tasks.scancodeio_submit_project.delay( scancodeproject_uuid=scancode_project.uuid, user_uuid=user.uuid, + pipeline_name=self.pipeline_name, ) ) +class LoadSBOMsForm(BaseProductImportFormView): + project_type = ScanCodeProject.ProjectType.LOAD_SBOMS + input_label = "SBOM" + pipeline_name = "load_sbom" + + +class ImportManifestsForm(BaseProductImportFormView): + project_type = ScanCodeProject.ProjectType.IMPORT_FROM_MANIFEST + input_label = "Manifest" + pipeline_name = "resolve_dependencies" + + class StrongTextWidget(forms.Widget): def render(self, name, value, attrs=None, renderer=None): if value: diff --git a/product_portfolio/importers.py b/product_portfolio/importers.py index bbc8cde1..d831f1bc 100644 --- a/product_portfolio/importers.py +++ b/product_portfolio/importers.py @@ -13,6 +13,7 @@ from django.core.exceptions import MultipleObjectsReturned from django.core.exceptions import ValidationError from django.core.validators import EMPTY_VALUES +from django.db import IntegrityError from django.db import transaction from django.db.models import ObjectDoesNotExist from django.db.models import Q @@ -26,11 +27,13 @@ from component_catalog.models import Component from component_catalog.models import Package from dejacode_toolkit.scancodeio import ScanCodeIO +from dje.copier import copy_object from dje.importers import BaseImporter from dje.importers import BaseImportModelForm from dje.importers import BaseImportModelFormSet from dje.importers import ComponentRelatedFieldImportMixin from dje.importers import ModelChoiceFieldForImport +from dje.models import Dataspace from dje.utils import get_help_text from dje.utils import is_uuid4 from product_portfolio.forms import ProductComponentLicenseExpressionFormMixin @@ -667,12 +670,26 @@ def import_package(self, package_data): if (value := package_data.get(field)) } + # Check if the Package already exists in the local Dataspace try: package = Package.objects.scope(self.user.dataspace).get(**unique_together_lookups) self.existing.append(package) except (ObjectDoesNotExist, MultipleObjectsReturned): package = None + # Check if the Package already exists in the reference Dataspace + reference_dataspace = Dataspace.objects.get_reference() + user_dataspace = self.user.dataspace + if not package and user_dataspace != reference_dataspace: + qs = Package.objects.scope(reference_dataspace).filter(**unique_together_lookups) + if qs.exists(): + reference_object = qs.first() + try: + package = copy_object(reference_object, user_dataspace, self.user, update=False) + self.created.append(package) + except IntegrityError as error: + self.errors.append(error) + if license_expression := package_data.get("declared_license_expression"): license_expression = str(self.licensing.dedup(license_expression)) package_data["license_expression"] = license_expression @@ -683,8 +700,8 @@ def import_package(self, package_data): if not package: try: package = Package.create_from_data(self.user, package_data, validate=True) - except ValidationError as e: - self.errors.append(e) + except ValidationError as errors: + self.errors.append(errors) return self.created.append(package) diff --git a/product_portfolio/models.py b/product_portfolio/models.py index baa72ac2..61f6573f 100644 --- a/product_portfolio/models.py +++ b/product_portfolio/models.py @@ -254,6 +254,9 @@ def get_check_package_version_url(self): def get_load_sboms_url(self): return self.get_url("load_sboms") + def get_import_manifests_url(self): + return self.get_url("import_manifests") + def get_pull_project_data_url(self): return self.get_url("pull_project_data") @@ -1119,7 +1122,6 @@ class ScanCodeProject(HistoryFieldsMixin, DataspacedModel): """Wrap a ScanCode.io Project.""" class ProjectType(models.TextChoices): - # This type was replaced by LOAD_SBOMS but is kept for backward compatibility IMPORT_FROM_MANIFEST = "IMPORT_FROM_MANIFEST", _("Import from Manifest") LOAD_SBOMS = "LOAD_SBOMS", _("Load SBOMs") PULL_FROM_SCANCODEIO = "PULL_FROM_SCANCODEIO", _("Pull from ScanCode.io") diff --git a/product_portfolio/templates/product_portfolio/import_manifests_form.html b/product_portfolio/templates/product_portfolio/import_manifests_form.html new file mode 100644 index 00000000..f806f77a --- /dev/null +++ b/product_portfolio/templates/product_portfolio/import_manifests_form.html @@ -0,0 +1,77 @@ +{% extends "bootstrap_base.html" %} +{% load i18n static crispy_forms_tags %} +{% load inject_preserved_filters from dje_tags %} + +{% block page_title %}{% trans "Import Packages from manifests" %}{% endblock %} + +{% block content %} +
+
+
+
+
+ {% trans "Products" %} + / {{ object.get_absolute_link }} +
+

+ {% trans "Import Packages from manifests" %} +

+
+
+
+
+ + {% include 'includes/messages_alert.html' %} + +
+
+ Supports resolving packages for: +
    +
  • Python: requirements.txt and setup.py manifest files.
    • +
+
+ Multiple Manifests: + You can provide multiple Manifests by packaging them into a zip archive. + DejaCode will handle and process them accordingly. +
+ + + +
+
+ {{ form.errors }} + {% crispy form %} +
+
+{% endblock %} + +{% block javascripts %} + +{% endblock %} \ No newline at end of file diff --git a/product_portfolio/templates/product_portfolio/product_details.html b/product_portfolio/templates/product_portfolio/product_details.html index c68411d0..da4c0ae4 100644 --- a/product_portfolio/templates/product_portfolio/product_details.html +++ b/product_portfolio/templates/product_portfolio/product_details.html @@ -39,6 +39,7 @@ {% trans 'Import data from Scan' %} {% if request.user.dataspace.enable_package_scanning %} {% trans 'Load Packages from SBOMs' %} + {% trans 'Import Packages from manifests' %} {% endif %} {% if pull_project_data_form %} {% trans 'Pull ScanCode.io Project data' %} diff --git a/product_portfolio/templates/product_portfolio/scancodeio_project_status.html b/product_portfolio/templates/product_portfolio/scancodeio_project_status.html index 822d746b..e3e03309 100644 --- a/product_portfolio/templates/product_portfolio/scancodeio_project_status.html +++ b/product_portfolio/templates/product_portfolio/scancodeio_project_status.html @@ -1,6 +1,37 @@ +{% load humanize %} + +{% if scancode_project.results.errors %} +
{{ scancode_project.results.errors|length }} errors:
+ +
+{% endif %} +{% if scancode_project.results.created %} +
{{ scancode_project.results.created|length }} packages created:
+ +
+{% endif %} +{% if scancode_project.results.existing %} +
{{ scancode_project.results.existing|length }} packages updated/existing:
+ +
+{% endif %} + {% for run in scan_data.runs %}