Add support to import packages from manifest #65 (#67)

Signed-off-by: tdruez <tdruez@nexb.com>

CHANGELOG.rst

@@ -31,6 +31,9 @@ Release notes
 - Refactor the "Import manifest" feature as "Load SBOMs".
   https://github.com/nexB/dejacode/issues/61
 
+- Add support to import packages from manifest.
+  https://github.com/nexB/dejacode/issues/65
+
 - Add a vulnerability link to the VulnerableCode app in the Vulnerability tab.
   https://github.com/nexB/dejacode/issues/4
 

dejacode_toolkit/scancodeio.py

@@ -76,10 +76,12 @@ def submit_scan(self, uri, user_uuid, dataspace_uuid):
         logger.debug(f'{self.label}: submit scan uri="{uri}" webhook_url="{webhook_url}"')
         return self.request_post(url=self.project_api_url, json=data)
 
-    def submit_load_sbom(self, project_name, file_location, user_uuid, execute_now=False):
+    def submit_project(
+        self, project_name, pipeline_name, file_location, user_uuid, execute_now=False
+    ):
         data = {
             "name": project_name,
-            "pipeline": "load_sbom",
+            "pipeline": pipeline_name,
             "execute_now": execute_now,
         }
         files = {
@@ -92,7 +94,7 @@ def submit_load_sbom(self, project_name, file_location, user_uuid, execute_now=F
         data["webhook_url"] = webhook_url
 
         logger.debug(
-            f"{self.label}: submit load sbom "
+            f"{self.label}: submit pipeline={pipeline_name} "
             f'project_name="{project_name}" webhook_url="{webhook_url}"'
         )
         return self.request_post(url=self.project_api_url, data=data, files=files)

dje/tasks.py

@@ -115,13 +115,14 @@ def scancodeio_submit_scan(uris, user_uuid, dataspace_uuid):
 
 
 @job
-def scancodeio_submit_load_sbom(scancodeproject_uuid, user_uuid):
+def scancodeio_submit_project(scancodeproject_uuid, user_uuid, pipeline_name):
     """Submit the provided SBOM file to ScanCode.io as an asynchronous task."""
     from dje.models import DejacodeUser
 
     logger.info(
-        f"Entering scancodeio_submit_load_sbom task with "
-        f"scancodeproject_uuid={scancodeproject_uuid} user_uuid={user_uuid}"
+        f"Entering scancodeio_submit_project task with "
+        f"scancodeproject_uuid={scancodeproject_uuid} user_uuid={user_uuid} "
+        f"pipeline_name={pipeline_name}"
     )
 
     ScanCodeProject = apps.get_model("product_portfolio", "scancodeproject")
@@ -137,24 +138,25 @@ def scancodeio_submit_load_sbom(scancodeproject_uuid, user_uuid):
     # Create a Project instance on ScanCode.io without immediate execution of the
     # pipeline. This allows to get instant feedback from ScanCode.io about the Project
     # creation status and its related data, even in SYNC mode.
-    response = scancodeio.submit_load_sbom(
+    response = scancodeio.submit_project(
         project_name=scancodeproject_uuid,
+        pipeline_name=pipeline_name,
         file_location=scancode_project.input_file.path,
         user_uuid=user_uuid,
         execute_now=False,
     )
 
     if not response:
-        logger.info("Error submitting the SBOM file to ScanCode.io server")
+        logger.info("Error submitting the file to ScanCode.io server")
         scancode_project.status = ScanCodeProject.Status.FAILURE
-        msg = "- Error: SBOM could not be submitted to ScanCode.io"
+        msg = "- Error: File could not be submitted to ScanCode.io"
         scancode_project.append_to_log(msg, save=True)
         return
 
     logger.info("Update the ScanCodeProject instance")
     scancode_project.status = ScanCodeProject.Status.SUBMITTED
     scancode_project.project_uuid = response.get("uuid")
-    msg = "- SBOM file submitted to ScanCode.io for inspection"
+    msg = "- File submitted to ScanCode.io for inspection"
     scancode_project.append_to_log(msg, save=True)
 
     # Delay the execution of the pipeline after the ScancodeProject instance was
@@ -164,7 +166,7 @@ def scancodeio_submit_load_sbom(scancodeproject_uuid, user_uuid):
         transaction.on_commit(lambda: scancodeio.start_pipeline(run_url=runs[0]["url"]))
 
 
-@job
+@job("default", timeout=1200)
 def pull_project_data_from_scancodeio(scancodeproject_uuid):
     """
     Pull Project data from ScanCode.io as an asynchronous task for the provided

product_portfolio/api.py

@@ -35,6 +35,7 @@
 from dje.permissions import assign_all_object_permissions
 from product_portfolio.filters import ComponentCompletenessAPIFilter
 from product_portfolio.forms import ImportFromScanForm
+from product_portfolio.forms import ImportManifestsForm
 from product_portfolio.forms import LoadSBOMsForm
 from product_portfolio.forms import PullProjectDataForm
 from product_portfolio.models import CodebaseResource
@@ -216,6 +217,25 @@ class LoadSBOMsFormSerializer(serializers.Serializer):
     )
 
 
+class ImportManifestsFormSerializer(serializers.Serializer):
+    """Serializer equivalent of ImportManifestsForm, used for API documentation."""
+
+    input_file = serializers.FileField(
+        required=True,
+        help_text=ImportManifestsForm.base_fields["input_file"].label,
+    )
+    update_existing_packages = serializers.BooleanField(
+        required=False,
+        default=False,
+        help_text=ImportManifestsForm.base_fields["update_existing_packages"].help_text,
+    )
+    scan_all_packages = serializers.BooleanField(
+        required=False,
+        default=False,
+        help_text=ImportManifestsForm.base_fields["scan_all_packages"].help_text,
+    )
+
+
 class ImportFromScanSerializer(serializers.Serializer):
     """Serializer equivalent of ImportFromScanForm, used for API documentation."""
 
@@ -319,6 +339,23 @@ def load_sboms(self, request, *args, **kwargs):
         form.submit(product=product, user=request.user)
         return Response({"status": "SBOM file submitted to ScanCode.io for inspection."})
 
+    @action(detail=True, methods=["post"], serializer_class=ImportManifestsFormSerializer)
+    def import_manifests(self, request, *args, **kwargs):
+        """
+        Import Packages from Manifests.
+
+        Multiple Manifests: You can provide multiple files by packaging them into a zip
+        archive. DejaCode will handle and process them accordingly.
+        """
+        product = self.get_object()
+
+        form = ImportManifestsForm(data=request.POST, files=request.FILES)
+        if not form.is_valid():
+            return Response(form.errors, status=status.HTTP_400_BAD_REQUEST)
+
+        form.submit(product=product, user=request.user)
+        return Response({"status": "Manifest file submitted to ScanCode.io for inspection."})
+
     @action(detail=True, methods=["post"], serializer_class=ImportFromScanSerializer)
     def import_from_scan(self, request, *args, **kwargs):
         """

product_portfolio/forms.py

@@ -587,11 +587,15 @@ def save(self, product):
         return warnings, created_counts
 
 
-class LoadSBOMsForm(forms.Form):
+class BaseProductImportFormView(forms.Form):
+    project_type = None
+    input_label = ""
+
     input_file = SmartFileField(
-        label=_("SBOM file or zip archive"),
+        label=_("file or zip archive"),
         required=True,
     )
+
     update_existing_packages = forms.BooleanField(
         label=_("Update existing packages with discovered packages data"),
         required=False,
@@ -614,6 +618,10 @@ class LoadSBOMsForm(forms.Form):
         ),
     )
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.fields["input_file"].label = _(f"{self.input_label} file or zip archive")
+
     @property
     def helper(self):
         helper = FormHelper()
@@ -627,21 +635,34 @@ def submit(self, product, user):
         scancode_project = ScanCodeProject.objects.create(
             product=product,
             dataspace=product.dataspace,
-            type=ScanCodeProject.ProjectType.LOAD_SBOMS,
+            type=self.project_type,
             input_file=self.cleaned_data.get("input_file"),
             update_existing_packages=self.cleaned_data.get("update_existing_packages"),
             scan_all_packages=self.cleaned_data.get("scan_all_packages"),
             created_by=user,
         )
 
         transaction.on_commit(
-            lambda: tasks.scancodeio_submit_load_sbom.delay(
+            lambda: tasks.scancodeio_submit_project.delay(
                 scancodeproject_uuid=scancode_project.uuid,
                 user_uuid=user.uuid,
+                pipeline_name=self.pipeline_name,
             )
         )
 
 
+class LoadSBOMsForm(BaseProductImportFormView):
+    project_type = ScanCodeProject.ProjectType.LOAD_SBOMS
+    input_label = "SBOM"
+    pipeline_name = "load_sbom"
+
+
+class ImportManifestsForm(BaseProductImportFormView):
+    project_type = ScanCodeProject.ProjectType.IMPORT_FROM_MANIFEST
+    input_label = "Manifest"
+    pipeline_name = "resolve_dependencies"
+
+
 class StrongTextWidget(forms.Widget):
     def render(self, name, value, attrs=None, renderer=None):
         if value:

product_portfolio/importers.py

@@ -13,6 +13,7 @@
 from django.core.exceptions import MultipleObjectsReturned
 from django.core.exceptions import ValidationError
 from django.core.validators import EMPTY_VALUES
+from django.db import IntegrityError
 from django.db import transaction
 from django.db.models import ObjectDoesNotExist
 from django.db.models import Q
@@ -26,11 +27,13 @@
 from component_catalog.models import Component
 from component_catalog.models import Package
 from dejacode_toolkit.scancodeio import ScanCodeIO
+from dje.copier import copy_object
 from dje.importers import BaseImporter
 from dje.importers import BaseImportModelForm
 from dje.importers import BaseImportModelFormSet
 from dje.importers import ComponentRelatedFieldImportMixin
 from dje.importers import ModelChoiceFieldForImport
+from dje.models import Dataspace
 from dje.utils import get_help_text
 from dje.utils import is_uuid4
 from product_portfolio.forms import ProductComponentLicenseExpressionFormMixin
@@ -667,12 +670,26 @@ def import_package(self, package_data):
             if (value := package_data.get(field))
         }
 
+        # Check if the Package already exists in the local Dataspace
         try:
             package = Package.objects.scope(self.user.dataspace).get(**unique_together_lookups)
             self.existing.append(package)
         except (ObjectDoesNotExist, MultipleObjectsReturned):
             package = None
 
+        # Check if the Package already exists in the reference Dataspace
+        reference_dataspace = Dataspace.objects.get_reference()
+        user_dataspace = self.user.dataspace
+        if not package and user_dataspace != reference_dataspace:
+            qs = Package.objects.scope(reference_dataspace).filter(**unique_together_lookups)
+            if qs.exists():
+                reference_object = qs.first()
+                try:
+                    package = copy_object(reference_object, user_dataspace, self.user, update=False)
+                    self.created.append(package)
+                except IntegrityError as error:
+                    self.errors.append(error)
+
         if license_expression := package_data.get("declared_license_expression"):
             license_expression = str(self.licensing.dedup(license_expression))
             package_data["license_expression"] = license_expression
@@ -683,8 +700,8 @@ def import_package(self, package_data):
         if not package:
             try:
                 package = Package.create_from_data(self.user, package_data, validate=True)
-            except ValidationError as e:
-                self.errors.append(e)
+            except ValidationError as errors:
+                self.errors.append(errors)
                 return
             self.created.append(package)
 

product_portfolio/models.py

@@ -254,6 +254,9 @@ def get_check_package_version_url(self):
     def get_load_sboms_url(self):
         return self.get_url("load_sboms")
 
+    def get_import_manifests_url(self):
+        return self.get_url("import_manifests")
+
     def get_pull_project_data_url(self):
         return self.get_url("pull_project_data")
 
@@ -1119,7 +1122,6 @@ class ScanCodeProject(HistoryFieldsMixin, DataspacedModel):
     """Wrap a ScanCode.io Project."""
 
     class ProjectType(models.TextChoices):
-        # This type was replaced by LOAD_SBOMS but is kept for backward compatibility
         IMPORT_FROM_MANIFEST = "IMPORT_FROM_MANIFEST", _("Import from Manifest")
         LOAD_SBOMS = "LOAD_SBOMS", _("Load SBOMs")
         PULL_FROM_SCANCODEIO = "PULL_FROM_SCANCODEIO", _("Pull from ScanCode.io")

product_portfolio/templates/product_portfolio/import_manifests_form.html

@@ -0,0 +1,77 @@
+{% extends "bootstrap_base.html" %}
+{% load i18n static crispy_forms_tags %}
+{% load inject_preserved_filters from dje_tags %}
+
+{% block page_title %}{% trans "Import Packages from manifests" %}{% endblock %}
+
+{% block content %}
+  <div class="header">
+    <div class="header-body">
+      <div class="row align-items-center">
+        <div class="col">
+          <div class="header-pretitle">
+            <a href="{% inject_preserved_filters 'product_portfolio:product_list' %}">{% trans "Products" %}</a>
+            / {{ object.get_absolute_link }}
+          </div>
+          <h1 class="header-title">
+            {% trans "Import Packages from manifests" %}
+          </h1>
+        </div>
+      </div>
+    </div>
+  </div>
+
+  {% include 'includes/messages_alert.html' %}
+
+  <div class="alert alert-success">
+    <div>
+      Supports resolving packages for:
+      <ul class="mt-2">
+        <li><strong>Python</strong>: requirements.txt and setup.py manifest files.</li>
+      </ul>
+    </div>
+    <strong>Multiple Manifests:</strong>
+    You can provide multiple Manifests by packaging them into a <strong>zip archive</strong>.
+    DejaCode will handle and process them accordingly.
+  </div>
+
+  <div class="alert alert-primary" role="alert">
+    When you upload your <strong>Manifest file to DejaCode</strong>,
+    the following process will occur:
+    <ul class="mb-0 mt-2">
+      <li>
+        <strong>Submission to ScanCode.io</strong>
+        Your Manifest file will be submitted to ScanCode.io for thorough scan inspection.
+      </li>
+      <li>
+        <strong>Package Discovery</strong>
+        ScanCode.io will identify and discover packages within your Manifest.
+      </li>
+      <li>
+        <strong>Package Importation</strong>
+        DejaCode will retrieve the discovered packages from ScanCode.io and import them into its system.
+      </li>
+      <li>
+        <strong>Package Assignment</strong>
+        The imported packages will be assigned to the corresponding product within DejaCode.
+      </li>
+    </ul>
+  </div>
+
+  <div class="row">
+    <div class="col-8">
+      {{ form.errors }}
+      {% crispy form %}
+    </div>
+  </div>
+{% endblock %}
+
+{% block javascripts %}
+  <script>
+    $(document).ready(function () {
+      $('form#import-manifest-form').on('submit', function () {
+        NEXB.displayOverlay("Load Packages from Manifest...");
+      })
+    });
+  </script>
+{% endblock %}

product_portfolio/templates/product_portfolio/product_details.html

@@ -39,6 +39,7 @@
             <a class="dropdown-item" href="{{ object.get_import_from_scan_url }}"><i class="fas fa-file-upload"></i> {% trans 'Import data from Scan' %}</a>
             {% if request.user.dataspace.enable_package_scanning %}
               <a class="dropdown-item" href="{{ object.get_load_sboms_url }}"><i class="fas fa-file-upload"></i> {% trans 'Load Packages from SBOMs' %}</a>
+              <a class="dropdown-item" href="{{ object.get_import_manifests_url }}"><i class="fas fa-file-upload"></i> {% trans 'Import Packages from manifests' %}</a>
             {% endif %}
             {% if pull_project_data_form %}
               <a class="dropdown-item" style="margin-left: -3px;" href="#" data-bs-toggle="modal" data-bs-target="#pull-project-data-modal"><i class="fas fa-cloud-download-alt"></i> {% trans 'Pull ScanCode.io Project data' %}</a>