From 578d706dfa4b7feca646024811f8df5c1397281c Mon Sep 17 00:00:00 2001 From: tdruez Date: Mon, 2 Sep 2024 20:21:48 +0400 Subject: [PATCH] Provide feedback on the "Improve" action through the import system #45 Signed-off-by: tdruez --- component_catalog/models.py | 2 +- dje/tasks.py | 28 ++++++++++++++++--- .../0007_alter_scancodeproject_type.py | 18 ++++++++++++ product_portfolio/models.py | 12 ++++++++ product_portfolio/tests/test_views.py | 20 +++++++++++-- product_portfolio/views.py | 21 +++++++++----- 6 files changed, 87 insertions(+), 14 deletions(-) create mode 100644 product_portfolio/migrations/0007_alter_scancodeproject_type.py diff --git a/component_catalog/models.py b/component_catalog/models.py index e23e092d..eb099fc9 100644 --- a/component_catalog/models.py +++ b/component_catalog/models.py @@ -2529,7 +2529,7 @@ def update_from_purldb(self, user): when available. """ purldb_entries = self.get_purldb_entries(user) - if not purldb_entries or len(purldb_entries) != 1: + if not purldb_entries: return package_data = purldb_entries[0] diff --git a/dje/tasks.py b/dje/tasks.py index b5c4574b..a13f1cba 100644 --- a/dje/tasks.py +++ b/dje/tasks.py @@ -241,6 +241,7 @@ def improve_packages_from_purldb(product_uuid, user_uuid): DejacodeUser = apps.get_model("dje", "DejacodeUser") History = apps.get_model("dje", "History") Product = apps.get_model("product_portfolio", "product") + ScanCodeProject = apps.get_model("product_portfolio", "scancodeproject") try: user = DejacodeUser.objects.get(uuid=user_uuid) @@ -262,16 +263,35 @@ def improve_packages_from_purldb(product_uuid, user_uuid): logger.error("[improve_packages_from_purldb]: Permission denied.") return - updated_packages = product.improve_packages_from_purldb(user) - logger.info(f"[improve_packages_from_purldb]: {len(updated_packages)} updated from PurlDB.") + scancode_project = ScanCodeProject.objects.create( + product=product, + dataspace=product.dataspace, + type=ScanCodeProject.ProjectType.IMPROVE_FROM_PURLDB, + status=ScanCodeProject.Status.IMPORT_STARTED, + created_by=user, + ) + + try: + updated_packages = product.improve_packages_from_purldb(user) + except Exception as e: + scancode_project.update( + status=ScanCodeProject.Status.FAILURE, + import_log=str(e), + ) - verb = "Improved packages from PurlDB" + logger.info(f"[improve_packages_from_purldb]: {len(updated_packages)} updated from PurlDB.") + verb = "Improved packages from PurlDB:" if updated_packages: description = ", ".join([str(package) for package in updated_packages]) - History.log_change(user, product, message=f"{verb}: {description}") + History.log_change(user, product, message=f"{verb} {description}") else: description = "No packages updated from PurlDB data." + scancode_project.update( + status=ScanCodeProject.Status.SUCCESS, + import_log=[verb, description], + ) + notify.send( sender=user, verb=verb, diff --git a/product_portfolio/migrations/0007_alter_scancodeproject_type.py b/product_portfolio/migrations/0007_alter_scancodeproject_type.py new file mode 100644 index 00000000..52dd9681 --- /dev/null +++ b/product_portfolio/migrations/0007_alter_scancodeproject_type.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.6 on 2024-09-02 15:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('product_portfolio', '0006_productdependency'), + ] + + operations = [ + migrations.AlterField( + model_name='scancodeproject', + name='type', + field=models.CharField(choices=[('IMPORT_FROM_MANIFEST', 'Import from Manifest'), ('LOAD_SBOMS', 'Load SBOMs'), ('PULL_FROM_SCANCODEIO', 'Pull from ScanCode.io'), ('IMPROVE_FROM_PURLDB', 'Improve from PurlDB')], db_index=True, help_text='The type of import, for the ProjectType choices.', max_length=50), + ), + ] diff --git a/product_portfolio/models.py b/product_portfolio/models.py index 544d7fd8..2715bf47 100644 --- a/product_portfolio/models.py +++ b/product_portfolio/models.py @@ -1216,6 +1216,15 @@ def generate_input_file_path(instance, filename): return f"{dataspace}/scancode_project/{instance.uuid}/{filename}" +class ScanCodeProjectQuerySet(ProductSecuredQuerySet): + def in_progress(self): + in_progress_statuses = [ + ScanCodeProject.Status.SUBMITTED, + ScanCodeProject.Status.IMPORT_STARTED, + ] + return self.filter(status__in=in_progress_statuses) + + class ScanCodeProject(HistoryFieldsMixin, DataspacedModel): """Wrap a ScanCode.io Project.""" @@ -1223,6 +1232,7 @@ class ProjectType(models.TextChoices): IMPORT_FROM_MANIFEST = "IMPORT_FROM_MANIFEST", _("Import from Manifest") LOAD_SBOMS = "LOAD_SBOMS", _("Load SBOMs") PULL_FROM_SCANCODEIO = "PULL_FROM_SCANCODEIO", _("Pull from ScanCode.io") + IMPROVE_FROM_PURLDB = "IMPROVE_FROM_PURLDB", _("Improve from PurlDB") class Status(models.TextChoices): SUBMITTED = "submitted" @@ -1278,6 +1288,8 @@ class Status(models.TextChoices): default=dict, ) + objects = DataspacedManager.from_queryset(ScanCodeProjectQuerySet)() + class Meta: unique_together = ("dataspace", "uuid") ordering = ["-created_date"] diff --git a/product_portfolio/tests/test_views.py b/product_portfolio/tests/test_views.py index 1811e267..23daf1c7 100644 --- a/product_portfolio/tests/test_views.py +++ b/product_portfolio/tests/test_views.py @@ -135,7 +135,7 @@ def test_product_portfolio_detail_view_tab_inventory_and_hierarchy_availability( ProductComponent.objects.create( product=self.product1, component=self.component1, dataspace=self.dataspace ) - with self.assertNumQueries(29): + with self.assertNumQueries(30): response = self.client.get(url) self.assertContains(response, expected1) self.assertContains(response, expected2) @@ -161,7 +161,7 @@ def test_product_portfolio_detail_view_tab_inventory_availability(self): ProductPackage.objects.create( product=self.product1, package=self.package1, dataspace=self.dataspace ) - with self.assertNumQueries(26): + with self.assertNumQueries(27): response = self.client.get(url) self.assertContains(response, expected) @@ -3142,6 +3142,16 @@ def test_product_portfolio_improve_packages_from_purldb_view(self, mock_is_confi self.assertEqual(200, response.status_code) self.assertContains(response, "Improve Packages from PurlDB in progress...") + ScanCodeProject.objects.create( + product=self.product1, + dataspace=self.product1.dataspace, + type=ScanCodeProject.ProjectType.IMPROVE_FROM_PURLDB, + status=ScanCodeProject.Status.IMPORT_STARTED, + ) + response = self.client.get(url, follow=True) + self.assertEqual(200, response.status_code) + self.assertContains(response, "Improve Packages already in progress...") + @mock.patch("product_portfolio.models.Product.improve_packages_from_purldb") def test_product_portfolio_improve_packages_from_purldb_task(self, mock_improve): mock_improve.return_value = ["pkg1", "pkg2"] @@ -3176,3 +3186,9 @@ def test_product_portfolio_improve_packages_from_purldb_task(self, mock_improve) "INFO:dje.tasks:[improve_packages_from_purldb]: 2 updated from PurlDB.", ] self.assertEqual(expected, cm.output) + + import_project = self.product1.scancodeprojects.get() + self.assertEqual(import_project.type, ScanCodeProject.ProjectType.IMPROVE_FROM_PURLDB) + self.assertEqual(import_project.status, ScanCodeProject.Status.SUCCESS) + expected = ["Improved packages from PurlDB:", "pkg1, pkg2"] + self.assertEqual(expected, import_project.import_log) diff --git a/product_portfolio/views.py b/product_portfolio/views.py index af67a0b9..679a277f 100644 --- a/product_portfolio/views.py +++ b/product_portfolio/views.py @@ -2401,11 +2401,18 @@ def improve_packages_from_purldb_view(request, dataspace, name, version=""): if not product.packages.count(): raise Http404("No packages available for this product.") - transaction.on_commit( - lambda: tasks.improve_packages_from_purldb( - product_uuid=product.uuid, - user_uuid=user.uuid, - ) + improve_in_progress = product.scancodeprojects.in_progress().filter( + type=ScanCodeProject.ProjectType.IMPROVE_FROM_PURLDB, ) - messages.success(request, "Improve Packages from PurlDB in progress...") - return redirect(f"{product.get_absolute_url()}#history") + + if improve_in_progress.exists(): + messages.error(request, "Improve Packages already in progress...") + else: + transaction.on_commit( + lambda: tasks.improve_packages_from_purldb( + product_uuid=product.uuid, + user_uuid=user.uuid, + ) + ) + messages.success(request, "Improve Packages from PurlDB in progress...") + return redirect(f"{product.get_absolute_url()}#imports")