From 5b7b5cf4bcfd4e5e6877b767d4355747f4ec4e6a Mon Sep 17 00:00:00 2001 From: tdruez Date: Fri, 5 Jul 2024 17:27:33 +0400 Subject: [PATCH] Add prototype for Dependency model and view #138 Signed-off-by: tdruez --- dejacode_toolkit/scancodeio.py | 22 ++ product_portfolio/filters.py | 33 +++ product_portfolio/forms.py | 4 +- product_portfolio/importers.py | 37 +++ .../0006_productdependency_and_more.py | 219 ++++++++++++++++++ product_portfolio/models.py | 96 ++++++++ .../tabs/tab_dependencies.html | 93 ++++++++ product_portfolio/urls.py | 2 + product_portfolio/views.py | 100 ++++++++ 9 files changed, 604 insertions(+), 2 deletions(-) create mode 100644 product_portfolio/migrations/0006_productdependency_and_more.py create mode 100644 product_portfolio/templates/product_portfolio/tabs/tab_dependencies.html diff --git a/dejacode_toolkit/scancodeio.py b/dejacode_toolkit/scancodeio.py index 7356f1de..f58a14b5 100644 --- a/dejacode_toolkit/scancodeio.py +++ b/dejacode_toolkit/scancodeio.py @@ -46,6 +46,10 @@ def get_scan_summary_url(self, project_uuid): def get_project_packages_url(self, project_uuid): return f"{self.project_api_url}{project_uuid}/packages/" + # TODO: Remove duplication with get_project_packages_url + def get_project_dependencies_url(self, project_uuid): + return f"{self.project_api_url}{project_uuid}/dependencies/" + def get_scan_results(self, download_url, dataspace): scan_info = self.fetch_scan_info(uri=download_url, dataspace=dataspace) @@ -248,6 +252,24 @@ def fetch_project_packages(self, project_uuid): return packages + # TODO: Remove duplication with fetch_project_packages + def fetch_project_dependencies(self, project_uuid): + """Return the list of dependencies for the provided `project_uuid`.""" + api_url = self.get_project_dependencies_url(project_uuid) + dependencies = [] + + next_url = api_url + while next_url: + logger.debug(f"{self.label}: fetch dependencies from project_packages_url={next_url}") + response = self.request_get(url=next_url) + if not response: + raise Exception("Error fetching project dependencies") + + dependencies.extend(response["results"]) + next_url = response["next"] + + return dependencies + # (label, scan_field, model_field, input_type) SCAN_SUMMARY_FIELDS = [ ( diff --git a/product_portfolio/filters.py b/product_portfolio/filters.py index a1273b9b..f6ec7db5 100644 --- a/product_portfolio/filters.py +++ b/product_portfolio/filters.py @@ -26,6 +26,7 @@ from product_portfolio.models import CodebaseResource from product_portfolio.models import Product from product_portfolio.models import ProductComponent +from product_portfolio.models import ProductDependency from product_portfolio.models import ProductPackage from product_portfolio.models import ProductStatus @@ -314,3 +315,35 @@ class Meta: "related_deployed_from", "related_deployed_to", ] + + +class DependencyFilterSet(DataspacedFilterSet): + q = SearchFilter( + label=_("Search"), + search_fields=[ + "for_package__filename", + "for_package__type", + "for_package__namespace", + "for_package__name", + "for_package__version", + "resolved_to_package__filename", + "resolved_to_package__type", + "resolved_to_package__namespace", + "resolved_to_package__name", + "resolved_to_package__version", + ], + ) + is_deployment_path = BooleanChoiceFilter( + widget=DropDownWidget(anchor="#codebase"), + ) + + class Meta: + model = ProductDependency + fields = [ + "scope", + "datasource_id", + "is_runtime", + "is_optional", + "is_resolved", + "is_direct", + ] diff --git a/product_portfolio/forms.py b/product_portfolio/forms.py index edd173c4..8ecabbab 100644 --- a/product_portfolio/forms.py +++ b/product_portfolio/forms.py @@ -599,7 +599,7 @@ class BaseProductImportFormView(forms.Form): initial=False, help_text=_( "If checked, the discovered packages from the manifest that are already " - "existing in your Dataspace will be updated with ScanCode data" + "existing in your Dataspace will be updated with ScanCode data. " "Note that only the empty fields will be updated. " "By default (un-checked), existing packages will be assign to the product " "without any modification." @@ -894,7 +894,7 @@ class PullProjectDataForm(forms.Form): initial=False, help_text=_( "If checked, the discovered packages from the Project that are already " - "existing in your Dataspace will be updated with ScanCode.io data." + "existing in your Dataspace will be updated with ScanCode.io data. " "Note that only the empty fields will be updated. " "By default (un-checked), existing packages will be assign to the product " "without any modification." diff --git a/product_portfolio/importers.py b/product_portfolio/importers.py index df1e45ec..7039e7ef 100644 --- a/product_portfolio/importers.py +++ b/product_portfolio/importers.py @@ -41,6 +41,7 @@ from product_portfolio.models import CodebaseResourceUsage from product_portfolio.models import Product from product_portfolio.models import ProductComponent +from product_portfolio.models import ProductDependency from product_portfolio.models import ProductItemPurpose from product_portfolio.models import ProductPackage from product_portfolio.models import ProductRelationStatus @@ -639,6 +640,8 @@ def __init__(self, user, project_uuid, product, update_existing=False, scan_all_ self.created = [] self.existing = [] self.errors = [] + # Use to assign dependencies to the correct Package instance + self.package_uid_mapping = {} self.user = user self.project_uuid = project_uuid @@ -650,9 +653,11 @@ def __init__(self, user, project_uuid, product, update_existing=False, scan_all_ self.packages = scancodeio.fetch_project_packages(self.project_uuid) if not self.packages: raise Exception("Packages could not be fetched from ScanCode.io") + self.dependencies = scancodeio.fetch_project_dependencies(self.project_uuid) def save(self): self.import_packages() + self.import_dependencies() if self.scan_all_packages: transaction.on_commit(lambda: self.product.scan_all_packages_task(self.user)) @@ -663,7 +668,12 @@ def import_packages(self): for package_data in self.packages: self.import_package(package_data) + def import_dependencies(self): + for dependency_data in self.dependencies: + self.import_dependency(dependency_data) + def import_package(self, package_data): + package_uid = package_data.get("package_uid") unique_together_lookups = { field: value for field in self.unique_together_fields @@ -715,3 +725,30 @@ def import_package(self, package_data): "created_by": self.user, }, ) + + self.package_uid_mapping[package_uid] = package + + def import_dependency(self, dependency_data): + dependency = None + # TODO: Check if the Dependency already exists in the local Dataspace + # try: + # dependency = ProductDependency.objects.scope(self.user.dataspace) + # .get(**unique_together_lookups) + # self.existing.append(package) + # except (ObjectDoesNotExist, MultipleObjectsReturned): + # dependency = None + + dependency_data["product"] = self.product + for_package_uid = dependency_data["for_package_uid"] + dependency_data["for_package"] = self.package_uid_mapping.get(for_package_uid) + resolved_to_package_uid = dependency_data["resolved_to_package_uid"] + dependency_data["resolved_to_package"] = self.package_uid_mapping.get( + resolved_to_package_uid + ) + + if not dependency: + try: + ProductDependency.create_from_data(self.user, dependency_data, validate=True) + except ValidationError as errors: + print(errors) + return diff --git a/product_portfolio/migrations/0006_productdependency_and_more.py b/product_portfolio/migrations/0006_productdependency_and_more.py new file mode 100644 index 00000000..eb20629a --- /dev/null +++ b/product_portfolio/migrations/0006_productdependency_and_more.py @@ -0,0 +1,219 @@ +# Generated by Django 5.0.6 on 2024-07-05 12:49 + +import django.db.models.deletion +import dje.fields +import uuid +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("component_catalog", "0005_remove_component_concluded_license_and_more"), + ("dje", "0003_alter_dejacodeuser_homepage_layout"), + ("product_portfolio", "0005_alter_product_license_expression_and_more"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="ProductDependency", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "uuid", + models.UUIDField( + default=uuid.uuid4, editable=False, verbose_name="UUID" + ), + ), + ( + "created_date", + models.DateTimeField( + auto_now_add=True, + db_index=True, + help_text="The date and time the object was created.", + ), + ), + ( + "last_modified_date", + models.DateTimeField( + auto_now=True, + db_index=True, + help_text="The date and time the object was last modified.", + ), + ), + ( + "dependency_uid", + models.CharField( + help_text="The unique identifier of this dependency.", + max_length=1024, + ), + ), + ( + "extracted_requirement", + models.CharField( + blank=True, + help_text="The version requirements of this dependency.", + max_length=256, + ), + ), + ( + "scope", + models.CharField( + blank=True, + help_text="The scope of this dependency, how it is used in a project.", + max_length=64, + ), + ), + ( + "datasource_id", + models.CharField( + blank=True, + help_text="The identifier for the datafile handler used to obtain this dependency.", + max_length=64, + ), + ), + ( + "is_runtime", + models.BooleanField( + default=False, + help_text="True if this dependency is a runtime dependency.", + ), + ), + ( + "is_optional", + models.BooleanField( + default=False, + help_text="True if this dependency is an optional dependency", + ), + ), + ( + "is_resolved", + models.BooleanField( + default=False, + help_text="True if this dependency version requirement has been pinned and this dependency points to an exact version.", + ), + ), + ( + "is_direct", + models.BooleanField( + default=False, + help_text="True if this is a direct, first-level dependency relationship for a package.", + ), + ), + ( + "created_by", + models.ForeignKey( + editable=False, + help_text="The application user who created the object.", + null=True, + on_delete=django.db.models.deletion.PROTECT, + related_name="created_%(class)ss", + serialize=False, + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "dataspace", + models.ForeignKey( + editable=False, + help_text="A Dataspace is an independent, exclusive set of DejaCode data, which can be either nexB master reference data or installation-specific data.", + on_delete=django.db.models.deletion.PROTECT, + to="dje.dataspace", + ), + ), + ( + "for_package", + models.ForeignKey( + blank=True, + editable=False, + help_text="The package that declares this dependency.", + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="declared_dependencies", + to="component_catalog.package", + ), + ), + ( + "last_modified_by", + dje.fields.LastModifiedByField( + editable=False, + help_text="The application user who last modified the object.", + null=True, + on_delete=django.db.models.deletion.PROTECT, + related_name="modified_%(class)ss", + serialize=False, + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "product", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="dependencies", + to="product_portfolio.product", + ), + ), + ( + "resolved_to_package", + models.ForeignKey( + blank=True, + editable=False, + help_text="The resolved package for this dependency. If empty, it indicates the dependency is unresolved.", + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="resolved_from_dependencies", + to="component_catalog.package", + ), + ), + ], + options={ + "verbose_name": "product dependency", + "verbose_name_plural": "product dependencies", + "ordering": ["dependency_uid"], + "indexes": [ + models.Index(fields=["scope"], name="product_por_scope_eb3e33_idx"), + models.Index( + fields=["is_runtime"], name="product_por_is_runt_3476a9_idx" + ), + models.Index( + fields=["is_optional"], name="product_por_is_opti_9b7ae0_idx" + ), + models.Index( + fields=["is_resolved"], name="product_por_is_reso_368f34_idx" + ), + models.Index( + fields=["is_direct"], name="product_por_is_dire_3abb9d_idx" + ), + ], + }, + ), + migrations.AddConstraint( + model_name="productdependency", + constraint=models.UniqueConstraint( + condition=models.Q(("dependency_uid", ""), _negated=True), + fields=("product", "dependency_uid"), + name="product_portfolio_productdependency_unique_dependency_uid_within_product", + ), + ), + migrations.AddConstraint( + model_name="productdependency", + constraint=models.UniqueConstraint( + fields=("dataspace", "uuid"), + name="product_portfolio_productdependency_unique_uuid_within_dataspace", + ), + ), + migrations.AlterUniqueTogether( + name="productdependency", + unique_together={("dataspace", "uuid"), ("product", "dependency_uid")}, + ), + ] diff --git a/product_portfolio/models.py b/product_portfolio/models.py index 81301d82..94fc36ab 100644 --- a/product_portfolio/models.py +++ b/product_portfolio/models.py @@ -1316,3 +1316,99 @@ def notify(self, verb, description): recipient=self.created_by, description=description, ) + + +class ProductDependency(HistoryFieldsMixin, DataspacedModel): + product = models.ForeignKey( + to="product_portfolio.Product", + related_name="dependencies", + on_delete=models.CASCADE, + ) + dependency_uid = models.CharField( + max_length=1024, + help_text=_("The unique identifier of this dependency."), + ) + for_package = models.ForeignKey( + to="component_catalog.Package", + related_name="declared_dependencies", + help_text=_("The package that declares this dependency."), + on_delete=models.CASCADE, + editable=False, + blank=True, + null=True, + ) + resolved_to_package = models.ForeignKey( + to="component_catalog.Package", + related_name="resolved_from_dependencies", + help_text=_( + "The resolved package for this dependency. " + "If empty, it indicates the dependency is unresolved." + ), + on_delete=models.SET_NULL, + editable=False, + blank=True, + null=True, + ) + extracted_requirement = models.CharField( + max_length=256, + blank=True, + help_text=_("The version requirements of this dependency."), + ) + scope = models.CharField( + max_length=64, + blank=True, + help_text=_("The scope of this dependency, how it is used in a project."), + ) + datasource_id = models.CharField( + max_length=64, + blank=True, + help_text=_("The identifier for the datafile handler used to obtain this dependency."), + ) + is_runtime = models.BooleanField( + default=False, + help_text=_("True if this dependency is a runtime dependency."), + ) + is_optional = models.BooleanField( + default=False, + help_text=_("True if this dependency is an optional dependency"), + ) + is_resolved = models.BooleanField( + default=False, + help_text=_( + "True if this dependency version requirement has been pinned " + "and this dependency points to an exact version." + ), + ) + is_direct = models.BooleanField( + default=False, + help_text=_( + "True if this is a direct, first-level dependency relationship " "for a package." + ), + ) + + class Meta: + unique_together = (("product", "dependency_uid"), ("dataspace", "uuid")) + verbose_name = "product dependency" + verbose_name_plural = "product dependencies" + ordering = ["dependency_uid"] + indexes = [ + models.Index(fields=["scope"]), + models.Index(fields=["is_runtime"]), + models.Index(fields=["is_optional"]), + models.Index(fields=["is_resolved"]), + models.Index(fields=["is_direct"]), + ] + constraints = [ + models.UniqueConstraint( + fields=["product", "dependency_uid"], + condition=~models.Q(dependency_uid=""), + name="%(app_label)s_%(class)s_unique_dependency_uid_within_product", + ), + models.UniqueConstraint( + fields=["dataspace", "uuid"], + name="%(app_label)s_%(class)s_unique_uuid_within_dataspace", + ), + ] + + def __str__(self): + return self.dependency_uid diff --git a/product_portfolio/templates/product_portfolio/tabs/tab_dependencies.html b/product_portfolio/templates/product_portfolio/tabs/tab_dependencies.html new file mode 100644 index 00000000..610629e2 --- /dev/null +++ b/product_portfolio/templates/product_portfolio/tabs/tab_dependencies.html @@ -0,0 +1,93 @@ +{% load i18n %} +{% load as_icon from dje_tags %} +{% load humanize %} + +{% spaceless %} +
+
+
    +
  • +
    + +
    +
    • +
  • +
    {{ page_obj.paginator.count|intcomma }} results
    +
    • +
+
+
+ {% include 'pagination/object_list_pagination.html' %} +
+
+ + + + + + + + + + + + + + + {% for dependency in filter_dependency.qs %} + + + + + + + + + + {% empty %} + + + + {% endfor %} + +
+ {% trans 'For package' %} + + {% trans 'Resolved to package' %} + + {% trans 'Scope' %} + + {% trans 'Is runtime' %} + + {% trans 'Is optional' %} + + {% trans 'Is resolved' %} + + {% trans 'Is direct' %} +
+ + {{ dependency.for_package }} + + + {% if dependency.resolved_to_package %} + + {{ dependency.resolved_to_package }} + + {% endif %} + + {{ dependency.scope }} + + {{ dependency.is_runtime|as_icon }} + + {{ dependency.is_optional|as_icon }} + + {{ dependency.is_resolved|as_icon }} + + {{ dependency.is_direct|as_icon }} +
+ No results. + {% if filter_dependency.is_active %} + Clear search and filters + {% endif %} +
+{% endspaceless %} \ No newline at end of file diff --git a/product_portfolio/urls.py b/product_portfolio/urls.py index b04afdc1..cd3d0b6f 100644 --- a/product_portfolio/urls.py +++ b/product_portfolio/urls.py @@ -21,6 +21,7 @@ from product_portfolio.views import ProductListView from product_portfolio.views import ProductSendAboutFilesView from product_portfolio.views import ProductTabCodebaseView +from product_portfolio.views import ProductTabDependenciesView from product_portfolio.views import ProductTabImportsView from product_portfolio.views import ProductTabInventoryView from product_portfolio.views import ProductTreeComparisonView @@ -92,6 +93,7 @@ def product_path(path_segment, view): *product_path("load_sboms", LoadSBOMsView.as_view()), *product_path("import_manifests", ImportManifestsView.as_view()), *product_path("tab_codebase", ProductTabCodebaseView.as_view()), + *product_path("tab_dependencies", ProductTabDependenciesView.as_view()), *product_path("tab_imports", ProductTabImportsView.as_view()), *product_path("tab_inventory", ProductTabInventoryView.as_view()), *product_path("pull_project_data", PullProjectDataFromScanCodeIOView.as_view()), diff --git a/product_portfolio/views.py b/product_portfolio/views.py index 10137323..4cb198a5 100644 --- a/product_portfolio/views.py +++ b/product_portfolio/views.py @@ -96,6 +96,7 @@ from license_library.models import License from license_library.models import LicenseAssignedTag from product_portfolio.filters import CodebaseResourceFilterSet +from product_portfolio.filters import DependencyFilterSet from product_portfolio.filters import ProductComponentFilterSet from product_portfolio.filters import ProductFilterSet from product_portfolio.filters import ProductPackageFilterSet @@ -283,6 +284,11 @@ class ProductDetailsView( "owner", ], }, + "dependencies": { + "fields": [ + "dependencies", + ], + }, "activity": {}, "imports": {}, "history": { @@ -463,6 +469,50 @@ def tab_inventory(self): "fields": [(None, tab_context, None, template)], } + # def tab_dependencies(self): + # dependencies_count = self.object.dependencies.count() + # if not dependencies_count: + # return + # + # label = f'Dependencies {dependencies_count}' + # template = "product_portfolio/tabs/tab_dependencies.html" + # + # dependencies = self.object.dependencies.select_related( + # "for_package", + # "resolved_to_package", + # ) + # tab_context = { + # "dependencies": dependencies, + # } + # + # return { + # "label": format_html(label), + # "fields": [(None, tab_context, None, template)], + # } + + def tab_dependencies(self): + dependencies_count = self.object.dependencies.count() + if not dependencies_count: + return + + label = f'Dependencies {dependencies_count}' + template = "tabs/tab_async_loader.html" + + # Pass the current request query context to the async request + tab_view_url = self.object.get_url("tab_dependencies") + if full_query_string := self.request.META["QUERY_STRING"]: + tab_view_url += f"?{full_query_string}" + + tab_context = { + "tab_view_url": tab_view_url, + "tab_object_name": "dependencies", + } + + return { + "label": format_html(label), + "fields": [(None, tab_context, None, template)], + } + def tab_codebase(self): codebaseresources_count = self.object.codebaseresources.count() if not codebaseresources_count: @@ -880,6 +930,56 @@ def has_any_values(field_name): return context_data +class ProductTabDependenciesView( + LoginRequiredMixin, + BaseProductView, + PreviousNextPaginationMixin, + TabContentView, +): + template_name = "product_portfolio/tabs/tab_dependencies.html" + paginate_by = 50 + query_dict_page_param = "dependencies-page" + tab_id = "dependencies" + + def get_context_data(self, **kwargs): + context_data = super().get_context_data(**kwargs) + + dependency_qs = self.object.dependencies.select_related( + "for_package", + "resolved_to_package", + ) + + filter_dependency = DependencyFilterSet( + self.request.GET, + queryset=dependency_qs, + dataspace=self.object.dataspace, + prefix="dependencies", + ) + + paginator = Paginator(filter_dependency.qs, self.paginate_by) + page_number = self.request.GET.get(self.query_dict_page_param) + page_obj = paginator.get_page(page_number) + + context_data.update( + { + "filter_dependency": filter_dependency, + "page_obj": page_obj, + "search_query": self.request.GET.get("dependencies-q", ""), + } + ) + + if page_obj: + previous_url, next_url = self.get_previous_next(page_obj) + context_data.update( + { + "previous_url": (previous_url or "") + f"#{self.tab_id}", + "next_url": (next_url or "") + f"#{self.tab_id}", + } + ) + + return context_data + + class ProductTabImportsView( LoginRequiredMixin, BaseProductView,