diff --git a/.github/workflows/ci-docker.yml b/.github/workflows/ci-docker.yml index 3333879d..09f0aa89 100644 --- a/.github/workflows/ci-docker.yml +++ b/.github/workflows/ci-docker.yml @@ -1,6 +1,10 @@ name: Test on Docker CI -on: [push, pull_request] +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] jobs: test: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3dbb5e3c..21bdac9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,10 @@ name: Test CI -on: [push, pull_request] +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] env: DATABASE_NAME: dejacode diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2be142b3..69e25cc3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,6 +3,57 @@ Release notes ### Version 5.1.1-dev +- Add visual indicator in hierarchy views, when an object on the far left or far right + also belong or have a hierarchy (relationship tree). + https://github.com/nexB/dejacode/issues/70 + +- Add search and pagination on the Product Inventory tab. + https://github.com/nexB/dejacode/issues/3 + https://github.com/nexB/dejacode/issues/112 + +- Fix an issue displaying the "Delete" button in the "Edit Product Relationship" + modal form. + https://github.com/nexB/dejacode/issues/128 + +- Add support for PURL(s) in the "Add Package" modal. + If the PURL type is supported by the packageurl_python library, a download URL + will be generated for creating the package and submitting a scan. + https://github.com/nexB/dejacode/issues/131 + +- Leverage PurlDB during the "Add Package" process. + DejaCode will look up the PurlDB to retrieve and fetch all available data to + create the package. + https://github.com/nexB/dejacode/issues/131 + +- Populate the Package notice_text using "*NOTICE*" file content from Scan "key files". + https://github.com/nexB/dejacode/issues/136 + +- Added 2 new license related fields on the Component and Package models: + * declared_license_expression + * other_license_expression + https://github.com/nexB/dejacode/issues/63 + +- Added 2 properties on the Component and Package models: + * declared_license_expression_spdx (computed from declared_license_expression) + * other_license_expression_spdx (computed from other_license_expression) + https://github.com/nexB/dejacode/issues/63 + +- Removed 2 fields: Package.declared_license and Component.concluded_license + https://github.com/nexB/dejacode/issues/63 + +- The new license fields are automatically populated from the Package scan + "Update packages automatically from scan". + The new license fields are pre-filled in the Package form when using the + "Add Package" from a PurlDB entry. + The new license fields are pre-filled in the Component form when using the + "Add Component from Package data". + The license expression values provided in the form for the new field is now + properly checked and return a validation error when incorrect. + https://github.com/nexB/dejacode/issues/63 + +- Use the declared_license_expression_spdx value in SPDX outputs. + https://github.com/nexB/dejacode/issues/63 + ### Version 5.1.0 - Upgrade Python version to 3.12 and Django to 5.0.x diff --git a/Makefile b/Makefile index 1ccbbf21..f914f489 100644 --- a/Makefile +++ b/Makefile @@ -126,6 +126,7 @@ postgresdb: @dropdb ${DB_NAME} || true @echo "-> Create ${DB_NAME} database" @createdb --owner=${DB_USERNAME} ${POSTGRES_INITDB_ARGS} ${DB_NAME} + @gunzip < ${DB_INIT_FILE} | psql --username=${DB_USERNAME} ${DB_NAME} run: ${MANAGE} runserver 8000 --insecure diff --git a/component_catalog/admin.py b/component_catalog/admin.py index 6f745659..4b2751a4 100644 --- a/component_catalog/admin.py +++ b/component_catalog/admin.py @@ -324,6 +324,8 @@ class ComponentAdmin( "copyright", "holder", "license_expression", + "declared_license_expression", + "other_license_expression", "reference_notes", "release_date", "description", @@ -394,7 +396,6 @@ class ComponentAdmin( "ip_sensitivity_approved", "affiliate_obligations", "affiliate_obligation_triggers", - "concluded_license", "legal_comments", "sublicense_allowed", "express_patent_grant", @@ -418,7 +419,10 @@ class ComponentAdmin( autocomplete_lookup_fields = {"fk": ["owner"]} # We have to use 'completion_level' rather than the 'completion_level_pct' # callable to keep the help_text available during render in the template. - readonly_fields = DataspacedAdmin.readonly_fields + ("urn_link", "completion_level") + readonly_fields = DataspacedAdmin.readonly_fields + ( + "urn_link", + "completion_level", + ) form = ComponentAdminForm inlines = [ SubcomponentChildInline, @@ -826,6 +830,8 @@ class PackageAdmin( { "fields": ( "license_expression", + "declared_license_expression", + "other_license_expression", "copyright", "holder", "author", @@ -864,7 +870,6 @@ class PackageAdmin( "Others", { "fields": ( - "declared_license", "parties", "datasource_id", "file_references", @@ -880,7 +885,10 @@ class PackageAdmin( ), get_additional_information_fieldset(), ] - readonly_fields = DataspacedAdmin.readonly_fields + ("package_url", "inferred_url") + readonly_fields = DataspacedAdmin.readonly_fields + ( + "package_url", + "inferred_url", + ) form = PackageAdminForm importer_class = PackageImporter mass_update_form = PackageMassUpdateForm diff --git a/component_catalog/api.py b/component_catalog/api.py index e9ad4fbf..f19f71a7 100644 --- a/component_catalog/api.py +++ b/component_catalog/api.py @@ -140,6 +140,8 @@ class Meta: "holder", "author", "license_expression", + "declared_license_expression", + "other_license_expression", "reference_notes", "homepage_url", "vcs_url", @@ -305,7 +307,6 @@ class Meta: "ip_sensitivity_approved", "affiliate_obligations", "affiliate_obligation_triggers", - "concluded_license", "legal_comments", "sublicense_allowed", "express_patent_grant", @@ -323,6 +324,8 @@ class Meta: "licenses_summary", "license_choices_expression", "license_choices", + "declared_license_expression", + "other_license_expression", "created_date", "last_modified_date", ) @@ -524,6 +527,8 @@ class Meta(ComponentSerializer.Meta): "copyright", "holder", "license_expression", + "declared_license_expression", + "other_license_expression", "reference_notes", "release_date", "description", @@ -551,7 +556,6 @@ class Meta(ComponentSerializer.Meta): "ip_sensitivity_approved", "affiliate_obligations", "affiliate_obligation_triggers", - "concluded_license", "legal_comments", "sublicense_allowed", "express_patent_grant", @@ -639,6 +643,8 @@ class Meta: "licenses_summary", "license_choices_expression", "license_choices", + "declared_license_expression", + "other_license_expression", "reference_notes", "homepage_url", "vcs_url", @@ -656,7 +662,6 @@ class Meta: "version", "qualifiers", "subpath", - "declared_license", "parties", "datasource_id", "file_references", @@ -682,6 +687,7 @@ class Meta: def create(self, validated_data): """Collect data, purl, and submit scan if `collect_data` is provided.""" user = self.context["request"].user + dataspace = user.dataspace collect_data = validated_data.pop("collect_data", None) download_url = validated_data.get("download_url") @@ -701,14 +707,14 @@ def create(self, validated_data): package = super().create(validated_data) # Submit the scan if Package was properly created - scancodeio = ScanCodeIO(user) - if scancodeio.is_configured() and user.dataspace.enable_package_scanning: + scancodeio = ScanCodeIO(dataspace) + if scancodeio.is_configured() and dataspace.enable_package_scanning: # Ensure the task is executed after the transaction is successfully committed transaction.on_commit( lambda: tasks.scancodeio_submit_scan.delay( uris=download_url, user_uuid=user.uuid, - dataspace_uuid=user.dataspace.uuid, + dataspace_uuid=dataspace.uuid, ) ) @@ -799,7 +805,8 @@ class Meta: def collect_create_scan(download_url, user): - package_qs = Package.objects.filter(download_url=download_url, dataspace=user.dataspace) + dataspace = user.dataspace + package_qs = Package.objects.filter(download_url=download_url, dataspace=dataspace) if package_qs.exists(): return False @@ -814,12 +821,12 @@ def collect_create_scan(download_url, user): package = Package.create_from_data(user, package_data) - scancodeio = ScanCodeIO(user) - if scancodeio.is_configured() and user.dataspace.enable_package_scanning: + scancodeio = ScanCodeIO(dataspace) + if scancodeio.is_configured() and dataspace.enable_package_scanning: tasks.scancodeio_submit_scan.delay( uris=download_url, user_uuid=user.uuid, - dataspace_uuid=user.dataspace.uuid, + dataspace_uuid=dataspace.uuid, ) return package diff --git a/component_catalog/forms.py b/component_catalog/forms.py index e6c3ddf8..e2c2cb9f 100644 --- a/component_catalog/forms.py +++ b/component_catalog/forms.py @@ -14,6 +14,7 @@ from django.urls import reverse from django.urls import reverse_lazy from django.utils.functional import cached_property +from django.utils.text import Truncator import packageurl from crispy_forms.helper import FormHelper @@ -82,6 +83,11 @@ class ComponentForm( DataspacedModelForm, ): default_on_addition_fields = ["configuration_status"] + expression_field_names = [ + "license_expression", + "declared_license_expression", + "other_license_expression", + ] save_as = True clone_m2m_classes = [ ComponentAssignedPackage, @@ -106,6 +112,8 @@ class Meta: "holder", "notice_text", "license_expression", + "declared_license_expression", + "other_license_expression", "release_date", "description", "homepage_url", @@ -130,6 +138,8 @@ class Meta: "owner": OwnerChoiceField, } widgets = { + "declared_license_expression": forms.Textarea(attrs={"rows": 2}), + "other_license_expression": forms.Textarea(attrs={"rows": 2}), "copyright": forms.Textarea(attrs={"rows": 2}), "notice_text": forms.Textarea(attrs={"rows": 2}), "description": forms.Textarea(attrs={"rows": 2}), @@ -188,6 +198,7 @@ def helper(self): Group("name", "version", "owner"), HTML("
"), "license_expression", + Group("declared_license_expression", "other_license_expression"), Group("copyright", "holder"), "notice_text", Group("notice_filename", "notice_url"), @@ -267,6 +278,11 @@ class PackageForm( PackageFieldsValidationMixin, DataspacedModelForm, ): + expression_field_names = [ + "license_expression", + "declared_license_expression", + "other_license_expression", + ] save_as = True color_initial = True @@ -299,6 +315,8 @@ class Meta: "notes", "usage_policy", "license_expression", + "declared_license_expression", + "other_license_expression", "copyright", "holder", "author", @@ -320,6 +338,8 @@ class Meta: "collect_data", ] widgets = { + "declared_license_expression": forms.Textarea(attrs={"rows": 2}), + "other_license_expression": forms.Textarea(attrs={"rows": 2}), "description": forms.Textarea(attrs={"rows": 2}), "notes": forms.Textarea(attrs={"rows": 2}), "copyright": forms.Textarea(attrs={"rows": 2}), @@ -337,7 +357,7 @@ class Meta: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - scancodeio = ScanCodeIO(self.user) + scancodeio = ScanCodeIO(self.dataspace) self.submit_scan_enabled = all( [ self.is_addition, @@ -377,6 +397,7 @@ def helper(self): Group("version", "qualifiers", "subpath"), HTML("
"), "license_expression", + Group("declared_license_expression", "other_license_expression"), Group("copyright", "notice_text"), Group("holder", "author"), HTML("
"), @@ -429,6 +450,12 @@ def save(self, *args, **kwargs): class BaseScanToPackageForm(LicenseExpressionFormMixin, DataspacedModelForm): + expression_field_names = [ + "license_expression", + "declared_license_expression", + "other_license_expression", + ] + @property def helper(self): helper = FormHelper() @@ -451,6 +478,8 @@ class Meta: fields = [ "package_url", "license_expression", + "declared_license_expression", + "other_license_expression", "copyright", "primary_language", "description", @@ -482,6 +511,12 @@ def __init__(self, *args, **kwargs): def fields_with_initial_value(self): kept_fields = {} + # Duplicate the declared_license_expression into the license_expression field + # if currently empty on the Package instance. + if not self.instance.license_expression: + if declared_license_expression := self.initial.get("declared_license_expression"): + self.initial["license_expression"] = declared_license_expression + for field_name, field in self.fields.items(): if not self.initial.get(field_name): continue @@ -489,7 +524,7 @@ def fields_with_initial_value(self): instance_value = getattr(self.instance, field_name, None) help_text = "No current value" if instance_value: - help_text = f"Current value: {instance_value}" + help_text = f"Current value: {Truncator(instance_value).chars(200)}" field.help_text = help_text kept_fields[field_name] = field @@ -524,6 +559,8 @@ class Meta: model = Package fields = [ "license_expression", + "declared_license_expression", + "other_license_expression", "primary_language", "holder", ] @@ -542,7 +579,7 @@ def set_help_text_with_initial_value(self): instance_value = getattr(self.instance, field_name, None) help_text = "No current value" if instance_value: - help_text = f"Current value: {instance_value}" + help_text = f"Current value: {Truncator(instance_value).chars(200)}" field.help_text = help_text @@ -922,6 +959,12 @@ class ComponentAdminForm( SetKeywordsChoicesFormMixin, DataspacedAdminForm, ): + expression_field_names = [ + "license_expression", + "declared_license_expression", + "other_license_expression", + ] + keywords = JSONListField( required=False, widget=AdminAwesompleteInputWidget(attrs=autocomplete_placeholder), @@ -956,6 +999,12 @@ class PackageAdminForm( SetKeywordsChoicesFormMixin, DataspacedAdminForm, ): + expression_field_names = [ + "license_expression", + "declared_license_expression", + "other_license_expression", + ] + keywords = JSONListField( required=False, widget=AdminAwesompleteInputWidget(attrs=autocomplete_placeholder), @@ -1043,7 +1092,6 @@ class Meta: "ip_sensitivity_approved", "affiliate_obligations", "affiliate_obligation_triggers", - "concluded_license", "legal_comments", "sublicense_allowed", "express_patent_grant", diff --git a/component_catalog/license_expression_dje.py b/component_catalog/license_expression_dje.py index 386e0e92..1085dcf0 100644 --- a/component_catalog/license_expression_dje.py +++ b/component_catalog/license_expression_dje.py @@ -9,6 +9,7 @@ from collections import defaultdict from itertools import chain +from django.core.cache import caches from django.core.exceptions import ValidationError from django.forms import widgets from django.urls import reverse @@ -23,6 +24,8 @@ from dje.widgets import AwesompleteInputWidgetMixin from license_library.models import License +licensing_cache = caches["licensing"] + def build_licensing(licenses=None): """ @@ -34,6 +37,39 @@ def build_licensing(licenses=None): return Licensing(licenses) +def fetch_licensing_for_dataspace(dataspace, license_keys=None): + """ + Return a Licensing object for the provided ``dataspace``. + An optional list of ``license_keys`` can be provided to limit the licenses + included in the Licensing object. + """ + license_qs = License.objects.scope(dataspace).for_expression(license_keys) + licensing = build_licensing(license_qs) + return licensing + + +def get_dataspace_licensing(dataspace, license_keys=None): + """ + Return a Licensing object for the provided ``dataspace``. + The Licensing object is put in the cache for 5 minutes. + Note that the cache is not used when ``license_keys`` are provided. + """ + if license_keys is not None: + # Bypass cache if license_keys is provided + return fetch_licensing_for_dataspace(dataspace, license_keys) + + cache_key = str({dataspace.name}) + # First look in the cache for an existing Licensing for this Dataspace + licensing = licensing_cache.get(cache_key) + + if licensing is None: + # If not cached, compute the value and cache it + licensing = fetch_licensing_for_dataspace(dataspace, license_keys) + licensing_cache.set(cache_key, licensing, timeout=600) # 10 minutes + + return licensing + + def parse_expression( expression, licenses=None, validate_known=True, validate_strict=False, simple=False ): @@ -276,6 +312,14 @@ def clean_license_expression(self): # or without `license_expression` value. return self.clean_expression_base(expression) + def clean_declared_license_expression(self): + expression = self.cleaned_data.get("declared_license_expression") + return self.clean_expression_base(expression) + + def clean_other_license_expression(self): + expression = self.cleaned_data.get("other_license_expression") + return self.clean_expression_base(expression) + def clean_expression_base(self, expression): """ Return a normalized license expression string validated against @@ -373,12 +417,6 @@ def get_unique_license_keys(license_expression): return {symbol.key for symbol in symbols} -def get_licensing_for_formatted_render(dataspace, show_policy=False, license_keys=None): - license_qs = License.objects.scope(dataspace).for_expression(show_policy, license_keys) - licensing = build_licensing(license_qs) - return licensing - - def get_formatted_expression(licensing, license_expression, show_policy, show_category=False): normalized = parse_expression( license_expression, licenses=licensing, validate_known=False, validate_strict=False @@ -386,3 +424,19 @@ def get_formatted_expression(licensing, license_expression, show_policy, show_ca return normalized.render_as_readable( as_link=True, show_policy=show_policy, show_category=show_category ) + + +def render_expression_as_html(expression, dataspace): + """Return the ``expression`` as rendered HTML content.""" + show_policy = dataspace.show_usage_policy_in_user_views + licensing = get_dataspace_licensing(dataspace) + + formatted_expression = get_formatted_expression(licensing, expression, show_policy) + return format_html(formatted_expression) + + +def get_expression_as_spdx(expression, dataspace): + """Return an SPDX license expression built from the ``expression``.""" + licensing = get_dataspace_licensing(dataspace) + parsed_expression = parse_expression(expression, licensing) + return parsed_expression.render(template="{symbol.spdx_id}") diff --git a/component_catalog/migrations/0005_remove_component_concluded_license_and_more.py b/component_catalog/migrations/0005_remove_component_concluded_license_and_more.py new file mode 100644 index 00000000..e332e569 --- /dev/null +++ b/component_catalog/migrations/0005_remove_component_concluded_license_and_more.py @@ -0,0 +1,75 @@ +# Generated by Django 5.0.6 on 2024-06-14 07:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("component_catalog", "0004_initial"), + ] + + operations = [ + migrations.RemoveField( + model_name="component", + name="concluded_license", + ), + migrations.RemoveField( + model_name="package", + name="declared_license", + ), + migrations.AddField( + model_name="component", + name="declared_license_expression", + field=models.TextField( + blank=True, + help_text="A license expression derived from statements in the manifests or key files of a software project, such as the NOTICE, COPYING, README, and LICENSE files.", + ), + ), + migrations.AddField( + model_name="component", + name="other_license_expression", + field=models.TextField( + blank=True, + help_text="A license expression derived from detected licenses in the non-key files of a software project, which are often third-party software used by the project, or test, sample and documentation files.", + ), + ), + migrations.AddField( + model_name="package", + name="declared_license_expression", + field=models.TextField( + blank=True, + help_text="A license expression derived from statements in the manifests or key files of a software project, such as the NOTICE, COPYING, README, and LICENSE files.", + ), + ), + migrations.AddField( + model_name="package", + name="other_license_expression", + field=models.TextField( + blank=True, + help_text="A license expression derived from detected licenses in the non-key files of a software project, which are often third-party software used by the project, or test, sample and documentation files.", + ), + ), + migrations.AlterField( + model_name="component", + name="license_expression", + field=models.CharField( + blank=True, + db_index=True, + help_text='The License Expression assigned to a DejaCode Package or Component is an editable value equivalent to a "concluded license" as determined by a curator who has performed analysis to clarify or correct the declared license expression, which may have been assigned automatically (from a scan or an associated package definition) when the Package or Component was originally created. A license expression defines the relationship of one or more licenses to a software object. More than one applicable license can be expressed as "license-key-a AND license-key-b". A choice of applicable licenses can be expressed as "license-key-a OR license-key-b", and you can indicate the primary (preferred) license by placing it first, on the left-hand side of the OR relationship. The relationship words (OR, AND) can be combined as needed, and the use of parentheses can be applied to clarify the meaning; for example "((license-key-a AND license-key-b) OR (license-key-c))". An exception to a license can be expressed as "license-key WITH license-exception-key".', + max_length=1024, + verbose_name="Concluded license expression", + ), + ), + migrations.AlterField( + model_name="package", + name="license_expression", + field=models.CharField( + blank=True, + db_index=True, + help_text='The License Expression assigned to a DejaCode Package or Component is an editable value equivalent to a "concluded license" as determined by a curator who has performed analysis to clarify or correct the declared license expression, which may have been assigned automatically (from a scan or an associated package definition) when the Package or Component was originally created. A license expression defines the relationship of one or more licenses to a software object. More than one applicable license can be expressed as "license-key-a AND license-key-b". A choice of applicable licenses can be expressed as "license-key-a OR license-key-b", and you can indicate the primary (preferred) license by placing it first, on the left-hand side of the OR relationship. The relationship words (OR, AND) can be combined as needed, and the use of parentheses can be applied to clarify the meaning; for example "((license-key-a AND license-key-b) OR (license-key-c))". An exception to a license can be expressed as "license-key WITH license-exception-key".', + max_length=1024, + verbose_name="Concluded license expression", + ), + ), + ] diff --git a/component_catalog/models.py b/component_catalog/models.py index 52236809..eaa05d2e 100644 --- a/component_catalog/models.py +++ b/component_catalog/models.py @@ -36,6 +36,7 @@ from cyclonedx.model import component as cyclonedx_component from cyclonedx.model import contact as cyclonedx_contact from cyclonedx.model import license as cyclonedx_license +from license_expression import ExpressionError from packageurl import PackageURL from packageurl.contrib import purl2url from packageurl.contrib import url2purl @@ -44,12 +45,14 @@ from packageurl.contrib.django.utils import without_empty_values from component_catalog.license_expression_dje import build_licensing +from component_catalog.license_expression_dje import get_expression_as_spdx from component_catalog.license_expression_dje import get_license_objects from component_catalog.license_expression_dje import parse_expression from dejacode_toolkit import spdx from dejacode_toolkit.download import DataCollectionException from dejacode_toolkit.download import collect_package_data from dejacode_toolkit.purldb import PurlDB +from dejacode_toolkit.purldb import pick_purldb_entry from dje import urn from dje.copier import post_copy from dje.copier import post_update @@ -65,6 +68,7 @@ from dje.models import ParentChildRelationshipModel from dje.models import ReferenceNotesMixin from dje.tasks import logger as tasks_logger +from dje.utils import is_purl_str from dje.utils import set_fields_from_object from dje.validators import generic_uri_validator from dje.validators import validate_url_segment @@ -92,6 +96,29 @@ "version", ] +LICENSE_EXPRESSION_HELP_TEXT = _( + "The License Expression assigned to a DejaCode Package or Component is an editable " + 'value equivalent to a "concluded license" as determined by a curator who has ' + "performed analysis to clarify or correct the declared license expression, which " + "may have been assigned automatically (from a scan or an associated package " + "definition) when the Package or Component was originally created. " + "A license expression defines the relationship of one or more licenses to a " + "software object. More than one applicable license can be expressed as " + '"license-key-a AND license-key-b". A choice of applicable licenses can be ' + 'expressed as "license-key-a OR license-key-b", and you can indicate the primary ' + "(preferred) license by placing it first, on the left-hand side of the OR " + "relationship. The relationship words (OR, AND) can be combined as needed, " + "and the use of parentheses can be applied to clarify the meaning; " + 'for example "((license-key-a AND license-key-b) OR (license-key-c))". ' + "An exception to a license can be expressed as " + '"license-key WITH license-exception-key".' +) + + +class PackageAlreadyExistsWarning(Exception): + def __init__(self, message): + self.message = message + def validate_filename(value): invalid_chars = ["/", "\\", ":"] @@ -160,7 +187,21 @@ def get_license_expression_linked_with_policy(self): if license_expression: return format_html('{}', license_expression) - def get_license_expression_spdx_id(self): + def _get_primary_license(self): + """ + Return the primary license key of this instance or None. The primary license is + the left most license of the expression. It can be the combination of a + license WITH an exception and therefore may contain more than one key. + + WARNING: This does not support exception as primary_license. + """ + if self.license_expression: + licensing = build_licensing() + return licensing.primary_license_key(self.license_expression) + + primary_license = cached_property(_get_primary_license) + + def get_expression_as_spdx(self, expression): """ Return the license_expression formatted for SPDX compatibility. @@ -172,23 +213,20 @@ def get_license_expression_spdx_id(self): See discussion at https://github.com/spdx/tools-java/issues/73 """ - expression = self.get_license_expression("{symbol.spdx_id}") - if expression: - return expression.replace("WITH LicenseRef-", "AND LicenseRef-") + if not expression: + return - def _get_primary_license(self): - """ - Return the primary license key of this instance or None. The primary license is - the left most license of the expression. It can be the combination of a - license WITH an exception and therefore may contain more than one key. + try: + expression_as_spdx = get_expression_as_spdx(expression, self.dataspace) + except ExpressionError as e: + return str(e) - WARNING: This does not support exception as primary_license. - """ - if self.license_expression: - licensing = build_licensing() - return licensing.primary_license_key(self.license_expression) + if expression_as_spdx: + return expression_as_spdx.replace("WITH LicenseRef-", "AND LicenseRef-") - primary_license = cached_property(_get_primary_license) + @property + def concluded_license_expression_spdx(self): + return self.get_expression_as_spdx(self.license_expression) def save(self, *args, **kwargs): """ @@ -294,6 +332,37 @@ def compliance_table_class(self): return "table-warning" +class LicenseFieldsMixin(models.Model): + declared_license_expression = models.TextField( + blank=True, + help_text=_( + "A license expression derived from statements in the manifests or key " + "files of a software project, such as the NOTICE, COPYING, README, and " + "LICENSE files." + ), + ) + + other_license_expression = models.TextField( + blank=True, + help_text=_( + "A license expression derived from detected licenses in the non-key files " + "of a software project, which are often third-party software used by the " + "project, or test, sample and documentation files." + ), + ) + + class Meta: + abstract = True + + @property + def declared_license_expression_spdx(self): + return self.get_expression_as_spdx(self.declared_license_expression) + + @property + def other_license_expression_spdx(self): + return self.get_expression_as_spdx(self.other_license_expression) + + def get_cyclonedx_properties(instance): """ Return fields not supported natively by CycloneDX as properties. @@ -691,25 +760,6 @@ class BaseComponentMixin( ), ) - license_expression = models.CharField( - _("License expression"), - max_length=1024, - blank=True, - db_index=True, - help_text=_( - "On a component or a product in DejaCode, a license expression defines the " - "relationship of one or more licenses to that software as declared by its " - 'licensor. More than one applicable license can be expressed as "license-key-a ' - 'AND license-key-b". A choice of applicable licenses can be expressed as ' - '"license-key-a OR license-key-b", and you can indicate the primary (preferred) ' - "license by placing it first, on the left-hand side of the OR relationship. " - "The relationship words (OR, AND) can be combined as needed, and the use of " - 'parentheses can be applied to clarify the meaning; for example "((license-key-a ' - 'AND license-key-b) OR (license-key-c))". An exception to a license can be ' - 'expressed as “license-key WITH license-exception-key".' - ), - ) - class Meta: abstract = True unique_together = (("dataspace", "name", "version"), ("dataspace", "uuid")) @@ -765,7 +815,7 @@ def as_cyclonedx(self, license_expression_spdx=None): urls=[self.owner.homepage_url], ) - expression_spdx = license_expression_spdx or self.get_license_expression_spdx_id() + expression_spdx = license_expression_spdx or self.concluded_license_expression_spdx licenses = [] if expression_spdx: # Using the LicenseExpression directly as the make_with_expression method @@ -819,10 +869,19 @@ class Component( HolderMixin, KeywordsMixin, CPEMixin, + LicenseFieldsMixin, ParentChildModelMixin, BaseComponentMixin, DataspacedModel, ): + license_expression = models.CharField( + _("Concluded license expression"), + max_length=1024, + blank=True, + db_index=True, + help_text=LICENSE_EXPRESSION_HELP_TEXT, + ) + configuration_status = models.ForeignKey( to="component_catalog.ComponentStatus", on_delete=models.PROTECT, @@ -1020,19 +1079,6 @@ class Component( ), ) - concluded_license = models.CharField( - max_length=1024, - blank=True, - db_index=True, - help_text=_( - "This is a memo field to record the conclusions of the legal team after full review " - "and scanning of the component package, and is only intended to document that " - "decision. The main value of the field is to clarify the company interpretation of " - "the license that should apply to a component when there is a choice, or when there " - "is ambiguity in the original component documentation." - ), - ) - legal_comments = models.TextField( blank=True, help_text=_( @@ -1279,7 +1325,12 @@ def aboutcode_data(self): return without_empty_values(component_data) def as_spdx(self, license_concluded=None): - """Return this Component as an SPDX Package entry.""" + """ + Return this Component as an SPDX Package entry. + An optional ``license_concluded`` can be provided to override the + ``license_expression`` value defined on this instance. + This can be a license choice applied to a Product relationship. + """ external_refs = [] if cpe_external_ref := self.get_spdx_cpe_external_ref(): @@ -1289,14 +1340,12 @@ def as_spdx(self, license_concluded=None): if self.notice_text: attribution_texts.append(self.notice_text) - license_expression_spdx = self.get_license_expression_spdx_id() - return spdx.Package( name=self.name, spdx_id=f"dejacode-{self._meta.model_name}-{self.uuid}", supplier=self.owner.as_spdx() if self.owner else "", - license_concluded=license_concluded or license_expression_spdx, - license_declared=license_expression_spdx, + license_concluded=license_concluded or self.concluded_license_expression_spdx, + license_declared=self.declared_license_expression_spdx, copyright_text=self.copyright, version=self.version, homepage=self.homepage_url, @@ -1584,6 +1633,7 @@ class Package( UsagePolicyMixin, SetPolicyFromLicenseMixin, LicenseExpressionMixin, + LicenseFieldsMixin, RequestMixin, HistoryFieldsMixin, ReferenceNotesMixin, @@ -1672,21 +1722,11 @@ class Package( ) license_expression = models.CharField( - _("License expression"), + _("Concluded license expression"), max_length=1024, blank=True, db_index=True, - help_text=_( - "On a package in DejaCode, a license expression defines the relationship of one or " - "more licenses to that software as declared by its licensor. More than one " - 'applicable license can be expressed as "license-key-a AND license-key-b". A choice ' - 'of applicable licenses can be expressed as "license-key-a OR license-key-b", and you ' - "can indicate the primary (preferred) license by placing it first, on the left-hand " - "side of the OR relationship. The relationship words (OR, AND) can be combined as " - "needed, and the use of parentheses can be applied to clarify the meaning; for " - 'example "((license-key-a AND license-key-b) OR (license-key-c))". An exception to ' - 'a license can be expressed as “license-key WITH license-exception-key".' - ), + help_text=LICENSE_EXPRESSION_HELP_TEXT, ) copyright = models.TextField( @@ -1753,15 +1793,6 @@ class Package( ), ) - declared_license = models.TextField( - blank=True, - help_text=_( - "The declared license mention, tag or text as found in a package manifest. " - "This can be a string, a list or dict of strings possibly nested, " - "as found originally in the manifest." - ), - ) - datasource_id = models.CharField( max_length=64, blank=True, @@ -2180,7 +2211,12 @@ def get_about_files(self): return about_files def as_spdx(self, license_concluded=None): - """Return this Package as an SPDX Package entry.""" + """ + Return this Package as an SPDX Package entry. + An optional ``license_concluded`` can be provided to override the + ``license_expression`` value defined on this instance. + This can be a license choice applied to a Product relationship. + """ checksums = [ spdx.Checksum(algorithm=algorithm, value=checksum_value) for algorithm in ["sha1", "md5"] @@ -2205,14 +2241,12 @@ def as_spdx(self, license_concluded=None): if cpe_external_ref := self.get_spdx_cpe_external_ref(): external_refs.append(cpe_external_ref) - license_expression_spdx = self.get_license_expression_spdx_id() - return spdx.Package( name=self.name or self.filename, spdx_id=f"dejacode-{self._meta.model_name}-{self.uuid}", download_location=self.download_url, - license_declared=license_expression_spdx, - license_concluded=license_concluded or license_expression_spdx, + license_concluded=license_concluded or self.concluded_license_expression_spdx, + license_declared=self.declared_license_expression_spdx, copyright_text=self.copyright, version=self.version, homepage=self.homepage_url, @@ -2230,7 +2264,7 @@ def get_spdx_packages(self): def as_cyclonedx(self, license_expression_spdx=None): """Return this Package as an CycloneDX Component entry.""" - expression_spdx = license_expression_spdx or self.get_license_expression_spdx_id() + expression_spdx = license_expression_spdx or self.concluded_license_expression_spdx licenses = [] if expression_spdx: @@ -2291,6 +2325,72 @@ def where_used(self, user): f"Component {self.component_set.count()}\n" ) + @classmethod + def create_from_url(cls, url, user): + """ + Create a package from the given URL for the specified user. + + This function processes the URL to create a package entry. It handles + both direct download URLs and Package URLs (purls), checking for + existing packages to avoid duplicates. If the package is not already + present, it collects necessary package data and creates a new package + entry. + """ + url = url.strip() + if not url: + return + + package_data = {} + scoped_packages_qs = cls.objects.scope(user.dataspace) + + if is_purl_str(url): + download_url = purl2url.get_download_url(url) + package_url = PackageURL.from_string(url) + existing_packages = scoped_packages_qs.for_package_url(url, exact_match=True) + else: + download_url = url + package_url = url2purl.get_purl(url) + existing_packages = scoped_packages_qs.filter(download_url=url) + + if existing_packages: + package_links = [package.get_absolute_link() for package in existing_packages] + raise PackageAlreadyExistsWarning( + f"{url} already exists in your Dataspace as {', '.join(package_links)}" + ) + + # Matching in PurlDB early to avoid more processing in case of a match. + purldb_data = None + if user.dataspace.enable_purldb_access: + package_for_match = cls(download_url=download_url) + package_for_match.set_package_url(package_url) + purldb_entries = package_for_match.get_purldb_entries(user) + # Look for one ith the same exact purl in that case + if purldb_data := pick_purldb_entry(purldb_entries, purl=url): + # The format from PurlDB is "2019-11-18T00:00:00Z" from DateTimeField + if release_date := purldb_data.get("release_date"): + purldb_data["release_date"] = release_date.split("T")[0] + package_data.update(purldb_data) + + if download_url and not purldb_data: + package_data = collect_package_data(download_url) + + if sha1 := package_data.get("sha1"): + if sha1_match := scoped_packages_qs.filter(sha1=sha1): + package_link = sha1_match[0].get_absolute_link() + raise PackageAlreadyExistsWarning( + f"{url} already exists in your Dataspace as {package_link}" + ) + + # Duplicate the declared_license_expression into the license_expression field. + if declared_license_expression := package_data.get("declared_license_expression"): + package_data["license_expression"] = declared_license_expression + + if package_url: + package_data.update(package_url.to_dict(encode=True, empty="")) + + package = cls.create_from_data(user, package_data) + return package + def get_purldb_entries(self, user, max_request_call=0, timeout=10): """ Return the PurlDB entries that correspond to this Package instance. @@ -2316,13 +2416,15 @@ def get_purldb_entries(self, user, max_request_call=0, timeout=10): if self.download_url: payloads.append({"download_url": self.download_url}) + purldb = PurlDB(user.dataspace) for index, payload in enumerate(payloads): if max_request_call and index >= max_request_call: return - if packages_data := PurlDB(user).find_packages(payload, timeout): + if packages_data := purldb.find_packages(payload, timeout): return packages_data + def update_from_purldb(self, user): """ Find this Package in the PurlDB and update empty fields with PurlDB data diff --git a/component_catalog/templates/component_catalog/component_details.html b/component_catalog/templates/component_catalog/component_details.html index 092b918f..94640261 100644 --- a/component_catalog/templates/component_catalog/component_details.html +++ b/component_catalog/templates/component_catalog/component_details.html @@ -58,7 +58,7 @@ {% block javascripts %} {{ block.super }} - + {% include 'component_catalog/includes/component_hierarchy.js.html' with related_parents=tabsets.Hierarchy.fields.0.1.related_parents related_children=tabsets.Hierarchy.fields.0.1.related_children productcomponents=tabsets.Hierarchy.fields.0.1.productcomponents %} {% if tabsets.Owner.extra %} {% include 'organization/includes/owner_hierarchy.js.html' with current_owner=object.owner parents=tabsets.Owner.extra.context.owner_parents children=tabsets.Owner.extra.context.owner_children tab_name="tab_owner" %} diff --git a/component_catalog/templates/component_catalog/includes/add_package_modal.html b/component_catalog/templates/component_catalog/includes/add_package_modal.html index 194816d6..8f282819 100644 --- a/component_catalog/templates/component_catalog/includes/add_package_modal.html +++ b/component_catalog/templates/component_catalog/includes/add_package_modal.html @@ -10,19 +10,28 @@
Add Package
{% csrf_token %}
- - + + - Enter the download URL for obtaining the package.
+ Enter the download URL(s) or Package URL(s) of your packages.

You can provide multiple URLs separated by a new-line.
A download URL is one that will immediately initiate the download of a software package if you click it on a browser page or paste it into a browser address field.

Note that this feature is intended only for publicly available open source packages, not your private code.
DejaCode will automatically collect the filename, sha1, md5, and size and apply them to the package definition. {% if user.dataspace.enable_package_scanning %} - - Package scanning is enabled in your Dataspace, DejaCode will also submit the package to ScanCode.io and the results will be returned to the "Scan" detail tab of the package when that scan is complete. - +
+ + Package scanning is enabled in your Dataspace, DejaCode will also submit the package to ScanCode.io and the results will be returned to the "Scan" detail tab of the package when that scan is complete. + +
+ {% endif %} + {% if user.dataspace.enable_purldb_access %} +
+ + PurlDB access is enabled in your Dataspace, DejaCode will look up the PurlDB to retrieve and fetch all available data to create the package. + +
{% endif %}
@@ -66,8 +75,13 @@
Add Package
}, error: function(data) { $('#overlay').remove(); - let error_msg = data; - if (data.responseJSON) error_msg = data.responseJSON.error_message; + let error_msg; + if (data.responseJSON) { + error_msg = data.responseJSON.error_message; + } + else { + error_msg = data.statusText; + } $('#package-add-error').html('Error: ' + error_msg).show(); } }); diff --git a/component_catalog/templates/component_catalog/includes/component_hierarchy.js.html b/component_catalog/templates/component_catalog/includes/component_hierarchy.js.html index 49478251..c37e010b 100644 --- a/component_catalog/templates/component_catalog/includes/component_hierarchy.js.html +++ b/component_catalog/templates/component_catalog/includes/component_hierarchy.js.html @@ -1,84 +1,66 @@ - \ No newline at end of file + {% if related_child.child_count > 0 %} + var linkUrl = '{{ related_child.child.get_absolute_url }}#hierarchy'; + addEndpointWithLink(jsPlumbHierarchy, source_id, 'RightMiddle', linkUrl); + {% endif %} + {% endfor %} +{% endblock %} \ No newline at end of file diff --git a/component_catalog/templates/component_catalog/includes/scan_matches_modal.html b/component_catalog/templates/component_catalog/includes/scan_matches_modal.html index e94b2365..79b2c860 100644 --- a/component_catalog/templates/component_catalog/includes/scan_matches_modal.html +++ b/component_catalog/templates/component_catalog/includes/scan_matches_modal.html @@ -17,14 +17,12 @@
\ No newline at end of file diff --git a/component_catalog/templates/component_catalog/includes/scan_summary_to_package_modal.html b/component_catalog/templates/component_catalog/includes/scan_summary_to_package_modal.html index e6af3aa6..4c1ff58b 100644 --- a/component_catalog/templates/component_catalog/includes/scan_summary_to_package_modal.html +++ b/component_catalog/templates/component_catalog/includes/scan_summary_to_package_modal.html @@ -20,36 +20,65 @@
Set values from Scan Summary to Package
\ No newline at end of file diff --git a/component_catalog/templates/component_catalog/includes/scan_to_package_modal.html b/component_catalog/templates/component_catalog/includes/scan_to_package_modal.html index 9b68d722..5380dea9 100644 --- a/component_catalog/templates/component_catalog/includes/scan_to_package_modal.html +++ b/component_catalog/templates/component_catalog/includes/scan_to_package_modal.html @@ -20,7 +20,16 @@
Set values from Scan to Package
\ No newline at end of file diff --git a/component_catalog/templates/component_catalog/tabs/tab_scan.html b/component_catalog/templates/component_catalog/tabs/tab_scan.html index a19f63dc..200fda71 100644 --- a/component_catalog/templates/component_catalog/tabs/tab_scan.html +++ b/component_catalog/templates/component_catalog/tabs/tab_scan.html @@ -17,7 +17,6 @@ NEXB.displayOverlay("Submitting Scan Request..."); }); - {% else %} {% include 'tabs/tab_content.html' %} {% if object.has_license_matches %} diff --git a/component_catalog/templates/component_catalog/tabs/tab_subcomponents.html b/component_catalog/templates/component_catalog/tabs/tab_subcomponents.html index dc97a864..d98f4857 100644 --- a/component_catalog/templates/component_catalog/tabs/tab_subcomponents.html +++ b/component_catalog/templates/component_catalog/tabs/tab_subcomponents.html @@ -34,7 +34,7 @@ {% endif %} {{ data.child.version }} - {{ data.child.owner }} + {{ data.child.owner|default_if_none:"" }} {{ data.subcomponent.purpose }} {% if data.subcomponent.license_expression %} @@ -50,7 +50,7 @@ {{ data.subcomponent.is_modified|as_icon }} {% if component.is_active %} -