Merge branch 'main' into 45-improve-from-purldb

.dockerignore

@@ -1,3 +1,4 @@
+**/.*
 **/.git
 **/.gitignore
 **/.github
@@ -9,6 +10,7 @@
 **/.DS_Store
 **/.aof
 **/venv
+**/.venv
 **/env
 **/bin
 # **/docs we want to keep dje/templates/rest_framework/docs/
@@ -22,6 +24,7 @@ docs/
 **/*.egg-info
 **/*.log
 **/__pycache__
+**/.*cache
 *.pyc
 .dockerignore
 .readthedocs.yaml
@@ -30,6 +33,8 @@ docker.env
 Makefile
 Dockerfile
 README.rst
+CHANGELOG.rst
+CONTRIBUTING.rst
+MANIFEST.in
 docker-compose.yml
-fabfile.py
 pyvenv.cfg

.github/workflows/ci.yml

@@ -42,7 +42,9 @@ jobs:
           python-version: "3.12"
 
       - name: Install python-ldap OS dependencies
-        run: sudo apt-get install -y libsasl2-dev libldap2-dev libssl-dev
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libldap2-dev libsasl2-dev
 
       - name: Install dependencies
         run: make dev envfile
@@ -60,4 +62,4 @@ jobs:
         run: make docs
 
       - name: Run tests
-        run: bin/python manage.py test --verbosity=2 --noinput --parallel
+        run: .venv/bin/python manage.py test --verbosity=2 --noinput --parallel

.gitignore

@@ -36,4 +36,5 @@ local
 /dist/
 /.cache/
 /.python-version
-/enterprise.egg-info/
+/.venv/
+/.*cache/

CHANGELOG.rst

@@ -54,6 +54,10 @@ Release notes
 - Use the declared_license_expression_spdx value in SPDX outputs.
   https://github.com/nexB/dejacode/issues/63
 
+- Add new ProductDependency model to support relating Packages in the context of a
+  Product.
+  https://github.com/nexB/dejacode/issues/138
+
 ### Version 5.1.0
 
 - Upgrade Python version to 3.12 and Django to 5.0.x

Dockerfile

@@ -15,7 +15,7 @@ LABEL org.opencontainers.image.licenses="AGPL-3.0-only"
 ENV APP_NAME dejacode
 ENV APP_USER app
 ENV APP_DIR /opt/$APP_NAME
-ENV VIRTUAL_ENV /opt/$APP_NAME/venv
+ENV VENV_LOCATION /opt/$APP_NAME/.venv
 
 # Force Python unbuffered stdout and stderr (they are flushed to terminal immediately)
 ENV PYTHONUNBUFFERED 1
@@ -51,9 +51,9 @@ USER $APP_USER
 RUN mkdir -p /var/$APP_NAME/static/ /var/$APP_NAME/media/
 
 # Create the virtualenv
-RUN python -m venv $VIRTUAL_ENV
+RUN python -m venv $VENV_LOCATION
 # Enable the virtualenv, similar effect as "source activate"
-ENV PATH $VIRTUAL_ENV/bin:$PATH
+ENV PATH $VENV_LOCATION/bin:$PATH
 
 # Install the dependencies before the codebase COPY for proper Docker layer caching
 COPY --chown=$APP_USER:$APP_USER setup.cfg setup.py $APP_DIR/

Makefile

@@ -7,16 +7,14 @@
 #
 
 PYTHON_EXE=python3.12
-MANAGE=bin/python manage.py
-ACTIVATE?=. bin/activate;
+VENV_LOCATION=.venv
+ACTIVATE?=. ${VENV_LOCATION}/bin/activate;
+MANAGE=${VENV_LOCATION}/bin/python manage.py
 PIP_ARGS=--find-links=./thirdparty/dist/ --no-index --no-cache-dir
 GET_SECRET_KEY=`cat /dev/urandom | head -c 50  | base64`
 # Customize with `$ make envfile ENV_FILE=/etc/dejacode/.env`
 ENV_FILE=.env
-FIXTURES_LOCATION=./dje/fixtures
 DOCS_LOCATION=./docs
-MODIFIED_PYTHON_FILES=`git ls-files -m "*.py"`
-BLACK_ARGS=--exclude="migrations|data|lib/|lib64|bin|var|dist|.cache" -l 100
 DOCKER_COMPOSE=docker compose -f docker-compose.yml
 DOCKER_EXEC=${DOCKER_COMPOSE} exec
 DB_NAME=dejacode_db
@@ -28,7 +26,7 @@ TIMESTAMP=$(shell date +"%Y-%m-%d_%H%M")
 
 virtualenv:
 	@echo "-> Bootstrap the virtualenv with PYTHON_EXE=${PYTHON_EXE}"
-	${PYTHON_EXE} -m venv .
+	${PYTHON_EXE} -m venv ${VENV_LOCATION}
 
 conf: virtualenv
 	@echo "-> Install dependencies"
@@ -46,53 +44,33 @@ envfile:
 	@mkdir -p $(shell dirname ${ENV_FILE}) && touch ${ENV_FILE}
 	@echo "SECRET_KEY=${GET_SECRET_KEY}" > ${ENV_FILE}
 
-isort:
-	@echo "-> Apply isort changes to ensure proper imports ordering"
-	@${ACTIVATE} isort .
-
-black:
-	@echo "-> Apply black code formatter"
-	@${ACTIVATE} black ${BLACK_ARGS} .
-
 doc8:
 	@echo "-> Run doc8 validation"
 	@${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ \
 	  --ignore-path docs/installation_and_sysadmin/ --quiet docs/
 
-valid: isort black doc8 check
-
-bandit:
-	@echo "-> Run source code security analyzer"
-	@${ACTIVATE} pip install bandit
-	@${ACTIVATE} bandit --recursive . \
-	  --exclude ./bin,./data,./dist,./docs,./include,./lib,./share,./thirdparty,./var,tests \
-	  --quiet
-
-check: doc8 bandit
-	@echo "-> Run flake8 (pycodestyle, pyflakes, mccabe) validation"
-	@${ACTIVATE} flake8 .
-	@echo "-> Run isort imports ordering validation"
-	@${ACTIVATE} isort --check-only .
-	@echo "-> Run black validation"
-	@${ACTIVATE} black --check ${BLACK_ARGS} .
+valid:
+	@echo "-> Run Ruff format"
+	@${ACTIVATE} ruff format
+	@echo "-> Run Ruff linter"
+	@${ACTIVATE} ruff check --fix
+
+check:
+	@echo "-> Run Ruff linter validation (pycodestyle, bandit, isort, and more)"
+	@${ACTIVATE} ruff check
+	@echo "-> Run Ruff format validation"
+	@${ACTIVATE} ruff format --check
 	@echo "-> Running ABOUT files validation"
 	@${ACTIVATE} about check ./thirdparty/
-	@$(MAKE) check-docstrings
-
-check-docstrings:
-	@echo "-> Run docstring validation"
-	@${ACTIVATE} pip install pydocstyle
-	@${ACTIVATE} pydocstyle component_catalog dejacode dejacode_toolkit dje \
-	  license_library notification organization policy product_portfolio purldb \
-	  reporting workflow
+	@$(MAKE) doc8
 
 check-deploy:
 	@echo "-> Check Django deployment settings"
 	${MANAGE} check --deploy
 
 clean:
 	@echo "-> Cleaning the Python env"
-	rm -rf bin/ lib/ lib64/ include/ build/ dist/ share/ pip-selfcheck.json pyvenv.cfg
+	rm -rf .venv/ .*_cache/ *.egg-info/ build/ dist/
 	find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete
 
 initdb:
@@ -165,4 +143,4 @@ log:
 createsuperuser:
 	${DOCKER_EXEC} web ./manage.py createsuperuser
 
-.PHONY: virtualenv conf dev envfile check bandit isort black doc8 valid check-docstrings check-deploy clean initdb postgresdb migrate run test docs build psql bash shell log createsuperuser
+.PHONY: virtualenv conf dev envfile check doc8 valid check-deploy clean initdb postgresdb migrate run test docs build psql bash shell log createsuperuser

component_catalog/filters.py

@@ -159,13 +159,22 @@ def filter(self, qs, value):
 class PackageFilterSet(DataspacedFilterSet):
     q = PackageSearchFilter(
         label=_("Search"),
-        match_order_fields=["filename"],
-        search_fields=[
+        match_order_fields=[
+            "type",
+            "namespace",
+            "name",
+            "version",
             "filename",
+            "download_url",
+            "sha1",
+            "md5",
+        ],
+        search_fields=[
             "type",
             "namespace",
             "name",
             "version",
+            "filename",
             "download_url",
             "sha1",
             "md5",

component_catalog/forms.py

@@ -1063,6 +1063,8 @@ class Meta:
             "copyright",
             "holder",
             "license_expression",
+            "declared_license_expression",
+            "other_license_expression",
             "reference_notes",
             "description",
             "homepage_url",
@@ -1140,6 +1142,8 @@ class Meta:
             "holder",
             "author",
             "license_expression",
+            "declared_license_expression",
+            "other_license_expression",
             "reference_notes",
             "usage_policy",
             "homepage_url",

component_catalog/license_expression_dje.py

@@ -14,6 +14,7 @@
 from django.forms import widgets
 from django.urls import reverse
 from django.utils.html import format_html
+from django.utils.safestring import mark_safe
 
 from boolean.boolean import PARSE_ERRORS
 from license_expression import ExpressionError
@@ -432,7 +433,10 @@ def render_expression_as_html(expression, dataspace):
     licensing = get_dataspace_licensing(dataspace)
 
     formatted_expression = get_formatted_expression(licensing, expression, show_policy)
-    return format_html(formatted_expression)
+    return format_html(
+        '<span class="license-expression">{}</span>',
+        mark_safe(formatted_expression),
+    )
 
 
 def get_expression_as_spdx(expression, dataspace):

component_catalog/management/commands/collectpackagesdata.py

@@ -13,40 +13,42 @@
 
 
 class Command(DataspacedCommand):
-    help = ('Collects and saves md5, sha1, and size values where one of those '
-            'are missing in the given Dataspace on Package instances.')
+    help = (
+        "Collects and saves md5, sha1, and size values where one of those "
+        "are missing in the given Dataspace on Package instances."
+    )
 
     def add_arguments(self, parser):
         super().add_arguments(parser)
         parser.add_argument(
-            '--save',
-            action='store_true',
-            dest='save',
+            "--save",
+            action="store_true",
+            dest="save",
             default=False,
-            help='Use save() in place of update() (default) to trigger all '
-                 'associated logic and signals. Fields such as last_modified_date '
-                 'will be updated.',
+            help="Use save() in place of update() (default) to trigger all "
+            "associated logic and signals. Fields such as last_modified_date "
+            "will be updated.",
         )
 
     def handle(self, *args, **options):
         super().handle(*args, **options)
 
         packages = (
             Package.objects.scope(self.dataspace)
-            .exclude(download_url='')
-            .filter(Q(md5='') | Q(sha1='') | Q(size__isnull=True))
+            .exclude(download_url="")
+            .filter(Q(md5="") | Q(sha1="") | Q(size__isnull=True))
         )
 
-        self.stdout.write(f'{packages.count()} Packages in the queue.')
+        self.stdout.write(f"{packages.count()} Packages in the queue.")
 
         update_count = 0
         for package in packages:
-            self.stdout.write(f'Collecting: {package.download_url}')
+            self.stdout.write(f"Collecting: {package.download_url}")
             update_fields = package.collect_data(save=False)
             if not update_fields:
                 continue
 
-            if options['save']:
+            if options["save"]:
                 package.save()
             else:
                 Package.objects.filter(pk=package.pk).update(
@@ -56,5 +58,5 @@ def handle(self, *args, **options):
             self.stdout.write(f"{', '.join(update_fields)} updated")
             update_count += 1
 
-        msg = f'{update_count} Package(s) updated.'
+        msg = f"{update_count} Package(s) updated."
         self.stdout.write(self.style.SUCCESS(msg))

component_catalog/models.py

@@ -48,6 +48,7 @@
 from component_catalog.license_expression_dje import get_expression_as_spdx
 from component_catalog.license_expression_dje import get_license_objects
 from component_catalog.license_expression_dje import parse_expression
+from component_catalog.license_expression_dje import render_expression_as_html
 from dejacode_toolkit import spdx
 from dejacode_toolkit.download import DataCollectionException
 from dejacode_toolkit.download import collect_package_data
@@ -228,6 +229,11 @@ def get_expression_as_spdx(self, expression):
     def concluded_license_expression_spdx(self):
         return self.get_expression_as_spdx(self.license_expression)
 
+    @property
+    def license_expression_html(self):
+        if self.license_expression:
+            return render_expression_as_html(self.license_expression, self.dataspace)
+
     def save(self, *args, **kwargs):
         """
         Call the handle_assigned_licenses method on save, except during copy.
@@ -1627,6 +1633,30 @@ def annotate_sortable_identifier(self):
             sortable_identifier=Concat(*PACKAGE_URL_FIELDS, "filename", output_field=CharField())
         )
 
+    def only_rendering_fields(self):
+        """Minimum requirements to render a Package element in the UI."""
+        return self.only(
+            "uuid",
+            *PACKAGE_URL_FIELDS,
+            "filename",
+            "license_expression",
+            "dataspace__name",
+            "dataspace__show_usage_policy_in_user_views",
+        )
+
+    def declared_dependencies_count(self, product):
+        """
+        Annotate the QuerySet with this each Package declared_dependencies count.
+        A ``product`` context need to be provided to get the proper counts as
+        dependencies are always scoped to a Product.
+        """
+        return self.annotate(
+            declared_dependencies_count=models.Count(
+                "declared_dependencies",
+                filter=models.Q(declared_dependencies__product=product),
+            )
+        )
+
 
 class Package(
     ExternalReferenceMixin,

component_catalog/templates/component_catalog/includes/hierarchy_instance_box.html

@@ -28,6 +28,12 @@
             <button type="button" data-bs-toggle="tooltip" title="Edit" class="btn btn-link p-0" aria-label="Edit object"><i class="far fa-edit fa-sm"></i></button>
           </span>
         {% endif %}
+        {% if relation.package_id and relation.package.declared_dependencies.all %}
+          <a class="btn badge text-bg-primary rounded-pill ms-1"
+             href="{{ product.get_absolute_url }}?dependencies-for_package__uuid={{ relation.package.uuid }}#dependencies" class="ms-1" data-bs-toggle="tooltip" title="Dependencies" aria-label="Dependencies">
+            {{ relation.package.declared_dependencies.all|length }}<i class="fa-solid fa-share-nodes ms-1"></i>
+          </a>
+        {% endif %}
       {% elif instance.is_active or is_product %}
         <a href="{{ instance.get_absolute_url }}#hierarchy">{{ instance }}</a>
         {% if relation.component_id and has_edit_productcomponent %}