diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 356eba1e..7f422676 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -97,7 +97,7 @@ if [ "$TEST" = "s3" ]; then sed -i -e '$a s3_test: true\ minio_access_key: "'$MINIO_ACCESS_KEY'"\ minio_secret_key: "'$MINIO_SECRET_KEY'"\ -pulp_scenario_settings: null\ +pulp_scenario_settings: {"domain_enabled": true}\ pulp_scenario_env: {}\ ' vars/main.yaml export PULP_API_ROOT="/rerouted/djnd/" @@ -111,7 +111,7 @@ if [ "$TEST" = "azure" ]; then - ./azurite:/etc/pulp\ command: "azurite-blob --blobHost 0.0.0.0"' vars/main.yaml sed -i -e '$a azure_test: true\ -pulp_scenario_settings: null\ +pulp_scenario_settings: {"domain_enabled": true}\ pulp_scenario_env: {}\ ' vars/main.yaml fi diff --git a/CHANGES/668.feature b/CHANGES/668.feature new file mode 100644 index 00000000..b59531ad --- /dev/null +++ b/CHANGES/668.feature @@ -0,0 +1 @@ +Added Domain support. diff --git a/docs/tech-preview.rst b/docs/tech-preview.rst index 0ba3d9f0..3204611a 100644 --- a/docs/tech-preview.rst +++ b/docs/tech-preview.rst @@ -9,3 +9,4 @@ The following features are currently being released as part of a tech preview * Fully mirror Python repositories provided PyPI and Pulp itself. * ``Twine`` upload packages to indexes at endpoints '/simple` or '/legacy'. * Create pull-through caches of remote sources. +* Pulp Domain support diff --git a/pulp_python/app/__init__.py b/pulp_python/app/__init__.py index d4f0f3de..11f5af69 100644 --- a/pulp_python/app/__init__.py +++ b/pulp_python/app/__init__.py @@ -10,3 +10,4 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig): label = "python" version = "3.12.0.dev" python_package_name = "pulp-python" + domain_compatible = True diff --git a/pulp_python/app/migrations/0012_add_domain.py b/pulp_python/app/migrations/0012_add_domain.py new file mode 100644 index 00000000..aefa1c0f --- /dev/null +++ b/pulp_python/app/migrations/0012_add_domain.py @@ -0,0 +1,37 @@ +# Generated by Django 4.2.10 on 2024-05-30 17:53 + +from django.db import migrations, models +import django.db.models.deletion +import pulpcore.app.util + + +class Migration(migrations.Migration): + + dependencies = [ + ("python", "0011_alter_pythondistribution_distribution_ptr_and_more"), + ] + + operations = [ + migrations.AlterUniqueTogether( + name="pythonpackagecontent", + unique_together=set(), + ), + migrations.AddField( + model_name="pythonpackagecontent", + name="_pulp_domain", + field=models.ForeignKey( + default=pulpcore.app.util.get_domain_pk, + on_delete=django.db.models.deletion.PROTECT, + to="core.domain", + ), + ), + migrations.AlterField( + model_name="pythonpackagecontent", + name="sha256", + field=models.CharField(db_index=True, max_length=64), + ), + migrations.AlterUniqueTogether( + name="pythonpackagecontent", + unique_together={("sha256", "_pulp_domain")}, + ), + ] diff --git a/pulp_python/app/modelresource.py b/pulp_python/app/modelresource.py index 7fd75bd0..6533c881 100644 --- a/pulp_python/app/modelresource.py +++ b/pulp_python/app/modelresource.py @@ -1,5 +1,6 @@ from pulpcore.plugin.importexport import BaseContentResource from pulpcore.plugin.modelresources import RepositoryResource +from pulpcore.plugin.util import get_domain from pulp_python.app.models import ( PythonPackageContent, PythonRepository, @@ -15,7 +16,9 @@ def set_up_queryset(self): """ :return: PythonPackageContent specific to a specified repo-version. """ - return PythonPackageContent.objects.filter(pk__in=self.repo_version.content) + return PythonPackageContent.objects.filter( + pk__in=self.repo_version.content, _pulp_domain=get_domain() + ) class Meta: model = PythonPackageContent diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index 0e8b29fc..38c5948b 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -23,6 +23,7 @@ PYPI_SERIAL_CONSTANT, ) from pulpcore.plugin.repo_version_utils import remove_duplicates, validate_repo_version +from pulpcore.plugin.util import get_domain_pk, get_domain log = getLogger(__name__) @@ -68,6 +69,7 @@ def content_handler(self, path): path = PurePath(path) name = None version = None + domain = get_domain() if path.match("pypi/*/*/json"): version = path.parts[2] name = path.parts[1] @@ -76,7 +78,7 @@ def content_handler(self, path): elif len(path.parts) and path.parts[0] == "simple": # Temporary fix for PublishedMetadata not being properly served from remote storage # https://github.com/pulp/pulp_python/issues/413 - if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + if domain.storage_class != "pulpcore.app.models.storage.FileSystem": if self.publication or self.repository: try: publication = self.publication or Publication.objects.filter( @@ -105,7 +107,11 @@ def content_handler(self, path): ) # TODO Change this value to the Repo's serial value when implemented headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT)} - json_body = python_content_to_json(self.base_path, package_content, version=version) + if not settings.DOMAIN_ENABLED: + domain = None + json_body = python_content_to_json( + self.base_path, package_content, version=version, domain=domain + ) if json_body: return json_response(json_body, headers=headers) @@ -143,7 +149,7 @@ class PythonPackageContent(Content): name = models.TextField() name.register_lookup(NormalizeName) version = models.TextField() - sha256 = models.CharField(unique=True, db_index=True, max_length=64) + sha256 = models.CharField(db_index=True, max_length=64) # Optional metadata python_version = models.TextField() metadata_version = models.TextField() @@ -168,6 +174,8 @@ class PythonPackageContent(Content): classifiers = models.JSONField(default=list) project_urls = models.JSONField(default=dict) description_content_type = models.TextField() + # Pulp Domains + _pulp_domain = models.ForeignKey("core.Domain", default=get_domain_pk, on_delete=models.PROTECT) @staticmethod def init_from_artifact_and_relative_path(artifact, relative_path): @@ -179,6 +187,8 @@ def init_from_artifact_and_relative_path(artifact, relative_path): data["version"] = metadata.version data["filename"] = path.name data["sha256"] = artifact.sha256 + data["pulp_domain_id"] = artifact.pulp_domain_id + data["_pulp_domain_id"] = artifact.pulp_domain_id return PythonPackageContent(**data) def __str__(self): @@ -200,7 +210,7 @@ def __str__(self): class Meta: default_related_name = "%(app_label)s_%(model_name)s" - unique_together = ("sha256",) + unique_together = ("sha256", "_pulp_domain") class PythonPublication(Publication): diff --git a/pulp_python/app/pypi/serializers.py b/pulp_python/app/pypi/serializers.py index d5f6d607..f296a38b 100644 --- a/pulp_python/app/pypi/serializers.py +++ b/pulp_python/app/pypi/serializers.py @@ -4,6 +4,7 @@ from rest_framework import serializers from pulp_python.app.utils import DIST_EXTENSIONS from pulpcore.plugin.models import Artifact +from pulpcore.plugin.util import get_domain from django.db.utils import IntegrityError log = logging.getLogger(__name__) @@ -76,7 +77,7 @@ def validate(self, data): try: artifact.save() except IntegrityError: - artifact = Artifact.objects.get(sha256=artifact.sha256) + artifact = Artifact.objects.get(sha256=artifact.sha256, pulp_domain=get_domain()) artifact.touch() log.info(f"Artifact for {file.name} already existed in database") data["content"] = (artifact, file.name) diff --git a/pulp_python/app/pypi/views.py b/pulp_python/app/pypi/views.py index ab276558..b094a0c5 100644 --- a/pulp_python/app/pypi/views.py +++ b/pulp_python/app/pypi/views.py @@ -26,7 +26,9 @@ from pathlib import PurePath from pypi_simple.parse_stream import parse_links_stream_response +from pulpcore.plugin.viewsets import OperationPostponedResponse from pulpcore.plugin.tasking import dispatch +from pulpcore.plugin.util import get_domain from pulp_python.app.models import ( PythonDistribution, PythonPackageContent, @@ -75,7 +77,7 @@ def get_distribution(path): "repository", "publication", "publication__repository_version", "remote" ) try: - return distro_qs.get(base_path=path) + return distro_qs.get(base_path=path, pulp_domain=get_domain()) except ObjectDoesNotExist: raise Http404(f"No PythonDistribution found for base_path {path}") @@ -100,6 +102,14 @@ def get_drvc(self, path): content = self.get_content(repo_ver) return distro, repo_ver, content + def initial(self, request, *args, **kwargs): + """Perform common initialization tasks for PyPI endpoints.""" + super().initial(request, *args, **kwargs) + if settings.DOMAIN_ENABLED: + self.base_content_url = urljoin(BASE_CONTENT_URL, f"{get_domain().name}/") + else: + self.base_content_url = BASE_CONTENT_URL + class PackageUploadMixin(PyPIMixin): """A Mixin to provide package upload support.""" @@ -137,7 +147,7 @@ def upload(self, request, path): kwargs={"artifact_sha256": artifact.sha256, "filename": filename, "repository_pk": str(repo.pk)}) - return Response(data={"task": reverse('tasks-detail', args=[result.pk], request=None)}) + return OperationPostponedResponse(result, request) def upload_package_group(self, repo, artifact, filename, session): """Steps 4 & 5, spawns tasks to add packages to index.""" @@ -176,7 +186,7 @@ def create_group_upload_task(self, cur_session, repository, artifact, filename, return reverse('tasks-detail', args=[result.pk], request=None) -class SimpleView(ViewSet, PackageUploadMixin): +class SimpleView(PackageUploadMixin, ViewSet): """View for the PyPI simple API.""" permission_classes = [IsAuthenticatedOrReadOnly] @@ -186,7 +196,7 @@ def list(self, request, path): """Gets the simple api html page for the index.""" distro, repo_version, content = self.get_drvc(path) if self.should_redirect(distro, repo_version=repo_version): - return redirect(urljoin(BASE_CONTENT_URL, f'{path}/simple/')) + return redirect(urljoin(self.base_content_url, f'{path}/simple/')) names = content.order_by('name').values_list('name', flat=True).distinct().iterator() return StreamingHttpResponse(write_simple_index(names, streamed=True)) @@ -197,7 +207,7 @@ def parse_url(link): digest, _, value = parsed.fragment.partition('=') stripped_url = urlunsplit(chain(parsed[:3], ("", ""))) redirect = f'{path}/{link.text}?redirect={stripped_url}' - d_url = urljoin(BASE_CONTENT_URL, redirect) + d_url = urljoin(self.base_content_url, redirect) return link.text, d_url, value if digest == 'sha256' else '' url = remote.get_remote_artifact_url(f'simple/{package}/') @@ -224,7 +234,7 @@ def retrieve(self, request, path, package): if not repo_ver or not content.filter(name__normalize=normalized).exists(): return self.pull_through_package_simple(normalized, path, distro.remote) if self.should_redirect(distro, repo_version=repo_ver): - return redirect(urljoin(BASE_CONTENT_URL, f'{path}/simple/{normalized}/')) + return redirect(urljoin(self.base_content_url, f'{path}/simple/{normalized}/')) packages = ( content.filter(name__normalize=normalized) .values_list('filename', 'sha256', 'name') @@ -237,7 +247,7 @@ def retrieve(self, request, path, package): else: packages = chain([present], packages) name = present[2] - releases = ((f, urljoin(BASE_CONTENT_URL, f'{path}/{f}'), d) for f, d, _ in packages) + releases = ((f, urljoin(self.base_content_url, f'{path}/{f}'), d) for f, d, _ in packages) return StreamingHttpResponse(write_simple_detail(name, releases, streamed=True)) @extend_schema(request=PackageUploadSerializer, @@ -253,7 +263,7 @@ def create(self, request, path): return self.upload(request, path) -class MetadataView(ViewSet, PyPIMixin): +class MetadataView(PyPIMixin, ViewSet): """View for the PyPI JSON metadata endpoint.""" authentication_classes = [] @@ -272,6 +282,7 @@ def retrieve(self, request, path, meta): meta_path = PurePath(meta) name = None version = None + domain = None if meta_path.match("*/*/json"): version = meta_path.parts[1] name = meta_path.parts[0] @@ -281,13 +292,17 @@ def retrieve(self, request, path, meta): package_content = content.filter(name__iexact=name) # TODO Change this value to the Repo's serial value when implemented headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT)} - json_body = python_content_to_json(path, package_content, version=version) + if settings.DOMAIN_ENABLED: + domain = get_domain() + json_body = python_content_to_json( + path, package_content, version=version, domain=domain + ) if json_body: return Response(data=json_body, headers=headers) return Response(status="404") -class PyPIView(ViewSet, PyPIMixin): +class PyPIView(PyPIMixin, ViewSet): """View for base_url of distribution.""" authentication_classes = [] @@ -305,7 +320,7 @@ def retrieve(self, request, path): return Response(data=data) -class UploadView(ViewSet, PackageUploadMixin): +class UploadView(PackageUploadMixin, ViewSet): """View for the `/legacy` upload endpoint.""" @extend_schema(request=PackageUploadSerializer, diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index 2573a1e1..ff789b53 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -5,6 +5,7 @@ from pulpcore.plugin import models as core_models from pulpcore.plugin import serializers as core_serializers +from pulpcore.plugin.util import get_domain from pulp_python.app import models as python_models from pulp_python.app.utils import get_project_metadata_from_artifact, parse_project_metadata @@ -56,6 +57,8 @@ class PythonDistributionSerializer(core_serializers.DistributionSerializer): def get_base_url(self, obj): """Gets the base url.""" + if settings.DOMAIN_ENABLED: + return f"{settings.PYPI_API_HOSTNAME}/pypi/{get_domain().name}/{obj.base_path}/" return f"{settings.PYPI_API_HOSTNAME}/pypi/{obj.base_path}/" class Meta: @@ -232,13 +235,17 @@ def deferred_validate(self, data): _data['version'] = metadata.version _data['filename'] = filename _data['sha256'] = artifact.sha256 + data["pulp_domain_id"] = artifact.pulp_domain_id + data["_pulp_domain_id"] = artifact.pulp_domain_id data.update(_data) return data def retrieve(self, validated_data): - content = python_models.PythonPackageContent.objects.filter(sha256=validated_data["sha256"]) + content = python_models.PythonPackageContent.objects.filter( + sha256=validated_data["sha256"], _pulp_domain=get_domain() + ) return content.first() class Meta: diff --git a/pulp_python/app/tasks/publish.py b/pulp_python/app/tasks/publish.py index c4fb0478..136b511f 100644 --- a/pulp_python/app/tasks/publish.py +++ b/pulp_python/app/tasks/publish.py @@ -6,6 +6,7 @@ from packaging.utils import canonicalize_name from pulpcore.plugin import models +from pulpcore.plugin.util import get_domain from pulp_python.app import models as python_models from pulp_python.app.utils import write_simple_index, write_simple_detail @@ -49,11 +50,12 @@ def write_simple_api(publication): publication (pulpcore.plugin.models.Publication): A publication to generate metadata for """ + domain = get_domain() simple_dir = 'simple/' os.mkdir(simple_dir) project_names = ( python_models.PythonPackageContent.objects.filter( - pk__in=publication.repository_version.content + pk__in=publication.repository_version.content, _pulp_domain=domain ) .order_by('name') .values_list('name', flat=True) @@ -76,7 +78,7 @@ def write_simple_api(publication): return packages = python_models.PythonPackageContent.objects.filter( - pk__in=publication.repository_version.content + pk__in=publication.repository_version.content, _pulp_domain=domain ) releases = packages.order_by("name").values("name", "filename", "sha256") diff --git a/pulp_python/app/tasks/upload.py b/pulp_python/app/tasks/upload.py index de9e89ba..9f72c364 100644 --- a/pulp_python/app/tasks/upload.py +++ b/pulp_python/app/tasks/upload.py @@ -4,6 +4,7 @@ from django.db import transaction from django.contrib.sessions.models import Session from pulpcore.plugin.models import Artifact, CreatedResource, ContentArtifact +from pulpcore.plugin.util import get_domain from pulp_python.app.models import PythonPackageContent, PythonRepository from pulp_python.app.utils import get_project_metadata_from_artifact, parse_project_metadata @@ -18,8 +19,9 @@ def upload(artifact_sha256, filename, repository_pk=None): filename: the full filename of the package to create repository_pk: the optional pk of the repository to add the content to """ - pre_check = PythonPackageContent.objects.filter(sha256=artifact_sha256) - content_to_add = pre_check or create_content(artifact_sha256, filename) + domain = get_domain() + pre_check = PythonPackageContent.objects.filter(sha256=artifact_sha256, _pulp_domain=domain) + content_to_add = pre_check or create_content(artifact_sha256, filename, domain) content_to_add.get().touch() if repository_pk: repository = PythonRepository.objects.get(pk=repository_pk) @@ -36,6 +38,7 @@ def upload_group(session_pk, repository_pk=None): repository_pk: optional repository to add Content to """ s_query = Session.objects.select_for_update().filter(pk=session_pk) + domain = get_domain() while True: with transaction.atomic(): session_data = s_query.first().get_decoded() @@ -44,8 +47,10 @@ def upload_group(session_pk, repository_pk=None): if now >= start_time: content_to_add = PythonPackageContent.objects.none() for artifact_sha256, filename in session_data['artifacts']: - pre_check = PythonPackageContent.objects.filter(sha256=artifact_sha256) - content = pre_check or create_content(artifact_sha256, filename) + pre_check = PythonPackageContent.objects.filter( + sha256=artifact_sha256, _pulp_domain=domain + ) + content = pre_check or create_content(artifact_sha256, filename, domain) content.get().touch() content_to_add |= content @@ -59,17 +64,18 @@ def upload_group(session_pk, repository_pk=None): time.sleep(sleep_time.seconds) -def create_content(artifact_sha256, filename): +def create_content(artifact_sha256, filename, domain): """ Creates PythonPackageContent from artifact. Args: artifact_sha256: validated artifact filename: file name + domain: the pulp_domain to perform this task in Returns: queryset of the new created content """ - artifact = Artifact.objects.get(sha256=artifact_sha256) + artifact = Artifact.objects.get(sha256=artifact_sha256, pulp_domain=domain) metadata = get_project_metadata_from_artifact(filename, artifact) data = parse_project_metadata(vars(metadata)) @@ -77,6 +83,8 @@ def create_content(artifact_sha256, filename): data['version'] = metadata.version data['filename'] = filename data['sha256'] = artifact.sha256 + data['pulp_domain'] = domain + data['_pulp_domain'] = domain @transaction.atomic() def create(): diff --git a/pulp_python/app/urls.py b/pulp_python/app/urls.py index d6a51378..0a786333 100644 --- a/pulp_python/app/urls.py +++ b/pulp_python/app/urls.py @@ -1,8 +1,12 @@ +from django.conf import settings from django.urls import path from pulp_python.app.pypi.views import SimpleView, MetadataView, PyPIView, UploadView -PYPI_API_URL = 'pypi//' +if settings.DOMAIN_ENABLED: + PYPI_API_URL = "pypi///" +else: + PYPI_API_URL = "pypi//" # TODO: Implement remaining PyPI endpoints # path("project/", PackageProject.as_view()), # Endpoints to nicely see contents of index # path("search/", PackageSearch.as_view()), diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index eb1082de..f66dcbd7 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -3,7 +3,6 @@ import tempfile import json from collections import defaultdict -from django.core.files.storage import default_storage as storage from django.conf import settings from jinja2 import Template from packaging.utils import canonicalize_name @@ -144,15 +143,14 @@ def get_project_metadata_from_artifact(filename, artifact): # because pkginfo validates that the filename has a valid extension before # reading it with tempfile.NamedTemporaryFile('wb', dir=".", suffix=filename) as temp_file: - artifact_file = storage.open(artifact.file.name) - shutil.copyfileobj(artifact_file, temp_file) + shutil.copyfileobj(artifact.file, temp_file) temp_file.flush() metadata = DIST_TYPES[packagetype](temp_file.name) metadata.packagetype = packagetype return metadata -def python_content_to_json(base_path, content_query, version=None): +def python_content_to_json(base_path, content_query, version=None, domain=None): """ Converts a QuerySet of PythonPackageContent into the PyPi JSON format https://www.python.org/dev/peps/pep-0566/ @@ -169,8 +167,8 @@ def python_content_to_json(base_path, content_query, version=None): if not latest_content: return None full_metadata.update({"info": python_content_to_info(latest_content[0])}) - full_metadata.update({"releases": python_content_to_releases(content_query, base_path)}) - full_metadata.update({"urls": python_content_to_urls(latest_content, base_path)}) + full_metadata.update({"releases": python_content_to_releases(content_query, base_path, domain)}) + full_metadata.update({"urls": python_content_to_urls(latest_content, base_path, domain)}) return full_metadata @@ -245,25 +243,27 @@ def python_content_to_info(content): } -def python_content_to_releases(content_query, base_path): +def python_content_to_releases(content_query, base_path, domain=None): """ Takes a QuerySet of PythonPackageContent and returns a dictionary of releases with each key being a version and value being a list of content for that version of the package """ releases = defaultdict(lambda: []) for content in content_query: - releases[content.version].append(python_content_to_download_info(content, base_path)) + releases[content.version].append( + python_content_to_download_info(content, base_path, domain) + ) return releases -def python_content_to_urls(contents, base_path): +def python_content_to_urls(contents, base_path, domain=None): """ Takes the latest content in contents and returns a list of download information """ - return [python_content_to_download_info(content, base_path) for content in contents] + return [python_content_to_download_info(content, base_path, domain) for content in contents] -def python_content_to_download_info(content, base_path): +def python_content_to_download_info(content, base_path, domain=None): """ Takes in a PythonPackageContent and base path of the distribution to create a dictionary of download information for that content. This dictionary is used by Releases and Urls. @@ -280,7 +280,10 @@ def find_artifact(): origin = settings.CONTENT_ORIGIN.strip("/") prefix = settings.CONTENT_PATH_PREFIX.strip("/") base_path = base_path.strip("/") - url = "/".join((origin, prefix, base_path, content.filename)) + components = [origin, prefix, base_path, content.filename] + if domain: + components.insert(2, domain.name) + url = "/".join(components) return { "comment_text": "", "digests": {"md5": artifact.md5, "sha256": artifact.sha256}, diff --git a/pulp_python/pytest_plugin.py b/pulp_python/pytest_plugin.py index 4657bea5..f04a7d4b 100644 --- a/pulp_python/pytest_plugin.py +++ b/pulp_python/pytest_plugin.py @@ -35,11 +35,14 @@ def python_bindings(_api_client_set, bindings_cfg): @pytest.fixture def python_repo_factory(python_bindings, gen_object_with_cleanup): """A factory to generate a Python Repository with auto-cleanup.""" - def _gen_python_repo(remote=None, **body): + def _gen_python_repo(remote=None, pulp_domain=None, **body): body.setdefault("name", str(uuid.uuid4())) + kwargs = {} + if pulp_domain: + kwargs["pulp_domain"] = pulp_domain if remote: body["remote"] = remote if isinstance(remote, str) else remote.pulp_href - return gen_object_with_cleanup(python_bindings.RepositoriesPythonApi, body) + return gen_object_with_cleanup(python_bindings.RepositoriesPythonApi, body, **kwargs) return _gen_python_repo @@ -53,7 +56,9 @@ def python_repo(python_repo_factory): @pytest.fixture def python_distribution_factory(python_bindings, gen_object_with_cleanup): """A factory to generate a Python Distribution with auto-cleanup.""" - def _gen_python_distribution(publication=None, repository=None, version=None, **body): + def _gen_python_distribution( + publication=None, repository=None, version=None, pulp_domain=None, **body + ): name = str(uuid.uuid4()) body.setdefault("name", name) body.setdefault("base_path", name) @@ -69,7 +74,10 @@ def _gen_python_distribution(publication=None, repository=None, version=None, ** body = {"repository_version": ver_href} else: body["repository"] = repo_href - return gen_object_with_cleanup(python_bindings.DistributionsPypiApi, body) + kwargs = {} + if pulp_domain: + kwargs["pulp_domain"] = pulp_domain + return gen_object_with_cleanup(python_bindings.DistributionsPypiApi, body, **kwargs) yield _gen_python_distribution @@ -77,7 +85,7 @@ def _gen_python_distribution(publication=None, repository=None, version=None, ** @pytest.fixture def python_publication_factory(python_bindings, gen_object_with_cleanup): """A factory to generate a Python Publication with auto-cleanup.""" - def _gen_python_publication(repository, version=None): + def _gen_python_publication(repository, version=None, pulp_domain=None): repo_href = get_href(repository) if version: if version.isnumeric(): @@ -87,7 +95,10 @@ def _gen_python_publication(repository, version=None): body = {"repository_version": ver_href} else: body = {"repository": repo_href} - return gen_object_with_cleanup(python_bindings.PublicationsPypiApi, body) + kwargs = {} + if pulp_domain: + kwargs["pulp_domain"] = pulp_domain + return gen_object_with_cleanup(python_bindings.PublicationsPypiApi, body, **kwargs) yield _gen_python_publication @@ -95,13 +106,16 @@ def _gen_python_publication(repository, version=None): @pytest.fixture def python_remote_factory(python_bindings, gen_object_with_cleanup): """A factory to generate a Python Remote with auto-cleanup.""" - def _gen_python_remote(url=PYTHON_FIXTURE_URL, includes=None, **body): + def _gen_python_remote(url=PYTHON_FIXTURE_URL, includes=None, pulp_domain=None, **body): body.setdefault("name", str(uuid.uuid4())) body.setdefault("url", url) if includes is None: includes = PYTHON_XS_PROJECT_SPECIFIER body["includes"] = includes - return gen_object_with_cleanup(python_bindings.RemotesPythonApi, body) + kwargs = {} + if pulp_domain: + kwargs["pulp_domain"] = pulp_domain + return gen_object_with_cleanup(python_bindings.RemotesPythonApi, body, **kwargs) yield _gen_python_remote @@ -112,7 +126,10 @@ def python_repo_with_sync( ): """A factory to generate a Python Repository synced with the passed in Remote.""" def _gen_python_repo_sync(remote=None, mirror=False, repository=None, **body): - remote = remote or python_remote_factory() + kwargs = {} + if pulp_domain := body.get("pulp_domain"): + kwargs["pulp_domain"] = pulp_domain + remote = remote or python_remote_factory(**kwargs) repo = repository or python_repo_factory(**body) sync_body = {"mirror": mirror, "remote": remote.pulp_href} monitor_task(python_bindings.RepositoriesPythonApi.sync(repo.pulp_href, sync_body).task) @@ -190,5 +207,5 @@ def _gen_summary(repository_version=None, repository=None, version=None): def get_href(item): - """Tries to get the href from the given item, wether it is a string or object.""" + """Tries to get the href from the given item, whether it is a string or object.""" return item if isinstance(item, str) else item.pulp_href diff --git a/pulp_python/tests/functional/api/test_consume_content.py b/pulp_python/tests/functional/api/test_consume_content.py index 46aea008..9c153cd1 100644 --- a/pulp_python/tests/functional/api/test_consume_content.py +++ b/pulp_python/tests/functional/api/test_consume_content.py @@ -24,8 +24,6 @@ def test_pip_consume_content( "--force-reinstall", "--trusted-host", urlsplit(distro.base_url).hostname, - "--trusted-host", - "ci-azurite", "-i", distro.base_url + "simple/", "shelf-reader", diff --git a/pulp_python/tests/functional/api/test_domains.py b/pulp_python/tests/functional/api/test_domains.py new file mode 100644 index 00000000..74316dec --- /dev/null +++ b/pulp_python/tests/functional/api/test_domains.py @@ -0,0 +1,264 @@ +import pytest +import uuid +import json +import subprocess + +from pulpcore.app import settings + +from pulp_python.tests.functional.constants import PYTHON_URL, PYTHON_EGG_FILENAME +from urllib.parse import urlsplit + + +pytestmark = pytest.mark.skipif(not settings.DOMAIN_ENABLED, reason="Domain not enabled") + + +@pytest.mark.parallel +def test_domain_object_creation( + domain_factory, + python_bindings, + python_repo_factory, + python_remote_factory, + python_distribution_factory, +): + """Test basic object creation in a separate domain.""" + domain = domain_factory() + domain_name = domain.name + + repo = python_repo_factory(pulp_domain=domain_name) + assert f"{domain_name}/api/v3/" in repo.pulp_href + + repos = python_bindings.RepositoriesPythonApi.list(pulp_domain=domain_name) + assert repos.count == 1 + assert repo.pulp_href == repos.results[0].pulp_href + + # Check that distribution's base_url reflects second domain's name + distro = python_distribution_factory(repository=repo.pulp_href, pulp_domain=domain_name) + assert distro.repository == repo.pulp_href + assert domain_name in distro.base_url + + # Will list repos on default domain + default_repos = python_bindings.RepositoriesPythonApi.list(name=repo.name) + assert default_repos.count == 0 + + # Try to create an object w/ cross domain relations + default_remote = python_remote_factory(policy="immediate") + with pytest.raises(python_bindings.ApiException) as e: + repo_body = {"name": str(uuid.uuid4()), "remote": default_remote.pulp_href} + python_bindings.RepositoriesPythonApi.create(repo_body, pulp_domain=domain.name) + assert e.value.status == 400 + assert json.loads(e.value.body) == { + "non_field_errors": [f"Objects must all be apart of the {domain_name} domain."] + } + + with pytest.raises(python_bindings.ApiException) as e: + sync_body = {"remote": default_remote.pulp_href} + python_bindings.RepositoriesPythonApi.sync(repo.pulp_href, sync_body) + assert e.value.status == 400 + assert json.loads(e.value.body) == { + "non_field_errors": [f"Objects must all be apart of the {domain_name} domain."] + } + + with pytest.raises(python_bindings.ApiException) as e: + publish_body = {"repository": repo.pulp_href} + python_bindings.PublicationsPypiApi.create(publish_body) + assert e.value.status == 400 + assert json.loads(e.value.body) == { + "non_field_errors": ["Objects must all be apart of the default domain."] + } + + with pytest.raises(python_bindings.ApiException) as e: + distro_body = { + "name": str(uuid.uuid4()), "base_path": str(uuid.uuid4()), "repository": repo.pulp_href + } + python_bindings.DistributionsPypiApi.create(distro_body) + assert e.value.status == 400 + assert json.loads(e.value.body) == { + "non_field_errors": ["Objects must all be apart of the default domain."] + } + + +@pytest.fixture +def python_file(tmp_path, http_get): + filename = tmp_path / PYTHON_EGG_FILENAME + with open(filename, mode="wb") as f: + f.write(http_get(PYTHON_URL)) + yield filename + + +@pytest.mark.parallel +def test_domain_content_upload( + domain_factory, + pulpcore_bindings, + python_bindings, + python_file, + monitor_task, +): + """Test uploading of file content with domains.""" + domain = domain_factory() + + content_body = {"relative_path": PYTHON_EGG_FILENAME, "file": python_file} + task = python_bindings.ContentPackagesApi.create(**content_body).task + response = monitor_task(task) + default_content = python_bindings.ContentPackagesApi.read(response.created_resources[0]) + default_artifact_href = default_content.artifact + + # Try to create content in second domain with default domain's artifact + with pytest.raises(python_bindings.ApiException) as e: + content_body = {"relative_path": PYTHON_EGG_FILENAME, "artifact": default_artifact_href} + python_bindings.ContentPackagesApi.create(**content_body, pulp_domain=domain.name) + assert e.value.status == 400 + assert json.loads(e.value.body) == { + "non_field_errors": [f"Objects must all be apart of the {domain.name} domain."] + } + + # Now create the same content in the second domain + content_body = {"relative_path": PYTHON_EGG_FILENAME, "file": python_file} + task2 = python_bindings.ContentPackagesApi.create(**content_body, pulp_domain=domain.name).task + response = monitor_task(task2) + domain_content = python_bindings.ContentPackagesApi.read(response.created_resources[0]) + domain_artifact_href = domain_content.artifact + + assert default_content.pulp_href != domain_content.pulp_href + assert default_artifact_href != domain_artifact_href + assert default_content.sha256 == domain_content.sha256 + assert default_content.filename == domain_content.filename + + domain_contents = python_bindings.ContentPackagesApi.list(pulp_domain=domain.name) + assert domain_contents.count == 1 + + # Content needs to be deleted for the domain to be deleted + body = {"orphan_protection_time": 0} + task = pulpcore_bindings.OrphansCleanupApi.cleanup(body, pulp_domain=domain.name).task + monitor_task(task) + + domain_contents = python_bindings.ContentPackagesApi.list(pulp_domain=domain.name) + assert domain_contents.count == 0 + + +@pytest.mark.parallel +def test_domain_content_replication( + domain_factory, + bindings_cfg, + pulp_settings, + pulpcore_bindings, + python_bindings, + python_file, + python_repo_factory, + python_publication_factory, + python_distribution_factory, + monitor_task, + monitor_task_group, + gen_object_with_cleanup, + add_to_cleanup, +): + """Test replication feature through the usage of domains.""" + # Set up source domain to replicate from + source_domain = domain_factory() + repo = python_repo_factory(pulp_domain=source_domain.name) + body = {"relative_path": PYTHON_EGG_FILENAME, "file": python_file, "repository": repo.pulp_href} + monitor_task( + python_bindings.ContentPackagesApi.create(pulp_domain=source_domain.name, **body).task + ) + pub = python_publication_factory(repository=repo, pulp_domain=source_domain.name) + python_distribution_factory(publication=pub.pulp_href, pulp_domain=source_domain.name) + + # Create the replica domain + replica_domain = domain_factory() + upstream_pulp_body = { + "name": str(uuid.uuid4()), + "base_url": bindings_cfg.host, + "api_root": pulp_settings.API_ROOT, + "domain": source_domain.name, + "username": bindings_cfg.username, + "password": bindings_cfg.password, + } + upstream_pulp = gen_object_with_cleanup( + pulpcore_bindings.UpstreamPulpsApi, upstream_pulp_body, pulp_domain=replica_domain.name + ) + # Run the replicate task and assert that all tasks successfully complete. + response = pulpcore_bindings.UpstreamPulpsApi.replicate(upstream_pulp.pulp_href) + monitor_task_group(response.task_group) + + counts = {} + for api_client in ( + python_bindings.ContentPackagesApi, + python_bindings.RepositoriesPythonApi, + python_bindings.RemotesPythonApi, + python_bindings.PublicationsPypiApi, + python_bindings.DistributionsPypiApi, + ): + result = api_client.list(pulp_domain=replica_domain.name) + counts[api_client] = result.count + for item in result.results: + add_to_cleanup(api_client, item.pulp_href) + + assert all(1 == x for x in counts.values()), f"Replica had more than 1 object {counts}" + + +@pytest.fixture +def shelf_reader_cleanup(): + """Take care of uninstalling shelf-reader before/after the test.""" + cmd = ("pip", "uninstall", "shelf-reader", "-y") + subprocess.run(cmd) + yield + subprocess.run(cmd) + + +@pytest.mark.parallel +def test_domain_pypi_apis( + domain_factory, + pulpcore_bindings, + monitor_task, + python_file, + python_repo_factory, + python_distribution_factory, + pulp_admin_user, + http_get, + shelf_reader_cleanup, +): + """Test the PyPI apis with upload & download through python tooling (twine/pip).""" + domain = domain_factory() + repo = python_repo_factory(pulp_domain=domain.name) + distro = python_distribution_factory(repository=repo.pulp_href, pulp_domain=domain.name) + + response = json.loads(http_get(distro.base_url)) + assert response["projects"] == response["releases"] == response["files"] == 0 + + # Test upload + subprocess.run( + ( + "twine", + "upload", + "--repository-url", + distro.base_url + "simple/", + python_file, + "-u", + pulp_admin_user.username, + "-p", + pulp_admin_user.password, + ), + capture_output=True, + check=True, + ) + results = pulpcore_bindings.TasksApi.list( + reserved_resources=repo.pulp_href, pulp_domain=domain.name + ) + monitor_task(results.results[0].pulp_href) + response = json.loads(http_get(distro.base_url)) + assert response["projects"] == response["releases"] == response["files"] == 1 + + # Test download + subprocess.run( + ( + "pip", + "install", + "--no-deps", + "--trusted-host", + urlsplit(distro.base_url).hostname, + "-i", + distro.base_url + "simple/", + "shelf-reader", + ), + capture_output=True, + check=True, + ) diff --git a/pulp_python/tests/functional/api/test_download_content.py b/pulp_python/tests/functional/api/test_download_content.py index 67204326..8c58fb81 100644 --- a/pulp_python/tests/functional/api/test_download_content.py +++ b/pulp_python/tests/functional/api/test_download_content.py @@ -15,7 +15,7 @@ def test_basic_pulp_to_pulp_sync( python_content_summary, python_publication_factory, python_distribution_factory, - pulp_settings, + pulp_content_url, ): """ This test checks that the JSON endpoint is setup correctly to allow one Pulp @@ -27,14 +27,7 @@ def test_basic_pulp_to_pulp_sync( repo = python_repo_with_sync(remote) pub = python_publication_factory(repository=repo) distro = python_distribution_factory(publication=pub) - # TODO Add if check if domains are enabled. - url_fragments = [ - pulp_settings.CONTENT_ORIGIN, - pulp_settings.CONTENT_PATH_PREFIX.strip("/"), - distro.base_path, - "" - ] - unit_url = "/".join(url_fragments) + unit_url = f"{pulp_content_url}{distro.base_path}/" # Sync using old Pulp content api endpoints remote2 = python_remote_factory(url=unit_url, **remote_body) diff --git a/pulp_python/tests/functional/api/test_export_import.py b/pulp_python/tests/functional/api/test_export_import.py index ddc91abd..15a21e04 100644 --- a/pulp_python/tests/functional/api/test_export_import.py +++ b/pulp_python/tests/functional/api/test_export_import.py @@ -7,11 +7,22 @@ import pytest import uuid +from pulpcore.app import settings from pulp_python.tests.functional.constants import ( PYTHON_XS_PROJECT_SPECIFIER, PYTHON_SM_PROJECT_SPECIFIER ) +pytestmark = [ + pytest.mark.skipif(settings.DOMAIN_ENABLED, reason="Domains do not support export."), + pytest.mark.skipif( + "/tmp" not in settings.ALLOWED_EXPORT_PATHS, + reason="Cannot run export-tests unless /tmp is in ALLOWED_EXPORT_PATHS " + f"({settings.ALLOWED_EXPORT_PATHS}).", + ), +] + + @pytest.mark.parallel def test_export_then_import( python_bindings, diff --git a/pulp_python/tests/functional/api/test_full_mirror.py b/pulp_python/tests/functional/api/test_full_mirror.py index 5357ea3c..c14c145c 100644 --- a/pulp_python/tests/functional/api/test_full_mirror.py +++ b/pulp_python/tests/functional/api/test_full_mirror.py @@ -38,7 +38,7 @@ def test_pull_through_install( @pytest.mark.parallel -def test_pull_through_simple(python_remote_factory, python_distribution_factory, pulp_settings): +def test_pull_through_simple(python_remote_factory, python_distribution_factory, pulp_content_url): """Tests that the simple page is properly modified when requesting a pull-through.""" remote = python_remote_factory(url=PYPI_URL) distro = python_distribution_factory(remote=remote.pulp_href) @@ -46,13 +46,11 @@ def test_pull_through_simple(python_remote_factory, python_distribution_factory, url = f"{distro.base_url}simple/shelf-reader/" project_page = parse_repo_project_response("shelf-reader", requests.get(url)) - # TODO ADD check for pulp_domains when added - pulp_content_base_url = urljoin(pulp_settings.CONTENT_ORIGIN, pulp_settings.CONTENT_PATH_PREFIX) assert len(project_page.packages) == 2 for package in project_page.packages: assert package.filename in PYTHON_XS_FIXTURE_CHECKSUMS relative_path = f"{distro.base_path}/{package.filename}?redirect=" - assert urljoin(pulp_content_base_url, relative_path) in package.url + assert urljoin(pulp_content_url, relative_path) in package.url digests = package.get_digests() assert PYTHON_XS_FIXTURE_CHECKSUMS[package.filename] == digests["sha256"] diff --git a/pulp_python/tests/functional/api/test_pypi_apis.py b/pulp_python/tests/functional/api/test_pypi_apis.py index 096dc850..1e557f36 100644 --- a/pulp_python/tests/functional/api/test_pypi_apis.py +++ b/pulp_python/tests/functional/api/test_pypi_apis.py @@ -106,7 +106,7 @@ def test_package_upload( files={"content": open(egg_file, "rb")}, auth=("admin", "password"), ) - assert response.status_code == 200 + assert response.status_code == 202 monitor_task(response.json()["task"]) summary = python_content_summary(repository=repo) assert summary.added["python.python"]["count"] == 1 @@ -136,7 +136,7 @@ def test_package_upload_session( files={"content": open(egg_file, "rb")}, auth=("admin", "password"), ) - assert response.status_code == 200 + assert response.status_code == 202 task = monitor_task(response.json()["task"]) response2 = session.post( url, @@ -144,7 +144,7 @@ def test_package_upload_session( files={"content": open(wheel_file, "rb")}, auth=("admin", "password"), ) - assert response2.status_code == 200 + assert response2.status_code == 202 task2 = monitor_task(response2.json()["task"]) assert task != task2 summary = python_content_summary(repository=repo) @@ -165,7 +165,7 @@ def test_package_upload_simple( files={"content": open(egg_file, "rb")}, auth=("admin", "password"), ) - assert response.status_code == 200 + assert response.status_code == 202 monitor_task(response.json()["task"]) summary = python_content_summary(repository=repo) assert summary.added["python.python"]["count"] == 1 diff --git a/pulp_python/tests/functional/constants.py b/pulp_python/tests/functional/constants.py index 8c24687f..e449fe92 100644 --- a/pulp_python/tests/functional/constants.py +++ b/pulp_python/tests/functional/constants.py @@ -201,33 +201,34 @@ "classifiers": "[]", } -# Info data for Shelf-reader +# Info data for Shelf-reader, Not all the fields are the same whether uploaded or synced :( +# pkginfo filters out 'UNKNOWN' values & fails to find a couple others due to package description PYTHON_INFO_DATA = { "name": "shelf-reader", "version": "0.1", # "metadata_version": "", # Maybe program "1.1" into parse_metadata of app/utils.py "summary": "Make sure your collections are in call number order.", - "keywords": "library barcode call number shelf collection", + # "keywords": "library barcode call number shelf collection", "home_page": "https://github.com/asmacdo/shelf-reader", - "download_url": "UNKNOWN", + # "download_url": "UNKNOWN", "author": "Austin Macdonald", "author_email": "asmacdo@gmail.com", "maintainer": "", "maintainer_email": "", # "license": "GNU GENERAL PUBLIC LICENSE Version 2, June 1991", "requires_python": None, - "project_url": "https://pypi.org/project/shelf-reader/", - "platform": "UNKNOWN", + # "project_url": "https://pypi.org/project/shelf-reader/", + # "platform": "UNKNOWN", # "supported_platform": None, "requires_dist": None, # "provides_dist": [], # "obsoletes_dist": [], # "requires_external": [], - "classifiers": ['Development Status :: 4 - Beta', 'Environment :: Console', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)', - 'Natural Language :: English', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7'], + # "classifiers": ['Development Status :: 4 - Beta', 'Environment :: Console', + # 'Intended Audience :: Developers', + # 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)', + # 'Natural Language :: English', 'Programming Language :: Python :: 2', + # 'Programming Language :: Python :: 2.7'], "downloads": {"last_day": -1, "last_month": -1, "last_week": -1}, # maybe add description, license is long for this one } diff --git a/requirements.txt b/requirements.txt index 6c1b1723..91bc710c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -pulpcore>=3.49.0,<3.55 +pulpcore>=3.49.0,<3.70.0 pkginfo>=1.8.2,<1.9.7 bandersnatch>=6.1,<6.2 pypi-simple>=0.9.0,<1.0.0 diff --git a/staging_docs/user/learn/tech-preview.md b/staging_docs/user/learn/tech-preview.md index 027fb932..4c6ccc91 100644 --- a/staging_docs/user/learn/tech-preview.md +++ b/staging_docs/user/learn/tech-preview.md @@ -8,3 +8,4 @@ The following features are currently being released as part of a tech preview - Fully mirror Python repositories provided PyPI and Pulp itself. - `Twine` upload packages to indexes at endpoints '/simple\` or '/legacy'. - Create pull-through caches of remote sources. +- Pulp Domain Support diff --git a/template_config.yml b/template_config.yml index 2efc0e68..39538a37 100644 --- a/template_config.yml +++ b/template_config.yml @@ -52,9 +52,11 @@ pulp_settings: allowed_import_paths: /tmp orphan_protection_time: 0 pypi_api_hostname: https://pulp:443 -pulp_settings_azure: null +pulp_settings_azure: + domain_enabled: true pulp_settings_gcp: null -pulp_settings_s3: null +pulp_settings_s3: + domain_enabled: true pydocstyle: true release_email: pulp-infra@redhat.com release_user: pulpbot