Skip to content

Commit

Permalink
Merge pull request #673 from gerrod3/domains
Browse files Browse the repository at this point in the history
Add domain support
  • Loading branch information
gerrod3 authored Jun 19, 2024
2 parents 66ada93 + a3068e4 commit 5f58544
Show file tree
Hide file tree
Showing 25 changed files with 461 additions and 83 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/scripts/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ if [ "$TEST" = "s3" ]; then
sed -i -e '$a s3_test: true\
minio_access_key: "'$MINIO_ACCESS_KEY'"\
minio_secret_key: "'$MINIO_SECRET_KEY'"\
pulp_scenario_settings: null\
pulp_scenario_settings: {"domain_enabled": true}\
pulp_scenario_env: {}\
' vars/main.yaml
export PULP_API_ROOT="/rerouted/djnd/"
Expand All @@ -111,7 +111,7 @@ if [ "$TEST" = "azure" ]; then
- ./azurite:/etc/pulp\
command: "azurite-blob --blobHost 0.0.0.0"' vars/main.yaml
sed -i -e '$a azure_test: true\
pulp_scenario_settings: null\
pulp_scenario_settings: {"domain_enabled": true}\
pulp_scenario_env: {}\
' vars/main.yaml
fi
Expand Down
1 change: 1 addition & 0 deletions CHANGES/668.feature
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Added Domain support.
1 change: 1 addition & 0 deletions docs/tech-preview.rst
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,4 @@ The following features are currently being released as part of a tech preview
* Fully mirror Python repositories provided PyPI and Pulp itself.
* ``Twine`` upload packages to indexes at endpoints '/simple` or '/legacy'.
* Create pull-through caches of remote sources.
* Pulp Domain support
1 change: 1 addition & 0 deletions pulp_python/app/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,4 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig):
label = "python"
version = "3.12.0.dev"
python_package_name = "pulp-python"
domain_compatible = True
37 changes: 37 additions & 0 deletions pulp_python/app/migrations/0012_add_domain.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# Generated by Django 4.2.10 on 2024-05-30 17:53

from django.db import migrations, models
import django.db.models.deletion
import pulpcore.app.util


class Migration(migrations.Migration):

dependencies = [
("python", "0011_alter_pythondistribution_distribution_ptr_and_more"),
]

operations = [
migrations.AlterUniqueTogether(
name="pythonpackagecontent",
unique_together=set(),
),
migrations.AddField(
model_name="pythonpackagecontent",
name="_pulp_domain",
field=models.ForeignKey(
default=pulpcore.app.util.get_domain_pk,
on_delete=django.db.models.deletion.PROTECT,
to="core.domain",
),
),
migrations.AlterField(
model_name="pythonpackagecontent",
name="sha256",
field=models.CharField(db_index=True, max_length=64),
),
migrations.AlterUniqueTogether(
name="pythonpackagecontent",
unique_together={("sha256", "_pulp_domain")},
),
]
5 changes: 4 additions & 1 deletion pulp_python/app/modelresource.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from pulpcore.plugin.importexport import BaseContentResource
from pulpcore.plugin.modelresources import RepositoryResource
from pulpcore.plugin.util import get_domain
from pulp_python.app.models import (
PythonPackageContent,
PythonRepository,
Expand All @@ -15,7 +16,9 @@ def set_up_queryset(self):
"""
:return: PythonPackageContent specific to a specified repo-version.
"""
return PythonPackageContent.objects.filter(pk__in=self.repo_version.content)
return PythonPackageContent.objects.filter(
pk__in=self.repo_version.content, _pulp_domain=get_domain()
)

class Meta:
model = PythonPackageContent
Expand Down
18 changes: 14 additions & 4 deletions pulp_python/app/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
PYPI_SERIAL_CONSTANT,
)
from pulpcore.plugin.repo_version_utils import remove_duplicates, validate_repo_version
from pulpcore.plugin.util import get_domain_pk, get_domain

log = getLogger(__name__)

Expand Down Expand Up @@ -68,6 +69,7 @@ def content_handler(self, path):
path = PurePath(path)
name = None
version = None
domain = get_domain()
if path.match("pypi/*/*/json"):
version = path.parts[2]
name = path.parts[1]
Expand All @@ -76,7 +78,7 @@ def content_handler(self, path):
elif len(path.parts) and path.parts[0] == "simple":
# Temporary fix for PublishedMetadata not being properly served from remote storage
# https://github.com/pulp/pulp_python/issues/413
if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem":
if domain.storage_class != "pulpcore.app.models.storage.FileSystem":
if self.publication or self.repository:
try:
publication = self.publication or Publication.objects.filter(
Expand Down Expand Up @@ -105,7 +107,11 @@ def content_handler(self, path):
)
# TODO Change this value to the Repo's serial value when implemented
headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT)}
json_body = python_content_to_json(self.base_path, package_content, version=version)
if not settings.DOMAIN_ENABLED:
domain = None
json_body = python_content_to_json(
self.base_path, package_content, version=version, domain=domain
)
if json_body:
return json_response(json_body, headers=headers)

Expand Down Expand Up @@ -143,7 +149,7 @@ class PythonPackageContent(Content):
name = models.TextField()
name.register_lookup(NormalizeName)
version = models.TextField()
sha256 = models.CharField(unique=True, db_index=True, max_length=64)
sha256 = models.CharField(db_index=True, max_length=64)
# Optional metadata
python_version = models.TextField()
metadata_version = models.TextField()
Expand All @@ -168,6 +174,8 @@ class PythonPackageContent(Content):
classifiers = models.JSONField(default=list)
project_urls = models.JSONField(default=dict)
description_content_type = models.TextField()
# Pulp Domains
_pulp_domain = models.ForeignKey("core.Domain", default=get_domain_pk, on_delete=models.PROTECT)

@staticmethod
def init_from_artifact_and_relative_path(artifact, relative_path):
Expand All @@ -179,6 +187,8 @@ def init_from_artifact_and_relative_path(artifact, relative_path):
data["version"] = metadata.version
data["filename"] = path.name
data["sha256"] = artifact.sha256
data["pulp_domain_id"] = artifact.pulp_domain_id
data["_pulp_domain_id"] = artifact.pulp_domain_id
return PythonPackageContent(**data)

def __str__(self):
Expand All @@ -200,7 +210,7 @@ def __str__(self):

class Meta:
default_related_name = "%(app_label)s_%(model_name)s"
unique_together = ("sha256",)
unique_together = ("sha256", "_pulp_domain")


class PythonPublication(Publication):
Expand Down
3 changes: 2 additions & 1 deletion pulp_python/app/pypi/serializers.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from rest_framework import serializers
from pulp_python.app.utils import DIST_EXTENSIONS
from pulpcore.plugin.models import Artifact
from pulpcore.plugin.util import get_domain
from django.db.utils import IntegrityError

log = logging.getLogger(__name__)
Expand Down Expand Up @@ -76,7 +77,7 @@ def validate(self, data):
try:
artifact.save()
except IntegrityError:
artifact = Artifact.objects.get(sha256=artifact.sha256)
artifact = Artifact.objects.get(sha256=artifact.sha256, pulp_domain=get_domain())
artifact.touch()
log.info(f"Artifact for {file.name} already existed in database")
data["content"] = (artifact, file.name)
Expand Down
37 changes: 26 additions & 11 deletions pulp_python/app/pypi/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@
from pathlib import PurePath
from pypi_simple.parse_stream import parse_links_stream_response

from pulpcore.plugin.viewsets import OperationPostponedResponse
from pulpcore.plugin.tasking import dispatch
from pulpcore.plugin.util import get_domain
from pulp_python.app.models import (
PythonDistribution,
PythonPackageContent,
Expand Down Expand Up @@ -75,7 +77,7 @@ def get_distribution(path):
"repository", "publication", "publication__repository_version", "remote"
)
try:
return distro_qs.get(base_path=path)
return distro_qs.get(base_path=path, pulp_domain=get_domain())
except ObjectDoesNotExist:
raise Http404(f"No PythonDistribution found for base_path {path}")

Expand All @@ -100,6 +102,14 @@ def get_drvc(self, path):
content = self.get_content(repo_ver)
return distro, repo_ver, content

def initial(self, request, *args, **kwargs):
"""Perform common initialization tasks for PyPI endpoints."""
super().initial(request, *args, **kwargs)
if settings.DOMAIN_ENABLED:
self.base_content_url = urljoin(BASE_CONTENT_URL, f"{get_domain().name}/")
else:
self.base_content_url = BASE_CONTENT_URL


class PackageUploadMixin(PyPIMixin):
"""A Mixin to provide package upload support."""
Expand Down Expand Up @@ -137,7 +147,7 @@ def upload(self, request, path):
kwargs={"artifact_sha256": artifact.sha256,
"filename": filename,
"repository_pk": str(repo.pk)})
return Response(data={"task": reverse('tasks-detail', args=[result.pk], request=None)})
return OperationPostponedResponse(result, request)

def upload_package_group(self, repo, artifact, filename, session):
"""Steps 4 & 5, spawns tasks to add packages to index."""
Expand Down Expand Up @@ -176,7 +186,7 @@ def create_group_upload_task(self, cur_session, repository, artifact, filename,
return reverse('tasks-detail', args=[result.pk], request=None)


class SimpleView(ViewSet, PackageUploadMixin):
class SimpleView(PackageUploadMixin, ViewSet):
"""View for the PyPI simple API."""

permission_classes = [IsAuthenticatedOrReadOnly]
Expand All @@ -186,7 +196,7 @@ def list(self, request, path):
"""Gets the simple api html page for the index."""
distro, repo_version, content = self.get_drvc(path)
if self.should_redirect(distro, repo_version=repo_version):
return redirect(urljoin(BASE_CONTENT_URL, f'{path}/simple/'))
return redirect(urljoin(self.base_content_url, f'{path}/simple/'))
names = content.order_by('name').values_list('name', flat=True).distinct().iterator()
return StreamingHttpResponse(write_simple_index(names, streamed=True))

Expand All @@ -197,7 +207,7 @@ def parse_url(link):
digest, _, value = parsed.fragment.partition('=')
stripped_url = urlunsplit(chain(parsed[:3], ("", "")))
redirect = f'{path}/{link.text}?redirect={stripped_url}'
d_url = urljoin(BASE_CONTENT_URL, redirect)
d_url = urljoin(self.base_content_url, redirect)
return link.text, d_url, value if digest == 'sha256' else ''

url = remote.get_remote_artifact_url(f'simple/{package}/')
Expand All @@ -224,7 +234,7 @@ def retrieve(self, request, path, package):
if not repo_ver or not content.filter(name__normalize=normalized).exists():
return self.pull_through_package_simple(normalized, path, distro.remote)
if self.should_redirect(distro, repo_version=repo_ver):
return redirect(urljoin(BASE_CONTENT_URL, f'{path}/simple/{normalized}/'))
return redirect(urljoin(self.base_content_url, f'{path}/simple/{normalized}/'))
packages = (
content.filter(name__normalize=normalized)
.values_list('filename', 'sha256', 'name')
Expand All @@ -237,7 +247,7 @@ def retrieve(self, request, path, package):
else:
packages = chain([present], packages)
name = present[2]
releases = ((f, urljoin(BASE_CONTENT_URL, f'{path}/{f}'), d) for f, d, _ in packages)
releases = ((f, urljoin(self.base_content_url, f'{path}/{f}'), d) for f, d, _ in packages)
return StreamingHttpResponse(write_simple_detail(name, releases, streamed=True))

@extend_schema(request=PackageUploadSerializer,
Expand All @@ -253,7 +263,7 @@ def create(self, request, path):
return self.upload(request, path)


class MetadataView(ViewSet, PyPIMixin):
class MetadataView(PyPIMixin, ViewSet):
"""View for the PyPI JSON metadata endpoint."""

authentication_classes = []
Expand All @@ -272,6 +282,7 @@ def retrieve(self, request, path, meta):
meta_path = PurePath(meta)
name = None
version = None
domain = None
if meta_path.match("*/*/json"):
version = meta_path.parts[1]
name = meta_path.parts[0]
Expand All @@ -281,13 +292,17 @@ def retrieve(self, request, path, meta):
package_content = content.filter(name__iexact=name)
# TODO Change this value to the Repo's serial value when implemented
headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT)}
json_body = python_content_to_json(path, package_content, version=version)
if settings.DOMAIN_ENABLED:
domain = get_domain()
json_body = python_content_to_json(
path, package_content, version=version, domain=domain
)
if json_body:
return Response(data=json_body, headers=headers)
return Response(status="404")


class PyPIView(ViewSet, PyPIMixin):
class PyPIView(PyPIMixin, ViewSet):
"""View for base_url of distribution."""

authentication_classes = []
Expand All @@ -305,7 +320,7 @@ def retrieve(self, request, path):
return Response(data=data)


class UploadView(ViewSet, PackageUploadMixin):
class UploadView(PackageUploadMixin, ViewSet):
"""View for the `/legacy` upload endpoint."""

@extend_schema(request=PackageUploadSerializer,
Expand Down
9 changes: 8 additions & 1 deletion pulp_python/app/serializers.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

from pulpcore.plugin import models as core_models
from pulpcore.plugin import serializers as core_serializers
from pulpcore.plugin.util import get_domain

from pulp_python.app import models as python_models
from pulp_python.app.utils import get_project_metadata_from_artifact, parse_project_metadata
Expand Down Expand Up @@ -56,6 +57,8 @@ class PythonDistributionSerializer(core_serializers.DistributionSerializer):

def get_base_url(self, obj):
"""Gets the base url."""
if settings.DOMAIN_ENABLED:
return f"{settings.PYPI_API_HOSTNAME}/pypi/{get_domain().name}/{obj.base_path}/"
return f"{settings.PYPI_API_HOSTNAME}/pypi/{obj.base_path}/"

class Meta:
Expand Down Expand Up @@ -232,13 +235,17 @@ def deferred_validate(self, data):
_data['version'] = metadata.version
_data['filename'] = filename
_data['sha256'] = artifact.sha256
data["pulp_domain_id"] = artifact.pulp_domain_id
data["_pulp_domain_id"] = artifact.pulp_domain_id

data.update(_data)

return data

def retrieve(self, validated_data):
content = python_models.PythonPackageContent.objects.filter(sha256=validated_data["sha256"])
content = python_models.PythonPackageContent.objects.filter(
sha256=validated_data["sha256"], _pulp_domain=get_domain()
)
return content.first()

class Meta:
Expand Down
6 changes: 4 additions & 2 deletions pulp_python/app/tasks/publish.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from packaging.utils import canonicalize_name

from pulpcore.plugin import models
from pulpcore.plugin.util import get_domain

from pulp_python.app import models as python_models
from pulp_python.app.utils import write_simple_index, write_simple_detail
Expand Down Expand Up @@ -49,11 +50,12 @@ def write_simple_api(publication):
publication (pulpcore.plugin.models.Publication): A publication to generate metadata for
"""
domain = get_domain()
simple_dir = 'simple/'
os.mkdir(simple_dir)
project_names = (
python_models.PythonPackageContent.objects.filter(
pk__in=publication.repository_version.content
pk__in=publication.repository_version.content, _pulp_domain=domain
)
.order_by('name')
.values_list('name', flat=True)
Expand All @@ -76,7 +78,7 @@ def write_simple_api(publication):
return

packages = python_models.PythonPackageContent.objects.filter(
pk__in=publication.repository_version.content
pk__in=publication.repository_version.content, _pulp_domain=domain
)
releases = packages.order_by("name").values("name", "filename", "sha256")

Expand Down
Loading

0 comments on commit 5f58544

Please sign in to comment.