From 95584a58c4d70a5121098872b59d5fdf632f5e9c Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Tue, 4 Aug 2020 09:28:57 -0400 Subject: [PATCH] add model support, an API, and a migration for Org -> Galaxy credentials see: https://github.com/ansible/awx/issues/7813 --- awx/api/serializers.py | 1 + awx/api/urls/organization.py | 2 + awx/api/views/__init__.py | 1 + awx/api/views/organization.py | 21 ++- awx/main/conf.py | 162 ------------------ awx/main/constants.py | 4 - .../migrations/0118_galaxy_credentials.py | 34 ++++ awx/main/models/credential/__init__.py | 33 ++++ awx/main/models/organization.py | 23 +++ awx/main/redact.py | 14 +- awx/main/tasks.py | 49 ++---- .../functional/api/test_credential_type.py | 2 +- .../functional/api/test_organizations.py | 89 +++++++++- .../tests/functional/models/test_project.py | 30 +++- awx/main/tests/functional/test_credential.py | 1 + awx/main/tests/unit/test_tasks.py | 95 +++++++++- awx/settings/defaults.py | 19 -- .../jobs-form/configuration-jobs.form.js | 25 --- docs/collections.md | 30 ++-- 19 files changed, 363 insertions(+), 272 deletions(-) create mode 100644 awx/main/migrations/0118_galaxy_credentials.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index be6a9d640b9d..0cc8eba0149b 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1269,6 +1269,7 @@ def get_related(self, obj): object_roles = self.reverse('api:organization_object_roles_list', kwargs={'pk': obj.pk}), access_list = self.reverse('api:organization_access_list', kwargs={'pk': obj.pk}), instance_groups = self.reverse('api:organization_instance_groups_list', kwargs={'pk': obj.pk}), + galaxy_credentials = self.reverse('api:organization_galaxy_credentials_list', kwargs={'pk': obj.pk}), )) return res diff --git a/awx/api/urls/organization.py b/awx/api/urls/organization.py index 3d172f136097..12b2807905bc 100644 --- a/awx/api/urls/organization.py +++ b/awx/api/urls/organization.py @@ -21,6 +21,7 @@ OrganizationNotificationTemplatesSuccessList, OrganizationNotificationTemplatesApprovalList, OrganizationInstanceGroupsList, + OrganizationGalaxyCredentialsList, OrganizationObjectRolesList, OrganizationAccessList, OrganizationApplicationList, @@ -49,6 +50,7 @@ url(r'^(?P[0-9]+)/notification_templates_approvals/$', OrganizationNotificationTemplatesApprovalList.as_view(), name='organization_notification_templates_approvals_list'), url(r'^(?P[0-9]+)/instance_groups/$', OrganizationInstanceGroupsList.as_view(), name='organization_instance_groups_list'), + url(r'^(?P[0-9]+)/galaxy_credentials/$', OrganizationGalaxyCredentialsList.as_view(), name='organization_galaxy_credentials_list'), url(r'^(?P[0-9]+)/object_roles/$', OrganizationObjectRolesList.as_view(), name='organization_object_roles_list'), url(r'^(?P[0-9]+)/access_list/$', OrganizationAccessList.as_view(), name='organization_access_list'), url(r'^(?P[0-9]+)/applications/$', OrganizationApplicationList.as_view(), name='organization_applications_list'), diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index 77481a3917b7..33a989cd5134 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -122,6 +122,7 @@ OrganizationNotificationTemplatesSuccessList, OrganizationNotificationTemplatesApprovalList, OrganizationInstanceGroupsList, + OrganizationGalaxyCredentialsList, OrganizationAccessList, OrganizationObjectRolesList, ) diff --git a/awx/api/views/organization.py b/awx/api/views/organization.py index cb929ec5b52b..06172af79fe8 100644 --- a/awx/api/views/organization.py +++ b/awx/api/views/organization.py @@ -7,6 +7,7 @@ # Django from django.db.models import Count from django.contrib.contenttypes.models import ContentType +from django.utils.translation import ugettext_lazy as _ # AWX from awx.main.models import ( @@ -20,7 +21,8 @@ Role, User, Team, - InstanceGroup + InstanceGroup, + Credential ) from awx.api.generics import ( ListCreateAPIView, @@ -42,7 +44,8 @@ RoleSerializer, NotificationTemplateSerializer, InstanceGroupSerializer, - ProjectSerializer, JobTemplateSerializer, WorkflowJobTemplateSerializer + ProjectSerializer, JobTemplateSerializer, WorkflowJobTemplateSerializer, + CredentialSerializer ) from awx.api.views.mixin import ( RelatedJobsPreventDeleteMixin, @@ -214,6 +217,20 @@ class OrganizationInstanceGroupsList(SubListAttachDetachAPIView): relationship = 'instance_groups' +class OrganizationGalaxyCredentialsList(SubListAttachDetachAPIView): + + model = Credential + serializer_class = CredentialSerializer + parent_model = Organization + relationship = 'galaxy_credentials' + + def is_valid_relation(self, parent, sub, created=False): + if sub.kind != 'galaxy_api_token': + return {'msg': _( + f"Credential must be a Galaxy credential, not {sub.credential_type.name}." + )} + + class OrganizationAccessList(ResourceAccessList): model = User # needs to be User for AccessLists's diff --git a/awx/main/conf.py b/awx/main/conf.py index 8d091894d607..ae3ad4533d8b 100644 --- a/awx/main/conf.py +++ b/awx/main/conf.py @@ -2,7 +2,6 @@ import json import logging import os -from distutils.version import LooseVersion as Version # Django from django.utils.translation import ugettext_lazy as _ @@ -436,87 +435,6 @@ def _load_default_license_from_file(): category_slug='jobs', ) -register( - 'PRIMARY_GALAXY_URL', - field_class=fields.URLField, - required=False, - allow_blank=True, - label=_('Primary Galaxy Server URL'), - help_text=_( - 'For organizations that run their own Galaxy service, this gives the option to specify a ' - 'host as the primary galaxy server. Requirements will be downloaded from the primary if the ' - 'specific role or collection is available there. If the content is not avilable in the primary, ' - 'or if this field is left blank, it will default to galaxy.ansible.com.' - ), - category=_('Jobs'), - category_slug='jobs' -) - -register( - 'PRIMARY_GALAXY_USERNAME', - field_class=fields.CharField, - required=False, - allow_blank=True, - label=_('Primary Galaxy Server Username'), - help_text=_('(This setting is deprecated and will be removed in a future release) ' - 'For using a galaxy server at higher precedence than the public Ansible Galaxy. ' - 'The username to use for basic authentication against the Galaxy instance, ' - 'this is mutually exclusive with PRIMARY_GALAXY_TOKEN.'), - category=_('Jobs'), - category_slug='jobs' -) - -register( - 'PRIMARY_GALAXY_PASSWORD', - field_class=fields.CharField, - encrypted=True, - required=False, - allow_blank=True, - label=_('Primary Galaxy Server Password'), - help_text=_('(This setting is deprecated and will be removed in a future release) ' - 'For using a galaxy server at higher precedence than the public Ansible Galaxy. ' - 'The password to use for basic authentication against the Galaxy instance, ' - 'this is mutually exclusive with PRIMARY_GALAXY_TOKEN.'), - category=_('Jobs'), - category_slug='jobs' -) - -register( - 'PRIMARY_GALAXY_TOKEN', - field_class=fields.CharField, - encrypted=True, - required=False, - allow_blank=True, - label=_('Primary Galaxy Server Token'), - help_text=_('For using a galaxy server at higher precedence than the public Ansible Galaxy. ' - 'The token to use for connecting with the Galaxy instance, ' - 'this is mutually exclusive with corresponding username and password settings.'), - category=_('Jobs'), - category_slug='jobs' -) - -register( - 'PRIMARY_GALAXY_AUTH_URL', - field_class=fields.CharField, - required=False, - allow_blank=True, - label=_('Primary Galaxy Authentication URL'), - help_text=_('For using a galaxy server at higher precedence than the public Ansible Galaxy. ' - 'The token_endpoint of a Keycloak server.'), - category=_('Jobs'), - category_slug='jobs' -) - -register( - 'PUBLIC_GALAXY_ENABLED', - field_class=fields.BooleanField, - default=True, - label=_('Allow Access to Public Galaxy'), - help_text=_('Allow or deny access to the public Ansible Galaxy during project updates.'), - category=_('Jobs'), - category_slug='jobs' -) - register( 'GALAXY_IGNORE_CERTS', field_class=fields.BooleanField, @@ -856,84 +774,4 @@ def logging_validate(serializer, attrs): return attrs -def galaxy_validate(serializer, attrs): - """Ansible Galaxy config options have mutual exclusivity rules, these rules - are enforced here on serializer validation so that users will not be able - to save settings which obviously break all project updates. - """ - prefix = 'PRIMARY_GALAXY_' - errors = {} - - def _new_value(setting_name): - if setting_name in attrs: - return attrs[setting_name] - elif not serializer.instance: - return '' - return getattr(serializer.instance, setting_name, '') - - if not _new_value('PRIMARY_GALAXY_URL'): - if _new_value('PUBLIC_GALAXY_ENABLED') is False: - msg = _('A URL for Primary Galaxy must be defined before disabling public Galaxy.') - # put error in both keys because UI has trouble with errors in toggles - for key in ('PRIMARY_GALAXY_URL', 'PUBLIC_GALAXY_ENABLED'): - errors.setdefault(key, []) - errors[key].append(msg) - raise serializers.ValidationError(errors) - - from awx.main.constants import GALAXY_SERVER_FIELDS - if not any('{}{}'.format(prefix, subfield.upper()) in attrs for subfield in GALAXY_SERVER_FIELDS): - return attrs - - galaxy_data = {} - for subfield in GALAXY_SERVER_FIELDS: - galaxy_data[subfield] = _new_value('{}{}'.format(prefix, subfield.upper())) - if not galaxy_data['url']: - for k, v in galaxy_data.items(): - if v: - setting_name = '{}{}'.format(prefix, k.upper()) - errors.setdefault(setting_name, []) - errors[setting_name].append(_( - 'Cannot provide field if PRIMARY_GALAXY_URL is not set.' - )) - for k in GALAXY_SERVER_FIELDS: - if galaxy_data[k]: - setting_name = '{}{}'.format(prefix, k.upper()) - if (not serializer.instance) or (not getattr(serializer.instance, setting_name, '')): - # new auth is applied, so check if compatible with version - from awx.main.utils import get_ansible_version - current_version = get_ansible_version() - min_version = '2.9' - if Version(current_version) < Version(min_version): - errors.setdefault(setting_name, []) - errors[setting_name].append(_( - 'Galaxy server settings are not available until Ansible {min_version}, ' - 'you are running {current_version}.' - ).format(min_version=min_version, current_version=current_version)) - if (galaxy_data['password'] or galaxy_data['username']) and (galaxy_data['token'] or galaxy_data['auth_url']): - for k in ('password', 'username', 'token', 'auth_url'): - setting_name = '{}{}'.format(prefix, k.upper()) - if setting_name in attrs: - errors.setdefault(setting_name, []) - errors[setting_name].append(_( - 'Setting Galaxy token and authentication URL is mutually exclusive with username and password.' - )) - if bool(galaxy_data['username']) != bool(galaxy_data['password']): - msg = _('If authenticating via username and password, both must be provided.') - for k in ('username', 'password'): - setting_name = '{}{}'.format(prefix, k.upper()) - errors.setdefault(setting_name, []) - errors[setting_name].append(msg) - if bool(galaxy_data['token']) != bool(galaxy_data['auth_url']): - msg = _('If authenticating via token, both token and authentication URL must be provided.') - for k in ('token', 'auth_url'): - setting_name = '{}{}'.format(prefix, k.upper()) - errors.setdefault(setting_name, []) - errors[setting_name].append(msg) - - if errors: - raise serializers.ValidationError(errors) - return attrs - - register_validate('logging', logging_validate) -register_validate('jobs', galaxy_validate) diff --git a/awx/main/constants.py b/awx/main/constants.py index c0382c7504f5..323f61f31101 100644 --- a/awx/main/constants.py +++ b/awx/main/constants.py @@ -50,7 +50,3 @@ # loggers that may be called getting logging settings 'awx.conf' ) - -# these correspond to both AWX and Ansible settings to keep naming consistent -# for instance, settings.PRIMARY_GALAXY_AUTH_URL vs env var ANSIBLE_GALAXY_SERVER_FOO_AUTH_URL -GALAXY_SERVER_FIELDS = ('url', 'username', 'password', 'token', 'auth_url') diff --git a/awx/main/migrations/0118_galaxy_credentials.py b/awx/main/migrations/0118_galaxy_credentials.py new file mode 100644 index 000000000000..f61434d7d1bd --- /dev/null +++ b/awx/main/migrations/0118_galaxy_credentials.py @@ -0,0 +1,34 @@ +# Generated by Django 2.2.11 on 2020-08-04 15:19 + +import awx.main.fields +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0117_v400_remove_cloudforms_inventory'), + ] + + operations = [ + migrations.AlterField( + model_name='credentialtype', + name='kind', + field=models.CharField(choices=[('ssh', 'Machine'), ('vault', 'Vault'), ('net', 'Network'), ('scm', 'Source Control'), ('cloud', 'Cloud'), ('token', 'Personal Access Token'), ('insights', 'Insights'), ('external', 'External'), ('kubernetes', 'Kubernetes'), ('galaxy', 'Galaxy/Automation Hub')], max_length=32), + ), + migrations.CreateModel( + name='OrganizationGalaxyCredentialMembership', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('position', models.PositiveIntegerField(db_index=True, default=None, null=True)), + ('credential', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='main.Credential')), + ('organization', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='main.Organization')), + ], + ), + migrations.AddField( + model_name='organization', + name='galaxy_credentials', + field=awx.main.fields.OrderedManyToManyField(blank=True, related_name='organization_galaxy_credentials', through='main.OrganizationGalaxyCredentialMembership', to='main.Credential'), + ), + ] diff --git a/awx/main/models/credential/__init__.py b/awx/main/models/credential/__init__.py index 36bb2684ea4a..be4a21a99d73 100644 --- a/awx/main/models/credential/__init__.py +++ b/awx/main/models/credential/__init__.py @@ -331,6 +331,7 @@ class Meta: ('insights', _('Insights')), ('external', _('External')), ('kubernetes', _('Kubernetes')), + ('galaxy', _('Galaxy/Automation Hub')), ) kind = models.CharField( @@ -1173,6 +1174,38 @@ def create(self): ) +ManagedCredentialType( + namespace='galaxy_api_token', + kind='galaxy', + name=ugettext_noop('Ansible Galaxy Automation Hub API Token'), + inputs={ + 'fields': [{ + 'id': 'url', + 'label': ugettext_noop('Galaxy Server URL'), + 'type': 'string', + 'help_text': ugettext_noop('The URL of the galaxy instance to connect to.') + },{ + 'id': 'auth_url', + 'label': ugettext_noop('Auth Server URL'), + 'type': 'string', + 'help_text': ugettext_noop( + 'The URL of a Keycloak server token_endpoint, if using ' + 'SSO auth.' + ) + },{ + 'id': 'token', + 'label': ugettext_noop('API Token'), + 'type': 'string', + 'secret': True, + 'help_text': ugettext_noop( + 'A token to use for authentication against the Galaxy instance.' + ) + }], + 'required': ['url'], + } +) + + class CredentialInputSource(PrimordialModel): class Meta: diff --git a/awx/main/models/organization.py b/awx/main/models/organization.py index 23ce65f5e938..bf2e07d255c7 100644 --- a/awx/main/models/organization.py +++ b/awx/main/models/organization.py @@ -45,6 +45,12 @@ class Meta: blank=True, through='OrganizationInstanceGroupMembership' ) + galaxy_credentials = OrderedManyToManyField( + 'Credential', + blank=True, + through='OrganizationGalaxyCredentialMembership', + related_name='%(class)s_galaxy_credentials' + ) max_hosts = models.PositiveIntegerField( blank=True, default=0, @@ -108,6 +114,23 @@ def _get_related_jobs(self): return UnifiedJob.objects.non_polymorphic().filter(organization=self) +class OrganizationGalaxyCredentialMembership(models.Model): + + organization = models.ForeignKey( + 'Organization', + on_delete=models.CASCADE + ) + credential = models.ForeignKey( + 'Credential', + on_delete=models.CASCADE + ) + position = models.PositiveIntegerField( + null=True, + default=None, + db_index=True, + ) + + class Team(CommonModelNameNotUnique, ResourceMixin): ''' A team is a group of users that work on common projects. diff --git a/awx/main/redact.py b/awx/main/redact.py index 4c286eb9a87d..c0e23659415c 100644 --- a/awx/main/redact.py +++ b/awx/main/redact.py @@ -1,8 +1,6 @@ import re import urllib.parse as urlparse -from django.conf import settings - REPLACE_STR = '$encrypted$' @@ -13,11 +11,13 @@ class UriCleaner(object): @staticmethod def remove_sensitive(cleartext): # exclude_list contains the items that will _not_ be redacted - exclude_list = [settings.PUBLIC_GALAXY_SERVER['url']] - if settings.PRIMARY_GALAXY_URL: - exclude_list += [settings.PRIMARY_GALAXY_URL] - if settings.FALLBACK_GALAXY_SERVERS: - exclude_list += [server['url'] for server in settings.FALLBACK_GALAXY_SERVERS] + # TODO: replace this with the server URLs from proj.org.credentials + exclude_list = [] + #exclude_list = [settings.PUBLIC_GALAXY_SERVER['url']] + #if settings.PRIMARY_GALAXY_URL: + # exclude_list += [settings.PRIMARY_GALAXY_URL] + #if settings.FALLBACK_GALAXY_SERVERS: + # exclude_list += [server['url'] for server in settings.FALLBACK_GALAXY_SERVERS] redactedtext = cleartext text_index = 0 while True: diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 06c740c1297d..8e94a93cec58 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -50,7 +50,7 @@ # AWX from awx import __version__ as awx_application_version -from awx.main.constants import PRIVILEGE_ESCALATION_METHODS, STANDARD_INVENTORY_UPDATE_ENV, GALAXY_SERVER_FIELDS +from awx.main.constants import PRIVILEGE_ESCALATION_METHODS, STANDARD_INVENTORY_UPDATE_ENV from awx.main.access import access_registry from awx.main.redact import UriCleaner from awx.main.models import ( @@ -2046,35 +2046,24 @@ def build_env(self, project_update, private_data_dir, isolated=False, private_da env['ANSIBLE_CALLBACK_PLUGINS'] = self.get_path_to('..', 'plugins', 'callback') if settings.GALAXY_IGNORE_CERTS: env['ANSIBLE_GALAXY_IGNORE'] = True - # Set up the public Galaxy server, if enabled - galaxy_configured = False - if settings.PUBLIC_GALAXY_ENABLED: - galaxy_servers = [settings.PUBLIC_GALAXY_SERVER] # static setting - else: - galaxy_configured = True - galaxy_servers = [] - # Set up fallback Galaxy servers, if configured - if settings.FALLBACK_GALAXY_SERVERS: - galaxy_configured = True - galaxy_servers = settings.FALLBACK_GALAXY_SERVERS + galaxy_servers - # Set up the primary Galaxy server, if configured - if settings.PRIMARY_GALAXY_URL: - galaxy_configured = True - galaxy_servers = [{'id': 'primary_galaxy'}] + galaxy_servers - for key in GALAXY_SERVER_FIELDS: - value = getattr(settings, 'PRIMARY_GALAXY_{}'.format(key.upper())) - if value: - galaxy_servers[0][key] = value - if galaxy_configured: - for server in galaxy_servers: - for key in GALAXY_SERVER_FIELDS: - if not server.get(key): - continue - env_key = ('ANSIBLE_GALAXY_SERVER_{}_{}'.format(server.get('id', 'unnamed'), key)).upper() - env[env_key] = server[key] - if galaxy_servers: - # now set the precedence of galaxy servers - env['ANSIBLE_GALAXY_SERVER_LIST'] = ','.join([server.get('id', 'unnamed') for server in galaxy_servers]) + + # build out env vars for Galaxy credentials (in order) + galaxy_server_list = [] + for i, cred in enumerate( + project_update.project.organization.galaxy_credentials.all() + ): + env[f'ANSIBLE_GALAXY_SERVER_SERVER{i}_URL'] = cred.get_input('url') + auth_url = cred.get_input('auth_url', default=None) + token = cred.get_input('token', default=None) + if token: + env[f'ANSIBLE_GALAXY_SERVER_SERVER{i}_TOKEN'] = token + if auth_url: + env[f'ANSIBLE_GALAXY_SERVER_SERVER{i}_AUTH_URL'] = auth_url + galaxy_server_list.append(f'server{i}') + + if galaxy_server_list: + env['ANSIBLE_GALAXY_SERVER_LIST'] = ','.join(galaxy_server_list) + return env def _build_scm_url_extra_vars(self, project_update): diff --git a/awx/main/tests/functional/api/test_credential_type.py b/awx/main/tests/functional/api/test_credential_type.py index c8f87f0c5755..bf7aa4ceffae 100644 --- a/awx/main/tests/functional/api/test_credential_type.py +++ b/awx/main/tests/functional/api/test_credential_type.py @@ -220,7 +220,7 @@ def test_create_valid_kind(kind, get, post, admin): @pytest.mark.django_db -@pytest.mark.parametrize('kind', ['ssh', 'vault', 'scm', 'insights', 'kubernetes']) +@pytest.mark.parametrize('kind', ['ssh', 'vault', 'scm', 'insights', 'kubernetes', 'galaxy']) def test_create_invalid_kind(kind, get, post, admin): response = post(reverse('api:credential_type_list'), { 'kind': kind, diff --git a/awx/main/tests/functional/api/test_organizations.py b/awx/main/tests/functional/api/test_organizations.py index 082719967631..6c45c0c68197 100644 --- a/awx/main/tests/functional/api/test_organizations.py +++ b/awx/main/tests/functional/api/test_organizations.py @@ -9,7 +9,7 @@ import pytest # AWX -from awx.main.models import ProjectUpdate +from awx.main.models import ProjectUpdate, CredentialType, Credential from awx.api.versioning import reverse @@ -288,3 +288,90 @@ def sort_keys(x): assert resp.data['error'] == u"Resource is being used by running jobs." assert resp_sorted == expect_sorted + + +@pytest.mark.django_db +def test_galaxy_credential_association_forbidden(alice, organization, post): + galaxy = CredentialType.defaults['galaxy_api_token']() + galaxy.save() + + cred = Credential.objects.create( + credential_type=galaxy, + name='Public Galaxy', + organization=organization, + inputs={ + 'url': 'https://galaxy.ansible.com/' + } + ) + url = reverse('api:organization_galaxy_credentials_list', kwargs={'pk': organization.id}) + post( + url, + {'associate': True, 'id': cred.pk}, + user=alice, + expect=403 + ) + + +@pytest.mark.django_db +def test_galaxy_credential_type_enforcement(admin, organization, post): + ssh = CredentialType.defaults['ssh']() + ssh.save() + + cred = Credential.objects.create( + credential_type=ssh, + name='SSH Credential', + organization=organization, + ) + url = reverse('api:organization_galaxy_credentials_list', kwargs={'pk': organization.id}) + resp = post( + url, + {'associate': True, 'id': cred.pk}, + user=admin, + expect=400 + ) + assert resp.data['msg'] == 'Credential must be a Galaxy credential, not Machine.' + + +@pytest.mark.django_db +def test_galaxy_credential_association(alice, admin, organization, post, get): + galaxy = CredentialType.defaults['galaxy_api_token']() + galaxy.save() + + for i in range(5): + cred = Credential.objects.create( + credential_type=galaxy, + name=f'Public Galaxy {i + 1}', + organization=organization, + inputs={ + 'url': 'https://galaxy.ansible.com/' + } + ) + url = reverse('api:organization_galaxy_credentials_list', kwargs={'pk': organization.id}) + post( + url, + {'associate': True, 'id': cred.pk}, + user=admin, + expect=204 + ) + resp = get(url, user=admin) + assert [cred['name'] for cred in resp.data['results']] == [ + 'Public Galaxy 1', + 'Public Galaxy 2', + 'Public Galaxy 3', + 'Public Galaxy 4', + 'Public Galaxy 5', + ] + + post( + url, + {'disassociate': True, 'id': Credential.objects.get(name='Public Galaxy 3').pk}, + user=admin, + expect=204 + ) + resp = get(url, user=admin) + assert [cred['name'] for cred in resp.data['results']] == [ + 'Public Galaxy 1', + 'Public Galaxy 2', + 'Public Galaxy 4', + 'Public Galaxy 5', + ] diff --git a/awx/main/tests/functional/models/test_project.py b/awx/main/tests/functional/models/test_project.py index 2cf43c5690cf..d3c34498b021 100644 --- a/awx/main/tests/functional/models/test_project.py +++ b/awx/main/tests/functional/models/test_project.py @@ -1,7 +1,7 @@ import pytest from unittest import mock -from awx.main.models import Project +from awx.main.models import Project, Credential, CredentialType from awx.main.models.organization import Organization @@ -57,3 +57,31 @@ def test_foreign_key_change_changes_modified_by(project, organization): def test_project_related_jobs(project): update = project.create_unified_job() assert update.id in [u.id for u in project._get_related_jobs()] + + +@pytest.mark.django_db +def test_galaxy_credentials(project): + org = project.organization + galaxy = CredentialType.defaults['galaxy_api_token']() + galaxy.save() + for i in range(5): + cred = Credential.objects.create( + name=f'Ansible Galaxy {i + 1}', + organization=org, + credential_type=galaxy, + inputs={ + 'url': 'https://galaxy.ansible.com/' + } + ) + cred.save() + org.galaxy_credentials.add(cred) + + assert [ + cred.name for cred in org.galaxy_credentials.all() + ] == [ + 'Ansible Galaxy 1', + 'Ansible Galaxy 2', + 'Ansible Galaxy 3', + 'Ansible Galaxy 4', + 'Ansible Galaxy 5', + ] diff --git a/awx/main/tests/functional/test_credential.py b/awx/main/tests/functional/test_credential.py index 721bf5c043ac..684f9dd5a79b 100644 --- a/awx/main/tests/functional/test_credential.py +++ b/awx/main/tests/functional/test_credential.py @@ -81,6 +81,7 @@ def test_default_cred_types(): 'azure_rm', 'cloudforms', 'conjur', + 'galaxy_api_token', 'gce', 'github_token', 'gitlab_token', diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 71bcd8d03c8f..dc4d7088e184 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -25,6 +25,7 @@ Job, JobTemplate, Notification, + Organization, Project, ProjectUpdate, UnifiedJob, @@ -59,6 +60,18 @@ def patch_Job(): yield +@pytest.fixture +def patch_Organization(): + _credentials = [] + credentials_mock = mock.Mock(**{ + 'all': lambda: _credentials, + 'add': _credentials.append, + 'spec_set': ['all', 'add'], + }) + with mock.patch.object(Organization, 'galaxy_credentials', credentials_mock): + yield + + @pytest.fixture def job(): return Job( @@ -131,7 +144,6 @@ def test_send_notifications_list(mock_notifications_filter, mock_job_get, mocker ('SECRET_KEY', 'SECRET'), ('VMWARE_PASSWORD', 'SECRET'), ('API_SECRET', 'SECRET'), - ('ANSIBLE_GALAXY_SERVER_PRIMARY_GALAXY_PASSWORD', 'SECRET'), ('ANSIBLE_GALAXY_SERVER_PRIMARY_GALAXY_TOKEN', 'SECRET'), ]) def test_safe_env_filtering(key, value): @@ -1780,10 +1792,89 @@ def test_awx_task_env(self, settings, private_data_dir, job): assert env['FOO'] == 'BAR' +@pytest.mark.usefixtures("patch_Organization") +class TestProjectUpdateGalaxyCredentials(TestJobExecution): + + @pytest.fixture + def project_update(self): + org = Organization(pk=1) + proj = Project(pk=1, organization=org) + project_update = ProjectUpdate(pk=1, project=proj) + project_update.websocket_emit_status = mock.Mock() + return project_update + + parametrize = { + 'test_galaxy_credentials_ignore_certs': [ + dict(ignore=True), + dict(ignore=False), + ], + } + + def test_galaxy_credentials_ignore_certs(self, private_data_dir, project_update, ignore): + settings.GALAXY_IGNORE_CERTS = ignore + task = tasks.RunProjectUpdate() + env = task.build_env(project_update, private_data_dir) + if ignore: + assert env['ANSIBLE_GALAXY_IGNORE'] is True + else: + assert 'ANSIBLE_GALAXY_IGNORE' not in env + + def test_galaxy_credentials_empty(self, private_data_dir, project_update): + task = tasks.RunProjectUpdate() + env = task.build_env(project_update, private_data_dir) + for k in env: + assert not k.startswith('ANSIBLE_GALAXY_SERVER') + + def test_single_public_galaxy(self, private_data_dir, project_update): + credential_type = CredentialType.defaults['galaxy_api_token']() + public_galaxy = Credential(pk=1, credential_type=credential_type, inputs={ + 'url': 'https://galaxy.ansible.com/', + }) + project_update.project.organization.galaxy_credentials.add(public_galaxy) + task = tasks.RunProjectUpdate() + env = task.build_env(project_update, private_data_dir) + assert sorted([ + (k, v) for k, v in env.items() + if k.startswith('ANSIBLE_GALAXY') + ]) == [ + ('ANSIBLE_GALAXY_SERVER_LIST', 'server0'), + ('ANSIBLE_GALAXY_SERVER_SERVER0_URL', 'https://galaxy.ansible.com/'), + ] + + def test_multiple_galaxy_endpoints(self, private_data_dir, project_update): + credential_type = CredentialType.defaults['galaxy_api_token']() + public_galaxy = Credential(pk=1, credential_type=credential_type, inputs={ + 'url': 'https://galaxy.ansible.com/', + }) + rh = Credential(pk=2, credential_type=credential_type, inputs={ + 'url': 'https://cloud.redhat.com/api/automation-hub/', + 'auth_url': 'https://sso.redhat.com/example/openid-connect/token/', + 'token': 'secret123' + }) + project_update.project.organization.galaxy_credentials.add(public_galaxy) + project_update.project.organization.galaxy_credentials.add(rh) + task = tasks.RunProjectUpdate() + env = task.build_env(project_update, private_data_dir) + assert sorted([ + (k, v) for k, v in env.items() + if k.startswith('ANSIBLE_GALAXY') + ]) == [ + ('ANSIBLE_GALAXY_SERVER_LIST', 'server0,server1'), + ('ANSIBLE_GALAXY_SERVER_SERVER0_URL', 'https://galaxy.ansible.com/'), + ('ANSIBLE_GALAXY_SERVER_SERVER1_AUTH_URL', 'https://sso.redhat.com/example/openid-connect/token/'), # noqa + ('ANSIBLE_GALAXY_SERVER_SERVER1_TOKEN', 'secret123'), + ('ANSIBLE_GALAXY_SERVER_SERVER1_URL', 'https://cloud.redhat.com/api/automation-hub/'), + ] + + +@pytest.mark.usefixtures("patch_Organization") class TestProjectUpdateCredentials(TestJobExecution): @pytest.fixture def project_update(self): - project_update = ProjectUpdate(pk=1, project=Project(pk=1)) + project_update = ProjectUpdate( + pk=1, + project=Project(pk=1, organization=Organization(pk=1)), + ) project_update.websocket_emit_status = mock.Mock() return project_update diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index fe7c8c0ba345..8b43eb151d90 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -569,28 +569,9 @@ def IS_TESTING(argv=None): # Follow symlinks when scanning for playbooks AWX_SHOW_PLAYBOOK_LINKS = False -# Settings for primary galaxy server, should be set in the UI -PRIMARY_GALAXY_URL = '' -PRIMARY_GALAXY_USERNAME = '' -PRIMARY_GALAXY_TOKEN = '' -PRIMARY_GALAXY_PASSWORD = '' -PRIMARY_GALAXY_AUTH_URL = '' - -# Settings for the public galaxy server(s). -PUBLIC_GALAXY_ENABLED = True -PUBLIC_GALAXY_SERVER = { - 'id': 'galaxy', - 'url': 'https://galaxy.ansible.com' -} - # Applies to any galaxy server GALAXY_IGNORE_CERTS = False -# List of dicts of fallback (additional) Galaxy servers. If configured, these -# will be higher precedence than public Galaxy, but lower than primary Galaxy. -# Available options: 'id', 'url', 'username', 'password', 'token', 'auth_url' -FALLBACK_GALAXY_SERVERS = [] - # Enable bubblewrap support for running jobs (playbook runs only). # Note: This setting may be overridden by database settings. AWX_PROOT_ENABLED = True diff --git a/awx/ui/client/src/configuration/forms/jobs-form/configuration-jobs.form.js b/awx/ui/client/src/configuration/forms/jobs-form/configuration-jobs.form.js index faff8e23661a..0f40144d4fb2 100644 --- a/awx/ui/client/src/configuration/forms/jobs-form/configuration-jobs.form.js +++ b/awx/ui/client/src/configuration/forms/jobs-form/configuration-jobs.form.js @@ -74,31 +74,6 @@ export default ['i18n', function(i18n) { AWX_SHOW_PLAYBOOK_LINKS: { type: 'toggleSwitch', }, - PRIMARY_GALAXY_URL: { - type: 'text', - reset: 'PRIMARY_GALAXY_URL', - }, - PRIMARY_GALAXY_USERNAME: { - type: 'text', - reset: 'PRIMARY_GALAXY_USERNAME', - }, - PRIMARY_GALAXY_PASSWORD: { - type: 'sensitive', - hasShowInputButton: true, - reset: 'PRIMARY_GALAXY_PASSWORD', - }, - PRIMARY_GALAXY_TOKEN: { - type: 'sensitive', - hasShowInputButton: true, - reset: 'PRIMARY_GALAXY_TOKEN', - }, - PRIMARY_GALAXY_AUTH_URL: { - type: 'text', - reset: 'PRIMARY_GALAXY_AUTH_URL', - }, - PUBLIC_GALAXY_ENABLED: { - type: 'toggleSwitch', - }, AWX_TASK_ENV: { type: 'textarea', reset: 'AWX_TASK_ENV', diff --git a/docs/collections.md b/docs/collections.md index 68c11950f8c2..0bb90feaa67c 100644 --- a/docs/collections.md +++ b/docs/collections.md @@ -85,28 +85,22 @@ job runs. It will populate the cache id set by the last "check" type update. ### Galaxy Server Selection -Ansible core default settings will download collections from the public -Galaxy server at `https://galaxy.ansible.com`. For details on -how Galaxy servers are configured in Ansible in general see: +For details on how Galaxy servers are configured in Ansible in general see: https://docs.ansible.com/ansible/devel/user_guide/collections_using.html (if "devel" link goes stale in the future, it is for Ansible 2.9) -You can set a different server to be the primary Galaxy server to download -roles and collections from in AWX project updates. -This is done via the setting `PRIMARY_GALAXY_URL` and similar -`PRIMARY_GALAXY_xxxx` settings for authentication. - -If the `PRIMARY_GALAXY_URL` setting is not blank, then the server list is defined -to be `primary_galaxy,galaxy`. The `primary_galaxy` server definition uses the URL -from those settings, as well as username, password, and/or token and auth_url if applicable. -the `galaxy` server definition uses public Galaxy (`https://galaxy.ansible.com`) -with no authentication. - -This configuration causes requirements to be downloaded from the user-specified -primary galaxy server if they are available there. If a requirement is -not available from the primary galaxy server, then it will fallback to -downloading it from the public Galaxy server. +You can specify a list of zero or more servers to download roles and +collections from for AWX Project Updates. This is done by associating Galaxy +credentials (in sequential order) via the API at +`/api/v2/organizations/N/galaxy_credentials/`. Authentication +via an API token is optional (i.e., https://galaxy.ansible.com/), but other +content sources (such as Red Hat Ansible Automation Hub) require proper +configuration of the Auth URL and Token. + +If no credentials are defined at this endpoint for an Organization, then roles and +collections will *not* be installed based on requirements.yml for Project Updates +in that Organization. Even when these settings are enabled, this can still be bypassed for a specific requirement by using the `source:` option, as described in Ansible documentation.