From dfaa69be51573a424048ea29128b1f7bd24ca330 Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Thu, 22 Apr 2021 13:45:33 -0400 Subject: [PATCH 01/11] add an awx-manage command that gets pip freeze data from custom_venvs and outputs to command line stdout remove analytics tests for counts of custom venvs, bump collector version, and remove list of custom venvs from API --- awx/api/views/root.py | 8 +---- awx/main/analytics/collectors.py | 7 ++--- awx/main/analytics/metrics.py | 2 -- awx/main/management/commands/custom_venvs.py | 28 ++++++++++++++++++ .../tests/functional/analytics/test_counts.py | 1 - .../functional/analytics/test_metrics.py | 1 - awx/main/utils/common.py | 29 +++++++++++-------- 7 files changed, 48 insertions(+), 28 deletions(-) create mode 100644 awx/main/management/commands/custom_venvs.py diff --git a/awx/api/views/root.py b/awx/api/views/root.py index ac5592207f7f..9ab8ff084064 100644 --- a/awx/api/views/root.py +++ b/awx/api/views/root.py @@ -300,13 +300,7 @@ def get(self, request, format=None): or Organization.accessible_objects(request.user, 'auditor_role').exists() or Organization.accessible_objects(request.user, 'project_admin_role').exists() ): - data.update( - dict( - project_base_dir=settings.PROJECTS_ROOT, project_local_paths=Project.get_local_path_choices(), custom_virtualenvs=get_custom_venv_choices() - ) - ) - elif JobTemplate.accessible_objects(request.user, 'admin_role').exists(): - data['custom_virtualenvs'] = get_custom_venv_choices() + data.update(dict(project_base_dir=settings.PROJECTS_ROOT, project_local_paths=Project.get_local_path_choices())) return Response(data) diff --git a/awx/main/analytics/collectors.py b/awx/main/analytics/collectors.py index 9755beeac94a..9455c73bf03b 100644 --- a/awx/main/analytics/collectors.py +++ b/awx/main/analytics/collectors.py @@ -15,7 +15,7 @@ from psycopg2.errors import UntranslatableCharacter from awx.conf.license import get_license -from awx.main.utils import get_awx_version, get_custom_venv_choices, camelcase_to_underscore, datetime_hook +from awx.main.utils import get_awx_version, camelcase_to_underscore, datetime_hook from awx.main import models from awx.main.analytics import register @@ -120,7 +120,7 @@ def config(since, **kwargs): } -@register('counts', '1.0', description=_('Counts of objects such as organizations, inventories, and projects')) +@register('counts', '1.1', description=_('Counts of objects such as organizations, inventories, and projects')) def counts(since, **kwargs): counts = {} for cls in ( @@ -138,9 +138,6 @@ def counts(since, **kwargs): ): counts[camelcase_to_underscore(cls.__name__)] = cls.objects.count() - venvs = get_custom_venv_choices() - counts['custom_virtualenvs'] = len([v for v in venvs if os.path.basename(v.rstrip('/')) != 'ansible']) - inv_counts = dict(models.Inventory.objects.order_by().values_list('kind').annotate(Count('kind'))) inv_counts['normal'] = inv_counts.get('', 0) inv_counts.pop('', None) diff --git a/awx/main/analytics/metrics.py b/awx/main/analytics/metrics.py index b9cd5153cc65..a32a7fc9595a 100644 --- a/awx/main/analytics/metrics.py +++ b/awx/main/analytics/metrics.py @@ -39,7 +39,6 @@ def metrics(): ], registry=REGISTRY, ) - CUSTOM_VENVS = Gauge('awx_custom_virtualenvs_total', 'Number of virtualenvs', registry=REGISTRY) RUNNING_JOBS = Gauge('awx_running_jobs_total', 'Number of running jobs on the system', registry=REGISTRY) PENDING_JOBS = Gauge('awx_pending_jobs_total', 'Number of pending jobs on the system', registry=REGISTRY) STATUS = Gauge( @@ -159,7 +158,6 @@ def metrics(): HOST_COUNT.labels(type='active').set(current_counts['active_host_count']) SCHEDULE_COUNT.set(current_counts['schedule']) - CUSTOM_VENVS.set(current_counts['custom_virtualenvs']) USER_SESSIONS.labels(type='all').set(current_counts['active_sessions']) USER_SESSIONS.labels(type='user').set(current_counts['active_user_sessions']) diff --git a/awx/main/management/commands/custom_venvs.py b/awx/main/management/commands/custom_venvs.py new file mode 100644 index 000000000000..5a0489c7ee38 --- /dev/null +++ b/awx/main/management/commands/custom_venvs.py @@ -0,0 +1,28 @@ +# Copyright (c) 2021 Ansible, Inc. +# All Rights Reserved + +from awx.main.utils.common import get_custom_venv_choices, get_custom_venv_pip_freeze +from django.core.management.base import BaseCommand + + +class Command(BaseCommand): + """Returns either a list of custom venv paths or outputs the pip freeze from the path passed in the argument""" + + def add_arguments(self, parser): + parser.add_argument( + '--path', + dest='path', + type=str, + default='', + help='run without arguments to see a list of paths, run with one of those paths as an argument and see the pip freeze data', + ) + + def handle(self, *args, **options): + super(Command, self).__init__() + if options.get('path'): + pip_data = get_custom_venv_pip_freeze(options.get('path')) + print(pip_data) + else: + venvs = get_custom_venv_choices() + for venv in venvs: + print(venv) diff --git a/awx/main/tests/functional/analytics/test_counts.py b/awx/main/tests/functional/analytics/test_counts.py index 146afb604c62..a55065b6bd73 100644 --- a/awx/main/tests/functional/analytics/test_counts.py +++ b/awx/main/tests/functional/analytics/test_counts.py @@ -12,7 +12,6 @@ def test_empty(): "active_sessions": 0, "active_host_count": 0, "credential": 0, - "custom_virtualenvs": 0, # dev env ansible3 "host": 0, "inventory": 0, "inventories": {"normal": 0, "smart": 0}, diff --git a/awx/main/tests/functional/analytics/test_metrics.py b/awx/main/tests/functional/analytics/test_metrics.py index 2a04adcfcea5..3a8d1c441181 100644 --- a/awx/main/tests/functional/analytics/test_metrics.py +++ b/awx/main/tests/functional/analytics/test_metrics.py @@ -21,7 +21,6 @@ 'awx_sessions_total': 0.0, 'awx_sessions_total': 0.0, 'awx_sessions_total': 0.0, - 'awx_custom_virtualenvs_total': 0.0, 'awx_running_jobs_total': 0.0, 'awx_instance_capacity': 100.0, 'awx_instance_consumed_capacity': 0.0, diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index 543f351d4ef4..644097b32acb 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -6,6 +6,7 @@ import yaml import logging import os +import subprocess import re import stat import urllib.parse @@ -890,30 +891,34 @@ def get_current_apps(): return current_apps -def get_custom_venv_choices(custom_paths=None): +def get_custom_venv_choices(): from django.conf import settings - custom_paths = custom_paths or settings.CUSTOM_VENV_PATHS - all_venv_paths = [settings.BASE_VENV_PATH] + custom_paths + all_venv_paths = settings.CUSTOM_VENV_PATHS + [settings.BASE_VENV_PATH] custom_venv_choices = [] - for custom_venv_path in all_venv_paths: + for venv_path in all_venv_paths: try: - if os.path.exists(custom_venv_path): + if os.path.exists(venv_path): custom_venv_choices.extend( - [ - os.path.join(custom_venv_path, x, '') - for x in os.listdir(custom_venv_path) - if x != 'awx' - and os.path.isdir(os.path.join(custom_venv_path, x)) - and os.path.exists(os.path.join(custom_venv_path, x, 'bin', 'activate')) - ] + [os.path.join(venv_path, x) for x in os.listdir(venv_path) if os.path.exists(os.path.join(venv_path, x, 'bin', 'pip'))] ) except Exception: logger.exception("Encountered an error while discovering custom virtual environments.") return custom_venv_choices +def get_custom_venv_pip_freeze(venv_path): + venv_path = os.path.join(venv_path, 'bin', 'pip') + try: + if os.path.exists(venv_path): + freeze_data = subprocess.run([venv_path, "freeze"], capture_output=True) + pip_data = (freeze_data.stdout).decode('UTF-8') + return pip_data + except Exception: + logger.exception("Encountered an error while discovering Pip Freeze data for custom virtual environments.") + + def is_ansible_variable(key): return key.startswith('ansible_') From 256a47618fb02977c977d2b97293d47235d88e87 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Tue, 11 May 2021 18:15:51 -0400 Subject: [PATCH 02/11] Ignore awx venv - General usability / readibility improvements --- awx/main/utils/common.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index 644097b32acb..e33d1322456a 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -14,6 +14,8 @@ import contextlib import tempfile import psutil +import traceback +import sys from functools import reduce, wraps from decimal import Decimal @@ -898,25 +900,26 @@ def get_custom_venv_choices(): custom_venv_choices = [] for venv_path in all_venv_paths: - try: - if os.path.exists(venv_path): - custom_venv_choices.extend( - [os.path.join(venv_path, x) for x in os.listdir(venv_path) if os.path.exists(os.path.join(venv_path, x, 'bin', 'pip'))] - ) - except Exception: - logger.exception("Encountered an error while discovering custom virtual environments.") + if os.path.exists(venv_path): + for d in os.listdir(venv_path): + if venv_path == settings.BASE_VENV_PATH and d == 'awx': + continue + + if os.path.exists(os.path.join(venv_path, d, 'bin', 'pip')): + custom_venv_choices.append(os.path.join(venv_path, d)) + return custom_venv_choices def get_custom_venv_pip_freeze(venv_path): - venv_path = os.path.join(venv_path, 'bin', 'pip') + pip_path = os.path.join(venv_path, 'bin', 'pip') + try: - if os.path.exists(venv_path): - freeze_data = subprocess.run([venv_path, "freeze"], capture_output=True) - pip_data = (freeze_data.stdout).decode('UTF-8') - return pip_data + freeze_data = subprocess.run([pip_path, "freeze"], capture_output=True) + pip_data = (freeze_data.stdout).decode('UTF-8') + return pip_data except Exception: - logger.exception("Encountered an error while discovering Pip Freeze data for custom virtual environments.") + logger.exception("Encountered an error while trying to run 'pip freeze' for custom virtual environments:") def is_ansible_variable(key): From 137fedfc9be63e3fe5493d8ef2d2eeb7904e1770 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Tue, 11 May 2021 18:18:27 -0400 Subject: [PATCH 03/11] Rename custom venv export command and add usability tweaks - Uses a positional argument instead of a named arg - More helpful output --- awx/main/management/commands/custom_venvs.py | 28 ----------- .../management/commands/export_custom_venv.py | 47 +++++++++++++++++++ 2 files changed, 47 insertions(+), 28 deletions(-) delete mode 100644 awx/main/management/commands/custom_venvs.py create mode 100644 awx/main/management/commands/export_custom_venv.py diff --git a/awx/main/management/commands/custom_venvs.py b/awx/main/management/commands/custom_venvs.py deleted file mode 100644 index 5a0489c7ee38..000000000000 --- a/awx/main/management/commands/custom_venvs.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) 2021 Ansible, Inc. -# All Rights Reserved - -from awx.main.utils.common import get_custom_venv_choices, get_custom_venv_pip_freeze -from django.core.management.base import BaseCommand - - -class Command(BaseCommand): - """Returns either a list of custom venv paths or outputs the pip freeze from the path passed in the argument""" - - def add_arguments(self, parser): - parser.add_argument( - '--path', - dest='path', - type=str, - default='', - help='run without arguments to see a list of paths, run with one of those paths as an argument and see the pip freeze data', - ) - - def handle(self, *args, **options): - super(Command, self).__init__() - if options.get('path'): - pip_data = get_custom_venv_pip_freeze(options.get('path')) - print(pip_data) - else: - venvs = get_custom_venv_choices() - for venv in venvs: - print(venv) diff --git a/awx/main/management/commands/export_custom_venv.py b/awx/main/management/commands/export_custom_venv.py new file mode 100644 index 000000000000..1ea1d2c0b79a --- /dev/null +++ b/awx/main/management/commands/export_custom_venv.py @@ -0,0 +1,47 @@ +# Copyright (c) 2021 Ansible, Inc. +# All Rights Reserved +import sys + +from awx.main.utils.common import get_custom_venv_choices, get_custom_venv_pip_freeze +from django.core.management.base import BaseCommand +from django.conf import settings + + +class Command(BaseCommand): + """Returns either a list of custom venv paths or outputs the pip freeze from the path passed in the argument""" + + def add_arguments(self, parser): + parser.add_argument( + 'path', + type=str, + nargs='?', + default='', + help='run without arguments to see a list of paths, run with one of those paths as an argument and see the pip freeze data', + ) + + def handle(self, *args, **options): + super(Command, self).__init__() + if options.get('path'): + pip_data = get_custom_venv_pip_freeze(options.get('path')) + if pip_data: + print(pip_data) + else: + venvs = get_custom_venv_choices() + if venvs: + print('# {}'.format("Discovered virtual environments:")) + for venv in venvs: + print(venv) + + msg = [ + '', + 'To export the contents of a virtual environment, ' 're-run while supplying the path as an argument:', + 'awx-manage export_custom_venv /path/to/venv', + ] + print('\n'.join(msg)) + else: + msg = ["No custom virtual environments detected in:", settings.BASE_VENV_PATH] + + for path in settings.CUSTOM_VENV_PATHS: + msg.append(path) + + print('\n'.join(msg), file=sys.stderr) From 779ca8b260018d13df22c83d29e7819d41073301 Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Wed, 12 May 2021 11:48:40 -0400 Subject: [PATCH 04/11] split the one command into two for clarity and remove unused imports --- awx/api/views/root.py | 4 +-- .../management/commands/export_custom_venv.py | 29 ++++------------ .../management/commands/list_custom_venvs.py | 33 +++++++++++++++++++ awx/main/utils/common.py | 2 -- 4 files changed, 42 insertions(+), 26 deletions(-) create mode 100644 awx/main/management/commands/list_custom_venvs.py diff --git a/awx/api/views/root.py b/awx/api/views/root.py index 9ab8ff084064..393440fe5c13 100644 --- a/awx/api/views/root.py +++ b/awx/api/views/root.py @@ -24,11 +24,11 @@ from awx.conf.registry import settings_registry from awx.main.analytics import all_collectors from awx.main.ha import is_ha_environment -from awx.main.utils import get_awx_version, get_custom_venv_choices +from awx.main.utils import get_awx_version, to_python_boolean, get_custom_venv_choices from awx.main.utils.licensing import validate_entitlement_manifest from awx.api.versioning import reverse, drf_reverse from awx.main.constants import PRIVILEGE_ESCALATION_METHODS -from awx.main.models import Project, Organization, Instance, InstanceGroup, JobTemplate +from awx.main.models import Project, Organization, Instance, InstanceGroup from awx.main.utils import set_environ logger = logging.getLogger('awx.api.views.root') diff --git a/awx/main/management/commands/export_custom_venv.py b/awx/main/management/commands/export_custom_venv.py index 1ea1d2c0b79a..93ef6d71f47d 100644 --- a/awx/main/management/commands/export_custom_venv.py +++ b/awx/main/management/commands/export_custom_venv.py @@ -1,14 +1,12 @@ # Copyright (c) 2021 Ansible, Inc. # All Rights Reserved -import sys -from awx.main.utils.common import get_custom_venv_choices, get_custom_venv_pip_freeze +from awx.main.utils.common import get_custom_venv_pip_freeze from django.core.management.base import BaseCommand -from django.conf import settings class Command(BaseCommand): - """Returns either a list of custom venv paths or outputs the pip freeze from the path passed in the argument""" + """Returns the pip freeze from the path passed in the argument""" def add_arguments(self, parser): parser.add_argument( @@ -16,7 +14,7 @@ def add_arguments(self, parser): type=str, nargs='?', default='', - help='run without arguments to see a list of paths, run with one of those paths as an argument and see the pip freeze data', + help='run this with a path to a virutal environment as an argument to see the pip freeze data', ) def handle(self, *args, **options): @@ -25,23 +23,10 @@ def handle(self, *args, **options): pip_data = get_custom_venv_pip_freeze(options.get('path')) if pip_data: print(pip_data) - else: - venvs = get_custom_venv_choices() - if venvs: - print('# {}'.format("Discovered virtual environments:")) - for venv in venvs: - print(venv) - msg = [ - '', - 'To export the contents of a virtual environment, ' 're-run while supplying the path as an argument:', - 'awx-manage export_custom_venv /path/to/venv', + 'To list all available custom virtual environments run:', + 'awx-manage list_custom_venvs', ] print('\n'.join(msg)) - else: - msg = ["No custom virtual environments detected in:", settings.BASE_VENV_PATH] - - for path in settings.CUSTOM_VENV_PATHS: - msg.append(path) - - print('\n'.join(msg), file=sys.stderr) + else: + print("missing argument: please include a path argument following the command.") diff --git a/awx/main/management/commands/list_custom_venvs.py b/awx/main/management/commands/list_custom_venvs.py new file mode 100644 index 000000000000..bec712eaf9fd --- /dev/null +++ b/awx/main/management/commands/list_custom_venvs.py @@ -0,0 +1,33 @@ +# Copyright (c) 2021 Ansible, Inc. +# All Rights Reserved +import sys + +from awx.main.utils.common import get_custom_venv_choices +from django.core.management.base import BaseCommand +from django.conf import settings + + +class Command(BaseCommand): + """Returns either a list of custom venv paths from the path passed in the argument""" + + def handle(self, *args, **options): + super(Command, self).__init__() + venvs = get_custom_venv_choices() + if venvs: + print('# {}'.format("Discovered virtual environments:")) + for venv in venvs: + print(venv) + + msg = [ + '', + 'To export the contents of a virtual environment, ' 'run the following command while supplying the path as an argument:', + 'awx-manage export_custom_venv /path/to/venv', + ] + print('\n'.join(msg)) + else: + msg = ["No custom virtual environments detected in:", settings.BASE_VENV_PATH] + + for path in settings.CUSTOM_VENV_PATHS: + msg.append(path) + + print('\n'.join(msg), file=sys.stderr) diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index e33d1322456a..873717b5fa1f 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -14,8 +14,6 @@ import contextlib import tempfile import psutil -import traceback -import sys from functools import reduce, wraps from decimal import Decimal From 2ed3038a5c0e1781888c16d999773d0d130fd3db Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Wed, 19 May 2021 09:52:37 -0400 Subject: [PATCH 05/11] fixed what would be a rather embarrassing misspeeeling --- awx/main/management/commands/export_custom_venv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/management/commands/export_custom_venv.py b/awx/main/management/commands/export_custom_venv.py index 93ef6d71f47d..b63a7492aff8 100644 --- a/awx/main/management/commands/export_custom_venv.py +++ b/awx/main/management/commands/export_custom_venv.py @@ -14,7 +14,7 @@ def add_arguments(self, parser): type=str, nargs='?', default='', - help='run this with a path to a virutal environment as an argument to see the pip freeze data', + help='run this with a path to a virtual environment as an argument to see the pip freeze data', ) def handle(self, *args, **options): From b256e5b79d7dbe16800212e05f357c496edd6964 Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Fri, 21 May 2021 11:36:34 -0400 Subject: [PATCH 06/11] adding in a 3rd command that shows the associated templates, orgs, and invs the venv is tied to --- .../management/commands/export_custom_venv.py | 3 +- .../commands/get_custom_venv_associations.py | 35 +++++++++++++++++++ .../management/commands/list_custom_venvs.py | 4 +-- 3 files changed, 38 insertions(+), 4 deletions(-) create mode 100644 awx/main/management/commands/get_custom_venv_associations.py diff --git a/awx/main/management/commands/export_custom_venv.py b/awx/main/management/commands/export_custom_venv.py index b63a7492aff8..fac39a052ae2 100644 --- a/awx/main/management/commands/export_custom_venv.py +++ b/awx/main/management/commands/export_custom_venv.py @@ -12,7 +12,6 @@ def add_arguments(self, parser): parser.add_argument( 'path', type=str, - nargs='?', default='', help='run this with a path to a virtual environment as an argument to see the pip freeze data', ) @@ -24,7 +23,7 @@ def handle(self, *args, **options): if pip_data: print(pip_data) msg = [ - 'To list all available custom virtual environments run:', + 'To list all (now deprecated) custom virtual environments run:', 'awx-manage list_custom_venvs', ] print('\n'.join(msg)) diff --git a/awx/main/management/commands/get_custom_venv_associations.py b/awx/main/management/commands/get_custom_venv_associations.py new file mode 100644 index 000000000000..4ae877503f14 --- /dev/null +++ b/awx/main/management/commands/get_custom_venv_associations.py @@ -0,0 +1,35 @@ +# Copyright (c) 2021 Ansible, Inc. +# All Rights Reserved + +from django.core.management.base import BaseCommand +from awx.main.models import Organization, InventorySource, UnifiedJobTemplate + + +class Command(BaseCommand): + """Returns the pip freeze from the path passed in the argument""" + + def add_arguments(self, parser): + parser.add_argument( + 'path', + type=str, + nargs='?', + default='', + help='run this with a path to a virtual environment as an argument to see the pip freeze data', + ) + + def handle(self, *args, **options): + # look organiztions and unified job templates (which include JTs, workflows, and Inventory updates) + super(Command, self).__init__() + results = {} + if options.get('path'): + # sanity check here - is path in list + path = options.get('path') + orgs = [{"name": org.name, "id": org.id} for org in Organization.objects.filter(custom_virtualenv=path)] + ujts = [{"name": org.name, "id": org.id} for org in UnifiedJobTemplate.objects.filter(custom_virtualenv=path)] + invsrc = [{"name": inv.name, "id": inv.id} for inv in InventorySource.objects.filter(custom_virtualenv=path)] + results["Organizations"] = orgs + results["Unified Job Templates"] = ujts + results["Inventory Sources"] = invsrc + print(results) + else: + print("missing argument: please include a path argument following the command.") diff --git a/awx/main/management/commands/list_custom_venvs.py b/awx/main/management/commands/list_custom_venvs.py index bec712eaf9fd..b621ea3a3bec 100644 --- a/awx/main/management/commands/list_custom_venvs.py +++ b/awx/main/management/commands/list_custom_venvs.py @@ -8,7 +8,7 @@ class Command(BaseCommand): - """Returns either a list of custom venv paths from the path passed in the argument""" + """Returns a list of custom venv paths from the path passed in the argument""" def handle(self, *args, **options): super(Command, self).__init__() @@ -20,7 +20,7 @@ def handle(self, *args, **options): msg = [ '', - 'To export the contents of a virtual environment, ' 'run the following command while supplying the path as an argument:', + 'To export the contents of a (deprecated) virtual environment, ' 'run the following command while supplying the path as an argument:', 'awx-manage export_custom_venv /path/to/venv', ] print('\n'.join(msg)) From c0b812c47a8a2580ab6f112ca528c1d9c3d05355 Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Fri, 21 May 2021 12:11:40 -0400 Subject: [PATCH 07/11] split unified job templates out into jts and projects because workflows don't have custom virtualenvs & rename file so the command is shorter update copypasta and add sanity check --- .../commands/custom_venv_associations.py | 37 +++++++++++++++++++ .../management/commands/export_custom_venv.py | 5 +-- .../commands/get_custom_venv_associations.py | 35 ------------------ 3 files changed, 39 insertions(+), 38 deletions(-) create mode 100644 awx/main/management/commands/custom_venv_associations.py delete mode 100644 awx/main/management/commands/get_custom_venv_associations.py diff --git a/awx/main/management/commands/custom_venv_associations.py b/awx/main/management/commands/custom_venv_associations.py new file mode 100644 index 000000000000..bcb0431d6896 --- /dev/null +++ b/awx/main/management/commands/custom_venv_associations.py @@ -0,0 +1,37 @@ +# Copyright (c) 2021 Ansible, Inc. +# All Rights Reserved + +from django.core.management.base import BaseCommand +from awx.main.utils.common import get_custom_venv_choices +from awx.main.models import Organization, InventorySource, JobTemplate, Project + + +class Command(BaseCommand): + """Returns the pip freeze from the path passed in the argument""" + + def add_arguments(self, parser): + parser.add_argument( + 'path', + type=str, + nargs=1, + default='', + help='run this with a path to a virtual environment as an argument to see the associated Job Templates, Organizations, Projects, and Inventory Sources.', + ) + + def handle(self, *args, **options): + # look organiztions and unified job templates (which include JTs, workflows, and Inventory updates) + super(Command, self).__init__() + results = {} + path = options.get('path') + if path: + if path in get_custom_venv_choices(): # verify this is a valid path + path = options.get('path')[0] + orgs = [{"name": org.name, "id": org.id} for org in Organization.objects.filter(custom_virtualenv=path)] + jts = [{"name": jt.name, "id": jt.id} for jt in JobTemplate.objects.filter(custom_virtualenv=path)] + proj = [{"name": proj.name, "id": proj.id} for proj in Project.objects.filter(custom_virtualenv=path)] + invsrc = [{"name": inv.name, "id": inv.id} for inv in InventorySource.objects.filter(custom_virtualenv=path)] + results["Organizations"] = orgs + results["Job Templates"] = jts + results["Projects"] = proj + results["Inventory Sources"] = invsrc + print(results) diff --git a/awx/main/management/commands/export_custom_venv.py b/awx/main/management/commands/export_custom_venv.py index fac39a052ae2..34acfc90d831 100644 --- a/awx/main/management/commands/export_custom_venv.py +++ b/awx/main/management/commands/export_custom_venv.py @@ -12,6 +12,7 @@ def add_arguments(self, parser): parser.add_argument( 'path', type=str, + nargs=1, default='', help='run this with a path to a virtual environment as an argument to see the pip freeze data', ) @@ -19,7 +20,7 @@ def add_arguments(self, parser): def handle(self, *args, **options): super(Command, self).__init__() if options.get('path'): - pip_data = get_custom_venv_pip_freeze(options.get('path')) + pip_data = get_custom_venv_pip_freeze(options.get('path')[0]) if pip_data: print(pip_data) msg = [ @@ -27,5 +28,3 @@ def handle(self, *args, **options): 'awx-manage list_custom_venvs', ] print('\n'.join(msg)) - else: - print("missing argument: please include a path argument following the command.") diff --git a/awx/main/management/commands/get_custom_venv_associations.py b/awx/main/management/commands/get_custom_venv_associations.py deleted file mode 100644 index 4ae877503f14..000000000000 --- a/awx/main/management/commands/get_custom_venv_associations.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (c) 2021 Ansible, Inc. -# All Rights Reserved - -from django.core.management.base import BaseCommand -from awx.main.models import Organization, InventorySource, UnifiedJobTemplate - - -class Command(BaseCommand): - """Returns the pip freeze from the path passed in the argument""" - - def add_arguments(self, parser): - parser.add_argument( - 'path', - type=str, - nargs='?', - default='', - help='run this with a path to a virtual environment as an argument to see the pip freeze data', - ) - - def handle(self, *args, **options): - # look organiztions and unified job templates (which include JTs, workflows, and Inventory updates) - super(Command, self).__init__() - results = {} - if options.get('path'): - # sanity check here - is path in list - path = options.get('path') - orgs = [{"name": org.name, "id": org.id} for org in Organization.objects.filter(custom_virtualenv=path)] - ujts = [{"name": org.name, "id": org.id} for org in UnifiedJobTemplate.objects.filter(custom_virtualenv=path)] - invsrc = [{"name": inv.name, "id": inv.id} for inv in InventorySource.objects.filter(custom_virtualenv=path)] - results["Organizations"] = orgs - results["Unified Job Templates"] = ujts - results["Inventory Sources"] = invsrc - print(results) - else: - print("missing argument: please include a path argument following the command.") From cece7ff741234d4c3995d78ea1a6697485c8c221 Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Wed, 2 Jun 2021 12:05:20 -0400 Subject: [PATCH 08/11] add a -q flag to make scripting easier and general improvements for readability --- awx/api/views/root.py | 2 +- .../commands/custom_venv_associations.py | 28 ++++++++++++++----- .../management/commands/export_custom_venv.py | 17 +++++++---- .../management/commands/list_custom_venvs.py | 24 +++++++++++----- 4 files changed, 51 insertions(+), 20 deletions(-) diff --git a/awx/api/views/root.py b/awx/api/views/root.py index 393440fe5c13..558d512010cb 100644 --- a/awx/api/views/root.py +++ b/awx/api/views/root.py @@ -24,7 +24,7 @@ from awx.conf.registry import settings_registry from awx.main.analytics import all_collectors from awx.main.ha import is_ha_environment -from awx.main.utils import get_awx_version, to_python_boolean, get_custom_venv_choices +from awx.main.utils import get_awx_version, get_custom_venv_choices from awx.main.utils.licensing import validate_entitlement_manifest from awx.api.versioning import reverse, drf_reverse from awx.main.constants import PRIVILEGE_ESCALATION_METHODS diff --git a/awx/main/management/commands/custom_venv_associations.py b/awx/main/management/commands/custom_venv_associations.py index bcb0431d6896..e1f749b79815 100644 --- a/awx/main/management/commands/custom_venv_associations.py +++ b/awx/main/management/commands/custom_venv_associations.py @@ -4,6 +4,7 @@ from django.core.management.base import BaseCommand from awx.main.utils.common import get_custom_venv_choices from awx.main.models import Organization, InventorySource, JobTemplate, Project +import yaml class Command(BaseCommand): @@ -17,6 +18,7 @@ def add_arguments(self, parser): default='', help='run this with a path to a virtual environment as an argument to see the associated Job Templates, Organizations, Projects, and Inventory Sources.', ) + parser.add_argument('-q', action='store_true', help='run with -q to output only the results of the query.') def handle(self, *args, **options): # look organiztions and unified job templates (which include JTs, workflows, and Inventory updates) @@ -24,14 +26,26 @@ def handle(self, *args, **options): results = {} path = options.get('path') if path: - if path in get_custom_venv_choices(): # verify this is a valid path - path = options.get('path')[0] + if path[0] in get_custom_venv_choices(): # verify this is a valid path + path = path[0] orgs = [{"name": org.name, "id": org.id} for org in Organization.objects.filter(custom_virtualenv=path)] jts = [{"name": jt.name, "id": jt.id} for jt in JobTemplate.objects.filter(custom_virtualenv=path)] proj = [{"name": proj.name, "id": proj.id} for proj in Project.objects.filter(custom_virtualenv=path)] invsrc = [{"name": inv.name, "id": inv.id} for inv in InventorySource.objects.filter(custom_virtualenv=path)] - results["Organizations"] = orgs - results["Job Templates"] = jts - results["Projects"] = proj - results["Inventory Sources"] = invsrc - print(results) + results["organizations"] = orgs + results["job_templates"] = jts + results["projects"] = proj + results["inventory_sources"] = invsrc + print('\n', '# {}'.format("Virtual environments associations:")) + print(yaml.dump(results)) + if not options.get('q'): + msg = [ + '', + 'To list all (now deprecated) custom virtual environments run:', + 'awx-manage list_custom_venvs', + '', + 'To export the contents of a (deprecated) virtual environment, ' 'run the following command while supplying the path as an argument:', + 'awx-manage export_custom_venv /path/to/venv', + '', + ] + print('\n'.join(msg)) diff --git a/awx/main/management/commands/export_custom_venv.py b/awx/main/management/commands/export_custom_venv.py index 34acfc90d831..246bf151ef04 100644 --- a/awx/main/management/commands/export_custom_venv.py +++ b/awx/main/management/commands/export_custom_venv.py @@ -16,15 +16,22 @@ def add_arguments(self, parser): default='', help='run this with a path to a virtual environment as an argument to see the pip freeze data', ) + parser.add_argument('-q', action='store_true', help='run with -q to output only the results of the query.') def handle(self, *args, **options): super(Command, self).__init__() if options.get('path'): pip_data = get_custom_venv_pip_freeze(options.get('path')[0]) if pip_data: + print('\n', '# {}'.format("Virtual environment contents:")) print(pip_data) - msg = [ - 'To list all (now deprecated) custom virtual environments run:', - 'awx-manage list_custom_venvs', - ] - print('\n'.join(msg)) + if not options.get('q'): + msg = [ + 'To list all (now deprecated) custom virtual environments run:', + 'awx-manage list_custom_venvs', + '', + 'To view the connections a (deprecated) virtual environment had in the database, run the following command while supplying the path as an argument:', + 'awx-manage custom_venv_associations /path/to/venv', + '', + ] + print('\n'.join(msg)) diff --git a/awx/main/management/commands/list_custom_venvs.py b/awx/main/management/commands/list_custom_venvs.py index b621ea3a3bec..6262d2300a0e 100644 --- a/awx/main/management/commands/list_custom_venvs.py +++ b/awx/main/management/commands/list_custom_venvs.py @@ -10,20 +10,30 @@ class Command(BaseCommand): """Returns a list of custom venv paths from the path passed in the argument""" + def add_arguments(self, parser): + parser.add_argument('-q', action='store_true', help='run with -q to output only the results of the query.') + def handle(self, *args, **options): super(Command, self).__init__() venvs = get_custom_venv_choices() if venvs: - print('# {}'.format("Discovered virtual environments:")) + print('\n', '# {}'.format("Discovered virtual environments:")) for venv in venvs: print(venv) + if not options.get('q'): + msg = [ + '', + 'To export the contents of a (deprecated) virtual environment, ' 'run the following command while supplying the path as an argument:', + 'awx-manage export_custom_venv /path/to/venv', + '', + 'To view the connections a (deprecated) virtual environment had in the database, run the following command while supplying the path as an argument:', + 'awx-manage custom_venv_associations /path/to/venv', + '', + ] + print('\n'.join(msg)) + else: + print('\n') - msg = [ - '', - 'To export the contents of a (deprecated) virtual environment, ' 'run the following command while supplying the path as an argument:', - 'awx-manage export_custom_venv /path/to/venv', - ] - print('\n'.join(msg)) else: msg = ["No custom virtual environments detected in:", settings.BASE_VENV_PATH] From 550ab82f63f119da465fb1a666287c1fe2e996f1 Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Wed, 2 Jun 2021 13:54:23 -0400 Subject: [PATCH 09/11] add the conditionals for incorrect paths and helpful info for if a user does that and remove unused import --- awx/api/views/root.py | 2 +- .../commands/custom_venv_associations.py | 32 +++++++++------ .../management/commands/export_custom_venv.py | 39 ++++++++++++------- .../management/commands/list_custom_venvs.py | 12 +++--- 4 files changed, 50 insertions(+), 35 deletions(-) diff --git a/awx/api/views/root.py b/awx/api/views/root.py index 558d512010cb..e464ca86bd10 100644 --- a/awx/api/views/root.py +++ b/awx/api/views/root.py @@ -24,7 +24,7 @@ from awx.conf.registry import settings_registry from awx.main.analytics import all_collectors from awx.main.ha import is_ha_environment -from awx.main.utils import get_awx_version, get_custom_venv_choices +from awx.main.utils import get_awx_version from awx.main.utils.licensing import validate_entitlement_manifest from awx.api.versioning import reverse, drf_reverse from awx.main.constants import PRIVILEGE_ESCALATION_METHODS diff --git a/awx/main/management/commands/custom_venv_associations.py b/awx/main/management/commands/custom_venv_associations.py index e1f749b79815..e80c97aad000 100644 --- a/awx/main/management/commands/custom_venv_associations.py +++ b/awx/main/management/commands/custom_venv_associations.py @@ -26,7 +26,8 @@ def handle(self, *args, **options): results = {} path = options.get('path') if path: - if path[0] in get_custom_venv_choices(): # verify this is a valid path + all_venvs = get_custom_venv_choices() + if path[0] in all_venvs: # verify this is a valid path path = path[0] orgs = [{"name": org.name, "id": org.id} for org in Organization.objects.filter(custom_virtualenv=path)] jts = [{"name": jt.name, "id": jt.id} for jt in JobTemplate.objects.filter(custom_virtualenv=path)] @@ -38,14 +39,21 @@ def handle(self, *args, **options): results["inventory_sources"] = invsrc print('\n', '# {}'.format("Virtual environments associations:")) print(yaml.dump(results)) - if not options.get('q'): - msg = [ - '', - 'To list all (now deprecated) custom virtual environments run:', - 'awx-manage list_custom_venvs', - '', - 'To export the contents of a (deprecated) virtual environment, ' 'run the following command while supplying the path as an argument:', - 'awx-manage export_custom_venv /path/to/venv', - '', - ] - print('\n'.join(msg)) + + if not options.get('q'): + msg = [ + '', + '- To list all (now deprecated) custom virtual environments run:', + 'awx-manage list_custom_venvs', + '', + '- To export the contents of a (deprecated) virtual environment, ' 'run the following command while supplying the path as an argument:', + 'awx-manage export_custom_venv /path/to/venv', + '', + '- Run these commands with `-q` to remove tool tips.', + '', + ] + print('\n'.join(msg)) + + else: + print('\n', '# Incorrect path, verify your path is from the following list:') + print('\n'.join(all_venvs), '\n') diff --git a/awx/main/management/commands/export_custom_venv.py b/awx/main/management/commands/export_custom_venv.py index 246bf151ef04..f267baa6bea5 100644 --- a/awx/main/management/commands/export_custom_venv.py +++ b/awx/main/management/commands/export_custom_venv.py @@ -1,7 +1,7 @@ # Copyright (c) 2021 Ansible, Inc. # All Rights Reserved -from awx.main.utils.common import get_custom_venv_pip_freeze +from awx.main.utils.common import get_custom_venv_pip_freeze, get_custom_venv_choices from django.core.management.base import BaseCommand @@ -21,17 +21,26 @@ def add_arguments(self, parser): def handle(self, *args, **options): super(Command, self).__init__() if options.get('path'): - pip_data = get_custom_venv_pip_freeze(options.get('path')[0]) - if pip_data: - print('\n', '# {}'.format("Virtual environment contents:")) - print(pip_data) - if not options.get('q'): - msg = [ - 'To list all (now deprecated) custom virtual environments run:', - 'awx-manage list_custom_venvs', - '', - 'To view the connections a (deprecated) virtual environment had in the database, run the following command while supplying the path as an argument:', - 'awx-manage custom_venv_associations /path/to/venv', - '', - ] - print('\n'.join(msg)) + path = options.get('path') + all_venvs = get_custom_venv_choices() + if path[0] in all_venvs: + pip_data = get_custom_venv_pip_freeze(options.get('path')[0]) + if pip_data: + print('\n', '# {}'.format("Virtual environment contents:")) + print(pip_data) + if not options.get('q'): + msg = [ + '- To list all (now deprecated) custom virtual environments run:', + 'awx-manage list_custom_venvs', + '', + '- To view the connections a (deprecated) virtual environment had in the database, run the following command while supplying the path as an argument:', + 'awx-manage custom_venv_associations /path/to/venv', + '', + '- Run these commands with `-q` to remove tool tips.', + '', + ] + print('\n'.join(msg)) + + else: + print('\n', '# Incorrect path, verify your path is from the following list:') + print('\n'.join(all_venvs)) diff --git a/awx/main/management/commands/list_custom_venvs.py b/awx/main/management/commands/list_custom_venvs.py index 6262d2300a0e..03534881b99e 100644 --- a/awx/main/management/commands/list_custom_venvs.py +++ b/awx/main/management/commands/list_custom_venvs.py @@ -18,21 +18,19 @@ def handle(self, *args, **options): venvs = get_custom_venv_choices() if venvs: print('\n', '# {}'.format("Discovered virtual environments:")) - for venv in venvs: - print(venv) + print('\n'.join(venvs), '\n') if not options.get('q'): msg = [ - '', - 'To export the contents of a (deprecated) virtual environment, ' 'run the following command while supplying the path as an argument:', + '- To export the contents of a (deprecated) virtual environment, ' 'run the following command while supplying the path as an argument:', 'awx-manage export_custom_venv /path/to/venv', '', - 'To view the connections a (deprecated) virtual environment had in the database, run the following command while supplying the path as an argument:', + '- To view the connections a (deprecated) virtual environment had in the database, run the following command while supplying the path as an argument:', 'awx-manage custom_venv_associations /path/to/venv', '', + '- Run these commands with `-q` to remove tool tips.', + '', ] print('\n'.join(msg)) - else: - print('\n') else: msg = ["No custom virtual environments detected in:", settings.BASE_VENV_PATH] From bd2da80cea4cd0c2df4af316da34c001a00a18b4 Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Thu, 3 Jun 2021 15:46:04 -0400 Subject: [PATCH 10/11] restructure so the -q is actually completely quiet --- awx/main/management/commands/custom_venv_associations.py | 8 ++++---- awx/main/management/commands/export_custom_venv.py | 6 ++++-- awx/main/management/commands/list_custom_venvs.py | 8 +++++--- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/awx/main/management/commands/custom_venv_associations.py b/awx/main/management/commands/custom_venv_associations.py index e80c97aad000..89c298f589a5 100644 --- a/awx/main/management/commands/custom_venv_associations.py +++ b/awx/main/management/commands/custom_venv_associations.py @@ -37,12 +37,10 @@ def handle(self, *args, **options): results["job_templates"] = jts results["projects"] = proj results["inventory_sources"] = invsrc - print('\n', '# {}'.format("Virtual environments associations:")) - print(yaml.dump(results)) - if not options.get('q'): msg = [ - '', + '# Virtual Environments Associations:', + yaml.dump(results), '- To list all (now deprecated) custom virtual environments run:', 'awx-manage list_custom_venvs', '', @@ -53,6 +51,8 @@ def handle(self, *args, **options): '', ] print('\n'.join(msg)) + else: + print(yaml.dump(results)) else: print('\n', '# Incorrect path, verify your path is from the following list:') diff --git a/awx/main/management/commands/export_custom_venv.py b/awx/main/management/commands/export_custom_venv.py index f267baa6bea5..82065b789d00 100644 --- a/awx/main/management/commands/export_custom_venv.py +++ b/awx/main/management/commands/export_custom_venv.py @@ -26,10 +26,10 @@ def handle(self, *args, **options): if path[0] in all_venvs: pip_data = get_custom_venv_pip_freeze(options.get('path')[0]) if pip_data: - print('\n', '# {}'.format("Virtual environment contents:")) - print(pip_data) if not options.get('q'): msg = [ + '# Virtual environment contents:', + pip_data, '- To list all (now deprecated) custom virtual environments run:', 'awx-manage list_custom_venvs', '', @@ -40,6 +40,8 @@ def handle(self, *args, **options): '', ] print('\n'.join(msg)) + else: + print(pip_data) else: print('\n', '# Incorrect path, verify your path is from the following list:') diff --git a/awx/main/management/commands/list_custom_venvs.py b/awx/main/management/commands/list_custom_venvs.py index 03534881b99e..0d84b5e3fd56 100644 --- a/awx/main/management/commands/list_custom_venvs.py +++ b/awx/main/management/commands/list_custom_venvs.py @@ -17,10 +17,11 @@ def handle(self, *args, **options): super(Command, self).__init__() venvs = get_custom_venv_choices() if venvs: - print('\n', '# {}'.format("Discovered virtual environments:")) - print('\n'.join(venvs), '\n') if not options.get('q'): msg = [ + '# Discovered Virtual Environments:', + '\n'.join(venvs), + '', '- To export the contents of a (deprecated) virtual environment, ' 'run the following command while supplying the path as an argument:', 'awx-manage export_custom_venv /path/to/venv', '', @@ -31,7 +32,8 @@ def handle(self, *args, **options): '', ] print('\n'.join(msg)) - + else: + print('\n'.join(venvs), '\n') else: msg = ["No custom virtual environments detected in:", settings.BASE_VENV_PATH] From baade775ab5e17698219ad6ea5851d322042a12d Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Thu, 3 Jun 2021 16:51:58 -0400 Subject: [PATCH 11/11] remove changes to root.py to keep the custom virtualenvs listed in api/config --- awx/api/views/root.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/awx/api/views/root.py b/awx/api/views/root.py index e464ca86bd10..59a62375d089 100644 --- a/awx/api/views/root.py +++ b/awx/api/views/root.py @@ -24,11 +24,11 @@ from awx.conf.registry import settings_registry from awx.main.analytics import all_collectors from awx.main.ha import is_ha_environment -from awx.main.utils import get_awx_version +from awx.main.utils import get_awx_version, get_custom_venv_choices from awx.main.utils.licensing import validate_entitlement_manifest from awx.api.versioning import reverse, drf_reverse from awx.main.constants import PRIVILEGE_ESCALATION_METHODS -from awx.main.models import Project, Organization, Instance, InstanceGroup +from awx.main.models import Project, Organization, Instance, InstanceGroup, JobTemplate from awx.main.utils import set_environ logger = logging.getLogger('awx.api.views.root') @@ -300,7 +300,15 @@ def get(self, request, format=None): or Organization.accessible_objects(request.user, 'auditor_role').exists() or Organization.accessible_objects(request.user, 'project_admin_role').exists() ): - data.update(dict(project_base_dir=settings.PROJECTS_ROOT, project_local_paths=Project.get_local_path_choices())) + data.update( + dict( + project_base_dir=settings.PROJECTS_ROOT, + project_local_paths=Project.get_local_path_choices(), + custom_virtualenvs=get_custom_venv_choices(), + ) + ) + elif JobTemplate.accessible_objects(request.user, 'admin_role').exists(): + data['custom_virtualenvs'] = get_custom_venv_choices() return Response(data)