diff --git a/awx/api/views/root.py b/awx/api/views/root.py index e8530aab1058..8784fddf6fc4 100644 --- a/awx/api/views/root.py +++ b/awx/api/views/root.py @@ -301,7 +301,9 @@ def get(self, request, format=None): ): data.update( dict( - project_base_dir=settings.PROJECTS_ROOT, project_local_paths=Project.get_local_path_choices(), custom_virtualenvs=get_custom_venv_choices() + project_base_dir=settings.PROJECTS_ROOT, + project_local_paths=Project.get_local_path_choices(), + custom_virtualenvs=get_custom_venv_choices(), ) ) elif JobTemplate.accessible_objects(request.user, 'admin_role').exists(): diff --git a/awx/main/analytics/collectors.py b/awx/main/analytics/collectors.py index d60c2ea46ced..b71cec45d678 100644 --- a/awx/main/analytics/collectors.py +++ b/awx/main/analytics/collectors.py @@ -15,7 +15,7 @@ from psycopg2.errors import UntranslatableCharacter from awx.conf.license import get_license -from awx.main.utils import get_awx_version, get_custom_venv_choices, camelcase_to_underscore, datetime_hook +from awx.main.utils import get_awx_version, camelcase_to_underscore, datetime_hook from awx.main import models from awx.main.analytics import register @@ -115,7 +115,7 @@ def config(since, **kwargs): } -@register('counts', '1.0', description=_('Counts of objects such as organizations, inventories, and projects')) +@register('counts', '1.1', description=_('Counts of objects such as organizations, inventories, and projects')) def counts(since, **kwargs): counts = {} for cls in ( @@ -133,9 +133,6 @@ def counts(since, **kwargs): ): counts[camelcase_to_underscore(cls.__name__)] = cls.objects.count() - venvs = get_custom_venv_choices() - counts['custom_virtualenvs'] = len([v for v in venvs if os.path.basename(v.rstrip('/')) != 'ansible']) - inv_counts = dict(models.Inventory.objects.order_by().values_list('kind').annotate(Count('kind'))) inv_counts['normal'] = inv_counts.get('', 0) inv_counts.pop('', None) diff --git a/awx/main/analytics/metrics.py b/awx/main/analytics/metrics.py index b9cd5153cc65..a32a7fc9595a 100644 --- a/awx/main/analytics/metrics.py +++ b/awx/main/analytics/metrics.py @@ -39,7 +39,6 @@ def metrics(): ], registry=REGISTRY, ) - CUSTOM_VENVS = Gauge('awx_custom_virtualenvs_total', 'Number of virtualenvs', registry=REGISTRY) RUNNING_JOBS = Gauge('awx_running_jobs_total', 'Number of running jobs on the system', registry=REGISTRY) PENDING_JOBS = Gauge('awx_pending_jobs_total', 'Number of pending jobs on the system', registry=REGISTRY) STATUS = Gauge( @@ -159,7 +158,6 @@ def metrics(): HOST_COUNT.labels(type='active').set(current_counts['active_host_count']) SCHEDULE_COUNT.set(current_counts['schedule']) - CUSTOM_VENVS.set(current_counts['custom_virtualenvs']) USER_SESSIONS.labels(type='all').set(current_counts['active_sessions']) USER_SESSIONS.labels(type='user').set(current_counts['active_user_sessions']) diff --git a/awx/main/management/commands/custom_venv_associations.py b/awx/main/management/commands/custom_venv_associations.py new file mode 100644 index 000000000000..89c298f589a5 --- /dev/null +++ b/awx/main/management/commands/custom_venv_associations.py @@ -0,0 +1,59 @@ +# Copyright (c) 2021 Ansible, Inc. +# All Rights Reserved + +from django.core.management.base import BaseCommand +from awx.main.utils.common import get_custom_venv_choices +from awx.main.models import Organization, InventorySource, JobTemplate, Project +import yaml + + +class Command(BaseCommand): + """Returns the pip freeze from the path passed in the argument""" + + def add_arguments(self, parser): + parser.add_argument( + 'path', + type=str, + nargs=1, + default='', + help='run this with a path to a virtual environment as an argument to see the associated Job Templates, Organizations, Projects, and Inventory Sources.', + ) + parser.add_argument('-q', action='store_true', help='run with -q to output only the results of the query.') + + def handle(self, *args, **options): + # look organiztions and unified job templates (which include JTs, workflows, and Inventory updates) + super(Command, self).__init__() + results = {} + path = options.get('path') + if path: + all_venvs = get_custom_venv_choices() + if path[0] in all_venvs: # verify this is a valid path + path = path[0] + orgs = [{"name": org.name, "id": org.id} for org in Organization.objects.filter(custom_virtualenv=path)] + jts = [{"name": jt.name, "id": jt.id} for jt in JobTemplate.objects.filter(custom_virtualenv=path)] + proj = [{"name": proj.name, "id": proj.id} for proj in Project.objects.filter(custom_virtualenv=path)] + invsrc = [{"name": inv.name, "id": inv.id} for inv in InventorySource.objects.filter(custom_virtualenv=path)] + results["organizations"] = orgs + results["job_templates"] = jts + results["projects"] = proj + results["inventory_sources"] = invsrc + if not options.get('q'): + msg = [ + '# Virtual Environments Associations:', + yaml.dump(results), + '- To list all (now deprecated) custom virtual environments run:', + 'awx-manage list_custom_venvs', + '', + '- To export the contents of a (deprecated) virtual environment, ' 'run the following command while supplying the path as an argument:', + 'awx-manage export_custom_venv /path/to/venv', + '', + '- Run these commands with `-q` to remove tool tips.', + '', + ] + print('\n'.join(msg)) + else: + print(yaml.dump(results)) + + else: + print('\n', '# Incorrect path, verify your path is from the following list:') + print('\n'.join(all_venvs), '\n') diff --git a/awx/main/management/commands/export_custom_venv.py b/awx/main/management/commands/export_custom_venv.py new file mode 100644 index 000000000000..82065b789d00 --- /dev/null +++ b/awx/main/management/commands/export_custom_venv.py @@ -0,0 +1,48 @@ +# Copyright (c) 2021 Ansible, Inc. +# All Rights Reserved + +from awx.main.utils.common import get_custom_venv_pip_freeze, get_custom_venv_choices +from django.core.management.base import BaseCommand + + +class Command(BaseCommand): + """Returns the pip freeze from the path passed in the argument""" + + def add_arguments(self, parser): + parser.add_argument( + 'path', + type=str, + nargs=1, + default='', + help='run this with a path to a virtual environment as an argument to see the pip freeze data', + ) + parser.add_argument('-q', action='store_true', help='run with -q to output only the results of the query.') + + def handle(self, *args, **options): + super(Command, self).__init__() + if options.get('path'): + path = options.get('path') + all_venvs = get_custom_venv_choices() + if path[0] in all_venvs: + pip_data = get_custom_venv_pip_freeze(options.get('path')[0]) + if pip_data: + if not options.get('q'): + msg = [ + '# Virtual environment contents:', + pip_data, + '- To list all (now deprecated) custom virtual environments run:', + 'awx-manage list_custom_venvs', + '', + '- To view the connections a (deprecated) virtual environment had in the database, run the following command while supplying the path as an argument:', + 'awx-manage custom_venv_associations /path/to/venv', + '', + '- Run these commands with `-q` to remove tool tips.', + '', + ] + print('\n'.join(msg)) + else: + print(pip_data) + + else: + print('\n', '# Incorrect path, verify your path is from the following list:') + print('\n'.join(all_venvs)) diff --git a/awx/main/management/commands/list_custom_venvs.py b/awx/main/management/commands/list_custom_venvs.py new file mode 100644 index 000000000000..0d84b5e3fd56 --- /dev/null +++ b/awx/main/management/commands/list_custom_venvs.py @@ -0,0 +1,43 @@ +# Copyright (c) 2021 Ansible, Inc. +# All Rights Reserved +import sys + +from awx.main.utils.common import get_custom_venv_choices +from django.core.management.base import BaseCommand +from django.conf import settings + + +class Command(BaseCommand): + """Returns a list of custom venv paths from the path passed in the argument""" + + def add_arguments(self, parser): + parser.add_argument('-q', action='store_true', help='run with -q to output only the results of the query.') + + def handle(self, *args, **options): + super(Command, self).__init__() + venvs = get_custom_venv_choices() + if venvs: + if not options.get('q'): + msg = [ + '# Discovered Virtual Environments:', + '\n'.join(venvs), + '', + '- To export the contents of a (deprecated) virtual environment, ' 'run the following command while supplying the path as an argument:', + 'awx-manage export_custom_venv /path/to/venv', + '', + '- To view the connections a (deprecated) virtual environment had in the database, run the following command while supplying the path as an argument:', + 'awx-manage custom_venv_associations /path/to/venv', + '', + '- Run these commands with `-q` to remove tool tips.', + '', + ] + print('\n'.join(msg)) + else: + print('\n'.join(venvs), '\n') + else: + msg = ["No custom virtual environments detected in:", settings.BASE_VENV_PATH] + + for path in settings.CUSTOM_VENV_PATHS: + msg.append(path) + + print('\n'.join(msg), file=sys.stderr) diff --git a/awx/main/tests/functional/analytics/test_counts.py b/awx/main/tests/functional/analytics/test_counts.py index 146afb604c62..a55065b6bd73 100644 --- a/awx/main/tests/functional/analytics/test_counts.py +++ b/awx/main/tests/functional/analytics/test_counts.py @@ -12,7 +12,6 @@ def test_empty(): "active_sessions": 0, "active_host_count": 0, "credential": 0, - "custom_virtualenvs": 0, # dev env ansible3 "host": 0, "inventory": 0, "inventories": {"normal": 0, "smart": 0}, diff --git a/awx/main/tests/functional/analytics/test_metrics.py b/awx/main/tests/functional/analytics/test_metrics.py index 2a04adcfcea5..3a8d1c441181 100644 --- a/awx/main/tests/functional/analytics/test_metrics.py +++ b/awx/main/tests/functional/analytics/test_metrics.py @@ -21,7 +21,6 @@ 'awx_sessions_total': 0.0, 'awx_sessions_total': 0.0, 'awx_sessions_total': 0.0, - 'awx_custom_virtualenvs_total': 0.0, 'awx_running_jobs_total': 0.0, 'awx_instance_capacity': 100.0, 'awx_instance_consumed_capacity': 0.0, diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index 6218baf7b309..c0f8de4abce0 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -7,6 +7,7 @@ import yaml import logging import os +import subprocess import re import stat import subprocess @@ -916,30 +917,35 @@ def get_current_apps(): return current_apps -def get_custom_venv_choices(custom_paths=None): +def get_custom_venv_choices(): from django.conf import settings - custom_paths = custom_paths or settings.CUSTOM_VENV_PATHS - all_venv_paths = [settings.BASE_VENV_PATH] + custom_paths + all_venv_paths = settings.CUSTOM_VENV_PATHS + [settings.BASE_VENV_PATH] custom_venv_choices = [] - for custom_venv_path in all_venv_paths: - try: - if os.path.exists(custom_venv_path): - custom_venv_choices.extend( - [ - os.path.join(custom_venv_path, x, '') - for x in os.listdir(custom_venv_path) - if x != 'awx' - and os.path.isdir(os.path.join(custom_venv_path, x)) - and os.path.exists(os.path.join(custom_venv_path, x, 'bin', 'activate')) - ] - ) - except Exception: - logger.exception("Encountered an error while discovering custom virtual environments.") + for venv_path in all_venv_paths: + if os.path.exists(venv_path): + for d in os.listdir(venv_path): + if venv_path == settings.BASE_VENV_PATH and d == 'awx': + continue + + if os.path.exists(os.path.join(venv_path, d, 'bin', 'pip')): + custom_venv_choices.append(os.path.join(venv_path, d)) + return custom_venv_choices +def get_custom_venv_pip_freeze(venv_path): + pip_path = os.path.join(venv_path, 'bin', 'pip') + + try: + freeze_data = subprocess.run([pip_path, "freeze"], capture_output=True) + pip_data = (freeze_data.stdout).decode('UTF-8') + return pip_data + except Exception: + logger.exception("Encountered an error while trying to run 'pip freeze' for custom virtual environments:") + + def is_ansible_variable(key): return key.startswith('ansible_')