Merge pull request #10090 from rebeccahhh/custom_venv_command

add a new awx-manage command `custom_venvs`

add an awx-manage command that gets pip freeze data from custom_venv and outputs to command line stdout
SUMMARY

part of #7062  - this command is a glorified pip freeze + some extra stuff, people could navigate to each of their custom virtual environments themselves and run a pip freeze, but this allows them to not, and everyone likes their life to be easier. The extra stuff allows users to see the connections that their existing virtual envs have in awx to things like organizations, jobs, inventory updates, and projects.

ISSUE TYPE


Feature Pull Request

COMPONENT NAME


API

AWX VERSION

awx: 19.1.0

ADDITIONAL INFORMATION

This is built off of existing code and there is a line that gets custom venv paths from the settings module, that line does not seem to be working. I have written around that but want to make a note of it.

Reviewed-by: Alan Rominger <arominge@redhat.com>
Reviewed-by: Rebeccah Hunter <rhunter@redhat.com>
Reviewed-by: Jeff Bradberry <None>
Reviewed-by: Shane McDonald <me@shanemcd.com>
Reviewed-by: Elijah DeLee <kdelee@redhat.com>

awx/api/views/root.py

@@ -301,7 +301,9 @@ def get(self, request, format=None):
         ):
             data.update(
                 dict(
-                    project_base_dir=settings.PROJECTS_ROOT, project_local_paths=Project.get_local_path_choices(), custom_virtualenvs=get_custom_venv_choices()
+                    project_base_dir=settings.PROJECTS_ROOT,
+                    project_local_paths=Project.get_local_path_choices(),
+                    custom_virtualenvs=get_custom_venv_choices(),
                 )
             )
         elif JobTemplate.accessible_objects(request.user, 'admin_role').exists():

awx/main/analytics/collectors.py

@@ -15,7 +15,7 @@
 from psycopg2.errors import UntranslatableCharacter
 
 from awx.conf.license import get_license
-from awx.main.utils import get_awx_version, get_custom_venv_choices, camelcase_to_underscore, datetime_hook
+from awx.main.utils import get_awx_version, camelcase_to_underscore, datetime_hook
 from awx.main import models
 from awx.main.analytics import register
 
@@ -115,7 +115,7 @@ def config(since, **kwargs):
     }
 
 
-@register('counts', '1.0', description=_('Counts of objects such as organizations, inventories, and projects'))
+@register('counts', '1.1', description=_('Counts of objects such as organizations, inventories, and projects'))
 def counts(since, **kwargs):
     counts = {}
     for cls in (
@@ -133,9 +133,6 @@ def counts(since, **kwargs):
     ):
         counts[camelcase_to_underscore(cls.__name__)] = cls.objects.count()
 
-    venvs = get_custom_venv_choices()
-    counts['custom_virtualenvs'] = len([v for v in venvs if os.path.basename(v.rstrip('/')) != 'ansible'])
-
     inv_counts = dict(models.Inventory.objects.order_by().values_list('kind').annotate(Count('kind')))
     inv_counts['normal'] = inv_counts.get('', 0)
     inv_counts.pop('', None)

awx/main/analytics/metrics.py

@@ -39,7 +39,6 @@ def metrics():
         ],
         registry=REGISTRY,
     )
-    CUSTOM_VENVS = Gauge('awx_custom_virtualenvs_total', 'Number of virtualenvs', registry=REGISTRY)
     RUNNING_JOBS = Gauge('awx_running_jobs_total', 'Number of running jobs on the system', registry=REGISTRY)
     PENDING_JOBS = Gauge('awx_pending_jobs_total', 'Number of pending jobs on the system', registry=REGISTRY)
     STATUS = Gauge(
@@ -159,7 +158,6 @@ def metrics():
     HOST_COUNT.labels(type='active').set(current_counts['active_host_count'])
 
     SCHEDULE_COUNT.set(current_counts['schedule'])
-    CUSTOM_VENVS.set(current_counts['custom_virtualenvs'])
 
     USER_SESSIONS.labels(type='all').set(current_counts['active_sessions'])
     USER_SESSIONS.labels(type='user').set(current_counts['active_user_sessions'])

awx/main/management/commands/custom_venv_associations.py

@@ -0,0 +1,59 @@
+# Copyright (c) 2021 Ansible, Inc.
+# All Rights Reserved
+
+from django.core.management.base import BaseCommand
+from awx.main.utils.common import get_custom_venv_choices
+from awx.main.models import Organization, InventorySource, JobTemplate, Project
+import yaml
+
+
+class Command(BaseCommand):
+    """Returns the pip freeze from the path passed in the argument"""
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            'path',
+            type=str,
+            nargs=1,
+            default='',
+            help='run this with a path to a virtual environment as an argument to see the associated Job Templates, Organizations, Projects, and Inventory Sources.',
+        )
+        parser.add_argument('-q', action='store_true', help='run with -q to output only the results of the query.')
+
+    def handle(self, *args, **options):
+        # look organiztions and unified job templates (which include JTs, workflows, and Inventory updates)
+        super(Command, self).__init__()
+        results = {}
+        path = options.get('path')
+        if path:
+            all_venvs = get_custom_venv_choices()
+            if path[0] in all_venvs:  # verify this is a valid path
+                path = path[0]
+                orgs = [{"name": org.name, "id": org.id} for org in Organization.objects.filter(custom_virtualenv=path)]
+                jts = [{"name": jt.name, "id": jt.id} for jt in JobTemplate.objects.filter(custom_virtualenv=path)]
+                proj = [{"name": proj.name, "id": proj.id} for proj in Project.objects.filter(custom_virtualenv=path)]
+                invsrc = [{"name": inv.name, "id": inv.id} for inv in InventorySource.objects.filter(custom_virtualenv=path)]
+                results["organizations"] = orgs
+                results["job_templates"] = jts
+                results["projects"] = proj
+                results["inventory_sources"] = invsrc
+                if not options.get('q'):
+                    msg = [
+                        '# Virtual Environments Associations:',
+                        yaml.dump(results),
+                        '- To list all (now deprecated) custom virtual environments run:',
+                        'awx-manage list_custom_venvs',
+                        '',
+                        '- To export the contents of a (deprecated) virtual environment, ' 'run the following command while supplying the path as an argument:',
+                        'awx-manage export_custom_venv /path/to/venv',
+                        '',
+                        '- Run these commands with `-q` to remove tool tips.',
+                        '',
+                    ]
+                    print('\n'.join(msg))
+                else:
+                    print(yaml.dump(results))
+
+            else:
+                print('\n', '# Incorrect path, verify your path is from the following list:')
+                print('\n'.join(all_venvs), '\n')

awx/main/management/commands/export_custom_venv.py

@@ -0,0 +1,48 @@
+# Copyright (c) 2021 Ansible, Inc.
+# All Rights Reserved
+
+from awx.main.utils.common import get_custom_venv_pip_freeze, get_custom_venv_choices
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+    """Returns the pip freeze from the path passed in the argument"""
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            'path',
+            type=str,
+            nargs=1,
+            default='',
+            help='run this with a path to a virtual environment as an argument to see the pip freeze data',
+        )
+        parser.add_argument('-q', action='store_true', help='run with -q to output only the results of the query.')
+
+    def handle(self, *args, **options):
+        super(Command, self).__init__()
+        if options.get('path'):
+            path = options.get('path')
+            all_venvs = get_custom_venv_choices()
+            if path[0] in all_venvs:
+                pip_data = get_custom_venv_pip_freeze(options.get('path')[0])
+                if pip_data:
+                    if not options.get('q'):
+                        msg = [
+                            '# Virtual environment contents:',
+                            pip_data,
+                            '- To list all (now deprecated) custom virtual environments run:',
+                            'awx-manage list_custom_venvs',
+                            '',
+                            '- To view the connections a (deprecated) virtual environment had in the database, run the following command while supplying the path as an argument:',
+                            'awx-manage custom_venv_associations /path/to/venv',
+                            '',
+                            '- Run these commands with `-q` to remove tool tips.',
+                            '',
+                        ]
+                        print('\n'.join(msg))
+                    else:
+                        print(pip_data)
+
+            else:
+                print('\n', '# Incorrect path, verify your path is from the following list:')
+                print('\n'.join(all_venvs))

awx/main/management/commands/list_custom_venvs.py

@@ -0,0 +1,43 @@
+# Copyright (c) 2021 Ansible, Inc.
+# All Rights Reserved
+import sys
+
+from awx.main.utils.common import get_custom_venv_choices
+from django.core.management.base import BaseCommand
+from django.conf import settings
+
+
+class Command(BaseCommand):
+    """Returns a list of custom venv paths from the path passed in the argument"""
+
+    def add_arguments(self, parser):
+        parser.add_argument('-q', action='store_true', help='run with -q to output only the results of the query.')
+
+    def handle(self, *args, **options):
+        super(Command, self).__init__()
+        venvs = get_custom_venv_choices()
+        if venvs:
+            if not options.get('q'):
+                msg = [
+                    '# Discovered Virtual Environments:',
+                    '\n'.join(venvs),
+                    '',
+                    '- To export the contents of a (deprecated) virtual environment, ' 'run the following command while supplying the path as an argument:',
+                    'awx-manage export_custom_venv /path/to/venv',
+                    '',
+                    '- To view the connections a (deprecated) virtual environment had in the database, run the following command while supplying the path as an argument:',
+                    'awx-manage custom_venv_associations /path/to/venv',
+                    '',
+                    '- Run these commands with `-q` to remove tool tips.',
+                    '',
+                ]
+                print('\n'.join(msg))
+            else:
+                print('\n'.join(venvs), '\n')
+        else:
+            msg = ["No custom virtual environments detected in:", settings.BASE_VENV_PATH]
+
+            for path in settings.CUSTOM_VENV_PATHS:
+                msg.append(path)
+
+            print('\n'.join(msg), file=sys.stderr)

awx/main/tests/functional/analytics/test_counts.py

@@ -12,7 +12,6 @@ def test_empty():
         "active_sessions": 0,
         "active_host_count": 0,
         "credential": 0,
-        "custom_virtualenvs": 0,  # dev env ansible3
         "host": 0,
         "inventory": 0,
         "inventories": {"normal": 0, "smart": 0},

awx/main/tests/functional/analytics/test_metrics.py

@@ -21,7 +21,6 @@
     'awx_sessions_total': 0.0,
     'awx_sessions_total': 0.0,
     'awx_sessions_total': 0.0,
-    'awx_custom_virtualenvs_total': 0.0,
     'awx_running_jobs_total': 0.0,
     'awx_instance_capacity': 100.0,
     'awx_instance_consumed_capacity': 0.0,

awx/main/utils/common.py

@@ -7,6 +7,7 @@
 import yaml
 import logging
 import os
+import subprocess
 import re
 import stat
 import subprocess
@@ -916,30 +917,35 @@ def get_current_apps():
     return current_apps
 
 
-def get_custom_venv_choices(custom_paths=None):
+def get_custom_venv_choices():
     from django.conf import settings
 
-    custom_paths = custom_paths or settings.CUSTOM_VENV_PATHS
-    all_venv_paths = [settings.BASE_VENV_PATH] + custom_paths
+    all_venv_paths = settings.CUSTOM_VENV_PATHS + [settings.BASE_VENV_PATH]
     custom_venv_choices = []
 
-    for custom_venv_path in all_venv_paths:
-        try:
-            if os.path.exists(custom_venv_path):
-                custom_venv_choices.extend(
-                    [
-                        os.path.join(custom_venv_path, x, '')
-                        for x in os.listdir(custom_venv_path)
-                        if x != 'awx'
-                        and os.path.isdir(os.path.join(custom_venv_path, x))
-                        and os.path.exists(os.path.join(custom_venv_path, x, 'bin', 'activate'))
-                    ]
-                )
-        except Exception:
-            logger.exception("Encountered an error while discovering custom virtual environments.")
+    for venv_path in all_venv_paths:
+        if os.path.exists(venv_path):
+            for d in os.listdir(venv_path):
+                if venv_path == settings.BASE_VENV_PATH and d == 'awx':
+                    continue
+
+                if os.path.exists(os.path.join(venv_path, d, 'bin', 'pip')):
+                    custom_venv_choices.append(os.path.join(venv_path, d))
+
     return custom_venv_choices
 
 
+def get_custom_venv_pip_freeze(venv_path):
+    pip_path = os.path.join(venv_path, 'bin', 'pip')
+
+    try:
+        freeze_data = subprocess.run([pip_path, "freeze"], capture_output=True)
+        pip_data = (freeze_data.stdout).decode('UTF-8')
+        return pip_data
+    except Exception:
+        logger.exception("Encountered an error while trying to run 'pip freeze' for custom virtual environments:")
+
+
 def is_ansible_variable(key):
     return key.startswith('ansible_')