From 31c572b58719d5b310a0b3a6e5872cfd6b4da4d9 Mon Sep 17 00:00:00 2001
From: Wes Bonelli <wpb3hw@virginia.edu>
Date: Wed, 28 Jul 2021 12:46:34 -0400
Subject: [PATCH] show agent health status

---
 .../front_end/src/components/agents/agent.vue | 121 ++++++++++++++----
 plantit/plantit/agents/views.py               |   3 +-
 plantit/plantit/celery_tasks.py               |   3 +-
 plantit/plantit/utils.py                      |  15 ++-
 4 files changed, 111 insertions(+), 31 deletions(-)

diff --git a/plantit/front_end/src/components/agents/agent.vue b/plantit/front_end/src/components/agents/agent.vue
index f410e46c..67997163 100644
--- a/plantit/front_end/src/components/agents/agent.vue
+++ b/plantit/front_end/src/components/agents/agent.vue
@@ -110,7 +110,7 @@
                                     v-if="getAgent.logo"
                                     rounded
                                     class="card-img-right overflow-hidden"
-                                    style="max-height: 5rem;position: absolute;right: 20px;top: 20px;z-index:1"
+                                    style="max-height: 3rem;position: absolute;right: 20px;top: 20px;z-index:1"
                                     right
                                     :src="getAgent.logo"
                                 ></b-img>
@@ -118,7 +118,7 @@
                                     <b-col>
                                         <b-row>
                                             <b-col md="auto">
-                                                <h2
+                                                <h1
                                                     :class="
                                                         profile.darkMode
                                                             ? 'text-white'
@@ -126,10 +126,42 @@
                                                     "
                                                 >
                                                     <i
+                                                        v-if="
+                                                            !getAgent.disabled
+                                                        "
                                                         class="fas fa-server fa-fw"
                                                     ></i>
+                                                    <span
+                                                        v-b-tooltip:hover
+                                                        title="Disabled"
+                                                        class="fa-stack fa-1x m-0 p-0"
+                                                        v-else
+                                                        ><i
+                                                            class="fas fa-server fa-stack-1x"
+                                                        ></i
+                                                        ><i
+                                                            class="fas fa-slash fa-stack-1x text-danger"
+                                                        ></i
+                                                        ><i
+                                                            class="far fa-circle fa-stack-2x text-danger"
+                                                        ></i
+                                                    ></span>
+                                                    <i
+                                                        v-if="
+                                                            getAgent.is_healthy
+                                                        "
+                                                        v-b-tooltip:hover
+                                                        title="Healthy"
+                                                        class="fas fa-heartbeat text-warning fa-fw"
+                                                    ></i
+                                                    ><i
+                                                        v-else
+                                                        v-b-tooltip:hover
+                                                        title="Unhealthy"
+                                                        class="fas fa-medkit text-danger fa-fw"
+                                                    ></i>
                                                     {{ getAgent.name }}
-                                                </h2>
+                                                </h1>
                                                 <b-badge
                                                     v-if="
                                                         getAgent.role ===
@@ -146,14 +178,48 @@
                                                     variant="warning"
                                                     >Owner</b-badge
                                                 >
-                                                <br />
-                                                <small>{{
-                                                    getAgent.description
-                                                }}</small>
                                             </b-col>
-                                          <b-col style="position: relative; left: -20px" md="auto" align-self="middle"><i v-if="getAgent.is_healthy" v-b-tooltip:hover title="Healthy" class="fas fa-heartbeat text-warning fa-fw"></i><i v-else v-b-tooltip:hover title="Unhealthy" class="far fa-heart text-danger fa-fw"></i></b-col>
                                         </b-row>
-                                        <hr />
+                                        <b-row
+                                            ><b-col
+                                                ><small>{{
+                                                    getAgent.description
+                                                }}</small></b-col
+                                            ></b-row
+                                        >
+
+                                        <div
+                                            v-if="
+                                                !getAgent.is_healthy &&
+                                                    healthcheckOutput.length > 0
+                                            "
+                                        >
+                                            <br />
+                                            <b-row
+                                                :class="
+                                                    profile.darkMode
+                                                        ? 'theme-container-dark m-0 p-1'
+                                                        : 'theme-container-light m-0 p-1'
+                                                "
+                                            >
+                                                <b-col
+                                                    class="m-0 p-0 pl-3 pr-3 pt-1 text-danger"
+                                                    style="white-space: pre-line;"
+                                                >
+                                                    <span
+                                                        v-for="line in healthcheckOutput"
+                                                        v-bind:key="line"
+                                                        v-show="
+                                                            line !==
+                                                                undefined &&
+                                                                line !== null
+                                                        "
+                                                        >{{ line + '\n' }}</span
+                                                    >
+                                                </b-col></b-row
+                                            >
+                                        </div>
+                                        <br />
                                         <b-row
                                             ><b-col>
                                                 <h5
@@ -207,6 +273,9 @@
                                                     >
                                                 </b-row>
                                             </b-col>
+                                        </b-row>
+                                        <br />
+                                        <b-row>
                                             <b-col>
                                                 <h5
                                                     :class="
@@ -1754,7 +1823,8 @@ export default {
             searchWorkflows: false,
             authorizingWorkflow: false,
             blockingWorkflow: false,
-            authorizingUser: false
+            authorizingUser: false,
+            healthcheckOutput: []
         };
     },
     computed: {
@@ -2458,20 +2528,23 @@ export default {
                 headers: { 'Content-Type': 'application/json' }
             })
                 .then(async response => {
-                    if (response.status === 200 && response.data.healthy)
-                        await this.$store.dispatch('alerts/add', {
-                            variant: 'success',
-                            message: `Connection to ${this.getAgent.name} succeeded`,
-                            guid: guid().toString(),
-                            time: moment().format()
-                        });
-                    else
-                        await this.$store.dispatch('alerts/add', {
-                            variant: 'danger',
-                            message: `Failed to connect to ${this.getAgent.name}`,
-                            guid: guid().toString(),
-                            time: moment().format()
-                        });
+                    if (response.status === 200) {
+                        if (response.data.healthy)
+                            await this.$store.dispatch('alerts/add', {
+                                variant: 'success',
+                                message: `Connection to ${this.getAgent.name} succeeded`,
+                                guid: guid().toString(),
+                                time: moment().format()
+                            });
+                        else
+                            await this.$store.dispatch('alerts/add', {
+                                variant: 'danger',
+                                message: `Failed to connect to ${this.getAgent.name}`,
+                                guid: guid().toString(),
+                                time: moment().format()
+                            });
+                    }
+                    this.healthcheckOutput = response.data.output;
                     this.checkingConnection = false;
                 })
                 .catch(async error => {
diff --git a/plantit/plantit/agents/views.py b/plantit/plantit/agents/views.py
index 850c20c4..f33cc1e0 100644
--- a/plantit/plantit/agents/views.py
+++ b/plantit/plantit/agents/views.py
@@ -320,7 +320,8 @@ def healthcheck(request, name):
     except: return HttpResponseNotFound()
 
     body = json.loads(request.body.decode('utf-8'))
-    return JsonResponse({'healthy': is_healthy(agent, body['auth'])})
+    healthy, output = is_healthy(agent, body['auth'])
+    return JsonResponse({'healthy': healthy, 'output': output})
 
 
 @login_required
diff --git a/plantit/plantit/celery_tasks.py b/plantit/plantit/celery_tasks.py
index 005aadd9..45858db8 100644
--- a/plantit/plantit/celery_tasks.py
+++ b/plantit/plantit/celery_tasks.py
@@ -540,7 +540,8 @@ def refresh_all_user_cyverse_tokens():
 def agents_healthchecks():
     agents = Agent.objects.filter(authentication=AgentAuthentication.KEY)
     for agent in agents:
-        agent.is_healthy = is_healthy(agent, {'username': agent.user.username, 'port': agent.port})
+        health, _ = is_healthy(agent, {'username': agent.user.username, 'port': agent.port})
+        agent.is_healthy = health
         agent.disabled = not agent.is_healthy
         agent.save()
 
diff --git a/plantit/plantit/utils.py b/plantit/plantit/utils.py
index 853af0c6..f1427895 100644
--- a/plantit/plantit/utils.py
+++ b/plantit/plantit/utils.py
@@ -1757,7 +1757,7 @@ def has_virtual_memory(agent: Agent) -> bool:
     return agent.header_skip == '--mem'
 
 
-def is_healthy(agent: Agent, auth: dict) -> bool:
+def is_healthy(agent: Agent, auth: dict) -> (bool, List[str]):
     """
     Checks agent health
 
@@ -1767,6 +1767,7 @@ def is_healthy(agent: Agent, auth: dict) -> bool:
 
     Returns: True if the agent was successfully reached, otherwise false.
     """
+    output = []
     try:
         if agent.authentication == AgentAuthentication.PASSWORD:
             ssh = SSH(host=agent.hostname, port=int(auth['port']), username=auth['username'], password=auth['password'])
@@ -1775,12 +1776,16 @@ def is_healthy(agent: Agent, auth: dict) -> bool:
 
         with ssh:
             logger.info(f"Checking agent {agent.name}'s health")
-            for line in execute_command(ssh=ssh, precommand=':', command=f"pwd", directory=agent.workdir): logger.info(line)
+            for line in execute_command(ssh=ssh, precommand=':', command=f"pwd", directory=agent.workdir):
+                logger.info(line)
+                output.append(line)
             logger.info(f"Agent {agent.name} healthcheck succeeded")
-            return True
+            return True, output
     except:
-        logger.warning(f"Agent {agent.name} healthcheck failed:\n{traceback.format_exc()}")
-        return False
+        msg = f"Agent {agent.name} healthcheck failed:\n{traceback.format_exc()}"
+        logger.warning(msg)
+        output.append(msg)
+        return False, output
 
 
 # MIAPPE