Skip to content

Commit

Permalink
show agent health status
Browse files Browse the repository at this point in the history
  • Loading branch information
@wpbonelli
wpbonelli committed Jul 28, 2021
1 parent eaa248d commit 31c572b
Show file tree
Hide file tree
Showing 4 changed files with 111 additions and 31 deletions.
121 changes: 97 additions & 24 deletions plantit/front_end/src/components/agents/agent.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,26 +110,58 @@
v-if="getAgent.logo"
rounded
class="card-img-right overflow-hidden"
style="max-height: 5rem;position: absolute;right: 20px;top: 20px;z-index:1"
style="max-height: 3rem;position: absolute;right: 20px;top: 20px;z-index:1"
right
:src="getAgent.logo"
></b-img>
<b-row no-gutters>
<b-col>
<b-row>
<b-col md="auto">
<h2
<h1
:class="
profile.darkMode
? 'text-white'
: 'text-dark'
"
>
<i
v-if="
!getAgent.disabled
"
class="fas fa-server fa-fw"
></i>
<span
v-b-tooltip:hover
title="Disabled"
class="fa-stack fa-1x m-0 p-0"
v-else
><i
class="fas fa-server fa-stack-1x"
></i
><i
class="fas fa-slash fa-stack-1x text-danger"
></i
><i
class="far fa-circle fa-stack-2x text-danger"
></i
></span>
<i
v-if="
getAgent.is_healthy
"
v-b-tooltip:hover
title="Healthy"
class="fas fa-heartbeat text-warning fa-fw"
></i
><i
v-else
v-b-tooltip:hover
title="Unhealthy"
class="fas fa-medkit text-danger fa-fw"
></i>
{{ getAgent.name }}
</h2>
</h1>
<b-badge
v-if="
getAgent.role ===
Expand All @@ -146,14 +178,48 @@
variant="warning"
>Owner</b-badge
>
<br />
<small>{{
getAgent.description
}}</small>
</b-col>
<b-col style="position: relative; left: -20px" md="auto" align-self="middle"><i v-if="getAgent.is_healthy" v-b-tooltip:hover title="Healthy" class="fas fa-heartbeat text-warning fa-fw"></i><i v-else v-b-tooltip:hover title="Unhealthy" class="far fa-heart text-danger fa-fw"></i></b-col>
</b-row>
<hr />
<b-row
><b-col
><small>{{
getAgent.description
}}</small></b-col
></b-row
>

<div
v-if="
!getAgent.is_healthy &&
healthcheckOutput.length > 0
"
>
<br />
<b-row
:class="
profile.darkMode
? 'theme-container-dark m-0 p-1'
: 'theme-container-light m-0 p-1'
"
>
<b-col
class="m-0 p-0 pl-3 pr-3 pt-1 text-danger"
style="white-space: pre-line;"
>
<span
v-for="line in healthcheckOutput"
v-bind:key="line"
v-show="
line !==
undefined &&
line !== null
"
>{{ line + '\n' }}</span
>
</b-col></b-row
>
</div>
<br />
<b-row
><b-col>
<h5
Expand Down Expand Up @@ -207,6 +273,9 @@
>
</b-row>
</b-col>
</b-row>
<br />
<b-row>
<b-col>
<h5
:class="
Expand Down Expand Up @@ -1754,7 +1823,8 @@ export default {
searchWorkflows: false,
authorizingWorkflow: false,
blockingWorkflow: false,
authorizingUser: false
authorizingUser: false,
healthcheckOutput: []
};
},
computed: {
Expand Down Expand Up @@ -2458,20 +2528,23 @@ export default {
headers: { 'Content-Type': 'application/json' }
})
.then(async response => {
if (response.status === 200 && response.data.healthy)
await this.$store.dispatch('alerts/add', {
variant: 'success',
message: `Connection to ${this.getAgent.name} succeeded`,
guid: guid().toString(),
time: moment().format()
});
else
await this.$store.dispatch('alerts/add', {
variant: 'danger',
message: `Failed to connect to ${this.getAgent.name}`,
guid: guid().toString(),
time: moment().format()
});
if (response.status === 200) {
if (response.data.healthy)
await this.$store.dispatch('alerts/add', {
variant: 'success',
message: `Connection to ${this.getAgent.name} succeeded`,
guid: guid().toString(),
time: moment().format()
});
else
await this.$store.dispatch('alerts/add', {
variant: 'danger',
message: `Failed to connect to ${this.getAgent.name}`,
guid: guid().toString(),
time: moment().format()
});
}
this.healthcheckOutput = response.data.output;
this.checkingConnection = false;
})
.catch(async error => {
Expand Down
3 changes: 2 additions & 1 deletion plantit/plantit/agents/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,8 @@ def healthcheck(request, name):
except: return HttpResponseNotFound()

body = json.loads(request.body.decode('utf-8'))
return JsonResponse({'healthy': is_healthy(agent, body['auth'])})
healthy, output = is_healthy(agent, body['auth'])
return JsonResponse({'healthy': healthy, 'output': output})


@login_required
Expand Down
3 changes: 2 additions & 1 deletion plantit/plantit/celery_tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -540,7 +540,8 @@ def refresh_all_user_cyverse_tokens():
def agents_healthchecks():
agents = Agent.objects.filter(authentication=AgentAuthentication.KEY)
for agent in agents:
agent.is_healthy = is_healthy(agent, {'username': agent.user.username, 'port': agent.port})
health, _ = is_healthy(agent, {'username': agent.user.username, 'port': agent.port})
agent.is_healthy = health
agent.disabled = not agent.is_healthy
agent.save()

Expand Down
15 changes: 10 additions & 5 deletions plantit/plantit/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1757,7 +1757,7 @@ def has_virtual_memory(agent: Agent) -> bool:
return agent.header_skip == '--mem'


def is_healthy(agent: Agent, auth: dict) -> bool:
def is_healthy(agent: Agent, auth: dict) -> (bool, List[str]):
"""
Checks agent health
Expand All @@ -1767,6 +1767,7 @@ def is_healthy(agent: Agent, auth: dict) -> bool:
Returns: True if the agent was successfully reached, otherwise false.
"""
output = []
try:
if agent.authentication == AgentAuthentication.PASSWORD:
ssh = SSH(host=agent.hostname, port=int(auth['port']), username=auth['username'], password=auth['password'])
Expand All @@ -1775,12 +1776,16 @@ def is_healthy(agent: Agent, auth: dict) -> bool:

with ssh:
logger.info(f"Checking agent {agent.name}'s health")
for line in execute_command(ssh=ssh, precommand=':', command=f"pwd", directory=agent.workdir): logger.info(line)
for line in execute_command(ssh=ssh, precommand=':', command=f"pwd", directory=agent.workdir):
logger.info(line)
output.append(line)
logger.info(f"Agent {agent.name} healthcheck succeeded")
return True
return True, output
except:
logger.warning(f"Agent {agent.name} healthcheck failed:\n{traceback.format_exc()}")
return False
msg = f"Agent {agent.name} healthcheck failed:\n{traceback.format_exc()}"
logger.warning(msg)
output.append(msg)
return False, output


# MIAPPE
Expand Down

0 comments on commit 31c572b

Please sign in to comment.