[Stack Monitoring] Add stale status reporting for Kibana

docs/settings/monitoring-settings.asciidoc

@@ -137,6 +137,10 @@ represent. Defaults to 10. If you modify the
 `monitoring.ui.collection.interval` in `elasticsearch.yml`, use the same
 value in this setting.
 
+`monitoring.ui.kibana.reporting.stale_status_threshold_seconds`::
+Specifies how many seconds can pass before the Kibana status reports are considered stale.
+Defaults to `120`.
+
 [float]
 [[monitoring-ui-cgroup-settings]]
 ===== Monitoring UI container settings

test/plugin_functional/test_suites/core_plugins/rendering.ts

@@ -130,6 +130,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) {
         'monitoring.kibana.collection.enabled (boolean)',
         'monitoring.kibana.collection.interval (number)',
         'monitoring.ui.ccs.enabled (boolean)',
+        'monitoring.ui.kibana.reporting.stale_status_threshold_seconds (number)',
         'monitoring.ui.container.apm.enabled (boolean)',
         'monitoring.ui.container.elasticsearch.enabled (boolean)',
         'monitoring.ui.container.logstash.enabled (boolean)',

x-pack/plugins/monitoring/public/application/contexts/external_config_context.tsx

@@ -12,6 +12,7 @@ export interface ExternalConfig {
   showCgroupMetricsElasticsearch: boolean;
   showCgroupMetricsLogstash: boolean;
   renderReactApp: boolean;
+  staleStatusThresholdSeconds: number;
 }
 
 export const ExternalConfigContext = createContext({} as ExternalConfig);

x-pack/plugins/monitoring/public/components/cluster/overview/helpers.js

@@ -70,7 +70,7 @@ export function HealthStatusIndicator(props) {
   return (
     <EuiFlexGroup alignItems="center" gutterSize="s">
       <EuiFlexItem grow={false}>
-        <EuiHealth color={statusColor} data-test-subj="statusIcon">
+        <EuiHealth color={statusColor} data-test-subj="status">
           <HealthLabel {...props} />
         </EuiHealth>
       </EuiFlexItem>

x-pack/plugins/monitoring/public/components/cluster/overview/kibana_panel.js

@@ -5,52 +5,54 @@
  * 2.0.
  */
 
-import React from 'react';
-import { formatNumber } from '../../../lib/format_number';
-import {
-  ClusterItemContainer,
-  HealthStatusIndicator,
-  BytesPercentageUsage,
-  DisabledIfNoDataAndInSetupModeLink,
-} from './helpers';
-import { get } from 'lodash';
 import {
+  EuiBadge,
+  EuiDescriptionList,
+  EuiDescriptionListDescription,
+  EuiDescriptionListTitle,
   EuiFlexGrid,
   EuiFlexGroup,
   EuiFlexItem,
+  EuiHorizontalRule,
   EuiLink,
-  EuiTitle,
   EuiPanel,
-  EuiDescriptionList,
-  EuiDescriptionListTitle,
-  EuiDescriptionListDescription,
-  EuiHorizontalRule,
+  EuiTitle,
+  EuiToolTip,
 } from '@elastic/eui';
-import { FormattedMessage } from '@kbn/i18n-react';
 import { i18n } from '@kbn/i18n';
-import { SetupModeTooltip } from '../../setup_mode/tooltip';
+import { FormattedMessage } from '@kbn/i18n-react';
+import { get } from 'lodash';
+import React from 'react';
 import { KIBANA_SYSTEM_ID, RULE_KIBANA_VERSION_MISMATCH } from '../../../../common/constants';
-import { getSafeForExternalLink } from '../../../lib/get_safe_for_external_link';
+import { SetupModeFeature } from '../../../../common/enums';
 import { AlertsBadge } from '../../../alerts/badge';
 import { shouldShowAlertBadge } from '../../../alerts/lib/should_show_alert_badge';
+import { ExternalConfigContext } from '../../../application/contexts/external_config_context';
+import { formatNumber } from '../../../lib/format_number';
+import { getSafeForExternalLink } from '../../../lib/get_safe_for_external_link';
 import { isSetupModeFeatureEnabled } from '../../../lib/setup_mode';
-import { SetupModeFeature } from '../../../../common/enums';
 import { SetupModeContext } from '../../setup_mode/setup_mode_context';
+import { SetupModeTooltip } from '../../setup_mode/tooltip';
+import {
+  BytesPercentageUsage,
+  ClusterItemContainer,
+  DisabledIfNoDataAndInSetupModeLink,
+  HealthStatusIndicator,
+} from './helpers';
 
 const INSTANCES_PANEL_ALERTS = [RULE_KIBANA_VERSION_MISMATCH];
 
 export function KibanaPanel(props) {
   const setupMode = props.setupMode;
   const alerts = props.alerts;
   const setupModeContext = React.useContext(SetupModeContext);
+  const { staleStatusThresholdSeconds } = React.useContext(ExternalConfigContext);
   const showDetectedKibanas =
     setupMode.enabled && get(setupMode.data, 'kibana.detected.doesExist', false);
   if (!props.count && !showDetectedKibanas) {
     return null;
   }
 
-  const statusIndicator = <HealthStatusIndicator status={props.status} product={'kb'} />;
-
   const goToKibana = () => getSafeForExternalLink('#/kibana');
   const goToInstances = () => getSafeForExternalLink('#/kibana/instances');
 
@@ -78,7 +80,12 @@ export function KibanaPanel(props) {
   return (
     <ClusterItemContainer
       {...props}
-      statusIndicator={statusIndicator}
+      statusIndicator={statusIndicator(
+        props.status,
+        props.some_status_is_stale,
+        goToInstances(),
+        staleStatusThresholdSeconds
+      )}
       url="kibana"
       title={i18n.translate('xpack.monitoring.cluster.overview.kibanaPanel.kibanaTitle', {
         defaultMessage: 'Kibana',
@@ -203,3 +210,42 @@ export function KibanaPanel(props) {
     </ClusterItemContainer>
   );
 }
+
+function statusIndicator(status, someStatusIsStale, instancesHref, staleStatusThresholdSeconds) {
+  if (!someStatusIsStale) {
+    return <HealthStatusIndicator status={status} product={'kb'} />;
+  }
+
+  const staleMessage = i18n.translate(
+    'xpack.monitoring.cluster.overview.kibanaPanel.staleStatusTooltip',
+    {
+      defaultMessage:
+        "It's been more than {staleStatusThresholdSeconds} seconds since we have heard from some instances.",
+      values: {
+        staleStatusThresholdSeconds,
+      },
+    }
+  );
+
+  return (
+    <>
+      <div style={{ marginBottom: '8px' }}>
+        <EuiToolTip position="top" content={staleMessage}>
+          <EuiBadge iconType="alert" color="warning" data-test-subj="status">
+            {i18n.translate('xpack.monitoring.cluster.overview.kibanaPanel.staleStatusLabel', {
+              defaultMessage: 'Stale',
+            })}
+          </EuiBadge>
+        </EuiToolTip>
+      </div>
+      <EuiLink href={instancesHref}>
+        {i18n.translate(
+          'xpack.monitoring.cluster.overview.kibanaPanel.staleStatusLinkToInstancesLabel',
+          {
+            defaultMessage: 'View all instances',
+          }
+        )}
+      </EuiLink>
+    </>
+  );
+}

x-pack/plugins/monitoring/public/components/kibana/cluster_status/index.js

@@ -5,11 +5,15 @@
  * 2.0.
  */
 
+import { EuiBadge, EuiLink, EuiStat, EuiToolTip } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
 import React from 'react';
-import { SummaryStatus } from '../../summary_status';
-import { KibanaStatusIcon } from '../status_icon';
+import { useLocation } from 'react-router-dom';
+import { ExternalConfigContext } from '../../../application/contexts/external_config_context';
 import { formatMetric } from '../../../lib/format_number';
-import { i18n } from '@kbn/i18n';
+import { getSafeForExternalLink } from '../../../lib/get_safe_for_external_link';
+import { DefaultStatusIndicator, SummaryStatus } from '../../summary_status';
+import { KibanaStatusIcon } from '../status_icon';
 
 export function ClusterStatus({ stats, alerts }) {
   const {
@@ -20,8 +24,12 @@ export function ClusterStatus({ stats, alerts }) {
     requests_total: requests,
     response_time_max: maxResponseTime,
     status,
+    some_status_is_stale: someStatusIsStale,
   } = stats;
 
+  const { staleStatusThresholdSeconds } = React.useContext(ExternalConfigContext);
+  const location = useLocation();
+
   const metrics = [
     {
       label: i18n.translate('xpack.monitoring.kibana.clusterStatus.instancesLabel', {
@@ -60,15 +68,107 @@ export function ClusterStatus({ stats, alerts }) {
     },
   ];
 
-  const IconComponent = ({ status }) => <KibanaStatusIcon status={status} />;
+  const StatusIndicator = () => {
+    if (!someStatusIsStale) {
+      return (
+        <DefaultStatusIndicator status={status} isOnline={true} IconComponent={KibanaStatusIcon} />
+      );
+    }
+
+    const staleMessage = i18n.translate(
+      'xpack.monitoring.kibana.clusterStatus.staleStatusTooltip',
+      {
+        defaultMessage:
+          "It's been more than {staleStatusThresholdSeconds} seconds since we have heard from some instances.",
+        values: {
+          staleStatusThresholdSeconds,
+        },
+      }
+    );
+
+    if (location.pathname === '/kibana') {
+      return <OverviewPageStatusIndicator staleMessage={staleMessage} />;
+    }
+
+    return <InstancesPageStatusIndicator staleMessage={staleMessage} />;
+  };
 
   return (
     <SummaryStatus
-      metrics={metrics}
-      status={status}
+      StatusIndicator={StatusIndicator}
       alerts={alerts}
-      IconComponent={IconComponent}
+      metrics={metrics}
       data-test-subj="kibanaClusterStatus"
     />
   );
 }
+
+function OverviewPageStatusIndicator({ staleMessage }) {
+  const instancesHref = getSafeForExternalLink('#/kibana/instances');
+
+  const title = (
+    <>
+      <div style={{ marginBottom: '8px' }}>
+        <EuiToolTip position="top" content={staleMessage}>
+          <EuiBadge iconType="alert" color="warning">
+            {i18n.translate(
+              'xpack.monitoring.kibana.clusterStatus.overview.staleStatusInstancesLabel',
+              {
+                defaultMessage: 'Stale',
+              }
+            )}
+          </EuiBadge>
+        </EuiToolTip>
+      </div>
+      <EuiLink href={instancesHref}>
+        {i18n.translate(
+          'xpack.monitoring.kibana.clusterStatus.overview.staleStatusLinkToInstancesLabel',
+          {
+            defaultMessage: 'View all instances',
+          }
+        )}
+      </EuiLink>
+    </>
+  );
+
+  return (
+    <EuiStat
+      data-test-subj="status"
+      description={i18n.translate('xpack.monitoring.kibana.clusterStatus.overview.statusLabel', {
+        defaultMessage: 'Status',
+      })}
+      title={title}
+      titleSize="xxxs"
+      textAlign="left"
+      className="monSummaryStatusNoWrap__stat"
+    />
+  );
+}
+
+function InstancesPageStatusIndicator({ staleMessage }) {
+  const title = (
+    <EuiToolTip position="top" content={staleMessage}>
+      <EuiBadge iconType="alert" color="warning">
+        {i18n.translate(
+          'xpack.monitoring.kibana.clusterStatus.instances.staleStatusInstancesLabel',
+          {
+            defaultMessage: 'Stale',
+          }
+        )}
+      </EuiBadge>
+    </EuiToolTip>
+  );
+
+  return (
+    <EuiStat
+      data-test-subj="status"
+      description={i18n.translate('xpack.monitoring.kibana.clusterStatus.instances.statusLabel', {
+        defaultMessage: 'Status',
+      })}
+      title={title}
+      titleSize="xxxs"
+      textAlign="left"
+      className="monSummaryStatusNoWrap__stat"
+    />
+  );
+}