APPSERV-19 Adds monitoring of stuck and hogging threads to monitoring console

appserver/monitoring-console/core/src/main/java/fish/payara/monitoring/model/SeriesAnnotation.java

@@ -60,13 +60,15 @@ public final class SeriesAnnotation implements Serializable, Iterable<Entry<Stri
     private final Series series;
     private final String instance;
     private final long value;
+    private final boolean keyed;
     private final String[] attrs;
 
-    public SeriesAnnotation(long time, Series series, String instance, long value, String[] attrs) {
+    public SeriesAnnotation(long time, Series series, String instance, long value, boolean keyed, String[] attrs) {
         this.time = time;
         this.series = series;
         this.instance = instance;
         this.value = value;
+        this.keyed = keyed && attrs.length >= 2;
         this.attrs = attrs;
         if (attrs.length % 2 == 1) {
             throw new IllegalArgumentException(
@@ -90,6 +92,17 @@ public String getInstance() {
         return instance;
     }
 
+    public boolean isKeyed() {
+        return keyed;
+    }
+
+    /**
+     * @return By convention the first attribute is the key, null if not defined
+     */
+    public String getKeyAttribute() {
+        return isKeyed() ? attrs[1] : null;
+    }
+
     public int getAttriuteCount() {
         return attrs.length / 2;
     }

appserver/monitoring-console/core/src/main/java/fish/payara/monitoring/store/ConsumingMonitoringDataCollector.java

@@ -78,10 +78,10 @@ public MonitoringDataCollector collect(CharSequence metric, long value) {
     }
 
     @Override
-    public MonitoringDataCollector annotate(CharSequence metric, long value, String... attrs) {
+    public MonitoringDataCollector annotate(CharSequence metric, long value, boolean keyed, String... attrs) {
         int length = tags.length();
         appendMetricName(tags, metric);
-        annotationConsumer.accept(tags, value, attrs);
+        annotationConsumer.accept(tags, value, keyed, attrs);
         tags.setLength(length);
         return this;
     }

appserver/monitoring-console/core/src/main/java/fish/payara/monitoring/store/InMemoryMonitoringDataRepository.java

@@ -56,6 +56,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Objects;
 import java.util.Queue;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
@@ -102,6 +103,7 @@
 @RunLevel(StartupRunLevel.VAL)
 public class InMemoryMonitoringDataRepository extends AbstractMonitoringService implements MonitoringDataRepository {
 
+    private static final int MAX_ANNOTATIONS_PER_SERIES = 20;
     /**
      * The topic name used to share data of instances with the DAS.
      */
@@ -289,19 +291,22 @@ private void addLocalPoint(CharSequence key, long value) {
         }
     }
 
-    private void addLocalAnnotation(CharSequence series, long value, String[] annotations) {
+    private void addLocalAnnotation(CharSequence series, long value, boolean keyed, String[] annotations) {
         Series s = seriesOrNull(series);
         if (s != null) {
-            addAnnotation(new SeriesAnnotation(collectedSecond, s, instanceName, value, annotations));
+            addAnnotation(new SeriesAnnotation(collectedSecond, s, instanceName, value, keyed, annotations));
         }
     }
 
     private void addAnnotation(SeriesAnnotation annotation) {
         Queue<SeriesAnnotation> annotations = annotationsBySeries.computeIfAbsent(annotation.getSeries(), //
                 key -> new ConcurrentLinkedQueue<>());
+        if (annotation.isKeyed()) {
+            annotations.removeIf(a -> Objects.equals(a.getKeyAttribute(), annotation.getKeyAttribute()));
+        }
         annotations.add(annotation);
-        if (annotations.size() > 20) {
-            annotations.poll();
+        if (annotations.size() > MAX_ANNOTATIONS_PER_SERIES) {
+                annotations.poll();
         }
     }
 
@@ -441,13 +446,13 @@ public void accept(CharSequence series, long value) {
         }
 
         @Override
-        public void accept(CharSequence series, long value, String[] attrs) {
+        public void accept(CharSequence series, long value, boolean keyed, String[] attrs) {
             if (this.annotations == null) {
                 this.annotations = new ArrayList<>();
             }
             Series s = seriesOrNull(series.toString());
             if (s != null) {
-                this.annotations.add(new SeriesAnnotation(time, s, instance, value, attrs));
+                this.annotations.add(new SeriesAnnotation(time, s, instance, value, keyed, attrs));
             }
         }
     }

appserver/monitoring-console/core/src/main/java/fish/payara/monitoring/store/MonitoringAnnotationConsumer.java

@@ -17,8 +17,10 @@ public interface MonitoringAnnotationConsumer {
      *
      * @param series      the full metric name, e.g. <code>x:y a:b xCount</code>
      * @param value       numeric value of the metric
+     * @param keyed       true, if the first attribute value given refers to the key used to identify duplicate 
+     *                    annotations, else false. Annotations for same key replace each other.
      * @param attrs       a sequence of key-value pairs the value is annotated with. 
      *                    For example: ["name", "Foo", "age", "7"]
      */
-    void accept(CharSequence series, long value, String[] attrs);
+    void accept(CharSequence series, long value, boolean keyed, String[] attrs);
 }

appserver/monitoring-console/core/src/test/java/fish/payara/monitoring/collect/ConsumingMonitoringDataCollectorTest.java

@@ -147,7 +147,7 @@ public void collectDataPoints() {
         annotationsBySeries.clear();
         MonitoringDataCollector collector = new ConsumingMonitoringDataCollector(
                 (series, value) -> dataPointsBySeries.put(series.toString(), value),
-                (series, value, attrs) -> annotationsBySeries.put(series.toString(), attrs));
+                (series, value, keyed, attrs) -> annotationsBySeries.put(series.toString(), attrs));
         collect(collector);
     }
 

appserver/monitoring-console/core/src/test/java/fish/payara/monitoring/model/SeriesAnnotationTest.java

@@ -42,6 +42,7 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -78,8 +79,32 @@ public void annotationAttributesCanBeIterated() {
         assertAttributesCanBeIterated("Key1", "Val1", "Key2", "Val2", "Key3", "Val3");
     }
 
+    @Test
+    public void annotationsWithKeyAreRecognised() {
+        assertFalse(newAnnotation(false).isKeyed());
+        assertFalse(newAnnotation(true).isKeyed());
+        assertCanBeNonKeyed("Key1", "Val1");
+        assertCanBeNonKeyed("Key1", "Val1", "Key2", "Val2");
+        assertCanBeNonKeyed("Key1", "Val1", "Key2", "Val2", "Key3", "Val3");
+        assertCanBeKeyed("Key1", "Val1");
+        assertCanBeKeyed("Key1", "Val1", "Key2", "Val2");
+        assertCanBeKeyed("Key1", "Val1", "Key2", "Val2", "Key3", "Val3");
+    }
+
+    private static void assertCanBeNonKeyed(String... attrs) {
+        SeriesAnnotation annotation = newAnnotation(false, attrs);
+        assertFalse(annotation.isKeyed());
+        assertNull(annotation.getKeyAttribute());
+    }
+
+    private static void assertCanBeKeyed(String... attrs) {
+        SeriesAnnotation annotation = newAnnotation(true, attrs);
+        assertTrue(annotation.isKeyed());
+        assertEquals(attrs[1], annotation.getKeyAttribute());
+    }
+
     private static void assertAttributesCanBeIterated(String... attrs) {
-        SeriesAnnotation annotation = new SeriesAnnotation(1L, Series.ANY, "instance", 1L, attrs);
+        SeriesAnnotation annotation = newAnnotation(false, attrs);
         assertEquals(attrs.length / 2, annotation.getAttriuteCount());
         Iterator<Entry<String, String>> iter = annotation.iterator();
         for (int i = 0; i < attrs.length; i+=2) {
@@ -94,15 +119,19 @@ private static void assertAttributesCanBeIterated(String... attrs) {
     private static void assertInvalidAttributes(String...attrs) {
         String attrsString = Arrays.toString(attrs);
         try {
-            assertNotNull(new SeriesAnnotation(1L, Series.ANY, "instance", 1L, attrs));
+            assertNotNull(newAnnotation(false, attrs));
             fail("Expected attributes cause exception but were accepted: " + attrsString);
         } catch (IllegalArgumentException ex) {
             assertEquals("Annotation attributes always must be given in pairs but got: " + attrsString, ex.getMessage());
         }
     }
 
     private static void assertValidAttributes(String...attrs) {
-        assertNotNull(new SeriesAnnotation(1L, Series.ANY, "instance", 1L, attrs));
+        assertNotNull(newAnnotation(false, attrs));
+    }
+
+    public static SeriesAnnotation newAnnotation(boolean keyed, String... attrs) {
+        return new SeriesAnnotation(1L, Series.ANY, "instance", 1L, keyed, attrs);
     }
 
 }

appserver/monitoring-console/webapp/src/main/webapp/js/mc-model.js

@@ -262,6 +262,24 @@ MonitoringConsole.Model = (function() {
 							decorations: { alerts: { noOngoing: true, noAcknowledged: true}},
 							options: { noAnnotations: true}},
 					],
+				},
+				threads: {
+					name: 'Threads',
+					numberOfColumns: 4,
+					widgets: [
+						{ series: 'ns:health StuckThreadDuration', type: 'annotation', mode: 'table', unit: 'ms',
+							displayName: 'Stuck Thread Incidents',
+							grid: {column: 0, item: 1, colspan: 3, rowspan: 1},
+							fields: ["Thread", "Started", "Value", "Threshold", "Suspended", "Locked", "State"]},
+						{ series: 'ns:health HoggingThreadDuration', type: 'annotation', mode: 'table', unit: 'ms',
+							displayName: 'Hogging Thread Incidents',
+							grid: {column: 0, item: 2, colspan: 3, rowspan: 1},
+							fields: ["Thread", "When", "Value", "Usage%", "Threshold%", "Method", "Exited"]},
+						{ series: 'ns:jvm ThreadCount', displayName: 'Live Threads', 
+							grid: {column: 3, item: 1}},
+						{ series: 'ns:jvm DaemonThreadCount', displayName: 'Daemon Threads', 
+							grid: {column: 3, item: 2}},							
+					],
 				}
 			},
 	};

nucleus/common/internal-api/src/main/java/fish/payara/monitoring/collect/MonitoringDataCollector.java

@@ -95,16 +95,25 @@ public interface MonitoringDataCollector {
      * @param value       value for the annotation instance, usually identical to current collected value but
      *                    potentially different in case of multiple instances that were aggregated to current collected
      *                    value.
+     * @param keyed       true, if the first attribute value given refers to the key used to identify duplicate 
+     *                    annotations, else false. Annotations for same key replace each other.
      * @param attrs       a sequence of key-value pairs the value is annotated with. 
      *                    For example: ["name", "Foo", "age", "7"]
      * @return this collector for chaining (with unchanged tags)
      */
-    MonitoringDataCollector annotate(CharSequence metric, long value, String... attrs);
+    MonitoringDataCollector annotate(CharSequence metric, long value, boolean keyed, String... attrs);
 
     /*
      * Helper methods for convenience and consistent tagging.
      */
 
+    /**
+     * Same as {@link #annotate(CharSequence, long, boolean, String...)} with {@code keyed} being {@code false}.
+     */
+    default MonitoringDataCollector annotate(CharSequence metric, long value, String... attrs) {
+        return annotate(metric, value, false, attrs);
+    }
+
     default MonitoringDataCollector prefix(CharSequence prefix) {
         MonitoringDataCollector self = this;
         return new MonitoringDataCollector() {
@@ -125,9 +134,9 @@ public MonitoringDataCollector collect(CharSequence metric, long value) {
             }
 
             @Override
-            public MonitoringDataCollector annotate(CharSequence metric, long value, String... attrs) {
+            public MonitoringDataCollector annotate(CharSequence metric, long value, boolean keyed, String... attrs) {
                 prefixed.setLength(prefix.length());
-                self.annotate(prefixed.append(metric), value, attrs);
+                self.annotate(prefixed.append(metric), value, keyed, attrs);
                 return this;
             }
         };

nucleus/payara-modules/healthcheck-core/src/main/java/fish/payara/nucleus/healthcheck/admin/HoggingThreadsConfigurer.java

@@ -2,7 +2,7 @@
  DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 
 
- Copyright (c) 2016-2019 Payara Foundation. All rights reserved.
+ Copyright (c) 2016-2020 Payara Foundation. All rights reserved.
 
 
  The contents of this file are subject to the terms of the Common Development
@@ -107,7 +107,7 @@ public class HoggingThreadsConfigurer implements AdminCommand {
     @Max(value = 100, message ="Threshold is a percentage so must be less than 100")
     private String threshold;
 
-    @Min(value = 1, message = "Retry count must be 1 or more")
+    @Min(value = 0, message = "Retry count must be zero or more")
     @Param(name = "retry-count")
     private String retryCount;
 

nucleus/payara-modules/healthcheck-core/src/main/java/fish/payara/nucleus/healthcheck/admin/SetHealthCheckServiceConfiguration.java

@@ -1,6 +1,6 @@
 /*
  *
- * Copyright (c) 2019 Payara Foundation and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019-2020 Payara Foundation and/or its affiliates. All rights reserved.
  *
  * The contents of this file are subject to the terms of either the GNU
  * General Public License Version 2 only ("GPL") or the Common Development
@@ -160,7 +160,7 @@ public class SetHealthCheckServiceConfiguration implements AdminCommand {
     @Max(value = 100, message ="Hogging threads threshold is a percentage so must be less than 100")
     private String hogginThreadsThreshold;
 
-    @Min(value = 1, message = "Hogging threads retry count must be 1 or more")
+    @Min(value = 0, message = "Hogging threads retry count must be zero or more")
     @Param(name = "hogging-threads-retry-count", optional = true)
     private String hogginThreadsRetryCount;