From d0796e64c121129d27d66436066fa42844ebf3a7 Mon Sep 17 00:00:00 2001 From: Surya Sashank Nistala Date: Mon, 24 Jan 2022 23:43:19 +0530 Subject: [PATCH 1/3] Enables thread contention monitoring. Fixes calculation of average thread blocked time and average thread waited time Signed-off-by: Surya Sashank Nistala --- .../performanceanalyzer/jvm/ThreadList.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/java/org/opensearch/performanceanalyzer/jvm/ThreadList.java b/src/main/java/org/opensearch/performanceanalyzer/jvm/ThreadList.java index cfaae9d40..d38619365 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/jvm/ThreadList.java +++ b/src/main/java/org/opensearch/performanceanalyzer/jvm/ThreadList.java @@ -52,6 +52,12 @@ public class ThreadList { private static Lock vmAttachLock = new ReentrantLock(); + static { + if (threadBean.isThreadContentionMonitoringSupported()) { + threadBean.setThreadContentionMonitoringEnabled(true); + } + } + public static class ThreadState { public long javaTid; public long nativeTid; @@ -263,6 +269,9 @@ private static void parseThreadInfo(final ThreadInfo info) { 1.0e-3 * (t.blockedTime - oldt.blockedTime) / (t.blockedCount - oldt.blockedCount); + } else if (t.blockedCount == oldt.blockedCount && t.blockedTime > oldt.blockedTime) { + t.avgBlockedTime = + 1.0e-3 * (t.blockedTime - oldt.blockedTime + oldt.avgBlockedTime); } else { CircularLongArray arr = ThreadHistory.blockedTidHistoryMap.get(t.nativeTid); // NOTE: this is an upper bound @@ -275,6 +284,8 @@ private static void parseThreadInfo(final ThreadInfo info) { 1.0e-3 * (t.waitedTime - oldt.waitedTime) / (t.waitedCount - oldt.waitedCount); + } else if (t.waitedCount == oldt.waitedCount && t.waitedTime > oldt.waitedTime) { + t.avgWaitedTime = 1.0e-3 * (t.waitedTime - oldt.waitedTime + oldt.avgWaitedTime); } else { CircularLongArray arr = ThreadHistory.waitedTidHistoryMap.get(t.nativeTid); // NOTE: this is an upper bound From 5c78fd11c3f3f9c8aa3999fe598a8f950f7344ec Mon Sep 17 00:00:00 2001 From: Surya Sashank Nistala Date: Mon, 4 Apr 2022 17:48:54 +0530 Subject: [PATCH 2/3] check thread contention monitoring enabled/disabled via PA setting Signed-off-by: Surya Sashank Nistala --- null/data/batch_metrics_enabled.conf | 1 + null/data/thread_contention_monitoring_enabled.conf | 1 + .../collectors/OSMetricsCollector.java | 3 ++- .../PerformanceAnalyzerMetricsCollector.java | 9 +++++++++ .../ScheduledMetricCollectorsExecutor.java | 13 +++++++++++++ .../performanceanalyzer/jvm/ThreadList.java | 13 ++++++------- .../performanceanalyzer/jvm/ThreadListTests.java | 2 +- 7 files changed, 33 insertions(+), 9 deletions(-) create mode 100644 null/data/batch_metrics_enabled.conf create mode 100644 null/data/thread_contention_monitoring_enabled.conf diff --git a/null/data/batch_metrics_enabled.conf b/null/data/batch_metrics_enabled.conf new file mode 100644 index 000000000..f32a5804e --- /dev/null +++ b/null/data/batch_metrics_enabled.conf @@ -0,0 +1 @@ +true \ No newline at end of file diff --git a/null/data/thread_contention_monitoring_enabled.conf b/null/data/thread_contention_monitoring_enabled.conf new file mode 100644 index 000000000..f32a5804e --- /dev/null +++ b/null/data/thread_contention_monitoring_enabled.conf @@ -0,0 +1 @@ +true \ No newline at end of file diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/OSMetricsCollector.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/OSMetricsCollector.java index 97f6cf9ef..c154a4dc5 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/collectors/OSMetricsCollector.java +++ b/src/main/java/org/opensearch/performanceanalyzer/collectors/OSMetricsCollector.java @@ -45,7 +45,8 @@ public void collectMetrics(long startTime) { SchedMetricsGenerator schedMetricsGenerator = osMetricsGenerator.getSchedMetricsGenerator(); schedMetricsGenerator.addSample(); - Map threadStates = ThreadList.getNativeTidMap(); + Map threadStates = + ThreadList.getNativeTidMap(getThreadContentionMonitoringEnabled()); DiskIOMetricsGenerator diskIOMetricsGenerator = osMetricsGenerator.getDiskIOMetricsGenerator(); diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/PerformanceAnalyzerMetricsCollector.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/PerformanceAnalyzerMetricsCollector.java index fdff3afdf..00b559ab5 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/collectors/PerformanceAnalyzerMetricsCollector.java +++ b/src/main/java/org/opensearch/performanceanalyzer/collectors/PerformanceAnalyzerMetricsCollector.java @@ -33,6 +33,7 @@ enum State { private String collectorName; protected StringBuilder value; protected State state; + private boolean threadContentionMonitoringEnabled; protected PerformanceAnalyzerMetricsCollector(int timeInterval, String collectorName) { this.timeInterval = timeInterval; @@ -92,4 +93,12 @@ public State getState() { public void setState(State state) { this.state = state; } + + public void setThreadContentionMonitoringEnabled(boolean enabled) { + this.threadContentionMonitoringEnabled = enabled; + } + + public boolean getThreadContentionMonitoringEnabled() { + return threadContentionMonitoringEnabled; + } } diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/ScheduledMetricCollectorsExecutor.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/ScheduledMetricCollectorsExecutor.java index f1580bd79..afc69c6e6 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/collectors/ScheduledMetricCollectorsExecutor.java +++ b/src/main/java/org/opensearch/performanceanalyzer/collectors/ScheduledMetricCollectorsExecutor.java @@ -25,6 +25,7 @@ public class ScheduledMetricCollectorsExecutor extends Thread { private static final int COLLECTOR_THREAD_KEEPALIVE_SECS = 1000; private final boolean checkFeatureDisabledFlag; private boolean paEnabled = false; + private boolean threadContentionMonitoringEnabled = false; private int minTimeIntervalToSleep = Integer.MAX_VALUE; private Map metricsCollectors; @@ -52,7 +53,19 @@ public synchronized boolean getEnabled() { return paEnabled; } + public synchronized void setThreadContentionMonitoringEnabled(final boolean enabled) { + metricsCollectors + .keySet() + .forEach(collector -> collector.setThreadContentionMonitoringEnabled(enabled)); + threadContentionMonitoringEnabled = enabled; + } + + private synchronized boolean getThreadContentionMonitoringEnabled() { + return threadContentionMonitoringEnabled; + } + public void addScheduledMetricCollector(PerformanceAnalyzerMetricsCollector task) { + task.setThreadContentionMonitoringEnabled(getThreadContentionMonitoringEnabled()); metricsCollectors.put(task, System.currentTimeMillis() + task.getTimeInterval()); if (task.getTimeInterval() < minTimeIntervalToSleep) { minTimeIntervalToSleep = task.getTimeInterval(); diff --git a/src/main/java/org/opensearch/performanceanalyzer/jvm/ThreadList.java b/src/main/java/org/opensearch/performanceanalyzer/jvm/ThreadList.java index d38619365..f792c3d27 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/jvm/ThreadList.java +++ b/src/main/java/org/opensearch/performanceanalyzer/jvm/ThreadList.java @@ -52,12 +52,6 @@ public class ThreadList { private static Lock vmAttachLock = new ReentrantLock(); - static { - if (threadBean.isThreadContentionMonitoringSupported()) { - threadBean.setThreadContentionMonitoringEnabled(true); - } - } - public static class ThreadState { public long javaTid; public long nativeTid; @@ -123,8 +117,13 @@ public String toString() { * acquire this lock and move on if we could not get it. * * @return A hashmap of threadId to threadState. + * @param threadContentionMonitoringEnabled */ - public static Map getNativeTidMap() { + public static Map getNativeTidMap( + boolean threadContentionMonitoringEnabled) { + if (threadBean.isThreadContentionMonitoringSupported()) { + threadBean.setThreadContentionMonitoringEnabled(threadContentionMonitoringEnabled); + } if (vmAttachLock.tryLock()) { try { // Thread dumps are expensive and therefore we make sure that at least diff --git a/src/test/java/org/opensearch/performanceanalyzer/jvm/ThreadListTests.java b/src/test/java/org/opensearch/performanceanalyzer/jvm/ThreadListTests.java index f1d8137ba..dba96cd14 100644 --- a/src/test/java/org/opensearch/performanceanalyzer/jvm/ThreadListTests.java +++ b/src/test/java/org/opensearch/performanceanalyzer/jvm/ThreadListTests.java @@ -36,7 +36,7 @@ private static void runOnce() throws InterruptedException { String params[] = new String[0]; while (true) { ThreadList.runThreadDump(OSGlobals.getPid(), params); - ThreadList.LOGGER.info(ThreadList.getNativeTidMap().values()); + ThreadList.LOGGER.info(ThreadList.getNativeTidMap(false).values()); /*GCMetrics.runOnce(); HeapMetrics.runOnce(); From 40cc6b02a56898c4b8e86fe35f3be5e8b182713c Mon Sep 17 00:00:00 2001 From: Surya Sashank Nistala Date: Thu, 7 Apr 2022 00:06:11 +0530 Subject: [PATCH 3/3] remove test generated files Signed-off-by: Surya Sashank Nistala --- null/data/batch_metrics_enabled.conf | 1 - null/data/thread_contention_monitoring_enabled.conf | 1 - 2 files changed, 2 deletions(-) delete mode 100644 null/data/batch_metrics_enabled.conf delete mode 100644 null/data/thread_contention_monitoring_enabled.conf diff --git a/null/data/batch_metrics_enabled.conf b/null/data/batch_metrics_enabled.conf deleted file mode 100644 index f32a5804e..000000000 --- a/null/data/batch_metrics_enabled.conf +++ /dev/null @@ -1 +0,0 @@ -true \ No newline at end of file diff --git a/null/data/thread_contention_monitoring_enabled.conf b/null/data/thread_contention_monitoring_enabled.conf deleted file mode 100644 index f32a5804e..000000000 --- a/null/data/thread_contention_monitoring_enabled.conf +++ /dev/null @@ -1 +0,0 @@ -true \ No newline at end of file