From abc3652b003a3e14a7eae3e0f81c1a28d2b6106c Mon Sep 17 00:00:00 2001 From: Simon Murray Date: Mon, 14 Aug 2017 16:20:45 +0100 Subject: [PATCH] Fix TLS Race Connecting to InfluxDB Rather than leaving stale connections about we tried to poll for data coming in from InfluxDB and timeout if it didn't repond in a timely manner. This introduced a race where the timeout triggers, a context switch occurs where data is actually available and the TlsStream spins trying to asynchronously notify that data is available, but which never gets read. Not only does this use up 100% of a core, but it also slowly starves the system of handler threads at which point metrics stop being delivered. This basically removes the poll and timeout, any TLS socket erros should be detected by TCP keep-alives. Fixes #5460 #5469 --- doc/09-object-types.md | 1 - lib/perfdata/influxdbwriter.cpp | 15 +++------------ lib/perfdata/influxdbwriter.hpp | 2 +- lib/perfdata/influxdbwriter.ti | 3 --- 4 files changed, 4 insertions(+), 17 deletions(-) diff --git a/doc/09-object-types.md b/doc/09-object-types.md index 78b59192ce7..cceed0bb799 100644 --- a/doc/09-object-types.md +++ b/doc/09-object-types.md @@ -926,7 +926,6 @@ Configuration Attributes: enable_send_metadata | **Optional.** Whether to send check metadata e.g. states, execution time, latency etc. flush_interval | **Optional.** How long to buffer data points before transfering to InfluxDB. Defaults to `10s`. flush_threshold | **Optional.** How many data points to buffer before forcing a transfer to InfluxDB. Defaults to `1024`. - socket_timeout | **Optional.** How long to wait for InfluxDB to respond. Defaults to `5s`. Note: If `flush_threshold` is set too low, this will always force the feature to flush all data to InfluxDB. Experiment with the setting, if you are processing more than 1024 metrics per second diff --git a/lib/perfdata/influxdbwriter.cpp b/lib/perfdata/influxdbwriter.cpp index c6fa2c2c396..0a7f506fdab 100644 --- a/lib/perfdata/influxdbwriter.cpp +++ b/lib/perfdata/influxdbwriter.cpp @@ -134,9 +134,9 @@ void InfluxdbWriter::ExceptionHandler(boost::exception_ptr exp) //TODO: Close the connection, if we keep it open. } -Stream::Ptr InfluxdbWriter::Connect(TcpSocket::Ptr& socket) +Stream::Ptr InfluxdbWriter::Connect() { - socket = new TcpSocket(); + TcpSocket::Ptr socket = new TcpSocket(); Log(LogNotice, "InfluxdbWriter") << "Reconnecting to InfluxDB on host '" << GetHost() << "' port '" << GetPort() << "'."; @@ -423,8 +423,7 @@ void InfluxdbWriter::Flush(void) String body = boost::algorithm::join(m_DataBuffer, "\n"); m_DataBuffer.clear(); - TcpSocket::Ptr socket; - Stream::Ptr stream = Connect(socket); + Stream::Ptr stream = Connect(); if (!stream) return; @@ -462,14 +461,6 @@ void InfluxdbWriter::Flush(void) HttpResponse resp(stream, req); StreamReadContext context; - struct timeval timeout = { GetSocketTimeout(), 0 }; - - if (!socket->Poll(true, false, &timeout)) { - Log(LogWarning, "InfluxdbWriter") - << "Response timeout of TCP socket from host '" << GetHost() << "' port '" << GetPort() << "'."; - return; - } - try { resp.Parse(context, true); } catch (const std::exception& ex) { diff --git a/lib/perfdata/influxdbwriter.hpp b/lib/perfdata/influxdbwriter.hpp index dba570c45b1..e0a149c3a15 100644 --- a/lib/perfdata/influxdbwriter.hpp +++ b/lib/perfdata/influxdbwriter.hpp @@ -74,7 +74,7 @@ class InfluxdbWriter : public ObjectImpl static String EscapeKey(const String& str); static String EscapeField(const String& str); - Stream::Ptr Connect(TcpSocket::Ptr& socket); + Stream::Ptr Connect(); void AssertOnWorkQueue(void); diff --git a/lib/perfdata/influxdbwriter.ti b/lib/perfdata/influxdbwriter.ti index 94188471f64..a2bf1924802 100644 --- a/lib/perfdata/influxdbwriter.ti +++ b/lib/perfdata/influxdbwriter.ti @@ -90,9 +90,6 @@ class InfluxdbWriter : ConfigObject [config] int flush_threshold { default {{{ return 1024; }}} }; - [config] int socket_timeout { - default {{{ return 5; }}} - }; }; validator InfluxdbWriter {