Fix TLS Race Connecting to InfluxDB

Rather than leaving stale connections about we tried to poll for data coming in
from InfluxDB and timeout if it didn't repond in a timely manner.  This introduced
a race where the timeout triggers, a context switch occurs where data is actually
available and the TlsStream spins trying to asynchronously notify that data is
available, but which never gets read.  Not only does this use up 100% of a core,
but it also slowly starves the system of handler threads at which point metrics
stop being delivered.

This basically removes the poll and timeout, any TLS socket erros should be
detected by TCP keep-alives.

Fixes #5460 #5469

refs #5504

doc/09-object-types.md

@@ -926,7 +926,6 @@ Configuration Attributes:
   enable_send_metadata   | **Optional.** Whether to send check metadata e.g. states, execution time, latency etc.
   flush_interval         | **Optional.** How long to buffer data points before transfering to InfluxDB. Defaults to `10s`.
   flush_threshold        | **Optional.** How many data points to buffer before forcing a transfer to InfluxDB.  Defaults to `1024`.
-  socket_timeout         | **Optional.** How long to wait for InfluxDB to respond.  Defaults to `5s`.
 
 Note: If `flush_threshold` is set too low, this will always force the feature to flush all data
 to InfluxDB. Experiment with the setting, if you are processing more than 1024 metrics per second

lib/perfdata/influxdbwriter.cpp

@@ -134,9 +134,9 @@ void InfluxdbWriter::ExceptionHandler(boost::exception_ptr exp)
 	//TODO: Close the connection, if we keep it open.
 }
 
-Stream::Ptr InfluxdbWriter::Connect(TcpSocket::Ptr& socket)
+Stream::Ptr InfluxdbWriter::Connect()
 {
-	socket = new TcpSocket();
+	TcpSocket::Ptr socket = new TcpSocket();
 
 	Log(LogNotice, "InfluxdbWriter")
 	    << "Reconnecting to InfluxDB on host '" << GetHost() << "' port '" << GetPort() << "'.";
@@ -423,8 +423,7 @@ void InfluxdbWriter::Flush(void)
 	String body = boost::algorithm::join(m_DataBuffer, "\n");
 	m_DataBuffer.clear();
 
-	TcpSocket::Ptr socket;
-	Stream::Ptr stream = Connect(socket);
+	Stream::Ptr stream = Connect();
 
 	if (!stream)
 		return;
@@ -462,14 +461,6 @@ void InfluxdbWriter::Flush(void)
 	HttpResponse resp(stream, req);
 	StreamReadContext context;
 
-	struct timeval timeout = { GetSocketTimeout(), 0 };
-
-	if (!socket->Poll(true, false, &timeout)) {
-		Log(LogWarning, "InfluxdbWriter")
-		    << "Response timeout of TCP socket from host '" << GetHost() << "' port '" << GetPort() << "'.";
-		return;
-	}
-
 	try {
 		resp.Parse(context, true);
 	} catch (const std::exception& ex) {

lib/perfdata/influxdbwriter.hpp

@@ -74,7 +74,7 @@ class InfluxdbWriter : public ObjectImpl<InfluxdbWriter>
 	static String EscapeKey(const String& str);
 	static String EscapeField(const String& str);
 
-	Stream::Ptr Connect(TcpSocket::Ptr& socket);
+	Stream::Ptr Connect();
 
 	void AssertOnWorkQueue(void);
 

lib/perfdata/influxdbwriter.ti

@@ -90,9 +90,6 @@ class InfluxdbWriter : ConfigObject
 	[config] int flush_threshold {
 		default {{{ return 1024; }}}
 	};
-	[config] int socket_timeout {
-		default {{{ return 5; }}}
-	};
 };
 
 validator InfluxdbWriter {