Better prometheus support (#414)

* fix collector init

* Added promhttp server

* Fix memory leak

* Fix memory leak

* Minimum viable server

* Port is now set via config

* Port is now set via config

* Port is now set via config

* Better code organisation & config procession

* Fix docs

* Fix old typo in label_count

* More reliable init

* Shuffle metrics by label for more appropriate collection

* Done with shuffling

* Done, not tested

* Done, now rebase

* Format

* Fix port

Co-authored-by: tim-shlyap <tim-shlyap@yandex-team.ru>

CMakeLists.txt

@@ -125,6 +125,15 @@ set(od_libraries ${od_libraries} ${od_ldap})
 
 # use Prom
 find_package(Prom)
+
+if (PROMHTTP_FOUND)
+    set(od_promhttp ${PROMHTTP_LIBRARY})
+    include_directories(${PROMHTTP_INCLUDE_DIR})
+endif()
+
+set(od_libraries ${od_libraries} ${od_promhttp})
+
+
 if (PROM_FOUND)
     set(od_prom ${PROM_LIBRARY})
     include_directories(${PROM_INCLUDE_DIR})

cmake/FindProm.cmake

@@ -13,3 +13,12 @@ find_package_handle_standard_args(PROM REQUIRED_VARS PROM_LIBRARY PROM_INCLUDE_D
 if (PROM_FOUND)
     message(STATUS "Found PROM: ${PROM_LIBRARY}")
 endif()
+
+find_path(PROMHTTP_INCLUDE_DIR NAMES promhttp.h)
+find_library(PROMHTTP_LIBRARY promhttp)
+
+find_package_handle_standard_args(PROMHTTP REQUIRED_VARS PROMHTTP_LIBRARY PROMHTTP_INCLUDE_DIR)
+
+if (PROMHTTP_FOUND)
+    message(STATUS "Found PROMHTTP: ${PROMHTTP_LIBRARY}")
+endif()

config-examples/odyssey-dev-ldap.conf

@@ -18,7 +18,7 @@ log_session yes
 log_query yes
 log_stats yes
 stats_interval 60
-log_stats_prom yes
+log_stats_general_prom yes
 
 workers "auto"
 resolvers 1

config-examples/odyssey-dev-with-watchdog.conf

@@ -18,7 +18,7 @@ log_session yes
 log_query yes
 log_stats yes
 stats_interval 60
-log_stats_prom yes
+log_general_stats_prom yes
 
 workers "auto"
 resolvers 1

config-examples/odyssey-dev.conf

@@ -18,7 +18,9 @@ log_session yes
 log_query yes
 log_stats yes
 stats_interval 60
-log_stats_prom yes
+promhttp_server_port 7777
+log_general_stats_prom yes
+log_route_stats_prom no
 
 workers "auto"
 resolvers 1
@@ -37,7 +39,6 @@ keepalive_probes 9
 
 keepalive_usr_timeout 0
 
-
 listen {
 	host "*"
 	port 6432

documentation/configuration.md

@@ -150,11 +150,17 @@ Periodically display information about active routes.
 
 `log_stats yes`
 
-#### log\_stats_prom *yes|no*
+#### promhttp_server_port *integer*
 
-Write information about active routes in Prometheus format in addition to ordinary format. Requires [C Prometheus client library](https://github.com/digitalocean/prometheus-client-c) installed.
+Port on which metrics server listen. *http://localhost:port/* -- check is port running. *http://localhost:port/metrics* -- get metrics as a response.
 
-`log_stats_prom no`
+#### log\_general\_stats_prom *yes|no*
+
+Write information about active routes in Prometheus format in addition to ordinary format. Requires [C Prometheus client library](https://github.com/digitalocean/prometheus-client-c) installed. Log only info not specific to route
+
+### log\_route\_stats_prom *yes|no*
+
+Write information about active routes in Prometheus format in addition to ordinary format. Requires [C Prometheus client library](https://github.com/digitalocean/prometheus-client-c) installed. Log all available info
 
 #### stats\_interval *integer*
 

odyssey.conf

@@ -183,12 +183,26 @@ log_stats yes
 #
 stats_interval 60
 
+#
+# Port for Prometheus metrics http-server to listen
+# Require Prometheus C library and Promhttp library
+# Once port is set and server is started change require restart
+#
+# promhttp_server_port 7777
+
 #
 # Log stats in Prometheus format.
 #
 # Write stats in Prometheus format in addition to ordinary logs.
 #
-log_stats_prom no
+# Enable log general odyssey's log, not related to concrete routes
+#
+log_stats_general_prom no
+
+#
+# Enable log route related logs
+# Note, this option also turn on general prometheus logging
+log_stats_route_prom no
 
 ###
 ### PERFORMANCE

sources/build.h.cmake

@@ -14,5 +14,6 @@
 #cmakedefine PAM_FOUND @PAM_FOUND@
 #cmakedefine LDAP_FOUND @LDAP_FOUND@
 #cmakedefine PROM_FOUND @PROM_FOUND@
+#cmakedefine PROMHTTP_FOUND @PROMHTTP_FOUND@
 
 #endif /* ODYSSEY_BUILD_H */

sources/config.c

@@ -21,7 +21,8 @@ void od_config_init(od_config_t *config)
 	config->log_query = 0;
 	config->log_file = NULL;
 	config->log_stats = 1;
-	config->log_stats_prom = 0;
+	config->log_general_stats_prom = 0;
+	config->log_route_stats_prom = 0;
 	config->stats_interval = 3;
 	config->log_format = NULL;
 	config->pid_file = NULL;

sources/config.h

@@ -35,7 +35,8 @@ struct od_config {
 	char *log_file;
 	char *log_format;
 	int log_stats;
-	int log_stats_prom;
+	int log_general_stats_prom;
+	int log_route_stats_prom;
 	int log_syslog;
 	char *log_syslog_ident;
 	char *log_syslog_facility;

sources/config_reader.c

@@ -24,7 +24,12 @@ typedef enum {
 	OD_LLOG_FILE,
 	OD_LLOG_FORMAT,
 	OD_LLOG_STATS,
-	OD_LLOG_STATS_PROM,
+
+	/* Prometheus */
+	OD_LLOG_GENERAL_STATS_PROM,
+	OD_LLOG_ROUTE_STATS_PROM,
+	OD_LPROMHTTP_PORT,
+
 	OD_LPID_FILE,
 	OD_LUNIX_SOCKET_DIR,
 	OD_LUNIX_SOCKET_MODE,
@@ -156,12 +161,16 @@ static od_keyword_t od_config_keywords[] = {
 	od_keyword("log_file", OD_LLOG_FILE),
 	od_keyword("log_format", OD_LLOG_FORMAT),
 	od_keyword("log_stats", OD_LLOG_STATS),
-	od_keyword("log_stats_prom", OD_LLOG_STATS_PROM),
 	od_keyword("log_syslog", OD_LLOG_SYSLOG),
 	od_keyword("log_syslog_ident", OD_LLOG_SYSLOG_IDENT),
 	od_keyword("log_syslog_facility", OD_LLOG_SYSLOG_FACILITY),
 	od_keyword("stats_interval", OD_LSTATS_INTERVAL),
 
+	/* Prometheus */
+	od_keyword("log_general_stats_prom", OD_LLOG_GENERAL_STATS_PROM),
+	od_keyword("log_route_stats_prom", OD_LLOG_ROUTE_STATS_PROM),
+	od_keyword("promhttp_server_port", OD_LPROMHTTP_PORT),
+
 	/* listen */
 	od_keyword("listen", OD_LLISTEN),
 	od_keyword("host", OD_LHOST),
@@ -1762,13 +1771,6 @@ static int od_config_reader_parse(od_config_reader_t *reader,
 				goto error;
 			}
 			continue;
-		/* log_stats_prom */
-		case OD_LLOG_STATS_PROM:
-			if (!od_config_reader_yes_no(reader,
-						     &config->log_stats_prom)) {
-				goto error;
-			}
-			continue;
 		/* log_format */
 		case OD_LLOG_FORMAT:
 			if (!od_config_reader_string(reader,
@@ -1877,6 +1879,31 @@ static int od_config_reader_parse(od_config_reader_t *reader,
 				goto error;
 			}
 			continue;
+		/* log_stats_prom */
+		case OD_LLOG_GENERAL_STATS_PROM: {
+			if (!od_config_reader_yes_no(
+				    reader, &config->log_general_stats_prom))
+				goto error;
+			continue;
+		}
+		case OD_LLOG_ROUTE_STATS_PROM: {
+			if (!od_config_reader_yes_no(
+				    reader, &config->log_route_stats_prom))
+				goto error;
+			continue;
+		}
+		case OD_LPROMHTTP_PORT: {
+			int port;
+			if (!od_config_reader_number(reader, &port))
+				goto error;
+#ifdef PROMHTTP_FOUND
+			if (od_prom_set_port(
+				    port, ((od_cron_t *)(reader->global->cron))
+						  ->metrics) != OK_RESPONSE)
+				goto error;
+#endif
+			continue;
+		}
 		/* workers */
 		case OD_LWORKERS: {
 			od_token_t tok;

sources/cron.c

@@ -66,18 +66,21 @@ static int od_cron_stat_cb(od_route_t *route, od_stat_t *current,
 	od_route_unlock(route);
 
 #ifdef PROM_FOUND
-	if (instance->config.log_stats_prom) {
+	if (instance->config.log_general_stats_prom) {
 		od_prom_metrics_write_stat_cb(
 			metrics, info.user, info.database, info.database_len,
 			info.user_len, info.client_pool_total,
 			info.server_pool_active, info.server_pool_idle,
 			info.avg_count_tx, info.avg_tx_time,
 			info.avg_count_query, info.avg_query_time,
 			info.avg_recv_client, info.avg_recv_server);
-		const char *prom_log = od_prom_metrics_get_stat_cb(metrics);
-		od_logger_write_plain(&instance->logger, OD_LOG, "stats", NULL,
-				      NULL, prom_log);
-		od_prom_free(prom_log);
+		if (instance->config.log_route_stats_prom) {
+			const char *prom_log =
+				od_prom_metrics_get_stat_cb(metrics);
+			od_logger_write_plain(&instance->logger, OD_LOG,
+					      "stats", NULL, NULL, prom_log);
+			free(prom_log);
+		}
 	}
 #endif
 	od_log(&instance->logger, "stats", NULL, NULL,
@@ -119,7 +122,7 @@ static inline void od_cron_stat(od_cron_t *cron)
 			     &msg_allocated, &msg_cache_count,
 			     &msg_cache_gc_count, &msg_cache_size);
 #ifdef PROM_FOUND
-		if (instance->config.log_stats_prom) {
+		if (instance->config.log_general_stats_prom) {
 			od_prom_metrics_write_stat(
 				cron->metrics, msg_allocated, msg_cache_count,
 				msg_cache_gc_count, msg_cache_size,
@@ -128,7 +131,7 @@ static inline void od_cron_stat(od_cron_t *cron)
 				od_prom_metrics_get_stat(cron->metrics);
 			od_logger_write_plain(&instance->logger, OD_LOG,
 					      "stats", NULL, NULL, prom_log);
-			od_prom_free(prom_log);
+			free(prom_log);
 		}
 #endif
 		od_log(&instance->logger, "stats", NULL, NULL,
@@ -278,10 +281,8 @@ void od_cron_init(od_cron_t *cron)
 
 #ifdef PROM_FOUND
 	cron->metrics = (od_prom_metrics_t *)malloc(sizeof(od_prom_metrics_t));
-	int err = od_prom_metrics_init(cron->metrics);
-	if (err) {
-		fprintf(stdout, "Could not initialize metrics");
-	}
+	cron->metrics->port = 0;
+	cron->metrics->http_server = NULL;
 #endif
 
 	cron->online = 0;

sources/instance.c

@@ -117,6 +117,32 @@ int od_instance_main(od_instance_t *instance, int argc, char **argv)
 		goto error;
 	}
 
+#ifdef PROM_FOUND
+	rc = od_prom_metrics_init(cron.metrics);
+	if (rc != OK_RESPONSE) {
+		od_error(&instance->logger, "metrics", NULL, NULL,
+			 "failed to initialize metrics");
+		goto error;
+	}
+#ifdef PROMHTTP_FOUND
+	if (instance->config.log_route_stats_prom) {
+		rc = od_prom_activate_route_metrics(cron.metrics);
+		if (rc != OK_RESPONSE) {
+			od_error(&instance->logger, "promhttp", NULL, NULL,
+				 "%s", "could not activate prom_http server");
+			goto error;
+		}
+	} else if (instance->config.log_general_stats_prom) {
+		rc = od_prom_activate_general_metrics(cron.metrics);
+		if (rc != OK_RESPONSE) {
+			od_error(&instance->logger, "promhttp", NULL, NULL,
+				 "%s", "could not activate prom_http server");
+			goto error;
+		}
+	}
+#endif
+#endif
+
 	rc = od_apply_validate_cli_args(&instance->logger, &instance->config,
 					&args, &router.rules);
 	if (rc != OK_RESPONSE) {