From c8b792c0366a9aadc241de67250e85c36ddca8fb Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 24 Mar 2021 19:38:24 +0100 Subject: [PATCH] Add docs in PostgreSQL modules about recommended configuration (#24588) (cherry picked from commit 7f5a358cce09c050ead4d935905c1af515d00048) --- filebeat/docs/modules/postgresql.asciidoc | 75 +++++++++++++++++-- .../module/postgresql/_meta/docs.asciidoc | 75 +++++++++++++++++-- metricbeat/docs/modules/postgresql.asciidoc | 4 + metricbeat/metricbeat.reference.yml | 4 + .../postgresql/_meta/config.reference.yml | 4 + .../postgresql/statement/_meta/docs.asciidoc | 40 ++++++++++ x-pack/metricbeat/metricbeat.reference.yml | 4 + 7 files changed, 190 insertions(+), 16 deletions(-) diff --git a/filebeat/docs/modules/postgresql.asciidoc b/filebeat/docs/modules/postgresql.asciidoc index d13a54d11c58..66f292eaabb1 100644 --- a/filebeat/docs/modules/postgresql.asciidoc +++ b/filebeat/docs/modules/postgresql.asciidoc @@ -22,19 +22,63 @@ on Debian. include::../include/running-modules.asciidoc[] [float] -=== Example dashboards +=== Supported log formats -This module comes with two sample dashboards. +This module can collect any logs from PostgreSQL servers, but to be able to +better analyze their contents and extract more information, they should be +formatted in a determined way. -The first dashboard is for regular logs. +There are some settings to take into account for the log format. -[role="screenshot"] -image::./images/filebeat-postgresql-overview.png[] +Log lines should be preffixed with the timestamp in milliseconds, the process +id, the user id and the database name. This uses to be the default in most +distributions, and is translated to this setting in the configuration file: -The second one shows the slowlogs of PostgreSQL. +["source","sh"] +---------------------------- +log_line_prefix = '%m [%p] %q%u@%d ' +---------------------------- + +PostgreSQL server can be configured to log statements and their durations and +this module is able to collect this information. To be able to correlate each +duration with their statements, they must be logged in the same line. This +happens when the following options are used: + +["source","sh"] +---------------------------- +log_duration = 'on' +log_statement = 'none' +log_min_duration_statement = 0 +---------------------------- + +Setting a zero value in `log_min_duration_statement` will log all statements +executed by a client. You probably want to configure it to a higher value, so it +logs only slower statements. This value is configured in milliseconds. + +When using `log_statement` and `log_duration` together, statements and durations +are logged in different lines, and {beatname_uc} is not able to correlate both +values, for this reason it is recommended to disable `log_statement`. + +NOTE: The PostgreSQL module of Metricbeat is also able to collect information +about all statements executed in the server. You may chose which one is better +for your needings. An important difference is that the Metricbeat module +collects aggregated information when the statement is executed several times, +but cannot know when each statement was executed. This information can be +obtained from logs. + +Other logging options that you may consider to enable are the following ones: + +["source","sh"] +---------------------------- +log_checkpoints = 'on'; +log_connections = 'on'; +log_disconnections = 'on'; +log_lock_waits = 'on'; +---------------------------- + +Both `log_connections` and `log_disconnections` can cause a lot of events if you +don't have persistent connections, so enable with care. -[role="screenshot"] -image::./images/filebeat-postgresql-slowlog-overview.png[] include::../include/configuring-intro.asciidoc[] @@ -69,6 +113,21 @@ include::../include/config-option-intro.asciidoc[] include::../include/var-paths.asciidoc[] +[float] +=== Example dashboards + +This module comes with two sample dashboards. + +The first dashboard is for regular logs. + +[role="screenshot"] +image::./images/filebeat-postgresql-overview.png[] + +The second one shows the slowlogs of PostgreSQL. If `log_min_duration_statement` +is not used, this dashboard will show incomplete or no data. + +[role="screenshot"] +image::./images/filebeat-postgresql-slowlog-overview.png[] :has-dashboards!: diff --git a/filebeat/module/postgresql/_meta/docs.asciidoc b/filebeat/module/postgresql/_meta/docs.asciidoc index 7360720da065..11f264b06985 100644 --- a/filebeat/module/postgresql/_meta/docs.asciidoc +++ b/filebeat/module/postgresql/_meta/docs.asciidoc @@ -17,19 +17,63 @@ on Debian. include::../include/running-modules.asciidoc[] [float] -=== Example dashboards +=== Supported log formats -This module comes with two sample dashboards. +This module can collect any logs from PostgreSQL servers, but to be able to +better analyze their contents and extract more information, they should be +formatted in a determined way. -The first dashboard is for regular logs. +There are some settings to take into account for the log format. -[role="screenshot"] -image::./images/filebeat-postgresql-overview.png[] +Log lines should be preffixed with the timestamp in milliseconds, the process +id, the user id and the database name. This uses to be the default in most +distributions, and is translated to this setting in the configuration file: -The second one shows the slowlogs of PostgreSQL. +["source","sh"] +---------------------------- +log_line_prefix = '%m [%p] %q%u@%d ' +---------------------------- + +PostgreSQL server can be configured to log statements and their durations and +this module is able to collect this information. To be able to correlate each +duration with their statements, they must be logged in the same line. This +happens when the following options are used: + +["source","sh"] +---------------------------- +log_duration = 'on' +log_statement = 'none' +log_min_duration_statement = 0 +---------------------------- + +Setting a zero value in `log_min_duration_statement` will log all statements +executed by a client. You probably want to configure it to a higher value, so it +logs only slower statements. This value is configured in milliseconds. + +When using `log_statement` and `log_duration` together, statements and durations +are logged in different lines, and {beatname_uc} is not able to correlate both +values, for this reason it is recommended to disable `log_statement`. + +NOTE: The PostgreSQL module of Metricbeat is also able to collect information +about all statements executed in the server. You may chose which one is better +for your needings. An important difference is that the Metricbeat module +collects aggregated information when the statement is executed several times, +but cannot know when each statement was executed. This information can be +obtained from logs. + +Other logging options that you may consider to enable are the following ones: + +["source","sh"] +---------------------------- +log_checkpoints = 'on'; +log_connections = 'on'; +log_disconnections = 'on'; +log_lock_waits = 'on'; +---------------------------- + +Both `log_connections` and `log_disconnections` can cause a lot of events if you +don't have persistent connections, so enable with care. -[role="screenshot"] -image::./images/filebeat-postgresql-slowlog-overview.png[] include::../include/configuring-intro.asciidoc[] @@ -64,6 +108,21 @@ include::../include/config-option-intro.asciidoc[] include::../include/var-paths.asciidoc[] +[float] +=== Example dashboards + +This module comes with two sample dashboards. + +The first dashboard is for regular logs. + +[role="screenshot"] +image::./images/filebeat-postgresql-overview.png[] + +The second one shows the slowlogs of PostgreSQL. If `log_min_duration_statement` +is not used, this dashboard will show incomplete or no data. + +[role="screenshot"] +image::./images/filebeat-postgresql-slowlog-overview.png[] :has-dashboards!: diff --git a/metricbeat/docs/modules/postgresql.asciidoc b/metricbeat/docs/modules/postgresql.asciidoc index 2761cbbb1f08..7782b17321b2 100644 --- a/metricbeat/docs/modules/postgresql.asciidoc +++ b/metricbeat/docs/modules/postgresql.asciidoc @@ -77,6 +77,10 @@ metricbeat.modules: # Stats about every PostgreSQL process - activity + # Stats about every statement executed in the server. It requires the + # `pg_stats_statement` library to be configured in the server. + #- statement + period: 10s # The host must be passed as PostgreSQL URL. Example: diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 0f441e958dc6..51ca4b915ea0 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -559,6 +559,10 @@ metricbeat.modules: # Stats about every PostgreSQL process - activity + # Stats about every statement executed in the server. It requires the + # `pg_stats_statement` library to be configured in the server. + #- statement + period: 10s # The host must be passed as PostgreSQL URL. Example: diff --git a/metricbeat/module/postgresql/_meta/config.reference.yml b/metricbeat/module/postgresql/_meta/config.reference.yml index f27874eee36a..3b4ed4579d11 100644 --- a/metricbeat/module/postgresql/_meta/config.reference.yml +++ b/metricbeat/module/postgresql/_meta/config.reference.yml @@ -10,6 +10,10 @@ # Stats about every PostgreSQL process - activity + # Stats about every statement executed in the server. It requires the + # `pg_stats_statement` library to be configured in the server. + #- statement + period: 10s # The host must be passed as PostgreSQL URL. Example: diff --git a/metricbeat/module/postgresql/statement/_meta/docs.asciidoc b/metricbeat/module/postgresql/statement/_meta/docs.asciidoc index 6c188dce2d99..20f295c11707 100644 --- a/metricbeat/module/postgresql/statement/_meta/docs.asciidoc +++ b/metricbeat/module/postgresql/statement/_meta/docs.asciidoc @@ -1 +1,41 @@ This is the `statement` metricset of the PostgreSQL module. + +This module collects information from the `pg_stat_statements` view, that keeps +track of planning and execution statistics of all SQL statements executed by +the server. + +`pg_stat_statements` is included by an additional module in PostgreSQL. This +module requires additional shared memory, and is disabled by default. + +You can enable it by adding this module to the configuration as a shared +preloaded library. + +["source"] +------------------------------------------- +shared_preload_libraries = 'pg_stat_statements' +pg_stat_statements.max = 10000 +pg_stat_statements.track = all +------------------------------------------- + +NOTE: Preloading this library in your server will increase the memory usage of +your PostgreSQL server. Use it with care. + +Once the server is started with this module, it starts collecting statistics +about all statements executed. To make these statistics available in the +`pg_stat_statements` view, the following statement needs to be executed in the +server: + +["source","sql"] +------------------------------------------- +CREATE EXTENSION pg_stat_statements; +------------------------------------------- + +You can read more about the available options for this module in the +https://www.postgresql.org/docs/13/pgstatstatements.html[official documentation]. + +NOTE: The PostgreSQL module of Filebeat is also able to collect information +about statements executed in the server from its logs. You may chose which one +is better for your needings. An important difference is that the Metricbeat +module collects aggregated information when the statement is executed several +times, but cannot know when each statement was executed. This information can be +obtained from logs. diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 775e36af08c1..e9269da4213b 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -559,6 +559,10 @@ metricbeat.modules: # Stats about every PostgreSQL process - activity + # Stats about every statement executed in the server. It requires the + # `pg_stats_statement` library to be configured in the server. + #- statement + period: 10s # The host must be passed as PostgreSQL URL. Example: