diff --git a/packages/postgresql/_dev/build/docs/README.md b/packages/postgresql/_dev/build/docs/README.md index 6ba032bb7d6..a54728b252b 100644 --- a/packages/postgresql/_dev/build/docs/README.md +++ b/packages/postgresql/_dev/build/docs/README.md @@ -4,7 +4,7 @@ This integration periodically fetches logs and metrics from [PostgreSQL](https:/ ## Compatibility -The `log` dataset was tested with logs from versions 9.5 on Ubuntu, 9.6 on Debian, and finally 10.11, 11.4 and 12.2 on Arch Linux 9.3. +The `log` dataset was tested with logs from versions 9.5 on Ubuntu, 9.6 on Debian, and finally 10.11, 11.4 and 12.2 on Arch Linux 9.3. CSV format was tested using versions 11 and 13 (distro is not relevant here). The `activity`, `bgwriter`, `database` and `statement` datasets were tested with PostgreSQL 9.5.3 and is expected to work with all versions >= 9. @@ -12,7 +12,32 @@ The `activity`, `bgwriter`, `database` and `statement` datasets were tested with ### log -The `log` dataset collects the PostgreSQL logs. +The `log` dataset collects the PostgreSQL logs in plain text format or CSV. + +#### Using CSV logs + +Since the PostgreSQL CSV log file is a well-defined format, +there is almost no configuration to be done in Fleet, just the filepath. + +On the other hand, it's necessary to configure PostgreSQL to emit `.csv` logs. + +The recommended parameters are: +``` +logging_collector = 'on'; +log_destination = 'csvlog'; +log_statement = 'none'; +log_checkpoints = on; +log_connections = on; +log_disconnections = on; +log_lock_waits = on; +log_min_duration_statement = 0; +``` + +In busy servers, `log_min_duration_statement` can cause contention, so you can assign +a value greater than 0. + +Both `log_connections` and `log_disconnections` can cause a lot of events if you don't have +persistent connections, so enable with care. {{fields "log"}} @@ -48,4 +73,4 @@ The `statement` dataset periodically fetches metrics from PostgreSQL servers. {{event "statement"}} -{{fields "statement"}} \ No newline at end of file +{{fields "statement"}} diff --git a/packages/postgresql/_dev/deploy/docker/Dockerfile b/packages/postgresql/_dev/deploy/docker/Dockerfile new file mode 100644 index 00000000000..0252a80a7d5 --- /dev/null +++ b/packages/postgresql/_dev/deploy/docker/Dockerfile @@ -0,0 +1,4 @@ +ARG SERVICE_VERSION=${SERVICE_VERSION:-9.5.3} +FROM postgres:${SERVICE_VERSION} +COPY docker-entrypoint-initdb.d /docker-entrypoint-initdb.d +HEALTHCHECK --interval=10s --retries=6 CMD psql -h localhost -U postgres -l diff --git a/packages/postgresql/_dev/deploy/docker/docker-compose.yml b/packages/postgresql/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 00000000000..e0367afafab --- /dev/null +++ b/packages/postgresql/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,11 @@ +version: '2.3' +services: + postgresql: + # Commented out `image:` below until we have a process to refresh the hosted images from + # Dockerfiles in this repo. Until then, we build the image locally using `build:` below. + # image: docker.elastic.co/integrations-ci/beats-postgresql:${POSTGRESQL_VERSION:-9.5.3}-1 + build: . + ports: + - 5432 + volumes: + - ${SERVICE_LOGS_DIR}/postgresql:/var/log/postgresql diff --git a/packages/postgresql/_dev/deploy/docker/docker-entrypoint-initdb.d/0-enable-logging.sh b/packages/postgresql/_dev/deploy/docker/docker-entrypoint-initdb.d/0-enable-logging.sh new file mode 100644 index 00000000000..283909d0dc8 --- /dev/null +++ b/packages/postgresql/_dev/deploy/docker/docker-entrypoint-initdb.d/0-enable-logging.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +chmod a+wx /var/log/postgresql + +cat <<-EOF >> $PGDATA/postgresql.conf +# Enable some log facilities. +log_duration = 'on' +log_connections = 'on' +log_disconnections = 'on' + +# Ensure that statements are logged, with their durations. +log_statement = 'none' +log_min_duration_statement = 0 + +# Give agent read permissions. In NO case for production usage. +log_file_mode = '0666' + +# Try to imitate logging behaviour in Debian/Ubuntu, but there the logging collector +# is not used. +logging_collector = 'on' +log_directory = '/var/log/postgresql' +log_line_prefix = '%m [%p] %q%u@%d ' +EOF diff --git a/packages/postgresql/_dev/deploy/docker/docker-entrypoint-initdb.d/1-enable-pg_stat_statements.sh b/packages/postgresql/_dev/deploy/docker/docker-entrypoint-initdb.d/1-enable-pg_stat_statements.sh new file mode 100644 index 00000000000..03b17c78789 --- /dev/null +++ b/packages/postgresql/_dev/deploy/docker/docker-entrypoint-initdb.d/1-enable-pg_stat_statements.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +cat <<-EOF >> $PGDATA/postgresql.conf +shared_preload_libraries = 'pg_stat_statements' +pg_stat_statements.max = 10000 +pg_stat_statements.track = all +EOF diff --git a/packages/postgresql/_dev/deploy/docker/docker-entrypoint-initdb.d/2-create-extension-pg_stat_statements.sql b/packages/postgresql/_dev/deploy/docker/docker-entrypoint-initdb.d/2-create-extension-pg_stat_statements.sql new file mode 100644 index 00000000000..41424ded1ef --- /dev/null +++ b/packages/postgresql/_dev/deploy/docker/docker-entrypoint-initdb.d/2-create-extension-pg_stat_statements.sql @@ -0,0 +1 @@ +create extension pg_stat_statements; \ No newline at end of file diff --git a/packages/postgresql/_dev/deploy/variants.yml b/packages/postgresql/_dev/deploy/variants.yml new file mode 100644 index 00000000000..6118a543c88 --- /dev/null +++ b/packages/postgresql/_dev/deploy/variants.yml @@ -0,0 +1,4 @@ +variants: + v9_5_3: + SERVICE_VERSION: 9.5.3 +default: v9_5_3 diff --git a/packages/postgresql/changelog.yml b/packages/postgresql/changelog.yml index 9e447ab2f1c..67eeb60e051 100644 --- a/packages/postgresql/changelog.yml +++ b/packages/postgresql/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.3.0" + changes: + - description: Add support for logs in CSV format + type: enhancement # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/integrations/pull/747 - version: "0.2.7" changes: - description: Updating package owner diff --git a/packages/postgresql/data_stream/activity/_dev/test/system/test-default-config.yml b/packages/postgresql/data_stream/activity/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..e01b0b57ac8 --- /dev/null +++ b/packages/postgresql/data_stream/activity/_dev/test/system/test-default-config.yml @@ -0,0 +1,7 @@ +vars: + hosts: + - postgres://{{Hostname}}:{{Port}}?sslmode=disable + username: postgres + password: postgres +data_stream: + vars: ~ diff --git a/packages/postgresql/data_stream/activity/fields/ecs.yml b/packages/postgresql/data_stream/activity/fields/ecs.yml index 50e1946dc41..4eee3428d3b 100644 --- a/packages/postgresql/data_stream/activity/fields/ecs.yml +++ b/packages/postgresql/data_stream/activity/fields/ecs.yml @@ -1,3 +1,67 @@ +- name: ecs + title: ECS + group: 2 + description: Meta-information specific to ECS. + type: group + fields: + - name: version + level: core + required: true + type: keyword + ignore_above: 1024 + description: 'ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. + + When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events.' + example: 1.0.0 +- name: error + title: Error + group: 2 + description: |- + These fields can represent errors of any kind. + Use them for errors that happen while fetching events or in cases where the + event itself contains an error. + type: group + fields: + - name: message + level: core + type: text + description: Error message. +- name: event + title: Event + group: 2 + description: 'The event fields are used for context information about the log or metric event itself. + + A log is defined as an event containing details of something that happened. Log events must include the time at which the thing happened. Examples of log events include a process starting on a host, a network packet being sent from a source to a destination, or a network connection between a client and a server being initiated or closed. A metric is defined as an event containing one or more numerical measurements and the time at which the measurement was taken. Examples of metric events include memory pressure measured on a host and device temperature. See the `event.kind` definition in this section for additional details about metric and state events.' + type: group + fields: + - name: dataset + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the dataset. + + If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. + + It''s recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name.' + example: apache.access + - name: duration + level: core + type: long + format: duration + input_format: nanoseconds + output_format: asMilliseconds + output_precision: 1 + description: 'Duration of the event in nanoseconds. + + If event.start and event.end are known this value should be the difference between the end and start time.' + - name: module + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the module this data is coming from. + + If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module.' + example: apache - name: service.address type: keyword description: Service address diff --git a/packages/postgresql/data_stream/bgwriter/_dev/test/system/test-default-config.yml b/packages/postgresql/data_stream/bgwriter/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..e01b0b57ac8 --- /dev/null +++ b/packages/postgresql/data_stream/bgwriter/_dev/test/system/test-default-config.yml @@ -0,0 +1,7 @@ +vars: + hosts: + - postgres://{{Hostname}}:{{Port}}?sslmode=disable + username: postgres + password: postgres +data_stream: + vars: ~ diff --git a/packages/postgresql/data_stream/bgwriter/fields/ecs.yml b/packages/postgresql/data_stream/bgwriter/fields/ecs.yml index 50e1946dc41..4eee3428d3b 100644 --- a/packages/postgresql/data_stream/bgwriter/fields/ecs.yml +++ b/packages/postgresql/data_stream/bgwriter/fields/ecs.yml @@ -1,3 +1,67 @@ +- name: ecs + title: ECS + group: 2 + description: Meta-information specific to ECS. + type: group + fields: + - name: version + level: core + required: true + type: keyword + ignore_above: 1024 + description: 'ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. + + When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events.' + example: 1.0.0 +- name: error + title: Error + group: 2 + description: |- + These fields can represent errors of any kind. + Use them for errors that happen while fetching events or in cases where the + event itself contains an error. + type: group + fields: + - name: message + level: core + type: text + description: Error message. +- name: event + title: Event + group: 2 + description: 'The event fields are used for context information about the log or metric event itself. + + A log is defined as an event containing details of something that happened. Log events must include the time at which the thing happened. Examples of log events include a process starting on a host, a network packet being sent from a source to a destination, or a network connection between a client and a server being initiated or closed. A metric is defined as an event containing one or more numerical measurements and the time at which the measurement was taken. Examples of metric events include memory pressure measured on a host and device temperature. See the `event.kind` definition in this section for additional details about metric and state events.' + type: group + fields: + - name: dataset + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the dataset. + + If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. + + It''s recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name.' + example: apache.access + - name: duration + level: core + type: long + format: duration + input_format: nanoseconds + output_format: asMilliseconds + output_precision: 1 + description: 'Duration of the event in nanoseconds. + + If event.start and event.end are known this value should be the difference between the end and start time.' + - name: module + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the module this data is coming from. + + If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module.' + example: apache - name: service.address type: keyword description: Service address diff --git a/packages/postgresql/data_stream/database/_dev/test/system/test-default-config.yml b/packages/postgresql/data_stream/database/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..e01b0b57ac8 --- /dev/null +++ b/packages/postgresql/data_stream/database/_dev/test/system/test-default-config.yml @@ -0,0 +1,7 @@ +vars: + hosts: + - postgres://{{Hostname}}:{{Port}}?sslmode=disable + username: postgres + password: postgres +data_stream: + vars: ~ diff --git a/packages/postgresql/data_stream/database/fields/ecs.yml b/packages/postgresql/data_stream/database/fields/ecs.yml index 50e1946dc41..d65cb5d1f80 100644 --- a/packages/postgresql/data_stream/database/fields/ecs.yml +++ b/packages/postgresql/data_stream/database/fields/ecs.yml @@ -1,3 +1,31 @@ +- name: ecs + title: ECS + group: 2 + description: Meta-information specific to ECS. + type: group + fields: + - name: version + level: core + required: true + type: keyword + ignore_above: 1024 + description: 'ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. + + When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events.' + example: 1.0.0 +- name: error + title: Error + group: 2 + description: |- + These fields can represent errors of any kind. + Use them for errors that happen while fetching events or in cases where the + event itself contains an error. + type: group + fields: + - name: message + level: core + type: text + description: Error message. - name: service.address type: keyword description: Service address diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-10-default.log b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-10-default.log new file mode 100644 index 00000000000..9d7ef78f5eb --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-10-default.log @@ -0,0 +1,20 @@ +2020-04-15 12:02:55.244 CEST [23922] LOG: database system was shut down at 2020-04-15 12:02:52 CEST +2020-04-15 12:02:55.247 CEST [23920] LOG: database system is ready to accept connections +2020-04-15 12:04:45.416 CEST [24981] FATAL: password authentication failed for user "root" +2020-04-15 12:04:45.416 CEST [24981] DETAIL: Role "root" does not exist. + Connection matched pg_hba.conf line 80: "local all all md5" +2020-04-15 12:04:45.416 CEST [24981] LOG: could not send data to client: Broken pipe +2020-04-15 12:06:36.719 CEST [25143] ERROR: syntax error at or near "l" at character 1 +2020-04-15 12:56:29.569 CEST [25143] STATEMENT: SELECT al.id, al.tenant_id, al.created_by_id, al.create_ip, al.audit_date, al.audit_table, al.entity_id, al.entity_name, al.reason_for_change, al.audit_log_event_type_id, + aet.lookup_code, al.old_value, al.new_value, al.event_crf_id, al.event_crf_version_id, al.study_id, al.study_site_id, ss.rc_oid, al.subject_id, s.unique_identifier, + al.study_event_id, sed.name AS studyEventName, al.user_id, al.value_index, al.crf_version_id, al.global_logs, cv.version_name, crf.id AS crfId, crf.name AS crfName + FROM public.rc_audit_log_events AS al + LEFT JOIN rc_crf_versions AS cv ON cv.id=al.crf_version_id + LEFT JOIN rc_crfs AS crf ON crf.id=cv.crf_id + LEFT JOIN ad_lookup_codes AS aet ON aet.id=al.audit_log_event_type_id + LEFT JOIN rc_study_sites AS ss ON ss.id=al.study_site_id + LEFT JOIN rc_subjects AS s ON s.id=al.subject_id + LEFT JOIN rc_study_events AS se ON se.id=al.study_event_id + LEFT JOIN rc_study_event_definitions AS sed ON sed.id=se.study_event_definition_id + WHERE al.tenant_id=$1 AND al.study_id=$2 AND aet.lookup_code IN ($3, $4, $5, $6) AND al.audit_date >= $7 ORDER BY al.id DESC limit $8 + ; diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-10-default.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-10-default.log-config.yml new file mode 100644 index 00000000000..860a8bfb242 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-10-default.log-config.yml @@ -0,0 +1,4 @@ +dynamic_fields: + event.ingested: ".*" +multiline: + first_line_pattern: '^\d{4}-\d{2}-\d{2} ' diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-10-default.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-10-default.log-expected.json new file mode 100644 index 00000000000..30b8a5c305a --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-10-default.log-expected.json @@ -0,0 +1,186 @@ +{ + "expected": [ + { + "process": { + "pid": 23922 + }, + "@timestamp": "2020-04-15T10:02:55.244Z", + "postgresql": { + "log": { + "timestamp": "2020-04-15 12:02:55.244 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system was shut down at 2020-04-15 12:02:52 CEST", + "event": { + "ingested": "2021-03-03T14:56:50.279010861Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 23920 + }, + "@timestamp": "2020-04-15T10:02:55.247Z", + "postgresql": { + "log": { + "timestamp": "2020-04-15 12:02:55.247 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system is ready to accept connections", + "event": { + "ingested": "2021-03-03T14:56:50.279015846Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 24981 + }, + "@timestamp": "2020-04-15T10:04:45.416Z", + "postgresql": { + "log": { + "timestamp": "2020-04-15 12:04:45.416 CEST" + } + }, + "log": { + "level": "FATAL" + }, + "message": "password authentication failed for user \"root\"", + "event": { + "ingested": "2021-03-03T14:56:50.279017436Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 24981 + }, + "@timestamp": "2020-04-15T10:04:45.416Z", + "postgresql": { + "log": { + "timestamp": "2020-04-15 12:04:45.416 CEST" + } + }, + "log": { + "level": "DETAIL" + }, + "message": "Role \"root\" does not exist.\n\tConnection matched pg_hba.conf line 80: \"local all all md5\"", + "event": { + "ingested": "2021-03-03T14:56:50.279021876Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 24981 + }, + "@timestamp": "2020-04-15T10:04:45.416Z", + "postgresql": { + "log": { + "timestamp": "2020-04-15 12:04:45.416 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "could not send data to client: Broken pipe", + "event": { + "ingested": "2021-03-03T14:56:50.279023548Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 25143 + }, + "@timestamp": "2020-04-15T10:06:36.719Z", + "postgresql": { + "log": { + "timestamp": "2020-04-15 12:06:36.719 CEST" + } + }, + "log": { + "level": "ERROR" + }, + "message": "syntax error at or near \"l\" at character 1", + "event": { + "ingested": "2021-03-03T14:56:50.279024940Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 25143 + }, + "@timestamp": "2020-04-15T10:56:29.569Z", + "postgresql": { + "log": { + "timestamp": "2020-04-15 12:56:29.569 CEST" + } + }, + "log": { + "level": "STATEMENT" + }, + "message": "SELECT al.id, al.tenant_id, al.created_by_id, al.create_ip, al.audit_date, al.audit_table, al.entity_id, al.entity_name, al.reason_for_change, al.audit_log_event_type_id,\n\t aet.lookup_code, al.old_value, al.new_value, al.event_crf_id, al.event_crf_version_id, al.study_id, al.study_site_id, ss.rc_oid, al.subject_id, s.unique_identifier,\n\t al.study_event_id, sed.name AS studyEventName, al.user_id, al.value_index, al.crf_version_id, al.global_logs, cv.version_name, crf.id AS crfId, crf.name AS crfName\n\t FROM public.rc_audit_log_events AS al\n\t LEFT JOIN rc_crf_versions AS cv ON cv.id=al.crf_version_id\n\t LEFT JOIN rc_crfs AS crf ON crf.id=cv.crf_id\n\t LEFT JOIN ad_lookup_codes AS aet ON aet.id=al.audit_log_event_type_id\n\t LEFT JOIN rc_study_sites AS ss ON ss.id=al.study_site_id\n\t LEFT JOIN rc_subjects AS s ON s.id=al.subject_id\n\t LEFT JOIN rc_study_events AS se ON se.id=al.study_event_id\n\t LEFT JOIN rc_study_event_definitions AS sed ON sed.id=se.study_event_definition_id\n\t WHERE al.tenant_id=$1 AND al.study_id=$2 AND aet.lookup_code IN ($3, $4, $5, $6) AND al.audit_date \u003e= $7 ORDER BY al.id DESC limit $8\n\t;", + "event": { + "ingested": "2021-03-03T14:56:50.279026894Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + } + ] +} \ No newline at end of file diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-10-min-duration-statement.log b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-10-min-duration-statement.log new file mode 100644 index 00000000000..1cc9817b195 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-10-min-duration-statement.log @@ -0,0 +1,17 @@ +2019-09-22 06:28:24 UTC DETAIL: parameters: $1 = '86', $2 = '575', $3 = 'Item Inserted', $4 = 'Item Updated', $5 = 'Subject Updated', $6 = 'Subject Created', $7 = '2019-01-22 00:00:00+00' +2019-09-22 06:28:24 UTC LOG: duration: 112.337 ms execute S_59: UPDATE qrtz_TRIGGERS SET TRIGGER_STATE = $1 WHERE SCHED_NAME = 'Scheduler_1' AND TRIGGER_NAME = $2 AND TRIGGER_GROUP = $3 AND TRIGGER_STATE = $4 +2019-09-22 06:28:24 UTC DETAIL: parameters: $1 = 'ACQUIRED', $2 = 'surveyInvitation_3Prbn85DiBWe8wHa_158802_77133_1260104', $3 = 'ExecutorsService', $4 = 'WAITING' +2019-09-22 06:28:24 UTC LOG: duration: 2474.307 ms execute S_30: SELECT * FROM qrtz_LOCKS WHERE SCHED_NAME = 'Scheduler_1' AND LOCK_NAME = $1 FOR UPDATE +2019-09-22 06:28:24 UTC DETAIL: parameters: $1 = 'TRIGGER_ACCESS' +2019-09-22 06:28:24 UTC LOG: duration: 18.327 ms execute S_32: SELECT al.id, al.tenant_id, al.created_by_id, al.create_ip, al.audit_date, al.audit_table, al.entity_id, al.entity_name, al.reason_for_change, al.audit_log_event_type_id, + aet.lookup_code, al.old_value, al.new_value, al.event_crf_id, al.event_crf_version_id, al.study_id, al.study_site_id, ss.rc_oid, al.subject_id, s.unique_identifier, + al.study_event_id, sed.name AS studyEventName, al.user_id, al.value_index, al.crf_version_id, al.global_logs, cv.version_name, crf.id AS crfId, crf.name AS crfName + FROM public.rc_audit_log_events AS al + LEFT JOIN rc_crf_versions AS cv ON cv.id=al.crf_version_id + LEFT JOIN rc_crfs AS crf ON crf.id=cv.crf_id + LEFT JOIN ad_lookup_codes AS aet ON aet.id=al.audit_log_event_type_id + LEFT JOIN rc_study_sites AS ss ON ss.id=al.study_site_id + LEFT JOIN rc_subjects AS s ON s.id=al.subject_id + LEFT JOIN rc_study_events AS se ON se.id=al.study_event_id + LEFT JOIN rc_study_event_definitions AS sed ON sed.id=se.study_event_definition_id + WHERE al.tenant_id=$1 AND al.study_id=$2 AND aet.lookup_code IN ($3, $4, $5, $6) AND al.audit_date >= $7 ORDER BY al.id DESC limit $8 diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-10-min-duration-statement.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-10-min-duration-statement.log-config.yml new file mode 100644 index 00000000000..860a8bfb242 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-10-min-duration-statement.log-config.yml @@ -0,0 +1,4 @@ +dynamic_fields: + event.ingested: ".*" +multiline: + first_line_pattern: '^\d{4}-\d{2}-\d{2} ' diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-10-min-duration-statement.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-10-min-duration-statement.log-expected.json new file mode 100644 index 00000000000..4bc9932ec19 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-10-min-duration-statement.log-expected.json @@ -0,0 +1,154 @@ +{ + "expected": [ + { + "@timestamp": "2019-09-22T06:28:24.000Z", + "postgresql": { + "log": { + "timestamp": "2019-09-22 06:28:24 UTC" + } + }, + "log": { + "level": "DETAIL" + }, + "message": "parameters: $1 = '86', $2 = '575', $3 = 'Item Inserted', $4 = 'Item Updated', $5 = 'Subject Updated', $6 = 'Subject Created', $7 = '2019-01-22 00:00:00+00'", + "event": { + "ingested": "2021-03-03T14:56:50.330489948Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "@timestamp": "2019-09-22T06:28:24.000Z", + "postgresql": { + "log": { + "query_step": "execute", + "query_name": "S_59", + "query": "UPDATE qrtz_TRIGGERS SET TRIGGER_STATE = $1 WHERE SCHED_NAME = 'Scheduler_1' AND TRIGGER_NAME = $2 AND TRIGGER_GROUP = $3 AND TRIGGER_STATE = $4", + "timestamp": "2019-09-22 06:28:24 UTC" + } + }, + "log": { + "level": "LOG" + }, + "message": "2019-09-22 06:28:24 UTC LOG: duration: 112.337 ms execute S_59: UPDATE qrtz_TRIGGERS SET TRIGGER_STATE = $1 WHERE SCHED_NAME = 'Scheduler_1' AND TRIGGER_NAME = $2 AND TRIGGER_GROUP = $3 AND TRIGGER_STATE = $4", + "event": { + "duration": 112337000, + "ingested": "2021-03-03T14:56:50.330493386Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "@timestamp": "2019-09-22T06:28:24.000Z", + "postgresql": { + "log": { + "timestamp": "2019-09-22 06:28:24 UTC" + } + }, + "log": { + "level": "DETAIL" + }, + "message": "parameters: $1 = 'ACQUIRED', $2 = 'surveyInvitation_3Prbn85DiBWe8wHa_158802_77133_1260104', $3 = 'ExecutorsService', $4 = 'WAITING'", + "event": { + "ingested": "2021-03-03T14:56:50.330494897Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "@timestamp": "2019-09-22T06:28:24.000Z", + "postgresql": { + "log": { + "query_step": "execute", + "query_name": "S_30", + "query": "SELECT * FROM qrtz_LOCKS WHERE SCHED_NAME = 'Scheduler_1' AND LOCK_NAME = $1 FOR UPDATE", + "timestamp": "2019-09-22 06:28:24 UTC" + } + }, + "log": { + "level": "LOG" + }, + "message": "2019-09-22 06:28:24 UTC LOG: duration: 2474.307 ms execute S_30: SELECT * FROM qrtz_LOCKS WHERE SCHED_NAME = 'Scheduler_1' AND LOCK_NAME = $1 FOR UPDATE", + "event": { + "duration": 2474306816, + "ingested": "2021-03-03T14:56:50.330496291Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "@timestamp": "2019-09-22T06:28:24.000Z", + "postgresql": { + "log": { + "timestamp": "2019-09-22 06:28:24 UTC" + } + }, + "log": { + "level": "DETAIL" + }, + "message": "parameters: $1 = 'TRIGGER_ACCESS'", + "event": { + "ingested": "2021-03-03T14:56:50.330497733Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "@timestamp": "2019-09-22T06:28:24.000Z", + "postgresql": { + "log": { + "query_step": "execute", + "query_name": "S_32", + "query": "SELECT al.id, al.tenant_id, al.created_by_id, al.create_ip, al.audit_date, al.audit_table, al.entity_id, al.entity_name, al.reason_for_change, al.audit_log_event_type_id,\n aet.lookup_code, al.old_value, al.new_value, al.event_crf_id, al.event_crf_version_id, al.study_id, al.study_site_id, ss.rc_oid, al.subject_id, s.unique_identifier,\n al.study_event_id, sed.name AS studyEventName, al.user_id, al.value_index, al.crf_version_id, al.global_logs, cv.version_name, crf.id AS crfId, crf.name AS crfName\n FROM public.rc_audit_log_events AS al\n LEFT JOIN rc_crf_versions AS cv ON cv.id=al.crf_version_id\n LEFT JOIN rc_crfs AS crf ON crf.id=cv.crf_id\n LEFT JOIN ad_lookup_codes AS aet ON aet.id=al.audit_log_event_type_id\n LEFT JOIN rc_study_sites AS ss ON ss.id=al.study_site_id\n LEFT JOIN rc_subjects AS s ON s.id=al.subject_id\n LEFT JOIN rc_study_events AS se ON se.id=al.study_event_id\n LEFT JOIN rc_study_event_definitions AS sed ON sed.id=se.study_event_definition_id\n WHERE al.tenant_id=$1 AND al.study_id=$2 AND aet.lookup_code IN ($3, $4, $5, $6) AND al.audit_date \u003e= $7 ORDER BY al.id DESC limit $8", + "timestamp": "2019-09-22 06:28:24 UTC" + } + }, + "log": { + "level": "LOG" + }, + "message": "2019-09-22 06:28:24 UTC LOG: duration: 18.327 ms execute S_32: SELECT al.id, al.tenant_id, al.created_by_id, al.create_ip, al.audit_date, al.audit_table, al.entity_id, al.entity_name, al.reason_for_change, al.audit_log_event_type_id,\n aet.lookup_code, al.old_value, al.new_value, al.event_crf_id, al.event_crf_version_id, al.study_id, al.study_site_id, ss.rc_oid, al.subject_id, s.unique_identifier,\n al.study_event_id, sed.name AS studyEventName, al.user_id, al.value_index, al.crf_version_id, al.global_logs, cv.version_name, crf.id AS crfId, crf.name AS crfName\n FROM public.rc_audit_log_events AS al\n LEFT JOIN rc_crf_versions AS cv ON cv.id=al.crf_version_id\n LEFT JOIN rc_crfs AS crf ON crf.id=cv.crf_id\n LEFT JOIN ad_lookup_codes AS aet ON aet.id=al.audit_log_event_type_id\n LEFT JOIN rc_study_sites AS ss ON ss.id=al.study_site_id\n LEFT JOIN rc_subjects AS s ON s.id=al.subject_id\n LEFT JOIN rc_study_events AS se ON se.id=al.study_event_id\n LEFT JOIN rc_study_event_definitions AS sed ON sed.id=se.study_event_definition_id\n WHERE al.tenant_id=$1 AND al.study_id=$2 AND aet.lookup_code IN ($3, $4, $5, $6) AND al.audit_date \u003e= $7 ORDER BY al.id DESC limit $8", + "event": { + "duration": 18327000, + "ingested": "2021-03-03T14:56:50.330504519Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + } + ] +} \ No newline at end of file diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-4.log b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-4.log new file mode 100644 index 00000000000..dedc3285aa3 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-4.log @@ -0,0 +1,29 @@ +2019-07-23 12:06:24.406 UTC [25] LOG: 00000: database system was interrupted; last known up at 2019-07-23 12:03:20 UTC +2019-07-23 12:06:24.406 UTC [25] LOCATION: StartupXLOG, xlog.c:6388 +2019-07-23 12:06:24.478 UTC [25] LOG: 00000: database system was not properly shut down; automatic recovery in progress +2019-07-23 12:06:24.478 UTC [25] LOCATION: StartupXLOG, xlog.c:6879 +2019-07-23 12:06:24.485 UTC [25] LOG: 00000: redo starts at 0/1651398 +2019-07-23 12:06:24.485 UTC [25] LOCATION: StartupXLOG, xlog.c:7150 +2019-07-23 12:06:24.485 UTC [25] LOG: 00000: invalid record length at 0/16513D0: wanted 24, got 0 +2019-07-23 12:06:24.485 UTC [25] LOCATION: ReadRecord, xlog.c:4233 +2019-07-23 12:06:24.485 UTC [25] LOG: 00000: redo done at 0/1651398 +2019-07-23 12:06:24.485 UTC [25] LOCATION: StartupXLOG, xlog.c:7422 +2019-07-23 12:06:24.507 UTC [1] LOG: 00000: database system is ready to accept connections +2019-07-23 12:06:24.507 UTC [1] LOCATION: reaper, postmaster.c:2889 +2019-07-23 12:06:30.536 UTC [44] LOG: 00000: connection received: host=[local] +2019-07-23 12:06:30.536 UTC [44] LOCATION: BackendInitialize, postmaster.c:4206 +2019-07-23 12:06:30.537 UTC [44] LOG: 00000: connection authorized: user=postgres database=postgres +2019-07-23 12:06:30.537 UTC [44] LOCATION: PerformAuthentication, postinit.c:279 +2019-07-23 12:06:33.732 UTC [44] LOG: 00000: statement: show config_filel; +2019-07-23 12:06:33.732 UTC [44] LOCATION: exec_simple_query, postgres.c:975 +2019-07-23 12:06:33.732 UTC [44] ERROR: 42704: unrecognized configuration parameter "config_filel" +2019-07-23 12:06:33.732 UTC [44] LOCATION: GetConfigOptionByName, guc.c:8342 +2019-07-23 12:06:33.732 UTC [44] STATEMENT: show config_filel; +2019-07-23 12:06:34.877 UTC [44] LOG: 00000: statement: show config_file; +2019-07-23 12:06:34.877 UTC [44] LOCATION: exec_simple_query, postgres.c:975 +2019-07-23 12:06:34.878 UTC [44] LOG: 00000: duration: 0.524 ms +2019-07-23 12:06:34.878 UTC [44] LOCATION: exec_simple_query, postgres.c:1219 +2019-07-23 12:09:57.563 UTC [44] LOG: 00000: statement: SELECT * FROM pg_catalog.pg_tables; +2019-07-23 12:09:57.563 UTC [44] LOCATION: exec_simple_query, postgres.c:975 +2019-07-23 12:09:57.565 UTC [44] LOG: 00000: duration: 2.139 ms +2019-07-23 12:09:57.565 UTC [44] LOCATION: exec_simple_query, postgres.c:1219 diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-4.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-4.log-config.yml new file mode 100644 index 00000000000..860a8bfb242 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-4.log-config.yml @@ -0,0 +1,4 @@ +dynamic_fields: + event.ingested: ".*" +multiline: + first_line_pattern: '^\d{4}-\d{2}-\d{2} ' diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-4.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-4.log-expected.json new file mode 100644 index 00000000000..7cc82dd2407 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-4.log-expected.json @@ -0,0 +1,772 @@ +{ + "expected": [ + { + "process": { + "pid": 25 + }, + "@timestamp": "2019-07-23T12:06:24.406Z", + "postgresql": { + "log": { + "sql_state_code": "00000", + "timestamp": "2019-07-23 12:06:24.406 UTC" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system was interrupted; last known up at 2019-07-23 12:03:20 UTC", + "event": { + "ingested": "2021-03-03T14:56:50.949559101Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 25 + }, + "@timestamp": "2019-07-23T12:06:24.406Z", + "postgresql": { + "log": { + "timestamp": "2019-07-23 12:06:24.406 UTC" + } + }, + "log": { + "level": "LOCATION" + }, + "message": "StartupXLOG, xlog.c:6388", + "event": { + "ingested": "2021-03-03T14:56:50.949565185Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 25 + }, + "@timestamp": "2019-07-23T12:06:24.478Z", + "postgresql": { + "log": { + "sql_state_code": "00000", + "timestamp": "2019-07-23 12:06:24.478 UTC" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system was not properly shut down; automatic recovery in progress", + "event": { + "ingested": "2021-03-03T14:56:50.949567554Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 25 + }, + "@timestamp": "2019-07-23T12:06:24.478Z", + "postgresql": { + "log": { + "timestamp": "2019-07-23 12:06:24.478 UTC" + } + }, + "log": { + "level": "LOCATION" + }, + "message": "StartupXLOG, xlog.c:6879", + "event": { + "ingested": "2021-03-03T14:56:50.949569899Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 25 + }, + "@timestamp": "2019-07-23T12:06:24.485Z", + "postgresql": { + "log": { + "sql_state_code": "00000", + "timestamp": "2019-07-23 12:06:24.485 UTC" + } + }, + "log": { + "level": "LOG" + }, + "message": "redo starts at 0/1651398", + "event": { + "ingested": "2021-03-03T14:56:50.949572205Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 25 + }, + "@timestamp": "2019-07-23T12:06:24.485Z", + "postgresql": { + "log": { + "timestamp": "2019-07-23 12:06:24.485 UTC" + } + }, + "log": { + "level": "LOCATION" + }, + "message": "StartupXLOG, xlog.c:7150", + "event": { + "ingested": "2021-03-03T14:56:50.949575127Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 25 + }, + "@timestamp": "2019-07-23T12:06:24.485Z", + "postgresql": { + "log": { + "sql_state_code": "00000", + "timestamp": "2019-07-23 12:06:24.485 UTC" + } + }, + "log": { + "level": "LOG" + }, + "message": "invalid record length at 0/16513D0: wanted 24, got 0", + "event": { + "ingested": "2021-03-03T14:56:50.949577389Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 25 + }, + "@timestamp": "2019-07-23T12:06:24.485Z", + "postgresql": { + "log": { + "timestamp": "2019-07-23 12:06:24.485 UTC" + } + }, + "log": { + "level": "LOCATION" + }, + "message": "ReadRecord, xlog.c:4233", + "event": { + "ingested": "2021-03-03T14:56:50.949579688Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 25 + }, + "@timestamp": "2019-07-23T12:06:24.485Z", + "postgresql": { + "log": { + "sql_state_code": "00000", + "timestamp": "2019-07-23 12:06:24.485 UTC" + } + }, + "log": { + "level": "LOG" + }, + "message": "redo done at 0/1651398", + "event": { + "ingested": "2021-03-03T14:56:50.949582401Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 25 + }, + "@timestamp": "2019-07-23T12:06:24.485Z", + "postgresql": { + "log": { + "timestamp": "2019-07-23 12:06:24.485 UTC" + } + }, + "log": { + "level": "LOCATION" + }, + "message": "StartupXLOG, xlog.c:7422", + "event": { + "ingested": "2021-03-03T14:56:50.949584672Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 1 + }, + "@timestamp": "2019-07-23T12:06:24.507Z", + "postgresql": { + "log": { + "sql_state_code": "00000", + "timestamp": "2019-07-23 12:06:24.507 UTC" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system is ready to accept connections", + "event": { + "ingested": "2021-03-03T14:56:50.949586861Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 1 + }, + "@timestamp": "2019-07-23T12:06:24.507Z", + "postgresql": { + "log": { + "timestamp": "2019-07-23 12:06:24.507 UTC" + } + }, + "log": { + "level": "LOCATION" + }, + "message": "reaper, postmaster.c:2889", + "event": { + "ingested": "2021-03-03T14:56:50.949589462Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 44 + }, + "@timestamp": "2019-07-23T12:06:30.536Z", + "postgresql": { + "log": { + "sql_state_code": "00000", + "timestamp": "2019-07-23 12:06:30.536 UTC" + } + }, + "log": { + "level": "LOG" + }, + "message": "connection received: host=[local]", + "event": { + "ingested": "2021-03-03T14:56:50.949591909Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 44 + }, + "@timestamp": "2019-07-23T12:06:30.536Z", + "postgresql": { + "log": { + "timestamp": "2019-07-23 12:06:30.536 UTC" + } + }, + "log": { + "level": "LOCATION" + }, + "message": "BackendInitialize, postmaster.c:4206", + "event": { + "ingested": "2021-03-03T14:56:50.949594467Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 44 + }, + "@timestamp": "2019-07-23T12:06:30.537Z", + "postgresql": { + "log": { + "sql_state_code": "00000", + "timestamp": "2019-07-23 12:06:30.537 UTC" + } + }, + "log": { + "level": "LOG" + }, + "message": "connection authorized: user=postgres database=postgres", + "event": { + "ingested": "2021-03-03T14:56:50.949596845Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 44 + }, + "@timestamp": "2019-07-23T12:06:30.537Z", + "postgresql": { + "log": { + "timestamp": "2019-07-23 12:06:30.537 UTC" + } + }, + "log": { + "level": "LOCATION" + }, + "message": "PerformAuthentication, postinit.c:279", + "event": { + "ingested": "2021-03-03T14:56:50.949599139Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 44 + }, + "@timestamp": "2019-07-23T12:06:33.732Z", + "postgresql": { + "log": { + "sql_state_code": "00000", + "timestamp": "2019-07-23 12:06:33.732 UTC" + } + }, + "log": { + "level": "LOG" + }, + "message": "statement: show config_filel;", + "event": { + "ingested": "2021-03-03T14:56:50.949601522Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 44 + }, + "@timestamp": "2019-07-23T12:06:33.732Z", + "postgresql": { + "log": { + "timestamp": "2019-07-23 12:06:33.732 UTC" + } + }, + "log": { + "level": "LOCATION" + }, + "message": "exec_simple_query, postgres.c:975", + "event": { + "ingested": "2021-03-03T14:56:50.949604440Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 44 + }, + "@timestamp": "2019-07-23T12:06:33.732Z", + "postgresql": { + "log": { + "sql_state_code": "42704", + "timestamp": "2019-07-23 12:06:33.732 UTC" + } + }, + "log": { + "level": "ERROR" + }, + "message": "unrecognized configuration parameter \"config_filel\"", + "event": { + "ingested": "2021-03-03T14:56:50.949606751Z", + "category": [ + "database" + ], + "type": [ + "error" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 44 + }, + "@timestamp": "2019-07-23T12:06:33.732Z", + "postgresql": { + "log": { + "timestamp": "2019-07-23 12:06:33.732 UTC" + } + }, + "log": { + "level": "LOCATION" + }, + "message": "GetConfigOptionByName, guc.c:8342", + "event": { + "ingested": "2021-03-03T14:56:50.949609269Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 44 + }, + "@timestamp": "2019-07-23T12:06:33.732Z", + "postgresql": { + "log": { + "timestamp": "2019-07-23 12:06:33.732 UTC" + } + }, + "log": { + "level": "STATEMENT" + }, + "message": "show config_filel;", + "event": { + "ingested": "2021-03-03T14:56:50.949611496Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 44 + }, + "@timestamp": "2019-07-23T12:06:34.877Z", + "postgresql": { + "log": { + "sql_state_code": "00000", + "timestamp": "2019-07-23 12:06:34.877 UTC" + } + }, + "log": { + "level": "LOG" + }, + "message": "statement: show config_file;", + "event": { + "ingested": "2021-03-03T14:56:50.949614056Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 44 + }, + "@timestamp": "2019-07-23T12:06:34.877Z", + "postgresql": { + "log": { + "timestamp": "2019-07-23 12:06:34.877 UTC" + } + }, + "log": { + "level": "LOCATION" + }, + "message": "exec_simple_query, postgres.c:975", + "event": { + "ingested": "2021-03-03T14:56:50.949616376Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 44 + }, + "@timestamp": "2019-07-23T12:06:34.878Z", + "postgresql": { + "log": { + "sql_state_code": "00000", + "timestamp": "2019-07-23 12:06:34.878 UTC" + } + }, + "log": { + "level": "LOG" + }, + "message": "duration: 0.524 ms", + "event": { + "ingested": "2021-03-03T14:56:50.949618659Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 44 + }, + "@timestamp": "2019-07-23T12:06:34.878Z", + "postgresql": { + "log": { + "timestamp": "2019-07-23 12:06:34.878 UTC" + } + }, + "log": { + "level": "LOCATION" + }, + "message": "exec_simple_query, postgres.c:1219", + "event": { + "ingested": "2021-03-03T14:56:50.949621114Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 44 + }, + "@timestamp": "2019-07-23T12:09:57.563Z", + "postgresql": { + "log": { + "sql_state_code": "00000", + "timestamp": "2019-07-23 12:09:57.563 UTC" + } + }, + "log": { + "level": "LOG" + }, + "message": "statement: SELECT * FROM pg_catalog.pg_tables;", + "event": { + "ingested": "2021-03-03T14:56:50.949623466Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 44 + }, + "@timestamp": "2019-07-23T12:09:57.563Z", + "postgresql": { + "log": { + "timestamp": "2019-07-23 12:09:57.563 UTC" + } + }, + "log": { + "level": "LOCATION" + }, + "message": "exec_simple_query, postgres.c:975", + "event": { + "ingested": "2021-03-03T14:56:50.949625951Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 44 + }, + "@timestamp": "2019-07-23T12:09:57.565Z", + "postgresql": { + "log": { + "sql_state_code": "00000", + "timestamp": "2019-07-23 12:09:57.565 UTC" + } + }, + "log": { + "level": "LOG" + }, + "message": "duration: 2.139 ms", + "event": { + "ingested": "2021-03-03T14:56:50.949628314Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 44 + }, + "@timestamp": "2019-07-23T12:09:57.565Z", + "postgresql": { + "log": { + "timestamp": "2019-07-23 12:09:57.565 UTC" + } + }, + "log": { + "level": "LOCATION" + }, + "message": "exec_simple_query, postgres.c:1219", + "event": { + "ingested": "2021-03-03T14:56:50.949630978Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + } + ] +} \ No newline at end of file diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-autovacuum-csv.log b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-autovacuum-csv.log new file mode 100644 index 00000000000..9fff82ab697 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-autovacuum-csv.log @@ -0,0 +1,6 @@ +2021-01-04 00:37:32.862 UTC,,,87,,5ff2634c.57,1,,2021-01-04 00:37:32 UTC,4/43,0,LOG,00000,"automatic vacuum of table ""postgres.public.t"": index scans: 1 +pages: 0 removed, 89 remain, 0 skipped due to pins, 0 skipped frozen +tuples: 10000 removed, 10000 remain, 0 are dead but not yet removable, oldest xmin: 578 +buffer usage: 316 hits, 2 misses, 4 dirtied +avg read rate: 0.569 MB/s, avg write rate: 1.138 MB/s +system usage: CPU: user: 0.00 s, system: 0.00 s, elapsed: 0.02 s",,,,,,,,,"" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-autovacuum-csv.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-autovacuum-csv.log-config.yml new file mode 100644 index 00000000000..860a8bfb242 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-autovacuum-csv.log-config.yml @@ -0,0 +1,4 @@ +dynamic_fields: + event.ingested: ".*" +multiline: + first_line_pattern: '^\d{4}-\d{2}-\d{2} ' diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-autovacuum-csv.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-autovacuum-csv.log-expected.json new file mode 100644 index 00000000000..fa68a85e344 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-autovacuum-csv.log-expected.json @@ -0,0 +1,35 @@ +{ + "expected": [ + { + "process": { + "pid": 87 + }, + "@timestamp": "2021-01-04T00:37:32.862Z", + "postgresql": { + "log": { + "transaction_id": 0, + "session_start_time": "2021-01-04T00:37:32.000Z", + "session_id": "5ff2634c.57", + "virtual_transaction_id": "4/43", + "sql_state_code": "00000", + "timestamp": "2021-01-04 00:37:32.862 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:50.373912886Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "automatic vacuum of table \"postgres.public.t\": index scans: 1\npages: 0 removed, 89 remain, 0 skipped due to pins, 0 skipped frozen\ntuples: 10000 removed, 10000 remain, 0 are dead but not yet removable, oldest xmin: 578\nbuffer usage: 316 hits, 2 misses, 4 dirtied\navg read rate: 0.569 MB/s, avg write rate: 1.138 MB/s\nsystem usage: CPU: user: 0.00 s, system: 0.00 s, elapsed: 0.02 s" + } + ] +} \ No newline at end of file diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-connection-disconnection-csv.log b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-connection-disconnection-csv.log new file mode 100644 index 00000000000..077332271d1 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-connection-disconnection-csv.log @@ -0,0 +1,28 @@ +2021-01-04 00:04:50.554 UTC,,,27,,5ff25ba2.1b,1,,2021-01-04 00:04:50 UTC,,0,LOG,00000,"database system was shut down at 2021-01-03 20:01:08 UTC",,,,,,,,,"" +2021-01-04 00:04:50.568 UTC,,,1,,5ff25ba2.1,1,,2021-01-04 00:04:50 UTC,,0,LOG,00000,"database system is ready to accept connections",,,,,,,,,"" +2021-01-04 00:05:06.011 UTC,,,34,"172.24.0.1:42304",5ff25bb2.22,1,"",2021-01-04 00:05:06 UTC,,0,LOG,00000,"connection received: host=172.24.0.1 port=42304",,,,,,,,,"" +2021-01-04 00:05:06.086 UTC,"postgres","postgres",34,"172.24.0.1:42304",5ff25bb2.22,2,"authentication",2021-01-04 00:05:06 UTC,3/1,0,LOG,00000,"connection authorized: user=postgres database=postgres",,,,,,,,,"" +2021-01-04 00:05:12.999 UTC,"postgres","postgres",34,"172.24.0.1:42304",5ff25bb2.22,3,"idle",2021-01-04 00:05:06 UTC,3/2,0,LOG,00000,"statement: SELECT name FROM (SELECT pg_catalog.lower(name) AS name FROM pg_catalog.pg_settings WHERE context != 'internal' UNION ALL SELECT 'all') ss WHERE substring(name,1,10)='log_connec' +LIMIT 1000",,,,,,,,,"psql" +2021-01-04 00:05:17.146 UTC,"postgres","postgres",34,"172.24.0.1:42304",5ff25bb2.22,4,"idle",2021-01-04 00:05:06 UTC,3/3,0,LOG,00000,"statement: ALTER SYSTEM SET log_connections = on;",,,,,,,,,"psql" +2021-01-04 00:05:23.242 UTC,"postgres","postgres",34,"172.24.0.1:42304",5ff25bb2.22,5,"idle",2021-01-04 00:05:06 UTC,3/4,0,LOG,00000,"statement: ALTER SYSTEM SET log_disconnections = on;",,,,,,,,,"psql" +2021-01-04 00:05:28.166 UTC,"postgres","postgres",34,"172.24.0.1:42304",5ff25bb2.22,6,"idle",2021-01-04 00:05:06 UTC,3/5,0,LOG,00000,"statement: SELECT name FROM (SELECT pg_catalog.lower(name) AS name FROM pg_catalog.pg_settings WHERE context != 'internal' UNION ALL SELECT 'all') ss WHERE substring(name,1,3)='log' +LIMIT 1000",,,,,,,,,"psql" +2021-01-04 00:05:29.434 UTC,"postgres","postgres",34,"172.24.0.1:42304",5ff25bb2.22,7,"idle",2021-01-04 00:05:06 UTC,3/6,0,LOG,00000,"statement: SELECT name FROM (SELECT pg_catalog.lower(name) AS name FROM pg_catalog.pg_settings WHERE context != 'internal' UNION ALL SELECT 'all') ss WHERE substring(name,1,6)='log_mi' +LIMIT 1000",,,,,,,,,"psql" +2021-01-04 00:05:31.342 UTC,"postgres","postgres",34,"172.24.0.1:42304",5ff25bb2.22,8,"idle",2021-01-04 00:05:06 UTC,3/7,0,LOG,00000,"statement: SELECT name FROM (SELECT pg_catalog.lower(name) AS name FROM pg_catalog.pg_settings WHERE context != 'internal' UNION ALL SELECT 'all') ss WHERE substring(name,1,12)='log_min_dura' +LIMIT 1000",,,,,,,,,"psql" +2021-01-04 00:05:37.670 UTC,"postgres","postgres",34,"172.24.0.1:42304",5ff25bb2.22,9,"idle",2021-01-04 00:05:06 UTC,3/8,0,LOG,00000,"statement: alter system set log_min_duration_statement = 0;",,,,,,,,,"psql" +2021-01-04 00:05:51.418 UTC,"postgres","postgres",34,"172.24.0.1:42304",5ff25bb2.22,10,"idle",2021-01-04 00:05:06 UTC,,0,LOG,00000,"disconnection: session time: 0:00:45.407 user=postgres database=postgres host=172.24.0.1 port=42304",,,,,,,,,"psql" +2021-01-04 00:05:58.207 UTC,,,36,"172.24.0.1:42326",5ff25be6.24,1,"",2021-01-04 00:05:58 UTC,,0,LOG,00000,"connection received: host=172.24.0.1 port=42326",,,,,,,,,"" +2021-01-04 00:05:58.232 UTC,"azlev","azlev",36,"172.24.0.1:42326",5ff25be6.24,2,"authentication",2021-01-04 00:05:58 UTC,3/9,0,FATAL,28P01,"password authentication failed for user ""azlev""","Role ""azlev"" does not exist. +Connection matched pg_hba.conf line 95: ""host all all all md5""",,,,,,,,"" +2021-01-04 00:05:59.807 UTC,,,37,"172.24.0.1:42330",5ff25be7.25,1,"",2021-01-04 00:05:59 UTC,,0,LOG,00000,"connection received: host=172.24.0.1 port=42330",,,,,,,,,"" +2021-01-04 00:05:59.833 UTC,"azlev","azlev",37,"172.24.0.1:42330",5ff25be7.25,2,"authentication",2021-01-04 00:05:59 UTC,3/10,0,FATAL,28P01,"password authentication failed for user ""azlev""","Role ""azlev"" does not exist. +Connection matched pg_hba.conf line 95: ""host all all all md5""",,,,,,,,"" +2021-01-04 00:06:03.347 UTC,,,38,"172.24.0.1:42336",5ff25beb.26,1,"",2021-01-04 00:06:03 UTC,,0,LOG,00000,"connection received: host=172.24.0.1 port=42336",,,,,,,,,"" +2021-01-04 00:06:03.370 UTC,"azlev","azlev",38,"172.24.0.1:42336",5ff25beb.26,2,"authentication",2021-01-04 00:06:03 UTC,3/11,0,FATAL,28P01,"password authentication failed for user ""azlev""","Role ""azlev"" does not exist. +Connection matched pg_hba.conf line 95: ""host all all all md5""",,,,,,,,"" +2021-01-04 00:06:04.765 UTC,,,39,"172.24.0.1:42340",5ff25bec.27,1,"",2021-01-04 00:06:04 UTC,,0,LOG,00000,"connection received: host=172.24.0.1 port=42340",,,,,,,,,"" +2021-01-04 00:06:04.799 UTC,"azlev","azlev",39,"172.24.0.1:42340",5ff25bec.27,2,"authentication",2021-01-04 00:06:04 UTC,3/12,0,FATAL,28P01,"password authentication failed for user ""azlev""","Role ""azlev"" does not exist. +Connection matched pg_hba.conf line 95: ""host all all all md5""",,,,,,,,"" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-connection-disconnection-csv.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-connection-disconnection-csv.log-config.yml new file mode 100644 index 00000000000..860a8bfb242 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-connection-disconnection-csv.log-config.yml @@ -0,0 +1,4 @@ +dynamic_fields: + event.ingested: ".*" +multiline: + first_line_pattern: '^\d{4}-\d{2}-\d{2} ' diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-connection-disconnection-csv.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-connection-disconnection-csv.log-expected.json new file mode 100644 index 00000000000..67f32364dbf --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-connection-disconnection-csv.log-expected.json @@ -0,0 +1,801 @@ +{ + "expected": [ + { + "process": { + "pid": 27 + }, + "@timestamp": "2021-01-04T00:04:50.554Z", + "postgresql": { + "log": { + "transaction_id": 0, + "session_start_time": "2021-01-04T00:04:50.000Z", + "session_id": "5ff25ba2.1b", + "sql_state_code": "00000", + "timestamp": "2021-01-04 00:04:50.554 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:50.391226367Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "database system was shut down at 2021-01-03 20:01:08 UTC" + }, + { + "process": { + "pid": 1 + }, + "@timestamp": "2021-01-04T00:04:50.568Z", + "postgresql": { + "log": { + "transaction_id": 0, + "session_start_time": "2021-01-04T00:04:50.000Z", + "session_id": "5ff25ba2.1", + "sql_state_code": "00000", + "timestamp": "2021-01-04 00:04:50.568 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:50.391231535Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "database system is ready to accept connections" + }, + { + "process": { + "pid": 34 + }, + "@timestamp": "2021-01-04T00:05:06.011Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42304, + "sql_state_code": "00000", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:05:06.000Z", + "session_id": "5ff25bb2.22", + "timestamp": "2021-01-04 00:05:06.011 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:50.391233921Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "connection received: host=172.24.0.1 port=42304" + }, + { + "process": { + "pid": 34 + }, + "@timestamp": "2021-01-04T00:05:06.086Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42304, + "database": "postgres", + "virtual_transaction_id": "3/1", + "sql_state_code": "00000", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:05:06.000Z", + "session_id": "5ff25bb2.22", + "command_tag": "authentication", + "timestamp": "2021-01-04 00:05:06.086 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "connection authorized: user=postgres database=postgres", + "event": { + "ingested": "2021-03-03T14:56:50.391236209Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 34 + }, + "@timestamp": "2021-01-04T00:05:12.999Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42304, + "database": "postgres", + "virtual_transaction_id": "3/2", + "sql_state_code": "00000", + "application_name": "psql", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:05:06.000Z", + "session_id": "5ff25bb2.22", + "command_tag": "idle", + "query_step": "statement", + "timestamp": "2021-01-04 00:05:12.999 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "SELECT name FROM (SELECT pg_catalog.lower(name) AS name FROM pg_catalog.pg_settings WHERE context != 'internal' UNION ALL SELECT 'all') ss WHERE substring(name,1,10)='log_connec'\nLIMIT 1000", + "event": { + "ingested": "2021-03-03T14:56:50.391238418Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 34 + }, + "@timestamp": "2021-01-04T00:05:17.146Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42304, + "database": "postgres", + "virtual_transaction_id": "3/3", + "sql_state_code": "00000", + "application_name": "psql", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:05:06.000Z", + "session_id": "5ff25bb2.22", + "command_tag": "idle", + "query_step": "statement", + "timestamp": "2021-01-04 00:05:17.146 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "ALTER SYSTEM SET log_connections = on;", + "event": { + "ingested": "2021-03-03T14:56:50.391240512Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 34 + }, + "@timestamp": "2021-01-04T00:05:23.242Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42304, + "database": "postgres", + "virtual_transaction_id": "3/4", + "sql_state_code": "00000", + "application_name": "psql", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:05:06.000Z", + "session_id": "5ff25bb2.22", + "command_tag": "idle", + "query_step": "statement", + "timestamp": "2021-01-04 00:05:23.242 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "ALTER SYSTEM SET log_disconnections = on;", + "event": { + "ingested": "2021-03-03T14:56:50.391242403Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 34 + }, + "@timestamp": "2021-01-04T00:05:28.166Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42304, + "database": "postgres", + "virtual_transaction_id": "3/5", + "sql_state_code": "00000", + "application_name": "psql", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:05:06.000Z", + "session_id": "5ff25bb2.22", + "command_tag": "idle", + "query_step": "statement", + "timestamp": "2021-01-04 00:05:28.166 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "SELECT name FROM (SELECT pg_catalog.lower(name) AS name FROM pg_catalog.pg_settings WHERE context != 'internal' UNION ALL SELECT 'all') ss WHERE substring(name,1,3)='log'\nLIMIT 1000", + "event": { + "ingested": "2021-03-03T14:56:50.391245727Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 34 + }, + "@timestamp": "2021-01-04T00:05:29.434Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42304, + "database": "postgres", + "virtual_transaction_id": "3/6", + "sql_state_code": "00000", + "application_name": "psql", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:05:06.000Z", + "session_id": "5ff25bb2.22", + "command_tag": "idle", + "query_step": "statement", + "timestamp": "2021-01-04 00:05:29.434 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "SELECT name FROM (SELECT pg_catalog.lower(name) AS name FROM pg_catalog.pg_settings WHERE context != 'internal' UNION ALL SELECT 'all') ss WHERE substring(name,1,6)='log_mi'\nLIMIT 1000", + "event": { + "ingested": "2021-03-03T14:56:50.391248582Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 34 + }, + "@timestamp": "2021-01-04T00:05:31.342Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42304, + "database": "postgres", + "virtual_transaction_id": "3/7", + "sql_state_code": "00000", + "application_name": "psql", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:05:06.000Z", + "session_id": "5ff25bb2.22", + "command_tag": "idle", + "query_step": "statement", + "timestamp": "2021-01-04 00:05:31.342 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "SELECT name FROM (SELECT pg_catalog.lower(name) AS name FROM pg_catalog.pg_settings WHERE context != 'internal' UNION ALL SELECT 'all') ss WHERE substring(name,1,12)='log_min_dura'\nLIMIT 1000", + "event": { + "ingested": "2021-03-03T14:56:50.391250041Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 34 + }, + "@timestamp": "2021-01-04T00:05:37.670Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42304, + "database": "postgres", + "virtual_transaction_id": "3/8", + "sql_state_code": "00000", + "application_name": "psql", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:05:06.000Z", + "session_id": "5ff25bb2.22", + "command_tag": "idle", + "query_step": "statement", + "timestamp": "2021-01-04 00:05:37.670 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "alter system set log_min_duration_statement = 0;", + "event": { + "ingested": "2021-03-03T14:56:50.391251499Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 34 + }, + "@timestamp": "2021-01-04T00:05:51.418Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42304, + "database": "postgres", + "sql_state_code": "00000", + "application_name": "psql", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:05:06.000Z", + "session_id": "5ff25bb2.22", + "command_tag": "idle", + "timestamp": "2021-01-04 00:05:51.418 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "disconnection: session time: 0:00:45.407 user=postgres database=postgres host=172.24.0.1 port=42304", + "event": { + "ingested": "2021-03-03T14:56:50.391253188Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 36 + }, + "@timestamp": "2021-01-04T00:05:58.207Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42326, + "sql_state_code": "00000", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:05:58.000Z", + "session_id": "5ff25be6.24", + "timestamp": "2021-01-04 00:05:58.207 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:50.391254668Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "connection received: host=172.24.0.1 port=42326" + }, + { + "process": { + "pid": 36 + }, + "@timestamp": "2021-01-04T00:05:58.232Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42326, + "database": "azlev", + "virtual_transaction_id": "3/9", + "sql_state_code": "28P01", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:05:58.000Z", + "session_id": "5ff25be6.24", + "command_tag": "authentication", + "detail": "Role \"azlev\" does not exist.\nConnection matched pg_hba.conf line 95: \"host all all all md5\"", + "timestamp": "2021-01-04 00:05:58.232 UTC" + } + }, + "related": { + "user": [ + "azlev" + ] + }, + "log": { + "level": "FATAL" + }, + "message": "password authentication failed for user \"azlev\"", + "event": { + "ingested": "2021-03-03T14:56:50.391256101Z", + "category": [ + "database" + ], + "type": [ + "error" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "azlev" + } + }, + { + "process": { + "pid": 37 + }, + "@timestamp": "2021-01-04T00:05:59.807Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42330, + "sql_state_code": "00000", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:05:59.000Z", + "session_id": "5ff25be7.25", + "timestamp": "2021-01-04 00:05:59.807 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:50.391257543Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "connection received: host=172.24.0.1 port=42330" + }, + { + "process": { + "pid": 37 + }, + "@timestamp": "2021-01-04T00:05:59.833Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42330, + "database": "azlev", + "virtual_transaction_id": "3/10", + "sql_state_code": "28P01", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:05:59.000Z", + "session_id": "5ff25be7.25", + "command_tag": "authentication", + "detail": "Role \"azlev\" does not exist.\nConnection matched pg_hba.conf line 95: \"host all all all md5\"", + "timestamp": "2021-01-04 00:05:59.833 UTC" + } + }, + "related": { + "user": [ + "azlev" + ] + }, + "log": { + "level": "FATAL" + }, + "message": "password authentication failed for user \"azlev\"", + "event": { + "ingested": "2021-03-03T14:56:50.391259003Z", + "category": [ + "database" + ], + "type": [ + "error" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "azlev" + } + }, + { + "process": { + "pid": 38 + }, + "@timestamp": "2021-01-04T00:06:03.347Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42336, + "sql_state_code": "00000", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:06:03.000Z", + "session_id": "5ff25beb.26", + "timestamp": "2021-01-04 00:06:03.347 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:50.391260726Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "connection received: host=172.24.0.1 port=42336" + }, + { + "process": { + "pid": 38 + }, + "@timestamp": "2021-01-04T00:06:03.370Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42336, + "database": "azlev", + "virtual_transaction_id": "3/11", + "sql_state_code": "28P01", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:06:03.000Z", + "session_id": "5ff25beb.26", + "command_tag": "authentication", + "detail": "Role \"azlev\" does not exist.\nConnection matched pg_hba.conf line 95: \"host all all all md5\"", + "timestamp": "2021-01-04 00:06:03.370 UTC" + } + }, + "related": { + "user": [ + "azlev" + ] + }, + "log": { + "level": "FATAL" + }, + "message": "password authentication failed for user \"azlev\"", + "event": { + "ingested": "2021-03-03T14:56:50.391262243Z", + "category": [ + "database" + ], + "type": [ + "error" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "azlev" + } + }, + { + "process": { + "pid": 39 + }, + "@timestamp": "2021-01-04T00:06:04.765Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42340, + "sql_state_code": "00000", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:06:04.000Z", + "session_id": "5ff25bec.27", + "timestamp": "2021-01-04 00:06:04.765 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:50.391263701Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "connection received: host=172.24.0.1 port=42340" + }, + { + "process": { + "pid": 39 + }, + "@timestamp": "2021-01-04T00:06:04.799Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42340, + "database": "azlev", + "virtual_transaction_id": "3/12", + "sql_state_code": "28P01", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:06:04.000Z", + "session_id": "5ff25bec.27", + "command_tag": "authentication", + "detail": "Role \"azlev\" does not exist.\nConnection matched pg_hba.conf line 95: \"host all all all md5\"", + "timestamp": "2021-01-04 00:06:04.799 UTC" + } + }, + "related": { + "user": [ + "azlev" + ] + }, + "log": { + "level": "FATAL" + }, + "message": "password authentication failed for user \"azlev\"", + "event": { + "ingested": "2021-03-03T14:56:50.391265154Z", + "category": [ + "database" + ], + "type": [ + "error" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "azlev" + } + } + ] +} \ No newline at end of file diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-duration-csv.log b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-duration-csv.log new file mode 100644 index 00000000000..b2bf57edf9f --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-duration-csv.log @@ -0,0 +1,3 @@ +2021-01-04 00:17:53.742 UTC,"postgres","postgres",54,"172.24.0.1:42608",5ff25ea4.36,6,"ALTER SYSTEM",2021-01-04 00:17:40 UTC,3/0,0,LOG,00000,"duration: 6.589 ms",,,,,,,,,"psql" +2021-01-04 00:18:01.055 UTC,"postgres","postgres",54,"172.24.0.1:42608",5ff25ea4.36,7,"idle",2021-01-04 00:17:40 UTC,3/39,0,LOG,00000,"statement: select pg_reload_conf();",,,,,,,,,"psql" +2021-01-04 00:18:04.650 UTC,"postgres","postgres",54,"172.24.0.1:42608",5ff25ea4.36,9,"SELECT",2021-01-04 00:17:40 UTC,3/0,0,LOG,00000,"duration: 148.472 ms statement: select generate_series(1, 1000000);",,,,,,,,,"psql" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-duration-csv.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-duration-csv.log-config.yml new file mode 100644 index 00000000000..860a8bfb242 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-duration-csv.log-config.yml @@ -0,0 +1,4 @@ +dynamic_fields: + event.ingested: ".*" +multiline: + first_line_pattern: '^\d{4}-\d{2}-\d{2} ' diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-duration-csv.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-duration-csv.log-expected.json new file mode 100644 index 00000000000..2fa0d2f00b1 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-duration-csv.log-expected.json @@ -0,0 +1,139 @@ +{ + "expected": [ + { + "process": { + "pid": 54 + }, + "@timestamp": "2021-01-04T00:17:53.742Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42608, + "database": "postgres", + "virtual_transaction_id": "3/0", + "sql_state_code": "00000", + "application_name": "psql", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:17:40.000Z", + "session_id": "5ff25ea4.36", + "command_tag": "ALTER SYSTEM", + "timestamp": "2021-01-04 00:17:53.742 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "event": { + "duration": 6589000, + "ingested": "2021-03-03T14:56:50.628169714Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 54 + }, + "@timestamp": "2021-01-04T00:18:01.055Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42608, + "database": "postgres", + "virtual_transaction_id": "3/39", + "sql_state_code": "00000", + "application_name": "psql", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:17:40.000Z", + "session_id": "5ff25ea4.36", + "command_tag": "idle", + "query_step": "statement", + "timestamp": "2021-01-04 00:18:01.055 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "select pg_reload_conf();", + "event": { + "ingested": "2021-03-03T14:56:50.628173662Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 54 + }, + "@timestamp": "2021-01-04T00:18:04.650Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42608, + "database": "postgres", + "virtual_transaction_id": "3/0", + "sql_state_code": "00000", + "application_name": "psql", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:17:40.000Z", + "session_id": "5ff25ea4.36", + "command_tag": "SELECT", + "query_step": "statement", + "timestamp": "2021-01-04 00:18:04.650 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "select generate_series(1, 1000000);", + "event": { + "duration": 148472000, + "ingested": "2021-03-03T14:56:50.628175083Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + } + ] +} \ No newline at end of file diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-ipv6-csv.log b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-ipv6-csv.log new file mode 100644 index 00000000000..92293a8d6be --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-ipv6-csv.log @@ -0,0 +1 @@ +2021-01-03 17:45:17.612 UTC,"jose.villanova","postgres",20460,"::1:16790",5ff202ad.4fec,1,"startup",2021-01-03 17:45:17 UTC,3/15493200,0,FATAL,28000,"role ""jose.villanova"" does not exist",,,,,,,,,"" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-ipv6-csv.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-ipv6-csv.log-config.yml new file mode 100644 index 00000000000..860a8bfb242 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-ipv6-csv.log-config.yml @@ -0,0 +1,4 @@ +dynamic_fields: + event.ingested: ".*" +multiline: + first_line_pattern: '^\d{4}-\d{2}-\d{2} ' diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-ipv6-csv.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-ipv6-csv.log-expected.json new file mode 100644 index 00000000000..c511f124aab --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-ipv6-csv.log-expected.json @@ -0,0 +1,47 @@ +{ + "expected": [ + { + "process": { + "pid": 20460 + }, + "@timestamp": "2021-01-03T17:45:17.612Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 16790, + "database": "postgres", + "virtual_transaction_id": "3/15493200", + "sql_state_code": "28000", + "client_addr": "::1", + "session_start_time": "2021-01-03T17:45:17.000Z", + "session_id": "5ff202ad.4fec", + "command_tag": "startup", + "timestamp": "2021-01-03 17:45:17.612 UTC" + } + }, + "related": { + "user": [ + "jose.villanova" + ] + }, + "log": { + "level": "FATAL" + }, + "message": "role \"jose.villanova\" does not exist", + "event": { + "ingested": "2021-03-03T14:56:50.668839732Z", + "category": [ + "database" + ], + "type": [ + "error" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "jose.villanova" + } + } + ] +} \ No newline at end of file diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-multi-line-csv.log b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-multi-line-csv.log new file mode 100644 index 00000000000..dea8d6c082c --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-multi-line-csv.log @@ -0,0 +1,2 @@ +2021-01-04 00:22:01.903 UTC,"postgres","postgres",59,"172.24.0.1:42642",5ff25f96.3b,3,"SELECT",2021-01-04 00:21:42 UTC,3/0,0,LOG,00000,"duration: 0.658 ms statement: SELECT 'multi', +'line';",,,,,,,,,"psql" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-multi-line-csv.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-multi-line-csv.log-config.yml new file mode 100644 index 00000000000..860a8bfb242 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-multi-line-csv.log-config.yml @@ -0,0 +1,4 @@ +dynamic_fields: + event.ingested: ".*" +multiline: + first_line_pattern: '^\d{4}-\d{2}-\d{2} ' diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-multi-line-csv.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-multi-line-csv.log-expected.json new file mode 100644 index 00000000000..de551be6c6f --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-multi-line-csv.log-expected.json @@ -0,0 +1,50 @@ +{ + "expected": [ + { + "process": { + "pid": 59 + }, + "@timestamp": "2021-01-04T00:22:01.903Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42642, + "database": "postgres", + "virtual_transaction_id": "3/0", + "sql_state_code": "00000", + "application_name": "psql", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:21:42.000Z", + "session_id": "5ff25f96.3b", + "command_tag": "SELECT", + "query_step": "statement", + "timestamp": "2021-01-04 00:22:01.903 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "SELECT 'multi',\n'line';", + "event": { + "duration": 658000, + "ingested": "2021-03-03T14:56:50.683421638Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + } + ] +} \ No newline at end of file diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-parse-bind-csv.log b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-parse-bind-csv.log new file mode 100644 index 00000000000..b1f5f1de566 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-parse-bind-csv.log @@ -0,0 +1,3 @@ +2021-01-04 00:51:56.837 UTC,"postgres","postgres",105,"172.24.0.1:44618",5ff26691.69,6,"PARSE",2021-01-04 00:51:29 UTC,3/135,0,LOG,00000,"duration: 7.775 ms parse py:0x7fde12d61b80: SELECT * from information_schema.tables WHERE table_name = $1",,,,,,,,,"" +2021-01-04 00:51:56.843 UTC,"postgres","postgres",105,"172.24.0.1:44618",5ff26691.69,7,"BIND",2021-01-04 00:51:29 UTC,3/136,0,LOG,00000,"duration: 4.091 ms bind py:0x7fde12d61b80: SELECT * from information_schema.tables WHERE table_name = $1","parameters: $1 = 'tables'",,,,,,,,"" +2021-01-04 00:51:56.843 UTC,"postgres","postgres",105,"172.24.0.1:44618",5ff26691.69,8,"SELECT",2021-01-04 00:51:29 UTC,3/136,0,LOG,00000,"duration: 0.455 ms execute py:0x7fde12d61b80: SELECT * from information_schema.tables WHERE table_name = $1","parameters: $1 = 'tables'",,,,,,,,"" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-parse-bind-csv.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-parse-bind-csv.log-config.yml new file mode 100644 index 00000000000..860a8bfb242 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-parse-bind-csv.log-config.yml @@ -0,0 +1,4 @@ +dynamic_fields: + event.ingested: ".*" +multiline: + first_line_pattern: '^\d{4}-\d{2}-\d{2} ' diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-parse-bind-csv.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-parse-bind-csv.log-expected.json new file mode 100644 index 00000000000..36c27355c55 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-parse-bind-csv.log-expected.json @@ -0,0 +1,144 @@ +{ + "expected": [ + { + "process": { + "pid": 105 + }, + "@timestamp": "2021-01-04T00:51:56.837Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 44618, + "database": "postgres", + "virtual_transaction_id": "3/135", + "sql_state_code": "00000", + "query_name": "py:0x7fde12d61b80", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:51:29.000Z", + "session_id": "5ff26691.69", + "command_tag": "PARSE", + "query_step": "parse", + "timestamp": "2021-01-04 00:51:56.837 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "SELECT * from information_schema.tables WHERE table_name = $1", + "event": { + "duration": 7775000, + "ingested": "2021-03-03T14:56:50.699628371Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 105 + }, + "@timestamp": "2021-01-04T00:51:56.843Z", + "postgresql": { + "log": { + "transaction_id": 0, + "sql_state_code": "00000", + "session_start_time": "2021-01-04T00:51:29.000Z", + "session_id": "5ff26691.69", + "command_tag": "BIND", + "client_port": 44618, + "database": "postgres", + "virtual_transaction_id": "3/136", + "query_name": "py:0x7fde12d61b80", + "client_addr": "172.24.0.1", + "query_step": "bind", + "detail": "parameters: $1 = 'tables'", + "timestamp": "2021-01-04 00:51:56.843 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "SELECT * from information_schema.tables WHERE table_name = $1", + "event": { + "duration": 4091000, + "ingested": "2021-03-03T14:56:50.699631987Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 105 + }, + "@timestamp": "2021-01-04T00:51:56.843Z", + "postgresql": { + "log": { + "transaction_id": 0, + "sql_state_code": "00000", + "session_start_time": "2021-01-04T00:51:29.000Z", + "session_id": "5ff26691.69", + "command_tag": "SELECT", + "client_port": 44618, + "database": "postgres", + "virtual_transaction_id": "3/136", + "query_name": "py:0x7fde12d61b80", + "client_addr": "172.24.0.1", + "query_step": "execute", + "detail": "parameters: $1 = 'tables'", + "timestamp": "2021-01-04 00:51:56.843 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "SELECT * from information_schema.tables WHERE table_name = $1", + "event": { + "duration": 455000, + "ingested": "2021-03-03T14:56:50.699633495Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + } + ] +} \ No newline at end of file diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-start-stop-csv.log b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-start-stop-csv.log new file mode 100644 index 00000000000..1a4e1b082f1 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-start-stop-csv.log @@ -0,0 +1,13 @@ +2021-01-03 20:00:46.695 UTC,,,28,,5ff2226e.1c,1,,2021-01-03 20:00:46 UTC,,0,LOG,00000,"database system was shut down at 2021-01-03 20:00:42 UTC",,,,,,,,,"" +2021-01-03 20:00:46.708 UTC,,,1,,5ff2226e.1,1,,2021-01-03 20:00:46 UTC,,0,LOG,00000,"database system is ready to accept connections",,,,,,,,,"" +2021-01-03 20:01:00.349 UTC,,,35,"172.24.0.1:38352",5ff2227c.23,1,"",2021-01-03 20:01:00 UTC,,0,LOG,00000,"connection received: host=172.24.0.1 port=38352",,,,,,,,,"" +2021-01-03 20:01:02.701 UTC,,,36,"172.24.0.1:38356",5ff2227e.24,1,"",2021-01-03 20:01:02 UTC,,0,LOG,00000,"connection received: host=172.24.0.1 port=38356",,,,,,,,,"" +2021-01-03 20:01:02.727 UTC,"postgres","postgres",36,"172.24.0.1:38356",5ff2227e.24,2,"authentication",2021-01-03 20:01:02 UTC,3/2,0,LOG,00000,"connection authorized: user=postgres database=postgres",,,,,,,,,"" +2021-01-03 20:01:07.094 UTC,"postgres","postgres",36,"172.24.0.1:38356",5ff2227e.24,3,"idle",2021-01-03 20:01:02 UTC,3/3,0,LOG,00000,"statement: SELECT 1;",,,,,,,,,"psql" +2021-01-03 20:01:07.724 UTC,"postgres","postgres",36,"172.24.0.1:38356",5ff2227e.24,4,"idle",2021-01-03 20:01:02 UTC,,0,LOG,00000,"disconnection: session time: 0:00:05.023 user=postgres database=postgres host=172.24.0.1 port=38356",,,,,,,,,"psql" +2021-01-03 20:01:08.894 UTC,,,1,,5ff2226e.1,2,,2021-01-03 20:00:46 UTC,,0,LOG,00000,"received smart shutdown request",,,,,,,,,"" +2021-01-03 20:01:08.899 UTC,,,1,,5ff2226e.1,3,,2021-01-03 20:00:46 UTC,,0,LOG,00000,"background worker ""logical replication launcher"" (PID 34) exited with exit code 1",,,,,,,,,"" +2021-01-03 20:01:08.899 UTC,,,29,,5ff2226e.1d,1,,2021-01-03 20:00:46 UTC,,0,LOG,00000,"shutting down",,,,,,,,,"" +2021-01-03 20:01:08.901 UTC,,,29,,5ff2226e.1d,2,,2021-01-03 20:00:46 UTC,,0,LOG,00000,"checkpoint starting: shutdown immediate",,,,,,,,,"" +2021-01-03 20:01:08.910 UTC,,,29,,5ff2226e.1d,3,,2021-01-03 20:00:46 UTC,,0,LOG,00000,"checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.000 s, sync=0.000 s, total=0.010 s; sync files=0, longest=0.000 s, average=0.000 s; distance=0 kB, estimate=0 kB",,,,,,,,,"" +2021-01-03 20:01:08.919 UTC,,,1,,5ff2226e.1,4,,2021-01-03 20:00:46 UTC,,0,LOG,00000,"database system is shut down",,,,,,,,,"" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-start-stop-csv.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-start-stop-csv.log-config.yml new file mode 100644 index 00000000000..860a8bfb242 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-start-stop-csv.log-config.yml @@ -0,0 +1,4 @@ +dynamic_fields: + event.ingested: ".*" +multiline: + first_line_pattern: '^\d{4}-\d{2}-\d{2} ' diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-start-stop-csv.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-start-stop-csv.log-expected.json new file mode 100644 index 00000000000..dfbe1e8395b --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-start-stop-csv.log-expected.json @@ -0,0 +1,439 @@ +{ + "expected": [ + { + "process": { + "pid": 28 + }, + "@timestamp": "2021-01-03T20:00:46.695Z", + "postgresql": { + "log": { + "transaction_id": 0, + "session_start_time": "2021-01-03T20:00:46.000Z", + "session_id": "5ff2226e.1c", + "sql_state_code": "00000", + "timestamp": "2021-01-03 20:00:46.695 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:50.746611594Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "database system was shut down at 2021-01-03 20:00:42 UTC" + }, + { + "process": { + "pid": 1 + }, + "@timestamp": "2021-01-03T20:00:46.708Z", + "postgresql": { + "log": { + "transaction_id": 0, + "session_start_time": "2021-01-03T20:00:46.000Z", + "session_id": "5ff2226e.1", + "sql_state_code": "00000", + "timestamp": "2021-01-03 20:00:46.708 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:50.746615556Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "database system is ready to accept connections" + }, + { + "process": { + "pid": 35 + }, + "@timestamp": "2021-01-03T20:01:00.349Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 38352, + "sql_state_code": "00000", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-03T20:01:00.000Z", + "session_id": "5ff2227c.23", + "timestamp": "2021-01-03 20:01:00.349 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:50.746617121Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "connection received: host=172.24.0.1 port=38352" + }, + { + "process": { + "pid": 36 + }, + "@timestamp": "2021-01-03T20:01:02.701Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 38356, + "sql_state_code": "00000", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-03T20:01:02.000Z", + "session_id": "5ff2227e.24", + "timestamp": "2021-01-03 20:01:02.701 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:50.746618552Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "connection received: host=172.24.0.1 port=38356" + }, + { + "process": { + "pid": 36 + }, + "@timestamp": "2021-01-03T20:01:02.727Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 38356, + "database": "postgres", + "virtual_transaction_id": "3/2", + "sql_state_code": "00000", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-03T20:01:02.000Z", + "session_id": "5ff2227e.24", + "command_tag": "authentication", + "timestamp": "2021-01-03 20:01:02.727 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "connection authorized: user=postgres database=postgres", + "event": { + "ingested": "2021-03-03T14:56:50.746619977Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 36 + }, + "@timestamp": "2021-01-03T20:01:07.094Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 38356, + "database": "postgres", + "virtual_transaction_id": "3/3", + "sql_state_code": "00000", + "application_name": "psql", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-03T20:01:02.000Z", + "session_id": "5ff2227e.24", + "command_tag": "idle", + "query_step": "statement", + "timestamp": "2021-01-03 20:01:07.094 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "SELECT 1;", + "event": { + "ingested": "2021-03-03T14:56:50.746621376Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 36 + }, + "@timestamp": "2021-01-03T20:01:07.724Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 38356, + "database": "postgres", + "sql_state_code": "00000", + "application_name": "psql", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-03T20:01:02.000Z", + "session_id": "5ff2227e.24", + "command_tag": "idle", + "timestamp": "2021-01-03 20:01:07.724 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "disconnection: session time: 0:00:05.023 user=postgres database=postgres host=172.24.0.1 port=38356", + "event": { + "ingested": "2021-03-03T14:56:50.746622775Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 1 + }, + "@timestamp": "2021-01-03T20:01:08.894Z", + "postgresql": { + "log": { + "transaction_id": 0, + "session_start_time": "2021-01-03T20:00:46.000Z", + "session_id": "5ff2226e.1", + "sql_state_code": "00000", + "timestamp": "2021-01-03 20:01:08.894 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:50.746624164Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "received smart shutdown request" + }, + { + "process": { + "pid": 1 + }, + "@timestamp": "2021-01-03T20:01:08.899Z", + "postgresql": { + "log": { + "transaction_id": 0, + "session_start_time": "2021-01-03T20:00:46.000Z", + "session_id": "5ff2226e.1", + "sql_state_code": "00000", + "timestamp": "2021-01-03 20:01:08.899 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:50.746625530Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "background worker \"logical replication launcher\" (PID 34) exited with exit code 1" + }, + { + "process": { + "pid": 29 + }, + "@timestamp": "2021-01-03T20:01:08.899Z", + "postgresql": { + "log": { + "transaction_id": 0, + "session_start_time": "2021-01-03T20:00:46.000Z", + "session_id": "5ff2226e.1d", + "sql_state_code": "00000", + "timestamp": "2021-01-03 20:01:08.899 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:50.746626886Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "shutting down" + }, + { + "process": { + "pid": 29 + }, + "@timestamp": "2021-01-03T20:01:08.901Z", + "postgresql": { + "log": { + "transaction_id": 0, + "session_start_time": "2021-01-03T20:00:46.000Z", + "session_id": "5ff2226e.1d", + "sql_state_code": "00000", + "timestamp": "2021-01-03 20:01:08.901 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:50.746628271Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "checkpoint starting: shutdown immediate" + }, + { + "process": { + "pid": 29 + }, + "@timestamp": "2021-01-03T20:01:08.910Z", + "postgresql": { + "log": { + "transaction_id": 0, + "session_start_time": "2021-01-03T20:00:46.000Z", + "session_id": "5ff2226e.1d", + "sql_state_code": "00000", + "timestamp": "2021-01-03 20:01:08.910 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:50.746629784Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "checkpoint complete: wrote 0 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.000 s, sync=0.000 s, total=0.010 s; sync files=0, longest=0.000 s, average=0.000 s; distance=0 kB, estimate=0 kB" + }, + { + "process": { + "pid": 1 + }, + "@timestamp": "2021-01-03T20:01:08.919Z", + "postgresql": { + "log": { + "transaction_id": 0, + "session_start_time": "2021-01-03T20:00:46.000Z", + "session_id": "5ff2226e.1", + "sql_state_code": "00000", + "timestamp": "2021-01-03 20:01:08.919 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:50.746631208Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "database system is shut down" + } + ] +} \ No newline at end of file diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-tempfile-csv.log b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-tempfile-csv.log new file mode 100644 index 00000000000..0d04c57102c --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-tempfile-csv.log @@ -0,0 +1,5 @@ +2021-01-04 00:33:05.565 UTC,"postgres","postgres",81,"172.24.0.1:42798",5ff26239.51,5,"SET",2021-01-04 00:32:57 UTC,3/0,0,LOG,00000,"duration: 0.189 ms statement: set log_temp_files = 0;",,,,,,,,,"psql" +2021-01-04 00:33:15.885 UTC,"postgres","postgres",81,"172.24.0.1:42798",5ff26239.51,6,"SELECT",2021-01-04 00:32:57 UTC,3/81,0,LOG,00000,"temporary file: path ""base/pgsql_tmp/pgsql_tmp81.3"", size 162381824",,,,,,"select * from generate_series(1, 3000) as t1(a), generate_series(1, 3000) as t2(a) order by 1 desc, 2;",,,"psql" +2021-01-04 00:33:15.885 UTC,"postgres","postgres",81,"172.24.0.1:42798",5ff26239.51,7,"SELECT",2021-01-04 00:32:57 UTC,3/81,0,LOG,00000,"temporary file: path ""base/pgsql_tmp/pgsql_tmp81.1"", size 42000",,,,,,"select * from generate_series(1, 3000) as t1(a), generate_series(1, 3000) as t2(a) order by 1 desc, 2;",,,"psql" +2021-01-04 00:33:15.885 UTC,"postgres","postgres",81,"172.24.0.1:42798",5ff26239.51,8,"SELECT",2021-01-04 00:32:57 UTC,3/81,0,LOG,00000,"temporary file: path ""base/pgsql_tmp/pgsql_tmp81.2"", size 42000",,,,,,"select * from generate_series(1, 3000) as t1(a), generate_series(1, 3000) as t2(a) order by 1 desc, 2;",,,"psql" +2021-01-04 00:33:15.885 UTC,"postgres","postgres",81,"172.24.0.1:42798",5ff26239.51,9,"SELECT",2021-01-04 00:32:57 UTC,3/0,0,LOG,00000,"duration: 6921.284 ms statement: select * from generate_series(1, 3000) as t1(a), generate_series(1, 3000) as t2(a) order by 1 desc, 2;",,,,,,,,,"psql" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-tempfile-csv.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-tempfile-csv.log-config.yml new file mode 100644 index 00000000000..860a8bfb242 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-tempfile-csv.log-config.yml @@ -0,0 +1,4 @@ +dynamic_fields: + event.ingested: ".*" +multiline: + first_line_pattern: '^\d{4}-\d{2}-\d{2} ' diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-tempfile-csv.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-tempfile-csv.log-expected.json new file mode 100644 index 00000000000..8a3543e258a --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-11-tempfile-csv.log-expected.json @@ -0,0 +1,231 @@ +{ + "expected": [ + { + "process": { + "pid": 81 + }, + "@timestamp": "2021-01-04T00:33:05.565Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42798, + "database": "postgres", + "virtual_transaction_id": "3/0", + "sql_state_code": "00000", + "application_name": "psql", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:32:57.000Z", + "session_id": "5ff26239.51", + "command_tag": "SET", + "query_step": "statement", + "timestamp": "2021-01-04 00:33:05.565 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "set log_temp_files = 0;", + "event": { + "duration": 189000, + "ingested": "2021-03-03T14:56:50.877242736Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 81 + }, + "@timestamp": "2021-01-04T00:33:15.885Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42798, + "database": "postgres", + "virtual_transaction_id": "3/81", + "sql_state_code": "00000", + "application_name": "psql", + "query": "select * from generate_series(1, 3000) as t1(a), generate_series(1, 3000) as t2(a) order by 1 desc, 2;", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:32:57.000Z", + "session_id": "5ff26239.51", + "command_tag": "SELECT", + "timestamp": "2021-01-04 00:33:15.885 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "temporary file: path \"base/pgsql_tmp/pgsql_tmp81.3\", size 162381824", + "event": { + "ingested": "2021-03-03T14:56:50.877246145Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 81 + }, + "@timestamp": "2021-01-04T00:33:15.885Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42798, + "database": "postgres", + "virtual_transaction_id": "3/81", + "sql_state_code": "00000", + "application_name": "psql", + "query": "select * from generate_series(1, 3000) as t1(a), generate_series(1, 3000) as t2(a) order by 1 desc, 2;", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:32:57.000Z", + "session_id": "5ff26239.51", + "command_tag": "SELECT", + "timestamp": "2021-01-04 00:33:15.885 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "temporary file: path \"base/pgsql_tmp/pgsql_tmp81.1\", size 42000", + "event": { + "ingested": "2021-03-03T14:56:50.877247700Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 81 + }, + "@timestamp": "2021-01-04T00:33:15.885Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42798, + "database": "postgres", + "virtual_transaction_id": "3/81", + "sql_state_code": "00000", + "application_name": "psql", + "query": "select * from generate_series(1, 3000) as t1(a), generate_series(1, 3000) as t2(a) order by 1 desc, 2;", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:32:57.000Z", + "session_id": "5ff26239.51", + "command_tag": "SELECT", + "timestamp": "2021-01-04 00:33:15.885 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "temporary file: path \"base/pgsql_tmp/pgsql_tmp81.2\", size 42000", + "event": { + "ingested": "2021-03-03T14:56:50.877249160Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 81 + }, + "@timestamp": "2021-01-04T00:33:15.885Z", + "postgresql": { + "log": { + "transaction_id": 0, + "client_port": 42798, + "database": "postgres", + "virtual_transaction_id": "3/0", + "sql_state_code": "00000", + "application_name": "psql", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T00:32:57.000Z", + "session_id": "5ff26239.51", + "command_tag": "SELECT", + "query_step": "statement", + "timestamp": "2021-01-04 00:33:15.885 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "select * from generate_series(1, 3000) as t1(a), generate_series(1, 3000) as t2(a) order by 1 desc, 2;", + "event": { + "duration": 6921284096, + "ingested": "2021-03-03T14:56:50.877250548Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + } + ] +} \ No newline at end of file diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-12-default.log b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-12-default.log new file mode 100644 index 00000000000..770f384134b --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-12-default.log @@ -0,0 +1,10 @@ +2020-04-16 11:45:11.844 CEST [26953] LOG: starting PostgreSQL 12.2 on x86_64-pc-linux-gnu, compiled by gcc (Arch Linux 9.3.0-1) 9.3.0, 64-bit +2020-04-16 11:45:11.844 CEST [26953] LOG: listening on IPv6 address "::1", port 5432 +2020-04-16 11:45:11.844 CEST [26953] LOG: listening on IPv4 address "127.0.0.1", port 5432 +2020-04-16 11:45:11.846 CEST [26953] LOG: listening on Unix socket "/tmp/.s.PGSQL.5432" +2020-04-16 11:45:11.861 CEST [26954] LOG: database system was shut down at 2020-04-16 11:45:01 CEST +2020-04-16 11:45:11.864 CEST [26953] LOG: database system is ready to accept connections +2020-04-16 12:22:22.579 CEST [26953] LOG: received smart shutdown request +2020-04-16 12:22:22.582 CEST [26953] LOG: background worker "logical replication launcher" (PID 26960) exited with exit code 1 +2020-04-16 12:22:22.582 CEST [26955] LOG: shutting down +2020-04-16 12:22:22.596 CEST [26953] LOG: database system is shut down diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-12-default.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-12-default.log-config.yml new file mode 100644 index 00000000000..860a8bfb242 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-12-default.log-config.yml @@ -0,0 +1,4 @@ +dynamic_fields: + event.ingested: ".*" +multiline: + first_line_pattern: '^\d{4}-\d{2}-\d{2} ' diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-12-default.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-12-default.log-expected.json new file mode 100644 index 00000000000..fa50396c7fd --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-12-default.log-expected.json @@ -0,0 +1,264 @@ +{ + "expected": [ + { + "process": { + "pid": 26953 + }, + "@timestamp": "2020-04-16T09:45:11.844Z", + "postgresql": { + "log": { + "timestamp": "2020-04-16 11:45:11.844 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "starting PostgreSQL 12.2 on x86_64-pc-linux-gnu, compiled by gcc (Arch Linux 9.3.0-1) 9.3.0, 64-bit", + "event": { + "ingested": "2021-03-03T14:56:51.443624368Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 26953 + }, + "@timestamp": "2020-04-16T09:45:11.844Z", + "postgresql": { + "log": { + "timestamp": "2020-04-16 11:45:11.844 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "listening on IPv6 address \"::1\", port 5432", + "event": { + "ingested": "2021-03-03T14:56:51.443629799Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 26953 + }, + "@timestamp": "2020-04-16T09:45:11.844Z", + "postgresql": { + "log": { + "timestamp": "2020-04-16 11:45:11.844 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "listening on IPv4 address \"127.0.0.1\", port 5432", + "event": { + "ingested": "2021-03-03T14:56:51.443632388Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 26953 + }, + "@timestamp": "2020-04-16T09:45:11.846Z", + "postgresql": { + "log": { + "timestamp": "2020-04-16 11:45:11.846 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "listening on Unix socket \"/tmp/.s.PGSQL.5432\"", + "event": { + "ingested": "2021-03-03T14:56:51.443634763Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 26954 + }, + "@timestamp": "2020-04-16T09:45:11.861Z", + "postgresql": { + "log": { + "timestamp": "2020-04-16 11:45:11.861 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system was shut down at 2020-04-16 11:45:01 CEST", + "event": { + "ingested": "2021-03-03T14:56:51.443636820Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 26953 + }, + "@timestamp": "2020-04-16T09:45:11.864Z", + "postgresql": { + "log": { + "timestamp": "2020-04-16 11:45:11.864 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system is ready to accept connections", + "event": { + "ingested": "2021-03-03T14:56:51.443638995Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 26953 + }, + "@timestamp": "2020-04-16T10:22:22.579Z", + "postgresql": { + "log": { + "timestamp": "2020-04-16 12:22:22.579 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "received smart shutdown request", + "event": { + "ingested": "2021-03-03T14:56:51.443641247Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 26953 + }, + "@timestamp": "2020-04-16T10:22:22.582Z", + "postgresql": { + "log": { + "timestamp": "2020-04-16 12:22:22.582 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "background worker \"logical replication launcher\" (PID 26960) exited with exit code 1", + "event": { + "ingested": "2021-03-03T14:56:51.443643454Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 26955 + }, + "@timestamp": "2020-04-16T10:22:22.582Z", + "postgresql": { + "log": { + "timestamp": "2020-04-16 12:22:22.582 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "shutting down", + "event": { + "ingested": "2021-03-03T14:56:51.443645477Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 26953 + }, + "@timestamp": "2020-04-16T10:22:22.596Z", + "postgresql": { + "log": { + "timestamp": "2020-04-16 12:22:22.596 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system is shut down", + "event": { + "ingested": "2021-03-03T14:56:51.443647468Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + } + ] +} \ No newline at end of file diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-12-min-duration-statement.log b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-12-min-duration-statement.log new file mode 100644 index 00000000000..0d92eeb4122 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-12-min-duration-statement.log @@ -0,0 +1,21 @@ +2020-04-16 12:48:36.677 CEST [34492] LOG: duration: 0.327 ms statement: select 1; +2020-04-16 12:48:40.316 CEST [34492] LOG: duration: 0.320 ms statement: select version(); +2020-04-16 12:48:44.696 CEST [34492] ERROR: column "name" does not exist at character 8 +2020-04-16 12:48:44.696 CEST [34492] STATEMENT: select name from user; +2020-04-16 12:49:16.871 CEST [34492] LOG: duration: 3.431 ms statement: CREATE TABLE weather ( + city varchar(80), + temp_lo int, + temp_hi int, + prcp real, + date date + ); +2020-04-16 12:49:19.866 CEST [34492] ERROR: relation "weather" already exists +2020-04-16 12:49:54.907 CEST [34492] LOG: duration: 3.039 ms statement: SELECT pg_catalog.quote_ident(c.relname) FROM pg_catalog.pg_class c WHERE c.relkind IN ('r', 'S', 'v', 'm', 'f', 'p') AND substring(pg_catalog.quote_ident(c.relname),1,2)='we' AND pg_catalog.pg_table_is_visible(c.oid) AND c.relnamespace <> (SELECT oid FROM pg_catalog.pg_namespace WHERE nspname = 'pg_catalog') + UNION + SELECT pg_catalog.quote_ident(n.nspname) || '.' FROM pg_catalog.pg_namespace n WHERE substring(pg_catalog.quote_ident(n.nspname) || '.',1,2)='we' AND (SELECT pg_catalog.count(*) FROM pg_catalog.pg_namespace WHERE substring(pg_catalog.quote_ident(nspname) || '.',1,2) = substring('we',1,pg_catalog.length(pg_catalog.quote_ident(nspname))+1)) > 1 + UNION + SELECT pg_catalog.quote_ident(n.nspname) || '.' || pg_catalog.quote_ident(c.relname) FROM pg_catalog.pg_class c, pg_catalog.pg_namespace n WHERE c.relnamespace = n.oid AND c.relkind IN ('r', 'S', 'v', 'm', 'f', 'p') AND substring(pg_catalog.quote_ident(n.nspname) || '.' || pg_catalog.quote_ident(c.relname),1,2)='we' AND substring(pg_catalog.quote_ident(n.nspname) || '.',1,2) = substring('we',1,pg_catalog.length(pg_catalog.quote_ident(n.nspname))+1) AND (SELECT pg_catalog.count(*) FROM pg_catalog.pg_namespace WHERE substring(pg_catalog.quote_ident(nspname) || '.',1,2) = substring('we',1,pg_catalog.length(pg_catalog.quote_ident(nspname))+1)) = 1 + LIMIT 1000 +2020-04-16 12:49:55.464 CEST [34492] LOG: duration: 0.179 ms statement: select * From weather ; +2020-04-16 12:50:05.322 CEST [34492] LOG: duration: 1.661 ms statement: INSERT INTO weather VALUES ('San Francisco', 46, 50, 0.25, '1994-11-27'); +2020-04-16 12:50:06.741 CEST [34492] LOG: duration: 0.144 ms statement: select * From weather ; diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-12-min-duration-statement.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-12-min-duration-statement.log-config.yml new file mode 100644 index 00000000000..860a8bfb242 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-12-min-duration-statement.log-config.yml @@ -0,0 +1,4 @@ +dynamic_fields: + event.ingested: ".*" +multiline: + first_line_pattern: '^\d{4}-\d{2}-\d{2} ' diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-12-min-duration-statement.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-12-min-duration-statement.log-expected.json new file mode 100644 index 00000000000..a0fb8b3c7fe --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-12-min-duration-statement.log-expected.json @@ -0,0 +1,285 @@ +{ + "expected": [ + { + "process": { + "pid": 34492 + }, + "@timestamp": "2020-04-16T10:48:36.677Z", + "postgresql": { + "log": { + "query_step": "statement", + "query": "select 1;", + "timestamp": "2020-04-16 12:48:36.677 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "2020-04-16 12:48:36.677 CEST [34492] LOG: duration: 0.327 ms statement: select 1;", + "event": { + "duration": 327000, + "ingested": "2021-03-03T14:56:51.517923931Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 34492 + }, + "@timestamp": "2020-04-16T10:48:40.316Z", + "postgresql": { + "log": { + "query_step": "statement", + "query": "select version();", + "timestamp": "2020-04-16 12:48:40.316 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "2020-04-16 12:48:40.316 CEST [34492] LOG: duration: 0.320 ms statement: select version();", + "event": { + "duration": 320000, + "ingested": "2021-03-03T14:56:51.517929353Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 34492 + }, + "@timestamp": "2020-04-16T10:48:44.696Z", + "postgresql": { + "log": { + "timestamp": "2020-04-16 12:48:44.696 CEST" + } + }, + "log": { + "level": "ERROR" + }, + "message": "column \"name\" does not exist at character 8", + "event": { + "ingested": "2021-03-03T14:56:51.517931830Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 34492 + }, + "@timestamp": "2020-04-16T10:48:44.696Z", + "postgresql": { + "log": { + "timestamp": "2020-04-16 12:48:44.696 CEST" + } + }, + "log": { + "level": "STATEMENT" + }, + "message": "select name from user;", + "event": { + "ingested": "2021-03-03T14:56:51.517934367Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 34492 + }, + "@timestamp": "2020-04-16T10:49:16.871Z", + "postgresql": { + "log": { + "query_step": "statement", + "query": "CREATE TABLE weather (\n\t city varchar(80),\n\t temp_lo int,\n\t temp_hi int,\n\t prcp real,\n\t date date\n\t);", + "timestamp": "2020-04-16 12:49:16.871 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "2020-04-16 12:49:16.871 CEST [34492] LOG: duration: 3.431 ms statement: CREATE TABLE weather (\n\t city varchar(80),\n\t temp_lo int,\n\t temp_hi int,\n\t prcp real,\n\t date date\n\t);", + "event": { + "duration": 3431000, + "ingested": "2021-03-03T14:56:51.517936695Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 34492 + }, + "@timestamp": "2020-04-16T10:49:19.866Z", + "postgresql": { + "log": { + "timestamp": "2020-04-16 12:49:19.866 CEST" + } + }, + "log": { + "level": "ERROR" + }, + "message": "relation \"weather\" already exists", + "event": { + "ingested": "2021-03-03T14:56:51.517938944Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 34492 + }, + "@timestamp": "2020-04-16T10:49:54.907Z", + "postgresql": { + "log": { + "query_step": "statement", + "query": "SELECT pg_catalog.quote_ident(c.relname) FROM pg_catalog.pg_class c WHERE c.relkind IN ('r', 'S', 'v', 'm', 'f', 'p') AND substring(pg_catalog.quote_ident(c.relname),1,2)='we' AND pg_catalog.pg_table_is_visible(c.oid) AND c.relnamespace \u003c\u003e (SELECT oid FROM pg_catalog.pg_namespace WHERE nspname = 'pg_catalog')\n\tUNION\n\tSELECT pg_catalog.quote_ident(n.nspname) || '.' FROM pg_catalog.pg_namespace n WHERE substring(pg_catalog.quote_ident(n.nspname) || '.',1,2)='we' AND (SELECT pg_catalog.count(*) FROM pg_catalog.pg_namespace WHERE substring(pg_catalog.quote_ident(nspname) || '.',1,2) = substring('we',1,pg_catalog.length(pg_catalog.quote_ident(nspname))+1)) \u003e 1\n\tUNION\n\tSELECT pg_catalog.quote_ident(n.nspname) || '.' || pg_catalog.quote_ident(c.relname) FROM pg_catalog.pg_class c, pg_catalog.pg_namespace n WHERE c.relnamespace = n.oid AND c.relkind IN ('r', 'S', 'v', 'm', 'f', 'p') AND substring(pg_catalog.quote_ident(n.nspname) || '.' || pg_catalog.quote_ident(c.relname),1,2)='we' AND substring(pg_catalog.quote_ident(n.nspname) || '.',1,2) = substring('we',1,pg_catalog.length(pg_catalog.quote_ident(n.nspname))+1) AND (SELECT pg_catalog.count(*) FROM pg_catalog.pg_namespace WHERE substring(pg_catalog.quote_ident(nspname) || '.',1,2) = substring('we',1,pg_catalog.length(pg_catalog.quote_ident(nspname))+1)) = 1\n\tLIMIT 1000", + "timestamp": "2020-04-16 12:49:54.907 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "2020-04-16 12:49:54.907 CEST [34492] LOG: duration: 3.039 ms statement: SELECT pg_catalog.quote_ident(c.relname) FROM pg_catalog.pg_class c WHERE c.relkind IN ('r', 'S', 'v', 'm', 'f', 'p') AND substring(pg_catalog.quote_ident(c.relname),1,2)='we' AND pg_catalog.pg_table_is_visible(c.oid) AND c.relnamespace \u003c\u003e (SELECT oid FROM pg_catalog.pg_namespace WHERE nspname = 'pg_catalog')\n\tUNION\n\tSELECT pg_catalog.quote_ident(n.nspname) || '.' FROM pg_catalog.pg_namespace n WHERE substring(pg_catalog.quote_ident(n.nspname) || '.',1,2)='we' AND (SELECT pg_catalog.count(*) FROM pg_catalog.pg_namespace WHERE substring(pg_catalog.quote_ident(nspname) || '.',1,2) = substring('we',1,pg_catalog.length(pg_catalog.quote_ident(nspname))+1)) \u003e 1\n\tUNION\n\tSELECT pg_catalog.quote_ident(n.nspname) || '.' || pg_catalog.quote_ident(c.relname) FROM pg_catalog.pg_class c, pg_catalog.pg_namespace n WHERE c.relnamespace = n.oid AND c.relkind IN ('r', 'S', 'v', 'm', 'f', 'p') AND substring(pg_catalog.quote_ident(n.nspname) || '.' || pg_catalog.quote_ident(c.relname),1,2)='we' AND substring(pg_catalog.quote_ident(n.nspname) || '.',1,2) = substring('we',1,pg_catalog.length(pg_catalog.quote_ident(n.nspname))+1) AND (SELECT pg_catalog.count(*) FROM pg_catalog.pg_namespace WHERE substring(pg_catalog.quote_ident(nspname) || '.',1,2) = substring('we',1,pg_catalog.length(pg_catalog.quote_ident(nspname))+1)) = 1\n\tLIMIT 1000", + "event": { + "duration": 3039000, + "ingested": "2021-03-03T14:56:51.517940679Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 34492 + }, + "@timestamp": "2020-04-16T10:49:55.464Z", + "postgresql": { + "log": { + "query_step": "statement", + "query": "select * From weather ;", + "timestamp": "2020-04-16 12:49:55.464 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "2020-04-16 12:49:55.464 CEST [34492] LOG: duration: 0.179 ms statement: select * From weather ;", + "event": { + "duration": 179000, + "ingested": "2021-03-03T14:56:51.517942273Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 34492 + }, + "@timestamp": "2020-04-16T10:50:05.322Z", + "postgresql": { + "log": { + "query_step": "statement", + "query": "INSERT INTO weather VALUES ('San Francisco', 46, 50, 0.25, '1994-11-27');", + "timestamp": "2020-04-16 12:50:05.322 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "2020-04-16 12:50:05.322 CEST [34492] LOG: duration: 1.661 ms statement: INSERT INTO weather VALUES ('San Francisco', 46, 50, 0.25, '1994-11-27');", + "event": { + "duration": 1661000, + "ingested": "2021-03-03T14:56:51.517943850Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 34492 + }, + "@timestamp": "2020-04-16T10:50:06.741Z", + "postgresql": { + "log": { + "query_step": "statement", + "query": "select * From weather ;", + "timestamp": "2020-04-16 12:50:06.741 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "2020-04-16 12:50:06.741 CEST [34492] LOG: duration: 0.144 ms statement: select * From weather ;", + "event": { + "duration": 144000, + "ingested": "2021-03-03T14:56:51.517945378Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + } + ] +} \ No newline at end of file diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-13-csv.log b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-13-csv.log new file mode 100644 index 00000000000..10912d64eb7 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-13-csv.log @@ -0,0 +1,30 @@ +2021-01-04 01:06:13.270 UTC,,,1,,5ff26a05.1,1,,2021-01-04 01:06:13 UTC,,0,LOG,00000,"starting PostgreSQL 13.1 (Debian 13.1-1.pgdg100+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 8.3.0-6) 8.3.0, 64-bit",,,,,,,,,"","postmaster" +2021-01-04 01:06:13.270 UTC,,,1,,5ff26a05.1,2,,2021-01-04 01:06:13 UTC,,0,LOG,00000,"listening on IPv4 address ""0.0.0.0"", port 5432",,,,,,,,,"","postmaster" +2021-01-04 01:06:13.270 UTC,,,1,,5ff26a05.1,3,,2021-01-04 01:06:13 UTC,,0,LOG,00000,"listening on IPv6 address ""::"", port 5432",,,,,,,,,"","postmaster" +2021-01-04 01:06:13.273 UTC,,,1,,5ff26a05.1,4,,2021-01-04 01:06:13 UTC,,0,LOG,00000,"listening on Unix socket ""/var/run/postgresql/.s.PGSQL.5432""",,,,,,,,,"","postmaster" +2021-01-04 01:06:13.281 UTC,,,79,,5ff26a05.4f,1,,2021-01-04 01:06:13 UTC,,0,LOG,00000,"database system was shut down at 2021-01-04 01:06:13 UTC",,,,,,,,,"","startup" +2021-01-04 01:06:13.289 UTC,,,1,,5ff26a05.1,5,,2021-01-04 01:06:13 UTC,,0,LOG,00000,"database system is ready to accept connections",,,,,,,,,"","postmaster" +2021-01-04 01:06:20.982 UTC,,,86,"172.24.0.1:45126",5ff26a0c.56,1,"",2021-01-04 01:06:20 UTC,,0,LOG,00000,"connection received: host=172.24.0.1 port=45126",,,,,,,,,"","not initialized" +2021-01-04 01:06:21.083 UTC,"postgres","postgres",86,"172.24.0.1:45126",5ff26a0c.56,2,"authentication",2021-01-04 01:06:20 UTC,3/1,0,LOG,00000,"connection authorized: user=postgres database=postgres application_name=psql",,,,,,,,,"","client backend" +2021-01-04 01:06:25.161 UTC,"postgres","postgres",86,"172.24.0.1:45126",5ff26a0c.56,3,"idle",2021-01-04 01:06:20 UTC,3/2,0,LOG,00000,"statement: select 1;",,,,,,,,,"psql","client backend" +2021-01-04 01:06:41.115 UTC,"postgres","postgres",86,"172.24.0.1:45126",5ff26a0c.56,4,"idle",2021-01-04 01:06:20 UTC,3/3,0,LOG,00000,"statement: select name, setting from pg_settings where name like 'log%';",,,,,,,,,"psql","client backend" +2021-01-04 01:06:54.227 UTC,"postgres","postgres",86,"172.24.0.1:45126",5ff26a0c.56,5,"idle",2021-01-04 01:06:20 UTC,3/4,0,LOG,00000,"statement: SELECT name FROM (SELECT pg_catalog.lower(name) AS name FROM pg_catalog.pg_settings WHERE context != 'internal' UNION ALL SELECT 'all') ss WHERE substring(name,1,7)='log_min' +LIMIT 1000",,,,,,,,,"psql","client backend" +2021-01-04 01:06:55.502 UTC,"postgres","postgres",86,"172.24.0.1:45126",5ff26a0c.56,6,"idle",2021-01-04 01:06:20 UTC,3/5,0,LOG,00000,"statement: SELECT name FROM (SELECT pg_catalog.lower(name) AS name FROM pg_catalog.pg_settings WHERE context != 'internal' UNION ALL SELECT 'all') ss WHERE substring(name,1,10)='log_min_du' +LIMIT 1000",,,,,,,,,"psql","client backend" +2021-01-04 01:06:58.297 UTC,"postgres","postgres",86,"172.24.0.1:45126",5ff26a0c.56,7,"idle",2021-01-04 01:06:20 UTC,3/6,0,LOG,00000,"statement: SELECT name FROM (SELECT pg_catalog.lower(name) AS name FROM pg_catalog.pg_settings WHERE context != 'internal' UNION ALL SELECT 'all') ss WHERE substring(name,1,19)='log_min_duration_st' +LIMIT 1000",,,,,,,,,"psql","client backend" +2021-01-04 01:07:01.116 UTC,"postgres","postgres",86,"172.24.0.1:45126",5ff26a0c.56,8,"idle",2021-01-04 01:06:20 UTC,3/7,0,LOG,00000,"statement: alter system set log_min_duration_statement = 0;",,,,,,,,,"psql","client backend" +2021-01-04 01:07:04.364 UTC,"postgres","postgres",86,"172.24.0.1:45126",5ff26a0c.56,9,"idle",2021-01-04 01:06:20 UTC,3/8,0,LOG,00000,"statement: SELECT name FROM (SELECT pg_catalog.lower(name) AS name FROM pg_catalog.pg_settings WHERE context != 'internal' UNION ALL SELECT 'all') ss WHERE substring(name,1,3)='log' +LIMIT 1000",,,,,,,,,"psql","client backend" +2021-01-04 01:07:07.070 UTC,"postgres","postgres",86,"172.24.0.1:45126",5ff26a0c.56,10,"idle",2021-01-04 01:06:20 UTC,3/9,0,LOG,00000,"statement: SELECT name FROM (SELECT pg_catalog.lower(name) AS name FROM pg_catalog.pg_settings WHERE context != 'internal' UNION ALL SELECT 'all') ss WHERE substring(name,1,6)='log_st' +LIMIT 1000",,,,,,,,,"psql","client backend" +2021-01-04 01:07:13.725 UTC,"postgres","postgres",86,"172.24.0.1:45126",5ff26a0c.56,11,"idle",2021-01-04 01:06:20 UTC,3/10,0,LOG,00000,"statement: alter system set log_statement = 'none';",,,,,,,,,"psql","client backend" +2021-01-04 01:07:19.998 UTC,"postgres","postgres",86,"172.24.0.1:45126",5ff26a0c.56,12,"idle",2021-01-04 01:06:20 UTC,3/11,0,LOG,00000,"statement: select pg_reload_conf();",,,,,,,,,"psql","client backend" +2021-01-04 01:07:19.999 UTC,,,1,,5ff26a05.1,6,,2021-01-04 01:06:13 UTC,,0,LOG,00000,"received SIGHUP, reloading configuration files",,,,,,,,,"","postmaster" +2021-01-04 01:07:20.001 UTC,,,1,,5ff26a05.1,7,,2021-01-04 01:06:13 UTC,,0,LOG,00000,"parameter ""log_min_duration_statement"" changed to ""0""",,,,,,,,,"","postmaster" +2021-01-04 01:07:20.001 UTC,,,1,,5ff26a05.1,8,,2021-01-04 01:06:13 UTC,,0,LOG,00000,"parameter ""log_statement"" changed to ""none""",,,,,,,,,"","postmaster" +2021-01-04 01:07:24.360 UTC,,,80,,5ff26a05.50,1,,2021-01-04 01:06:13 UTC,,0,LOG,00000,"checkpoint starting: immediate force wait",,,,,,,,,"","checkpointer" +2021-01-04 01:07:24.374 UTC,,,80,,5ff26a05.50,2,,2021-01-04 01:06:13 UTC,,0,LOG,00000,"checkpoint complete: wrote 1 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.014 s; sync files=1, longest=0.001 s, average=0.001 s; distance=0 kB, estimate=0 kB",,,,,,,,,"","checkpointer" +2021-01-04 01:07:24.374 UTC,"postgres","postgres",86,"172.24.0.1:45126",5ff26a0c.56,13,"CHECKPOINT",2021-01-04 01:06:20 UTC,3/0,0,LOG,00000,"duration: 15.136 ms statement: checkpoint;",,,,,,,,,"psql","client backend" +2021-01-04 01:07:25.950 UTC,"postgres","postgres",86,"172.24.0.1:45126",5ff26a0c.56,14,"idle",2021-01-04 01:06:20 UTC,,0,LOG,00000,"disconnection: session time: 0:01:04.968 user=postgres database=postgres host=172.24.0.1 port=45126",,,,,,,,,"psql","client backend" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-13-csv.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-13-csv.log-config.yml new file mode 100644 index 00000000000..860a8bfb242 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-13-csv.log-config.yml @@ -0,0 +1,4 @@ +dynamic_fields: + event.ingested: ".*" +multiline: + first_line_pattern: '^\d{4}-\d{2}-\d{2} ' diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-13-csv.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-13-csv.log-expected.json new file mode 100644 index 00000000000..7e425a1afae --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-13-csv.log-expected.json @@ -0,0 +1,973 @@ +{ + "expected": [ + { + "process": { + "pid": 1 + }, + "@timestamp": "2021-01-04T01:06:13.270Z", + "postgresql": { + "log": { + "transaction_id": 0, + "session_start_time": "2021-01-04T01:06:13.000Z", + "backend_type": "postmaster", + "session_id": "5ff26a05.1", + "sql_state_code": "00000", + "timestamp": "2021-01-04 01:06:13.270 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:51.655770571Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "starting PostgreSQL 13.1 (Debian 13.1-1.pgdg100+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 8.3.0-6) 8.3.0, 64-bit" + }, + { + "process": { + "pid": 1 + }, + "@timestamp": "2021-01-04T01:06:13.270Z", + "postgresql": { + "log": { + "transaction_id": 0, + "session_start_time": "2021-01-04T01:06:13.000Z", + "backend_type": "postmaster", + "session_id": "5ff26a05.1", + "sql_state_code": "00000", + "timestamp": "2021-01-04 01:06:13.270 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:51.655774048Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "listening on IPv4 address \"0.0.0.0\", port 5432" + }, + { + "process": { + "pid": 1 + }, + "@timestamp": "2021-01-04T01:06:13.270Z", + "postgresql": { + "log": { + "transaction_id": 0, + "session_start_time": "2021-01-04T01:06:13.000Z", + "backend_type": "postmaster", + "session_id": "5ff26a05.1", + "sql_state_code": "00000", + "timestamp": "2021-01-04 01:06:13.270 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:51.655775511Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "listening on IPv6 address \"::\", port 5432" + }, + { + "process": { + "pid": 1 + }, + "@timestamp": "2021-01-04T01:06:13.273Z", + "postgresql": { + "log": { + "transaction_id": 0, + "session_start_time": "2021-01-04T01:06:13.000Z", + "backend_type": "postmaster", + "session_id": "5ff26a05.1", + "sql_state_code": "00000", + "timestamp": "2021-01-04 01:06:13.273 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:51.655776933Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "listening on Unix socket \"/var/run/postgresql/.s.PGSQL.5432\"" + }, + { + "process": { + "pid": 79 + }, + "@timestamp": "2021-01-04T01:06:13.281Z", + "postgresql": { + "log": { + "transaction_id": 0, + "session_start_time": "2021-01-04T01:06:13.000Z", + "backend_type": "startup", + "session_id": "5ff26a05.4f", + "sql_state_code": "00000", + "timestamp": "2021-01-04 01:06:13.281 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:51.655778356Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "database system was shut down at 2021-01-04 01:06:13 UTC" + }, + { + "process": { + "pid": 1 + }, + "@timestamp": "2021-01-04T01:06:13.289Z", + "postgresql": { + "log": { + "transaction_id": 0, + "session_start_time": "2021-01-04T01:06:13.000Z", + "backend_type": "postmaster", + "session_id": "5ff26a05.1", + "sql_state_code": "00000", + "timestamp": "2021-01-04 01:06:13.289 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:51.655779770Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "database system is ready to accept connections" + }, + { + "process": { + "pid": 86 + }, + "@timestamp": "2021-01-04T01:06:20.982Z", + "postgresql": { + "log": { + "transaction_id": 0, + "backend_type": "not initialized", + "client_port": 45126, + "sql_state_code": "00000", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T01:06:20.000Z", + "session_id": "5ff26a0c.56", + "timestamp": "2021-01-04 01:06:20.982 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:51.655781183Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "connection received: host=172.24.0.1 port=45126" + }, + { + "process": { + "pid": 86 + }, + "@timestamp": "2021-01-04T01:06:21.083Z", + "postgresql": { + "log": { + "transaction_id": 0, + "backend_type": "client backend", + "client_port": 45126, + "database": "postgres", + "virtual_transaction_id": "3/1", + "sql_state_code": "00000", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T01:06:20.000Z", + "session_id": "5ff26a0c.56", + "command_tag": "authentication", + "timestamp": "2021-01-04 01:06:21.083 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "connection authorized: user=postgres database=postgres application_name=psql", + "event": { + "ingested": "2021-03-03T14:56:51.655782653Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 86 + }, + "@timestamp": "2021-01-04T01:06:25.161Z", + "postgresql": { + "log": { + "transaction_id": 0, + "backend_type": "client backend", + "sql_state_code": "00000", + "session_start_time": "2021-01-04T01:06:20.000Z", + "session_id": "5ff26a0c.56", + "command_tag": "idle", + "client_port": 45126, + "database": "postgres", + "virtual_transaction_id": "3/2", + "application_name": "psql", + "client_addr": "172.24.0.1", + "query_step": "statement", + "timestamp": "2021-01-04 01:06:25.161 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "select 1;", + "event": { + "ingested": "2021-03-03T14:56:51.655784075Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 86 + }, + "@timestamp": "2021-01-04T01:06:41.115Z", + "postgresql": { + "log": { + "transaction_id": 0, + "backend_type": "client backend", + "sql_state_code": "00000", + "session_start_time": "2021-01-04T01:06:20.000Z", + "session_id": "5ff26a0c.56", + "command_tag": "idle", + "client_port": 45126, + "database": "postgres", + "virtual_transaction_id": "3/3", + "application_name": "psql", + "client_addr": "172.24.0.1", + "query_step": "statement", + "timestamp": "2021-01-04 01:06:41.115 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "select name, setting from pg_settings where name like 'log%';", + "event": { + "ingested": "2021-03-03T14:56:51.655785575Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 86 + }, + "@timestamp": "2021-01-04T01:06:54.227Z", + "postgresql": { + "log": { + "transaction_id": 0, + "backend_type": "client backend", + "sql_state_code": "00000", + "session_start_time": "2021-01-04T01:06:20.000Z", + "session_id": "5ff26a0c.56", + "command_tag": "idle", + "client_port": 45126, + "database": "postgres", + "virtual_transaction_id": "3/4", + "application_name": "psql", + "client_addr": "172.24.0.1", + "query_step": "statement", + "timestamp": "2021-01-04 01:06:54.227 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "SELECT name FROM (SELECT pg_catalog.lower(name) AS name FROM pg_catalog.pg_settings WHERE context != 'internal' UNION ALL SELECT 'all') ss WHERE substring(name,1,7)='log_min'\nLIMIT 1000", + "event": { + "ingested": "2021-03-03T14:56:51.655786995Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 86 + }, + "@timestamp": "2021-01-04T01:06:55.502Z", + "postgresql": { + "log": { + "transaction_id": 0, + "backend_type": "client backend", + "sql_state_code": "00000", + "session_start_time": "2021-01-04T01:06:20.000Z", + "session_id": "5ff26a0c.56", + "command_tag": "idle", + "client_port": 45126, + "database": "postgres", + "virtual_transaction_id": "3/5", + "application_name": "psql", + "client_addr": "172.24.0.1", + "query_step": "statement", + "timestamp": "2021-01-04 01:06:55.502 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "SELECT name FROM (SELECT pg_catalog.lower(name) AS name FROM pg_catalog.pg_settings WHERE context != 'internal' UNION ALL SELECT 'all') ss WHERE substring(name,1,10)='log_min_du'\nLIMIT 1000", + "event": { + "ingested": "2021-03-03T14:56:51.655788551Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 86 + }, + "@timestamp": "2021-01-04T01:06:58.297Z", + "postgresql": { + "log": { + "transaction_id": 0, + "backend_type": "client backend", + "sql_state_code": "00000", + "session_start_time": "2021-01-04T01:06:20.000Z", + "session_id": "5ff26a0c.56", + "command_tag": "idle", + "client_port": 45126, + "database": "postgres", + "virtual_transaction_id": "3/6", + "application_name": "psql", + "client_addr": "172.24.0.1", + "query_step": "statement", + "timestamp": "2021-01-04 01:06:58.297 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "SELECT name FROM (SELECT pg_catalog.lower(name) AS name FROM pg_catalog.pg_settings WHERE context != 'internal' UNION ALL SELECT 'all') ss WHERE substring(name,1,19)='log_min_duration_st'\nLIMIT 1000", + "event": { + "ingested": "2021-03-03T14:56:51.655789951Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 86 + }, + "@timestamp": "2021-01-04T01:07:01.116Z", + "postgresql": { + "log": { + "transaction_id": 0, + "backend_type": "client backend", + "sql_state_code": "00000", + "session_start_time": "2021-01-04T01:06:20.000Z", + "session_id": "5ff26a0c.56", + "command_tag": "idle", + "client_port": 45126, + "database": "postgres", + "virtual_transaction_id": "3/7", + "application_name": "psql", + "client_addr": "172.24.0.1", + "query_step": "statement", + "timestamp": "2021-01-04 01:07:01.116 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "alter system set log_min_duration_statement = 0;", + "event": { + "ingested": "2021-03-03T14:56:51.655791334Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 86 + }, + "@timestamp": "2021-01-04T01:07:04.364Z", + "postgresql": { + "log": { + "transaction_id": 0, + "backend_type": "client backend", + "sql_state_code": "00000", + "session_start_time": "2021-01-04T01:06:20.000Z", + "session_id": "5ff26a0c.56", + "command_tag": "idle", + "client_port": 45126, + "database": "postgres", + "virtual_transaction_id": "3/8", + "application_name": "psql", + "client_addr": "172.24.0.1", + "query_step": "statement", + "timestamp": "2021-01-04 01:07:04.364 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "SELECT name FROM (SELECT pg_catalog.lower(name) AS name FROM pg_catalog.pg_settings WHERE context != 'internal' UNION ALL SELECT 'all') ss WHERE substring(name,1,3)='log'\nLIMIT 1000", + "event": { + "ingested": "2021-03-03T14:56:51.655792710Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 86 + }, + "@timestamp": "2021-01-04T01:07:07.070Z", + "postgresql": { + "log": { + "transaction_id": 0, + "backend_type": "client backend", + "sql_state_code": "00000", + "session_start_time": "2021-01-04T01:06:20.000Z", + "session_id": "5ff26a0c.56", + "command_tag": "idle", + "client_port": 45126, + "database": "postgres", + "virtual_transaction_id": "3/9", + "application_name": "psql", + "client_addr": "172.24.0.1", + "query_step": "statement", + "timestamp": "2021-01-04 01:07:07.070 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "SELECT name FROM (SELECT pg_catalog.lower(name) AS name FROM pg_catalog.pg_settings WHERE context != 'internal' UNION ALL SELECT 'all') ss WHERE substring(name,1,6)='log_st'\nLIMIT 1000", + "event": { + "ingested": "2021-03-03T14:56:51.655794095Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 86 + }, + "@timestamp": "2021-01-04T01:07:13.725Z", + "postgresql": { + "log": { + "transaction_id": 0, + "backend_type": "client backend", + "sql_state_code": "00000", + "session_start_time": "2021-01-04T01:06:20.000Z", + "session_id": "5ff26a0c.56", + "command_tag": "idle", + "client_port": 45126, + "database": "postgres", + "virtual_transaction_id": "3/10", + "application_name": "psql", + "client_addr": "172.24.0.1", + "query_step": "statement", + "timestamp": "2021-01-04 01:07:13.725 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "alter system set log_statement = 'none';", + "event": { + "ingested": "2021-03-03T14:56:51.655795599Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 86 + }, + "@timestamp": "2021-01-04T01:07:19.998Z", + "postgresql": { + "log": { + "transaction_id": 0, + "backend_type": "client backend", + "sql_state_code": "00000", + "session_start_time": "2021-01-04T01:06:20.000Z", + "session_id": "5ff26a0c.56", + "command_tag": "idle", + "client_port": 45126, + "database": "postgres", + "virtual_transaction_id": "3/11", + "application_name": "psql", + "client_addr": "172.24.0.1", + "query_step": "statement", + "timestamp": "2021-01-04 01:07:19.998 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "select pg_reload_conf();", + "event": { + "ingested": "2021-03-03T14:56:51.655797003Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 1 + }, + "@timestamp": "2021-01-04T01:07:19.999Z", + "postgresql": { + "log": { + "transaction_id": 0, + "session_start_time": "2021-01-04T01:06:13.000Z", + "backend_type": "postmaster", + "session_id": "5ff26a05.1", + "sql_state_code": "00000", + "timestamp": "2021-01-04 01:07:19.999 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:51.655798394Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "received SIGHUP, reloading configuration files" + }, + { + "process": { + "pid": 1 + }, + "@timestamp": "2021-01-04T01:07:20.001Z", + "postgresql": { + "log": { + "transaction_id": 0, + "session_start_time": "2021-01-04T01:06:13.000Z", + "backend_type": "postmaster", + "session_id": "5ff26a05.1", + "sql_state_code": "00000", + "timestamp": "2021-01-04 01:07:20.001 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:51.655799783Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "parameter \"log_min_duration_statement\" changed to \"0\"" + }, + { + "process": { + "pid": 1 + }, + "@timestamp": "2021-01-04T01:07:20.001Z", + "postgresql": { + "log": { + "transaction_id": 0, + "session_start_time": "2021-01-04T01:06:13.000Z", + "backend_type": "postmaster", + "session_id": "5ff26a05.1", + "sql_state_code": "00000", + "timestamp": "2021-01-04 01:07:20.001 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:51.655801169Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "parameter \"log_statement\" changed to \"none\"" + }, + { + "process": { + "pid": 80 + }, + "@timestamp": "2021-01-04T01:07:24.360Z", + "postgresql": { + "log": { + "transaction_id": 0, + "session_start_time": "2021-01-04T01:06:13.000Z", + "backend_type": "checkpointer", + "session_id": "5ff26a05.50", + "sql_state_code": "00000", + "timestamp": "2021-01-04 01:07:24.360 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:51.655802567Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "checkpoint starting: immediate force wait" + }, + { + "process": { + "pid": 80 + }, + "@timestamp": "2021-01-04T01:07:24.374Z", + "postgresql": { + "log": { + "transaction_id": 0, + "session_start_time": "2021-01-04T01:06:13.000Z", + "backend_type": "checkpointer", + "session_id": "5ff26a05.50", + "sql_state_code": "00000", + "timestamp": "2021-01-04 01:07:24.374 UTC" + } + }, + "log": { + "level": "LOG" + }, + "event": { + "ingested": "2021-03-03T14:56:51.655803947Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "message": "checkpoint complete: wrote 1 buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.001 s, sync=0.001 s, total=0.014 s; sync files=1, longest=0.001 s, average=0.001 s; distance=0 kB, estimate=0 kB" + }, + { + "process": { + "pid": 86 + }, + "@timestamp": "2021-01-04T01:07:24.374Z", + "postgresql": { + "log": { + "transaction_id": 0, + "backend_type": "client backend", + "sql_state_code": "00000", + "session_start_time": "2021-01-04T01:06:20.000Z", + "session_id": "5ff26a0c.56", + "command_tag": "CHECKPOINT", + "client_port": 45126, + "database": "postgres", + "virtual_transaction_id": "3/0", + "application_name": "psql", + "client_addr": "172.24.0.1", + "query_step": "statement", + "timestamp": "2021-01-04 01:07:24.374 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "checkpoint;", + "event": { + "duration": 15136000, + "ingested": "2021-03-03T14:56:51.655805483Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 86 + }, + "@timestamp": "2021-01-04T01:07:25.950Z", + "postgresql": { + "log": { + "transaction_id": 0, + "backend_type": "client backend", + "client_port": 45126, + "database": "postgres", + "sql_state_code": "00000", + "application_name": "psql", + "client_addr": "172.24.0.1", + "session_start_time": "2021-01-04T01:06:20.000Z", + "session_id": "5ff26a0c.56", + "command_tag": "idle", + "timestamp": "2021-01-04 01:07:25.950 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "disconnection: session time: 0:01:04.968 user=postgres database=postgres host=172.24.0.1 port=45126", + "event": { + "ingested": "2021-03-03T14:56:51.655806891Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + } + ] +} \ No newline at end of file diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-13-error-code-csv.log b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-13-error-code-csv.log new file mode 100644 index 00000000000..52289a29f82 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-13-error-code-csv.log @@ -0,0 +1,4 @@ +2021-02-14 10:45:33.257 UTC,"postgres","postgres",86,"172.24.0.1:48978",6028ff3a.56,4,"idle",2021-02-14 10:45:14 UTC,3/3,0,LOG,00000,"statement: SET idle_in_transaction_session_timeout = 50;",,,,,,,,,"psql","client backend" +2021-02-14 10:45:48.113 UTC,"postgres","postgres",86,"172.24.0.1:48978",6028ff3a.56,5,"idle",2021-02-14 10:45:14 UTC,3/4,0,LOG,00000,"statement: BEGIN;",,,,,,,,,"psql","client backend" +2021-02-14 10:45:48.164 UTC,"postgres","postgres",86,"172.24.0.1:48978",6028ff3a.56,6,"idle in transaction",2021-02-14 10:45:14 UTC,3/4,0,FATAL,25P03,"terminating connection due to idle-in-transaction timeout",,,,,,,,,"psql","client backend" +2021-02-14 10:45:48.164 UTC,"postgres","postgres",86,"172.24.0.1:48978",6028ff3a.56,7,"idle in transaction",2021-02-14 10:45:14 UTC,,0,LOG,00000,"disconnection: session time: 0:00:33.289 user=postgres database=postgres host=172.24.0.1 port=48978",,,,,,,,,"psql","client backend" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-13-error-code-csv.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-13-error-code-csv.log-config.yml new file mode 100644 index 00000000000..860a8bfb242 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-13-error-code-csv.log-config.yml @@ -0,0 +1,4 @@ +dynamic_fields: + event.ingested: ".*" +multiline: + first_line_pattern: '^\d{4}-\d{2}-\d{2} ' diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-13-error-code-csv.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-13-error-code-csv.log-expected.json new file mode 100644 index 00000000000..8cf17321632 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-13-error-code-csv.log-expected.json @@ -0,0 +1,185 @@ +{ + "expected": [ + { + "process": { + "pid": 86 + }, + "@timestamp": "2021-02-14T10:45:33.257Z", + "postgresql": { + "log": { + "transaction_id": 0, + "backend_type": "client backend", + "sql_state_code": "00000", + "session_start_time": "2021-02-14T10:45:14.000Z", + "session_id": "6028ff3a.56", + "command_tag": "idle", + "client_port": 48978, + "database": "postgres", + "virtual_transaction_id": "3/3", + "application_name": "psql", + "client_addr": "172.24.0.1", + "query_step": "statement", + "timestamp": "2021-02-14 10:45:33.257 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "SET idle_in_transaction_session_timeout = 50;", + "event": { + "ingested": "2021-03-03T14:56:51.598546149Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 86 + }, + "@timestamp": "2021-02-14T10:45:48.113Z", + "postgresql": { + "log": { + "transaction_id": 0, + "backend_type": "client backend", + "sql_state_code": "00000", + "session_start_time": "2021-02-14T10:45:14.000Z", + "session_id": "6028ff3a.56", + "command_tag": "idle", + "client_port": 48978, + "database": "postgres", + "virtual_transaction_id": "3/4", + "application_name": "psql", + "client_addr": "172.24.0.1", + "query_step": "statement", + "timestamp": "2021-02-14 10:45:48.113 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "BEGIN;", + "event": { + "ingested": "2021-03-03T14:56:51.598549798Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 86 + }, + "@timestamp": "2021-02-14T10:45:48.164Z", + "postgresql": { + "log": { + "transaction_id": 0, + "backend_type": "client backend", + "client_port": 48978, + "database": "postgres", + "virtual_transaction_id": "3/4", + "sql_state_code": "25P03", + "application_name": "psql", + "client_addr": "172.24.0.1", + "session_start_time": "2021-02-14T10:45:14.000Z", + "session_id": "6028ff3a.56", + "command_tag": "idle in transaction", + "timestamp": "2021-02-14 10:45:48.164 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "FATAL" + }, + "message": "terminating connection due to idle-in-transaction timeout", + "event": { + "ingested": "2021-03-03T14:56:51.598551359Z", + "category": [ + "database" + ], + "type": [ + "error" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 86 + }, + "@timestamp": "2021-02-14T10:45:48.164Z", + "postgresql": { + "log": { + "transaction_id": 0, + "backend_type": "client backend", + "client_port": 48978, + "database": "postgres", + "sql_state_code": "00000", + "application_name": "psql", + "client_addr": "172.24.0.1", + "session_start_time": "2021-02-14T10:45:14.000Z", + "session_id": "6028ff3a.56", + "command_tag": "idle in transaction", + "timestamp": "2021-02-14 10:45:48.164 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "disconnection: session time: 0:00:33.289 user=postgres database=postgres host=172.24.0.1 port=48978", + "event": { + "ingested": "2021-03-03T14:56:51.598552833Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + } + ] +} \ No newline at end of file diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-debian-with-slowlog.log b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-debian-with-slowlog.log new file mode 100644 index 00000000000..59416a1898c --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-debian-with-slowlog.log @@ -0,0 +1,64 @@ +2017-07-31 13:36:42.585 CEST [4974] LOG: database system was shut down at 2017-06-17 16:58:04 CEST +2017-07-31 13:36:42.605 CEST [4974] LOG: MultiXact member wraparound protections are now enabled +2017-07-31 13:36:42.615 CEST [4978] LOG: autovacuum launcher started +2017-07-31 13:36:42.616 CEST [4973] LOG: database system is ready to accept connections +2017-07-31 13:36:42.956 CEST [4980] [unknown]@[unknown] LOG: incomplete startup packet +2017-07-31 13:36:43.557 CEST [4983] postgres@postgres LOG: duration: 37.118 ms statement: SELECT d.datname as "Name", + pg_catalog.pg_get_userbyid(d.datdba) as "Owner", + pg_catalog.pg_encoding_to_char(d.encoding) as "Encoding", + d.datcollate as "Collate", + d.datctype as "Ctype", + pg_catalog.array_to_string(d.datacl, E'\n') AS "Access privileges" + FROM pg_catalog.pg_database d + ORDER BY 1; +2017-07-31 13:36:44.104 CEST [4986] postgres@postgres LOG: duration: 2.895 ms statement: SELECT d.datname as "Name", + pg_catalog.pg_get_userbyid(d.datdba) as "Owner", + pg_catalog.pg_encoding_to_char(d.encoding) as "Encoding", + d.datcollate as "Collate", + d.datctype as "Ctype", + pg_catalog.array_to_string(d.datacl, E'\n') AS "Access privileges" + FROM pg_catalog.pg_database d + ORDER BY 1; +2017-07-31 13:36:44.642 CEST [4989] postgres@postgres LOG: duration: 2.809 ms statement: SELECT d.datname as "Name", + pg_catalog.pg_get_userbyid(d.datdba) as "Owner", + pg_catalog.pg_encoding_to_char(d.encoding) as "Encoding", + d.datcollate as "Collate", + d.datctype as "Ctype", + pg_catalog.array_to_string(d.datacl, E'\n') AS "Access privileges" + FROM pg_catalog.pg_database d + ORDER BY 1; +2017-07-31 13:39:16.249 CEST [5407] postgres@users FATAL: database "users" does not exist +2017-07-31 13:39:17.945 CEST [5500] postgres@user FATAL: database "user" does not exist +2017-07-31 13:39:21.025 CEST [5404] postgres@postgres LOG: duration: 37.598 ms statement: SELECT n.nspname as "Schema", + c.relname as "Name", + CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as "Type", + pg_catalog.pg_get_userbyid(c.relowner) as "Owner" + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r','') + AND n.nspname <> 'pg_catalog' + AND n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_toast' + AND pg_catalog.pg_table_is_visible(c.oid) + ORDER BY 1,2; +2017-07-31 13:39:31.619 CEST [5502] postgres@clients LOG: duration: 9.482 ms statement: select * from clients; +2017-07-31 13:39:40.147 CEST [5502] postgres@clients LOG: duration: 0.765 ms statement: select id from clients; +2017-07-31 13:40:54.310 CEST [5502] postgres@clients LOG: duration: 26.082 ms statement: SELECT n.nspname as "Schema", + c.relname as "Name", + CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as "Type", + pg_catalog.pg_get_userbyid(c.relowner) as "Owner" + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r','') + AND n.nspname <> 'pg_catalog' + AND n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_toast' + AND pg_catalog.pg_table_is_visible(c.oid) + ORDER BY 1,2; +2017-07-31 13:43:22.645 CEST [5502] postgres@clients LOG: duration: 36.162 ms statement: create table cats(name varchar(50) primary key, toy varchar (50) not null, born timestamp not null); +2017-07-31 13:46:02.670 CEST [5502] postgres@c$lients LOG: duration: 10.540 ms statement: insert into cats(name, toy, born) values('kate', 'ball', now()); +2017-07-31 13:46:23.016 CEST [5502] postgres@_clients$db LOG: duration: 5.156 ms statement: insert into cats(name, toy, born) values('frida', 'horse', now()); +2017-07-31 13:46:55.637 CEST [5502] postgres@clients_db LOG: duration: 25.871 ms statement: create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null); +2019-05-06 19:00:04.511 UTC [913763] elastic@opbeans LOG: duration: 0.753 ms statement: SELECT p.id, p.sku, p.name, p.stock, t.name AS type_name FROM products p LEFT JOIN product_types t ON p.type_id=t.id + FROM orders JOIN customers ON orders.customer_id=customers.id + FROM products JOIN product_types ON type_id=product_types.id diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-debian-with-slowlog.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-debian-with-slowlog.log-config.yml new file mode 100644 index 00000000000..860a8bfb242 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-debian-with-slowlog.log-config.yml @@ -0,0 +1,4 @@ +dynamic_fields: + event.ingested: ".*" +multiline: + first_line_pattern: '^\d{4}-\d{2}-\d{2} ' diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-debian-with-slowlog.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-debian-with-slowlog.log-expected.json new file mode 100644 index 00000000000..a4718dbf78f --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-debian-with-slowlog.log-expected.json @@ -0,0 +1,669 @@ +{ + "expected": [ + { + "process": { + "pid": 4974 + }, + "@timestamp": "2017-07-31T11:36:42.585Z", + "postgresql": { + "log": { + "timestamp": "2017-07-31 13:36:42.585 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system was shut down at 2017-06-17 16:58:04 CEST", + "event": { + "ingested": "2021-03-03T14:56:51.934300905Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 4974 + }, + "@timestamp": "2017-07-31T11:36:42.605Z", + "postgresql": { + "log": { + "timestamp": "2017-07-31 13:36:42.605 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "MultiXact member wraparound protections are now enabled", + "event": { + "ingested": "2021-03-03T14:56:51.934304348Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 4978 + }, + "@timestamp": "2017-07-31T11:36:42.615Z", + "postgresql": { + "log": { + "timestamp": "2017-07-31 13:36:42.615 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "autovacuum launcher started", + "event": { + "ingested": "2021-03-03T14:56:51.934305855Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 4973 + }, + "@timestamp": "2017-07-31T11:36:42.616Z", + "postgresql": { + "log": { + "timestamp": "2017-07-31 13:36:42.616 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system is ready to accept connections", + "event": { + "ingested": "2021-03-03T14:56:51.934307290Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 4980 + }, + "@timestamp": "2017-07-31T11:36:42.956Z", + "postgresql": { + "log": { + "database": "unknown", + "timestamp": "2017-07-31 13:36:42.956 CEST" + } + }, + "related": { + "user": [ + "unknown" + ] + }, + "log": { + "level": "LOG" + }, + "message": "incomplete startup packet", + "event": { + "ingested": "2021-03-03T14:56:51.934308739Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "unknown" + } + }, + { + "process": { + "pid": 4983 + }, + "@timestamp": "2017-07-31T11:36:43.557Z", + "postgresql": { + "log": { + "database": "postgres", + "query_step": "statement", + "query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", + "timestamp": "2017-07-31 13:36:43.557 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "2017-07-31 13:36:43.557 CEST [4983] postgres@postgres LOG: duration: 37.118 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", + "event": { + "duration": 37118000, + "ingested": "2021-03-03T14:56:51.934310185Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 4986 + }, + "@timestamp": "2017-07-31T11:36:44.104Z", + "postgresql": { + "log": { + "database": "postgres", + "query_step": "statement", + "query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", + "timestamp": "2017-07-31 13:36:44.104 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "2017-07-31 13:36:44.104 CEST [4986] postgres@postgres LOG: duration: 2.895 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", + "event": { + "duration": 2895000, + "ingested": "2021-03-03T14:56:51.934311565Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 4989 + }, + "@timestamp": "2017-07-31T11:36:44.642Z", + "postgresql": { + "log": { + "database": "postgres", + "query_step": "statement", + "query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", + "timestamp": "2017-07-31 13:36:44.642 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "2017-07-31 13:36:44.642 CEST [4989] postgres@postgres LOG: duration: 2.809 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", + "event": { + "duration": 2809000, + "ingested": "2021-03-03T14:56:51.934312969Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 5407 + }, + "@timestamp": "2017-07-31T11:39:16.249Z", + "postgresql": { + "log": { + "database": "users", + "timestamp": "2017-07-31 13:39:16.249 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "FATAL" + }, + "message": "database \"users\" does not exist", + "event": { + "ingested": "2021-03-03T14:56:51.934314375Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 5500 + }, + "@timestamp": "2017-07-31T11:39:17.945Z", + "postgresql": { + "log": { + "database": "user", + "timestamp": "2017-07-31 13:39:17.945 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "FATAL" + }, + "message": "database \"user\" does not exist", + "event": { + "ingested": "2021-03-03T14:56:51.934315865Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 5404 + }, + "@timestamp": "2017-07-31T11:39:21.025Z", + "postgresql": { + "log": { + "database": "postgres", + "query_step": "statement", + "query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname \u003c\u003e 'pg_catalog'\n\t AND n.nspname \u003c\u003e 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", + "timestamp": "2017-07-31 13:39:21.025 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "2017-07-31 13:39:21.025 CEST [5404] postgres@postgres LOG: duration: 37.598 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname \u003c\u003e 'pg_catalog'\n\t AND n.nspname \u003c\u003e 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", + "event": { + "duration": 37598000, + "ingested": "2021-03-03T14:56:51.934317319Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 5502 + }, + "@timestamp": "2017-07-31T11:39:31.619Z", + "postgresql": { + "log": { + "database": "clients", + "query_step": "statement", + "query": "select * from clients;", + "timestamp": "2017-07-31 13:39:31.619 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "2017-07-31 13:39:31.619 CEST [5502] postgres@clients LOG: duration: 9.482 ms statement: select * from clients;", + "event": { + "duration": 9482000, + "ingested": "2021-03-03T14:56:51.934318881Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 5502 + }, + "@timestamp": "2017-07-31T11:39:40.147Z", + "postgresql": { + "log": { + "database": "clients", + "query_step": "statement", + "query": "select id from clients;", + "timestamp": "2017-07-31 13:39:40.147 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "2017-07-31 13:39:40.147 CEST [5502] postgres@clients LOG: duration: 0.765 ms statement: select id from clients;", + "event": { + "duration": 765000, + "ingested": "2021-03-03T14:56:51.934320290Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 5502 + }, + "@timestamp": "2017-07-31T11:40:54.310Z", + "postgresql": { + "log": { + "database": "clients", + "query_step": "statement", + "query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname \u003c\u003e 'pg_catalog'\n\t AND n.nspname \u003c\u003e 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", + "timestamp": "2017-07-31 13:40:54.310 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "2017-07-31 13:40:54.310 CEST [5502] postgres@clients LOG: duration: 26.082 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname \u003c\u003e 'pg_catalog'\n\t AND n.nspname \u003c\u003e 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", + "event": { + "duration": 26082000, + "ingested": "2021-03-03T14:56:51.934321719Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 5502 + }, + "@timestamp": "2017-07-31T11:43:22.645Z", + "postgresql": { + "log": { + "database": "clients", + "query_step": "statement", + "query": "create table cats(name varchar(50) primary key, toy varchar (50) not null, born timestamp not null);", + "timestamp": "2017-07-31 13:43:22.645 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "2017-07-31 13:43:22.645 CEST [5502] postgres@clients LOG: duration: 36.162 ms statement: create table cats(name varchar(50) primary key, toy varchar (50) not null, born timestamp not null);", + "event": { + "duration": 36162000, + "ingested": "2021-03-03T14:56:51.934323088Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 5502 + }, + "@timestamp": "2017-07-31T11:46:02.670Z", + "postgresql": { + "log": { + "database": "c$lients", + "query_step": "statement", + "query": "insert into cats(name, toy, born) values('kate', 'ball', now());", + "timestamp": "2017-07-31 13:46:02.670 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "2017-07-31 13:46:02.670 CEST [5502] postgres@c$lients LOG: duration: 10.540 ms statement: insert into cats(name, toy, born) values('kate', 'ball', now());", + "event": { + "duration": 10540000, + "ingested": "2021-03-03T14:56:51.934324471Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 5502 + }, + "@timestamp": "2017-07-31T11:46:23.016Z", + "postgresql": { + "log": { + "database": "_clients$db", + "query_step": "statement", + "query": "insert into cats(name, toy, born) values('frida', 'horse', now());", + "timestamp": "2017-07-31 13:46:23.016 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "2017-07-31 13:46:23.016 CEST [5502] postgres@_clients$db LOG: duration: 5.156 ms statement: insert into cats(name, toy, born) values('frida', 'horse', now());", + "event": { + "duration": 5156000, + "ingested": "2021-03-03T14:56:51.934325971Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 5502 + }, + "@timestamp": "2017-07-31T11:46:55.637Z", + "postgresql": { + "log": { + "database": "clients_db", + "query_step": "statement", + "query": "create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null);", + "timestamp": "2017-07-31 13:46:55.637 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "2017-07-31 13:46:55.637 CEST [5502] postgres@clients_db LOG: duration: 25.871 ms statement: create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null);", + "event": { + "duration": 25871000, + "ingested": "2021-03-03T14:56:51.934335911Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 913763 + }, + "@timestamp": "2019-05-06T19:00:04.511Z", + "postgresql": { + "log": { + "database": "opbeans", + "query_step": "statement", + "query": "SELECT p.id, p.sku, p.name, p.stock, t.name AS type_name FROM products p LEFT JOIN product_types t ON p.type_id=t.id\n FROM orders JOIN customers ON orders.customer_id=customers.id\n FROM products JOIN product_types ON type_id=product_types.id", + "timestamp": "2019-05-06 19:00:04.511 UTC" + } + }, + "related": { + "user": [ + "elastic" + ] + }, + "log": { + "level": "LOG" + }, + "message": "2019-05-06 19:00:04.511 UTC [913763] elastic@opbeans LOG: duration: 0.753 ms statement: SELECT p.id, p.sku, p.name, p.stock, t.name AS type_name FROM products p LEFT JOIN product_types t ON p.type_id=t.id\n FROM orders JOIN customers ON orders.customer_id=customers.id\n FROM products JOIN product_types ON type_id=product_types.id", + "event": { + "duration": 753000, + "ingested": "2021-03-03T14:56:51.934337887Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "elastic" + } + } + ] +} \ No newline at end of file diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-multi-core.log b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-multi-core.log new file mode 100644 index 00000000000..75b8188ee55 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-multi-core.log @@ -0,0 +1,19 @@ +2017-04-03 22:32:14.322 CEST [12975-1] [unknown]@[unknown] LOG: incomplete startup packet +2017-04-03 22:32:14.322 CEST [5404-1] postgres@user FATAL: database "user" does not exist +2017-04-03 22:35:22.389 CEST [5404-2] postgres@postgres LOG: duration: 37.598 ms statement: SELECT n.nspname as "Schema", + c.relname as "Name", + CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as "Type", + pg_catalog.pg_get_userbyid(c.relowner) as "Owner" + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r','') + AND n.nspname <> 'pg_catalog' + AND n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_toast' + AND pg_catalog.pg_table_is_visible(c.oid) + ORDER BY 1,2; +2017-07-31 13:36:43.557 EST [835-1] LOG: autovacuum launcher started +2017-07-31 13:36:44.227 EST [832-1] LOG: checkpoints are occurring too frequently (25 seconds apart) +2017-07-31 13:46:02.670 EST [832-2] HINT: Consider increasing the configuration parameter "max_wal_size". +2017-07-31 13:46:23.016 EST [768-1] postgres@postgres FATAL: the database system is starting up +2017-07-31 13:46:55.637 EST [771-1] postgres@postgres FATAL: the database system is starting up diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-multi-core.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-multi-core.log-config.yml new file mode 100644 index 00000000000..860a8bfb242 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-multi-core.log-config.yml @@ -0,0 +1,4 @@ +dynamic_fields: + event.ingested: ".*" +multiline: + first_line_pattern: '^\d{4}-\d{2}-\d{2} ' diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-multi-core.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-multi-core.log-expected.json new file mode 100644 index 00000000000..832781082f9 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-multi-core.log-expected.json @@ -0,0 +1,268 @@ +{ + "expected": [ + { + "process": { + "pid": 12975 + }, + "@timestamp": "2017-04-03T20:32:14.322Z", + "postgresql": { + "log": { + "database": "unknown", + "session_line_number": 1, + "timestamp": "2017-04-03 22:32:14.322 CEST" + } + }, + "related": { + "user": [ + "unknown" + ] + }, + "log": { + "level": "LOG" + }, + "message": "incomplete startup packet", + "event": { + "ingested": "2021-03-03T14:56:52.103687679Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "unknown" + } + }, + { + "process": { + "pid": 5404 + }, + "@timestamp": "2017-04-03T20:32:14.322Z", + "postgresql": { + "log": { + "database": "user", + "session_line_number": 1, + "timestamp": "2017-04-03 22:32:14.322 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "FATAL" + }, + "message": "database \"user\" does not exist", + "event": { + "ingested": "2021-03-03T14:56:52.103692080Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 5404 + }, + "@timestamp": "2017-04-03T20:35:22.389Z", + "postgresql": { + "log": { + "database": "postgres", + "session_line_number": 2, + "query_step": "statement", + "query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname \u003c\u003e 'pg_catalog'\n\t AND n.nspname \u003c\u003e 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", + "timestamp": "2017-04-03 22:35:22.389 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "2017-04-03 22:35:22.389 CEST [5404-2] postgres@postgres LOG: duration: 37.598 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname \u003c\u003e 'pg_catalog'\n\t AND n.nspname \u003c\u003e 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", + "event": { + "duration": 37598000, + "ingested": "2021-03-03T14:56:52.103693612Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 835 + }, + "@timestamp": "2017-07-31T17:36:43.557Z", + "postgresql": { + "log": { + "session_line_number": 1, + "timestamp": "2017-07-31 13:36:43.557 EST" + } + }, + "log": { + "level": "LOG" + }, + "message": "autovacuum launcher started", + "event": { + "ingested": "2021-03-03T14:56:52.103695023Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "EST", + "kind": "event" + } + }, + { + "process": { + "pid": 832 + }, + "@timestamp": "2017-07-31T17:36:44.227Z", + "postgresql": { + "log": { + "session_line_number": 1, + "timestamp": "2017-07-31 13:36:44.227 EST" + } + }, + "log": { + "level": "LOG" + }, + "message": "checkpoints are occurring too frequently (25 seconds apart)", + "event": { + "ingested": "2021-03-03T14:56:52.103696412Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "EST", + "kind": "event" + } + }, + { + "process": { + "pid": 832 + }, + "@timestamp": "2017-07-31T17:46:02.670Z", + "postgresql": { + "log": { + "session_line_number": 2, + "timestamp": "2017-07-31 13:46:02.670 EST" + } + }, + "log": { + "level": "HINT" + }, + "message": "Consider increasing the configuration parameter \"max_wal_size\".", + "event": { + "ingested": "2021-03-03T14:56:52.103697810Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "EST", + "kind": "event" + } + }, + { + "process": { + "pid": 768 + }, + "@timestamp": "2017-07-31T17:46:23.016Z", + "postgresql": { + "log": { + "database": "postgres", + "session_line_number": 1, + "timestamp": "2017-07-31 13:46:23.016 EST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "FATAL" + }, + "message": "the database system is starting up", + "event": { + "ingested": "2021-03-03T14:56:52.103699294Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "EST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 771 + }, + "@timestamp": "2017-07-31T17:46:55.637Z", + "postgresql": { + "log": { + "database": "postgres", + "session_line_number": 1, + "timestamp": "2017-07-31 13:46:55.637 EST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "FATAL" + }, + "message": "the database system is starting up", + "event": { + "ingested": "2021-03-03T14:56:52.103700696Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "EST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + } + ] +} \ No newline at end of file diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-new-timestamp.log b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-new-timestamp.log new file mode 100644 index 00000000000..2042fb9f0a6 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-new-timestamp.log @@ -0,0 +1,5 @@ +2017-07-31 13:36:43 EST [835-1] LOG: autovacuum launcher started +2017-07-31 13:36:44 EST [832-1] LOG: checkpoints are occurring too frequently (25 seconds apart) +2017-07-31 13:46:02 EST [832-2] HINT: Consider increasing the configuration parameter "max_wal_size". +2017-07-31 13:46:23 EST [768-1] postgres@postgres FATAL: the database system is starting up +2017-07-31 13:46:55 EST [771-1] postgres@postgres FATAL: the database system is starting up diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-new-timestamp.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-new-timestamp.log-config.yml new file mode 100644 index 00000000000..860a8bfb242 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-new-timestamp.log-config.yml @@ -0,0 +1,4 @@ +dynamic_fields: + event.ingested: ".*" +multiline: + first_line_pattern: '^\d{4}-\d{2}-\d{2} ' diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-new-timestamp.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-new-timestamp.log-expected.json new file mode 100644 index 00000000000..5461ec8d030 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-9-6-new-timestamp.log-expected.json @@ -0,0 +1,157 @@ +{ + "expected": [ + { + "process": { + "pid": 835 + }, + "@timestamp": "2017-07-31T17:36:43.000Z", + "postgresql": { + "log": { + "session_line_number": 1, + "timestamp": "2017-07-31 13:36:43 EST" + } + }, + "log": { + "level": "LOG" + }, + "message": "autovacuum launcher started", + "event": { + "ingested": "2021-03-03T14:56:52.220813250Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "EST", + "kind": "event" + } + }, + { + "process": { + "pid": 832 + }, + "@timestamp": "2017-07-31T17:36:44.000Z", + "postgresql": { + "log": { + "session_line_number": 1, + "timestamp": "2017-07-31 13:36:44 EST" + } + }, + "log": { + "level": "LOG" + }, + "message": "checkpoints are occurring too frequently (25 seconds apart)", + "event": { + "ingested": "2021-03-03T14:56:52.220817413Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "EST", + "kind": "event" + } + }, + { + "process": { + "pid": 832 + }, + "@timestamp": "2017-07-31T17:46:02.000Z", + "postgresql": { + "log": { + "session_line_number": 2, + "timestamp": "2017-07-31 13:46:02 EST" + } + }, + "log": { + "level": "HINT" + }, + "message": "Consider increasing the configuration parameter \"max_wal_size\".", + "event": { + "ingested": "2021-03-03T14:56:52.220818986Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "EST", + "kind": "event" + } + }, + { + "process": { + "pid": 768 + }, + "@timestamp": "2017-07-31T17:46:23.000Z", + "postgresql": { + "log": { + "database": "postgres", + "session_line_number": 1, + "timestamp": "2017-07-31 13:46:23 EST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "FATAL" + }, + "message": "the database system is starting up", + "event": { + "ingested": "2021-03-03T14:56:52.220820420Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "EST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 771 + }, + "@timestamp": "2017-07-31T17:46:55.000Z", + "postgresql": { + "log": { + "database": "postgres", + "session_line_number": 1, + "timestamp": "2017-07-31 13:46:55 EST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "FATAL" + }, + "message": "the database system is starting up", + "event": { + "ingested": "2021-03-03T14:56:52.220821859Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "EST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + } + ] +} \ No newline at end of file diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-query-steps-slowlog.log b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-query-steps-slowlog.log new file mode 100644 index 00000000000..faba3c3ba62 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-query-steps-slowlog.log @@ -0,0 +1,3 @@ +2019-09-04 15:52:38.004 CEST [31136] user@host LOG: duration: 12.437 ms parse : select * from table +2019-09-04 15:52:38.004 CEST [31136] user@host LOG: duration: 12.437 ms execute pdo_stmt_00000002: select * from table + diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-query-steps-slowlog.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-query-steps-slowlog.log-config.yml new file mode 100644 index 00000000000..860a8bfb242 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-query-steps-slowlog.log-config.yml @@ -0,0 +1,4 @@ +dynamic_fields: + event.ingested: ".*" +multiline: + first_line_pattern: '^\d{4}-\d{2}-\d{2} ' diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-query-steps-slowlog.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-query-steps-slowlog.log-expected.json new file mode 100644 index 00000000000..a050e095479 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-query-steps-slowlog.log-expected.json @@ -0,0 +1,81 @@ +{ + "expected": [ + { + "process": { + "pid": 31136 + }, + "@timestamp": "2019-09-04T13:52:38.004Z", + "postgresql": { + "log": { + "database": "host", + "query_step": "parse", + "query": "select * from table", + "timestamp": "2019-09-04 15:52:38.004 CEST" + } + }, + "related": { + "user": [ + "user" + ] + }, + "log": { + "level": "LOG" + }, + "message": "2019-09-04 15:52:38.004 CEST [31136] user@host LOG: duration: 12.437 ms parse \u003cunnamed\u003e: select * from table", + "event": { + "duration": 12437000, + "ingested": "2021-03-03T14:56:52.260498355Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "user" + } + }, + { + "process": { + "pid": 31136 + }, + "@timestamp": "2019-09-04T13:52:38.004Z", + "postgresql": { + "log": { + "database": "host", + "query_step": "execute", + "query_name": "pdo_stmt_00000002", + "query": "select * from table\n", + "timestamp": "2019-09-04 15:52:38.004 CEST" + } + }, + "related": { + "user": [ + "user" + ] + }, + "log": { + "level": "LOG" + }, + "message": "2019-09-04 15:52:38.004 CEST [31136] user@host LOG: duration: 12.437 ms execute pdo_stmt_00000002: select * from table\n", + "event": { + "duration": 12437000, + "ingested": "2021-03-03T14:56:52.260504601Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "user" + } + } + ] +} \ No newline at end of file diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-ubuntu-9-5.log b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-ubuntu-9-5.log new file mode 100644 index 00000000000..5a3df37b16a --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-ubuntu-9-5.log @@ -0,0 +1,78 @@ +2017-04-03 22:32:14.322 CEST [31225] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-04-03 22:32:14.322 CEST [31225] postgres@mydb LOG: unexpected EOF on client connection with an open transaction +2017-04-03 22:35:22.389 CEST [3474] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-04-03 22:36:56.464 CEST [3525] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-04-03 22:37:12.961 CEST [3570] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-04-07 21:05:28.549 CEST [21483] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-04-07 21:09:41.345 CEST [21597] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-04-07 22:45:30.218 CEST [22603] postgres@mydb ERROR: operator does not exist: jsonb @> at character 49 +2017-04-07 22:45:30.218 CEST [22603] postgres@mydb HINT: No operator matches the given name and argument type(s). You might need to add explicit type casts. +2017-04-07 22:45:30.218 CEST [22603] postgres@mydb STATEMENT: SELECT id, user FROM users WHERE NOT user @> %s +2017-04-07 22:46:09.751 CEST [22608] postgres@mydb ERROR: column "%s" does not exist at character 52 +2017-04-07 22:46:09.751 CEST [22608] postgres@mydb STATEMENT: SELECT id, user FROM users WHERE NOT user @> "%s" +2017-04-07 23:02:51.199 CEST [24341] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-04-07 23:02:51.199 CEST [24341] postgres@mydb LOG: unexpected EOF on client connection with an open transaction +2017-04-07 23:04:36.087 CEST [20730] postgres@mydb ERROR: syntax error at or near "{" at character 49 +2017-04-07 23:04:36.087 CEST [20730] postgres@mydb STATEMENT: INSERT INTO users (id, user) VALUES (1, {"attr": "yes"}); +2017-04-07 23:04:51.462 CEST [20730] postgres@mydb ERROR: syntax error at or near "{" at character 49 +2017-04-07 23:04:51.462 CEST [20730] postgres@mydb STATEMENT: INSERT INTO users (id, user) VALUES (1, {attr: "yes"}); +2017-04-07 23:05:06.217 CEST [20730] postgres@mydb ERROR: column "a" does not exist at character 42 +2017-04-07 23:05:06.217 CEST [20730] postgres@mydb STATEMENT: INSERT INTO users (id, user) VALUES (1, '{"attr": "yes"}'); +2017-04-07 23:05:18.295 CEST [20730] postgres@mydb ERROR: column "attr" does not exist at character 42 +2017-04-07 23:05:18.295 CEST [20730] postgres@mydb STATEMENT: INSERT INTO users (id, user) VALUES ("1", '{"attr": "no"}'); +2017-04-07 23:13:47.505 CEST [24489] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-04-07 23:13:47.505 CEST [24489] postgres@mydb LOG: unexpected EOF on client connection with an open transaction +2017-04-08 12:32:51.056 CEST [20730] postgres@mydb ERROR: duplicate key value violates unique constraint "users_pkey" +2017-04-08 12:32:51.056 CEST [20730] postgres@mydb DETAIL: Key (id)=(1) already exists. +2017-04-08 12:32:51.056 CEST [20730] postgres@mydb STATEMENT: INSERT INTO users (id, user) VALUES ('1', '{"attr": "yes"}'); +2017-04-08 21:54:37.443 CEST [30630] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-04-08 21:54:37.468 CEST [30502] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-04-08 21:54:37.618 CEST [20769] LOG: received fast shutdown request +2017-04-08 21:54:37.618 CEST [20769] LOG: aborting any active transactions +2017-04-08 21:54:37.618 CEST [20774] LOG: autovacuum launcher shutting down +2017-04-08 21:54:37.622 CEST [20771] LOG: shutting down +2017-04-08 21:54:37.644 CEST [20769] LOG: database system is shut down +2017-04-08 21:56:02.932 CEST [797] LOG: database system was shut down at 2017-04-08 21:54:37 CEST +2017-04-08 21:56:02.944 CEST [797] LOG: MultiXact member wraparound protections are now enabled +2017-04-08 21:56:02.946 CEST [780] LOG: database system is ready to accept connections +2017-04-08 21:56:02.947 CEST [802] LOG: autovacuum launcher started +2017-04-08 21:56:03.362 CEST [891] [unknown]@[unknown] LOG: incomplete startup packet +2017-05-27 14:07:53.007 UTC [32567] LOG: database system was shut down at 2017-05-27 14:07:52 UTC +2017-05-27 14:07:53.010 UTC [32567] LOG: MultiXact member wraparound protections are now enabled +2017-05-27 14:07:53.015 UTC [32566] LOG: database system is ready to accept connections +2017-05-27 14:07:53.016 UTC [32571] LOG: autovacuum launcher started +2017-05-27 14:07:53.463 UTC [32573] [unknown]@[unknown] LOG: incomplete startup packet +2017-05-27 14:08:13.661 UTC [1308] postgres@mydb FATAL: database "mydb" does not exist +2017-05-27 14:59:26.553 UTC [1994] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-05-27 14:59:26.555 UTC [1989] postgres@mydb LOG: could not receive data from client: Connection reset by peer +2017-06-06 07:54:13.753 CEST [9110] LOG: received fast shutdown request +2017-06-06 07:54:13.753 CEST [9110] LOG: aborting any active transactions +2017-06-06 07:54:13.753 CEST [9115] LOG: autovacuum launcher shutting down +2017-06-06 07:54:13.755 CEST [9112] LOG: shutting down +2017-06-06 07:54:13.816 CEST [9110] LOG: database system is shut down +2017-06-06 07:55:39.725 CEST [12969] LOG: database system was shut down at 2017-06-06 07:54:13 CEST +2017-06-06 07:55:39.736 CEST [12969] LOG: MultiXact member wraparound protections are now enabled +2017-06-06 07:55:39.739 CEST [12968] LOG: database system is ready to accept connections +2017-06-06 07:55:39.739 CEST [12973] LOG: autovacuum launcher started +2017-06-06 07:55:40.155 CEST [12975] [unknown]@[unknown] LOG: incomplete startup packet +2017-06-06 07:55:40.156 CEST [12975] [unknown]@[unknown] LOG: incomplete startup packet +2017-06-10 19:37:30.681 CEST [17398] LOG: database system was shut down at 2017-06-10 19:37:29 CEST +2017-06-10 19:37:30.695 CEST [17398] LOG: MultiXact member wraparound protections are now enabled +2017-06-10 19:37:30.702 CEST [17397] LOG: database system is ready to accept connections +2017-06-10 19:37:30.702 CEST [17402] LOG: autovacuum launcher started +2017-06-10 19:37:31.104 CEST [17404] [unknown]@[unknown] LOG: incomplete startup packet +2017-06-10 20:27:55.911 CEST [17397] LOG: received fast shutdown request +2017-06-10 20:27:55.911 CEST [17397] LOG: aborting any active transactions +2017-06-10 20:27:55.911 CEST [17402] LOG: autovacuum launcher shutting down +2017-06-10 20:27:55.914 CEST [17399] LOG: shutting down +2017-06-10 20:27:55.973 CEST [17397] LOG: database system is shut down +2017-06-10 20:27:57.022 CEST [24490] LOG: database system was shut down at 2017-06-10 20:27:55 CEST +2017-06-10 20:27:57.032 CEST [24490] LOG: MultiXact member wraparound protections are now enabled +2017-06-10 20:27:57.035 CEST [24494] LOG: autovacuum launcher started +2017-06-10 20:27:57.035 CEST [24489] LOG: database system is ready to accept connections +2017-06-10 20:27:57.475 CEST [24496] [unknown]@[unknown] LOG: incomplete startup packet +2017-06-17 16:58:03.937 CEST [24489] LOG: received fast shutdown request +2017-06-17 16:58:03.937 CEST [24489] LOG: aborting any active transactions +2017-06-17 16:58:03.938 CEST [24494] LOG: autovacuum launcher shutting down +2017-06-17 16:58:03.940 CEST [24491] LOG: shutting down +2017-06-17 16:58:04.040 CEST [24489] LOG: database system is shut down diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-ubuntu-9-5.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-ubuntu-9-5.log-config.yml new file mode 100644 index 00000000000..860a8bfb242 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-ubuntu-9-5.log-config.yml @@ -0,0 +1,4 @@ +dynamic_fields: + event.ingested: ".*" +multiline: + first_line_pattern: '^\d{4}-\d{2}-\d{2} ' diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-ubuntu-9-5.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-ubuntu-9-5.log-expected.json new file mode 100644 index 00000000000..fb340502261 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/test-postgresql-ubuntu-9-5.log-expected.json @@ -0,0 +1,2374 @@ +{ + "expected": [ + { + "process": { + "pid": 31225 + }, + "@timestamp": "2017-04-03T20:32:14.322Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-03 22:32:14.322 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "could not receive data from client: Connection reset by peer", + "event": { + "ingested": "2021-03-03T14:56:52.287698653Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 31225 + }, + "@timestamp": "2017-04-03T20:32:14.322Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-03 22:32:14.322 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "unexpected EOF on client connection with an open transaction", + "event": { + "ingested": "2021-03-03T14:56:52.287702611Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 3474 + }, + "@timestamp": "2017-04-03T20:35:22.389Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-03 22:35:22.389 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "could not receive data from client: Connection reset by peer", + "event": { + "ingested": "2021-03-03T14:56:52.287704192Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 3525 + }, + "@timestamp": "2017-04-03T20:36:56.464Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-03 22:36:56.464 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "could not receive data from client: Connection reset by peer", + "event": { + "ingested": "2021-03-03T14:56:52.287705643Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 3570 + }, + "@timestamp": "2017-04-03T20:37:12.961Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-03 22:37:12.961 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "could not receive data from client: Connection reset by peer", + "event": { + "ingested": "2021-03-03T14:56:52.287707063Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 21483 + }, + "@timestamp": "2017-04-07T19:05:28.549Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-07 21:05:28.549 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "could not receive data from client: Connection reset by peer", + "event": { + "ingested": "2021-03-03T14:56:52.287708465Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 21597 + }, + "@timestamp": "2017-04-07T19:09:41.345Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-07 21:09:41.345 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "could not receive data from client: Connection reset by peer", + "event": { + "ingested": "2021-03-03T14:56:52.287709943Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 22603 + }, + "@timestamp": "2017-04-07T20:45:30.218Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-07 22:45:30.218 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "ERROR" + }, + "message": "operator does not exist: jsonb @\u003e at character 49", + "event": { + "ingested": "2021-03-03T14:56:52.287711379Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 22603 + }, + "@timestamp": "2017-04-07T20:45:30.218Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-07 22:45:30.218 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "HINT" + }, + "message": "No operator matches the given name and argument type(s). You might need to add explicit type casts.", + "event": { + "ingested": "2021-03-03T14:56:52.287712808Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 22603 + }, + "@timestamp": "2017-04-07T20:45:30.218Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-07 22:45:30.218 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "STATEMENT" + }, + "message": "SELECT id, user FROM users WHERE NOT user @\u003e %s", + "event": { + "ingested": "2021-03-03T14:56:52.287714224Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 22608 + }, + "@timestamp": "2017-04-07T20:46:09.751Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-07 22:46:09.751 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "ERROR" + }, + "message": "column \"%s\" does not exist at character 52", + "event": { + "ingested": "2021-03-03T14:56:52.287715654Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 22608 + }, + "@timestamp": "2017-04-07T20:46:09.751Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-07 22:46:09.751 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "STATEMENT" + }, + "message": "SELECT id, user FROM users WHERE NOT user @\u003e \"%s\"", + "event": { + "ingested": "2021-03-03T14:56:52.287717260Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 24341 + }, + "@timestamp": "2017-04-07T21:02:51.199Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-07 23:02:51.199 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "could not receive data from client: Connection reset by peer", + "event": { + "ingested": "2021-03-03T14:56:52.287718740Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 24341 + }, + "@timestamp": "2017-04-07T21:02:51.199Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-07 23:02:51.199 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "unexpected EOF on client connection with an open transaction", + "event": { + "ingested": "2021-03-03T14:56:52.287720281Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 20730 + }, + "@timestamp": "2017-04-07T21:04:36.087Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-07 23:04:36.087 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "ERROR" + }, + "message": "syntax error at or near \"{\" at character 49", + "event": { + "ingested": "2021-03-03T14:56:52.287721771Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 20730 + }, + "@timestamp": "2017-04-07T21:04:36.087Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-07 23:04:36.087 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "STATEMENT" + }, + "message": "INSERT INTO users (id, user) VALUES (1, {\"attr\": \"yes\"});", + "event": { + "ingested": "2021-03-03T14:56:52.287723220Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 20730 + }, + "@timestamp": "2017-04-07T21:04:51.462Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-07 23:04:51.462 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "ERROR" + }, + "message": "syntax error at or near \"{\" at character 49", + "event": { + "ingested": "2021-03-03T14:56:52.287724769Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 20730 + }, + "@timestamp": "2017-04-07T21:04:51.462Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-07 23:04:51.462 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "STATEMENT" + }, + "message": "INSERT INTO users (id, user) VALUES (1, {attr: \"yes\"});", + "event": { + "ingested": "2021-03-03T14:56:52.287726186Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 20730 + }, + "@timestamp": "2017-04-07T21:05:06.217Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-07 23:05:06.217 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "ERROR" + }, + "message": "column \"a\" does not exist at character 42", + "event": { + "ingested": "2021-03-03T14:56:52.287727599Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 20730 + }, + "@timestamp": "2017-04-07T21:05:06.217Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-07 23:05:06.217 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "STATEMENT" + }, + "message": "INSERT INTO users (id, user) VALUES (1, '{\"attr\": \"yes\"}');", + "event": { + "ingested": "2021-03-03T14:56:52.287729018Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 20730 + }, + "@timestamp": "2017-04-07T21:05:18.295Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-07 23:05:18.295 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "ERROR" + }, + "message": "column \"attr\" does not exist at character 42", + "event": { + "ingested": "2021-03-03T14:56:52.287730436Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 20730 + }, + "@timestamp": "2017-04-07T21:05:18.295Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-07 23:05:18.295 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "STATEMENT" + }, + "message": "INSERT INTO users (id, user) VALUES (\"1\", '{\"attr\": \"no\"}');", + "event": { + "ingested": "2021-03-03T14:56:52.287731853Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 24489 + }, + "@timestamp": "2017-04-07T21:13:47.505Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-07 23:13:47.505 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "could not receive data from client: Connection reset by peer", + "event": { + "ingested": "2021-03-03T14:56:52.287733278Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 24489 + }, + "@timestamp": "2017-04-07T21:13:47.505Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-07 23:13:47.505 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "unexpected EOF on client connection with an open transaction", + "event": { + "ingested": "2021-03-03T14:56:52.287734878Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 20730 + }, + "@timestamp": "2017-04-08T10:32:51.056Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-08 12:32:51.056 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "ERROR" + }, + "message": "duplicate key value violates unique constraint \"users_pkey\"", + "event": { + "ingested": "2021-03-03T14:56:52.287736336Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 20730 + }, + "@timestamp": "2017-04-08T10:32:51.056Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-08 12:32:51.056 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "DETAIL" + }, + "message": "Key (id)=(1) already exists.", + "event": { + "ingested": "2021-03-03T14:56:52.287737742Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 20730 + }, + "@timestamp": "2017-04-08T10:32:51.056Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-08 12:32:51.056 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "STATEMENT" + }, + "message": "INSERT INTO users (id, user) VALUES ('1', '{\"attr\": \"yes\"}');", + "event": { + "ingested": "2021-03-03T14:56:52.287739136Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 30630 + }, + "@timestamp": "2017-04-08T19:54:37.443Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-08 21:54:37.443 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "could not receive data from client: Connection reset by peer", + "event": { + "ingested": "2021-03-03T14:56:52.287740584Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 30502 + }, + "@timestamp": "2017-04-08T19:54:37.468Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-04-08 21:54:37.468 CEST" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "could not receive data from client: Connection reset by peer", + "event": { + "ingested": "2021-03-03T14:56:52.287741970Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 20769 + }, + "@timestamp": "2017-04-08T19:54:37.618Z", + "postgresql": { + "log": { + "timestamp": "2017-04-08 21:54:37.618 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "received fast shutdown request", + "event": { + "ingested": "2021-03-03T14:56:52.287743380Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 20769 + }, + "@timestamp": "2017-04-08T19:54:37.618Z", + "postgresql": { + "log": { + "timestamp": "2017-04-08 21:54:37.618 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "aborting any active transactions", + "event": { + "ingested": "2021-03-03T14:56:52.287744783Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 20774 + }, + "@timestamp": "2017-04-08T19:54:37.618Z", + "postgresql": { + "log": { + "timestamp": "2017-04-08 21:54:37.618 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "autovacuum launcher shutting down", + "event": { + "ingested": "2021-03-03T14:56:52.287746174Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 20771 + }, + "@timestamp": "2017-04-08T19:54:37.622Z", + "postgresql": { + "log": { + "timestamp": "2017-04-08 21:54:37.622 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "shutting down", + "event": { + "ingested": "2021-03-03T14:56:52.287747639Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 20769 + }, + "@timestamp": "2017-04-08T19:54:37.644Z", + "postgresql": { + "log": { + "timestamp": "2017-04-08 21:54:37.644 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system is shut down", + "event": { + "ingested": "2021-03-03T14:56:52.287749063Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 797 + }, + "@timestamp": "2017-04-08T19:56:02.932Z", + "postgresql": { + "log": { + "timestamp": "2017-04-08 21:56:02.932 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system was shut down at 2017-04-08 21:54:37 CEST", + "event": { + "ingested": "2021-03-03T14:56:52.287750534Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 797 + }, + "@timestamp": "2017-04-08T19:56:02.944Z", + "postgresql": { + "log": { + "timestamp": "2017-04-08 21:56:02.944 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "MultiXact member wraparound protections are now enabled", + "event": { + "ingested": "2021-03-03T14:56:52.287751960Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 780 + }, + "@timestamp": "2017-04-08T19:56:02.946Z", + "postgresql": { + "log": { + "timestamp": "2017-04-08 21:56:02.946 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system is ready to accept connections", + "event": { + "ingested": "2021-03-03T14:56:52.287753373Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 802 + }, + "@timestamp": "2017-04-08T19:56:02.947Z", + "postgresql": { + "log": { + "timestamp": "2017-04-08 21:56:02.947 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "autovacuum launcher started", + "event": { + "ingested": "2021-03-03T14:56:52.287754785Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 891 + }, + "@timestamp": "2017-04-08T19:56:03.362Z", + "postgresql": { + "log": { + "database": "unknown", + "timestamp": "2017-04-08 21:56:03.362 CEST" + } + }, + "related": { + "user": [ + "unknown" + ] + }, + "log": { + "level": "LOG" + }, + "message": "incomplete startup packet", + "event": { + "ingested": "2021-03-03T14:56:52.287756201Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "unknown" + } + }, + { + "process": { + "pid": 32567 + }, + "@timestamp": "2017-05-27T14:07:53.007Z", + "postgresql": { + "log": { + "timestamp": "2017-05-27 14:07:53.007 UTC" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system was shut down at 2017-05-27 14:07:52 UTC", + "event": { + "ingested": "2021-03-03T14:56:52.287757613Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 32567 + }, + "@timestamp": "2017-05-27T14:07:53.010Z", + "postgresql": { + "log": { + "timestamp": "2017-05-27 14:07:53.010 UTC" + } + }, + "log": { + "level": "LOG" + }, + "message": "MultiXact member wraparound protections are now enabled", + "event": { + "ingested": "2021-03-03T14:56:52.287759047Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 32566 + }, + "@timestamp": "2017-05-27T14:07:53.015Z", + "postgresql": { + "log": { + "timestamp": "2017-05-27 14:07:53.015 UTC" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system is ready to accept connections", + "event": { + "ingested": "2021-03-03T14:56:52.287760468Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 32571 + }, + "@timestamp": "2017-05-27T14:07:53.016Z", + "postgresql": { + "log": { + "timestamp": "2017-05-27 14:07:53.016 UTC" + } + }, + "log": { + "level": "LOG" + }, + "message": "autovacuum launcher started", + "event": { + "ingested": "2021-03-03T14:56:52.287761871Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + } + }, + { + "process": { + "pid": 32573 + }, + "@timestamp": "2017-05-27T14:07:53.463Z", + "postgresql": { + "log": { + "database": "unknown", + "timestamp": "2017-05-27 14:07:53.463 UTC" + } + }, + "related": { + "user": [ + "unknown" + ] + }, + "log": { + "level": "LOG" + }, + "message": "incomplete startup packet", + "event": { + "ingested": "2021-03-03T14:56:52.287763265Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "unknown" + } + }, + { + "process": { + "pid": 1308 + }, + "@timestamp": "2017-05-27T14:08:13.661Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-05-27 14:08:13.661 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "FATAL" + }, + "message": "database \"mydb\" does not exist", + "event": { + "ingested": "2021-03-03T14:56:52.287764668Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 1994 + }, + "@timestamp": "2017-05-27T14:59:26.553Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-05-27 14:59:26.553 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "could not receive data from client: Connection reset by peer", + "event": { + "ingested": "2021-03-03T14:56:52.287766067Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 1989 + }, + "@timestamp": "2017-05-27T14:59:26.555Z", + "postgresql": { + "log": { + "database": "mydb", + "timestamp": "2017-05-27 14:59:26.555 UTC" + } + }, + "related": { + "user": [ + "postgres" + ] + }, + "log": { + "level": "LOG" + }, + "message": "could not receive data from client: Connection reset by peer", + "event": { + "ingested": "2021-03-03T14:56:52.287767469Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "UTC", + "kind": "event" + }, + "user": { + "name": "postgres" + } + }, + { + "process": { + "pid": 9110 + }, + "@timestamp": "2017-06-06T05:54:13.753Z", + "postgresql": { + "log": { + "timestamp": "2017-06-06 07:54:13.753 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "received fast shutdown request", + "event": { + "ingested": "2021-03-03T14:56:52.287768854Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 9110 + }, + "@timestamp": "2017-06-06T05:54:13.753Z", + "postgresql": { + "log": { + "timestamp": "2017-06-06 07:54:13.753 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "aborting any active transactions", + "event": { + "ingested": "2021-03-03T14:56:52.287770281Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 9115 + }, + "@timestamp": "2017-06-06T05:54:13.753Z", + "postgresql": { + "log": { + "timestamp": "2017-06-06 07:54:13.753 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "autovacuum launcher shutting down", + "event": { + "ingested": "2021-03-03T14:56:52.287771734Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 9112 + }, + "@timestamp": "2017-06-06T05:54:13.755Z", + "postgresql": { + "log": { + "timestamp": "2017-06-06 07:54:13.755 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "shutting down", + "event": { + "ingested": "2021-03-03T14:56:52.287773338Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 9110 + }, + "@timestamp": "2017-06-06T05:54:13.816Z", + "postgresql": { + "log": { + "timestamp": "2017-06-06 07:54:13.816 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system is shut down", + "event": { + "ingested": "2021-03-03T14:56:52.287774748Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 12969 + }, + "@timestamp": "2017-06-06T05:55:39.725Z", + "postgresql": { + "log": { + "timestamp": "2017-06-06 07:55:39.725 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system was shut down at 2017-06-06 07:54:13 CEST", + "event": { + "ingested": "2021-03-03T14:56:52.287776147Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 12969 + }, + "@timestamp": "2017-06-06T05:55:39.736Z", + "postgresql": { + "log": { + "timestamp": "2017-06-06 07:55:39.736 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "MultiXact member wraparound protections are now enabled", + "event": { + "ingested": "2021-03-03T14:56:52.287777540Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 12968 + }, + "@timestamp": "2017-06-06T05:55:39.739Z", + "postgresql": { + "log": { + "timestamp": "2017-06-06 07:55:39.739 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system is ready to accept connections", + "event": { + "ingested": "2021-03-03T14:56:52.287778954Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 12973 + }, + "@timestamp": "2017-06-06T05:55:39.739Z", + "postgresql": { + "log": { + "timestamp": "2017-06-06 07:55:39.739 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "autovacuum launcher started", + "event": { + "ingested": "2021-03-03T14:56:52.287780336Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 12975 + }, + "@timestamp": "2017-06-06T05:55:40.155Z", + "postgresql": { + "log": { + "database": "unknown", + "timestamp": "2017-06-06 07:55:40.155 CEST" + } + }, + "related": { + "user": [ + "unknown" + ] + }, + "log": { + "level": "LOG" + }, + "message": "incomplete startup packet", + "event": { + "ingested": "2021-03-03T14:56:52.287781780Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "unknown" + } + }, + { + "process": { + "pid": 12975 + }, + "@timestamp": "2017-06-06T05:55:40.156Z", + "postgresql": { + "log": { + "database": "unknown", + "timestamp": "2017-06-06 07:55:40.156 CEST" + } + }, + "related": { + "user": [ + "unknown" + ] + }, + "log": { + "level": "LOG" + }, + "message": "incomplete startup packet", + "event": { + "ingested": "2021-03-03T14:56:52.287783204Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "unknown" + } + }, + { + "process": { + "pid": 17398 + }, + "@timestamp": "2017-06-10T17:37:30.681Z", + "postgresql": { + "log": { + "timestamp": "2017-06-10 19:37:30.681 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system was shut down at 2017-06-10 19:37:29 CEST", + "event": { + "ingested": "2021-03-03T14:56:52.287784652Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 17398 + }, + "@timestamp": "2017-06-10T17:37:30.695Z", + "postgresql": { + "log": { + "timestamp": "2017-06-10 19:37:30.695 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "MultiXact member wraparound protections are now enabled", + "event": { + "ingested": "2021-03-03T14:56:52.287786069Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 17397 + }, + "@timestamp": "2017-06-10T17:37:30.702Z", + "postgresql": { + "log": { + "timestamp": "2017-06-10 19:37:30.702 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system is ready to accept connections", + "event": { + "ingested": "2021-03-03T14:56:52.287787450Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 17402 + }, + "@timestamp": "2017-06-10T17:37:30.702Z", + "postgresql": { + "log": { + "timestamp": "2017-06-10 19:37:30.702 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "autovacuum launcher started", + "event": { + "ingested": "2021-03-03T14:56:52.287788864Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 17404 + }, + "@timestamp": "2017-06-10T17:37:31.104Z", + "postgresql": { + "log": { + "database": "unknown", + "timestamp": "2017-06-10 19:37:31.104 CEST" + } + }, + "related": { + "user": [ + "unknown" + ] + }, + "log": { + "level": "LOG" + }, + "message": "incomplete startup packet", + "event": { + "ingested": "2021-03-03T14:56:52.287790303Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "unknown" + } + }, + { + "process": { + "pid": 17397 + }, + "@timestamp": "2017-06-10T18:27:55.911Z", + "postgresql": { + "log": { + "timestamp": "2017-06-10 20:27:55.911 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "received fast shutdown request", + "event": { + "ingested": "2021-03-03T14:56:52.287791752Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 17397 + }, + "@timestamp": "2017-06-10T18:27:55.911Z", + "postgresql": { + "log": { + "timestamp": "2017-06-10 20:27:55.911 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "aborting any active transactions", + "event": { + "ingested": "2021-03-03T14:56:52.287793192Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 17402 + }, + "@timestamp": "2017-06-10T18:27:55.911Z", + "postgresql": { + "log": { + "timestamp": "2017-06-10 20:27:55.911 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "autovacuum launcher shutting down", + "event": { + "ingested": "2021-03-03T14:56:52.287794618Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 17399 + }, + "@timestamp": "2017-06-10T18:27:55.914Z", + "postgresql": { + "log": { + "timestamp": "2017-06-10 20:27:55.914 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "shutting down", + "event": { + "ingested": "2021-03-03T14:56:52.287796026Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 17397 + }, + "@timestamp": "2017-06-10T18:27:55.973Z", + "postgresql": { + "log": { + "timestamp": "2017-06-10 20:27:55.973 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system is shut down", + "event": { + "ingested": "2021-03-03T14:56:52.287797448Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 24490 + }, + "@timestamp": "2017-06-10T18:27:57.022Z", + "postgresql": { + "log": { + "timestamp": "2017-06-10 20:27:57.022 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system was shut down at 2017-06-10 20:27:55 CEST", + "event": { + "ingested": "2021-03-03T14:56:52.287798871Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 24490 + }, + "@timestamp": "2017-06-10T18:27:57.032Z", + "postgresql": { + "log": { + "timestamp": "2017-06-10 20:27:57.032 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "MultiXact member wraparound protections are now enabled", + "event": { + "ingested": "2021-03-03T14:56:52.287800283Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 24494 + }, + "@timestamp": "2017-06-10T18:27:57.035Z", + "postgresql": { + "log": { + "timestamp": "2017-06-10 20:27:57.035 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "autovacuum launcher started", + "event": { + "ingested": "2021-03-03T14:56:52.287801696Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 24489 + }, + "@timestamp": "2017-06-10T18:27:57.035Z", + "postgresql": { + "log": { + "timestamp": "2017-06-10 20:27:57.035 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system is ready to accept connections", + "event": { + "ingested": "2021-03-03T14:56:52.287803091Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 24496 + }, + "@timestamp": "2017-06-10T18:27:57.475Z", + "postgresql": { + "log": { + "database": "unknown", + "timestamp": "2017-06-10 20:27:57.475 CEST" + } + }, + "related": { + "user": [ + "unknown" + ] + }, + "log": { + "level": "LOG" + }, + "message": "incomplete startup packet", + "event": { + "ingested": "2021-03-03T14:56:52.287804450Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + }, + "user": { + "name": "unknown" + } + }, + { + "process": { + "pid": 24489 + }, + "@timestamp": "2017-06-17T14:58:03.937Z", + "postgresql": { + "log": { + "timestamp": "2017-06-17 16:58:03.937 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "received fast shutdown request", + "event": { + "ingested": "2021-03-03T14:56:52.287805844Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 24489 + }, + "@timestamp": "2017-06-17T14:58:03.937Z", + "postgresql": { + "log": { + "timestamp": "2017-06-17 16:58:03.937 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "aborting any active transactions", + "event": { + "ingested": "2021-03-03T14:56:52.287807355Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 24494 + }, + "@timestamp": "2017-06-17T14:58:03.938Z", + "postgresql": { + "log": { + "timestamp": "2017-06-17 16:58:03.938 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "autovacuum launcher shutting down", + "event": { + "ingested": "2021-03-03T14:56:52.287808756Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 24491 + }, + "@timestamp": "2017-06-17T14:58:03.940Z", + "postgresql": { + "log": { + "timestamp": "2017-06-17 16:58:03.940 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "shutting down", + "event": { + "ingested": "2021-03-03T14:56:52.287810176Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + }, + { + "process": { + "pid": 24489 + }, + "@timestamp": "2017-06-17T14:58:04.040Z", + "postgresql": { + "log": { + "timestamp": "2017-06-17 16:58:04.040 CEST" + } + }, + "log": { + "level": "LOG" + }, + "message": "database system is shut down", + "event": { + "ingested": "2021-03-03T14:56:52.287811573Z", + "category": [ + "database" + ], + "type": [ + "info" + ], + "timezone": "CEST", + "kind": "event" + } + } + ] +} \ No newline at end of file diff --git a/packages/postgresql/data_stream/log/agent/stream/log.yml.hbs b/packages/postgresql/data_stream/log/agent/stream/log.yml.hbs index 3fab24ae165..7bbea40b30a 100644 --- a/packages/postgresql/data_stream/log/agent/stream/log.yml.hbs +++ b/packages/postgresql/data_stream/log/agent/stream/log.yml.hbs @@ -11,4 +11,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 \ No newline at end of file + ecs.version: 1.8.0 \ No newline at end of file diff --git a/packages/postgresql/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/postgresql/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 1088387dbe8..3326dbc27fe 100644 --- a/packages/postgresql/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/postgresql/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -1,18 +1,27 @@ --- description: Pipeline for parsing PostgreSQL logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' + - grok: field: message - ignore_missing: true patterns: - - '^%{DATETIME:postgresql.log.timestamp} (\[%{NUMBER:process.pid:long}(-%{BASE16FLOAT:postgresql.log.core_id:long})?\] ((\[%{USERNAME:user.name}\]@\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\]|%{USERNAME:user.name}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?)?%{WORD:log.level}: (?:%{NUMBER:postgresql.log.error.code:long}|%{SPACE})(duration: %{NUMBER:temp.duration:float} ms %{POSTGRESQL_QUERY_STEP}: %{GREEDYDATA:postgresql.log.query}|: %{GREEDYDATA:message}|%{GREEDYDATA:message})' + - '^%{DATETIME:postgresql.log.timestamp}%{CHAR:separator}%{GREEDYDATA:raw_message}' pattern_definitions: DATETIME: '[-0-9]+ %{TIME} %{WORD:event.timezone}' + CHAR: . GREEDYDATA: |- (.| | )* - POSTGRESQL_DB_NAME: '[a-zA-Z0-9_]+[a-zA-Z0-9_\$]*' - POSTGRESQL_QUERY_STEP: '%{WORD:postgresql.log.query_step}(?: | %{WORD:postgresql.log.query_name})?' +- pipeline: + name: '{{ IngestPipeline "pipeline-log" }}' + if: ctx.separator != ',' +- pipeline: + name: '{{ IngestPipeline "pipeline-csv" }}' + if: ctx.separator == ',' + - date: field: postgresql.log.timestamp target_field: '@timestamp' @@ -35,19 +44,27 @@ processors: field: event.category value: - database -- append: +- set: field: event.type value: - info -- append: + if: "ctx?.postgresql?.log?.sql_state_code == null || (ctx.postgresql.log.sql_state_code ==~ /^0[012].*/)" +- set: field: event.type value: - error - if: "ctx?.postgresql?.log?.error?.code != null && ctx.postgresql.log.error.code >= 02000" + if: "ctx?.postgresql?.log?.sql_state_code != null && ! (ctx.postgresql.log.sql_state_code ==~ /^0[012].*/)" - append: field: related.user value: "{{user.name}}" if: "ctx?.user?.name != null" +- remove: + field: + - separator + - raw_message +- remove: + field: temp + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/postgresql/data_stream/log/elasticsearch/ingest_pipeline/pipeline-csv.yml b/packages/postgresql/data_stream/log/elasticsearch/ingest_pipeline/pipeline-csv.yml new file mode 100644 index 00000000000..416d3f8aff6 --- /dev/null +++ b/packages/postgresql/data_stream/log/elasticsearch/ingest_pipeline/pipeline-csv.yml @@ -0,0 +1,85 @@ +--- +description: Pipeline for parsing PostgreSQL CSV logs. +processors: + +- csv: + field: raw_message + separator: "," + target_fields: ["user.name", + "postgresql.log.database", + "process.pid", + "tempcsv.connection_from", + "postgresql.log.session_id", + "tempcsv.session_line_num", + "postgresql.log.command_tag", + "tempcsv.session_start_time", + "postgresql.log.virtual_transaction_id", + "postgresql.log.transaction_id", + "log.level", + "postgresql.log.sql_state_code", + "tempcsv.message", + "postgresql.log.detail", + "postgresql.log.hint", + "postgresql.internal_query", + "tempcsv.internal_query_pos", + "postgresql.log.context", + "postgresql.log.query", + "tempcsv.query_pos", + "postgresql.log.location", + "postgresql.log.application_name", + "postgresql.log.backend_type"] + ignore_missing: true + trim: true +- remove: + field: message + ignore_missing: false + +- grok: + field: tempcsv.connection_from + ignore_missing: true + patterns: + - '^%{DATA:postgresql.log.client_addr}(:%{NUMBER:postgresql.log.client_port:int})?$' + +- convert: + field: "postgresql.log.session_line_num" + type: long + ignore_missing: true + +- convert: + field: "process.pid" + type: long + ignore_missing: true + +- date: + field: tempcsv.session_start_time + target_field: postgresql.log.session_start_time + formats: + - yyyy-MM-dd HH:mm:ss.SSS zz + - yyyy-MM-dd HH:mm:ss zz + +- convert: + field: postgresql.log.transaction_id + type: long + ignore_missing: true +- grok: + field: tempcsv.message + ignore_missing: true + patterns: + - '^duration: %{NUMBER:temp.duration:float} ms$' + - '^duration: %{NUMBER:temp.duration:float} ms %{POSTGRESQL_QUERY_STEP:postgresql.log.query_step} %{DATA:postgresql.log.query_name}: %{GREEDYDATA:message}$' + - '^duration: %{NUMBER:temp.duration:float} ms %{POSTGRESQL_QUERY_STEP:postgresql.log.query_step}: %{GREEDYDATA:message}$' + - '^(%{POSTGRESQL_QUERY_STEP:postgresql.log.query_step}: )?%{GREEDYDATA:message}$' + pattern_definitions: + GREEDYDATA: |- + (.| + | )* + POSTGRESQL_QUERY_STEP: '(parse|bind|statement|fastpath function call|execute|execute fetch from)' + +- grok: + field: tempcsv.connection_from + ignore_missing: true + patterns: + - '^%{DATA:postgresql.log.client_addr}(:%{NUMBER:postgresql.log.client_port:int})?$' + +- remove: + field: tempcsv diff --git a/packages/postgresql/data_stream/log/elasticsearch/ingest_pipeline/pipeline-log.yml b/packages/postgresql/data_stream/log/elasticsearch/ingest_pipeline/pipeline-log.yml new file mode 100644 index 00000000000..31c52277b40 --- /dev/null +++ b/packages/postgresql/data_stream/log/elasticsearch/ingest_pipeline/pipeline-log.yml @@ -0,0 +1,15 @@ +--- +description: Pipeline for parsing PostgreSQL logs. +processors: +- grok: + field: raw_message + ignore_missing: true + patterns: + - '^(\[%{NUMBER:process.pid:long}(-%{BASE16FLOAT:postgresql.log.session_line_number:long})?\] ((\[%{USERNAME:user.name}\]@\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\]|%{USERNAME:user.name}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?)?%{WORD:log.level}: (?:%{POSTGRESQL_ERROR:postgresql.log.sql_state_code}|%{SPACE})(duration: %{NUMBER:temp.duration:float} ms %{POSTGRESQL_QUERY_STEP}: %{GREEDYDATA:postgresql.log.query}|: %{GREEDYDATA:message}|%{GREEDYDATA:message})' + pattern_definitions: + GREEDYDATA: |- + (.| + | )* + POSTGRESQL_DB_NAME: '[a-zA-Z0-9_]+[a-zA-Z0-9_\$]*' + POSTGRESQL_QUERY_STEP: '%{WORD:postgresql.log.query_step}(?: | %{WORD:postgresql.log.query_name})?' + POSTGRESQL_ERROR: '\b[A-Z0-9]{5}\b' diff --git a/packages/postgresql/data_stream/log/fields/ecs.yml b/packages/postgresql/data_stream/log/fields/ecs.yml index 749e28dcf3e..936b5ae23da 100644 --- a/packages/postgresql/data_stream/log/fields/ecs.yml +++ b/packages/postgresql/data_stream/log/fields/ecs.yml @@ -10,6 +10,46 @@ group: 2 type: group fields: + - name: category + level: core + type: keyword + ignore_above: 1024 + description: 'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. + + `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. + + This field is an array. This will allow proper categorization of some events that fall in multiple categories.' + example: authentication + - name: duration + level: core + type: long + format: duration + input_format: nanoseconds + output_format: asMilliseconds + output_precision: 1 + description: 'Duration of the event in nanoseconds. + + If event.start and event.end are known this value should be the difference between the end and start time.' + - name: ingested + level: core + type: date + description: 'Timestamp when an event arrived in the central data store. + + This is different from `@timestamp`, which is when the event originally occurred. It''s also different from `event.created`, which is meant to capture the first time an agent saw the event. + + In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` < `event.created` < `event.ingested`.' + example: '2016-05-23T08:05:35.101Z' + default_field: false + - name: kind + level: core + type: keyword + ignore_above: 1024 + description: 'This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. + + `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. + + The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not.' + example: alert - name: timezone level: extended type: keyword @@ -30,9 +70,29 @@ If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. ignore_above: 1024 +- name: user + title: User + group: 2 + type: group + fields: + - name: name + level: core + type: keyword + description: Short name or login of the user. + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false - name: process title: Process group: 2 + description: |- + These fields contain information about a process. + These fields can help you correlate metrics information with a process id/name + from a log message. The `process.pid` often stays in the metric itself and + is copied to the global field for correlation. type: group fields: - name: pid @@ -40,18 +100,48 @@ type: long format: string description: Process id. -- name: user - title: User + example: 4242 +- name: related + title: Related group: 2 + description: |- + This field set is meant to facilitate pivoting around a piece of data. + Some pieces of information can be seen in many places in an ECS event. To facilitate + searching for them, store an array of all seen values to their corresponding + field in `related.`. + + A concrete example is IP addresses, which can be under host, observer, source, + destination, client, server, and network.forwarded_ip. If you append all IPs + to `related.ip`, you can then search for a given IP trivially, no matter where + it appeared, by querying `related.ip:192.0.2.15`. type: group fields: - - name: name + - name: user + level: extended + type: keyword + ignore_above: 1024 + description: All the user names seen on your event. + default_field: false +- name: error + title: Error + group: 2 + description: |- + These fields can represent errors of any kind. + Use them for errors that happen while fetching events or in cases where the + event itself contains an error. + type: group + fields: + - name: code level: core type: keyword - description: Short name or login of the user. ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false + description: Error code describing the error. + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique identifier for the error. + - name: message + level: core + type: text + description: Error message. diff --git a/packages/postgresql/data_stream/log/fields/fields.yml b/packages/postgresql/data_stream/log/fields/fields.yml index 590a625d6aa..f0686531fc4 100644 --- a/packages/postgresql/data_stream/log/fields/fields.yml +++ b/packages/postgresql/data_stream/log/fields/fields.yml @@ -5,29 +5,94 @@ type: keyword description: | The timestamp from the log line. - - name: core_id + - name: client_addr + type: keyword + description: | + Host where the connection originated from. + - name: client_port + type: long + description: | + Port where the connection originated from. + - name: session_id + type: keyword + description: | + PostgreSQL session. + - name: session_line_number type: long description: | - Core id + Line number inside a session. (%l in `log_line_prefix`). - name: database type: keyword description: | - Name of database + Name of database. - name: query type: keyword description: | - Query statement. + Query statement. In the case of CSV parse, look at command_tag to get more context. - name: query_step type: keyword description: | - Statement step when using extended query protocol (one of statement, parse, bind or execute) + Statement step when using extended query protocol (one of statement, parse, bind or execute). - name: query_name type: keyword description: | Name given to a query when using extended query protocol. If it is "", or not present, this field is ignored. - - name: error.code + - name: command_tag + type: keyword + description: | + Type of session's current command. The complete list can be found at: src/include/tcop/cmdtaglist.h + - name: session_start_time + type: date + description: | + Time when this session started. + - name: virtual_transaction_id + type: keyword + description: | + Backend local transaction id. + - name: transaction_id + type: long + description: | + The id of current transaction. + - name: sql_state_code + type: keyword + description: | + State code returned by Postgres (if any). See also https://www.postgresql.org/docs/current/errcodes-appendix.html + - name: detail + type: keyword + description: | + More information about the message, parameters in case of a parametrized query. e.g. 'Role \"user\" does not exist.', 'parameters: $1 = 42', etc. + - name: hint + type: keyword + description: | + A possible solution to solve an error. + - name: internal_query + type: keyword + description: | + Internal query that led to the error (if any). + - name: internal_query_pos + type: long + description: | + Character count of the internal query (if any). + - name: context + type: keyword + description: | + Error context. + - name: query_pos type: long - description: Error code returned by Postgres (if any) + description: | + Character count of the error position (if any). + - name: location + type: keyword + description: | + Location of the error in the PostgreSQL source code (if log_error_verbosity is set to verbose). + - name: application_name + type: keyword + description: | + Name of the application of this event. It is defined by the client. + - name: backend_type + type: keyword + description: | + Type of backend of this event. Possible types are autovacuum launcher, autovacuum worker, logical replication launcher, logical replication worker, parallel worker, background writer, client backend, checkpointer, startup, walreceiver, walsender and walwriter. In addition, background workers registered by extensions may have additional types. - name: event.kind type: keyword description: Event kind (e.g. event) diff --git a/packages/postgresql/data_stream/log/manifest.yml b/packages/postgresql/data_stream/log/manifest.yml index 25befde30ce..a50905aa063 100644 --- a/packages/postgresql/data_stream/log/manifest.yml +++ b/packages/postgresql/data_stream/log/manifest.yml @@ -1,6 +1,6 @@ -title: PostgreSQL application logs -release: experimental type: logs +title: PostgreSQL logs +release: experimental streams: - input: logfile vars: @@ -12,6 +12,7 @@ streams: show_user: true default: - /var/log/postgresql/postgresql-*-*.log* + - /var/log/postgresql/postgresql-*-*.csv* template_path: log.yml.hbs - title: PostgreSQL application logs - description: Collect PostgreSQL application logs using log input + title: PostgreSQL logs + description: Collect PostgreSQL log logs using log input diff --git a/packages/postgresql/data_stream/statement/_dev/test/system/test-default-config.yml b/packages/postgresql/data_stream/statement/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..e01b0b57ac8 --- /dev/null +++ b/packages/postgresql/data_stream/statement/_dev/test/system/test-default-config.yml @@ -0,0 +1,7 @@ +vars: + hosts: + - postgres://{{Hostname}}:{{Port}}?sslmode=disable + username: postgres + password: postgres +data_stream: + vars: ~ diff --git a/packages/postgresql/data_stream/statement/fields/ecs.yml b/packages/postgresql/data_stream/statement/fields/ecs.yml index 50e1946dc41..4eee3428d3b 100644 --- a/packages/postgresql/data_stream/statement/fields/ecs.yml +++ b/packages/postgresql/data_stream/statement/fields/ecs.yml @@ -1,3 +1,67 @@ +- name: ecs + title: ECS + group: 2 + description: Meta-information specific to ECS. + type: group + fields: + - name: version + level: core + required: true + type: keyword + ignore_above: 1024 + description: 'ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. + + When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events.' + example: 1.0.0 +- name: error + title: Error + group: 2 + description: |- + These fields can represent errors of any kind. + Use them for errors that happen while fetching events or in cases where the + event itself contains an error. + type: group + fields: + - name: message + level: core + type: text + description: Error message. +- name: event + title: Event + group: 2 + description: 'The event fields are used for context information about the log or metric event itself. + + A log is defined as an event containing details of something that happened. Log events must include the time at which the thing happened. Examples of log events include a process starting on a host, a network packet being sent from a source to a destination, or a network connection between a client and a server being initiated or closed. A metric is defined as an event containing one or more numerical measurements and the time at which the measurement was taken. Examples of metric events include memory pressure measured on a host and device temperature. See the `event.kind` definition in this section for additional details about metric and state events.' + type: group + fields: + - name: dataset + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the dataset. + + If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. + + It''s recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name.' + example: apache.access + - name: duration + level: core + type: long + format: duration + input_format: nanoseconds + output_format: asMilliseconds + output_precision: 1 + description: 'Duration of the event in nanoseconds. + + If event.start and event.end are known this value should be the difference between the end and start time.' + - name: module + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the module this data is coming from. + + If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module.' + example: apache - name: service.address type: keyword description: Service address diff --git a/packages/postgresql/data_stream/statement/manifest.yml b/packages/postgresql/data_stream/statement/manifest.yml index b96e58a717e..39d28e740a9 100644 --- a/packages/postgresql/data_stream/statement/manifest.yml +++ b/packages/postgresql/data_stream/statement/manifest.yml @@ -1,6 +1,6 @@ +type: metrics title: PostgreSQL statement metrics release: experimental -type: metrics streams: - input: postgresql/metrics title: PostgreSQL statement metrics diff --git a/packages/postgresql/docs/README.md b/packages/postgresql/docs/README.md index 6b9f34fc897..267f169ac6c 100644 --- a/packages/postgresql/docs/README.md +++ b/packages/postgresql/docs/README.md @@ -4,7 +4,7 @@ This integration periodically fetches logs and metrics from [PostgreSQL](https:/ ## Compatibility -The `log` dataset was tested with logs from versions 9.5 on Ubuntu, 9.6 on Debian, and finally 10.11, 11.4 and 12.2 on Arch Linux 9.3. +The `log` dataset was tested with logs from versions 9.5 on Ubuntu, 9.6 on Debian, and finally 10.11, 11.4 and 12.2 on Arch Linux 9.3. CSV format was tested using versions 11 and 13 (distro is not relevant here). The `activity`, `bgwriter`, `database` and `statement` datasets were tested with PostgreSQL 9.5.3 and is expected to work with all versions >= 9. @@ -12,7 +12,32 @@ The `activity`, `bgwriter`, `database` and `statement` datasets were tested with ### log -The `log` dataset collects the PostgreSQL logs. +The `log` dataset collects the PostgreSQL logs in plain text format or CSV. + +#### Using CSV logs + +Since the PostgreSQL CSV log file is a well-defined format, +there is almost no configuration to be done in Fleet, just the filepath. + +On the other hand, it's necessary to configure PostgreSQL to emit `.csv` logs. + +The recommended parameters are: +``` +logging_collector = 'on'; +log_destination = 'csvlog'; +log_statement = 'none'; +log_checkpoints = on; +log_connections = on; +log_disconnections = on; +log_lock_waits = on; +log_min_duration_statement = 0; +``` + +In busy servers, `log_min_duration_statement` can cause contention, so you can assign +a value greater than 0. + +Both `log_connections` and `log_disconnections` can cause a lot of events if you don't have +persistent connections, so enable with care. **Exported fields** @@ -35,8 +60,13 @@ The `log` dataset collects the PostgreSQL logs. | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| error.code | Error code describing the error. | keyword | +| error.id | Unique identifier for the error. | keyword | +| error.message | Error message. | text | | event.category | Event category (e.g. database) | keyword | | event.code | Identification code for this event | keyword | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | +| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` < `event.created` < `event.ingested`. | date | | event.kind | Event kind (e.g. event) | keyword | | event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | | event.type | Event severity (e.g. info, error) | keyword | @@ -58,14 +88,31 @@ The `log` dataset collects the PostgreSQL logs. | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | | message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | text | -| postgresql.log.core_id | Core id | long | -| postgresql.log.database | Name of database | keyword | -| postgresql.log.error.code | Error code returned by Postgres (if any) | long | -| postgresql.log.query | Query statement. | keyword | +| postgresql.log.application_name | Name of the application of this event. It is defined by the client. | keyword | +| postgresql.log.backend_type | Type of backend of this event. Possible types are autovacuum launcher, autovacuum worker, logical replication launcher, logical replication worker, parallel worker, background writer, client backend, checkpointer, startup, walreceiver, walsender and walwriter. In addition, background workers registered by extensions may have additional types. | keyword | +| postgresql.log.client_addr | Host where the connection originated from. | keyword | +| postgresql.log.client_port | Port where the connection originated from. | long | +| postgresql.log.command_tag | Type of session's current command. The complete list can be found at: src/include/tcop/cmdtaglist.h | keyword | +| postgresql.log.context | Error context. | keyword | +| postgresql.log.database | Name of database. | keyword | +| postgresql.log.detail | More information about the message, parameters in case of a parametrized query. e.g. 'Role \"user\" does not exist.', 'parameters: $1 = 42', etc. | keyword | +| postgresql.log.hint | A possible solution to solve an error. | keyword | +| postgresql.log.internal_query | Internal query that led to the error (if any). | keyword | +| postgresql.log.internal_query_pos | Character count of the internal query (if any). | long | +| postgresql.log.location | Location of the error in the PostgreSQL source code (if log_error_verbosity is set to verbose). | keyword | +| postgresql.log.query | Query statement. In the case of CSV parse, look at command_tag to get more context. | keyword | | postgresql.log.query_name | Name given to a query when using extended query protocol. If it is "", or not present, this field is ignored. | keyword | -| postgresql.log.query_step | Statement step when using extended query protocol (one of statement, parse, bind or execute) | keyword | +| postgresql.log.query_pos | Character count of the error position (if any). | long | +| postgresql.log.query_step | Statement step when using extended query protocol (one of statement, parse, bind or execute). | keyword | +| postgresql.log.session_id | PostgreSQL session. | keyword | +| postgresql.log.session_line_number | Line number inside a session. (%l in `log_line_prefix`). | long | +| postgresql.log.session_start_time | Time when this session started. | date | +| postgresql.log.sql_state_code | State code returned by Postgres (if any). See also https://www.postgresql.org/docs/current/errcodes-appendix.html | keyword | | postgresql.log.timestamp | The timestamp from the log line. | keyword | +| postgresql.log.transaction_id | The id of current transaction. | long | +| postgresql.log.virtual_transaction_id | Backend local transaction id. | keyword | | process.pid | Process id. | long | +| related.user | All the user names seen on your event. | keyword | | user.name | Short name or login of the user. | keyword | @@ -146,6 +193,11 @@ An example event for `activity` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.message | Error message. | text | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -256,6 +308,11 @@ An example event for `bgwriter` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.message | Error message. | text | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -362,6 +419,8 @@ An example event for `database` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.message | Error message. | text | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -501,6 +560,11 @@ An example event for `statement` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.message | Error message. | text | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -540,3 +604,4 @@ An example event for `statement` looks as following: | postgresql.statement.user.id | OID of the user logged into the backend that ran the query. | long | | service.address | Service address | keyword | | service.type | Service type | keyword | + diff --git a/packages/postgresql/img/filebeat-postgresql-overview.png b/packages/postgresql/img/filebeat-postgresql-overview.png deleted file mode 100644 index b4fc428c4d0..00000000000 Binary files a/packages/postgresql/img/filebeat-postgresql-overview.png and /dev/null differ diff --git a/packages/postgresql/img/filebeat-postgresql-slowlog-overview.png b/packages/postgresql/img/filebeat-postgresql-slowlog-overview.png deleted file mode 100644 index 944a2577852..00000000000 Binary files a/packages/postgresql/img/filebeat-postgresql-slowlog-overview.png and /dev/null differ diff --git a/packages/postgresql/img/logs-overview.png b/packages/postgresql/img/logs-overview.png new file mode 100644 index 00000000000..aa1c75c6f46 Binary files /dev/null and b/packages/postgresql/img/logs-overview.png differ diff --git a/packages/postgresql/img/logs-slowlog-overview.png b/packages/postgresql/img/logs-slowlog-overview.png new file mode 100644 index 00000000000..180bd90c64d Binary files /dev/null and b/packages/postgresql/img/logs-slowlog-overview.png differ diff --git a/packages/postgresql/img/metricbeat-postgresql-overview.png b/packages/postgresql/img/metricbeat-postgresql-overview.png deleted file mode 100644 index 9d75e9addb2..00000000000 Binary files a/packages/postgresql/img/metricbeat-postgresql-overview.png and /dev/null differ diff --git a/packages/postgresql/img/metrics-overview.png b/packages/postgresql/img/metrics-overview.png new file mode 100644 index 00000000000..c040c4eab13 Binary files /dev/null and b/packages/postgresql/img/metrics-overview.png differ diff --git a/packages/postgresql/kibana/dashboard/postgresql-158be870-87f4-11e7-ad9c-db80de0bf8d3.json b/packages/postgresql/kibana/dashboard/postgresql-158be870-87f4-11e7-ad9c-db80de0bf8d3.json index 4e7fdea097f..ee67d7ec368 100644 --- a/packages/postgresql/kibana/dashboard/postgresql-158be870-87f4-11e7-ad9c-db80de0bf8d3.json +++ b/packages/postgresql/kibana/dashboard/postgresql-158be870-87f4-11e7-ad9c-db80de0bf8d3.json @@ -84,16 +84,19 @@ }, "id": "postgresql-158be870-87f4-11e7-ad9c-db80de0bf8d3", "migrationVersion": { - "dashboard": "7.3.0" + "dashboard": "7.9.3" }, + "namespaces": [ + "default" + ], "references": [ { - "id": "postgresql-log-level-count", + "id": "postgresql-PostgreSQL Log Level Count", "name": "panel_0", "type": "visualization" }, { - "id": "postgresql-all-logs", + "id": "postgresql-PostgreSQL All Logs", "name": "panel_1", "type": "search" }, diff --git a/packages/postgresql/kibana/dashboard/postgresql-4288b790-b79f-11e9-a579-f5c0a5d81340.json b/packages/postgresql/kibana/dashboard/postgresql-4288b790-b79f-11e9-a579-f5c0a5d81340.json index 8a98aed147c..958a518be35 100644 --- a/packages/postgresql/kibana/dashboard/postgresql-4288b790-b79f-11e9-a579-f5c0a5d81340.json +++ b/packages/postgresql/kibana/dashboard/postgresql-4288b790-b79f-11e9-a579-f5c0a5d81340.json @@ -183,8 +183,11 @@ }, "id": "postgresql-4288b790-b79f-11e9-a579-f5c0a5d81340", "migrationVersion": { - "dashboard": "7.3.0" + "dashboard": "7.9.3" }, + "namespaces": [ + "default" + ], "references": [ { "id": "postgresql-fbfa67e0-b796-11e9-a579-f5c0a5d81340", diff --git a/packages/postgresql/kibana/dashboard/postgresql-e4c5f230-87f3-11e7-ad9c-db80de0bf8d3.json b/packages/postgresql/kibana/dashboard/postgresql-e4c5f230-87f3-11e7-ad9c-db80de0bf8d3.json index a12950e3296..c459109a442 100644 --- a/packages/postgresql/kibana/dashboard/postgresql-e4c5f230-87f3-11e7-ad9c-db80de0bf8d3.json +++ b/packages/postgresql/kibana/dashboard/postgresql-e4c5f230-87f3-11e7-ad9c-db80de0bf8d3.json @@ -85,21 +85,24 @@ }, "id": "postgresql-e4c5f230-87f3-11e7-ad9c-db80de0bf8d3", "migrationVersion": { - "dashboard": "7.3.0" + "dashboard": "7.9.3" }, + "namespaces": [ + "default" + ], "references": [ { - "id": "postgresql-query-count-and-duration", + "id": "postgresql-PostgreSQL Query Count and Duration", "name": "panel_0", "type": "visualization" }, { - "id": "postgresql-slow-postgresql-queries", + "id": "postgresql-Slow PostgreSQL Queries", "name": "panel_1", "type": "search" }, { - "id": "postgresql-query-durations", + "id": "postgresql-PostgreSQL Query Durations", "name": "panel_2", "type": "search" } diff --git a/packages/postgresql/kibana/search/postgresql-PostgreSQL All Logs.json b/packages/postgresql/kibana/search/postgresql-PostgreSQL All Logs.json new file mode 100644 index 00000000000..c18517a511c --- /dev/null +++ b/packages/postgresql/kibana/search/postgresql-PostgreSQL All Logs.json @@ -0,0 +1,73 @@ +{ + "attributes": { + "columns": [ + "user.name", + "postgresql.log.database", + "log.level", + "message", + "postgresql.log.query" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "query", + "negate": false, + "type": "custom", + "value": "{\"prefix\":{\"data_stream.dataset\":\"postgresql.\"}}" + }, + "query": { + "prefix": { + "data_stream.dataset": "postgresql." + } + } + } + ], + "highlightAll": true, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + }, + "version": true + } + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "title": "All Logs [Logs PostgreSQL]", + "version": 1 + }, + "id": "postgresql-PostgreSQL All Logs", + "migrationVersion": { + "search": "7.9.3" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + } + ], + "type": "search" +} \ No newline at end of file diff --git a/packages/postgresql/kibana/search/postgresql-PostgreSQL Query Durations.json b/packages/postgresql/kibana/search/postgresql-PostgreSQL Query Durations.json new file mode 100644 index 00000000000..a1a128ea5ad --- /dev/null +++ b/packages/postgresql/kibana/search/postgresql-PostgreSQL Query Durations.json @@ -0,0 +1,47 @@ +{ + "attributes": { + "columns": [ + "user.name", + "postgresql.log.database", + "event.duration", + "postgresql.log.query" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "event.duration:*" + }, + "version": true + } + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "title": "Query Durations [Logs PostgreSQL]", + "version": 1 + }, + "id": "postgresql-PostgreSQL Query Durations", + "migrationVersion": { + "search": "7.9.3" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "search" +} \ No newline at end of file diff --git a/packages/postgresql/kibana/search/postgresql-Slow PostgreSQL Queries.json b/packages/postgresql/kibana/search/postgresql-Slow PostgreSQL Queries.json new file mode 100644 index 00000000000..853cf384de7 --- /dev/null +++ b/packages/postgresql/kibana/search/postgresql-Slow PostgreSQL Queries.json @@ -0,0 +1,47 @@ +{ + "attributes": { + "columns": [ + "user.name", + "postgresql.log.database", + "event.duration", + "postgresql.log.query" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "event.duration\u003e30000000" + }, + "version": true + } + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "title": "Slow Queries [Logs PostgreSQL]", + "version": 1 + }, + "id": "postgresql-Slow PostgreSQL Queries", + "migrationVersion": { + "search": "7.9.3" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "search" +} \ No newline at end of file diff --git a/packages/postgresql/kibana/visualization/postgresql-0cb65170-b909-11e9-a579-f5c0a5d81340.json b/packages/postgresql/kibana/visualization/postgresql-0cb65170-b909-11e9-a579-f5c0a5d81340.json index 4b0f1aa958e..b98cc2b5456 100644 --- a/packages/postgresql/kibana/visualization/postgresql-0cb65170-b909-11e9-a579-f5c0a5d81340.json +++ b/packages/postgresql/kibana/visualization/postgresql-0cb65170-b909-11e9-a579-f5c0a5d81340.json @@ -2,13 +2,7 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } + "searchSourceJSON": {} }, "title": "Local block cache stats [Metrics PostgreSQL]", "uiStateJSON": {}, @@ -111,8 +105,11 @@ }, "id": "postgresql-0cb65170-b909-11e9-a579-f5c0a5d81340", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.10.0" }, + "namespaces": [ + "default" + ], "references": [], "type": "visualization" } \ No newline at end of file diff --git a/packages/postgresql/kibana/visualization/postgresql-147875b0-b903-11e9-a579-f5c0a5d81340.json b/packages/postgresql/kibana/visualization/postgresql-147875b0-b903-11e9-a579-f5c0a5d81340.json index 9517cd4a8b6..7b9552a46b2 100644 --- a/packages/postgresql/kibana/visualization/postgresql-147875b0-b903-11e9-a579-f5c0a5d81340.json +++ b/packages/postgresql/kibana/visualization/postgresql-147875b0-b903-11e9-a579-f5c0a5d81340.json @@ -2,13 +2,7 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } + "searchSourceJSON": {} }, "title": "Query Calls Count [Metrics PostgreSQL]", "uiStateJSON": {}, @@ -94,8 +88,11 @@ }, "id": "postgresql-147875b0-b903-11e9-a579-f5c0a5d81340", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.10.0" }, + "namespaces": [ + "default" + ], "references": [], "type": "visualization" } \ No newline at end of file diff --git a/packages/postgresql/kibana/visualization/postgresql-20931ef0-b79a-11e9-a579-f5c0a5d81340.json b/packages/postgresql/kibana/visualization/postgresql-20931ef0-b79a-11e9-a579-f5c0a5d81340.json index 906523feeb7..9cab195306b 100644 --- a/packages/postgresql/kibana/visualization/postgresql-20931ef0-b79a-11e9-a579-f5c0a5d81340.json +++ b/packages/postgresql/kibana/visualization/postgresql-20931ef0-b79a-11e9-a579-f5c0a5d81340.json @@ -2,13 +2,7 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } + "searchSourceJSON": {} }, "title": "Rows Inserted/Deleted/Updated [Metrics PostgreSQL]", "uiStateJSON": {}, @@ -144,8 +138,11 @@ }, "id": "postgresql-20931ef0-b79a-11e9-a579-f5c0a5d81340", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.10.0" }, + "namespaces": [ + "default" + ], "references": [], "type": "visualization" } \ No newline at end of file diff --git a/packages/postgresql/kibana/visualization/postgresql-3dbd5370-87f3-11e7-ad9c-db80de0bf8d3.json b/packages/postgresql/kibana/visualization/postgresql-3dbd5370-87f3-11e7-ad9c-db80de0bf8d3.json index 8f4f5dab931..0574d5746eb 100644 --- a/packages/postgresql/kibana/visualization/postgresql-3dbd5370-87f3-11e7-ad9c-db80de0bf8d3.json +++ b/packages/postgresql/kibana/visualization/postgresql-3dbd5370-87f3-11e7-ad9c-db80de0bf8d3.json @@ -120,11 +120,14 @@ }, "id": "postgresql-3dbd5370-87f3-11e7-ad9c-db80de0bf8d3", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.10.0" }, + "namespaces": [ + "default" + ], "references": [ { - "id": "postgresql-all-logs", + "id": "postgresql-PostgreSQL All Logs", "name": "search_0", "type": "search" } diff --git a/packages/postgresql/kibana/visualization/postgresql-570973a0-b798-11e9-a579-f5c0a5d81340.json b/packages/postgresql/kibana/visualization/postgresql-570973a0-b798-11e9-a579-f5c0a5d81340.json index 626d265b8dd..6be70b680cd 100644 --- a/packages/postgresql/kibana/visualization/postgresql-570973a0-b798-11e9-a579-f5c0a5d81340.json +++ b/packages/postgresql/kibana/visualization/postgresql-570973a0-b798-11e9-a579-f5c0a5d81340.json @@ -2,13 +2,7 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } + "searchSourceJSON": {} }, "title": "Fileblock IO Rate [Metrics PostgreSQL]", "uiStateJSON": {}, @@ -112,8 +106,11 @@ }, "id": "postgresql-570973a0-b798-11e9-a579-f5c0a5d81340", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.10.0" }, + "namespaces": [ + "default" + ], "references": [], "type": "visualization" } \ No newline at end of file diff --git a/packages/postgresql/kibana/visualization/postgresql-66d67200-b799-11e9-a579-f5c0a5d81340.json b/packages/postgresql/kibana/visualization/postgresql-66d67200-b799-11e9-a579-f5c0a5d81340.json index e41ed373e6b..c34da1aa60c 100644 --- a/packages/postgresql/kibana/visualization/postgresql-66d67200-b799-11e9-a579-f5c0a5d81340.json +++ b/packages/postgresql/kibana/visualization/postgresql-66d67200-b799-11e9-a579-f5c0a5d81340.json @@ -2,13 +2,7 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } + "searchSourceJSON": {} }, "title": "Rows Fetched/Returned [Metrics PostgreSQL]", "uiStateJSON": {}, @@ -144,8 +138,11 @@ }, "id": "postgresql-66d67200-b799-11e9-a579-f5c0a5d81340", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.10.0" }, + "namespaces": [ + "default" + ], "references": [], "type": "visualization" } \ No newline at end of file diff --git a/packages/postgresql/kibana/visualization/postgresql-960ecdf0-b79a-11e9-a579-f5c0a5d81340.json b/packages/postgresql/kibana/visualization/postgresql-960ecdf0-b79a-11e9-a579-f5c0a5d81340.json index 1d8e657bf30..ae893fb1f28 100644 --- a/packages/postgresql/kibana/visualization/postgresql-960ecdf0-b79a-11e9-a579-f5c0a5d81340.json +++ b/packages/postgresql/kibana/visualization/postgresql-960ecdf0-b79a-11e9-a579-f5c0a5d81340.json @@ -2,13 +2,7 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } + "searchSourceJSON": {} }, "title": "Conflict/Deadlock Rates [Metrics PostgreSQL]", "uiStateJSON": {}, @@ -112,8 +106,11 @@ }, "id": "postgresql-960ecdf0-b79a-11e9-a579-f5c0a5d81340", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.10.0" }, + "namespaces": [ + "default" + ], "references": [], "type": "visualization" } \ No newline at end of file diff --git a/packages/postgresql/kibana/visualization/postgresql-98e6b0a0-b79b-11e9-a579-f5c0a5d81340.json b/packages/postgresql/kibana/visualization/postgresql-98e6b0a0-b79b-11e9-a579-f5c0a5d81340.json index d60a5dc7530..2e4416e0df3 100644 --- a/packages/postgresql/kibana/visualization/postgresql-98e6b0a0-b79b-11e9-a579-f5c0a5d81340.json +++ b/packages/postgresql/kibana/visualization/postgresql-98e6b0a0-b79b-11e9-a579-f5c0a5d81340.json @@ -43,8 +43,11 @@ }, "id": "postgresql-98e6b0a0-b79b-11e9-a579-f5c0a5d81340", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.10.0" }, + "namespaces": [ + "default" + ], "references": [ { "id": "metrics-*", diff --git a/packages/postgresql/kibana/visualization/postgresql-PostgreSQL Log Level Count.json b/packages/postgresql/kibana/visualization/postgresql-PostgreSQL Log Level Count.json new file mode 100644 index 00000000000..bfcfbc4e995 --- /dev/null +++ b/packages/postgresql/kibana/visualization/postgresql-PostgreSQL Log Level Count.json @@ -0,0 +1,74 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Log Level Count [Logs PostgreSQL]", + "uiStateJSON": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "log.level", + "order": "desc", + "orderBy": "1", + "size": 12 + }, + "schema": "bucket", + "type": "terms" + } + ], + "params": { + "perPage": 10, + "showMeticsAtAllLevels": false, + "showPartialRows": false, + "showTotal": false, + "sort": { + "columnIndex": null, + "direction": null + }, + "totalFunc": "sum" + }, + "title": "Log Level Count [Logs PostgreSQL]", + "type": "table" + } + }, + "id": "postgresql-PostgreSQL Log Level Count", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "postgresql-PostgreSQL All Logs", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/packages/postgresql/kibana/visualization/postgresql-PostgreSQL Query Count and Duration.json b/packages/postgresql/kibana/visualization/postgresql-PostgreSQL Query Count and Duration.json new file mode 100644 index 00000000000..b868bb1d4a2 --- /dev/null +++ b/packages/postgresql/kibana/visualization/postgresql-PostgreSQL Query Count and Duration.json @@ -0,0 +1,164 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "savedSearchRefName": "search_0", + "title": "Query count and cumulated duration [Logs PostgreSQL]", + "uiStateJSON": { + "vis": { + "colors": { + "Number of queries": "#0A437C", + "Sum of query duration": "#6ED0E0" + }, + "legendOpen": true + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "3", + "params": { + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1 + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Number of queries" + }, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Sum of query duration", + "field": "event.duration" + }, + "schema": "metric", + "type": "sum" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "@timestamp per 3 hours" + }, + "type": "category" + } + ], + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "legendPosition": "right", + "seriesParams": [ + { + "data": { + "id": "4", + "label": "Number of queries" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": true, + "type": "histogram", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "2", + "label": "Sum of query duration" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": true, + "type": "histogram", + "valueAxis": "ValueAxis-1" + } + ], + "times": [], + "type": "histogram", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "" + }, + "type": "value" + } + ] + }, + "title": "Query count and cumulated duration [Logs PostgreSQL]", + "type": "histogram" + } + }, + "id": "postgresql-PostgreSQL Query Count and Duration", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "postgresql-PostgreSQL Query Durations", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/packages/postgresql/kibana/visualization/postgresql-d733c630-b797-11e9-a579-f5c0a5d81340.json b/packages/postgresql/kibana/visualization/postgresql-d733c630-b797-11e9-a579-f5c0a5d81340.json index cb07fcd9b52..2e2c128dfbf 100644 --- a/packages/postgresql/kibana/visualization/postgresql-d733c630-b797-11e9-a579-f5c0a5d81340.json +++ b/packages/postgresql/kibana/visualization/postgresql-d733c630-b797-11e9-a579-f5c0a5d81340.json @@ -2,13 +2,7 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } + "searchSourceJSON": {} }, "title": "Database Transactions [Metrics PostgreSQL]", "uiStateJSON": {}, @@ -112,8 +106,11 @@ }, "id": "postgresql-d733c630-b797-11e9-a579-f5c0a5d81340", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.10.0" }, + "namespaces": [ + "default" + ], "references": [], "type": "visualization" } \ No newline at end of file diff --git a/packages/postgresql/kibana/visualization/postgresql-e2b28ce0-b908-11e9-a579-f5c0a5d81340.json b/packages/postgresql/kibana/visualization/postgresql-e2b28ce0-b908-11e9-a579-f5c0a5d81340.json index 1290f53bdca..49d92b794a7 100644 --- a/packages/postgresql/kibana/visualization/postgresql-e2b28ce0-b908-11e9-a579-f5c0a5d81340.json +++ b/packages/postgresql/kibana/visualization/postgresql-e2b28ce0-b908-11e9-a579-f5c0a5d81340.json @@ -2,13 +2,7 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } + "searchSourceJSON": {} }, "title": "Shared block cache stats [Metrics PostgreSQL]", "uiStateJSON": {}, @@ -109,8 +103,11 @@ }, "id": "postgresql-e2b28ce0-b908-11e9-a579-f5c0a5d81340", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.10.0" }, + "namespaces": [ + "default" + ], "references": [], "type": "visualization" } \ No newline at end of file diff --git a/packages/postgresql/kibana/visualization/postgresql-fbfa67e0-b796-11e9-a579-f5c0a5d81340.json b/packages/postgresql/kibana/visualization/postgresql-fbfa67e0-b796-11e9-a579-f5c0a5d81340.json index 92b7e8b8283..14fdd17ea8f 100644 --- a/packages/postgresql/kibana/visualization/postgresql-fbfa67e0-b796-11e9-a579-f5c0a5d81340.json +++ b/packages/postgresql/kibana/visualization/postgresql-fbfa67e0-b796-11e9-a579-f5c0a5d81340.json @@ -2,13 +2,7 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } + "searchSourceJSON": {} }, "title": "Query Latency [Metrics PostgreSQL]", "uiStateJSON": {}, @@ -85,8 +79,11 @@ }, "id": "postgresql-fbfa67e0-b796-11e9-a579-f5c0a5d81340", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.10.0" }, + "namespaces": [ + "default" + ], "references": [], "type": "visualization" } \ No newline at end of file diff --git a/packages/postgresql/manifest.yml b/packages/postgresql/manifest.yml index 91a238efcfb..961774eca36 100644 --- a/packages/postgresql/manifest.yml +++ b/packages/postgresql/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: postgresql title: PostgreSQL -version: 0.2.6 +version: 0.3.0 license: basic description: PostgreSQL Integration type: integration @@ -9,19 +9,19 @@ categories: - datastore release: experimental conditions: - kibana.version: '^7.9.0' + kibana.version: '^7.13.0' screenshots: - - src: /img/filebeat-postgresql-overview.png - title: Filebeat PostgreSQL overview - size: 1364x655 + - src: /img/logs-overview.png + title: PostgreSQL logs overview + size: 1919x951 type: image/png - - src: /img/filebeat-postgresql-slowlog-overview.png + - src: /img/logs-slowlog-overview.png title: Filebeat PostgreSQL slowlog overview - size: 1363x656 + size: 1918x789 type: image/png - - src: /img/metricbeat-postgresql-overview.png + - src: /img/metrics-overview.png title: Metricbeat PostgreSQL overview - size: 5004x2672 + size: 1919x952 type: image/png icons: - src: /img/logo_postgres.svg