From 9d3ac0d04f2f3f59f0e5613d89d6acac4ba35deb Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 2 Mar 2021 18:53:36 +0100 Subject: [PATCH] Add config --- .../postgresql-10-default.log-config.yml | 2 + .../postgresql-10-default.log-expected.json | 40 ++--- ...l-10-min-duration-statement.log-config.yml | 2 + ...0-min-duration-statement.log-expected.json | 34 ++-- ...ostgresql-11-autovacuum.csv.log-config.yml | 5 + ...gresql-11-autovacuum.csv.log-expected.json | 12 +- ...onnection-disconnection.csv.log-config.yml | 4 + ...ection-disconnection.csv.log-expected.json | 56 +++---- .../postgresql-11-duration.csv.log-config.yml | 2 + ...stgresql-11-duration.csv.log-expected.json | 6 +- .../postgresql-11-ipv6.csv.log-config.yml | 2 + .../postgresql-11-ipv6.csv.log-expected.json | 2 +- ...ostgresql-11-multi-line.csv.log-config.yml | 2 + ...gresql-11-multi-line.csv.log-expected.json | 4 +- ...ostgresql-11-parse-bind.csv.log-config.yml | 2 + ...gresql-11-parse-bind.csv.log-expected.json | 6 +- ...ostgresql-11-start-stop.csv.log-config.yml | 2 + ...gresql-11-start-stop.csv.log-expected.json | 26 +-- .../postgresql-11-tempfile.csv.log-config.yml | 2 + ...stgresql-11-tempfile.csv.log-expected.json | 10 +- .../pipeline/postgresql-11.4.log-config.yml | 2 + .../postgresql-11.4.log-expected.json | 58 +++---- .../postgresql-12-default.log-config.yml | 2 + .../postgresql-12-default.log-expected.json | 20 +-- ...l-12-min-duration-statement.log-config.yml | 2 + ...2-min-duration-statement.log-expected.json | 42 ++--- ...ostgresql-13-error-code.csv.log-config.yml | 2 + ...gresql-13-error-code.csv.log-expected.json | 8 +- .../pipeline/postgresql-13.csv.log-config.yml | 6 + .../postgresql-13.csv.log-expected.json | 60 +++---- ...sql-9.6-debian-with-slowlog.log-config.yml | 2 + ...-9.6-debian-with-slowlog.log-expected.json | 128 +++++++------- .../postgresql-9.6-multi-core.log-config.yml | 2 + ...ostgresql-9.6-multi-core.log-expected.json | 38 ++--- ...ostgresql-9.6-new-timestamp.log-config.yml | 2 + ...gresql-9.6-new-timestamp.log-expected.json | 10 +- ...tgresql-query-steps-slowlog.log-config.yml | 2 + ...esql-query-steps-slowlog.log-expected.json | 6 +- .../postgresql-ubuntu-9.5.log-config.yml | 2 + .../postgresql-ubuntu-9.5.log-expected.json | 156 +++++++++--------- .../postgresql/data_stream/log/fields/ecs.yml | 23 +++ 41 files changed, 433 insertions(+), 361 deletions(-) create mode 100644 packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-10-default.log-config.yml create mode 100644 packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-10-min-duration-statement.log-config.yml create mode 100644 packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-autovacuum.csv.log-config.yml create mode 100644 packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-connection-disconnection.csv.log-config.yml create mode 100644 packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-duration.csv.log-config.yml create mode 100644 packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-ipv6.csv.log-config.yml create mode 100644 packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-multi-line.csv.log-config.yml create mode 100644 packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-parse-bind.csv.log-config.yml create mode 100644 packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-start-stop.csv.log-config.yml create mode 100644 packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-tempfile.csv.log-config.yml create mode 100644 packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11.4.log-config.yml create mode 100644 packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-12-default.log-config.yml create mode 100644 packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-12-min-duration-statement.log-config.yml create mode 100644 packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-13-error-code.csv.log-config.yml create mode 100644 packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-13.csv.log-config.yml create mode 100644 packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-debian-with-slowlog.log-config.yml create mode 100644 packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-multi-core.log-config.yml create mode 100644 packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-new-timestamp.log-config.yml create mode 100644 packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-query-steps-slowlog.log-config.yml create mode 100644 packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-ubuntu-9.5.log-config.yml diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-10-default.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-10-default.log-config.yml new file mode 100644 index 00000000000..c39dc386179 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-10-default.log-config.yml @@ -0,0 +1,2 @@ +dynamic_fields: + event.ingested: ".*" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-10-default.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-10-default.log-expected.json index 1bb7bd1dc7a..9cfbe0047fa 100644 --- a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-10-default.log-expected.json +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-10-default.log-expected.json @@ -15,7 +15,7 @@ }, "message": "database system was shut down at 2020-04-15 12:02:52 CEST", "event": { - "ingested": "2021-03-02T16:28:58.307179014Z", + "ingested": "2021-03-02T18:00:24.895666821Z", "category": [ "database" ], @@ -41,7 +41,7 @@ }, "message": "database system is ready to accept connections", "event": { - "ingested": "2021-03-02T16:28:58.307184002Z", + "ingested": "2021-03-02T18:00:24.895670340Z", "category": [ "database" ], @@ -67,7 +67,7 @@ }, "message": "password authentication failed for user \"root\"", "event": { - "ingested": "2021-03-02T16:28:58.307185422Z", + "ingested": "2021-03-02T18:00:24.895671356Z", "category": [ "database" ], @@ -93,7 +93,7 @@ }, "message": "Role \"root\" does not exist.", "event": { - "ingested": "2021-03-02T16:28:58.307186614Z", + "ingested": "2021-03-02T18:00:24.895677357Z", "category": [ "database" ], @@ -107,7 +107,7 @@ { "message": "\tConnection matched pg_hba.conf line 80: \"local all all md5\"", "event": { - "ingested": "2021-03-02T16:28:58.307187706Z" + "ingested": "2021-03-02T18:00:24.895677996Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\tConnection matched pg_hba.conf line 80: \\\"local all all md5\\\"]" @@ -128,7 +128,7 @@ }, "message": "could not send data to client: Broken pipe", "event": { - "ingested": "2021-03-02T16:28:58.307188841Z", + "ingested": "2021-03-02T18:00:24.895678689Z", "category": [ "database" ], @@ -154,7 +154,7 @@ }, "message": "syntax error at or near \"l\" at character 1", "event": { - "ingested": "2021-03-02T16:28:58.307189891Z", + "ingested": "2021-03-02T18:00:24.895679223Z", "category": [ "database" ], @@ -180,7 +180,7 @@ }, "message": "SELECT al.id, al.tenant_id, al.created_by_id, al.create_ip, al.audit_date, al.audit_table, al.entity_id, al.entity_name, al.reason_for_change, al.audit_log_event_type_id,", "event": { - "ingested": "2021-03-02T16:28:58.307191102Z", + "ingested": "2021-03-02T18:00:24.895679762Z", "category": [ "database" ], @@ -194,7 +194,7 @@ { "message": "\t aet.lookup_code, al.old_value, al.new_value, al.event_crf_id, al.event_crf_version_id, al.study_id, al.study_site_id, ss.rc_oid, al.subject_id, s.unique_identifier,", "event": { - "ingested": "2021-03-02T16:28:58.307192192Z" + "ingested": "2021-03-02T18:00:24.895680287Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t aet.lookup_code, al.old_value, al.new_value, al.event_crf_id, al.event_crf_version_id, al.study_id, al.study_site_id, ss.rc_oid, al.subject_id, s.unique_identifier,]" @@ -203,7 +203,7 @@ { "message": "\t al.study_event_id, sed.name AS studyEventName, al.user_id, al.value_index, al.crf_version_id, al.global_logs, cv.version_name, crf.id AS crfId, crf.name AS crfName", "event": { - "ingested": "2021-03-02T16:28:58.307193302Z" + "ingested": "2021-03-02T18:00:24.895680882Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t al.study_event_id, sed.name AS studyEventName, al.user_id, al.value_index, al.crf_version_id, al.global_logs, cv.version_name, crf.id AS crfId, crf.name AS crfName]" @@ -212,7 +212,7 @@ { "message": "\t FROM public.rc_audit_log_events AS al", "event": { - "ingested": "2021-03-02T16:28:58.307194550Z" + "ingested": "2021-03-02T18:00:24.895681479Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t FROM public.rc_audit_log_events AS al]" @@ -221,7 +221,7 @@ { "message": "\t LEFT JOIN rc_crf_versions AS cv ON cv.id=al.crf_version_id", "event": { - "ingested": "2021-03-02T16:28:58.307195990Z" + "ingested": "2021-03-02T18:00:24.895682274Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t LEFT JOIN rc_crf_versions AS cv ON cv.id=al.crf_version_id]" @@ -230,7 +230,7 @@ { "message": "\t LEFT JOIN rc_crfs AS crf ON crf.id=cv.crf_id", "event": { - "ingested": "2021-03-02T16:28:58.307197190Z" + "ingested": "2021-03-02T18:00:24.895682816Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t LEFT JOIN rc_crfs AS crf ON crf.id=cv.crf_id]" @@ -239,7 +239,7 @@ { "message": "\t LEFT JOIN ad_lookup_codes AS aet ON aet.id=al.audit_log_event_type_id", "event": { - "ingested": "2021-03-02T16:28:58.307198348Z" + "ingested": "2021-03-02T18:00:24.895683352Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t LEFT JOIN ad_lookup_codes AS aet ON aet.id=al.audit_log_event_type_id]" @@ -248,7 +248,7 @@ { "message": "\t LEFT JOIN rc_study_sites AS ss ON ss.id=al.study_site_id", "event": { - "ingested": "2021-03-02T16:28:58.307199396Z" + "ingested": "2021-03-02T18:00:24.895683885Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t LEFT JOIN rc_study_sites AS ss ON ss.id=al.study_site_id]" @@ -257,7 +257,7 @@ { "message": "\t LEFT JOIN rc_subjects AS s ON s.id=al.subject_id", "event": { - "ingested": "2021-03-02T16:28:58.307205864Z" + "ingested": "2021-03-02T18:00:24.895684421Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t LEFT JOIN rc_subjects AS s ON s.id=al.subject_id]" @@ -266,7 +266,7 @@ { "message": "\t LEFT JOIN rc_study_events AS se ON se.id=al.study_event_id", "event": { - "ingested": "2021-03-02T16:28:58.307207269Z" + "ingested": "2021-03-02T18:00:24.895685073Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t LEFT JOIN rc_study_events AS se ON se.id=al.study_event_id]" @@ -275,7 +275,7 @@ { "message": "\t LEFT JOIN rc_study_event_definitions AS sed ON sed.id=se.study_event_definition_id", "event": { - "ingested": "2021-03-02T16:28:58.307208395Z" + "ingested": "2021-03-02T18:00:24.895685603Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t LEFT JOIN rc_study_event_definitions AS sed ON sed.id=se.study_event_definition_id]" @@ -284,7 +284,7 @@ { "message": "\t WHERE al.tenant_id=$1 AND al.study_id=$2 AND aet.lookup_code IN ($3, $4, $5, $6) AND al.audit_date \u003e= $7 ORDER BY al.id DESC limit $8", "event": { - "ingested": "2021-03-02T16:28:58.307209525Z" + "ingested": "2021-03-02T18:00:24.895686138Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t WHERE al.tenant_id=$1 AND al.study_id=$2 AND aet.lookup_code IN ($3, $4, $5, $6) AND al.audit_date \u003e= $7 ORDER BY al.id DESC limit $8]" @@ -293,7 +293,7 @@ { "message": "\t;", "event": { - "ingested": "2021-03-02T16:28:58.307210711Z" + "ingested": "2021-03-02T18:00:24.895686673Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t;]" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-10-min-duration-statement.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-10-min-duration-statement.log-config.yml new file mode 100644 index 00000000000..c39dc386179 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-10-min-duration-statement.log-config.yml @@ -0,0 +1,2 @@ +dynamic_fields: + event.ingested: ".*" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-10-min-duration-statement.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-10-min-duration-statement.log-expected.json index e8d0fce7b94..63b7b5e0993 100644 --- a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-10-min-duration-statement.log-expected.json +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-10-min-duration-statement.log-expected.json @@ -12,7 +12,7 @@ }, "message": "parameters: $1 = '86', $2 = '575', $3 = 'Item Inserted', $4 = 'Item Updated', $5 = 'Subject Updated', $6 = 'Subject Created', $7 = '2019-01-22 00:00:00+00'", "event": { - "ingested": "2021-03-02T16:28:58.746470834Z", + "ingested": "2021-03-02T18:00:24.971645777Z", "category": [ "database" ], @@ -40,7 +40,7 @@ "message": "2019-09-22 06:28:24 UTC LOG: duration: 112.337 ms execute S_59: UPDATE qrtz_TRIGGERS SET TRIGGER_STATE = $1 WHERE SCHED_NAME = 'Scheduler_1' AND TRIGGER_NAME = $2 AND TRIGGER_GROUP = $3 AND TRIGGER_STATE = $4", "event": { "duration": 112337000, - "ingested": "2021-03-02T16:28:58.746504770Z", + "ingested": "2021-03-02T18:00:24.971649364Z", "category": [ "database" ], @@ -63,7 +63,7 @@ }, "message": "parameters: $1 = 'ACQUIRED', $2 = 'surveyInvitation_3Prbn85DiBWe8wHa_158802_77133_1260104', $3 = 'ExecutorsService', $4 = 'WAITING'", "event": { - "ingested": "2021-03-02T16:28:58.746507099Z", + "ingested": "2021-03-02T18:00:24.971650097Z", "category": [ "database" ], @@ -91,7 +91,7 @@ "message": "2019-09-22 06:28:24 UTC LOG: duration: 2474.307 ms execute S_30: SELECT * FROM qrtz_LOCKS WHERE SCHED_NAME = 'Scheduler_1' AND LOCK_NAME = $1 FOR UPDATE", "event": { "duration": 2474306816, - "ingested": "2021-03-02T16:28:58.746510822Z", + "ingested": "2021-03-02T18:00:24.971650668Z", "category": [ "database" ], @@ -114,7 +114,7 @@ }, "message": "parameters: $1 = 'TRIGGER_ACCESS'", "event": { - "ingested": "2021-03-02T16:28:58.746511969Z", + "ingested": "2021-03-02T18:00:24.971651235Z", "category": [ "database" ], @@ -142,7 +142,7 @@ "message": "2019-09-22 06:28:24 UTC LOG: duration: 18.327 ms execute S_32: SELECT al.id, al.tenant_id, al.created_by_id, al.create_ip, al.audit_date, al.audit_table, al.entity_id, al.entity_name, al.reason_for_change, al.audit_log_event_type_id,", "event": { "duration": 18327000, - "ingested": "2021-03-02T16:28:58.746513027Z", + "ingested": "2021-03-02T18:00:24.971651804Z", "category": [ "database" ], @@ -156,7 +156,7 @@ { "message": " aet.lookup_code, al.old_value, al.new_value, al.event_crf_id, al.event_crf_version_id, al.study_id, al.study_site_id, ss.rc_oid, al.subject_id, s.unique_identifier,", "event": { - "ingested": "2021-03-02T16:28:58.746513977Z" + "ingested": "2021-03-02T18:00:24.971652351Z" }, "error": { "message": "Provided Grok expressions do not match field value: [ aet.lookup_code, al.old_value, al.new_value, al.event_crf_id, al.event_crf_version_id, al.study_id, al.study_site_id, ss.rc_oid, al.subject_id, s.unique_identifier,]" @@ -165,7 +165,7 @@ { "message": " al.study_event_id, sed.name AS studyEventName, al.user_id, al.value_index, al.crf_version_id, al.global_logs, cv.version_name, crf.id AS crfId, crf.name AS crfName", "event": { - "ingested": "2021-03-02T16:28:58.746514943Z" + "ingested": "2021-03-02T18:00:24.971652903Z" }, "error": { "message": "Provided Grok expressions do not match field value: [ al.study_event_id, sed.name AS studyEventName, al.user_id, al.value_index, al.crf_version_id, al.global_logs, cv.version_name, crf.id AS crfId, crf.name AS crfName]" @@ -174,7 +174,7 @@ { "message": " FROM public.rc_audit_log_events AS al", "event": { - "ingested": "2021-03-02T16:28:58.746515935Z" + "ingested": "2021-03-02T18:00:24.971653454Z" }, "error": { "message": "Provided Grok expressions do not match field value: [ FROM public.rc_audit_log_events AS al]" @@ -183,7 +183,7 @@ { "message": " LEFT JOIN rc_crf_versions AS cv ON cv.id=al.crf_version_id", "event": { - "ingested": "2021-03-02T16:28:58.746517053Z" + "ingested": "2021-03-02T18:00:24.971654003Z" }, "error": { "message": "Provided Grok expressions do not match field value: [ LEFT JOIN rc_crf_versions AS cv ON cv.id=al.crf_version_id]" @@ -192,7 +192,7 @@ { "message": " LEFT JOIN rc_crfs AS crf ON crf.id=cv.crf_id", "event": { - "ingested": "2021-03-02T16:28:58.746518092Z" + "ingested": "2021-03-02T18:00:24.971654555Z" }, "error": { "message": "Provided Grok expressions do not match field value: [ LEFT JOIN rc_crfs AS crf ON crf.id=cv.crf_id]" @@ -201,7 +201,7 @@ { "message": " LEFT JOIN ad_lookup_codes AS aet ON aet.id=al.audit_log_event_type_id", "event": { - "ingested": "2021-03-02T16:28:58.746519302Z" + "ingested": "2021-03-02T18:00:24.971655262Z" }, "error": { "message": "Provided Grok expressions do not match field value: [ LEFT JOIN ad_lookup_codes AS aet ON aet.id=al.audit_log_event_type_id]" @@ -210,7 +210,7 @@ { "message": " LEFT JOIN rc_study_sites AS ss ON ss.id=al.study_site_id", "event": { - "ingested": "2021-03-02T16:28:58.746520180Z" + "ingested": "2021-03-02T18:00:24.971655826Z" }, "error": { "message": "Provided Grok expressions do not match field value: [ LEFT JOIN rc_study_sites AS ss ON ss.id=al.study_site_id]" @@ -219,7 +219,7 @@ { "message": " LEFT JOIN rc_subjects AS s ON s.id=al.subject_id", "event": { - "ingested": "2021-03-02T16:28:58.746521100Z" + "ingested": "2021-03-02T18:00:24.971656374Z" }, "error": { "message": "Provided Grok expressions do not match field value: [ LEFT JOIN rc_subjects AS s ON s.id=al.subject_id]" @@ -228,7 +228,7 @@ { "message": " LEFT JOIN rc_study_events AS se ON se.id=al.study_event_id", "event": { - "ingested": "2021-03-02T16:28:58.746522025Z" + "ingested": "2021-03-02T18:00:24.971656926Z" }, "error": { "message": "Provided Grok expressions do not match field value: [ LEFT JOIN rc_study_events AS se ON se.id=al.study_event_id]" @@ -237,7 +237,7 @@ { "message": " LEFT JOIN rc_study_event_definitions AS sed ON sed.id=se.study_event_definition_id", "event": { - "ingested": "2021-03-02T16:28:58.746523020Z" + "ingested": "2021-03-02T18:00:24.971657480Z" }, "error": { "message": "Provided Grok expressions do not match field value: [ LEFT JOIN rc_study_event_definitions AS sed ON sed.id=se.study_event_definition_id]" @@ -246,7 +246,7 @@ { "message": " WHERE al.tenant_id=$1 AND al.study_id=$2 AND aet.lookup_code IN ($3, $4, $5, $6) AND al.audit_date \u003e= $7 ORDER BY al.id DESC limit $8", "event": { - "ingested": "2021-03-02T16:28:58.746524337Z" + "ingested": "2021-03-02T18:00:24.971658155Z" }, "error": { "message": "Provided Grok expressions do not match field value: [ WHERE al.tenant_id=$1 AND al.study_id=$2 AND aet.lookup_code IN ($3, $4, $5, $6) AND al.audit_date \u003e= $7 ORDER BY al.id DESC limit $8]" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-autovacuum.csv.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-autovacuum.csv.log-config.yml new file mode 100644 index 00000000000..2029946898a --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-autovacuum.csv.log-config.yml @@ -0,0 +1,5 @@ +numeric_keyword_fields: + - "postgresql.log.transaction_id" + - "process.pid" +dynamic_fields: + event.ingested: ".*" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-autovacuum.csv.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-autovacuum.csv.log-expected.json index d94c2cf157c..fc1fc411759 100644 --- a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-autovacuum.csv.log-expected.json +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-autovacuum.csv.log-expected.json @@ -23,7 +23,7 @@ }, "message": "2021-01-04 00:37:32.862 UTC,,,87,,5ff2634c.57,1,,2021-01-04 00:37:32 UTC,4/43,0,LOG,00000,\"automatic vacuum of table \"\"postgres.public.t\"\": index scans: 1", "event": { - "ingested": "2021-03-02T16:28:58.878393489Z", + "ingested": "2021-03-02T18:00:25.038453613Z", "timezone": "UTC" }, "error": { @@ -34,7 +34,7 @@ { "message": "pages: 0 removed, 89 remain, 0 skipped due to pins, 0 skipped frozen", "event": { - "ingested": "2021-03-02T16:28:58.878398109Z" + "ingested": "2021-03-02T18:00:25.038456088Z" }, "error": { "message": "Provided Grok expressions do not match field value: [pages: 0 removed, 89 remain, 0 skipped due to pins, 0 skipped frozen]" @@ -43,7 +43,7 @@ { "message": "tuples: 10000 removed, 10000 remain, 0 are dead but not yet removable, oldest xmin: 578", "event": { - "ingested": "2021-03-02T16:28:58.878399409Z" + "ingested": "2021-03-02T18:00:25.038456779Z" }, "error": { "message": "Provided Grok expressions do not match field value: [tuples: 10000 removed, 10000 remain, 0 are dead but not yet removable, oldest xmin: 578]" @@ -52,7 +52,7 @@ { "message": "buffer usage: 316 hits, 2 misses, 4 dirtied", "event": { - "ingested": "2021-03-02T16:28:58.878405759Z" + "ingested": "2021-03-02T18:00:25.038457380Z" }, "error": { "message": "Provided Grok expressions do not match field value: [buffer usage: 316 hits, 2 misses, 4 dirtied]" @@ -61,7 +61,7 @@ { "message": "avg read rate: 0.569 MB/s, avg write rate: 1.138 MB/s", "event": { - "ingested": "2021-03-02T16:28:58.878407196Z" + "ingested": "2021-03-02T18:00:25.038457990Z" }, "error": { "message": "Provided Grok expressions do not match field value: [avg read rate: 0.569 MB/s, avg write rate: 1.138 MB/s]" @@ -70,7 +70,7 @@ { "message": "system usage: CPU: user: 0.00 s, system: 0.00 s, elapsed: 0.02 s\",,,,,,,,,\"\"", "event": { - "ingested": "2021-03-02T16:28:58.878408481Z" + "ingested": "2021-03-02T18:00:25.038458594Z" }, "error": { "message": "Provided Grok expressions do not match field value: [system usage: CPU: user: 0.00 s, system: 0.00 s, elapsed: 0.02 s\\\",,,,,,,,,\\\"\\\"]" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-connection-disconnection.csv.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-connection-disconnection.csv.log-config.yml new file mode 100644 index 00000000000..f379ce73ca7 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-connection-disconnection.csv.log-config.yml @@ -0,0 +1,4 @@ +numeric_keyword_fields: + - "process.pid" +dynamic_fields: + event.ingested: ".*" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-connection-disconnection.csv.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-connection-disconnection.csv.log-expected.json index 85eb7d782ac..25e8f4ff18c 100644 --- a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-connection-disconnection.csv.log-expected.json +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-connection-disconnection.csv.log-expected.json @@ -18,7 +18,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:28:58.912002249Z", + "ingested": "2021-03-02T18:00:25.072246452Z", "category": [ "database" ], @@ -48,7 +48,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:28:58.912007487Z", + "ingested": "2021-03-02T18:00:25.072250532Z", "category": [ "database" ], @@ -80,7 +80,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:28:58.912008614Z", + "ingested": "2021-03-02T18:00:25.072251933Z", "category": [ "database" ], @@ -121,7 +121,7 @@ }, "message": "connection authorized: user=postgres database=postgres", "event": { - "ingested": "2021-03-02T16:28:58.912009845Z", + "ingested": "2021-03-02T18:00:25.072253051Z", "category": [ "database" ], @@ -165,7 +165,7 @@ } }, "event": { - "ingested": "2021-03-02T16:28:58.912011285Z", + "ingested": "2021-03-02T18:00:25.072254118Z", "timezone": "UTC" }, "user": { @@ -175,7 +175,7 @@ { "message": "LIMIT 1000\",,,,,,,,,\"psql\"", "event": { - "ingested": "2021-03-02T16:28:58.912012512Z" + "ingested": "2021-03-02T18:00:25.072255307Z" }, "error": { "message": "Provided Grok expressions do not match field value: [LIMIT 1000\\\",,,,,,,,,\\\"psql\\\"]" @@ -212,7 +212,7 @@ }, "message": "ALTER SYSTEM SET log_connections = on;", "event": { - "ingested": "2021-03-02T16:28:58.912013475Z", + "ingested": "2021-03-02T18:00:25.072256417Z", "category": [ "database" ], @@ -257,7 +257,7 @@ }, "message": "ALTER SYSTEM SET log_disconnections = on;", "event": { - "ingested": "2021-03-02T16:28:58.912014667Z", + "ingested": "2021-03-02T18:00:25.072257497Z", "category": [ "database" ], @@ -301,7 +301,7 @@ } }, "event": { - "ingested": "2021-03-02T16:28:58.912015936Z", + "ingested": "2021-03-02T18:00:25.072258684Z", "timezone": "UTC" }, "user": { @@ -311,7 +311,7 @@ { "message": "LIMIT 1000\",,,,,,,,,\"psql\"", "event": { - "ingested": "2021-03-02T16:28:58.912017146Z" + "ingested": "2021-03-02T18:00:25.072259777Z" }, "error": { "message": "Provided Grok expressions do not match field value: [LIMIT 1000\\\",,,,,,,,,\\\"psql\\\"]" @@ -347,7 +347,7 @@ } }, "event": { - "ingested": "2021-03-02T16:28:58.912018243Z", + "ingested": "2021-03-02T18:00:25.072260813Z", "timezone": "UTC" }, "user": { @@ -357,7 +357,7 @@ { "message": "LIMIT 1000\",,,,,,,,,\"psql\"", "event": { - "ingested": "2021-03-02T16:28:58.912019650Z" + "ingested": "2021-03-02T18:00:25.072262219Z" }, "error": { "message": "Provided Grok expressions do not match field value: [LIMIT 1000\\\",,,,,,,,,\\\"psql\\\"]" @@ -393,7 +393,7 @@ } }, "event": { - "ingested": "2021-03-02T16:28:58.912020802Z", + "ingested": "2021-03-02T18:00:25.072263312Z", "timezone": "UTC" }, "user": { @@ -403,7 +403,7 @@ { "message": "LIMIT 1000\",,,,,,,,,\"psql\"", "event": { - "ingested": "2021-03-02T16:28:58.912021903Z" + "ingested": "2021-03-02T18:00:25.072264465Z" }, "error": { "message": "Provided Grok expressions do not match field value: [LIMIT 1000\\\",,,,,,,,,\\\"psql\\\"]" @@ -440,7 +440,7 @@ }, "message": "alter system set log_min_duration_statement = 0;", "event": { - "ingested": "2021-03-02T16:28:58.912023232Z", + "ingested": "2021-03-02T18:00:25.072265591Z", "category": [ "database" ], @@ -483,7 +483,7 @@ }, "message": "disconnection: session time: 0:00:45.407 user=postgres database=postgres host=172.24.0.1 port=42304", "event": { - "ingested": "2021-03-02T16:28:58.912024450Z", + "ingested": "2021-03-02T18:00:25.072266627Z", "category": [ "database" ], @@ -517,7 +517,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:28:58.912025729Z", + "ingested": "2021-03-02T18:00:25.072267904Z", "category": [ "database" ], @@ -560,7 +560,7 @@ } }, "event": { - "ingested": "2021-03-02T16:28:58.912026974Z", + "ingested": "2021-03-02T18:00:25.072268986Z", "timezone": "UTC" }, "user": { @@ -570,7 +570,7 @@ { "message": "Connection matched pg_hba.conf line 95: \"\"host all all all md5\"\"\",,,,,,,,\"\"", "event": { - "ingested": "2021-03-02T16:28:58.912028157Z" + "ingested": "2021-03-02T18:00:25.072270051Z" }, "error": { "message": "Provided Grok expressions do not match field value: [Connection matched pg_hba.conf line 95: \\\"\\\"host all all all md5\\\"\\\"\\\",,,,,,,,\\\"\\\"]" @@ -596,7 +596,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:28:58.912029289Z", + "ingested": "2021-03-02T18:00:25.072271177Z", "category": [ "database" ], @@ -639,7 +639,7 @@ } }, "event": { - "ingested": "2021-03-02T16:28:58.912030553Z", + "ingested": "2021-03-02T18:00:25.072272279Z", "timezone": "UTC" }, "user": { @@ -649,7 +649,7 @@ { "message": "Connection matched pg_hba.conf line 95: \"\"host all all all md5\"\"\",,,,,,,,\"\"", "event": { - "ingested": "2021-03-02T16:28:58.912031772Z" + "ingested": "2021-03-02T18:00:25.072273270Z" }, "error": { "message": "Provided Grok expressions do not match field value: [Connection matched pg_hba.conf line 95: \\\"\\\"host all all all md5\\\"\\\"\\\",,,,,,,,\\\"\\\"]" @@ -675,7 +675,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:28:58.912032986Z", + "ingested": "2021-03-02T18:00:25.072274257Z", "category": [ "database" ], @@ -718,7 +718,7 @@ } }, "event": { - "ingested": "2021-03-02T16:28:58.912034255Z", + "ingested": "2021-03-02T18:00:25.072275543Z", "timezone": "UTC" }, "user": { @@ -728,7 +728,7 @@ { "message": "Connection matched pg_hba.conf line 95: \"\"host all all all md5\"\"\",,,,,,,,\"\"", "event": { - "ingested": "2021-03-02T16:28:58.912035339Z" + "ingested": "2021-03-02T18:00:25.072276890Z" }, "error": { "message": "Provided Grok expressions do not match field value: [Connection matched pg_hba.conf line 95: \\\"\\\"host all all all md5\\\"\\\"\\\",,,,,,,,\\\"\\\"]" @@ -754,7 +754,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:28:58.912036850Z", + "ingested": "2021-03-02T18:00:25.072278077Z", "category": [ "database" ], @@ -797,7 +797,7 @@ } }, "event": { - "ingested": "2021-03-02T16:28:58.912037783Z", + "ingested": "2021-03-02T18:00:25.072279146Z", "timezone": "UTC" }, "user": { @@ -807,7 +807,7 @@ { "message": "Connection matched pg_hba.conf line 95: \"\"host all all all md5\"\"\",,,,,,,,\"\"", "event": { - "ingested": "2021-03-02T16:28:58.912038944Z" + "ingested": "2021-03-02T18:00:25.072280316Z" }, "error": { "message": "Provided Grok expressions do not match field value: [Connection matched pg_hba.conf line 95: \\\"\\\"host all all all md5\\\"\\\"\\\",,,,,,,,\\\"\\\"]" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-duration.csv.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-duration.csv.log-config.yml new file mode 100644 index 00000000000..c39dc386179 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-duration.csv.log-config.yml @@ -0,0 +1,2 @@ +dynamic_fields: + event.ingested: ".*" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-duration.csv.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-duration.csv.log-expected.json index 243cfafd797..ceef024b104 100644 --- a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-duration.csv.log-expected.json +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-duration.csv.log-expected.json @@ -31,7 +31,7 @@ }, "event": { "duration": 6589000, - "ingested": "2021-03-02T16:28:59.431382943Z", + "ingested": "2021-03-02T18:00:25.456799154Z", "category": [ "database" ], @@ -76,7 +76,7 @@ }, "message": "select pg_reload_conf();", "event": { - "ingested": "2021-03-02T16:28:59.431385985Z", + "ingested": "2021-03-02T18:00:25.456801712Z", "category": [ "database" ], @@ -123,7 +123,7 @@ "message": "select generate_series(1, 1000000);", "event": { "duration": 148472000, - "ingested": "2021-03-02T16:28:59.431392025Z", + "ingested": "2021-03-02T18:00:25.456802353Z", "category": [ "database" ], diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-ipv6.csv.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-ipv6.csv.log-config.yml new file mode 100644 index 00000000000..c39dc386179 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-ipv6.csv.log-config.yml @@ -0,0 +1,2 @@ +dynamic_fields: + event.ingested: ".*" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-ipv6.csv.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-ipv6.csv.log-expected.json index 4abdf6cf269..c289035bc1c 100644 --- a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-ipv6.csv.log-expected.json +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-ipv6.csv.log-expected.json @@ -29,7 +29,7 @@ }, "message": "role \"jose.villanova\" does not exist", "event": { - "ingested": "2021-03-02T16:28:59.496697968Z", + "ingested": "2021-03-02T18:00:25.503565473Z", "category": [ "database" ], diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-multi-line.csv.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-multi-line.csv.log-config.yml new file mode 100644 index 00000000000..c39dc386179 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-multi-line.csv.log-config.yml @@ -0,0 +1,2 @@ +dynamic_fields: + event.ingested: ".*" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-multi-line.csv.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-multi-line.csv.log-expected.json index 864087c0dc7..c051c5b0f0c 100644 --- a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-multi-line.csv.log-expected.json +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-multi-line.csv.log-expected.json @@ -30,7 +30,7 @@ } }, "event": { - "ingested": "2021-03-02T16:28:59.522269536Z", + "ingested": "2021-03-02T18:00:25.519477605Z", "timezone": "UTC" }, "user": { @@ -40,7 +40,7 @@ { "message": "'line';\",,,,,,,,,\"psql\"", "event": { - "ingested": "2021-03-02T16:28:59.522274305Z" + "ingested": "2021-03-02T18:00:25.519480103Z" }, "error": { "message": "Provided Grok expressions do not match field value: ['line';\\\",,,,,,,,,\\\"psql\\\"]" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-parse-bind.csv.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-parse-bind.csv.log-config.yml new file mode 100644 index 00000000000..c39dc386179 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-parse-bind.csv.log-config.yml @@ -0,0 +1,2 @@ +dynamic_fields: + event.ingested: ".*" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-parse-bind.csv.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-parse-bind.csv.log-expected.json index b727a03df19..15d0644662b 100644 --- a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-parse-bind.csv.log-expected.json +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-parse-bind.csv.log-expected.json @@ -33,7 +33,7 @@ "message": "SELECT * from information_schema.tables WHERE table_name = $1", "event": { "duration": 7775000, - "ingested": "2021-03-02T16:28:59.619146364Z", + "ingested": "2021-03-02T18:00:25.536033883Z", "category": [ "database" ], @@ -81,7 +81,7 @@ "message": "SELECT * from information_schema.tables WHERE table_name = $1", "event": { "duration": 4091000, - "ingested": "2021-03-02T16:28:59.619151466Z", + "ingested": "2021-03-02T18:00:25.536036355Z", "category": [ "database" ], @@ -129,7 +129,7 @@ "message": "SELECT * from information_schema.tables WHERE table_name = $1", "event": { "duration": 455000, - "ingested": "2021-03-02T16:28:59.619152926Z", + "ingested": "2021-03-02T18:00:25.536036986Z", "category": [ "database" ], diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-start-stop.csv.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-start-stop.csv.log-config.yml new file mode 100644 index 00000000000..c39dc386179 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-start-stop.csv.log-config.yml @@ -0,0 +1,2 @@ +dynamic_fields: + event.ingested: ".*" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-start-stop.csv.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-start-stop.csv.log-expected.json index 221200450c5..32d08b9609a 100644 --- a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-start-stop.csv.log-expected.json +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-start-stop.csv.log-expected.json @@ -18,7 +18,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:28:59.999153986Z", + "ingested": "2021-03-02T18:00:25.581996715Z", "category": [ "database" ], @@ -48,7 +48,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:28:59.999162929Z", + "ingested": "2021-03-02T18:00:25.581999140Z", "category": [ "database" ], @@ -80,7 +80,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:28:59.999163895Z", + "ingested": "2021-03-02T18:00:25.581999755Z", "category": [ "database" ], @@ -112,7 +112,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:28:59.999165511Z", + "ingested": "2021-03-02T18:00:25.582000291Z", "category": [ "database" ], @@ -153,7 +153,7 @@ }, "message": "connection authorized: user=postgres database=postgres", "event": { - "ingested": "2021-03-02T16:28:59.999166833Z", + "ingested": "2021-03-02T18:00:25.582000832Z", "category": [ "database" ], @@ -198,7 +198,7 @@ }, "message": "SELECT 1;", "event": { - "ingested": "2021-03-02T16:28:59.999167852Z", + "ingested": "2021-03-02T18:00:25.582001372Z", "category": [ "database" ], @@ -241,7 +241,7 @@ }, "message": "disconnection: session time: 0:00:05.023 user=postgres database=postgres host=172.24.0.1 port=38356", "event": { - "ingested": "2021-03-02T16:28:59.999168788Z", + "ingested": "2021-03-02T18:00:25.582001904Z", "category": [ "database" ], @@ -273,7 +273,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:28:59.999169863Z", + "ingested": "2021-03-02T18:00:25.582002437Z", "category": [ "database" ], @@ -303,7 +303,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:28:59.999170947Z", + "ingested": "2021-03-02T18:00:25.582002968Z", "category": [ "database" ], @@ -333,7 +333,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:28:59.999172182Z", + "ingested": "2021-03-02T18:00:25.582003505Z", "category": [ "database" ], @@ -363,7 +363,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:28:59.999173388Z", + "ingested": "2021-03-02T18:00:25.582004041Z", "category": [ "database" ], @@ -393,7 +393,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:28:59.999175850Z", + "ingested": "2021-03-02T18:00:25.582004743Z", "category": [ "database" ], @@ -423,7 +423,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:28:59.999176861Z", + "ingested": "2021-03-02T18:00:25.582005289Z", "category": [ "database" ], diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-tempfile.csv.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-tempfile.csv.log-config.yml new file mode 100644 index 00000000000..c39dc386179 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-tempfile.csv.log-config.yml @@ -0,0 +1,2 @@ +dynamic_fields: + event.ingested: ".*" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-tempfile.csv.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-tempfile.csv.log-expected.json index e4f484b54dc..e7ae5ef8f92 100644 --- a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-tempfile.csv.log-expected.json +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-tempfile.csv.log-expected.json @@ -33,7 +33,7 @@ "message": "set log_temp_files = 0;", "event": { "duration": 189000, - "ingested": "2021-03-02T16:29:00.228280143Z", + "ingested": "2021-03-02T18:00:25.912935720Z", "category": [ "database" ], @@ -78,7 +78,7 @@ }, "message": "temporary file: path \"base/pgsql_tmp/pgsql_tmp81.3\", size 162381824", "event": { - "ingested": "2021-03-02T16:29:00.228283646Z", + "ingested": "2021-03-02T18:00:25.912941046Z", "category": [ "database" ], @@ -123,7 +123,7 @@ }, "message": "temporary file: path \"base/pgsql_tmp/pgsql_tmp81.1\", size 42000", "event": { - "ingested": "2021-03-02T16:29:00.228284387Z", + "ingested": "2021-03-02T18:00:25.912942338Z", "category": [ "database" ], @@ -168,7 +168,7 @@ }, "message": "temporary file: path \"base/pgsql_tmp/pgsql_tmp81.2\", size 42000", "event": { - "ingested": "2021-03-02T16:29:00.228285118Z", + "ingested": "2021-03-02T18:00:25.912943353Z", "category": [ "database" ], @@ -215,7 +215,7 @@ "message": "select * from generate_series(1, 3000) as t1(a), generate_series(1, 3000) as t2(a) order by 1 desc, 2;", "event": { "duration": 6921284096, - "ingested": "2021-03-02T16:29:00.228285754Z", + "ingested": "2021-03-02T18:00:25.912944447Z", "category": [ "database" ], diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11.4.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11.4.log-config.yml new file mode 100644 index 00000000000..c39dc386179 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11.4.log-config.yml @@ -0,0 +1,2 @@ +dynamic_fields: + event.ingested: ".*" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11.4.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11.4.log-expected.json index 44a18fe8893..d81b6b6bc74 100644 --- a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11.4.log-expected.json +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-11.4.log-expected.json @@ -16,7 +16,7 @@ }, "message": "database system was interrupted; last known up at 2019-07-23 12:03:20 UTC", "event": { - "ingested": "2021-03-02T16:29:00.481255656Z", + "ingested": "2021-03-02T18:00:26.065569136Z", "category": [ "database" ], @@ -42,7 +42,7 @@ }, "message": "StartupXLOG, xlog.c:6388", "event": { - "ingested": "2021-03-02T16:29:00.481260442Z", + "ingested": "2021-03-02T18:00:26.065573856Z", "category": [ "database" ], @@ -69,7 +69,7 @@ }, "message": "database system was not properly shut down; automatic recovery in progress", "event": { - "ingested": "2021-03-02T16:29:00.481262192Z", + "ingested": "2021-03-02T18:00:26.065575013Z", "category": [ "database" ], @@ -95,7 +95,7 @@ }, "message": "StartupXLOG, xlog.c:6879", "event": { - "ingested": "2021-03-02T16:29:00.481263296Z", + "ingested": "2021-03-02T18:00:26.065575947Z", "category": [ "database" ], @@ -122,7 +122,7 @@ }, "message": "redo starts at 0/1651398", "event": { - "ingested": "2021-03-02T16:29:00.481264456Z", + "ingested": "2021-03-02T18:00:26.065576812Z", "category": [ "database" ], @@ -148,7 +148,7 @@ }, "message": "StartupXLOG, xlog.c:7150", "event": { - "ingested": "2021-03-02T16:29:00.481265570Z", + "ingested": "2021-03-02T18:00:26.065577731Z", "category": [ "database" ], @@ -175,7 +175,7 @@ }, "message": "invalid record length at 0/16513D0: wanted 24, got 0", "event": { - "ingested": "2021-03-02T16:29:00.481266619Z", + "ingested": "2021-03-02T18:00:26.065578754Z", "category": [ "database" ], @@ -201,7 +201,7 @@ }, "message": "ReadRecord, xlog.c:4233", "event": { - "ingested": "2021-03-02T16:29:00.481267598Z", + "ingested": "2021-03-02T18:00:26.065579637Z", "category": [ "database" ], @@ -228,7 +228,7 @@ }, "message": "redo done at 0/1651398", "event": { - "ingested": "2021-03-02T16:29:00.481268551Z", + "ingested": "2021-03-02T18:00:26.065580455Z", "category": [ "database" ], @@ -254,7 +254,7 @@ }, "message": "StartupXLOG, xlog.c:7422", "event": { - "ingested": "2021-03-02T16:29:00.481269503Z", + "ingested": "2021-03-02T18:00:26.065581557Z", "category": [ "database" ], @@ -281,7 +281,7 @@ }, "message": "database system is ready to accept connections", "event": { - "ingested": "2021-03-02T16:29:00.481270456Z", + "ingested": "2021-03-02T18:00:26.065582502Z", "category": [ "database" ], @@ -307,7 +307,7 @@ }, "message": "reaper, postmaster.c:2889", "event": { - "ingested": "2021-03-02T16:29:00.481271722Z", + "ingested": "2021-03-02T18:00:26.065583761Z", "category": [ "database" ], @@ -334,7 +334,7 @@ }, "message": "connection received: host=[local]", "event": { - "ingested": "2021-03-02T16:29:00.481272694Z", + "ingested": "2021-03-02T18:00:26.065584773Z", "category": [ "database" ], @@ -360,7 +360,7 @@ }, "message": "BackendInitialize, postmaster.c:4206", "event": { - "ingested": "2021-03-02T16:29:00.481273642Z", + "ingested": "2021-03-02T18:00:26.065585802Z", "category": [ "database" ], @@ -387,7 +387,7 @@ }, "message": "connection authorized: user=postgres database=postgres", "event": { - "ingested": "2021-03-02T16:29:00.481274614Z", + "ingested": "2021-03-02T18:00:26.065586737Z", "category": [ "database" ], @@ -413,7 +413,7 @@ }, "message": "PerformAuthentication, postinit.c:279", "event": { - "ingested": "2021-03-02T16:29:00.481275574Z", + "ingested": "2021-03-02T18:00:26.065587633Z", "category": [ "database" ], @@ -440,7 +440,7 @@ }, "message": "statement: show config_filel;", "event": { - "ingested": "2021-03-02T16:29:00.481276658Z", + "ingested": "2021-03-02T18:00:26.065588360Z", "category": [ "database" ], @@ -466,7 +466,7 @@ }, "message": "exec_simple_query, postgres.c:975", "event": { - "ingested": "2021-03-02T16:29:00.481277701Z", + "ingested": "2021-03-02T18:00:26.065588964Z", "category": [ "database" ], @@ -493,7 +493,7 @@ }, "message": "unrecognized configuration parameter \"config_filel\"", "event": { - "ingested": "2021-03-02T16:29:00.481278789Z", + "ingested": "2021-03-02T18:00:26.065589517Z", "category": [ "database" ], @@ -519,7 +519,7 @@ }, "message": "GetConfigOptionByName, guc.c:8342", "event": { - "ingested": "2021-03-02T16:29:00.481280096Z", + "ingested": "2021-03-02T18:00:26.065590069Z", "category": [ "database" ], @@ -545,7 +545,7 @@ }, "message": "show config_filel;", "event": { - "ingested": "2021-03-02T16:29:00.481281148Z", + "ingested": "2021-03-02T18:00:26.065590813Z", "category": [ "database" ], @@ -572,7 +572,7 @@ }, "message": "statement: show config_file;", "event": { - "ingested": "2021-03-02T16:29:00.481282457Z", + "ingested": "2021-03-02T18:00:26.065591692Z", "category": [ "database" ], @@ -598,7 +598,7 @@ }, "message": "exec_simple_query, postgres.c:975", "event": { - "ingested": "2021-03-02T16:29:00.481283442Z", + "ingested": "2021-03-02T18:00:26.065592508Z", "category": [ "database" ], @@ -625,7 +625,7 @@ }, "message": "duration: 0.524 ms", "event": { - "ingested": "2021-03-02T16:29:00.481284490Z", + "ingested": "2021-03-02T18:00:26.065593620Z", "category": [ "database" ], @@ -651,7 +651,7 @@ }, "message": "exec_simple_query, postgres.c:1219", "event": { - "ingested": "2021-03-02T16:29:00.481285471Z", + "ingested": "2021-03-02T18:00:26.065596577Z", "category": [ "database" ], @@ -678,7 +678,7 @@ }, "message": "statement: SELECT * FROM pg_catalog.pg_tables;", "event": { - "ingested": "2021-03-02T16:29:00.481286501Z", + "ingested": "2021-03-02T18:00:26.065597817Z", "category": [ "database" ], @@ -704,7 +704,7 @@ }, "message": "exec_simple_query, postgres.c:975", "event": { - "ingested": "2021-03-02T16:29:00.481287583Z", + "ingested": "2021-03-02T18:00:26.065598648Z", "category": [ "database" ], @@ -731,7 +731,7 @@ }, "message": "duration: 2.139 ms", "event": { - "ingested": "2021-03-02T16:29:00.481288788Z", + "ingested": "2021-03-02T18:00:26.065599835Z", "category": [ "database" ], @@ -757,7 +757,7 @@ }, "message": "exec_simple_query, postgres.c:1219", "event": { - "ingested": "2021-03-02T16:29:00.481290013Z", + "ingested": "2021-03-02T18:00:26.065601162Z", "category": [ "database" ], diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-12-default.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-12-default.log-config.yml new file mode 100644 index 00000000000..c39dc386179 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-12-default.log-config.yml @@ -0,0 +1,2 @@ +dynamic_fields: + event.ingested: ".*" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-12-default.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-12-default.log-expected.json index 6411e5c967a..0f05b750b9a 100644 --- a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-12-default.log-expected.json +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-12-default.log-expected.json @@ -15,7 +15,7 @@ }, "message": "starting PostgreSQL 12.2 on x86_64-pc-linux-gnu, compiled by gcc (Arch Linux 9.3.0-1) 9.3.0, 64-bit", "event": { - "ingested": "2021-03-02T16:29:00.974844851Z", + "ingested": "2021-03-02T18:00:26.343292646Z", "category": [ "database" ], @@ -41,7 +41,7 @@ }, "message": "listening on IPv6 address \"::1\", port 5432", "event": { - "ingested": "2021-03-02T16:29:00.974850411Z", + "ingested": "2021-03-02T18:00:26.343296821Z", "category": [ "database" ], @@ -67,7 +67,7 @@ }, "message": "listening on IPv4 address \"127.0.0.1\", port 5432", "event": { - "ingested": "2021-03-02T16:29:00.974851675Z", + "ingested": "2021-03-02T18:00:26.343298226Z", "category": [ "database" ], @@ -93,7 +93,7 @@ }, "message": "listening on Unix socket \"/tmp/.s.PGSQL.5432\"", "event": { - "ingested": "2021-03-02T16:29:00.974852862Z", + "ingested": "2021-03-02T18:00:26.343299464Z", "category": [ "database" ], @@ -119,7 +119,7 @@ }, "message": "database system was shut down at 2020-04-16 11:45:01 CEST", "event": { - "ingested": "2021-03-02T16:29:00.974854287Z", + "ingested": "2021-03-02T18:00:26.343300741Z", "category": [ "database" ], @@ -145,7 +145,7 @@ }, "message": "database system is ready to accept connections", "event": { - "ingested": "2021-03-02T16:29:00.974855502Z", + "ingested": "2021-03-02T18:00:26.343301902Z", "category": [ "database" ], @@ -171,7 +171,7 @@ }, "message": "received smart shutdown request", "event": { - "ingested": "2021-03-02T16:29:00.974858388Z", + "ingested": "2021-03-02T18:00:26.343302887Z", "category": [ "database" ], @@ -197,7 +197,7 @@ }, "message": "background worker \"logical replication launcher\" (PID 26960) exited with exit code 1", "event": { - "ingested": "2021-03-02T16:29:00.974860222Z", + "ingested": "2021-03-02T18:00:26.343303799Z", "category": [ "database" ], @@ -223,7 +223,7 @@ }, "message": "shutting down", "event": { - "ingested": "2021-03-02T16:29:00.974861291Z", + "ingested": "2021-03-02T18:00:26.343304806Z", "category": [ "database" ], @@ -249,7 +249,7 @@ }, "message": "database system is shut down", "event": { - "ingested": "2021-03-02T16:29:00.974862153Z", + "ingested": "2021-03-02T18:00:26.343305757Z", "category": [ "database" ], diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-12-min-duration-statement.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-12-min-duration-statement.log-config.yml new file mode 100644 index 00000000000..c39dc386179 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-12-min-duration-statement.log-config.yml @@ -0,0 +1,2 @@ +dynamic_fields: + event.ingested: ".*" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-12-min-duration-statement.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-12-min-duration-statement.log-expected.json index 05cab1ac570..465861e4336 100644 --- a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-12-min-duration-statement.log-expected.json +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-12-min-duration-statement.log-expected.json @@ -19,7 +19,7 @@ "message": "2020-04-16 12:48:36.677 CEST [34492] LOG: duration: 0.327 ms statement: select 1;", "event": { "duration": 327000, - "ingested": "2021-03-02T16:29:01.258261418Z", + "ingested": "2021-03-02T18:00:26.507648394Z", "category": [ "database" ], @@ -49,7 +49,7 @@ "message": "2020-04-16 12:48:40.316 CEST [34492] LOG: duration: 0.320 ms statement: select version();", "event": { "duration": 320000, - "ingested": "2021-03-02T16:29:01.258264776Z", + "ingested": "2021-03-02T18:00:26.507650924Z", "category": [ "database" ], @@ -75,7 +75,7 @@ }, "message": "column \"name\" does not exist at character 8", "event": { - "ingested": "2021-03-02T16:29:01.258265487Z", + "ingested": "2021-03-02T18:00:26.507651573Z", "category": [ "database" ], @@ -101,7 +101,7 @@ }, "message": "select name from user;", "event": { - "ingested": "2021-03-02T16:29:01.258266069Z", + "ingested": "2021-03-02T18:00:26.507652112Z", "category": [ "database" ], @@ -131,7 +131,7 @@ "message": "2020-04-16 12:49:16.871 CEST [34492] LOG: duration: 3.431 ms statement: CREATE TABLE weather (", "event": { "duration": 3431000, - "ingested": "2021-03-02T16:29:01.258266649Z", + "ingested": "2021-03-02T18:00:26.507652645Z", "category": [ "database" ], @@ -145,7 +145,7 @@ { "message": "\t city varchar(80),", "event": { - "ingested": "2021-03-02T16:29:01.258267235Z" + "ingested": "2021-03-02T18:00:26.507653183Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t city varchar(80),]" @@ -154,7 +154,7 @@ { "message": "\t temp_lo int,", "event": { - "ingested": "2021-03-02T16:29:01.258267817Z" + "ingested": "2021-03-02T18:00:26.507653717Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t temp_lo int,]" @@ -163,7 +163,7 @@ { "message": "\t temp_hi int,", "event": { - "ingested": "2021-03-02T16:29:01.258268403Z" + "ingested": "2021-03-02T18:00:26.507654260Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t temp_hi int,]" @@ -172,7 +172,7 @@ { "message": "\t prcp real,", "event": { - "ingested": "2021-03-02T16:29:01.258268990Z" + "ingested": "2021-03-02T18:00:26.507654791Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t prcp real,]" @@ -181,7 +181,7 @@ { "message": "\t date date", "event": { - "ingested": "2021-03-02T16:29:01.258269581Z" + "ingested": "2021-03-02T18:00:26.507655325Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t date date]" @@ -190,7 +190,7 @@ { "message": "\t);", "event": { - "ingested": "2021-03-02T16:29:01.258270167Z" + "ingested": "2021-03-02T18:00:26.507655861Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t);]" @@ -211,7 +211,7 @@ }, "message": "relation \"weather\" already exists", "event": { - "ingested": "2021-03-02T16:29:01.258270904Z", + "ingested": "2021-03-02T18:00:26.507656537Z", "category": [ "database" ], @@ -241,7 +241,7 @@ "message": "2020-04-16 12:49:54.907 CEST [34492] LOG: duration: 3.039 ms statement: SELECT pg_catalog.quote_ident(c.relname) FROM pg_catalog.pg_class c WHERE c.relkind IN ('r', 'S', 'v', 'm', 'f', 'p') AND substring(pg_catalog.quote_ident(c.relname),1,2)='we' AND pg_catalog.pg_table_is_visible(c.oid) AND c.relnamespace \u003c\u003e (SELECT oid FROM pg_catalog.pg_namespace WHERE nspname = 'pg_catalog')", "event": { "duration": 3039000, - "ingested": "2021-03-02T16:29:01.258271490Z", + "ingested": "2021-03-02T18:00:26.507657073Z", "category": [ "database" ], @@ -255,7 +255,7 @@ { "message": "\tUNION", "event": { - "ingested": "2021-03-02T16:29:01.258272080Z" + "ingested": "2021-03-02T18:00:26.507657697Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\tUNION]" @@ -264,7 +264,7 @@ { "message": "\tSELECT pg_catalog.quote_ident(n.nspname) || '.' FROM pg_catalog.pg_namespace n WHERE substring(pg_catalog.quote_ident(n.nspname) || '.',1,2)='we' AND (SELECT pg_catalog.count(*) FROM pg_catalog.pg_namespace WHERE substring(pg_catalog.quote_ident(nspname) || '.',1,2) = substring('we',1,pg_catalog.length(pg_catalog.quote_ident(nspname))+1)) \u003e 1", "event": { - "ingested": "2021-03-02T16:29:01.258272663Z" + "ingested": "2021-03-02T18:00:26.507658299Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\tSELECT pg_catalog.quote_ident(n.nspname) || '.' FROM pg_catalog.pg_namespace n WHERE substring(pg_catalog.quote_ident(n.nspname) || '.',1,2)='we' AND (SELECT pg_catalog.count(*) FROM pg_catalog.pg_namespace WHERE substring(pg_catalog.quote_ident(nspname) || '.',1,2) = substring('we',1,pg_catalog.length(pg_catalog.quote_ident(nspname))+1)) \u003e 1]" @@ -273,7 +273,7 @@ { "message": "\tUNION", "event": { - "ingested": "2021-03-02T16:29:01.258273251Z" + "ingested": "2021-03-02T18:00:26.507658834Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\tUNION]" @@ -282,7 +282,7 @@ { "message": "\tSELECT pg_catalog.quote_ident(n.nspname) || '.' || pg_catalog.quote_ident(c.relname) FROM pg_catalog.pg_class c, pg_catalog.pg_namespace n WHERE c.relnamespace = n.oid AND c.relkind IN ('r', 'S', 'v', 'm', 'f', 'p') AND substring(pg_catalog.quote_ident(n.nspname) || '.' || pg_catalog.quote_ident(c.relname),1,2)='we' AND substring(pg_catalog.quote_ident(n.nspname) || '.',1,2) = substring('we',1,pg_catalog.length(pg_catalog.quote_ident(n.nspname))+1) AND (SELECT pg_catalog.count(*) FROM pg_catalog.pg_namespace WHERE substring(pg_catalog.quote_ident(nspname) || '.',1,2) = substring('we',1,pg_catalog.length(pg_catalog.quote_ident(nspname))+1)) = 1", "event": { - "ingested": "2021-03-02T16:29:01.258273940Z" + "ingested": "2021-03-02T18:00:26.507659493Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\tSELECT pg_catalog.quote_ident(n.nspname) || '.' || pg_catalog.quote_ident(c.relname) FROM pg_catalog.pg_class c, pg_catalog.pg_namespace n WHERE c.relnamespace = n.oid AND c.relkind IN ('r', 'S', 'v', 'm', 'f', 'p') AND substring(pg_catalog.quote_ident(n.nspname) || '.' || pg_catalog.quote_ident(c.relname),1,2)='we' AND substring(pg_catalog.quote_ident(n.nspname) || '.',1,2) = substring('we',1,pg_catalog.length(pg_catalog.quote_ident(n.nspname))+1) AND (SELECT pg_catalog.count(*) FROM pg_catalog.pg_namespace WHERE substring(pg_catalog.quote_ident(nspname) || '.',1,2) = substring('we',1,pg_catalog.length(pg_catalog.quote_ident(nspname))+1)) = 1]" @@ -291,7 +291,7 @@ { "message": "\tLIMIT 1000", "event": { - "ingested": "2021-03-02T16:29:01.258274525Z" + "ingested": "2021-03-02T18:00:26.507660023Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\tLIMIT 1000]" @@ -316,7 +316,7 @@ "message": "2020-04-16 12:49:55.464 CEST [34492] LOG: duration: 0.179 ms statement: select * From weather ;", "event": { "duration": 179000, - "ingested": "2021-03-02T16:29:01.258275112Z", + "ingested": "2021-03-02T18:00:26.507660545Z", "category": [ "database" ], @@ -346,7 +346,7 @@ "message": "2020-04-16 12:50:05.322 CEST [34492] LOG: duration: 1.661 ms statement: INSERT INTO weather VALUES ('San Francisco', 46, 50, 0.25, '1994-11-27');", "event": { "duration": 1661000, - "ingested": "2021-03-02T16:29:01.258275701Z", + "ingested": "2021-03-02T18:00:26.507661072Z", "category": [ "database" ], @@ -376,7 +376,7 @@ "message": "2020-04-16 12:50:06.741 CEST [34492] LOG: duration: 0.144 ms statement: select * From weather ;", "event": { "duration": 144000, - "ingested": "2021-03-02T16:29:01.258276283Z", + "ingested": "2021-03-02T18:00:26.507661604Z", "category": [ "database" ], diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-13-error-code.csv.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-13-error-code.csv.log-config.yml new file mode 100644 index 00000000000..c39dc386179 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-13-error-code.csv.log-config.yml @@ -0,0 +1,2 @@ +dynamic_fields: + event.ingested: ".*" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-13-error-code.csv.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-13-error-code.csv.log-expected.json index 5cf61a0d4a2..b12b0e66af4 100644 --- a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-13-error-code.csv.log-expected.json +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-13-error-code.csv.log-expected.json @@ -32,7 +32,7 @@ }, "message": "SET idle_in_transaction_session_timeout = 50;", "event": { - "ingested": "2021-03-02T16:29:01.426765206Z", + "ingested": "2021-03-02T18:00:26.632764911Z", "category": [ "database" ], @@ -78,7 +78,7 @@ }, "message": "BEGIN;", "event": { - "ingested": "2021-03-02T16:29:01.426770046Z", + "ingested": "2021-03-02T18:00:26.632767808Z", "category": [ "database" ], @@ -123,7 +123,7 @@ }, "message": "terminating connection due to idle-in-transaction timeout", "event": { - "ingested": "2021-03-02T16:29:01.426771210Z", + "ingested": "2021-03-02T18:00:26.632768536Z", "category": [ "database" ], @@ -167,7 +167,7 @@ }, "message": "disconnection: session time: 0:00:33.289 user=postgres database=postgres host=172.24.0.1 port=48978", "event": { - "ingested": "2021-03-02T16:29:01.426772182Z", + "ingested": "2021-03-02T18:00:26.632769133Z", "category": [ "database" ], diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-13.csv.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-13.csv.log-config.yml new file mode 100644 index 00000000000..334301d86b9 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-13.csv.log-config.yml @@ -0,0 +1,6 @@ +numeric_keyword_fields: + - "process.pid" + - "postgresql.log.client_port" + - "postgresql.log.transaction_id" +dynamic_fields: + event.ingested: ".*" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-13.csv.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-13.csv.log-expected.json index 13b96d178e6..3bd46fab1ed 100644 --- a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-13.csv.log-expected.json +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-13.csv.log-expected.json @@ -19,7 +19,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:29:01.629955687Z", + "ingested": "2021-03-02T18:00:26.710621779Z", "category": [ "database" ], @@ -50,7 +50,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:29:01.629963546Z", + "ingested": "2021-03-02T18:00:26.710624726Z", "category": [ "database" ], @@ -81,7 +81,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:29:01.629964332Z", + "ingested": "2021-03-02T18:00:26.710625421Z", "category": [ "database" ], @@ -112,7 +112,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:29:01.629965213Z", + "ingested": "2021-03-02T18:00:26.710626021Z", "category": [ "database" ], @@ -143,7 +143,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:29:01.629965931Z", + "ingested": "2021-03-02T18:00:26.710626608Z", "category": [ "database" ], @@ -174,7 +174,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:29:01.629966582Z", + "ingested": "2021-03-02T18:00:26.710627184Z", "category": [ "database" ], @@ -207,7 +207,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:29:01.629967226Z", + "ingested": "2021-03-02T18:00:26.710627773Z", "category": [ "database" ], @@ -249,7 +249,7 @@ }, "message": "connection authorized: user=postgres database=postgres application_name=psql", "event": { - "ingested": "2021-03-02T16:29:01.629967866Z", + "ingested": "2021-03-02T18:00:26.710628360Z", "category": [ "database" ], @@ -295,7 +295,7 @@ }, "message": "select 1;", "event": { - "ingested": "2021-03-02T16:29:01.629968498Z", + "ingested": "2021-03-02T18:00:26.710628946Z", "category": [ "database" ], @@ -341,7 +341,7 @@ }, "message": "select name, setting from pg_settings where name like 'log%';", "event": { - "ingested": "2021-03-02T16:29:01.629969142Z", + "ingested": "2021-03-02T18:00:26.710629529Z", "category": [ "database" ], @@ -385,7 +385,7 @@ } }, "event": { - "ingested": "2021-03-02T16:29:01.629969791Z", + "ingested": "2021-03-02T18:00:26.710630121Z", "timezone": "UTC" }, "user": { @@ -395,7 +395,7 @@ { "message": "LIMIT 1000\",,,,,,,,,\"psql\",\"client backend\"", "event": { - "ingested": "2021-03-02T16:29:01.629970606Z" + "ingested": "2021-03-02T18:00:26.710630868Z" }, "error": { "message": "Provided Grok expressions do not match field value: [LIMIT 1000\\\",,,,,,,,,\\\"psql\\\",\\\"client backend\\\"]" @@ -431,7 +431,7 @@ } }, "event": { - "ingested": "2021-03-02T16:29:01.629971258Z", + "ingested": "2021-03-02T18:00:26.710631461Z", "timezone": "UTC" }, "user": { @@ -441,7 +441,7 @@ { "message": "LIMIT 1000\",,,,,,,,,\"psql\",\"client backend\"", "event": { - "ingested": "2021-03-02T16:29:01.629971898Z" + "ingested": "2021-03-02T18:00:26.710632040Z" }, "error": { "message": "Provided Grok expressions do not match field value: [LIMIT 1000\\\",,,,,,,,,\\\"psql\\\",\\\"client backend\\\"]" @@ -477,7 +477,7 @@ } }, "event": { - "ingested": "2021-03-02T16:29:01.629972537Z", + "ingested": "2021-03-02T18:00:26.710632622Z", "timezone": "UTC" }, "user": { @@ -487,7 +487,7 @@ { "message": "LIMIT 1000\",,,,,,,,,\"psql\",\"client backend\"", "event": { - "ingested": "2021-03-02T16:29:01.629973174Z" + "ingested": "2021-03-02T18:00:26.710633200Z" }, "error": { "message": "Provided Grok expressions do not match field value: [LIMIT 1000\\\",,,,,,,,,\\\"psql\\\",\\\"client backend\\\"]" @@ -525,7 +525,7 @@ }, "message": "alter system set log_min_duration_statement = 0;", "event": { - "ingested": "2021-03-02T16:29:01.629973930Z", + "ingested": "2021-03-02T18:00:26.710633903Z", "category": [ "database" ], @@ -569,7 +569,7 @@ } }, "event": { - "ingested": "2021-03-02T16:29:01.629974601Z", + "ingested": "2021-03-02T18:00:26.710634484Z", "timezone": "UTC" }, "user": { @@ -579,7 +579,7 @@ { "message": "LIMIT 1000\",,,,,,,,,\"psql\",\"client backend\"", "event": { - "ingested": "2021-03-02T16:29:01.629975312Z" + "ingested": "2021-03-02T18:00:26.710635063Z" }, "error": { "message": "Provided Grok expressions do not match field value: [LIMIT 1000\\\",,,,,,,,,\\\"psql\\\",\\\"client backend\\\"]" @@ -615,7 +615,7 @@ } }, "event": { - "ingested": "2021-03-02T16:29:01.629975948Z", + "ingested": "2021-03-02T18:00:26.710635636Z", "timezone": "UTC" }, "user": { @@ -625,7 +625,7 @@ { "message": "LIMIT 1000\",,,,,,,,,\"psql\",\"client backend\"", "event": { - "ingested": "2021-03-02T16:29:01.629976582Z" + "ingested": "2021-03-02T18:00:26.710636223Z" }, "error": { "message": "Provided Grok expressions do not match field value: [LIMIT 1000\\\",,,,,,,,,\\\"psql\\\",\\\"client backend\\\"]" @@ -663,7 +663,7 @@ }, "message": "alter system set log_statement = 'none';", "event": { - "ingested": "2021-03-02T16:29:01.629977226Z", + "ingested": "2021-03-02T18:00:26.710636799Z", "category": [ "database" ], @@ -709,7 +709,7 @@ }, "message": "select pg_reload_conf();", "event": { - "ingested": "2021-03-02T16:29:01.629977862Z", + "ingested": "2021-03-02T18:00:26.710637381Z", "category": [ "database" ], @@ -742,7 +742,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:29:01.629978640Z", + "ingested": "2021-03-02T18:00:26.710638099Z", "category": [ "database" ], @@ -773,7 +773,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:29:01.629979364Z", + "ingested": "2021-03-02T18:00:26.710638673Z", "category": [ "database" ], @@ -804,7 +804,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:29:01.629980002Z", + "ingested": "2021-03-02T18:00:26.710639256Z", "category": [ "database" ], @@ -835,7 +835,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:29:01.629980638Z", + "ingested": "2021-03-02T18:00:26.710639839Z", "category": [ "database" ], @@ -866,7 +866,7 @@ "level": "LOG" }, "event": { - "ingested": "2021-03-02T16:29:01.629981274Z", + "ingested": "2021-03-02T18:00:26.710640430Z", "category": [ "database" ], @@ -912,7 +912,7 @@ "message": "checkpoint;", "event": { "duration": 15136000, - "ingested": "2021-03-02T16:29:01.629981940Z", + "ingested": "2021-03-02T18:00:26.710641017Z", "category": [ "database" ], @@ -956,7 +956,7 @@ }, "message": "disconnection: session time: 0:01:04.968 user=postgres database=postgres host=172.24.0.1 port=45126", "event": { - "ingested": "2021-03-02T16:29:01.629982589Z", + "ingested": "2021-03-02T18:00:26.710641599Z", "category": [ "database" ], diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-debian-with-slowlog.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-debian-with-slowlog.log-config.yml new file mode 100644 index 00000000000..c39dc386179 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-debian-with-slowlog.log-config.yml @@ -0,0 +1,2 @@ +dynamic_fields: + event.ingested: ".*" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-debian-with-slowlog.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-debian-with-slowlog.log-expected.json index 4a97dc081d7..1f7add10161 100644 --- a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-debian-with-slowlog.log-expected.json +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-debian-with-slowlog.log-expected.json @@ -15,7 +15,7 @@ }, "message": "database system was shut down at 2017-06-17 16:58:04 CEST", "event": { - "ingested": "2021-03-02T16:29:02.444121814Z", + "ingested": "2021-03-02T18:00:27.663143945Z", "category": [ "database" ], @@ -41,7 +41,7 @@ }, "message": "MultiXact member wraparound protections are now enabled", "event": { - "ingested": "2021-03-02T16:29:02.444143743Z", + "ingested": "2021-03-02T18:00:27.663152260Z", "category": [ "database" ], @@ -67,7 +67,7 @@ }, "message": "autovacuum launcher started", "event": { - "ingested": "2021-03-02T16:29:02.444147910Z", + "ingested": "2021-03-02T18:00:27.663152943Z", "category": [ "database" ], @@ -93,7 +93,7 @@ }, "message": "database system is ready to accept connections", "event": { - "ingested": "2021-03-02T16:29:02.444148535Z", + "ingested": "2021-03-02T18:00:27.663153554Z", "category": [ "database" ], @@ -125,7 +125,7 @@ }, "message": "incomplete startup packet", "event": { - "ingested": "2021-03-02T16:29:02.444149125Z", + "ingested": "2021-03-02T18:00:27.663154725Z", "category": [ "database" ], @@ -164,7 +164,7 @@ "message": "2017-07-31 13:36:43.557 CEST [4983] postgres@postgres LOG: duration: 37.118 ms statement: SELECT d.datname as \"Name\",", "event": { "duration": 37118000, - "ingested": "2021-03-02T16:29:02.444149706Z", + "ingested": "2021-03-02T18:00:27.663155813Z", "category": [ "database" ], @@ -181,7 +181,7 @@ { "message": "\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",", "event": { - "ingested": "2021-03-02T16:29:02.444150284Z" + "ingested": "2021-03-02T18:00:27.663156850Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t pg_catalog.pg_get_userbyid(d.datdba) as \\\"Owner\\\",]" @@ -190,7 +190,7 @@ { "message": "\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",", "event": { - "ingested": "2021-03-02T16:29:02.444150858Z" + "ingested": "2021-03-02T18:00:27.663157417Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t pg_catalog.pg_encoding_to_char(d.encoding) as \\\"Encoding\\\",]" @@ -199,7 +199,7 @@ { "message": "\t d.datcollate as \"Collate\",", "event": { - "ingested": "2021-03-02T16:29:02.444151435Z" + "ingested": "2021-03-02T18:00:27.663158344Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t d.datcollate as \\\"Collate\\\",]" @@ -208,7 +208,7 @@ { "message": "\t d.datctype as \"Ctype\",", "event": { - "ingested": "2021-03-02T16:29:02.444152026Z" + "ingested": "2021-03-02T18:00:27.663159266Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t d.datctype as \\\"Ctype\\\",]" @@ -217,7 +217,7 @@ { "message": "\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"", "event": { - "ingested": "2021-03-02T16:29:02.444152747Z" + "ingested": "2021-03-02T18:00:27.663159938Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t pg_catalog.array_to_string(d.datacl, E'\\\\n') AS \\\"Access privileges\\\"]" @@ -226,7 +226,7 @@ { "message": "\tFROM pg_catalog.pg_database d", "event": { - "ingested": "2021-03-02T16:29:02.444153590Z" + "ingested": "2021-03-02T18:00:27.663161829Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\tFROM pg_catalog.pg_database d]" @@ -235,7 +235,7 @@ { "message": "\tORDER BY 1;", "event": { - "ingested": "2021-03-02T16:29:02.444154177Z" + "ingested": "2021-03-02T18:00:27.663162399Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\tORDER BY 1;]" @@ -266,7 +266,7 @@ "message": "2017-07-31 13:36:44.104 CEST [4986] postgres@postgres LOG: duration: 2.895 ms statement: SELECT d.datname as \"Name\",", "event": { "duration": 2895000, - "ingested": "2021-03-02T16:29:02.444154752Z", + "ingested": "2021-03-02T18:00:27.663162966Z", "category": [ "database" ], @@ -283,7 +283,7 @@ { "message": "\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",", "event": { - "ingested": "2021-03-02T16:29:02.444155328Z" + "ingested": "2021-03-02T18:00:27.663164100Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t pg_catalog.pg_get_userbyid(d.datdba) as \\\"Owner\\\",]" @@ -292,7 +292,7 @@ { "message": "\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",", "event": { - "ingested": "2021-03-02T16:29:02.444155903Z" + "ingested": "2021-03-02T18:00:27.663165188Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t pg_catalog.pg_encoding_to_char(d.encoding) as \\\"Encoding\\\",]" @@ -301,7 +301,7 @@ { "message": "\t d.datcollate as \"Collate\",", "event": { - "ingested": "2021-03-02T16:29:02.444156593Z" + "ingested": "2021-03-02T18:00:27.663166070Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t d.datcollate as \\\"Collate\\\",]" @@ -310,7 +310,7 @@ { "message": "\t d.datctype as \"Ctype\",", "event": { - "ingested": "2021-03-02T16:29:02.444157486Z" + "ingested": "2021-03-02T18:00:27.663166759Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t d.datctype as \\\"Ctype\\\",]" @@ -319,7 +319,7 @@ { "message": "\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"", "event": { - "ingested": "2021-03-02T16:29:02.444158065Z" + "ingested": "2021-03-02T18:00:27.663167329Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t pg_catalog.array_to_string(d.datacl, E'\\\\n') AS \\\"Access privileges\\\"]" @@ -328,7 +328,7 @@ { "message": "\tFROM pg_catalog.pg_database d", "event": { - "ingested": "2021-03-02T16:29:02.444158650Z" + "ingested": "2021-03-02T18:00:27.663167911Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\tFROM pg_catalog.pg_database d]" @@ -337,7 +337,7 @@ { "message": "\tORDER BY 1;", "event": { - "ingested": "2021-03-02T16:29:02.444159228Z" + "ingested": "2021-03-02T18:00:27.663168479Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\tORDER BY 1;]" @@ -368,7 +368,7 @@ "message": "2017-07-31 13:36:44.642 CEST [4989] postgres@postgres LOG: duration: 2.809 ms statement: SELECT d.datname as \"Name\",", "event": { "duration": 2809000, - "ingested": "2021-03-02T16:29:02.444160079Z", + "ingested": "2021-03-02T18:00:27.663169038Z", "category": [ "database" ], @@ -385,7 +385,7 @@ { "message": "\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",", "event": { - "ingested": "2021-03-02T16:29:02.444160714Z" + "ingested": "2021-03-02T18:00:27.663170303Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t pg_catalog.pg_get_userbyid(d.datdba) as \\\"Owner\\\",]" @@ -394,7 +394,7 @@ { "message": "\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",", "event": { - "ingested": "2021-03-02T16:29:02.444161416Z" + "ingested": "2021-03-02T18:00:27.663171341Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t pg_catalog.pg_encoding_to_char(d.encoding) as \\\"Encoding\\\",]" @@ -403,7 +403,7 @@ { "message": "\t d.datcollate as \"Collate\",", "event": { - "ingested": "2021-03-02T16:29:02.444162012Z" + "ingested": "2021-03-02T18:00:27.663172122Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t d.datcollate as \\\"Collate\\\",]" @@ -412,7 +412,7 @@ { "message": "\t d.datctype as \"Ctype\",", "event": { - "ingested": "2021-03-02T16:29:02.444162600Z" + "ingested": "2021-03-02T18:00:27.663173229Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t d.datctype as \\\"Ctype\\\",]" @@ -421,7 +421,7 @@ { "message": "\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"", "event": { - "ingested": "2021-03-02T16:29:02.444163182Z" + "ingested": "2021-03-02T18:00:27.663173779Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t pg_catalog.array_to_string(d.datacl, E'\\\\n') AS \\\"Access privileges\\\"]" @@ -430,7 +430,7 @@ { "message": "\tFROM pg_catalog.pg_database d", "event": { - "ingested": "2021-03-02T16:29:02.444163780Z" + "ingested": "2021-03-02T18:00:27.663174347Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\tFROM pg_catalog.pg_database d]" @@ -439,7 +439,7 @@ { "message": "\tORDER BY 1;", "event": { - "ingested": "2021-03-02T16:29:02.444164370Z" + "ingested": "2021-03-02T18:00:27.663175040Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\tORDER BY 1;]" @@ -466,7 +466,7 @@ }, "message": "database \"users\" does not exist", "event": { - "ingested": "2021-03-02T16:29:02.444164949Z", + "ingested": "2021-03-02T18:00:27.663175749Z", "category": [ "database" ], @@ -501,7 +501,7 @@ }, "message": "database \"user\" does not exist", "event": { - "ingested": "2021-03-02T16:29:02.444165532Z", + "ingested": "2021-03-02T18:00:27.663176317Z", "category": [ "database" ], @@ -540,7 +540,7 @@ "message": "2017-07-31 13:39:21.025 CEST [5404] postgres@postgres LOG: duration: 37.598 ms statement: SELECT n.nspname as \"Schema\",", "event": { "duration": 37598000, - "ingested": "2021-03-02T16:29:02.444166209Z", + "ingested": "2021-03-02T18:00:27.663177611Z", "category": [ "database" ], @@ -557,7 +557,7 @@ { "message": "\t c.relname as \"Name\",", "event": { - "ingested": "2021-03-02T16:29:02.444166786Z" + "ingested": "2021-03-02T18:00:27.663178179Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t c.relname as \\\"Name\\\",]" @@ -566,7 +566,7 @@ { "message": "\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",", "event": { - "ingested": "2021-03-02T16:29:02.444167377Z" + "ingested": "2021-03-02T18:00:27.663179161Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \\\"Type\\\",]" @@ -575,7 +575,7 @@ { "message": "\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"", "event": { - "ingested": "2021-03-02T16:29:02.444168083Z" + "ingested": "2021-03-02T18:00:27.663179881Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t pg_catalog.pg_get_userbyid(c.relowner) as \\\"Owner\\\"]" @@ -584,7 +584,7 @@ { "message": "\tFROM pg_catalog.pg_class c", "event": { - "ingested": "2021-03-02T16:29:02.444168664Z" + "ingested": "2021-03-02T18:00:27.663180828Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\tFROM pg_catalog.pg_class c]" @@ -593,7 +593,7 @@ { "message": "\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace", "event": { - "ingested": "2021-03-02T16:29:02.444169242Z" + "ingested": "2021-03-02T18:00:27.663182348Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace]" @@ -602,7 +602,7 @@ { "message": "\tWHERE c.relkind IN ('r','')", "event": { - "ingested": "2021-03-02T16:29:02.444169822Z" + "ingested": "2021-03-02T18:00:27.663183350Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\tWHERE c.relkind IN ('r','')]" @@ -611,7 +611,7 @@ { "message": "\t AND n.nspname \u003c\u003e 'pg_catalog'", "event": { - "ingested": "2021-03-02T16:29:02.444170407Z" + "ingested": "2021-03-02T18:00:27.663183913Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t AND n.nspname \u003c\u003e 'pg_catalog']" @@ -620,7 +620,7 @@ { "message": "\t AND n.nspname \u003c\u003e 'information_schema'", "event": { - "ingested": "2021-03-02T16:29:02.444171062Z" + "ingested": "2021-03-02T18:00:27.663184475Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t AND n.nspname \u003c\u003e 'information_schema']" @@ -629,7 +629,7 @@ { "message": "\t AND n.nspname !~ '^pg_toast'", "event": { - "ingested": "2021-03-02T16:29:02.444171651Z" + "ingested": "2021-03-02T18:00:27.663185207Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t AND n.nspname !~ '^pg_toast']" @@ -638,7 +638,7 @@ { "message": "\t AND pg_catalog.pg_table_is_visible(c.oid)", "event": { - "ingested": "2021-03-02T16:29:02.444172322Z" + "ingested": "2021-03-02T18:00:27.663185971Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t AND pg_catalog.pg_table_is_visible(c.oid)]" @@ -647,7 +647,7 @@ { "message": "\tORDER BY 1,2;", "event": { - "ingested": "2021-03-02T16:29:02.444173102Z" + "ingested": "2021-03-02T18:00:27.663187077Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\tORDER BY 1,2;]" @@ -678,7 +678,7 @@ "message": "2017-07-31 13:39:31.619 CEST [5502] postgres@clients LOG: duration: 9.482 ms statement: select * from clients;", "event": { "duration": 9482000, - "ingested": "2021-03-02T16:29:02.444173690Z", + "ingested": "2021-03-02T18:00:27.663188221Z", "category": [ "database" ], @@ -717,7 +717,7 @@ "message": "2017-07-31 13:39:40.147 CEST [5502] postgres@clients LOG: duration: 0.765 ms statement: select id from clients;", "event": { "duration": 765000, - "ingested": "2021-03-02T16:29:02.444174272Z", + "ingested": "2021-03-02T18:00:27.663189755Z", "category": [ "database" ], @@ -756,7 +756,7 @@ "message": "2017-07-31 13:40:54.310 CEST [5502] postgres@clients LOG: duration: 26.082 ms statement: SELECT n.nspname as \"Schema\",", "event": { "duration": 26082000, - "ingested": "2021-03-02T16:29:02.444175043Z", + "ingested": "2021-03-02T18:00:27.663190467Z", "category": [ "database" ], @@ -773,7 +773,7 @@ { "message": "\t c.relname as \"Name\",", "event": { - "ingested": "2021-03-02T16:29:02.444175623Z" + "ingested": "2021-03-02T18:00:27.663191256Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t c.relname as \\\"Name\\\",]" @@ -782,7 +782,7 @@ { "message": "\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",", "event": { - "ingested": "2021-03-02T16:29:02.444176202Z" + "ingested": "2021-03-02T18:00:27.663192609Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \\\"Type\\\",]" @@ -791,7 +791,7 @@ { "message": "\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"", "event": { - "ingested": "2021-03-02T16:29:02.444176778Z" + "ingested": "2021-03-02T18:00:27.663194552Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t pg_catalog.pg_get_userbyid(c.relowner) as \\\"Owner\\\"]" @@ -800,7 +800,7 @@ { "message": "\tFROM pg_catalog.pg_class c", "event": { - "ingested": "2021-03-02T16:29:02.444177361Z" + "ingested": "2021-03-02T18:00:27.663196478Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\tFROM pg_catalog.pg_class c]" @@ -809,7 +809,7 @@ { "message": "\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace", "event": { - "ingested": "2021-03-02T16:29:02.444178048Z" + "ingested": "2021-03-02T18:00:27.663197929Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace]" @@ -818,7 +818,7 @@ { "message": "\tWHERE c.relkind IN ('r','')", "event": { - "ingested": "2021-03-02T16:29:02.444178630Z" + "ingested": "2021-03-02T18:00:27.663199716Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\tWHERE c.relkind IN ('r','')]" @@ -827,7 +827,7 @@ { "message": "\t AND n.nspname \u003c\u003e 'pg_catalog'", "event": { - "ingested": "2021-03-02T16:29:02.444179229Z" + "ingested": "2021-03-02T18:00:27.663200888Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t AND n.nspname \u003c\u003e 'pg_catalog']" @@ -836,7 +836,7 @@ { "message": "\t AND n.nspname \u003c\u003e 'information_schema'", "event": { - "ingested": "2021-03-02T16:29:02.444179811Z" + "ingested": "2021-03-02T18:00:27.663201778Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t AND n.nspname \u003c\u003e 'information_schema']" @@ -845,7 +845,7 @@ { "message": "\t AND n.nspname !~ '^pg_toast'", "event": { - "ingested": "2021-03-02T16:29:02.444180393Z" + "ingested": "2021-03-02T18:00:27.663202510Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t AND n.nspname !~ '^pg_toast']" @@ -854,7 +854,7 @@ { "message": "\t AND pg_catalog.pg_table_is_visible(c.oid)", "event": { - "ingested": "2021-03-02T16:29:02.444180978Z" + "ingested": "2021-03-02T18:00:27.663203267Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t AND pg_catalog.pg_table_is_visible(c.oid)]" @@ -863,7 +863,7 @@ { "message": "\tORDER BY 1,2;", "event": { - "ingested": "2021-03-02T16:29:02.444181787Z" + "ingested": "2021-03-02T18:00:27.663204471Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\tORDER BY 1,2;]" @@ -894,7 +894,7 @@ "message": "2017-07-31 13:43:22.645 CEST [5502] postgres@clients LOG: duration: 36.162 ms statement: create table cats(name varchar(50) primary key, toy varchar (50) not null, born timestamp not null);", "event": { "duration": 36162000, - "ingested": "2021-03-02T16:29:02.444182369Z", + "ingested": "2021-03-02T18:00:27.663206066Z", "category": [ "database" ], @@ -933,7 +933,7 @@ "message": "2017-07-31 13:46:02.670 CEST [5502] postgres@c$lients LOG: duration: 10.540 ms statement: insert into cats(name, toy, born) values('kate', 'ball', now());", "event": { "duration": 10540000, - "ingested": "2021-03-02T16:29:02.444183070Z", + "ingested": "2021-03-02T18:00:27.663206904Z", "category": [ "database" ], @@ -972,7 +972,7 @@ "message": "2017-07-31 13:46:23.016 CEST [5502] postgres@_clients$db LOG: duration: 5.156 ms statement: insert into cats(name, toy, born) values('frida', 'horse', now());", "event": { "duration": 5156000, - "ingested": "2021-03-02T16:29:02.444183653Z", + "ingested": "2021-03-02T18:00:27.663207782Z", "category": [ "database" ], @@ -1011,7 +1011,7 @@ "message": "2017-07-31 13:46:55.637 CEST [5502] postgres@clients_db LOG: duration: 25.871 ms statement: create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null);", "event": { "duration": 25871000, - "ingested": "2021-03-02T16:29:02.444184235Z", + "ingested": "2021-03-02T18:00:27.663208846Z", "category": [ "database" ], @@ -1050,7 +1050,7 @@ "message": "2019-05-06 19:00:04.511 UTC [913763] elastic@opbeans LOG: duration: 0.753 ms statement: SELECT p.id, p.sku, p.name, p.stock, t.name AS type_name FROM products p LEFT JOIN product_types t ON p.type_id=t.id", "event": { "duration": 753000, - "ingested": "2021-03-02T16:29:02.444184814Z", + "ingested": "2021-03-02T18:00:27.663210123Z", "category": [ "database" ], @@ -1067,7 +1067,7 @@ { "message": " FROM orders JOIN customers ON orders.customer_id=customers.id", "event": { - "ingested": "2021-03-02T16:29:02.444185409Z" + "ingested": "2021-03-02T18:00:27.663210806Z" }, "error": { "message": "Provided Grok expressions do not match field value: [ FROM orders JOIN customers ON orders.customer_id=customers.id]" @@ -1076,7 +1076,7 @@ { "message": " FROM products JOIN product_types ON type_id=product_types.id", "event": { - "ingested": "2021-03-02T16:29:02.444185997Z" + "ingested": "2021-03-02T18:00:27.663211480Z" }, "error": { "message": "Provided Grok expressions do not match field value: [ FROM products JOIN product_types ON type_id=product_types.id]" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-multi-core.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-multi-core.log-config.yml new file mode 100644 index 00000000000..c39dc386179 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-multi-core.log-config.yml @@ -0,0 +1,2 @@ +dynamic_fields: + event.ingested: ".*" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-multi-core.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-multi-core.log-expected.json index b3f38161d69..84acb8413f1 100644 --- a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-multi-core.log-expected.json +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-multi-core.log-expected.json @@ -22,7 +22,7 @@ }, "message": "incomplete startup packet", "event": { - "ingested": "2021-03-02T16:29:03.068026115Z", + "ingested": "2021-03-02T18:00:28.195781392Z", "category": [ "database" ], @@ -58,7 +58,7 @@ }, "message": "database \"user\" does not exist", "event": { - "ingested": "2021-03-02T16:29:03.068031495Z", + "ingested": "2021-03-02T18:00:28.195784369Z", "category": [ "database" ], @@ -98,7 +98,7 @@ "message": "2017-04-03 22:35:22.389 CEST [5404-2] postgres@postgres LOG: duration: 37.598 ms statement: SELECT n.nspname as \"Schema\",", "event": { "duration": 37598000, - "ingested": "2021-03-02T16:29:03.068032926Z", + "ingested": "2021-03-02T18:00:28.195785107Z", "category": [ "database" ], @@ -115,7 +115,7 @@ { "message": "\t c.relname as \"Name\",", "event": { - "ingested": "2021-03-02T16:29:03.068034028Z" + "ingested": "2021-03-02T18:00:28.195789056Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t c.relname as \\\"Name\\\",]" @@ -124,7 +124,7 @@ { "message": "\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",", "event": { - "ingested": "2021-03-02T16:29:03.068035207Z" + "ingested": "2021-03-02T18:00:28.195789680Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \\\"Type\\\",]" @@ -133,7 +133,7 @@ { "message": "\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"", "event": { - "ingested": "2021-03-02T16:29:03.068036298Z" + "ingested": "2021-03-02T18:00:28.195790245Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t pg_catalog.pg_get_userbyid(c.relowner) as \\\"Owner\\\"]" @@ -142,7 +142,7 @@ { "message": "\tFROM pg_catalog.pg_class c", "event": { - "ingested": "2021-03-02T16:29:03.068037272Z" + "ingested": "2021-03-02T18:00:28.195790824Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\tFROM pg_catalog.pg_class c]" @@ -151,7 +151,7 @@ { "message": "\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace", "event": { - "ingested": "2021-03-02T16:29:03.068038367Z" + "ingested": "2021-03-02T18:00:28.195791395Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace]" @@ -160,7 +160,7 @@ { "message": "\tWHERE c.relkind IN ('r','')", "event": { - "ingested": "2021-03-02T16:29:03.068039510Z" + "ingested": "2021-03-02T18:00:28.195791968Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\tWHERE c.relkind IN ('r','')]" @@ -169,7 +169,7 @@ { "message": "\t AND n.nspname \u003c\u003e 'pg_catalog'", "event": { - "ingested": "2021-03-02T16:29:03.068040645Z" + "ingested": "2021-03-02T18:00:28.195792534Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t AND n.nspname \u003c\u003e 'pg_catalog']" @@ -178,7 +178,7 @@ { "message": "\t AND n.nspname \u003c\u003e 'information_schema'", "event": { - "ingested": "2021-03-02T16:29:03.068041689Z" + "ingested": "2021-03-02T18:00:28.195793103Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t AND n.nspname \u003c\u003e 'information_schema']" @@ -187,7 +187,7 @@ { "message": "\t AND n.nspname !~ '^pg_toast'", "event": { - "ingested": "2021-03-02T16:29:03.068043219Z" + "ingested": "2021-03-02T18:00:28.195793872Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t AND n.nspname !~ '^pg_toast']" @@ -196,7 +196,7 @@ { "message": "\t AND pg_catalog.pg_table_is_visible(c.oid)", "event": { - "ingested": "2021-03-02T16:29:03.068044260Z" + "ingested": "2021-03-02T18:00:28.195794457Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\t AND pg_catalog.pg_table_is_visible(c.oid)]" @@ -205,7 +205,7 @@ { "message": "\tORDER BY 1,2;", "event": { - "ingested": "2021-03-02T16:29:03.068045597Z" + "ingested": "2021-03-02T18:00:28.195795025Z" }, "error": { "message": "Provided Grok expressions do not match field value: [\\tORDER BY 1,2;]" @@ -227,7 +227,7 @@ }, "message": "autovacuum launcher started", "event": { - "ingested": "2021-03-02T16:29:03.068046815Z", + "ingested": "2021-03-02T18:00:28.195795590Z", "category": [ "database" ], @@ -254,7 +254,7 @@ }, "message": "checkpoints are occurring too frequently (25 seconds apart)", "event": { - "ingested": "2021-03-02T16:29:03.068047777Z", + "ingested": "2021-03-02T18:00:28.195796168Z", "category": [ "database" ], @@ -281,7 +281,7 @@ }, "message": "Consider increasing the configuration parameter \"max_wal_size\".", "event": { - "ingested": "2021-03-02T16:29:03.068049087Z", + "ingested": "2021-03-02T18:00:28.195796859Z", "category": [ "database" ], @@ -314,7 +314,7 @@ }, "message": "the database system is starting up", "event": { - "ingested": "2021-03-02T16:29:03.068050233Z", + "ingested": "2021-03-02T18:00:28.195797446Z", "category": [ "database" ], @@ -350,7 +350,7 @@ }, "message": "the database system is starting up", "event": { - "ingested": "2021-03-02T16:29:03.068051399Z", + "ingested": "2021-03-02T18:00:28.195798015Z", "category": [ "database" ], diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-new-timestamp.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-new-timestamp.log-config.yml new file mode 100644 index 00000000000..c39dc386179 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-new-timestamp.log-config.yml @@ -0,0 +1,2 @@ +dynamic_fields: + event.ingested: ".*" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-new-timestamp.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-new-timestamp.log-expected.json index 0866c874d3c..6f14bc1e7a9 100644 --- a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-new-timestamp.log-expected.json +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-9.6-new-timestamp.log-expected.json @@ -16,7 +16,7 @@ }, "message": "autovacuum launcher started", "event": { - "ingested": "2021-03-02T16:29:03.342082328Z", + "ingested": "2021-03-02T18:00:28.446381811Z", "category": [ "database" ], @@ -43,7 +43,7 @@ }, "message": "checkpoints are occurring too frequently (25 seconds apart)", "event": { - "ingested": "2021-03-02T16:29:03.342087955Z", + "ingested": "2021-03-02T18:00:28.446386382Z", "category": [ "database" ], @@ -70,7 +70,7 @@ }, "message": "Consider increasing the configuration parameter \"max_wal_size\".", "event": { - "ingested": "2021-03-02T16:29:03.342089922Z", + "ingested": "2021-03-02T18:00:28.446387360Z", "category": [ "database" ], @@ -103,7 +103,7 @@ }, "message": "the database system is starting up", "event": { - "ingested": "2021-03-02T16:29:03.342091180Z", + "ingested": "2021-03-02T18:00:28.446387979Z", "category": [ "database" ], @@ -139,7 +139,7 @@ }, "message": "the database system is starting up", "event": { - "ingested": "2021-03-02T16:29:03.342092167Z", + "ingested": "2021-03-02T18:00:28.446388699Z", "category": [ "database" ], diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-query-steps-slowlog.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-query-steps-slowlog.log-config.yml new file mode 100644 index 00000000000..c39dc386179 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-query-steps-slowlog.log-config.yml @@ -0,0 +1,2 @@ +dynamic_fields: + event.ingested: ".*" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-query-steps-slowlog.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-query-steps-slowlog.log-expected.json index 7e5252e6edb..8dfadd1f1cb 100644 --- a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-query-steps-slowlog.log-expected.json +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-query-steps-slowlog.log-expected.json @@ -25,7 +25,7 @@ "message": "2019-09-04 15:52:38.004 CEST [31136] user@host LOG: duration: 12.437 ms parse \u003cunnamed\u003e: select * from table", "event": { "duration": 12437000, - "ingested": "2021-03-02T16:29:03.619611067Z", + "ingested": "2021-03-02T18:00:28.514863735Z", "category": [ "database" ], @@ -65,7 +65,7 @@ "message": "2019-09-04 15:52:38.004 CEST [31136] user@host LOG: duration: 12.437 ms execute pdo_stmt_00000002: select * from table", "event": { "duration": 12437000, - "ingested": "2021-03-02T16:29:03.619617030Z", + "ingested": "2021-03-02T18:00:28.514866963Z", "category": [ "database" ], @@ -82,7 +82,7 @@ { "message": "", "event": { - "ingested": "2021-03-02T16:29:03.619619231Z" + "ingested": "2021-03-02T18:00:28.514867624Z" }, "error": { "message": "Provided Grok expressions do not match field value: []" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-ubuntu-9.5.log-config.yml b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-ubuntu-9.5.log-config.yml new file mode 100644 index 00000000000..c39dc386179 --- /dev/null +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-ubuntu-9.5.log-config.yml @@ -0,0 +1,2 @@ +dynamic_fields: + event.ingested: ".*" diff --git a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-ubuntu-9.5.log-expected.json b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-ubuntu-9.5.log-expected.json index 659dd8f2836..50b64aea8bf 100644 --- a/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-ubuntu-9.5.log-expected.json +++ b/packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-ubuntu-9.5.log-expected.json @@ -21,7 +21,7 @@ }, "message": "could not receive data from client: Connection reset by peer", "event": { - "ingested": "2021-03-02T16:29:03.685948617Z", + "ingested": "2021-03-02T18:00:28.545617077Z", "category": [ "database" ], @@ -56,7 +56,7 @@ }, "message": "unexpected EOF on client connection with an open transaction", "event": { - "ingested": "2021-03-02T16:29:03.685953892Z", + "ingested": "2021-03-02T18:00:28.545621442Z", "category": [ "database" ], @@ -91,7 +91,7 @@ }, "message": "could not receive data from client: Connection reset by peer", "event": { - "ingested": "2021-03-02T16:29:03.685955455Z", + "ingested": "2021-03-02T18:00:28.545623161Z", "category": [ "database" ], @@ -126,7 +126,7 @@ }, "message": "could not receive data from client: Connection reset by peer", "event": { - "ingested": "2021-03-02T16:29:03.685956676Z", + "ingested": "2021-03-02T18:00:28.545624146Z", "category": [ "database" ], @@ -161,7 +161,7 @@ }, "message": "could not receive data from client: Connection reset by peer", "event": { - "ingested": "2021-03-02T16:29:03.685957787Z", + "ingested": "2021-03-02T18:00:28.545625230Z", "category": [ "database" ], @@ -196,7 +196,7 @@ }, "message": "could not receive data from client: Connection reset by peer", "event": { - "ingested": "2021-03-02T16:29:03.685958906Z", + "ingested": "2021-03-02T18:00:28.545626121Z", "category": [ "database" ], @@ -231,7 +231,7 @@ }, "message": "could not receive data from client: Connection reset by peer", "event": { - "ingested": "2021-03-02T16:29:03.685960004Z", + "ingested": "2021-03-02T18:00:28.545627146Z", "category": [ "database" ], @@ -266,7 +266,7 @@ }, "message": "operator does not exist: jsonb @\u003e at character 49", "event": { - "ingested": "2021-03-02T16:29:03.685961159Z", + "ingested": "2021-03-02T18:00:28.545628271Z", "category": [ "database" ], @@ -301,7 +301,7 @@ }, "message": "No operator matches the given name and argument type(s). You might need to add explicit type casts.", "event": { - "ingested": "2021-03-02T16:29:03.685962306Z", + "ingested": "2021-03-02T18:00:28.545629091Z", "category": [ "database" ], @@ -336,7 +336,7 @@ }, "message": "SELECT id, user FROM users WHERE NOT user @\u003e %s", "event": { - "ingested": "2021-03-02T16:29:03.685963453Z", + "ingested": "2021-03-02T18:00:28.545629905Z", "category": [ "database" ], @@ -371,7 +371,7 @@ }, "message": "column \"%s\" does not exist at character 52", "event": { - "ingested": "2021-03-02T16:29:03.685964769Z", + "ingested": "2021-03-02T18:00:28.545630804Z", "category": [ "database" ], @@ -406,7 +406,7 @@ }, "message": "SELECT id, user FROM users WHERE NOT user @\u003e \"%s\"", "event": { - "ingested": "2021-03-02T16:29:03.685966309Z", + "ingested": "2021-03-02T18:00:28.545631988Z", "category": [ "database" ], @@ -441,7 +441,7 @@ }, "message": "could not receive data from client: Connection reset by peer", "event": { - "ingested": "2021-03-02T16:29:03.685967789Z", + "ingested": "2021-03-02T18:00:28.545632830Z", "category": [ "database" ], @@ -476,7 +476,7 @@ }, "message": "unexpected EOF on client connection with an open transaction", "event": { - "ingested": "2021-03-02T16:29:03.685968965Z", + "ingested": "2021-03-02T18:00:28.545633711Z", "category": [ "database" ], @@ -511,7 +511,7 @@ }, "message": "syntax error at or near \"{\" at character 49", "event": { - "ingested": "2021-03-02T16:29:03.685970107Z", + "ingested": "2021-03-02T18:00:28.545634821Z", "category": [ "database" ], @@ -546,7 +546,7 @@ }, "message": "INSERT INTO users (id, user) VALUES (1, {\"attr\": \"yes\"});", "event": { - "ingested": "2021-03-02T16:29:03.685971097Z", + "ingested": "2021-03-02T18:00:28.545635739Z", "category": [ "database" ], @@ -581,7 +581,7 @@ }, "message": "syntax error at or near \"{\" at character 49", "event": { - "ingested": "2021-03-02T16:29:03.685972413Z", + "ingested": "2021-03-02T18:00:28.545636711Z", "category": [ "database" ], @@ -616,7 +616,7 @@ }, "message": "INSERT INTO users (id, user) VALUES (1, {attr: \"yes\"});", "event": { - "ingested": "2021-03-02T16:29:03.685973428Z", + "ingested": "2021-03-02T18:00:28.545637596Z", "category": [ "database" ], @@ -651,7 +651,7 @@ }, "message": "column \"a\" does not exist at character 42", "event": { - "ingested": "2021-03-02T16:29:03.685974451Z", + "ingested": "2021-03-02T18:00:28.545638420Z", "category": [ "database" ], @@ -686,7 +686,7 @@ }, "message": "INSERT INTO users (id, user) VALUES (1, '{\"attr\": \"yes\"}');", "event": { - "ingested": "2021-03-02T16:29:03.685975363Z", + "ingested": "2021-03-02T18:00:28.545639381Z", "category": [ "database" ], @@ -721,7 +721,7 @@ }, "message": "column \"attr\" does not exist at character 42", "event": { - "ingested": "2021-03-02T16:29:03.685976307Z", + "ingested": "2021-03-02T18:00:28.545640293Z", "category": [ "database" ], @@ -756,7 +756,7 @@ }, "message": "INSERT INTO users (id, user) VALUES (\"1\", '{\"attr\": \"no\"}');", "event": { - "ingested": "2021-03-02T16:29:03.685977213Z", + "ingested": "2021-03-02T18:00:28.545641207Z", "category": [ "database" ], @@ -791,7 +791,7 @@ }, "message": "could not receive data from client: Connection reset by peer", "event": { - "ingested": "2021-03-02T16:29:03.685978154Z", + "ingested": "2021-03-02T18:00:28.545642082Z", "category": [ "database" ], @@ -826,7 +826,7 @@ }, "message": "unexpected EOF on client connection with an open transaction", "event": { - "ingested": "2021-03-02T16:29:03.685979199Z", + "ingested": "2021-03-02T18:00:28.545643090Z", "category": [ "database" ], @@ -861,7 +861,7 @@ }, "message": "duplicate key value violates unique constraint \"users_pkey\"", "event": { - "ingested": "2021-03-02T16:29:03.685980104Z", + "ingested": "2021-03-02T18:00:28.545643996Z", "category": [ "database" ], @@ -896,7 +896,7 @@ }, "message": "Key (id)=(1) already exists.", "event": { - "ingested": "2021-03-02T16:29:03.685980982Z", + "ingested": "2021-03-02T18:00:28.545644802Z", "category": [ "database" ], @@ -931,7 +931,7 @@ }, "message": "INSERT INTO users (id, user) VALUES ('1', '{\"attr\": \"yes\"}');", "event": { - "ingested": "2021-03-02T16:29:03.685981963Z", + "ingested": "2021-03-02T18:00:28.545645632Z", "category": [ "database" ], @@ -966,7 +966,7 @@ }, "message": "could not receive data from client: Connection reset by peer", "event": { - "ingested": "2021-03-02T16:29:03.685982949Z", + "ingested": "2021-03-02T18:00:28.545646538Z", "category": [ "database" ], @@ -1001,7 +1001,7 @@ }, "message": "could not receive data from client: Connection reset by peer", "event": { - "ingested": "2021-03-02T16:29:03.685983848Z", + "ingested": "2021-03-02T18:00:28.545647448Z", "category": [ "database" ], @@ -1030,7 +1030,7 @@ }, "message": "received fast shutdown request", "event": { - "ingested": "2021-03-02T16:29:03.685984753Z", + "ingested": "2021-03-02T18:00:28.545648444Z", "category": [ "database" ], @@ -1056,7 +1056,7 @@ }, "message": "aborting any active transactions", "event": { - "ingested": "2021-03-02T16:29:03.685985971Z", + "ingested": "2021-03-02T18:00:28.545649357Z", "category": [ "database" ], @@ -1082,7 +1082,7 @@ }, "message": "autovacuum launcher shutting down", "event": { - "ingested": "2021-03-02T16:29:03.685987095Z", + "ingested": "2021-03-02T18:00:28.545650151Z", "category": [ "database" ], @@ -1108,7 +1108,7 @@ }, "message": "shutting down", "event": { - "ingested": "2021-03-02T16:29:03.685992883Z", + "ingested": "2021-03-02T18:00:28.545651067Z", "category": [ "database" ], @@ -1134,7 +1134,7 @@ }, "message": "database system is shut down", "event": { - "ingested": "2021-03-02T16:29:03.685993742Z", + "ingested": "2021-03-02T18:00:28.545651976Z", "category": [ "database" ], @@ -1160,7 +1160,7 @@ }, "message": "database system was shut down at 2017-04-08 21:54:37 CEST", "event": { - "ingested": "2021-03-02T16:29:03.685994845Z", + "ingested": "2021-03-02T18:00:28.545653036Z", "category": [ "database" ], @@ -1186,7 +1186,7 @@ }, "message": "MultiXact member wraparound protections are now enabled", "event": { - "ingested": "2021-03-02T16:29:03.685995680Z", + "ingested": "2021-03-02T18:00:28.545653966Z", "category": [ "database" ], @@ -1212,7 +1212,7 @@ }, "message": "database system is ready to accept connections", "event": { - "ingested": "2021-03-02T16:29:03.685996557Z", + "ingested": "2021-03-02T18:00:28.545654912Z", "category": [ "database" ], @@ -1238,7 +1238,7 @@ }, "message": "autovacuum launcher started", "event": { - "ingested": "2021-03-02T16:29:03.685997397Z", + "ingested": "2021-03-02T18:00:28.545655779Z", "category": [ "database" ], @@ -1270,7 +1270,7 @@ }, "message": "incomplete startup packet", "event": { - "ingested": "2021-03-02T16:29:03.685998428Z", + "ingested": "2021-03-02T18:00:28.545656745Z", "category": [ "database" ], @@ -1299,7 +1299,7 @@ }, "message": "database system was shut down at 2017-05-27 14:07:52 UTC", "event": { - "ingested": "2021-03-02T16:29:03.685999636Z", + "ingested": "2021-03-02T18:00:28.545657672Z", "category": [ "database" ], @@ -1325,7 +1325,7 @@ }, "message": "MultiXact member wraparound protections are now enabled", "event": { - "ingested": "2021-03-02T16:29:03.686000590Z", + "ingested": "2021-03-02T18:00:28.545658488Z", "category": [ "database" ], @@ -1351,7 +1351,7 @@ }, "message": "database system is ready to accept connections", "event": { - "ingested": "2021-03-02T16:29:03.686001521Z", + "ingested": "2021-03-02T18:00:28.545659304Z", "category": [ "database" ], @@ -1377,7 +1377,7 @@ }, "message": "autovacuum launcher started", "event": { - "ingested": "2021-03-02T16:29:03.686002474Z", + "ingested": "2021-03-02T18:00:28.545660440Z", "category": [ "database" ], @@ -1409,7 +1409,7 @@ }, "message": "incomplete startup packet", "event": { - "ingested": "2021-03-02T16:29:03.686003784Z", + "ingested": "2021-03-02T18:00:28.545661386Z", "category": [ "database" ], @@ -1444,7 +1444,7 @@ }, "message": "database \"mydb\" does not exist", "event": { - "ingested": "2021-03-02T16:29:03.686005291Z", + "ingested": "2021-03-02T18:00:28.545662727Z", "category": [ "database" ], @@ -1479,7 +1479,7 @@ }, "message": "could not receive data from client: Connection reset by peer", "event": { - "ingested": "2021-03-02T16:29:03.686006439Z", + "ingested": "2021-03-02T18:00:28.545664336Z", "category": [ "database" ], @@ -1514,7 +1514,7 @@ }, "message": "could not receive data from client: Connection reset by peer", "event": { - "ingested": "2021-03-02T16:29:03.686007486Z", + "ingested": "2021-03-02T18:00:28.545665651Z", "category": [ "database" ], @@ -1543,7 +1543,7 @@ }, "message": "received fast shutdown request", "event": { - "ingested": "2021-03-02T16:29:03.686008341Z", + "ingested": "2021-03-02T18:00:28.545666555Z", "category": [ "database" ], @@ -1569,7 +1569,7 @@ }, "message": "aborting any active transactions", "event": { - "ingested": "2021-03-02T16:29:03.686009619Z", + "ingested": "2021-03-02T18:00:28.545667545Z", "category": [ "database" ], @@ -1595,7 +1595,7 @@ }, "message": "autovacuum launcher shutting down", "event": { - "ingested": "2021-03-02T16:29:03.686010593Z", + "ingested": "2021-03-02T18:00:28.545668397Z", "category": [ "database" ], @@ -1621,7 +1621,7 @@ }, "message": "shutting down", "event": { - "ingested": "2021-03-02T16:29:03.686011989Z", + "ingested": "2021-03-02T18:00:28.545669339Z", "category": [ "database" ], @@ -1647,7 +1647,7 @@ }, "message": "database system is shut down", "event": { - "ingested": "2021-03-02T16:29:03.686013411Z", + "ingested": "2021-03-02T18:00:28.545670187Z", "category": [ "database" ], @@ -1673,7 +1673,7 @@ }, "message": "database system was shut down at 2017-06-06 07:54:13 CEST", "event": { - "ingested": "2021-03-02T16:29:03.686014825Z", + "ingested": "2021-03-02T18:00:28.545671020Z", "category": [ "database" ], @@ -1699,7 +1699,7 @@ }, "message": "MultiXact member wraparound protections are now enabled", "event": { - "ingested": "2021-03-02T16:29:03.686016165Z", + "ingested": "2021-03-02T18:00:28.545671930Z", "category": [ "database" ], @@ -1725,7 +1725,7 @@ }, "message": "database system is ready to accept connections", "event": { - "ingested": "2021-03-02T16:29:03.686018673Z", + "ingested": "2021-03-02T18:00:28.545672925Z", "category": [ "database" ], @@ -1751,7 +1751,7 @@ }, "message": "autovacuum launcher started", "event": { - "ingested": "2021-03-02T16:29:03.686019710Z", + "ingested": "2021-03-02T18:00:28.545673745Z", "category": [ "database" ], @@ -1783,7 +1783,7 @@ }, "message": "incomplete startup packet", "event": { - "ingested": "2021-03-02T16:29:03.686020684Z", + "ingested": "2021-03-02T18:00:28.545674656Z", "category": [ "database" ], @@ -1818,7 +1818,7 @@ }, "message": "incomplete startup packet", "event": { - "ingested": "2021-03-02T16:29:03.686021829Z", + "ingested": "2021-03-02T18:00:28.545675626Z", "category": [ "database" ], @@ -1847,7 +1847,7 @@ }, "message": "database system was shut down at 2017-06-10 19:37:29 CEST", "event": { - "ingested": "2021-03-02T16:29:03.686023340Z", + "ingested": "2021-03-02T18:00:28.545676616Z", "category": [ "database" ], @@ -1873,7 +1873,7 @@ }, "message": "MultiXact member wraparound protections are now enabled", "event": { - "ingested": "2021-03-02T16:29:03.686024659Z", + "ingested": "2021-03-02T18:00:28.545677628Z", "category": [ "database" ], @@ -1899,7 +1899,7 @@ }, "message": "database system is ready to accept connections", "event": { - "ingested": "2021-03-02T16:29:03.686026066Z", + "ingested": "2021-03-02T18:00:28.545678697Z", "category": [ "database" ], @@ -1925,7 +1925,7 @@ }, "message": "autovacuum launcher started", "event": { - "ingested": "2021-03-02T16:29:03.686027115Z", + "ingested": "2021-03-02T18:00:28.545679796Z", "category": [ "database" ], @@ -1957,7 +1957,7 @@ }, "message": "incomplete startup packet", "event": { - "ingested": "2021-03-02T16:29:03.686028794Z", + "ingested": "2021-03-02T18:00:28.545680883Z", "category": [ "database" ], @@ -1986,7 +1986,7 @@ }, "message": "received fast shutdown request", "event": { - "ingested": "2021-03-02T16:29:03.686032255Z", + "ingested": "2021-03-02T18:00:28.545681831Z", "category": [ "database" ], @@ -2012,7 +2012,7 @@ }, "message": "aborting any active transactions", "event": { - "ingested": "2021-03-02T16:29:03.686033596Z", + "ingested": "2021-03-02T18:00:28.545682839Z", "category": [ "database" ], @@ -2038,7 +2038,7 @@ }, "message": "autovacuum launcher shutting down", "event": { - "ingested": "2021-03-02T16:29:03.686034533Z", + "ingested": "2021-03-02T18:00:28.545683903Z", "category": [ "database" ], @@ -2064,7 +2064,7 @@ }, "message": "shutting down", "event": { - "ingested": "2021-03-02T16:29:03.686036062Z", + "ingested": "2021-03-02T18:00:28.545684974Z", "category": [ "database" ], @@ -2090,7 +2090,7 @@ }, "message": "database system is shut down", "event": { - "ingested": "2021-03-02T16:29:03.686036871Z", + "ingested": "2021-03-02T18:00:28.545685996Z", "category": [ "database" ], @@ -2116,7 +2116,7 @@ }, "message": "database system was shut down at 2017-06-10 20:27:55 CEST", "event": { - "ingested": "2021-03-02T16:29:03.686037721Z", + "ingested": "2021-03-02T18:00:28.545687040Z", "category": [ "database" ], @@ -2142,7 +2142,7 @@ }, "message": "MultiXact member wraparound protections are now enabled", "event": { - "ingested": "2021-03-02T16:29:03.686038653Z", + "ingested": "2021-03-02T18:00:28.545688145Z", "category": [ "database" ], @@ -2168,7 +2168,7 @@ }, "message": "autovacuum launcher started", "event": { - "ingested": "2021-03-02T16:29:03.686039523Z", + "ingested": "2021-03-02T18:00:28.545689345Z", "category": [ "database" ], @@ -2194,7 +2194,7 @@ }, "message": "database system is ready to accept connections", "event": { - "ingested": "2021-03-02T16:29:03.686040563Z", + "ingested": "2021-03-02T18:00:28.545690454Z", "category": [ "database" ], @@ -2226,7 +2226,7 @@ }, "message": "incomplete startup packet", "event": { - "ingested": "2021-03-02T16:29:03.686041717Z", + "ingested": "2021-03-02T18:00:28.545691662Z", "category": [ "database" ], @@ -2255,7 +2255,7 @@ }, "message": "received fast shutdown request", "event": { - "ingested": "2021-03-02T16:29:03.686042673Z", + "ingested": "2021-03-02T18:00:28.545692822Z", "category": [ "database" ], @@ -2281,7 +2281,7 @@ }, "message": "aborting any active transactions", "event": { - "ingested": "2021-03-02T16:29:03.686044263Z", + "ingested": "2021-03-02T18:00:28.545694120Z", "category": [ "database" ], @@ -2307,7 +2307,7 @@ }, "message": "autovacuum launcher shutting down", "event": { - "ingested": "2021-03-02T16:29:03.686045303Z", + "ingested": "2021-03-02T18:00:28.545697111Z", "category": [ "database" ], @@ -2333,7 +2333,7 @@ }, "message": "shutting down", "event": { - "ingested": "2021-03-02T16:29:03.686046146Z", + "ingested": "2021-03-02T18:00:28.545698369Z", "category": [ "database" ], @@ -2359,7 +2359,7 @@ }, "message": "database system is shut down", "event": { - "ingested": "2021-03-02T16:29:03.686047117Z", + "ingested": "2021-03-02T18:00:28.545699158Z", "category": [ "database" ], diff --git a/packages/postgresql/data_stream/log/fields/ecs.yml b/packages/postgresql/data_stream/log/fields/ecs.yml index ebd6d914156..e62ba0963e1 100644 --- a/packages/postgresql/data_stream/log/fields/ecs.yml +++ b/packages/postgresql/data_stream/log/fields/ecs.yml @@ -63,6 +63,29 @@ description: Process id. example: 4242 +- name: related + title: Related + group: 2 + description: 'This field set is meant to facilitate pivoting around a piece of + data. + + Some pieces of information can be seen in many places in an ECS event. To facilitate + searching for them, store an array of all seen values to their corresponding + field in `related.`. + + A concrete example is IP addresses, which can be under host, observer, source, + destination, client, server, and network.forwarded_ip. If you append all IPs + to `related.ip`, you can then search for a given IP trivially, no matter where + it appeared, by querying `related.ip:192.0.2.15`.' + type: group + fields: + - name: user + level: extended + type: keyword + ignore_above: 1024 + description: All the user names seen on your event. + default_field: false + - name: error title: Error group: 2