Skip to content

Commit

Permalink
Add config
Browse files Browse the repository at this point in the history
  • Loading branch information
@jsoriano
jsoriano committed Mar 2, 2021
1 parent efc85f5 commit 9d3ac0d
Show file tree
Hide file tree
Showing 41 changed files with 433 additions and 361 deletions.
2 changes: 2 additions & 0 deletions packages/postgresql/data_stream/log/_dev/test/pipeline/postgresql-10-default.log-config.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
dynamic_fields:
event.ingested: ".*"
40 changes: 20 additions & 20 deletions ...ges/postgresql/data_stream/log/_dev/test/pipeline/postgresql-10-default.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
},
"message": "database system was shut down at 2020-04-15 12:02:52 CEST",
"event": {
"ingested": "2021-03-02T16:28:58.307179014Z",
"ingested": "2021-03-02T18:00:24.895666821Z",
"category": [
"database"
],
Expand All @@ -41,7 +41,7 @@
},
"message": "database system is ready to accept connections",
"event": {
"ingested": "2021-03-02T16:28:58.307184002Z",
"ingested": "2021-03-02T18:00:24.895670340Z",
"category": [
"database"
],
Expand All @@ -67,7 +67,7 @@
},
"message": "password authentication failed for user \"root\"",
"event": {
"ingested": "2021-03-02T16:28:58.307185422Z",
"ingested": "2021-03-02T18:00:24.895671356Z",
"category": [
"database"
],
Expand All @@ -93,7 +93,7 @@
},
"message": "Role \"root\" does not exist.",
"event": {
"ingested": "2021-03-02T16:28:58.307186614Z",
"ingested": "2021-03-02T18:00:24.895677357Z",
"category": [
"database"
],
Expand All @@ -107,7 +107,7 @@
{
"message": "\tConnection matched pg_hba.conf line 80: \"local all all md5\"",
"event": {
"ingested": "2021-03-02T16:28:58.307187706Z"
"ingested": "2021-03-02T18:00:24.895677996Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [\\tConnection matched pg_hba.conf line 80: \\\"local all all md5\\\"]"
Expand All @@ -128,7 +128,7 @@
},
"message": "could not send data to client: Broken pipe",
"event": {
"ingested": "2021-03-02T16:28:58.307188841Z",
"ingested": "2021-03-02T18:00:24.895678689Z",
"category": [
"database"
],
Expand All @@ -154,7 +154,7 @@
},
"message": "syntax error at or near \"l\" at character 1",
"event": {
"ingested": "2021-03-02T16:28:58.307189891Z",
"ingested": "2021-03-02T18:00:24.895679223Z",
"category": [
"database"
],
Expand All @@ -180,7 +180,7 @@
},
"message": "SELECT al.id, al.tenant_id, al.created_by_id, al.create_ip, al.audit_date, al.audit_table, al.entity_id, al.entity_name, al.reason_for_change, al.audit_log_event_type_id,",
"event": {
"ingested": "2021-03-02T16:28:58.307191102Z",
"ingested": "2021-03-02T18:00:24.895679762Z",
"category": [
"database"
],
Expand All @@ -194,7 +194,7 @@
{
"message": "\t aet.lookup_code, al.old_value, al.new_value, al.event_crf_id, al.event_crf_version_id, al.study_id, al.study_site_id, ss.rc_oid, al.subject_id, s.unique_identifier,",
"event": {
"ingested": "2021-03-02T16:28:58.307192192Z"
"ingested": "2021-03-02T18:00:24.895680287Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [\\t aet.lookup_code, al.old_value, al.new_value, al.event_crf_id, al.event_crf_version_id, al.study_id, al.study_site_id, ss.rc_oid, al.subject_id, s.unique_identifier,]"
Expand All @@ -203,7 +203,7 @@
{
"message": "\t al.study_event_id, sed.name AS studyEventName, al.user_id, al.value_index, al.crf_version_id, al.global_logs, cv.version_name, crf.id AS crfId, crf.name AS crfName",
"event": {
"ingested": "2021-03-02T16:28:58.307193302Z"
"ingested": "2021-03-02T18:00:24.895680882Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [\\t al.study_event_id, sed.name AS studyEventName, al.user_id, al.value_index, al.crf_version_id, al.global_logs, cv.version_name, crf.id AS crfId, crf.name AS crfName]"
Expand All @@ -212,7 +212,7 @@
{
"message": "\t FROM public.rc_audit_log_events AS al",
"event": {
"ingested": "2021-03-02T16:28:58.307194550Z"
"ingested": "2021-03-02T18:00:24.895681479Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [\\t FROM public.rc_audit_log_events AS al]"
Expand All @@ -221,7 +221,7 @@
{
"message": "\t LEFT JOIN rc_crf_versions AS cv ON cv.id=al.crf_version_id",
"event": {
"ingested": "2021-03-02T16:28:58.307195990Z"
"ingested": "2021-03-02T18:00:24.895682274Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [\\t LEFT JOIN rc_crf_versions AS cv ON cv.id=al.crf_version_id]"
Expand All @@ -230,7 +230,7 @@
{
"message": "\t LEFT JOIN rc_crfs AS crf ON crf.id=cv.crf_id",
"event": {
"ingested": "2021-03-02T16:28:58.307197190Z"
"ingested": "2021-03-02T18:00:24.895682816Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [\\t LEFT JOIN rc_crfs AS crf ON crf.id=cv.crf_id]"
Expand All @@ -239,7 +239,7 @@
{
"message": "\t LEFT JOIN ad_lookup_codes AS aet ON aet.id=al.audit_log_event_type_id",
"event": {
"ingested": "2021-03-02T16:28:58.307198348Z"
"ingested": "2021-03-02T18:00:24.895683352Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [\\t LEFT JOIN ad_lookup_codes AS aet ON aet.id=al.audit_log_event_type_id]"
Expand All @@ -248,7 +248,7 @@
{
"message": "\t LEFT JOIN rc_study_sites AS ss ON ss.id=al.study_site_id",
"event": {
"ingested": "2021-03-02T16:28:58.307199396Z"
"ingested": "2021-03-02T18:00:24.895683885Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [\\t LEFT JOIN rc_study_sites AS ss ON ss.id=al.study_site_id]"
Expand All @@ -257,7 +257,7 @@
{
"message": "\t LEFT JOIN rc_subjects AS s ON s.id=al.subject_id",
"event": {
"ingested": "2021-03-02T16:28:58.307205864Z"
"ingested": "2021-03-02T18:00:24.895684421Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [\\t LEFT JOIN rc_subjects AS s ON s.id=al.subject_id]"
Expand All @@ -266,7 +266,7 @@
{
"message": "\t LEFT JOIN rc_study_events AS se ON se.id=al.study_event_id",
"event": {
"ingested": "2021-03-02T16:28:58.307207269Z"
"ingested": "2021-03-02T18:00:24.895685073Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [\\t LEFT JOIN rc_study_events AS se ON se.id=al.study_event_id]"
Expand All @@ -275,7 +275,7 @@
{
"message": "\t LEFT JOIN rc_study_event_definitions AS sed ON sed.id=se.study_event_definition_id",
"event": {
"ingested": "2021-03-02T16:28:58.307208395Z"
"ingested": "2021-03-02T18:00:24.895685603Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [\\t LEFT JOIN rc_study_event_definitions AS sed ON sed.id=se.study_event_definition_id]"
Expand All @@ -284,7 +284,7 @@
{
"message": "\t WHERE al.tenant_id=$1 AND al.study_id=$2 AND aet.lookup_code IN ($3, $4, $5, $6) AND al.audit_date \u003e= $7 ORDER BY al.id DESC limit $8",
"event": {
"ingested": "2021-03-02T16:28:58.307209525Z"
"ingested": "2021-03-02T18:00:24.895686138Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [\\t WHERE al.tenant_id=$1 AND al.study_id=$2 AND aet.lookup_code IN ($3, $4, $5, $6) AND al.audit_date \u003e= $7 ORDER BY al.id DESC limit $8]"
Expand All @@ -293,7 +293,7 @@
{
"message": "\t;",
"event": {
"ingested": "2021-03-02T16:28:58.307210711Z"
"ingested": "2021-03-02T18:00:24.895686673Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [\\t;]"
Expand Down
2 changes: 2 additions & 0 deletions ...ql/data_stream/log/_dev/test/pipeline/postgresql-10-min-duration-statement.log-config.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
dynamic_fields:
event.ingested: ".*"
34 changes: 17 additions & 17 deletions ...data_stream/log/_dev/test/pipeline/postgresql-10-min-duration-statement.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
},
"message": "parameters: $1 = '86', $2 = '575', $3 = 'Item Inserted', $4 = 'Item Updated', $5 = 'Subject Updated', $6 = 'Subject Created', $7 = '2019-01-22 00:00:00+00'",
"event": {
"ingested": "2021-03-02T16:28:58.746470834Z",
"ingested": "2021-03-02T18:00:24.971645777Z",
"category": [
"database"
],
Expand Down Expand Up @@ -40,7 +40,7 @@
"message": "2019-09-22 06:28:24 UTC LOG: duration: 112.337 ms execute S_59: UPDATE qrtz_TRIGGERS SET TRIGGER_STATE = $1 WHERE SCHED_NAME = 'Scheduler_1' AND TRIGGER_NAME = $2 AND TRIGGER_GROUP = $3 AND TRIGGER_STATE = $4",
"event": {
"duration": 112337000,
"ingested": "2021-03-02T16:28:58.746504770Z",
"ingested": "2021-03-02T18:00:24.971649364Z",
"category": [
"database"
],
Expand All @@ -63,7 +63,7 @@
},
"message": "parameters: $1 = 'ACQUIRED', $2 = 'surveyInvitation_3Prbn85DiBWe8wHa_158802_77133_1260104', $3 = 'ExecutorsService', $4 = 'WAITING'",
"event": {
"ingested": "2021-03-02T16:28:58.746507099Z",
"ingested": "2021-03-02T18:00:24.971650097Z",
"category": [
"database"
],
Expand Down Expand Up @@ -91,7 +91,7 @@
"message": "2019-09-22 06:28:24 UTC LOG: duration: 2474.307 ms execute S_30: SELECT * FROM qrtz_LOCKS WHERE SCHED_NAME = 'Scheduler_1' AND LOCK_NAME = $1 FOR UPDATE",
"event": {
"duration": 2474306816,
"ingested": "2021-03-02T16:28:58.746510822Z",
"ingested": "2021-03-02T18:00:24.971650668Z",
"category": [
"database"
],
Expand All @@ -114,7 +114,7 @@
},
"message": "parameters: $1 = 'TRIGGER_ACCESS'",
"event": {
"ingested": "2021-03-02T16:28:58.746511969Z",
"ingested": "2021-03-02T18:00:24.971651235Z",
"category": [
"database"
],
Expand Down Expand Up @@ -142,7 +142,7 @@
"message": "2019-09-22 06:28:24 UTC LOG: duration: 18.327 ms execute S_32: SELECT al.id, al.tenant_id, al.created_by_id, al.create_ip, al.audit_date, al.audit_table, al.entity_id, al.entity_name, al.reason_for_change, al.audit_log_event_type_id,",
"event": {
"duration": 18327000,
"ingested": "2021-03-02T16:28:58.746513027Z",
"ingested": "2021-03-02T18:00:24.971651804Z",
"category": [
"database"
],
Expand All @@ -156,7 +156,7 @@
{
"message": " aet.lookup_code, al.old_value, al.new_value, al.event_crf_id, al.event_crf_version_id, al.study_id, al.study_site_id, ss.rc_oid, al.subject_id, s.unique_identifier,",
"event": {
"ingested": "2021-03-02T16:28:58.746513977Z"
"ingested": "2021-03-02T18:00:24.971652351Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [ aet.lookup_code, al.old_value, al.new_value, al.event_crf_id, al.event_crf_version_id, al.study_id, al.study_site_id, ss.rc_oid, al.subject_id, s.unique_identifier,]"
Expand All @@ -165,7 +165,7 @@
{
"message": " al.study_event_id, sed.name AS studyEventName, al.user_id, al.value_index, al.crf_version_id, al.global_logs, cv.version_name, crf.id AS crfId, crf.name AS crfName",
"event": {
"ingested": "2021-03-02T16:28:58.746514943Z"
"ingested": "2021-03-02T18:00:24.971652903Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [ al.study_event_id, sed.name AS studyEventName, al.user_id, al.value_index, al.crf_version_id, al.global_logs, cv.version_name, crf.id AS crfId, crf.name AS crfName]"
Expand All @@ -174,7 +174,7 @@
{
"message": " FROM public.rc_audit_log_events AS al",
"event": {
"ingested": "2021-03-02T16:28:58.746515935Z"
"ingested": "2021-03-02T18:00:24.971653454Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [ FROM public.rc_audit_log_events AS al]"
Expand All @@ -183,7 +183,7 @@
{
"message": " LEFT JOIN rc_crf_versions AS cv ON cv.id=al.crf_version_id",
"event": {
"ingested": "2021-03-02T16:28:58.746517053Z"
"ingested": "2021-03-02T18:00:24.971654003Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [ LEFT JOIN rc_crf_versions AS cv ON cv.id=al.crf_version_id]"
Expand All @@ -192,7 +192,7 @@
{
"message": " LEFT JOIN rc_crfs AS crf ON crf.id=cv.crf_id",
"event": {
"ingested": "2021-03-02T16:28:58.746518092Z"
"ingested": "2021-03-02T18:00:24.971654555Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [ LEFT JOIN rc_crfs AS crf ON crf.id=cv.crf_id]"
Expand All @@ -201,7 +201,7 @@
{
"message": " LEFT JOIN ad_lookup_codes AS aet ON aet.id=al.audit_log_event_type_id",
"event": {
"ingested": "2021-03-02T16:28:58.746519302Z"
"ingested": "2021-03-02T18:00:24.971655262Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [ LEFT JOIN ad_lookup_codes AS aet ON aet.id=al.audit_log_event_type_id]"
Expand All @@ -210,7 +210,7 @@
{
"message": " LEFT JOIN rc_study_sites AS ss ON ss.id=al.study_site_id",
"event": {
"ingested": "2021-03-02T16:28:58.746520180Z"
"ingested": "2021-03-02T18:00:24.971655826Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [ LEFT JOIN rc_study_sites AS ss ON ss.id=al.study_site_id]"
Expand All @@ -219,7 +219,7 @@
{
"message": " LEFT JOIN rc_subjects AS s ON s.id=al.subject_id",
"event": {
"ingested": "2021-03-02T16:28:58.746521100Z"
"ingested": "2021-03-02T18:00:24.971656374Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [ LEFT JOIN rc_subjects AS s ON s.id=al.subject_id]"
Expand All @@ -228,7 +228,7 @@
{
"message": " LEFT JOIN rc_study_events AS se ON se.id=al.study_event_id",
"event": {
"ingested": "2021-03-02T16:28:58.746522025Z"
"ingested": "2021-03-02T18:00:24.971656926Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [ LEFT JOIN rc_study_events AS se ON se.id=al.study_event_id]"
Expand All @@ -237,7 +237,7 @@
{
"message": " LEFT JOIN rc_study_event_definitions AS sed ON sed.id=se.study_event_definition_id",
"event": {
"ingested": "2021-03-02T16:28:58.746523020Z"
"ingested": "2021-03-02T18:00:24.971657480Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [ LEFT JOIN rc_study_event_definitions AS sed ON sed.id=se.study_event_definition_id]"
Expand All @@ -246,7 +246,7 @@
{
"message": " WHERE al.tenant_id=$1 AND al.study_id=$2 AND aet.lookup_code IN ($3, $4, $5, $6) AND al.audit_date \u003e= $7 ORDER BY al.id DESC limit $8",
"event": {
"ingested": "2021-03-02T16:28:58.746524337Z"
"ingested": "2021-03-02T18:00:24.971658155Z"
},
"error": {
"message": "Provided Grok expressions do not match field value: [ WHERE al.tenant_id=$1 AND al.study_id=$2 AND aet.lookup_code IN ($3, $4, $5, $6) AND al.audit_date \u003e= $7 ORDER BY al.id DESC limit $8]"
Expand Down
5 changes: 5 additions & 0 deletions ...postgresql/data_stream/log/_dev/test/pipeline/postgresql-11-autovacuum.csv.log-config.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
numeric_keyword_fields:
- "postgresql.log.transaction_id"
- "process.pid"
dynamic_fields:
event.ingested: ".*"
Loading

0 comments on commit 9d3ac0d

Please sign in to comment.