From bab56c7aa6833422c3d4718d62c140942966ef85 Mon Sep 17 00:00:00 2001 From: Marius Iversen Date: Mon, 14 Jun 2021 17:27:34 +0200 Subject: [PATCH] [Bluecoat] updating bluecoat ECS version and adding event.original options (#1072) * updating bluecoat ECS version and adding event.original options * linting, update changelog and manifest * adding checks for processors in template * linting --- packages/bluecoat/changelog.yml | 5 + .../_dev/test/pipeline/test-common-config.yml | 5 + .../_dev/test/pipeline/test-generated.log | 100 ++ .../pipeline/test-generated.log-expected.json | 1204 +++++++++++++++++ .../director/agent/stream/stream.yml.hbs | 13 +- .../director/agent/stream/tcp.yml.hbs | 15 +- .../director/agent/stream/udp.yml.hbs | 13 +- .../elasticsearch/ingest_pipeline/default.yml | 89 +- .../data_stream/director/manifest.yml | 51 + packages/bluecoat/manifest.yml | 2 +- 10 files changed, 1436 insertions(+), 61 deletions(-) create mode 100644 packages/bluecoat/data_stream/director/_dev/test/pipeline/test-common-config.yml create mode 100644 packages/bluecoat/data_stream/director/_dev/test/pipeline/test-generated.log create mode 100644 packages/bluecoat/data_stream/director/_dev/test/pipeline/test-generated.log-expected.json diff --git a/packages/bluecoat/changelog.yml b/packages/bluecoat/changelog.yml index 6584ebef1cf..b508d24742e 100644 --- a/packages/bluecoat/changelog.yml +++ b/packages/bluecoat/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.2.0" + changes: + - description: update to ECS 1.10.0, add event.original options, and preparing for fleet GA. + type: enhancement + link: https://github.com/elastic/integrations/pull/1072 - version: "0.1.4" changes: - description: update to ECS 1.9.0 diff --git a/packages/bluecoat/data_stream/director/_dev/test/pipeline/test-common-config.yml b/packages/bluecoat/data_stream/director/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..5622947e4b8 --- /dev/null +++ b/packages/bluecoat/data_stream/director/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,5 @@ +dynamic_fields: + event.ingested: ".*" +fields: + tags: + - preserve_original_event diff --git a/packages/bluecoat/data_stream/director/_dev/test/pipeline/test-generated.log b/packages/bluecoat/data_stream/director/_dev/test/pipeline/test-generated.log new file mode 100644 index 00000000000..6bf53ab9040 --- /dev/null +++ b/packages/bluecoat/data_stream/director/_dev/test/pipeline/test-generated.log @@ -0,0 +1,100 @@ +ntpd[1001]: kernel time sync enabled utl +restorecond: : Reset file context quasiarc: liqua +auditd[5699]: Audit daemon rotating log files +anacron[5066]: Normal exit ehend +restorecond: : Reset file context vol: luptat +heartbeat: : < Processing command: accept +restorecond: : Reset file context nci: ofdeFin +auditd[6668]: Audit daemon rotating log files +anacron[1613]: Normal exit mvolu +ntpd[2959]: ntpd gelit-r tatno +anacron[654]: Updated timestamp for job rmagni to sit +dmd: : < Health state for metric"seq3874.mail.domain" "quid" changed to "fug", reason: "success" +auditd[2067]: Audit daemon rotating log files +pm[5969]: < check_license_validity(), tae +logrotate: : ALERT exited abnormally with temUten +sshd: : < error: Bind to port Duisau on psum failed: failure +configd: : < itaut@rveli: command: accept +authd: : < authd_signal_handler(), quam +xinetd[6547]: Started working: onproide available services +logrotate: : ALERT exited abnormally with tfug +heartbeat: : < Processing command: deny +rsyslogd: : Warning: rehe +sshd: : < error: Bind to port erc on amqu failed: unknown +ntpd[4515]: ntpd emp-r aperia +restorecond: : Reset file context run: vol +logrotate: : ALERT exited abnormally with mporain +heartbeat: : < connect: atu +cmd: : < cmd starting adeseru +cli[7108]: <<-uam.low> tmo@::fficiade:10.2.53.125 : CLI launched +pm[7061]: < ntpd will start in tlabo +poller[795]: < Querying content system for job results. +runner[6134]: < Processing command: allow +epmd: : epmd: epmd running orpor +runner[602]: < Failed to exec olup +shutdown[2807]: shutting down non +configd: : < sperna@sintocc: command: cancel +auditd[2986]: Audit daemon rotating log files +configd: : < CREATE onsequ +auditd[1243]: Audit daemon rotating log files +xinetd[6599]: Started working: naal available services +xinetd[5850]: Started working: rQu available services +heartbeat: : < queips: undefined symbol: ncidi +authd: : < authd_close(): npr +anacron[6373]: Anacron 1.3962 started on epre +cli[3979]: <<-iduntu.medium> temUt@avol752.www5.test : Processing command accept +cmd: : < cmd starting isiuta +sshd[5227]: dutp(psaquaea:taevita): pam_putenv: ameiusm +ccd: : < Device elitse6672.internal.localdomain: mquisno +runner[1859]: < Failed to exec umSe +shutdown[6110]: shutting down itau +sshd[2415]: PAM lorsita more authentication failure; dolore +rsyslogd: : Warning: tio +cli[802]: <<-gnaaliqu.very-high> velillu@::cteturad:10.18.204.87 : Processing a secure command... +heartbeat: : < connect: inimveni +authd: : < authd_close(): psumqu +runner[2558]: < Failed to exec edquiac +anacron[4538]: Updated timestamp for job remips to uisaute +auditd[6837]: Audit daemon rotating log files +pm[1493]: < print_msg(), dic +configd: : < Device "itation4168.api.domain" completed command(s) accept ;; CPL generated by Visual Policy Manager: isciv ;rroqu ; nofd ; dipisci +epmd: : epmd: invalid packet size (mquae) +runner[429]: < File reading failed +shutdown[7595]: shutting down emqu +heartbeat: : < The HB command is accept +authd: : < authd_signal_handler(), isetquas +authd: : < authd_signal_handler(), gnaal +logrotate: : ALERT exited abnormally with voluptas +ntpd[627]: ntpd exiting on signal orin +restorecond: : Reset file context ecillu: mmodoc +cli[1140]: <<-abore.high> modocon@ipsu3680.mail.test : Processing command: deny +sshd: : bad username mquisn +ntpd[1313]: ntpd derit-r orese +ccd: : < Device Communication Daemon online +rsyslogd: : Warning: moles +restorecond: : Reset file context olup: aco +shutdown[609]: shutting down ser +ntpd[2991]: ntpd orinrep-r quiavol +dmd: : < inserted device id = sBonor2001.www5.example and serial number = amc into DB +ccd: : < ccd_handle_read_failure(), uid +cmd: : < cmd starting lmolesti +dmd: : < inserted device id = ersp6625.internal.domain and serial number = seq into DB +cmd: : < cmd starting uipexe +heartbeat: : < The HB command is cancel +anacron[7360]: Normal exit tperspic +dmd: : < Filter on (tetura) things. riosamni +ccd: : < Device eleumiu2454.api.local: tat +schedulerd: : < System time changed, recomputing job run times. +xinetd[3450]: Started working: aconsequ available services +authd: : < handle_authd unknown message =utemvel +rsyslogd: : Warning: iusm +ntpd[16]: time reset stquido +ccd: : < Device olu5333.www.domain: orumSe +anacron[80]: Normal exit ici +ntpd[7612]: kernel time sync enabled nturmag +cli[7128]: eseruntm(lpaquiof:oloreeu): pam_putenv: olor +schedulerd: : < Executing Job "tquo" execution iatnu +logrotate: : ALERT exited abnormally with ntut +poller[7151]: < Querying content system for job results. +ntpd[2314]: ntpd litanim-r rQuisaut +heartbeat: : < Processing command: block diff --git a/packages/bluecoat/data_stream/director/_dev/test/pipeline/test-generated.log-expected.json b/packages/bluecoat/data_stream/director/_dev/test/pipeline/test-generated.log-expected.json new file mode 100644 index 00000000000..cd50916a450 --- /dev/null +++ b/packages/bluecoat/data_stream/director/_dev/test/pipeline/test-generated.log-expected.json @@ -0,0 +1,1204 @@ +{ + "expected": [ + { + "ecs": { + "version": "1.10.0" + }, + "message": "ntpd[1001]: kernel time sync enabled utl", + "event": { + "ingested": "2021-06-09T09:50:38.142122500Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "restorecond: : Reset file context quasiarc: liqua", + "event": { + "ingested": "2021-06-09T09:50:38.142166900Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "auditd[5699]: Audit daemon rotating log files", + "event": { + "ingested": "2021-06-09T09:50:38.142174400Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "anacron[5066]: Normal exit ehend", + "event": { + "ingested": "2021-06-09T09:50:38.142200100Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "restorecond: : Reset file context vol: luptat", + "event": { + "ingested": "2021-06-09T09:50:38.142206600Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "heartbeat: : \u003c\u003ceumiu.medium\u003e Processing command: accept", + "event": { + "ingested": "2021-06-09T09:50:38.142212600Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "restorecond: : Reset file context nci: ofdeFin", + "event": { + "ingested": "2021-06-09T09:50:38.142219100Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "auditd[6668]: Audit daemon rotating log files", + "event": { + "ingested": "2021-06-09T09:50:38.142224500Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "anacron[1613]: Normal exit mvolu", + "event": { + "ingested": "2021-06-09T09:50:38.142229400Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "ntpd[2959]: ntpd gelit-r tatno", + "event": { + "ingested": "2021-06-09T09:50:38.142234600Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "anacron[654]: Updated timestamp for job rmagni to sit", + "event": { + "ingested": "2021-06-09T09:50:38.142240900Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "dmd: : \u003c\u003ctenima.very-high\u003e Health state for metric\"seq3874.mail.domain\" \"quid\" changed to \"fug\", reason: \"success\"", + "event": { + "ingested": "2021-06-09T09:50:38.142246200Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "auditd[2067]: Audit daemon rotating log files", + "event": { + "ingested": "2021-06-09T09:50:38.142251Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "pm[5969]: \u003c\u003ctquovol.very-high\u003e check_license_validity(), tae", + "event": { + "ingested": "2021-06-09T09:50:38.142256Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "logrotate: : ALERT exited abnormally with temUten", + "event": { + "ingested": "2021-06-09T09:50:38.142260800Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "sshd: : \u003c\u003cdun.medium\u003e error: Bind to port Duisau on psum failed: failure", + "event": { + "ingested": "2021-06-09T09:50:38.142265900Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "configd: : \u003c\u003cend.medium\u003e itaut@rveli: command: accept", + "event": { + "ingested": "2021-06-09T09:50:38.142270700Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "authd: : \u003c\u003cluptat.low\u003e authd_signal_handler(), quam", + "event": { + "ingested": "2021-06-09T09:50:38.142282300Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "xinetd[6547]: Started working: onproide available services", + "event": { + "ingested": "2021-06-09T09:50:38.142287Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "logrotate: : ALERT exited abnormally with tfug", + "event": { + "ingested": "2021-06-09T09:50:38.142292Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "heartbeat: : \u003c\u003curE.medium\u003e Processing command: deny", + "event": { + "ingested": "2021-06-09T09:50:38.142297200Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "rsyslogd: : Warning: rehe", + "event": { + "ingested": "2021-06-09T09:50:38.142302700Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "sshd: : \u003c\u003cstiae.medium\u003e error: Bind to port erc on amqu failed: unknown", + "event": { + "ingested": "2021-06-09T09:50:38.142369500Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "ntpd[4515]: ntpd emp-r aperia", + "event": { + "ingested": "2021-06-09T09:50:38.142377200Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "restorecond: : Reset file context run: vol", + "event": { + "ingested": "2021-06-09T09:50:38.142383100Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "logrotate: : ALERT exited abnormally with mporain", + "event": { + "ingested": "2021-06-09T09:50:38.142388100Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "heartbeat: : \u003c\u003cmpori.very-high\u003e connect: atu", + "event": { + "ingested": "2021-06-09T09:50:38.142393Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "cmd: : \u003c\u003ctexp.medium\u003e cmd starting adeseru", + "event": { + "ingested": "2021-06-09T09:50:38.142398200Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "cli[7108]: \u003c\u003c-uam.low\u003e tmo@::fficiade:10.2.53.125 : CLI launched", + "event": { + "ingested": "2021-06-09T09:50:38.142402700Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "pm[7061]: \u003c\u003cihilmo.very-high\u003e ntpd will start in tlabo", + "event": { + "ingested": "2021-06-09T09:50:38.142407300Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "poller[795]: \u003c\u003coluptate.low\u003e Querying content system for job results.", + "event": { + "ingested": "2021-06-09T09:50:38.142412Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "runner[6134]: \u003c\u003cedo.very-high\u003e Processing command: allow", + "event": { + "ingested": "2021-06-09T09:50:38.142416400Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "epmd: : epmd: epmd running orpor", + "event": { + "ingested": "2021-06-09T09:50:38.142420900Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "runner[602]: \u003c\u003cemvel.very-high\u003e Failed to exec olup", + "event": { + "ingested": "2021-06-09T09:50:38.142427300Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "shutdown[2807]: shutting down non", + "event": { + "ingested": "2021-06-09T09:50:38.142431800Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "configd: : \u003c\u003cugiatnu.high\u003e sperna@sintocc: command: cancel", + "event": { + "ingested": "2021-06-09T09:50:38.142436Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "auditd[2986]: Audit daemon rotating log files", + "event": { + "ingested": "2021-06-09T09:50:38.142440200Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "configd: : \u003c\u003cccaecat.medium\u003e CREATE onsequ", + "event": { + "ingested": "2021-06-09T09:50:38.142444200Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "auditd[1243]: Audit daemon rotating log files", + "event": { + "ingested": "2021-06-09T09:50:38.142448100Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "xinetd[6599]: Started working: naal available services", + "event": { + "ingested": "2021-06-09T09:50:38.142452100Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "xinetd[5850]: Started working: rQu available services", + "event": { + "ingested": "2021-06-09T09:50:38.142456100Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "heartbeat: : \u003c\u003cboree.low\u003e queips: undefined symbol: ncidi", + "event": { + "ingested": "2021-06-09T09:50:38.142460100Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "authd: : \u003c\u003color.very-high\u003e authd_close(): npr", + "event": { + "ingested": "2021-06-09T09:50:38.142469300Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "anacron[6373]: Anacron 1.3962 started on epre", + "event": { + "ingested": "2021-06-09T09:50:38.142473500Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "cli[3979]: \u003c\u003c-iduntu.medium\u003e temUt@avol752.www5.test : Processing command accept", + "event": { + "ingested": "2021-06-09T09:50:38.142477500Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "cmd: : \u003c\u003camc.medium\u003e cmd starting isiuta", + "event": { + "ingested": "2021-06-09T09:50:38.142481300Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "sshd[5227]: dutp(psaquaea:taevita): pam_putenv: ameiusm", + "event": { + "ingested": "2021-06-09T09:50:38.142485Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "ccd: : \u003c\u003colab.low\u003e Device elitse6672.internal.localdomain: mquisno", + "event": { + "ingested": "2021-06-09T09:50:38.142488800Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "runner[1859]: \u003c\u003ctasnulap.high\u003e Failed to exec umSe", + "event": { + "ingested": "2021-06-09T09:50:38.142492600Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "shutdown[6110]: shutting down itau", + "event": { + "ingested": "2021-06-09T09:50:38.142496400Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "sshd[2415]: PAM lorsita more authentication failure; dolore", + "event": { + "ingested": "2021-06-09T09:50:38.142500700Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "rsyslogd: : Warning: tio", + "event": { + "ingested": "2021-06-09T09:50:38.142504800Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "cli[802]: \u003c\u003c-gnaaliqu.very-high\u003e velillu@::cteturad:10.18.204.87 : Processing a secure command...", + "event": { + "ingested": "2021-06-09T09:50:38.142508700Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "heartbeat: : \u003c\u003creprehe.high\u003e connect: inimveni", + "event": { + "ingested": "2021-06-09T09:50:38.142512400Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "authd: : \u003c\u003clitani.low\u003e authd_close(): psumqu", + "event": { + "ingested": "2021-06-09T09:50:38.142516200Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "runner[2558]: \u003c\u003cicabo.high\u003e Failed to exec edquiac", + "event": { + "ingested": "2021-06-09T09:50:38.142520100Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "anacron[4538]: Updated timestamp for job remips to uisaute", + "event": { + "ingested": "2021-06-09T09:50:38.142523900Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "auditd[6837]: Audit daemon rotating log files", + "event": { + "ingested": "2021-06-09T09:50:38.142527800Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "pm[1493]: \u003c\u003cetdolor.high\u003e print_msg(), dic", + "event": { + "ingested": "2021-06-09T09:50:38.142531800Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "configd: : \u003c\u003cavolupt.low\u003e Device \"itation4168.api.domain\" completed command(s) accept ;; CPL generated by Visual Policy Manager: isciv ;rroqu ; nofd ; dipisci", + "event": { + "ingested": "2021-06-09T09:50:38.142535500Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "epmd: : epmd: invalid packet size (mquae)", + "event": { + "ingested": "2021-06-09T09:50:38.142539300Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "runner[429]: \u003c\u003ccorpori.very-high\u003e File reading failed", + "event": { + "ingested": "2021-06-09T09:50:38.142543Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "shutdown[7595]: shutting down emqu", + "event": { + "ingested": "2021-06-09T09:50:38.142547100Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "heartbeat: : \u003c\u003cleumiur.low\u003e The HB command is accept", + "event": { + "ingested": "2021-06-09T09:50:38.142550900Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "authd: : \u003c\u003cest.very-high\u003e authd_signal_handler(), isetquas", + "event": { + "ingested": "2021-06-09T09:50:38.142554800Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "authd: : \u003c\u003cpsaqua.medium\u003e authd_signal_handler(), gnaal", + "event": { + "ingested": "2021-06-09T09:50:38.142558700Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "logrotate: : ALERT exited abnormally with voluptas", + "event": { + "ingested": "2021-06-09T09:50:38.142562800Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "ntpd[627]: ntpd exiting on signal orin", + "event": { + "ingested": "2021-06-09T09:50:38.142566500Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "restorecond: : Reset file context ecillu: mmodoc", + "event": { + "ingested": "2021-06-09T09:50:38.142570200Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "cli[1140]: \u003c\u003c-abore.high\u003e modocon@ipsu3680.mail.test : Processing command: deny", + "event": { + "ingested": "2021-06-09T09:50:38.142573900Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "sshd: : bad username mquisn", + "event": { + "ingested": "2021-06-09T09:50:38.142577700Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "ntpd[1313]: ntpd derit-r orese", + "event": { + "ingested": "2021-06-09T09:50:38.142581500Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "ccd: : \u003c\u003cleumiur.medium\u003e Device Communication Daemon online", + "event": { + "ingested": "2021-06-09T09:50:38.142586500Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "rsyslogd: : Warning: moles", + "event": { + "ingested": "2021-06-09T09:50:38.142590500Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "restorecond: : Reset file context olup: aco", + "event": { + "ingested": "2021-06-09T09:50:38.142595400Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "shutdown[609]: shutting down ser", + "event": { + "ingested": "2021-06-09T09:50:38.142599200Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "ntpd[2991]: ntpd orinrep-r quiavol", + "event": { + "ingested": "2021-06-09T09:50:38.142603100Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "dmd: : \u003c\u003cquin.medium\u003e inserted device id = sBonor2001.www5.example and serial number = amc into DB", + "event": { + "ingested": "2021-06-09T09:50:38.142607Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "ccd: : \u003c\u003came.very-high\u003e ccd_handle_read_failure(), uid", + "event": { + "ingested": "2021-06-09T09:50:38.142610700Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "cmd: : \u003c\u003cscivel.high\u003e cmd starting lmolesti", + "event": { + "ingested": "2021-06-09T09:50:38.142614500Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "dmd: : \u003c\u003cemaperia.high\u003e inserted device id = ersp6625.internal.domain and serial number = seq into DB", + "event": { + "ingested": "2021-06-09T09:50:38.142618Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "cmd: : \u003c\u003ctanimid.medium\u003e cmd starting uipexe", + "event": { + "ingested": "2021-06-09T09:50:38.142621800Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "heartbeat: : \u003c\u003core.low\u003e The HB command is cancel", + "event": { + "ingested": "2021-06-09T09:50:38.142625300Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "anacron[7360]: Normal exit tperspic", + "event": { + "ingested": "2021-06-09T09:50:38.142629Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "dmd: : \u003c\u003cict.very-high\u003e Filter on (tetura) things. riosamni", + "event": { + "ingested": "2021-06-09T09:50:38.142633200Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "ccd: : \u003c\u003cumetMa.low\u003e Device eleumiu2454.api.local: tat", + "event": { + "ingested": "2021-06-09T09:50:38.142637800Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "schedulerd: : \u003c\u003clumqu.very-high\u003e System time changed, recomputing job run times.", + "event": { + "ingested": "2021-06-09T09:50:38.142642100Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "xinetd[3450]: Started working: aconsequ available services", + "event": { + "ingested": "2021-06-09T09:50:38.142646300Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "authd: : \u003c\u003csequat.high\u003e handle_authd unknown message =utemvel", + "event": { + "ingested": "2021-06-09T09:50:38.142650400Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "rsyslogd: : Warning: iusm", + "event": { + "ingested": "2021-06-09T09:50:38.142671300Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "ntpd[16]: time reset stquido", + "event": { + "ingested": "2021-06-09T09:50:38.142683500Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "ccd: : \u003c\u003caaliq.high\u003e Device olu5333.www.domain: orumSe", + "event": { + "ingested": "2021-06-09T09:50:38.142689Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "anacron[80]: Normal exit ici", + "event": { + "ingested": "2021-06-09T09:50:38.142693600Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "ntpd[7612]: kernel time sync enabled nturmag", + "event": { + "ingested": "2021-06-09T09:50:38.142698Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "cli[7128]: eseruntm(lpaquiof:oloreeu): pam_putenv: olor", + "event": { + "ingested": "2021-06-09T09:50:38.142702Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "schedulerd: : \u003c\u003cici.very-high\u003e Executing Job \"tquo\" execution iatnu", + "event": { + "ingested": "2021-06-09T09:50:38.142705800Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "logrotate: : ALERT exited abnormally with ntut", + "event": { + "ingested": "2021-06-09T09:50:38.142709500Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "poller[7151]: \u003c\u003cess.high\u003e Querying content system for job results.", + "event": { + "ingested": "2021-06-09T09:50:38.142713200Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "ntpd[2314]: ntpd litanim-r rQuisaut", + "event": { + "ingested": "2021-06-09T09:50:38.142716900Z" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "ecs": { + "version": "1.10.0" + }, + "message": "heartbeat: : \u003c\u003cmetco.high\u003e Processing command: block", + "event": { + "ingested": "2021-06-09T09:50:38.142724Z" + }, + "tags": [ + "preserve_original_event" + ] + } + ] +} \ No newline at end of file diff --git a/packages/bluecoat/data_stream/director/agent/stream/stream.yml.hbs b/packages/bluecoat/data_stream/director/agent/stream/stream.yml.hbs index dd2a81aa35e..6e7b34d9097 100644 --- a/packages/bluecoat/data_stream/director/agent/stream/stream.yml.hbs +++ b/packages/bluecoat/data_stream/director/agent/stream/stream.yml.hbs @@ -4,8 +4,11 @@ paths: {{/each}} exclude_files: [".gz$"] tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#each tags as |tag i|}} - - {{tag}} + - {{tag}} {{/each}} fields_under_root: true fields: @@ -16,8 +19,10 @@ fields: {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} - processors: +{{#if processors}} +{{processors}} +{{/if}} - script: lang: javascript params: @@ -3810,7 +3815,3 @@ processors: target_subdomain_field: url.subdomain target_etld_field: url.top_level_domain - add_locale: ~ -- add_fields: - target: '' - fields: - ecs.version: 1.9.0 diff --git a/packages/bluecoat/data_stream/director/agent/stream/tcp.yml.hbs b/packages/bluecoat/data_stream/director/agent/stream/tcp.yml.hbs index 69230198b79..64143bb9307 100644 --- a/packages/bluecoat/data_stream/director/agent/stream/tcp.yml.hbs +++ b/packages/bluecoat/data_stream/director/agent/stream/tcp.yml.hbs @@ -1,8 +1,11 @@ tcp: host: "{{tcp_host}}:{{tcp_port}}" tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#each tags as |tag i|}} - - {{tag}} + - {{tag}} {{/each}} fields_under_root: true fields: @@ -13,8 +16,10 @@ fields: {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} - processors: +{{#if processors}} +{{processors}} +{{/if}} - script: lang: javascript params: @@ -3806,8 +3811,4 @@ processors: target_field: url.registered_domain target_subdomain_field: url.subdomain target_etld_field: url.top_level_domain -- add_locale: ~ -- add_fields: - target: '' - fields: - ecs.version: 1.9.0 +- add_locale: ~ \ No newline at end of file diff --git a/packages/bluecoat/data_stream/director/agent/stream/udp.yml.hbs b/packages/bluecoat/data_stream/director/agent/stream/udp.yml.hbs index 8bac284ac89..b6f20c28cb6 100644 --- a/packages/bluecoat/data_stream/director/agent/stream/udp.yml.hbs +++ b/packages/bluecoat/data_stream/director/agent/stream/udp.yml.hbs @@ -1,8 +1,11 @@ udp: host: "{{udp_host}}:{{udp_port}}" tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#each tags as |tag i|}} - - {{tag}} + - {{tag}} {{/each}} fields_under_root: true fields: @@ -13,8 +16,10 @@ fields: {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} - processors: +{{#if processors}} +{{processors}} +{{/if}} - script: lang: javascript params: @@ -3807,7 +3812,3 @@ processors: target_subdomain_field: url.subdomain target_etld_field: url.top_level_domain - add_locale: ~ -- add_fields: - target: '' - fields: - ecs.version: 1.9.0 diff --git a/packages/bluecoat/data_stream/director/elasticsearch/ingest_pipeline/default.yml b/packages/bluecoat/data_stream/director/elasticsearch/ingest_pipeline/default.yml index 5a0a60c8971..d28e60fe292 100644 --- a/packages/bluecoat/data_stream/director/elasticsearch/ingest_pipeline/default.yml +++ b/packages/bluecoat/data_stream/director/elasticsearch/ingest_pipeline/default.yml @@ -4,60 +4,67 @@ description: Pipeline for Blue Coat Director processors: # ECS event.ingested - set: - field: event.ingested - value: '{{_ingest.timestamp}}' + field: event.ingested + value: '{{_ingest.timestamp}}' + - set: + field: ecs.version + value: '1.10.0' # User agent - user_agent: - field: user_agent.original - ignore_missing: true + field: user_agent.original + ignore_missing: true # IP Geolocation Lookup - geoip: - field: source.ip - target_field: source.geo - ignore_missing: true + field: source.ip + target_field: source.geo + ignore_missing: true - geoip: - field: destination.ip - target_field: destination.geo - ignore_missing: true - + field: destination.ip + target_field: destination.geo + ignore_missing: true # IP Autonomous System (AS) Lookup - geoip: - database_file: GeoLite2-ASN.mmdb - field: source.ip - target_field: source.as - properties: - - asn - - organization_name - ignore_missing: true + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true - geoip: - database_file: GeoLite2-ASN.mmdb - field: destination.ip - target_field: destination.as - properties: - - asn - - organization_name - ignore_missing: true + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true - rename: - field: source.as.asn - target_field: source.as.number - ignore_missing: true + field: source.as.asn + target_field: source.as.number + ignore_missing: true - rename: - field: source.as.organization_name - target_field: source.as.organization.name - ignore_missing: true + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true - rename: - field: destination.as.asn - target_field: destination.as.number - ignore_missing: true + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true - rename: - field: destination.as.organization_name - target_field: destination.as.organization.name - ignore_missing: true + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true - append: - field: related.hosts - value: '{{host.name}}' - allow_duplicates: false - if: ctx.host?.name != null && ctx.host?.name != '' + field: related.hosts + value: '{{host.name}}' + allow_duplicates: false + if: ctx.host?.name != null && ctx.host?.name != '' + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - append: field: error.message diff --git a/packages/bluecoat/data_stream/director/manifest.yml b/packages/bluecoat/data_stream/director/manifest.yml index 50359be2912..26ed4bbbd8e 100644 --- a/packages/bluecoat/data_stream/director/manifest.yml +++ b/packages/bluecoat/data_stream/director/manifest.yml @@ -54,6 +54,23 @@ streams: required: false show_user: false default: false + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - input: tcp title: Blue Coat Director logs description: Collect Blue Coat Director logs @@ -106,6 +123,23 @@ streams: required: false show_user: false default: false + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - input: logfile enabled: false title: Blue Coat Director logs @@ -152,3 +186,20 @@ streams: required: false show_user: false default: false + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + diff --git a/packages/bluecoat/manifest.yml b/packages/bluecoat/manifest.yml index 478b761ab16..00a9af14994 100644 --- a/packages/bluecoat/manifest.yml +++ b/packages/bluecoat/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: bluecoat title: Blue Coat Director -version: 0.1.4 +version: 0.2.0 description: Blue Coat Director Integration categories: ["network", "security"] release: experimental