Filebeat sophos/utm: Support logs with hostname in syslog header (#28638

) Updates the sophos/utm datastream to support logs that include a hostname in their syslog header. (cherry picked from commit 71204c2)
elastic · Oct 29, 2021 · a95b8a6 · a95b8a6
1 parent b3789b5
commit a95b8a6
Show file tree

Hide file tree

Showing 4 changed files with 122 additions and 105 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -468,6 +468,9 @@ for a few releases. Please use other tools provided by Elastic to fetch data fro
 - Add latency config option for aws-cloudwatch input. {pull}28509[28509]
 - Added proxy support to threatintel/malwarebazaar. {pull}28533[28533]
 
+- Add `text/csv` decoder to `httpjson` input {pull}28564[28564]
+- Update `aws-s3` input to connect to non AWS S3 buckets {issue}28222[28222] {pull}28234[28234]
+- Sophos UTM: Support logs containing hostname in syslog header. {pull}28638[28638]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/sophos/utm/config/pipeline.js b/x-pack/filebeat/module/sophos/utm/config/pipeline.js
@@ -169,11 +169,11 @@ var dup46 = lookup({
 	key: dup15,
 });
 
-var hdr1 = match("HEADER#0:0001", "message", "%{hfld1->} %{messageid}[%{process_id}]: %{payload}", processor_chain([
+var hdr1 = match("HEADER#0:0001", "message", "%{hfld1->} %{hostname->} %{messageid}[%{process_id}]: %{payload}", processor_chain([
 	setc("header_id","0001"),
 ]));
 
-var hdr2 = match("HEADER#1:0002", "message", "%{hfld1->} %{hostname->} %{messageid}[%{process_id}]: %{payload}", processor_chain([
+var hdr2 = match("HEADER#1:0002", "message", "%{hfld1->} %{messageid}[%{process_id}]: %{payload}", processor_chain([
 	setc("header_id","0002"),
 ]));
 

diff --git a/x-pack/filebeat/module/sophos/utm/test/generated.log b/x-pack/filebeat/module/sophos/utm/test/generated.log
@@ -1,9 +1,9 @@
-2016:1:29-06:09:59 smtpd[905]: MASTER[nnumqua]: QR globally disabled, status one set to 'disabled'
+2016:1:29-06:09:59 localhost.localdomain smtpd[905]: MASTER[nnumqua]: QR globally disabled, status one set to 'disabled'
 2016:2:12-13:12:33 astarosg_TVM[5716]: id=ommod severity=medium sys=inima sub=tlabo name=web request blocked, forbidden application detectedaction=accept method=ugiatnu client=stiae facility=nofdeF user=sunt srcip=10.57.170.140 dstip=10.213.231.72 version=1.5102 storage=emips ad_domain=imadmi object=ostrume class=molest type=upt attributes=uiineavocount=tisetq node=irati account=icistatuscode=giatquov cached=eritquii profile=dexeac filteraction=iscinge size=6992 request=oreseos url=https://mail.example.net/tati/utaliqu.html?iquaUten=santium#iciatisu referer=https://www5.example.org/eporroqu/uat.txt?atquovo=suntinc#xeac error=nidolo authtime=tatn dnstime=eli cattime=nnu avscantime=dolo fullreqtime=Loremip device=idolor auth=emeumfu ua=CSed exceptions=lupt group=psaquae category=oinBCSe categoryname=mnisist content-type=sedd reputation=uatD application=iunt app-id=temveleu reason=colabo filename=eme file=numqu extension=qui time=civeli function=block line=agnaali message=gnam fwrule=tat seq=ipitla initf=enp0s7281 outitf=enp0s7084 dstmac=01:00:5e:de:94:f6 srcmac=01:00:5e:1d:c1:c0 proto=den length=tutla tos=olorema prec=;iades ttl=siarchi srcport=2289 dstport=3920 tcpflags=mqu info=apariat prec=tlabore caller=untmolli engine=remi localip=saute host=ercit2385.internal.home extra=run server=10.47.202.102 cookie=quirat set-cookie=llu
 2016:2:26-20:15:08 eirure7587.internal.localhost reverseproxy: [mpori] [aaliquaU:medium] [pid 3905:lpaqui] (22)No form context found: [client sitame] No form context found when parsing iadese tag, referer: https://api.example.com/utla/utei.htm?oei=tlabori#oin
 2016:3:12-03:17:42 data4478.api.lan confd: id=iquipex severity=very-high sys=uradip sub=wri name=bor client=occa facility=stquidol user=itquiin srcip=10.106.239.55 version=1.3129 storage=atevel object=nsecte class=itame type=eumfug attributes=litcount=asun node=estia account=eaq
 2016:3:26-10:20:16 ctetura3009.www5.corp reverseproxy: [lita] [adeseru:medium] [pid 7692:eaq] amest configured -- corp normal operations
-2016:4:9-17:22:51 smtpd[1411]: MASTER[inculpa]: QR globally disabled, status one set to 'disabled'
+2016:4:9-17:22:51 localhost smtpd[1411]: MASTER[inculpa]: QR globally disabled, status one set to 'disabled'
 2016:4:24-00:25:25 httpproxy[176]: [nse] disk_cache_zap (non) paquioff
 2016:5:8-07:27:59 ptasnu6684.mail.lan reverseproxy: [orumSe] [boree:low] [pid 945:rQuisau] AH01915: Init: (10.18.13.211:205) You configured ofdeFini(irat) on the onev(aturauto) port!
 2016:5:22-14:30:33 ssecillu7166.internal.lan barnyard: Initializing daemon mode