Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add autonomous system info to Filebeat modules #13036

Merged
merged 1 commit into from
Jul 23, 2019
Merged

Add autonomous system info to Filebeat modules #13036

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Report host metadata for Filebeat logs in Kubernetes. {pull}12790[12790]
- Add netflow dashboards based on Logstash netflow. {pull}12857[12857]
- Parse more fields from Elasticsearch slowlogs. {pull}11939[11939]
- Update module pipelines to enrich events with autonomous system fields. {pull}13036[13036]

*Heartbeat*

Expand Down
155 changes: 97 additions & 58 deletions filebeat/module/apache/access/ingest/default.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,60 +1,99 @@
{
"description": "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins.",
"processors": [{
"grok": {
"field": "message",
"patterns":[
"%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?",
"%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"-\" %{NUMBER:http.response.status_code:long} -",
"\\[%{HTTPDATE:apache.access.time}\\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} %{DATA:apache.access.ssl.cipher} \"%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}\" %{NUMBER:http.response.body.bytes:long}"
],
"ignore_missing": true
}
},{
"remove":{
"field": "message"
}
}, {
"grok": {
"field": "source.address",
"ignore_missing": true,
"patterns": [
"^(%{IP:source.ip}|%{HOSTNAME:source.domain})$"
]
}
}, {
"rename": {
"field": "@timestamp",
"target_field": "event.created"
}
}, {
"date": {
"field": "apache.access.time",
"target_field": "@timestamp",
"formats": ["dd/MMM/yyyy:H:m:s Z"],
"ignore_failure": true
}
}, {
"remove": {
"field": "apache.access.time",
"ignore_failure": true
}
}, {
"user_agent": {
"field": "user_agent.original",
"ignore_failure": true
}
}, {
"geoip": {
"field": "source.ip",
"target_field": "source.geo",
"ignore_missing": true
}
}],
"on_failure" : [{
"set" : {
"field" : "error.message",
"value" : "{{ _ingest.on_failure_message }}"
}
}]
"description": "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins.",
"processors": [
{
"grok": {
"field": "message",
"patterns": [
"%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?",
"%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"-\" %{NUMBER:http.response.status_code:long} -",
"\\[%{HTTPDATE:apache.access.time}\\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} %{DATA:apache.access.ssl.cipher} \"%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}\" %{NUMBER:http.response.body.bytes:long}"
],
"ignore_missing": true
}
},
{
"remove": {
"field": "message"
}
},
{
"grok": {
"field": "source.address",
"ignore_missing": true,
"patterns": [
"^(%{IP:source.ip}|%{HOSTNAME:source.domain})$"
]
}
},
{
"rename": {
"field": "@timestamp",
"target_field": "event.created"
}
},
{
"date": {
"field": "apache.access.time",
"target_field": "@timestamp",
"formats": [
"dd/MMM/yyyy:H:m:s Z"
],
"ignore_failure": true
}
},
{
"remove": {
"field": "apache.access.time",
"ignore_failure": true
}
},
{
"user_agent": {
"field": "user_agent.original",
"ignore_failure": true
}
},
{
"geoip": {
"field": "source.ip",
"target_field": "source.geo",
"ignore_missing": true
}
},
{
"geoip": {
"database_file": "GeoLite2-ASN.mmdb",
"field": "source.ip",
"target_field": "source.as",
"properties": [
"asn",
"organization_name"
],
"ignore_missing": true
}
},
{
"rename": {
"field": "source.as.asn",
"target_field": "source.as.number",
"ignore_missing": true
}
},
{
"rename": {
"field": "source.as.organization_name",
"target_field": "source.as.organization.name",
"ignore_missing": true
}
}
],
"on_failure": [
{
"set": {
"field": "error.message",
"value": "{{ _ingest.on_failure_message }}"
}
}
]
}
6 changes: 6 additions & 0 deletions filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@
"log.offset": 181,
"service.type": "apache",
"source.address": "77.179.66.156",
"source.as.number": 6805,
"source.as.organization.name": "Telefonica Germany",
"source.geo.continent_name": "Europe",
"source.geo.country_iso_code": "DE",
"source.geo.location.lat": 51.2993,
Expand All @@ -80,6 +82,8 @@
"log.offset": 252,
"service.type": "apache",
"source.address": "77.179.66.156",
"source.as.number": 6805,
"source.as.organization.name": "Telefonica Germany",
"source.geo.continent_name": "Europe",
"source.geo.country_iso_code": "DE",
"source.geo.location.lat": 51.2993,
Expand All @@ -101,6 +105,8 @@
"log.offset": 332,
"service.type": "apache",
"source.address": "77.179.66.156",
"source.as.number": 6805,
"source.as.organization.name": "Telefonica Germany",
"source.geo.continent_name": "Europe",
"source.geo.country_iso_code": "DE",
"source.geo.location.lat": 51.2993,
Expand Down
130 changes: 80 additions & 50 deletions filebeat/module/apache/error/ingest/pipeline.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,55 +1,85 @@
{
"description": "Pipeline for parsing apache error logs",
"processors": [
{
"grok": {
"field": "message",
"patterns": [
"\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{LOGLEVEL:log.level}\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}",
"\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{DATA:apache.error.module}:%{LOGLEVEL:log.level}\\] \\[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}"
],
"pattern_definitions": {
"APACHE_TIME": "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
"description": "Pipeline for parsing apache error logs",
"processors": [
{
"grok": {
"field": "message",
"patterns": [
"\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{LOGLEVEL:log.level}\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}",
"\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{DATA:apache.error.module}:%{LOGLEVEL:log.level}\\] \\[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}"
],
"pattern_definitions": {
"APACHE_TIME": "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
},
"ignore_missing": true
}
},
"ignore_missing": true
}
},
{
"date": {
"field": "apache.error.timestamp",
"target_field": "@timestamp",
"formats": ["EEE MMM dd H:m:s yyyy", "EEE MMM dd H:m:s.SSSSSS yyyy"],
"ignore_failure": true
}
},
{
"remove": {
"field": "apache.error.timestamp",
"ignore_failure": true
}
},

{
"grok": {
"field": "source.address",
"ignore_missing": true,
"patterns": [
"^(%{IP:source.ip}|%{HOSTNAME:source.domain})$"
]
{
"date": {
"field": "apache.error.timestamp",
"target_field": "@timestamp",
"formats": [
"EEE MMM dd H:m:s yyyy",
"EEE MMM dd H:m:s.SSSSSS yyyy"
],
"ignore_failure": true
}
},
{
"remove": {
"field": "apache.error.timestamp",
"ignore_failure": true
}
},
{
"grok": {
"field": "source.address",
"ignore_missing": true,
"patterns": [
"^(%{IP:source.ip}|%{HOSTNAME:source.domain})$"
]
}
},
{
"geoip": {
"field": "source.ip",
"target_field": "source.geo",
"ignore_missing": true
}
},
{
"geoip": {
"database_file": "GeoLite2-ASN.mmdb",
"field": "source.ip",
"target_field": "source.as",
"properties": [
"asn",
"organization_name"
],
"ignore_missing": true
}
},
{
"rename": {
"field": "source.as.asn",
"target_field": "source.as.number",
"ignore_missing": true
}
},
{
"rename": {
"field": "source.as.organization_name",
"target_field": "source.as.organization.name",
"ignore_missing": true
}
}
},
{
"geoip": {
"field": "source.ip",
"target_field": "source.geo",
"ignore_missing": true
],
"on_failure": [
{
"set": {
"field": "error.message",
"value": "{{ _ingest.on_failure_message }}"
}
}
}
],
"on_failure" : [{
"set" : {
"field" : "error.message",
"value" : "{{ _ingest.on_failure_message }}"
}
}]
]
}
6 changes: 5 additions & 1 deletion filebeat/module/apache/error/test/test.log-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@
"process.thread.id": 4328636416,
"service.type": "apache",
"source.address": "72.15.99.187",
"source.as.number": 11693,
"source.as.organization.name": "HC Cable OPCO, LLC",
"source.geo.city_name": "Newnan",
"source.geo.continent_name": "North America",
"source.geo.country_iso_code": "US",
Expand All @@ -61,6 +63,8 @@
"process.pid": 15934,
"service.type": "apache",
"source.address": "123.123.123.123",
"source.as.number": 4808,
"source.as.organization.name": "China Unicom Beijing Province Network",
"source.geo.city_name": "Beijing",
"source.geo.continent_name": "Asia",
"source.geo.country_iso_code": "CN",
Expand All @@ -71,4 +75,4 @@
"source.ip": "123.123.123.123",
"source.port": "12345"
}
]
]
26 changes: 26 additions & 0 deletions filebeat/module/auditd/log/ingest/pipeline.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,32 @@
"target_field": "source.geo",
"ignore_failure": true
}
},
{
"geoip": {
"database_file": "GeoLite2-ASN.mmdb",
"field": "source.ip",
"target_field": "source.as",
"properties": [
"asn",
"organization_name"
],
"ignore_missing": true
}
},
{
"rename": {
"field": "source.as.asn",
"target_field": "source.as.number",
"ignore_missing": true
}
},
{
"rename": {
"field": "source.as.organization_name",
"target_field": "source.as.organization.name",
"ignore_missing": true
}
}
],
"on_failure": [
Expand Down
Loading