From 56512799348a1ef26db5d510fdcb2352c610cac4 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Mon, 29 May 2017 15:00:48 +0200 Subject: [PATCH] Change grok pattern to fetch correct IP from X-Forwarded-For list (#4351) * Change grok pattern to fetch correct IP from X-Forwarded-For list. * Document change to the nginx module in the changelog. * Add tests for nginx access log. (cherry picked from commit b6194b4354e0acc49078759fc6056452f13647b1) --- CHANGELOG.asciidoc | 1 + filebeat/module/nginx/access/test/test.log | 6 + .../nginx/access/test/test.log-expected.json | 344 ++++++++++++++++++ 3 files changed, 351 insertions(+) create mode 100644 filebeat/module/nginx/access/test/test.log create mode 100644 filebeat/module/nginx/access/test/test.log-expected.json diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index cd59c690215..e267f6074ec 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -72,6 +72,7 @@ https://github.com/elastic/beats/compare/v5.4.1...master[Check the HEAD diff] *Filebeat* - Add support for loading Xpack Machine Learning configurations from the modules, and added sample configurations for the Nginx module. {pull}4506[4506] {pull}4609[4609] +- Add ability to parse nginx logs exposing the X-Forwarded-For header instead of the remote address. {pull}4351[4351] *Heartbeat* diff --git a/filebeat/module/nginx/access/test/test.log b/filebeat/module/nginx/access/test/test.log new file mode 100644 index 00000000000..e303a6d516d --- /dev/null +++ b/filebeat/module/nginx/access/test/test.log @@ -0,0 +1,6 @@ +10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] "GET /ocelot HTTP/1.1" 200 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" +172.17.0.1 - - [29/May/2017:19:02:48 +0000] "GET /stringpatch HTTP/1.1" 404 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" +10.0.0.2, 10.0.0.1, 85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] "GET /ocelot HTTP/1.1" 200 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" +85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] "GET /ocelot HTTP/1.1" 200 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" +"10.5.102.222, 199.96.1.1, 204.246.1.1" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] "GET /assets/xxxx?q=100 HTTP/1.1" 200 25507 "-" "Amazon CloudFront" +2a03:0000:10ff:f00f:0000:0000:0:8000, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] "GET /test.html HTTP/1.1" 404 8571 "-" "Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)" diff --git a/filebeat/module/nginx/access/test/test.log-expected.json b/filebeat/module/nginx/access/test/test.log-expected.json new file mode 100644 index 00000000000..9b31bf3fa70 --- /dev/null +++ b/filebeat/module/nginx/access/test/test.log-expected.json @@ -0,0 +1,344 @@ +[ + { + "_index" : "filebeat-6.0.0-alpha2-2017.05.30", + "_type" : "doc", + "_id" : "AVxWUuZ8OMOtQBaTipsE", + "_score" : 1.0, + "_source" : { + "@timestamp" : "2016-12-07T10:05:07.000Z", + "offset" : 527, + "nginx" : { + "access" : { + "referrer" : "-", + "response_code" : "200", + "remote_ip" : "85.181.35.98", + "geoip" : { + "continent_name" : "Europe", + "country_iso_code" : "DE", + "location" : { + "lon" : 9.0, + "lat" : 51.0 + } + }, + "method" : "GET", + "user_name" : "-", + "http_version" : "1.1", + "body_sent" : { + "bytes" : "571" + }, + "remote_ip_list" : [ + "10.0.0.2", + "10.0.0.1", + "85.181.35.98" + ], + "url" : "/ocelot", + "user_agent" : { + "major" : "49", + "minor" : "0", + "os" : "Mac OS X 10.12", + "os_minor" : "12", + "os_major" : "10", + "name" : "Firefox", + "os_name" : "Mac OS X", + "device" : "Other" + } + } + }, + "beat" : { + "hostname" : "a-mac-with-esc-key-2.local", + "name" : "a-mac-with-esc-key-2.local", + "version" : "6.0.0-alpha2" + }, + "prospector" : { + "type" : "log" + }, + "read_timestamp" : "2017-05-29T22:28:06.246Z", + "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/nginx/access/test/test.log", + "fileset" : { + "module" : "nginx", + "name" : "access" + } + } + }, + { + "_index" : "filebeat-6.0.0-alpha2-2017.05.30", + "_type" : "doc", + "_id" : "AVxWUuZ8OMOtQBaTipsD", + "_score" : 1.0, + "_source" : { + "@timestamp" : "2017-05-29T19:02:48.000Z", + "offset" : 341, + "nginx" : { + "access" : { + "referrer" : "-", + "response_code" : "404", + "remote_ip" : "172.17.0.1", + "method" : "GET", + "user_name" : "-", + "http_version" : "1.1", + "body_sent" : { + "bytes" : "612" + }, + "remote_ip_list" : [ + "172.17.0.1" + ], + "url" : "/stringpatch", + "user_agent" : { + "patch" : "a2", + "major" : "15", + "minor" : "0", + "os" : "Windows 7", + "name" : "Firefox Alpha", + "os_name" : "Windows 7", + "device" : "Other" + } + } + }, + "beat" : { + "hostname" : "a-mac-with-esc-key-2.local", + "name" : "a-mac-with-esc-key-2.local", + "version" : "6.0.0-alpha2" + }, + "prospector" : { + "type" : "log" + }, + "read_timestamp" : "2017-05-29T22:28:06.246Z", + "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/nginx/access/test/test.log", + "fileset" : { + "module" : "nginx", + "name" : "access" + } + } + }, + { + "_index" : "filebeat-6.0.0-alpha2-2017.05.30", + "_type" : "doc", + "_id" : "AVxWUuZ8OMOtQBaTipsF", + "_score" : 1.0, + "_source" : { + "@timestamp" : "2016-12-07T10:05:07.000Z", + "offset" : 693, + "nginx" : { + "access" : { + "referrer" : "-", + "response_code" : "200", + "remote_ip" : "85.181.35.98", + "geoip" : { + "continent_name" : "Europe", + "country_iso_code" : "DE", + "location" : { + "lon" : 9.0, + "lat" : 51.0 + } + }, + "method" : "GET", + "user_name" : "-", + "http_version" : "1.1", + "body_sent" : { + "bytes" : "571" + }, + "remote_ip_list" : [ + "85.181.35.98" + ], + "url" : "/ocelot", + "user_agent" : { + "major" : "49", + "minor" : "0", + "os" : "Mac OS X 10.12", + "os_minor" : "12", + "os_major" : "10", + "name" : "Firefox", + "os_name" : "Mac OS X", + "device" : "Other" + } + } + }, + "beat" : { + "hostname" : "a-mac-with-esc-key-2.local", + "name" : "a-mac-with-esc-key-2.local", + "version" : "6.0.0-alpha2" + }, + "prospector" : { + "type" : "log" + }, + "read_timestamp" : "2017-05-29T22:28:06.246Z", + "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/nginx/access/test/test.log", + "fileset" : { + "module" : "nginx", + "name" : "access" + } + } + }, + { + "_index" : "filebeat-6.0.0-alpha2-2017.05.30", + "_type" : "doc", + "_id" : "AVxWUuZ8OMOtQBaTipsC", + "_score" : 1.0, + "_source" : { + "@timestamp" : "2016-12-07T10:05:07.000Z", + "offset" : 183, + "nginx" : { + "access" : { + "referrer" : "-", + "response_code" : "200", + "remote_ip" : "10.0.0.2", + "method" : "GET", + "user_name" : "-", + "http_version" : "1.1", + "body_sent" : { + "bytes" : "571" + }, + "remote_ip_list" : [ + "10.0.0.2", + "10.0.0.1", + "127.0.0.1" + ], + "url" : "/ocelot", + "user_agent" : { + "major" : "49", + "minor" : "0", + "os" : "Mac OS X 10.12", + "os_minor" : "12", + "os_major" : "10", + "name" : "Firefox", + "os_name" : "Mac OS X", + "device" : "Other" + } + } + }, + "beat" : { + "hostname" : "a-mac-with-esc-key-2.local", + "name" : "a-mac-with-esc-key-2.local", + "version" : "6.0.0-alpha2" + }, + "prospector" : { + "type" : "log" + }, + "read_timestamp" : "2017-05-29T22:28:06.245Z", + "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/nginx/access/test/test.log", + "fileset" : { + "module" : "nginx", + "name" : "access" + } + } + }, + { + "_index" : "filebeat-6.0.0-alpha2-2017.05.30", + "_type" : "doc", + "_id" : "AVxWUuZ8OMOtQBaTipsG", + "_score" : 1.0, + "_source" : { + "@timestamp" : "2016-01-22T13:18:29.000Z", + "offset" : 845, + "nginx" : { + "access" : { + "referrer" : "-", + "response_code" : "200", + "remote_ip" : "199.96.1.1", + "geoip" : { + "continent_name" : "North America", + "city_name" : "Springfield", + "country_iso_code" : "US", + "region_name" : "Illinois", + "location" : { + "lon" : -89.6859, + "lat" : 39.772 + } + }, + "method" : "GET", + "user_name" : "-", + "http_version" : "1.1", + "body_sent" : { + "bytes" : "25507" + }, + "remote_ip_list" : [ + "10.5.102.222", + "199.96.1.1", + "204.246.1.1", + "10.2.1.185" + ], + "url" : "/assets/xxxx?q=100", + "user_agent" : { + "os" : "Other", + "name" : "Other", + "os_name" : "Other", + "device" : "Other" + } + } + }, + "beat" : { + "hostname" : "a-mac-with-esc-key-2.local", + "name" : "a-mac-with-esc-key-2.local", + "version" : "6.0.0-alpha2" + }, + "prospector" : { + "type" : "log" + }, + "read_timestamp" : "2017-05-29T22:28:06.246Z", + "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/nginx/access/test/test.log", + "fileset" : { + "module" : "nginx", + "name" : "access" + } + } + }, + { + "_index" : "filebeat-6.0.0-alpha2-2017.05.30", + "_type" : "doc", + "_id" : "AVxWUuZ8OMOtQBaTipsH", + "_score" : 1.0, + "_source" : { + "@timestamp" : "2016-12-30T06:47:09.000Z", + "offset" : 1085, + "nginx" : { + "access" : { + "referrer" : "-", + "response_code" : "404", + "remote_ip" : "2a03:0000:10ff:f00f:0000:0000:0:8000", + "geoip" : { + "continent_name" : "Europe", + "country_iso_code" : "PT", + "location" : { + "lon" : -8.13057, + "lat" : 39.6945 + } + }, + "method" : "GET", + "user_name" : "-", + "http_version" : "1.1", + "body_sent" : { + "bytes" : "8571" + }, + "remote_ip_list" : [ + "2a03:0000:10ff:f00f:0000:0000:0:8000", + "10.225.192.17", + "10.2.2.121" + ], + "url" : "/test.html", + "user_agent" : { + "major" : "1", + "minor" : "0", + "os" : "Other", + "name" : "Facebot", + "os_name" : "Other", + "device" : "Spider" + } + } + }, + "beat" : { + "hostname" : "a-mac-with-esc-key-2.local", + "name" : "a-mac-with-esc-key-2.local", + "version" : "6.0.0-alpha2" + }, + "prospector" : { + "type" : "log" + }, + "read_timestamp" : "2017-05-29T22:28:06.246Z", + "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/nginx/access/test/test.log", + "fileset" : { + "module" : "nginx", + "name" : "access" + } + } + } +]