elastic · monicasarbu · Dec 19, 2016 · Nov 23, 2016 · Nov 28, 2016 · Nov 28, 2016
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -8,36 +8,31 @@
 // Template, add newest changes here
 
 === Beats version HEAD
-https://github.com/elastic/beats/compare/v5.0.1...master[Check the HEAD diff]
+https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff]
 
 ==== Breaking changes
 
 *Affecting all Beats*
 
 *Metricbeat*
-- Change data structure of experimental haproxy module. {pull}3003[3003]
 
 *Packetbeat*
 
 *Topbeat*
 
 *Filebeat*
-- If a file is falling under ignore_older during startup, offset is now set to end of file instead of 0.
-  With the previous logic the whole file was sent in case a line was added and it was inconsitent with
-  files which were harvested previously. {pull}2907[2907]
-- tail_files is now only applied on the first scan and not for all new files. {pull}2932[2932]
 
 *Winlogbeat*
 
 
 ==== Bugfixes
 
 *Affecting all Beats*
-- Fix empty benign errors logged by processor actions. {pull}3046[3046]
 
 *Metricbeat*
 
-- Calculate the fsstat values per mounting point, and not filesystem. {pull}2777[2777]
+- Fix service times-out at startup. {pull}3056[3056]
+- Kafka module case sensitive host name matching. {pull}3193[3193]
 
 *Packetbeat*
 
@@ -47,40 +42,31 @@ https://github.com/elastic/beats/compare/v5.0.1...master[Check the HEAD diff]
 - Fix registry cleanup issue when files falling under ignore_older after restart. {issue}2818[2818]
 
 *Winlogbeat*
+- Fix for "The array bounds are invalid" error when reading large events. {issue}3076[3076]
 
 ==== Added
 
 *Affecting all Beats*
-- Add add_cloud_metadata processor for collecting cloud provider metadata. {pull}2728[2728]
-- Added decode_json_fields processor for decoding fields containing JSON strings. {pull}2605[2605]
+- Add support for passing list and dictionary settings via -E flag.
+- Support for parsing list and dictionary setting from environment variables.
+- Added new flags to import_dashboards (-cacert, -cert, -key, -insecure). {pull}3139[3139] {pull}3163[3163]
 
 *Metricbeat*
+- Kafka module broker matching enhancements. {pull}3129[3129]
+- Add a couchbase module with metricsets for node, cluster and bucker. {pull}3081[3081]
+- Export number of cores for cpu module. {pull}3192[3192]
 
-- Add experimental filebeat metricset in the beats module. {pull}2297[2297]
-- Add experimental libbeat metricset in the beats module. {pull}2339[2339]
-- Add experimental docker module. Provided by Ingensi and @douaejeouit based on dockbeat.
-- Add username and password config options to the MongoDB module. {pull}2889[2889]
-- Add username and password config options to the PostgreSQL module. {pull}2889[2890]
-- Add system core metricset for Windows. {pull}2883[2883]
-- Add a sample Redis Kibana dashboard. {pull}2916[2916]
-- Add support for MongoDB 3.4 and WiredTiger metrics. {pull}2999[2999]
-- Add experimental kafka module with partition metricset. {pull}2969[2969]
-- Add raw config option for mysql/status metricset. {pull}3001[3001]
 
 *Packetbeat*
 
 *Topbeat*
 
 *Filebeat*
-- Add command line option -once to run filebeat only once and then close. {pull}2456[2456]
-- Only load matching states into prospector to improve state handling {pull}2840[2840]
-- Reset all states ttl on startup to make sure it is overwritten by new config {pull}2840[2840]
-- Persist all states for files which fall under ignore_older to have consistent behaviour {pull}2859[2859]
-- Improve shutdown behaviour with large number of files. {pull}3035[3035]
+- Add enabled config option to prospectors. {pull}3157[3157]
 
 *Winlogbeat*
 
-- Add `event_logs.batch_read_size` configuration option. {pull}2641[2641]
+- Reduced amount of memory allocated while reading event log records. {pull}3113[3113] {pull}3118[3113]
 
 ==== Deprecated
 
@@ -91,13 +77,87 @@ https://github.com/elastic/beats/compare/v5.0.1...master[Check the HEAD diff]
 *Packetbeat*
 
 *Topbeat*
+- Fix error on importing dashboards due to colons in the Caassandra dashboard. {issue}3140[3140]
 
 *Filebeat*
 
 *Winlogbeat*
 
 ////////////////////////////////////////////////////////////
 
+[[release-notes-5.1.1]]
+=== Beats version 5.1.1
+https://github.com/elastic/beats/compare/v5.0.2...v5.1.1[View commits]
+
+==== Breaking changes
+
+*Metricbeat*
+
+- Change data structure of experimental haproxy module. {pull}3003[3003]
+
+*Filebeat*
+
+- If a file is falling under `ignore_older` during startup, offset is now set to end of file instead of 0.
+  With the previous logic the whole file was sent in case a line was added and it was inconsistent with
+  files which were harvested previously. {pull}2907[2907]
+- `tail_files` is now only applied on the first scan and not for all new files. {pull}2932[2932]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Fix empty benign errors logged by processor actions. {pull}3046[3046]
+
+*Metricbeat*
+
+- Calculate the fsstat values per mounting point, and not filesystem. {pull}2777[2777]
+
+==== Added
+
+*Affecting all Beats*
+
+- Add add_cloud_metadata processor for collecting cloud provider metadata. {pull}2728[2728]
+- Added decode_json_fields processor for decoding fields containing JSON strings. {pull}2605[2605]
+
+*Metricbeat*
+
+- Add experimental Docker module. Provided by Ingensi and @douaejeouit based on dockbeat.
+- Add a sample Redis Kibana dashboard. {pull}2916[2916]
+- Add support for MongoDB 3.4 and WiredTiger metrics. {pull}2999[2999]
+- Add experimental kafka module with partition metricset. {pull}2969[2969]
+- Add raw config option for mysql/status metricset. {pull}3001[3001]
+
+*Filebeat*
+
+- Add command line option `-once` to run Filebeat only once and then close. {pull}2456[2456]
+- Only load matching states into prospector to improve state handling {pull}2840[2840]
+- Reset all states ttl on startup to make sure it is overwritten by new config {pull}2840[2840]
+- Persist all states for files which fall under `ignore_older` to have consistent behaviour {pull}2859[2859]
+- Improve shutdown behaviour with large number of files. {pull}3035[3035]
+
+*Winlogbeat*
+
+- Add `event_logs.batch_read_size` configuration option. {pull}2641[2641]
+
+[[release-notes-5.1.0]]
+=== Beats version 5.1.0 (skipped)
+
+Version 5.1.0 doesn't exist because, for a short period of time, the Elastic
+Yum and Apt repositories included unreleased binaries labeled 5.1.0. To avoid
+confusion and upgrade issues for the people that have installed these without
+realizing, we decided to skip the 5.1.0 version and release 5.1.1 instead.
+
+[[release-notes-5.0.2]]
+=== Beats version 5.0.2
+https://github.com/elastic/beats/compare/v5.0.1...v5.0.2[View commits]
+
+==== Bugfixes
+
+*Metricbeat*
+
+- Fix the `password` option in the MongoDB module. {pull}2995[2995]
+
+
 [[release-notes-5.0.1]]
 === Beats version 5.0.1
 https://github.com/elastic/beats/compare/v5.0.0...v5.0.1[View commits]

diff --git a/Dockerfile b/Dockerfile
@@ -11,5 +11,5 @@ COPY libbeat/scripts/docker-entrypoint.sh /entrypoint.sh
 RUN mkdir -p /etc/pki/tls/certs
 COPY testing/environments/docker/logstash/pki/tls/certs/logstash.crt /etc/pki/tls/certs/logstash.crt
 
-# Create a copy of the respository inside the container.
+# Create a copy of the repository inside the container.
 COPY . /go/src/github.com/elastic/beats/
diff --git a/Makefile b/Makefile
@@ -3,6 +3,7 @@ BUILD_DIR=build
 COVERAGE_DIR=${BUILD_DIR}/coverage
 BEATS=packetbeat filebeat winlogbeat metricbeat heartbeat
 PROJECTS=libbeat ${BEATS}
+PROJECTS_ENV=libbeat metricbeat
 SNAPSHOT?=yes
 
 # Runs complete testsuites (unit, system, integration) for all beats with coverage and race detection.
@@ -12,6 +13,9 @@ testsuite:
 	$(foreach var,$(PROJECTS),$(MAKE) -C $(var) testsuite || exit 1;)
 	#$(MAKE) -C generate test
 
+stop-environments:
+	$(foreach var,$(PROJECTS_ENV),$(MAKE) -C $(var) stop-environment || exit 0;)
+
 # Runs unit and system tests without coverage and race detection.
 .PHONY: test
 test:

diff --git a/dev-tools/packer/platforms/centos/run.sh.j2 b/dev-tools/packer/platforms/centos/run.sh.j2
@@ -24,8 +24,8 @@ RPM_VERSION=`echo ${VERSION} | sed 's/-/_/g'`
 fpm --force -s dir -t rpm \
         -n {{.beat_name}} -v ${RPM_VERSION} \
         --architecture {{.rpm_arch}} \
-        --vendor "Elastic" \
-        --license "ASL 2.0" \
+        --vendor "{{.beat_vendor}}"  \
+        --license "{{.beat_license}}"  \
         --description "{{.beat_description}}" \
         --url {{.beat_url}} \
         --rpm-init /tmp/{{.beat_name}}.init \

diff --git a/dev-tools/packer/platforms/centos/systemd.j2 b/dev-tools/packer/platforms/centos/systemd.j2
@@ -1,6 +1,6 @@
 [Unit]
 Description={{.beat_name}}
-Documentation=https://www.elastic.co/guide/en/beats/{{.beat_name}}/current/index.html
+Documentation={{.beat_doc_url}}
 Wants=network-online.target
 After=network-online.target
 

diff --git a/dev-tools/packer/platforms/debian/run.sh.j2 b/dev-tools/packer/platforms/debian/run.sh.j2
@@ -20,8 +20,8 @@ fi
 # create deb
 fpm --force -s dir -t deb \
         -n {{.beat_name}} -v ${VERSION} \
-        --vendor "Elastic" \
-        --license "ASL 2.0" \
+        --vendor "{{.beat_vendor}}"  \
+        --license "{{.beat_license}}"  \
         --architecture {{.deb_arch}} \
         --description "{{.beat_description}}" \
         --url {{.beat_url}} \

diff --git a/dev-tools/packer/platforms/debian/systemd.j2 b/dev-tools/packer/platforms/debian/systemd.j2
@@ -1,6 +1,6 @@
 [Unit]
 Description={{.beat_name}}
-Documentation=https://www.elastic.co/guide/en/beats/{{.beat_name}}/current/index.html
+Documentation={{.beat_doc_url}}
 Wants=network-online.target
 After=network-online.target
 

diff --git a/dev-tools/packer/xgo-scripts/before_build.sh b/dev-tools/packer/xgo-scripts/before_build.sh
@@ -46,14 +46,15 @@ fi
 # with git 1.7
 git clone https://github.com/tsg/gotpl.git /go/src/github.com/tsg/gotpl
 mkdir -p /go/src/gopkg.in/yaml.v2
-cp -r /go/src/github.com/elastic/beats/vendor/gopkg.in/yaml.v2 /go/src/gopkg.in/
+
+cp -r $LIBBEAT_PATH/../vendor/gopkg.in/yaml.v2 /go/src/gopkg.in/
 go install github.com/tsg/gotpl
 
 # Append doc versions to package.yml
 cat ${LIBBEAT_PATH}/docs/version.asciidoc >> ${PREFIX}/package.yml
 # Make variable naming of doc-branch compatible with gotpl. Generate and copy README.md into homedir
 sed -i -e 's/:doc-branch/doc_branch/g' ${PREFIX}/package.yml
-/go/bin/gotpl ${LIBBEAT_PATH}/../dev-tools/packer/readme.md.j2 < ${PREFIX}/package.yml > ${PREFIX}/homedir/README.md
+/go/bin/gotpl /templates/readme.md.j2 < ${PREFIX}/package.yml > ${PREFIX}/homedir/README.md
 
 # Copy template
 cp $BEATNAME.template.json $PREFIX/$BEATNAME.template.json

diff --git a/filebeat/Makefile b/filebeat/Makefile
@@ -12,6 +12,14 @@ include ../libbeat/scripts/Makefile
 .PHONY: before-build
 before-build:
 
-# Collects all dependencies and then calls update
+# Collects all module and dataset fields
+.PHONY: fields
+fields:
+	mkdir -p _meta/
+	cat ${ES_BEATS}/filebeat/_meta/fields.common.yml > _meta/fields.generated.yml
+	. ${PYTHON_ENV}/bin/activate; python ${ES_BEATS}/metricbeat/scripts/fields_collector.py >> _meta/fields.generated.yml
+
+
+# Runs all collection steps and updates afterwards
 .PHONY: collect
-collect:
+collect: fields update
diff --git a/filebeat/_meta/beat.full.yml b/filebeat/_meta/beat.full.yml
@@ -205,6 +205,9 @@ filebeat.prospectors:
   # Note: Potential data loss. Make sure to read and understand the docs for this option.
   #close_timeout: 0
 
+  # Defines if prospectors is enabled
+  #enabled: true
+
 #----------------------------- Stdin prospector -------------------------------
 # Configuration to use stdin input
 #- input_type: stdin

diff --git a/filebeat/_meta/fields.yml → filebeat/_meta/fields.common.yml b/filebeat/_meta/fields.yml → filebeat/_meta/fields.common.yml
@@ -33,3 +33,8 @@
       description: >
         The input type from which the event was generated. This field is set to the value specified for the `input_type` option in the prospector section of the Filebeat config file.
 
+    - name: error
+      description: >
+        Ingestion pipeline error message, added in case there are errors reported by
+        the Ingest Node in Elasticsearch.
+
diff --git a/filebeat/_meta/kibana/dashboard/Filebeat-MySQL-Dashboard.json b/filebeat/_meta/kibana/dashboard/Filebeat-MySQL-Dashboard.json
@@ -0,0 +1,13 @@
+{
+  "hits": 0, 
+  "timeRestore": false, 
+  "description": "", 
+  "title": "Filebeat MySQL Dashboard", 
+  "uiStateJSON": "{\"P-1\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", 
+  "panelsJSON": "[{\"col\":1,\"id\":\"MySQL-slowest-queries\",\"panelIndex\":1,\"row\":8,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"MySQL-Slow-queries-over-time\",\"panelIndex\":2,\"row\":1,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"MySQL-error-logs\",\"panelIndex\":3,\"row\":1,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"columns\":[\"mysql.error.level\",\"mysql.error.message\"],\"id\":\"Filebeat-MySQL-error-log\",\"panelIndex\":4,\"row\":8,\"size_x\":6,\"size_y\":5,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":7,\"id\":\"MySQL-Error-logs-levels\",\"panelIndex\":5,\"row\":4,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"MySQL-Slow-logs-by-count\",\"panelIndex\":6,\"row\":4,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"}]", 
+  "optionsJSON": "{\"darkTheme\":false}", 
+  "version": 1, 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}"
+  }
+}
diff --git a/filebeat/_meta/kibana/dashboard/Filebeat-Nginx-Dashboard.json b/filebeat/_meta/kibana/dashboard/Filebeat-Nginx-Dashboard.json
@@ -0,0 +1,13 @@
+{
+  "hits": 0, 
+  "timeRestore": false, 
+  "description": "", 
+  "title": "Filebeat Nginx Dashboard", 
+  "uiStateJSON": "{\"P-4\":{\"vis\":{\"legendOpen\":true}},\"P-8\":{\"mapCenter\":[50.51342652633956,-0.17578125]}}", 
+  "panelsJSON": "[{\"col\":9,\"id\":\"Errors-over-time\",\"panelIndex\":2,\"row\":4,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Nginx-Access-Browsers\",\"panelIndex\":3,\"row\":10,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":5,\"id\":\"Nginx-Access-OSes\",\"panelIndex\":4,\"row\":10,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"New-Visualization\",\"panelIndex\":5,\"row\":4,\"size_x\":8,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Nginx-Access-Response-codes-by-top-URLs\",\"panelIndex\":6,\"row\":7,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"col\":9,\"id\":\"Sent-sizes\",\"panelIndex\":7,\"row\":10,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"id\":\"Nginx-Access-Map\",\"type\":\"visualization\",\"panelIndex\":8,\"size_x\":12,\"size_y\":3,\"col\":1,\"row\":1}]", 
+  "optionsJSON": "{\"darkTheme\":false}", 
+  "version": 1, 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}"
+  }
+}
diff --git a/filebeat/_meta/kibana/dashboard/Filebeat-syslog-dashboard.json b/filebeat/_meta/kibana/dashboard/Filebeat-syslog-dashboard.json
@@ -0,0 +1,13 @@
+{
+  "hits": 0, 
+  "timeRestore": false, 
+  "description": "", 
+  "title": "Filebeat syslog dashboard", 
+  "uiStateJSON": "{}", 
+  "panelsJSON": "[{\"id\":\"Syslog-events-by-hostname\",\"type\":\"visualization\",\"panelIndex\":1,\"size_x\":8,\"size_y\":4,\"col\":1,\"row\":1},{\"id\":\"Syslog-hostnames-and-processes\",\"type\":\"visualization\",\"panelIndex\":2,\"size_x\":4,\"size_y\":4,\"col\":9,\"row\":1},{\"id\":\"Syslog-system-logs\",\"type\":\"search\",\"panelIndex\":3,\"size_x\":12,\"size_y\":7,\"col\":1,\"row\":5,\"columns\":[\"syslog.system.hostname\",\"syslog.system.program\",\"syslog.system.message\"],\"sort\":[\"@timestamp\",\"desc\"]}]", 
+  "optionsJSON": "{\"darkTheme\":false}", 
+  "version": 1, 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}}}]}"
+  }
+}
diff --git a/filebeat/_meta/kibana/search/Filebeat-MySQL-Slow-log.json b/filebeat/_meta/kibana/search/Filebeat-MySQL-Slow-log.json
@@ -0,0 +1,16 @@
+{
+  "sort": [
+    "@timestamp", 
+    "desc"
+  ], 
+  "hits": 0, 
+  "description": "", 
+  "title": "Filebeat MySQL Slow log", 
+  "version": 1, 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"_exists_:mysql.slowlog\"}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}"
+  }, 
+  "columns": [
+    "_source"
+  ]
+}
diff --git a/filebeat/_meta/kibana/search/Filebeat-MySQL-error-log.json b/filebeat/_meta/kibana/search/Filebeat-MySQL-error-log.json
@@ -0,0 +1,17 @@
+{
+  "sort": [
+    "@timestamp", 
+    "desc"
+  ], 
+  "hits": 0, 
+  "description": "", 
+  "title": "Filebeat MySQL error log", 
+  "version": 1, 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"query\":\"_exists_:mysql.error\",\"analyze_wildcard\":true}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}"
+  }, 
+  "columns": [
+    "mysql.error.level", 
+    "mysql.error.message"
+  ]
+}
diff --git a/filebeat/_meta/kibana/search/Filebeat-Nginx-module.json b/filebeat/_meta/kibana/search/Filebeat-Nginx-module.json
@@ -0,0 +1,16 @@
+{
+  "sort": [
+    "@timestamp", 
+    "desc"
+  ], 
+  "hits": 0, 
+  "description": "", 
+  "title": "Filebeat Nginx module", 
+  "version": 1, 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"query\":\"_exists_:nginx\",\"analyze_wildcard\":true}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}"
+  }, 
+  "columns": [
+    "_source"
+  ]
+}