From 28f6bfa28be899834afab374875c565a41687876 Mon Sep 17 00:00:00 2001 From: ruflin Date: Thu, 21 Dec 2017 10:36:50 +1100 Subject: [PATCH 1/6] Add dashboard test loading for auditbeat In combination with adding the dashboard loading a libbeat.yml.j2 was created so all beats can share the common config options and we do not have to add each option for each beat. So far this is applied to metricbeat, auditbeat and filebeat. Other beats will follow in a follow up PR. --- .../tests/system/config/auditbeat.yml.j2 | 9 +- auditbeat/tests/system/test_base.py | 22 ++++ filebeat/tests/system/config/filebeat.yml.j2 | 99 +---------------- libbeat/tests/system/beat/beat.py | 5 +- libbeat/tests/system/config/libbeat.yml.j2 | 103 ++++++++++++++++++ .../tests/system/config/metricbeat.yml.j2 | 79 +------------- 6 files changed, 132 insertions(+), 185 deletions(-) create mode 100644 libbeat/tests/system/config/libbeat.yml.j2 diff --git a/auditbeat/tests/system/config/auditbeat.yml.j2 b/auditbeat/tests/system/config/auditbeat.yml.j2 index 8ec0ba1037e0..3c4f9ac30ae7 100644 --- a/auditbeat/tests/system/config/auditbeat.yml.j2 +++ b/auditbeat/tests/system/config/auditbeat.yml.j2 @@ -22,11 +22,4 @@ queue.mem: flush.min_events: 8 flush.timeout: 0.1s -{%- if elasticsearch %} -output.elasticsearch: - hosts: ["{{ elasticsearch.host }}"] -{%- else %} -output.file: - path: '{{ output_file_path|default(beat.working_dir + "/output") }}' - filename: {{ output_file_filename|default("auditbeat") }} -{%- endif %} +{% include './tests/system/config/libbeat.yml.j2' %} diff --git a/auditbeat/tests/system/test_base.py b/auditbeat/tests/system/test_base.py index ef19427e449c..7d1ef3ea9c1b 100644 --- a/auditbeat/tests/system/test_base.py +++ b/auditbeat/tests/system/test_base.py @@ -49,3 +49,25 @@ def test_template(self): assert exit_code == 0 assert self.log_contains('Loaded index template') assert len(es.cat.templates(name='auditbeat-*', h='name')) > 0 + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + def test_dashboards(self): + """ + Test that the dashboards can be loaded with `setup --dashboards` + """ + + kibana_dir = os.path.join(self.beat_path, "_meta", "kibana") + shutil.copytree(kibana_dir, os.path.join(self.working_dir, "kibana")) + + es = Elasticsearch([self.get_elasticsearch_url()]) + self.render_config_template( + modules=[{ + "name": "auditd", + }], + elasticsearch={"host": self.get_elasticsearch_url()}, + kibana={"host": self.get_kibana_url()}, + ) + exit_code = self.run_beat(extra_args=["setup", "--dashboards"]) + + assert exit_code == 0 + assert self.log_contains("Kibana dashboards successfully loaded.") diff --git a/filebeat/tests/system/config/filebeat.yml.j2 b/filebeat/tests/system/config/filebeat.yml.j2 index 56ddae6d81fe..d6b66239f330 100644 --- a/filebeat/tests/system/config/filebeat.yml.j2 +++ b/filebeat/tests/system/config/filebeat.yml.j2 @@ -119,101 +119,4 @@ filebeat.autodiscover: {%- endfor %} {% endif %} -#================================ General ===================================== - -# The name of the shipper that publishes the network data. It can be used to group -# all the transactions sent by a single shipper in the web interface. -# If this options is not defined, the hostname is used. -name: {{shipper_name}} - -# The tags of the shipper are included in their own field with each -# transaction published. Tags make it easy to group servers by different -# logical properties. -tags: [ - {%- if agent_tags -%} - {%- for tag in agent_tags -%} - "{{ tag }}" - {%- if not loop.last %}, {% endif -%} - {%- endfor -%} - {%- endif -%} -] - -{% if geoip_paths is not none %} -geoip: - paths: [ - {%- for path in geoip_paths -%} - "{{ beat.working_dir + '/' + path }}" - {%- if not loop.last %}, {% endif -%} - {%- endfor -%} - ] -{%- endif %} - -{% if setup_template_name %} -setup.template.name: setup_template_name -setup.template.pattern: setup_template_pattern -{%- endif %} - -{%- if processors %} - -#================================ Filters ===================================== - -processors: -{%- for processor in processors %} -{%- for name, settings in processor.items() %} -- {{name}}: - {%- if settings %} - {%- for k, v in settings.items() %} - {{k}}: - {{v | default([])}} - {%- endfor %} - {%- endif %} -{%- endfor %} -{%- endfor %} - -{%- endif %} - -#================================ Queue ===================================== - -queue.mem: - events: 4096 - flush.min_events: 8 - flush.timeout: 0.1s - -#================================ Outputs ===================================== - -# Configure what outputs to use when sending the data collected by the beat. -# Multiple outputs may be used. - -{%- if elasticsearch %} -#------------------------------- Elasticsearch output ---------------------------- -output.elasticsearch: - hosts: ["{{ elasticsearch.host }}"] -{% if elasticsearch.pipeline %} - pipeline: {{elasticsearch.pipeline}} -{% endif %} -{% if elasticsearch.index %} - index: {{elasticsearch.index}} -{% endif %} -{%- elif logstash %} -#------------------------------- Logstash output --------------------------------- -output.logstash: - hosts: ["{{ logstash.host }}"] -{%- else %} -#------------------------------- File output ---------------------------------- -output.file: - path: {{ output_file_path|default(beat.working_dir + "/output") }} - filename: "{{ output_file_filename|default("filebeat") }}" - rotate_every_kb: {{ rotate_every_kb | default(1000) }} - #number_of_files: 7 -{%- endif %} - -{% if path_data %} -#================================ Paths ===================================== -path: - data: {{path_data}} -{% endif %} - -{% if keystore_path %} -#================================ keystore ===================================== -keystore.path: {{keystore_path}} -{% endif %} +{% include './tests/system/config/libbeat.yml.j2' %} diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index 067989f80e31..697130a9c858 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -277,7 +277,10 @@ def copy_files(self, files, source_dir="files/"): def setUp(self): self.template_env = jinja2.Environment( - loader=jinja2.FileSystemLoader(self.beat_path) + loader=jinja2.FileSystemLoader([ + self.beat_path, + os.path.abspath(os.path.join(self.beat_path, "../libbeat")) + ]) ) # create working dir diff --git a/libbeat/tests/system/config/libbeat.yml.j2 b/libbeat/tests/system/config/libbeat.yml.j2 new file mode 100644 index 000000000000..09a35394dafa --- /dev/null +++ b/libbeat/tests/system/config/libbeat.yml.j2 @@ -0,0 +1,103 @@ +#================================ General ===================================== + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +# If this options is not defined, the hostname is used. +name: {{shipper_name}} + +# The tags of the shipper are included in their own field with each +# transaction published. Tags make it easy to group servers by different +# logical properties. +tags: [ + {%- if agent_tags -%} + {%- for tag in agent_tags -%} + "{{ tag }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + {%- endif -%} +] + +{% if geoip_paths is not none %} +geoip: + paths: [ + {%- for path in geoip_paths -%} + "{{ beat.working_dir + '/' + path }}" + {%- if not loop.last %}, {% endif -%} + {%- endfor -%} + ] +{%- endif %} + +{% if setup_template_name %} +setup.template.name: setup_template_name +setup.template.pattern: setup_template_pattern +{%- endif %} + +#================================ Processors ===================================== + +{%- if processors %} +processors: +{%- for processor in processors %} +{%- for name, settings in processor.items() %} +- {{name}}: + {%- if settings %} + {%- for k, v in settings.items() %} + {{k}}: + {{v | default([])}} + {%- endfor %} + {%- endif %} +{%- endfor %} +{%- endfor %} + +{%- endif %} + +#================================ Queue ===================================== + +queue.mem: + events: 4096 + flush.min_events: {{ flush_min_events|default(8) }} + flush.timeout: 0.1s + +{% if kibana -%} +setup.kibana.host: "{{ kibana.host }}" +{%- endif %} + +#================================ Outputs ===================================== + +# Configure what outputs to use when sending the data collected by the beat. +# Multiple outputs may be used. + +{% if elasticsearch -%} +output: + elasticsearch: + hosts: ["{{ elasticsearch.host }}"] + {% if elasticsearch.pipeline %} + pipeline: {{elasticsearch.pipeline}} + {% endif %} + {% if elasticsearch.index %} + index: {{elasticsearch.index}} + {% endif %} +{%- endif %} + +{% if logstash %} +output.logstash: + hosts: ["{{ logstash.host }}"] +{%- endif %} + +{% if not (console or elasticsearch or logstash) -%} +output.file: + path: {{ output_file_path|default(beat.working_dir + "/output") }} + filename: "{{ output_file_filename|default(beat.beat_name) }}" + rotate_every_kb: {{ rotate_every_kb | default(1000) }} + #number_of_files: 7 +{%- endif %} + +{% if path_data %} +#================================ Paths ===================================== +path: + data: {{path_data}} +{%endif%} + +{% if keystore_path %} +#================================ keystore ===================================== +keystore.path: {{keystore_path}} +{% endif %} diff --git a/metricbeat/tests/system/config/metricbeat.yml.j2 b/metricbeat/tests/system/config/metricbeat.yml.j2 index 3ed3879fc49e..7d0f7c299d98 100644 --- a/metricbeat/tests/system/config/metricbeat.yml.j2 +++ b/metricbeat/tests/system/config/metricbeat.yml.j2 @@ -101,43 +101,6 @@ metricbeat.config.modules: # Disable random start delay for metricsets. metricbeat.max_start_delay: 0 -#================================ General ===================================== - -# The name of the shipper that publishes the network data. It can be used to group -# all the transactions sent by a single shipper in the web interface. -# If this options is not defined, the hostname is used. -name: {{shipper_name}} - -# The tags of the shipper are included in their own field with each -# transaction published. Tags make it easy to group servers by different -# logical properties. -tags: [ - {%- if agent_tags -%} - {%- for tag in agent_tags -%} - "{{ tag }}" - {%- if not loop.last %}, {% endif -%} - {%- endfor -%} - {%- endif -%} -] - - -#================================ Processors ===================================== - -{%- if processors %} -processors: -{%- for processor in processors %} -{%- for name, settings in processor.items() %} -- {{name}}: - {%- if settings %} - {%- for k, v in settings.items() %} - {{k}}: - {{v | default([])}} - {%- endfor %} - {%- endif %} -{%- endfor %} -{%- endfor %} - -{%- endif %} #============================== Autodiscover ================================== @@ -155,44 +118,4 @@ metricbeat.autodiscover: {%- endfor %} {% endif %} -#================================ Queue ===================================== - -queue.mem: - events: 4096 - flush.min_events: {{ flush_min_events|default(8) }} - flush.timeout: 0.1s - -{% if kibana -%} -setup.kibana.host: "{{ kibana.host }}" -{%- endif %} - -#================================ Outputs ===================================== - -# Configure what outputs to use when sending the data collected by the beat. -# Multiple outputs may be used. - -output: - {% if elasticsearch -%} - elasticsearch: - hosts: ["{{ elasticsearch.host }}"] - {%- endif %} - - # File as output - # Options - # path: where to save the files - # filename: name of the files - # rotate_every_kb: maximum size of the files in path - # number of files: maximum number of files in path - {% if not (console or elasticsearch) -%} - file: - path: {{ output_file_path|default(beat.working_dir + "/output") }} - filename: "{{ output_file_filename|default("metricbeat") }}" - rotate_every_kb: 1000 - #number_of_files: 7 - {%- endif %} - -{% if path_data %} -#================================ Paths ===================================== -path: - data: {{path_data}} -{%endif%} +{% include './tests/system/config/libbeat.yml.j2' %} From bee250434b421177fb6a9065938b17fcb52aa4d2 Mon Sep 17 00:00:00 2001 From: ruflin Date: Thu, 21 Dec 2017 11:21:20 +1100 Subject: [PATCH 2/6] fix review --- auditbeat/tests/system/test_base.py | 6 ++++-- libbeat/tests/system/config/libbeat.yml.j2 | 10 ---------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/auditbeat/tests/system/test_base.py b/auditbeat/tests/system/test_base.py index 7d1ef3ea9c1b..347b374944c0 100644 --- a/auditbeat/tests/system/test_base.py +++ b/auditbeat/tests/system/test_base.py @@ -62,8 +62,10 @@ def test_dashboards(self): es = Elasticsearch([self.get_elasticsearch_url()]) self.render_config_template( modules=[{ - "name": "auditd", - }], + "name": "file_integrity", + "extras": { + "paths": ["file.example"], + }], elasticsearch={"host": self.get_elasticsearch_url()}, kibana={"host": self.get_kibana_url()}, ) diff --git a/libbeat/tests/system/config/libbeat.yml.j2 b/libbeat/tests/system/config/libbeat.yml.j2 index 09a35394dafa..687468183722 100644 --- a/libbeat/tests/system/config/libbeat.yml.j2 +++ b/libbeat/tests/system/config/libbeat.yml.j2 @@ -17,16 +17,6 @@ tags: [ {%- endif -%} ] -{% if geoip_paths is not none %} -geoip: - paths: [ - {%- for path in geoip_paths -%} - "{{ beat.working_dir + '/' + path }}" - {%- if not loop.last %}, {% endif -%} - {%- endfor -%} - ] -{%- endif %} - {% if setup_template_name %} setup.template.name: setup_template_name setup.template.pattern: setup_template_pattern From 060f9a158f5e0bac5b9d7b2f804340faedc08005 Mon Sep 17 00:00:00 2001 From: ruflin Date: Thu, 21 Dec 2017 15:26:08 +1100 Subject: [PATCH 3/6] Enable system tests and environment. Change testing. --- auditbeat/tests/system/test_base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/auditbeat/tests/system/test_base.py b/auditbeat/tests/system/test_base.py index 347b374944c0..139908760d63 100644 --- a/auditbeat/tests/system/test_base.py +++ b/auditbeat/tests/system/test_base.py @@ -1,5 +1,7 @@ import re import sys +import os +import shutil import unittest from auditbeat import BaseTest from elasticsearch import Elasticsearch @@ -65,7 +67,8 @@ def test_dashboards(self): "name": "file_integrity", "extras": { "paths": ["file.example"], - }], + } + }], elasticsearch={"host": self.get_elasticsearch_url()}, kibana={"host": self.get_kibana_url()}, ) From 099cc8ede0b0c46e8014e126bf788600bbf81ebc Mon Sep 17 00:00:00 2001 From: ruflin Date: Wed, 14 Mar 2018 15:53:24 +0100 Subject: [PATCH 4/6] add kibana environment --- auditbeat/docker-compose.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/auditbeat/docker-compose.yml b/auditbeat/docker-compose.yml index 6ba1a7cc7039..f3db9be04d89 100644 --- a/auditbeat/docker-compose.yml +++ b/auditbeat/docker-compose.yml @@ -7,6 +7,9 @@ services: env_file: - ${PWD}/build/test.env working_dir: /go/src/github.com/elastic/beats/auditbeat + environment: + - KIBANA_HOST=kibana + - KIBANA_PORT=5601 volumes: - ${PWD}/..:/go/src/github.com/elastic/beats/ command: make @@ -17,8 +20,14 @@ services: image: busybox depends_on: elasticsearch: { condition: service_healthy } + kibana: { condition: service_healthy } elasticsearch: extends: file: ../testing/environments/${TESTING_ENVIRONMENT}.yml service: elasticsearch + + kibana: + extends: + file: ../testing/environments/${TESTING_ENVIRONMENT}.yml + service: kibana From db667ce7b8a6de7199754a6a45f5ad4a72dadda9 Mon Sep 17 00:00:00 2001 From: ruflin Date: Thu, 15 Mar 2018 13:19:44 +0100 Subject: [PATCH 5/6] fix template variables --- libbeat/tests/system/config/libbeat.yml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/tests/system/config/libbeat.yml.j2 b/libbeat/tests/system/config/libbeat.yml.j2 index 687468183722..a8e494e63eb9 100644 --- a/libbeat/tests/system/config/libbeat.yml.j2 +++ b/libbeat/tests/system/config/libbeat.yml.j2 @@ -18,8 +18,8 @@ tags: [ ] {% if setup_template_name %} -setup.template.name: setup_template_name -setup.template.pattern: setup_template_pattern +setup.template.name: "{{setup_template_name}}" +setup.template.pattern: "{{setup_template_pattern}}" {%- endif %} #================================ Processors ===================================== From 1fdc523bf6ef331feff1377554460d3a656621b9 Mon Sep 17 00:00:00 2001 From: ruflin Date: Thu, 15 Mar 2018 13:26:01 +0100 Subject: [PATCH 6/6] fix filebeat tests with console --- filebeat/tests/system/test_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/tests/system/test_base.py b/filebeat/tests/system/test_base.py index 7a64c1f98df3..105be678390d 100644 --- a/filebeat/tests/system/test_base.py +++ b/filebeat/tests/system/test_base.py @@ -31,7 +31,7 @@ def test_invalid_config_with_removed_settings(self): """ Checks if filebeat fails to load if removed settings have been used: """ - self.render_config_template(console={"pretty": "false"}) + self.render_config_template() exit_code = self.run_beat(extra_args=[ "-E", "filebeat.spool_size=2048",