From b7de771f5309e3b084620b40e11dbc7aeffac27d Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 6 Feb 2024 13:27:53 -0700 Subject: [PATCH] work in progress for idaholab/Malcolm#395, malcolm reporting capture statistics from zeek/suricata --- config/logstash.env.example | 2 + logstash/pipelines/zeek/10_zeek_prep.conf | 24 +- logstash/pipelines/zeek/11_zeek_parse.conf | 260 +++++++++++++++++++++ shared/bin/suricata_config_populate.py | 3 +- 4 files changed, 271 insertions(+), 18 deletions(-) diff --git a/config/logstash.env.example b/config/logstash.env.example index 48ce73ff3..a9436ad51 100644 --- a/config/logstash.env.example +++ b/config/logstash.env.example @@ -18,5 +18,7 @@ LOGSTASH_NETBOX_AUTO_POPULATE=false # Caching parameters for NetBox's LogStash lookups LOGSTASH_NETBOX_CACHE_SIZE=1000 LOGSTASH_NETBOX_CACHE_TTL=30 +# Zeek log types that will be ignored (dropped) by LogStash +LOGSTASH_ZEEK_IGNORED_LOGS=analyzer,broker,bsap_ip_unknown,bsap_serial_unknown,capture_loss,cluster,config,ecat_arp_info,loaded_scripts,packet_filter,png,print,prof,reporter,stats,stderr,stdout # Logstash memory allowance and other Java options LS_JAVA_OPTS=-server -Xms2500m -Xmx2500m -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true \ No newline at end of file diff --git a/logstash/pipelines/zeek/10_zeek_prep.conf b/logstash/pipelines/zeek/10_zeek_prep.conf index 6e0785a35..c54668ce9 100644 --- a/logstash/pipelines/zeek/10_zeek_prep.conf +++ b/logstash/pipelines/zeek/10_zeek_prep.conf @@ -23,24 +23,14 @@ filter { end" } - # report types we're going to ignore - if (([log_source] == "analyzer") or - ([log_source] == "bsap_ip_unknown") or - ([log_source] == "bsap_serial_unknown") or - ([log_source] == "ecat_arp_info") or - ([log_source] == "reporter") or - ([log_source] == "broker") or - ([log_source] == "cluster") or - ([log_source] == "capture_loss") or - ([log_source] == "communication") or - ([log_source] == "packet_filter") or - ([log_source] == "png") or - ([log_source] == "stats") or - ([log_source] == "stderr") or - ([log_source] == "stdout") or - ([log_source] == "loaded_scripts")) { - drop { id => "drop_zeek_ignored_source" } + # Zeek logs we're going to ignore + ruby { + id => "ruby_zeek_log_type_determine_drop" + init => "logtypesStr = ENV['LOGSTASH_ZEEK_IGNORED_LOGS'] || 'analyzer,broker,bsap_ip_unknown,bsap_serial_unknown,capture_loss,cluster,config,ecat_arp_info,loaded_scripts,packet_filter,png,print,prof,reporter,stats,stderr,stdout' ; $logtypes = logtypesStr.gsub(/\s+/, '').split(',');" + code => "event.set('[@metadata][drop_zeek_log]', true) if $logtypes.include?(event.get('[log_source]').to_s)" } + if [@metadata][drop_zeek_log] { drop { id => "drop_zeek_ignored_source" } } + # remove some tags pulled from the filename we might not want if ([@metadata][zeek_log_tags]) { diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index 1ce4a0074..99fee1b6e 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -5439,6 +5439,266 @@ filter { } # if / else if for opcua log types + } else if ([log_source] == "analyzer") { + ############################################################################################################################# + # analyzer.log + # Zeek Logging analyzer confirmations and violations into analyzer.log + # https://docs.zeek.org/en/master/scripts/base/frameworks/analyzer/logging.zeek.html + + dissect { + id => "dissect_zeek_diagnostic_analyzer" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][cause]} %{[zeek_cols][analyzer_kind]} %{[zeek_cols][analyzer_name]} %{[zeek_cols][uid]} %{[zeek_cols][fuid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][failure_reason]} %{[zeek_cols][failure_data]}" + } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_analyzer" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_analyzer" + init => "$zeek_diagnostic_analyzer_field_names = [ 'ts', 'cause', 'analyzer_kind', 'analyzer_name', 'uid', 'fuid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'failure_reason', 'failure_data' ]" + code => "event.set('[zeek_cols]', $zeek_diagnostic_analyzer_field_names.zip(event.get('[message]')).to_h)" + } + } + + mutate { id => "mutate_add_tag_zeek_diagnostic_analyzer" + add_tag => [ "_zeekdiagnostic" ] } + + } else if ([log_source] == "broker") { + ############################################################################################################################# + # broker.log + # https://docs.zeek.org/en/master/scripts/base/frameworks/broker/log.zeek.html + + dissect { + id => "dissect_zeek_diagnostic_broker" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][event_type]} %{[zeek_cols][event_action]} %{[zeek_cols][peer_ip]} %{[zeek_cols][peer_port]} %{[zeek_cols][peer_message]}" + } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_broker" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_broker" + init => "$zeek_diagnostic_broker_field_names = [ 'ts', 'event_type', 'event_action', 'peer_ip', 'peer_port', 'peer_message' ]" + code => "event.set('[zeek_cols]', $zeek_diagnostic_broker_field_names.zip(event.get('[message]')).to_h)" + } + } + + mutate { id => "mutate_add_tag_zeek_diagnostic_broker" + add_tag => [ "_zeekdiagnostic" ] } + + } else if ([log_source] == "capture_loss") { + ############################################################################################################################# + # capture_loss.log + # Reports analysis of missing traffic. Zeek bases its conclusions on analysis of TCP sequence numbers. + # https://docs.zeek.org/en/master/logs/capture-loss-and-reporter.html + + dissect { + id => "dissect_zeek_diagnostic_capture_loss" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][ts_delta]} %{[zeek_cols][peer]} %{[zeek_cols][gaps]} %{[zeek_cols][acks]} %{[zeek_cols][percent_lost]}" + } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_capture_loss" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_capture_loss" + init => "$zeek_diagnostic_capture_loss_field_names = [ 'ts', 'ts_delta', 'peer', 'gaps', 'acks', 'percent_lost' ]" + code => "event.set('[zeek_cols]', $zeek_diagnostic_capture_loss_field_names.zip(event.get('[message]')).to_h)" + } + } + + mutate { id => "mutate_add_tag_zeek_diagnostic_capture_loss" + add_tag => [ "_zeekdiagnostic" ] } + + } else if ([log_source] == "cluster") { + ############################################################################################################################# + # cluster.log + # Logging for establishing and controlling a cluster of Zeek instances + # https://docs.zeek.org/en/master/scripts/base/frameworks/cluster/main.zeek.html#type-Cluster::Info + + dissect { + id => "dissect_zeek_diagnostic_cluster" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][node]} %{[zeek_cols][node_message]}" + } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_cluster" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_cluster" + init => "$zeek_diagnostic_cluster_field_names = [ 'ts', 'node', 'node_message' ]" + code => "event.set('[zeek_cols]', $zeek_diagnostic_cluster_field_names.zip(event.get('[message]')).to_h)" + } + } + + mutate { id => "mutate_add_tag_zeek_diagnostic_cluster" + add_tag => [ "_zeekdiagnostic" ] } + + } else if ([log_source] == "config") { + ############################################################################################################################# + # config.log + # Logging for Zeek configuration changes + # https://docs.zeek.org/en/master/scripts/base/frameworks/config/main.zeek.html#type-Config::Info + + dissect { + id => "dissect_zeek_diagnostic_config" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][value_name]} %{[zeek_cols][value_old]} %{[zeek_cols][value_new]} %{[zeek_cols][location]}" + } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_config" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_config" + init => "$zeek_diagnostic_config_field_names = [ 'ts', 'value_name', 'value_old', 'value_new', 'location' ]" + code => "event.set('[zeek_cols]', $zeek_diagnostic_config_field_names.zip(event.get('[message]')).to_h)" + } + } + + mutate { id => "mutate_add_tag_zeek_diagnostic_config" + add_tag => [ "_zeekdiagnostic" ] } + + } else if ([log_source] == "packet_filter") { + ############################################################################################################################# + # packet_filter.log + # https://docs.zeek.org/en/master/scripts/base/frameworks/packet-filter/main.zeek.html#type-PacketFilter::Info + + dissect { + id => "dissect_zeek_diagnostic_packet_filter" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][node]} %{[zeek_cols][filter]} %{[zeek_cols][init]} %{[zeek_cols][success]} %{[zeek_cols][failure_reason]}" + } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_packet_filter" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_packet_filter" + init => "$zeek_diagnostic_packet_filter_field_names = [ 'ts', 'node', 'filter', 'init', 'success', 'failure_reason' ]" + code => "event.set('[zeek_cols]', $zeek_diagnostic_packet_filter_field_names.zip(event.get('[message]')).to_h)" + } + } + + mutate { id => "mutate_add_tag_zeek_diagnostic_packet_filter" + add_tag => [ "_zeekdiagnostic" ] } + + } else if ([log_source] == "print") { + ############################################################################################################################# + # print.log + # https://docs.zeek.org/en/master/scripts/base/frameworks/logging/main.zeek.html#type-Log::PrintLogInfo + + dissect { + id => "dissect_zeek_diagnostic_print" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][vals]}" + } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_print" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_print" + init => "$zeek_diagnostic_print_field_names = [ 'ts', 'vals' ]" + code => "event.set('[zeek_cols]', $zeek_diagnostic_print_field_names.zip(event.get('[message]')).to_h)" + } + } + + mutate { id => "split_zeek_diagnostic_print_vals" + split => { "[zeek_cols][vals]" => "," } } + + mutate { id => "mutate_add_tag_zeek_diagnostic_print" + add_tag => [ "_zeekdiagnostic" ] } + + + } else if ([log_source] == "reporter") { + ############################################################################################################################# + # reporter.log + # https://docs.zeek.org/en/master/scripts/base/frameworks/reporter/main.zeek.html#type-Reporter::Info + + dissect { + id => "dissect_zeek_diagnostic_reporter" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][level]} %{[zeek_cols][msg]} %{[zeek_cols][location]}" + } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_reporter" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_reporter" + init => "$zeek_diagnostic_reporter_field_names = [ 'ts', 'node', 'filter', 'init', 'success', 'failure_reason' ]" + code => "event.set('[zeek_cols]', $zeek_diagnostic_reporter_field_names.zip(event.get('[message]')).to_h)" + } + } + + mutate { id => "mutate_add_tag_zeek_diagnostic_reporter" + add_tag => [ "_zeekdiagnostic" ] } + + } else if ([log_source] == "stats") { + ############################################################################################################################# + # stats.log + # https://docs.zeek.org/en/master/scripts/policy/misc/stats.zeek.html#type-Stats::Info + + dissect { + id => "dissect_zeek_diagnostic_stats" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][peer]} %{[zeek_cols][mem]} %{[zeek_cols][pkts_proc]} %{[zeek_cols][bytes_recv]} %{[zeek_cols][pkts_dropped]} %{[zeek_cols][pkts_link]} %{[zeek_cols][pkt_lag]} %{[zeek_cols][pkts_filtered]} %{[zeek_cols][events_proc]} %{[zeek_cols][events_queued]} %{[zeek_cols][active_tcp_conns]} %{[zeek_cols][active_udp_conns]} %{[zeek_cols][active_icmp_conns]} %{[zeek_cols][tcp_conns]} %{[zeek_cols][udp_conns]} %{[zeek_cols][icmp_conns]} %{[zeek_cols][timers]} %{[zeek_cols][active_timers]} %{[zeek_cols][files]} %{[zeek_cols][active_files]} %{[zeek_cols][dns_requests]} %{[zeek_cols][active_dns_requests]} %{[zeek_cols][reassem_tcp_size]} %{[zeek_cols][reassem_file_size]} %{[zeek_cols][reassem_frag_size]} %{[zeek_cols][reassem_unknown_size]}" + } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_stats" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_stats" + init => "$zeek_diagnostic_stats_field_names = [ 'ts', 'peer', 'mem', 'pkts_proc', 'bytes_recv', 'pkts_dropped', 'pkts_link', 'pkt_lag', 'pkts_filtered', 'events_proc', 'events_queued', 'active_tcp_conns', 'active_udp_conns', 'active_icmp_conns', 'tcp_conns', 'udp_conns', 'icmp_conns', 'timers', 'active_timers', 'files', 'active_files', 'dns_requests', 'active_dns_requests', 'reassem_tcp_size', 'reassem_file_size', 'reassem_frag_size', 'reassem_unknown_size' ]" + code => "event.set('[zeek_cols]', $zeek_diagnostic_stats_field_names.zip(event.get('[message]')).to_h)" + } + } + + mutate { id => "mutate_add_tag_zeek_diagnostic_stats" + add_tag => [ "_zeekdiagnostic" ] } + } else { # some other unknown zeek log file. should start with ts at least! csv { diff --git a/shared/bin/suricata_config_populate.py b/shared/bin/suricata_config_populate.py index 50c365304..00998116b 100755 --- a/shared/bin/suricata_config_populate.py +++ b/shared/bin/suricata_config_populate.py @@ -187,6 +187,7 @@ def __call__(self, repr, data): 'SSH_EVE_ENABLED': False, 'SSH_HASSH': True, 'SSH_PORTS': 22, + 'STATS': False, 'STREAM_CHECKSUM_VALIDATION': False, 'STREAM_INLINE': 'auto', 'STREAM_MEMCAP': '64mb', @@ -1170,7 +1171,7 @@ def main(): logging.error(output) # final tweaks - deep_set(cfg, ['stats', 'enabled'], False) + deep_set(cfg, ['stats', 'enabled'], val2bool(DEFAULT_VARS['STATS'])) cfg.pop('rule-files', None) deep_set(cfg, ['rule-files'], GetRuleFiles())