work in progress for idaholab#331, improvements to extracted_files_ht…

…tp_server.py and the setting/creation of ACL rules on hedgehog

arkime/etc/config.ini

@@ -516,6 +516,7 @@ zeek.files.sha256=db:zeek.files.sha256;group:zeek_files;kind:termfield;viewerOnl
 zeek.files.extracted=db:zeek.files.extracted;group:zeek_files;kind:termfield;viewerOnly:true;friendly:Extracted Filename;help:Extracted Filename
 zeek.files.extracted_cutoff=db:zeek.files.extracted_cutoff;group:zeek_files;kind:termfield;viewerOnly:true;friendly:Truncated;help:Truncated
 zeek.files.extracted_size=db:zeek.files.extracted_size;group:zeek_files;kind:integer;viewerOnly:true;friendly:Extracted Bytes;help:Extracted Bytes
+zeek.files.extracted_uri=db:zeek.files.extracted_uri;group:zeek_files;kind:termfield;viewerOnly:true;friendly:Extracted Filename URL;help:Extracted File URL
 
 # ftp.log
 # https://docs.zeek.org/en/stable/scripts/base/protocols/ftp/info.zeek.html#type-FTP::Info
@@ -2661,7 +2662,7 @@ o_zeek_ecat_log_address=require:zeek.ecat_log_address;title:Zeek ecat_log_addres
 o_zeek_ecat_registers=require:zeek.ecat_registers;title:Zeek ecat_registers.log;fields:zeek.ecat_registers.command,zeek.ecat_registers.server_addr,zeek.ecat_registers.register_type,zeek.ecat_registers.register_addr,zeek.ecat_registers.data
 o_zeek_ecat_soe_info=require:zeek.ecat_soe_info;title:Zeek ecat_soe_info.log;fields:zeek.ecat_soe_info.opcode,zeek.ecat_soe_info.incomplete,zeek.ecat_soe_info.error,zeek.ecat_soe_info.drive_num,zeek.ecat_soe_info.element,zeek.ecat_soe_info.index
 o_zeek_enip=require:zeek.enip;title:Zeek enip.log;fields:zeek.enip.enip_command,zeek.enip.enip_command_code,zeek.enip.length,zeek.enip.session_handle,zeek.enip.enip_status,zeek.enip.sender_context,zeek.enip.options
-o_zeek_files=require:zeek.files;title:Zeek files.log;fields:zeek.files.tx_hosts,zeek.files.rx_hosts,zeek.files.conn_uids,zeek.files.source,zeek.files.depth,zeek.files.analyzers,zeek.files.mime_type,zeek.files.filename,zeek.files.ftime,zeek.files.duration,zeek.files.local_orig,zeek.files.seen_bytes,zeek.files.total_bytes,zeek.files.missing_bytes,zeek.files.overflow_bytes,zeek.files.timedout,zeek.files.parent_fuid,zeek.files.md5,zeek.files.sha1,zeek.files.sha256,zeek.files.extracted,zeek.files.extracted_cutoff,zeek.files.extracted_size
+o_zeek_files=require:zeek.files;title:Zeek files.log;fields:zeek.files.tx_hosts,zeek.files.rx_hosts,zeek.files.conn_uids,zeek.files.source,zeek.files.depth,zeek.files.analyzers,zeek.files.mime_type,zeek.files.filename,zeek.files.ftime,zeek.files.duration,zeek.files.local_orig,zeek.files.seen_bytes,zeek.files.total_bytes,zeek.files.missing_bytes,zeek.files.overflow_bytes,zeek.files.timedout,zeek.files.parent_fuid,zeek.files.md5,zeek.files.sha1,zeek.files.sha256,zeek.files.extracted,zeek.files.extracted_cutoff,zeek.files.extracted_size,zeek.files.extracted_uri
 o_zeek_ftp=require:zeek.ftp;title:Zeek ftp.log;fields:zeek.ftp.command,zeek.ftp.arg,zeek.ftp.mime_type,zeek.ftp.file_size,zeek.ftp.reply_code,zeek.ftp.reply_msg,zeek.ftp.data_channel.passive,zeek.ftp.data_channel.orig_h,zeek.ftp.data_channel.resp_h,zeek.ftp.data_channel.resp_p
 o_zeek_genisys=require:zeek.genisys;title:Zeek genisys.log;fields:zeek.genisys.header,zeek.genisys.server,zeek.genisys.direction,zeek.genisys.crc_transmitted,zeek.genisys.crc_calculated,zeek.genisys.payload.address,zeek.genisys.payload.data
 o_zeek_gquic=require:zeek.gquic;title:Zeek gquic.log;fields:zeek.gquic.version,zeek.gquic.server_name,zeek.gquic.user_agent,zeek.gquic.tag_count,zeek.gquic.cyu,zeek.gquic.cyutags

arkime/wise/source.zeeklogs.js

@@ -966,6 +966,7 @@ class MalcolmSource extends WISESource {
       "zeek.files.extracted",
       "zeek.files.extracted_cutoff",
       "zeek.files.extracted_size",
+      "zeek.files.extracted_uri",
       "zeek.files.filename",
       "zeek.files.ftime",
       "zeek.files.local_orig",
@@ -2253,8 +2254,10 @@ class MalcolmSource extends WISESource {
     this.api.addValueAction("malcolm_websearch_mime", { name: "Media Type Registry", url: 'https://www.iana.org/assignments/media-types/%TEXT%', fields: mimeFieldsStr });
 
     // add right-click for extracted files from zeek
-    var carvedFieldsStr = allFields.filter(value => /^zeek\.files\.extracted$/i.test(value)).join(',');
-    this.api.addValueAction("malcolm_carved_file_quarantined", { name: "Download", url: "/extracted-files/%TEXT%", fields: carvedFieldsStr });
+    // var carvedFieldsStr = allFields.filter(value => /^zeek\.files\.extracted$/i.test(value)).join(',');
+    // this.api.addValueAction("malcolm_carved_file", { name: "Download", url: "/extracted-files/%TEXT%", fields: carvedFieldsStr });
+    var carvedFieldsUrlStr = allFields.filter(value => /^zeek\.files\.extracted_uri$/i.test(value)).join(',');
+    this.api.addValueAction("malcolm_carved_file_url", { name: "Download", url: "/%TEXT%", fields: carvedFieldsUrlStr });
 
     // add right-clicks for pivoting into dashboards from Arkime (see nginx.conf)
     var filterLabel = "OpenSearch Dashboards %DBFIELD%";

dashboards/templates/composable/component/zeek.json

@@ -67,6 +67,7 @@
         "zeek.files.extracted": { "type": "keyword" },
         "zeek.files.extracted_cutoff": { "type": "keyword" },
         "zeek.files.extracted_size": { "type": "long" },
+        "zeek.files.extracted_uri": { "type": "keyword" },
         "zeek.files.filename": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } },
         "zeek.files.ftime": { "type": "date" },
         "zeek.files.local_orig": { "type": "keyword" },

logstash/pipelines/zeek/12_zeek_mutate.conf

@@ -16,6 +16,11 @@ filter {
   if (![event][provider]) { mutate { id => "mutate_add_field_event_provider_zeek"
                                      add_field => { "[event][provider]" => "zeek" } } }
 
+  if (![host][name]) and ([agent][hostname]) {
+    mutate { id => "mutate_zeek_add_field_host_name_agent_hostname"
+             add_field => { "[host][name]" => "%{[agent][hostname]}" } }
+  }
+
   # rename the zeek child array to match the log type
   mutate { id => "mutate_rename_zeek_log_type"
            rename => { "[zeek_cols]" => "[zeek][%{[log_source]}]" } }
@@ -560,6 +565,25 @@ filter {
     if ([zeek][files][sha256]) {       mutate { id => "mutate_add_field_ecs_files_hash_sha256"
                                       add_field => { "[file][hash][sha256]" => "%{[zeek][files][sha256]}" } } }
 
+    if ([zeek][files][extracted]) {
+      ruby {
+        id => "ruby_zeek_files_extracted_uri_build"
+        code => "
+          uri = nil
+          if (fName = event.get('[zeek][files][extracted]')) then
+            if (tags = event.get('[tags]')) && tags.include?('_filebeat_zeek_hedgehog_live') then
+              if (hName = event.get('[host][name]')) then
+                uri = 'hh-extracted-files/' + hName + '/' + fName
+              end
+            else
+              uri = 'extracted-files/' + fName
+            end
+          end
+          event.set('[zeek][files][extracted_uri]', uri) unless uri.nil? or (uri.length == 0)
+        "
+      }
+    }
+
   } else if ([log_source] == "ftp") {
     #############################################################################################################################
     # ftp.log specific logic