for idaholab#415, beginning work on using DNS for populating hostname…

…s in netbox (work in progress, probably broken)
mmguero-dev · Apr 18, 2024 · 5439f6c · 5439f6c
1 parent 7d44153
commit 5439f6c
Show file tree

Hide file tree

Showing 3 changed files with 212 additions and 141 deletions.
diff --git a/config/logstash.env.example b/config/logstash.env.example
@@ -10,7 +10,7 @@ LOGSTASH_SEVERITY_SCORING=true
 # Whether or not Logstash will perform a reverse DNS lookup for external IP addresses
 LOGSTASH_REVERSE_DNS=false
 # Which types of logs will be enriched via NetBox (comma-separated list of provider.dataset, or the string all to enrich all logs)
-LOGSTASH_NETBOX_ENRICHMENT_DATASETS=suricata.alert,zeek.conn,zeek.known_hosts,zeek.known_services,zeek.notice,zeek.signatures,zeek.software,zeek.weird
+LOGSTASH_NETBOX_ENRICHMENT_DATASETS=suricata.alert,zeek.conn,zeek.dhcp,zeek.dns,zeek.known_hosts,zeek.known_services,zeek.ntlm,zeek.notice,zeek.signatures,zeek.software,zeek.weird
 # Zeek log types that will be ignored (dropped) by LogStash
 LOGSTASH_ZEEK_IGNORED_LOGS=analyzer,broker,cluster,config,loaded_scripts,packet_filter,png,print,prof,reporter,stderr,stdout
 # Logstash memory allowance and other Java options

diff --git a/logstash/pipelines/enrichment/21_netbox.conf b/logstash/pipelines/enrichment/21_netbox.conf
@@ -13,7 +13,7 @@ filter {
   ruby {
     id => "ruby_determine_netbox_suitability"
     # @logtypes = {"suricata"=>["alert"], "zeek"=>["conn", "known_hosts", "known_services", "notice", "signatures", "software", "weird"]}
-    init => "logtypesStr = ENV['LOGSTASH_NETBOX_ENRICHMENT_DATASETS'] || 'suricata.alert,zeek.conn,zeek.known_hosts,zeek.known_services,zeek.notice,zeek.signatures,zeek.software,zeek.weird' ; logtypesArr = logtypesStr.gsub(/\s+/, '').split(','); @logtypes = logtypesArr.group_by { |logtype| logtype.split('.').first }.transform_values { |values| values.map { |v| v.split('.')[1] } }"
+    init => "logtypesStr = ENV['LOGSTASH_NETBOX_ENRICHMENT_DATASETS'] || 'suricata.alert,zeek.conn,zeek.dhcp,zeek.dns,zeek.known_hosts,zeek.known_services,zeek.ntlm,zeek.notice,zeek.signatures,zeek.software,zeek.weird' ; logtypesArr = logtypesStr.gsub(/\s+/, '').split(','); @logtypes = logtypesArr.group_by { |logtype| logtype.split('.').first }.transform_values { |values| values.map { |v| v.split('.')[1] } }"
     code => "
       provider = event.get('[event][provider]').to_s
       dataset = event.get('[event][dataset]').to_s
@@ -28,18 +28,48 @@ filter {
   }
 
   if ([@metadata][do_netbox_enrichment]) {
+
+    if ([dns][question][name]) and ([dns][resolved_ip]) {
+      ruby {
+        id => "ruby_netbox_enrich_dns_ip_to_host"
+        path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb"
+        # the "planned" status indicates that while we'll create the device
+        #   entry with an IP address and hostname, additional details (such
+        #   as the manufacturer based on MAC address) will need to be updated
+        #   later (also note the blank "target" which means this record will be
+        #   used to populate the netbox database, but nothing will actually
+        #   be stored in a field of the record itself as a result of this filter)
+        script_params => {
+          "lookup_type" => "ip_device"
+          "default_status" => "planned"
+          "source" => "[dns][resolved_ip]"
+          "source_hostname" => "[dns][question][name]"
+          "enabled_env" => "NETBOX_ENRICHMENT"
+          "verbose_env" => "NETBOX_ENRICHMENT_VERBOSE"
+          "lookup_site_env" => "NETBOX_DEFAULT_SITE"
+          "netbox_token_env" => "SUPERUSER_API_TOKEN"
+          "cache_size_env" => "NETBOX_CACHE_SIZE"
+          "cache_ttl_env" => "NETBOX_CACHE_TTL"
+          "autopopulate_env" => "NETBOX_AUTO_POPULATE"
+          "default_manuf_env" => "NETBOX_DEFAULT_MANUFACTURER"
+          "default_dtype_env" => "NETBOX_DEFAULT_DEVICE_TYPE"
+          "default_role_env" => "NETBOX_DEFAULT_ROLE"
+        }
+      }
+    }
+
     if ([source][ip]) and
        (([network][direction] == "internal") or ([network][direction] == "outbound")) {
       ruby {
         id => "ruby_netbox_enrich_source_ip_segment"
         path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb"
         script_params => {
-          "enabled_env" => "NETBOX_ENRICHMENT"
-          "verbose_env" => "NETBOX_ENRICHMENT_VERBOSE"
           "source" => "[source][ip]"
           "target" => "[source][segment]"
-          "auto_prefix_env" => "NETBOX_AUTO_CREATE_PREFIX"
           "lookup_type" => "ip_prefix"
+          "enabled_env" => "NETBOX_ENRICHMENT"
+          "verbose_env" => "NETBOX_ENRICHMENT_VERBOSE"
+          "auto_prefix_env" => "NETBOX_AUTO_CREATE_PREFIX"
           "lookup_site_env" => "NETBOX_DEFAULT_SITE"
           "netbox_token_env" => "SUPERUSER_API_TOKEN"
           "cache_size_env" => "NETBOX_CACHE_SIZE"
@@ -50,13 +80,15 @@ filter {
         id => "ruby_netbox_enrich_source_ip_device"
         path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb"
         script_params => {
-          "enabled_env" => "NETBOX_ENRICHMENT"
-          "verbose_env" => "NETBOX_ENRICHMENT_VERBOSE"
+          "lookup_type" => "ip_device"
+          "default_status" => "staged"
           "source" => "[source][ip]"
           "target" => "[source][device]"
-          "lookup_type" => "ip_device"
+          "source_oui" => "[source][oui]"
+          "source_mac" => "[source][mac]"
+          "enabled_env" => "NETBOX_ENRICHMENT"
+          "verbose_env" => "NETBOX_ENRICHMENT_VERBOSE"
           "lookup_site_env" => "NETBOX_DEFAULT_SITE"
-          "lookup_service" => "false"
           "netbox_token_env" => "SUPERUSER_API_TOKEN"
           "cache_size_env" => "NETBOX_CACHE_SIZE"
           "cache_ttl_env" => "NETBOX_CACHE_TTL"
@@ -66,8 +98,6 @@ filter {
           "default_role_env" => "NETBOX_DEFAULT_ROLE"
           "autopopulate_fuzzy_threshold_env" => "NETBOX_DEFAULT_FUZZY_THRESHOLD"
           "autopopulate_create_manuf_env" => "NETBOX_DEFAULT_AUTOCREATE_MANUFACTURER"
-          "source_oui" => "[source][oui]"
-          "source_mac" => "[source][mac]"
         }
       }
     }
@@ -77,12 +107,12 @@ filter {
         id => "ruby_netbox_enrich_destination_ip_segment"
         path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb"
         script_params => {
-          "enabled_env" => "NETBOX_ENRICHMENT"
-          "verbose_env" => "NETBOX_ENRICHMENT_VERBOSE"
+          "lookup_type" => "ip_prefix"
           "source" => "[destination][ip]"
           "target" => "[destination][segment]"
+          "enabled_env" => "NETBOX_ENRICHMENT"
+          "verbose_env" => "NETBOX_ENRICHMENT_VERBOSE"
           "auto_prefix_env" => "NETBOX_AUTO_CREATE_PREFIX"
-          "lookup_type" => "ip_prefix"
           "lookup_site_env" => "NETBOX_DEFAULT_SITE"
           "netbox_token_env" => "SUPERUSER_API_TOKEN"
           "cache_size_env" => "NETBOX_CACHE_SIZE"
@@ -93,11 +123,14 @@ filter {
         id => "ruby_netbox_enrich_destination_ip_device"
         path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb"
         script_params => {
-          "enabled_env" => "NETBOX_ENRICHMENT"
-          "verbose_env" => "NETBOX_ENRICHMENT_VERBOSE"
+          "lookup_type" => "ip_device"
+          "default_status" => "staged"
           "source" => "[destination][ip]"
           "target" => "[destination][device]"
-          "lookup_type" => "ip_device"
+          "source_oui" => "[destination][oui]"
+          "source_mac" => "[destination][mac]"
+          "enabled_env" => "NETBOX_ENRICHMENT"
+          "verbose_env" => "NETBOX_ENRICHMENT_VERBOSE"
           "lookup_site_env" => "NETBOX_DEFAULT_SITE"
           "lookup_service_env" => "NETBOX_ENRICHMENT_LOOKUP_SERVICE"
           "lookup_service_port_source" => "[destination][port]"
@@ -110,8 +143,6 @@ filter {
           "default_role_env" => "NETBOX_DEFAULT_ROLE"
           "autopopulate_fuzzy_threshold_env" => "NETBOX_DEFAULT_FUZZY_THRESHOLD"
           "autopopulate_create_manuf_env" => "NETBOX_DEFAULT_AUTOCREATE_MANUFACTURER"
-          "source_oui" => "[destination][oui]"
-          "source_mac" => "[destination][mac]"
         }
       }
     }