From dbf33678679065b5d52e5066b7ab3b6f9e302b74 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 25 Jun 2024 15:13:54 -0600 Subject: [PATCH] tweaks for idaholab/Malcolm#419, testing ja4+ merge --- arkime/wise/source.zeeklogs.js | 4 -- .../composable/component/malcolm_common.json | 4 +- logstash/pipelines/zeek/12_zeek_mutate.conf | 55 +++++++------------ 3 files changed, 22 insertions(+), 41 deletions(-) diff --git a/arkime/wise/source.zeeklogs.js b/arkime/wise/source.zeeklogs.js index d12224941..8d79c275e 100644 --- a/arkime/wise/source.zeeklogs.js +++ b/arkime/wise/source.zeeklogs.js @@ -715,10 +715,6 @@ class MalcolmSource extends WISESource { "threat.technique.id", "threat.technique.name", "threat.technique.reference", - "tcp.client.ja4l", - "tcp.server.ja4ls", - "tcp.client.ja4t", - "tcp.server.ja4ts", "tls.cipher", "tls.client.issuer", "tls.client.ja4", diff --git a/dashboards/templates/composable/component/malcolm_common.json b/dashboards/templates/composable/component/malcolm_common.json index 5189218b9..88a942e26 100644 --- a/dashboards/templates/composable/component/malcolm_common.json +++ b/dashboards/templates/composable/component/malcolm_common.json @@ -71,7 +71,9 @@ "url": { "type": "keyword" }, "details": { "type": "nested" } } - } + }, + "tls.client.ja4": { "type": "keyword" }, + "tls.server.ja4s": { "type": "keyword" } } } } diff --git a/logstash/pipelines/zeek/12_zeek_mutate.conf b/logstash/pipelines/zeek/12_zeek_mutate.conf index 8e4bccde4..707c679f0 100644 --- a/logstash/pipelines/zeek/12_zeek_mutate.conf +++ b/logstash/pipelines/zeek/12_zeek_mutate.conf @@ -180,6 +180,7 @@ filter { } } + # rename conn.log's TCP JA4 fields to what Arkime uses if ([zeek][conn][ja4l]) { mutate { id => "mutate_merge_zeek_tcp_ja4l" merge => { "[tcp][ja4l]" => "[zeek][conn][ja4l]" } } @@ -200,27 +201,6 @@ filter { merge => { "[tcp][ja4ts]" => "[zeek][conn][ja4ts]" } } } - # ECS uses the client/server convention with JA4. This is parallelled with the JA4 location fields - if ([zeek][conn][ja4l]) { - mutate { id => "mutate_add_field_zeek_tcp_ja4l" - add_field => { "[tcp][client][ja4l]" => "[zeek][conn][ja4l]" } } - } - - if ([zeek][conn][ja4ls]) { - mutate { id => "mutate_add_field_zeek_tcp_ja4ls" - add_field => { "[tcp][server][ja4ls]" => "[zeek][conn][ja4ls]" } } - } - - if ([zeek][conn][ja4t]) { - mutate { id => "mutate_add_field_zeek_tcp_ja4t" - add_field => { "[tcp][client][ja4t]" => "[zeek][conn][ja4t]" } } - } - - if ([zeek][conn][ja4ts]) { - mutate { id => "mutate_add_field_zeek_tcp_ja4ts" - add_field => { "[tcp][server][ja4ts]" => "[zeek][conn][ja4ts]" } } - } - # aggregate total bytes and packets ruby { id => "ruby_zeek_bytes_and_packets_calc" @@ -920,18 +900,23 @@ filter { ############################################################################################################################# # ja4ssh.log specific logic - mutate { - id => "mutate_rename_ja4ssh_fields" - rename => { "[zeek][ja4ssh][ja4ssh]" => "[zeek][ssh][ja4ssh]" } - } - - mutate { - id => "mutate_merge_ja4ssh_fields" - merge => { "[ssh][ja4ssh]" => "[zeek][ssh][ja4ssh]" } + if ([zeek][ja4ssh][ja4ssh]) { + mutate { + id => "mutate_rename_ja4ssh_fields" + rename => { "[zeek][ja4ssh][ja4ssh]" => "[zeek][ssh][ja4ssh]" } + } + mutate { + id => "mutate_merge_ja4ssh_fields" + merge => { "[ssh][ja4ssh]" => "[zeek][ssh][ja4ssh]" } + } } mutate { id => "mutate_remove_fields_zeek_ja4ssh" - remove_field => [ "[zeek][ja4ssh]" ] } + remove_field => [ "[zeek][ja4ssh][is_ssh]", + "[zeek][ja4ssh][orig_pack_len]", + "[zeek][ja4ssh][resp_pack_len]", + "[zeek][ja4ssh][orig_ack]", + "[zeek][ja4ssh][resp_ack]" ] } } else if ([log_source] == "kerberos") { ############################################################################################################################# @@ -2070,8 +2055,6 @@ filter { if ([zeek][ssl][ja4]) { mutate { id => "mutate_merge_zeek_ssl_ja4" merge => { "[tls][ja4]" => "[zeek][ssl][ja4]" } } - mutate { id => "mutate_merge_zeek_ssl_ja4_ecs" - merge => { "[tls][ja4]" => "[zeek][ssl][ja4]" } } } if ([zeek][ssl][ja4s]) { @@ -2131,8 +2114,8 @@ filter { add_field => { "[tls][curve]" => "%{[zeek][ssl][curve]}" } } } # ECS - zeek.ssl.ja4 -> tls.client.ja4 - if ([zeek][ssl][ja4]) { mutate { id => "mutate_add_field_ecs_zeek_tls_client_ja4" - add_field => { "[tls][client][ja4]" => "%{[zeek][ssl][ja4]}" } } } + if ([zeek][ssl][ja4]) { mutate { id => "mutate_rename_ecs_zeek_tls_client_ja4" + rename => { "[zeek][ssl][ja4]" => "[tls][client][ja4]" } } } # ECS - zeek.ssl.client_issuer_full -> tls.client.issuer if ([zeek][ssl][client_issuer_full]) { mutate { id => "mutate_add_field_ecs_zeek_tls_client_issuer_full" @@ -2155,8 +2138,8 @@ filter { add_field => { "[tls][server][issuer]" => "%{[zeek][ssl][issuer_full]}" } } } # ECS - zeek.ssl.ja4s -> tls.server.ja4s - if ([zeek][ssl][ja4s]) { mutate { id => "mutate_add_field_ecs_zeek_tls_server_ja4s" - add_field => { "[tls][server][ja4s]" => "%{[zeek][ssl][ja4s]}" } } } + if ([zeek][ssl][ja4s]) { mutate { id => "mutate_rename_field_ecs_zeek_tls_server_ja4s" + rename => { "[zeek][ssl][ja4s]" => "[tls][server][ja4s]" } } } # ECS - zeek.ssl.subject_full -> tls.server.subject if ([zeek][ssl][subject_full]) { mutate { id => "mutate_add_field_ecs_zeek_tls_subject_full"