From 0fd6ab89b7578f289060392c64aa8cc8875e0a42 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 29 Mar 2024 13:53:45 -0600 Subject: [PATCH] work in progress for idaholab/Malcolm#331, improvements to extracted_files_http_server.py and the setting/creation of ACL rules on hedgehog --- Dockerfiles/file-monitor.Dockerfile | 30 ++++---- file-monitor/scripts/.gitignore | 1 - hedgehog-iso/build.sh | 10 +++ hedgehog-iso/build_via_vagrant.sh | 5 +- .../normal/0169-pip-installs.hook.chroot | 2 + ...{ufw_arkime_viewer => ufw_sensor_services} | 2 +- .../interface/sensor_ctl/control_vars.conf | 14 +++- .../sensor_ctl/supervisor.d/zeek.conf | 20 ++++++ .../supervisor.init/arkime_config_populate.sh | 2 +- hedgehog-raspi/generate-recipe.py | 6 +- hedgehog-raspi/sensor_install.sh | 4 ++ shared/bin/configure-capture.py | 14 ++-- .../bin}/extracted_files_http_server.py | 71 ++++++++++++++----- shared/bin/ufw_allow_requests.sh | 67 +++++++++++++++++ shared/bin/ufw_allow_viewer.sh | 62 ---------------- shared/bin/web-ui-asset-download.sh | 34 +++++++++ 16 files changed, 236 insertions(+), 108 deletions(-) delete mode 100644 file-monitor/scripts/.gitignore rename hedgehog-iso/config/includes.chroot/etc/sudoers.d/{ufw_arkime_viewer => ufw_sensor_services} (53%) rename {file-monitor/scripts => shared/bin}/extracted_files_http_server.py (94%) create mode 100755 shared/bin/ufw_allow_requests.sh delete mode 100755 shared/bin/ufw_allow_viewer.sh create mode 100755 shared/bin/web-ui-asset-download.sh diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile index 45f7f29ff..ac500cda7 100644 --- a/Dockerfiles/file-monitor.Dockerfile +++ b/Dockerfiles/file-monitor.Dockerfile @@ -103,6 +103,11 @@ ENV SUPERCRONIC_SHA1SUM "cd48d45c4b10f3f0bfdd3a57d054cd05ac96812b" ENV SUPERCRONIC_CRONTAB "/etc/crontab" COPY --chmod=755 shared/bin/yara_rules_setup.sh /usr/local/bin/ +ADD nginx/landingpage/css "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/css" +ADD nginx/landingpage/js "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/js" +ADD --chmod=644 docs/images/logo/Malcolm_background.png "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/assets/img/bg-masthead.png" +COPY --chmod=644 docs/images/icon/favicon.ico "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/favicon.ico" +COPY --chmod=755 shared/bin/web-ui-asset-download.sh /usr/local/bin/ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sources && \ apt-get -q update && \ @@ -170,6 +175,8 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour rm -rf "${SRC_BASE_DIR}"/yara* && \ cd "${YARA_RULES_SRC_DIR}" && \ /usr/local/bin/yara_rules_setup.sh -r "${YARA_RULES_SRC_DIR}" -y "${YARA_RULES_DIR}" && \ + cd /tmp && \ + /usr/local/bin/web-ui-asset-download.sh -o "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/css" && \ cd /tmp && \ curl -fsSL -o ./capa.zip "${CAPA_URL}" && \ unzip ./capa.zip && \ @@ -190,9 +197,6 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour libtool \ make \ python3-dev && \ - apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages autoremove && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* /tmp/* && \ mkdir -p /var/log/clamav "${CLAMAV_RULES_DIR}" && \ groupadd --gid ${DEFAULT_GID} ${PGROUP} && \ useradd -m --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER} && \ @@ -214,7 +218,10 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/clam_scan.py && \ ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/yara_scan.py && \ ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/capa_scan.py && \ - echo "0 */6 * * * /bin/bash /usr/local/bin/capa-update.sh\n0 */6 * * * /usr/local/bin/yara_rules_setup.sh -r \"${YARA_RULES_SRC_DIR}\" -y \"${YARA_RULES_DIR}\"" > ${SUPERCRONIC_CRONTAB} + echo "0 */6 * * * /bin/bash /usr/local/bin/capa-update.sh\n0 */6 * * * /usr/local/bin/yara_rules_setup.sh -r \"${YARA_RULES_SRC_DIR}\" -y \"${YARA_RULES_DIR}\"" > ${SUPERCRONIC_CRONTAB} && \ + apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages autoremove && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* && \ USER ${PUSER} @@ -222,23 +229,10 @@ RUN /usr/bin/freshclam freshclam --config-file=/etc/clamav/freshclam.conf USER root -ADD nginx/landingpage/css "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/css" -ADD nginx/landingpage/js "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/js" -ADD --chmod=644 docs/images/logo/Malcolm_background.png "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/assets/img/bg-masthead.png" -ADD --chmod=644 https://fonts.gstatic.com/s/lato/v24/S6u_w4BMUTPHjxsI9w2_Gwfo.ttf "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/css/" -ADD --chmod=644 https://fonts.gstatic.com/s/lato/v24/S6u8w4BMUTPHjxsAXC-v.ttf "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/css/" -ADD --chmod=644 https://fonts.gstatic.com/s/lato/v24/S6u_w4BMUTPHjxsI5wq_Gwfo.ttf "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/css/" -ADD --chmod=644 https://fonts.gstatic.com/s/lato/v24/S6u9w4BMUTPHh7USSwiPHA.ttf "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/css/" -ADD --chmod=644 https://fonts.gstatic.com/s/lato/v24/S6uyw4BMUTPHjx4wWw.ttf "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/css/" -ADD --chmod=644 https://fonts.gstatic.com/s/lato/v24/S6u9w4BMUTPHh6UVSwiPHA.ttf "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/css/" -ADD --chmod=644 'https://cdn.jsdelivr.net/npm/bootstrap-icons@1.5.0/font/fonts/bootstrap-icons.woff2?856008caa5eb66df68595e734e59580d' "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/css/bootstrap-icons.woff2" -ADD --chmod=644 'https://cdn.jsdelivr.net/npm/bootstrap-icons@1.5.0/font/fonts/bootstrap-icons.woff?856008caa5eb66df68595e734e59580d' "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/css/bootstrap-icons.woff" - -COPY --chmod=644 docs/images/icon/favicon.ico "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/favicon.ico" COPY --chmod=755 shared/bin/docker-uid-gid-setup.sh /usr/local/bin/ COPY --chmod=755 shared/bin/service_check_passthrough.sh /usr/local/bin/ COPY --chmod=755 shared/bin/zeek_carve*.py /usr/local/bin/ -COPY --chmod=755 file-monitor/scripts/*.py /usr/local/bin/ +COPY --chmod=644 shared/bin/extracted_files_http_server.py /usr/local/bin/ COPY --chmod=644 shared/bin/watch_common.py /usr/local/bin/ COPY --chmod=644 scripts/malcolm_utils.py /usr/local/bin/ COPY --chmod=644 file-monitor/supervisord.conf /etc/supervisord.conf diff --git a/file-monitor/scripts/.gitignore b/file-monitor/scripts/.gitignore deleted file mode 100644 index a2d7c8915..000000000 --- a/file-monitor/scripts/.gitignore +++ /dev/null @@ -1 +0,0 @@ -malcolm_utils.py diff --git a/hedgehog-iso/build.sh b/hedgehog-iso/build.sh index 10de19435..2a6e16fe8 100755 --- a/hedgehog-iso/build.sh +++ b/hedgehog-iso/build.sh @@ -114,6 +114,16 @@ if [ -d "$WORKDIR" ]; then chown -R root:root ./config/includes.chroot/usr/local/bin/ ./config/includes.chroot/opt/zeek/bin/ rsync -a "$SCRIPT_PATH/suricata/" ./config/includes.chroot/opt/sensor/sensor_ctl/suricata/ + # assets for extracted file server + mkdir -p ./config/includes.chroot/opt/sensor/assets/img/ + rsync -a "$SCRIPT_PATH/nginx/" ./config/includes.chroot/opt/sensor/assets/ + cp "$SCRIPT_PATH"/docs/images/icon/favicon.ico ./config/includes.chroot/opt/sensor/assets/ + cp "$SCRIPT_PATH"/docs/images/logo/Malcolm_background.png ./config/includes.chroot/opt/sensor/assets/img/bg-masthead.png + bash "$SCRIPT_PATH/shared/bin/web-ui-asset-download.sh" -o ./config/includes.chroot/opt/sensor/assets/css/ + chown -R root:root ./config/includes.chroot/opt/sensor/assets/css/ + find ./config/includes.chroot/opt/sensor/assets/ -type d -exec chmod 755 "{}" \; + find ./config/includes.chroot/opt/sensor/assets/ -type f -exec chmod 644 "{}" \; + # write out some version stuff specific to this installation version echo "BUILD_ID=\"$(date +'%Y-%m-%d')-${IMAGE_VERSION}\"" > ./config/includes.chroot/opt/sensor/.os-info echo "VARIANT=\"Hedgehog Linux (Sensor) v${IMAGE_VERSION}\"" >> ./config/includes.chroot/opt/sensor/.os-info diff --git a/hedgehog-iso/build_via_vagrant.sh b/hedgehog-iso/build_via_vagrant.sh index 73c557598..e9ba553b5 100755 --- a/hedgehog-iso/build_via_vagrant.sh +++ b/hedgehog-iso/build_via_vagrant.sh @@ -30,6 +30,7 @@ function cleanup_shared_and_docs { "$SCRIPT_PATH"/Gemfile \ "$SCRIPT_PATH"/README.md \ "$SCRIPT_PATH"/suricata \ + "$SCRIPT_PATH"/nginx \ "$SCRIPT_PATH"/htpdate } @@ -90,8 +91,10 @@ cp -r "$SCRIPT_PATH"/../shared \ "$SCRIPT_PATH"/../README.md "$SCRIPT_PATH"/ cp "$SCRIPT_PATH"/../scripts/documentation_build.sh "$SCRIPT_PATH"/docs/ cp "$SCRIPT_PATH"/../scripts/malcolm_utils.py "$SCRIPT_PATH"/shared/bin/ -mkdir "$SCRIPT_PATH"/suricata +mkdir "$SCRIPT_PATH"/nginx "$SCRIPT_PATH"/suricata cp -r "$SCRIPT_PATH"/../suricata/rules-default "$SCRIPT_PATH"/suricata/ +cp -r "$SCRIPT_PATH"/../nginx/css "$SCRIPT_PATH"/nginx/ +cp -r "$SCRIPT_PATH"/../nginx/js "$SCRIPT_PATH"/nginx/ YML_IMAGE_VERSION="$(grep -P "^\s+image:.*/malcolm/" "$SCRIPT_PATH"/../docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" [[ -n $YML_IMAGE_VERSION ]] && echo "$YML_IMAGE_VERSION" > "$SCRIPT_PATH"/shared/version.txt diff --git a/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot b/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot index 3a217625c..37bbf8d5c 100755 --- a/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot +++ b/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot @@ -12,9 +12,11 @@ pip3 install --break-system-packages --no-compile --no-cache-dir --force-reinsta clamd \ dateparser \ debinterface \ + dominate \ pymisp \ python-dotenv \ ruamel.yaml \ stix2 \ + stream-zip \ taxii2-client \ watchdog diff --git a/hedgehog-iso/config/includes.chroot/etc/sudoers.d/ufw_arkime_viewer b/hedgehog-iso/config/includes.chroot/etc/sudoers.d/ufw_sensor_services similarity index 53% rename from hedgehog-iso/config/includes.chroot/etc/sudoers.d/ufw_arkime_viewer rename to hedgehog-iso/config/includes.chroot/etc/sudoers.d/ufw_sensor_services index 55f79d5e2..f89d43f29 100644 --- a/hedgehog-iso/config/includes.chroot/etc/sudoers.d/ufw_arkime_viewer +++ b/hedgehog-iso/config/includes.chroot/etc/sudoers.d/ufw_sensor_services @@ -1,2 +1,2 @@ # allow unprivileged mgmt of UFW access for the local Arkime viewer instance -%netdev ALL=(root) NOPASSWD: /usr/local/bin/ufw_allow_viewer.sh +%netdev ALL=(root) NOPASSWD: /usr/local/bin/ufw_allow_requests.sh diff --git a/hedgehog-iso/interface/sensor_ctl/control_vars.conf b/hedgehog-iso/interface/sensor_ctl/control_vars.conf index 66a9ec266..04d324989 100644 --- a/hedgehog-iso/interface/sensor_ctl/control_vars.conf +++ b/hedgehog-iso/interface/sensor_ctl/control_vars.conf @@ -12,7 +12,6 @@ export PCAP_PRUNE_CHECK_SECONDS=60 export ARKIME_VIEWER_PORT=8005 export ARKIME_PACKET_THREADS=5 -export ARKIME_PACKET_ACL= export ARKIME_ECS_PROVIDER=arkime export ARKIME_ECS_DATASET=session export ARKIME_COMPRESSION_TYPE=zstd @@ -26,6 +25,19 @@ export ARKIME_FREESPACEG=7% export ARKIME_ROTATE_INDEX=daily export ARKIME_DEBUG_LEVEL=0 +# Whether or not to serve the directory containing Zeek-extracted over HTTP at ./extracted-files/ +export EXTRACTED_FILE_HTTP_SERVER_ENABLE=false +export EXTRACTED_FILE_HTTP_SERVER_PORT=8006 +export EXTRACTED_FILE_HTTP_ASSETS_DIR=/opt/assets +# Whether or not Zeek-extracted files served over HTTP will be archived in a Zip file +export EXTRACTED_FILE_HTTP_SERVER_ZIP=false +# Whether or not to use libmagic to show MIME types for Zeek-extracted files served +export EXTRACTED_FILE_HTTP_SERVER_MAGIC=false +# HTTP server will look in subdirectories for requested filename (e.g., in "/quarantined" and "/preserved") +export EXTRACTED_FILE_HTTP_SERVER_RECURSIVE=true + +export MALCOLM_REQUEST_ACL= +export MALCOLM_REQUEST_PORTS=$ARKIME_VIEWER_PORT,$EXTRACTED_FILE_HTTP_SERVER_PORT export DOCUMENTATION_PORT=8420 export MISCBEAT_PORT=9516 export FLUENTBIT_METRICS_INTERVAL=30 diff --git a/hedgehog-iso/interface/sensor_ctl/supervisor.d/zeek.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/zeek.conf index 6148fa342..e3787b232 100644 --- a/hedgehog-iso/interface/sensor_ctl/supervisor.d/zeek.conf +++ b/hedgehog-iso/interface/sensor_ctl/supervisor.d/zeek.conf @@ -101,3 +101,23 @@ autostart=%(ENV_ZEEK_FILE_WATCH)s autorestart=%(ENV_ZEEK_FILE_WATCH)s directory=%(ENV_ZEEK_LOG_PATH)s user=sensor + +[program:fileserve] +command=/usr/local/bin/extracted_files_http_server.py + --port %(ENV_EXTRACTED_FILE_HTTP_SERVER_PORT)s + --zip %(ENV_EXTRACTED_FILE_HTTP_SERVER_ZIP)s + --recursive %(ENV_EXTRACTED_FILE_HTTP_SERVER_RECURSIVE)s + --directory "%(ENV_ZEEK_LOG_PATH)s/extract_files" + --assets-directory "%(ENV_EXTRACTED_FILE_HTTP_ASSETS_DIR)s" +startsecs=30 +startretries=3 +stopasgroup=true +killasgroup=true +autostart=%(ENV_EXTRACTED_FILE_HTTP_SERVER_ENABLE)s +autorestart=%(ENV_EXTRACTED_FILE_HTTP_SERVER_ENABLE)s +directory=%(ENV_ZEEK_LOG_PATH)s/extract_files +user=sensor + +; #TODO: assets? +; --assets-directory-req-replacer /assets +; --assets-directory-resp-replacer /extracted-files/assets diff --git a/hedgehog-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh b/hedgehog-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh index 41273b6b9..6643040df 100644 --- a/hedgehog-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh +++ b/hedgehog-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh @@ -112,7 +112,7 @@ if [[ -n $SUPERVISOR_PATH ]] && [[ -r "$SUPERVISOR_PATH"/arkime/config.ini ]]; t fi # update the firewall ACL (via ufw) to allow retrieval of packets - sudo --non-interactive /usr/local/bin/ufw_allow_viewer.sh + sudo --non-interactive /usr/local/bin/ufw_allow_requests.sh # make sure interface flags are set appropriately for capture if [[ -n $CAPTURE_INTERFACE ]]; then diff --git a/hedgehog-raspi/generate-recipe.py b/hedgehog-raspi/generate-recipe.py index 4ad1535c7..7dfbc98a3 100755 --- a/hedgehog-raspi/generate-recipe.py +++ b/hedgehog-raspi/generate-recipe.py @@ -92,7 +92,7 @@ # Nothing yet! extra_root_shell_cmds = [ 'cp sensor_install.sh "${ROOT?}/root/"', - '/bin/bash -c \'mkdir -p "${ROOT?}/opt/"{buildshared,deps,hooks,patches,sensor/sensor_ctl/suricata/rules-default,arkime/etc,zeek/bin}\'', + '/bin/bash -c \'mkdir -p "${ROOT?}/opt/"{sensor/assets/img,buildshared,deps,hooks,patches,sensor/sensor_ctl/suricata/rules-default,arkime/etc,zeek/bin}\'', 'cp "%s/arkime/patch/"* "${ROOT?}/opt/patches/" || true' % MALCOLM_DIR, 'cp "%s/arkime/etc/"* "${ROOT?}/opt/arkime/etc" || true' % SENSOR_DIR, 'cp -r "%s/suricata/rules-default/"* "${ROOT?}/opt/sensor/sensor_ctl/suricata/rules-default/" || true' @@ -110,6 +110,10 @@ 'cp -r "%s/config/hooks/normal/"* "${ROOT?}/opt/hooks/"' % SENSOR_DIR, 'cp -r "%s/config/package-lists/"* "${ROOT?}/opt/deps/"' % SENSOR_DIR, 'cp -r "%s/docs/images/hedgehog/logo/hedgehog-ascii-text.txt"* "${ROOT?}/root/"' % MALCOLM_DIR, + 'cp -r "%s/nginx/css/" "${ROOT?}/opt/sensor/assets/"' % MALCOLM_DIR, + 'cp -r "%s/nginx/js/" "${ROOT?}/opt/sensor/assets/"' % MALCOLM_DIR, + 'cp -r "%s/docs/images/icon/favicon.ico" "${ROOT?}/opt/sensor/assets/"' % MALCOLM_DIR, + 'cp -r "%s/docs/images/logo/Malcolm_background.png" "${ROOT?}/opt/sensor/assets/img/bg-masthead.png"' % MALCOLM_DIR, ] # Extend list just in case version is 4 diff --git a/hedgehog-raspi/sensor_install.sh b/hedgehog-raspi/sensor_install.sh index c377b8433..d1f95924f 100644 --- a/hedgehog-raspi/sensor_install.sh +++ b/hedgehog-raspi/sensor_install.sh @@ -397,6 +397,10 @@ install_files() { curl -s -S -L -o ./oui.txt "https://www.wireshark.org/download/automated/data/manuf" popd >/dev/null 2>&1 + # download assets for extracted file server + /usr/local/bin/web-ui-asset-download.sh -o /opt/sensor/assets/css + find /opt/sensor/assets -type f + # Prepare Fluentbit and Beats repo GPG keys local apt_lists='/etc/apt/sources.list.d' local apt_keys='/etc/apt/keyrings' diff --git a/shared/bin/configure-capture.py b/shared/bin/configure-capture.py index 44cc1b01c..7f06908d4 100755 --- a/shared/bin/configure-capture.py +++ b/shared/bin/configure-capture.py @@ -105,7 +105,7 @@ class Constants: # specific to arkime ARKIME_PASSWORD_SECRET = "ARKIME_PASSWORD_SECRET" - ARKIME_PACKET_ACL = "ARKIME_PACKET_ACL" + MALCOLM_REQUEST_ACL = "MALCOLM_REQUEST_ACL" ARKIME_COMPRESSION_TYPE = "ARKIME_COMPRESSION_TYPE" ARKIME_COMPRESSION_LEVEL = "ARKIME_COMPRESSION_LEVEL" ARKIME_COMPRESSION_TYPES = ( @@ -376,10 +376,12 @@ def main(): "OS_USERNAME" in capture_config_dict.keys() ): previous_config_values[Constants.BEAT_HTTP_USERNAME] = capture_config_dict["OS_USERNAME"] - if (Constants.ARKIME_PACKET_ACL not in previous_config_values.keys()) and ( - "ARKIME_PACKET_ACL" in capture_config_dict.keys() + if (Constants.MALCOLM_REQUEST_ACL not in previous_config_values.keys()) and ( + "MALCOLM_REQUEST_ACL" in capture_config_dict.keys() ): - previous_config_values[Constants.ARKIME_PACKET_ACL] = capture_config_dict[Constants.ARKIME_PACKET_ACL] + previous_config_values[Constants.MALCOLM_REQUEST_ACL] = capture_config_dict[ + Constants.MALCOLM_REQUEST_ACL + ] if (Constants.ARKIME_PASSWORD_SECRET not in previous_config_values.keys()) and ( "ARKIME_PASSWORD_SECRET" in capture_config_dict.keys() ): @@ -894,14 +896,14 @@ def main(): arkime_config_dict[Constants.ARKIME_PASSWORD_SECRET] = arkime_password # get list of IP addresses allowed for packet payload retrieval - lines = previous_config_values[Constants.ARKIME_PACKET_ACL].split(",") + lines = previous_config_values[Constants.MALCOLM_REQUEST_ACL].split(",") lines.append(opensearch_config_dict[Constants.BEAT_OS_HOST]) code, lines = d.editbox_str( "\n".join(list(filter(None, list(set(lines))))), title=Constants.MSG_CONFIG_ARKIME_PCAP_ACL ) if code != Dialog.OK: raise CancelledError - arkime_config_dict[Constants.ARKIME_PACKET_ACL] = ','.join( + arkime_config_dict[Constants.MALCOLM_REQUEST_ACL] = ','.join( [ ip for ip in list(set(filter(None, [x.strip() for x in lines.split('\n')]))) diff --git a/file-monitor/scripts/extracted_files_http_server.py b/shared/bin/extracted_files_http_server.py similarity index 94% rename from file-monitor/scripts/extracted_files_http_server.py rename to shared/bin/extracted_files_http_server.py index e1f2e0743..26af3b4c5 100755 --- a/file-monitor/scripts/extracted_files_http_server.py +++ b/shared/bin/extracted_files_http_server.py @@ -7,21 +7,23 @@ # be aes-256-cbc encrypted in a way that's compatible with: # openssl enc -aes-256-cbc -d -in encrypted.data -out decrypted.data +import atexit import argparse import dominate +import functools import hashlib import magic import os import re +import ssl import sys +import time from Crypto.Cipher import AES from datetime import datetime, timedelta, UTC from dominate.tags import * -from http.server import HTTPServer, SimpleHTTPRequestHandler -from socketserver import ThreadingMixIn +from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler from stat import S_IFREG from stream_zip import ZIP_32, stream_zip -from threading import Thread from malcolm_utils import ( eprint, @@ -70,7 +72,7 @@ class HTTPHandler(SimpleHTTPRequestHandler): def translate_path(self, path): path = SimpleHTTPRequestHandler.translate_path(self, path) relpath = os.path.relpath(path, os.getcwd()) - fullpath = os.path.join(self.server.base_path, relpath) + fullpath = os.path.join(self.directory, relpath) return fullpath, relpath # override do_GET for fancy directory listing and so that files are encrypted/zipped, if requested @@ -417,17 +419,21 @@ def do_GET(self): ################################################################################################### # -class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): - def __init__(self, base_path, server_address, RequestHandlerClass=HTTPHandler): - self.base_path = base_path - HTTPServer.__init__(self, server_address, RequestHandlerClass) - - -################################################################################################### -# -def serve_on_port(path: str, port: int): - server = ThreadingHTTPServer(path, ("", port)) - print(f"serving {path} at port {port}") +def serve_on_port( + path, + port, + tls=False, + tls_key_file=None, + tls_cert_file=None, + server_class=ThreadingHTTPServer, + handler_class=HTTPHandler, +): + server = server_class(("", port), functools.partial(handler_class, directory=path)) + if tlsOk := (tls and os.path.isfile(str(tls_key_file)) and os.path.isfile(str(tls_cert_file))): + ctx = ssl.SSLContext(protocol=ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(certfile=tls_cert_file, keyfile=tls_key_file) + server.socket = ctx.wrap_socket(server.socket, server_side=True) + print(f"serving {path} at port {port}{' over TLS' if tlsOk else ''}") server.serve_forever() @@ -474,6 +480,33 @@ def main(): type=int, default=defaultPort, ) + parser.add_argument( + '-t', + '--tls', + dest='tls', + type=str2bool, + nargs='?', + const=True, + default=defaultMagic, + metavar='true|false', + help=f"Serve with TLS (must specify --tls-keyfile and --tls-certfile)", + ) + parser.add_argument( + '--tls-keyfile', + dest='tlsKeyFile', + help=f'TLS Key File', + metavar='', + type=str, + default=None, + ) + parser.add_argument( + '--tls-certfile', + dest='tlsCertFile', + help=f'TLS Certificate File', + metavar='', + type=str, + default=None, + ) parser.add_argument( '-d', '--directory', @@ -591,7 +624,13 @@ def main(): if args.assetsDirRespReplacer: args.assetsDirRespReplacer = os.path.join(args.assetsDirRespReplacer, '') - Thread(target=serve_on_port, args=[args.serveDir, args.port]).start() + serve_on_port( + path=args.serveDir, + port=args.port, + tls=args.tls, + tls_key_file=args.tlsKeyFile, + tls_cert_file=args.tlsCertFile, + ) ################################################################################################### diff --git a/shared/bin/ufw_allow_requests.sh b/shared/bin/ufw_allow_requests.sh new file mode 100755 index 000000000..bec7776a0 --- /dev/null +++ b/shared/bin/ufw_allow_requests.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. + +# manage a UFW rule for allowing a remote Malcolm instance to connect to +# services hosted on the sensor + +# works with a comma-separated list of IP addresses in $MALCOLM_REQUEST_ACL, or +# if that variable is not set, a single IP address in $OS_HOST + +[[ "$(uname -s)" = 'Darwin' ]] && REALPATH=grealpath || REALPATH=realpath +[[ "$(uname -s)" = 'Darwin' ]] && DIRNAME=gdirname || DIRNAME=dirname +if ! (type "$REALPATH" && type "$DIRNAME") > /dev/null; then + echo "$(basename "${BASH_SOURCE[0]}") requires $REALPATH and $DIRNAME" + exit 1 +fi +export SCRIPT_PATH="$($DIRNAME $($REALPATH -e "${BASH_SOURCE[0]}"))" + +# control_vars.conf file must be specified as argument to script or be found in an expected place +# source configuration variables file if found (precedence: pwd, script directory, /opt/sensor/sensor_ctl) +if [[ -n "$1" ]]; then + source "$1" +else + CONTROL_VARS_FILE="control_vars.conf" + if [[ -r ./"$CONTROL_VARS_FILE" ]]; then + source ./"$CONTROL_VARS_FILE" + elif [[ -r "$SCRIPT_PATH"/"$CONTROL_VARS_FILE" ]]; then + source "$SCRIPT_PATH"/"$CONTROL_VARS_FILE" + elif [[ -r /opt/sensor/sensor_ctl/"$CONTROL_VARS_FILE" ]]; then + source /opt/sensor/sensor_ctl/"$CONTROL_VARS_FILE" + fi +fi + +if [[ -z $MALCOLM_REQUEST_PORTS ]] || ( [[ -z $MALCOLM_REQUEST_ACL ]] && [[ -z $OS_HOST ]] ); then + echo "Either the remote host (\$MALCOLM_REQUEST_ACL or \$OS_HOST) or the request ports (\$MALCOLM_REQUEST_PORTS) is undefined" + exit 1 +elif [[ ! -x /usr/sbin/ufw ]]; then + echo "/usr/sbin/ufw does not exist or is not executable" + exit 1 +fi + +while read SERVICE_PORT; do + + # delete previous UFW rule(s) + while read LINE; do + if [[ -n $LINE ]] && [[ "$LINE" =~ ^[0-9]+$ ]]; then + /usr/sbin/ufw --force delete $LINE + fi + done <<< "$(/usr/sbin/ufw status numbered | tac | grep "${SERVICE_PORT}/tcp" | sed "s/].*//" | sed "s/[^0-9]*//g")" + + # add new UFW rule(s) + if [[ -n $MALCOLM_REQUEST_ACL ]]; then + # loop over ACL IP addresses + IFS="," + for IP in $MALCOLM_REQUEST_ACL; do + /usr/sbin/ufw allow proto tcp from $IP to any port $SERVICE_PORT + done + unset IFS + elif [[ -n $OS_HOST ]]; then + # ACL not defined, create a rule for $OS_HOST + /usr/sbin/ufw allow proto tcp from $OS_HOST to any port $SERVICE_PORT + fi + + # output status of rule + /usr/sbin/ufw status | grep "${SERVICE_PORT}/tcp" + +done < <(echo "${MALCOLM_REQUEST_PORTS}" | tr ',' '\n') # loop over ',' separated MALCOLM_REQUEST_PORTS values \ No newline at end of file diff --git a/shared/bin/ufw_allow_viewer.sh b/shared/bin/ufw_allow_viewer.sh deleted file mode 100755 index 90c06fbaf..000000000 --- a/shared/bin/ufw_allow_viewer.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. - -# manage a UFW rule for allowing a remote Arkime viewer instance (on the same host -# to which arkime's capture is forwarding session logs) to connect to and -# retrieve PCAP segments from the local Arkime viewer instance - -# works with a comma-separated list of IP addresses in $ARKIME_PACKET_ACL, or -# if that variable is not set, a single IP address in $OS_HOST - -[[ "$(uname -s)" = 'Darwin' ]] && REALPATH=grealpath || REALPATH=realpath -[[ "$(uname -s)" = 'Darwin' ]] && DIRNAME=gdirname || DIRNAME=dirname -if ! (type "$REALPATH" && type "$DIRNAME") > /dev/null; then - echo "$(basename "${BASH_SOURCE[0]}") requires $REALPATH and $DIRNAME" - exit 1 -fi -export SCRIPT_PATH="$($DIRNAME $($REALPATH -e "${BASH_SOURCE[0]}"))" - -# control_vars.conf file must be specified as argument to script or be found in an expected place -# source configuration variables file if found (precedence: pwd, script directory, /opt/sensor/sensor_ctl) -if [[ -n "$1" ]]; then - source "$1" -else - CONTROL_VARS_FILE="control_vars.conf" - if [[ -r ./"$CONTROL_VARS_FILE" ]]; then - source ./"$CONTROL_VARS_FILE" - elif [[ -r "$SCRIPT_PATH"/"$CONTROL_VARS_FILE" ]]; then - source "$SCRIPT_PATH"/"$CONTROL_VARS_FILE" - elif [[ -r /opt/sensor/sensor_ctl/"$CONTROL_VARS_FILE" ]]; then - source /opt/sensor/sensor_ctl/"$CONTROL_VARS_FILE" - fi -fi - -if [[ -z $ARKIME_VIEWER_PORT ]] || ( [[ -z $ARKIME_PACKET_ACL ]] && [[ -z $OS_HOST ]] ); then - echo "Either the remote Arkime viewer host (\$ARKIME_PACKET_ACL or \$OS_HOST) or the local Arkime viewer port (\$ARKIME_VIEWER_PORT) is undefined" - exit 1 -elif [[ ! -x /usr/sbin/ufw ]]; then - echo "/usr/sbin/ufw does not exist or is not executable" - exit 1 -fi - -# delete previous UFW rule(s) -while read LINE; do - if [[ -n $LINE ]] && [[ "$LINE" =~ ^[0-9]+$ ]]; then - /usr/sbin/ufw --force delete $LINE - fi -done <<< "$(/usr/sbin/ufw status numbered | tac | grep "${ARKIME_VIEWER_PORT}/tcp" | sed "s/].*//" | sed "s/[^0-9]*//g")" - -# add new UFW rule(s) -if [[ -n $ARKIME_PACKET_ACL ]]; then - IFS="," - for IP in $ARKIME_PACKET_ACL; do - /usr/sbin/ufw allow proto tcp from $IP to any port $ARKIME_VIEWER_PORT - done - unset IFS -elif [[ -n $OS_HOST ]]; then - /usr/sbin/ufw allow proto tcp from $OS_HOST to any port $ARKIME_VIEWER_PORT -fi - -# output status of rule -/usr/sbin/ufw status | grep "${ARKIME_VIEWER_PORT}/tcp" diff --git a/shared/bin/web-ui-asset-download.sh b/shared/bin/web-ui-asset-download.sh new file mode 100755 index 000000000..5200b40e1 --- /dev/null +++ b/shared/bin/web-ui-asset-download.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +unset VERBOSE +OUTPUT_DIR=/tmp + +while getopts o:v opts; do + case ${opts} in + o) OUTPUT_DIR=${OPTARG} ;; + v) VERBOSE=1 ;; + esac +done + +set -e +if [[ -n $VERBOSE ]]; then + set -x +fi + +mkdir -p "$OUTPUT_DIR" +pushd "$OUTPUT_DIR" >/dev/null 2>&1 +curl --fail-early -fsSL --remote-name-all \ + https://fonts.gstatic.com/s/lato/v24/S6u_w4BMUTPHjxsI9w2_Gwfo.ttf \ + https://fonts.gstatic.com/s/lato/v24/S6u8w4BMUTPHjxsAXC-v.ttf \ + https://fonts.gstatic.com/s/lato/v24/S6u_w4BMUTPHjxsI5wq_Gwfo.ttf \ + https://fonts.gstatic.com/s/lato/v24/S6u9w4BMUTPHh7USSwiPHA.ttf \ + https://fonts.gstatic.com/s/lato/v24/S6uyw4BMUTPHjx4wWw.ttf \ + https://fonts.gstatic.com/s/lato/v24/S6u9w4BMUTPHh6UVSwiPHA.ttf \ + 'https://cdn.jsdelivr.net/npm/bootstrap-icons@1.5.0/font/fonts/bootstrap-icons.woff2?856008caa5eb66df68595e734e59580d' \ + 'https://cdn.jsdelivr.net/npm/bootstrap-icons@1.5.0/font/fonts/bootstrap-icons.woff?856008caa5eb66df68595e734e59580d' +popd >/dev/null 2>&1 + +if [[ -n $VERBOSE ]]; then + set +x +fi +set +e