From d99b6c08a81b2d70c74e4be223ce34514905f0cd Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Wed, 11 Sep 2019 16:42:30 +0100 Subject: [PATCH 1/5] move testing to docker --- .travis.yml | 29 ++++++++---------- ci/build.sh | 21 ------------- ci/setup.sh | 26 ----------------- ci/unit/Dockerfile | 12 ++++++++ ci/unit/docker-compose.yml | 21 +++++++++++++ ci/unit/docker-run.sh | 5 ++++ ci/unit/docker-setup.sh | 60 ++++++++++++++++++++++++++++++++++++++ ci/unit/run.sh | 6 ++++ 8 files changed, 116 insertions(+), 64 deletions(-) delete mode 100755 ci/build.sh delete mode 100755 ci/setup.sh create mode 100644 ci/unit/Dockerfile create mode 100644 ci/unit/docker-compose.yml create mode 100755 ci/unit/docker-run.sh create mode 100755 ci/unit/docker-setup.sh create mode 100755 ci/unit/run.sh diff --git a/.travis.yml b/.travis.yml index 3b61247..22a845a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,21 +1,16 @@ --- -sudo: false -language: ruby -cache: bundler +sudo: required +services: docker +addons: + apt: + packages: + - docker-ce matrix: include: - - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=master - - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=7.0 - - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=6.7 - - rvm: jruby-9.1.13.0 - env: LOGSTASH_BRANCH=6.6 - - rvm: jruby-1.7.27 - env: LOGSTASH_BRANCH=5.6 + - env: ELASTIC_STACK_VERSION=7.x + - env: ELASTIC_STACK_VERSION=6.x + - env: SNAPSHOT=true ELASTIC_STACK_VERSION=7.x + - env: SNAPSHOT=true ELASTIC_STACK_VERSION=8.x fast_finish: true -install: true -script: ci/build.sh -jdk: openjdk8 -before_install: gem install bundler -v '< 2' +install: ci/unit/docker-setup.sh +script: ci/unit/docker-run.sh diff --git a/ci/build.sh b/ci/build.sh deleted file mode 100755 index 06caffd..0000000 --- a/ci/build.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# version: 1 -######################################################## -# -# AUTOMATICALLY GENERATED! DO NOT EDIT -# -######################################################## -set -e - -echo "Starting build process in: `pwd`" -source ./ci/setup.sh - -if [[ -f "ci/run.sh" ]]; then - echo "Running custom build script in: `pwd`/ci/run.sh" - source ./ci/run.sh -else - echo "Running default build scripts in: `pwd`/ci/build.sh" - bundle install - bundle exec rake vendor - bundle exec rspec spec -fi diff --git a/ci/setup.sh b/ci/setup.sh deleted file mode 100755 index 835fa43..0000000 --- a/ci/setup.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# version: 1 -######################################################## -# -# AUTOMATICALLY GENERATED! DO NOT EDIT -# -######################################################## -set -e -if [ "$LOGSTASH_BRANCH" ]; then - echo "Building plugin using Logstash source" - BASE_DIR=`pwd` - echo "Checking out branch: $LOGSTASH_BRANCH" - git clone -b $LOGSTASH_BRANCH https://github.com/elastic/logstash.git ../../logstash --depth 1 - printf "Checked out Logstash revision: %s\n" "$(git -C ../../logstash rev-parse HEAD)" - cd ../../logstash - echo "Building plugins with Logstash version:" - cat versions.yml - echo "---" - # We need to build the jars for that specific version - echo "Running gradle assemble in: `pwd`" - ./gradlew assemble - cd $BASE_DIR - export LOGSTASH_SOURCE=1 -else - echo "Building plugin using released gems on rubygems" -fi diff --git a/ci/unit/Dockerfile b/ci/unit/Dockerfile new file mode 100644 index 0000000..934fe24 --- /dev/null +++ b/ci/unit/Dockerfile @@ -0,0 +1,12 @@ +ARG ELASTIC_STACK_VERSION +FROM docker.elastic.co/logstash/logstash:$ELASTIC_STACK_VERSION +COPY --chown=logstash:logstash Gemfile /usr/share/plugins/this/Gemfile +COPY --chown=logstash:logstash *.gemspec /usr/share/plugins/this/ +RUN cp /usr/share/logstash/logstash-core/versions-gem-copy.yml /usr/share/logstash/versions.yml +ENV PATH="${PATH}:/usr/share/logstash/vendor/jruby/bin" +ENV LOGSTASH_SOURCE=1 +ENV JARS_SKIP="true" +RUN gem install bundler -v '< 2' +WORKDIR /usr/share/plugins/this +RUN bundle install +COPY --chown=logstash:logstash . /usr/share/plugins/this diff --git a/ci/unit/docker-compose.yml b/ci/unit/docker-compose.yml new file mode 100644 index 0000000..557fd99 --- /dev/null +++ b/ci/unit/docker-compose.yml @@ -0,0 +1,21 @@ +version: '3' + +# run tests: cd ci/unit; docker-compose up --build --force-recreate +# manual: cd ci/unit; docker-compose run logstash bash +services: + + logstash: + build: + context: ../../ + dockerfile: ci/unit/Dockerfile + args: + - ELASTIC_STACK_VERSION=$ELASTIC_STACK_VERSION + command: /usr/share/plugins/this/ci/unit/run.sh + environment: + LS_JAVA_OPTS: "-Xmx256m -Xms256m" + LOGSTASH_SOURCE: 1 + JARS_SKIP: "true" + OSS: "true" + tty: true + #volumes: + # - ./:/usr/share/plugins/this diff --git a/ci/unit/docker-run.sh b/ci/unit/docker-run.sh new file mode 100755 index 0000000..e73aedf --- /dev/null +++ b/ci/unit/docker-run.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +# This is intended to be run inside the docker container as the command of the docker-compose. +set -ex +docker-compose -f ci/unit/docker-compose.yml up --exit-code-from logstash diff --git a/ci/unit/docker-setup.sh b/ci/unit/docker-setup.sh new file mode 100755 index 0000000..7035b01 --- /dev/null +++ b/ci/unit/docker-setup.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# This is intended to be run the plugin's root directory. `ci/unit/docker-test.sh` +# Ensure you have Docker installed locally and set the ELASTIC_STACK_VERSION environment variable. +set -e + +VERSION_URL="https://raw.githubusercontent.com/elastic/logstash/master/ci/logstash_releases.json" + +if [ "$ELASTIC_STACK_VERSION" ]; then + echo "Fetching versions from $VERSION_URL" + VERSIONS=$(curl $VERSION_URL) + if [[ "$SNAPSHOT" = "true" ]]; then + ELASTIC_STACK_RETRIEVED_VERSION=$(echo $VERSIONS | jq '.snapshots."'"$ELASTIC_STACK_VERSION"'"') + echo $ELASTIC_STACK_RETRIEVED_VERSION + else + ELASTIC_STACK_RETRIEVED_VERSION=$(echo $VERSIONS | jq '.releases."'"$ELASTIC_STACK_VERSION"'"') + fi + if [[ "$ELASTIC_STACK_RETRIEVED_VERSION" != "null" ]]; then + # remove starting and trailing double quotes + ELASTIC_STACK_RETRIEVED_VERSION="${ELASTIC_STACK_RETRIEVED_VERSION%\"}" + ELASTIC_STACK_RETRIEVED_VERSION="${ELASTIC_STACK_RETRIEVED_VERSION#\"}" + echo "Translated $ELASTIC_STACK_VERSION to ${ELASTIC_STACK_RETRIEVED_VERSION}" + export ELASTIC_STACK_VERSION=$ELASTIC_STACK_RETRIEVED_VERSION + fi + + echo "Testing against version: $ELASTIC_STACK_VERSION" + + if [[ "$ELASTIC_STACK_VERSION" = *"-SNAPSHOT" ]]; then + cd /tmp + + if [[ $ELASTIC_STACK_VERSION == 8* ]]; then + jq=".build.projects.logstash.packages.\"logstash-$ELASTIC_STACK_VERSION-docker-image.tar.gz\".url" + else + jq=".build.projects.\"logstash\".packages.\"logstash-$ELASTIC_STACK_VERSION-docker-image.tar.gz\".url" + fi + echo "curl --silent https://artifacts-api.elastic.co/v1/versions/$ELASTIC_STACK_VERSION/builds/latest | jq -r $jq)" + result=$(curl --silent https://artifacts-api.elastic.co/v1/versions/$ELASTIC_STACK_VERSION/builds/latest | jq -r $jq) + echo $result + curl $result > logstash-docker-image.tar.gz + tar xfvz logstash-docker-image.tar.gz repositories + echo "Loading docker image: " + cat repositories + docker load < logstash-docker-image.tar.gz + rm logstash-docker-image.tar.gz + cd - + fi + + if [ -f Gemfile.lock ]; then + rm Gemfile.lock + fi + + docker-compose -f ci/unit/docker-compose.yml down + docker-compose -f ci/unit/docker-compose.yml build + #docker-compose -f ci/unit/docker-compose.yml up --exit-code-from logstash --force-recreate +else + echo "Please set the ELASTIC_STACK_VERSION environment variable" + echo "For example: export ELASTIC_STACK_VERSION=6.2.4" + exit 1 +fi + diff --git a/ci/unit/run.sh b/ci/unit/run.sh new file mode 100755 index 0000000..91e54bb --- /dev/null +++ b/ci/unit/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# This is intended to be run inside the docker container as the command of the docker-compose. +set -ex + +bundle exec rspec -fd --pattern spec/**/*_spec.rb,spec/**/*_specs.rb From af7d9fb54eeb863d7543dc40acf66d881d6dced2 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Fri, 26 Jul 2019 23:13:52 +0000 Subject: [PATCH 2/5] allow deep dot notation in extension field names --- lib/logstash/codecs/cef.rb | 13 ++++++++----- spec/codecs/cef_spec.rb | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/lib/logstash/codecs/cef.rb b/lib/logstash/codecs/cef.rb index 30a53ba..121c97f 100644 --- a/lib/logstash/codecs/cef.rb +++ b/lib/logstash/codecs/cef.rb @@ -189,13 +189,16 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base # commas, periods, and square-bracketed index offsets. # # To support this, we look for a specific sequence of characters that are followed by an equals sign. This pattern - # will correctly identify all strictly-legal keys, and will also match those that include a dot "subkey" + # will correctly identify all strictly-legal keys, and will also match those that include a dot-joined "subkeys" and + # square-bracketed array indexing # # That sequence must begin with one or more `\w` (word: alphanumeric + underscore), which _optionally_ may be followed - # by "subkey" sequence consisting of a literal dot (`.`) followed by a non-whitespace character, then one or more word - # characters, and then one or more characters that do not convey semantic meaning within CEF (e.g., literal-pipe (`|`), - # whitespace (`\s`), literal-dot (`.`), literal-equals (`=`), or literal-backslash ('\')). - EXTENSION_KEY_PATTERN = /(?:\w+(?:\.[^\s]\w+[^\|\s\.\=\\]+)?(?==))/ + # by one or more "subkey" sequences and an optional square-bracketed index. + # + # To be understood by this implementation, a "subkey" sequence must consist of a literal dot (`.`) followed by one or + # more characters that do not convey semantic meaning within CEF (e.g., literal-dot (`.`), literal-equals (`=`), + # whitespace (`\s`), literal-pipe (`|`), literal-backslash ('\'), or literal-square brackets (`[` or `]`)). + EXTENSION_KEY_PATTERN = /(?:\w+(?:\.[^\.=\s\|\\\[\]]+)*(?:\[[0-9]+\])?(?==))/ # Some CEF extension keys seen in the wild use an undocumented array-like syntax that may not be compatible with # the Event API's strict-mode FieldReference parser (e.g., `fieldname[0]`). diff --git a/spec/codecs/cef_spec.rb b/spec/codecs/cef_spec.rb index f7ad424..4c270c7 100644 --- a/spec/codecs/cef_spec.rb +++ b/spec/codecs/cef_spec.rb @@ -580,6 +580,20 @@ def do_decode(codec, data) end end + let(:preserve_complex_multiple_dot_notation_in_extension_fields) { 'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.foo.name[1]=new_name' } + it "should keep ad.fields" do + decode_one(subject, preserve_complex_multiple_dot_notation_in_extension_fields) do |e| + validate(e) + insist { e.get("sourceAddress") } == "10.0.0.192" + insist { e.get("destinationAddress") } == "12.121.122.82" + insist { e.get("[ad.field][0]") } == "field0" + insist { e.get("[ad.foo.name][1]") } == "new_name" + insist { e.get("ad.Authentification") } == "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0" + insist { e.get('ad.Error_,Code') } == "3221225578" + insist { e.get("additional.dotfieldName") } == "new_value" + end + end + let (:preserve_random_values_key_value_pairs_alongside_with_additional_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 cs4=401 random.user Admin 0 23041A10181C0000 23041810181C0000 /CN\=random.user/OU\=User Login End-Entity /CN\=TEST/OU\=Login CA TEST 34 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.name[1]=new_name'} it "should correctly parse random values even with additional fields in message" do decode_one(subject, preserve_random_values_key_value_pairs_alongside_with_additional_fields) do |e| From 5ec1c4e25b710f2a13793ceaa94397151adac919 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Wed, 11 Sep 2019 16:44:28 +0100 Subject: [PATCH 3/5] add another test to dot notation --- spec/codecs/cef_spec.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/spec/codecs/cef_spec.rb b/spec/codecs/cef_spec.rb index 4c270c7..b79c455 100644 --- a/spec/codecs/cef_spec.rb +++ b/spec/codecs/cef_spec.rb @@ -555,6 +555,16 @@ def do_decode(codec, data) end end + let (:dots_in_keys) {'CEF:0|Vendor|Device|Version|13|my message|5|dvchost=loghost cat=traffic deviceSeverity=notice ad.nn=TEST src=192.168.0.1 destinationPort=53'} + it "should be OK with dots in keys" do + decode_one(subject, dots_in_keys) do |e| + insist { e.get("deviceHostName") } == "loghost" + insist { e.get("ad.nn") } == 'TEST' + insist { e.get("sourceAddress") } == '192.168.0.1' + insist { e.get("destinationPort") } == '53' + end + end + let (:allow_spaces_in_values) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232 dproc=InternetExplorer x.x.x.x'} it "should be OK to have one or more spaces in values" do decode_one(subject, allow_spaces_in_values) do |e| From 68a035a04fa866ebe879a5205ea73467decf5a24 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Wed, 11 Sep 2019 17:19:44 +0100 Subject: [PATCH 4/5] bump to 6.0.1 --- CHANGELOG.md | 3 +++ logstash-codec-cef.gemspec | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa542e0..85c4b06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 6.0.1 + - Fix support for deep dot notation [#73](https://github.com/logstash-plugins/logstash-codec-cef/pull/73) + ## 6.0.0 - Removed obsolete `sev` and `deprecated_v1_fields` fields diff --git a/logstash-codec-cef.gemspec b/logstash-codec-cef.gemspec index 003898e..80c647a 100644 --- a/logstash-codec-cef.gemspec +++ b/logstash-codec-cef.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-codec-cef' - s.version = '6.0.0' + s.version = '6.0.1' s.platform = 'java' s.licenses = ['Apache License (2.0)'] s.summary = "Reads the ArcSight Common Event Format (CEF)." From 7b5a383d1b5669177eb810a57df397367d3497ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Duarte?= Date: Wed, 11 Sep 2019 18:42:13 +0100 Subject: [PATCH 5/5] Update CHANGELOG.md Co-Authored-By: Colin Surprenant --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85c4b06..752434f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ ## 6.0.1 - - Fix support for deep dot notation [#73](https://github.com/logstash-plugins/logstash-codec-cef/pull/73) + - Fixed support for deep dot notation [#73](https://github.com/logstash-plugins/logstash-codec-cef/pull/73) ## 6.0.0 - Removed obsolete `sev` and `deprecated_v1_fields` fields