diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dcfd2c..13e304e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.1.0 + - Feat: target configuration + event-factory support [#6](https://github.com/logstash-plugins/logstash-codec-edn/pull/6) + ## 3.0.6 - Update gemspec summary diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 8277628..092510b 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -22,4 +22,36 @@ include::{include_path}/plugin_header.asciidoc[] Reads and produces EDN format data. +[id="plugins-{type}s-{plugin}-options"] +==== Edn Codec configuration options + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <> |<>|No +|======================================================================= + +  + +[id="plugins-{type}s-{plugin}-target"] +===== `target` + + * Value type is <> + * There is no default value for this setting. + * The option is only relevant while decoding. + +Define the target field for placing the decoded fields. +If this setting is not set, data will be stored at the root (top level) of the event. + +For example, if you want data to be put under the `document` field: +[source,ruby] + input { + tcp { + port => 4242 + codec => edn { + target => "[document]" + } + } + } + diff --git a/lib/logstash/codecs/edn.rb b/lib/logstash/codecs/edn.rb index 200cb46..6e0b016 100644 --- a/lib/logstash/codecs/edn.rb +++ b/lib/logstash/codecs/edn.rb @@ -1,9 +1,22 @@ require "logstash/codecs/base" require "logstash/util" +require 'logstash/plugin_mixins/event_support/event_factory_adapter' +require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter' + class LogStash::Codecs::EDN < LogStash::Codecs::Base + + extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter + + include LogStash::PluginMixins::EventSupport::EventFactoryAdapter + config_name "edn" + # Defines a target field for placing decoded fields. + # If this setting is omitted, data gets stored at the root (top level) of the event. + # + # NOTE: the target is only relevant while decoding data into a new event. + config :target, :validate => :field_reference def register require "edn" @@ -12,10 +25,10 @@ def register public def decode(data) begin - yield LogStash::Event.new(EDN.read(data)) + yield targeted_event_factory.new_event(EDN.read(data)) rescue => e @logger.warn("EDN parse failure. Falling back to plain-text", :error => e, :data => data) - yield LogStash::Event.new("message" => data) + yield event_factory.new_event("message" => data) end end diff --git a/logstash-codec-edn.gemspec b/logstash-codec-edn.gemspec index 71b1fe9..2027ae6 100644 --- a/logstash-codec-edn.gemspec +++ b/logstash-codec-edn.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-codec-edn' - s.version = '3.0.6' + s.version = '3.1.0' s.licenses = ['Apache License (2.0)'] s.summary = "Reads EDN format data" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -21,6 +21,8 @@ Gem::Specification.new do |s| # Gem dependencies s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" + s.add_runtime_dependency 'logstash-mixin-event_support', '~> 1.0' + s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0' s.add_runtime_dependency 'edn' diff --git a/spec/codecs/edn_spec.rb b/spec/codecs/edn_spec.rb index f2db20d..415c789 100644 --- a/spec/codecs/edn_spec.rb +++ b/spec/codecs/edn_spec.rb @@ -6,26 +6,48 @@ require "edn" describe LogStash::Codecs::EDN do - subject do - next LogStash::Codecs::EDN.new - end + + subject { LogStash::Codecs::EDN.new(config) } + + let(:config) { Hash.new } + + let(:data) { {"foo" => "bar", "baz" => {"bah" => ["a", "b", "c"]}, "@timestamp" => "2014-05-30T02:52:17.929Z"} } context "#decode" do it "should return an event from edn data" do - data = {"foo" => "bar", "baz" => {"bah" => ["a", "b", "c"]}, "@timestamp" => "2014-05-30T02:52:17.929Z"} + event_count = 0 subject.decode(data.to_edn) do |event| + event_count += 1 insist { event }.is_a?(LogStash::Event) insist { event.get("foo") } == data["foo"] insist { event.get("baz") } == data["baz"] insist { event.get("bah") } == data["bah"] insist { event.get("@timestamp").to_iso8601 } == data["@timestamp"] end + expect(event_count).to eql 1 + end + + context 'with target' do + + let(:config) { super().merge('target' => '[doc]') } + + it "decodes an event" do + event_count = 0 + subject.decode(data.to_edn) do |event| + event_count += 1 + expect(event.include?("foo")).to be false + expect(event.get("[doc][foo]")).to eql 'bar' + expect(event.get("[doc][baz]")).to eql "bah" => ["a","b","c"] + expect(event.get("[doc][@timestamp]")).to eql data["@timestamp"] + end + expect(event_count).to eql 1 + end + end end context "#encode" do it "should return edn data from pure ruby hash" do - data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}, "@timestamp" => "2014-05-30T02:52:17.929Z"} event = LogStash::Event.new(data) got_event = false subject.on_event do |e, d| @@ -43,7 +65,6 @@ # deserialization using JrJackson in :raw mode which creates Java LinkedHashMap # and not Ruby Hash which will not be monkey patched with the #to_edn method it "should return edn data from deserialized json with normalization" do - data = LogStash::Json.load('{"foo": "bar", "baz": {"bah": ["a","b","c"]}, "@timestamp": "2014-05-30T02:52:17.929Z"}') event = LogStash::Event.new(data) got_event = false subject.on_event do |e, d|