From 19caf7b4fbd0d54132e1c82b4873e720b2cb507a Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 15:13:47 +0900 Subject: [PATCH 01/36] use latest ruby for CI --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b472ac18ae..0cadaf0906 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,8 @@ language: ruby rvm: - - 2.1 - - 2.2.3 + - 2.1.10 + - 2.2.4 - 2.3.0 - ruby-head - rbx From f601453903e1fa64d3fca5d77eb894c55ac2efa7 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 15:26:34 +0900 Subject: [PATCH 02/36] fix plugin base API to add "owned" plugins by other (input/output/filter) plugins * Buffer/Storage/Parser/Formatter are candidates for this feature * Input/Output/Filter can have these own default values for owned plugins * RetryState plugin helper is added at same time to fix helpers API --- lib/fluent/config/configure_proxy.rb | 35 ++- lib/fluent/configurable.rb | 13 + lib/fluent/plugin.rb | 39 ++- lib/fluent/plugin/base.rb | 55 ++-- lib/fluent/plugin/owned_by_mixin.rb | 38 +++ lib/fluent/plugin/storage.rb | 43 +-- lib/fluent/plugin_helper.rb | 2 + lib/fluent/plugin_helper/retry_state.rb | 172 ++++++++++ lib/fluent/plugin_helper/storage.rb | 3 +- test/config/test_configurable.rb | 41 +++ test/config/test_configure_proxy.rb | 51 +++ test/helper.rb | 25 ++ test/plugin/test_owned_by.rb | 35 +++ test/plugin/test_storage.rb | 2 +- test/plugin_helper/test_retry_state.rb | 398 ++++++++++++++++++++++++ test/plugin_helper/test_storage.rb | 2 +- 16 files changed, 874 insertions(+), 80 deletions(-) create mode 100644 lib/fluent/plugin/owned_by_mixin.rb create mode 100644 lib/fluent/plugin_helper/retry_state.rb create mode 100644 test/plugin/test_owned_by.rb create mode 100644 test/plugin_helper/test_retry_state.rb diff --git a/lib/fluent/config/configure_proxy.rb b/lib/fluent/config/configure_proxy.rb index 3e23c3e8c0..a54fae4b05 100644 --- a/lib/fluent/config/configure_proxy.rb +++ b/lib/fluent/config/configure_proxy.rb @@ -19,7 +19,8 @@ module Fluent module Config class ConfigureProxy - attr_accessor :name, :final, :param_name, :init, :required, :multi, :alias, :argument, :params, :defaults, :descriptions, :sections + attr_accessor :name, :final, :param_name, :init, :required, :multi, :alias, :configured_in_section + attr_accessor :argument, :params, :defaults, :descriptions, :sections # config_param :desc, :string, :default => '....' # config_set_default :buffer_type, :memory # @@ -50,6 +51,11 @@ def initialize(name, opts = {}) raise "init and required are exclusive" if @init && @required + # specify section name for viewpoint of owner(parent) plugin + # for buffer plugins: all params are in section of owner + # others: , (formatter/parser), ... + @configured_in_section = nil + @argument = nil # nil: ignore argument @params = {} @defaults = {} @@ -89,6 +95,9 @@ def merge(other) # self is base class, other is subclass if overwrite?(other, :alias) raise ConfigError, "BUG: subclass cannot overwrite base class's config_section: alias" end + if overwrite?(other, :configured_in_section) + raise ConfigError, "BUG: subclass cannot overwrite base class's config_section: configured_in" + end options = {} # param_name is used not to ovewrite plugin's instance @@ -103,6 +112,9 @@ def merge(other) # self is base class, other is subclass merged = self.class.new(other.name, options) + # configured_in MUST be kept + merged.configured_in_section = self.configured_in_section + merged.argument = other.argument || self.argument merged.params = other.params.merge(self.params) merged.defaults = self.defaults.merge(other.defaults) @@ -144,6 +156,9 @@ def merge_for_finalized(other) if overwrite?(other, :alias) raise ConfigError, "BUG: subclass cannot overwrite base class's config_section: alias" end + if overwrite?(other, :configured_in_section) + raise ConfigError, "BUG: subclass cannot overwrite base class's config_section: configured_in" + end options = {} options[:param_name] = other.param_name @@ -155,6 +170,8 @@ def merge_for_finalized(other) merged = self.class.new(other.name, options) + merged.configured_in_section = self.configured_in_section + merged.argument = self.argument || other.argument merged.params = other.params.merge(self.params) merged.defaults = other.defaults.merge(self.defaults) @@ -175,6 +192,15 @@ def merge_for_finalized(other) merged end + def overwrite_defaults(other) # other is owner plugin's corresponding proxy + self.defaults = self.defaults.merge(other.defaults) + self.sections.keys.each do |section_key| + if other.sections.has_key?(section_key) + self.sections[section_key].overwrite_defaults(other.sections[section_key]) + end + end + end + def parameter_configuration(name, *args, &block) name = name.to_sym @@ -213,6 +239,13 @@ def parameter_configuration(name, *args, &block) [name, block, opts] end + def configured_in(section_name) + if @configured_in_section + raise ArgumentError, "#{self.name}: configured_in called twice" + end + @configured_in_section = section_name.to_sym + end + def config_argument(name, *args, &block) if @argument raise ArgumentError, "#{self.name}: config_argument called twice" diff --git a/lib/fluent/configurable.rb b/lib/fluent/configurable.rb index 9d7322b0ec..f762a795b6 100644 --- a/lib/fluent/configurable.rb +++ b/lib/fluent/configurable.rb @@ -27,6 +27,7 @@ def self.included(mod) end def initialize + super # to simulate implicit 'attr_accessor' by config_param / config_section and its value by config_set_default proxy = self.class.merged_configure_proxy proxy.params.keys.each do |name| @@ -51,6 +52,14 @@ def configure(conf) proxy = self.class.merged_configure_proxy conf.corresponding_proxies << proxy + if self.respond_to?(:owner) && self.owner + owner_proxy = owner.class.merged_configure_proxy + if proxy.configured_in_section + owner_proxy = owner_proxy.sections[proxy.configured_in_section] + end + proxy.overwrite_defaults(owner_proxy) if owner_proxy + end + # In the nested section, can't get plugin class through proxies so get plugin class here plugin_class = Fluent::Plugin.lookup_type_from_class(proxy.name.to_s) root = Fluent::Config::SectionGenerator.generate(proxy, conf, logger, plugin_class) @@ -97,6 +106,10 @@ def configure_proxy(mod_name) map[mod_name] end + def configured_in(section_name) + configure_proxy(self.name).configured_in(section_name) + end + def config_param(name, *args, &block) configure_proxy(self.name).config_param(name, *args, &block) attr_accessor name diff --git a/lib/fluent/plugin.rb b/lib/fluent/plugin.rb index e474174051..e970580446 100644 --- a/lib/fluent/plugin.rb +++ b/lib/fluent/plugin.rb @@ -25,9 +25,12 @@ module Plugin # plugins for fluentd plugins: fluent/plugin/type/NAME.rb # ex: storage, buffer chunk, ... + # first class plugins (instantiated by Engine) INPUT_REGISTRY = Registry.new(:input, 'fluent/plugin/in_') OUTPUT_REGISTRY = Registry.new(:output, 'fluent/plugin/out_') FILTER_REGISTRY = Registry.new(:filter, 'fluent/plugin/filter_') + + # feature plugin: second class plugins (instanciated by Plugins or Helpers) BUFFER_REGISTRY = Registry.new(:buffer, 'fluent/plugin/buf_') PARSER_REGISTRY = Registry.new(:parser, 'fluent/plugin/parser_') FORMATTER_REGISTRY = Registry.new(:formatter, 'fluent/plugin/formatter_') @@ -105,11 +108,11 @@ def self.new_filter(type) new_impl('filter', FILTER_REGISTRY, type) end - def self.new_buffer(type) - new_impl('buffer', BUFFER_REGISTRY, type) + def self.new_buffer(type, parent: nil) + new_impl('buffer', BUFFER_REGISTRY, type, parent) end - def self.new_parser(type) + def self.new_parser(type, parent: nil) require 'fluent/parser' if type[0] == '/' && type[-1] == '/' @@ -117,16 +120,16 @@ def self.new_parser(type) require 'fluent/parser' Fluent::TextParser.lookup(type) else - new_impl('parser', PARSER_REGISTRY, type) + new_impl('parser', PARSER_REGISTRY, type, parent) end end - def self.new_formatter(type) - new_impl('formatter', FORMATTER_REGISTRY, type) + def self.new_formatter(type, parent: nil) + new_impl('formatter', FORMATTER_REGISTRY, type, parent) end - def self.new_storage(type) - new_impl('storage', STORAGE_REGISTRY, type) + def self.new_storage(type, parent: nil) + new_impl('storage', STORAGE_REGISTRY, type, parent) end def self.register_impl(kind, registry, type, value) @@ -138,17 +141,21 @@ def self.register_impl(kind, registry, type, value) nil end - def self.new_impl(kind, registry, type) + def self.new_impl(kind, registry, type, parent=nil) # "'type' not found" is handled by registry obj = registry.lookup(type) - case - when obj.is_a?(Class) - obj.new - when obj.respond_to?(:call) && obj.arity == 0 - obj.call - else - raise Fluent::ConfigError, "#{kind} plugin '#{type}' is not a Class nor callable (without arguments)." + impl = case + when obj.is_a?(Class) + obj.new + when obj.respond_to?(:call) && obj.arity == 0 + obj.call + else + raise Fluent::ConfigError, "#{kind} plugin '#{type}' is not a Class nor callable (without arguments)." + end + if parent && impl.respond_to?("owner=") + impl.owner = parent end + impl end end end diff --git a/lib/fluent/plugin/base.rb b/lib/fluent/plugin/base.rb index e58770c7f8..1f715dc2cc 100644 --- a/lib/fluent/plugin/base.rb +++ b/lib/fluent/plugin/base.rb @@ -16,25 +16,19 @@ require 'fluent/plugin' require 'fluent/configurable' -require 'fluent/plugin_id' -require 'fluent/log' -require 'fluent/plugin_helper' require 'fluent/system_config' module Fluent module Plugin class Base include Configurable - include PluginId include SystemConfig::Mixin - include PluginLoggerMixin - include PluginHelper::Mixin - State = Struct.new(:configure, :start, :stop, :shutdown, :close, :terminate) + State = Struct.new(:configure, :start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate) def initialize super - @state = State.new(false, false, false, false, false, false) + @_state = State.new(false, false, false, false, false, false, false, false) end def has_router? @@ -43,59 +37,76 @@ def has_router? def configure(conf) super - @state.configure = true + @_state ||= State.new(false, false, false, false, false, false, false, false) + @_state.configure = true self end def start - @log.reset - @state.start = true + @_state.start = true self end def stop - @state.stop = true + @_state.stop = true + self + end + + def before_shutdown + @_state.before_shutdown = true self end def shutdown - @state.shutdown = true + @_state.shutdown = true + self + end + + def after_shutdown + @_state.after_shutdown = true self end def close - @state.close = true + @_state.close = true self end def terminate - @state.terminate = true - @log.reset + @_state.terminate = true self end def configured? - @state.configure + @_state.configure end def started? - @state.start + @_state.start end def stopped? - @state.stop + @_state.stop + end + + def before_shutdown? + @_state.before_shutdown end def shutdown? - @state.shutdown + @_state.shutdown + end + + def after_shutdown? + @_state.after_shutdown end def closed? - @state.close + @_state.close end def terminated? - @state.terminate + @_state.terminate end end end diff --git a/lib/fluent/plugin/owned_by_mixin.rb b/lib/fluent/plugin/owned_by_mixin.rb new file mode 100644 index 0000000000..aac9581f12 --- /dev/null +++ b/lib/fluent/plugin/owned_by_mixin.rb @@ -0,0 +1,38 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Fluent + module Plugin + module OwnedByMixin + def owner=(plugin) + @_owner = plugin + + @_plugin_id = plugin.plugin_id + @_plugin_id_configured = plugin.plugin_id_configured? + + @log = plugin.log + end + + def owner + @_owner + end + + def log + @log + end + end + end +end diff --git a/lib/fluent/plugin/storage.rb b/lib/fluent/plugin/storage.rb index cead81ec58..962755593b 100644 --- a/lib/fluent/plugin/storage.rb +++ b/lib/fluent/plugin/storage.rb @@ -14,17 +14,18 @@ # limitations under the License. # -require 'fluent/plugin' -require 'fluent/configurable' +require 'fluent/plugin/base' +require 'fluent/plugin/owned_by_mixin' module Fluent module Plugin - class Storage - include Fluent::Configurable - include Fluent::SystemConfig::Mixin + class Storage < Base + include OwnedByMixin DEFAULT_TYPE = 'local' + configured_in :storage + config_param :persistent, :bool, default: false # load/save with all operations config_param :autosave, :bool, default: true config_param :autosave_interval, :time, default: 10 @@ -37,30 +38,6 @@ def self.validate_key(key) attr_accessor :log - def configure(conf) - super(conf) - - @_owner = nil - end - - def plugin_id(id, configured) - @_plugin_id = id - @_plugin_id_configured = configured - end - - def owner=(plugin) - @_owner = plugin - - @_plugin_id = plugin.plugin_id - @_plugin_id_configured = plugin.plugin_id_configured? - - @log = plugin.log - end - - def owner - @_owner - end - def persistent_always? false end @@ -102,14 +79,6 @@ def delete(key) def update(key, &block) # transactional get-and-update raise NotImplementedError, "Implement this method in child class" end - - # storage plugins has only 'close' and 'terminate' - # stop: used in helper to stop autosave - # shutdown: used in helper to call #save finally if needed - def close; end - def terminate - @_owner = nil - end end end end diff --git a/lib/fluent/plugin_helper.rb b/lib/fluent/plugin_helper.rb index 49b7e896e4..6913e91dc9 100644 --- a/lib/fluent/plugin_helper.rb +++ b/lib/fluent/plugin_helper.rb @@ -19,6 +19,8 @@ require 'fluent/plugin_helper/event_loop' require 'fluent/plugin_helper/timer' require 'fluent/plugin_helper/child_process' +require 'fluent/plugin_helper/storage' +require 'fluent/plugin_helper/retry_state' module Fluent module PluginHelper diff --git a/lib/fluent/plugin_helper/retry_state.rb b/lib/fluent/plugin_helper/retry_state.rb new file mode 100644 index 0000000000..e5c090e60b --- /dev/null +++ b/lib/fluent/plugin_helper/retry_state.rb @@ -0,0 +1,172 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Fluent + module PluginHelper + module RetryState + def retry_state_create( + title, retry_type, wait, timeout, + forever: false, max_steps: nil, backoff_base: 2, max_interval: nil, randomize: true, randomize_width: 0.125, + secondary: false, secondary_threshold: 0.8 + ) + case retry_type + when :expbackoff + ExponentialBackOffRetry.new(title, wait, timeout, forever, max_steps, randomize, randomize_width, backoff_base, max_interval, secondary, secondary_threshold) + when :periodic + PeriodicRetry.new(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold) + else + raise "BUG: unknown retry_type specified: '#{retry_type}'" + end + end + + class RetryStateMachine + attr_reader :title, :start, :steps, :next_time, :timeout_at, :current, :secondary_transition_at, :secondary_transition_steps + + def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold) + @title = title + + @start = current_time + @steps = 0 + @next_time = nil # should be initialized for first retry by child class + + @timeout = timeout + @timeout_at = @start + timeout + @current = :primary + + if randomize_width < 0 || randomize_width > 0.5 + raise "BUG: randomize_width MUST be between 0 and 0.5" + end + + @randomize = randomize + @randomize_width = randomize_width + + if forever && secondary + raise "BUG: forever and secondary are exclusive to each other" + end + + @forever = forever + @max_steps = max_steps + + @secondary = secondary + @secondary_threshold = secondary_threshold + if @secondary + raise "BUG: secondary_transition_threshold MUST be between 0 and 1" if @secondary_threshold <= 0 || @secondary_threshold >= 1 + @secondary_transition_at = @start + timeout * @secondary_threshold + @secondary_transition_steps = nil + end + end + + def current_time + Time.now + end + + def randomize(interval) + return interval unless @randomize + + interval + (interval * @randomize_width * (2 * rand - 1.0)) + end + + def calc_next_time + if @forever || !@secondary # primary + naive = naive_next_time(@steps) + if @forever + naive + elsif naive >= @timeout_at + @timeout_at + else + naive + end + elsif @current == :primary && @secondary + naive = naive_next_time(@steps) + if naive >= @secondary_transition_at + @secondary_transition_at + else + naive + end + elsif @current == :secondary + naive = naive_next_time(@steps - @secondary_transition_steps + 1) + if naive >= @timeout_at + @timeout_at + else + naive + end + else + raise "BUG: it's out of design" + end + end + + def naive_next_time(retry_times) + raise NotImplementedError + end + + def secondary? + @secondary && (@current == :secondary || current_time >= @secondary_transition_at) + end + + def step + @steps += 1 + if @secondary && @current != :secondary && current_time >= @secondary_transition_at + @current = :secondary + @secondary_transition_steps = @steps + end + @next_time = calc_next_time + nil + end + + def limit? + if @forever + false + else + @next_time >= @timeout_at || !!(@max_steps && @steps >= @max_steps) + end + end + end + + class ExponentialBackOffRetry < RetryStateMachine + def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, backoff_base, max_interval, secondary, secondary_threathold) + super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threathold) + @constant_factor = wait + @backoff_base = backoff_base + @max_interval = max_interval + + @next_time = @start + @constant_factor + end + + def naive_next_time(retry_next_times) + interval = @constant_factor * ( @backoff_base ** ( retry_next_times - 1 ) ) + intr = if @max_interval && interval > @max_interval + @max_interval + else + interval + end + current_time + randomize(intr) + end + end + + class PeriodicRetry < RetryStateMachine + def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threathold) + super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threathold) + @retry_wait = wait + @next_time = @start + @retry_wait + end + + def naive_next_time(retry_next_times) + current_time + randomize(@retry_wait) + end + end + end + end +end diff --git a/lib/fluent/plugin_helper/storage.rb b/lib/fluent/plugin_helper/storage.rb index c7f5245e04..cc89b83916 100644 --- a/lib/fluent/plugin_helper/storage.rb +++ b/lib/fluent/plugin_helper/storage.rb @@ -39,8 +39,7 @@ def storage_create(usage: '', type: nil, conf: nil) unless type raise ArgumentError, "BUG: type not specified without configuration" end - storage = Plugin.new_storage(type) - storage.owner = self + storage = Plugin.new_storage(type, parent: self) config = case conf when Fluent::Config::Element conf diff --git a/test/config/test_configurable.rb b/test/config/test_configurable.rb index d1d7fe3411..3f26824bd0 100644 --- a/test/config/test_configurable.rb +++ b/test/config/test_configurable.rb @@ -286,6 +286,29 @@ class OverwriteAlias < FinalizedBase end end end + + module OverwriteDefaults + class Owner + include Fluent::Configurable + config_set_default :key1, "V1" + config_section :buffer do + config_set_default :size_of_something, 1024 + end + end + + class FlatChild + include Fluent::Configurable + attr_accessor :owner + config_param :key1, :string, default: "v1" + end + + class BufferChild + include Fluent::Configurable + attr_accessor :owner + configured_in :buffer + config_param :size_of_something, :size, default: 128 + end + end end module Fluent::Config @@ -930,6 +953,24 @@ class TestConfigurable < ::Test::Unit::TestCase end end + sub_test_case 'defaults can be overwritten by owner' do + test 'for feature plugin which has flat parameters with parent' do + owner = ConfigurableSpec::OverwriteDefaults::Owner.new + child = ConfigurableSpec::OverwriteDefaults::FlatChild.new + child.owner = owner + child.configure(config_element('ROOT', '', {}, [])) + assert_equal "V1", child.key1 + end + + test 'for feature plugin which has parameters in subsection of parent' do + owner = ConfigurableSpec::OverwriteDefaults::Owner.new + child = ConfigurableSpec::OverwriteDefaults::BufferChild.new + child.owner = owner + child.configure(config_element('ROOT', '', {}, [])) + assert_equal 1024, child.size_of_something + end + end + sub_test_case ':secret option' do setup do @conf = config_element('ROOT', '', diff --git a/test/config/test_configure_proxy.rb b/test/config/test_configure_proxy.rb index 8c6c4c5654..07aac42632 100644 --- a/test/config/test_configure_proxy.rb +++ b/test/config/test_configure_proxy.rb @@ -53,6 +53,7 @@ class TestConfigureProxy < ::Test::Unit::TestCase assert_false(proxy.required?) assert_nil(proxy.multi) assert_true(proxy.multi?) + assert_nil(proxy.configured_in_section) p2 = Fluent::Config::ConfigureProxy.new(:section, param_name: :sections, init: false, required: true, multi: false) proxy = p1.merge(p2) @@ -64,10 +65,12 @@ class TestConfigureProxy < ::Test::Unit::TestCase assert_true(proxy.required?) assert_false(proxy.multi) assert_false(proxy.multi?) + assert_nil(proxy.configured_in_section) end test 'does not overwrite with argument object without any specifications of required/multi' do p1 = Fluent::Config::ConfigureProxy.new(:section1) + p1.configured_in_section = :subsection p2 = Fluent::Config::ConfigureProxy.new(:section2, param_name: :sections, init: false, required: true, multi: false) p3 = Fluent::Config::ConfigureProxy.new(:section3) proxy = p1.merge(p2).merge(p3) @@ -79,6 +82,54 @@ class TestConfigureProxy < ::Test::Unit::TestCase assert_true(proxy.required?) assert_false(proxy.multi) assert_false(proxy.multi?) + assert_equal :subsection, proxy.configured_in_section + end + end + + sub_test_case '#overwrite_defaults' do + test 'overwrites only defaults with others defaults' do + p1 = Fluent::Config::ConfigureProxy.new(:mychild) + p1.configured_in_section = :child + p1.config_param(:k1a, :string) + p1.config_param(:k1b, :string) + p1.config_param(:k2a, :integer, default: 0) + p1.config_param(:k2b, :integer, default: 0) + p1.config_section(:sub1) do + config_param :k3, :time, default: 30 + end + + p0 = Fluent::Config::ConfigureProxy.new(:myparent) + p0.config_section(:child) do + config_set_default :k1a, "v1a" + config_param :k1b, :string, default: "v1b" + config_set_default :k2a, 21 + config_param :k2b, :integer, default: 22 + config_section :sub1 do + config_set_default :k3, 60 + end + end + + p1.overwrite_defaults(p0.sections[:child]) + + assert_equal "v1a", p1.defaults[:k1a] + assert_equal "v1b", p1.defaults[:k1b] + assert_equal 21, p1.defaults[:k2a] + assert_equal 22, p1.defaults[:k2b] + assert_equal 60, p1.sections[:sub1].defaults[:k3] + end + end + + sub_test_case '#configured_in' do + test 'sets a section name which have configuration parameters of target plugin in owners configuration' do + proxy = Fluent::Config::ConfigureProxy.new(:section) + proxy.configured_in(:mysection) + assert_equal :mysection, proxy.configured_in_section + end + + test 'do not permit to be called twice' do + proxy = Fluent::Config::ConfigureProxy.new(:section) + proxy.configured_in(:mysection) + assert_raise(ArgumentError) { proxy.configured_in(:myothersection) } end end diff --git a/test/helper.rb b/test/helper.rb index d1fb1fd583..47b0f32aca 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -42,6 +42,23 @@ def to_masked_element require 'fluent/config/element' require 'fluent/log' require 'fluent/test' +require 'fluent/plugin/base' +require 'fluent/log' +require 'fluent/plugin_id' +require 'fluent/plugin_helper' +require 'fluent/time' + +module Fluent + module Plugin + class TestBase < Base + # a base plugin class, but not input nor output + # mainly for helpers and owned plugins + include PluginId + include PluginLoggerMixin + include PluginHelper::Mixin + end + end +end unless defined?(Test::Unit::AssertionFailedError) class Test::Unit::AssertionFailedError < StandardError @@ -52,6 +69,14 @@ def config_element(name = 'test', argument = '', params = {}, elements = []) Fluent::Config::Element.new(name, argument, params, elements) end +def event_time(str=nil) + if str + Fluent::EventTime.parse(str) + else + Fluent::EventTime.now + end +end + def unused_port(num = 1) ports = [] sockets = [] diff --git a/test/plugin/test_owned_by.rb b/test/plugin/test_owned_by.rb new file mode 100644 index 0000000000..50bdd323d2 --- /dev/null +++ b/test/plugin/test_owned_by.rb @@ -0,0 +1,35 @@ +require_relative '../helper' +require 'fluent/plugin/base' +require 'fluent/plugin/input' +require 'fluent/plugin/owned_by_mixin' + +module OwnedByMixinTestEnv + class DummyParent < Fluent::Plugin::Input + Fluent::Plugin.register_input('dummy_parent', self) + end + class DummyChild < Fluent::Plugin::Base + include Fluent::Plugin::OwnedByMixin + Fluent::Plugin.register_parser('dummy_child', self) + end +end + +class OwnedByMixinTest < Test::Unit::TestCase + sub_test_case 'Owned plugins' do + setup do + Fluent::Test.setup + end + + test 'inherits plugin id and logger from parent' do + parent = Fluent::Plugin.new_input('dummy_parent') + parent.configure(config_element('ROOT', '', {'@id' => 'my_parent_id', '@log_level' => 'trace'})) + child = Fluent::Plugin.new_parser('dummy_child', parent: parent) + + assert_equal parent.object_id, child.owner.object_id + + assert child.instance_eval{ @_plugin_id_configured } + assert_equal 'my_parent_id', child.instance_eval{ @_plugin_id } + + assert_equal Fluent::Log::LEVEL_TRACE, child.log.level + end + end +end diff --git a/test/plugin/test_storage.rb b/test/plugin/test_storage.rb index 901cbae9be..430ed21032 100644 --- a/test/plugin/test_storage.rb +++ b/test/plugin/test_storage.rb @@ -2,7 +2,7 @@ require 'fluent/plugin/storage' require 'fluent/plugin/base' -class DummyPlugin < Fluent::Plugin::Base +class DummyPlugin < Fluent::Plugin::TestBase end class BareStorage < Fluent::Plugin::Storage diff --git a/test/plugin_helper/test_retry_state.rb b/test/plugin_helper/test_retry_state.rb new file mode 100644 index 0000000000..5108042f32 --- /dev/null +++ b/test/plugin_helper/test_retry_state.rb @@ -0,0 +1,398 @@ +require_relative '../helper' +require 'fluent/plugin_helper/retry_state' +require 'fluent/plugin/base' + +require 'time' + +class Fluent::PluginHelper::RetryState::RetryStateMachine + def override_current_time(time) + (class << self; self; end).module_eval do + define_method(:current_time){ time } + end + end +end + +class RetryStateHelperTest < Test::Unit::TestCase + class Dummy < Fluent::Plugin::TestBase + helpers :retry_state + end + + setup do + @d = Dummy.new + end + + test 'randomize can generate value within specified +/- range' do + s = @d.retry_state_create(:t1, :expbackoff, 0.1, 30) # default enabled w/ 0.125 + 500.times do + r = s.randomize(1000) + assert{ r >= 875 && r < 1125 } + end + + s = @d.retry_state_create(:t1, :expbackoff, 0.1, 30, randomize_width: 0.25) + 500.times do + r = s.randomize(1000) + assert{ r >= 750 && r < 1250 } + end + end + + test 'plugin can create retry_state machine' do + s = @d.retry_state_create(:t1, :expbackoff, 0.1, 30) + # attr_reader :title, :start, :steps, :next_time, :timeout_at, :current, :secondary_transition_at, :secondary_transition_times + + assert_equal :t1, s.title + start_time = s.start + + assert_equal 0, s.steps + assert_equal (start_time + 0.1).to_i, s.next_time.to_i + assert_equal (start_time + 0.1).nsec, s.next_time.nsec + assert_equal (start_time + 30), s.timeout_at + + assert_equal :primary, s.current + assert{ s.is_a? Fluent::PluginHelper::RetryState::ExponentialBackOffRetry } + end + + test 'periodic retries' do + s = @d.retry_state_create(:t2, :periodic, 3, 29, randomize: false) + dummy_current_time = s.start + s.override_current_time(dummy_current_time) + + assert_equal dummy_current_time, s.current_time + assert_equal (dummy_current_time + 29), s.timeout_at + assert_equal (dummy_current_time + 3), s.next_time + + i = 1 + while i < 9 + s.override_current_time(s.next_time) + s.step + assert_equal i, s.steps + assert_equal (s.current_time + 3), s.next_time + assert !s.limit? + i += 1 + end + + assert_equal 9, i + s.override_current_time(s.next_time) + s.step + assert_equal s.timeout_at, s.next_time + assert s.limit? + end + + test 'periodic retries with max_steps' do + s = @d.retry_state_create(:t2, :periodic, 3, 29, randomize: false, max_steps: 5) + dummy_current_time = s.start + s.override_current_time(dummy_current_time) + + assert_equal dummy_current_time, s.current_time + assert_equal (dummy_current_time + 29), s.timeout_at + assert_equal (dummy_current_time + 3), s.next_time + + i = 1 + while i < 5 + s.override_current_time(s.next_time) + s.step + assert_equal i, s.steps + assert_equal (s.current_time + 3), s.next_time + assert !s.limit? + i += 1 + end + + assert_equal 5, i + s.override_current_time(s.next_time) + s.step + assert_equal (s.current_time + 3), s.next_time + assert s.limit? + end + + test 'periodic retries with secondary' do + s = @d.retry_state_create(:t3, :periodic, 3, 100, randomize: false, secondary: true) # threshold 0.8 + dummy_current_time = s.start + s.override_current_time(dummy_current_time) + + assert_equal dummy_current_time, s.current_time + assert_equal (dummy_current_time + 100), s.timeout_at + assert_equal (dummy_current_time + 100 * 0.8), s.secondary_transition_at + + assert_equal (dummy_current_time + 3), s.next_time + assert !s.secondary? + + i = 1 + while i < 26 + s.override_current_time(s.next_time) + assert !s.secondary? + + s.step + assert_equal i, s.steps + assert_equal (s.current_time + 3), s.next_time + assert !s.limit? + i += 1 + end + + assert_equal 26, i + s.override_current_time(s.next_time) # 78 + assert !s.secondary? + + s.step + assert_equal 26, s.steps + assert_equal s.secondary_transition_at, s.next_time + assert !s.limit? + + i += 1 + assert_equal 27, i + s.override_current_time(s.next_time) # 80 + assert s.secondary? + + s.step + assert_equal (s.current_time + 3), s.next_time + assert_equal s.steps, s.secondary_transition_steps + assert !s.limit? + + i += 1 + + while i < 33 + s.override_current_time(s.next_time) + assert s.secondary? + + s.step + assert_equal (s.current_time + 3), s.next_time + assert !s.limit? + i += 1 + end + + assert_equal 33, i + s.override_current_time(s.next_time) # 98 + assert s.secondary? + + s.step + assert_equal s.timeout_at, s.next_time + assert s.limit? + end + + test 'periodic retries with secondary and specified threshold' do + s = @d.retry_state_create(:t3, :periodic, 3, 100, randomize: false, secondary: true, secondary_threshold: 0.75) + dummy_current_time = s.start + s.override_current_time(dummy_current_time) + + assert_equal dummy_current_time, s.current_time + assert_equal (dummy_current_time + 100), s.timeout_at + assert_equal (dummy_current_time + 100 * 0.75), s.secondary_transition_at + end + + test 'exponential backoff forever without randomization' do + s = @d.retry_state_create(:t11, :expbackoff, 0.1, 300, randomize: false, forever: true, backoff_base: 2) + dummy_current_time = s.start + s.override_current_time(dummy_current_time) + + assert_equal dummy_current_time, s.current_time + + assert_equal 0, s.steps + assert_equal (dummy_current_time + 0.1), s.next_time + + i = 1 + while i < 300 + s.step + assert_equal i, s.steps + assert_equal (dummy_current_time + 0.1 * (2 ** (i - 1))), s.next_time + assert !s.limit? + i += 1 + end + end + + test 'exponential backoff with max_interval' do + s = @d.retry_state_create(:t12, :expbackoff, 0.1, 300, randomize: false, forever: true, backoff_base: 2, max_interval: 100) + dummy_current_time = s.start + s.override_current_time(dummy_current_time) + + assert_equal dummy_current_time, s.current_time + + assert_equal 0, s.steps + assert_equal (dummy_current_time + 0.1), s.next_time + + # 0.1 * (2 ** (10 - 1)) == 0.1 * 2 ** 9 == 51.2 + # 0.1 * (2 ** (11 - 1)) == 0.1 * 2 ** 10 == 102.4 + i = 1 + while i < 11 + s.step + assert_equal i, s.steps + assert_equal (dummy_current_time + 0.1 * (2 ** (i - 1))), s.next_time, "start:#{dummy_current_time}, i:#{i}" + i += 1 + end + + s.step + assert_equal 11, s.steps + assert_equal (dummy_current_time + 100), s.next_time + + s.step + assert_equal 12, s.steps + assert_equal (dummy_current_time + 100), s.next_time + end + + test 'exponential backoff with shorter timeout' do + s = @d.retry_state_create(:t13, :expbackoff, 1, 12, randomize: false, backoff_base: 2, max_interval: 10) + dummy_current_time = s.start + s.override_current_time(dummy_current_time) + + assert_equal dummy_current_time, s.current_time + + assert_equal (dummy_current_time + 12), s.timeout_at + + assert_equal 0, s.steps + assert_equal (dummy_current_time + 1), s.next_time + + # 1 + 1 + 2 + 4 (=8) + + s.override_current_time(s.next_time) + s.step + assert_equal 1, s.steps + assert_equal (s.current_time + 1), s.next_time + + s.override_current_time(s.next_time) + s.step + assert_equal 2, s.steps + assert_equal (s.current_time + 2), s.next_time + + s.override_current_time(s.next_time) + s.step + assert_equal 3, s.steps + assert_equal (s.current_time + 4), s.next_time + + assert !s.limit? + + # + 8 (=16) > 12 + + s.override_current_time(s.next_time) + s.step + assert_equal 4, s.steps + assert_equal s.timeout_at, s.next_time + + assert s.limit? + end + + test 'exponential backoff with max_steps' do + s = @d.retry_state_create(:t14, :expbackoff, 1, 120, randomize: false, backoff_base: 2, max_interval: 10, max_steps: 6) + dummy_current_time = s.start + s.override_current_time(dummy_current_time) + + assert_equal dummy_current_time, s.current_time + + assert_equal (dummy_current_time + 120), s.timeout_at + + assert_equal 0, s.steps + assert_equal (dummy_current_time + 1), s.next_time + + s.override_current_time(s.next_time) + s.step + assert_equal 1, s.steps + assert_equal (s.current_time + 1), s.next_time + + s.override_current_time(s.next_time) + s.step + assert_equal 2, s.steps + assert_equal (s.current_time + 2), s.next_time + + s.override_current_time(s.next_time) + s.step + assert_equal 3, s.steps + assert_equal (s.current_time + 4), s.next_time + + assert !s.limit? + + s.override_current_time(s.next_time) + s.step + assert_equal 4, s.steps + assert_equal (s.current_time + 8), s.next_time + + assert !s.limit? + + s.override_current_time(s.next_time) + s.step + assert_equal 5, s.steps + assert_equal (s.current_time + 10), s.next_time + + assert !s.limit? + + s.override_current_time(s.next_time) + s.step + assert_equal 6, s.steps + assert_equal (s.current_time + 10), s.next_time + + assert s.limit? + end + + test 'exponential backoff retries with secondary' do + s = @d.retry_state_create(:t15, :expbackoff, 1, 100, randomize: false, backoff_base: 2, secondary: true) # threshold 0.8 + dummy_current_time = s.start + s.override_current_time(dummy_current_time) + + assert_equal dummy_current_time, s.current_time + assert_equal (dummy_current_time + 100), s.timeout_at + assert_equal (dummy_current_time + 100 * 0.8), s.secondary_transition_at + + assert_equal (dummy_current_time + 1), s.next_time + assert !s.secondary? + + # 1, 1(2), 2(4), 4(8), 8(16), 16(32), 32(64), (80), (81), (83), (87), (95), (100) + i = 1 + while i < 7 + s.override_current_time(s.next_time) + assert !s.secondary? + + s.step + assert_equal i, s.steps + assert_equal (s.current_time + 1 * (2 ** (i - 1))), s.next_time + assert !s.limit? + i += 1 + end + + assert_equal 7, i + s.override_current_time(s.next_time) # 64 + assert !s.secondary? + + s.step + assert_equal 7, s.steps + assert_equal s.secondary_transition_at, s.next_time + assert !s.limit? + + i += 1 + assert_equal 8, i + s.override_current_time(s.next_time) # 80 + assert s.secondary? + + s.step + assert_equal 8, s.steps + assert_equal s.steps, s.secondary_transition_steps + assert_equal (s.secondary_transition_at + 1.0), s.next_time + assert !s.limit? + + # 81, 82, 84, 88, 96, 100 + j = 1 + while j < 4 + s.override_current_time(s.next_time) + assert s.secondary? + assert_equal :secondary, s.current + + s.step + assert_equal (8 + j), s.steps + assert_equal (s.current_time + (1 * (2 ** j))), s.next_time + assert !s.limit?, "j:#{j}" + j += 1 + end + + assert_equal 4, j + s.override_current_time(s.next_time) # 96 + assert s.secondary? + + s.step + assert_equal s.timeout_at, s.next_time + assert s.limit? + end + + test 'exponential backoff retries with secondary and specified threshold' do + s = @d.retry_state_create(:t16, :expbackoff, 1, 100, randomize: false, secondary: true, backoff_base: 2, secondary_threshold: 0.75) + dummy_current_time = s.start + s.override_current_time(dummy_current_time) + + assert_equal dummy_current_time, s.current_time + assert_equal (dummy_current_time + 100), s.timeout_at + assert_equal (dummy_current_time + 100 * 0.75), s.secondary_transition_at + end +end diff --git a/test/plugin_helper/test_storage.rb b/test/plugin_helper/test_storage.rb index b6822f8b46..9f9d3a6347 100644 --- a/test/plugin_helper/test_storage.rb +++ b/test/plugin_helper/test_storage.rb @@ -69,7 +69,7 @@ def synchronized? end class StorageHelperTest < Test::Unit::TestCase - class Dummy < Fluent::Plugin::Base + class Dummy < Fluent::Plugin::TestBase helpers :storage end From 5c237ac619d9ec6cb147b0b1fb034cabff95891c Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 15:29:44 +0900 Subject: [PATCH 03/36] add some utility methods to write plugins --- lib/fluent/plugin_helper/event_emitter.rb | 33 +++++++++++++++++++---- lib/fluent/plugin_helper/thread.rb | 23 +++++++++++++--- test/plugin_helper/test_event_emitter.rb | 4 +-- test/plugin_helper/test_thread.rb | 2 +- 4 files changed, 50 insertions(+), 12 deletions(-) diff --git a/lib/fluent/plugin_helper/event_emitter.rb b/lib/fluent/plugin_helper/event_emitter.rb index 767b345999..103a8ef47b 100644 --- a/lib/fluent/plugin_helper/event_emitter.rb +++ b/lib/fluent/plugin_helper/event_emitter.rb @@ -15,19 +15,36 @@ # require 'fluent/engine' +require 'fluent/time' module Fluent module PluginHelper module EventEmitter - attr_accessor :router - # stop : [-] # shutdown : disable @router # close : [-] # terminate: [-] + def router + @_event_emitter_used_actually = true + @router + end + + def router=(r) + @router = r + end + + def has_router? + true + end + + def event_emitter_used_actually? + @_event_emitter_used_actually + end + def initialize super + @_event_emitter_used_actually = false @router = nil end @@ -42,13 +59,19 @@ def configure(conf) end end - def has_router? - true + def after_shutdown + @router = nil + super end - def close + def close # unset router many times to reduce test cost + @router = nil super + end + + def terminate @router = nil + super end end end diff --git a/lib/fluent/plugin_helper/thread.rb b/lib/fluent/plugin_helper/thread.rb index 3a250e51fe..4d0831b205 100644 --- a/lib/fluent/plugin_helper/thread.rb +++ b/lib/fluent/plugin_helper/thread.rb @@ -74,6 +74,20 @@ def thread_create(title, *args) thread end + def thread_exist?(title) + @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.size > 0 + end + + def thread_started?(title) + t = @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.first + t && t[:_fluentd_plugin_helper_thread_started] + end + + def thread_running?(title) + t = @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.first + t && t[:_fluentd_plugin_helper_thread_running] + end + def initialize super @_threads_mutex = Mutex.new @@ -105,10 +119,11 @@ def terminate super @_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id| thread = @_threads[obj_id] - if thread - thread.kill - thread.join - end + thread.kill if thread + end + @_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id| + thread = @_threads[obj_id] + thread.join @_threads_mutex.synchronize{ @_threads.delete(obj_id) } end @_thread_wait_seconds = nil diff --git a/test/plugin_helper/test_event_emitter.rb b/test/plugin_helper/test_event_emitter.rb index a4ebf0d654..3c2d457c01 100644 --- a/test/plugin_helper/test_event_emitter.rb +++ b/test/plugin_helper/test_event_emitter.rb @@ -7,10 +7,10 @@ class EventEmitterTest < Test::Unit::TestCase Fluent::Test.setup end - class Dummy0 < Fluent::Plugin::Base + class Dummy0 < Fluent::Plugin::TestBase end - class Dummy < Fluent::Plugin::Base + class Dummy < Fluent::Plugin::TestBase helpers :event_emitter end diff --git a/test/plugin_helper/test_thread.rb b/test/plugin_helper/test_thread.rb index 7a08f5d9bc..488239260a 100644 --- a/test/plugin_helper/test_thread.rb +++ b/test/plugin_helper/test_thread.rb @@ -4,7 +4,7 @@ require 'timeout' class ThreadTest < Test::Unit::TestCase - class Dummy < Fluent::Plugin::Base + class Dummy < Fluent::Plugin::TestBase helpers :thread def configure(conf) super From e2cb738d49edb410c06b09abbe902e5b732432dd Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 15:31:13 +0900 Subject: [PATCH 04/36] fix to use plugin base class only for tests --- test/plugin_helper/test_child_process.rb | 2 +- test/plugin_helper/test_event_loop.rb | 2 +- test/plugin_helper/test_timer.rb | 2 +- test/test_plugin_helper.rb | 17 ++++++++++------- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/test/plugin_helper/test_child_process.rb b/test/plugin_helper/test_child_process.rb index 0b0ceb1837..f1a971c753 100644 --- a/test/plugin_helper/test_child_process.rb +++ b/test/plugin_helper/test_child_process.rb @@ -25,7 +25,7 @@ class ChildProcessTest < Test::Unit::TestCase end end - class Dummy < Fluent::Plugin::Base + class Dummy < Fluent::Plugin::TestBase helpers :child_process def configure(conf) super diff --git a/test/plugin_helper/test_event_loop.rb b/test/plugin_helper/test_event_loop.rb index c9f9763815..1fa6147474 100644 --- a/test/plugin_helper/test_event_loop.rb +++ b/test/plugin_helper/test_event_loop.rb @@ -3,7 +3,7 @@ require 'fluent/plugin/base' class EventLoopTest < Test::Unit::TestCase - class Dummy < Fluent::Plugin::Base + class Dummy < Fluent::Plugin::TestBase helpers :event_loop def configure(conf) super diff --git a/test/plugin_helper/test_timer.rb b/test/plugin_helper/test_timer.rb index 0cd76c9ae3..060d7ebbc6 100644 --- a/test/plugin_helper/test_timer.rb +++ b/test/plugin_helper/test_timer.rb @@ -3,7 +3,7 @@ require 'fluent/plugin/base' class TimerTest < Test::Unit::TestCase - class Dummy < Fluent::Plugin::Base + class Dummy < Fluent::Plugin::TestBase helpers :timer end diff --git a/test/test_plugin_helper.rb b/test/test_plugin_helper.rb index 6403db8e46..479a0dc63b 100644 --- a/test/test_plugin_helper.rb +++ b/test/test_plugin_helper.rb @@ -6,23 +6,26 @@ class ConfigTest < Test::Unit::TestCase module FluentTest; end sub_test_case 'Fluent::Plugin::Base.helpers method works as shortcut to include helper modules' do - class FluentTest::PluginTest1 < Fluent::Plugin::Base + class FluentTest::PluginTest1 < Fluent::Plugin::TestBase helpers :event_emitter end - class FluentTest::PluginTest2 < Fluent::Plugin::Base + class FluentTest::PluginTest2 < Fluent::Plugin::TestBase helpers :thread end - class FluentTest::PluginTest3 < Fluent::Plugin::Base + class FluentTest::PluginTest3 < Fluent::Plugin::TestBase helpers :event_loop end - class FluentTest::PluginTest4 < Fluent::Plugin::Base + class FluentTest::PluginTest4 < Fluent::Plugin::TestBase helpers :timer end - class FluentTest::PluginTest5 < Fluent::Plugin::Base + class FluentTest::PluginTest5 < Fluent::Plugin::TestBase helpers :child_process end - class FluentTest::PluginTest0 < Fluent::Plugin::Base - helpers :event_emitter, :thread, :event_loop, :timer, :child_process, :child_process + class FluentTest::PluginTest6 < Fluent::Plugin::TestBase + helpers :retry_state + end + class FluentTest::PluginTest0 < Fluent::Plugin::TestBase + helpers :event_emitter, :thread, :event_loop, :timer, :child_process, :retry_state end test 'plugin can include helper event_emitter' do From 36e13f6170f7d64d6771340bc8a482441e28ee62 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 15:31:59 +0900 Subject: [PATCH 05/36] add utility method to overwrite system config in tests --- lib/fluent/system_config.rb | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/fluent/system_config.rb b/lib/fluent/system_config.rb index 1e2b2463c0..2fd15feec2 100644 --- a/lib/fluent/system_config.rb +++ b/lib/fluent/system_config.rb @@ -52,6 +52,16 @@ def self.blank_system_config Fluent::Config::Element.new('', '', {}, []) end + def self.overwrite_system_config(hash) + older = $_system_config || nil + begin + $_system_config = SystemConfig.new(Fluent::Config::Element.new('system', '', hash, [])) + yield + ensure + $_system_config = older + end + end + def initialize(conf=nil) super() conf ||= SystemConfig.blank_system_config @@ -93,13 +103,16 @@ def apply(supervisor) module Mixin def system_config require 'fluent/engine' - @_system_config || Fluent::Engine.system_config + unless $_system_config + $_system_config = nil + end + @_system_config || $_system_config || Fluent::Engine.system_config end def system_config_override(opts={}) require 'fluent/engine' unless @_system_config - @_system_config = Fluent::Engine.system_config.dup + @_system_config = ($_system_config || Fluent::Engine.system_config).dup end opts.each_pair do |key, value| @_system_config.send(:"#{key.to_s}=", value) From 6912c05d251fdac142ee4f55181082f07b6e3c72 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 15:32:49 +0900 Subject: [PATCH 06/36] fix to reset internal state (especially for tests) --- lib/fluent/log.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/fluent/log.rb b/lib/fluent/log.rb index 8d3530c75a..f50b2b98be 100644 --- a/lib/fluent/log.rb +++ b/lib/fluent/log.rb @@ -390,5 +390,15 @@ def configure(conf) @log.optional_attrs = {} end end + + def start + @log.reset + super + end + + def terminate + super + @log.reset + end end end From 4087110d238b5ea4aed9cc24eb77c20aa4400a33 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 15:34:11 +0900 Subject: [PATCH 07/36] add module/mixin to handle unique_id in right way (not to implement it by each plugins) --- lib/fluent/unique_id.rb | 39 ++++++++++++++++++++++++++++++++++ test/test_unique_id.rb | 47 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 lib/fluent/unique_id.rb create mode 100644 test/test_unique_id.rb diff --git a/lib/fluent/unique_id.rb b/lib/fluent/unique_id.rb new file mode 100644 index 0000000000..060c2e2290 --- /dev/null +++ b/lib/fluent/unique_id.rb @@ -0,0 +1,39 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Fluent + module UniqueId + def self.generate + now = Time.now.utc + u1 = ((now.to_i * 1000 * 1000 + now.usec) << 12 | rand(0xfff)) + [u1 >> 32, u1 & 0xffffffff, rand(0xffffffff), rand(0xffffffff)].pack('NNNN') + end + + def self.hex(unique_id) + unique_id.unpack('H*').first + end + + module Mixin + def generate_unique_id + Fluent::UniqueId.generate + end + + def dump_unique_id_hex(unique_id) + Fluent::UniqueId.hex(unique_id) + end + end + end +end diff --git a/test/test_unique_id.rb b/test/test_unique_id.rb new file mode 100644 index 0000000000..5a888137d9 --- /dev/null +++ b/test/test_unique_id.rb @@ -0,0 +1,47 @@ +require_relative 'helper' +require 'fluent/plugin/base' +require 'fluent/unique_id' + +module UniqueIdTestEnv + class Dummy < Fluent::Plugin::Base + include Fluent::UniqueId::Mixin + end +end + +class UniqueIdTest < Test::Unit::TestCase + sub_test_case 'module used directly' do + test '.generate generates 128bit length unique id (16bytes)' do + assert_equal 16, Fluent::UniqueId.generate.bytesize + ary = [] + 100_000.times do + ary << Fluent::UniqueId.generate + end + assert_equal 100_000, ary.uniq.size + end + + test '.hex dumps 16bytes id into 32 chars' do + assert_equal 32, Fluent::UniqueId.hex(Fluent::UniqueId.generate).size + assert(Fluent::UniqueId.hex(Fluent::UniqueId.generate) =~ /^[0-9a-z]{32}$/) + end + end + + sub_test_case 'mixin' do + setup do + @i = UniqueIdTestEnv::Dummy.new + end + + test '#generate_unique_id generates 128bit length id (16bytes)' do + assert_equal 16, @i.generate_unique_id.bytesize + ary = [] + 100_000.times do + ary << @i.generate_unique_id + end + assert_equal 100_000, ary.uniq.size + end + + test '#dump_unique_id_hex dumps 16bytes id into 32 chars' do + assert_equal 32, @i.dump_unique_id_hex(@i.generate_unique_id).size + assert(@i.dump_unique_id_hex(@i.generate_unique_id) =~ /^[0-9a-z]{32}$/) + end + end +end From e54b5bb41f42e267392d4712fee63a666e7b8702 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 15:35:11 +0900 Subject: [PATCH 08/36] add v0.14 Buffer API --- lib/fluent/plugin/buffer.rb | 374 ++++++++++++++++++++++++++++++ lib/fluent/plugin/buffer/chunk.rb | 124 ++++++++++ 2 files changed, 498 insertions(+) create mode 100644 lib/fluent/plugin/buffer.rb create mode 100644 lib/fluent/plugin/buffer/chunk.rb diff --git a/lib/fluent/plugin/buffer.rb b/lib/fluent/plugin/buffer.rb new file mode 100644 index 0000000000..38fb4a1db0 --- /dev/null +++ b/lib/fluent/plugin/buffer.rb @@ -0,0 +1,374 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'fluent/plugin/base' +require 'fluent/plugin/owned_by_mixin' +require 'fluent/unique_id' + +require 'monitor' + +module Fluent + module Plugin + class Buffer < Base + include OwnedByMixin + include UniqueId::Mixin + include MonitorMixin + + class BufferError < StandardError; end + class BufferOverflowError < BufferError; end + class BufferChunkOverflowError < BufferError; end # A record size is larger than chunk size limit + + MINIMUM_APPEND_ATTEMPT_RECORDS = 10 + + DEFAULT_CHUNK_BYTES_LIMIT = 8 * 1024 * 1024 # 8MB + DEFAULT_TOTAL_BYTES_LIMIT = 512 * 1024 * 1024 # 512MB, same with v0.12 (BufferedOutput + buf_memory: 64 x 8MB) + + configured_in :buffer + + # TODO: system total buffer bytes limit by SystemConfig + + config_param :chunk_bytes_limit, :size, default: DEFAULT_CHUNK_BYTES_LIMIT + config_param :total_bytes_limit, :size, default: DEFAULT_TOTAL_BYTES_LIMIT + + # If user specify this value and (chunk_size * queue_length) is smaller than total_size, + # then total_size is automatically configured to that value + config_param :queue_length_limit, :integer, default: nil + + # optional new limitations + config_param :chunk_records_limit, :integer, default: nil + + Metadata = Struct.new(:timekey, :tag, :variables) + + # for tests + attr_accessor :stage_size, :queue_size + attr_reader :stage, :queue, :dequeued, :queued_num + + def initialize + super + + @chunk_bytes_limit = nil + @total_bytes_limit = nil + @queue_length_limit = nil + @chunk_records_limit = nil + + @stage = {} #=> Hash (metadata -> chunk) : not flushed yet + @queue = [] #=> Array (chunks) : already flushed (not written) + @dequeued = {} #=> Hash (unique_id -> chunk): already written (not purged) + @queued_num = {} # metadata => int (number of queued chunks) + + @stage_size = @queue_size = 0 + @metadata_list = [] # keys of @stage + end + + def persistent? + false + end + + def configure(conf) + super + + unless @queue_length_limit.nil? + @total_bytes_limit = @chunk_bytes_limit * @queue_length_limit + end + end + + def start + super + + @stage, @queue = resume + @stage.each_pair do |metadata, chunk| + @metadata_list << metadata unless @metadata_list.include?(metadata) + @stage_size += chunk.size + end + @queue.each do |chunk| + @metadata_list << chunk.metadata unless @metadata_list.include?(chunk.metadata) + @queued_num[chunk.metadata] ||= 0 + @queued_num[chunk.metadata] += 1 + @queue_size += chunk.size + end + end + + def close + super + synchronize do + @dequeued.each_pair do |chunk_id, chunk| + chunk.close + end + until @queue.empty? + @queue.shift.close + end + @stage.each_pair do |metadata, chunk| + chunk.close + end + end + end + + def terminate + super + @dequeued = @stage = @queue = @queued_num = @metadata_list = nil + @stage_size = @queue_size = 0 + end + + def storable? + @total_bytes_limit > @stage_size + @queue_size + end + + ## TODO: for back pressure feature + # def used?(ratio) + # @total_size_limit * ratio > @stage_size + @queue_size + # end + + def resume + # return {}, [] + raise NotImplementedError, "Implement this method in child class" + end + + def generate_chunk(metadata) + raise NotImplementedError, "Implement this method in child class" + end + + def metadata_list + synchronize do + @metadata_list.dup + end + end + + def new_metadata(timekey: nil, tag: nil, variables: nil) + Metadata.new(timekey, tag, variables) + end + + def add_metadata(metadata) + synchronize do + if i = @metadata_list.index(metadata) + @metadata_list[i] + else + @metadata_list << metadata + metadata + end + end + end + + def metadata(timekey: nil, tag: nil, variables: nil) + meta = new_metadata(timekey: timekey, tag: tag, variables: variables) + add_metadata(meta) + end + + # metadata MUST have consistent object_id for each variation + # data MUST be Array of serialized events + def emit(metadata, data, force: false) + return if data.size < 1 + raise BufferOverflowError unless storable? + + stored = false + + # the case whole data can be stored in staged chunk: almost all emits will success + chunk = synchronize { @stage[metadata] ||= generate_chunk(metadata) } + original_size = chunk.size + chunk.synchronize do + begin + chunk.append(data) + if !size_over?(chunk) || force + chunk.commit + stored = true + @stage_size += (chunk.size - original_size) + else + chunk.rollback + end + rescue + chunk.rollback + raise + end + end + return if stored + + # try step-by-step appending if data can't be stored into existing a chunk + emit_step_by_step(metadata, data) + end + + def queued_records + synchronize { @queue.reduce(0){|r, chunk| r + chunk.records } } + end + + def queued?(metadata=nil) + synchronize do + if metadata + n = @queued_num[metadata] + n && n.nonzero? + else + !@queue.empty? + end + end + end + + def enqueue_chunk(metadata) + synchronize do + chunk = @stage.delete(metadata) + return nil unless chunk + + chunk.synchronize do + if chunk.empty? + chunk.close + else + @queue << chunk + @queued_num[metadata] = @queued_num.fetch(metadata, 0) + 1 + chunk.enqueued! if chunk.respond_to?(:enqueued!) + end + end + size = chunk.size + @stage_size -= size + @queue_size += size + end + nil + end + + def enqueue_all + synchronize do + if block_given? + @stage.keys.each do |metadata| + chunk = @stage[metadata] + v = yield metadata, chunk + enqueue_chunk(metadata) if v + end + else + @stage.keys.each do |metadata| + enqueue_chunk(metadata) + end + end + end + end + + def dequeue_chunk + return nil if @queue.empty? + synchronize do + chunk = @queue.shift + + # this buffer is dequeued by other thread just before "synchronize" in this thread + return nil unless chunk + + @dequeued[chunk.unique_id] = chunk + @queued_num[chunk.metadata] -= 1 # BUG if nil, 0 or subzero + chunk + end + end + + def takeback_chunk(chunk_id) + synchronize do + chunk = @dequeued.delete(chunk_id) + return false unless chunk # already purged by other thread + @queue.unshift(chunk) + @queued_num[chunk.metadata] += 1 # BUG if nil + end + true + end + + def purge_chunk(chunk_id) + synchronize do + chunk = @dequeued.delete(chunk_id) + return nil unless chunk # purged by other threads + + metadata = chunk.metadata + begin + size = chunk.size + chunk.purge + @queue_size -= size + rescue => e + log.error "failed to purge buffer chunk", chunk_id: dump_unique_id_hex(chunk_id), error_class: e.class, error: e + end + + if metadata && !@stage[metadata] && (!@queued_num[metadata] || @queued_num[metadata] < 1) + @metadata_list.delete(metadata) + end + end + nil + end + + def clear_queue! + synchronize do + until @queue.empty? + begin + q = @queue.shift + log.debug("purging a chunk in queue"){ {id: dump_unique_id_hex(chunk.unique_id), size: chunk.size, records: chunk.records} } + q.purge + rescue => e + log.error "unexpected error while clearing buffer queue", error_class: e.class, error: e + end + end + @queue_size = 0 + end + end + + def size_over?(chunk) + chunk.size > @chunk_bytes_limit || (@chunk_records_limit && chunk.records > @chunk_records_limit) + end + + def emit_step_by_step(metadata, data) + attempt_records = data.size / 3 + + synchronize do # critical section for buffer (stage/queue) + while data.size > 0 + if attempt_records < MINIMUM_APPEND_ATTEMPT_RECORDS + attempt_records = MINIMUM_APPEND_ATTEMPT_RECORDS + end + + chunk = @stage[metadata] + unless chunk + chunk = @stage[metadata] = generate_chunk(metadata) + end + + chunk.synchronize do # critical section for chunk (chunk append/commit/rollback) + begin + empty_chunk = chunk.empty? + original_size = chunk.size + + attempt = data.slice(0, attempt_records) + chunk.append(attempt) + + if size_over?(chunk) + chunk.rollback + + if attempt_records <= MINIMUM_APPEND_ATTEMPT_RECORDS + if empty_chunk # record is too large even for empty chunk + raise BufferChunkOverflowError, "minimum append butch exceeds chunk bytes limit" + end + # no more records for this chunk -> enqueue -> to be flushed + enqueue_chunk(metadata) # `chunk` will be removed from stage + attempt_records = data.size # fresh chunk may have enough space + else + # whole data can be processed by twice operation + # ( by using apttempt /= 2, 3 operations required for odd numbers of data) + attempt_records = (attempt_records / 2) + 1 + end + + next + end + + chunk.commit + @stage_size += (chunk.size - original_size) + data.slice!(0, attempt_records) + # same attempt size + nil # discard return value of data.slice!() immediately + rescue + chunk.rollback + raise + end + end + end + end + nil + end # emit_step_by_step + end + end +end diff --git a/lib/fluent/plugin/buffer/chunk.rb b/lib/fluent/plugin/buffer/chunk.rb new file mode 100644 index 0000000000..e094d22075 --- /dev/null +++ b/lib/fluent/plugin/buffer/chunk.rb @@ -0,0 +1,124 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'fluent/msgpack_factory' +require 'fluent/plugin/buffer' +require 'fluent/unique_id' + +require 'fileutils' +require 'monitor' + +module Fluent + module Plugin + class Buffer # fluent/plugin/buffer is alread loaded + class Chunk + include MonitorMixin + include MessagePackFactory::Mixin + include UniqueId::Mixin + + # Chunks has 2 part: + # * metadata: contains metadata which should be restored after resume (if possible) + # v: {key=>value,key=>value,...} (optional) + # t: tag as string (optional) + # k: time slice key (optional) + # + # id: unique_id of chunk (*) + # r: number of records (*) + # c: created_at as unix time (*) + # m: modified_at as unix time (*) + # (*): fields automatically injected by chunk itself + # * data: binary data, combined records represented as String, maybe compressed + + # NOTE: keys of metadata are named with a single letter + # to decread bytesize of metadata I/O + + # TODO: CompressedPackedMessage of forward protocol? + + def initialize(metadata) + super() + @unique_id = generate_unique_id + @metadata = metadata + + @records = 0 + @created_at = Time.now + @modified_at = Time.now + end + + attr_reader :unique_id, :metadata, :created_at, :modified_at + + # data is array of formatted record string + def append(data) + raise NotImplementedError, "Implement this method in child class" + end + + def commit + raise NotImplementedError, "Implement this method in child class" + end + + def rollback + raise NotImplementedError, "Implement this method in child class" + end + + def size + raise NotImplementedError, "Implement this method in child class" + end + + def records + raise NotImplementedError, "Implement this method in child class" + end + + def empty? + size == 0 + end + + ## method for post-process of enqueue (e.g., renaming file for file chunks) + # def enqueued! + + def close + raise NotImplementedError, "Implement this method in child class" + end + + def purge + raise NotImplementedError, "Implement this method in child class" + end + + def read + raise NotImplementedError, "Implement this method in child class" + end + + def open(&block) + raise NotImplementedError, "Implement this method in child class" + end + + def write_to(io) + open do |i| + FileUtils.copy_stream(i, io) + end + end + + def msgpack_each(&block) + open do |io| + u = msgpack_factory.unpacker(io) + begin + u.each(&block) + rescue EOFError + end + end + end + end + end + end +end From 0fffaf34f68f1dea0f27c4c331d172c0ee8dd5bc Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 15:35:38 +0900 Subject: [PATCH 09/36] add MemoryBuffer implementation for v0.14 API --- lib/fluent/plugin/buf_memory2.rb | 34 ++++++++++ lib/fluent/plugin/buffer/memory_chunk.rb | 86 ++++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 lib/fluent/plugin/buf_memory2.rb create mode 100644 lib/fluent/plugin/buffer/memory_chunk.rb diff --git a/lib/fluent/plugin/buf_memory2.rb b/lib/fluent/plugin/buf_memory2.rb new file mode 100644 index 0000000000..ceda281b9a --- /dev/null +++ b/lib/fluent/plugin/buf_memory2.rb @@ -0,0 +1,34 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'fluent/plugin/buffer' +require 'fluent/plugin/buffer/memory_chunk' + +module Fluent + module Plugin + class MemoryBuffer < Fluent::Plugin::Buffer + Plugin.register_buffer('memory2', self) + + def resume + return {}, [] + end + + def generate_chunk(metadata) + Fluent::Plugin::Buffer::MemoryChunk.new(metadata) + end + end + end +end diff --git a/lib/fluent/plugin/buffer/memory_chunk.rb b/lib/fluent/plugin/buffer/memory_chunk.rb new file mode 100644 index 0000000000..7368b927c2 --- /dev/null +++ b/lib/fluent/plugin/buffer/memory_chunk.rb @@ -0,0 +1,86 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'fluent/plugin/buffer/chunk' + +module Fluent + module Plugin + class Buffer + class MemoryChunk < Chunk + def initialize(metadata) + super + @chunk = ''.force_encoding('ASCII-8BIT') + @chunk_bytes = 0 + @adding_bytes = 0 + @adding_records = 0 + end + + def append(data) + adding = data.join.force_encoding('ASCII-8BIT') + @chunk << adding + @adding_bytes += adding.bytesize + @adding_records += data.size + true + end + + def commit + @records += @adding_records + @chunk_bytes += @adding_bytes + + @adding_bytes = @adding_records = 0 + @modified_at = Time.now + true + end + + def rollback + @chunk.slice!(@chunk_bytes, @adding_bytes) + @adding_bytes = @adding_records = 0 + true + end + + def size + @chunk_bytes + @adding_bytes + end + + def records + @records + @adding_records + end + + def empty? + @chunk.empty? + end + + def close + true + end + + def purge + @chunk = ''.force_encoding("ASCII-8BIT") + @chunk_bytes = @records = @adding_bytes = @adding_records = 0 + true + end + + def read + @chunk + end + + def open(&block) + StringIO.open(@chunk, &block) + end + end + end + end +end From 2612e9f8a7af897d3662d5565fabce2df62d6497 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 15:36:20 +0900 Subject: [PATCH 10/36] add FileBuffer implementation for v0.14 Buffer API --- lib/fluent/plugin/buf_file2.rb | 146 +++++++++++ lib/fluent/plugin/buffer/file_chunk.rb | 330 +++++++++++++++++++++++++ 2 files changed, 476 insertions(+) create mode 100644 lib/fluent/plugin/buf_file2.rb create mode 100644 lib/fluent/plugin/buffer/file_chunk.rb diff --git a/lib/fluent/plugin/buf_file2.rb b/lib/fluent/plugin/buf_file2.rb new file mode 100644 index 0000000000..7292c4896f --- /dev/null +++ b/lib/fluent/plugin/buf_file2.rb @@ -0,0 +1,146 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'fileutils' + +require 'fluent/plugin/buffer' +require 'fluent/plugin/buffer/file_chunk' +require 'fluent/system_config' + +module Fluent + module Plugin + class FileBuffer < Fluent::Plugin::Buffer + Plugin.register_buffer('file2', self) + + include SystemConfig::Mixin + + DEFAULT_CHUNK_BYTES_LIMIT = 256 * 1024 * 1024 # 256MB + DEFAULT_TOTAL_BYTES_LIMIT = 64 * 1024 * 1024 * 1024 # 64GB, same with v0.12 (TimeSlicedOutput + buf_file) + + DIR_PERMISSION = 0755 + + # TODO: buffer_path based on system config + desc 'The path where buffer chunks are stored.' + config_param :path, :string + + config_set_default :chunk_bytes_limit, DEFAULT_CHUNK_BYTES_LIMIT + config_set_default :total_bytes_limit, DEFAULT_TOTAL_BYTES_LIMIT + + config_param :file_permission, :string, default: nil # '0644' + config_param :dir_permission, :string, default: nil # '0755' + + ##TODO: Buffer plugin cannot handle symlinks because new API @stage has many writing buffer chunks + ## re-implement this feature on out_file, w/ enqueue_chunk(or generate_chunk) hook + chunk.path + # attr_accessor :symlink_path + + @@buffer_paths = {} + + def initialize + super + @symlink_path = nil + end + + def configure(conf) + super + + type_of_owner = Plugin.lookup_type_from_class(@_owner.class) + if @@buffer_paths.has_key?(@path) && !buffer_path_for_test? + type_using_this_path = @@buffer_paths[@path] + raise ConfigError, "Other '#{type_using_this_path}' plugin already use same buffer path: type = #{type_of_owner}, buffer path = #{@path}" + end + + @@buffer_paths[@path] = type_of_owner + + # TODO: create buffer path with plugin_id, under directory specified by system config + if File.exist?(@path) + if File.directory?(@path) + @path = File.join(@path, 'buffer.*.log') + elsif File.basename(@path).include?('.*.') + # valid path (buffer.*.log will be ignored) + else + @path = @path + '.*.log' + end + else # path doesn't exist + if File.basename(@path).include?('.*.') + # valid path + else + # path is handled as directory, and it will be created at #start + @path = File.join(@path, 'buffer.*.log') + end + end + + unless @dir_permission + @dir_permission = system_config.dir_permission || DIR_PERMISSION + end + end + + def buffer_path_for_test? + caller_locations.each do |location| + # Thread::Backtrace::Location#path returns base filename or absolute path. + # #absolute_path returns absolute_path always. + # https://bugs.ruby-lang.org/issues/12159 + if location.absolute_path =~ /\/test_[^\/]+\.rb$/ # location.path =~ /test_.+\.rb$/ + return true + end + end + false + end + + def start + FileUtils.mkdir_p File.dirname(@path), mode: @dir_permission + + super + end + + def persistent? + true + end + + def resume + stage = {} + queue = [] + + Dir.glob(@path) do |path| + m = new_metadata() # this metadata will be overwritten by resuming .meta file content + # so it should not added into @metadata_list for now + mode = Fluent::Plugin::Buffer::FileChunk.assume_chunk_state(path) + chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode) # file chunk resumes contents of metadata + case chunk.state + when :staged + stage[chunk.metadata] = chunk + when :queued + queue << chunk + else + raise "BUG: unexpected chunk state '#{chunk.state}' for path '#{path}'" + end + end + + queue.sort_by!{ |chunk| chunk.modified_at } + + return stage, queue + end + + def generate_chunk(metadata) + # FileChunk generates real path with unique_id + if @file_permission + Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, perm: @file_permission) + else + Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create) + end + end + end + end +end diff --git a/lib/fluent/plugin/buffer/file_chunk.rb b/lib/fluent/plugin/buffer/file_chunk.rb new file mode 100644 index 0000000000..6ec6f1585d --- /dev/null +++ b/lib/fluent/plugin/buffer/file_chunk.rb @@ -0,0 +1,330 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'fluent/plugin/buffer/chunk' +require 'fluent/unique_id' + +module Fluent + module Plugin + class Buffer + class FileChunk < Chunk + ### buffer path user specified : /path/to/directory/user_specified_prefix.*.log + ### buffer chunk path : /path/to/directory/user_specified_prefix.b513b61c9791029c2513b61c9791029c2.log + ### buffer chunk metadata path : /path/to/directory/user_specified_prefix.b513b61c9791029c2513b61c9791029c2.log.meta + + # NOTE: Old style buffer path of time sliced output plugins had a part of key: prefix.20150414.b513b61...suffix + # But this part is not used now for any purpose. (Now metadata is used instead.) + + # state: b/q - 'b'(on stage, compatible with v0.12), 'q'(enqueued) + # path_prefix: path prefix string, ended with '.' + # path_suffix: path suffix string, like '.log' (or any other user specified) + + include SystemConfig::Mixin + + FILE_PERMISSION = 0644 + + attr_reader :path, :state, :permission + + def initialize(metadata, path, mode, perm: system_config.file_permission || FILE_PERMISSION) + super(metadata) + # state: staged/queued/closed + @state = nil + + @permission = perm + + @bytesize = @records = @adding_bytes = @adding_records = 0 + + case mode + when :create then create_new_chunk(path, perm) + when :staged then load_existing_staged_chunk(path) + when :queued then load_existing_enqueued_chunk(path) + else + raise ArgumentError, "Invalid file chunk mode: #{mode}" + end + end + + def append(data) + raise "BUG: appending to non-staged chunk, now '#{@state}'" unless @state == :staged + + bytes = 0 + adding = ''.force_encoding(Encoding::ASCII_8BIT) + data.each do |d| + x = d.force_encoding(Encoding::ASCII_8BIT) + bytes += x.bytesize + adding << x + end + @chunk.write adding + + @adding_bytes += bytes + @adding_records += data.size + + true + end + + def commit + write_metadata # this should be at first: of course, this operation may fail + + @commit_position = @chunk.pos + @records += @adding_records + @bytesize += @adding_bytes + @adding_bytes = @adding_records = 0 + @modified_at = Time.now + + true + end + + def rollback + if @chunk.pos != @commit_position + @chunk.seek(@commit_position, IO::SEEK_SET) + @chunk.truncate(@commit_position) + end + @adding_bytes = @adding_records = 0 + true + end + + def size + @bytesize + @adding_bytes + end + + def records + @records + @adding_records + end + + def empty? + @bytesize == 0 + end + + def close + @state = :closed + size = @chunk.size + @chunk.close + @meta.close if @meta # meta may be missing if chunk is queued at first + if size == 0 + File.unlink(@path, @meta_path) + end + end + + def purge + @state = :closed + @chunk.close + @meta.close if @meta + @bytesize = @records = @adding_bytes = @adding_records = 0 + File.unlink(@path, @meta_path) + end + + def read + @chunk.seek(0, IO::SEEK_SET) + @chunk.read + end + + def open(&block) + @chunk.seek(0, IO::SEEK_SET) + val = yield @chunk + @chunk.seek(0, IO::SEEK_END) if @state == :staged + val + end + + def self.assume_chunk_state(path) + if /\.(b|q)([0-9a-f]+)\.[^\/]*\Z/n =~ path # //n switch means explicit 'ASCII-8BIT' pattern + $1 == 'b' ? :staged : :queued + else + :queued + end + end + + def self.generate_stage_chunk_path(path, unique_id) + pos = path.index('.*.') + raise "BUG: buffer chunk path on stage MUST have '.*.'" unless pos + + prefix = path[0...pos] + suffix = path[(pos+3)..-1] + + chunk_id = Fluent::UniqueId.hex(unique_id) + state = 'b' + "#{prefix}.#{state}#{chunk_id}.#{suffix}" + end + + def self.generate_queued_chunk_path(path, unique_id) + chunk_id = Fluent::UniqueId.hex(unique_id) + if pos = path.index(".b#{chunk_id}.") + path.sub(".b#{chunk_id}.", ".q#{chunk_id}.") + else # for unexpected cases (ex: users rename files while opened by fluentd) + path + ".q#{chunk_id}.chunk" + end + end + + # used only for queued v0.12 buffer path + def self.unique_id_from_path(path) + if /\.(b|q)([0-9a-f]+)\.[^\/]*\Z/n =~ path # //n switch means explicit 'ASCII-8BIT' pattern + return $2.scan(/../).map{|x| x.to_i(16) }.pack('C*') + end + nil + end + + def restore_metadata(bindata) + data = msgpack_unpacker(symbolize_keys: true).feed(bindata).read rescue {} + + now = Time.now + + @unique_id = data[:id] || self.class.unique_id_from_path(@path) || @unique_id + @records = data[:r] || 0 + @created_at = Time.at(data.fetch(:c, now.to_i)) + @modified_at = Time.at(data.fetch(:m, now.to_i)) + + @metadata.timekey = data[:timekey] + @metadata.tag = data[:tag] + @metadata.variables = data[:variables] + end + + def restore_metadata_partially(chunk) + @unique_id = self.class.unique_id_from_path(chunk.path) || @unique_id + @records = 0 + @created_at = chunk.ctime # birthtime isn't supported on Windows (and Travis?) + @modified_at = chunk.mtime + + @metadata.timekey = nil + @metadata.tag = nil + @metadata.variables = nil + end + + def write_metadata(update: true) + data = @metadata.to_h.merge({ + id: @unique_id, + r: (update ? @records + @adding_records : @records), + c: @created_at.to_i, + m: (update ? Time.now : @modified_at).to_i, + }) + @meta.seek(0, IO::SEEK_SET) + @meta.truncate(0) + @meta.write(msgpack_packer.pack(data)) + end + + def enqueued! + return unless @state == :staged + + new_chunk_path = self.class.generate_queued_chunk_path(@path, @unique_id) + new_meta_path = new_chunk_path + '.meta' + + write_metadata(update: false) # re-write metadata w/ finalized records + + file_rename(@chunk, @path, new_chunk_path, ->(new_io){ @chunk = new_io }) + @path = new_chunk_path + + file_rename(@meta, @meta_path, new_meta_path) + @meta_path = new_meta_path + + @state = :queued + end + + def file_rename(file, old_path, new_path, callback=nil) + pos = file.pos + if Fluent.windows? + file.close + File.rename(old_path, new_path) + file = File.open(new_path, 'rb', @permission) + else + File.rename(old_path, new_path) + file.reopen(new_path, 'rb') + end + file.set_encoding(Encoding::ASCII_8BIT) + file.sync = true + file.binmode + file.pos = pos + callback.call(file) if callback + end + + def create_new_chunk(path, perm) + @path = self.class.generate_stage_chunk_path(path, @unique_id) + @meta_path = @path + '.meta' + @chunk = File.open(@path, 'w+', perm) + @chunk.set_encoding(Encoding::ASCII_8BIT) + @chunk.sync = true + @chunk.binmode + @meta = File.open(@meta_path, 'w', perm) + @meta.set_encoding(Encoding::ASCII_8BIT) + @meta.sync = true + @meta.binmode + + @state = :staged + @bytesize = 0 + @commit_position = @chunk.pos # must be 0 + @adding_bytes = 0 + @adding_records = 0 + end + + def load_existing_staged_chunk(path) + @path = path + @meta_path = @path + '.meta' + + @meta = nil + # staging buffer chunk without metadata is classic buffer chunk file + # and it should be enqueued immediately + if File.exist?(@meta_path) + @chunk = File.open(@path, 'r+') + @chunk.set_encoding(Encoding::ASCII_8BIT) + @chunk.sync = true + @chunk.seek(0, IO::SEEK_END) + @chunk.binmode + + @meta = File.open(@meta_path, 'r+') + @meta.set_encoding(Encoding::ASCII_8BIT) + @meta.sync = true + @meta.binmode + restore_metadata(@meta.read) + @meta.seek(0, IO::SEEK_SET) + + @state = :staged + @bytesize = @chunk.size + @commit_position = @chunk.pos + @adding_bytes = 0 + @adding_records = 0 + else + # classic buffer chunk - read only chunk + @chunk = File.open(@path, 'rb') + @chunk.set_encoding(Encoding::ASCII_8BIT) + @chunk.binmode + @chunk.seek(0, IO::SEEK_SET) + @state = :queued + @bytesize = @chunk.size + + restore_metadata_partially(@chunk) + + @commit_position = @chunk.size + @unique_id = self.class.unique_id_from_path(@path) || @unique_id + end + end + + def load_existing_enqueued_chunk(path) + @path = path + @chunk = File.open(@path, 'rb') + @chunk.set_encoding(Encoding::ASCII_8BIT) + @chunk.binmode + @chunk.seek(0, IO::SEEK_SET) + @bytesize = @chunk.size + @commit_position = @chunk.size + + @meta_path = @path + '.meta' + if File.readable?(@meta_path) + restore_metadata(File.open(@meta_path){|f| f.set_encoding(Encoding::ASCII_8BIT); f.binmode; f.read }) + else + restore_metadata_partially(@chunk) + end + @state = :queued + end + end + end + end +end From 013ea4d731c4e96a7f554dd8e29d5e3ddcc9a38b Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 15:36:58 +0900 Subject: [PATCH 11/36] add tests for v0.14 plugin base class --- test/plugin/test_base.rb | 75 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 test/plugin/test_base.rb diff --git a/test/plugin/test_base.rb b/test/plugin/test_base.rb new file mode 100644 index 0000000000..78d117a47f --- /dev/null +++ b/test/plugin/test_base.rb @@ -0,0 +1,75 @@ +require_relative '../helper' +require 'fluent/plugin/base' + +module FluentPluginBaseTest + class DummyPlugin < Fluent::Plugin::Base + end +end + +class BaseTest < Test::Unit::TestCase + setup do + @p = FluentPluginBaseTest::DummyPlugin.new + end + + test 'has methods for phases of plugin life cycle, and methods to know "super"s were correctly called or not' do + assert !@p.configured? + @p.configure(config_element()) + assert @p.configured? + + assert !@p.started? + @p.start + assert @p.start + + assert !@p.stopped? + @p.stop + assert @p.stopped? + + assert !@p.before_shutdown? + @p.before_shutdown + assert @p.before_shutdown? + + assert !@p.shutdown? + @p.shutdown + assert @p.shutdown? + + assert !@p.after_shutdown? + @p.after_shutdown + assert @p.after_shutdown? + + assert !@p.closed? + @p.close + assert @p.closed? + + assert !@p.terminated? + @p.terminate + assert @p.terminated? + end + + test 'can access system config' do + assert @p.system_config + + @p.system_config_override({'process_name' => 'mytest'}) + assert_equal 'mytest', @p.system_config.process_name + end + + test 'does not have router in default' do + assert !@p.has_router? + end + + test 'is configurable by config_param and config_section' do + assert_nothing_raised do + class FluentPluginBaseTest::DummyPlugin2 < Fluent::Plugin::TestBase + config_param :myparam1, :string + config_section :mysection, multi: false do + config_param :myparam2, :integer + end + end + end + p2 = FluentPluginBaseTest::DummyPlugin2.new + assert_nothing_raised do + p2.configure(config_element('ROOT', '', {'myparam1' => 'myvalue1'}, [config_element('mysection', '', {'myparam2' => 99})])) + end + assert_equal 'myvalue1', p2.myparam1 + assert_equal 99, p2.mysection.myparam2 + end +end From 44e2bdc9d84db81e1cee029ba9f664cc4b535bb7 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 15:37:26 +0900 Subject: [PATCH 12/36] add Input plugin for v0.14 plugin API --- lib/fluent/plugin/input.rb | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 lib/fluent/plugin/input.rb diff --git a/lib/fluent/plugin/input.rb b/lib/fluent/plugin/input.rb new file mode 100644 index 0000000000..a00facdf65 --- /dev/null +++ b/lib/fluent/plugin/input.rb @@ -0,0 +1,33 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'fluent/plugin/base' + +require 'fluent/log' +require 'fluent/plugin_id' +require 'fluent/plugin_helper' + +module Fluent + module Plugin + class Input < Base + include PluginId + include PluginLoggerMixin + include PluginHelper::Mixin + + helpers :event_emitter + end + end +end From e82c40d64188a66b049a9a2ef04911454621a104 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 15:37:58 +0900 Subject: [PATCH 13/36] add tests for Input plugin for v0.14 API --- test/plugin/test_input.rb | 122 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 test/plugin/test_input.rb diff --git a/test/plugin/test_input.rb b/test/plugin/test_input.rb new file mode 100644 index 0000000000..2e0a93a64a --- /dev/null +++ b/test/plugin/test_input.rb @@ -0,0 +1,122 @@ +require_relative '../helper' +require 'fluent/plugin/input' +require 'flexmock/test_unit' + +module FluentPluginInputTest + class DummyPlugin < Fluent::Plugin::Input + end +end + +class InputTest < Test::Unit::TestCase + setup do + Fluent::Test.setup + @p = FluentPluginInputTest::DummyPlugin.new + end + + test 'has healthy lifecycle' do + assert !@p.configured? + @p.configure(config_element()) + assert @p.configured? + + assert !@p.started? + @p.start + assert @p.start + + assert !@p.stopped? + @p.stop + assert @p.stopped? + + assert !@p.before_shutdown? + @p.before_shutdown + assert @p.before_shutdown? + + assert !@p.shutdown? + @p.shutdown + assert @p.shutdown? + + assert !@p.after_shutdown? + @p.after_shutdown + assert @p.after_shutdown? + + assert !@p.closed? + @p.close + assert @p.closed? + + assert !@p.terminated? + @p.terminate + assert @p.terminated? + end + + test 'has plugin_id automatically generated' do + assert @p.respond_to?(:plugin_id_configured?) + assert @p.respond_to?(:plugin_id) + + @p.configure(config_element()) + + assert !@p.plugin_id_configured? + assert @p.plugin_id + assert{ @p.plugin_id != 'mytest' } + end + + test 'has plugin_id manually configured' do + @p.configure(config_element('ROOT', '', {'@id' => 'mytest'})) + assert @p.plugin_id_configured? + assert_equal 'mytest', @p.plugin_id + end + + test 'has plugin logger' do + assert @p.respond_to?(:log) + assert @p.log + + # default logger + original_logger = @p.log + + @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'})) + + assert{ @p.log.object_id != original_logger.object_id } + assert_equal Fluent::Log::LEVEL_DEBUG, @p.log.level + end + + test 'can load plugin helpers' do + assert_nothing_raised do + class FluentPluginInputTest::DummyPlugin2 < Fluent::Plugin::Input + helpers :storage + end + end + end + + test 'has router and can emit into it' do + assert @p.has_router? + + @p.configure(config_element()) + assert @p.router + + DummyRouter = Struct.new(:emits) do + def emit(tag, es) + self.emits << [tag, es] + end + end + @p.router = DummyRouter.new([]) + @p.router.emit('mytag', []) + @p.router.emit('mytag.testing', ['it is not es, but no problem for tests']) + + assert_equal ['mytag', []], @p.router.emits[0] + assert_equal ['mytag.testing', ['it is not es, but no problem for tests']], @p.router.emits[1] + end + + test 'has router for specified label if configured' do + @p.configure(config_element()) + original_router = @p.router + + router_mock = flexmock('mytest') + router_mock.should_receive(:emit).once.with('mytag.testing', ['for mock']) + label_mock = flexmock('mylabel') + label_mock.should_receive(:event_router).once.and_return(router_mock) + Fluent::Engine.root_agent.labels['@mytest'] = label_mock + + @p.configure(config_element('ROOT', '', {'@label' => '@mytest'})) + assert{ @p.router.object_id != original_router.object_id } + + @p.router.emit('mytag.testing', ['for mock']) + end +end From 1f53273438dbecdee50f55836d3afbe5df7b3c26 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 15:38:46 +0900 Subject: [PATCH 14/36] add Output plugin implementation (including buffering) for v0.14 plugin API --- lib/fluent/plugin/output.rb | 827 ++++++++++++++++++++++++++++++++++++ 1 file changed, 827 insertions(+) create mode 100644 lib/fluent/plugin/output.rb diff --git a/lib/fluent/plugin/output.rb b/lib/fluent/plugin/output.rb new file mode 100644 index 0000000000..376daf6a9f --- /dev/null +++ b/lib/fluent/plugin/output.rb @@ -0,0 +1,827 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'fluent/plugin/base' +require 'fluent/log' +require 'fluent/plugin_id' +require 'fluent/plugin_helper' +require 'fluent/timezone' +require 'fluent/unique_id' + +require 'time' +require 'monitor' + +module Fluent + module Plugin + class Output < Base + include PluginId + include PluginLoggerMixin + include PluginHelper::Mixin + include UniqueId::Mixin + + helpers :thread, :retry_state + + CHUNK_KEY_PATTERN = /^[-_.@a-zA-Z0-9]+$/ + CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{[-_.@a-zA-Z0-9]+\}/ + + # `` and `` sections are available only when '#format' and '#write' are implemented + config_section :buffer, param_name: :buffer_config, init: true, required: false, multi: false, final: true do + config_argument :chunk_keys, :array, value_type: :string, default: [] + config_param :@type, :string, default: 'memory2' + + config_param :timekey_range, :time, default: nil # range size to be used: `time.to_i / @timekey_range` + config_param :timekey_use_utc, :bool, default: false # default is localtime + config_param :timekey_zone, :string, default: Time.now.strftime('%z') # '+0900' + config_param :timekey_wait, :time, default: 600 + + desc 'If true, plugin will try to flush buffer just before shutdown.' + config_param :flush_at_shutdown, :bool, default: nil # change default by buffer_plugin.persistent? + + desc 'How to enqueue chunks to be flushed. "fast" flushes per flush_interval, "immediate" flushes just after event arrival.' + config_param :flush_mode, :enum, list: [:default, :none, :fast, :immediate], default: :default + config_param :flush_interval, :time, default: 60, desc: 'The interval between buffer chunk flushes.' + + config_param :flush_threads, :integer, default: 1, desc: 'The number of threads to flush the buffer.' + + config_param :flush_thread_interval, :float, default: 1.0, desc: 'Seconds to sleep between checks for buffer flushes in flush threads.' + config_param :flush_burst_interval, :float, default: 1.0, desc: 'Seconds to sleep between flushes when many buffer chunks are queued.' + + config_param :delayed_commit_timeout, :time, default: 60, desc: 'Seconds of timeout for buffer chunks to be committed by plugins later.' + + config_param :retry_forever, :bool, default: false, desc: 'If true, plugin will ignore retry_timeout and retry_max_times options and retry flushing forever.' + config_param :retry_timeout, :time, default: 72 * 60 * 60, desc: 'The maximum seconds to retry to flush while failing, until plugin discards buffer chunks.' + # 72hours == 17 times with exponential backoff (not to change default behavior) + config_param :retry_max_times, :integer, default: nil, desc: 'The maximum number of times to retry to flush while failing.' + + config_param :retry_secondary_threshold, :float, default: 0.8, desc: 'ratio of retry_timeout to switch to use secondary while failing.' + # expornential backoff sequence will be initialized at the time of this threshold + + desc 'How to wait next retry to flush buffer.' + config_param :retry_type, :enum, list: [:expbackoff, :periodic], default: :expbackoff + ### Periodic -> fixed :retry_wait + ### Exponencial backoff: k is number of retry times + # c: constant factor, @retry_wait + # b: base factor, @retry_backoff_base + # k: times + # total retry time: c + c * b^1 + (...) + c*b^k = c*b^(k+1) - 1 + config_param :retry_wait, :time, default: 1, desc: 'Seconds to wait before next retry to flush, or constant factor of exponential backoff.' + config_param :retry_backoff_base, :float, default: 2, desc: 'The base number of exponencial backoff for retries.' + config_param :retry_max_interval, :time, default: nil, desc: 'The maximum interval seconds for exponencial backoff between retries while failing.' + + config_param :retry_randomize, :bool, default: true, desc: 'If true, output plugin will retry after randomized interval not to do burst retries.' + end + + config_section :secondary, param_name: :secondary_config, required: false, multi: false, final: true do + config_param :@type, :string, default: nil + config_section :buffer, required: false, multi: false do + # dummy to detect invalid specification for here + end + config_section :secondary, required: false, multi: false do + # dummy to detect invalid specification for here + end + end + + def process(tag, es) + raise NotImplementedError, "BUG: output plugins MUST implement this method" + end + + def format(tag, time, record) + raise NotImplementedError, "BUG: output plugins MUST implement this method" + end + + def write(chunk) + raise NotImplementedError, "BUG: output plugins MUST implement this method" + end + + def try_write(chunk) + raise NotImplementedError, "BUG: output plugins MUST implement this method" + end + + # TODO: add a way to do rollback_chunk + mark_as_failure, and rollback_chunk_automatically + mark_as_failure (by conf) + + def prefer_buffered_processing + # override this method to return false only when all of these are true: + # * plugin has both implementation for buffered and non-buffered methods + # * plugin is expected to work as non-buffered plugin if no `` sections specified + true + end + + def prefer_delayed_commit + # override this method to decide which is used of `write` or `try_write` if both are implemented + true + end + + # Internal states + FlushThreadState = Struct.new(:thread, :next_time) + DequeuedChunkInfo = Struct.new(:chunk_id, :time, :timeout) do + def expired? + time + timeout < Time.now + end + end + + attr_reader :as_secondary, :delayed_commit, :delayed_commit_timeout + attr_reader :num_errors, :emit_count, :emit_records, :write_count, :rollback_count + + # for tests + attr_reader :buffer, :retry, :secondary, :chunk_keys, :chunk_key_time, :chunk_key_tag + attr_accessor :output_enqueue_thread_waiting + + def initialize + super + @buffering = false + @delayed_commit = false + @as_secondary = false + @primary_instance = nil + end + + def acts_as_secondary(primary) + @as_secondary = true + @primary_instance = primary + (class << self; self; end).module_eval do + define_method(:extract_placeholders){ |str, metadata| @primary_instance.extract_placeholders(str, metadata) } + define_method(:commit_write){ |chunk_id| @primary_instance.commit_write(chunk_id, delayed: delayed_commit, secondary: true) } + define_method(:rollback_write){ |chunk_id| @primary_instance.rollback_write(chunk_id) } + end + end + + def configure(conf) + unless implement?(:synchronous) || implement?(:buffered) || implement?(:delayed_commit) + raise "BUG: output plugin must implement some methods. see developer documents." + end + + has_buffer_section = (conf.elements.select{|e| e.name == 'buffer' }.size > 0) + + super + + if has_buffer_section + unless implement?(:buffered) || implement?(:delayed_commit) + raise Fluent::ConfigError, " section is configured, but plugin '#{self.class}' doesn't support buffering" + end + @buffering = true + else # no buffer sections + if implement?(:synchronous) + if !implement?(:buffered) && !implement?(:delayed_commit) + if @as_secondary + raise Fluent::ConfigError, "secondary plugin '#{self.class}' must support buffering, but doesn't." + end + @buffering = false + else + if @as_secondary + # secondary plugin always works as buffered plugin without buffer instance + @buffering = true + else + # @buffering.nil? shows that enabling buffering or not will be decided in lazy way in #start + @buffering = nil + end + end + else # buffered or delayed_commit is supported by `unless` of first line in this method + @buffering = true + end + end + + if @as_secondary + if !@buffering && !@buffering.nil? + raise Fluent::ConfigError, "secondary plugin '#{self.class}' must support buffering, but doesn't" + end + end + + if (@buffering || @buffering.nil?) && !@as_secondary + # When @buffering.nil?, @buffer_config was initialized with default value for all parameters. + # If so, this configuration MUST success. + @chunk_keys = @buffer_config.chunk_keys + @chunk_key_time = !!@chunk_keys.delete('time') + @chunk_key_tag = !!@chunk_keys.delete('tag') + if @chunk_keys.any?{ |key| key !~ CHUNK_KEY_PATTERN } + raise Fluent::ConfigError, "chunk_keys specification includes invalid char" + end + + if @chunk_key_time + raise Fluent::ConfigError, " argument includes 'time', but timekey_range is not configured" unless @buffer_config.timekey_range + Fluent::Timezone.validate!(@buffer_config.timekey_zone) + @buffer_config.timekey_zone = '+0000' if @buffer_config.timekey_use_utc + @output_time_formatter_cache = {} + end + + @flush_mode = @buffer_config.flush_mode + if @flush_mode == :default + @flush_mode = (@chunk_key_time ? :none : :fast) + end + + buffer_type = @buffer_config[:@type] + buffer_conf = conf.elements.select{|e| e.name == 'buffer' }.first || Fluent::Config::Element.new('buffer', '', {}, []) + @buffer = Plugin.new_buffer(buffer_type, parent: self) + @buffer.configure(buffer_conf) + + @flush_at_shutdown = @buffer_config.flush_at_shutdown + if @flush_at_shutdown.nil? + @flush_at_shutdown = if @buffer.persistent? + false + else + true # flush_at_shutdown is true in default for on-memory buffer + end + elsif !@flush_at_shutdown && !@buffer.persistent? + buf_type = Plugin.lookup_type_from_class(@buffer.class) + log.warn "'flush_at_shutdown' is false, and buffer plugin '#{buf_type}' is not persistent buffer." + log.warn "your configuration will lose buffered data at shutdown. please confirm your configuration again." + end + end + + if @secondary_config + raise Fluent::ConfigError, "Invalid section for non-buffered plugin" unless @buffering + raise Fluent::ConfigError, " section cannot have section" if @secondary_config.buffer + raise Fluent::ConfigError, " section cannot have section" if @secondary_config.secondary + raise Fluent::ConfigError, " section and 'retry_forever' are exclusive" if @buffer_config.retry_forever + + secondary_type = @secondary_config[:@type] + secondary_conf = conf.elements.select{|e| e.name == 'secondary' }.first + @secondary = Plugin.new_output(secondary_type) + @secondary.acts_as_secondary(self) + @secondary.configure(secondary_conf) + @secondary.router = router if @secondary.has_router? + if self.class != @secondary.class + log.warn "secondary type should be same with primary one", primary: self.class.to_s, secondary: @secondary.class.to_s + end + else + @secondary = nil + end + + self + end + + def start + super + # TODO: well organized counters + @counters_monitor = Monitor.new + @num_errors = 0 + @emit_count = 0 + @emit_records = 0 + @write_count = 0 + @rollback_count = 0 + + if @buffering.nil? + @buffering = prefer_buffered_processing + if !@buffering && @buffer + @buffer.terminate # it's not started, so terminate will be enough + end + end + + if @buffering + m = method(:emit_buffered) + (class << self; self; end).module_eval do + define_method(:emit, m) + end + + @delayed_commit = if implement?(:buffered) && implement?(:delayed_commit) + prefer_delayed_commit + else + implement?(:delayed_commit) + end + @delayed_commit_timeout = @buffer_config.delayed_commit_timeout + else # !@buffered + m = method(:emit_sync) + (class << self; self; end).module_eval do + define_method(:emit, m) + end + end + + if @buffering && !@as_secondary + @retry = nil + @retry_mutex = Mutex.new + + @buffer.start + + @output_flush_threads = [] + @output_flush_threads_mutex = Mutex.new + @output_flush_threads_running = true + + # mainly for test: detect enqueue works as code below: + # @output.interrupt_flushes + # # emits + # @output.enqueue_thread_wait + @output_flush_interrupted = false + @output_enqueue_thread_mutex = Mutex.new + @output_enqueue_thread_waiting = false + + @dequeued_chunks = [] + @dequeued_chunks_mutex = Mutex.new + + @buffer_config.flush_threads.times do |i| + thread_title = "flush_thread_#{i}".to_sym + thread_state = FlushThreadState.new(nil, nil) + thread = thread_create(thread_title, thread_state, &method(:flush_thread_run)) + thread_state.thread = thread + @output_flush_threads_mutex.synchronize do + @output_flush_threads << thread_state + end + end + @output_flush_thread_current_position = 0 + + if @flush_mode == :fast || @chunk_key_time + thread_create(:enqueue_thread, &method(:enqueue_thread_run)) + end + end + @secondary.start if @secondary + end + + def stop + super + @secondary.stop if @secondary + @buffer.stop if @buffering && @buffer + end + + def before_shutdown + super + @secondary.before_shutdown if @secondary + + if @buffering && @buffer + if @flush_at_shutdown + force_flush + end + @buffer.before_shutdown + end + end + + def shutdown + super + @secondary.shutdown if @secondary + @buffer.shutdown if @buffering && @buffer + end + + def after_shutdown + super + try_rollback_all if @buffering && !@as_secondary # rollback regardless with @delayed_commit, because secondary may do it + @secondary.after_shutdown if @secondary + + if @buffering && @buffer + @buffer.after_shutdown + + @output_flush_threads_running = false + @output_flush_threads.each do |state| + state.thread.run if state.thread.alive? # to wakeup thread and make it to stop by itself + end + @output_flush_threads.each do |state| + state.thread.join + end + end + end + + def close + super + @buffer.close if @buffering && @buffer + @secondary.close if @secondary + end + + def terminate + super + @buffer.terminate if @buffering && @buffer + @secondary.terminate if @secondary + end + + def support_in_v12_style?(feature) + # for plugins written in v0.12 styles + case feature + when :synchronous then false + when :buffered then false + when :delayed_commit then false + end + end + + def implement?(feature) + methods_of_plugin = self.class.instance_methods(false) + case feature + when :synchronous then methods_of_plugin.include?(:process) || support_in_v12_style?(:synchronous) + when :buffered then methods_of_plugin.include?(:format) && methods_of_plugin.include?(:write) || support_in_v12_style?(:buffered) + when :delayed_commit then methods_of_plugin.include?(:format) && methods_of_plugin.include?(:try_write) + else + raise ArgumentError, "Unknown feature for output plugin: #{feature}" + end + end + + # TODO: optimize this code + def extract_placeholders(str, metadata) + if metadata.timekey.nil? && metadata.tag.nil? && metadata.variables.nil? + str + else + rvalue = str + # strftime formatting + if @chunk_key_time # this section MUST be earlier than rest to use raw 'str' + @output_time_formatter_cache[str] ||= Fluent::Timezone.formatter(@buffer_config.timekey_zone, str) + rvalue = @output_time_formatter_cache[str].call(metadata.timekey) + end + # ${tag}, ${tag[0]}, ${tag[1]}, ... + if @chunk_key_tag + if str =~ /\$\{tag\[\d+\]\}/ + hash = {'${tag}' => metadata.tag} + metadata.tag.split('.').each_with_index do |part, i| + hash["${tag[#{i}]}"] = part + end + rvalue = rvalue.gsub(/\$\{tag(\[\d+\])?\}/, hash) + elsif str.include?('${tag}') + rvalue = rvalue.gsub('${tag}', metadata.tag) + end + end + # ${a_chunk_key}, ... + if !@chunk_keys.empty? && metadata.variables + hash = {'${tag}' => '${tag}'} # not to erase this wrongly + @chunk_keys.each do |key| + hash["${#{key}}"] = metadata.variables[key.to_sym] + end + rvalue = rvalue.gsub(CHUNK_KEY_PLACEHOLDER_PATTERN, hash) + end + rvalue + end + end + + def emit(tag, es) + # actually this method will be overwritten by #configure + if @buffering + emit_buffered(tag, es) + else + emit_sync(tag, es) + end + end + + def emit_sync(tag, es) + @counters_monitor.synchronize{ @emit_count += 1 } + begin + process(tag, es) + # TODO: how to count records of es? add API to event streams? + # @counters_monitor.synchronize{ @emit_records += records } + rescue + @counters_monitor.synchronize{ @num_errors += 1 } + raise + end + end + + def emit_buffered(tag, es) + @counters_monitor.synchronize{ @emit_count += 1 } + begin + metalist = handle_stream(tag, es) + if @flush_mode == :immediate + metalist.each do |meta| + @buffer.enqueue_chunk(meta) + end + end + if !@retry && @buffer.queued? + submit_flush_once + end + rescue + # TODO: separate number of errors into emit errors and write/flush errors + @counters_monitor.synchronize{ @num_errors += 1 } + raise + end + end + + # TODO: optimize this code + def metadata(tag, time, record) + # this arguments are ordered in output plugin's rule + # Metadata 's argument order is different from this one (timekey, tag, variables) + timekey_range = @buffer_config.timekey_range + if @chunk_keys.empty? + if !@chunk_key_time && !@chunk_key_tag + @buffer.metadata() + elsif @chunk_key_time && @chunk_key_tag + time_int = time.to_i + timekey = time_int - (time_int % timekey_range) + @buffer.metadata(timekey: timekey, tag: tag) + elsif @chunk_key_time + time_int = time.to_i + timekey = time_int - (time_int % timekey_range) + @buffer.metadata(timekey: timekey) + else + @buffer.metadata(tag: tag) + end + else + timekey = if @chunk_key_time + time_int = time.to_i + time_int - (time_int % timekey_range) + else + nil + end + pairs = Hash[@chunk_keys.map{|k| [k.to_sym, record[k]]}] + @buffer.metadata(timekey: timekey, tag: (@chunk_key_tag ? tag : nil), variables: pairs) + end + end + + def handle_stream(tag, es) + meta_and_data = {} + records = 0 + es.each do |time, record| + meta = metadata(tag, time, record) + meta_and_data[meta] ||= [] + meta_and_data[meta] << format(tag, time, record) + records += 1 + end + meta_and_data.each_pair do |meta, data| + @buffer.emit(meta, data) + end + @counters_monitor.synchronize{ @emit_records += records } + meta_and_data.keys + end + + def commit_write(chunk_id, delayed: @delayed_commit, secondary: false) + if delayed + @dequeued_chunks_mutex.synchronize do + @dequeued_chunks.delete_if{ |info| info.chunk_id == chunk_id } + end + end + @buffer.purge_chunk(chunk_id) + + @retry_mutex.synchronize do + if @retry # success to flush chunks in retries + if secondary + log.warn "retry succeeded by secondary.", plugin_id: plugin_id, chunk_id: dump_unique_id_hex(chunk_id) + else + log.warn "retry succeeded.", plugin_id: plugin_id, chunk_id: dump_unique_id_hex(chunk_id) + end + @retry = nil + end + end + end + + def rollback_write(chunk_id) + # This API is to rollback chunks explicitly from plugins. + # 3rd party plugins can depend it on automatic rollback of #try_rollback_write + @dequeued_chunks_mutex.synchronize do + @dequeued_chunks.delete_if{ |info| info.chunk_id == chunk_id } + end + # returns true if chunk was rollbacked as expected + # false if chunk was already flushed and couldn't be rollbacked unexpectedly + # in many cases, false can be just ignored + if @buffer.takeback_chunk(chunk_id) + @counters_monitor.synchronize{ @rollback_count += 1 } + true + else + false + end + end + + def try_rollback_write + now = Time.now + @dequeued_chunks_mutex.synchronize do + while @dequeued_chunks.first && @dequeued_chunks.first.expired? + info = @dequeued_chunks.shift + if @buffer.takeback_chunk(info.chunk_id) + @counters_monitor.synchronize{ @rollback_count += 1 } + log.warn "failed to flush the buffer chunk, timeout to commit.", plugin_id: plugin_id, chunk_id: dump_unique_id_hex(info.chunk_id), flushed_at: info.time + end + end + end + end + + def try_rollback_all + return unless @dequeued_chunks + @dequeued_chunks_mutex.synchronize do + until @dequeued_chunks.empty? + info = @dequeued_chunks.shift + if @buffer.takeback_chunk(info.chunk_id) + @counters_monitor.synchronize{ @rollback_count += 1 } + log.info "delayed commit for buffer chunks was cancelled in shutdown", plugin_id: plugin_id, chunk_id: dump_unique_id_hex(info.chunk_id) + end + end + end + end + + def next_flush_time + if @buffer.queued? + @retry_mutex.synchronize do + @retry ? @retry.next_time : Time.now + @buffer_config.flush_burst_interval + end + else + Time.now + @buffer_config.flush_thread_interval + end + end + + def try_flush + chunk = @buffer.dequeue_chunk + return unless chunk + + output = self + using_secondary = false + if @retry_mutex.synchronize{ @retry && @retry.secondary? } + output = @secondary + using_secondary = true + end + + begin + if output.delayed_commit + @counters_monitor.synchronize{ @write_count += 1 } + output.try_write(chunk) + @dequeued_chunks_mutex.synchronize do + # delayed_commit_timeout for secondary is configured in of primary ( don't get ) + @dequeued_chunks << DequeuedChunkInfo.new(chunk.unique_id, Time.now, self.delayed_commit_timeout) + end + else # output plugin without delayed purge + chunk_id = chunk.unique_id + @counters_monitor.synchronize{ @write_count += 1 } + output.write(chunk) + commit_write(chunk_id, secondary: using_secondary) + end + rescue => e + log.debug "taking back chunk for errors.", plugin_id: plugin_id, chunk: dump_unique_id_hex(chunk.unique_id) + @buffer.takeback_chunk(chunk.unique_id) + + @retry_mutex.synchronize do + if @retry + @counters_monitor.synchronize{ @num_errors += 1 } + if @retry.limit? + records = @buffer.queued_records + log.error "failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.", plugin_id: plugin_id, retry_times: @retry.steps, records: records, error: e + log.error_backtrace e.backtrace + @buffer.clear_queue! + log.debug "buffer queue cleared", plugin_id: plugin_id + @retry = nil + else + @retry.step + msg = if using_secondary + "failed to flush the buffer with secondary output." + else + "failed to flush the buffer." + end + log.warn msg, plugin_id: plugin_id, retry_time: @retry.steps, next_retry: @retry.next_time, chunk: dump_unique_id_hex(chunk.unique_id), error: e + log.warn_backtrace e.backtrace + end + else + @retry = retry_state(@buffer_config.retry_randomize) + @counters_monitor.synchronize{ @num_errors += 1 } + log.warn "failed to flush the buffer.", plugin_id: plugin_id, retry_time: @retry.steps, next_retry: @retry.next_time, chunk: dump_unique_id_hex(chunk.unique_id), error: e + log.warn_backtrace e.backtrace + end + end + end + end + + def retry_state(randomize) + if @secondary + retry_state_create( + :output_retries, @buffer_config.retry_type, @buffer_config.retry_wait, @buffer_config.retry_timeout, + forever: @buffer_config.retry_forever, max_steps: @buffer_config.retry_max_times, backoff_base: @buffer_config.retry_backoff_base, + max_interval: @buffer_config.retry_max_interval, + secondary: true, secondary_threshold: @buffer_config.retry_secondary_threshold, + randomize: randomize + ) + else + retry_state_create( + :output_retries, @buffer_config.retry_type, @buffer_config.retry_wait, @buffer_config.retry_timeout, + forever: @buffer_config.retry_forever, max_steps: @buffer_config.retry_max_times, backoff_base: @buffer_config.retry_backoff_base, + max_interval: @buffer_config.retry_max_interval, + randomize: randomize + ) + end + end + + def submit_flush_once + # Without locks: it is rough but enough to select "next" writer selection + @output_flush_thread_current_position = (@output_flush_thread_current_position + 1) % @buffer_config.flush_threads + state = @output_flush_threads[@output_flush_thread_current_position] + state.next_time = 0 + state.thread.run + end + + def force_flush + @buffer.enqueue_all + submit_flush_all + end + + def submit_flush_all + while !@retry && @buffer.queued? + submit_flush_once + sleep @buffer_config.flush_burst_interval + end + end + + # only for tests of output plugin + def interrupt_flushes + @output_flush_interrupted = true + end + + # only for tests of output plugin + def enqueue_thread_wait + @output_enqueue_thread_mutex.synchronize do + @output_flush_interrupted = false + @output_enqueue_thread_waiting = true + end + require 'timeout' + Timeout.timeout(10) do + Thread.pass while @output_enqueue_thread_waiting + end + end + + # only for tests of output plugin + def flush_thread_wakeup + @output_flush_threads.each do |state| + state.next_time = 0 + state.thread.run + end + end + + def enqueue_thread_run + value_for_interval = nil + if @flush_mode == :fast + value_for_interval = @buffer_config.flush_interval + end + if @chunk_key_time + if !value_for_interval || @buffer_config.timekey_range < value_for_interval + value_for_interval = @buffer_config.timekey_range + end + end + unless value_for_interval + raise "BUG: both of flush_interval and timekey are disabled" + end + interval = value_for_interval / 11.0 + if interval < @buffer_config.flush_thread_interval + interval = @buffer_config.flush_thread_interval + end + + begin + while @output_flush_threads_running + now = Time.now + if @output_flush_interrupted + sleep interval + next + end + + @output_enqueue_thread_mutex.lock + begin + if @flush_mode == :fast + flush_interval = @buffer_config.flush_interval + @buffer.enqueue_all{ |metadata, chunk| chunk.created_at + flush_interval <= now } + end + + if @chunk_key_time + timekey_range = @buffer_config.timekey_range + timekey_wait = @buffer_config.timekey_wait + current_time_int = now.to_i + current_time_range = current_time_int - current_time_int % timekey_range + @buffer.enqueue_all{ |metadata, chunk| metadata.timekey < current_time_range && metadata.timekey + timekey_range + timekey_wait <= current_time_int } + end + rescue => e + log.error "unexpected error while checking flushed chunks. ignored.", plugin_id: plugin_id, error_class: e.class, error: e + log.error_backtrace + end + @output_enqueue_thread_waiting = false + @output_enqueue_thread_mutex.unlock + sleep interval + end + rescue => e + # normal errors are rescued by inner begin-rescue clause. + log.error "error on enqueue thread", plugin_id: plugin_id, error_class: e.class, error: e + log.error_backtrace + raise + end + end + + def flush_thread_run(state) + flush_thread_interval = @buffer_config.flush_thread_interval + + # If the given clock_id is not supported, Errno::EINVAL is raised. + clock_id = Process::CLOCK_MONOTONIC rescue Process::CLOCK_MONOTONIC_RAW + state.next_time = Process.clock_gettime(clock_id) + flush_thread_interval + + begin + # This thread don't use `thread_current_running?` because this thread should run in `before_shutdown` phase + while @output_flush_threads_running + time = Process.clock_gettime(clock_id) + interval = state.next_time - time + + if state.next_time <= time + try_flush + # next_flush_interval uses flush_thread_interval or flush_burst_interval (or retrying) + interval = next_flush_time.to_f - Time.now.to_f + # TODO: if secondary && delayed-commit, next_flush_time will be much longer than expected (because @retry still exists) + # @retry should be cleard if delayed commit is enabled? Or any other solution? + state.next_time = Process.clock_gettime(clock_id) + interval + end + + if @dequeued_chunks_mutex.synchronize{ !@dequeued_chunks.empty? && @dequeued_chunks.first.expired? } + unless @output_flush_interrupted + try_rollback_write + end + end + + sleep interval if interval > 0 + end + rescue => e + # normal errors are rescued by output plugins in #try_flush + # so this rescue section is for critical & unrecoverable errors + log.error "error on output thread", plugin_id: plugin_id, error_class: e.class, error: e + log.error_backtrace + raise + end + end + end + end +end From 1ef535fd96b09bceef0c7863cc8d1502f4de00af Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 15:39:32 +0900 Subject: [PATCH 15/36] add tests for Buffer plugin for v0.14 API --- test/plugin/test_buf_file2.rb | 323 ++++++++++ test/plugin/test_buf_memory2.rb | 42 ++ test/plugin/test_buffer.rb | 760 ++++++++++++++++++++++++ test/plugin/test_buffer_chunk.rb | 90 +++ test/plugin/test_buffer_file_chunk.rb | 694 ++++++++++++++++++++++ test/plugin/test_buffer_memory_chunk.rb | 193 ++++++ 6 files changed, 2102 insertions(+) create mode 100644 test/plugin/test_buf_file2.rb create mode 100644 test/plugin/test_buf_memory2.rb create mode 100644 test/plugin/test_buffer.rb create mode 100644 test/plugin/test_buffer_chunk.rb create mode 100644 test/plugin/test_buffer_file_chunk.rb create mode 100644 test/plugin/test_buffer_memory_chunk.rb diff --git a/test/plugin/test_buf_file2.rb b/test/plugin/test_buf_file2.rb new file mode 100644 index 0000000000..975abf256e --- /dev/null +++ b/test/plugin/test_buf_file2.rb @@ -0,0 +1,323 @@ +require_relative '../helper' +require 'fluent/plugin/buf_file2' +require 'fluent/plugin/output' +require 'fluent/unique_id' +require 'fluent/system_config' + +require 'msgpack' + +module FluentPluginFileBufferTest + class DummyOutputPlugin < Fluent::Plugin::Output + Fluent::Plugin.register_output('buffer_file_test_output', self) + end +end + +class FileBufferTest < Test::Unit::TestCase + def metadata(timekey: nil, tag: nil, variables: nil) + Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables) + end + + def write_metadata(path, chunk_id, metadata, records, ctime, mtime) + metadata = { + timekey: metadata.timekey, tag: metadata.tag, variables: metadata.variables, + id: chunk_id, + r: records, + c: ctime, + m: mtime, + } + File.open(path, 'w') do |f| + f.write metadata.to_msgpack + end + end + + teardown do + if @p + @p.stop unless @p.stopped? + @p.before_shutdown unless @p.before_shutdown? + @p.shutdown unless @p.shutdown? + @p.after_shutdown unless @p.after_shutdown? + @p.close unless @p.closed? + @p.terminate unless @p.terminated? + end + end + + sub_test_case 'non configured buffer plugin instance' do + # tests for path + test '' + end + + sub_test_case 'buffer plugin configured only with path' do + setup do + @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__) + @bufpath = File.join(@bufdir, 'testbuf.*.log') + FileUtils.rm_r @bufdir if File.exist?(@bufdir) + + Fluent::Test.setup + @d = FluentPluginFileBufferTest::DummyOutputPlugin.new + @p = Fluent::Plugin::FileBuffer.new + @p.owner = @d + @p.configure(config_element('buffer', '', {'path' => @bufpath})) + @p.start + end + + test 'this is persistent plugin' do + assert @p.persistent? + end + + test '#start creates directory for buffer chunks' do + plugin = Fluent::Plugin::FileBuffer.new + plugin.owner = @d + rand_num = rand(0..100) + bufpath = File.join(File.expand_path("../../tmp/buffer_file_#{rand_num}", __FILE__), 'testbuf.*.log') + bufdir = File.dirname(bufpath) + + FileUtils.rm_r bufdir if File.exist?(bufdir) + assert !File.exist?(bufdir) + + plugin.configure(config_element('buffer', '', {'path' => bufpath})) + assert !File.exist?(bufdir) + + plugin.start + assert File.exist?(bufdir) + assert{ File.stat(bufdir).mode.to_s(8).end_with?('755') } + + plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate + FileUtils.rm_r bufdir + end + + test '#start creates directory for buffer chunks with specified permission' do + plugin = Fluent::Plugin::FileBuffer.new + plugin.owner = @d + rand_num = rand(0..100) + bufpath = File.join(File.expand_path("../../tmp/buffer_file_#{rand_num}", __FILE__), 'testbuf.*.log') + bufdir = File.dirname(bufpath) + + FileUtils.rm_r bufdir if File.exist?(bufdir) + assert !File.exist?(bufdir) + + plugin.configure(config_element('buffer', '', {'path' => bufpath, 'dir_permission' => 0700})) + assert !File.exist?(bufdir) + + plugin.start + assert File.exist?(bufdir) + assert{ File.stat(bufdir).mode.to_s(8).end_with?('700') } + + plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate + FileUtils.rm_r bufdir + end + + test '#start creates directory for buffer chunks with specified permission via system config' do + sysconf = {'dir_permission' => '700'} + Fluent::SystemConfig.overwrite_system_config(sysconf) do + plugin = Fluent::Plugin::FileBuffer.new + plugin.owner = @d + rand_num = rand(0..100) + bufpath = File.join(File.expand_path("../../tmp/buffer_file_#{rand_num}", __FILE__), 'testbuf.*.log') + bufdir = File.dirname(bufpath) + + FileUtils.rm_r bufdir if File.exist?(bufdir) + assert !File.exist?(bufdir) + + plugin.configure(config_element('buffer', '', {'path' => bufpath})) + assert !File.exist?(bufdir) + + plugin.start + assert File.exist?(bufdir) + assert{ File.stat(bufdir).mode.to_s(8).end_with?('700') } + + plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate + FileUtils.rm_r bufdir + end + end + + test '#generate_chunk generates blank file chunk on path from unique_id of metadata' do + m1 = metadata() + c1 = @p.generate_chunk(m1) + assert c1.is_a? Fluent::Plugin::Buffer::FileChunk + assert_equal m1, c1.metadata + assert c1.empty? + assert_equal :staged, c1.state + assert_equal Fluent::Plugin::Buffer::FileChunk::FILE_PERMISSION, c1.permission + assert_equal @bufpath.gsub('.*.', ".b#{Fluent::UniqueId.hex(c1.unique_id)}."), c1.path + assert{ File.stat(c1.path).mode.to_s(8).end_with?('644') } + + m2 = metadata(timekey: event_time('2016-04-17 11:15:00 -0700').to_i) + c2 = @p.generate_chunk(m2) + assert c2.is_a? Fluent::Plugin::Buffer::FileChunk + assert_equal m2, c2.metadata + assert c2.empty? + assert_equal :staged, c2.state + assert_equal Fluent::Plugin::Buffer::FileChunk::FILE_PERMISSION, c2.permission + assert_equal @bufpath.gsub('.*.', ".b#{Fluent::UniqueId.hex(c2.unique_id)}."), c2.path + assert{ File.stat(c2.path).mode.to_s(8).end_with?('644') } + end + + test '#generate_chunk generates blank file chunk with specified permission' do + plugin = Fluent::Plugin::FileBuffer.new + plugin.owner = @d + rand_num = rand(0..100) + bufpath = File.join(File.expand_path("../../tmp/buffer_file_#{rand_num}", __FILE__), 'testbuf.*.log') + bufdir = File.dirname(bufpath) + + FileUtils.rm_r bufdir if File.exist?(bufdir) + assert !File.exist?(bufdir) + + plugin.configure(config_element('buffer', '', {'path' => bufpath, 'file_permission' => 0600})) + assert !File.exist?(bufdir) + plugin.start + + m = metadata() + c = plugin.generate_chunk(m) + assert c.is_a? Fluent::Plugin::Buffer::FileChunk + assert_equal m, c.metadata + assert c.empty? + assert_equal :staged, c.state + assert_equal 0600, c.permission + assert_equal bufpath.gsub('.*.', ".b#{Fluent::UniqueId.hex(c.unique_id)}."), c.path + assert{ File.stat(c.path).mode.to_s(8).end_with?('600') } + + plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate + FileUtils.rm_r bufdir + end + end + + + sub_test_case 'there are no existing file chunks' do + setup do + @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__) + @bufpath = File.join(@bufdir, 'testbuf.*.log') + FileUtils.rm_r @bufdir if File.exist?(@bufdir) + + Fluent::Test.setup + @d = FluentPluginFileBufferTest::DummyOutputPlugin.new + @p = Fluent::Plugin::FileBuffer.new + @p.owner = @d + @p.configure(config_element('buffer', '', {'path' => @bufpath})) + @p.start + end + + test '#resume returns empty buffer state' do + ary = @p.resume + assert_equal({}, ary[0]) + assert_equal([], ary[1]) + end + end + + sub_test_case 'there are some existing file chunks' do + setup do + @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__) + + @c1id = Fluent::UniqueId.generate + p1 = File.join(@bufdir, "etest.q#{Fluent::UniqueId.hex(@c1id)}.log") + File.open(p1, 'w') do |f| + f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" + f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" + f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" + f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n" + end + write_metadata( + p1 + '.meta', @c1id, metadata(timekey: event_time('2016-04-17 13:58:00 -0700').to_i), + 4, event_time('2016-04-17 13:58:00 -0700').to_i, event_time('2016-04-17 13:58:22 -0700').to_i + ) + + @c2id = Fluent::UniqueId.generate + p2 = File.join(@bufdir, "etest.q#{Fluent::UniqueId.hex(@c2id)}.log") + File.open(p2, 'w') do |f| + f.write ["t1.test", event_time('2016-04-17 13:59:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" + f.write ["t2.test", event_time('2016-04-17 13:59:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" + f.write ["t3.test", event_time('2016-04-17 13:59:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" + end + write_metadata( + p2 + '.meta', @c2id, metadata(timekey: event_time('2016-04-17 13:59:00 -0700').to_i), + 3, event_time('2016-04-17 13:59:00 -0700').to_i, event_time('2016-04-17 13:59:23 -0700').to_i + ) + + @c3id = Fluent::UniqueId.generate + p3 = File.join(@bufdir, "etest.b#{Fluent::UniqueId.hex(@c3id)}.log") + File.open(p3, 'w') do |f| + f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" + f.write ["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" + f.write ["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" + f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}].to_json + "\n" + end + write_metadata( + p3 + '.meta', @c3id, metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i), + 4, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i + ) + + @c4id = Fluent::UniqueId.generate + p4 = File.join(@bufdir, "etest.b#{Fluent::UniqueId.hex(@c4id)}.log") + File.open(p4, 'w') do |f| + f.write ["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" + f.write ["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" + f.write ["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" + end + write_metadata( + p4 + '.meta', @c4id, metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i), + 3, event_time('2016-04-17 14:01:00 -0700').to_i, event_time('2016-04-17 14:01:25 -0700').to_i + ) + + @bufpath = File.join(@bufdir, 'etest.*.log') + + Fluent::Test.setup + @d = FluentPluginFileBufferTest::DummyOutputPlugin.new + @p = Fluent::Plugin::FileBuffer.new + @p.owner = @d + @p.configure(config_element('buffer', '', {'path' => @bufpath})) + @p.start + end + + teardown do + Dir.glob(@bufpath).each do |path| + File.unlink(path + '.meta') if File.exist?(path + '.meta') + File.unlink(path) if File.exist?(path) + end + end + + test '#resume returns staged/queued chunks with metadata' do + stage,queue = @p.resume + assert_equal 2, stage.size + assert_equal 2, queue.size + + m3 = metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i) + assert_equal @c3id, stage[m3].unique_id + assert_equal 4, stage[m3].records + assert_equal :staged, stage[m3].state + + m4 = metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i) + assert_equal @c4id, stage[m4].unique_id + assert_equal 3, stage[m4].records + assert_equal :staged, stage[m4].state + end + + test '#resume returns queued chunks ordered by last modified time (FIFO)' do + stage,queue = @p.resume + assert_equal 2, stage.size + assert_equal 2, queue.size + + assert{ queue[0].modified_at < queue[1].modified_at } + + assert_equal @c1id, queue[0].unique_id + assert_equal :queued, queue[0].state + assert_equal event_time('2016-04-17 13:58:00 -0700').to_i, queue[0].metadata.timekey + assert_nil queue[0].metadata.tag + assert_nil queue[0].metadata.variables + assert_equal Time.parse('2016-04-17 13:58:00 -0700').localtime, queue[0].created_at + assert_equal Time.parse('2016-04-17 13:58:22 -0700').localtime, queue[0].modified_at + assert_equal 4, queue[0].records + + assert_equal @c2id, queue[1].unique_id + assert_equal :queued, queue[1].state + assert_equal event_time('2016-04-17 13:59:00 -0700').to_i, queue[1].metadata.timekey + assert_nil queue[1].metadata.tag + assert_nil queue[1].metadata.variables + assert_equal Time.parse('2016-04-17 13:59:00 -0700').localtime, queue[1].created_at + assert_equal Time.parse('2016-04-17 13:59:23 -0700').localtime, queue[1].modified_at + assert_equal 3, queue[1].records + end + end + + sub_test_case 'there are some existing file chunks without metadata file' do + test '#resume returns queued chunks for files without metadata' + end +end diff --git a/test/plugin/test_buf_memory2.rb b/test/plugin/test_buf_memory2.rb new file mode 100644 index 0000000000..267c33182a --- /dev/null +++ b/test/plugin/test_buf_memory2.rb @@ -0,0 +1,42 @@ +require_relative '../helper' +require 'fluent/plugin/buf_memory2' +require 'fluent/plugin/output' +require 'flexmock/test_unit' + +module FluentPluginMemoryBufferTest + class DummyOutputPlugin < Fluent::Plugin::Output + end +end + +class MemoryBufferTest < Test::Unit::TestCase + setup do + Fluent::Test.setup + @d = FluentPluginMemoryBufferTest::DummyOutputPlugin.new + @p = Fluent::Plugin::MemoryBuffer.new + @p.owner = @d + end + + test 'this is non persistent plugin' do + assert !@p.persistent? + end + + test '#resume always returns empty stage and queue' do + ary = @p.resume + assert_equal({}, ary[0]) + assert_equal([], ary[1]) + end + + test '#generate_chunk returns memory chunk instance' do + m1 = Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil) + c1 = @p.generate_chunk(m1) + assert c1.is_a? Fluent::Plugin::Buffer::MemoryChunk + assert_equal m1, c1.metadata + + require 'time' + t2 = Time.parse('2016-04-08 19:55:00 +0900').to_i + m2 = Fluent::Plugin::Buffer::Metadata.new(t2, 'test.tag', {k1: 'v1', k2: 0}) + c2 = @p.generate_chunk(m2) + assert c2.is_a? Fluent::Plugin::Buffer::MemoryChunk + assert_equal m2, c2.metadata + end +end diff --git a/test/plugin/test_buffer.rb b/test/plugin/test_buffer.rb new file mode 100644 index 0000000000..c5930a02be --- /dev/null +++ b/test/plugin/test_buffer.rb @@ -0,0 +1,760 @@ +require_relative '../helper' +require 'fluent/plugin/buffer' +require 'fluent/plugin/buffer/memory_chunk' +require 'flexmock/test_unit' + +require 'fluent/log' +require 'fluent/plugin_id' + +require 'time' + +module FluentPluginBufferTest + class DummyOutputPlugin < Fluent::Plugin::Base + include Fluent::PluginId + include Fluent::PluginLoggerMixin + end + class DummyMemoryChunk < Fluent::Plugin::Buffer::MemoryChunk + attr_reader :append_count, :rollbacked, :closed, :purged + def initialize(metadata) + super + @append_count = 0 + @rollbacked = false + @closed = false + @purged = false + end + def append(data) + @append_count += 1 + super + end + def rollback + super + @rollbacked = true + end + def close + super + @closed = true + end + def purge + super + @purged = true + end + end + class DummyPlugin < Fluent::Plugin::Buffer + def create_metadata(timekey=nil, tag=nil, variables=nil) + Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables) + end + def create_chunk(metadata, data) + c = FluentPluginBufferTest::DummyMemoryChunk.new(metadata) + c.append(data) + c.commit + c + end + def resume + dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil) + dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil) + dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil) + dm3 = create_metadata(Time.parse('2016-04-11 16:30:00 +0000').to_i, nil, nil) + staged = { + dm2 => create_chunk(dm2, ["b" * 100]), + dm3 => create_chunk(dm3, ["c" * 100]), + } + queued = [ + create_chunk(dm0, ["0" * 100]), + create_chunk(dm1, ["a" * 100]), + create_chunk(dm1, ["a" * 3]), + ] + return staged, queued + end + def generate_chunk(metadata) + DummyMemoryChunk.new(metadata) + end + end +end + +class BufferTest < Test::Unit::TestCase + def create_buffer(hash) + buffer_conf = config_element('buffer', '', hash, []) + owner = FluentPluginBufferTest::DummyOutputPlugin.new + owner.configure(config_element('ROOT', '', {}, [ buffer_conf ])) + p = FluentPluginBufferTest::DummyPlugin.new + p.owner = owner + p.configure(buffer_conf) + p + end + + def create_metadata(timekey=nil, tag=nil, variables=nil) + Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables) + end + + def create_chunk(metadata, data) + c = FluentPluginBufferTest::DummyMemoryChunk.new(metadata) + c.append(data) + c.commit + c + end + + setup do + Fluent::Test.setup + end + + sub_test_case 'using base buffer class' do + setup do + buffer_conf = config_element('buffer', '', {}, []) + owner = FluentPluginBufferTest::DummyOutputPlugin.new + owner.configure(config_element('ROOT', '', {}, [ buffer_conf ])) + p = Fluent::Plugin::Buffer.new + p.owner = owner + p.configure(buffer_conf) + @p = p + end + + test 'default persistency is false' do + assert !@p.persistent? + end + + test 'chunk bytes limit is 8MB, and total bytes limit is 512MB' do + assert_equal 8*1024*1024, @p.chunk_bytes_limit + assert_equal 512*1024*1024, @p.total_bytes_limit + end + + test 'chunk records limit is ignored in default' do + assert_nil @p.chunk_records_limit + end + + test '#storable? checks total size of staged and enqueued(includes dequeued chunks) against total_bytes_limit' do + assert_equal 512*1024*1024, @p.total_bytes_limit + assert_equal 0, @p.stage_size + assert_equal 0, @p.queue_size + assert @p.storable? + + @p.stage_size = 256 * 1024 * 1024 + @p.queue_size = 256 * 1024 * 1024 - 1 + assert @p.storable? + + @p.queue_size = 256 * 1024 * 1024 + assert !@p.storable? + end + + test '#resume must be implemented by subclass' do + assert_raise NotImplementedError do + @p.resume + end + end + + test '#generate_chunk must be implemented by subclass' do + assert_raise NotImplementedError do + @p.generate_chunk(Object.new) + end + end + end + + sub_test_case 'with default configuration and dummy implementation' do + setup do + @p = create_buffer({}) + @dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil) + @dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil) + @dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil) + @dm3 = create_metadata(Time.parse('2016-04-11 16:30:00 +0000').to_i, nil, nil) + @p.start + end + + test '#start resumes buffer states and update queued numbers per metadata' do + plugin = create_buffer({}) + + assert_equal({}, plugin.stage) + assert_equal([], plugin.queue) + assert_equal({}, plugin.dequeued) + assert_equal({}, plugin.queued_num) + assert_equal([], plugin.metadata_list) + + assert_equal 0, plugin.stage_size + assert_equal 0, plugin.queue_size + + # @p is started plugin + + assert_equal [@dm2,@dm3], @p.stage.keys + assert_equal "b" * 100, @p.stage[@dm2].read + assert_equal "c" * 100, @p.stage[@dm3].read + + assert_equal 200, @p.stage_size + + assert_equal 3, @p.queue.size + assert_equal "0" * 100, @p.queue[0].read + assert_equal "a" * 100, @p.queue[1].read + assert_equal "a" * 3, @p.queue[2].read + + assert_equal 203, @p.queue_size + + # staged, queued + assert_equal [@dm2,@dm3,@dm0,@dm1], @p.metadata_list + assert_equal 1, @p.queued_num[@dm0] + assert_equal 2, @p.queued_num[@dm1] + end + + test '#close closes all chunks in in dequeued, enqueued and staged' do + dmx = create_metadata(Time.parse('2016-04-11 15:50:00 +0000').to_i, nil, nil) + cx = create_chunk(dmx, ["x" * 1024]) + @p.dequeued[cx.unique_id] = cx + + staged_chunks = @p.stage.values.dup + queued_chunks = @p.queue.dup + + @p.close + + assert cx.closed + assert{ staged_chunks.all?{|c| c.closed } } + assert{ queued_chunks.all?{|c| c.closed } } + end + + test '#terminate initializes all internal states' do + dmx = create_metadata(Time.parse('2016-04-11 15:50:00 +0000').to_i, nil, nil) + cx = create_chunk(dmx, ["x" * 1024]) + @p.dequeued[cx.unique_id] = cx + + @p.close + + @p.terminate + + assert_nil @p.stage + assert_nil @p.queue + assert_nil @p.dequeued + assert_nil @p.queued_num + assert_nil @p.instance_eval{ @metadata_list } # #metadata_list does #dup for @metadata_list + assert_equal 0, @p.stage_size + assert_equal 0, @p.queue_size + end + + test '#metadata_list returns list of metadata on stage or in queue' do + assert_equal [@dm2,@dm3,@dm0,@dm1], @p.metadata_list + end + + test '#new_metadata creates metadata instance without inserting metadata_list' do + assert_equal [@dm2,@dm3,@dm0,@dm1], @p.metadata_list + m = @p.new_metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) + assert_equal [@dm2,@dm3,@dm0,@dm1], @p.metadata_list + end + + test '#add_metadata adds unknown metadata into list, or return known metadata if already exists' do + assert_equal [@dm2,@dm3,@dm0,@dm1], @p.metadata_list + + m = @p.new_metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) + mx = @p.add_metadata(m) + assert_equal [@dm2,@dm3,@dm0,@dm1,m], @p.metadata_list + assert_equal m.object_id, m.object_id + + my = @p.add_metadata(@dm1) + assert_equal [@dm2,@dm3,@dm0,@dm1,m], @p.metadata_list + assert_equal @dm1, my + assert{ @dm1.object_id != my.object_id } # 'my' is an object created in #resume + end + + test '#metadata is utility method to create-add-and-return metadata' do + assert_equal [@dm2,@dm3,@dm0,@dm1], @p.metadata_list + + m1 = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) + assert_equal [@dm2,@dm3,@dm0,@dm1,m1], @p.metadata_list + m2 = @p.metadata(timekey: @dm3.timekey) + assert_equal [@dm2,@dm3,@dm0,@dm1,m1], @p.metadata_list + assert_equal @dm3, m2 + end + + test '#queued_records returns total number of records in all chunks in queue' do + assert_equal 3, @p.queue.size + + r0 = @p.queue[0].records + assert_equal 1, r0 + r1 = @p.queue[1].records + assert_equal 1, r1 + r2 = @p.queue[2].records + assert_equal 1, r2 + + assert_equal (r0+r1+r2), @p.queued_records + end + + test '#queued? returns queue has any chunks or not without arguments' do + assert @p.queued? + + @p.queue.reject!{|c| true } + assert !@p.queued? + end + + test '#queued? returns queue has chunks for specified metadata with an argument' do + assert @p.queued?(@dm0) + assert @p.queued?(@dm1) + assert !@p.queued?(@dm2) + end + + test '#enqueue_chunk enqueues a chunk on stage with specified metadata' do + assert_equal 2, @p.stage.size + assert_equal [@dm2,@dm3], @p.stage.keys + assert_equal 3, @p.queue.size + assert_nil @p.queued_num[@dm2] + + assert_equal 200, @p.stage_size + assert_equal 203, @p.queue_size + + @p.enqueue_chunk(@dm2) + + assert_equal [@dm3], @p.stage.keys + assert_equal @dm2, @p.queue.last.metadata + assert_equal 1, @p.queued_num[@dm2] + assert_equal 100, @p.stage_size + assert_equal 303, @p.queue_size + end + + test '#enqueue_chunk ignores empty chunks' do + assert_equal 3, @p.queue.size + + m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) + c = create_chunk(m, ['']) + @p.stage[m] = c + assert @p.stage[m].empty? + assert !c.closed + + @p.enqueue_chunk(m) + + assert_nil @p.stage[m] + assert_equal 3, @p.queue.size + assert_nil @p.queued_num[m] + assert c.closed + end + + test '#enqueue_chunk calls #enqueued! if chunk responds to it' do + assert_equal 3, @p.queue.size + m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) + c = create_chunk(m, ['c' * 256]) + callback_called = false + (class << c; self; end).module_eval do + define_method(:enqueued!){ callback_called = true } + end + + @p.stage[m] = c + @p.enqueue_chunk(m) + + assert_equal c, @p.queue.last + assert callback_called + end + + test '#enqueue_all enqueues chunks on stage which given block returns true with' do + m1 = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) + c1 = create_chunk(m1, ['c' * 256]) + @p.stage[m1] = c1 + m2 = @p.metadata(timekey: Time.parse('2016-04-11 16:50:00 +0000').to_i) + c2 = create_chunk(m2, ['c' * 256]) + @p.stage[m2] = c2 + + assert_equal [@dm2,@dm3,m1,m2], @p.stage.keys + assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) + + @p.enqueue_all{ |m, c| m.timekey < Time.parse('2016-04-11 16:41:00 +0000').to_i } + + assert_equal [m2], @p.stage.keys + assert_equal [@dm0,@dm1,@dm1,@dm2,@dm3,m1], @p.queue.map(&:metadata) + end + + test '#enqueue_all enqueues all chunks on stage without block' do + m1 = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) + c1 = create_chunk(m1, ['c' * 256]) + @p.stage[m1] = c1 + m2 = @p.metadata(timekey: Time.parse('2016-04-11 16:50:00 +0000').to_i) + c2 = create_chunk(m2, ['c' * 256]) + @p.stage[m2] = c2 + + assert_equal [@dm2,@dm3,m1,m2], @p.stage.keys + assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) + + @p.enqueue_all + + assert_equal [], @p.stage.keys + assert_equal [@dm0,@dm1,@dm1,@dm2,@dm3,m1,m2], @p.queue.map(&:metadata) + end + + test '#dequeue_chunk dequeues a chunk from queue if a chunk exists' do + assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal({}, @p.dequeued) + + m1 = @p.dequeue_chunk + assert_equal @dm0, m1.metadata + assert_equal @dm0, @p.dequeued[m1.unique_id].metadata + + m2 = @p.dequeue_chunk + assert_equal @dm1, m2.metadata + assert_equal @dm1, @p.dequeued[m2.unique_id].metadata + + m3 = @p.dequeue_chunk + assert_equal @dm1, m3.metadata + assert_equal @dm1, @p.dequeued[m3.unique_id].metadata + + m4 = @p.dequeue_chunk + assert_nil m4 + end + + test '#takeback_chunk resumes a chunk from dequeued to queued at the head of queue, and returns true' do + assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal({}, @p.dequeued) + + m1 = @p.dequeue_chunk + assert_equal @dm0, m1.metadata + assert_equal @dm0, @p.dequeued[m1.unique_id].metadata + assert_equal [@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal({m1.unique_id => m1}, @p.dequeued) + + assert @p.takeback_chunk(m1.unique_id) + + assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal({}, @p.dequeued) + end + + test '#purge_chunk removes a chunk specified by argument id from dequeued chunks' do + assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal({}, @p.dequeued) + assert_equal [@dm2,@dm3,@dm0,@dm1], @p.metadata_list + + m0 = @p.dequeue_chunk + m1 = @p.dequeue_chunk + + assert @p.takeback_chunk(m0.unique_id) + + assert_equal [@dm0,@dm1], @p.queue.map(&:metadata) + assert_equal({m1.unique_id => m1}, @p.dequeued) + + assert !m1.purged + + @p.purge_chunk(m1.unique_id) + assert m1.purged + + assert_equal [@dm0,@dm1], @p.queue.map(&:metadata) + assert_equal({}, @p.dequeued) + assert_equal [@dm2,@dm3,@dm0,@dm1], @p.metadata_list + end + + test '#purge_chunk removes an argument metadata from metadata_list if no chunks exist on stage or in queue' do + assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal({}, @p.dequeued) + assert_equal [@dm2,@dm3,@dm0,@dm1], @p.metadata_list + + m0 = @p.dequeue_chunk + + assert_equal [@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal({m0.unique_id => m0}, @p.dequeued) + + assert !m0.purged + + @p.purge_chunk(m0.unique_id) + assert m0.purged + + assert_equal [@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal({}, @p.dequeued) + assert_equal [@dm2,@dm3,@dm1], @p.metadata_list + end + + test '#takeback_chunk returns false if specified chunk_id is already purged' do + assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal({}, @p.dequeued) + assert_equal [@dm2,@dm3,@dm0,@dm1], @p.metadata_list + + m0 = @p.dequeue_chunk + + assert_equal [@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal({m0.unique_id => m0}, @p.dequeued) + + assert !m0.purged + + @p.purge_chunk(m0.unique_id) + assert m0.purged + + assert_equal [@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal({}, @p.dequeued) + assert_equal [@dm2,@dm3,@dm1], @p.metadata_list + + assert !@p.takeback_chunk(m0.unique_id) + + assert_equal [@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal({}, @p.dequeued) + assert_equal [@dm2,@dm3,@dm1], @p.metadata_list + end + + test '#clear_queue! removes all chunks in queue, but leaves staged chunks' do + qchunks = @p.queue.dup + + assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal 2, @p.stage.size + assert_equal({}, @p.dequeued) + + @p.clear_queue! + + assert_equal [], @p.queue + assert_equal 0, @p.queue_size + assert_equal 2, @p.stage.size + assert_equal({}, @p.dequeued) + + assert{ qchunks.all?{ |c| c.purged } } + end + + test '#emit returns immediately if argument data is empty array' do + assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal [@dm2,@dm3], @p.stage.keys + + m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) + + @p.emit(m, []) + + assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal [@dm2,@dm3], @p.stage.keys + end + + test '#emit raises BufferOverflowError if buffer is not storable' do + @p.stage_size = 256 * 1024 * 1024 + @p.queue_size = 256 * 1024 * 1024 + + m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) + + assert_raise Fluent::Plugin::Buffer::BufferOverflowError do + @p.emit(m, ["x" * 256]) + end + end + + test '#emit stores data into an existing chunk with metadata specified' do + assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal [@dm2,@dm3], @p.stage.keys + + dm3data = @p.stage[@dm3].read.dup + prev_stage_size = @p.stage_size + + assert_equal 1, @p.stage[@dm3].append_count + + @p.emit(@dm3, ["x" * 256, "y" * 256, "z" * 256]) + + assert_equal 2, @p.stage[@dm3].append_count + assert_equal (dm3data + ("x" * 256) + ("y" * 256) + ("z" * 256)), @p.stage[@dm3].read + assert_equal (prev_stage_size + 768), @p.stage_size + + assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal [@dm2,@dm3], @p.stage.keys + end + + test '#emit creates new chunk and store data into it if there are no chunks for specified metadata' do + assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal [@dm2,@dm3], @p.stage.keys + + prev_stage_size = @p.stage_size + + m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) + + @p.emit(m, ["x" * 256, "y" * 256, "z" * 256]) + + assert_equal 1, @p.stage[m].append_count + assert_equal ("x" * 256 + "y" * 256 + "z" * 256), @p.stage[m].read + assert_equal (prev_stage_size + 768), @p.stage_size + + assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal [@dm2,@dm3,m], @p.stage.keys + end + + test '#emit tries to enqueue and store data into a new chunk if existing chunk is full' do + assert_equal 8 * 1024 * 1024, @p.chunk_bytes_limit + + assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal [@dm2,@dm3], @p.stage.keys + + m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) + + row = "x" * 1024 * 1024 + @p.emit(m, [row] * 8) + + assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal [@dm2,@dm3,m], @p.stage.keys + assert_equal 1, @p.stage[m].append_count + + prev_stage_size = @p.stage_size + + @p.emit(m, [row]) + + assert_equal [@dm0,@dm1,@dm1,m], @p.queue.map(&:metadata) + assert_equal [@dm2,@dm3,m], @p.stage.keys + assert_equal 1, @p.stage[m].append_count + assert_equal 1024*1024, @p.stage[m].size + assert_equal 3, @p.queue.last.append_count # 1 -> emit (2) -> emit_step_by_step (3) + assert @p.queue.last.rollbacked + end + + test '#emit rollbacks if commit raises errors' do + assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal [@dm2,@dm3], @p.stage.keys + + m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i) + + row = "x" * 1024 + @p.emit(m, [row] * 8) + + assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal [@dm2,@dm3,m], @p.stage.keys + + target_chunk = @p.stage[m] + + assert_equal 1, target_chunk.append_count + assert !target_chunk.rollbacked + + (class << target_chunk; self; end).module_eval do + define_method(:commit){ raise "yay" } + end + + assert_raise "yay" do + @p.emit(m, [row]) + end + + assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal [@dm2,@dm3,m], @p.stage.keys + + assert_equal 2, target_chunk.append_count + assert target_chunk.rollbacked + assert_equal row * 8, target_chunk.read + end + end + + sub_test_case 'with configuration for test with lower limits' do + setup do + @p = create_buffer({"chunk_bytes_limit" => 1024, "total_bytes_limit" => 10240}) + @dm0 = dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil) + @dm1 = dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil) + @dm2 = dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil) + @dm3 = dm3 = create_metadata(Time.parse('2016-04-11 16:30:00 +0000').to_i, nil, nil) + (class << @p; self; end).module_eval do + define_method(:resume) { + staged = { + dm2 => create_chunk(dm2, ["b" * 128] * 7), + dm3 => create_chunk(dm3, ["c" * 128] * 5), + } + queued = [ + create_chunk(dm0, ["0" * 128] * 8), + create_chunk(dm0, ["0" * 128] * 8), + create_chunk(dm0, ["0" * 128] * 8), + create_chunk(dm0, ["0" * 128] * 8), + create_chunk(dm0, ["0" * 128] * 8), + create_chunk(dm1, ["a" * 128] * 8), + create_chunk(dm1, ["a" * 128] * 8), + create_chunk(dm1, ["a" * 128] * 8), # 8 + create_chunk(dm1, ["a" * 128] * 3), + ] + return staged, queued + } + end + @p.start + end + + test '#storable? returns false when too many data exist' do + assert_equal [@dm0,@dm0,@dm0,@dm0,@dm0,@dm1,@dm1,@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal [@dm2,@dm3], @p.stage.keys + + assert_equal 128*8*8+128*3, @p.queue_size + assert_equal 128*7+128*5, @p.stage_size + + assert @p.storable? + + dm3 = @p.metadata(timekey: @dm3.timekey) + @p.emit(dm3, ["c" * 128]) + + assert_equal 10240, (@p.stage_size + @p.queue_size) + assert !@p.storable? + end + + test '#size_over? returns false if chunk size is bigger than limit' do + m = create_metadata(Time.parse('2016-04-11 16:40:00 +0000').to_i) + + c1 = create_chunk(m, ["a" * 128] * 8) + assert !@p.size_over?(c1) + + c2 = create_chunk(m, ["a" * 128] * 9) + assert @p.size_over?(c2) + + c3 = create_chunk(m, ["a" * 128] * 8 + ["a"]) + assert @p.size_over?(c3) + end + + test '#emit raises BufferChunkOverflowError if incoming data is bigger than chunk bytes limit' do + assert_equal [@dm0,@dm0,@dm0,@dm0,@dm0,@dm1,@dm1,@dm1,@dm1], @p.queue.map(&:metadata) + assert_equal [@dm2,@dm3], @p.stage.keys + + m = create_metadata(Time.parse('2016-04-11 16:40:00 +0000').to_i) + + assert_raise Fluent::Plugin::Buffer::BufferChunkOverflowError do + @p.emit(m, ["a" * 128] * 9) + end + end + end + + sub_test_case 'with configuration includes chunk_records_limit' do + setup do + @p = create_buffer({"chunk_bytes_limit" => 1024, "total_bytes_limit" => 10240, "chunk_records_limit" => 6}) + @dm0 = dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil) + @dm1 = dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil) + @dm2 = dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil) + @dm3 = dm3 = create_metadata(Time.parse('2016-04-11 16:30:00 +0000').to_i, nil, nil) + (class << @p; self; end).module_eval do + define_method(:resume) { + staged = { + dm2 => create_chunk(dm2, ["b" * 128] * 1), + dm3 => create_chunk(dm3, ["c" * 128] * 2), + } + queued = [ + create_chunk(dm0, ["0" * 128] * 6), + create_chunk(dm1, ["a" * 128] * 6), + create_chunk(dm1, ["a" * 128] * 6), + create_chunk(dm1, ["a" * 128] * 3), + ] + return staged, queued + } + end + @p.start + end + + test '#size_over? returns false if too many records exists in a chunk even if its bytes is less than limit' do + assert_equal 6, @p.chunk_records_limit + + m = create_metadata(Time.parse('2016-04-11 16:40:00 +0000').to_i) + + c1 = create_chunk(m, ["a" * 128] * 6) + assert_equal 6, c1.records + assert !@p.size_over?(c1) + + c2 = create_chunk(m, ["a" * 128] * 7) + assert @p.size_over?(c2) + + c3 = create_chunk(m, ["a" * 128] * 6 + ["a"]) + assert @p.size_over?(c3) + end + end + + sub_test_case 'with configuration includes queue_length_limit' do + setup do + @p = create_buffer({"chunk_bytes_limit" => 1024, "total_bytes_limit" => 10240, "queue_length_limit" => 5}) + @dm0 = dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil) + @dm1 = dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil) + @dm2 = dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil) + @dm3 = dm3 = create_metadata(Time.parse('2016-04-11 16:30:00 +0000').to_i, nil, nil) + (class << @p; self; end).module_eval do + define_method(:resume) { + staged = { + dm2 => create_chunk(dm2, ["b" * 128] * 1), + dm3 => create_chunk(dm3, ["c" * 128] * 2), + } + queued = [ + create_chunk(dm0, ["0" * 128] * 6), + create_chunk(dm1, ["a" * 128] * 6), + create_chunk(dm1, ["a" * 128] * 6), + create_chunk(dm1, ["a" * 128] * 3), + ] + return staged, queued + } + end + @p.start + end + + test '#configure will overwrite standard configuration if queue_length_limit' do + assert_equal 1024, @p.chunk_bytes_limit + assert_equal 5, @p.queue_length_limit + assert_equal (1024*5), @p.total_bytes_limit + end + end + +end diff --git a/test/plugin/test_buffer_chunk.rb b/test/plugin/test_buffer_chunk.rb new file mode 100644 index 0000000000..ed9b0560af --- /dev/null +++ b/test/plugin/test_buffer_chunk.rb @@ -0,0 +1,90 @@ +require_relative '../helper' +require 'fluent/plugin/buffer/chunk' + +class BufferChunkTest < Test::Unit::TestCase + sub_test_case 'blank buffer chunk' do + test 'has generated unique id, given metadata, created_at and modified_at' do + meta = Object.new + chunk = Fluent::Plugin::Buffer::Chunk.new(meta) + assert{ chunk.unique_id.bytesize == 16 } + assert{ chunk.metadata.object_id == meta.object_id } + assert{ chunk.created_at.is_a? Time } + assert{ chunk.modified_at.is_a? Time } + end + + test 'has many methods for chunks, but not implemented' do + meta = Object.new + chunk = Fluent::Plugin::Buffer::Chunk.new(meta) + + assert chunk.respond_to?(:append) + assert chunk.respond_to?(:commit) + assert chunk.respond_to?(:rollback) + assert chunk.respond_to?(:size) + assert chunk.respond_to?(:records) + assert chunk.respond_to?(:empty?) + assert chunk.respond_to?(:close) + assert chunk.respond_to?(:purge) + assert chunk.respond_to?(:read) + assert chunk.respond_to?(:open) + assert chunk.respond_to?(:write_to) + assert chunk.respond_to?(:msgpack_each) + assert_raise(NotImplementedError){ chunk.append(nil) } + assert_raise(NotImplementedError){ chunk.commit } + assert_raise(NotImplementedError){ chunk.rollback } + assert_raise(NotImplementedError){ chunk.size } + assert_raise(NotImplementedError){ chunk.records } + assert_raise(NotImplementedError){ chunk.empty? } + assert_raise(NotImplementedError){ chunk.close } + assert_raise(NotImplementedError){ chunk.purge } + assert_raise(NotImplementedError){ chunk.read } + assert_raise(NotImplementedError){ chunk.open(){} } + assert_raise(NotImplementedError){ chunk.write_to(nil) } + assert_raise(NotImplementedError){ chunk.msgpack_each(){|v| v} } + end + end + + class TestChunk < Fluent::Plugin::Buffer::Chunk + attr_accessor :data + def initialize(meta) + super + @data = '' + end + def size + @data.size + end + def open + require 'stringio' + io = StringIO.new(@data) + yield io + end + end + + sub_test_case 'minimum chunk implements #size and #open' do + test 'can respond to #empty? correctly' do + c = TestChunk.new(Object.new) + assert_equal 0, c.size + assert c.empty? + end + + test 'can write its contents to io object' do + c = TestChunk.new(Object.new) + c.data << "my data\nyour data\n" + io = StringIO.new + c.write_to(io) + assert "my data\nyour data\n", io.to_s + end + + test 'can feed objects into blocks with unpacking msgpack' do + require 'msgpack' + c = TestChunk.new(Object.new) + c.data << MessagePack.pack(['my data', 1]) + c.data << MessagePack.pack(['your data', 2]) + ary = [] + c.msgpack_each do |obj| + ary << obj + end + assert_equal ['my data', 1], ary[0] + assert_equal ['your data', 2], ary[1] + end + end +end diff --git a/test/plugin/test_buffer_file_chunk.rb b/test/plugin/test_buffer_file_chunk.rb new file mode 100644 index 0000000000..76ff0eca76 --- /dev/null +++ b/test/plugin/test_buffer_file_chunk.rb @@ -0,0 +1,694 @@ +require_relative '../helper' +require 'fluent/plugin/buffer/file_chunk' +require 'fluent/unique_id' + +require 'fileutils' +require 'msgpack' +require 'time' +require 'timecop' + +class BufferFileChunkTest < Test::Unit::TestCase + setup do + @klass = Fluent::Plugin::Buffer::FileChunk + @chunkdir = File.expand_path('../../tmp/buffer_file_chunk', __FILE__) + FileUtils.rm_r @chunkdir rescue nil + FileUtils.mkdir_p @chunkdir + end + teardown do + Timecop.return + end + + Metadata = Struct.new(:timekey, :tag, :variables) + def gen_metadata(timekey: nil, tag: nil, variables: nil) + Metadata.new(timekey, tag, variables) + end + + def read_metadata_file(path) + File.open(path){|f| MessagePack.unpack(f.read, symbolize_keys: true) } + end + + def gen_path(path) + File.join(@chunkdir, path) + end + + def gen_test_chunk_id + require 'time' + now = Time.parse('2016-04-07 14:31:33 +0900') + u1 = ((now.to_i * 1000 * 1000 + now.usec) << 12 | 1725) # 1725 is one of `rand(0xfff)` + u3 = 2979763054 # one of rand(0xffffffff) + u4 = 438020492 # ditto + [u1 >> 32, u1 & 0xffffffff, u3, u4].pack('NNNN') + # unique_id.unpack('N*').map{|n| n.to_s(16)}.join => "52fde6425d7406bdb19b936e1a1ba98c" + end + + def hex_id(id) + id.unpack('N*').map{|n| n.to_s(16)}.join + end + + sub_test_case 'classmethods' do + data( + correct_staged: ['/mydir/mypath/myfile.b00ff.log', :staged], + correct_queued: ['/mydir/mypath/myfile.q00ff.log', :queued], + incorrect_staged: ['/mydir/mypath/myfile.b00ff.log/unknown', :queued], + incorrect_queued: ['/mydir/mypath/myfile.q00ff.log/unknown', :queued], + ) + test 'can .assume_chunk_state' do |data| + path, expected = data + assert_equal expected, @klass.assume_chunk_state(path) + end + + test '.generate_stage_chunk_path generates path with staged mark & chunk unique_id' do + assert_equal gen_path("mychunk.b52fde6425d7406bdb19b936e1a1ba98c.log"), @klass.generate_stage_chunk_path(gen_path("mychunk.*.log"), gen_test_chunk_id) + assert_raise "BUG: buffer chunk path on stage MUST have '.*.'" do + @klass.generate_stage_chunk_path(gen_path("mychunk.log"), gen_test_chunk_id) + end + assert_raise "BUG: buffer chunk path on stage MUST have '.*.'" do + @klass.generate_stage_chunk_path(gen_path("mychunk.*"), gen_test_chunk_id) + end + assert_raise "BUG: buffer chunk path on stage MUST have '.*.'" do + @klass.generate_stage_chunk_path(gen_path("*.log"), gen_test_chunk_id) + end + end + + test '.generate_queued_chunk_path generates path with enqueued mark for staged chunk path' do + assert_equal( + gen_path("mychunk.q52fde6425d7406bdb19b936e1a1ba98c.log"), + @klass.generate_queued_chunk_path(gen_path("mychunk.b52fde6425d7406bdb19b936e1a1ba98c.log"), gen_test_chunk_id) + ) + end + + test '.generate_queued_chunk_path generates special path with chunk unique_id for non staged chunk path' do + assert_equal( + gen_path("mychunk.log.q52fde6425d7406bdb19b936e1a1ba98c.chunk"), + @klass.generate_queued_chunk_path(gen_path("mychunk.log"), gen_test_chunk_id) + ) + assert_equal( + gen_path("mychunk.q55555555555555555555555555555555.log.q52fde6425d7406bdb19b936e1a1ba98c.chunk"), + @klass.generate_queued_chunk_path(gen_path("mychunk.q55555555555555555555555555555555.log"), gen_test_chunk_id) + ) + end + + test '.unique_id_from_path recreates unique_id from file path to assume unique_id for v0.12 chunks' do + assert_equal gen_test_chunk_id, @klass.unique_id_from_path(gen_path("mychunk.q52fde6425d7406bdb19b936e1a1ba98c.log")) + end + end + + sub_test_case 'newly created chunk' do + setup do + @chunk_path = File.join(@chunkdir, 'test.*.log') + @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @chunk_path, :create) + end + + def gen_chunk_path(prefix, unique_id) + File.join(@chunkdir, "test.#{prefix}#{Fluent::UniqueId.hex(unique_id)}.log") + end + + teardown do + if @c + @c.purge rescue nil + end + if File.exist? @chunk_path + File.unlink @chunk_path + end + end + + test 'creates new files for chunk and metadata with specified path & permission' do + assert{ @c.unique_id.size == 16 } + assert_equal gen_chunk_path('b', @c.unique_id), @c.path + + assert File.exist?(gen_chunk_path('b', @c.unique_id)) + assert{ File.stat(gen_chunk_path('b', @c.unique_id)).mode.to_s(8).end_with?(@klass.const_get('FILE_PERMISSION').to_s(8)) } + + assert File.exist?(gen_chunk_path('b', @c.unique_id) + '.meta') + assert{ File.stat(gen_chunk_path('b', @c.unique_id) + '.meta').mode.to_s(8).end_with?(@klass.const_get('FILE_PERMISSION').to_s(8)) } + + assert_equal :staged, @c.state + assert @c.empty? + end + + test 'can #append, #commit and #read it' do + assert @c.empty? + + d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} + d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} + data = [d1.to_json + "\n", d2.to_json + "\n"] + @c.append(data) + @c.commit + + content = @c.read + ds = content.split("\n").select{|d| !d.empty? } + + assert_equal 2, ds.size + assert_equal d1, JSON.parse(ds[0]) + assert_equal d2, JSON.parse(ds[1]) + + d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} + d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} + @c.append([d3.to_json + "\n", d4.to_json + "\n"]) + @c.commit + + content = @c.read + ds = content.split("\n").select{|d| !d.empty? } + + assert_equal 4, ds.size + assert_equal d1, JSON.parse(ds[0]) + assert_equal d2, JSON.parse(ds[1]) + assert_equal d3, JSON.parse(ds[2]) + assert_equal d4, JSON.parse(ds[3]) + end + + test 'has its contents in binary (ascii-8bit)' do + data1 = "aaa bbb ccc".force_encoding('utf-8') + @c.append([data1]) + @c.commit + assert_equal Encoding::ASCII_8BIT, @c.instance_eval{ @chunk.external_encoding } + + content = @c.read + assert_equal Encoding::ASCII_8BIT, content.encoding + end + + test 'has #size and #records' do + assert @c.empty? + + d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} + d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} + data = [d1.to_json + "\n", d2.to_json + "\n"] + @c.append(data) + + assert_equal (d1.to_json + "\n" + d2.to_json + "\n").size, @c.size + assert_equal 2, @c.records + + @c.commit + + assert_equal (d1.to_json + "\n" + d2.to_json + "\n").size, @c.size + assert_equal 2, @c.records + + first_size = @c.size + + d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} + d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} + @c.append([d3.to_json + "\n", d4.to_json + "\n"]) + + assert_equal first_size + (d3.to_json + "\n" + d4.to_json + "\n").size, @c.size + assert_equal 4, @c.records + + @c.commit + + assert_equal first_size + (d3.to_json + "\n" + d4.to_json + "\n").size, @c.size + assert_equal 4, @c.records + end + + test 'can #rollback to revert non-committed data' do + assert @c.empty? + + d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} + d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} + data = [d1.to_json + "\n", d2.to_json + "\n"] + @c.append(data) + + assert_equal (d1.to_json + "\n" + d2.to_json + "\n").size, @c.size + assert_equal 2, @c.records + + @c.rollback + + assert @c.empty? + + assert_equal '', File.open(@c.path){|f| f.read } + + d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} + d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} + data = [d1.to_json + "\n", d2.to_json + "\n"] + @c.append(data) + @c.commit + + assert_equal (d1.to_json + "\n" + d2.to_json + "\n").size, @c.size + assert_equal 2, @c.records + + first_size = @c.size + + d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} + d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} + @c.append([d3.to_json + "\n", d4.to_json + "\n"]) + + assert_equal first_size + (d3.to_json + "\n" + d4.to_json + "\n").size, @c.size + assert_equal 4, @c.records + + @c.rollback + + assert_equal first_size, @c.size + assert_equal 2, @c.records + + assert_equal (d1.to_json + "\n" + d2.to_json + "\n"), File.open(@c.path){|f| f.read } + end + + test 'can store its data by #close' do + d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} + d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} + data = [d1.to_json + "\n", d2.to_json + "\n"] + @c.append(data) + @c.commit + d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} + d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} + @c.append([d3.to_json + "\n", d4.to_json + "\n"]) + @c.commit + + content = @c.read + + unique_id = @c.unique_id + records = @c.records + created_at = @c.created_at + modified_at = @c.modified_at + + @c.close + + assert_equal content, File.open(@c.path){|f| f.read } + + stored_meta = { + timekey: nil, tag: nil, variables: nil, + id: unique_id, + r: records, + c: @c.created_at.to_i, + m: @c.modified_at.to_i, + } + + assert_equal stored_meta, read_metadata_file(@c.path + '.meta') + end + + test 'deletes all data by #purge' do + d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} + d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} + data = [d1.to_json + "\n", d2.to_json + "\n"] + @c.append(data) + @c.commit + d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} + d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} + @c.append([d3.to_json + "\n", d4.to_json + "\n"]) + @c.commit + + @c.purge + + assert @c.empty? + assert_equal 0, @c.size + assert_equal 0, @c.records + + assert !File.exist?(@c.path) + assert !File.exist?(@c.path + '.meta') + end + + test 'can #open its contents as io' do + d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} + d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} + data = [d1.to_json + "\n", d2.to_json + "\n"] + @c.append(data) + @c.commit + d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} + d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} + @c.append([d3.to_json + "\n", d4.to_json + "\n"]) + @c.commit + + lines = [] + @c.open do |io| + assert io + io.readlines.each do |l| + lines << l + end + end + + assert_equal d1.to_json + "\n", lines[0] + assert_equal d2.to_json + "\n", lines[1] + assert_equal d3.to_json + "\n", lines[2] + assert_equal d4.to_json + "\n", lines[3] + end + + test 'can refer system config for file permission' do + chunk_path = File.join(@chunkdir, 'testperm.*.log') + Fluent::SystemConfig.overwrite_system_config("file_permission" => "600") do + c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, chunk_path, :create) + assert{ File.stat(c.path).mode.to_s(8).end_with?('600') } + assert{ File.stat(c.path + '.meta').mode.to_s(8).end_with?('600') } + end + end + + test '#write_metadata tries to store metadata on file' do + d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} + d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} + data = [d1.to_json + "\n", d2.to_json + "\n"] + @c.append(data) + @c.commit + + expected = { + timekey: nil, tag: nil, variables: nil, + id: @c.unique_id, + r: @c.records, + c: @c.created_at.to_i, + m: @c.modified_at.to_i, + } + assert_equal expected, read_metadata_file(@c.path + '.meta') + + d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} + d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} + @c.append([d3.to_json + "\n", d4.to_json + "\n"]) + # append does write_metadata + + dummy_now = Time.parse('2016-04-07 16:59:59 +0900') + Timecop.freeze(dummy_now) + @c.write_metadata + + expected = { + timekey: nil, tag: nil, variables: nil, + id: @c.unique_id, + r: @c.records, + c: @c.created_at.to_i, + m: dummy_now.to_i, + } + assert_equal expected, read_metadata_file(@c.path + '.meta') + + @c.commit + + expected = { + timekey: nil, tag: nil, variables: nil, + id: @c.unique_id, + r: @c.records, + c: @c.created_at.to_i, + m: @c.modified_at.to_i, + } + assert_equal expected, read_metadata_file(@c.path + '.meta') + + content = @c.read + + unique_id = @c.unique_id + records = @c.records + created_at = @c.created_at + modified_at = @c.modified_at + + @c.close + + assert_equal content, File.open(@c.path){|f| f.read } + + stored_meta = { + timekey: nil, tag: nil, variables: nil, + id: unique_id, + r: records, + c: @c.created_at.to_i, + m: @c.modified_at.to_i, + } + + assert_equal stored_meta, read_metadata_file(@c.path + '.meta') + end + end + + sub_test_case 'chunk with file for staged chunk' do + setup do + @chunk_id = gen_test_chunk_id + @chunk_path = File.join(@chunkdir, "test_staged.b#{hex_id(@chunk_id)}.log") + @enqueued_path = File.join(@chunkdir, "test_staged.q#{hex_id(@chunk_id)}.log") + + @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} + @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} + @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'} + @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'} + @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + "\n" }.join + File.open(@chunk_path, 'w') do |f| + f.write @d + end + + @metadata = { + timekey: nil, tag: 'testing', variables: {k: "x"}, + id: @chunk_id, + r: 4, + c: Time.parse('2016-04-07 17:44:00 +0900').to_i, + m: Time.parse('2016-04-07 17:44:13 +0900').to_i, + } + File.open(@chunk_path + '.meta', 'w') do |f| + f.write @metadata.to_msgpack + end + + @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @chunk_path, :staged) + end + + teardown do + if @c + @c.purge rescue nil + end + [@chunk_path, @chunk_path + '.meta', @enqueued_path, @enqueued_path + '.meta'].each do |path| + File.unlink path if File.exist? path + end + end + + test 'can load as staged chunk from file with metadata' do + assert_equal @chunk_path, @c.path + assert_equal :staged, @c.state + + assert_nil @c.metadata.timekey + assert_equal 'testing', @c.metadata.tag + assert_equal({k: "x"}, @c.metadata.variables) + + assert_equal 4, @c.records + assert_equal Time.parse('2016-04-07 17:44:00 +0900'), @c.created_at + assert_equal Time.parse('2016-04-07 17:44:13 +0900'), @c.modified_at + + content = @c.read + assert_equal @d, content + end + + test 'can be enqueued' do + stage_path = @c.path + queue_path = @enqueued_path + assert File.exist?(stage_path) + assert File.exist?(stage_path + '.meta') + assert !File.exist?(queue_path) + assert !File.exist?(queue_path + '.meta') + + @c.enqueued! + + assert_equal queue_path, @c.path + + assert !File.exist?(stage_path) + assert !File.exist?(stage_path + '.meta') + assert File.exist?(queue_path) + assert File.exist?(queue_path + '.meta') + + assert_nil @c.metadata.timekey + assert_equal 'testing', @c.metadata.tag + assert_equal({k: "x"}, @c.metadata.variables) + + assert_equal 4, @c.records + assert_equal Time.parse('2016-04-07 17:44:00 +0900'), @c.created_at + assert_equal Time.parse('2016-04-07 17:44:13 +0900'), @c.modified_at + + assert_equal @d, File.open(@c.path){|f| f.read } + assert_equal @metadata, read_metadata_file(@c.path + '.meta') + end + + test '#write_metadata tries to store metadata on file with non-committed data' do + d5 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'} + d5s = d5.to_json + "\n" + @c.append([d5s]) + + metadata = { + timekey: nil, tag: 'testing', variables: {k: "x"}, + id: @chunk_id, + r: 4, + c: Time.parse('2016-04-07 17:44:00 +0900').to_i, + m: Time.parse('2016-04-07 17:44:13 +0900').to_i, + } + assert_equal metadata, read_metadata_file(@c.path + '.meta') + + @c.write_metadata + + metadata = { + timekey: nil, tag: 'testing', variables: {k: "x"}, + id: @chunk_id, + r: 5, + c: Time.parse('2016-04-07 17:44:00 +0900').to_i, + m: Time.parse('2016-04-07 17:44:38 +0900').to_i, + } + + dummy_now = Time.parse('2016-04-07 17:44:38 +0900') + Timecop.freeze(dummy_now) + @c.write_metadata + + assert_equal metadata, read_metadata_file(@c.path + '.meta') + end + + test '#file_rename can rename chunk files even in windows, and call callback with file size' do + data = "aaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbccccccccccccccccccccccccccccc" + + testing_file1 = gen_path('rename1.test') + testing_file2 = gen_path('rename2.test') + f = File.open(testing_file1, 'w', @c.permission) + f.set_encoding(Encoding::ASCII_8BIT) + f.sync = true + f.binmode + f.write data + pos = f.pos + + assert f.binmode? + assert f.sync + assert_equal data.size, f.size + + io = nil + @c.file_rename(f, testing_file1, testing_file2, ->(new_io){ io = new_io }) + assert io + if Fluent.windows? + assert{ f != io } + else + assert_equal f, io + end + assert_equal Encoding::ASCII_8BIT, io.external_encoding + assert io.sync + assert io.binmode? + assert_equal data.size, io.size + + assert_equal pos, io.pos + + assert_equal '', io.read + + io.rewind + assert_equal data, io.read + end + end + + sub_test_case 'chunk with file for enqueued chunk' do + setup do + @chunk_id = gen_test_chunk_id + @enqueued_path = File.join(@chunkdir, "test_staged.q#{hex_id(@chunk_id)}.log") + + @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} + @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} + @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'} + @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'} + @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + "\n" }.join + File.open(@enqueued_path, 'w') do |f| + f.write @d + end + + @dummy_timekey = Time.parse('2016-04-07 17:40:00 +0900').to_i + + @metadata = { + timekey: @dummy_timekey, tag: 'testing', variables: {k: "x"}, + id: @chunk_id, + r: 4, + c: Time.parse('2016-04-07 17:44:00 +0900').to_i, + m: Time.parse('2016-04-07 17:44:13 +0900').to_i, + } + File.open(@enqueued_path + '.meta', 'w') do |f| + f.write @metadata.to_msgpack + end + + @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @enqueued_path, :queued) + end + + teardown do + if @c + @c.purge rescue nil + end + [@enqueued_path, @enqueued_path + '.meta'].each do |path| + File.unlink path if File.exist? path + end + end + + test 'can load as queued chunk (read only) with metadata' do + assert @c + assert_equal @chunk_id, @c.unique_id + assert_equal :queued, @c.state + assert_equal gen_metadata(timekey: @dummy_timekey, tag: 'testing', variables: {k: "x"}), @c.metadata + assert_equal Time.at(@metadata[:c]), @c.created_at + assert_equal Time.at(@metadata[:m]), @c.modified_at + assert_equal @metadata[:r], @c.records + assert_equal @d.size, @c.size + assert_equal @d, @c.read + + assert_raise "BUG: appending to non-staged chunk, now 'queued'" do + @c.append(["queued chunk is read only"]) + end + assert_raise IOError do + @c.instance_eval{ @chunk }.write "chunk io is opened as read only" + end + end + end + + sub_test_case 'chunk with queued chunk file of v0.12, without metadata' do + setup do + @chunk_id = gen_test_chunk_id + @chunk_path = File.join(@chunkdir, "test_v12.2016040811.q#{hex_id(@chunk_id)}.log") + + @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} + @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} + @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'} + @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'} + @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + "\n" }.join + File.open(@chunk_path, 'w') do |f| + f.write @d + end + + @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @chunk_path, :queued) + end + + teardown do + if @c + @c.purge rescue nil + end + File.unlink @chunk_path if File.exist? @chunk_path + end + + test 'can load as queued chunk from file without metadata' do + assert @c + assert_equal :queued, @c.state + assert_equal @chunk_id, @c.unique_id + assert_equal gen_metadata, @c.metadata + assert_equal @d.size, @c.size + assert_equal 0, @c.records + assert_equal @d, @c.read + + assert_raise "BUG: appending to non-staged chunk, now 'queued'" do + @c.append(["queued chunk is read only"]) + end + assert_raise IOError do + @c.instance_eval{ @chunk }.write "chunk io is opened as read only" + end + end + end + + sub_test_case 'chunk with staged chunk file of v0.12, without metadata' do + setup do + @chunk_id = gen_test_chunk_id + @chunk_path = File.join(@chunkdir, "test_v12.2016040811.b#{hex_id(@chunk_id)}.log") + + @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} + @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} + @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'} + @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'} + @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + "\n" }.join + File.open(@chunk_path, 'w') do |f| + f.write @d + end + + @c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, @chunk_path, :staged) + end + + teardown do + if @c + @c.purge rescue nil + end + File.unlink @chunk_path if File.exist? @chunk_path + end + + test 'can load as queued chunk from file without metadata even if it was loaded as staged chunk' do + assert @c + assert_equal :queued, @c.state + assert_equal @chunk_id, @c.unique_id + assert_equal gen_metadata, @c.metadata + assert_equal @d.size, @c.size + assert_equal 0, @c.records + assert_equal @d, @c.read + + assert_raise "BUG: appending to non-staged chunk, now 'queued'" do + @c.append(["queued chunk is read only"]) + end + assert_raise IOError do + @c.instance_eval{ @chunk }.write "chunk io is opened as read only" + end + end + end +end diff --git a/test/plugin/test_buffer_memory_chunk.rb b/test/plugin/test_buffer_memory_chunk.rb new file mode 100644 index 0000000000..62b38ddfd6 --- /dev/null +++ b/test/plugin/test_buffer_memory_chunk.rb @@ -0,0 +1,193 @@ +require_relative '../helper' +require 'fluent/plugin/buffer/memory_chunk' + +require 'json' + +class BufferMemoryChunkTest < Test::Unit::TestCase + setup do + @c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new) + end + + test 'has blank chunk initially' do + assert @c.empty? + assert_equal '', @c.instance_eval{ @chunk } + assert_equal 0, @c.instance_eval{ @chunk_bytes } + assert_equal 0, @c.instance_eval{ @adding_bytes } + assert_equal 0, @c.instance_eval{ @adding_records } + end + + test 'can #append, #commit and #read it' do + assert @c.empty? + + d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} + d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} + data = [d1.to_json + "\n", d2.to_json + "\n"] + @c.append(data) + @c.commit + + content = @c.read + ds = content.split("\n").select{|d| !d.empty? } + + assert_equal 2, ds.size + assert_equal d1, JSON.parse(ds[0]) + assert_equal d2, JSON.parse(ds[1]) + + d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} + d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} + @c.append([d3.to_json + "\n", d4.to_json + "\n"]) + @c.commit + + content = @c.read + ds = content.split("\n").select{|d| !d.empty? } + + assert_equal 4, ds.size + assert_equal d1, JSON.parse(ds[0]) + assert_equal d2, JSON.parse(ds[1]) + assert_equal d3, JSON.parse(ds[2]) + assert_equal d4, JSON.parse(ds[3]) + end + + test 'has its contents in binary (ascii-8bit)' do + data1 = "aaa bbb ccc".force_encoding('utf-8') + @c.append([data1]) + @c.commit + assert_equal Encoding::ASCII_8BIT, @c.instance_eval{ @chunk.encoding } + + content = @c.read + assert_equal Encoding::ASCII_8BIT, content.encoding + end + + test 'has #size and #records' do + assert @c.empty? + + d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} + d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} + data = [d1.to_json + "\n", d2.to_json + "\n"] + @c.append(data) + + assert_equal (d1.to_json + "\n" + d2.to_json + "\n").size, @c.size + assert_equal 2, @c.records + + @c.commit + + assert_equal (d1.to_json + "\n" + d2.to_json + "\n").size, @c.size + assert_equal 2, @c.records + + first_size = @c.size + + d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} + d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} + @c.append([d3.to_json + "\n", d4.to_json + "\n"]) + + assert_equal first_size + (d3.to_json + "\n" + d4.to_json + "\n").size, @c.size + assert_equal 4, @c.records + + @c.commit + + assert_equal first_size + (d3.to_json + "\n" + d4.to_json + "\n").size, @c.size + assert_equal 4, @c.records + end + + test 'can #rollback to revert non-committed data' do + assert @c.empty? + + d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} + d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} + data = [d1.to_json + "\n", d2.to_json + "\n"] + @c.append(data) + + assert_equal (d1.to_json + "\n" + d2.to_json + "\n").size, @c.size + assert_equal 2, @c.records + + @c.rollback + + assert @c.empty? + + assert @c.empty? + + d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} + d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} + data = [d1.to_json + "\n", d2.to_json + "\n"] + @c.append(data) + @c.commit + + assert_equal (d1.to_json + "\n" + d2.to_json + "\n").size, @c.size + assert_equal 2, @c.records + + first_size = @c.size + + d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} + d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} + @c.append([d3.to_json + "\n", d4.to_json + "\n"]) + + assert_equal first_size + (d3.to_json + "\n" + d4.to_json + "\n").size, @c.size + assert_equal 4, @c.records + + @c.rollback + + assert_equal first_size, @c.size + assert_equal 2, @c.records + end + + test 'does nothing for #close' do + d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} + d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} + data = [d1.to_json + "\n", d2.to_json + "\n"] + @c.append(data) + @c.commit + d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} + d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} + @c.append([d3.to_json + "\n", d4.to_json + "\n"]) + @c.commit + + content = @c.read + + @c.close + + assert_equal content, @c.read + end + + test 'deletes all data by #purge' do + d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} + d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} + data = [d1.to_json + "\n", d2.to_json + "\n"] + @c.append(data) + @c.commit + d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} + d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} + @c.append([d3.to_json + "\n", d4.to_json + "\n"]) + @c.commit + + @c.purge + + assert @c.empty? + assert_equal 0, @c.size + assert_equal 0, @c.records + assert_equal '', @c.read + end + + test 'can #open its contents as io' do + d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} + d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} + data = [d1.to_json + "\n", d2.to_json + "\n"] + @c.append(data) + @c.commit + d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'} + d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'} + @c.append([d3.to_json + "\n", d4.to_json + "\n"]) + @c.commit + + lines = [] + @c.open do |io| + assert io + io.readlines.each do |l| + lines << l + end + end + + assert_equal d1.to_json + "\n", lines[0] + assert_equal d2.to_json + "\n", lines[1] + assert_equal d3.to_json + "\n", lines[2] + assert_equal d4.to_json + "\n", lines[3] + end +end From 9ae1fc6ccb3d9ec38a95fef55745497168f03578 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 15:40:00 +0900 Subject: [PATCH 16/36] add tests for Output pluginn for v0.14 API --- test/plugin/test_output.rb | 439 +++++ test/plugin/test_output_as_buffered.rb | 1512 +++++++++++++++++ .../plugin/test_output_as_buffered_retries.rb | 786 +++++++++ .../test_output_as_buffered_secondary.rb | 766 +++++++++ 4 files changed, 3503 insertions(+) create mode 100644 test/plugin/test_output.rb create mode 100644 test/plugin/test_output_as_buffered.rb create mode 100644 test/plugin/test_output_as_buffered_retries.rb create mode 100644 test/plugin/test_output_as_buffered_secondary.rb diff --git a/test/plugin/test_output.rb b/test/plugin/test_output.rb new file mode 100644 index 0000000000..ef215e0f63 --- /dev/null +++ b/test/plugin/test_output.rb @@ -0,0 +1,439 @@ +require_relative '../helper' +require 'fluent/plugin/output' +require 'fluent/plugin/buffer' + +require 'json' +require 'time' +require 'timeout' + +module FluentPluginOutputTest + class DummyBareOutput < Fluent::Plugin::Output + def register(name, &block) + instance_variable_set("@#{name}", block) + end + end + class DummySyncOutput < DummyBareOutput + def process(tag, es) + @process ? @process.call(tag, es) : nil + end + end + class DummyAsyncOutput < DummyBareOutput + def format(tag, time, record) + @format ? @format.call(tag, time, record) : [tag, time, record].to_json + end + def write(chunk) + @write ? @write.call(chunk) : nil + end + end + class DummyDelayedOutput < DummyBareOutput + def format(tag, time, record) + @format ? @format.call(tag, time, record) : [tag, time, record].to_json + end + def try_write(chunk) + @try_write ? @try_write.call(chunk) : nil + end + end + class DummyFullFeatureOutput < DummyBareOutput + def prefer_buffered_processing + @prefer_buffered_processing ? @prefer_buffered_processing.call : false + end + def prefer_delayed_commit + @prefer_delayed_commit ? @prefer_delayed_commit.call : false + end + def process(tag, es) + @process ? @process.call(tag, es) : nil + end + def format(tag, time, record) + @format ? @format.call(tag, time, record) : [tag, time, record].to_json + end + def write(chunk) + @write ? @write.call(chunk) : nil + end + def try_write(chunk) + @try_write ? @try_write.call(chunk) : nil + end + end +end + +class OutputTest < Test::Unit::TestCase + def create_output(type=:full) + case type + when :bare then FluentPluginOutputTest::DummyBareOutput.new + when :sync then FluentPluginOutputTest::DummySyncOutput.new + when :buffered then FluentPluginOutputTest::DummyAsyncOutput.new + when :delayed then FluentPluginOutputTest::DummyDelayedOutput.new + when :full then FluentPluginOutputTest::DummyFullFeatureOutput.new + else + raise ArgumentError, "unknown type: #{type}" + end + end + def create_metadata(timekey: nil, tag: nil, variables: nil) + Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables) + end + def waiting(seconds) + Timeout.timeout(seconds) do + yield + end + end + + sub_test_case 'basic output feature' do + setup do + @i = create_output(:full) + end + + test '#implement? can return features for plugin instances' do + i1 = FluentPluginOutputTest::DummyBareOutput.new + assert !i1.implement?(:synchronous) + assert !i1.implement?(:buffered) + assert !i1.implement?(:delayed_commit) + + i2 = FluentPluginOutputTest::DummySyncOutput.new + assert i2.implement?(:synchronous) + assert !i2.implement?(:buffered) + assert !i2.implement?(:delayed_commit) + + i3 = FluentPluginOutputTest::DummyAsyncOutput.new + assert !i3.implement?(:synchronous) + assert i3.implement?(:buffered) + assert !i3.implement?(:delayed_commit) + + i4 = FluentPluginOutputTest::DummyDelayedOutput.new + assert !i4.implement?(:synchronous) + assert !i4.implement?(:buffered) + assert i4.implement?(:delayed_commit) + + i5 = FluentPluginOutputTest::DummyFullFeatureOutput.new + assert i5.implement?(:synchronous) + assert i5.implement?(:buffered) + assert i5.implement?(:delayed_commit) + end + + test 'plugin lifecycle for configure/start/stop/before_shutdown/shutdown/after_shutdown/close/terminate' do + assert !@i.configured? + @i.configure(config_element()) + assert @i.configured? + assert !@i.started? + @i.start + assert @i.started? + assert !@i.stopped? + @i.stop + assert @i.stopped? + assert !@i.before_shutdown? + @i.before_shutdown + assert @i.before_shutdown? + assert !@i.shutdown? + @i.shutdown + assert @i.shutdown? + assert !@i.after_shutdown? + @i.after_shutdown + assert @i.after_shutdown? + assert !@i.closed? + @i.close + assert @i.closed? + assert !@i.terminated? + @i.terminate + assert @i.terminated? + end + + test '#extract_placeholders does nothing if chunk key is not specified' do + @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')])) + assert !@i.chunk_key_time + assert !@i.chunk_key_tag + assert_equal [], @i.chunk_keys + tmpl = "/mypath/%Y/%m/%d/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail" + t = event_time('2016-04-11 20:30:00 +0900') + v = {key1: "value1", key2: "value2"} + m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v) + assert_equal tmpl, @i.extract_placeholders(tmpl, m) + end + + test '#extract_placeholders can extract time if time key and range are configured' do + @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {'timekey_range' => 60*30, 'timekey_zone' => "+0900"})])) + assert @i.chunk_key_time + assert !@i.chunk_key_tag + assert_equal [], @i.chunk_keys + tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail" + t = event_time('2016-04-11 20:30:00 +0900') + v = {key1: "value1", key2: "value2"} + m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v) + assert_equal "/mypath/2016/04/11/20-30/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail", @i.extract_placeholders(tmpl, m) + end + + test '#extract_placeholders can extract tag and parts of tag if tag is configured' do + @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag', {})])) + assert !@i.chunk_key_time + assert @i.chunk_key_tag + assert_equal [], @i.chunk_keys + tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail" + t = event_time('2016-04-11 20:30:00 +0900') + v = {key1: "value1", key2: "value2"} + m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v) + assert_equal "/mypath/%Y/%m/%d/%H-%M/fluentd.test.output/test/output/${key1}/${key2}/tail", @i.extract_placeholders(tmpl, m) + end + + test '#extract_placeholders can extract variables if variables are configured' do + @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'key1,key2', {})])) + assert !@i.chunk_key_time + assert !@i.chunk_key_tag + assert_equal ['key1','key2'], @i.chunk_keys + tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail" + t = event_time('2016-04-11 20:30:00 +0900') + v = {key1: "value1", key2: "value2"} + m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v) + assert_equal "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/value1/value2/tail", @i.extract_placeholders(tmpl, m) + end + + test '#extract_placeholders can extract all chunk keys if configured' do + @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,key1,key2', {'timekey_range' => 60*30, 'timekey_zone' => "+0900"})])) + assert @i.chunk_key_time + assert @i.chunk_key_tag + assert_equal ['key1','key2'], @i.chunk_keys + tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail" + t = event_time('2016-04-11 20:30:00 +0900') + v = {key1: "value1", key2: "value2"} + m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v) + assert_equal "/mypath/2016/04/11/20-30/fluentd.test.output/test/output/value1/value2/tail", @i.extract_placeholders(tmpl, m) + end + + test '#extract_placeholders removes out-of-range tag part and unknown variable placeholders' do + @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,key1,key2', {'timekey_range' => 60*30, 'timekey_zone' => "+0900"})])) + assert @i.chunk_key_time + assert @i.chunk_key_tag + assert_equal ['key1','key2'], @i.chunk_keys + tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[3]}/${tag[4]}/${key3}/${key4}/tail" + t = event_time('2016-04-11 20:30:00 +0900') + v = {key1: "value1", key2: "value2"} + m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v) + assert_equal "/mypath/2016/04/11/20-30/fluentd.test.output/////tail", @i.extract_placeholders(tmpl, m) + end + + test '#metadata returns object which contains tag/timekey/variables from records as specified in configuration' do + tag = 'test.output' + time = event_time('2016-04-12 15:31:23 -0700') + timekey = event_time('2016-04-12 15:00:00 -0700') + record = {"key1" => "value1", "num1" => 1, "message" => "my message"} + + i1 = create_output(:buffered) + i1.configure(config_element('ROOT','',{},[config_element('buffer', '')])) + assert_equal create_metadata(), i1.metadata(tag, time, record) + + i2 = create_output(:buffered) + i2.configure(config_element('ROOT','',{},[config_element('buffer', 'tag')])) + assert_equal create_metadata(tag: tag), i2.metadata(tag, time, record) + + i3 = create_output(:buffered) + i3.configure(config_element('ROOT','',{},[config_element('buffer', 'time', {"timekey_range" => 3600, "timekey_zone" => "-0700"})])) + assert_equal create_metadata(timekey: timekey), i3.metadata(tag, time, record) + + i4 = create_output(:buffered) + i4.configure(config_element('ROOT','',{},[config_element('buffer', 'key1', {})])) + assert_equal create_metadata(variables: {key1: "value1"}), i4.metadata(tag, time, record) + + i5 = create_output(:buffered) + i5.configure(config_element('ROOT','',{},[config_element('buffer', 'key1,num1', {})])) + assert_equal create_metadata(variables: {key1: "value1", num1: 1}), i5.metadata(tag, time, record) + + i6 = create_output(:buffered) + i6.configure(config_element('ROOT','',{},[config_element('buffer', 'tag,time', {"timekey_range" => 3600, "timekey_zone" => "-0700"})])) + assert_equal create_metadata(timekey: timekey, tag: tag), i6.metadata(tag, time, record) + + i7 = create_output(:buffered) + i7.configure(config_element('ROOT','',{},[config_element('buffer', 'tag,num1', {"timekey_range" => 3600, "timekey_zone" => "-0700"})])) + assert_equal create_metadata(tag: tag, variables: {num1: 1}), i7.metadata(tag, time, record) + + i8 = create_output(:buffered) + i8.configure(config_element('ROOT','',{},[config_element('buffer', 'time,tag,key1', {"timekey_range" => 3600, "timekey_zone" => "-0700"})])) + assert_equal create_metadata(timekey: timekey, tag: tag, variables: {key1: "value1"}), i8.metadata(tag, time, record) + end + + test '#emit calls #process via #emit_sync for non-buffered output' do + i = create_output(:sync) + process_called = false + i.register(:process){|tag, es| process_called = true } + i.configure(config_element()) + i.start + + t = event_time() + i.emit('tag', [ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]) + + assert process_called + + i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate + end + + test '#emit calls #format for buffered output' do + i = create_output(:buffered) + format_called_times = 0 + i.register(:format){|tag, time, record| format_called_times += 1; '' } + i.configure(config_element()) + i.start + + t = event_time() + i.emit('tag', [ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]) + + assert_equal 2, format_called_times + + i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate + end + + test '#prefer_buffered_processing (returns false) decides non-buffered without section' do + i = create_output(:full) + + process_called = false + format_called_times = 0 + i.register(:process){|tag, es| process_called = true } + i.register(:format){|tag, time, record| format_called_times += 1; '' } + + i.configure(config_element()) + i.register(:prefer_buffered_processing){ false } # delayed decision is possible to change after (output's) configure + i.start + + assert !i.prefer_buffered_processing + + t = event_time() + i.emit('tag', [ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]) + + assert process_called + assert_equal 0, format_called_times + + i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate + end + + test '#prefer_buffered_processing (returns true) decides buffered without section' do + i = create_output(:full) + + process_called = false + format_called_times = 0 + i.register(:process){|tag, es| process_called = true } + i.register(:format){|tag, time, record| format_called_times += 1; '' } + + i.configure(config_element()) + i.register(:prefer_buffered_processing){ true } # delayed decision is possible to change after (output's) configure + i.start + + assert i.prefer_buffered_processing + + t = event_time() + i.emit('tag', [ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]) + + assert !process_called + assert_equal 2, format_called_times + + i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate + end + + test 'output plugin will call #write for normal buffered plugin to flush buffer chunks' do + i = create_output(:buffered) + write_called = false + i.register(:write){ |chunk| write_called = true } + + i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})])) + i.start + + t = event_time() + i.emit('tag', [ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]) + i.force_flush + + assert write_called + + i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate + end + + test 'output plugin will call #try_write for plugin supports delayed commit only to flush buffer chunks' do + i = create_output(:delayed) + try_write_called = false + i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) } + + i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})])) + i.start + + t = event_time() + i.emit('tag', [ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]) + i.force_flush + + assert try_write_called + + i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate + end + + test '#prefer_delayed_commit (returns false) decides delayed commit is disabled if both are implemented' do + i = create_output(:full) + write_called = false + try_write_called = false + i.register(:write){ |chunk| write_called = true } + i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) } + + i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})])) + i.register(:prefer_delayed_commit){ false } # delayed decision is possible to change after (output's) configure + i.start + + assert !i.prefer_delayed_commit + + t = event_time() + i.emit('tag', [ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]) + i.force_flush + + assert write_called + assert !try_write_called + + i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate + end + + test '#prefer_delayed_commit (returns true) decides delayed commit is enabled if both are implemented' do + i = create_output(:full) + write_called = false + try_write_called = false + i.register(:write){ |chunk| write_called = true } + i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) } + + i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})])) + i.register(:prefer_delayed_commit){ true } # delayed decision is possible to change after (output's) configure + i.start + + assert i.prefer_delayed_commit + + t = event_time() + i.emit('tag', [ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]) + i.force_flush + + assert !write_called + assert try_write_called + + i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate + end + end + + sub_test_case 'sync output feature' do + setup do + @i = create_output(:sync) + end + + test 'raises configuration error if section is specified' do + assert_raise Fluent::ConfigError do + @i.configure(config_element('ROOT','',{},[config_element('buffer', '')])) + end + end + + test 'raises configuration error if section is specified' do + assert_raise Fluent::ConfigError do + @i.configure(config_element('ROOT','',{},[config_element('secondary','')])) + end + end + + test '#process is called for each event streams' do + ary = [] + @i.register(:process){|tag, es| ary << [tag, es] } + @i.configure(config_element()) + @i.start + + t = event_time() + es = [ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ] + 5.times do + @i.emit('tag', es) + end + assert_equal 5, ary.size + + @i.stop; @i.before_shutdown; @i.shutdown; @i.after_shutdown; @i.close; @i.terminate + end + end +end diff --git a/test/plugin/test_output_as_buffered.rb b/test/plugin/test_output_as_buffered.rb new file mode 100644 index 0000000000..7ae05e5805 --- /dev/null +++ b/test/plugin/test_output_as_buffered.rb @@ -0,0 +1,1512 @@ +require_relative '../helper' +require 'fluent/plugin/output' +require 'fluent/plugin/buffer' + +require 'json' +require 'time' +require 'timeout' +require 'timecop' + +module FluentPluginOutputAsBufferedTest + class DummyBareOutput < Fluent::Plugin::Output + def register(name, &block) + instance_variable_set("@#{name}", block) + end + end + class DummySyncOutput < DummyBareOutput + def process(tag, es) + @process ? @process.call(tag, es) : nil + end + end + class DummyAsyncOutput < DummyBareOutput + def format(tag, time, record) + @format ? @format.call(tag, time, record) : [tag, time, record].to_json + end + def write(chunk) + @write ? @write.call(chunk) : nil + end + end + class DummyDelayedOutput < DummyBareOutput + def format(tag, time, record) + @format ? @format.call(tag, time, record) : [tag, time, record].to_json + end + def try_write(chunk) + @try_write ? @try_write.call(chunk) : nil + end + def shutdown + if @shutdown_hook + @shutdown_hook.call + end + super + end + end + class DummyFullFeatureOutput < DummyBareOutput + def prefer_buffered_processing + @prefer_buffered_processing ? @prefer_buffered_processing.call : false + end + def prefer_delayed_commit + @prefer_delayed_commit ? @prefer_delayed_commit.call : false + end + def process(tag, es) + @process ? @process.call(tag, es) : nil + end + def format(tag, time, record) + @format ? @format.call(tag, time, record) : [tag, time, record].to_json + end + def write(chunk) + @write ? @write.call(chunk) : nil + end + def try_write(chunk) + @try_write ? @try_write.call(chunk) : nil + end + end +end + +class BufferedOutputTest < Test::Unit::TestCase + def create_output(type=:full) + case type + when :bare then FluentPluginOutputAsBufferedTest::DummyBareOutput.new + when :sync then FluentPluginOutputAsBufferedTest::DummySyncOutput.new + when :buffered then FluentPluginOutputAsBufferedTest::DummyAsyncOutput.new + when :delayed then FluentPluginOutputAsBufferedTest::DummyDelayedOutput.new + when :full then FluentPluginOutputAsBufferedTest::DummyFullFeatureOutput.new + else + raise ArgumentError, "unknown type: #{type}" + end + end + def create_metadata(timekey: nil, tag: nil, variables: nil) + Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables) + end + def waiting(seconds) + begin + Timeout.timeout(seconds) do + yield + end + rescue Timeout::Error + STDERR.print *(@i.log.out.logs) + raise + end + end + + teardown do + if @i + @i.stop unless @i.stopped? + @i.before_shutdown unless @i.before_shutdown? + @i.shutdown unless @i.shutdown? + @i.after_shutdown unless @i.after_shutdown? + @i.close unless @i.closed? + @i.terminate unless @i.terminated? + end + Timecop.return + end + + sub_test_case 'buffered output feature without any buffer key, flush_mode: none' do + setup do + hash = { + 'flush_mode' => 'none', + 'flush_threads' => 2, + 'chunk_bytes_limit' => 1024, + } + @i = create_output(:buffered) + @i.configure(config_element('ROOT','',{},[config_element('buffer','',hash)])) + @i.start + end + + test '#start does not create enqueue thread, but creates flush threads' do + @i.thread_wait_until_start + + assert @i.thread_exist?(:flush_thread_0) + assert @i.thread_exist?(:flush_thread_1) + assert !@i.thread_exist?(:enqueue_thread) + end + + test '#format is called for each events' do + ary = [] + @i.register(:format){|tag, time, record| ary << [tag, time, record]; '' } + + t = event_time() + es = [ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ] + + 5.times do + @i.emit('tag.test', es) + end + + assert_equal 10, ary.size + 5.times do |i| + assert_equal ["tag.test", t, {"key" => "value1"}], ary[i*2] + assert_equal ["tag.test", t, {"key" => "value2"}], ary[i*2+1] + end + end + + test '#write is called only when chunk bytes limit exceeded, and buffer chunk is purged' do + ary = [] + @i.register(:write){|chunk| ary << chunk.read } + + tag = "test.tag" + t = event_time() + r = {} + (0...10).each do |i| + r["key#{i}"] = "value #{i}" + end + event_size = [tag, t, r].to_json.size # 195 + + (1024 / event_size).times do |i| + @i.emit("test.tag", [ [t, r] ]) + end + assert{ @i.buffer.queue.size == 0 && ary.size == 0 } + + staged_chunk = @i.buffer.stage[@i.buffer.stage.keys.first] + assert{ staged_chunk.records != 0 } + + @i.emit("test.tag", [ [t, r] ]) + + assert{ @i.buffer.queue.size > 0 || @i.buffer.dequeued.size > 0 || ary.size > 0 } + + waiting(10) do + Thread.pass until @i.buffer.queue.size == 0 && @i.buffer.dequeued.size == 0 + Thread.pass until staged_chunk.records == 0 + end + + assert_equal 1, ary.size + assert_equal [tag,t,r].to_json * (1024 / event_size), ary.first + end + + test 'flush_at_shutdown work well when plugin is shutdown' do + ary = [] + @i.register(:write){|chunk| ary << chunk.read } + + tag = "test.tag" + t = event_time() + r = {} + (0...10).each do |i| + r["key#{i}"] = "value #{i}" + end + event_size = [tag, t, r].to_json.size # 195 + + (1024 / event_size).times do |i| + @i.emit("test.tag", [ [t, r] ]) + end + assert{ @i.buffer.queue.size == 0 && ary.size == 0 } + + @i.stop + @i.before_shutdown + @i.shutdown + @i.after_shutdown + + waiting(10) do + Thread.pass until ary.size == 1 + end + assert_equal [tag,t,r].to_json * (1024 / event_size), ary.first + end + end + + sub_test_case 'buffered output feature without any buffer key, flush_mode: fast' do + setup do + hash = { + 'flush_mode' => 'fast', + 'flush_interval' => 1, + 'flush_threads' => 1, + 'chunk_bytes_limit' => 1024, + } + @i = create_output(:buffered) + @i.configure(config_element('ROOT','',{},[config_element('buffer','',hash)])) + @i.start + end + + test '#start creates enqueue thread and flush threads' do + @i.thread_wait_until_start + + assert @i.thread_exist?(:flush_thread_0) + assert @i.thread_exist?(:enqueue_thread) + end + + test '#format is called for each event streams' do + ary = [] + @i.register(:format){|tag, time, record| ary << [tag, time, record]; '' } + + t = event_time() + es = [ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ] + + 5.times do + @i.emit('tag.test', es) + end + + assert_equal 10, ary.size + 5.times do |i| + assert_equal ["tag.test", t, {"key" => "value1"}], ary[i*2] + assert_equal ["tag.test", t, {"key" => "value2"}], ary[i*2+1] + end + end + + test '#write is called per flush_interval, and buffer chunk is purged' do + @i.thread_wait_until_start + + ary = [] + @i.register(:format){|tag,time,record| [tag,time,record].to_json + "\n" } + @i.register(:write){|chunk| chunk.read.split("\n").reject{|l| l.empty? }.each{|data| ary << data } } + + tag = "test.tag" + t = event_time() + r = {} + (0...10).each do |i| + r["key#{i}"] = "value #{i}" + end + event_size = [tag, t, r].to_json.size # 195 + + 3.times do |i| + rand_records = rand(1..5) + es = [ [t, r] ] * rand_records + assert_equal rand_records, es.size + + @i.interrupt_flushes + + @i.emit("test.tag", es) + + assert{ @i.buffer.stage.size == 1 } + + staged_chunk = @i.instance_eval{ @buffer.stage[@buffer.stage.keys.first] } + assert{ staged_chunk.records != 0 } + + @i.enqueue_thread_wait + + waiting(10) do + Thread.pass until @i.buffer.queue.size == 0 && @i.buffer.dequeued.size == 0 + Thread.pass until staged_chunk.records == 0 + end + + assert_equal rand_records, ary.size + ary.reject!{|e| true } + end + end + + test 'flush_at_shutdown work well when plugin is shutdown' do + ary = [] + @i.register(:write){|chunk| ary << chunk.read } + + tag = "test.tag" + t = event_time() + r = {} + (0...10).each do |i| + r["key#{i}"] = "value #{i}" + end + event_size = [tag, t, r].to_json.size # 195 + + (1024 / event_size).times do |i| + @i.emit("test.tag", [ [t, r] ]) + end + assert{ @i.buffer.queue.size == 0 && ary.size == 0 } + + @i.stop + @i.before_shutdown + @i.shutdown + @i.after_shutdown + + waiting(10) do + Thread.pass until ary.size == 1 + end + assert_equal [tag,t,r].to_json * (1024 / event_size), ary.first + end + end + + sub_test_case 'buffered output feature without any buffer key, flush_mode: immediate' do + setup do + hash = { + 'flush_mode' => 'immediate', + 'flush_threads' => 1, + 'chunk_bytes_limit' => 1024, + } + @i = create_output(:buffered) + @i.configure(config_element('ROOT','',{},[config_element('buffer','',hash)])) + @i.start + end + + test '#start does not create enqueue thread, but creates flush threads' do + @i.thread_wait_until_start + + assert @i.thread_exist?(:flush_thread_0) + assert !@i.thread_exist?(:enqueue_thread) + end + + test '#format is called for each event streams' do + ary = [] + @i.register(:format){|tag, time, record| ary << [tag, time, record]; '' } + + t = event_time() + es = [ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ] + + 5.times do + @i.emit('tag.test', es) + end + + assert_equal 10, ary.size + 5.times do |i| + assert_equal ["tag.test", t, {"key" => "value1"}], ary[i*2] + assert_equal ["tag.test", t, {"key" => "value2"}], ary[i*2+1] + end + end + + test '#write is called every time for each emits, and buffer chunk is purged' do + @i.thread_wait_until_start + + ary = [] + @i.register(:format){|tag,time,record| [tag,time,record].to_json + "\n" } + @i.register(:write){|chunk| chunk.read.split("\n").reject{|l| l.empty? }.each{|data| ary << data } } + + tag = "test.tag" + t = event_time() + r = {} + (0...10).each do |i| + r["key#{i}"] = "value #{i}" + end + event_size = [tag, t, r].to_json.size # 195 + + 3.times do |i| + rand_records = rand(1..5) + es = [ [t, r] ] * rand_records + assert_equal rand_records, es.size + @i.emit("test.tag", es) + + assert{ @i.buffer.stage.size == 0 && (@i.buffer.queue.size == 1 || @i.buffer.dequeued.size == 1 || ary.size > 0) } + + waiting(10) do + Thread.pass until @i.buffer.queue.size == 0 && @i.buffer.dequeued.size == 0 + end + + assert_equal rand_records, ary.size + ary.reject!{|e| true } + end + end + + test 'flush_at_shutdown work well when plugin is shutdown' do + ary = [] + @i.register(:write){|chunk| ary << chunk.read } + + tag = "test.tag" + t = event_time() + r = {} + (0...10).each do |i| + r["key#{i}"] = "value #{i}" + end + @i.emit("test.tag", [ [t, r] ]) + + @i.stop + @i.before_shutdown + @i.shutdown + @i.after_shutdown + + waiting(10) do + Thread.pass until ary.size == 1 + end + assert_equal [tag,t,r].to_json, ary.first + end + end + + sub_test_case 'buffered output feature with timekey and range' do + setup do + chunk_key = 'time' + hash = { + 'timekey_range' => 30, # per 30seconds + 'timekey_wait' => 5, # 5 second delay for flush + 'flush_threads' => 1, + } + @i = create_output(:buffered) + @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)])) + @i.start + end + + test '#configure raises config error if timekey_range is not specified' do + i = create_output(:buffered) + assert_raise Fluent::ConfigError do + i.configure(config_element('ROOT','',{},[config_element('buffer','time',)])) + end + end + + test 'default flush_mode is set to :none' do + assert_equal :none, @i.instance_eval{ @flush_mode } + end + + test '#start creates enqueue thread and flush threads' do + @i.thread_wait_until_start + + assert @i.thread_exist?(:flush_thread_0) + assert @i.thread_exist?(:enqueue_thread) + end + + test '#format is called for each event streams' do + ary = [] + @i.register(:format){|tag, time, record| ary << [tag, time, record]; '' } + + t = event_time() + es = [ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ] + + 5.times do + @i.emit('tag.test', es) + end + + assert_equal 10, ary.size + 5.times do |i| + assert_equal ["tag.test", t, {"key" => "value1"}], ary[i*2] + assert_equal ["tag.test", t, {"key" => "value2"}], ary[i*2+1] + end + end + + test '#write is called per time ranges after timekey_wait, and buffer chunk is purged' do + Timecop.freeze( Time.parse('2016-04-13 14:04:00 +0900') ) + + @i.thread_wait_until_start + + ary = [] + metachecks = [] + + @i.register(:format){|tag,time,record| [tag,time,record].to_json + "\n" } + @i.register(:write){|chunk| chunk.read.split("\n").reject{|l| l.empty? }.each{|data| e = JSON.parse(data); ary << e; metachecks << (chunk.metadata.timekey.to_i <= e[1].to_i && e[1].to_i < chunk.metadata.timekey.to_i + 30) } } + + r = {} + (0...10).each do |i| + r["key#{i}"] = "value #{i}" + end + ts = [ + Fluent::EventTime.parse('2016-04-13 14:03:21 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:23 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:29 +0900'), + Fluent::EventTime.parse('2016-04-13 14:03:30 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:33 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:38 +0900'), + Fluent::EventTime.parse('2016-04-13 14:03:43 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:49 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:51 +0900'), + Fluent::EventTime.parse('2016-04-13 14:04:00 +0900'), Fluent::EventTime.parse('2016-04-13 14:04:01 +0900'), + ] + events = [ + ["test.tag.1", ts[0], r], # range 14:03:00 - 03:29 + ["test.tag.2", ts[1], r], + ["test.tag.1", ts[2], r], + ["test.tag.1", ts[3], r], # range 14:03:30 - 04:00 + ["test.tag.1", ts[4], r], + ["test.tag.1", ts[5], r], + ["test.tag.1", ts[6], r], + ["test.tag.1", ts[7], r], + ["test.tag.2", ts[8], r], + ["test.tag.1", ts[9], r], # range 14:04:00 - 04:29 + ["test.tag.2", ts[10], r], + ] + + assert_equal 0, @i.write_count + + @i.interrupt_flushes + + events.shuffle.each do |tag, time, record| + @i.emit(tag, [ [time, record] ]) + end + assert{ @i.buffer.stage.size == 3 } + assert{ @i.write_count == 0 } + + @i.enqueue_thread_wait + + waiting(4) do + Thread.pass until @i.write_count > 0 + end + + assert{ @i.buffer.stage.size == 2 && @i.write_count == 1 } + + assert_equal 3, ary.size + assert_equal 2, ary.select{|e| e[0] == "test.tag.1" }.size + assert_equal 1, ary.select{|e| e[0] == "test.tag.2" }.size + + Timecop.freeze( Time.parse('2016-04-13 14:04:04 +0900') ) + + @i.enqueue_thread_wait + + assert{ @i.buffer.stage.size == 2 && @i.write_count == 1 } + + Timecop.freeze( Time.parse('2016-04-13 14:04:06 +0900') ) + + @i.enqueue_thread_wait + waiting(4) do + Thread.pass until @i.write_count > 1 + end + + assert{ @i.buffer.stage.size == 1 && @i.write_count == 2 } + + assert_equal 9, ary.size + assert_equal 7, ary.select{|e| e[0] == "test.tag.1" }.size + assert_equal 2, ary.select{|e| e[0] == "test.tag.2" }.size + + assert metachecks.all?{|e| e } + end + + test 'flush_at_shutdown work well when plugin is shutdown' do + Timecop.freeze( Time.parse('2016-04-13 14:04:00 +0900') ) + + @i.thread_wait_until_start + + ary = [] + metachecks = [] + + @i.register(:format){|tag,time,record| [tag,time,record].to_json + "\n" } + @i.register(:write){|chunk| chunk.read.split("\n").reject{|l| l.empty? }.each{|data| e = JSON.parse(data); ary << e; metachecks << (chunk.metadata.timekey.to_i <= e[1].to_i && e[1].to_i < chunk.metadata.timekey.to_i + 30) } } + + r = {} + (0...10).each do |i| + r["key#{i}"] = "value #{i}" + end + ts = [ + Fluent::EventTime.parse('2016-04-13 14:03:21 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:23 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:29 +0900'), + Fluent::EventTime.parse('2016-04-13 14:03:30 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:33 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:38 +0900'), + Fluent::EventTime.parse('2016-04-13 14:03:43 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:49 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:51 +0900'), + Fluent::EventTime.parse('2016-04-13 14:04:00 +0900'), Fluent::EventTime.parse('2016-04-13 14:04:01 +0900'), + ] + events = [ + ["test.tag.1", ts[0], r], # range 14:03:00 - 03:29 + ["test.tag.2", ts[1], r], + ["test.tag.1", ts[2], r], + ["test.tag.1", ts[3], r], # range 14:03:30 - 04:00 + ["test.tag.1", ts[4], r], + ["test.tag.1", ts[5], r], + ["test.tag.1", ts[6], r], + ["test.tag.1", ts[7], r], + ["test.tag.2", ts[8], r], + ["test.tag.1", ts[9], r], # range 14:04:00 - 04:29 + ["test.tag.2", ts[10], r], + ] + + assert_equal 0, @i.write_count + + @i.interrupt_flushes + + events.shuffle.each do |tag, time, record| + @i.emit(tag, [ [time, record] ]) + end + assert{ @i.buffer.stage.size == 3 } + assert{ @i.write_count == 0 } + + @i.enqueue_thread_wait + + waiting(4) do + Thread.pass until @i.write_count > 0 + end + + assert{ @i.buffer.stage.size == 2 && @i.write_count == 1 } + + Timecop.freeze( Time.parse('2016-04-13 14:04:04 +0900') ) + + @i.enqueue_thread_wait + + assert{ @i.buffer.stage.size == 2 && @i.write_count == 1 } + + Timecop.freeze( Time.parse('2016-04-13 14:04:06 +0900') ) + + @i.enqueue_thread_wait + waiting(4) do + Thread.pass until @i.write_count > 1 + end + + assert{ @i.buffer.stage.size == 1 && @i.write_count == 2 } + + Timecop.freeze( Time.parse('2016-04-13 14:04:13 +0900') ) + + assert_equal 9, ary.size + + @i.stop + @i.before_shutdown + @i.shutdown + @i.after_shutdown + + waiting(4) do + Thread.pass until @i.write_count > 2 + end + + assert_equal 11, ary.size + assert metachecks.all?{|e| e } + end + end + + sub_test_case 'buffered output feature with tag key' do + setup do + chunk_key = 'tag' + hash = { + 'flush_interval' => 10, + 'flush_threads' => 1, + 'flush_burst_interval' => 0.1, + 'chunk_bytes_limit' => 1024, + } + @i = create_output(:buffered) + @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)])) + @i.start + end + + test 'default flush_mode is set to :fast' do + assert_equal :fast, @i.instance_eval{ @flush_mode } + end + + test '#start creates enqueue thread and flush threads' do + @i.thread_wait_until_start + + assert @i.thread_exist?(:flush_thread_0) + assert @i.thread_exist?(:enqueue_thread) + end + + test '#format is called for each event streams' do + ary = [] + @i.register(:format){|tag, time, record| ary << [tag, time, record]; '' } + + t = event_time() + es = [ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ] + + 5.times do + @i.emit('tag.test', es) + end + + assert_equal 10, ary.size + 5.times do |i| + assert_equal ["tag.test", t, {"key" => "value1"}], ary[i*2] + assert_equal ["tag.test", t, {"key" => "value2"}], ary[i*2+1] + end + end + + test '#write is called per tags, per flush_interval & chunk sizes, and buffer chunk is purged' do + Timecop.freeze( Time.parse('2016-04-13 14:04:01 +0900') ) + + ary = [] + metachecks = [] + + @i.register(:format){|tag,time,record| [tag,time,record].to_json + "\n" } + @i.register(:write){|chunk| chunk.read.split("\n").reject{|l| l.empty? }.each{|data| e = JSON.parse(data); ary << e; metachecks << (chunk.metadata.tag == e[0]) } } + + @i.thread_wait_until_start + + r = {} + (0...10).each do |i| + r["key#{i}"] = "value #{i}" + end + ts = [ + event_time('2016-04-13 14:03:21 +0900'), event_time('2016-04-13 14:03:23 +0900'), event_time('2016-04-13 14:03:29 +0900'), + event_time('2016-04-13 14:03:30 +0900'), event_time('2016-04-13 14:03:33 +0900'), event_time('2016-04-13 14:03:38 +0900'), + event_time('2016-04-13 14:03:43 +0900'), event_time('2016-04-13 14:03:49 +0900'), event_time('2016-04-13 14:03:51 +0900'), + event_time('2016-04-13 14:04:00 +0900'), event_time('2016-04-13 14:04:01 +0900'), + ] + # size of a event is 197 + events = [ + ["test.tag.1", ts[0], r], + ["test.tag.2", ts[1], r], + ["test.tag.1", ts[2], r], + ["test.tag.1", ts[3], r], + ["test.tag.1", ts[4], r], + ["test.tag.1", ts[5], r], + ["test.tag.1", ts[6], r], + ["test.tag.1", ts[7], r], + ["test.tag.2", ts[8], r], + ["test.tag.1", ts[9], r], + ["test.tag.2", ts[10], r], + ] + + assert_equal 0, @i.write_count + + @i.interrupt_flushes + + events.shuffle.each do |tag, time, record| + @i.emit(tag, [ [time, record] ]) + end + assert{ @i.buffer.stage.size == 2 } # test.tag.1 x1, test.tag.2 x1 + + Timecop.freeze( Time.parse('2016-04-13 14:04:02 +0900') ) + + @i.enqueue_thread_wait + @i.flush_thread_wakeup + + waiting(4) do + Thread.pass until @i.write_count > 0 + end + + assert{ @i.buffer.stage.size == 2 } + assert{ @i.write_count == 1 } + assert{ @i.buffer.queue.size == 0 } + + # events fulfills a chunk (and queued immediately) + assert_equal 5, ary.size + assert_equal 5, ary.select{|e| e[0] == "test.tag.1" }.size + assert_equal 0, ary.select{|e| e[0] == "test.tag.2" }.size + + Timecop.freeze( Time.parse('2016-04-13 14:04:09 +0900') ) + + @i.enqueue_thread_wait + + assert{ @i.buffer.stage.size == 2 } + + # to trigger try_flush with flush_burst_interval + Timecop.freeze( Time.parse('2016-04-13 14:04:11 +0900') ) + @i.enqueue_thread_wait + Timecop.freeze( Time.parse('2016-04-13 14:04:15 +0900') ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + + assert{ @i.buffer.stage.size == 0 } + + waiting(4) do + Thread.pass until @i.write_count > 2 + end + + assert{ @i.buffer.stage.size == 0 && @i.write_count == 3 } + + assert_equal 11, ary.size + assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size + assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size + + assert metachecks.all?{|e| e } + end + + test 'flush_at_shutdown work well when plugin is shutdown' do + Timecop.freeze( Time.parse('2016-04-13 14:04:01 +0900') ) + + ary = [] + metachecks = [] + + @i.register(:format){|tag,time,record| [tag,time,record].to_json + "\n" } + @i.register(:write){|chunk| chunk.read.split("\n").reject{|l| l.empty? }.each{|data| e = JSON.parse(data); ary << e; metachecks << (chunk.metadata.tag == e[0]) } } + + @i.thread_wait_until_start + + r = {} + (0...10).each do |i| + r["key#{i}"] = "value #{i}" + end + ts = [ + event_time('2016-04-13 14:03:21 +0900'), event_time('2016-04-13 14:03:23 +0900'), event_time('2016-04-13 14:03:29 +0900'), + event_time('2016-04-13 14:03:30 +0900'), event_time('2016-04-13 14:03:33 +0900'), event_time('2016-04-13 14:03:38 +0900'), + event_time('2016-04-13 14:03:43 +0900'), event_time('2016-04-13 14:03:49 +0900'), event_time('2016-04-13 14:03:51 +0900'), + event_time('2016-04-13 14:04:00 +0900'), event_time('2016-04-13 14:04:01 +0900'), + ] + # size of a event is 197 + events = [ + ["test.tag.1", ts[0], r], + ["test.tag.2", ts[1], r], + ["test.tag.1", ts[2], r], + ["test.tag.1", ts[3], r], + ["test.tag.1", ts[4], r], + ["test.tag.1", ts[5], r], + ["test.tag.1", ts[6], r], + ["test.tag.1", ts[7], r], + ["test.tag.2", ts[8], r], + ["test.tag.1", ts[9], r], + ["test.tag.2", ts[10], r], + ] + + assert_equal 0, @i.write_count + + @i.interrupt_flushes + + events.shuffle.each do |tag, time, record| + @i.emit(tag, [ [time, record] ]) + end + assert{ @i.buffer.stage.size == 2 } # test.tag.1 x1, test.tag.2 x1 + + Timecop.freeze( Time.parse('2016-04-13 14:04:02 +0900') ) + + @i.enqueue_thread_wait + @i.flush_thread_wakeup + + waiting(4) do + Thread.pass until @i.write_count > 0 + end + + assert{ @i.buffer.stage.size == 2 } + assert{ @i.write_count == 1 } + assert{ @i.buffer.queue.size == 0 } + + # events fulfills a chunk (and queued immediately) + assert_equal 5, ary.size + assert_equal 5, ary.select{|e| e[0] == "test.tag.1" }.size + assert_equal 0, ary.select{|e| e[0] == "test.tag.2" }.size + + @i.stop + @i.before_shutdown + @i.shutdown + @i.after_shutdown + + waiting(4) do + Thread.pass until @i.write_count > 1 + end + + assert{ @i.buffer.stage.size == 0 && @i.buffer.queue.size == 0 && @i.write_count == 3 } + + assert_equal 11, ary.size + assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size + assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size + + assert metachecks.all?{|e| e } + end + end + + sub_test_case 'buffered output feature with variables' do + setup do + chunk_key = 'name,service' + hash = { + 'flush_interval' => 10, + 'flush_threads' => 1, + 'flush_burst_interval' => 0.1, + 'chunk_bytes_limit' => 1024, + } + @i = create_output(:buffered) + @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)])) + @i.start + end + + test 'default flush_mode is set to :fast' do + assert_equal :fast, @i.instance_eval{ @flush_mode } + end + + test '#start creates enqueue thread and flush threads' do + @i.thread_wait_until_start + + assert @i.thread_exist?(:flush_thread_0) + assert @i.thread_exist?(:enqueue_thread) + end + + test '#format is called for each event streams' do + ary = [] + @i.register(:format){|tag, time, record| ary << [tag, time, record]; '' } + + t = event_time() + es = [ + [t, {"key" => "value1", "name" => "moris", "service" => "a"}], + [t, {"key" => "value2", "name" => "moris", "service" => "b"}], + ] + + 5.times do + @i.emit('tag.test', es) + end + + assert_equal 10, ary.size + 5.times do |i| + assert_equal ["tag.test", t, {"key" => "value1", "name" => "moris", "service" => "a"}], ary[i*2] + assert_equal ["tag.test", t, {"key" => "value2", "name" => "moris", "service" => "b"}], ary[i*2+1] + end + end + + test '#write is called per value combination of variables, per flush_interval & chunk sizes, and buffer chunk is purged' do + Timecop.freeze( Time.parse('2016-04-13 14:04:01 +0900') ) + + ary = [] + metachecks = [] + + @i.register(:format){|tag,time,record| [tag,time,record].to_json + "\n" } + @i.register(:write){|chunk| chunk.read.split("\n").reject{|l| l.empty? }.each{|data| e = JSON.parse(data); ary << e; metachecks << (e[2]["name"] == chunk.metadata.variables[:name] && e[2]["service"] == chunk.metadata.variables[:service]) } } + + @i.thread_wait_until_start + + # size of a event is 195 + dummy_data = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + events = [ + ["test.tag.1", event_time('2016-04-13 14:03:21 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1) xxx-a (6 events) + ["test.tag.2", event_time('2016-04-13 14:03:23 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}], #(2) yyy-a (3 events) + ["test.tag.1", event_time('2016-04-13 14:03:29 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1) + ["test.tag.1", event_time('2016-04-13 14:03:30 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1) + ["test.tag.1", event_time('2016-04-13 14:03:33 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1) + ["test.tag.1", event_time('2016-04-13 14:03:38 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "b"}], #(3) xxx-b (2 events) + ["test.tag.1", event_time('2016-04-13 14:03:43 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1) + ["test.tag.1", event_time('2016-04-13 14:03:49 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "b"}], #(3) + ["test.tag.2", event_time('2016-04-13 14:03:51 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}], #(2) + ["test.tag.1", event_time('2016-04-13 14:04:00 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1) + ["test.tag.2", event_time('2016-04-13 14:04:01 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}], #(2) + ] + + assert_equal 0, @i.write_count + + @i.interrupt_flushes + + events.shuffle.each do |tag, time, record| + @i.emit(tag, [ [time, record] ]) + end + assert{ @i.buffer.stage.size == 3 } + + Timecop.freeze( Time.parse('2016-04-13 14:04:02 +0900') ) + + @i.enqueue_thread_wait + @i.flush_thread_wakeup + + waiting(4) do + Thread.pass until @i.write_count > 0 + end + + assert{ @i.buffer.stage.size == 3 } + assert{ @i.write_count == 1 } + assert{ @i.buffer.queue.size == 0 } + + # events fulfills a chunk (and queued immediately) + assert_equal 5, ary.size + assert_equal 5, ary.select{|e| e[0] == "test.tag.1" }.size + assert_equal 0, ary.select{|e| e[0] == "test.tag.2" }.size + assert ary[0...5].all?{|e| e[2]["name"] == "xxx" && e[2]["service"] == "a" } + + Timecop.freeze( Time.parse('2016-04-13 14:04:09 +0900') ) + + @i.enqueue_thread_wait + + assert{ @i.buffer.stage.size == 3 } + + # to trigger try_flush with flush_burst_interval + Timecop.freeze( Time.parse('2016-04-13 14:04:11 +0900') ) + @i.enqueue_thread_wait + Timecop.freeze( Time.parse('2016-04-13 14:04:12 +0900') ) + @i.enqueue_thread_wait + Timecop.freeze( Time.parse('2016-04-13 14:04:13 +0900') ) + @i.enqueue_thread_wait + Timecop.freeze( Time.parse('2016-04-13 14:04:14 +0900') ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + + assert{ @i.buffer.stage.size == 0 } + + waiting(4) do + Thread.pass until @i.write_count > 1 + end + + assert{ @i.buffer.stage.size == 0 && @i.write_count == 4 } + + assert_equal 11, ary.size + assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size + assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size + assert_equal 6, ary.select{|e| e[2]["name"] == "xxx" && e[2]["service"] == "a" }.size + assert_equal 3, ary.select{|e| e[2]["name"] == "yyy" && e[2]["service"] == "a" }.size + assert_equal 2, ary.select{|e| e[2]["name"] == "xxx" && e[2]["service"] == "b" }.size + + assert metachecks.all?{|e| e } + end + + test 'flush_at_shutdown work well when plugin is shutdown' do + Timecop.freeze( Time.parse('2016-04-13 14:04:01 +0900') ) + + ary = [] + metachecks = [] + + @i.register(:format){|tag,time,record| [tag,time,record].to_json + "\n" } + @i.register(:write){|chunk| chunk.read.split("\n").reject{|l| l.empty? }.each{|data| e = JSON.parse(data); ary << e; metachecks << (e[2]["name"] == chunk.metadata.variables[:name] && e[2]["service"] == chunk.metadata.variables[:service]) } } + + @i.thread_wait_until_start + + # size of a event is 195 + dummy_data = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + events = [ + ["test.tag.1", event_time('2016-04-13 14:03:21 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1) xxx-a (6 events) + ["test.tag.2", event_time('2016-04-13 14:03:23 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}], #(2) yyy-a (3 events) + ["test.tag.1", event_time('2016-04-13 14:03:29 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1) + ["test.tag.1", event_time('2016-04-13 14:03:30 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1) + ["test.tag.1", event_time('2016-04-13 14:03:33 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1) + ["test.tag.1", event_time('2016-04-13 14:03:38 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "b"}], #(3) xxx-b (2 events) + ["test.tag.1", event_time('2016-04-13 14:03:43 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1) + ["test.tag.1", event_time('2016-04-13 14:03:49 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "b"}], #(3) + ["test.tag.2", event_time('2016-04-13 14:03:51 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}], #(2) + ["test.tag.1", event_time('2016-04-13 14:04:00 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1) + ["test.tag.2", event_time('2016-04-13 14:04:01 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}], #(2) + ] + + assert_equal 0, @i.write_count + + @i.interrupt_flushes + + events.shuffle.each do |tag, time, record| + @i.emit(tag, [ [time, record] ]) + end + assert{ @i.buffer.stage.size == 3 } + + Timecop.freeze( Time.parse('2016-04-13 14:04:02 +0900') ) + + @i.enqueue_thread_wait + @i.flush_thread_wakeup + + waiting(4) do + Thread.pass until @i.write_count > 0 + end + + assert{ @i.buffer.stage.size == 3 } + assert{ @i.write_count == 1 } + assert{ @i.buffer.queue.size == 0 } + + # events fulfills a chunk (and queued immediately) + assert_equal 5, ary.size + assert_equal 5, ary.select{|e| e[0] == "test.tag.1" }.size + assert_equal 0, ary.select{|e| e[0] == "test.tag.2" }.size + + @i.stop + @i.before_shutdown + @i.shutdown + @i.after_shutdown + + waiting(4) do + Thread.pass until @i.write_count > 1 + end + + assert{ @i.buffer.stage.size == 0 && @i.buffer.queue.size == 0 && @i.write_count == 4 } + + assert_equal 11, ary.size + assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size + assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size + assert_equal 6, ary.select{|e| e[2]["name"] == "xxx" && e[2]["service"] == "a" }.size + assert_equal 3, ary.select{|e| e[2]["name"] == "yyy" && e[2]["service"] == "a" }.size + assert_equal 2, ary.select{|e| e[2]["name"] == "xxx" && e[2]["service"] == "b" }.size + + assert metachecks.all?{|e| e } + end + end + + sub_test_case 'buffered output feature with many keys' do + test 'default flush mode is set to :fast if keys does not include time' do + chunk_key = 'name,service,tag' + hash = { + 'flush_interval' => 10, + 'flush_threads' => 1, + 'flush_burst_interval' => 0.1, + 'chunk_bytes_limit' => 1024, + } + @i = create_output(:buffered) + @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)])) + @i.start + + assert_equal :fast, @i.instance_eval{ @flush_mode } + end + + test 'default flush mode is set to :none if keys includes time' do + chunk_key = 'name,service,tag,time' + hash = { + 'timekey_range' => 60, + 'flush_interval' => 10, + 'flush_threads' => 1, + 'flush_burst_interval' => 0.1, + 'chunk_bytes_limit' => 1024, + } + @i = create_output(:buffered) + @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)])) + @i.start + + assert_equal :none, @i.instance_eval{ @flush_mode } + end + end + + sub_test_case 'buffered output feature with delayed commit' do + setup do + chunk_key = 'tag' + hash = { + 'flush_interval' => 10, + 'flush_threads' => 1, + 'flush_burst_interval' => 0.1, + 'delayed_commit_timeout' => 30, + 'chunk_bytes_limit' => 1024, + } + @i = create_output(:delayed) + @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)])) + @i.start + end + + test '#format is called for each event streams' do + ary = [] + @i.register(:format){|tag, time, record| ary << [tag, time, record]; '' } + + t = event_time() + es = [ + [t, {"key" => "value1", "name" => "moris", "service" => "a"}], + [t, {"key" => "value2", "name" => "moris", "service" => "b"}], + ] + + 5.times do + @i.emit('tag.test', es) + end + + assert_equal 10, ary.size + 5.times do |i| + assert_equal ["tag.test", t, {"key" => "value1", "name" => "moris", "service" => "a"}], ary[i*2] + assert_equal ["tag.test", t, {"key" => "value2", "name" => "moris", "service" => "b"}], ary[i*2+1] + end + end + + test '#try_write is called per flush, buffer chunk is not purged until #commit_write is called' do + Timecop.freeze( Time.parse('2016-04-13 14:04:01 +0900') ) + + ary = [] + metachecks = [] + chunks = [] + + @i.register(:format){|tag,time,record| [tag,time,record].to_json + "\n" } + @i.register(:try_write) do |chunk| + chunks << chunk + chunk.read.split("\n").reject{|l| l.empty? }.each do |data| + e = JSON.parse(data) + ary << e + metachecks << (e[0] == chunk.metadata.tag) + end + end + + @i.thread_wait_until_start + + # size of a event is 195 + dummy_data = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + events = [ + ["test.tag.1", event_time('2016-04-13 14:03:21 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], + ["test.tag.2", event_time('2016-04-13 14:03:23 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}], + ["test.tag.1", event_time('2016-04-13 14:03:29 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], + ["test.tag.1", event_time('2016-04-13 14:03:30 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], + ["test.tag.1", event_time('2016-04-13 14:03:33 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], + ["test.tag.1", event_time('2016-04-13 14:03:38 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "b"}], + ["test.tag.1", event_time('2016-04-13 14:03:43 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], + ["test.tag.1", event_time('2016-04-13 14:03:49 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "b"}], + ["test.tag.2", event_time('2016-04-13 14:03:51 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}], + ["test.tag.1", event_time('2016-04-13 14:04:00 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], + ["test.tag.2", event_time('2016-04-13 14:04:01 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}], + ] + + assert_equal 0, @i.write_count + + @i.interrupt_flushes + + events.shuffle.each do |tag, time, record| + @i.emit(tag, [ [time, record] ]) + end + assert{ @i.buffer.stage.size == 2 } + + Timecop.freeze( Time.parse('2016-04-13 14:04:02 +0900') ) + + @i.enqueue_thread_wait + @i.flush_thread_wakeup + + waiting(4) do + Thread.pass until @i.write_count > 0 + end + + assert{ @i.buffer.stage.size == 2 } + assert{ @i.write_count == 1 } + assert{ @i.buffer.queue.size == 0 } + assert{ @i.buffer.dequeued.size == 1 } + + # events fulfills a chunk (and queued immediately) + assert_equal 5, ary.size + assert_equal 5, ary.select{|e| e[0] == "test.tag.1" }.size + assert_equal 0, ary.select{|e| e[0] == "test.tag.2" }.size + + assert_equal 1, chunks.size + assert !chunks.first.empty? + + Timecop.freeze( Time.parse('2016-04-13 14:04:09 +0900') ) + + @i.enqueue_thread_wait + + assert{ @i.buffer.stage.size == 2 } + + # to trigger try_flush with flush_burst_interval + Timecop.freeze( Time.parse('2016-04-13 14:04:11 +0900') ) + @i.enqueue_thread_wait + Timecop.freeze( Time.parse('2016-04-13 14:04:12 +0900') ) + @i.enqueue_thread_wait + Timecop.freeze( Time.parse('2016-04-13 14:04:13 +0900') ) + @i.enqueue_thread_wait + Timecop.freeze( Time.parse('2016-04-13 14:04:14 +0900') ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + + assert{ @i.buffer.stage.size == 0 } + + waiting(4) do + Thread.pass until @i.write_count > 1 + end + + assert{ @i.buffer.stage.size == 0 && @i.write_count == 3 } + assert{ @i.buffer.dequeued.size == 3 } + + assert_equal 11, ary.size + assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size + assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size + + assert_equal 3, chunks.size + assert chunks.all?{|c| !c.empty? } + + assert metachecks.all?{|e| e } + + @i.commit_write(chunks[0].unique_id) + assert{ @i.buffer.dequeued.size == 2 } + assert chunks[0].empty? + + @i.commit_write(chunks[1].unique_id) + assert{ @i.buffer.dequeued.size == 1 } + assert chunks[1].empty? + + @i.commit_write(chunks[2].unique_id) + assert{ @i.buffer.dequeued.size == 0 } + assert chunks[2].empty? + + # no problem to commit chunks already committed + assert_nothing_raised do + @i.commit_write(chunks[2].unique_id) + end + end + + test '#rollback_write and #try_rollback_write can rollback buffer chunks for delayed commit after timeout, and then be able to write it again' do + Timecop.freeze( Time.parse('2016-04-13 14:04:01 +0900') ) + + ary = [] + metachecks = [] + chunks = [] + + @i.register(:format){|tag,time,record| [tag,time,record].to_json + "\n" } + @i.register(:try_write) do |chunk| + chunks << chunk + chunk.read.split("\n").reject{|l| l.empty? }.each do |data| + e = JSON.parse(data) + ary << e + metachecks << (e[0] == chunk.metadata.tag) + end + end + + @i.thread_wait_until_start + + # size of a event is 195 + dummy_data = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + events = [ + ["test.tag.1", event_time('2016-04-13 14:03:21 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], + ["test.tag.2", event_time('2016-04-13 14:03:23 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}], + ["test.tag.1", event_time('2016-04-13 14:03:29 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], + ["test.tag.1", event_time('2016-04-13 14:03:30 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], + ["test.tag.1", event_time('2016-04-13 14:03:33 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], + ["test.tag.1", event_time('2016-04-13 14:03:38 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "b"}], + ["test.tag.1", event_time('2016-04-13 14:03:43 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], + ["test.tag.1", event_time('2016-04-13 14:03:49 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "b"}], + ["test.tag.2", event_time('2016-04-13 14:03:51 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}], + ["test.tag.1", event_time('2016-04-13 14:04:00 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], + ["test.tag.2", event_time('2016-04-13 14:04:01 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}], + ] + + assert_equal 0, @i.write_count + + @i.interrupt_flushes + + events.shuffle.each do |tag, time, record| + @i.emit(tag, [ [time, record] ]) + end + assert{ @i.buffer.stage.size == 2 } + + Timecop.freeze( Time.parse('2016-04-13 14:04:02 +0900') ) + + @i.enqueue_thread_wait + @i.flush_thread_wakeup + + waiting(4) do + Thread.pass until @i.write_count > 0 + end + + assert{ @i.buffer.stage.size == 2 } + assert{ @i.write_count == 1 } + assert{ @i.buffer.queue.size == 0 } + assert{ @i.buffer.dequeued.size == 1 } + + # events fulfills a chunk (and queued immediately) + assert_equal 5, ary.size + assert_equal 5, ary.select{|e| e[0] == "test.tag.1" }.size + assert_equal 0, ary.select{|e| e[0] == "test.tag.2" }.size + + assert_equal 1, chunks.size + assert !chunks.first.empty? + + Timecop.freeze( Time.parse('2016-04-13 14:04:09 +0900') ) + + @i.enqueue_thread_wait + + assert{ @i.buffer.stage.size == 2 } + + # to trigger try_flush with flush_burst_interval + Timecop.freeze( Time.parse('2016-04-13 14:04:11 +0900') ) + @i.enqueue_thread_wait + Timecop.freeze( Time.parse('2016-04-13 14:04:12 +0900') ) + @i.enqueue_thread_wait + Timecop.freeze( Time.parse('2016-04-13 14:04:13 +0900') ) + @i.enqueue_thread_wait + Timecop.freeze( Time.parse('2016-04-13 14:04:14 +0900') ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + + assert{ @i.buffer.stage.size == 0 } + + waiting(4) do + Thread.pass until @i.write_count > 2 + end + + assert{ @i.buffer.stage.size == 0 && @i.write_count == 3 } + assert{ @i.buffer.dequeued.size == 3 } + + assert_equal 11, ary.size + assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size + assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size + + assert_equal 3, chunks.size + assert chunks.all?{|c| !c.empty? } + + assert metachecks.all?{|e| e } + + @i.interrupt_flushes + + @i.rollback_write(chunks[2].unique_id) + + assert{ @i.buffer.dequeued.size == 2 } + assert{ @i.buffer.queue.size == 1 && @i.buffer.queue.first.unique_id == chunks[2].unique_id } + + Timecop.freeze( Time.parse('2016-04-13 14:04:15 +0900') ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + + waiting(4) do + Thread.pass until @i.write_count > 3 + end + + assert{ @i.write_count == 4 } + assert{ @i.rollback_count == 1 } + assert{ @i.instance_eval{ @dequeued_chunks.size } == 3 } + assert{ @i.buffer.dequeued.size == 3 } + assert{ @i.buffer.queue.size == 0 } + + assert_equal 4, chunks.size + assert chunks[2].unique_id == chunks[3].unique_id + + ary.reject!{|e| true } + chunks.reject!{|e| true } + + Timecop.freeze( Time.parse('2016-04-13 14:04:46 +0900') ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + + waiting(4) do + Thread.pass until @i.rollback_count == 4 + end + + assert{ chunks[0...3].all?{|c| !c.empty? } } + + # rollback is in progress, but some may be flushed again after rollback + Timecop.freeze( Time.parse('2016-04-13 14:04:46 +0900') ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + + waiting(4) do + Thread.pass until @i.write_count == 7 + end + + assert{ @i.write_count == 7 } + assert_equal 11, ary.size + assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size + assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size + assert{ chunks.size == 3 } + assert{ chunks.all?{|c| !c.empty? } } + + chunks.each{|c| @i.commit_write(c.unique_id) } + assert{ chunks.all?{|c| c.empty? } } + + assert{ @i.buffer.dequeued.size == 0 } + end + + test '#try_rollback_all will be called for all waiting chunks after shutdown' do + Timecop.freeze( Time.parse('2016-04-13 14:04:01 +0900') ) + + ary = [] + metachecks = [] + chunks = [] + + @i.register(:format){|tag,time,record| [tag,time,record].to_json + "\n" } + @i.register(:try_write) do |chunk| + chunks << chunk + chunk.read.split("\n").reject{|l| l.empty? }.each do |data| + e = JSON.parse(data) + ary << e + metachecks << (e[0] == chunk.metadata.tag) + end + end + + @i.thread_wait_until_start + + # size of a event is 195 + dummy_data = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + events = [ + ["test.tag.1", event_time('2016-04-13 14:03:21 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], + ["test.tag.2", event_time('2016-04-13 14:03:23 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}], + ["test.tag.1", event_time('2016-04-13 14:03:29 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], + ["test.tag.1", event_time('2016-04-13 14:03:30 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], + ["test.tag.1", event_time('2016-04-13 14:03:33 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], + ["test.tag.1", event_time('2016-04-13 14:03:38 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "b"}], + ["test.tag.1", event_time('2016-04-13 14:03:43 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], + ["test.tag.1", event_time('2016-04-13 14:03:49 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "b"}], + ["test.tag.2", event_time('2016-04-13 14:03:51 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}], + ["test.tag.1", event_time('2016-04-13 14:04:00 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], + ["test.tag.2", event_time('2016-04-13 14:04:01 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}], + ] + + assert_equal 0, @i.write_count + + @i.interrupt_flushes + + events.shuffle.each do |tag, time, record| + @i.emit(tag, [ [time, record] ]) + end + assert{ @i.buffer.stage.size == 2 } + + Timecop.freeze( Time.parse('2016-04-13 14:04:02 +0900') ) + + @i.enqueue_thread_wait + @i.flush_thread_wakeup + + waiting(4) do + Thread.pass until @i.write_count > 0 + end + + assert{ @i.buffer.stage.size == 2 } + assert{ @i.write_count == 1 } + assert{ @i.buffer.queue.size == 0 } + assert{ @i.buffer.dequeued.size == 1 } + + # events fulfills a chunk (and queued immediately) + assert_equal 5, ary.size + assert_equal 5, ary.select{|e| e[0] == "test.tag.1" }.size + assert_equal 0, ary.select{|e| e[0] == "test.tag.2" }.size + + assert_equal 1, chunks.size + assert !chunks.first.empty? + + Timecop.freeze( Time.parse('2016-04-13 14:04:09 +0900') ) + + @i.enqueue_thread_wait + + assert{ @i.buffer.stage.size == 2 } + + # to trigger try_flush with flush_burst_interval + Timecop.freeze( Time.parse('2016-04-13 14:04:11 +0900') ) + @i.enqueue_thread_wait + Timecop.freeze( Time.parse('2016-04-13 14:04:12 +0900') ) + @i.enqueue_thread_wait + Timecop.freeze( Time.parse('2016-04-13 14:04:13 +0900') ) + @i.enqueue_thread_wait + Timecop.freeze( Time.parse('2016-04-13 14:04:14 +0900') ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + + assert{ @i.buffer.stage.size == 0 } + + waiting(4) do + Thread.pass until @i.write_count > 2 + end + + assert{ @i.buffer.stage.size == 0 } + assert{ @i.buffer.queue.size == 0 } + assert{ @i.buffer.dequeued.size == 3 } + assert{ @i.write_count == 3 } + assert{ @i.rollback_count == 0 } + + assert_equal 11, ary.size + assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size + assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size + + assert{ chunks.size == 3 } + assert{ chunks.all?{|c| !c.empty? } } + + @i.register(:shutdown_hook){ @i.commit_write(chunks[1].unique_id) } + + @i.stop + @i.before_shutdown + @i.shutdown + + assert{ @i.buffer.dequeued.size == 2 } + assert{ !chunks[0].empty? } + assert{ chunks[1].empty? } + assert{ !chunks[2].empty? } + + @i.after_shutdown + + assert{ @i.rollback_count == 2 } + end + end +end diff --git a/test/plugin/test_output_as_buffered_retries.rb b/test/plugin/test_output_as_buffered_retries.rb new file mode 100644 index 0000000000..1332c8e88e --- /dev/null +++ b/test/plugin/test_output_as_buffered_retries.rb @@ -0,0 +1,786 @@ +require_relative '../helper' +require 'fluent/plugin/output' +require 'fluent/plugin/buffer' + +require 'json' +require 'time' +require 'timeout' +require 'timecop' + +module FluentPluginOutputAsBufferedRetryTest + class DummyBareOutput < Fluent::Plugin::Output + def register(name, &block) + instance_variable_set("@#{name}", block) + end + end + class DummySyncOutput < DummyBareOutput + def process(tag, es) + @process ? @process.call(tag, es) : nil + end + end + class DummyFullFeatureOutput < DummyBareOutput + def prefer_buffered_processing + @prefer_buffered_processing ? @prefer_buffered_processing.call : false + end + def prefer_delayed_commit + @prefer_delayed_commit ? @prefer_delayed_commit.call : false + end + def process(tag, es) + @process ? @process.call(tag, es) : nil + end + def format(tag, time, record) + @format ? @format.call(tag, time, record) : [tag, time, record].to_json + end + def write(chunk) + @write ? @write.call(chunk) : nil + end + def try_write(chunk) + @try_write ? @try_write.call(chunk) : nil + end + end + class DummyFullFeatureOutput2 < DummyFullFeatureOutput + def prefer_buffered_processing; true; end + def prefer_delayed_commit; super; end + def format(tag, time, record); super; end + def write(chunk); super; end + def try_write(chunk); super; end + end +end + +class BufferedOutputRetryTest < Test::Unit::TestCase + def create_output(type=:full) + case type + when :bare then FluentPluginOutputAsBufferedRetryTest::DummyBareOutput.new + when :sync then FluentPluginOutputAsBufferedRetryTest::DummySyncOutput.new + when :full then FluentPluginOutputAsBufferedRetryTest::DummyFullFeatureOutput.new + else + raise ArgumentError, "unknown type: #{type}" + end + end + def create_metadata(timekey: nil, tag: nil, variables: nil) + Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables) + end + def waiting(seconds) + begin + Timeout.timeout(seconds) do + yield + end + rescue Timeout::Error + STDERR.print *(@i.log.out.logs) + raise + end + end + def dummy_event_stream + [ + [ event_time('2016-04-13 18:33:00'), {"name" => "moris", "age" => 36, "message" => "data1"} ], + [ event_time('2016-04-13 18:33:13'), {"name" => "moris", "age" => 36, "message" => "data2"} ], + [ event_time('2016-04-13 18:33:32'), {"name" => "moris", "age" => 36, "message" => "data3"} ], + ] + end + + teardown do + if @i + @i.stop unless @i.stopped? + @i.before_shutdown unless @i.before_shutdown? + @i.shutdown unless @i.shutdown? + @i.after_shutdown unless @i.after_shutdown? + @i.close unless @i.closed? + @i.terminate unless @i.terminated? + end + Timecop.return + end + + sub_test_case 'buffered output for retries with exponential backoff' do + test 'exponential backoff is default strategy for retries' do + chunk_key = 'tag' + hash = { + 'flush_interval' => 1, + 'flush_burst_interval' => 0.1, + 'retry_randomize' => false, + } + @i = create_output() + @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)])) + @i.register(:prefer_buffered_processing){ true } + @i.start + + assert_equal :expbackoff, @i.buffer_config.retry_type + assert_equal 1, @i.buffer_config.retry_wait + assert_equal 2.0, @i.buffer_config.retry_backoff_base + assert !@i.buffer_config.retry_randomize + + now = Time.parse('2016-04-13 18:17:00 -0700') + Timecop.freeze( now ) + + retry_state = @i.retry_state( @i.buffer_config.retry_randomize ) + retry_state.step + assert_equal 1, (retry_state.next_time - now) + retry_state.step + assert_equal (1 * (2 ** 1)), (retry_state.next_time - now) + retry_state.step + assert_equal (1 * (2 ** 2)), (retry_state.next_time - now) + retry_state.step + assert_equal (1 * (2 ** 3)), (retry_state.next_time - now) + end + + test 'does retries correctly when #write fails' do + chunk_key = 'tag' + hash = { + 'flush_interval' => 1, + 'flush_burst_interval' => 0.1, + 'retry_randomize' => false, + 'retry_max_interval' => 60 * 60, + } + @i = create_output() + @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)])) + @i.register(:prefer_buffered_processing){ true } + @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" } + @i.register(:write){|chunk| raise "yay, your #write must fail" } + @i.start + + now = Time.parse('2016-04-13 18:33:30 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.1", dummy_event_stream()) + + now = Time.parse('2016-04-13 18:33:32 -0700') + Timecop.freeze( now ) + + @i.enqueue_thread_wait + + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > 0 } + + assert{ @i.write_count > 0 } + assert{ @i.num_errors > 0 } + + now = @i.next_flush_time + Timecop.freeze( now ) + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > 1 } + + assert{ @i.write_count > 1 } + assert{ @i.num_errors > 1 } + end + + test 'max retry interval is limited by retry_max_interval' do + chunk_key = 'tag' + hash = { + 'flush_interval' => 1, + 'flush_burst_interval' => 0.1, + 'retry_randomize' => false, + 'retry_max_interval' => 60, + } + @i = create_output() + @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)])) + @i.register(:prefer_buffered_processing){ true } + @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" } + @i.register(:write){|chunk| raise "yay, your #write must fail" } + @i.start + + now = Time.parse('2016-04-13 18:33:30 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.1", dummy_event_stream()) + + now = Time.parse('2016-04-13 18:33:32 -0700') + Timecop.freeze( now ) + + @i.enqueue_thread_wait + + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 } + + assert{ @i.write_count > 0 } + assert{ @i.num_errors > 0 } + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + + 10.times do + now = @i.next_flush_time + Timecop.freeze( now ) + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > prev_write_count && @i.num_errors > prev_num_errors } + + assert{ @i.write_count > prev_write_count } + assert{ @i.num_errors > prev_num_errors } + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + end + # expbackoff interval: 1 * 2 ** 10 == 1024 + # but it should be limited by retry_max_interval=60 + assert_equal 60, (@i.next_flush_time - now) + end + + test 'output plugin give retries up by retry_timeout, and clear queue in buffer' do + written_tags = [] + + chunk_key = 'tag' + hash = { + 'flush_interval' => 1, + 'flush_burst_interval' => 0.1, + 'retry_randomize' => false, + 'retry_timeout' => 3600, + } + @i = create_output() + @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)])) + @i.register(:prefer_buffered_processing){ true } + @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" } + @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise "yay, your #write must fail" } + @i.start + + now = Time.parse('2016-04-13 18:33:30 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.1", dummy_event_stream()) + + now = Time.parse('2016-04-13 18:33:31 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.2", dummy_event_stream()) + + assert_equal 0, @i.write_count + assert_equal 0, @i.num_errors + + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 } + + assert{ @i.buffer.queue.size > 0 } + assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' } + + assert{ @i.write_count > 0 } + assert{ @i.num_errors > 0 } + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + + first_failure = @i.retry.start + + 15.times do |i| # large enough + now = @i.next_flush_time + # p({i: i, now: now, diff: (now - Time.now)}) + # * if loop count is 12: + # {:i=>0, :now=>2016-04-13 18:33:32 -0700, :diff=>1.0} + # {:i=>1, :now=>2016-04-13 18:33:34 -0700, :diff=>2.0} + # {:i=>2, :now=>2016-04-13 18:33:38 -0700, :diff=>4.0} + # {:i=>3, :now=>2016-04-13 18:33:46 -0700, :diff=>8.0} + # {:i=>4, :now=>2016-04-13 18:34:02 -0700, :diff=>16.0} + # {:i=>5, :now=>2016-04-13 18:34:34 -0700, :diff=>32.0} + # {:i=>6, :now=>2016-04-13 18:35:38 -0700, :diff=>64.0} + # {:i=>7, :now=>2016-04-13 18:37:46 -0700, :diff=>128.0} + # {:i=>8, :now=>2016-04-13 18:42:02 -0700, :diff=>256.0} + # {:i=>9, :now=>2016-04-13 18:50:34 -0700, :diff=>512.0} + # {:i=>10, :now=>2016-04-13 19:07:38 -0700, :diff=>1024.0} + # {:i=>11, :now=>2016-04-13 19:33:31 -0700, :diff=>1553.0} # clear_queue! + + Timecop.freeze( now ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > prev_write_count && @i.num_errors > prev_num_errors } + + assert{ @i.write_count > prev_write_count } + assert{ @i.num_errors > prev_num_errors } + + break if @i.buffer.queue.size == 0 + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + + assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' } + end + assert{ now >= first_failure + 3600 } + + assert{ @i.buffer.stage.size == 0 } + assert{ written_tags.all?{|t| t == 'test.tag.1' } } + + @i.emit("test.tag.3", dummy_event_stream()) + + logs = @i.log.out.logs + assert{ logs.any?{|l| l.include?("[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.") } } + end + + test 'output plugin give retries up by retry_max_times, and clear queue in buffer' do + written_tags = [] + + chunk_key = 'tag' + hash = { + 'flush_interval' => 1, + 'flush_burst_interval' => 0.1, + 'retry_randomize' => false, + 'retry_max_times' => 10, + } + @i = create_output() + @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)])) + @i.register(:prefer_buffered_processing){ true } + @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" } + @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise "yay, your #write must fail" } + @i.start + + now = Time.parse('2016-04-13 18:33:30 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.1", dummy_event_stream()) + + now = Time.parse('2016-04-13 18:33:31 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.2", dummy_event_stream()) + + assert_equal 0, @i.write_count + assert_equal 0, @i.num_errors + + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 } + + assert{ @i.buffer.queue.size > 0 } + assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' } + + assert{ @i.write_count > 0 } + assert{ @i.num_errors > 0 } + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + + first_failure = @i.retry.start + + chunks = @i.buffer.queue.dup + + 20.times do |i| # large times enough + now = @i.next_flush_time + + Timecop.freeze( now ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > prev_write_count && @i.num_errors > prev_num_errors } + + assert{ @i.write_count > prev_write_count } + assert{ @i.num_errors > prev_num_errors } + + break if @i.buffer.queue.size == 0 + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + + assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' } + end + assert{ @i.buffer.stage.size == 0 } + assert{ written_tags.all?{|t| t == 'test.tag.1' } } + + @i.emit("test.tag.3", dummy_event_stream()) + + logs = @i.log.out.logs + assert{ logs.any?{|l| l.include?("[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=10") } } + + assert{ @i.buffer.queue.size == 0 } + assert{ @i.buffer.stage.size == 1 } + assert{ chunks.all?{|c| c.empty? } } + end + end + + sub_test_case 'bufferd output for retries with periodical retry' do + test 'periodical retries should retry to write in failing status per retry_wait' do + chunk_key = 'tag' + hash = { + 'flush_interval' => 1, + 'flush_burst_interval' => 0.1, + 'retry_type' => :periodic, + 'retry_wait' => 3, + 'retry_randomize' => false, + } + @i = create_output() + @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)])) + @i.register(:prefer_buffered_processing){ true } + @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" } + @i.register(:write){|chunk| raise "yay, your #write must fail" } + @i.start + + now = Time.parse('2016-04-13 18:33:30 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.1", dummy_event_stream()) + + now = Time.parse('2016-04-13 18:33:32 -0700') + Timecop.freeze( now ) + + @i.enqueue_thread_wait + + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > 0 } + + assert{ @i.write_count > 0 } + assert{ @i.num_errors > 0 } + + now = @i.next_flush_time + Timecop.freeze( now ) + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > 1 } + + assert{ @i.write_count > 1 } + assert{ @i.num_errors > 1 } + end + + test 'output plugin give retries up by retry_timeout, and clear queue in buffer' do + written_tags = [] + + chunk_key = 'tag' + hash = { + 'flush_interval' => 1, + 'flush_burst_interval' => 0.1, + 'retry_type' => :periodic, + 'retry_wait' => 30, + 'retry_randomize' => false, + 'retry_timeout' => 120, + } + @i = create_output() + @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)])) + @i.register(:prefer_buffered_processing){ true } + @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" } + @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise "yay, your #write must fail" } + @i.start + + now = Time.parse('2016-04-13 18:33:30 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.1", dummy_event_stream()) + + now = Time.parse('2016-04-13 18:33:31 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.2", dummy_event_stream()) + + assert_equal 0, @i.write_count + assert_equal 0, @i.num_errors + + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 } + + assert{ @i.buffer.queue.size > 0 } + assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' } + + assert{ @i.write_count > 0 } + assert{ @i.num_errors > 0 } + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + + first_failure = @i.retry.start + + 3.times do |i| + now = @i.next_flush_time + + Timecop.freeze( now ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > prev_write_count && @i.num_errors > prev_num_errors } + + assert{ @i.write_count > prev_write_count } + assert{ @i.num_errors > prev_num_errors } + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + end + + assert{ @i.next_flush_time >= first_failure + 120 } + + assert{ @i.buffer.queue.size == 2 } + assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' } + assert{ @i.buffer.stage.size == 0 } + + assert{ written_tags.all?{|t| t == 'test.tag.1' } } + + chunks = @i.buffer.queue.dup + + @i.emit("test.tag.3", dummy_event_stream()) + + now = @i.next_flush_time + Timecop.freeze( now ) + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > prev_write_count && @i.num_errors > prev_num_errors } + + logs = @i.log.out.logs + assert{ logs.any?{|l| l.start_with?("2016-04-13 18:35:31 -0700 [error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.") } } + assert{ @i.buffer.queue.size == 0 } + assert{ @i.buffer.stage.size == 1 } + assert{ chunks.all?{|c| c.empty? } } + end + + test 'retry_max_times can limit maximum times for retries' do + written_tags = [] + + chunk_key = 'tag' + hash = { + 'flush_interval' => 1, + 'flush_burst_interval' => 0.1, + 'retry_type' => :periodic, + 'retry_wait' => 3, + 'retry_randomize' => false, + 'retry_max_times' => 10, + } + @i = create_output() + @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)])) + @i.register(:prefer_buffered_processing){ true } + @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" } + @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise "yay, your #write must fail" } + @i.start + + now = Time.parse('2016-04-13 18:33:30 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.1", dummy_event_stream()) + + now = Time.parse('2016-04-13 18:33:31 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.2", dummy_event_stream()) + + assert_equal 0, @i.write_count + assert_equal 0, @i.num_errors + + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 } + + assert{ @i.buffer.queue.size > 0 } + assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' } + + assert{ @i.write_count > 0 } + assert{ @i.num_errors > 0 } + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + + first_failure = @i.retry.start + + chunks = @i.buffer.queue.dup + + 20.times do |i| + now = @i.next_flush_time + + Timecop.freeze( now ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > prev_write_count && @i.num_errors > prev_num_errors } + + assert{ @i.write_count > prev_write_count } + assert{ @i.num_errors > prev_num_errors } + + break if @i.buffer.queue.size == 0 + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + + assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' } + end + assert{ @i.buffer.stage.size == 0 } + assert{ written_tags.all?{|t| t == 'test.tag.1' } } + + + @i.emit("test.tag.3", dummy_event_stream()) + + logs = @i.log.out.logs + assert{ logs.any?{|l| l.include?("[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=10") } } + + assert{ @i.buffer.queue.size == 0 } + assert{ @i.buffer.stage.size == 1 } + assert{ chunks.all?{|c| c.empty? } } + end + end + + sub_test_case 'buffered output configured as retry_forever' do + test 'configuration error will be raised if secondary section is configured' do + chunk_key = 'tag' + hash = { + 'retry_forever' => true, + 'retry_randomize' => false, + } + i = create_output() + assert_raise Fluent::ConfigError do + i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash),config_element('secondary','')])) + end + end + + test 'retry_timeout and retry_max_times will be ignored if retry_forever is true for exponential backoff' do + written_tags = [] + + chunk_key = 'tag' + hash = { + 'flush_interval' => 1, + 'flush_burst_interval' => 0.1, + 'retry_type' => :expbackoff, + 'retry_forever' => true, + 'retry_randomize' => false, + 'retry_timeout' => 3600, + 'retry_max_times' => 10, + } + @i = create_output() + @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)])) + @i.register(:prefer_buffered_processing){ true } + @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" } + @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise "yay, your #write must fail" } + @i.start + + now = Time.parse('2016-04-13 18:33:30 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.1", dummy_event_stream()) + + now = Time.parse('2016-04-13 18:33:31 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.2", dummy_event_stream()) + + assert_equal 0, @i.write_count + assert_equal 0, @i.num_errors + + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 } + + assert{ @i.buffer.queue.size > 0 } + assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' } + + assert{ @i.write_count > 0 } + assert{ @i.num_errors > 0 } + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + + first_failure = @i.retry.start + + 15.times do |i| + now = @i.next_flush_time + + Timecop.freeze( now ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > prev_write_count && @i.num_errors > prev_num_errors } + + assert{ @i.write_count > prev_write_count } + assert{ @i.num_errors > prev_num_errors } + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + end + + assert{ @i.buffer.queue.size == 2 } + assert{ @i.retry.steps > 10 } + assert{ now > first_failure + 3600 } + end + + test 'retry_timeout and retry_max_times will be ignored if retry_forever is true for periodical retries' do + written_tags = [] + + chunk_key = 'tag' + hash = { + 'flush_interval' => 1, + 'flush_burst_interval' => 0.1, + 'retry_type' => :periodic, + 'retry_forever' => true, + 'retry_randomize' => false, + 'retry_wait' => 30, + 'retry_timeout' => 360, + 'retry_max_times' => 10, + } + @i = create_output() + @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)])) + @i.register(:prefer_buffered_processing){ true } + @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" } + @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise "yay, your #write must fail" } + @i.start + + now = Time.parse('2016-04-13 18:33:30 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.1", dummy_event_stream()) + + now = Time.parse('2016-04-13 18:33:31 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.2", dummy_event_stream()) + + assert_equal 0, @i.write_count + assert_equal 0, @i.num_errors + + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 } + + assert{ @i.buffer.queue.size > 0 } + assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' } + + assert{ @i.write_count > 0 } + assert{ @i.num_errors > 0 } + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + + first_failure = @i.retry.start + + 15.times do |i| + now = @i.next_flush_time + + Timecop.freeze( now ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > prev_write_count && @i.num_errors > prev_num_errors } + + assert{ @i.write_count > prev_write_count } + assert{ @i.num_errors > prev_num_errors } + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + end + + assert{ @i.buffer.queue.size == 2 } + assert{ @i.retry.steps > 10 } + assert{ now > first_failure + 360 } + end + end + + sub_test_case 'buffered output with delayed commit' do + test 'does retries correctly when #try_write fails' do + chunk_key = 'tag' + hash = { + 'flush_interval' => 1, + 'flush_burst_interval' => 0.1, + 'retry_randomize' => false, + 'retry_max_interval' => 60 * 60, + } + @i = create_output() + @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)])) + @i.register(:prefer_buffered_processing){ true } + @i.register(:prefer_delayed_commit){ true } + @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" } + @i.register(:try_write){|chunk| raise "yay, your #write must fail" } + @i.start + + now = Time.parse('2016-04-13 18:33:30 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.1", dummy_event_stream()) + + now = Time.parse('2016-04-13 18:33:32 -0700') + Timecop.freeze( now ) + + @i.enqueue_thread_wait + + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > 0 } + + assert{ @i.write_count > 0 } + assert{ @i.num_errors > 0 } + + now = @i.next_flush_time + Timecop.freeze( now ) + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > 1 } + + assert{ @i.write_count > 1 } + assert{ @i.num_errors > 1 } + end + end +end diff --git a/test/plugin/test_output_as_buffered_secondary.rb b/test/plugin/test_output_as_buffered_secondary.rb new file mode 100644 index 0000000000..e5af89f574 --- /dev/null +++ b/test/plugin/test_output_as_buffered_secondary.rb @@ -0,0 +1,766 @@ +require_relative '../helper' +require 'fluent/plugin/output' +require 'fluent/plugin/buffer' + +require 'json' +require 'time' +require 'timeout' +require 'timecop' + +module FluentPluginOutputAsBufferedSecondaryTest + class DummyBareOutput < Fluent::Plugin::Output + def register(name, &block) + instance_variable_set("@#{name}", block) + end + end + class DummySyncOutput < DummyBareOutput + def process(tag, es) + @process ? @process.call(tag, es) : nil + end + end + class DummyFullFeatureOutput < DummyBareOutput + def prefer_buffered_processing + @prefer_buffered_processing ? @prefer_buffered_processing.call : false + end + def prefer_delayed_commit + @prefer_delayed_commit ? @prefer_delayed_commit.call : false + end + def process(tag, es) + @process ? @process.call(tag, es) : nil + end + def format(tag, time, record) + @format ? @format.call(tag, time, record) : [tag, time, record].to_json + end + def write(chunk) + @write ? @write.call(chunk) : nil + end + def try_write(chunk) + @try_write ? @try_write.call(chunk) : nil + end + end + class DummyFullFeatureOutput2 < DummyFullFeatureOutput + def prefer_buffered_processing; true; end + def prefer_delayed_commit; super; end + def format(tag, time, record); super; end + def write(chunk); super; end + def try_write(chunk); super; end + end +end + +class BufferedOutputSecondaryTest < Test::Unit::TestCase + def create_output(type=:full) + case type + when :bare then FluentPluginOutputAsBufferedSecondaryTest::DummyBareOutput.new + when :sync then FluentPluginOutputAsBufferedSecondaryTest::DummySyncOutput.new + when :full then FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput.new + else + raise ArgumentError, "unknown type: #{type}" + end + end + def create_metadata(timekey: nil, tag: nil, variables: nil) + Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables) + end + def waiting(seconds) + begin + Timeout.timeout(seconds) do + yield + end + rescue Timeout::Error + STDERR.print *(@i.log.out.logs) + raise + end + end + def dummy_event_stream + [ + [ event_time('2016-04-13 18:33:00'), {"name" => "moris", "age" => 36, "message" => "data1"} ], + [ event_time('2016-04-13 18:33:13'), {"name" => "moris", "age" => 36, "message" => "data2"} ], + [ event_time('2016-04-13 18:33:32'), {"name" => "moris", "age" => 36, "message" => "data3"} ], + ] + end + + teardown do + if @i + @i.stop unless @i.stopped? + @i.before_shutdown unless @i.before_shutdown? + @i.shutdown unless @i.shutdown? + @i.after_shutdown unless @i.after_shutdown? + @i.close unless @i.closed? + @i.terminate unless @i.terminated? + end + Timecop.return + end + + sub_test_case 'secondary plugin feature for buffered output with periodical retry' do + setup do + Fluent::Plugin.register_output('output_secondary_test', FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput) + Fluent::Plugin.register_output('output_secondary_test2', FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput2) + end + + test 'raises configuration error if primary does not support buffering' do + i = create_output(:sync) + assert_raise Fluent::ConfigError do + i.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'output_secondary_test'})])) + end + end + + test 'raises configuration error if / section is specified in section' do + priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 30, 'retry_randomize' => false}) + secconf1 = config_element('secondary','',{'@type' => 'output_secondary_test'},[config_element('buffer', 'time')]) + secconf2 = config_element('secondary','',{'@type' => 'output_secondary_test'},[config_element('secondary', '')]) + i = create_output() + assert_raise Fluent::ConfigError do + i.configure(config_element('ROOT','',{},[priconf,secconf1])) + end + assert_raise Fluent::ConfigError do + i.configure(config_element('ROOT','',{},[priconf,secconf2])) + end + end + + test 'warns if secondary plugin is different type from primary one' do + priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 30, 'retry_randomize' => false}) + secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'}) + i = create_output() + i.configure(config_element('ROOT','',{},[priconf,secconf])) + logs = i.log.out.logs + assert{ logs.any?{|l| l.include?("secondary type should be same with primary one") } } + end + + test 'secondary plugin lifecycle is kicked by primary' do + priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 30, 'retry_randomize' => false}) + secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'}) + i = create_output() + i.configure(config_element('ROOT','',{},[priconf,secconf])) + logs = i.log.out.logs + assert{ logs.any?{|l| l.include?("secondary type should be same with primary one") } } + + assert i.secondary.configured? + + assert !i.secondary.started? + i.start + assert i.secondary.started? + + assert !i.secondary.stopped? + i.stop + assert i.secondary.stopped? + + assert !i.secondary.before_shutdown? + i.before_shutdown + assert i.secondary.before_shutdown? + + assert !i.secondary.shutdown? + i.shutdown + assert i.secondary.shutdown? + + assert !i.secondary.after_shutdown? + i.after_shutdown + assert i.secondary.after_shutdown? + + assert !i.secondary.closed? + i.close + assert i.secondary.closed? + + assert !i.secondary.terminated? + i.terminate + assert i.secondary.terminated? + end + + test 'primary plugin will emit event streams to secondary after retries for time of retry_timeout * retry_secondary_threshold' do + written = [] + priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'retry_randomize' => false}) + secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'}) + @i = create_output() + @i.configure(config_element('ROOT','',{},[priconf,secconf])) + @i.register(:prefer_buffered_processing){ true } + @i.register(:prefer_delayed_commit){ false } + @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" } + @i.register(:write){|chunk| raise "yay, your #write must fail" } + @i.secondary.register(:prefer_delayed_commit){ false } + @i.secondary.register(:write){|chunk| chunk.read.split("\n").each{|line| written << JSON.parse(line) } } + @i.start + + now = Time.parse('2016-04-13 18:33:30 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.1", dummy_event_stream()) + + now = Time.parse('2016-04-13 18:33:31 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.2", dummy_event_stream()) + + assert_equal 0, @i.write_count + assert_equal 0, @i.num_errors + + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 } + + assert{ @i.buffer.queue.size > 0 } + assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' } + + assert{ @i.write_count > 0 } + assert{ @i.num_errors > 0 } + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + + first_failure = @i.retry.start + + # retry_timeout == 60(sec), retry_secondary_threshold == 0.8 + + now = first_failure + 60 * 0.8 + 1 + + Timecop.freeze( now ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > prev_write_count } + + assert{ @i.write_count > prev_write_count } + assert{ @i.num_errors == prev_num_errors } + + assert_nil @i.retry + + assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {"name" => "moris", "age" => 36, "message" => "data1"} ], written[0] + assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1] + assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2] + + assert{ @i.log.out.logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } } + end + + test 'secondary can do non-delayed commit even if primary do delayed commit' do + written = [] + priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'retry_randomize' => false}) + secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'}) + @i = create_output() + @i.configure(config_element('ROOT','',{},[priconf,secconf])) + @i.register(:prefer_buffered_processing){ true } + @i.register(:prefer_delayed_commit){ true } + @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" } + @i.register(:try_write){|chunk| raise "yay, your #write must fail" } + @i.secondary.register(:prefer_delayed_commit){ false } + @i.secondary.register(:write){|chunk| chunk.read.split("\n").each{|line| written << JSON.parse(line) } } + @i.start + + now = Time.parse('2016-04-13 18:33:30 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.1", dummy_event_stream()) + + now = Time.parse('2016-04-13 18:33:31 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.2", dummy_event_stream()) + + assert_equal 0, @i.write_count + assert_equal 0, @i.num_errors + + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 } + + assert{ @i.buffer.queue.size > 0 } + assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' } + + assert{ @i.write_count > 0 } + assert{ @i.num_errors > 0 } + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + + first_failure = @i.retry.start + + # retry_timeout == 60(sec), retry_secondary_threshold == 0.8 + + now = first_failure + 60 * 0.8 + 1 + + Timecop.freeze( now ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > prev_write_count } + + assert{ @i.write_count > prev_write_count } + assert{ @i.num_errors == prev_num_errors } + + assert_nil @i.retry + + assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {"name" => "moris", "age" => 36, "message" => "data1"} ], written[0] + assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1] + assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2] + + assert{ @i.log.out.logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } } + end + + test 'secondary plugin can do delayed commit if primary do it' do + written = [] + chunks = [] + priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'retry_randomize' => false}) + secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'}) + @i = create_output() + @i.configure(config_element('ROOT','',{},[priconf,secconf])) + @i.register(:prefer_buffered_processing){ true } + @i.register(:prefer_delayed_commit){ true } + @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" } + @i.register(:try_write){|chunk| raise "yay, your #write must fail" } + @i.secondary.register(:prefer_delayed_commit){ true } + @i.secondary.register(:try_write){|chunk| chunks << chunk; chunk.read.split("\n").each{|line| written << JSON.parse(line) } } + @i.start + + now = Time.parse('2016-04-13 18:33:30 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.1", dummy_event_stream()) + + now = Time.parse('2016-04-13 18:33:31 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.2", dummy_event_stream()) + + assert_equal 0, @i.write_count + assert_equal 0, @i.num_errors + + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 } + + assert{ @i.buffer.queue.size > 0 } + assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' } + + assert{ @i.write_count > 0 } + assert{ @i.num_errors > 0 } + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + + first_failure = @i.retry.start + + # retry_timeout == 60(sec), retry_secondary_threshold == 0.8 + + now = first_failure + 60 * 0.8 + 1 + + Timecop.freeze( now ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > prev_write_count } + + assert{ @i.write_count > prev_write_count } + assert{ @i.num_errors == prev_num_errors } + + assert @i.retry + + assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {"name" => "moris", "age" => 36, "message" => "data1"} ], written[0] + assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1] + assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2] + + assert{ @i.buffer.dequeued.size > 0 } + assert{ chunks.size > 0 } + assert{ !chunks.first.empty? } + + @i.secondary.commit_write(chunks[0].unique_id) + + assert{ @i.buffer.dequeued[chunks[0].unique_id].nil? } + assert{ chunks.first.empty? } + + assert_nil @i.retry + + assert{ @i.log.out.logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } } + end + + test 'secondary plugin can do delayed commit even if primary does not do it' do + written = [] + chunks = [] + priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'retry_randomize' => false}) + secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'}) + @i = create_output() + @i.configure(config_element('ROOT','',{},[priconf,secconf])) + @i.register(:prefer_buffered_processing){ true } + @i.register(:prefer_delayed_commit){ false } + @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" } + @i.register(:write){|chunk| raise "yay, your #write must fail" } + @i.secondary.register(:prefer_delayed_commit){ true } + @i.secondary.register(:try_write){|chunk| chunks << chunk; chunk.read.split("\n").each{|line| written << JSON.parse(line) } } + @i.start + + now = Time.parse('2016-04-13 18:33:30 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.1", dummy_event_stream()) + + now = Time.parse('2016-04-13 18:33:31 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.2", dummy_event_stream()) + + assert_equal 0, @i.write_count + assert_equal 0, @i.num_errors + + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 } + + assert{ @i.buffer.queue.size > 0 } + assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' } + + assert{ @i.write_count > 0 } + assert{ @i.num_errors > 0 } + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + + first_failure = @i.retry.start + + # retry_timeout == 60(sec), retry_secondary_threshold == 0.8 + + now = first_failure + 60 * 0.8 + 1 + + Timecop.freeze( now ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > prev_write_count } + + assert{ @i.write_count > prev_write_count } + assert{ @i.num_errors == prev_num_errors } + + assert @i.retry + + assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {"name" => "moris", "age" => 36, "message" => "data1"} ], written[0] + assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1] + assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2] + + assert{ @i.buffer.dequeued.size > 0 } + assert{ chunks.size > 0 } + assert{ !chunks.first.empty? } + + @i.secondary.commit_write(chunks[0].unique_id) + + assert{ @i.buffer.dequeued[chunks[0].unique_id].nil? } + assert{ chunks.first.empty? } + + assert_nil @i.retry + + assert{ @i.log.out.logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } } + end + + test 'secondary plugin can do delayed commit even if primary does not do it, and non-committed chunks will be rollbacked by primary' do + written = [] + chunks = [] + priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'delayed_commit_timeout' => 2, 'retry_randomize' => false}) + secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'}) + @i = create_output() + @i.configure(config_element('ROOT','',{},[priconf,secconf])) + @i.register(:prefer_buffered_processing){ true } + @i.register(:prefer_delayed_commit){ false } + @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" } + @i.register(:write){|chunk| raise "yay, your #write must fail" } + @i.secondary.register(:prefer_delayed_commit){ true } + @i.secondary.register(:try_write){|chunk| chunks << chunk; chunk.read.split("\n").each{|line| written << JSON.parse(line) } } + @i.secondary.register(:write){|chunk| raise "don't use this" } + @i.start + + now = Time.parse('2016-04-13 18:33:30 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.1", dummy_event_stream()) + @i.emit("test.tag.2", dummy_event_stream()) + + now = Time.parse('2016-04-13 18:33:31 -0700') + Timecop.freeze( now ) + + assert_equal 0, @i.write_count + assert_equal 0, @i.num_errors + + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 } + + assert{ @i.buffer.queue.size == 2 } + assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' } + + assert{ @i.write_count > 0 } + assert{ @i.num_errors > 0 } + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + + first_failure = @i.retry.start + + # retry_timeout == 60(sec), retry_secondary_threshold == 0.8 + + now = first_failure + 60 * 0.8 + 1 + Timecop.freeze( now ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + now = first_failure + 60 * 0.8 + 2 + Timecop.freeze( now ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + + waiting(4){ Thread.pass until chunks.size == 2 } + + assert{ @i.write_count > prev_write_count } + assert{ @i.num_errors == prev_num_errors } + + assert @i.retry + + assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {"name" => "moris", "age" => 36, "message" => "data1"} ], written[0] + assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1] + assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2] + assert_equal [ 'test.tag.2', event_time('2016-04-13 18:33:00').to_i, {"name" => "moris", "age" => 36, "message" => "data1"} ], written[3] + assert_equal [ 'test.tag.2', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[4] + assert_equal [ 'test.tag.2', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[5] + + assert{ @i.buffer.dequeued.size == 2 } + assert{ chunks.size == 2 } + assert{ !chunks[0].empty? } + assert{ !chunks[1].empty? } + + 30.times do |i| # large enough + now = first_failure + 60 * 0.8 + 2 + i + Timecop.freeze( now ) + @i.flush_thread_wakeup + + break if @i.buffer.dequeued.size == 0 + end + + assert @i.retry + logs = @i.log.out.logs + assert{ logs.select{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") }.size == 2 } + end + + test 'retry_wait for secondary is same with one for primary' do + priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'retry_randomize' => false}) + secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'}) + @i = create_output() + @i.configure(config_element('ROOT','',{},[priconf,secconf])) + @i.register(:prefer_buffered_processing){ true } + @i.register(:prefer_delayed_commit){ false } + @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" } + @i.register(:write){|chunk| raise "yay, your #write must fail" } + @i.secondary.register(:prefer_delayed_commit){ false } + @i.secondary.register(:write){|chunk| raise "your secondary is also useless." } + @i.start + + now = Time.parse('2016-04-13 18:33:30 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.1", dummy_event_stream()) + + now = Time.parse('2016-04-13 18:33:31 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.2", dummy_event_stream()) + + assert_equal 0, @i.write_count + assert_equal 0, @i.num_errors + + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 } + + assert{ @i.buffer.queue.size > 0 } + assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' } + + assert{ @i.write_count > 0 } + assert{ @i.num_errors > 0 } + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + + first_failure = @i.retry.start + + # retry_timeout == 60(sec), retry_secondary_threshold == 0.8 + + now = first_failure + 60 * 0.8 + 1 + + Timecop.freeze( now ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > prev_write_count } + + assert{ @i.write_count > prev_write_count } + assert{ @i.num_errors > prev_num_errors } + + assert @i.retry + + assert_equal 3, (@i.next_flush_time - Time.now) + + logs = @i.log.out.logs + assert{ logs.any?{|l| l.include?("[warn]: failed to flush the buffer with secondary output.") } } + end + end + + sub_test_case 'secondary plugin feature for buffered output with exponential backoff' do + setup do + Fluent::Plugin.register_output('output_secondary_test', FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput) + Fluent::Plugin.register_output('output_secondary_test2', FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput2) + end + + test 'primary plugin will emit event streams to secondary after retries for time of retry_timeout * retry_secondary_threshold' do + written = [] + priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :expbackoff, 'retry_wait' => 1, 'retry_timeout' => 60, 'retry_randomize' => false}) + secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'}) + @i = create_output() + @i.configure(config_element('ROOT','',{},[priconf,secconf])) + @i.register(:prefer_buffered_processing){ true } + @i.register(:prefer_delayed_commit){ false } + @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" } + @i.register(:write){|chunk| raise "yay, your #write must fail" } + @i.secondary.register(:prefer_delayed_commit){ false } + @i.secondary.register(:write){|chunk| chunk.read.split("\n").each{|line| written << JSON.parse(line) } } + @i.start + + now = Time.parse('2016-04-13 18:33:30 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.1", dummy_event_stream()) + + now = Time.parse('2016-04-13 18:33:31 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.2", dummy_event_stream()) + + assert_equal 0, @i.write_count + assert_equal 0, @i.num_errors + + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 } + + assert{ @i.buffer.queue.size > 0 } + assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' } + + assert{ @i.write_count > 0 } + assert{ @i.num_errors > 0 } + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + + first_failure = @i.retry.start + + 20.times do |i| # large enough + now = @i.next_flush_time + Timecop.freeze( now ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > prev_write_count } + + assert{ @i.write_count > prev_write_count } + + break if @i.buffer.queue.size == 0 + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + end + + # retry_timeout == 60(sec), retry_secondary_threshold == 0.8 + + assert{ now >= first_failure + 60 * 0.8 } + + assert_nil @i.retry + + assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:00').to_i, {"name" => "moris", "age" => 36, "message" => "data1"} ], written[0] + assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1] + assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2] + + assert{ @i.log.out.logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } } + end + + test 'exponential backoff interval will be initialized when switched to secondary' do + priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :expbackoff, 'retry_wait' => 1, 'retry_timeout' => 60, 'retry_randomize' => false}) + secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'}) + @i = create_output() + @i.configure(config_element('ROOT','',{},[priconf,secconf])) + @i.register(:prefer_buffered_processing){ true } + @i.register(:prefer_delayed_commit){ false } + @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" } + @i.register(:write){|chunk| raise "yay, your #write must fail" } + @i.secondary.register(:prefer_delayed_commit){ false } + @i.secondary.register(:write){|chunk| raise "your secondary is also useless." } + @i.start + + now = Time.parse('2016-04-13 18:33:30 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.1", dummy_event_stream()) + + now = Time.parse('2016-04-13 18:33:31 -0700') + Timecop.freeze( now ) + + @i.emit("test.tag.2", dummy_event_stream()) + + assert_equal 0, @i.write_count + assert_equal 0, @i.num_errors + + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 } + + assert{ @i.buffer.queue.size > 0 } + assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' } + + assert{ @i.write_count > 0 } + assert{ @i.num_errors > 0 } + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + + first_failure = @i.retry.start + + 20.times do |i| # large enough + now = @i.next_flush_time + # p({i: i, now: now, diff: (now - Time.now)}) + # {:i=>0, :now=>2016-04-13 18:33:32 -0700, :diff=>1.0} + # {:i=>1, :now=>2016-04-13 18:33:34 -0700, :diff=>2.0} + # {:i=>2, :now=>2016-04-13 18:33:38 -0700, :diff=>4.0} + # {:i=>3, :now=>2016-04-13 18:33:46 -0700, :diff=>8.0} + # {:i=>4, :now=>2016-04-13 18:34:02 -0700, :diff=>16.0} + # {:i=>5, :now=>2016-04-13 18:34:19 -0700, :diff=>17.0} + Timecop.freeze( now ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > prev_write_count } + + assert{ @i.write_count > prev_write_count } + assert{ @i.num_errors > prev_num_errors } + + prev_write_count = @i.write_count + prev_num_errors = @i.num_errors + + break if @i.retry.secondary? + + assert{ @i.buffer.queue.first.metadata.tag == 'test.tag.1' } + end + + # retry_timeout == 60(sec), retry_secondary_threshold == 0.8 + + assert{ now >= first_failure + 60 * 0.8 } + assert @i.retry + logs = @i.log.out.logs + assert{ logs.any?{|l| l.include?("[warn]: failed to flush the buffer with secondary output.") } } + + assert{ (@i.next_flush_time - Time.now) <= 2 } # <= retry_wait (1s) * base (2) ** 1 + + 20.times do |i| # large enough again + now = @i.next_flush_time + # p({i: i, now: now, diff: (now - Time.now)}) + # {:i=>0, :now=>2016-04-13 18:34:20 -0700, :diff=>1.0} + # {:i=>1, :now=>2016-04-13 18:34:24 -0700, :diff=>4.0} + # {:i=>2, :now=>2016-04-13 18:34:31 -0700, :diff=>7.0} + + Timecop.freeze( now ) + @i.enqueue_thread_wait + @i.flush_thread_wakeup + waiting(4){ Thread.pass until @i.write_count > prev_write_count } + + assert{ @i.write_count > prev_write_count } + assert{ @i.num_errors > prev_num_errors } + + break if @i.buffer.queue.size == 0 + end + + logs = @i.log.out.logs + assert{ logs.any?{|l| l.include?("[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.") } } + + assert{ now >= first_failure + 60 } + end + end +end From 682b5743f1043093a25ce3491fa0ab2975c074a7 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 16:19:29 +0900 Subject: [PATCH 17/36] fix tests to wait callbacks surely --- test/plugin/test_output.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/plugin/test_output.rb b/test/plugin/test_output.rb index ef215e0f63..36f0b47122 100644 --- a/test/plugin/test_output.rb +++ b/test/plugin/test_output.rb @@ -293,6 +293,8 @@ def waiting(seconds) t = event_time() i.emit('tag', [ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]) + waiting(4){ Thread.pass until process_called } + assert process_called assert_equal 0, format_called_times @@ -316,6 +318,8 @@ def waiting(seconds) t = event_time() i.emit('tag', [ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]) + waiting(4){ Thread.pass until process_called } + assert !process_called assert_equal 2, format_called_times @@ -334,6 +338,8 @@ def waiting(seconds) i.emit('tag', [ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]) i.force_flush + waiting(4){ Thread.pass until write_called } + assert write_called i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate @@ -351,6 +357,8 @@ def waiting(seconds) i.emit('tag', [ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]) i.force_flush + waiting(4){ Thread.pass until try_write_called } + assert try_write_called i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate @@ -373,6 +381,8 @@ def waiting(seconds) i.emit('tag', [ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]) i.force_flush + waiting(4){ Thread.pass until write_called || try_write_called } + assert write_called assert !try_write_called @@ -396,6 +406,8 @@ def waiting(seconds) i.emit('tag', [ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]) i.force_flush + waiting(4){ Thread.pass until write_called || try_write_called } + assert !write_called assert try_write_called From 80c014d6d55f927aeeb6464a0034b4ce0891e98d Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 16:30:54 +0900 Subject: [PATCH 18/36] add requirement explicitly (Travis fails to build on ruby 2.1) --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0cadaf0906..c1cca4d613 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,3 +31,8 @@ matrix: - rvm: rbx - rvm: 2.3.0 os: osx + +addons: + apt: + packages: + - libgmp3-dev From 58f508ce033e81affd6c423441000551487f8ea3 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 17:57:59 +0900 Subject: [PATCH 19/36] fix to show logs in flushing threads when expected situation is not satisfied --- test/plugin/test_output.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/plugin/test_output.rb b/test/plugin/test_output.rb index 36f0b47122..afa8d16c26 100644 --- a/test/plugin/test_output.rb +++ b/test/plugin/test_output.rb @@ -71,8 +71,13 @@ def create_metadata(timekey: nil, tag: nil, variables: nil) Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables) end def waiting(seconds) - Timeout.timeout(seconds) do - yield + begin + Timeout.timeout(seconds) do + yield + end + rescue Timeout::Error + STDERR.print *(@i.log.out.logs) + raise end end From 3e628839e30ac9530f49c2cb79eb3b581c858590 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 17:58:15 +0900 Subject: [PATCH 20/36] fix to use parsed local time to check internal event time from logs --- test/plugin/test_output_as_buffered_retries.rb | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/test/plugin/test_output_as_buffered_retries.rb b/test/plugin/test_output_as_buffered_retries.rb index 1332c8e88e..ffdcfc6d8b 100644 --- a/test/plugin/test_output_as_buffered_retries.rb +++ b/test/plugin/test_output_as_buffered_retries.rb @@ -77,6 +77,14 @@ def dummy_event_stream [ event_time('2016-04-13 18:33:32'), {"name" => "moris", "age" => 36, "message" => "data3"} ], ] end + def get_log_time(msg, logs) + log_time = nil + log = logs.select{|l| l.include?(msg) }.first + if log && /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4}) \[error\]/ =~ log + log_time = Time.parse($1) + end + log_time + end teardown do if @i @@ -502,7 +510,14 @@ def dummy_event_stream waiting(4){ Thread.pass until @i.write_count > prev_write_count && @i.num_errors > prev_num_errors } logs = @i.log.out.logs - assert{ logs.any?{|l| l.start_with?("2016-04-13 18:35:31 -0700 [error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.") } } + + target_time = Time.parse("2016-04-13 18:35:31 -0700") + target_msg = "[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue." + assert{ logs.any?{|l| l.include?(target_msg) } } + + log_time = get_log_time(target_msg, logs) + assert_equal target_time.localtime, log_time.localtime + assert{ @i.buffer.queue.size == 0 } assert{ @i.buffer.stage.size == 1 } assert{ chunks.all?{|c| c.empty? } } From 2c825e84f78018c953e769d459a573f5a1b79c1c Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 17:58:53 +0900 Subject: [PATCH 21/36] specify osx_image to enable latest ruby on osx environment --- .travis.yml | 57 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index c1cca4d613..eb034ae1f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,47 @@ language: ruby -rvm: - - 2.1.10 - - 2.2.4 - - 2.3.0 - - ruby-head - - rbx +# http://rubies.travis-ci.org/ +# See here for osx_image -> OSX versions: https://docs.travis-ci.com/user/languages/objective-c +matrix: + include: + - rvm: 2.1.10 + os: linux + - rvm: 2.2.4 + os: linux + - rvm: 2.3.0 + os: linux + - rvm: ruby-head + os: linux + - rvm: 2.1.0 + os: osx + osx_image: xcode7.3 # OSX 10.11 + - rvm: 2.2.4 + os: osx + osx_image: xcode7.1 # OSX 10.10 + # - rvm: 2.3.0 + # os: osx + # osx_image: .... # no valid version/env for ruby 2.3 + - rvm: ruby-head + os: osx + osx_image: xcode 7.3 # OSX 10.11 + allow_failures: + - rvm: ruby-head + - rvm: rbx + - rvm: 2.3.0 + os: osx + +# rvm: +# - 2.1.10 +# - 2.2.4 +# - 2.3.0 +# - ruby-head +# - rbx + +# os: +# - linux +# - osx -os: - - linux - - osx +# osx_image: xcode7.3 branches: only: @@ -25,13 +57,6 @@ script: bundle exec rake sudo: false -matrix: - allow_failures: - - rvm: ruby-head - - rvm: rbx - - rvm: 2.3.0 - os: osx - addons: apt: packages: From a169cf789340706a36d954069648efba96b356b2 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 18:17:59 +0900 Subject: [PATCH 22/36] update flexmock (now it is missing on rubygems.org) --- fluentd.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fluentd.gemspec b/fluentd.gemspec index 43913526f1..3bae4edc57 100644 --- a/fluentd.gemspec +++ b/fluentd.gemspec @@ -36,7 +36,7 @@ Gem::Specification.new do |gem| gem.add_runtime_dependency("strptime", [">= 0.1.7"]) gem.add_development_dependency("rake", [">= 0.9.2"]) - gem.add_development_dependency("flexmock", ["~> 1.3.3"]) + gem.add_development_dependency("flexmock", ["~> 2.0.5"]) gem.add_development_dependency("parallel_tests", [">= 0.15.3"]) gem.add_development_dependency("simplecov", ["~> 0.6.4"]) gem.add_development_dependency("rr", [">= 1.0.0"]) From 0a6cd4f73af2ad347ef5432d0bfebd1f341f228d Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 18:26:55 +0900 Subject: [PATCH 23/36] install dependencies explicitly --- .travis.yml | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index eb034ae1f1..90eff26c18 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ language: ruby +cache: bundler # http://rubies.travis-ci.org/ # See here for osx_image -> OSX versions: https://docs.travis-ci.com/user/languages/objective-c @@ -18,30 +19,17 @@ matrix: - rvm: 2.2.4 os: osx osx_image: xcode7.1 # OSX 10.10 - # - rvm: 2.3.0 - # os: osx - # osx_image: .... # no valid version/env for ruby 2.3 - rvm: ruby-head os: osx osx_image: xcode 7.3 # OSX 10.11 allow_failures: - rvm: ruby-head - - rvm: rbx - - rvm: 2.3.0 - os: osx -# rvm: -# - 2.1.10 -# - 2.2.4 -# - 2.3.0 -# - ruby-head -# - rbx +# no valid version/env for ruby 2.3 right now +# - rvm: 2.3.0 +# os: osx +# osx_image: .... -# os: -# - linux -# - osx - -# osx_image: xcode7.3 branches: only: @@ -53,6 +41,8 @@ branches: gemfile: - Gemfile +install: bundle install --jobs=3 --retry=3 + script: bundle exec rake sudo: false From db19c737ea1c1fa4bc3b2862bc9c209d63eb0dd1 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 18:33:43 +0900 Subject: [PATCH 24/36] omit parameters same with defaults --- .travis.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 90eff26c18..beab954ee9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,6 @@ matrix: # os: osx # osx_image: .... - branches: only: - master @@ -38,13 +37,6 @@ branches: - v0.12 - v0.14 -gemfile: - - Gemfile - -install: bundle install --jobs=3 --retry=3 - -script: bundle exec rake - sudo: false addons: From 3a989380db1d029592e77111678fc62aa0e6e593 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 18:47:36 +0900 Subject: [PATCH 25/36] remove useless check --- test/plugin/test_output.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/plugin/test_output.rb b/test/plugin/test_output.rb index afa8d16c26..eb558d900b 100644 --- a/test/plugin/test_output.rb +++ b/test/plugin/test_output.rb @@ -323,8 +323,6 @@ def waiting(seconds) t = event_time() i.emit('tag', [ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]) - waiting(4){ Thread.pass until process_called } - assert !process_called assert_equal 2, format_called_times From f458123d09e2b241faaa50b6d7a60b779525673a Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 19:39:38 +0900 Subject: [PATCH 26/36] fix tests not to be affected by logger formatting --- test/plugin_helper/test_timer.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/plugin_helper/test_timer.rb b/test/plugin_helper/test_timer.rb index 060d7ebbc6..f8b50fec65 100644 --- a/test/plugin_helper/test_timer.rb +++ b/test/plugin_helper/test_timer.rb @@ -91,10 +91,8 @@ class Dummy < Fluent::Plugin::TestBase assert{ counter1 >= 4 && counter1 <= 6 } assert{ counter2 == 3 } - msg = "[error]: Unexpected error raised. Stopping the timer. title=:t2 error=# error_class=RuntimeError" - assert{ d1.log.out.logs.any?{|line| line.include?(msg) } } - msg = "[error]: Timer detached. title=:t2" - assert{ d1.log.out.logs.any?{|line| line.include?(msg) } } + assert{ d1.log.out.logs.any?{|line| line.include?("[error]:") && line.include?(msg) && line.include?("abort!!!!!!") } } + assert{ d1.log.out.logs.any?{|line| line.include?("[error]:") && line.include?("Timer detached. title=:t2") } } d1.shutdown; d1.close; d1.terminate end From fe56d936aee80f292b5b67b2ce87a1db79543f22 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Mon, 18 Apr 2016 19:44:24 +0900 Subject: [PATCH 27/36] add workaround to prohibit to pass arguments to thread_create helper method --- lib/fluent/plugin/output.rb | 4 +++- lib/fluent/plugin_helper/thread.rb | 13 ++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/fluent/plugin/output.rb b/lib/fluent/plugin/output.rb index 376daf6a9f..fb8f22fab0 100644 --- a/lib/fluent/plugin/output.rb +++ b/lib/fluent/plugin/output.rb @@ -321,7 +321,9 @@ def start @buffer_config.flush_threads.times do |i| thread_title = "flush_thread_#{i}".to_sym thread_state = FlushThreadState.new(nil, nil) - thread = thread_create(thread_title, thread_state, &method(:flush_thread_run)) + thread = thread_create(thread_title) do + flush_thread_run(thread_state) + end thread_state.thread = thread @output_flush_threads_mutex.synchronize do @output_flush_threads << thread_state diff --git a/lib/fluent/plugin_helper/thread.rb b/lib/fluent/plugin_helper/thread.rb index 4d0831b205..d3273cbd79 100644 --- a/lib/fluent/plugin_helper/thread.rb +++ b/lib/fluent/plugin_helper/thread.rb @@ -37,12 +37,19 @@ def thread_wait_until_start end end - def thread_create(title, *args) + # Ruby 2.2.3 or earlier (and all 2.1.x) cause bug about Threading ("Stack consistency error") + # by passing splatted argument to `yield` + # https://bugs.ruby-lang.org/issues/11027 + # We can enable to pass arguments after expire of Ruby 2.1 (& older 2.2.x) + # def thread_create(title, *args) + # Thread.new(*args) do |*t_args| + # yield *t_args + def thread_create(title) raise ArgumentError, "BUG: title must be a symbol" unless title.is_a? Symbol raise ArgumentError, "BUG: callback not specified" unless block_given? m = Mutex.new m.lock - thread = ::Thread.new(*args) do |*t_args| + thread = ::Thread.new do m.lock # run thread after that thread is successfully set into @_threads m.unlock thread_exit = false @@ -50,7 +57,7 @@ def thread_create(title, *args) ::Thread.current[:_fluentd_plugin_helper_thread_started] = true ::Thread.current[:_fluentd_plugin_helper_thread_running] = true begin - yield *t_args + yield thread_exit = true rescue => e log.warn "thread exited by unexpected error", plugin: self.class, title: title, error_class: e.class, error: e From dbb349221f687ea5fcaab138be90930b064edc27 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Tue, 19 Apr 2016 11:23:47 +0900 Subject: [PATCH 28/36] fix wrong test code --- test/plugin_helper/test_thread.rb | 5 +---- test/plugin_helper/test_timer.rb | 1 + 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/test/plugin_helper/test_thread.rb b/test/plugin_helper/test_thread.rb index 488239260a..0702d0bbf1 100644 --- a/test/plugin_helper/test_thread.rb +++ b/test/plugin_helper/test_thread.rb @@ -36,7 +36,6 @@ def configure(conf) d1.configure(config_element()) d1.start - thread_arguments = [] m1 = Mutex.new m2 = Mutex.new @@ -44,9 +43,8 @@ def configure(conf) thread_run = false Timeout.timeout(10) do - t = d1.thread_create(:test1, 'a', 'b') do |*args| + t = d1.thread_create(:test1) do m2.lock - thread_arguments += args assert !d1._threads.empty? # this must be true always assert d1.thread_current_running? @@ -62,7 +60,6 @@ def configure(conf) assert_equal :test1, t[:_fluentd_plugin_helper_thread_title] assert t[:_fluentd_plugin_helper_thread_running] - assert_equal ['a','b'], thread_arguments assert !d1._threads.empty? m1.unlock diff --git a/test/plugin_helper/test_timer.rb b/test/plugin_helper/test_timer.rb index f8b50fec65..bf011c3b07 100644 --- a/test/plugin_helper/test_timer.rb +++ b/test/plugin_helper/test_timer.rb @@ -91,6 +91,7 @@ class Dummy < Fluent::Plugin::TestBase assert{ counter1 >= 4 && counter1 <= 6 } assert{ counter2 == 3 } + msg = "Unexpected error raised. Stopping the timer. title=:t2" assert{ d1.log.out.logs.any?{|line| line.include?("[error]:") && line.include?(msg) && line.include?("abort!!!!!!") } } assert{ d1.log.out.logs.any?{|line| line.include?("[error]:") && line.include?("Timer detached. title=:t2") } } From d773e01e1ac8f7f2056a2dd6b48a95c24b5ac299 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Tue, 19 Apr 2016 14:37:40 +0900 Subject: [PATCH 29/36] add comment for paths --- lib/fluent/plugin/buf_file2.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/fluent/plugin/buf_file2.rb b/lib/fluent/plugin/buf_file2.rb index 7292c4896f..5c027df3c0 100644 --- a/lib/fluent/plugin/buf_file2.rb +++ b/lib/fluent/plugin/buf_file2.rb @@ -71,6 +71,7 @@ def configure(conf) elsif File.basename(@path).include?('.*.') # valid path (buffer.*.log will be ignored) else + # existing file will be ignored @path = @path + '.*.log' end else # path doesn't exist From 11db673a5ffab9092fe34aed51356209a50939dc Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Tue, 19 Apr 2016 14:38:53 +0900 Subject: [PATCH 30/36] getting file handler of metadata is required to close it cleanly (especially on Windows) --- lib/fluent/plugin/buffer/file_chunk.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fluent/plugin/buffer/file_chunk.rb b/lib/fluent/plugin/buffer/file_chunk.rb index 6ec6f1585d..1af339e153 100644 --- a/lib/fluent/plugin/buffer/file_chunk.rb +++ b/lib/fluent/plugin/buffer/file_chunk.rb @@ -223,7 +223,7 @@ def enqueued! file_rename(@chunk, @path, new_chunk_path, ->(new_io){ @chunk = new_io }) @path = new_chunk_path - file_rename(@meta, @meta_path, new_meta_path) + file_rename(@meta, @meta_path, new_meta_path, ->(new_io){ @meta = new_io }) @meta_path = new_meta_path @state = :queued From db901fa9d518a49dce820dac8bc20952814baa2f Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Tue, 19 Apr 2016 14:39:18 +0900 Subject: [PATCH 31/36] add mode not to convert newlines on Windows --- lib/fluent/plugin/buffer/file_chunk.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/fluent/plugin/buffer/file_chunk.rb b/lib/fluent/plugin/buffer/file_chunk.rb index 1af339e153..6f4615543c 100644 --- a/lib/fluent/plugin/buffer/file_chunk.rb +++ b/lib/fluent/plugin/buffer/file_chunk.rb @@ -249,11 +249,11 @@ def file_rename(file, old_path, new_path, callback=nil) def create_new_chunk(path, perm) @path = self.class.generate_stage_chunk_path(path, @unique_id) @meta_path = @path + '.meta' - @chunk = File.open(@path, 'w+', perm) + @chunk = File.open(@path, 'wb+', perm) @chunk.set_encoding(Encoding::ASCII_8BIT) @chunk.sync = true @chunk.binmode - @meta = File.open(@meta_path, 'w', perm) + @meta = File.open(@meta_path, 'wb', perm) @meta.set_encoding(Encoding::ASCII_8BIT) @meta.sync = true @meta.binmode @@ -273,13 +273,13 @@ def load_existing_staged_chunk(path) # staging buffer chunk without metadata is classic buffer chunk file # and it should be enqueued immediately if File.exist?(@meta_path) - @chunk = File.open(@path, 'r+') + @chunk = File.open(@path, 'rb+') @chunk.set_encoding(Encoding::ASCII_8BIT) @chunk.sync = true @chunk.seek(0, IO::SEEK_END) @chunk.binmode - @meta = File.open(@meta_path, 'r+') + @meta = File.open(@meta_path, 'rb+') @meta.set_encoding(Encoding::ASCII_8BIT) @meta.sync = true @meta.binmode From a3f3a7fd46253ee7c1c7213ae8842baa68d027b0 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Tue, 19 Apr 2016 14:42:26 +0900 Subject: [PATCH 32/36] fix tests for Windows environment --- test/plugin/test_buf_file2.rb | 65 ++++++++++++++++++++------- test/plugin/test_buffer_file_chunk.rb | 26 +++++------ 2 files changed, 61 insertions(+), 30 deletions(-) diff --git a/test/plugin/test_buf_file2.rb b/test/plugin/test_buf_file2.rb index 975abf256e..b1b11aaba8 100644 --- a/test/plugin/test_buf_file2.rb +++ b/test/plugin/test_buf_file2.rb @@ -25,21 +25,24 @@ def write_metadata(path, chunk_id, metadata, records, ctime, mtime) c: ctime, m: mtime, } - File.open(path, 'w') do |f| + File.open(path, 'wb') do |f| f.write metadata.to_msgpack end end - teardown do - if @p - @p.stop unless @p.stopped? - @p.before_shutdown unless @p.before_shutdown? - @p.shutdown unless @p.shutdown? - @p.after_shutdown unless @p.after_shutdown? - @p.close unless @p.closed? - @p.terminate unless @p.terminated? + sub_test_case 'non configured buffer plugin instance' do + setup do + Fluent::Test.setup + + @dir = File.expand_path('../../tmp/buffer_file_dir', __FILE__) + unless File.exist?(@dir) + FileUtils.mkdir_p @dir + end + Dir.glob(File.join(@dir, '*')).each do |path| + next if ['.', '..'].include?(File.basename(path)) + File.delete(path) + end end - end sub_test_case 'non configured buffer plugin instance' do # tests for path @@ -60,6 +63,23 @@ def write_metadata(path, chunk_id, metadata, records, ctime, mtime) @p.start end + teardown do + if @p + @p.stop unless @p.stopped? + @p.before_shutdown unless @p.before_shutdown? + @p.shutdown unless @p.shutdown? + @p.after_shutdown unless @p.after_shutdown? + @p.close unless @p.closed? + @p.terminate unless @p.terminated? + end + if @bufdir + Dir.glob(File.join(@bufdir, '*')).each do |path| + next if ['.', '..'].include?(File.basename(path)) + File.delete(path) + end + end + end + test 'this is persistent plugin' do assert @p.persistent? end @@ -206,10 +226,11 @@ def write_metadata(path, chunk_id, metadata, records, ctime, mtime) sub_test_case 'there are some existing file chunks' do setup do @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__) + FileUtils.mkdir_p @bufdir unless File.exist?(@bufdir) @c1id = Fluent::UniqueId.generate p1 = File.join(@bufdir, "etest.q#{Fluent::UniqueId.hex(@c1id)}.log") - File.open(p1, 'w') do |f| + File.open(p1, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" @@ -222,7 +243,7 @@ def write_metadata(path, chunk_id, metadata, records, ctime, mtime) @c2id = Fluent::UniqueId.generate p2 = File.join(@bufdir, "etest.q#{Fluent::UniqueId.hex(@c2id)}.log") - File.open(p2, 'w') do |f| + File.open(p2, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 13:59:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 13:59:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 13:59:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" @@ -234,7 +255,7 @@ def write_metadata(path, chunk_id, metadata, records, ctime, mtime) @c3id = Fluent::UniqueId.generate p3 = File.join(@bufdir, "etest.b#{Fluent::UniqueId.hex(@c3id)}.log") - File.open(p3, 'w') do |f| + File.open(p3, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" @@ -247,7 +268,7 @@ def write_metadata(path, chunk_id, metadata, records, ctime, mtime) @c4id = Fluent::UniqueId.generate p4 = File.join(@bufdir, "etest.b#{Fluent::UniqueId.hex(@c4id)}.log") - File.open(p4, 'w') do |f| + File.open(p4, 'wb') do |f| f.write ["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" f.write ["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" @@ -268,9 +289,19 @@ def write_metadata(path, chunk_id, metadata, records, ctime, mtime) end teardown do - Dir.glob(@bufpath).each do |path| - File.unlink(path + '.meta') if File.exist?(path + '.meta') - File.unlink(path) if File.exist?(path) + if @p + @p.stop unless @p.stopped? + @p.before_shutdown unless @p.before_shutdown? + @p.shutdown unless @p.shutdown? + @p.after_shutdown unless @p.after_shutdown? + @p.close unless @p.closed? + @p.terminate unless @p.terminated? + end + if @bufdir + Dir.glob(File.join(@bufdir, '*')).each do |path| + next if ['.', '..'].include?(File.basename(path)) + File.delete(path) + end end end diff --git a/test/plugin/test_buffer_file_chunk.rb b/test/plugin/test_buffer_file_chunk.rb index 76ff0eca76..ac72c5ce11 100644 --- a/test/plugin/test_buffer_file_chunk.rb +++ b/test/plugin/test_buffer_file_chunk.rb @@ -24,7 +24,7 @@ def gen_metadata(timekey: nil, tag: nil, variables: nil) end def read_metadata_file(path) - File.open(path){|f| MessagePack.unpack(f.read, symbolize_keys: true) } + File.open(path, 'rb'){|f| MessagePack.unpack(f.read, symbolize_keys: true) } end def gen_path(path) @@ -213,7 +213,7 @@ def gen_chunk_path(prefix, unique_id) assert @c.empty? - assert_equal '', File.open(@c.path){|f| f.read } + assert_equal '', File.open(@c.path, 'rb'){|f| f.read } d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'} d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'} @@ -238,7 +238,7 @@ def gen_chunk_path(prefix, unique_id) assert_equal first_size, @c.size assert_equal 2, @c.records - assert_equal (d1.to_json + "\n" + d2.to_json + "\n"), File.open(@c.path){|f| f.read } + assert_equal (d1.to_json + "\n" + d2.to_json + "\n"), File.open(@c.path, 'rb'){|f| f.read } end test 'can store its data by #close' do @@ -261,7 +261,7 @@ def gen_chunk_path(prefix, unique_id) @c.close - assert_equal content, File.open(@c.path){|f| f.read } + assert_equal content, File.open(@c.path, 'rb'){|f| f.read } stored_meta = { timekey: nil, tag: nil, variables: nil, @@ -383,7 +383,7 @@ def gen_chunk_path(prefix, unique_id) @c.close - assert_equal content, File.open(@c.path){|f| f.read } + assert_equal content, File.open(@c.path, 'rb'){|f| f.read } stored_meta = { timekey: nil, tag: nil, variables: nil, @@ -408,7 +408,7 @@ def gen_chunk_path(prefix, unique_id) @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'} @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'} @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + "\n" }.join - File.open(@chunk_path, 'w') do |f| + File.open(@chunk_path, 'wb') do |f| f.write @d end @@ -419,7 +419,7 @@ def gen_chunk_path(prefix, unique_id) c: Time.parse('2016-04-07 17:44:00 +0900').to_i, m: Time.parse('2016-04-07 17:44:13 +0900').to_i, } - File.open(@chunk_path + '.meta', 'w') do |f| + File.open(@chunk_path + '.meta', 'wb') do |f| f.write @metadata.to_msgpack end @@ -476,7 +476,7 @@ def gen_chunk_path(prefix, unique_id) assert_equal Time.parse('2016-04-07 17:44:00 +0900'), @c.created_at assert_equal Time.parse('2016-04-07 17:44:13 +0900'), @c.modified_at - assert_equal @d, File.open(@c.path){|f| f.read } + assert_equal @d, File.open(@c.path, 'rb'){|f| f.read } assert_equal @metadata, read_metadata_file(@c.path + '.meta') end @@ -516,7 +516,7 @@ def gen_chunk_path(prefix, unique_id) testing_file1 = gen_path('rename1.test') testing_file2 = gen_path('rename2.test') - f = File.open(testing_file1, 'w', @c.permission) + f = File.open(testing_file1, 'wb', @c.permission) f.set_encoding(Encoding::ASCII_8BIT) f.sync = true f.binmode @@ -559,7 +559,7 @@ def gen_chunk_path(prefix, unique_id) @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'} @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'} @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + "\n" }.join - File.open(@enqueued_path, 'w') do |f| + File.open(@enqueued_path, 'wb') do |f| f.write @d end @@ -572,7 +572,7 @@ def gen_chunk_path(prefix, unique_id) c: Time.parse('2016-04-07 17:44:00 +0900').to_i, m: Time.parse('2016-04-07 17:44:13 +0900').to_i, } - File.open(@enqueued_path + '.meta', 'w') do |f| + File.open(@enqueued_path + '.meta', 'wb') do |f| f.write @metadata.to_msgpack end @@ -618,7 +618,7 @@ def gen_chunk_path(prefix, unique_id) @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'} @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'} @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + "\n" }.join - File.open(@chunk_path, 'w') do |f| + File.open(@chunk_path, 'wb') do |f| f.write @d end @@ -660,7 +660,7 @@ def gen_chunk_path(prefix, unique_id) @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'} @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'} @d = [@d1,@d2,@d3,@d4].map{|d| d.to_json + "\n" }.join - File.open(@chunk_path, 'w') do |f| + File.open(@chunk_path, 'wb') do |f| f.write @d end From ff4212b465be07970d74e4c3bff6ee26898db0bb Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Tue, 19 Apr 2016 14:43:12 +0900 Subject: [PATCH 33/36] add more tests for buffer path configuration and #resume --- test/plugin/test_buf_file2.rb | 141 +++++++++++++++++++++++++++++++--- 1 file changed, 131 insertions(+), 10 deletions(-) diff --git a/test/plugin/test_buf_file2.rb b/test/plugin/test_buf_file2.rb index b1b11aaba8..8d88df6f2b 100644 --- a/test/plugin/test_buf_file2.rb +++ b/test/plugin/test_buf_file2.rb @@ -44,9 +44,29 @@ def write_metadata(path, chunk_id, metadata, records, ctime, mtime) end end - sub_test_case 'non configured buffer plugin instance' do - # tests for path - test '' + test 'path should include * normally' do + d = FluentPluginFileBufferTest::DummyOutputPlugin.new + p = Fluent::Plugin::FileBuffer.new + p.owner = d + p.configure(config_element('buffer', '', {'path' => File.join(@dir, 'buffer.*.file')})) + assert_equal File.join(@dir, 'buffer.*.file'), p.path + end + + test 'existing directory will be used with additional default file name' do + d = FluentPluginFileBufferTest::DummyOutputPlugin.new + p = Fluent::Plugin::FileBuffer.new + p.owner = d + p.configure(config_element('buffer', '', {'path' => @dir})) + assert_equal File.join(@dir, 'buffer.*.log'), p.path + end + + test 'unexisting path without * handled as directory' do + d = FluentPluginFileBufferTest::DummyOutputPlugin.new + p = Fluent::Plugin::FileBuffer.new + p.owner = d + p.configure(config_element('buffer', '', {'path' => File.join(@dir, 'buffer')})) + assert_equal File.join(@dir, 'buffer', 'buffer.*.log'), p.path + end end sub_test_case 'buffer plugin configured only with path' do @@ -306,9 +326,10 @@ def write_metadata(path, chunk_id, metadata, records, ctime, mtime) end test '#resume returns staged/queued chunks with metadata' do - stage,queue = @p.resume - assert_equal 2, stage.size - assert_equal 2, queue.size + assert_equal 2, @p.stage.size + assert_equal 2, @p.queue.size + + stage = @p.stage m3 = metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i) assert_equal @c3id, stage[m3].unique_id @@ -322,9 +343,10 @@ def write_metadata(path, chunk_id, metadata, records, ctime, mtime) end test '#resume returns queued chunks ordered by last modified time (FIFO)' do - stage,queue = @p.resume - assert_equal 2, stage.size - assert_equal 2, queue.size + assert_equal 2, @p.stage.size + assert_equal 2, @p.queue.size + + queue = @p.queue assert{ queue[0].modified_at < queue[1].modified_at } @@ -349,6 +371,105 @@ def write_metadata(path, chunk_id, metadata, records, ctime, mtime) end sub_test_case 'there are some existing file chunks without metadata file' do - test '#resume returns queued chunks for files without metadata' + setup do + @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__) + + @c1id = Fluent::UniqueId.generate + p1 = File.join(@bufdir, "etest.201604171358.q#{Fluent::UniqueId.hex(@c1id)}.log") + File.open(p1, 'wb') do |f| + f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" + f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" + f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" + f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n" + end + FileUtils.touch(p1, mtime: Time.parse('2016-04-17 13:58:28 -0700')) + + @c2id = Fluent::UniqueId.generate + p2 = File.join(@bufdir, "etest.201604171359.q#{Fluent::UniqueId.hex(@c2id)}.log") + File.open(p2, 'wb') do |f| + f.write ["t1.test", event_time('2016-04-17 13:59:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" + f.write ["t2.test", event_time('2016-04-17 13:59:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" + f.write ["t3.test", event_time('2016-04-17 13:59:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" + end + FileUtils.touch(p2, mtime: Time.parse('2016-04-17 13:59:30 -0700')) + + @c3id = Fluent::UniqueId.generate + p3 = File.join(@bufdir, "etest.201604171400.b#{Fluent::UniqueId.hex(@c3id)}.log") + File.open(p3, 'wb') do |f| + f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" + f.write ["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" + f.write ["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" + f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}].to_json + "\n" + end + FileUtils.touch(p3, mtime: Time.parse('2016-04-17 14:00:29 -0700')) + + @c4id = Fluent::UniqueId.generate + p4 = File.join(@bufdir, "etest.201604171401.b#{Fluent::UniqueId.hex(@c4id)}.log") + File.open(p4, 'wb') do |f| + f.write ["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}].to_json + "\n" + f.write ["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}].to_json + "\n" + f.write ["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}].to_json + "\n" + end + FileUtils.touch(p4, mtime: Time.parse('2016-04-17 14:01:22 -0700')) + + @bufpath = File.join(@bufdir, 'etest.*.log') + + Fluent::Test.setup + @d = FluentPluginFileBufferTest::DummyOutputPlugin.new + @p = Fluent::Plugin::FileBuffer.new + @p.owner = @d + @p.configure(config_element('buffer', '', {'path' => @bufpath})) + @p.start + end + + teardown do + if @p + @p.stop unless @p.stopped? + @p.before_shutdown unless @p.before_shutdown? + @p.shutdown unless @p.shutdown? + @p.after_shutdown unless @p.after_shutdown? + @p.close unless @p.closed? + @p.terminate unless @p.terminated? + end + if @bufdir + Dir.glob(File.join(@bufdir, '*')).each do |path| + next if ['.', '..'].include?(File.basename(path)) + File.delete(path) + end + end + end + + test '#resume returns queued chunks for files without metadata' do + assert_equal 0, @p.stage.size + assert_equal 4, @p.queue.size + + queue = @p.queue + + m = metadata() + + assert_equal @c1id, queue[0].unique_id + assert_equal m, queue[0].metadata + assert_equal 0, queue[0].records + assert_equal :queued, queue[0].state + assert_equal Time.parse('2016-04-17 13:58:28 -0700'), queue[0].modified_at + + assert_equal @c2id, queue[1].unique_id + assert_equal m, queue[1].metadata + assert_equal 0, queue[1].records + assert_equal :queued, queue[1].state + assert_equal Time.parse('2016-04-17 13:59:30 -0700'), queue[1].modified_at + + assert_equal @c3id, queue[2].unique_id + assert_equal m, queue[2].metadata + assert_equal 0, queue[2].records + assert_equal :queued, queue[2].state + assert_equal Time.parse('2016-04-17 14:00:29 -0700'), queue[2].modified_at + + assert_equal @c4id, queue[3].unique_id + assert_equal m, queue[3].metadata + assert_equal 0, queue[3].records + assert_equal :queued, queue[3].state + assert_equal Time.parse('2016-04-17 14:01:22 -0700'), queue[3].modified_at + end end end From 14d19d070856de144fe994ae818b872a82d3c5a6 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Tue, 19 Apr 2016 15:44:01 +0900 Subject: [PATCH 34/36] omit tests about file permissions on Windows --- test/plugin/test_buf_file2.rb | 3 +++ test/plugin/test_buffer_file_chunk.rb | 2 ++ 2 files changed, 5 insertions(+) diff --git a/test/plugin/test_buf_file2.rb b/test/plugin/test_buf_file2.rb index 8d88df6f2b..4882861a73 100644 --- a/test/plugin/test_buf_file2.rb +++ b/test/plugin/test_buf_file2.rb @@ -3,6 +3,7 @@ require 'fluent/plugin/output' require 'fluent/unique_id' require 'fluent/system_config' +require 'fluent/env' require 'msgpack' @@ -193,6 +194,8 @@ def write_metadata(path, chunk_id, metadata, records, ctime, mtime) end test '#generate_chunk generates blank file chunk with specified permission' do + omit "NTFS doesn't support UNIX like permissions" if Fluent.windows? + plugin = Fluent::Plugin::FileBuffer.new plugin.owner = @d rand_num = rand(0..100) diff --git a/test/plugin/test_buffer_file_chunk.rb b/test/plugin/test_buffer_file_chunk.rb index ac72c5ce11..cbb9e30d04 100644 --- a/test/plugin/test_buffer_file_chunk.rb +++ b/test/plugin/test_buffer_file_chunk.rb @@ -321,6 +321,8 @@ def gen_chunk_path(prefix, unique_id) end test 'can refer system config for file permission' do + omit "NTFS doesn't support UNIX like permissions" if Fluent.windows? + chunk_path = File.join(@chunkdir, 'testperm.*.log') Fluent::SystemConfig.overwrite_system_config("file_permission" => "600") do c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, chunk_path, :create) From af735c1be9b8f9ca0dc8d58b91e273a5b9c6feb3 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Tue, 19 Apr 2016 16:00:34 +0900 Subject: [PATCH 35/36] fix tests to cleanup generated files --- test/plugin/test_buf_file2.rb | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/plugin/test_buf_file2.rb b/test/plugin/test_buf_file2.rb index 4882861a73..6c1cfb4601 100644 --- a/test/plugin/test_buf_file2.rb +++ b/test/plugin/test_buf_file2.rb @@ -191,6 +191,9 @@ def write_metadata(path, chunk_id, metadata, records, ctime, mtime) assert_equal Fluent::Plugin::Buffer::FileChunk::FILE_PERMISSION, c2.permission assert_equal @bufpath.gsub('.*.', ".b#{Fluent::UniqueId.hex(c2.unique_id)}."), c2.path assert{ File.stat(c2.path).mode.to_s(8).end_with?('644') } + + c1.purge + c2.purge end test '#generate_chunk generates blank file chunk with specified permission' do @@ -219,6 +222,8 @@ def write_metadata(path, chunk_id, metadata, records, ctime, mtime) assert_equal bufpath.gsub('.*.', ".b#{Fluent::UniqueId.hex(c.unique_id)}."), c.path assert{ File.stat(c.path).mode.to_s(8).end_with?('600') } + c.purge + plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate FileUtils.rm_r bufdir end @@ -238,6 +243,22 @@ def write_metadata(path, chunk_id, metadata, records, ctime, mtime) @p.configure(config_element('buffer', '', {'path' => @bufpath})) @p.start end + teardown do + if @p + @p.stop unless @p.stopped? + @p.before_shutdown unless @p.before_shutdown? + @p.shutdown unless @p.shutdown? + @p.after_shutdown unless @p.after_shutdown? + @p.close unless @p.closed? + @p.terminate unless @p.terminated? + end + if @bufdir + Dir.glob(File.join(@bufdir, '*')).each do |path| + next if ['.', '..'].include?(File.basename(path)) + File.delete(path) + end + end + end test '#resume returns empty buffer state' do ary = @p.resume From e96dcddb104faaca2ed2ddb103b60f6886611861 Mon Sep 17 00:00:00 2001 From: TAGOMORI Satoshi Date: Tue, 19 Apr 2016 16:14:43 +0900 Subject: [PATCH 36/36] omit tests for permissions on Windows --- test/plugin/test_buf_file2.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/plugin/test_buf_file2.rb b/test/plugin/test_buf_file2.rb index 6c1cfb4601..b7f8de1732 100644 --- a/test/plugin/test_buf_file2.rb +++ b/test/plugin/test_buf_file2.rb @@ -127,6 +127,8 @@ def write_metadata(path, chunk_id, metadata, records, ctime, mtime) end test '#start creates directory for buffer chunks with specified permission' do + omit "NTFS doesn't support UNIX like permissions" if Fluent.windows? + plugin = Fluent::Plugin::FileBuffer.new plugin.owner = @d rand_num = rand(0..100) @@ -148,6 +150,8 @@ def write_metadata(path, chunk_id, metadata, records, ctime, mtime) end test '#start creates directory for buffer chunks with specified permission via system config' do + omit "NTFS doesn't support UNIX like permissions" if Fluent.windows? + sysconf = {'dir_permission' => '700'} Fluent::SystemConfig.overwrite_system_config(sysconf) do plugin = Fluent::Plugin::FileBuffer.new