From 58c5d42ddc596d29ab92445fa175a7b3d52b7a98 Mon Sep 17 00:00:00 2001 From: Aditya C S Date: Fri, 24 Jul 2020 13:32:50 +0530 Subject: [PATCH 1/2] logstash code refactor and doc improvements --- cmd/logstash/README.md | 11 +++++---- cmd/logstash/lib/logstash/outputs/loki.rb | 29 ++++++++++++++++------- cmd/logstash/logstash-output-loki.gemspec | 4 ++-- cmd/logstash/loki.conf | 2 ++ docs/sources/clients/logstash/_index.md | 5 ++++ 5 files changed, 35 insertions(+), 16 deletions(-) diff --git a/cmd/logstash/README.md b/cmd/logstash/README.md index 6fd462b21d02b..1c9c7f3e3d294 100644 --- a/cmd/logstash/README.md +++ b/cmd/logstash/README.md @@ -1,10 +1,12 @@ # Contributing to Loki Logstash Output Plugin -For information about hwo to use this plugin see this [documentation](../../docs/clients/logstash/README.md). +For information about how to use this plugin see this [documentation](../../docs/sources/clients/logstash/_index.md). ## Install dependencies -First you need to setup JRuby environment to build this plugin. Refer https://github.com/rbenv/rbenv for setting up your rbenv environment. +First, make sure you have JDK version `8` or `11` installed and you have set the `JAVA_HOME` environment variable. + +You need to setup JRuby environment to build this plugin. Refer https://github.com/rbenv/rbenv for setting up your rbenv environment. After setting up `rbenv`. Install JRuby @@ -20,7 +22,7 @@ ruby --version jruby 9.2.10 ``` -You should use make sure you are running jruby and not ruby. If the command below still shows ruby and not jruby, check that PATH contains `$HOME/.rbenv/shims` and `$HOME/.rbenv/bin`. Also verify that you have this in your bash profile: +You should make sure you are running `jruby` and not `ruby`. If the command `ruby --version` still shows `ruby` and not `jruby`, check that PATH contains `$HOME/.rbenv/shims` and `$HOME/.rbenv/bin`. Also verify that you have this in your bash profile: ```bash export PATH="$HOME/.rbenv/bin:$PATH" @@ -32,7 +34,7 @@ Then install bundler Follow those instructions to [install logstash](https://www.elastic.co/guide/en/logstash/current/installing-logstash.html) before moving to the next section. -## Install dependencies and Build plugin +## Build and test the plugin ### Install required packages @@ -43,7 +45,6 @@ git checkout tags/v7.6.2 export LOGSTASH_PATH=`pwd` export GEM_PATH=$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0 export GEM_HOME=$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0 -./gradlew assemble cd .. ruby -S bundle install ruby -S bundle exec rake vendor diff --git a/cmd/logstash/lib/logstash/outputs/loki.rb b/cmd/logstash/lib/logstash/outputs/loki.rb index 87b0b720f3178..9b9105a84c261 100644 --- a/cmd/logstash/lib/logstash/outputs/loki.rb +++ b/cmd/logstash/lib/logstash/outputs/loki.rb @@ -30,6 +30,9 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base ## 'TLS' config :ca_cert, :validate => :path, :required => false + ## 'Disable server certificate verification' + config :insecure_skip_verify, :validate => :boolean, :default => false, :required => false + ## 'Loki Tenant ID' config :tenant_id, :validate => :string, :required => false @@ -45,8 +48,8 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base ## 'Backoff configuration. Initial backoff time between retries. Default 1s' config :min_delay, :validate => :number, :default => 1, :required => false - ## 'Backoff configuration. Maximum backoff time between retries. Default 300s' - config :max_delay, :validate => :number, :default => 300, :required => false + ## 'Backoff configuration. Maximum backoff time between retries. Default 300s' + config :max_delay, :validate => :number, :default => 300, :required => false ## 'Backoff configuration. Maximum number of retries to do' config :retries, :validate => :number, :default => 10, :required => false @@ -102,6 +105,13 @@ def ssl_opts(uri) use_ssl: uri.scheme == 'https' } + # disable server certificate verification + if @insecure_skip_verify + opts = opts.merge( + verify_mode: OpenSSL::SSL::VERIFY_NONE + ) + end + if !@cert.nil? && !@key.nil? opts = opts.merge( verify_mode: OpenSSL::SSL::VERIFY_PEER, @@ -119,7 +129,8 @@ def ssl_opts(uri) end def run() - min_wait_checkfrequency = 1/100 #1 millisecond + # minimum wait frequency is 1 millisecond + min_wait_checkfrequency = 1/100 max_wait_checkfrequency = @batch_wait / 10 if max_wait_checkfrequency < min_wait_checkfrequency max_wait_checkfrequency = min_wait_checkfrequency @@ -139,7 +150,7 @@ def run() end } s.take(@max_wait_check) { - # Send batch if max wait time has been reached + # send batch if max wait time has been reached if is_batch_expired @logger.debug("Max batch_wait time is reached. Sending batch to loki") send(@batch) @@ -150,7 +161,7 @@ def run() end end - # add an entry to the current batch return false if the batch is full + # Add an entry to the current batch returns false if the batch is full # and the entry can't be added. def add_entry_to_batch(e) line = e.entry['line'] @@ -222,13 +233,13 @@ def loki_http_request(payload) raise StandardError.new res rescue StandardError => e retry_count += 1 - @logger.warn("Failed to send batch attempt: #{retry_count}/#{@retries}", :error_inspect => e.inspect, :error => e) + @logger.warn("Failed to send batch, attempt: #{retry_count}/#{@retries}", :error_inspect => e.inspect, :error => e) if retry_count < @retries sleep delay - if (delay * 2 - delay) > @max_delay - delay = delay - else + if delay * 2 <= @max_delay delay = delay * 2 + else + delay = @max_delay end retry else diff --git a/cmd/logstash/logstash-output-loki.gemspec b/cmd/logstash/logstash-output-loki.gemspec index bb9c51a3539f3..716b2fc065f0d 100644 --- a/cmd/logstash/logstash-output-loki.gemspec +++ b/cmd/logstash/logstash-output-loki.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| - s.name = 'logstash-output-loki' - s.version = '1.0.1' + s.name = 'logstash-output-loki' + s.version = '1.0.2' s.authors = ['Aditya C S','Cyril Tovena'] s.email = ['aditya.gnu@gmail.com','cyril.tovena@grafana.com'] diff --git a/cmd/logstash/loki.conf b/cmd/logstash/loki.conf index 6f156cbc75f37..f8ae19ba48eb9 100644 --- a/cmd/logstash/loki.conf +++ b/cmd/logstash/loki.conf @@ -33,5 +33,7 @@ output { # key => /path/to/key.key # ca_cert => /path/to/ca.pem + + # insecure_skip_verify => false } } diff --git a/docs/sources/clients/logstash/_index.md b/docs/sources/clients/logstash/_index.md index d131998b15798..d1aeb7b6ba893 100644 --- a/docs/sources/clients/logstash/_index.md +++ b/docs/sources/clients/logstash/_index.md @@ -74,6 +74,7 @@ output { [ca_cert => path | default = nil | required=false] + [insecure_skip_verify => boolean | default = fasle | required=false] } } ``` @@ -229,6 +230,10 @@ Loki is a multi-tenant log storage platform and all requests sent must include a Specify a pair of client certificate and private key with `cert` and `key` if a reverse proxy with client certificate verification is configured in front of Loki. `ca_cert` can also be specified if the server uses custom certificate authority. +### insecure_skip_verify + +A flag to disable server certificate verification. By default it is set to `false`. + ### Full configuration example ```conf From dcdaff5ae0dca0f0d2f838f6e4c6cca4f3f82ed7 Mon Sep 17 00:00:00 2001 From: Aditya C S Date: Fri, 24 Jul 2020 17:38:40 +0530 Subject: [PATCH 2/2] added back gradlew assemble --- cmd/logstash/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/logstash/README.md b/cmd/logstash/README.md index 1c9c7f3e3d294..e61f81adcc8ba 100644 --- a/cmd/logstash/README.md +++ b/cmd/logstash/README.md @@ -45,6 +45,7 @@ git checkout tags/v7.6.2 export LOGSTASH_PATH=`pwd` export GEM_PATH=$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0 export GEM_HOME=$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0 +./gradlew assemble cd .. ruby -S bundle install ruby -S bundle exec rake vendor