diff --git a/libraries/consul.rb b/libraries/consul.rb index 3634ca3..ac4d02c 100644 --- a/libraries/consul.rb +++ b/libraries/consul.rb @@ -18,5 +18,21 @@ def self.setup_consul(options) options[:token] = options[:consul_token] end end + + def self.update_backup_url(options) + require 'diplomat' + return unless options[:consul_backup_url] + + Diplomat.configure do |config| + config.url = @options[:consul_backup_url] + end + end + + def self.reset_url + require 'diplomat' + Diplomat.configure do |config| + config.url = Diplomat::Configuration.parse_consul_addr + end + end end end diff --git a/libraries/primitive_consul_lock.rb b/libraries/primitive_consul_lock.rb index 5cbcf26..761630e 100644 --- a/libraries/primitive_consul_lock.rb +++ b/libraries/primitive_consul_lock.rb @@ -117,7 +117,9 @@ def self.get_or_create(path, concurrency:, **kwargs) dc = kwargs[:dc] token = kwargs[:token] require 'diplomat' - retry_left = 5 + retry_total_attempts = 5 + connection_failed_count = 0 + retry_left = retry_total_attempts value = Mash.new({ version: 1, concurrency: concurrency, holders: {} }) current_lock = begin Chef::Log.info "Fetch lock state for #{path}" @@ -126,12 +128,15 @@ def self.get_or_create(path, concurrency:, **kwargs) retry_secs = 30 Chef::Log.info "Consul did not respond, wait #{retry_secs} seconds and retry to let it (re)start: #{e}" sleep retry_secs + connection_failed_count += 1 + ConsulCommon.update_backup_url(@options) if connection_failed_count == retry_total_attempts / 2 (retry_left -= 1).positive? ? retry : raise rescue Diplomat::KeyNotFound Chef::Log.info "Lock for #{path} did not exist, creating with value #{value}" Diplomat::Kv.put(path, value.to_json, cas: 0, dc: dc, token: token) # we ignore success/failure of CaS (retry_left -= 1).positive? ? retry : raise end.first + ConsulCommon.reset_url desired_lock = bootstrap_lock(value, current_lock) new(path, new_lock: desired_lock, dc: dc, token: token) end