From e38e259e345d11725d6f75a6d11d43d8efc91cca Mon Sep 17 00:00:00 2001 From: Jay Mundrawala Date: Fri, 28 Jul 2017 08:32:06 -0700 Subject: [PATCH 01/10] Add healthcheck for analytics rabbitmq vhost Signed-off-by: Jay Mundrawala --- .../templates/default/oc_erchef.config.erb | 3 +++ src/oc_erchef/Makefile | 2 +- .../src/chef_wm_rabbitmq_management.erl | 18 +++++++++++++++++- .../apps/oc_chef_wm/src/oc_chef_action.erl | 11 ++++++++++- 4 files changed, 31 insertions(+), 3 deletions(-) diff --git a/omnibus/files/private-chef-cookbooks/private-chef/templates/default/oc_erchef.config.erb b/omnibus/files/private-chef-cookbooks/private-chef/templates/default/oc_erchef.config.erb index 05737e7e3c..276022a4c5 100755 --- a/omnibus/files/private-chef-cookbooks/private-chef/templates/default/oc_erchef.config.erb +++ b/omnibus/files/private-chef-cookbooks/private-chef/templates/default/oc_erchef.config.erb @@ -127,6 +127,9 @@ data_collector, <% end %> oc_chef_authz, + <% if node['private_chef']['dark_launch']['actions'] %> + oc_chef_action, + <% end %> chef_sql, chef_<%= node['private_chef']['opscode-erchef']['search_provider'] %> ]}, diff --git a/src/oc_erchef/Makefile b/src/oc_erchef/Makefile index 3cfb0af42c..6a6e2d7cd0 100755 --- a/src/oc_erchef/Makefile +++ b/src/oc_erchef/Makefile @@ -74,7 +74,7 @@ bundle: @cd apps/chef_objects/priv/depselector_rb; bundle install --deployment --path .bundle -CHEFDK_GECODE_PATH:=/opt/chefdk/embedded/lib/ruby/gems/2.3.0/gems/dep-selector-libgecode-1.3.1/lib/dep-selector-libgecode/vendored-gecode +CHEFDK_GECODE_PATH:=/opt/chefdk/embedded/lib/ruby/gems/2.4.0/gems/dep-selector-libgecode-1.3.1/lib/dep-selector-libgecode/vendored-gecode travis_env: @echo export TRAVIS=1 @echo export USE_SYSTEM_GECODE=1 diff --git a/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_rabbitmq_management.erl b/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_rabbitmq_management.erl index b1a27d0129..856ae0c35c 100644 --- a/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_rabbitmq_management.erl +++ b/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_rabbitmq_management.erl @@ -40,7 +40,8 @@ get_rabbit_queue_monitor_setting/2, set_rabbit_management_setting/2, set_rabbit_queue_monitor_setting/2, - set_app_value/5 + set_app_value/5, + check_aliveness/1 ]). @@ -148,6 +149,10 @@ mk_max_length_path(Vhost) -> mk_current_length_path(Vhost) -> lists:flatten(io_lib:format("/queues/~s", [http_uri:encode(Vhost)])). +-spec mk_aliveness_check_path(string()) -> string(). +mk_aliveness_check_path(Vhost) -> + lists:flatten(io_lib:format("/aliveness-test/~s", [http_uri:encode(Vhost)])). + % make an http connection to the rabbitmq management console % and return a integer value or undefined -spec get_max_length(string()) -> integer() | undefined. @@ -302,3 +307,14 @@ parse_integer(Val) when is_list(Val) -> {Int, _Rest} -> Int end; parse_integer(_) -> undefined. + +-spec check_aliveness(string()) -> boolean(). +check_aliveness(Vhost) -> + Aliveness = rabbit_mgmt_server_request(mk_aliveness_check_path(Vhost)), + case Aliveness of + {ok, "200", _, _} -> + true; + Resp -> + lager:error("Error getting Rabbitmq aliveness: ~p", Resp), + false + end. diff --git a/src/oc_erchef/apps/oc_chef_wm/src/oc_chef_action.erl b/src/oc_erchef/apps/oc_chef_wm/src/oc_chef_action.erl index 78f2af4645..4cdb162557 100644 --- a/src/oc_erchef/apps/oc_chef_wm/src/oc_chef_action.erl +++ b/src/oc_erchef/apps/oc_chef_wm/src/oc_chef_action.erl @@ -26,7 +26,8 @@ -export([ log_action/2, - create_message/3 + create_message/3, + ping/0 ]). -ifdef(TEST). @@ -394,3 +395,11 @@ req_header(Name, Req) -> Header -> iolist_to_binary(Header) end. + +-spec ping() -> pong | pang. +ping() -> + VHost = envy:get(oc_chef_wm, actions_vhost, binary), + case chef_wm_rabbitmq_management:check_aliveness(binary_to_list(VHost)) of + true -> pong; + _ -> pang + end. From a2cdaeb027db2c2a76fa7080aed1ffc5745f930d Mon Sep 17 00:00:00 2001 From: Jay Mundrawala Date: Sun, 30 Jul 2017 17:08:00 -0500 Subject: [PATCH 02/10] Make dev env work without /etc/timezone Signed-off-by: Jay Mundrawala --- dev/Vagrantfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dev/Vagrantfile b/dev/Vagrantfile index 1bedc1d5df..9235a12ddb 100644 --- a/dev/Vagrantfile +++ b/dev/Vagrantfile @@ -452,7 +452,11 @@ def host_timezone end def host_timezone_linux - File.read("/etc/timezone").chomp + if File.exists?("/etc/timezone") + File.read("/etc/timezone").chomp + else + "UTC" + end end def host_timezone_osx From cef307ea14896dabf4a9011a7c2832fba8627a5e Mon Sep 17 00:00:00 2001 From: Jay Mundrawala Date: Sun, 30 Jul 2017 17:08:39 -0500 Subject: [PATCH 03/10] Refactor chef_wm_rabbitmq_management to be able to deal with multiple vips The current chef server configuration allows people to point actions at a rabbitmq that is not the same as the chef_index one. This change will allow us to check the health of each one. Signed-off-by: Jay Mundrawala --- ...chef_wm_actions_queue_monitoring_SUITE.erl | 10 +- .../src/chef_wm_actions_queue_monitoring.erl | 10 +- .../src/chef_wm_rabbitmq_management.erl | 136 +++++------------- .../apps/oc_chef_wm/src/chef_wm_status.erl | 6 +- .../apps/oc_chef_wm/src/oc_chef_action.erl | 7 +- .../src/oc_chef_action_queue_config.erl | 92 ++++++++++++ .../apps/oc_chef_wm/src/oc_chef_wm_sup.erl | 11 +- ...chef_wm_actions_queue_monitoring_tests.erl | 22 +-- .../oc_chef_wm/test/chef_wm_status_tests.erl | 14 +- 9 files changed, 169 insertions(+), 139 deletions(-) create mode 100644 src/oc_erchef/apps/oc_chef_wm/src/oc_chef_action_queue_config.erl diff --git a/src/oc_erchef/apps/oc_chef_wm/itest/chef_wm_actions_queue_monitoring_SUITE.erl b/src/oc_erchef/apps/oc_chef_wm/itest/chef_wm_actions_queue_monitoring_SUITE.erl index bf84a6c2e1..789d22f619 100644 --- a/src/oc_erchef/apps/oc_chef_wm/itest/chef_wm_actions_queue_monitoring_SUITE.erl +++ b/src/oc_erchef/apps/oc_chef_wm/itest/chef_wm_actions_queue_monitoring_SUITE.erl @@ -35,7 +35,7 @@ -define(ACTIONS_EXCHANGE, <<"actions">>). -define(Q_SETTING(K, V), - chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(K,V)). + oc_chef_action_queue_config:set_rabbit_queue_monitor_setting(K,V)). init_per_suite(InitialConfig) -> UseFakeRabbit = @@ -134,7 +134,7 @@ basic_queue_monitor(Config) -> % don't drop messages from now on - chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(drop_on_full_capacity, false), + oc_chef_action_queue_config:set_rabbit_queue_monitor_setting(drop_on_full_capacity, false), make_data_bag(?CLIENT_NAME, 13), make_data_bag(?CLIENT_NAME, 14), @@ -172,7 +172,7 @@ queue_full_dont_start(Config) -> case ?config(use_fake_rabbit, Config) of true -> meck:expect(chef_wm_rabbitmq_management, sync_check_queue_at_capacity, - fun(_Vhost, _Queue) -> + fun(_PoolNameAtom, _Vhost, _Queue) -> {MaxLength, MaxLength, true} end); false -> ok @@ -347,12 +347,12 @@ setup_rabbit_fake(Config) -> meck:expect(chef_wm_rabbitmq_management, get_max_length, - fun(_Vhost) -> + fun(_PoolNameAtom, _Vhost) -> ?config(max_length, Config) end), meck:expect(chef_wm_rabbitmq_management, get_current_length, - fun(_Vhost, _Queue) -> + fun(_PoolNameAtom, _Vhost, _Queue) -> fake_rabbit_current_length() end), ok. diff --git a/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_actions_queue_monitoring.erl b/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_actions_queue_monitoring.erl index 65f555ac66..d4c4bcfbd5 100644 --- a/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_actions_queue_monitoring.erl +++ b/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_actions_queue_monitoring.erl @@ -88,7 +88,7 @@ -define(QUEUE, <<"alaska">>). -define(GEN_SERVER_TIMEOUT_MILLIS, 5000). -define(QUEUE_MONITOR_RUN_EVERY_MILLIS, 60000). --define(QUEUE_MONITOR_SETTING(Key, Default), chef_wm_rabbitmq_management:get_rabbit_queue_monitor_setting(Key, Default)). +-define(QUEUE_MONITOR_SETTING(Key, Default), oc_chef_action_queue_config:get_rabbit_queue_monitor_setting(Key, Default)). -record(queue_monitor_state, { % name of the rabbitmq vhost to monitor, as a string @@ -306,9 +306,11 @@ handle_info(status_ping, #queue_monitor_state{ Pid = spawn_link( fun () -> Result = - chef_wm_rabbitmq_management:check_current_queue_state(Vhost, - Queue, - Dropped), + chef_wm_rabbitmq_management:check_current_queue_state( + oc_chef_action_queue_config:get_rabbit_management_pool_name(), + Vhost, + Queue, + Dropped), ParentPid ! Result end), {noreply, State#queue_monitor_state{worker_process=Pid}}; diff --git a/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_rabbitmq_management.erl b/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_rabbitmq_management.erl index 856ae0c35c..e971bcaf12 100644 --- a/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_rabbitmq_management.erl +++ b/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_rabbitmq_management.erl @@ -28,24 +28,17 @@ -endif. -export([calc_ratio_and_percent/2, - get_max_length/1, - get_current_length/2, - create_pool/0, - delete_pool/0, - get_pool_configs/0, - check_current_queue_state/3, - check_current_queue_length/4, - sync_check_queue_at_capacity/2, - get_rabbit_management_setting/2, - get_rabbit_queue_monitor_setting/2, - set_rabbit_management_setting/2, - set_rabbit_queue_monitor_setting/2, - set_app_value/5, - check_aliveness/1 + get_max_length/2, + get_current_length/3, + create_pool/2, + delete_pool/1, + check_current_queue_state/4, + check_current_queue_length/5, + sync_check_queue_at_capacity/3, + check_aliveness/2 ]). --define(POOLNAME, rabbitmq_management_service). -define(LOG_THRESHOLD, 0.8). -type max_length() :: integer(). @@ -56,72 +49,12 @@ % NOTE: oc_httpc client is configured to prepend /api %% oc_httpc pool functions -------------------------------------------- -create_pool() -> - Pools = get_pool_configs(), - [oc_httpc:add_pool(PoolNameAtom, Config) || {PoolNameAtom, Config} <- Pools, Config /= []], +create_pool(PoolNameAtom, Config) -> + oc_httpc:add_pool(PoolNameAtom, Config), ok. -delete_pool() -> - Pools = get_pool_configs(), - [ok = oc_httpc:delete_pool(PoolNameAtom) || {PoolNameAtom, _Config} <- Pools], - ok. - -get_pool_configs() -> - ServiceConfig = get_rabbit_management_setting(rabbitmq_management_service, []), - [{?POOLNAME, ServiceConfig}]. - -get_rabbit_management_setting(Key, Default) -> - try - RabbitConfig = envy:get(oc_chef_wm, rabbitmq, [], any), - MgmtConfig = proplists:get_value(management, RabbitConfig, []), - Config = proplists:get_value(Key, MgmtConfig, Default), - add_auth_to_ibrowse_options(Config, MgmtConfig) - catch Error:Reason -> - lager:info("Can't get configuration setting ~p ~p: ~p ~p", - [Key, Default, Error, Reason]), - Default - end. - -add_auth_to_ibrowse_options(Config, MgmtConfig) -> - Username = proplists:get_value(user, MgmtConfig), - {ok, Password} = chef_secrets:get(<<"rabbitmq">>, <<"management_password">>), - IbrowseOptions = proplists:get_value(ibrowse_options, Config), - Config1 = proplists:delete(ibrowse_options, Config), - IbrowseOptions1 = [{basic_auth, {Username, erlang:binary_to_list(Password)}} | IbrowseOptions], - [{ibrowse_options, IbrowseOptions1} | Config1]. - -get_rabbit_queue_monitor_setting(Key, Default) -> - try - RabbitConfig = envy:get(oc_chef_wm, rabbitmq, [], any), - MonitoringConfig = proplists:get_value(monitoring, RabbitConfig, []), - proplists:get_value(Key, MonitoringConfig, Default) - catch Error:Reason -> - lager:info("Can't get configuration setting ~p ~p: ~p ~p", - [Key, Default, Error, Reason]), - Default - end. - -set_rabbit_management_setting(Key, NewVal) -> - set_app_value(oc_chef_wm, rabbitmq, management, Key, NewVal). - -set_rabbit_queue_monitor_setting(Key, NewVal) -> - set_app_value(oc_chef_wm, rabbitmq, monitoring, Key, NewVal). - -%% value MUST already exist in the dict -set_app_value(App, ConfigKey, SubSectionKey, Prop, NewValue) -> - Cfg = envy:get(App, ConfigKey, [], any), - NewCfg = - lists:map(fun ({SectionKey,SectionValue}) - when SectionKey == SubSectionKey -> - {SectionKey, - lists:map(fun ({K, _V}) when K == Prop -> - {K, NewValue}; - (Val) -> Val - end, SectionValue) - }; - (Val) -> Val - end, Cfg), - application:set_env(App, ConfigKey, NewCfg). +delete_pool(PoolNameAtom) -> + ok = oc_httpc:delete_pool(PoolNameAtom). -spec calc_ratio_and_percent(integer(), integer()) -> {float(), float()}. @@ -136,9 +69,9 @@ calc_ratio_and_percent(CurrentLength, MaxLength) -> Pcnt = round(Ratio * 100.0), {Ratio, Pcnt}. --spec rabbit_mgmt_server_request(string()) -> oc_httpc:response(). -rabbit_mgmt_server_request(Path) -> - oc_httpc:request(?POOLNAME, Path, [], get, []). +-spec rabbit_mgmt_server_request(atom(), string()) -> oc_httpc:response(). +rabbit_mgmt_server_request(PoolNameAtom, Path) -> + oc_httpc:request(PoolNameAtom, Path, [], get, []). -spec mk_max_length_path(string()) -> string(). mk_max_length_path(Vhost) -> @@ -155,9 +88,9 @@ mk_aliveness_check_path(Vhost) -> % make an http connection to the rabbitmq management console % and return a integer value or undefined --spec get_max_length(string()) -> integer() | undefined. -get_max_length(Vhost) -> - MaxResult = rabbit_mgmt_server_request(mk_max_length_path(Vhost)), +-spec get_max_length(atom(), string()) -> integer() | undefined. +get_max_length(PoolNameAtom, Vhost) -> + MaxResult = rabbit_mgmt_server_request(PoolNameAtom, mk_max_length_path(Vhost)), case MaxResult of {ok, "200", _, MaxLengthJson} -> parse_max_length_response(MaxLengthJson); @@ -175,9 +108,9 @@ get_max_length(Vhost) -> % make an http connection to the rabbitmq management console % and return a integer value or undefined --spec get_current_length(string(), string()) -> integer() | undefined. -get_current_length(Vhost, Queue) -> - CurrentResult = rabbit_mgmt_server_request(mk_current_length_path(Vhost)), +-spec get_current_length(atom(), string(), string()) -> integer() | undefined. +get_current_length(PoolNameAtom, Vhost, Queue) -> + CurrentResult = rabbit_mgmt_server_request(PoolNameAtom, mk_current_length_path(Vhost)), case CurrentResult of {error, {conn_failed,_}} -> lager:info("Can't connect to RabbitMQ management console to fetch current length"), @@ -235,9 +168,9 @@ parse_max_length_response(Message) -> end. --spec sync_check_queue_at_capacity(string(), string()) -> {integer(), integer(), boolean()}. -sync_check_queue_at_capacity(Vhost, Queue) -> - Result = check_current_queue_state(Vhost, Queue, 0), +-spec sync_check_queue_at_capacity(atom(), string(), string()) -> {integer(), integer(), boolean()}. +sync_check_queue_at_capacity(PoolNameAtom, Vhost, Queue) -> + Result = check_current_queue_state(PoolNameAtom, Vhost, Queue, 0), case Result of skipped -> {0, 0, false}; {MaxLength, reset_dropped_since_last_check} -> {MaxLength, 0, false}; @@ -246,30 +179,31 @@ sync_check_queue_at_capacity(Vhost, Queue) -> --spec check_current_queue_state(string(), string(), integer()) -> +-spec check_current_queue_state(atom(), string(), string(), integer()) -> skipped | {max_length(), reset_dropped_since_last_check} | {max_length(), current_length(), queue_at_capacity()}. -check_current_queue_state(Vhost, Queue, DroppedSinceLastCheck) -> - case chef_wm_rabbitmq_management:get_max_length(Vhost) of +check_current_queue_state(PoolNameAtom, Vhost, Queue, DroppedSinceLastCheck) -> + case chef_wm_rabbitmq_management:get_max_length(PoolNameAtom, Vhost) of undefined -> skipped; % max length isn't configured, or something is broken % don't continue. MaxLength -> lager:debug("Queue Monitor max length = ~p", [MaxLength]), - check_current_queue_length(Vhost, + check_current_queue_length(PoolNameAtom, + Vhost, Queue, MaxLength, DroppedSinceLastCheck) end. --spec check_current_queue_length(string(), string(), integer(), integer()) -> +-spec check_current_queue_length(atom(), string(), string(), integer(), integer()) -> {max_length(), reset_dropped_since_last_check} | {max_length(), current_length(), queue_at_capacity()}. -check_current_queue_length(Vhost, Queue, MaxLength, DroppedSinceLastCheck) -> +check_current_queue_length(PoolNameAtom, Vhost, Queue, MaxLength, DroppedSinceLastCheck) -> % use ?MODULE here so I can use meck in integrations testing % https://github.com/eproxus/meck/issues/142 - CurrentLength = ?MODULE:get_current_length(Vhost, Queue), + CurrentLength = ?MODULE:get_current_length(PoolNameAtom, Vhost, Queue), case CurrentLength of undefined -> % a queue doesn't appear to be bound to the @@ -308,9 +242,9 @@ parse_integer(Val) when is_list(Val) -> end; parse_integer(_) -> undefined. --spec check_aliveness(string()) -> boolean(). -check_aliveness(Vhost) -> - Aliveness = rabbit_mgmt_server_request(mk_aliveness_check_path(Vhost)), +-spec check_aliveness(atom(), string()) -> boolean(). +check_aliveness(PoolNameAtom, Vhost) -> + Aliveness = rabbit_mgmt_server_request(PoolNameAtom, mk_aliveness_check_path(Vhost)), case Aliveness of {ok, "200", _, _} -> true; diff --git a/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_status.erl b/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_status.erl index 15bcbf7a62..24f65bded8 100644 --- a/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_status.erl +++ b/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_status.erl @@ -61,7 +61,7 @@ check_health() -> Status = overall_status(Pings), QueueMonStatus = - case chef_wm_rabbitmq_management:get_rabbit_queue_monitor_setting(queue_length_monitor_enabled, false) of + case oc_chef_action_queue_config:get_rabbit_queue_monitor_setting(queue_length_monitor_enabled, false) of false -> % chef_wm_actions_queue_monitoring isn't running, skip it []; true -> AnalyticsQ = chef_wm_actions_queue_monitoring:status(), @@ -92,7 +92,7 @@ overall_status(Pings) -> -spec is_analytics_queue_at_capacity() -> boolean(). is_analytics_queue_at_capacity() -> - case chef_wm_rabbitmq_management:get_rabbit_queue_monitor_setting(queue_length_monitor_enabled, false) of + case oc_chef_action_queue_config:get_rabbit_queue_monitor_setting(queue_length_monitor_enabled, false) of % don't try to connect to the queue monitor if it isn't running true -> chef_wm_actions_queue_monitoring:is_queue_at_capacity(); false -> false @@ -100,7 +100,7 @@ is_analytics_queue_at_capacity() -> -spec queue_at_capacity_affects_overall_status() -> boolean(). queue_at_capacity_affects_overall_status() -> - chef_wm_rabbitmq_management:get_rabbit_queue_monitor_setting(queue_at_capacity_affects_overall_status, false). + oc_chef_action_queue_config:get_rabbit_queue_monitor_setting(queue_at_capacity_affects_overall_status, false). -spec log_failure(fail | pong, [{binary(), <<_:32>>}], list()) -> ok. diff --git a/src/oc_erchef/apps/oc_chef_wm/src/oc_chef_action.erl b/src/oc_erchef/apps/oc_chef_wm/src/oc_chef_action.erl index 4cdb162557..7ca61281f7 100644 --- a/src/oc_erchef/apps/oc_chef_wm/src/oc_chef_action.erl +++ b/src/oc_erchef/apps/oc_chef_wm/src/oc_chef_action.erl @@ -174,7 +174,7 @@ maybe_add_remote_request_id(Msg, RemoteRequestId) -> Msg :: binary()) -> ok. publish(RoutingKey, Msg)-> QueueMonitorEnabled = - chef_wm_rabbitmq_management:get_rabbit_queue_monitor_setting(queue_length_monitor_enabled, false), + oc_chef_action_queue_config:get_rabbit_queue_monitor_setting(queue_length_monitor_enabled, false), publish(RoutingKey, Msg, QueueMonitorEnabled). @@ -185,7 +185,7 @@ publish(RoutingKey, Msg, false) -> oc_chef_action_queue:publish(RoutingKey, Msg); publish(RoutingKey, Msg, true) -> DropOnCapacity = - chef_wm_rabbitmq_management:get_rabbit_queue_monitor_setting(drop_on_full_capacity, true), + oc_chef_action_queue_config:get_rabbit_queue_monitor_setting(drop_on_full_capacity, true), case DropOnCapacity andalso chef_wm_actions_queue_monitoring:is_queue_at_capacity() of true -> chef_wm_actions_queue_monitoring:message_dropped(); @@ -399,7 +399,8 @@ req_header(Name, Req) -> -spec ping() -> pong | pang. ping() -> VHost = envy:get(oc_chef_wm, actions_vhost, binary), - case chef_wm_rabbitmq_management:check_aliveness(binary_to_list(VHost)) of + case chef_wm_rabbitmq_management:check_aliveness( + oc_chef_action_queue_config:get_rabbit_management_pool_name(), binary_to_list(VHost)) of true -> pong; _ -> pang end. diff --git a/src/oc_erchef/apps/oc_chef_wm/src/oc_chef_action_queue_config.erl b/src/oc_erchef/apps/oc_chef_wm/src/oc_chef_action_queue_config.erl new file mode 100644 index 0000000000..380683c751 --- /dev/null +++ b/src/oc_erchef/apps/oc_chef_wm/src/oc_chef_action_queue_config.erl @@ -0,0 +1,92 @@ +%% Copyright 2012-2014 Chef Software, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(oc_chef_action_queue_config). + +-export([ + get_rabbit_management_pool_name/0, + get_rabbit_management_pool_setting/0, + get_rabbit_queue_monitor_setting/2, + set_rabbit_management_setting/2, + set_rabbit_queue_monitor_setting/2, + set_app_value/5 + ]). + +-define(SERVER, ?MODULE). +-define(POOLNAME, rabbitmq_management_service). + +get_rabbit_management_pool_setting() -> + ServiceConfig = get_rabbit_management_setting(rabbitmq_management_service, []), + {?POOLNAME, ServiceConfig}. + +get_rabbit_management_pool_name() -> + ?POOLNAME. + +get_rabbit_management_setting(Key, Default) -> + try + RabbitConfig = envy:get(oc_chef_wm, rabbitmq, [], any), + MgmtConfig = proplists:get_value(management, RabbitConfig, []), + Config = proplists:get_value(Key, MgmtConfig, Default), + add_auth_to_ibrowse_options(Config, MgmtConfig) + catch Error:Reason -> + lager:info("Can't get configuration setting ~p ~p: ~p ~p", + [Key, Default, Error, Reason]), + Default + end. + +add_auth_to_ibrowse_options(Config, MgmtConfig) -> + Username = proplists:get_value(user, MgmtConfig), + {ok, Password} = chef_secrets:get(<<"rabbitmq">>, <<"management_password">>), + IbrowseOptions = proplists:get_value(ibrowse_options, Config), + Config1 = proplists:delete(ibrowse_options, Config), + IbrowseOptions1 = [{basic_auth, {Username, erlang:binary_to_list(Password)}} | IbrowseOptions], + [{ibrowse_options, IbrowseOptions1} | Config1]. + + +get_rabbit_queue_monitor_setting(Key, Default) -> + try + RabbitConfig = envy:get(oc_chef_wm, rabbitmq, [], any), + MonitoringConfig = proplists:get_value(monitoring, RabbitConfig, []), + proplists:get_value(Key, MonitoringConfig, Default) + catch Error:Reason -> + lager:info("Can't get configuration setting ~p ~p: ~p ~p", + [Key, Default, Error, Reason]), + Default + end. + +set_rabbit_management_setting(Key, NewVal) -> + set_app_value(oc_chef_wm, rabbitmq, management, Key, NewVal). + +set_rabbit_queue_monitor_setting(Key, NewVal) -> + set_app_value(oc_chef_wm, rabbitmq, monitoring, Key, NewVal). + +%% value MUST already exist in the dict +set_app_value(App, ConfigKey, SubSectionKey, Prop, NewValue) -> + Cfg = envy:get(App, ConfigKey, [], any), + NewCfg = + lists:map(fun ({SectionKey,SectionValue}) + when SectionKey == SubSectionKey -> + {SectionKey, + lists:map(fun ({K, _V}) when K == Prop -> + {K, NewValue}; + (Val) -> Val + end, SectionValue) + }; + (Val) -> Val + end, Cfg), + application:set_env(App, ConfigKey, NewCfg). + diff --git a/src/oc_erchef/apps/oc_chef_wm/src/oc_chef_wm_sup.erl b/src/oc_erchef/apps/oc_chef_wm/src/oc_chef_wm_sup.erl index 019b926b73..763a39f039 100644 --- a/src/oc_erchef/apps/oc_chef_wm/src/oc_chef_wm_sup.erl +++ b/src/oc_erchef/apps/oc_chef_wm/src/oc_chef_wm_sup.erl @@ -17,7 +17,7 @@ -include("oc_chef_wm.hrl"). --define(QUEUE_MONITOR_SETTING(Key, Default), chef_wm_rabbitmq_management:get_rabbit_queue_monitor_setting(Key, Default)). +-define(QUEUE_MONITOR_SETTING(Key, Default), oc_chef_action_queue_config:get_rabbit_queue_monitor_setting(Key, Default)). %% @spec start_link() -> ServerRet %% @doc API for starting the supervisor. @@ -74,8 +74,9 @@ maybe_start_action(true, Workers) -> case QMEnabled of true -> - chef_wm_rabbitmq_management:create_pool(), - {MaxLength, CurrentLength} = check_queue_at_capacity(Vhost, Queue), + {PoolNameAtom, PoolConfig} = oc_chef_action_queue_config:get_rabbit_management_pool_setting(), + chef_wm_rabbitmq_management:create_pool(PoolNameAtom, PoolConfig), + {MaxLength, CurrentLength} = check_actions_queue_at_capacity(PoolNameAtom, Vhost, Queue), ActionQueueMonitoringSpec = {chef_wm_actions_queue_monitoring, {chef_wm_actions_queue_monitoring, start_link, [Vhost, Queue, MaxLength, CurrentLength]}, @@ -91,10 +92,10 @@ maybe_start_action(false, Workers) -> Workers. -check_queue_at_capacity(Vhost, Queue) -> +check_actions_queue_at_capacity(PoolNameAtom, Vhost, Queue) -> PreventStartupOnCap = ?QUEUE_MONITOR_SETTING(prevent_erchef_startup_on_full_capacity, false), {MaxLength, CurrentLength, QueueAtCapacity} = - chef_wm_rabbitmq_management:sync_check_queue_at_capacity(Vhost, Queue), + chef_wm_rabbitmq_management:sync_check_queue_at_capacity(PoolNameAtom, Vhost, Queue), case QueueAtCapacity of true -> case PreventStartupOnCap of diff --git a/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_actions_queue_monitoring_tests.erl b/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_actions_queue_monitoring_tests.erl index b1cd666412..7204c1f406 100644 --- a/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_actions_queue_monitoring_tests.erl +++ b/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_actions_queue_monitoring_tests.erl @@ -332,8 +332,8 @@ queue_length_test_() -> {"check_publish_not_at_capacity", fun() -> % ensure the publish function is called and no messages are dropped - chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(queue_length_monitor_enabled, true), - chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(drop_on_full_capacity, true), + oc_chef_action_queue_config:set_rabbit_queue_monitor_setting(queue_length_monitor_enabled, true), + oc_chef_action_queue_config:set_rabbit_queue_monitor_setting(drop_on_full_capacity, true), meck:new(oc_chef_action_queue), meck:expect(oc_chef_action_queue, publish, fun (_, _) -> ok end), @@ -350,8 +350,8 @@ queue_length_test_() -> fun() -> %% ensure the publish function is called and 1 message is dropped %% due to queue being at capacity - chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(queue_length_monitor_enabled, true), - chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(drop_on_full_capacity, true), + oc_chef_action_queue_config:set_rabbit_queue_monitor_setting(queue_length_monitor_enabled, true), + oc_chef_action_queue_config:set_rabbit_queue_monitor_setting(drop_on_full_capacity, true), meck_response("200", max_length_json(), "200", at_capacity_json()), %% ensure that the queue is at capacity before calling @@ -382,8 +382,8 @@ queue_length_test_() -> %% due to queue being at capacity, reset queue length to 0 %% and 1 message should be published - chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(queue_length_monitor_enabled, true), - chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(drop_on_full_capacity, true), + oc_chef_action_queue_config:set_rabbit_queue_monitor_setting(queue_length_monitor_enabled, true), + oc_chef_action_queue_config:set_rabbit_queue_monitor_setting(drop_on_full_capacity, true), meck_response("200", max_length_json(), "200", at_capacity_json()), %% ensure that the queue is at capacity before calling @@ -425,8 +425,8 @@ queue_length_test_() -> {"check_publish_at_capacity_no_drop", fun() -> % queue is at capacity, but don't drop messages due to configuration - chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(queue_length_monitor_enabled, true), - chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(drop_on_full_capacity, false), + oc_chef_action_queue_config:set_rabbit_queue_monitor_setting(queue_length_monitor_enabled, true), + oc_chef_action_queue_config:set_rabbit_queue_monitor_setting(drop_on_full_capacity, false), meck_response("200", max_length_json(), "200", at_capacity_json()), %% ensure that the queue is at capacity before calling @@ -451,8 +451,8 @@ queue_length_test_() -> % queue length monitor is disabled catch(chef_wm_actions_queue_monitoring:stop()), - chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(queue_length_monitor_enabled, false), - chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(drop_on_full_capacity, false), + oc_chef_action_queue_config:set_rabbit_queue_monitor_setting(queue_length_monitor_enabled, false), + oc_chef_action_queue_config:set_rabbit_queue_monitor_setting(drop_on_full_capacity, false), undefined = whereis(chef_wm_actions_queue_monitoring), meck:new(oc_chef_action_queue), @@ -470,7 +470,7 @@ queue_length_test_() -> fun() -> % set oc_httpc to sleep for 100 millis, but have is_queue_at_capacity timeout % after 10 millis. - chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(queue_length_monitor_timeout_millis, 0), + oc_chef_action_queue_config:set_rabbit_queue_monitor_setting(queue_length_monitor_timeout_millis, 0), ?assertEqual(true, chef_wm_actions_queue_monitoring:is_queue_at_capacity()) end diff --git a/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_status_tests.erl b/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_status_tests.erl index 8e5c6c3c88..95cb1dcdd3 100644 --- a/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_status_tests.erl +++ b/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_status_tests.erl @@ -44,7 +44,7 @@ setup_env() -> application:set_env(oc_chef_wm, health_ping_timeout, ?PING_TIMEOUT), application:set_env(oc_chef_wm, health_ping_modules, ?CHECK_MODS), application:set_env(oc_chef_wm, rabbitmq, default_config()), - chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(queue_length_monitor_enabled, false), + oc_chef_action_queue_config:set_rabbit_queue_monitor_setting(queue_length_monitor_enabled, false), ok. cleanup_env() -> @@ -222,7 +222,7 @@ check_queue_mon_affects_overall_status_test_() -> fun() -> % overall_status will be fail as the queue is at capacity as % queue_at_capacity_affects_overall_status is true - chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(queue_at_capacity_affects_overall_status, true), + oc_chef_action_queue_config:set_rabbit_queue_monitor_setting(queue_at_capacity_affects_overall_status, true), {Status, Json} = chef_wm_status:check_health(), ?assertEqual(fail, Status), Ejson = chef_json:decode(Json), @@ -234,7 +234,7 @@ check_queue_mon_affects_overall_status_test_() -> fun() -> % overall_status will be pong as the queue is at capacity, but % queue_at_capacity_affects_overall_status is false - chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(queue_at_capacity_affects_overall_status, false), + oc_chef_action_queue_config:set_rabbit_queue_monitor_setting(queue_at_capacity_affects_overall_status, false), {Status, Json} = chef_wm_status:check_health(), ?assertEqual(pong, Status), Ejson = chef_json:decode(Json), @@ -246,8 +246,8 @@ check_queue_mon_affects_overall_status_test_() -> fun() -> % overall_status will be pong as the queue is at capacity, but the queue monitor is % disabled - chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(queue_at_capacity_affects_overall_status, false), - chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(queue_length_monitor_enabled, false), + oc_chef_action_queue_config:set_rabbit_queue_monitor_setting(queue_at_capacity_affects_overall_status, false), + oc_chef_action_queue_config:set_rabbit_queue_monitor_setting(queue_length_monitor_enabled, false), {Status, Json} = chef_wm_status:check_health(), ?assertEqual(pong, Status), Ejson = chef_json:decode(Json), @@ -258,8 +258,8 @@ check_queue_mon_affects_overall_status_test_() -> end, fun() -> % overall_status not affected as the queue monitor is disabled - chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(queue_at_capacity_affects_overall_status, true), - chef_wm_rabbitmq_management:set_rabbit_queue_monitor_setting(queue_length_monitor_enabled, false), + oc_chef_action_queue_config:set_rabbit_queue_monitor_setting(queue_at_capacity_affects_overall_status, true), + oc_chef_action_queue_config:set_rabbit_queue_monitor_setting(queue_length_monitor_enabled, false), {Status, Json} = chef_wm_status:check_health(), ?assertEqual(pong, Status), Ejson = chef_json:decode(Json), From d1b8796a980fc9d3659b3dddc3ab935e36a15036 Mon Sep 17 00:00:00 2001 From: Jay Mundrawala Date: Sun, 30 Jul 2017 20:47:40 -0500 Subject: [PATCH 04/10] Add ping for chef_index that checks rabbitmq Signed-off-by: Jay Mundrawala --- .../private-chef/recipes/rabbitmq.rb | 7 +++++ .../templates/default/oc_erchef.config.erb | 22 ++++++++++++++- .../apps/chef_index/src/chef_index.app.src | 3 +- .../apps/chef_index/src/chef_index.erl | 18 +++++++++++- .../apps/chef_index/src/chef_index_queue.erl | 28 ++++++++++++++++++- .../apps/chef_index/src/chef_index_sup.erl | 19 +++++++++++++ .../src/chef_wm_rabbitmq_management.erl | 2 +- 7 files changed, 94 insertions(+), 5 deletions(-) diff --git a/omnibus/files/private-chef-cookbooks/private-chef/recipes/rabbitmq.rb b/omnibus/files/private-chef-cookbooks/private-chef/recipes/rabbitmq.rb index cd605fde8b..940e4178d8 100644 --- a/omnibus/files/private-chef-cookbooks/private-chef/recipes/rabbitmq.rb +++ b/omnibus/files/private-chef-cookbooks/private-chef/recipes/rabbitmq.rb @@ -218,6 +218,13 @@ retries 10 end + execute "#{rmq_ctl} set_permissions -p #{rabbitmq['vhost']} #{rabbitmq['management_user']} \".*\" \".*\" \".*\"" do + environment (rabbitmq_env) + user opc_username + not_if "#{rmq_ctl_chpst} list_user_permissions #{rabbitmq['management_user']}|grep #{rabbitmq['vhost']}", :environment => rabbitmq_env, :user => "root" + retries 10 + end + execute "#{rmq_ctl} set_permissions -p / #{rabbitmq['management_user']} \".*\" \".*\" \".*\"" do environment (rabbitmq_env) user opc_username diff --git a/omnibus/files/private-chef-cookbooks/private-chef/templates/default/oc_erchef.config.erb b/omnibus/files/private-chef-cookbooks/private-chef/templates/default/oc_erchef.config.erb index 276022a4c5..7a49bd1294 100755 --- a/omnibus/files/private-chef-cookbooks/private-chef/templates/default/oc_erchef.config.erb +++ b/omnibus/files/private-chef-cookbooks/private-chef/templates/default/oc_erchef.config.erb @@ -127,9 +127,10 @@ data_collector, <% end %> oc_chef_authz, - <% if node['private_chef']['dark_launch']['actions'] %> + <% if node['private_chef']['dark_launch']['actions'] && node['private_chef']['rabbitmq']['management_enabled'] %> oc_chef_action, <% end %> + chef_index, chef_sql, chef_<%= node['private_chef']['opscode-erchef']['search_provider'] %> ]}, @@ -230,6 +231,25 @@ {max_age, <%= @solr_http_max_age %>}, {max_connection_duration, <%= @solr_http_max_connection_duration %>}, {ibrowse_options, <%= @solr_ibrowse_options %>} + ]}, + {rabbitmq_index_management_service, [ + {enabled, <%= @node['private_chef']['rabbitmq']['management_enabled'] %>}, + {user, "<%= @node['private_chef']['rabbitmq']['management_user'] %>"}, + <% if node['private_chef']['fips_enabled'] -%> + {root_url, "http://<%= node['private_chef']['rabbitmq']['vip'] %>:<%= @node['private_chef']['rabbitmq']['management_port'] %>/api"}, + <% else -%> + {root_url, "https://<%= node['private_chef']['rabbitmq']['vip'] %>:<%= @node['private_chef']['rabbitmq']['management_port'] %>/api"}, + <% end %> + {timeout, <%= @node['private_chef']['rabbitmq']['rabbit_mgmt_timeout'] %>}, + {init_count, <%= @node['private_chef']['rabbitmq']['rabbit_mgmt_http_init_count'] %>}, + {max_count, <%= @node['private_chef']['rabbitmq']['rabbit_mgmt_http_max_count'] %>}, + {cull_interval, {<%= @node['private_chef']['rabbitmq']['rabbit_mgmt_http_cull_interval'] %>, sec}}, + {max_age, {<%= @node['private_chef']['rabbitmq']['rabbit_mgmt_http_max_age'] %>, sec}}, + {max_connection_duration, {<%= @node['private_chef']['rabbitmq']['rabbit_mgmt_http_max_connection_duration'] %>, sec}}, + + {ibrowse_options, [ + <%= @node['private_chef']['rabbitmq']['rabbit_mgmt_ibrowse_options'] %> + ]} ]} ]}, diff --git a/src/oc_erchef/apps/chef_index/src/chef_index.app.src b/src/oc_erchef/apps/chef_index/src/chef_index.app.src index 99c4bb5b37..c722d069c5 100644 --- a/src/oc_erchef/apps/chef_index/src/chef_index.app.src +++ b/src/oc_erchef/apps/chef_index/src/chef_index.app.src @@ -24,7 +24,8 @@ kernel, stdlib, gen_bunny, - ibrowse + ibrowse, + chef_secrets ]}, {env, []} ]}. diff --git a/src/oc_erchef/apps/chef_index/src/chef_index.erl b/src/oc_erchef/apps/chef_index/src/chef_index.erl index 029315c98f..f9cc977e8d 100644 --- a/src/oc_erchef/apps/chef_index/src/chef_index.erl +++ b/src/oc_erchef/apps/chef_index/src/chef_index.erl @@ -29,7 +29,8 @@ delete/4, add/5, add_batch/1, - search_provider/0 + search_provider/0, + ping/0 ]). -include("chef_solr.hrl"). @@ -187,3 +188,18 @@ send_to_solr(batch, Doc) -> chef_index_batch:add_item(Doc); send_to_solr(inline, Doc) -> chef_index_expand:send_item(Doc). + +ping() -> + case queue_mode() of + rabbitmq -> + Config = envy:get(chef_index, rabbitmq_index_management_service, [], any), + Enabled = proplists:get_value(enabled, Config), + case Enabled of + true -> + chef_index_queue:ping(envy:get(chef_index, rabbitmq_vhost, binary)); + _ -> + pong + end; + _ -> + pong + end. diff --git a/src/oc_erchef/apps/chef_index/src/chef_index_queue.erl b/src/oc_erchef/apps/chef_index/src/chef_index_queue.erl index 6c250fbf33..b17f354448 100644 --- a/src/oc_erchef/apps/chef_index/src/chef_index_queue.erl +++ b/src/oc_erchef/apps/chef_index/src/chef_index_queue.erl @@ -21,7 +21,9 @@ delete/4, delete/5, set/5, - set/6 + set/6, + create_management_pool/3, + ping/1 ]). -ifdef(TEST). @@ -36,6 +38,8 @@ -type solr_url() :: [byte()] | binary() | undefined. -type vhost() :: binary(). +-define(POOLNAME, rabbitmq_index_management_service). + %%%% %% Public API %%%% @@ -103,10 +107,31 @@ package_for_delete(Type, ID, DatabaseName, SolrUrl) -> InnerEnvelope = inner_envelope(Type, ID, DatabaseName, {[]}, SolrUrl), {[{action, delete}, {payload, InnerEnvelope}]}. + +create_management_pool(Username, Password, Config) -> + chef_wm_rabbitmq_management:create_pool(?POOLNAME, add_basic_auth(Username, Password, Config)). + +-spec ping(binary()) -> pong | pang. +ping(VHost) -> + % TODO(jaym) 2017-08-02: chef_wm_rabbitmq_management should be moved to a shared app. + % The reason for this is because referencing chef_wm_rabbitmq_management from here + % creates a 2 way dependency between chef_index and oc_chef_wm. + case chef_wm_rabbitmq_management:check_aliveness( + ?POOLNAME, binary_to_list(VHost)) of + true -> pong; + _ -> pang + end. + %%%% %% Internal %%%% +add_basic_auth(Username, Password, Config) -> + IbrowseOptions = proplists:get_value(ibrowse_options, Config), + Config1 = proplists:delete(ibrowse_options, Config), + IbrowseOptions1 = [{basic_auth, {Username, erlang:binary_to_list(Password)}} | IbrowseOptions], + [{ibrowse_options, IbrowseOptions1} | Config1]. + -spec inner_envelope(chef_indexable_type(), uuid_binary(), chef_db_name(), ejson(), solr_url()) -> ejson(). inner_envelope(Type, ID, DatabaseName, Item, SolrUrl) -> %% SAMPLE ENVELOPE: @@ -157,3 +182,4 @@ object_id_to_i(UUID) -> unix_time() -> {MS, S, _US} = os:timestamp(), (1000000 * MS) + S. + diff --git a/src/oc_erchef/apps/chef_index/src/chef_index_sup.erl b/src/oc_erchef/apps/chef_index/src/chef_index_sup.erl index d27d1ed547..7804ce7b31 100644 --- a/src/oc_erchef/apps/chef_index/src/chef_index_sup.erl +++ b/src/oc_erchef/apps/chef_index/src/chef_index_sup.erl @@ -49,9 +49,28 @@ init([]) -> error_logger:info_msg("Starting chef_index_sup.~n", []), error_logger:info_msg("Creating HTTP pool for Solr.~n"), chef_index_http:create_pool(), + maybe_rabbitmq_monitoring(), Children = child_spec(), {ok, {{one_for_one, 60, 10}, Children}}. +maybe_rabbitmq_monitoring() -> + case envy:get(chef_index, search_queue_mode, rabbitmq, envy:one_of([rabbitmq, batch, inline])) of + rabbitmq -> + Config = envy:get(chef_index, rabbitmq_index_management_service, [], any), + {LocalConfig, HttpConfig} = proplists:split(Config, [enabled, user]), + case LocalConfig of + [[{enabled, true}], [{user, Username}]] -> + {ok, Password} = chef_secrets:get(<<"rabbitmq">>, <<"management_password">>), + chef_index_queue:create_management_pool(Username, Password, HttpConfig); + _ -> + error_logger:info_msg("Rabbitmq monitoring is disabled. " + "chef_index will not check rabbitmq health.~n"), + ok + end; + _ -> + ok + end. + %% Return a spec for a bunnyc gen_server or the chef_index_batch gen_server based on the %% search_queue_mode configuration. %% diff --git a/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_rabbitmq_management.erl b/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_rabbitmq_management.erl index e971bcaf12..8998a75eca 100644 --- a/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_rabbitmq_management.erl +++ b/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_rabbitmq_management.erl @@ -249,6 +249,6 @@ check_aliveness(PoolNameAtom, Vhost) -> {ok, "200", _, _} -> true; Resp -> - lager:error("Error getting Rabbitmq aliveness: ~p", Resp), + lager:error("Error getting Rabbitmq aliveness: ~p", [Resp]), false end. From eb01196aed030668d4279517836481ae5b55aa6c Mon Sep 17 00:00:00 2001 From: Jay Mundrawala Date: Tue, 1 Aug 2017 09:22:30 -0500 Subject: [PATCH 05/10] Add aliveness tests Signed-off-by: Jay Mundrawala --- ...chef_wm_actions_queue_monitoring_tests.erl | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_actions_queue_monitoring_tests.erl b/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_actions_queue_monitoring_tests.erl index 7204c1f406..dbce899e03 100644 --- a/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_actions_queue_monitoring_tests.erl +++ b/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_actions_queue_monitoring_tests.erl @@ -26,6 +26,7 @@ -define(QUEUE_LENGTH_REQ, "/queues/%2Fanalytics"). -define(MAX_LENGTH_REQ, "/policies/%2Fanalytics/max_length"). +-define(PING_REQ, "/aliveness-test/%2Fanalytics"). -define(EMPTY_STATUS, [{queue_at_capacity,false}, {dropped_since_last_check,0}, @@ -628,3 +629,37 @@ meck_conn_failure() -> {error,{conn_failed,undefined}} end). +aliveness_test_() -> + {foreach, + fun() -> + meck:new(oc_httpc) + end, + fun(_) -> + catch(meck:unload(oc_httpc)) + end, + [ + {"it's alive", + fun() -> + meck:expect(oc_httpc, request, + fun(_, ?PING_REQ, _, _, _) -> + dummy_response("200", <<"{\"status\": \"ok\"}">>) end), + Status = chef_wm_rabbitmq_management:check_aliveness(pool_name, ?VHOST), + ?assertMatch(true, Status) + end}, + {"something's wrong, but got response", + fun() -> + meck:expect(oc_httpc, request, + fun(_, ?PING_REQ, _, _, _) -> + dummy_response("500", <<"{\"status\": \"fail\"}">>) end), + Status = chef_wm_rabbitmq_management:check_aliveness(pool_name, ?VHOST), + ?assertMatch(false, Status) + end}, + {"could not connect", + fun() -> + meck:expect(oc_httpc, request, + fun(_, ?PING_REQ, _, _, _) -> + {error,{conn_failed,undefined}} end), + Status = chef_wm_rabbitmq_management:check_aliveness(pool_name, ?VHOST), + ?assertMatch(false, Status) + end} + ]}. From 2c280f01df0be0cfa756d76293c2427d2f69b6c0 Mon Sep 17 00:00:00 2001 From: Jay Mundrawala Date: Tue, 1 Aug 2017 12:31:29 -0700 Subject: [PATCH 06/10] Add oc_chef_action:ping() tests Signed-off-by: Jay Mundrawala --- .../oc_chef_wm/test/oc_chef_action_tests.erl | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/oc_erchef/apps/oc_chef_wm/test/oc_chef_action_tests.erl b/src/oc_erchef/apps/oc_chef_wm/test/oc_chef_action_tests.erl index 7970ee5b64..4aaaa19470 100755 --- a/src/oc_erchef/apps/oc_chef_wm/test/oc_chef_action_tests.erl +++ b/src/oc_erchef/apps/oc_chef_wm/test/oc_chef_action_tests.erl @@ -20,6 +20,8 @@ -include("oc_chef_wm.hrl"). -include_lib("eunit/include/eunit.hrl"). +-define(VHOST, <<"/actions">>). + msg(Task) -> {[{<<"message_type">>, <<"action">>}, {<<"message_version">>, <<"0.1.1">>}, @@ -462,6 +464,38 @@ end_to_end_test_() -> ] }. +ping_test_() -> + {foreach, + fun() -> + application:set_env(oc_chef_wm, actions_vhost, <<"/actions">>), + meck:new(chef_wm_rabbitmq_management) + end, + fun(_) -> + meck:unload(chef_wm_rabbitmq_management) + end, + [{"Aliveness returns true causes pong", + fun() -> + meck:expect(chef_wm_rabbitmq_management, check_aliveness, + fun(_, "/actions") -> + true + end), + Status = oc_chef_action:ping(), + ?assertEqual(pong, Status) + end + }, + {"Not Aliveness causes pang", + fun() -> + meck:expect(chef_wm_rabbitmq_management, check_aliveness, + fun(_, "/actions") -> + false + end), + Status = oc_chef_action:ping(), + ?assertEqual(pang, Status) + end + }] + }. + + %% %% Internal helper functions %% From a6867702807a1c90c2275891411eeb65b26e579f Mon Sep 17 00:00:00 2001 From: Jay Mundrawala Date: Tue, 1 Aug 2017 13:02:00 -0700 Subject: [PATCH 07/10] Add ping test for chef_index Signed-off-by: Jay Mundrawala --- .../apps/chef_index/test/chef_index_tests.erl | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/src/oc_erchef/apps/chef_index/test/chef_index_tests.erl b/src/oc_erchef/apps/chef_index/test/chef_index_tests.erl index 8b8bb29c54..42512c3c28 100644 --- a/src/oc_erchef/apps/chef_index/test/chef_index_tests.erl +++ b/src/oc_erchef/apps/chef_index/test/chef_index_tests.erl @@ -108,3 +108,63 @@ delete_assertion() -> meck:expect(chef_index_expand, send_delete, fun(?EXPECTED_DELETE_DOC) -> ok end), chef_index:delete(role, <<"a1">>, <<"db1">>, <<"abcd">>), ?assert(meck:validate(chef_index_expand)). + +ping_test_() -> + {foreach, + fun() -> + application:set_env(chef_index, rabbitmq_vhost, <<"/testvhost">>), + application:set_env(chef_index, search_queue_mode, rabbitmq), + application:set_env(chef_index, rabbitmq_index_management_service, [{enabled, true}]), + meck:new(chef_wm_rabbitmq_management) + end, + fun(_) -> + meck:unload(chef_wm_rabbitmq_management) + end, + [{"When rabbitmq is not used, the check returns pong", + % This is not a requirement, there's just nothing else that + % is currently checked. + fun() -> + application:set_env(chef_index, search_queue_mode, batch), + meck:expect(chef_wm_rabbitmq_management, check_aliveness, + fun(_, "/testvhost") -> + throw(shouldnt_be_called) + end), + Status = chef_index:ping(), + ?assertEqual(pong, Status) + end + }, + {"When rabbitmq is enabled but management is not, the check returns pong", + % This is not a requirement, there's just nothing else that + % is currently checked. + fun() -> + application:set_env(chef_index, rabbitmq_index_management_service, [{enabled, false}]), + meck:expect(chef_wm_rabbitmq_management, check_aliveness, + fun(_, "/testvhost") -> + throw(shouldnt_be_called) + end), + Status = chef_index:ping(), + ?assertEqual(pong, Status) + end + }, + {"When rabbitmq and management are enabled, and rabbit is alive, returns pong", + fun() -> + meck:expect(chef_wm_rabbitmq_management, check_aliveness, + fun(_, "/testvhost") -> + true + end), + Status = chef_index:ping(), + ?assertEqual(pong, Status) + end + }, + {"When rabbitmq and management are enabled, and rabbit is not alive, returns pang", + fun() -> + meck:expect(chef_wm_rabbitmq_management, check_aliveness, + fun(_, "/testvhost") -> + false + end), + Status = chef_index:ping(), + ?assertEqual(pang, Status) + end + } + ]}. + From e4efcdfa3ff97d5c64e92b70e9ef07f98eea01ad Mon Sep 17 00:00:00 2001 From: Jay Mundrawala Date: Wed, 2 Aug 2017 07:47:46 -0700 Subject: [PATCH 08/10] Save pool name as part of process state Saves us on having to do a look up every call Signed-off-by: Jay Mundrawala --- .../src/chef_wm_actions_queue_monitoring.erl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_actions_queue_monitoring.erl b/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_actions_queue_monitoring.erl index d4c4bcfbd5..159ec422f5 100644 --- a/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_actions_queue_monitoring.erl +++ b/src/oc_erchef/apps/oc_chef_wm/src/chef_wm_actions_queue_monitoring.erl @@ -134,7 +134,10 @@ sync_response_process = undefined, % total # of checks, even if unsuccessful - check_count = 0 + check_count = 0, + + % name of the http pool to use + pool_name = undefined }). @@ -198,9 +201,11 @@ init([Vhost, Queue, MaxLength, CurrentLength]) -> % used to catch worker msgs process_flag(trap_exit, true), TRef = start_update_timer(), + PoolNameAtom = oc_chef_action_queue_config:get_rabbit_management_pool_name(), {ok, #queue_monitor_state{timer=TRef, queue_at_capacity = MaxLength == CurrentLength andalso MaxLength > 0, worker_process = undefined, + pool_name = PoolNameAtom, vhost_to_monitor = Vhost, queue_to_monitor = Queue, max_length = MaxLength, @@ -300,6 +305,7 @@ handle_info({MaxLength, N, AtCap}, State) -> handle_info(status_ping, #queue_monitor_state{ worker_process = undefined, dropped_since_last_check = Dropped, + pool_name = PoolNameAtom, vhost_to_monitor = Vhost, queue_to_monitor = Queue} = State) -> ParentPid = self(), @@ -307,7 +313,7 @@ handle_info(status_ping, #queue_monitor_state{ fun () -> Result = chef_wm_rabbitmq_management:check_current_queue_state( - oc_chef_action_queue_config:get_rabbit_management_pool_name(), + PoolNameAtom, Vhost, Queue, Dropped), From e28760167e8fa8a255cff6f1a348957f0313eab6 Mon Sep 17 00:00:00 2001 From: Jay Mundrawala Date: Wed, 2 Aug 2017 07:52:48 -0700 Subject: [PATCH 09/10] Rename rabbitmq_management_service to rabbitmq_actions_management_service We have a management service for the internal queue as well now, so this will specifically describe it without additional context needed Signed-off-by: Jay Mundrawala --- .../private-chef/templates/default/oc_erchef.config.erb | 2 +- .../itest/chef_wm_actions_queue_monitoring_SUITE.erl | 2 +- .../apps/oc_chef_wm/src/oc_chef_action_queue_config.erl | 4 ++-- .../test/chef_wm_actions_queue_monitoring_statem.erl | 2 +- .../test/chef_wm_actions_queue_monitoring_tests.erl | 2 +- src/oc_erchef/apps/oc_chef_wm/test/chef_wm_status_tests.erl | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/omnibus/files/private-chef-cookbooks/private-chef/templates/default/oc_erchef.config.erb b/omnibus/files/private-chef-cookbooks/private-chef/templates/default/oc_erchef.config.erb index 7a49bd1294..d36fd1563a 100755 --- a/omnibus/files/private-chef-cookbooks/private-chef/templates/default/oc_erchef.config.erb +++ b/omnibus/files/private-chef-cookbooks/private-chef/templates/default/oc_erchef.config.erb @@ -143,7 +143,7 @@ {user, "<%= @node['private_chef']['rabbitmq']['management_user'] %>"}, {port, <%= @node['private_chef']['rabbitmq']['management_port'] %>}, % rabbitmq management http connection pool - {rabbitmq_management_service, [ + {rabbitmq_actions_management_service, [ <% if node['private_chef']['fips_enabled'] -%> %% See note about Bookshelf {root_url, "http://<%= @actions_vip %>:<%= @node['private_chef']['rabbitmq']['management_port'] %>/api"}, diff --git a/src/oc_erchef/apps/oc_chef_wm/itest/chef_wm_actions_queue_monitoring_SUITE.erl b/src/oc_erchef/apps/oc_chef_wm/itest/chef_wm_actions_queue_monitoring_SUITE.erl index 789d22f619..b09234f60e 100644 --- a/src/oc_erchef/apps/oc_chef_wm/itest/chef_wm_actions_queue_monitoring_SUITE.erl +++ b/src/oc_erchef/apps/oc_chef_wm/itest/chef_wm_actions_queue_monitoring_SUITE.erl @@ -243,7 +243,7 @@ default_rabbit_config(Config) -> {port, 15672}, {password, BinPassword}, % rabbitmq management http connection pool - {rabbitmq_management_service, + {rabbitmq_actions_management_service, [{root_url, "http://127.0.0.1:15672/api"}, {timeout, 30000}, {init_count, 25}, diff --git a/src/oc_erchef/apps/oc_chef_wm/src/oc_chef_action_queue_config.erl b/src/oc_erchef/apps/oc_chef_wm/src/oc_chef_action_queue_config.erl index 380683c751..41e41d04ba 100644 --- a/src/oc_erchef/apps/oc_chef_wm/src/oc_chef_action_queue_config.erl +++ b/src/oc_erchef/apps/oc_chef_wm/src/oc_chef_action_queue_config.erl @@ -27,10 +27,10 @@ ]). -define(SERVER, ?MODULE). --define(POOLNAME, rabbitmq_management_service). +-define(POOLNAME, rabbitmq_actions_management_service). get_rabbit_management_pool_setting() -> - ServiceConfig = get_rabbit_management_setting(rabbitmq_management_service, []), + ServiceConfig = get_rabbit_management_setting(rabbitmq_actions_management_service, []), {?POOLNAME, ServiceConfig}. get_rabbit_management_pool_name() -> diff --git a/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_actions_queue_monitoring_statem.erl b/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_actions_queue_monitoring_statem.erl index c9b694b4be..31a7c0bb52 100644 --- a/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_actions_queue_monitoring_statem.erl +++ b/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_actions_queue_monitoring_statem.erl @@ -48,7 +48,7 @@ default_rabbit_config() -> {port, 15672}, {password, <<"chef123">>}, % rabbitmq management http connection pool - {rabbitmq_management_service, + {rabbitmq_actions_management_service, [{root_url, "http://127.0.0.1:15672/api"}, {timeout, 30000}, {init_count, 25}, diff --git a/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_actions_queue_monitoring_tests.erl b/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_actions_queue_monitoring_tests.erl index dbce899e03..0bc2eef483 100644 --- a/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_actions_queue_monitoring_tests.erl +++ b/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_actions_queue_monitoring_tests.erl @@ -148,7 +148,7 @@ default_config() -> {port, 15672}, {password, <<"chef123">>}, % rabbitmq management http connection pool - {rabbitmq_management_service, + {rabbitmq_actions_management_service, [{root_url, "http://127.0.0.1:15672/api"}, {timeout, 30000}, {init_count, 25}, diff --git a/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_status_tests.erl b/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_status_tests.erl index 95cb1dcdd3..3da999ae2f 100644 --- a/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_status_tests.erl +++ b/src/oc_erchef/apps/oc_chef_wm/test/chef_wm_status_tests.erl @@ -142,7 +142,7 @@ default_config() -> {port, 15672}, {password, <<"chef123">>}, % rabbitmq management http connection pool - {rabbitmq_management_service, + {rabbitmq_actions_management_service, [{root_url, "http://127.0.0.1:15672/api"}, {timeout, 30000}, {init_count, 25}, From 96b687d67be5bfa9d9106f10b446943f8de41d20 Mon Sep 17 00:00:00 2001 From: Jay Mundrawala Date: Wed, 2 Aug 2017 08:25:05 -0700 Subject: [PATCH 10/10] Added release notes for RabbitMQ checks in the `_status` endpoint This feature adds checks for RabbiMQ vhosts. These are checked when the `_status` endpoint is called, and will use the RabbitMQ management plugin to check the health of the vhosts we use. Signed-off-by: Jay Mundrawala --- RELEASE_NOTES.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index bb2cf4fc5d..50e7abb956 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -50,6 +50,12 @@ For prior releases, see [PRIOR\_RELEASE\_NOTES.md](PRIOR_RELEASE_NOTES.md). The preflight check that was in place to catch these situations has been removed. +* [RabbitMQ health check in status endpoint](https://github.com/chef/chef-server/pull/1345): Chef Server's + `_status` endpoint now checks the health of the analytics and internal RabbitMQ vhosts. For these checks + to work, the RabbitMQ management plugin must be installed. If it is not, the checks are not done. If + Chef Server is configured not to use Actions, a check will not be performed against the Actions vhost. + If an indexing queue is not used, the `chef_index` RabbitMQ vhost will not be checked. + ## 12.15.8 (2017-06-20) * [Stricter validation of non-functional user record fields](https://github.com/chef/chef-server/pull/1294),