diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index fa47c73d49c41..addcac2ad0118 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -965,8 +965,15 @@ private void createConfiguration() { defaultConfig.put("discovery.initial_state_timeout", "0s"); // TODO: Remove these once https://github.com/elastic/elasticsearch/issues/46091 is fixed - defaultConfig.put("logger.org.elasticsearch.action.support.master", "DEBUG"); + defaultConfig.put("logger.org.elasticsearch.action.support.master.TransportMasterNodeAction", "TRACE"); + defaultConfig.put("logger.org.elasticsearch.cluster.metadata.MetaDataCreateIndexService", "TRACE"); + defaultConfig.put("logger.org.elasticsearch.cluster.service", "DEBUG"); defaultConfig.put("logger.org.elasticsearch.cluster.coordination", "DEBUG"); + defaultConfig.put("logger.org.elasticsearch.gateway.MetaStateService", "TRACE"); + if (getVersion().getMajor() >= 8) { + defaultConfig.put("cluster.service.slow_task_logging_threshold", "5s"); + defaultConfig.put("cluster.service.slow_master_task_logging_threshold", "5s"); + } HashSet overriden = new HashSet<>(defaultConfig.keySet()); overriden.retainAll(settings.keySet()); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java index 846f920b7b183..3533036209c16 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java @@ -19,6 +19,8 @@ package org.elasticsearch.action.admin.cluster.allocation; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeAction; @@ -56,6 +58,8 @@ public class TransportClusterAllocationExplainAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportClusterAllocationExplainAction.class); + private final ClusterInfoService clusterInfoService; private final AllocationDeciders allocationDeciders; private final ShardsAllocator shardAllocator; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java index 13270075806c3..7dcbd863d29e1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java @@ -18,6 +18,8 @@ */ package org.elasticsearch.action.admin.cluster.configuration; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.action.ActionListener; @@ -53,6 +55,8 @@ public class TransportAddVotingConfigExclusionsAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportAddVotingConfigExclusionsAction.class); + public static final Setting MAXIMUM_VOTING_CONFIG_EXCLUSIONS_SETTING = Setting.intSetting("cluster.max_voting_config_exclusions", 10, 1, Property.Dynamic, Property.NodeScope); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java index 78709c3db09ff..0be05b480c050 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java @@ -18,6 +18,8 @@ */ package org.elasticsearch.action.admin.cluster.configuration; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.action.ActionListener; @@ -49,6 +51,8 @@ public class TransportClearVotingConfigExclusionsAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportClearVotingConfigExclusionsAction.class); + @Inject public TransportClearVotingConfigExclusionsAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java index 777b2f4ca9d60..70fd23acfda92 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java @@ -19,6 +19,8 @@ package org.elasticsearch.action.admin.cluster.settings; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; @@ -47,6 +49,8 @@ public class TransportClusterUpdateSettingsAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportClusterUpdateSettingsAction.class); + private final AllocationService allocationService; private final ClusterSettings clusterSettings; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java index 83bfc800037b5..4cfc1bafe33b2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java @@ -21,6 +21,8 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.support.ActionFilters; @@ -62,6 +64,8 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportSnapshotsStatusAction.class); + private final SnapshotsService snapshotsService; private final NodeClient client; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java index 4eada5d0bc6ef..75e6447761561 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java @@ -19,6 +19,8 @@ package org.elasticsearch.action.admin.cluster.tasks; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; @@ -39,6 +41,8 @@ public class TransportPendingClusterTasksAction extends TransportMasterNodeReadAction { + private static final Logger logger = LogManager.getLogger(TransportPendingClusterTasksAction.class); + private final ClusterService clusterService; @Inject diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java index 978a5bc075a5b..d0ed81f8b7ac6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java @@ -20,6 +20,8 @@ package org.elasticsearch.action.admin.indices.alias; import com.carrotsearch.hppc.cursors.ObjectCursor; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.RequestValidators; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions; @@ -61,6 +63,8 @@ */ public class TransportIndicesAliasesAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportIndicesAliasesAction.class); + private final MetaDataIndexAliasesService indexAliasesService; private final RequestValidators requestValidators; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java index e28dddbbb0176..d6b17e70ad258 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java @@ -19,6 +19,8 @@ package org.elasticsearch.action.admin.indices.close; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; @@ -49,6 +51,8 @@ */ public class TransportCloseIndexAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportCloseIndexAction.class); + private final MetaDataIndexStateService indexStateService; private final DestructiveOperations destructiveOperations; private volatile boolean closeIndexEnabled; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java index f9449bae8bef3..af6b1e632df85 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java @@ -147,4 +147,19 @@ public boolean copySettings() { return copySettings; } + @Override + public String toString() { + return "CreateIndexClusterStateUpdateRequest{" + + "cause='" + cause + '\'' + + ", index='" + index + '\'' + + ", providedName='" + providedName + '\'' + + ", recoverFrom=" + recoverFrom + + ", resizeType=" + resizeType + + ", copySettings=" + copySettings + + ", settings=" + settings + + ", aliases=" + aliases + + ", blocks=" + blocks + + ", waitForActiveShards=" + waitForActiveShards + + '}'; + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java index 19341b63f172d..87a5362e04c84 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java @@ -19,6 +19,8 @@ package org.elasticsearch.action.admin.indices.delete; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; @@ -48,6 +50,8 @@ */ public class TransportDeleteIndexAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportDeleteIndexAction.class); + private final MetaDataDeleteIndexService deleteIndexService; private final DestructiveOperations destructiveOperations; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java index 0dda5fc8cfe16..6a53762461665 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java @@ -19,6 +19,8 @@ package org.elasticsearch.action.admin.indices.mapping.get; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.info.TransportClusterInfoAction; @@ -39,6 +41,8 @@ public class TransportGetMappingsAction extends TransportClusterInfoAction { + private static final Logger logger = LogManager.getLogger(TransportGetMappingsAction.class); + private final IndicesService indicesService; @Inject diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java index 41d807a0cf48c..4a9d833a29c54 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java @@ -19,6 +19,8 @@ package org.elasticsearch.action.admin.indices.mapping.put; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.RequestValidators; @@ -49,6 +51,8 @@ */ public class TransportPutMappingAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportPutMappingAction.class); + private final MetaDataMappingService metaDataMappingService; private final RequestValidators requestValidators; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java index 53c015596049c..b3eb1bcb64121 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java @@ -19,6 +19,8 @@ package org.elasticsearch.action.admin.indices.open; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; @@ -45,6 +47,8 @@ */ public class TransportOpenIndexAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportOpenIndexAction.class); + private final MetaDataIndexStateService indexStateService; private final DestructiveOperations destructiveOperations; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java index 7f69e8bc05bd6..f58f9ab943e0c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java @@ -19,6 +19,8 @@ package org.elasticsearch.action.admin.indices.settings.put; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; @@ -43,6 +45,8 @@ public class TransportUpdateSettingsAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportUpdateSettingsAction.class); + private final MetaDataUpdateSettingsService updateSettingsService; @Inject diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java index 5c23753d53bfc..6efb9c8e89bed 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.action.admin.indices.shards; +import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.CollectionUtil; import org.elasticsearch.action.ActionListener; @@ -70,6 +71,8 @@ public class TransportIndicesShardStoresAction extends TransportMasterNodeReadAction { + private static final Logger logger = LogManager.getLogger(TransportIndicesShardStoresAction.class); + private final NodeClient client; @Inject diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java index c6b2520678057..707d4ad71ebfa 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java @@ -18,6 +18,8 @@ */ package org.elasticsearch.action.admin.indices.template.delete; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; @@ -43,6 +45,8 @@ public class TransportDeleteIndexTemplateAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportDeleteIndexTemplateAction.class); + private final MetaDataIndexTemplateService indexTemplateService; @Inject diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java index ad18a4b8bc725..753b517a0cd9e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java @@ -18,6 +18,8 @@ */ package org.elasticsearch.action.admin.indices.template.put; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; @@ -45,6 +47,8 @@ */ public class TransportPutIndexTemplateAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportPutIndexTemplateAction.class); + private final MetaDataIndexTemplateService indexTemplateService; private final IndexScopedSettings indexScopedSettings; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java index a3283a1e33868..8a72d8ee8054e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java @@ -19,6 +19,8 @@ package org.elasticsearch.action.admin.indices.upgrade.post; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; @@ -41,6 +43,8 @@ public class TransportUpgradeSettingsAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportUpgradeSettingsAction.class); + private final MetaDataUpdateSettingsService updateSettingsService; @Inject diff --git a/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java b/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java index f325e09a9fbff..d4ac88c8b723c 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java @@ -58,7 +58,7 @@ public abstract class TransportMasterNodeAction, Response extends ActionResponse> extends HandledTransportAction { - protected static final Logger logger = LogManager.getLogger(TransportMasterNodeAction.class); + private static final Logger logger = LogManager.getLogger(TransportMasterNodeAction.class); protected final ThreadPool threadPool; protected final TransportService transportService; @@ -121,6 +121,7 @@ class AsyncSingleAction { public void start() { ClusterState state = clusterService.state(); + logger.trace("starting processing request [{}] with cluster state version [{}]", request, state.version()); this.observer = new ClusterStateObserver(state, clusterService, request.masterNodeTimeout(), logger, threadPool.getThreadContext()); doStart(state); @@ -135,16 +136,17 @@ protected void doStart(ClusterState clusterState) { final ClusterBlockException blockException = checkBlock(request, clusterState); if (blockException != null) { if (!blockException.retryable()) { + logger.trace("can't execute due to a non-retryable cluster block", blockException); listener.onFailure(blockException); } else { - logger.trace("can't execute due to a cluster block, retrying", blockException); + logger.debug("can't execute due to a cluster block, retrying", blockException); retry(blockException, newState -> { try { ClusterBlockException newException = checkBlock(request, newState); return (newException == null || !newException.retryable()); } catch (Exception e) { // accept state as block will be rechecked by doStart() and listener.onFailure() then called - logger.trace("exception occurred during cluster block checking, accepting state", e); + logger.debug("exception occurred during cluster block checking, accepting state", e); return true; } }); @@ -156,6 +158,7 @@ protected void doStart(ClusterState clusterState) { "stepped down before publishing action [{}], scheduling a retry", actionName), t); retry(t, masterChangePredicate); } else { + logger.debug("unexpected exception during publication", t); delegatedListener.onFailure(t); } }); @@ -169,6 +172,7 @@ protected void doStart(ClusterState clusterState) { } else { DiscoveryNode masterNode = nodes.getMasterNode(); final String actionName = getMasterActionName(masterNode); + logger.trace("forwarding request [{}] to master [{}]", actionName, masterNode); transportService.sendRequest(masterNode, actionName, request, new ActionListenerResponseHandler(listener, TransportMasterNodeAction.this::read) { @Override @@ -181,6 +185,8 @@ public void handleException(final TransportException exp) { actionName, nodes.getMasterNode(), exp.getDetailedMessage()); retry(cause, masterChangePredicate); } else { + logger.trace(new ParameterizedMessage("failure when forwarding request [{}] to master [{}]", + actionName, masterNode), exp); listener.onFailure(exp); } } @@ -188,6 +194,7 @@ public void handleException(final TransportException exp) { } } } catch (Exception e) { + logger.trace("top-level failure", e); listener.onFailure(e); } } @@ -197,6 +204,7 @@ private void retry(final Throwable failure, final Predicate stateP new ClusterStateObserver.Listener() { @Override public void onNewClusterState(ClusterState state) { + logger.trace("retrying with cluster state version [{}]", state.version()); doStart(state); } @@ -211,8 +219,7 @@ public void onTimeout(TimeValue timeout) { actionName, timeout), failure); listener.onFailure(new MasterNotDiscoveredException(failure)); } - }, statePredicate - ); + }, statePredicate); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 31806803931b8..9bfbec9f93df7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -202,17 +202,23 @@ public static void validateIndexOrAliasName(String index, BiFunction listener) { + logger.trace("createIndex[{}]", request); onlyCreateIndex(request, ActionListener.wrap(response -> { if (response.isAcknowledged()) { + logger.trace("[{}] index creation acknowledged, waiting for active shards [{}]", + request.index(), request.waitForActiveShards()); activeShardsObserver.waitForActiveShards(new String[]{request.index()}, request.waitForActiveShards(), request.ackTimeout(), shardsAcknowledged -> { if (shardsAcknowledged == false) { logger.debug("[{}] index created, but the operation timed out while waiting for " + "enough shards to be started.", request.index()); + } else { + logger.trace("[{}] index created and shards acknowledged", request.index()); } listener.onResponse(new CreateIndexClusterStateUpdateResponse(response.isAcknowledged(), shardsAcknowledged)); }, listener::onFailure); } else { + logger.trace("index creation not acknowledged for [{}]", request); listener.onResponse(new CreateIndexClusterStateUpdateResponse(false, false)); } }, listener::onFailure)); @@ -278,6 +284,7 @@ protected ClusterStateUpdateResponse newResponse(boolean acknowledged) { @Override public ClusterState execute(ClusterState currentState) throws Exception { + logger.trace("executing IndexCreationTask for [{}] against cluster state version [{}]", request, currentState.version()); Index createdIndex = null; String removalExtraInfo = null; IndexRemovalReason removalReason = IndexRemovalReason.FAILURE; diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java index b2d85f7dd7511..fb49136bb2341 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java @@ -6,6 +6,8 @@ package org.elasticsearch.xpack.ccr.action; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchSecurityException; @@ -49,6 +51,8 @@ public class TransportUnfollowAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportUnfollowAction.class); + private final Client client; @Inject diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrameUsageTransportAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrameUsageTransportAction.java index 7f0a79581907a..5280b8801de17 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrameUsageTransportAction.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrameUsageTransportAction.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.dataframe; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.SearchRequest; @@ -45,6 +47,8 @@ public class DataFrameUsageTransportAction extends XPackUsageFeatureTransportAction { + private static final Logger logger = LogManager.getLogger(DataFrameUsageTransportAction.class); + private final boolean enabled; private final XPackLicenseState licenseState; private final Client client; diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/action/TransportFreezeIndexAction.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/action/TransportFreezeIndexAction.java index a787a7945fc83..b412a47ea25ab 100644 --- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/action/TransportFreezeIndexAction.java +++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/action/TransportFreezeIndexAction.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.frozen.action; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; @@ -48,6 +50,8 @@ public final class TransportFreezeIndexAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportFreezeIndexAction.class); + private final DestructiveOperations destructiveOperations; private final MetaDataIndexStateService indexStateService; private final TransportCloseIndexAction transportCloseIndexAction; diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java index 9e344cefee22e..e5139c1b02f2a 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java @@ -6,6 +6,8 @@ package org.elasticsearch.xpack.ilm.action; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeAction; @@ -42,6 +44,8 @@ */ public class TransportPutLifecycleAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportPutLifecycleAction.class); + @Inject public TransportPutLifecycleAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRetryAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRetryAction.java index e308bea9f1d4d..1df695c47e5d7 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRetryAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRetryAction.java @@ -6,6 +6,8 @@ package org.elasticsearch.xpack.ilm.action; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeAction; @@ -32,6 +34,8 @@ public class TransportRetryAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportRetryAction.class); + IndexLifecycleService indexLifecycleService; @Inject diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java index 45b4e1805174a..70e0387d00f14 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java @@ -6,6 +6,8 @@ package org.elasticsearch.xpack.ml.action; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; @@ -90,6 +92,8 @@ public class TransportDeleteJobAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportDeleteJobAction.class); + private static final int MAX_SNAPSHOTS_TO_DELETE = 10000; private final Client client; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java index e31ff5c26fd59..bba6d3db8a0fe 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.ml.action; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.WriteRequest; @@ -40,6 +42,8 @@ public class TransportFinalizeJobExecutionAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportFinalizeJobExecutionAction.class); + private final Client client; @Inject diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java index c3a0683294437..8332d3cbfc6c3 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.ml.action; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; @@ -37,6 +39,8 @@ public class TransportGetDatafeedsAction extends TransportMasterNodeReadAction { + private static final Logger logger = LogManager.getLogger(TransportGetDatafeedsAction.class); + private final DatafeedConfigProvider datafeedConfigProvider; @Inject diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java index f50ce26132b9f..f2da0f65a287b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.ml.action; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; @@ -36,6 +38,8 @@ public class TransportGetDatafeedsStatsAction extends TransportMasterNodeReadAction { + private static final Logger logger = LogManager.getLogger(TransportGetDatafeedsStatsAction.class); + private final DatafeedConfigProvider datafeedConfigProvider; private final JobResultsProvider jobResultsProvider; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java index 98b7ece2dcb60..c6617d6d7dd73 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.ml.action; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; @@ -25,6 +27,8 @@ public class TransportGetJobsAction extends TransportMasterNodeReadAction { + private static final Logger logger = LogManager.getLogger(TransportGetJobsAction.class); + private final JobManager jobManager; @Inject diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java index b2f94020fbb9b..e7a1ece44a825 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java @@ -81,6 +81,8 @@ To ensure that a subsequent close job call will see that same task status (and s */ public class TransportOpenJobAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportOpenJobAction.class); + static final PersistentTasksCustomMetaData.Assignment AWAITING_MIGRATION = new PersistentTasksCustomMetaData.Assignment(null, "job cannot be assigned until it has been migrated."); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java index 1a5ba7629ca8e..7b394b580044f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.ml.action; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; @@ -42,6 +44,8 @@ public class TransportRevertModelSnapshotAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportRevertModelSnapshotAction.class); + private final Client client; private final JobManager jobManager; private final JobResultsProvider jobResultsProvider; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java index a65f7cc615df0..1f44d22c6681f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.ml.action; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ResourceAlreadyExistsException; @@ -79,6 +81,8 @@ To ensure that a subsequent stop datafeed call will see that same task status (a */ public class TransportStartDatafeedAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportStartDatafeedAction.class); + private final Client client; private final XPackLicenseState licenseState; private final PersistentTasksService persistentTasksService; diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java index 99111d43edde6..b18b917113802 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java @@ -62,6 +62,9 @@ import java.util.stream.Collectors; public class TransportPutRollupJobAction extends TransportMasterNodeAction { + + private static final Logger logger = LogManager.getLogger(TransportPutRollupJobAction.class); + private final XPackLicenseState licenseState; private final PersistentTasksService persistentTasksService; private final Client client; diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/service/TransportWatcherServiceAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/service/TransportWatcherServiceAction.java index 675d04dc3d0f1..92200bcc4f37c 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/service/TransportWatcherServiceAction.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/service/TransportWatcherServiceAction.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.watcher.transport.actions.service; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; @@ -34,6 +36,8 @@ public class TransportWatcherServiceAction extends TransportMasterNodeAction { + private static final Logger logger = LogManager.getLogger(TransportWatcherServiceAction.class); + private AckedRequest ackedRequest = new AckedRequest() { @Override public TimeValue ackTimeout() {