Skip to content

Commit

Permalink
Merge pull request brianfrankcooper#11 from Azure/users/ratella/excep…
Browse files Browse the repository at this point in the history 
…tion_logging

Cosmos operation exceptions logging into separate files
  • Loading branch information
@RaviTella
RaviTella authored Sep 16, 2022
2 parents e182773 + 436e65d commit e18b31d
Show file tree
Hide file tree
Showing 2 changed files with 118 additions and 20 deletions.
20 changes: 13 additions & 7 deletions azurecosmos/src/main/java/site/ycsb/db/AzureCosmosClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,12 @@ public class AzureCosmosClient extends DB {
private static final Marker PATCH_DIAGNOSTIC = MarkerFactory.getMarker("PATCH_DIAGNOSTIC");
private static final Marker DELETE_DIAGNOSTIC = MarkerFactory.getMarker("DELETE_DIAGNOSTIC");
private static final Marker QUERY_DIAGNOSTIC = MarkerFactory.getMarker("QUERY_DIAGNOSTIC");
private static final Marker CREATE_EXCEPTION = MarkerFactory.getMarker("CREATE_EXCEPTION");
private static final Marker READ_EXCEPTION = MarkerFactory.getMarker("READ_EXCEPTION");
private static final Marker PATCH_EXCEPTION = MarkerFactory.getMarker("PATCH_EXCEPTION");
private static final Marker DELETE_EXCEPTION = MarkerFactory.getMarker("DELETE_DIAGNOSTIC");
private static final Marker QUERY_EXCEPTION = MarkerFactory.getMarker("QUERY_EXCEPTION");


/**
* Count the number of times initialized to teardown on the last
Expand Down Expand Up @@ -383,7 +389,7 @@ public Status read(String table, String key, Set<String> fields, Map<String, Byt
if (!AzureCosmosClient.includeExceptionStackInLog) {
e = null;
}
LOGGER.error("Failed to read key {} in collection {} in database {} statusCode {}", key, table,
LOGGER.error(READ_EXCEPTION, "Failed to read key {} in collection {} in database {} statusCode {}", key, table,
AzureCosmosClient.databaseName, statusCode, e);
if (readFailureCounter != null) {
readFailureCounter.increment();
Expand Down Expand Up @@ -455,8 +461,8 @@ public Status scan(String table, String startkey, int recordcount, Set<String> f
if (!AzureCosmosClient.includeExceptionStackInLog) {
e = null;
}
LOGGER.error("Failed to query key {} from collection {} in database {} statusCode {}", startkey, table,
AzureCosmosClient.databaseName, statusCode, e);
LOGGER.error(QUERY_EXCEPTION, "Failed to query key {} from collection {} in database {} statusCode {}",
startkey, table, AzureCosmosClient.databaseName, statusCode, e);
}
if (scanFailureCounter != null) {
scanFailureCounter.increment();
Expand Down Expand Up @@ -508,7 +514,7 @@ public Status update(String table, String key, Map<String, ByteIterator> values)
if (!AzureCosmosClient.includeExceptionStackInLog) {
e = null;
}
LOGGER.error("Failed to update key {} to collection {} in database {} statusCode {}", key, table,
LOGGER.error(PATCH_EXCEPTION, "Failed to update key {} to collection {} in database {} statusCode {}", key, table,
AzureCosmosClient.databaseName, statusCode, e);
}

Expand Down Expand Up @@ -571,8 +577,8 @@ public Status insert(String table, String key, Map<String, ByteIterator> values)
if (!AzureCosmosClient.includeExceptionStackInLog) {
e = null;
}
LOGGER.error("Failed to insert key {} to collection {} in database {} statusCode {}", key, table,
AzureCosmosClient.databaseName, statusCode, e);
LOGGER.error(CREATE_EXCEPTION, "Failed to insert key {} to collection {} in database {} statusCode {}", key,
table, AzureCosmosClient.databaseName, statusCode, e);
}
if (writeFailureCounter != null) {
writeFailureCounter.increment();
Expand Down Expand Up @@ -605,7 +611,7 @@ public Status delete(String table, String key) {
if (!AzureCosmosClient.includeExceptionStackInLog) {
e = null;
}
LOGGER.error("Failed to delete key {} in collection {} database {} statusCode {}", key, table,
LOGGER.error(DELETE_EXCEPTION, "Failed to delete key {} in collection {} database {} statusCode {}", key, table,
AzureCosmosClient.databaseName, statusCode, e);
}
return Status.ERROR;
Expand Down
118 changes: 105 additions & 13 deletions azurecosmos/src/main/resources/log4j2.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,29 @@
<Configuration status="WARN" monitorInterval="30">
<Properties>
<Property name="LOG_PATTERN">%-4r [%t] %-5p %c %x -%m%n</Property>
<Property name="BASE_PATH_DIAGNOSTICS">/tmp/cosmos_client_logs/diagnostics</Property>
<Property name="BASE_PATH_EXCEPTION">/tmp/cosmos_client_logs/exceptions</Property>
</Properties>

<Appenders>
<Console name="console" target="SYSTEM_OUT" follow="true">
<PatternLayout pattern="${LOG_PATTERN}"/>
<Filters>
<MarkerFilter marker="QUERY_DIAGNOSTIC" onMatch="DENY" onMismatch="NEUTRAL" />
<MarkerFilter marker="PATCH_DIAGNOSTIC" onMatch="DENY" onMismatch="NEUTRAL" />
<MarkerFilter marker="CREATE_DIAGNOSTIC" onMatch="DENY" onMismatch="NEUTRAL" />
<MarkerFilter marker="READ_DIAGNOSTIC" onMatch="DENY" onMismatch="ACCEPT" />
<MarkerFilter marker="READ_DIAGNOSTIC" onMatch="DENY" onMismatch="NEUTRAL" />
<MarkerFilter marker="PATCH_DIAGNOSTIC" onMatch="DENY" onMismatch="NEUTRAL" />
<MarkerFilter marker="QUERY_DIAGNOSTIC" onMatch="DENY" onMismatch="NEUTRAL" />
<MarkerFilter marker="CREATE_EXCEPTION" onMatch="DENY" onMismatch="NEUTRAL" />
<MarkerFilter marker="READ_EXCEPTION" onMatch="DENY" onMismatch="NEUTRAL" />
<MarkerFilter marker="PATCH_EXCEPTION" onMatch="DENY" onMismatch="NEUTRAL" />
<MarkerFilter marker="QUERY_EXCEPTION" onMatch="DENY" onMismatch="NEUTRAL" />
<MarkerFilter marker="DELETE_EXCEPTION" onMatch="DENY" onMismatch="ACCEPT" />
</Filters>
</Console>
<RollingFile
name="writeRequestDiagnosticsRollingFileAppender"
fileName="/tmp/cosmos_client_diagnostic_logs/create/diagnostics.log"
filePattern="/tmp/cosmos_client_diagnostic_logs/create/diagnostics.%d{yyyy-MM-dd-hh-mm-ss}.log.gz"
name="createRequestDiagnosticsRollingFileAppender"
fileName="${BASE_PATH_DIAGNOSTICS}/create/diagnostics.log"
filePattern="${BASE_PATH_DIAGNOSTICS}/create/diagnostics.%d{yyyy-MM-dd-hh-mm-ss}.log.gz"
ignoreExceptions="false">
<PatternLayout pattern="${LOG_PATTERN}"/>
<MarkerFilter marker="CREATE_DIAGNOSTIC" onMatch="ACCEPT" onMismatch="DENY" />
Expand All @@ -29,8 +36,8 @@

<RollingFile
name="readRequestDiagnosticsRollingFileAppender"
fileName="/tmp/cosmos_client_diagnostic_logs/read/diagnostics.log"
filePattern="/tmp/cosmos_client_diagnostic_logs/read/diagnostics.%d{yyyy-MM-dd-hh-mm-ss}.log.gz"
fileName="${BASE_PATH_DIAGNOSTICS}/read/diagnostics.log"
filePattern="${BASE_PATH_DIAGNOSTICS}/read/diagnostics.%d{yyyy-MM-dd-hh-mm-ss}.log.gz"
ignoreExceptions="false">
<PatternLayout pattern="${LOG_PATTERN}"/>
<MarkerFilter marker="READ_DIAGNOSTIC" onMatch="ACCEPT" onMismatch="DENY" />
Expand All @@ -42,8 +49,8 @@

<RollingFile
name="patchRequestDiagnosticsRollingFileAppender"
fileName="/tmp/cosmos_client_diagnostic_logs/patch/diagnostics.log"
filePattern="/tmp/cosmos_client_diagnostic_logs/patch/diagnostics.%d{yyyy-MM-dd-hh-mm-ss}.log.gz"
fileName="${BASE_PATH_DIAGNOSTICS}/patch/diagnostics.log"
filePattern="${BASE_PATH_DIAGNOSTICS}/patch/diagnostics.%d{yyyy-MM-dd-hh-mm-ss}.log.gz"
ignoreExceptions="false">
<PatternLayout pattern="${LOG_PATTERN}"/>
<MarkerFilter marker="PATCH_DIAGNOSTIC" onMatch="ACCEPT" onMismatch="DENY" />
Expand All @@ -55,8 +62,8 @@

<RollingFile
name="queryRequestDiagnosticsRollingFileAppender"
fileName="/tmp/cosmos_client_diagnostic_logs/query/diagnostics.log"
filePattern="/tmp/cosmos_client_diagnostic_logs/query/diagnostics.%d{yyyy-MM-dd-hh-mm-ss}.log.gz"
fileName="${BASE_PATH_DIAGNOSTICS}/query/diagnostics.log"
filePattern="${BASE_PATH_DIAGNOSTICS}/query/diagnostics.%d{yyyy-MM-dd-hh-mm-ss}.log.gz"
ignoreExceptions="false">
<PatternLayout pattern="${LOG_PATTERN}"/>
<MarkerFilter marker="QUERY_DIAGNOSTIC" onMatch="ACCEPT" onMismatch="DENY" />
Expand All @@ -65,17 +72,102 @@
<SizeBasedTriggeringPolicy size="500MB" />
</Policies>
</RollingFile>

<RollingFile
name="deleteRequestDiagnosticsRollingFileAppender"
fileName="${BASE_PATH_DIAGNOSTICS}/delete/diagnostics.log"
filePattern="${BASE_PATH_DIAGNOSTICS}/delete/diagnostics.%d{yyyy-MM-dd-hh-mm-ss}.log.gz"
ignoreExceptions="false">
<PatternLayout pattern="${LOG_PATTERN}"/>
<MarkerFilter marker="QUERY_DIAGNOSTIC" onMatch="ACCEPT" onMismatch="DENY" />
<Policies>
<OnStartupTriggeringPolicy />
<SizeBasedTriggeringPolicy size="500MB" />
</Policies>
</RollingFile>

<RollingFile
name="createRequestExceptionRollingFileAppender"
fileName="${BASE_PATH_EXCEPTION}/create/exceptions.log"
filePattern="${BASE_PATH_EXCEPTION}/create/exceptions.%d{yyyy-MM-dd-hh-mm-ss}.log.gz"
ignoreExceptions="false">
<PatternLayout pattern="${LOG_PATTERN}"/>
<MarkerFilter marker="CREATE_EXCEPTION" onMatch="ACCEPT" onMismatch="DENY" />
<Policies>
<OnStartupTriggeringPolicy />
<SizeBasedTriggeringPolicy size="500MB" />
</Policies>
</RollingFile>

<RollingFile
name="readRequestExceptionRollingFileAppender"
fileName="${BASE_PATH_EXCEPTION}/read/exceptions.log"
filePattern="${BASE_PATH_EXCEPTION}/read/exceptions.%d{yyyy-MM-dd-hh-mm-ss}.log.gz"
ignoreExceptions="false">
<PatternLayout pattern="${LOG_PATTERN}"/>
<MarkerFilter marker="READ_EXCEPTION" onMatch="ACCEPT" onMismatch="DENY" />
<Policies>
<OnStartupTriggeringPolicy />
<SizeBasedTriggeringPolicy size="500MB" />
</Policies>
</RollingFile>

<RollingFile
name="patchRequestExceptionRollingFileAppender"
fileName="${BASE_PATH_EXCEPTION}/patch/exceptions.log"
filePattern="${BASE_PATH_EXCEPTION}/patch/exceptions.%d{yyyy-MM-dd-hh-mm-ss}.log.gz"
ignoreExceptions="false">
<PatternLayout pattern="${LOG_PATTERN}"/>
<MarkerFilter marker="PATCH_EXCEPTION" onMatch="ACCEPT" onMismatch="DENY" />
<Policies>
<OnStartupTriggeringPolicy />
<SizeBasedTriggeringPolicy size="500MB" />
</Policies>
</RollingFile>

<RollingFile
name="queryRequestExceptionRollingFileAppender"
fileName="${BASE_PATH_EXCEPTION}/query/exceptions.log"
filePattern="${BASE_PATH_EXCEPTION}/query/exceptions.%d{yyyy-MM-dd-hh-mm-ss}.log.gz"
ignoreExceptions="false">
<PatternLayout pattern="${LOG_PATTERN}"/>
<MarkerFilter marker="QUERY_EXCEPTION" onMatch="ACCEPT" onMismatch="DENY" />
<Policies>
<OnStartupTriggeringPolicy />
<SizeBasedTriggeringPolicy size="500MB" />
</Policies>
</RollingFile>

<RollingFile
name="deleteRequestExceptionRollingFileAppender"
fileName="${BASE_PATH_EXCEPTION}/delete/exceptions.log"
filePattern="${BASE_PATH_EXCEPTION}/delete/exceptions.%d{yyyy-MM-dd-hh-mm-ss}.log.gz"
ignoreExceptions="false">
<PatternLayout pattern="${LOG_PATTERN}"/>
<MarkerFilter marker="QUERY_EXCEPTION" onMatch="ACCEPT" onMismatch="DENY" />
<Policies>
<OnStartupTriggeringPolicy />
<SizeBasedTriggeringPolicy size="500MB" />
</Policies>
</RollingFile>

</Appenders>
<Loggers>
<Logger name="log4j.category.org.apache.http" level="INFO"/>
<Logger name="log4j.category.org.apache.http.wire" level="INFO"/>
<Logger name="log4j.category.org.apache.http.header" level="INFO"/>
<Root level="info">
<AppenderRef ref="console"/>
<AppenderRef ref="writeRequestDiagnosticsRollingFileAppender"/>
<AppenderRef ref="createRequestDiagnosticsRollingFileAppender"/>
<AppenderRef ref="readRequestDiagnosticsRollingFileAppender"/>
<AppenderRef ref="queryRequestDiagnosticsRollingFileAppender"/>
<AppenderRef ref="patchRequestDiagnosticsRollingFileAppender"/>
<AppenderRef ref="deleteRequestDiagnosticsRollingFileAppender"/>
<AppenderRef ref="createRequestExceptionRollingFileAppender"/>
<AppenderRef ref="readRequestExceptionRollingFileAppender"/>
<AppenderRef ref="patchRequestExceptionRollingFileAppender"/>
<AppenderRef ref="queryRequestExceptionRollingFileAppender"/>
<AppenderRef ref="deleteRequestExceptionRollingFileAppender"/>
</Root>
</Loggers>
</Configuration>

0 comments on commit e18b31d

Please sign in to comment.