Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(exceptions): Don't let errors in trackResult leak out #3843

Merged
merged 2 commits into from
Aug 6, 2020
Merged

fix(exceptions): Don't let errors in trackResult leak out #3843

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c67cfb8
fix(exceptions): Don't let errors in `trackResult` leak out
marchello2000 Aug 6, 2020
bacf624
Merge branch 'master' into mark/radar_trackresult
mergify[bot] Aug 6, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions orca-core/src/main/java/com/netflix/spinnaker/orca/clouddriver/utils/CloudProviderAware.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,12 @@ default String getCredentials(Map<String, Object> context) {

// may return a list with 0, 1 or more regions (no guarantees on the ordering)
default List<String> getRegions(Map<String, Object> context) {
String region = (String) context.getOrDefault("region", null);
Object region = context.getOrDefault("region", null);
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this sometimes is a string and sometimes a list. I tried to make it handle both. but also wrapped in try/catch in the other files

if (region != null) {
return ImmutableList.of(region);
if (region instanceof List) {
return ImmutableList.copyOf((List<String>) region);
}
return ImmutableList.of(region.toString());
}

try {
Expand Down
24 changes: 14 additions & 10 deletions orca-queue/src/main/kotlin/com/netflix/spinnaker/orca/q/handler/CompleteTaskHandler.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import com.netflix.spinnaker.orca.q.StartTask
import com.netflix.spinnaker.orca.q.get
import com.netflix.spinnaker.orca.q.metrics.MetricsTagHelper
import com.netflix.spinnaker.q.Queue
import java.lang.Exception
import java.time.Clock
import java.util.concurrent.TimeUnit
import org.springframework.beans.factory.annotation.Qualifier
Expand Down Expand Up @@ -118,18 +119,21 @@ class CompleteTaskHandler(
}

private fun trackResult(stage: StageExecution, taskModel: TaskExecution, status: ExecutionStatus) {
val commonTags = MetricsTagHelper.commonTags(stage, taskModel, status)
val detailedTags = MetricsTagHelper.detailedTaskTags(stage, taskModel, status)
try {
val commonTags = MetricsTagHelper.commonTags(stage, taskModel, status)
val detailedTags = MetricsTagHelper.detailedTaskTags(stage, taskModel, status)

// we are looking at the time it took to complete the whole execution, not just one invocation
val elapsedMillis = clock.millis() - (taskModel.startTime ?: 0)
// we are looking at the time it took to complete the whole execution, not just one invocation
val elapsedMillis = clock.millis() - (taskModel.startTime ?: 0)

hashMapOf(
"task.completions.duration" to commonTags + BasicTag("application", stage.execution.application),
"task.completions.duration.withType" to commonTags + detailedTags
).forEach {
name, tags ->
registry.timer(name, tags).record(elapsedMillis, TimeUnit.MILLISECONDS)
hashMapOf(
"task.completions.duration" to commonTags + BasicTag("application", stage.execution.application),
"task.completions.duration.withType" to commonTags + detailedTags
).forEach { name, tags ->
registry.timer(name, tags).record(elapsedMillis, TimeUnit.MILLISECONDS)
}
} catch (e: Exception) {
log.warn("Failed to track result for stage: ${stage.id}, task: ${taskModel.id}", e)
}
}
}
24 changes: 14 additions & 10 deletions orca-queue/src/main/kotlin/com/netflix/spinnaker/orca/q/handler/RunTaskHandler.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package com.netflix.spinnaker.orca.q.handler
import com.netflix.spectator.api.BasicTag
import com.netflix.spectator.api.Registry
import com.netflix.spinnaker.kork.dynamicconfig.DynamicConfigService
import com.netflix.spinnaker.kork.exceptions.UserException
import com.netflix.spinnaker.orca.TaskExecutionInterceptor
import com.netflix.spinnaker.orca.TaskResolver
import com.netflix.spinnaker.orca.api.pipeline.OverridableTimeoutRetryableTask
Expand Down Expand Up @@ -169,7 +170,7 @@ class RunTaskHandler(
} else if (e is TimeoutException && stage.context["markSuccessfulOnTimeout"] == true) {
queue.push(CompleteTask(message, SUCCEEDED))
} else {
if (e !is TimeoutException) {
if (e !is TimeoutException && e !is UserException) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh, this is actually from a related but separate change. but might as well go together. Basically, we shouldn't log errors for exceptiosn that are not in "our" control. E.g. bad SpEL expression and any other UserExceptions

log.error("Error running ${message.taskType.simpleName} for ${message.executionType}[${message.executionId}]", e)
}

Expand All @@ -186,17 +187,20 @@ class RunTaskHandler(
}

private fun trackResult(stage: StageExecution, thisInvocationStartTimeMs: Long, taskModel: TaskExecution, status: ExecutionStatus) {
val commonTags = MetricsTagHelper.commonTags(stage, taskModel, status)
val detailedTags = MetricsTagHelper.detailedTaskTags(stage, taskModel, status)
try {
val commonTags = MetricsTagHelper.commonTags(stage, taskModel, status)
val detailedTags = MetricsTagHelper.detailedTaskTags(stage, taskModel, status)

val elapsedMillis = clock.millis() - thisInvocationStartTimeMs
val elapsedMillis = clock.millis() - thisInvocationStartTimeMs

hashMapOf(
"task.invocations.duration" to commonTags + BasicTag("application", stage.execution.application),
"task.invocations.duration.withType" to commonTags + detailedTags
).forEach {
name, tags ->
registry.timer(name, tags).record(elapsedMillis, TimeUnit.MILLISECONDS)
hashMapOf(
"task.invocations.duration" to commonTags + BasicTag("application", stage.execution.application),
"task.invocations.duration.withType" to commonTags + detailedTags
).forEach { name, tags ->
registry.timer(name, tags).record(elapsedMillis, TimeUnit.MILLISECONDS)
}
} catch (e: java.lang.Exception) {
log.warn("Failed to track result for stage: ${stage.id}, task: ${taskModel.id}", e)
}
}

Expand Down