From 415bb5e6c80d3b0651da9bc906054d4ae1fc6a2d Mon Sep 17 00:00:00 2001 From: genbit Date: Tue, 9 Nov 2021 13:31:33 -0800 Subject: [PATCH] Separate ecs cloudwatch container insights into separate configs (#720) Signed-off-by: Sergey Generalov Co-authored-by: Sergey Generalov --- ...cs-cloudwatch-container-insights-xray.yaml | 225 ++++++++++++++++++ .../ecs-cloudwatch-container-insights.yaml | 217 +++++++++++++++++ config/ecs/ecs-cloudwatch-xray.yaml | 65 ----- config/ecs/ecs-cloudwatch.yaml | 65 ----- 4 files changed, 442 insertions(+), 130 deletions(-) create mode 100644 config/ecs/ecs-cloudwatch-container-insights-xray.yaml create mode 100644 config/ecs/ecs-cloudwatch-container-insights.yaml diff --git a/config/ecs/ecs-cloudwatch-container-insights-xray.yaml b/config/ecs/ecs-cloudwatch-container-insights-xray.yaml new file mode 100644 index 0000000000..f60abc1a7a --- /dev/null +++ b/config/ecs/ecs-cloudwatch-container-insights-xray.yaml @@ -0,0 +1,225 @@ +extensions: + health_check: + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + awsecscontainermetrics: + +processors: + batch/traces: + timeout: 1s + send_batch_size: 50 + batch/metrics: + timeout: 60s + resourcedetection: + detectors: + - env + - system + - ecs + - ec2 + filter: + metrics: + include: + match_type: strict + metric_names: + - ecs.task.memory.reserved + - ecs.task.memory.utilized + - ecs.task.cpu.reserved + - ecs.task.cpu.utilized + - ecs.task.network.rate.rx + - ecs.task.network.rate.tx + - ecs.task.storage.read_bytes + - ecs.task.storage.write_bytes + - container.duration + metricstransform: + transforms: + - metric_name: ecs.task.memory.utilized + action: update + new_name: MemoryUtilized + - metric_name: ecs.task.memory.reserved + action: update + new_name: MemoryReserved + - metric_name: ecs.task.cpu.utilized + action: update + new_name: CpuUtilized + - metric_name: ecs.task.cpu.reserved + action: update + new_name: CpuReserved + - metric_name: ecs.task.network.rate.rx + action: update + new_name: NetworkRxBytes + - metric_name: ecs.task.network.rate.tx + action: update + new_name: NetworkTxBytes + - metric_name: ecs.task.storage.read_bytes + action: update + new_name: StorageReadBytes + - metric_name: ecs.task.storage.write_bytes + action: update + new_name: StorageWriteBytes + + resource: + attributes: + - key: ClusterName + from_attribute: aws.ecs.cluster.name + action: insert + - key: aws.ecs.cluster.name + action: delete + - key: ServiceName + from_attribute: aws.ecs.service.name + action: insert + - key: aws.ecs.service.name + action: delete + - key: TaskId + from_attribute: aws.ecs.task.id + action: insert + - key: aws.ecs.task.id + action: delete + - key: TaskDefinitionFamily + from_attribute: aws.ecs.task.family + action: insert + - key: aws.ecs.task.family + action: delete + - key: TaskARN + from_attribute: aws.ecs.task.arn + action: insert + - key: aws.ecs.task.arn + action: delete + - key: DockerName + from_attribute: aws.ecs.docker.name + action: insert + - key: aws.ecs.docker.name + action: delete + - key: TaskDefinitionRevision + from_attribute: aws.ecs.task.version + action: insert + - key: aws.ecs.task.version + action: delete + - key: PullStartedAt + from_attribute: aws.ecs.task.pull_started_at + action: insert + - key: aws.ecs.task.pull_started_at + action: delete + - key: PullStoppedAt + from_attribute: aws.ecs.task.pull_stopped_at + action: insert + - key: aws.ecs.task.pull_stopped_at + action: delete + - key: AvailabilityZone + from_attribute: cloud.zone + action: insert + - key: cloud.zone + action: delete + - key: LaunchType + from_attribute: aws.ecs.task.launch_type + action: insert + - key: aws.ecs.task.launch_type + action: delete + - key: Region + from_attribute: cloud.region + action: insert + - key: cloud.region + action: delete + - key: AccountId + from_attribute: cloud.account.id + action: insert + - key: cloud.account.id + action: delete + - key: DockerId + from_attribute: container.id + action: insert + - key: container.id + action: delete + - key: ContainerName + from_attribute: container.name + action: insert + - key: container.name + action: delete + - key: Image + from_attribute: container.image.name + action: insert + - key: container.image.name + action: delete + - key: ImageId + from_attribute: aws.ecs.container.image.id + action: insert + - key: aws.ecs.container.image.id + action: delete + - key: ExitCode + from_attribute: aws.ecs.container.exit_code + action: insert + - key: aws.ecs.container.exit_code + action: delete + - key: CreatedAt + from_attribute: aws.ecs.container.created_at + action: insert + - key: aws.ecs.container.created_at + action: delete + - key: StartedAt + from_attribute: aws.ecs.container.started_at + action: insert + - key: aws.ecs.container.started_at + action: delete + - key: FinishedAt + from_attribute: aws.ecs.container.finished_at + action: insert + - key: aws.ecs.container.finished_at + action: delete + - key: ImageTag + from_attribute: container.image.tag + action: insert + - key: container.image.tag + action: delete + +exporters: + awsxray: + awsemf/application: + namespace: ECS/AWSOTel/Application + log_group_name: '/aws/ecs/application/metrics' + resource_to_telemetry_conversion: + enabled: true + dimension_rollup_option: NoDimensionRollup + metric_declarations: + - dimensions: [ [ TaskDefinitionFamily ] ] + metric_name_selectors: [.*] + awsemf/performance: + namespace: ECS/ContainerInsights + log_group_name: '/aws/ecs/containerinsights/{ClusterName}/performance' + log_stream_name: '{TaskId}' + resource_to_telemetry_conversion: + enabled: true + dimension_rollup_option: NoDimensionRollup + metric_declarations: + - dimensions: [ [ ClusterName ], [ ClusterName, TaskDefinitionFamily ] ] + metric_name_selectors: + - MemoryUtilized + - MemoryReserved + - CpuUtilized + - CpuReserved + - NetworkRxBytes + - NetworkTxBytes + - StorageReadBytes + - StorageWriteBytes + - metric_name_selectors: [container.*] + +service: + pipelines: + traces: + receivers: [otlp] + processors: [resourcedetection, batch/traces] + exporters: [awsxray] + metrics/application: + receivers: [otlp] + processors: [resourcedetection, resource, batch/metrics] + exporters: [awsemf/application] + metrics/performance: + receivers: [awsecscontainermetrics ] + processors: [filter, metricstransform, resource] + exporters: [ awsemf/performance ] + + extensions: [health_check] diff --git a/config/ecs/ecs-cloudwatch-container-insights.yaml b/config/ecs/ecs-cloudwatch-container-insights.yaml new file mode 100644 index 0000000000..60623d62b0 --- /dev/null +++ b/config/ecs/ecs-cloudwatch-container-insights.yaml @@ -0,0 +1,217 @@ +extensions: + health_check: + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + awsecscontainermetrics: + +processors: + batch/metrics: + timeout: 60s + resourcedetection: + detectors: + - env + - system + - ecs + - ec2 + filter: + metrics: + include: + match_type: strict + metric_names: + - ecs.task.memory.reserved + - ecs.task.memory.utilized + - ecs.task.cpu.reserved + - ecs.task.cpu.utilized + - ecs.task.network.rate.rx + - ecs.task.network.rate.tx + - ecs.task.storage.read_bytes + - ecs.task.storage.write_bytes + - container.duration + metricstransform: + transforms: + - metric_name: ecs.task.memory.utilized + action: update + new_name: MemoryUtilized + - metric_name: ecs.task.memory.reserved + action: update + new_name: MemoryReserved + - metric_name: ecs.task.cpu.utilized + action: update + new_name: CpuUtilized + - metric_name: ecs.task.cpu.reserved + action: update + new_name: CpuReserved + - metric_name: ecs.task.network.rate.rx + action: update + new_name: NetworkRxBytes + - metric_name: ecs.task.network.rate.tx + action: update + new_name: NetworkTxBytes + - metric_name: ecs.task.storage.read_bytes + action: update + new_name: StorageReadBytes + - metric_name: ecs.task.storage.write_bytes + action: update + new_name: StorageWriteBytes + + resource: + attributes: + - key: ClusterName + from_attribute: aws.ecs.cluster.name + action: insert + - key: aws.ecs.cluster.name + action: delete + - key: ServiceName + from_attribute: aws.ecs.service.name + action: insert + - key: aws.ecs.service.name + action: delete + - key: TaskId + from_attribute: aws.ecs.task.id + action: insert + - key: aws.ecs.task.id + action: delete + - key: TaskDefinitionFamily + from_attribute: aws.ecs.task.family + action: insert + - key: aws.ecs.task.family + action: delete + - key: TaskARN + from_attribute: aws.ecs.task.arn + action: insert + - key: aws.ecs.task.arn + action: delete + - key: DockerName + from_attribute: aws.ecs.docker.name + action: insert + - key: aws.ecs.docker.name + action: delete + - key: TaskDefinitionRevision + from_attribute: aws.ecs.task.version + action: insert + - key: aws.ecs.task.version + action: delete + - key: PullStartedAt + from_attribute: aws.ecs.task.pull_started_at + action: insert + - key: aws.ecs.task.pull_started_at + action: delete + - key: PullStoppedAt + from_attribute: aws.ecs.task.pull_stopped_at + action: insert + - key: aws.ecs.task.pull_stopped_at + action: delete + - key: AvailabilityZone + from_attribute: cloud.zone + action: insert + - key: cloud.zone + action: delete + - key: LaunchType + from_attribute: aws.ecs.task.launch_type + action: insert + - key: aws.ecs.task.launch_type + action: delete + - key: Region + from_attribute: cloud.region + action: insert + - key: cloud.region + action: delete + - key: AccountId + from_attribute: cloud.account.id + action: insert + - key: cloud.account.id + action: delete + - key: DockerId + from_attribute: container.id + action: insert + - key: container.id + action: delete + - key: ContainerName + from_attribute: container.name + action: insert + - key: container.name + action: delete + - key: Image + from_attribute: container.image.name + action: insert + - key: container.image.name + action: delete + - key: ImageId + from_attribute: aws.ecs.container.image.id + action: insert + - key: aws.ecs.container.image.id + action: delete + - key: ExitCode + from_attribute: aws.ecs.container.exit_code + action: insert + - key: aws.ecs.container.exit_code + action: delete + - key: CreatedAt + from_attribute: aws.ecs.container.created_at + action: insert + - key: aws.ecs.container.created_at + action: delete + - key: StartedAt + from_attribute: aws.ecs.container.started_at + action: insert + - key: aws.ecs.container.started_at + action: delete + - key: FinishedAt + from_attribute: aws.ecs.container.finished_at + action: insert + - key: aws.ecs.container.finished_at + action: delete + - key: ImageTag + from_attribute: container.image.tag + action: insert + - key: container.image.tag + action: delete + +exporters: + awsemf/application: + namespace: ECS/AWSOTel/Application + log_group_name: '/aws/ecs/application/metrics' + resource_to_telemetry_conversion: + enabled: true + dimension_rollup_option: NoDimensionRollup + metric_declarations: + - dimensions: [ [ TaskDefinitionFamily ] ] + metric_name_selectors: [.*] + awsemf/performance: + namespace: ECS/ContainerInsights + log_group_name: '/aws/ecs/containerinsights/{ClusterName}/performance' + log_stream_name: '{TaskId}' + resource_to_telemetry_conversion: + enabled: true + dimension_rollup_option: NoDimensionRollup + metric_declarations: + - dimensions: [ [ ClusterName ], [ ClusterName, TaskDefinitionFamily ] ] + metric_name_selectors: + - MemoryUtilized + - MemoryReserved + - CpuUtilized + - CpuReserved + - NetworkRxBytes + - NetworkTxBytes + - StorageReadBytes + - StorageWriteBytes + - metric_name_selectors: [container.*] + +service: + pipelines: + metrics/application: + receivers: [otlp] + processors: [resourcedetection, resource, batch/metrics] + exporters: [awsemf/application] + metrics/performance: + receivers: [awsecscontainermetrics ] + processors: [filter, metricstransform, resource] + exporters: [ awsemf/performance ] + + extensions: [health_check] diff --git a/config/ecs/ecs-cloudwatch-xray.yaml b/config/ecs/ecs-cloudwatch-xray.yaml index f60abc1a7a..e8e56543ae 100644 --- a/config/ecs/ecs-cloudwatch-xray.yaml +++ b/config/ecs/ecs-cloudwatch-xray.yaml @@ -8,7 +8,6 @@ receivers: endpoint: 0.0.0.0:4317 http: endpoint: 0.0.0.0:4318 - awsecscontainermetrics: processors: batch/traces: @@ -22,47 +21,6 @@ processors: - system - ecs - ec2 - filter: - metrics: - include: - match_type: strict - metric_names: - - ecs.task.memory.reserved - - ecs.task.memory.utilized - - ecs.task.cpu.reserved - - ecs.task.cpu.utilized - - ecs.task.network.rate.rx - - ecs.task.network.rate.tx - - ecs.task.storage.read_bytes - - ecs.task.storage.write_bytes - - container.duration - metricstransform: - transforms: - - metric_name: ecs.task.memory.utilized - action: update - new_name: MemoryUtilized - - metric_name: ecs.task.memory.reserved - action: update - new_name: MemoryReserved - - metric_name: ecs.task.cpu.utilized - action: update - new_name: CpuUtilized - - metric_name: ecs.task.cpu.reserved - action: update - new_name: CpuReserved - - metric_name: ecs.task.network.rate.rx - action: update - new_name: NetworkRxBytes - - metric_name: ecs.task.network.rate.tx - action: update - new_name: NetworkTxBytes - - metric_name: ecs.task.storage.read_bytes - action: update - new_name: StorageReadBytes - - metric_name: ecs.task.storage.write_bytes - action: update - new_name: StorageWriteBytes - resource: attributes: - key: ClusterName @@ -187,25 +145,6 @@ exporters: metric_declarations: - dimensions: [ [ TaskDefinitionFamily ] ] metric_name_selectors: [.*] - awsemf/performance: - namespace: ECS/ContainerInsights - log_group_name: '/aws/ecs/containerinsights/{ClusterName}/performance' - log_stream_name: '{TaskId}' - resource_to_telemetry_conversion: - enabled: true - dimension_rollup_option: NoDimensionRollup - metric_declarations: - - dimensions: [ [ ClusterName ], [ ClusterName, TaskDefinitionFamily ] ] - metric_name_selectors: - - MemoryUtilized - - MemoryReserved - - CpuUtilized - - CpuReserved - - NetworkRxBytes - - NetworkTxBytes - - StorageReadBytes - - StorageWriteBytes - - metric_name_selectors: [container.*] service: pipelines: @@ -217,9 +156,5 @@ service: receivers: [otlp] processors: [resourcedetection, resource, batch/metrics] exporters: [awsemf/application] - metrics/performance: - receivers: [awsecscontainermetrics ] - processors: [filter, metricstransform, resource] - exporters: [ awsemf/performance ] extensions: [health_check] diff --git a/config/ecs/ecs-cloudwatch.yaml b/config/ecs/ecs-cloudwatch.yaml index 60623d62b0..2269baa42c 100644 --- a/config/ecs/ecs-cloudwatch.yaml +++ b/config/ecs/ecs-cloudwatch.yaml @@ -8,7 +8,6 @@ receivers: endpoint: 0.0.0.0:4317 http: endpoint: 0.0.0.0:4318 - awsecscontainermetrics: processors: batch/metrics: @@ -19,47 +18,6 @@ processors: - system - ecs - ec2 - filter: - metrics: - include: - match_type: strict - metric_names: - - ecs.task.memory.reserved - - ecs.task.memory.utilized - - ecs.task.cpu.reserved - - ecs.task.cpu.utilized - - ecs.task.network.rate.rx - - ecs.task.network.rate.tx - - ecs.task.storage.read_bytes - - ecs.task.storage.write_bytes - - container.duration - metricstransform: - transforms: - - metric_name: ecs.task.memory.utilized - action: update - new_name: MemoryUtilized - - metric_name: ecs.task.memory.reserved - action: update - new_name: MemoryReserved - - metric_name: ecs.task.cpu.utilized - action: update - new_name: CpuUtilized - - metric_name: ecs.task.cpu.reserved - action: update - new_name: CpuReserved - - metric_name: ecs.task.network.rate.rx - action: update - new_name: NetworkRxBytes - - metric_name: ecs.task.network.rate.tx - action: update - new_name: NetworkTxBytes - - metric_name: ecs.task.storage.read_bytes - action: update - new_name: StorageReadBytes - - metric_name: ecs.task.storage.write_bytes - action: update - new_name: StorageWriteBytes - resource: attributes: - key: ClusterName @@ -183,25 +141,6 @@ exporters: metric_declarations: - dimensions: [ [ TaskDefinitionFamily ] ] metric_name_selectors: [.*] - awsemf/performance: - namespace: ECS/ContainerInsights - log_group_name: '/aws/ecs/containerinsights/{ClusterName}/performance' - log_stream_name: '{TaskId}' - resource_to_telemetry_conversion: - enabled: true - dimension_rollup_option: NoDimensionRollup - metric_declarations: - - dimensions: [ [ ClusterName ], [ ClusterName, TaskDefinitionFamily ] ] - metric_name_selectors: - - MemoryUtilized - - MemoryReserved - - CpuUtilized - - CpuReserved - - NetworkRxBytes - - NetworkTxBytes - - StorageReadBytes - - StorageWriteBytes - - metric_name_selectors: [container.*] service: pipelines: @@ -209,9 +148,5 @@ service: receivers: [otlp] processors: [resourcedetection, resource, batch/metrics] exporters: [awsemf/application] - metrics/performance: - receivers: [awsecscontainermetrics ] - processors: [filter, metricstransform, resource] - exporters: [ awsemf/performance ] extensions: [health_check]