wip 2

.github/workflows/appsignals-e2e-ec2-test.yml

@@ -166,10 +166,10 @@ jobs:
       - name: Call all test APIs
         continue-on-error: true
         run: |
-          curl -S -s http://${{ env.MAIN_SERVICE_ENDPOINT }}/outgoing-http-call/
-          curl -S -s http://${{ env.MAIN_SERVICE_ENDPOINT }}/aws-sdk-call/
-          curl -S -s http://${{ env.MAIN_SERVICE_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_IP }}/
-          curl -S -s http://${{ env.MAIN_SERVICE_ENDPOINT }}/client-call/
+          curl -S -s http://${{ env.MAIN_SERVICE_ENDPOINT }}/outgoing-http-call
+          curl -S -s http://${{ env.MAIN_SERVICE_ENDPOINT }}/aws-sdk-call?testingId=${{ env.TESTING_ID }}
+          curl -S -s http://${{ env.MAIN_SERVICE_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_IP }}
+          curl -S -s http://${{ env.MAIN_SERVICE_ENDPOINT }}/client-call
 
       # Validation for pulse telemetry data
       - name: Validate generated EMF logs
@@ -201,7 +201,7 @@ jobs:
           --log-group ${{ env.LOG_GROUP_NAME }}
           --service-name sample-application-${{ env.TESTING_ID }}
           --remote-service-name sample-remote-application-${{ env.TESTING_ID }}
-          --request-body ip=${{ env.REMOTE_SERVICE_IP }}
+          --request-body ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}
           --instance-ami ${{ env.EC2_INSTANCE_AMI }}
           --rollup'
 

.github/workflows/appsignals-e2e-eks-test.yml

@@ -253,10 +253,10 @@ jobs:
       - name: Call all test APIs
         continue-on-error: true
         run: |
-          curl -S -s http://${{ env.APP_ENDPOINT }}/outgoing-http-call/
-          curl -S -s http://${{ env.APP_ENDPOINT }}/aws-sdk-call/
-          curl -S -s http://${{ env.APP_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_POD_IP }}/
-          curl -S -s http://${{ env.APP_ENDPOINT }}/client-call/
+          curl -S -s http://${{ env.APP_ENDPOINT }}/outgoing-http-call
+          curl -S -s http://${{ env.APP_ENDPOINT }}/aws-sdk-call?testingId=${{ env.TESTING_ID }}
+          curl -S -s http://${{ env.APP_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_POD_IP }}
+          curl -S -s http://${{ env.APP_ENDPOINT }}/client-call
 
       # Validation for app signals telemetry data
       - name: Call endpoint and validate generated EMF logs
@@ -291,7 +291,7 @@ jobs:
           --service-name sample-application-${{ env.TESTING_ID }}
           --remote-service-name sample-remote-application-${{ env.TESTING_ID }}
           --remote-service-deployment-name ${{ env.REMOTE_SERVICE_DEPLOYMENT_NAME }}
-          --request-body ip=${{ env.REMOTE_SERVICE_POD_IP }}
+          --request-body ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }}
           --rollup'
 
       - name: Call endpoints and validate generated traces

.github/workflows/appsignals-python-e2e-ec2-test.yml

@@ -170,7 +170,7 @@ jobs:
         continue-on-error: true
         run: |
           curl -S -s -o /dev/null http://${{ env.MAIN_SERVICE_ENDPOINT }}/outgoing-http-call
-          curl -S -s -o /dev/null http://${{ env.MAIN_SERVICE_ENDPOINT }}/aws-sdk-call
+          curl -S -s -o /dev/null http://${{ env.MAIN_SERVICE_ENDPOINT }}/aws-sdk-call?testingId=${{ env.TESTING_ID }}
           curl -S -s -o /dev/null http://${{ env.MAIN_SERVICE_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_IP }}
           curl -S -s -o /dev/null http://${{ env.MAIN_SERVICE_ENDPOINT }}/client-call
 
@@ -204,7 +204,7 @@ jobs:
           --log-group ${{ env.LOG_GROUP_NAME }}
           --service-name python-sample-application-${{ env.TESTING_ID }}
           --remote-service-name python-sample-remote-application-${{ env.TESTING_ID }}
-          --request-body ip=${{ env.REMOTE_SERVICE_IP }}
+          --request-body ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}
           --instance-ami ${{ env.EC2_INSTANCE_AMI }}
           --rollup'
 

.github/workflows/appsignals-python-e2e-eks-test.yml

@@ -301,7 +301,7 @@ jobs:
         continue-on-error: true
         run: |
           curl -S -s -o /dev/null http://${{ env.APP_ENDPOINT }}/outgoing-http-call
-          curl -S -s -o /dev/null http://${{ env.APP_ENDPOINT }}/aws-sdk-call
+          curl -S -s -o /dev/null http://${{ env.APP_ENDPOINT }}/aws-sdk-call?testingId=${{ env.TESTING_ID }}
           curl -S -s -o /dev/null http://${{ env.APP_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_POD_IP }}
           curl -S -s -o /dev/null http://${{ env.APP_ENDPOINT }}/client-call
 
@@ -338,7 +338,7 @@ jobs:
           --service-name python-application-${{ env.TESTING_ID }}
           --remote-service-name python-remote-application-${{ env.TESTING_ID }}
           --remote-service-deployment-name ${{ env.REMOTE_SERVICE_DEPLOYMENT_NAME }}
-          --request-body ip=${{ env.REMOTE_SERVICE_POD_IP }}
+          --request-body ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }}
           --rollup'
 
       - name: Call endpoints and validate generated traces

terraform/ec2/main.tf

@@ -177,7 +177,7 @@ resource "null_resource" "remote_service_setup" {
       "echo $agent_config > amazon-cloudwatch-agent.json",
 
       # Get and run CW agent rpm
-       "${var.get_cw_agent_rpm_command}",
+      "${var.get_cw_agent_rpm_command}",
       "sudo rpm -U ./cw-agent.rpm",
       "sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:./amazon-cloudwatch-agent.json",
 

validator/src/main/java/com/amazon/aoc/App.java

@@ -75,6 +75,9 @@ public class App implements Callable<Integer> {
   @CommandLine.Option(names = {"--remote-service-deployment-name"})
   private String remoteServiceDeploymentName;
 
+  @CommandLine.Option(names = {"--remote-target-name"})
+  private String remoteTargetName;
+
   @CommandLine.Option(names = {"--endpoint"})
   private String endpoint;
 
@@ -160,6 +163,7 @@ public Integer call() throws Exception {
     context.setServiceName(this.serviceName);
     context.setRemoteServiceName(this.remoteServiceName);
     context.setRemoteServiceDeploymentName(this.remoteServiceDeploymentName);
+    context.setRemoteTargetName(this.remoteTargetName);
     context.setEndpoint(this.endpoint);
     context.setRequestBody(this.requestBody);
     context.setLogGroup(this.logGroup);

validator/src/main/java/com/amazon/aoc/models/Context.java

@@ -49,6 +49,8 @@ public class Context {
 
   private String remoteServiceDeploymentName;
 
+  private String remoteTargetName;
+
   private String endpoint;
 
   private String requestBody;

validator/src/main/java/com/amazon/aoc/services/CloudWatchService.java

@@ -28,13 +28,16 @@
 import com.amazonaws.services.logs.model.OutputLogEvent;
 import java.util.Date;
 import java.util.List;
+import java.util.stream.Collectors;
+import kotlin.Pair;
 import lombok.extern.log4j.Log4j2;
 
 /** a wrapper of cloudwatch client. */
 @Log4j2
 public class CloudWatchService {
   public static final String SERVICE_DIMENSION = "Service";
   public static final String REMOTE_SERVICE_DIMENSION = "RemoteService";
+  public static final String REMOTE_TARGET_DIMENSION = "RemoteTarget";
 
   private static final int MAX_QUERY_PERIOD = 60;
   private static final String REQUESTER = "integrationTest";
@@ -62,15 +65,20 @@ public CloudWatchService(String region) {
   public List<Metric> listMetrics(
       final String namespace,
       final String metricName,
-      final String dimensionKey,
-      final String dimensionValue) {
-    final DimensionFilter dimensionFilter =
-        new DimensionFilter().withName(dimensionKey).withValue(dimensionValue);
+      final List<Pair<String, String>> dimensionList) {
+    final List<DimensionFilter> dimensionFilters =
+        dimensionList.stream()
+            .map(
+                dimension ->
+                    new DimensionFilter()
+                        .withName(dimension.getFirst())
+                        .withValue(dimension.getSecond()))
+            .collect(Collectors.toList());
     final ListMetricsRequest listMetricsRequest =
         new ListMetricsRequest()
             .withNamespace(namespace)
             .withMetricName(metricName)
-            .withDimensions(dimensionFilter)
+            .withDimensions(dimensionFilters)
             .withRecentlyActive("PT3H");
     return amazonCloudWatch.listMetrics(listMetricsRequest).getMetrics();
   }

validator/src/main/java/com/amazon/aoc/validators/CWMetricValidator.java

@@ -30,17 +30,19 @@
 import com.google.common.collect.Lists;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeSet;
+import kotlin.Pair;
 import lombok.extern.log4j.Log4j2;
 
 @Log4j2
 public class CWMetricValidator implements IValidator {
-  private static int DEFAULT_MAX_RETRY_COUNT = 80;
+  private static int DEFAULT_MAX_RETRY_COUNT = 10;
 
   private MustacheHelper mustacheHelper = new MustacheHelper();
   private ICaller caller;
@@ -103,16 +105,30 @@ public void validate() throws Exception {
           }
 
           List<Metric> actualMetricList = Lists.newArrayList();
-          addMetrics(
-              CloudWatchService.SERVICE_DIMENSION,
-              serviceNames,
-              expectedMetricList,
-              actualMetricList);
-          addMetrics(
-              CloudWatchService.REMOTE_SERVICE_DIMENSION,
-              remoteServiceNames,
-              expectedMetricList,
-              actualMetricList);
+
+          // Add sets of dimesion filters to use for each query to CloudWatch
+          List<List<Pair<String, String>>> dimensionLists = Lists.newArrayList();
+          // Query metrics for rolled up into the [serviceDimension] dimension
+          for (String serviceName : serviceNames) {
+            dimensionLists.add(
+                Arrays.asList(new Pair<>(CloudWatchService.SERVICE_DIMENSION, serviceName)));
+          }
+          // Query metrics for rolled up into the [remoteServiceDimension] dimension
+          for (String remoteServiceName : remoteServiceNames) {
+            dimensionLists.add(
+                Arrays.asList(
+                    new Pair<>(CloudWatchService.REMOTE_SERVICE_DIMENSION, remoteServiceName)));
+          }
+          // Query metrics for rolled up into the [remoteServiceDimension, remoteTargetDemsion] dimensions
+          dimensionLists.add(
+              Arrays.asList(
+                  new Pair<>(CloudWatchService.REMOTE_SERVICE_DIMENSION, "AWS.SDK.S3"),
+                  new Pair<>(CloudWatchService.REMOTE_TARGET_DIMENSION, context.getRemoteTargetName())));
+
+          // Populate actualMetricList with metrics that pass through one of the dimension filters
+          for (List<Pair<String, String>> dimensionList : dimensionLists) {
+            addMetrics(dimensionList, expectedMetricList, actualMetricList);
+          }
 
           // remove the skip dimensions
           log.info("dimensions to be skipped in validation: {}", skippedDimensionNameList);
@@ -132,16 +148,12 @@ public void validate() throws Exception {
   }
 
   private void addMetrics(
-      String dimensionName,
-      List<String> dimensionValues,
+      List<Pair<String, String>> dimensionList,
       List<Metric> expectedMetricList,
       List<Metric> actualMetricList)
       throws Exception {
-    for (String dimensionValue : dimensionValues) {
-      actualMetricList.addAll(
-          this.listMetricFromCloudWatch(
-              cloudWatchService, expectedMetricList, dimensionName, dimensionValue));
-    }
+    actualMetricList.addAll(
+        this.listMetricFromCloudWatch(cloudWatchService, expectedMetricList, dimensionList));
   }
 
   /**
@@ -194,8 +206,7 @@ private void compareMetricLists(List<Metric> toBeCheckedMetricList, List<Metric>
   private List<Metric> listMetricFromCloudWatch(
       CloudWatchService cloudWatchService,
       List<Metric> expectedMetricList,
-      String dimensionKey,
-      String dimensionValue)
+      List<Pair<String, String>> dimensionList)
       throws IOException {
     // put namespace into the map key, so that we can use it to search metric
     HashMap<String, String> metricNameMap = new HashMap<>();
@@ -207,8 +218,7 @@ private List<Metric> listMetricFromCloudWatch(
     List<Metric> result = new ArrayList<>();
     for (String metricName : metricNameMap.keySet()) {
       result.addAll(
-          cloudWatchService.listMetrics(
-              metricNameMap.get(metricName), metricName, dimensionKey, dimensionValue));
+          cloudWatchService.listMetrics(metricNameMap.get(metricName), metricName, dimensionList));
     }
     return result;
   }