Merge pull request aws-observability#41 from jj22ee/metric-rollup-tests

Add Test Cases for Metric Rollup Addition of [RemoteService, RemoteTarget, ...]
zzhlogin · May 7, 2024 · 6a7b592 · 6a7b592
2 parents 379730b + 8a6086e
commit 6a7b592
Show file tree

Hide file tree

Showing 40 changed files with 488 additions and 92 deletions.
diff --git a/.github/workflows/application-signals-python-e2e-ec2-test.yml b/.github/workflows/application-signals-python-e2e-ec2-test.yml
@@ -168,7 +168,7 @@ jobs:
         continue-on-error: true
         run: |
           curl -S -s -o /dev/null http://${{ env.MAIN_SERVICE_ENDPOINT }}/outgoing-http-call; echo
-          curl -S -s -o /dev/null http://${{ env.MAIN_SERVICE_ENDPOINT }}/aws-sdk-call; echo
+          curl -S -s -o /dev/null http://${{ env.MAIN_SERVICE_ENDPOINT }}/aws-sdk-call?testingId=${{ env.TESTING_ID }}; echo
           curl -S -s -o /dev/null http://${{ env.MAIN_SERVICE_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_IP }}; echo
           curl -S -s -o /dev/null http://${{ env.MAIN_SERVICE_ENDPOINT }}/client-call; echo
 
@@ -184,36 +184,39 @@ jobs:
       - name: Validate generated EMF logs
         id: log-validation
         run: ./gradlew validator:run --args='-c python/ec2/log-validation.yml
+          --testing-id ${{ env.TESTING_ID }}
           --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}
           --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001
           --region ${{ inputs.aws-region }}
           --metric-namespace ${{ env.METRIC_NAMESPACE }}
           --log-group ${{ env.LOG_GROUP_NAME }}
           --service-name python-sample-application-${{ env.TESTING_ID }}
           --remote-service-name python-sample-remote-application-${{ env.TESTING_ID }}
-          --request-body ip=${{ env.REMOTE_SERVICE_IP }}
+          --query-string ip=${{ env.REMOTE_SERVICE_IP }}
           --instance-ami ${{ env.EC2_INSTANCE_AMI }}
           --rollup'
 
       - name: Validate generated metrics
         id: metric-validation
         if: (success() || steps.log-validation.outcome == 'failure') && !cancelled()
         run: ./gradlew validator:run --args='-c python/ec2/metric-validation.yml
+          --testing-id ${{ env.TESTING_ID }}
           --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}
           --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001
           --region ${{ inputs.aws-region }}
           --metric-namespace ${{ env.METRIC_NAMESPACE }}
           --log-group ${{ env.LOG_GROUP_NAME }}
           --service-name python-sample-application-${{ env.TESTING_ID }}
           --remote-service-name python-sample-remote-application-${{ env.TESTING_ID }}
-          --request-body ip=${{ env.REMOTE_SERVICE_IP }}
+          --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}
           --instance-ami ${{ env.EC2_INSTANCE_AMI }}
           --rollup'
 
       - name: Validate generated traces
         id: trace-validation
         if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled()
         run: ./gradlew validator:run --args='-c python/ec2/trace-validation.yml
+          --testing-id ${{ env.TESTING_ID }}
           --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}
           --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8001
           --region ${{ inputs.aws-region }}
@@ -222,7 +225,7 @@ jobs:
           --log-group ${{ env.LOG_GROUP_NAME }}
           --service-name python-sample-application-${{ env.TESTING_ID }}
           --remote-service-name python-sample-remote-application-${{ env.TESTING_ID }}
-          --request-body ip=${{ env.REMOTE_SERVICE_IP }}
+          --query-string ip=${{ env.REMOTE_SERVICE_IP }}
           --instance-ami ${{ env.EC2_INSTANCE_AMI }}
           --rollup'
 

diff --git a/.github/workflows/application-signals-python-e2e-eks-test.yml b/.github/workflows/application-signals-python-e2e-eks-test.yml
@@ -253,7 +253,7 @@ jobs:
         continue-on-error: true
         run: |
           curl -S -s -o /dev/null http://${{ env.APP_ENDPOINT }}/outgoing-http-call; echo
-          curl -S -s -o /dev/null http://${{ env.APP_ENDPOINT }}/aws-sdk-call; echo
+          curl -S -s -o /dev/null http://${{ env.APP_ENDPOINT }}/aws-sdk-call?testingId=${{ env.TESTING_ID }}; echo
           curl -S -s -o /dev/null http://${{ env.APP_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_POD_IP }}; echo
           curl -S -s -o /dev/null http://${{ env.APP_ENDPOINT }}/client-call; echo
 
@@ -280,7 +280,7 @@ jobs:
           --platform-info ${{ inputs.test-cluster-name }}
           --service-name python-application-${{ env.TESTING_ID }}
           --remote-service-deployment-name ${{ env.REMOTE_SERVICE_DEPLOYMENT_NAME }}
-          --request-body ip=${{ env.REMOTE_SERVICE_POD_IP }}
+          --query-string ip=${{ env.REMOTE_SERVICE_POD_IP }}
           --rollup'
 
       - name: Call endpoints and validate generated metrics
@@ -298,7 +298,7 @@ jobs:
           --service-name python-application-${{ env.TESTING_ID }}
           --remote-service-name python-remote-application-${{ env.TESTING_ID }}
           --remote-service-deployment-name ${{ env.REMOTE_SERVICE_DEPLOYMENT_NAME }}
-          --request-body ip=${{ env.REMOTE_SERVICE_POD_IP }}
+          --query-string ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }}
           --rollup'
 
       - name: Call endpoints and validate generated traces
@@ -314,7 +314,7 @@ jobs:
           --platform-info ${{ inputs.test-cluster-name }}
           --service-name python-application-${{ env.TESTING_ID }}
           --remote-service-deployment-name ${{ env.REMOTE_SERVICE_DEPLOYMENT_NAME }}
-          --request-body ip=${{ env.REMOTE_SERVICE_POD_IP }}
+          --query-string ip=${{ env.REMOTE_SERVICE_POD_IP }}
           --rollup'
 
       - name: Publish metric on test result

diff --git a/.github/workflows/appsignals-e2e-ec2-test.yml b/.github/workflows/appsignals-e2e-ec2-test.yml
@@ -165,10 +165,10 @@ jobs:
       - name: Call all test APIs
         continue-on-error: true
         run: |
-          curl -S -s http://${{ env.MAIN_SERVICE_ENDPOINT }}/outgoing-http-call/
-          curl -S -s http://${{ env.MAIN_SERVICE_ENDPOINT }}/aws-sdk-call/
-          curl -S -s http://${{ env.MAIN_SERVICE_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_IP }}/
-          curl -S -s http://${{ env.MAIN_SERVICE_ENDPOINT }}/client-call/
+          curl -S -s http://${{ env.MAIN_SERVICE_ENDPOINT }}/outgoing-http-call
+          curl -S -s http://${{ env.MAIN_SERVICE_ENDPOINT }}/aws-sdk-call?testingId=${{ env.TESTING_ID }}
+          curl -S -s http://${{ env.MAIN_SERVICE_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_IP }}
+          curl -S -s http://${{ env.MAIN_SERVICE_ENDPOINT }}/client-call
 
       - name: Initiate Gradlew Daemon
         uses: ./.github/workflows/actions/execute_and_retry
@@ -191,7 +191,7 @@ jobs:
           --log-group ${{ env.LOG_GROUP_NAME }}
           --service-name sample-application-${{ env.TESTING_ID }}
           --remote-service-name sample-remote-application-${{ env.TESTING_ID }}
-          --request-body ip=${{ env.REMOTE_SERVICE_IP }}
+          --query-string ip=${{ env.REMOTE_SERVICE_IP }}
           --instance-ami ${{ env.EC2_INSTANCE_AMI }}
           --rollup'
 
@@ -208,7 +208,7 @@ jobs:
           --log-group ${{ env.LOG_GROUP_NAME }}
           --service-name sample-application-${{ env.TESTING_ID }}
           --remote-service-name sample-remote-application-${{ env.TESTING_ID }}
-          --request-body ip=${{ env.REMOTE_SERVICE_IP }}
+          --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}
           --instance-ami ${{ env.EC2_INSTANCE_AMI }}
           --rollup'
 
@@ -225,7 +225,7 @@ jobs:
           --log-group ${{ env.LOG_GROUP_NAME }}
           --service-name sample-application-${{ env.TESTING_ID }}
           --remote-service-name sample-remote-application-${{ env.TESTING_ID }}
-          --request-body ip=${{ env.REMOTE_SERVICE_IP }}
+          --query-string ip=${{ env.REMOTE_SERVICE_IP }}
           --instance-ami ${{ env.EC2_INSTANCE_AMI }}
           --rollup'
 

diff --git a/.github/workflows/appsignals-e2e-eks-test.yml b/.github/workflows/appsignals-e2e-eks-test.yml
@@ -261,10 +261,10 @@ jobs:
       - name: Call all test APIs
         continue-on-error: true
         run: |
-          curl -S -s http://${{ env.APP_ENDPOINT }}/outgoing-http-call/
-          curl -S -s http://${{ env.APP_ENDPOINT }}/aws-sdk-call/
-          curl -S -s http://${{ env.APP_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_POD_IP }}/
-          curl -S -s http://${{ env.APP_ENDPOINT }}/client-call/
+          curl -S -s http://${{ env.APP_ENDPOINT }}/outgoing-http-call
+          curl -S -s http://${{ env.APP_ENDPOINT }}/aws-sdk-call?testingId=${{ env.TESTING_ID }}
+          curl -S -s http://${{ env.APP_ENDPOINT }}/remote-service?ip=${{ env.REMOTE_SERVICE_POD_IP }}
+          curl -S -s http://${{ env.APP_ENDPOINT }}/client-call
 
       - name: Initiate Gradlew Daemon
         uses: ./.github/workflows/actions/execute_and_retry
@@ -289,7 +289,7 @@ jobs:
           --platform-info ${{ inputs.test-cluster-name }}
           --service-name sample-application-${{ env.TESTING_ID }}
           --remote-service-deployment-name ${{ env.REMOTE_SERVICE_DEPLOYMENT_NAME }}
-          --request-body ip=${{ env.REMOTE_SERVICE_POD_IP }}
+          --query-string ip=${{ env.REMOTE_SERVICE_POD_IP }}
           --rollup'
 
       - name: Call endpoints and validate generated metrics
@@ -307,7 +307,7 @@ jobs:
           --service-name sample-application-${{ env.TESTING_ID }}
           --remote-service-name sample-remote-application-${{ env.TESTING_ID }}
           --remote-service-deployment-name ${{ env.REMOTE_SERVICE_DEPLOYMENT_NAME }}
-          --request-body ip=${{ env.REMOTE_SERVICE_POD_IP }}
+          --query-string ip=${{ env.REMOTE_SERVICE_POD_IP }}&testingId=${{ env.TESTING_ID }}
           --rollup'
 
       - name: Call endpoints and validate generated traces
@@ -324,7 +324,7 @@ jobs:
           --platform-info ${{ inputs.test-cluster-name }}
           --service-name sample-application-${{ env.TESTING_ID }}
           --remote-service-deployment-name ${{ env.REMOTE_SERVICE_DEPLOYMENT_NAME }}
-          --request-body ip=${{ env.REMOTE_SERVICE_POD_IP }}
+          --query-string ip=${{ env.REMOTE_SERVICE_POD_IP }}
           --rollup'
 
       - name: Publish metric on test result

diff --git a/.github/workflows/appsignals-e2e-k8s-test.yml b/.github/workflows/appsignals-e2e-k8s-test.yml
@@ -120,7 +120,7 @@ jobs:
           --app-namespace ${{ env.SAMPLE_APP_NAMESPACE }}
           --service-name sample-application-${{ env.TESTING_ID }}
           --remote-service-name sample-r-app-deployment-${{ env.TESTING_ID }}
-          --request-body ip=${{ env.REMOTE_SERVICE_IP }}
+          --query-string ip=${{ env.REMOTE_SERVICE_IP }}
           --rollup'
 
       - name: Validate generated metrics
@@ -138,7 +138,7 @@ jobs:
           --service-name sample-application-${{ env.TESTING_ID }}
           --remote-service-name sample-r-app-deployment-${{ env.TESTING_ID }}
           --remote-service-deployment-name sample-r-app-deployment-${{ env.TESTING_ID }}
-          --request-body ip=${{ env.REMOTE_SERVICE_IP }}
+          --query-string ip=${{ env.REMOTE_SERVICE_IP }}
           --rollup'
 
       - name: Validate generated traces
@@ -156,7 +156,7 @@ jobs:
           --service-name sample-application-${{ env.TESTING_ID }}
           --remote-service-name sample-r-app-deployment-${{ env.TESTING_ID }}
           --remote-service-deployment-name sample-r-app-deployment-${{ env.TESTING_ID }}
-          --request-body ip=${{ env.REMOTE_SERVICE_IP }}
+          --query-string ip=${{ env.REMOTE_SERVICE_IP }}
           --rollup'
 
       - name: Publish metric on test result

diff --git a/terraform/ec2/main.tf b/terraform/ec2/main.tf
@@ -177,7 +177,7 @@ resource "null_resource" "remote_service_setup" {
       "echo $agent_config > amazon-cloudwatch-agent.json",
 
       # Get and run CW agent rpm
-       "${var.get_cw_agent_rpm_command}",
+      "${var.get_cw_agent_rpm_command}",
       "sudo rpm -U ./cw-agent.rpm",
       "sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:./amazon-cloudwatch-agent.json",
 

diff --git a/validator/src/main/java/com/amazon/aoc/App.java b/validator/src/main/java/com/amazon/aoc/App.java
@@ -78,8 +78,8 @@ public class App implements Callable<Integer> {
   @CommandLine.Option(names = {"--endpoint"})
   private String endpoint;
 
-  @CommandLine.Option(names = {"--request-body"})
-  private String requestBody;
+  @CommandLine.Option(names = {"--query-string"})
+  private String queryString;
 
   @CommandLine.Option(
       names = {"--log-group"},
@@ -161,7 +161,7 @@ public Integer call() throws Exception {
     context.setRemoteServiceName(this.remoteServiceName);
     context.setRemoteServiceDeploymentName(this.remoteServiceDeploymentName);
     context.setEndpoint(this.endpoint);
-    context.setRequestBody(this.requestBody);
+    context.setQueryString(this.queryString);
     context.setLogGroup(this.logGroup);
     context.setEcsContext(buildJsonContext(ecsContext, ECSContext.class));
     context.setEc2Context(buildJsonContext(ec2Context, EC2Context.class));

diff --git a/validator/src/main/java/com/amazon/aoc/callers/HttpCaller.java b/validator/src/main/java/com/amazon/aoc/callers/HttpCaller.java
@@ -32,7 +32,7 @@
 public class HttpCaller implements ICaller {
   private String url;
   private String path;
-  private String requestBody;
+  private String queryString;
 
   /**
    * construct httpCaller.
@@ -51,11 +51,11 @@ public HttpCaller(String endpoint, String path) {
    *
    * @param endpoint the endpoint to call, for example "http://127.0.0.1:8080"
    * @param path the path to call, for example "/test"
-   * @param requestBody the request body, for example "key=value"
+   * @param queryString the query string, for example "key1=value1&key2=value2"
    */
-  public HttpCaller(String endpoint, String path, String requestBody) {
+  public HttpCaller(String endpoint, String path, String queryString) {
     this.path = path;
-    this.url = endpoint + path + "?" + requestBody + "/";
+    this.url = endpoint + path + "?" + queryString;
     log.info("validator is trying to hit this {} endpoint", this.url);
   }
 

diff --git a/validator/src/main/java/com/amazon/aoc/models/Context.java b/validator/src/main/java/com/amazon/aoc/models/Context.java
@@ -51,7 +51,7 @@ public class Context {
 
   private String endpoint;
 
-  private String requestBody;
+  private String queryString;
 
   private String logGroup;
 

diff --git a/validator/src/main/java/com/amazon/aoc/models/ValidationConfig.java b/validator/src/main/java/com/amazon/aoc/models/ValidationConfig.java
@@ -28,7 +28,7 @@ public class ValidationConfig {
 
   String httpPath;
   String httpMethod;
-  String requestBody;
+  String queryString;
 
   String expectedResultPath;
   Boolean shouldValidateMetricValue;

diff --git a/validator/src/main/java/com/amazon/aoc/services/CloudWatchService.java b/validator/src/main/java/com/amazon/aoc/services/CloudWatchService.java
@@ -35,6 +35,7 @@
 public class CloudWatchService {
   public static final String SERVICE_DIMENSION = "Service";
   public static final String REMOTE_SERVICE_DIMENSION = "RemoteService";
+  public static final String REMOTE_TARGET_DIMENSION = "RemoteTarget";
 
   private static final int MAX_QUERY_PERIOD = 60;
   private static final String REQUESTER = "integrationTest";

diff --git a/validator/src/main/java/com/amazon/aoc/validators/CWMetricValidator.java b/validator/src/main/java/com/amazon/aoc/validators/CWMetricValidator.java
@@ -90,17 +90,20 @@ public void validate() throws Exception {
     RetryHelper.retry(
         maxRetryCount,
         () -> {
-          // We will query both the Service and RemoteService dimensions to ensure we get all
-          // metrics from all aggregations, specifically the [RemoteService] aggregation.
+          // We will query the Service, RemoteService, and RemoteTarget dimensions to ensure we
+          // get all metrics from all aggregations, specifically the [RemoteService] aggregation.
           List<String> serviceNames =
               Lists.newArrayList(
                   context.getServiceName(), context.getRemoteServiceDeploymentName());
-          // TODO - Put the following back in: "www.amazon.com",  "AWS.SDK.S3"
           List<String> remoteServiceNames =
               Lists.newArrayList(context.getRemoteServiceDeploymentName());
+          List<String> remoteTargetNames = Lists.newArrayList();
           if (context.getRemoteServiceName() != null && !context.getRemoteServiceName().isEmpty()) {
             serviceNames.add(context.getRemoteServiceName());
           }
+          if (context.getTestingId() != null && !context.getTestingId().isEmpty()) {
+            remoteTargetNames.add("::s3:::e2e-test-bucket-name-" + context.getTestingId());
+          }
 
           List<Metric> actualMetricList = Lists.newArrayList();
           addMetrics(
@@ -113,6 +116,11 @@ public void validate() throws Exception {
               remoteServiceNames,
               expectedMetricList,
               actualMetricList);
+          addMetrics(
+              CloudWatchService.REMOTE_TARGET_DIMENSION,
+              remoteTargetNames,
+              expectedMetricList,
+              actualMetricList);
 
           // remove the skip dimensions
           log.info("dimensions to be skipped in validation: {}", skippedDimensionNameList);

diff --git a/validator/src/main/java/com/amazon/aoc/validators/ValidatorFactory.java b/validator/src/main/java/com/amazon/aoc/validators/ValidatorFactory.java
@@ -64,11 +64,11 @@ public IValidator launchValidator(ValidationConfig validationConfig) throws Exce
       case "http":
         caller = new HttpCaller(context.getEndpoint(), validationConfig.getHttpPath());
         break;
-      case "http-with-body":
+      case "http-with-query":
         // ONLY ONE OF THESE CAN BE USED PER VALIDATOR CALL
         caller =
             new HttpCaller(
-                context.getEndpoint(), validationConfig.getHttpPath(), context.getRequestBody());
+                context.getEndpoint(), validationConfig.getHttpPath(), context.getQueryString());
         break;
       case "none":
         caller = null;