Use localstack for AWS testing

.github/workflows/ci_main.yml

@@ -33,15 +33,30 @@ jobs:
         id: project-version
         uses: ./.github/actions/project-version
 
+      - name: Start LocalStack (AWS)
+        id: localstack
+        uses: LocalStack/setup-localstack@v0.2.2
+        with:
+          image-tag: 'latest'
+          env:
+            SERVICES=secretsmanager,kms 
+            EAGER_SERVICE_LOADING=1
+            AWS_ACCESS_KEY_ID=test
+            AWS_SECRET_ACCESS_KEY=test
+            AWS_DEFAULT_REGION=us-east-2
+            AWS_SESSION_TOKEN=test
+            AWS_SESSION_TOKEN=test
+
       - name: Build and Test
         id: build-test
         uses: ./.github/actions/build-test
+        need: localstack
         env:
-          AWS_REGION: ${{ secrets.AWS_REGION }}
-          AWS_ACCESS_KEY_ID: ${{ secrets.RO_AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.RO_AWS_SECRET_ACCESS_KEY }}
-          RW_AWS_ACCESS_KEY_ID: ${{ secrets.RW_AWS_ACCESS_KEY_ID }}
-          RW_AWS_SECRET_ACCESS_KEY: ${{ secrets.RW_AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: 'us-east-2'
+          AWS_ACCESS_KEY_ID: 'test'
+          AWS_SECRET_ACCESS_KEY: 'test'
+          RW_AWS_ACCESS_KEY_ID: 'test'
+          RW_AWS_SECRET_ACCESS_KEY: 'test'
           AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
           AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
           AZURE_INVALID_KEY_VAULT_NAME: ${{ secrets.AZURE_INVALID_KEY_VAULT_NAME }}

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/utils/MetadataFileHelpers.java

@@ -233,7 +233,8 @@ public void createAwsYamlFileAt(
       final String awsRegion,
       final String accessKeyId,
       final String secretAccessKey,
-      final String secretName) {
+      final String secretName,
+      final Optional<URI> awsEndpointOverride) {
     try {
       final Map<String, String> signingMetadata = new HashMap<>();
 
@@ -243,23 +244,29 @@ public void createAwsYamlFileAt(
       signingMetadata.put("accessKeyId", accessKeyId);
       signingMetadata.put("secretAccessKey", secretAccessKey);
       signingMetadata.put("secretName", secretName);
+      awsEndpointOverride.ifPresent(
+          endpoint -> signingMetadata.put("endpointOverride", endpoint.toString()));
 
       createYamlFile(metadataFilePath, signingMetadata);
     } catch (final Exception e) {
       throw new RuntimeException("Unable to construct aws yaml file", e);
     }
   }
 
-  public void createAwsYamlFileAt(
-      final Path metadataFilePath, final String awsRegion, final String secretName) {
+  public void createAwsYamlFileWithEnvironmentAt(
+      final Path metadataFilePath,
+      final String awsRegion,
+      final String secretName,
+      final Optional<URI> awsEndpointOverride) {
     try {
       final Map<String, String> signingMetadata = new HashMap<>();
 
       signingMetadata.put("type", "aws-secret");
       signingMetadata.put("authenticationMode", "ENVIRONMENT");
       signingMetadata.put("region", awsRegion);
       signingMetadata.put("secretName", secretName);
-
+      awsEndpointOverride.ifPresent(
+          endpoint -> signingMetadata.put("endpointOverride", endpoint.toString()));
       createYamlFile(metadataFilePath, signingMetadata);
     } catch (final Exception e) {
       throw new RuntimeException("Unable to construct aws yaml file", e);

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/publickeys/AwsKeyIdentifiersAcceptanceTest.java

@@ -94,18 +94,20 @@ public void specifiedAwsKeysReturnAppropriatePublicKey() {
         AWS_REGION,
         RO_AWS_ACCESS_KEY_ID,
         RO_AWS_SECRET_ACCESS_KEY,
-        awsSecretsManagerUtil.getSecretsManagerPrefix() + publicKey);
+        awsSecretsManagerUtil.getSecretsManagerPrefix() + publicKey,
+        awsEndpointOverride);
     initAndStartSigner("eth2");
     final Response response = callApiPublicKeysWithoutOpenApiClientSideFilter(BLS);
     validateApiResponse(response, containsInAnyOrder(publicKey));
   }
 
   @Test
   public void environmentAwsKeysReturnAppropriatePublicKey() {
-    METADATA_FILE_HELPERS.createAwsYamlFileAt(
+    METADATA_FILE_HELPERS.createAwsYamlFileWithEnvironmentAt(
         testDirectory.resolve(publicKey + ".yaml"),
         AWS_REGION,
-        awsSecretsManagerUtil.getSecretsManagerPrefix() + publicKey);
+        awsSecretsManagerUtil.getSecretsManagerPrefix() + publicKey,
+        awsEndpointOverride);
     initAndStartSigner("eth2");
     final Response response = callApiPublicKeysWithoutOpenApiClientSideFilter(BLS);
     validateApiResponse(response, containsInAnyOrder(publicKey));

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/signing/BlsSigningAcceptanceTest.java

@@ -223,7 +223,12 @@ public void ableToSignUsingAws() throws JsonProcessingException {
     final Path keyConfigFile = testDirectory.resolve(configFilename + ".yaml");
     try {
       METADATA_FILE_HELPERS.createAwsYamlFileAt(
-          keyConfigFile, region, roAwsAccessKeyId, roAwsSecretAccessKey, fullyPrefixKeyName);
+          keyConfigFile,
+          region,
+          roAwsAccessKeyId,
+          roAwsSecretAccessKey,
+          fullyPrefixKeyName,
+          awsEndpointOverride);
 
       signAndVerifySignature(ArtifactType.BLOCK);
     } finally {

localstack/README.md

@@ -0,0 +1,36 @@
+# Localstack for AWS services testing
+
+The AWS related unit, integration and acceptance tests can be tested against localstack.
+Use following command to start localstack services.
+
+```bash
+docker compose up
+```
+
+Use `CTRL-C` or run `docker compose down` from the same directory but from another shell instance.
+
+Export following environment variables before running Web3Signer tests.
+
+```bash 
+export RW_AWS_ACCESS_KEY_ID=test
+export RW_AWS_SECRET_ACCESS_KEY=test
+export AWS_ACCESS_KEY_ID=test
+export AWS_SECRET_ACCESS_KEY=test
+export AWS_REGION=us-east-2
+export AWS_ENDPOINT_OVERRIDE=http://127.0.0.1:4566
+```
+
+To import above in IntelliJ IDEA Run configurations:
+```
+RW_AWS_ACCESS_KEY_ID=test
+RW_AWS_SECRET_ACCESS_KEY=test
+AWS_ACCESS_KEY_ID=test
+AWS_SECRET_ACCESS_KEY=test
+AWS_REGION=us-east-2
+AWS_ENDPOINT_OVERRIDE=http://127.0.0.1:4566
+```
+
+Run gradle tests.
+```bash
+
+```

localstack/compose.yml

@@ -0,0 +1,26 @@
+services:
+  # see https://docs.localstack.cloud/getting-started/installation/#docker-compose
+  # see https://docs.localstack.cloud/references/configuration/
+  localstack:
+    container_name: "${LOCALSTACK_DOCKER_NAME-localstack_main}"
+    image: localstack/localstack
+    ports:
+      - "127.0.0.1:4566:4566"            # LocalStack Gateway
+      - "127.0.0.1:4510-4559:4510-4559"  # external services port range
+    environment:
+      # LocalStack configuration
+      - DEBUG=${DEBUG-}
+      - DOCKER_HOST=unix:///var/run/docker.sock
+      - SERVICES=secretsmanager,kms
+      - EAGER_SERVICE_LOADING=1
+      # AWS Configuration
+      - AWS_ACCESS_KEY_ID=test
+      - AWS_SECRET_ACCESS_KEY=test
+      - AWS_DEFAULT_REGION=us-east-2
+      - AWS_SESSION_TOKEN=test
+      - AWS_SESSION_TOKEN=test
+      # ready hook script configuration
+    volumes:
+      #- "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
+      #- "./init/ready.d:/etc/localstack/init/ready.d" # ready hooks
+      - "${DOCKER_HOST_PATH:-/var/run/docker.sock}:/var/run/docker.sock"