Add region, cleanup, README

Signed-off-by: Mikayla Thompson <thomika@amazon.com>

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/README.md

@@ -0,0 +1,80 @@
+# Console_link Library
+
+The console link library is designed to provide a unified interface for the many possible backend services involved in a migration. The interface can be used by multiple frontends--a CLI app and a web API, for instance.
+
+![alt text](diagram.png)
+
+
+The user defines their migration services in a `migration_services.yaml` file, by default found at `/etc/migration_services.yaml`.
+
+Currently the supported services are a source and target cluster and a metrics source. For example:
+
+```yaml
+source_cluster:
+	endpoint: "https://capture-proxy-es:9200"
+	allow_insecure: true
+target_cluster:
+	endpoint: "https://opensearchtarget:9200"
+	allow_insecure: true
+	authorization:
+		type: "basic"
+		details:
+			username: "admin"
+			password: "myStrongPassword123!"
+metrics_source:
+	type: "prometheus"
+	endpoint: "http://prometheus:9090"
+```
+
+### Services.yaml spec
+
+#### Cluster
+
+Source and target clusters have the following options:
+- `endpoint`: required, the endpoint to reach the cluster.
+- `authorization`: optional, if it is provided, type is required.
+	- `type`: required, the only currently implemented option is "basic", but "sigv4" should be available soon
+	- `details`: for basic auth, the details should be a `username` and `password`
+
+Having a `source_cluster` and `target_cluster` is required.
+
+#### Metrics Source
+
+Currently, the two supported metrics source types are `prometheus` and `cloudwatch`.
+
+- `type`: required, `prometheus` or `cloudwatch`
+- `endpoint`: required for `prometheus` (ignored for `cloudwatch`)
+- `region`: optional for `cloudwatch` (ignored for `prometheus`). if not provided, the usual rules are followed for determining aws region (`AWS_DEFAULT_REGION`, `~/.aws/config`)
+
+# Usage
+### Library
+The library can be imported and used within another application.
+Use `pip install .` from the top-level `console_link` directory to install it locally and then import it as, e.g. `from console_link.models.metrics_source import MetricsSource`
+
+#### CLI
+The CLI comes installed on the migration console. If you'd like to install it elsewhere, `pip install .` from the top-level `console_link` directory will install it and setup a `console` executable to access it.
+
+Autocomplete can be enabled by adding `eval "$(_CONSOLE_COMPLETE=bash_source console)"` to your `.bashrc` file, or `eval "$(_FOO_BAR_COMPLETE=zsh_source foo-bar)"` to your `.zshrc` and re-sourcing your shell.
+
+The structure of cli commands is:
+`console [--global-options] OBJECT COMMAND [--options]`
+
+##### Global Options
+The available global options are:
+- `--config-file FILE` to specify the path to a config file (default is `/etc/migration_services.yaml`)
+- `--json` to get output in JSON designed for machine consumption instead of printing to the console
+
+##### Objects
+Currently, the two objects available are `cluster` and `metrics`.
+
+##### Commands & options
+Each object has its own commands available, and each command has its own options. To see the available commands and options, use:
+```
+console OBJECT --help
+```
+
+## Testing
+```
+pip install -r tests/requirements.txt
+pytest
+```

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/cli.py

@@ -1,7 +1,8 @@
 import json
 from pprint import pprint
 import click
-import console_link.logic as logic
+import console_link.logic.clusters as logic_clusters
+import console_link.logic.metrics as logic_metrics
 from console_link.logic.instantiation import Environment
 from console_link.models.metrics_source import Component, MetricStatistic
 
@@ -47,20 +48,20 @@ def cat_indices_cmd(ctx):
         click.echo(
             json.dumps(
                 {
-                    "source_cluster": logic.clusters.cat_indices(
+                    "source_cluster": logic_clusters.cat_indices(
                         ctx.env.source_cluster, as_json=True
                     ),
-                    "target_cluster": logic.clusters.cat_indices(
+                    "target_cluster": logic_clusters.cat_indices(
                         ctx.env.target_cluster, as_json=True
                     ),
                 }
             )
         )
         return
     click.echo("SOURCE CLUSTER")
-    click.echo(logic.clusters.cat_indices(ctx.env.source_cluster))
+    click.echo(logic_clusters.cat_indices(ctx.env.source_cluster))
     click.echo("TARGET CLUSTER")
-    click.echo(logic.clusters.cat_indices(ctx.env.target_cluster))
+    click.echo(logic_clusters.cat_indices(ctx.env.target_cluster))
     pass
 
 
@@ -77,7 +78,7 @@ def replayer_group(ctx):
 @replayer_group.command(name="start")
 @click.pass_obj
 def start_replayer_cmd(ctx):
-    logic.services.start_replayer(ctx.env.replayer)
+    ctx.env.replayer.start()
 
 
 # ##################### METRICS ###################
@@ -109,7 +110,7 @@ def list_metrics_cmd(ctx):
 @click.option("--lookback", type=int, default=60, help="Lookback in minutes")
 @click.pass_obj
 def get_metrics_data_cmd(ctx, component, metric_name, statistic, lookback):
-    metric_data = logic.metrics.get_metric_data(
+    metric_data = logic_metrics.get_metric_data(
         ctx.env.metrics_source,
         component,
         metric_name,

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/logic/__init__.py

@@ -1,2 +0,0 @@
-import console_link.logic.clusters  # noqa: F401
-import console_link.logic.metrics  # noqa: F401

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/logic/utils.py

@@ -3,7 +3,8 @@
 
 
 class AWSAPIError(Exception):
-    pass
+    def __init__(self, message, status_code=None):
+        super().__init__("Error encountered calling an AWS API", message, status_code)
 
 
 def raise_for_aws_api_error(response: Dict) -> None:
@@ -13,10 +14,9 @@ def raise_for_aws_api_error(response: Dict) -> None:
     ):
         status_code = response["ResponseMetadata"]["HTTPStatusCode"]
     else:
+        raise AWSAPIError("ResponseMetadata was not found in the response")
+    if status_code not in range(200, 300):
         raise AWSAPIError(
-            "Error listing metrics from Cloudwatch"
-        )  # TODO: handle this better
-    if status_code != 200:
-        raise AWSAPIError(
-            "Error listing metrics from Cloudwatch"
-        )  # TODO: handle this better
+            "Non-2XX status code received",
+            status_code=status_code
+        )

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/cluster.py

@@ -47,7 +47,6 @@ def __init__(self, config: Dict) -> None:
         if "authorization" in config:
             self.auth_type = AuthMethod[config["authorization"]["type"].upper()]
             self.auth_details = config["authorization"]["details"]
-        pass
 
     def call_api(self, path, method: HttpMethod = HttpMethod.GET) -> requests.Response:
         """

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/metrics_source.py

@@ -3,6 +3,7 @@
 from typing import Any, Dict, List, Optional, Tuple
 
 import boto3
+import botocore
 from cerberus import Validator
 from console_link.logic.utils import raise_for_aws_api_error
 import requests
@@ -34,6 +35,10 @@ class Component(Enum):
         "type": "string",
         "required": False,
     },
+    "region": {
+        "type": "string",
+        "required": False
+    },
 }
 
 PROMETHEUS_SCHEMA = {k: v.copy() for k, v in SCHEMA.items()}
@@ -106,10 +111,16 @@ def __init__(self, list_metric_data: Dict[str, Any]):
 
 
 class CloudwatchMetricsSource(MetricsSource):
-    client = boto3.client("cloudwatch")
-
     def __init__(self, config: Dict) -> None:
         super().__init__(config)
+        if "region" in config:
+            self.region = config["region"]
+            self.boto_config = botocore.config.Config(region_name=self.region)
+            print("overriding client with a region-specific one")
+        else:
+            self.region = None
+            self.boto_config = None
+        self.client = boto3.client("cloudwatch", config=self.boto_config)
 
     def get_metrics(self, recent=True) -> Dict[str, List[str]]:
         response = self.client.list_metrics(  # TODO: implement pagination

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_metrics_source.py

@@ -1,9 +1,8 @@
 import json
 
 import requests
-from console_link.models.metrics_source import MetricsSource, CloudwatchMetricsSource, PrometheusMetricsSource
-from console_link.models.metrics_source import MetricStatistic, get_metrics_source, Component
-from console_link.models.metrics_source import UnsupportedMetricsSourceError
+from console_link.models.metrics_source import MetricsSource, CloudwatchMetricsSource, PrometheusMetricsSource, \
+    MetricStatistic, get_metrics_source, Component, UnsupportedMetricsSourceError
 import pytest
 import requests_mock