diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/README.md b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/README.md new file mode 100644 index 000000000..dccd1f28b --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/README.md @@ -0,0 +1,80 @@ +# Console_link Library + +The console link library is designed to provide a unified interface for the many possible backend services involved in a migration. The interface can be used by multiple frontends--a CLI app and a web API, for instance. + +![alt text](diagram.png) + + +The user defines their migration services in a `migration_services.yaml` file, by default found at `/etc/migration_services.yaml`. + +Currently the supported services are a source and target cluster and a metrics source. For example: + +```yaml +source_cluster: + endpoint: "https://capture-proxy-es:9200" + allow_insecure: true +target_cluster: + endpoint: "https://opensearchtarget:9200" + allow_insecure: true + authorization: + type: "basic" + details: + username: "admin" + password: "myStrongPassword123!" +metrics_source: + type: "prometheus" + endpoint: "http://prometheus:9090" +``` + +### Services.yaml spec + +#### Cluster + +Source and target clusters have the following options: +- `endpoint`: required, the endpoint to reach the cluster. +- `authorization`: optional, if it is provided, type is required. + - `type`: required, the only currently implemented option is "basic", but "sigv4" should be available soon + - `details`: for basic auth, the details should be a `username` and `password` + +Having a `source_cluster` and `target_cluster` is required. + +#### Metrics Source + +Currently, the two supported metrics source types are `prometheus` and `cloudwatch`. + +- `type`: required, `prometheus` or `cloudwatch` +- `endpoint`: required for `prometheus` (ignored for `cloudwatch`) +- `region`: optional for `cloudwatch` (ignored for `prometheus`). if not provided, the usual rules are followed for determining aws region (`AWS_DEFAULT_REGION`, `~/.aws/config`) + +# Usage +### Library +The library can be imported and used within another application. +Use `pip install .` from the top-level `console_link` directory to install it locally and then import it as, e.g. `from console_link.models.metrics_source import MetricsSource` + +#### CLI +The CLI comes installed on the migration console. If you'd like to install it elsewhere, `pip install .` from the top-level `console_link` directory will install it and setup a `console` executable to access it. + +Autocomplete can be enabled by adding `eval "$(_CONSOLE_COMPLETE=bash_source console)"` to your `.bashrc` file, or `eval "$(_FOO_BAR_COMPLETE=zsh_source foo-bar)"` to your `.zshrc` and re-sourcing your shell. + +The structure of cli commands is: +`console [--global-options] OBJECT COMMAND [--options]` + +##### Global Options +The available global options are: +- `--config-file FILE` to specify the path to a config file (default is `/etc/migration_services.yaml`) +- `--json` to get output in JSON designed for machine consumption instead of printing to the console + +##### Objects +Currently, the two objects available are `cluster` and `metrics`. + +##### Commands & options +Each object has its own commands available, and each command has its own options. To see the available commands and options, use: +``` +console OBJECT --help +``` + +## Testing +``` +pip install -r tests/requirements.txt +pytest +``` diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/cli.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/cli.py index 8f80b04e6..22f5714ca 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/cli.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/cli.py @@ -1,7 +1,8 @@ import json from pprint import pprint import click -import console_link.logic as logic +import console_link.logic.clusters as logic_clusters +import console_link.logic.metrics as logic_metrics from console_link.logic.instantiation import Environment from console_link.models.metrics_source import Component, MetricStatistic @@ -47,10 +48,10 @@ def cat_indices_cmd(ctx): click.echo( json.dumps( { - "source_cluster": logic.clusters.cat_indices( + "source_cluster": logic_clusters.cat_indices( ctx.env.source_cluster, as_json=True ), - "target_cluster": logic.clusters.cat_indices( + "target_cluster": logic_clusters.cat_indices( ctx.env.target_cluster, as_json=True ), } @@ -58,9 +59,9 @@ def cat_indices_cmd(ctx): ) return click.echo("SOURCE CLUSTER") - click.echo(logic.clusters.cat_indices(ctx.env.source_cluster)) + click.echo(logic_clusters.cat_indices(ctx.env.source_cluster)) click.echo("TARGET CLUSTER") - click.echo(logic.clusters.cat_indices(ctx.env.target_cluster)) + click.echo(logic_clusters.cat_indices(ctx.env.target_cluster)) pass @@ -77,7 +78,7 @@ def replayer_group(ctx): @replayer_group.command(name="start") @click.pass_obj def start_replayer_cmd(ctx): - logic.services.start_replayer(ctx.env.replayer) + ctx.env.replayer.start() # ##################### METRICS ################### @@ -109,7 +110,7 @@ def list_metrics_cmd(ctx): @click.option("--lookback", type=int, default=60, help="Lookback in minutes") @click.pass_obj def get_metrics_data_cmd(ctx, component, metric_name, statistic, lookback): - metric_data = logic.metrics.get_metric_data( + metric_data = logic_metrics.get_metric_data( ctx.env.metrics_source, component, metric_name, diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/logic/__init__.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/logic/__init__.py index 96e95c0a9..e69de29bb 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/logic/__init__.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/logic/__init__.py @@ -1,2 +0,0 @@ -import console_link.logic.clusters # noqa: F401 -import console_link.logic.metrics # noqa: F401 diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/logic/utils.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/logic/utils.py index 2eca290f3..7af59c925 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/logic/utils.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/logic/utils.py @@ -3,7 +3,8 @@ class AWSAPIError(Exception): - pass + def __init__(self, message, status_code=None): + super().__init__("Error encountered calling an AWS API", message, status_code) def raise_for_aws_api_error(response: Dict) -> None: @@ -13,10 +14,9 @@ def raise_for_aws_api_error(response: Dict) -> None: ): status_code = response["ResponseMetadata"]["HTTPStatusCode"] else: + raise AWSAPIError("ResponseMetadata was not found in the response") + if status_code not in range(200, 300): raise AWSAPIError( - "Error listing metrics from Cloudwatch" - ) # TODO: handle this better - if status_code != 200: - raise AWSAPIError( - "Error listing metrics from Cloudwatch" - ) # TODO: handle this better + "Non-2XX status code received", + status_code=status_code + ) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/cluster.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/cluster.py index 8408d74da..433cab7a4 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/cluster.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/cluster.py @@ -47,7 +47,6 @@ def __init__(self, config: Dict) -> None: if "authorization" in config: self.auth_type = AuthMethod[config["authorization"]["type"].upper()] self.auth_details = config["authorization"]["details"] - pass def call_api(self, path, method: HttpMethod = HttpMethod.GET) -> requests.Response: """ diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/metrics_source.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/metrics_source.py index d458ddc2c..87a204a19 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/metrics_source.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/metrics_source.py @@ -3,6 +3,7 @@ from typing import Any, Dict, List, Optional, Tuple import boto3 +import botocore from cerberus import Validator from console_link.logic.utils import raise_for_aws_api_error import requests @@ -34,6 +35,10 @@ class Component(Enum): "type": "string", "required": False, }, + "region": { + "type": "string", + "required": False + }, } PROMETHEUS_SCHEMA = {k: v.copy() for k, v in SCHEMA.items()} @@ -106,10 +111,16 @@ def __init__(self, list_metric_data: Dict[str, Any]): class CloudwatchMetricsSource(MetricsSource): - client = boto3.client("cloudwatch") - def __init__(self, config: Dict) -> None: super().__init__(config) + if "region" in config: + self.region = config["region"] + self.boto_config = botocore.config.Config(region_name=self.region) + print("overriding client with a region-specific one") + else: + self.region = None + self.boto_config = None + self.client = boto3.client("cloudwatch", config=self.boto_config) def get_metrics(self, recent=True) -> Dict[str, List[str]]: response = self.client.list_metrics( # TODO: implement pagination diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/diagram.png b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/diagram.png new file mode 100644 index 000000000..da6d3127c Binary files /dev/null and b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/diagram.png differ diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_metrics_source.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_metrics_source.py index d5a57e612..65af5b691 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_metrics_source.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_metrics_source.py @@ -1,9 +1,8 @@ import json import requests -from console_link.models.metrics_source import MetricsSource, CloudwatchMetricsSource, PrometheusMetricsSource -from console_link.models.metrics_source import MetricStatistic, get_metrics_source, Component -from console_link.models.metrics_source import UnsupportedMetricsSourceError +from console_link.models.metrics_source import MetricsSource, CloudwatchMetricsSource, PrometheusMetricsSource, \ + MetricStatistic, get_metrics_source, Component, UnsupportedMetricsSourceError import pytest import requests_mock