From 84b61193aedd1ec848faf44da183641663ccf820 Mon Sep 17 00:00:00 2001 From: Mikayla Thompson Date: Mon, 13 May 2024 14:43:53 -0600 Subject: [PATCH] Add rough script for setting up & running a demo Signed-off-by: Mikayla Thompson --- test/awsE2ESolutionSetup.sh | 3 +- test/demo.sh | 125 ++++++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 test/demo.sh diff --git a/test/awsE2ESolutionSetup.sh b/test/awsE2ESolutionSetup.sh index d48ab665f..24f00ef34 100755 --- a/test/awsE2ESolutionSetup.sh +++ b/test/awsE2ESolutionSetup.sh @@ -73,8 +73,9 @@ restore_and_record () { unbuffer aws ecs execute-command --cluster "migration-${STAGE}-ecs-cluster" --task "${console_task_arn}" --container "migration-console" --interactive --command "./catIndices.sh --source-endpoint $source_endpoint --source-no-auth --target-no-auth" unbuffer aws ecs execute-command --cluster "migration-${STAGE}-ecs-cluster" --task "${console_task_arn}" --container "migration-console" --interactive --command "./kafka-tools/kafka/bin/kafka-consumer-groups.sh --bootstrap-server ${kafka_brokers} --timeout 100000 --describe --group logging-group-default --command-config kafka-tools/aws/msk-iam-auth.properties" - # Turn off Replayer + # Turn off Replayer and RFS aws ecs update-service --cluster "migration-${deploy_stage}-ecs-cluster" --service "migration-${deploy_stage}-traffic-replayer-default" --desired-count 0 > /dev/null 2>&1 + aws ecs update-service --cluster "migration-${deploy_stage}-ecs-cluster" --service "migration-${deploy_stage}-reindex-from-snapshot" --desired-count 0 > /dev/null 2>&1 } # One-time required service-linked-role creation for AWS accounts which do not have these roles diff --git a/test/demo.sh b/test/demo.sh new file mode 100644 index 000000000..4f8029985 --- /dev/null +++ b/test/demo.sh @@ -0,0 +1,125 @@ +#!/bin/bash + +overall_setup() { + # This should be run before any other steps--it deploys the solution and sets it to a + # starting state without the capture proxy running + :</dev/null 2>&1 & + + # To verify + echo "es processes:" && pgrep -f "elasticsearch" && echo "proxy processes (should be empty):" && pgrep -f "trafficCaptureProxyServer" || curl localhost:9200 +RUN_ON_EC2_NODE + +} + +cleanup() { + # These commands are necessary if the environment may not be clean, but shouldn't be required otherwise. + + # Turn off the replayer + aws ecs update-service --cluster migration-${STAGE}-ecs-cluster --service migration-${STAGE}-traffic-replayer-default --desired-count 0 + + # Check that replayer args in the JSON task definition include `--speedup-factor 5.0 --remove-auth-header` -- if not modify and redeploy replayer + + # Turn off RFS + aws ecs update-service --cluster migration-${STAGE}-ecs-cluster --service migration-${STAGE}-reindex-from-snapshot --desired-count 0 + + # Get kafka brokers, source_url, and console task arn + MIGRATION_KAFKA_BROKERS=$(aws ssm get-parameter --name "/migration/$STAGE/default/kafkaBrokers" --query 'Parameter.Value' --output text | tr ',' '@') + SOURCE_URL="http://$(aws cloudformation describe-stacks --stack-name opensearch-infra-stack-Migration-Source --query "Stacks[0].Outputs[?OutputKey==\`loadbalancerurl\`].OutputValue" --output text):9200" + CONSOLE_TASK_ARN=$(aws ecs list-tasks --cluster migration-${STAGE}-ecs-cluster --family "migration-${STAGE}-migration-console" | jq --raw-output '.taskArns[0]') + + # Delete the Kafka topic + aws ecs execute-command --cluster "migration-${STAGE}-ecs-cluster" --task "${CONSOLE_TASK_ARN}" --container "migration-console" --interactive --command "./kafka-tools/kafka/bin/kafka-topics.sh --bootstrap-server $MIGRATION_KAFKA_BROKERS --delete --topic logging-traffic-topic --command-config kafka-tools/aws/msk-iam-auth.properties" + + # Verify that "logging-traffic-topic" does not appear in output from: + aws ecs execute-command --cluster "migration-${STAGE}-ecs-cluster" --task "${CONSOLE_TASK_ARN}" --container "migration-console" --interactive --command "./kafka-tools/kafka/bin/kafka-topics.sh --bootstrap-server $MIGRATION_KAFKA_BROKERS --list --command-config kafka-tools/aws/msk-iam-auth.properties" + + # Recreate the kafka topic + aws ecs execute-command --cluster "migration-${STAGE}-ecs-cluster" --task "${CONSOLE_TASK_ARN}" --container "migration-console" --interactive --command "./kafka-tools/kafka/bin/kafka-topics.sh --bootstrap-server $MIGRATION_KAFKA_BROKERS --create --topic logging-traffic-topic --command-config kafka-tools/aws/msk-iam-auth.properties" + + ./accessContainer.sh migration-console $STAGE $AWS_DEFAULT_REGION + :< +RUN_ON_EC2_NODE + + # Cat indices again and show that data is being written to Kafka + :< /dev/null + + # Show Kafka offsets + ./kafka-tools/kafka/bin/kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list "$MIGRATION_KAFKA_BROKER_ENDPOINTS" --topic logging-traffic-topic --time -1 --command-config kafka-tools/aws/msk-iam-auth.properties + + # Start an RFS migration + aws ecs update-service --cluster migration-${MIGRATION_STAGE}-ecs-cluster --service migration-${MIGRATION_STAGE}-reindex-from-snapshot --desired-count 1 + + # Check logs to find when it has finished + # TODO: add code snippet here for pulling logs + + # Start replayer + aws ecs update-service --cluster migration-${MIGRATION_STAGE}-ecs-cluster --service migration-${MIGRATION_STAGE}-traffic-replayer-default --desired-count 1 + + # Show tuples + ./humanReadableLogs.py /shared-replayer-output/traffic-replayer-default/tuples.log + + head -n 1 /shared-replayer-output/traffic-replayer-default/readable-tuples.log | jq + +RUN_ON_MIGRATION_CONSOLE +} + +overall_setup() +predemo_setup() +demo() \ No newline at end of file