From 174665db88c82dd3cebe97f7df07e4400a606406 Mon Sep 17 00:00:00 2001 From: "Jeffrey Jonathan Jennings (J3)" Date: Thu, 26 Dec 2024 11:57:42 -0500 Subject: [PATCH] Resolved #619. --- confluent-kafka-setup.tf | 2 +- main.tf | 2 +- .../avro_flight_consolidator_ccaf_app.py | 17 +++++---- ...ro-flight-consolidator-ccaf-app-locally.sh | 36 +++++++++++++------ 4 files changed, 37 insertions(+), 20 deletions(-) diff --git a/confluent-kafka-setup.tf b/confluent-kafka-setup.tf index 9e4c3c9..4d32d28 100644 --- a/confluent-kafka-setup.tf +++ b/confluent-kafka-setup.tf @@ -1,6 +1,6 @@ # Create the Kafka cluster resource "confluent_kafka_cluster" "kafka_cluster" { - display_name = "${local.secrets_insert}_kafka_cluster" + display_name = "${local.secrets_insert}" availability = "SINGLE_ZONE" cloud = local.cloud region = var.aws_region diff --git a/main.tf b/main.tf index f506a1e..adf0222 100644 --- a/main.tf +++ b/main.tf @@ -60,7 +60,7 @@ data "confluent_organization" "env" {} # Create the Confluent Cloud Environment resource "confluent_environment" "env" { - display_name = "${local.secrets_insert}_env" + display_name = "${local.secrets_insert}" stream_governance { package = "ESSENTIALS" diff --git a/python_ccaf/src/kickstarter/avro_flight_consolidator_ccaf_app.py b/python_ccaf/src/kickstarter/avro_flight_consolidator_ccaf_app.py index 36f7050..0cbcfb2 100644 --- a/python_ccaf/src/kickstarter/avro_flight_consolidator_ccaf_app.py +++ b/python_ccaf/src/kickstarter/avro_flight_consolidator_ccaf_app.py @@ -26,20 +26,25 @@ def run(): """ # The service account user is passed in as a command line argument. parser = argparse.ArgumentParser() - parser.add_argument('--service-account-user', - dest='service_account_user', + parser.add_argument('--catalog-name', + dest='catalog_name', required=True, - help='The Service Account User.') + help='The Catalog Name.') + parser.add_argument('--database-name', + dest='database_name', + required=True, + help='The Database Name.') parser.add_argument('--aws-region', dest='aws_region', required=True, help='The AWS Region.') known_args, _ = parser.parse_known_args() - service_account_user = known_args.service_account_user.lower() + catalog_name = known_args.catalog_name.lower() + database_name = known_args.database_name.lower() aws_region = known_args.aws_region.lower() # Retrieve the Confluent Cloud settings from AWS Secrets Manager. - secret_name = f"/confluent_cloud_resource/{service_account_user}/flink_compute_pool" + secret_name = f"/confluent_cloud_resource/{catalog_name}/flink_compute_pool" settings = get_secrets(aws_region, secret_name) # Build the ConfluentSettings object. @@ -58,8 +63,6 @@ def run(): tbl_env = TableEnvironment.create(confluent_settings) # The catalog name and database name are used to set the current catalog and database. - catalog_name = f"{service_account_user}_env" - database_name = f"{service_account_user}_kafka_cluster" tbl_env.use_catalog(catalog_name) tbl_env.use_database(database_name) catalog = tbl_env.get_catalog(catalog_name) diff --git a/scripts/run-avro-flight-consolidator-ccaf-app-locally.sh b/scripts/run-avro-flight-consolidator-ccaf-app-locally.sh index ff79581..cb802c3 100755 --- a/scripts/run-avro-flight-consolidator-ccaf-app-locally.sh +++ b/scripts/run-avro-flight-consolidator-ccaf-app-locally.sh @@ -3,8 +3,8 @@ # # *** Script Syntax *** # scripts/run-avro-flight-consolidator-ccaf-app-locally.sh --profile= -# --service-account-user= -# +# --catalog-name= +# --database-name= # for arg in "$@" # $@ sees arguments as separate words @@ -12,9 +12,12 @@ do case $arg in *"--profile="*) AWS_PROFILE=$arg;; - *"--service-account-user="*) - arg_length=23 - SERVICE_ACCOUNT_USER=${arg:$arg_length:$(expr ${#arg} - $arg_length)};; + *"--catalog-name="*) + arg_length=15 + CATALOG_NAME=${arg:$arg_length:$(expr ${#arg} - $arg_length)};; + *"--database-name="*) + arg_length=16 + DATABASE_NAME=${arg:$arg_length:$(expr ${#arg} - $arg_length)};; esac done @@ -24,18 +27,29 @@ then echo echo "(Error Message 001) You did not include the proper use of the --profile= argument in the call." echo - echo "Usage: Require ---> `basename $0` --profile= --service-account-user=" + echo "Usage: Require ---> `basename $0` --profile= --catalog-name= --database-name=" echo exit 85 # Common GNU/Linux Exit Code for 'Interrupted system call should be restarted' fi -# Check required --service-account-user argument was supplied -if [ -z $SERVICE_ACCOUNT_USER ] +# Check required --catalog-name argument was supplied +if [ -z $CATALOG_NAME ] then echo - echo "(Error Message 002) You did not include the proper use of the --service-account-user= argument in the call." + echo "(Error Message 002) You did not include the proper use of the --catalog-name= argument in the call." echo - echo "Usage: Require ---> `basename $0` --profile= --service-account-user=" + echo "Usage: Require ---> `basename $0` --profile= --catalog-name= --database-name=" + echo + exit 85 # Common GNU/Linux Exit Code for 'Interrupted system call should be restarted' +fi + +# Check required --database-name argument was supplied +if [ -z $DATABASE_NAME ] +then + echo + echo "(Error Message 003) You did not include the proper use of the --database-name= argument in the call." + echo + echo "Usage: Require ---> `basename $0` --profile= --catalog-name= --database-name=" echo exit 85 # Common GNU/Linux Exit Code for 'Interrupted system call should be restarted' fi @@ -48,4 +62,4 @@ export AWS_REGION=$(aws configure get sso_region $AWS_PROFILE) cd python_ccaf poetry shell -poetry run avro_flight_consolidator_ccaf_app --service-account-user $SERVICE_ACCOUNT_USER --aws-region $AWS_REGION +poetry run avro_flight_consolidator_ccaf_app --catalog-name $CATALOG_NAME --database-name $DATABASE_NAME --aws-region $AWS_REGION