Merge pull request #620 from j3-signalroom/619-pass-the-catalog_name-…

…and-database_name-as-arguments-to-the-avro_flight_consolidator_app-ccaf-flink-app

Resolved #619.

confluent-kafka-setup.tf

@@ -1,6 +1,6 @@
 # Create the Kafka cluster
 resource "confluent_kafka_cluster" "kafka_cluster" {
-    display_name = "${local.secrets_insert}_kafka_cluster"
+    display_name = "${local.secrets_insert}"
     availability = "SINGLE_ZONE"
     cloud        = local.cloud
     region       = var.aws_region

main.tf

@@ -60,7 +60,7 @@ data "confluent_organization" "env" {}
 
 # Create the Confluent Cloud Environment
 resource "confluent_environment" "env" {
-  display_name = "${local.secrets_insert}_env"
+  display_name = "${local.secrets_insert}"
 
   stream_governance {
     package = "ESSENTIALS"

python_ccaf/src/kickstarter/avro_flight_consolidator_ccaf_app.py

@@ -26,20 +26,25 @@ def run():
     """
     # The service account user is passed in as a command line argument.  
     parser = argparse.ArgumentParser()
-    parser.add_argument('--service-account-user',
-                        dest='service_account_user',
+    parser.add_argument('--catalog-name',
+                        dest='catalog_name',
                         required=True,
-                        help='The Service Account User.')
+                        help='The Catalog Name.')
+    parser.add_argument('--database-name',
+                        dest='database_name',
+                        required=True,
+                        help='The Database Name.')
     parser.add_argument('--aws-region',
                         dest='aws_region',
                         required=True,
                         help='The AWS Region.')
     known_args, _ = parser.parse_known_args()
-    service_account_user = known_args.service_account_user.lower()
+    catalog_name = known_args.catalog_name.lower()
+    database_name = known_args.database_name.lower()
     aws_region = known_args.aws_region.lower()
 
     # Retrieve the Confluent Cloud settings from AWS Secrets Manager.
-    secret_name = f"/confluent_cloud_resource/{service_account_user}/flink_compute_pool"
+    secret_name = f"/confluent_cloud_resource/{catalog_name}/flink_compute_pool"
     settings = get_secrets(aws_region, secret_name)
 
     # Build the ConfluentSettings object.
@@ -58,8 +63,6 @@ def run():
     tbl_env = TableEnvironment.create(confluent_settings)
 
     # The catalog name and database name are used to set the current catalog and database.
-    catalog_name = f"{service_account_user}_env"
-    database_name = f"{service_account_user}_kafka_cluster"
     tbl_env.use_catalog(catalog_name)
     tbl_env.use_database(database_name)
     catalog = tbl_env.get_catalog(catalog_name)

scripts/run-avro-flight-consolidator-ccaf-app-locally.sh

@@ -3,18 +3,21 @@
 #
 # *** Script Syntax ***
 # scripts/run-avro-flight-consolidator-ccaf-app-locally.sh --profile=<AWS_SSO_PROFILE_NAME>
-#                                                          --service-account-user=<SERVICE_ACCOUNT_USER>
-#                           
+#                                                          --catalog-name=<CATALOG_NAME>
+#                                                          --database-name=<DATABASE_NAME>
 #
 
 for arg in "$@" # $@ sees arguments as separate words
 do
     case $arg in
         *"--profile="*)
             AWS_PROFILE=$arg;;
-        *"--service-account-user="*)
-            arg_length=23
-            SERVICE_ACCOUNT_USER=${arg:$arg_length:$(expr ${#arg} - $arg_length)};;
+        *"--catalog-name="*)
+            arg_length=15
+            CATALOG_NAME=${arg:$arg_length:$(expr ${#arg} - $arg_length)};;
+        *"--database-name="*)
+            arg_length=16
+            DATABASE_NAME=${arg:$arg_length:$(expr ${#arg} - $arg_length)};;
     esac
 done
 
@@ -24,18 +27,29 @@ then
     echo
     echo "(Error Message 001)  You did not include the proper use of the --profile=<AWS_SSO_PROFILE_NAME> argument in the call."
     echo
-    echo "Usage:  Require ---> `basename $0` --profile=<AWS_SSO_PROFILE_NAME> --service-account-user=<SERVICE_ACCOUNT_USER>"
+    echo "Usage:  Require ---> `basename $0` --profile=<AWS_SSO_PROFILE_NAME> --catalog-name=<CATALOG_NAME> --database-name=<DATABASE_NAME>"
     echo
     exit 85 # Common GNU/Linux Exit Code for 'Interrupted system call should be restarted'
 fi
 
-# Check required --service-account-user argument was supplied
-if [ -z $SERVICE_ACCOUNT_USER ]
+# Check required --catalog-name argument was supplied
+if [ -z $CATALOG_NAME ]
 then
     echo
-    echo "(Error Message 002)  You did not include the proper use of the --service-account-user=<SERVICE_ACCOUNT_USER> argument in the call."
+    echo "(Error Message 002)  You did not include the proper use of the --catalog-name=<CATALOG_NAME> argument in the call."
     echo
-    echo "Usage:  Require ---> `basename $0` --profile=<AWS_SSO_PROFILE_NAME> --service-account-user=<SERVICE_ACCOUNT_USER>"
+    echo "Usage:  Require ---> `basename $0` --profile=<AWS_SSO_PROFILE_NAME> --catalog-name=<CATALOG_NAME> --database-name=<DATABASE_NAME>"
+    echo
+    exit 85 # Common GNU/Linux Exit Code for 'Interrupted system call should be restarted'
+fi
+
+# Check required --database-name argument was supplied
+if [ -z $DATABASE_NAME ]
+then
+    echo
+    echo "(Error Message 003)  You did not include the proper use of the --database-name=<DATABASE_NAME> argument in the call."
+    echo
+    echo "Usage:  Require ---> `basename $0` --profile=<AWS_SSO_PROFILE_NAME> --catalog-name=<CATALOG_NAME> --database-name=<DATABASE_NAME>"
     echo
     exit 85 # Common GNU/Linux Exit Code for 'Interrupted system call should be restarted'
 fi
@@ -48,4 +62,4 @@ export AWS_REGION=$(aws configure get sso_region $AWS_PROFILE)
 
 cd python_ccaf
 poetry shell
-poetry run avro_flight_consolidator_ccaf_app --service-account-user $SERVICE_ACCOUNT_USER --aws-region $AWS_REGION
+poetry run avro_flight_consolidator_ccaf_app --catalog-name $CATALOG_NAME --database-name $DATABASE_NAME --aws-region $AWS_REGION