Packaged solution labeling (#664)

GoogleCloudPlatform · Jan 6, 2023 · 433c99e · 433c99e
1 parent f4689ff
commit 433c99e
Show file tree

Hide file tree

Showing 16 changed files with 135 additions and 19 deletions.
diff --git a/api/v1/src/admin/dataManager.js b/api/v1/src/admin/dataManager.js
@@ -187,75 +187,90 @@ function sqlReplacements(projectId, text) {
 async function setupDatasharePrerequisites(projectId) {
     const bigqueryUtil = new BigQueryUtil(projectId);
 
+    const viewOptions = {
+        labels: { [cfg.googPackagedSolutionKey] : cfg.googPackagedSolutionValue }
+    };
+
     if (await bigqueryUtil.datasetExists(cfg.cdsDatasetId) === false) {
         console.log('Creating datashare dataset');
-        let labels = {};
-        labels[cfg.cdsMetadataLabelKey] = true;
+        let labels = { 
+            [cfg.cdsMetadataLabelKey]: true,
+            [cfg.googPackagedSolutionKey] : cfg.googPackagedSolutionValue
+        };
+
         const options = { description: 'Datashare Master Dataset', labels: labels };
         await bigqueryUtil.createDataset(cfg.cdsDatasetId, options);
     } else {
         console.log('Datashare dataset already exists');
         // Update existing dataset to add the label
         // This can be split out later to an update process if we need to make bigger changes
         await bigqueryUtil.setDatasetLabel(cfg.cdsDatasetId, cfg.cdsMetadataLabelKey, true);
+        await bigqueryUtil.setDatasetLabel(cfg.cdsDatasetId, cfg.googPackagedSolutionKey, cfg.googPackagedSolutionValue);
     }
 
     if (await bigqueryUtil.tableExists(cfg.cdsDatasetId, cfg.cdsPolicyTableId) === false) {
         console.log("Creating policy table");
-        const options = require('./bq/schema/policy.json');
+        const options = Object.assign(viewOptions, require('./bq/schema/policy.json'));
         await bigqueryUtil.createTable(cfg.cdsDatasetId, cfg.cdsPolicyTableId, options);
     } else {
         // Patch the schema
         const options = require('./bq/schema/policy.json');
         await bigqueryUtil.patchTableSchema(cfg.cdsDatasetId, cfg.cdsPolicyTableId, options.schema)
+        await bigqueryUtil.setTableLabel(cfg.cdsDatasetId, cfg.cdsPolicyTableId, cfg.googPackagedSolutionKey, cfg.googPackagedSolutionValue)
         console.log('Policy table already exists');
     }
 
     if (await bigqueryUtil.viewExists(cfg.cdsDatasetId, cfg.cdsPolicyViewId) === false) {
         console.log("Creating latest policies view");
         const sql = sqlReplacements(projectId, require('./bq/view/currentPolicy.sql'));
-        await bigqueryUtil.createView(cfg.cdsDatasetId, cfg.cdsPolicyViewId, sql);
+        await bigqueryUtil.createView(cfg.cdsDatasetId, cfg.cdsPolicyViewId, sql, viewOptions);
     } else {
+        await bigqueryUtil.setTableLabel(cfg.cdsDatasetId, cfg.cdsPolicyViewId, cfg.googPackagedSolutionKey, cfg.googPackagedSolutionValue)
         console.log('Policies view already exists');
     }
 
     if (await bigqueryUtil.tableExists(cfg.cdsDatasetId, cfg.cdsAccountTableId) === false) {
         console.log("Creating account table");
-        const options = require('./bq/schema/account.json');
+        const options = Object.assign(viewOptions, require('./bq/schema/account.json'));
         await bigqueryUtil.createTable(cfg.cdsDatasetId, cfg.cdsAccountTableId, options);
     } else {
+        await bigqueryUtil.setTableLabel(cfg.cdsDatasetId, cfg.cdsAccountTableId, cfg.googPackagedSolutionKey, cfg.googPackagedSolutionValue)
         console.log('Account table already exists');
     }
 
     if (await bigqueryUtil.viewExists(cfg.cdsDatasetId, cfg.cdsAccountViewId) === false) {
         console.log("Creating latest account view");
         const sql = sqlReplacements(projectId, require('./bq/view/currentAccount.sql'));
-        await bigqueryUtil.createView(cfg.cdsDatasetId, cfg.cdsAccountViewId, sql);
+        await bigqueryUtil.createView(cfg.cdsDatasetId, cfg.cdsAccountViewId, sql, viewOptions);
     } else {
+        await bigqueryUtil.setTableLabel(cfg.cdsDatasetId, cfg.cdsAccountViewId, cfg.googPackagedSolutionKey, cfg.googPackagedSolutionValue)
         console.log('Account view already exists');
     }
 
     if (await bigqueryUtil.tableExists(cfg.cdsDatasetId, cfg.cdsAuthorizedViewTableId) === false) {
         console.log("Creating authorizedView table");
-        const options = require('./bq/schema/authorizedView.json');
+        const options = Object.assign(viewOptions, require('./bq/schema/authorizedView.json'));
         await bigqueryUtil.createTable(cfg.cdsDatasetId, cfg.cdsAuthorizedViewTableId, options);
     } else {
+        await bigqueryUtil.setTableLabel(cfg.cdsDatasetId, cfg.cdsAuthorizedViewTableId, cfg.googPackagedSolutionKey, cfg.googPackagedSolutionValue)
         console.log('Authorized view table already exists');
     }
 
     if (await bigqueryUtil.viewExists(cfg.cdsDatasetId, cfg.cdsAuthorizedViewViewId) === false) {
         console.log("Creating latest authorizedView view");
         const sql = sqlReplacements(projectId, require('./bq/view/currentAuthorizedView.sql'));
-        await bigqueryUtil.createView(cfg.cdsDatasetId, cfg.cdsAuthorizedViewViewId, sql);
+        await bigqueryUtil.createView(cfg.cdsDatasetId, cfg.cdsAuthorizedViewViewId, sql, viewOptions);
     } else {
+        await bigqueryUtil.setTableLabel(cfg.cdsDatasetId, cfg.cdsAuthorizedViewViewId, cfg.googPackagedSolutionKey, cfg.googPackagedSolutionValue)
         console.log('Authorized view view already exists');
     }
 
     if (await bigqueryUtil.viewExists(cfg.cdsDatasetId, cfg.cdsCurrentUserPermissionViewId) === false) {
         console.log("Creating latest currentUserPermission view");
         const sql = sqlReplacements(projectId, require('./bq/view/currentUserPermission.sql'));
-        await bigqueryUtil.createView(cfg.cdsDatasetId, cfg.cdsCurrentUserPermissionViewId, sql);
+        await bigqueryUtil.createView(cfg.cdsDatasetId, cfg.cdsCurrentUserPermissionViewId, sql, viewOptions);
     } else {
+        await bigqueryUtil.setTableLabel(cfg.cdsDatasetId, cfg.cdsCurrentUserPermissionViewId, cfg.googPackagedSolutionKey, cfg.googPackagedSolutionValue)
         console.log('Current user dataset view already exists');
     }
 

diff --git a/api/v1/src/datasets/dataManager.js b/api/v1/src/datasets/dataManager.js
@@ -95,7 +95,10 @@ async function listDatasets(projectId, includeAll) {
  */
 async function createDataset(projectId, datasetId, description) {
     const options = {};
-    options.labels = { [cfg.cdsManagedLabelKey]: "true" };
+    options.labels = { 
+        [cfg.cdsManagedLabelKey]: "true",
+        [cfg.googPackagedSolutionKey] : cfg.googPackagedSolutionValue
+    };
     if (description) {
         options.description = description;
     }
@@ -519,8 +522,11 @@ async function createView(view, overrideSql) {
         let configuredExpirationTime = view.expiration && view.expiration.delete === true ? view.expiration.time : null;
 
         let viewDescription = `This view was generated by ${cfg.productName}. ${view.description}`;
-        let labels = {};
-        labels[cfg.cdsManagedLabelKey] = true;
+
+        let labels = { 
+            [cfg.cdsManagedLabelKey]: true,
+            [cfg.googPackagedSolutionKey] : cfg.googPackagedSolutionValue
+        };
 
         const viewOptions = {
             description: viewDescription,

diff --git a/api/v1/src/lib/config.js b/api/v1/src/lib/config.js
@@ -27,6 +27,8 @@ config.productName = 'Datashare';
 // TODO: Update through CI
 config.productVersion = '0.0.1';
 
+config.googPackagedSolutionKey = process.env.GOOG_PACKAGED_SOLUTION_KEY || 'goog_packaged_solution_key';
+config.googPackagedSolutionValue = process.env.GOOG_PACKAGED_SOLUTION_VALUE ||'datashare';
 config.cdsManagedLabelKey = 'datashare_managed';
 config.cdsMetadataLabelKey = 'datashare_metadata';
 config.cdsExclusionLabels = [config.cdsMetadataLabelKey];

diff --git a/api/v1/src/pubsub/dataManager.js b/api/v1/src/pubsub/dataManager.js
@@ -45,7 +45,10 @@ async function createTopic(projectId, name) {
     return pubsubUtil.createTopic(name).then(topicResponse => {
         const [topic] = topicResponse;
         let dict = {};
-        dict.labels = { [config.cdsManagedLabelKey]: "true" };
+        dict.labels = { 
+            [config.cdsManagedLabelKey]: "true",
+            [config.googPackagedSolutionKey] : config.googPackagedSolutionValue
+        };
         return topic.setMetadata(dict).then(() => {
             return {
                 success: true,

diff --git a/api/v1/src/storage/dataManager.js b/api/v1/src/storage/dataManager.js
@@ -45,7 +45,10 @@ async function createBucket(projectId, name) {
     const storageUtil = new StorageUtil(projectId);
     return storageUtil.createBucket(name).then(bucketResponse => {
         const [bucket] = bucketResponse;
-        let labels = { [config.cdsManagedLabelKey]: "true" };
+        let labels = { 
+            [config.cdsManagedLabelKey]: "true",
+            [config.googPackagedSolutionKey] : config.googPackagedSolutionValue
+        };
         return bucket.setLabels(labels).then(() => {
             return {
                 success: true,

diff --git a/terraform/deploy/main.tf b/terraform/deploy/main.tf
@@ -97,6 +97,8 @@ module "datashare-application" {
   api_domain                      = var.api_domain
   ui_domain                       = var.ui_domain
   secret_name_prefix              = var.secret_name_prefix
+  goog_packaged_solution_key      = var.goog_packaged_solution_key
+  goog_packaged_solution_value    = var.goog_packaged_solution_value
 }
 
 module "custom-domain" {
@@ -112,6 +114,8 @@ module "custom-domain" {
   cloud_run_ds_ui_name         = module.datashare-application[0].cloud_run_ds_ui_name
   ds_api_gateway_gateway_id    = module.datashare-application[0].ds_api_gateway_gateway_id
   update_cloud_dns             = var.update_cloud_dns
+  goog_packaged_solution_key   = var.goog_packaged_solution_key
+  goog_packaged_solution_value = var.goog_packaged_solution_value
 
   depends_on = [module.datashare-application]
 }
@@ -120,7 +124,9 @@ module "cloud-functions" {
   source = "../modules/ingestion-function"
   count  = var.deploy_ingestion_cloud_function ? 1 : 0
 
-  project_id = var.project_id
-  region     = var.region
-  tag        = var.tag
+  project_id                   = var.project_id
+  region                       = var.region
+  tag                          = var.tag
+  goog_packaged_solution_key   = var.goog_packaged_solution_key
+  goog_packaged_solution_value = var.goog_packaged_solution_value
 }
diff --git a/terraform/deploy/variables.tf b/terraform/deploy/variables.tf
@@ -274,4 +274,16 @@ variable "marketplace_integration_enabled" {
   type        = bool
   description = "Flag indicating if marketplace integration for datashare is enabled"
   default     = false
+}
+
+variable "goog_packaged_solution_key" {
+  type        = string
+  description = "The packaged solution key"
+  default     = "goog-packaged-solution"
+}
+
+variable "goog_packaged_solution_value" {
+  type        = string
+  description = "The packaged solution value"
+  default     = "datashare"
 }
diff --git a/terraform/modules/custom-domain/load-balancer.tf b/terraform/modules/custom-domain/load-balancer.tf
@@ -67,6 +67,11 @@ resource "google_compute_global_address" "default" {
   project    = var.project_id
   name       = "datashare-api-static-ip"
   ip_version = "IPV4"
+
+  // BETA only
+  /*labels = {
+    (var.goog_packaged_solution_key) = var.goog_packaged_solution_value
+  }*/
 }
 
 // TODO: Reserve an IP address first rather than allow auto-generation
@@ -79,4 +84,9 @@ resource "google_compute_global_forwarding_rule" "datashare-lb-forwarding-rule"
   port_range            = "443-443"
   project               = var.project_id
   target                = google_compute_target_https_proxy.datashare-target-http-proxy.id
+
+  // BETA only
+  /*labels = {
+    (var.goog_packaged_solution_key) = var.goog_packaged_solution_value
+  }*/
 }
diff --git a/terraform/modules/custom-domain/variables.tf b/terraform/modules/custom-domain/variables.tf
@@ -68,4 +68,14 @@ variable "ds_api_gateway_gateway_id" {
 variable "update_cloud_dns" {
   type        = bool
   description = "Flag indicating if the Cloud DNS zone should have its A record updated"
+}
+
+variable "goog_packaged_solution_key" {
+  type        = string
+  description = "The packaged solution key"
+}
+
+variable "goog_packaged_solution_value" {
+  type        = string
+  description = "The packaged solution value"
 }
diff --git a/terraform/modules/datashare-application/api-gateway.tf b/terraform/modules/datashare-application/api-gateway.tf
@@ -57,6 +57,10 @@ resource "google_api_gateway_api" "api_gw" {
   project  = var.project_id
   provider = google-beta
   api_id   = "api-gw-ds-api"
+
+  labels = {
+    (var.goog_packaged_solution_key) = var.goog_packaged_solution_value
+  }
 }
 
 resource "google_api_gateway_api_config" "api_cfg" {
@@ -77,6 +81,9 @@ resource "google_api_gateway_api_config" "api_cfg" {
       google_service_account = local.api_gateway_service_account_name
     }
   }
+  labels = {
+    (var.goog_packaged_solution_key) = var.goog_packaged_solution_value
+  }
   lifecycle {
     create_before_destroy = true
   }
@@ -89,6 +96,10 @@ resource "google_api_gateway_gateway" "gw" {
   api_config   = google_api_gateway_api_config.api_cfg.id
   gateway_id   = "api-gw-ds-api"
   display_name = "Datashare API Service Gateway"
+
+  labels = {
+    (var.goog_packaged_solution_key) = var.goog_packaged_solution_value
+  }
 }
 
 // Grant API Gateway account run.invoker access on the API service

diff --git a/terraform/modules/datashare-application/run-ds-api.tf b/terraform/modules/datashare-application/run-ds-api.tf
@@ -39,6 +39,9 @@ resource "google_cloud_run_service" "cloud-run-service-ds-api" {
     annotations = {
       "run.googleapis.com/ingress" = "all"
     }
+    labels = {
+      (var.goog_packaged_solution_key) = var.goog_packaged_solution_value
+    }
   }
 
   // TODO: Store and use secret manager functionality in Cloud Run to expose as env variables
@@ -129,6 +132,9 @@ resource "google_cloud_run_service" "cloud-run-service-ds-listener" {
     annotations = {
       "run.googleapis.com/ingress" = "internal"
     }
+    labels = {
+      (var.goog_packaged_solution_key) = var.goog_packaged_solution_value
+    }
   }
 
   template {

diff --git a/terraform/modules/datashare-application/run-ds-ui.tf b/terraform/modules/datashare-application/run-ds-ui.tf
@@ -34,6 +34,9 @@ resource "google_cloud_run_service" "cloud-run-ds-ui" {
     annotations = {
       "run.googleapis.com/ingress" = "all"
     }
+    labels = {
+      (var.goog_packaged_solution_key) = var.goog_packaged_solution_value
+    }
   }
 
   // TODO: Store and use secret manager functionality in Cloud Run to expose as env variables

diff --git a/terraform/modules/datashare-application/variables.tf b/terraform/modules/datashare-application/variables.tf
@@ -203,4 +203,14 @@ variable "secret_name_prefix" {
   type        = string
   description = "The prefix used for the configuration secret names"
   default     = "datashare"
+}
+
+variable "goog_packaged_solution_key" {
+  type        = string
+  description = "The packaged solution key"
+}
+
+variable "goog_packaged_solution_value" {
+  type        = string
+  description = "The packaged solution value"
 }
diff --git a/terraform/modules/ingestion-function/cloud-functions.tf b/terraform/modules/ingestion-function/cloud-functions.tf
@@ -64,8 +64,9 @@ resource "google_cloudfunctions_function" "datashare_cloud_function" {
   timeout               = 540
   entry_point           = "processEvent"
   labels = {
-    version  = replace(var.tag, ".", "_"),
-    md5_hash = data.archive_file.function_package.output_md5
+    version                          = replace(var.tag, ".", "_"),
+    md5_hash                         = data.archive_file.function_package.output_md5,
+    (var.goog_packaged_solution_key) = var.goog_packaged_solution_value
   }
   event_trigger {
     event_type = "google.storage.object.finalize"

diff --git a/terraform/modules/ingestion-function/gcs.tf b/terraform/modules/ingestion-function/gcs.tf
@@ -21,6 +21,10 @@ resource "google_storage_bucket" "install_bucket" {
   force_destroy               = true
   uniform_bucket_level_access = true
   storage_class               = var.ingestion_storage_bucket_storage_class
+
+  labels = {
+    (var.goog_packaged_solution_key) = var.goog_packaged_solution_value
+  }
 }
 
 // terraform import google_storage_bucket.ingestion_bucket cds-demo-1-271622-datashare-ingestion
@@ -30,4 +34,8 @@ resource "google_storage_bucket" "ingestion_bucket" {
   force_destroy               = true
   uniform_bucket_level_access = true
   storage_class               = var.ingestion_storage_bucket_storage_class
+
+  labels = {
+    (var.goog_packaged_solution_key) = var.goog_packaged_solution_value
+  }
 }
diff --git a/terraform/modules/ingestion-function/variables.tf b/terraform/modules/ingestion-function/variables.tf
@@ -63,4 +63,14 @@ variable "ingestion_function_description" {
 variable "datashare_ingestion_source_code_filename" {
   default     = "datashare-batch-cloud-function-src.zip"
   description = "The ingestion function source zip file path"
+}
+
+variable "goog_packaged_solution_key" {
+  type        = string
+  description = "The packaged solution key"
+}
+
+variable "goog_packaged_solution_value" {
+  type        = string
+  description = "The packaged solution value"
 }