From bc419da6cf29fb569a3946b448262da16287a0ec Mon Sep 17 00:00:00 2001 From: Graham Beckley Date: Mon, 21 Oct 2024 10:09:03 -0400 Subject: [PATCH 1/6] Delete Terraform directory --- terraform/README.md | 112 ------------------ terraform/example/main.tf | 8 -- terraform/example/terraform.tfvars.example | 4 - terraform/example/variables.tf | 49 -------- .../fourkeys-circleci-parser/README.md | 41 ------- .../modules/fourkeys-circleci-parser/main.tf | 89 -------------- .../fourkeys-circleci-parser/variables.tf | 26 ---- .../fourkeys-cloud-build-parser/README.md | 46 ------- .../fourkeys-cloud-build-parser/main.tf | 89 -------------- .../fourkeys-cloud-build-parser/variables.tf | 26 ---- .../modules/fourkeys-github-parser/README.md | 41 ------- .../modules/fourkeys-github-parser/main.tf | 89 -------------- .../fourkeys-github-parser/variables.tf | 26 ---- .../modules/fourkeys-gitlab-parser/README.md | 41 ------- .../modules/fourkeys-gitlab-parser/main.tf | 93 --------------- .../fourkeys-gitlab-parser/variables.tf | 26 ---- .../fourkeys-pagerduty-parser/README.md | 41 ------- .../modules/fourkeys-pagerduty-parser/main.tf | 89 -------------- .../fourkeys-pagerduty-parser/variables.tf | 26 ---- .../modules/fourkeys-tekton-parser/README.md | 41 ------- .../modules/fourkeys-tekton-parser/main.tf | 89 -------------- .../fourkeys-tekton-parser/variables.tf | 26 ---- terraform/modules/fourkeys/README.md | 84 ------------- terraform/modules/fourkeys/bigquery.tf | 105 ---------------- terraform/modules/fourkeys/dashboard.tf | 45 ------- terraform/modules/fourkeys/event-handler.tf | 70 ----------- .../fourkeys/files/events_raw_schema.json | 1 - terraform/modules/fourkeys/iam.tf | 41 ------- terraform/modules/fourkeys/locals.tf | 21 ---- terraform/modules/fourkeys/outputs.tf | 25 ---- terraform/modules/fourkeys/parsers.tf | 77 ------------ terraform/modules/fourkeys/provider.tf | 8 -- .../modules/fourkeys/queries/changes.sql | 1 - .../modules/fourkeys/queries/deployments.sql | 1 - .../fourkeys/queries/function_json2array.js | 1 - .../function_multiFormatParseTimestamp.sql | 20 ---- .../modules/fourkeys/queries/incidents.sql | 1 - terraform/modules/fourkeys/services.tf | 13 -- terraform/modules/fourkeys/variables.tf | 85 ------------- 39 files changed, 1717 deletions(-) delete mode 100644 terraform/README.md delete mode 100644 terraform/example/main.tf delete mode 100644 terraform/example/terraform.tfvars.example delete mode 100644 terraform/example/variables.tf delete mode 100644 terraform/modules/fourkeys-circleci-parser/README.md delete mode 100644 terraform/modules/fourkeys-circleci-parser/main.tf delete mode 100644 terraform/modules/fourkeys-circleci-parser/variables.tf delete mode 100644 terraform/modules/fourkeys-cloud-build-parser/README.md delete mode 100644 terraform/modules/fourkeys-cloud-build-parser/main.tf delete mode 100644 terraform/modules/fourkeys-cloud-build-parser/variables.tf delete mode 100644 terraform/modules/fourkeys-github-parser/README.md delete mode 100644 terraform/modules/fourkeys-github-parser/main.tf delete mode 100644 terraform/modules/fourkeys-github-parser/variables.tf delete mode 100644 terraform/modules/fourkeys-gitlab-parser/README.md delete mode 100644 terraform/modules/fourkeys-gitlab-parser/main.tf delete mode 100644 terraform/modules/fourkeys-gitlab-parser/variables.tf delete mode 100644 terraform/modules/fourkeys-pagerduty-parser/README.md delete mode 100644 terraform/modules/fourkeys-pagerduty-parser/main.tf delete mode 100644 terraform/modules/fourkeys-pagerduty-parser/variables.tf delete mode 100644 terraform/modules/fourkeys-tekton-parser/README.md delete mode 100644 terraform/modules/fourkeys-tekton-parser/main.tf delete mode 100644 terraform/modules/fourkeys-tekton-parser/variables.tf delete mode 100644 terraform/modules/fourkeys/README.md delete mode 100644 terraform/modules/fourkeys/bigquery.tf delete mode 100644 terraform/modules/fourkeys/dashboard.tf delete mode 100644 terraform/modules/fourkeys/event-handler.tf delete mode 120000 terraform/modules/fourkeys/files/events_raw_schema.json delete mode 100644 terraform/modules/fourkeys/iam.tf delete mode 100644 terraform/modules/fourkeys/locals.tf delete mode 100644 terraform/modules/fourkeys/outputs.tf delete mode 100644 terraform/modules/fourkeys/parsers.tf delete mode 100644 terraform/modules/fourkeys/provider.tf delete mode 120000 terraform/modules/fourkeys/queries/changes.sql delete mode 120000 terraform/modules/fourkeys/queries/deployments.sql delete mode 120000 terraform/modules/fourkeys/queries/function_json2array.js delete mode 100644 terraform/modules/fourkeys/queries/function_multiFormatParseTimestamp.sql delete mode 120000 terraform/modules/fourkeys/queries/incidents.sql delete mode 100644 terraform/modules/fourkeys/services.tf delete mode 100644 terraform/modules/fourkeys/variables.tf diff --git a/terraform/README.md b/terraform/README.md deleted file mode 100644 index 60de684d..00000000 --- a/terraform/README.md +++ /dev/null @@ -1,112 +0,0 @@ -# Four Keys Terraform - -This directory contains modules and examples for deploying Four Keys with Terraform. The primary module `modules/fourkeys` uses the other sub-modules to deploy resources to a provided Google Cloud Project. - -## Usage - -This is an example of deploying fourkeys as a remote Terraform module from [this GitHub project](https://github.com/dora-team/fourkeys): - -```hcl -module "fourkeys" { - source = "github.com/dora-team/fourkeys//terraform/modules/fourkeys" - project_id = "your-google-cloud-project-id" - parsers = ['github'] -} -``` - -The example above will deploy Four Keys with a GitHub parser for GitHub events. See the `terraform/example` directory for full example and options. - -Alternatively, you can fork the fourkeys project and deploy as a local module from the `terraform/example` directory: - -```hcl -module "fourkeys" { - source = "../modules/fourkeys" - project_id = "your-google-cloud-project-id" - parsers = ['github'] -} -``` - -## Before you begin - -To deploy Four Keys with Terraform, you will first need: - -* A Google Cloud project with billing enabled -* The owner role assigned to you on the project -* The [Google Cloud CLI](https://cloud.google.com/sdk/docs/install) and [Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) installed on your local machine. We recommend deploying from [Cloud Shell](https://shell.cloud.google.com/?show=ide%2Cterminal) on your Google Cloud project. - -## Deploying with Terraform - -1. Set an environment variable indicating your Google Cloud project ID: - ```sh - export PROJECT_ID="YOUR_PROJECT_ID" - ``` - -1. Clone the fourkeys git repository and change into the root directory - ``` - git clone https://github.com/dora-team/fourkeys.git && cd fourkeys - ``` - -1. Use Cloud Build to build and push containers to Google Container Registry for the dashboard, event-handler: - ``` - gcloud builds submit dashboard --config=dashboard/cloudbuild.yaml --project $PROJECT_ID && \ - gcloud builds submit event-handler --config=event-handler/cloudbuild.yaml --project $PROJECT_ID - ``` - -1. Use Cloud Build to build and push containers to Google Container Registry for the parsers you plan to use. See the [`bq-workers`](../bq-workers/) for available options. GitHub for example: - ``` - gcloud builds submit bq-workers --config=bq-workers/parsers.cloudbuild.yaml --project $PROJECT_ID --substitutions=_SERVICE=github - ``` - -1. Change your working directory to `terraform/example` and rename `terraform.tfvars.example` to `terraform.tfvars` - ``` - cd terraform/example && mv terraform.tfvars.example terraform.tfvars - ``` - -1. Edit `terraform.tfvars` with values for the required variables. See `variables.tf` for a list of the variables, along with their descriptions and default values. Values not defined in `terraform.tfvars` will use default values defined in `variables.tf` - -1. Run the following commands from the `example` directory: - - `terraform init` to inialize Terraform and download the module - - `terraform plan` to preview changes. - - `terraform apply` to deploy the resources. - -## Generating mock data - -To test your Four Keys deployment, you can generate mock data that simulates events from a GitHub repository. - -1. Export your event handler URL an environment variable. This is the webhook URL that will receive events: - - ```sh - export WEBHOOK=`gcloud run services list --project $PROJECT_ID | grep event-handler | awk '{print $4}'` - ``` - -1. Export your event handler secret to an environment variable. This is the secret used to authenticate events sent to the webhook: - - ```sh - export SECRET=`gcloud secrets versions access 1 --secret=event-handler --project $PROJECT_ID` - ``` - -1. From the root of the fourkeys project run: - - ```sh - python3 data-generator/generate_data.py --vc_system=github - ``` - - You can see these events being run through the pipeline: - * The event handler logs show successful requests - * The Pub/Sub topic show messages posted - * The BigQuery GitHub parser show successful requests - -1. View the generated data in the `events_raw` table in with bq: - - ```sh - bq query --project_id $PROJECT_ID 'SELECT * FROM four_keys.events_raw WHERE source = "githubmock";' - ``` - - Or query the table directly in [BigQuery](https://console.cloud.google.com/bigquery): - - ```sql - SELECT * FROM four_keys.events_raw WHERE source = 'githubmock'; - ``` diff --git a/terraform/example/main.tf b/terraform/example/main.tf deleted file mode 100644 index 272d1cb5..00000000 --- a/terraform/example/main.tf +++ /dev/null @@ -1,8 +0,0 @@ -module "fourkeys" { - source = "../modules/fourkeys" - project_id = var.project_id - enable_apis = var.enable_apis - region = var.region - bigquery_region = var.bigquery_region - parsers = var.parsers -} diff --git a/terraform/example/terraform.tfvars.example b/terraform/example/terraform.tfvars.example deleted file mode 100644 index 399b34e9..00000000 --- a/terraform/example/terraform.tfvars.example +++ /dev/null @@ -1,4 +0,0 @@ -project_id = "" -region = "" -bigquery_region = "" -parsers = ["", ""] \ No newline at end of file diff --git a/terraform/example/variables.tf b/terraform/example/variables.tf deleted file mode 100644 index 8b8ea2a8..00000000 --- a/terraform/example/variables.tf +++ /dev/null @@ -1,49 +0,0 @@ -variable "project_id" { - type = string - description = "project to deploy four keys resources to" -} - -variable "enable_apis" { - type = bool - description = "Toggle to include required APIs." - default = true -} - -variable "region" { - type = string - default = "us-central1" - description = "Region to deploy four keys resources in." -} - -variable "bigquery_region" { - type = string - default = "US" - description = "Region to deploy BigQuery resources in." - validation { - condition = can(regex("^(US|EU)$", var.bigquery_region)) - error_message = "The value for 'bigquery_region' must be one of: 'US','EU'." - } -} - -variable "parsers" { - type = list(string) - description = "List of data parsers to configure. Acceptable values are: 'github', 'gitlab', 'cloud-build', 'tekton', 'pagerduty'" -} - -variable "event_handler_container_url" { - type = string - description = "URL for the event_handler container image." - default = "" -} - -variable "dashboard_container_url" { - type = string - description = "URL for the dashboard container image." - default = "" -} - -variable "parser_container_urls" { - type = map(any) - description = "URL for the parser container images. e.g: {'github': 'gcr.io/youproject/github-parser', 'gitlab': 'gcr.io/youproject/gitlab-parser'} " - default = {} -} diff --git a/terraform/modules/fourkeys-circleci-parser/README.md b/terraform/modules/fourkeys-circleci-parser/README.md deleted file mode 100644 index 43106187..00000000 --- a/terraform/modules/fourkeys-circleci-parser/README.md +++ /dev/null @@ -1,41 +0,0 @@ - -## Requirements - -No requirements. - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | n/a | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_cloud_run_service.circleci_parser](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloud_run_service) | resource | -| [google_project_iam_member.pubsub_service_account_token_creator](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | -| [google_project_service.data_source_services](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_service) | resource | -| [google_pubsub_subscription.circleci](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_subscription) | resource | -| [google_pubsub_topic.circleci](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic) | resource | -| [google_pubsub_topic_iam_member.service_account_editor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic_iam_member) | resource | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [enable\_apis](#input\_enable\_apis) | Toggle to include required APIs. | `bool` | `false` | no | -| [fourkeys\_service\_account\_email](#input\_fourkeys\_service\_account\_email) | Service account for fourkeys. | `string` | n/a | yes | -| [parser\_container\_url](#input\_parser\_container\_url) | URL of image to use in Cloud Run service configuration. | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | Project ID of the target project. | `string` | n/a | yes | -| [region](#input\_region) | Region to deploy resources. | `string` | `"us-central1"` | no | - -## Outputs - -No outputs. - \ No newline at end of file diff --git a/terraform/modules/fourkeys-circleci-parser/main.tf b/terraform/modules/fourkeys-circleci-parser/main.tf deleted file mode 100644 index 9e2e5951..00000000 --- a/terraform/modules/fourkeys-circleci-parser/main.tf +++ /dev/null @@ -1,89 +0,0 @@ -data "google_project" "project" { - project_id = var.project_id -} - -locals { - services = var.enable_apis ? [ - "run.googleapis.com" - ] : [] -} - -resource "google_project_service" "data_source_services" { - project = var.project_id - for_each = toset(local.services) - service = each.value - disable_on_destroy = false -} - -resource "google_cloud_run_service" "circleci_parser" { - project = var.project_id - name = "fourkeys-circleci-parser" - location = var.region - - template { - spec { - containers { - image = var.parser_container_url - env { - name = "PROJECT_NAME" - value = var.project_id - } - } - service_account_name = var.fourkeys_service_account_email - } - } - - traffic { - percent = 100 - latest_revision = true - } - - metadata { - annotations = { - "run.googleapis.com/ingress" = "internal" - } - } - - lifecycle { - ignore_changes = [ - metadata[0].annotations, - ] - } - - autogenerate_revision_name = true - depends_on = [ - google_project_service.data_source_services - ] -} - -resource "google_pubsub_topic" "circleci" { - project = var.project_id - name = "circleci" -} - -resource "google_pubsub_topic_iam_member" "service_account_editor" { - project = var.project_id - topic = google_pubsub_topic.circleci.id - role = "roles/pubsub.editor" - member = "serviceAccount:${var.fourkeys_service_account_email}" -} - -resource "google_pubsub_subscription" "circleci" { - project = var.project_id - name = "circleci" - topic = google_pubsub_topic.circleci.id - - push_config { - push_endpoint = google_cloud_run_service.circleci_parser.status[0]["url"] - - oidc_token { - service_account_email = var.fourkeys_service_account_email - } - } -} -# This IAM role grant is for projects created before April 8, 2021. See: https://cloud.google.com/pubsub/docs/push -resource "google_project_iam_member" "pubsub_service_account_token_creator" { - project = var.project_id - member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" - role = "roles/iam.serviceAccountTokenCreator" -} \ No newline at end of file diff --git a/terraform/modules/fourkeys-circleci-parser/variables.tf b/terraform/modules/fourkeys-circleci-parser/variables.tf deleted file mode 100644 index 34ebfe1e..00000000 --- a/terraform/modules/fourkeys-circleci-parser/variables.tf +++ /dev/null @@ -1,26 +0,0 @@ -variable "project_id" { - type = string - description = "Project ID of the target project." -} - -variable "region" { - type = string - description = "Region to deploy resources." - default = "us-central1" -} - -variable "fourkeys_service_account_email" { - type = string - description = "Service account for fourkeys." -} - -variable "enable_apis" { - type = bool - description = "Toggle to include required APIs." - default = false -} - -variable "parser_container_url" { - type = string - description = "URL of image to use in Cloud Run service configuration." -} \ No newline at end of file diff --git a/terraform/modules/fourkeys-cloud-build-parser/README.md b/terraform/modules/fourkeys-cloud-build-parser/README.md deleted file mode 100644 index 216ce62d..00000000 --- a/terraform/modules/fourkeys-cloud-build-parser/README.md +++ /dev/null @@ -1,46 +0,0 @@ -This module creates the pubsub topic and parser for Cloud Build. The name of the pubsub topic is important! - -Cloud Build is designed so that if a topic exists named `cloud-builds`, build events are automatically written to it. But that topic is not created by default, [it has to be deliberately created](https://cloud.google.com/build/docs/subscribe-build-notifications#receiving_build_notifications). - - - -## Requirements - -No requirements. - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | n/a | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_cloud_run_service.cloudbuild_parser](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloud_run_service) | resource | -| [google_project_iam_member.pubsub_service_account_token_creator](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | -| [google_project_service.data_source_services](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_service) | resource | -| [google_pubsub_subscription.cloudbuild](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_subscription) | resource | -| [google_pubsub_topic.cloudbuild](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic) | resource | -| [google_pubsub_topic_iam_member.service_account_editor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic_iam_member) | resource | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [enable\_apis](#input\_enable\_apis) | Toggle to include required APIs. | `bool` | `false` | no | -| [fourkeys\_service\_account\_email](#input\_fourkeys\_service\_account\_email) | Service account for fourkeys. | `string` | n/a | yes | -| [parser\_container\_url](#input\_parser\_container\_url) | URL of image to use in Cloud Run service configuration. | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | Project ID of the target project. | `string` | n/a | yes | -| [region](#input\_region) | Region to deploy resources. | `string` | `"us-central1"` | no | - -## Outputs - -No outputs. - \ No newline at end of file diff --git a/terraform/modules/fourkeys-cloud-build-parser/main.tf b/terraform/modules/fourkeys-cloud-build-parser/main.tf deleted file mode 100644 index 8a7515dd..00000000 --- a/terraform/modules/fourkeys-cloud-build-parser/main.tf +++ /dev/null @@ -1,89 +0,0 @@ -data "google_project" "project" { - project_id = var.project_id -} - -locals { - services = var.enable_apis ? [ - "run.googleapis.com" - ] : [] -} - -resource "google_project_service" "data_source_services" { - project = var.project_id - for_each = toset(local.services) - service = each.value - disable_on_destroy = false -} - -resource "google_cloud_run_service" "cloudbuild_parser" { - project = var.project_id - name = "fourkeys-cloudbuild-parser" - location = var.region - - template { - spec { - containers { - image = var.parser_container_url - env { - name = "PROJECT_NAME" - value = var.project_id - } - } - service_account_name = var.fourkeys_service_account_email - } - } - - traffic { - percent = 100 - latest_revision = true - } - - metadata { - annotations = { - "run.googleapis.com/ingress" = "internal" - } - } - - lifecycle { - ignore_changes = [ - metadata[0].annotations, - ] - } - - autogenerate_revision_name = true - depends_on = [ - google_project_service.data_source_services - ] -} - -resource "google_pubsub_topic" "cloudbuild" { - project = var.project_id - name = "cloud-builds" -} - -resource "google_pubsub_topic_iam_member" "service_account_editor" { - project = var.project_id - topic = google_pubsub_topic.cloudbuild.id - role = "roles/editor" - member = "serviceAccount:${var.fourkeys_service_account_email}" -} - -resource "google_pubsub_subscription" "cloudbuild" { - project = var.project_id - name = "cloudbuild" - topic = google_pubsub_topic.cloudbuild.id - - push_config { - push_endpoint = google_cloud_run_service.cloudbuild_parser.status[0]["url"] - - oidc_token { - service_account_email = var.fourkeys_service_account_email - } - } -} - -resource "google_project_iam_member" "pubsub_service_account_token_creator" { - project = var.project_id - member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" - role = "roles/iam.serviceAccountTokenCreator" -} \ No newline at end of file diff --git a/terraform/modules/fourkeys-cloud-build-parser/variables.tf b/terraform/modules/fourkeys-cloud-build-parser/variables.tf deleted file mode 100644 index 34ebfe1e..00000000 --- a/terraform/modules/fourkeys-cloud-build-parser/variables.tf +++ /dev/null @@ -1,26 +0,0 @@ -variable "project_id" { - type = string - description = "Project ID of the target project." -} - -variable "region" { - type = string - description = "Region to deploy resources." - default = "us-central1" -} - -variable "fourkeys_service_account_email" { - type = string - description = "Service account for fourkeys." -} - -variable "enable_apis" { - type = bool - description = "Toggle to include required APIs." - default = false -} - -variable "parser_container_url" { - type = string - description = "URL of image to use in Cloud Run service configuration." -} \ No newline at end of file diff --git a/terraform/modules/fourkeys-github-parser/README.md b/terraform/modules/fourkeys-github-parser/README.md deleted file mode 100644 index 988ea721..00000000 --- a/terraform/modules/fourkeys-github-parser/README.md +++ /dev/null @@ -1,41 +0,0 @@ - -## Requirements - -No requirements. - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | n/a | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_cloud_run_service.github_parser](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloud_run_service) | resource | -| [google_project_iam_member.pubsub_service_account_token_creator](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | -| [google_project_service.data_source_services](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_service) | resource | -| [google_pubsub_subscription.github](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_subscription) | resource | -| [google_pubsub_topic.github](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic) | resource | -| [google_pubsub_topic_iam_member.service_account_editor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic_iam_member) | resource | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [enable\_apis](#input\_enable\_apis) | Toggle to include required APIs. | `bool` | `false` | no | -| [fourkeys\_service\_account\_email](#input\_fourkeys\_service\_account\_email) | Service account for fourkeys. | `string` | n/a | yes | -| [parser\_container\_url](#input\_parser\_container\_url) | URL of image to use in Cloud Run service configuration. | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | Project ID of the target project. | `string` | n/a | yes | -| [region](#input\_region) | Region to deploy resources. | `string` | `"us-central1"` | no | - -## Outputs - -No outputs. - \ No newline at end of file diff --git a/terraform/modules/fourkeys-github-parser/main.tf b/terraform/modules/fourkeys-github-parser/main.tf deleted file mode 100644 index 1aaebe9b..00000000 --- a/terraform/modules/fourkeys-github-parser/main.tf +++ /dev/null @@ -1,89 +0,0 @@ -data "google_project" "project" { - project_id = var.project_id -} - -locals { - services = var.enable_apis ? [ - "run.googleapis.com" - ] : [] -} - -resource "google_project_service" "data_source_services" { - project = var.project_id - for_each = toset(local.services) - service = each.value - disable_on_destroy = false -} - -resource "google_cloud_run_service" "github_parser" { - project = var.project_id - name = "fourkeys-github-parser" - location = var.region - - template { - spec { - containers { - image = var.parser_container_url - env { - name = "PROJECT_NAME" - value = var.project_id - } - } - service_account_name = var.fourkeys_service_account_email - } - } - - traffic { - percent = 100 - latest_revision = true - } - - metadata { - annotations = { - "run.googleapis.com/ingress" = "internal" - } - } - - lifecycle { - ignore_changes = [ - metadata[0].annotations, - ] - } - - autogenerate_revision_name = true - depends_on = [ - google_project_service.data_source_services - ] -} - -resource "google_pubsub_topic" "github" { - project = var.project_id - name = "github" -} - -resource "google_pubsub_topic_iam_member" "service_account_editor" { - project = var.project_id - topic = google_pubsub_topic.github.id - role = "roles/editor" - member = "serviceAccount:${var.fourkeys_service_account_email}" -} - -resource "google_pubsub_subscription" "github" { - project = var.project_id - name = "github" - topic = google_pubsub_topic.github.id - - push_config { - push_endpoint = google_cloud_run_service.github_parser.status[0]["url"] - - oidc_token { - service_account_email = var.fourkeys_service_account_email - } - } -} - -resource "google_project_iam_member" "pubsub_service_account_token_creator" { - project = var.project_id - member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" - role = "roles/iam.serviceAccountTokenCreator" -} \ No newline at end of file diff --git a/terraform/modules/fourkeys-github-parser/variables.tf b/terraform/modules/fourkeys-github-parser/variables.tf deleted file mode 100644 index 34ebfe1e..00000000 --- a/terraform/modules/fourkeys-github-parser/variables.tf +++ /dev/null @@ -1,26 +0,0 @@ -variable "project_id" { - type = string - description = "Project ID of the target project." -} - -variable "region" { - type = string - description = "Region to deploy resources." - default = "us-central1" -} - -variable "fourkeys_service_account_email" { - type = string - description = "Service account for fourkeys." -} - -variable "enable_apis" { - type = bool - description = "Toggle to include required APIs." - default = false -} - -variable "parser_container_url" { - type = string - description = "URL of image to use in Cloud Run service configuration." -} \ No newline at end of file diff --git a/terraform/modules/fourkeys-gitlab-parser/README.md b/terraform/modules/fourkeys-gitlab-parser/README.md deleted file mode 100644 index 8b092bf6..00000000 --- a/terraform/modules/fourkeys-gitlab-parser/README.md +++ /dev/null @@ -1,41 +0,0 @@ - -## Requirements - -No requirements. - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | n/a | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_cloud_run_service.gitlab_parser](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloud_run_service) | resource | -| [google_project_iam_member.pubsub_service_account_token_creator](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | -| [google_project_service.data_source_services](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_service) | resource | -| [google_pubsub_subscription.gitlab](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_subscription) | resource | -| [google_pubsub_topic.gitlab](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic) | resource | -| [google_pubsub_topic_iam_member.service_account_editor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic_iam_member) | resource | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [enable\_apis](#input\_enable\_apis) | Toggle to include required APIs. | `bool` | `false` | no | -| [fourkeys\_service\_account\_email](#input\_fourkeys\_service\_account\_email) | Service account for fourkeys. | `string` | n/a | yes | -| [parser\_container\_url](#input\_parser\_container\_url) | URL of image to use in Cloud Run service configuration. | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | Project ID of the target project. | `string` | n/a | yes | -| [region](#input\_region) | Region to deploy resources. | `string` | `"us-central1"` | no | - -## Outputs - -No outputs. - \ No newline at end of file diff --git a/terraform/modules/fourkeys-gitlab-parser/main.tf b/terraform/modules/fourkeys-gitlab-parser/main.tf deleted file mode 100644 index 1fc2931b..00000000 --- a/terraform/modules/fourkeys-gitlab-parser/main.tf +++ /dev/null @@ -1,93 +0,0 @@ -data "google_project" "project" { - project_id = var.project_id -} - -locals { - services = var.enable_apis ? [ - "run.googleapis.com" - ] : [] -} - -resource "google_project_service" "data_source_services" { - project = var.project_id - for_each = toset(local.services) - service = each.value - disable_on_destroy = false -} - -resource "google_cloud_run_service" "gitlab_parser" { - project = var.project_id - name = "fourkeys-gitlab-parser" - location = var.region - - template { - spec { - containers { - image = var.parser_container_url - env { - name = "PROJECT_NAME" - value = var.project_id - } - } - service_account_name = var.fourkeys_service_account_email - } - } - - traffic { - percent = 100 - latest_revision = true - } - - metadata { - annotations = { - "run.googleapis.com/ingress" = "internal" - } - } - - lifecycle { - ignore_changes = [ - metadata[0].annotations, - ] - } - - autogenerate_revision_name = true - depends_on = [ - google_project_service.data_source_services - ] -} - -resource "google_pubsub_topic" "gitlab" { - project = var.project_id - name = "gitlab" -} - -resource "google_pubsub_topic_iam_member" "service_account_editor" { - project = var.project_id - topic = google_pubsub_topic.gitlab.id - role = "roles/editor" - member = "serviceAccount:${var.fourkeys_service_account_email}" -} - -resource "google_pubsub_subscription" "gitlab" { - project = var.project_id - name = "gitlab" - topic = google_pubsub_topic.gitlab.id - - expiration_policy { - ttl = "" - } - - push_config { - push_endpoint = google_cloud_run_service.gitlab_parser.status[0]["url"] - - oidc_token { - service_account_email = var.fourkeys_service_account_email - } - } -} - -resource "google_project_iam_member" "pubsub_service_account_token_creator" { - project = var.project_id - member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" - role = "roles/iam.serviceAccountTokenCreator" -} diff --git a/terraform/modules/fourkeys-gitlab-parser/variables.tf b/terraform/modules/fourkeys-gitlab-parser/variables.tf deleted file mode 100644 index 34ebfe1e..00000000 --- a/terraform/modules/fourkeys-gitlab-parser/variables.tf +++ /dev/null @@ -1,26 +0,0 @@ -variable "project_id" { - type = string - description = "Project ID of the target project." -} - -variable "region" { - type = string - description = "Region to deploy resources." - default = "us-central1" -} - -variable "fourkeys_service_account_email" { - type = string - description = "Service account for fourkeys." -} - -variable "enable_apis" { - type = bool - description = "Toggle to include required APIs." - default = false -} - -variable "parser_container_url" { - type = string - description = "URL of image to use in Cloud Run service configuration." -} \ No newline at end of file diff --git a/terraform/modules/fourkeys-pagerduty-parser/README.md b/terraform/modules/fourkeys-pagerduty-parser/README.md deleted file mode 100644 index f3802b12..00000000 --- a/terraform/modules/fourkeys-pagerduty-parser/README.md +++ /dev/null @@ -1,41 +0,0 @@ - -## Requirements - -No requirements. - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | n/a | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_cloud_run_service.pagerduty_parser](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloud_run_service) | resource | -| [google_project_iam_member.pubsub_service_account_token_creator](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | -| [google_project_service.data_source_services](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_service) | resource | -| [google_pubsub_subscription.pagerduty](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_subscription) | resource | -| [google_pubsub_topic.pagerduty](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic) | resource | -| [google_pubsub_topic_iam_member.service_account_editor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic_iam_member) | resource | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [enable\_apis](#input\_enable\_apis) | Toggle to include required APIs. | `bool` | `false` | no | -| [fourkeys\_service\_account\_email](#input\_fourkeys\_service\_account\_email) | Service account for fourkeys. | `string` | n/a | yes | -| [parser\_container\_url](#input\_parser\_container\_url) | URL of image to use in Cloud Run service configuration. | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | Project ID of the target project. | `string` | n/a | yes | -| [region](#input\_region) | Region to deploy resources. | `string` | `"us-central1"` | no | - -## Outputs - -No outputs. - \ No newline at end of file diff --git a/terraform/modules/fourkeys-pagerduty-parser/main.tf b/terraform/modules/fourkeys-pagerduty-parser/main.tf deleted file mode 100644 index b91df506..00000000 --- a/terraform/modules/fourkeys-pagerduty-parser/main.tf +++ /dev/null @@ -1,89 +0,0 @@ -data "google_project" "project" { - project_id = var.project_id -} - -locals { - services = var.enable_apis ? [ - "run.googleapis.com" - ] : [] -} - -resource "google_project_service" "data_source_services" { - project = var.project_id - for_each = toset(local.services) - service = each.value - disable_on_destroy = false -} - -resource "google_cloud_run_service" "pagerduty_parser" { - project = var.project_id - name = "fourkeys-pagerduty-parser" - location = var.region - - template { - spec { - containers { - image = var.parser_container_url - env { - name = "PROJECT_NAME" - value = var.project_id - } - } - service_account_name = var.fourkeys_service_account_email - } - } - - traffic { - percent = 100 - latest_revision = true - } - - metadata { - annotations = { - "run.googleapis.com/ingress" = "internal" - } - } - - lifecycle { - ignore_changes = [ - metadata[0].annotations, - ] - } - - autogenerate_revision_name = true - depends_on = [ - google_project_service.data_source_services - ] -} - -resource "google_pubsub_topic" "pagerduty" { - project = var.project_id - name = "pagerduty" -} - -resource "google_pubsub_topic_iam_member" "service_account_editor" { - project = var.project_id - topic = google_pubsub_topic.pagerduty.id - role = "roles/pubsub.editor" - member = "serviceAccount:${var.fourkeys_service_account_email}" -} - -resource "google_pubsub_subscription" "pagerduty" { - project = var.project_id - name = "pagerduty" - topic = google_pubsub_topic.pagerduty.id - - push_config { - push_endpoint = google_cloud_run_service.pagerduty_parser.status[0]["url"] - - oidc_token { - service_account_email = var.fourkeys_service_account_email - } - } -} -# This IAM role grant is for projects created before April 8, 2021. See: https://cloud.google.com/pubsub/docs/push -resource "google_project_iam_member" "pubsub_service_account_token_creator" { - project = var.project_id - member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" - role = "roles/iam.serviceAccountTokenCreator" -} \ No newline at end of file diff --git a/terraform/modules/fourkeys-pagerduty-parser/variables.tf b/terraform/modules/fourkeys-pagerduty-parser/variables.tf deleted file mode 100644 index 34ebfe1e..00000000 --- a/terraform/modules/fourkeys-pagerduty-parser/variables.tf +++ /dev/null @@ -1,26 +0,0 @@ -variable "project_id" { - type = string - description = "Project ID of the target project." -} - -variable "region" { - type = string - description = "Region to deploy resources." - default = "us-central1" -} - -variable "fourkeys_service_account_email" { - type = string - description = "Service account for fourkeys." -} - -variable "enable_apis" { - type = bool - description = "Toggle to include required APIs." - default = false -} - -variable "parser_container_url" { - type = string - description = "URL of image to use in Cloud Run service configuration." -} \ No newline at end of file diff --git a/terraform/modules/fourkeys-tekton-parser/README.md b/terraform/modules/fourkeys-tekton-parser/README.md deleted file mode 100644 index 5be7b1e7..00000000 --- a/terraform/modules/fourkeys-tekton-parser/README.md +++ /dev/null @@ -1,41 +0,0 @@ - -## Requirements - -No requirements. - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | n/a | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_cloud_run_service.tekton_parser](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloud_run_service) | resource | -| [google_project_iam_member.pubsub_service_account_token_creator](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | -| [google_project_service.data_source_services](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_service) | resource | -| [google_pubsub_subscription.tekton](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_subscription) | resource | -| [google_pubsub_topic.tekton](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic) | resource | -| [google_pubsub_topic_iam_member.service_account_editor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic_iam_member) | resource | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [enable\_apis](#input\_enable\_apis) | Toggle to include required APIs. | `bool` | `false` | no | -| [fourkeys\_service\_account\_email](#input\_fourkeys\_service\_account\_email) | Service account for fourkeys. | `string` | n/a | yes | -| [parser\_container\_url](#input\_parser\_container\_url) | URL of image to use in Cloud Run service configuration. | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | Project ID of the target project. | `string` | n/a | yes | -| [region](#input\_region) | Region to deploy resources. | `string` | `"us-central1"` | no | - -## Outputs - -No outputs. - \ No newline at end of file diff --git a/terraform/modules/fourkeys-tekton-parser/main.tf b/terraform/modules/fourkeys-tekton-parser/main.tf deleted file mode 100644 index dfcd1bd0..00000000 --- a/terraform/modules/fourkeys-tekton-parser/main.tf +++ /dev/null @@ -1,89 +0,0 @@ -data "google_project" "project" { - project_id = var.project_id -} - -locals { - services = var.enable_apis ? [ - "run.googleapis.com" - ] : [] -} - -resource "google_project_service" "data_source_services" { - project = var.project_id - for_each = toset(local.services) - service = each.value - disable_on_destroy = false -} - -resource "google_cloud_run_service" "tekton_parser" { - project = var.project_id - name = "fourkeys-tekton-parser" - location = var.region - - template { - spec { - containers { - image = var.parser_container_url - env { - name = "PROJECT_NAME" - value = var.project_id - } - } - service_account_name = var.fourkeys_service_account_email - } - } - - traffic { - percent = 100 - latest_revision = true - } - - metadata { - annotations = { - "run.googleapis.com/ingress" = "internal" - } - } - - lifecycle { - ignore_changes = [ - metadata[0].annotations, - ] - } - - autogenerate_revision_name = true - depends_on = [ - google_project_service.data_source_services - ] -} - -resource "google_pubsub_topic" "tekton" { - project = var.project_id - name = "tekton" -} - -resource "google_pubsub_topic_iam_member" "service_account_editor" { - project = var.project_id - topic = google_pubsub_topic.tekton.id - role = "roles/editor" - member = "serviceAccount:${var.fourkeys_service_account_email}" -} - -resource "google_pubsub_subscription" "tekton" { - project = var.project_id - name = "tekton" - topic = google_pubsub_topic.tekton.id - - push_config { - push_endpoint = google_cloud_run_service.tekton_parser.status[0]["url"] - - oidc_token { - service_account_email = var.fourkeys_service_account_email - } - } -} - -resource "google_project_iam_member" "pubsub_service_account_token_creator" { - project = var.project_id - member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" - role = "roles/iam.serviceAccountTokenCreator" -} diff --git a/terraform/modules/fourkeys-tekton-parser/variables.tf b/terraform/modules/fourkeys-tekton-parser/variables.tf deleted file mode 100644 index 34ebfe1e..00000000 --- a/terraform/modules/fourkeys-tekton-parser/variables.tf +++ /dev/null @@ -1,26 +0,0 @@ -variable "project_id" { - type = string - description = "Project ID of the target project." -} - -variable "region" { - type = string - description = "Region to deploy resources." - default = "us-central1" -} - -variable "fourkeys_service_account_email" { - type = string - description = "Service account for fourkeys." -} - -variable "enable_apis" { - type = bool - description = "Toggle to include required APIs." - default = false -} - -variable "parser_container_url" { - type = string - description = "URL of image to use in Cloud Run service configuration." -} \ No newline at end of file diff --git a/terraform/modules/fourkeys/README.md b/terraform/modules/fourkeys/README.md deleted file mode 100644 index f1c8a7c0..00000000 --- a/terraform/modules/fourkeys/README.md +++ /dev/null @@ -1,84 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [google](#requirement\_google) | >= 4.17.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.17.0 | -| [random](#provider\_random) | n/a | -| [time](#provider\_time) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [circleci\_parser](#module\_circleci\_parser) | ../fourkeys-circleci-parser | n/a | -| [cloud\_build\_parser](#module\_cloud\_build\_parser) | ../fourkeys-cloud-build-parser | n/a | -| [github\_parser](#module\_github\_parser) | ../fourkeys-github-parser | n/a | -| [gitlab\_parser](#module\_gitlab\_parser) | ../fourkeys-gitlab-parser | n/a | -| [pagerduty\_parser](#module\_pagerduty\_parser) | ../fourkeys-pagerduty-parser | n/a | -| [tekton\_parser](#module\_tekton\_parser) | ../fourkeys-tekton-parser | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_bigquery_dataset.four_keys](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_dataset) | resource | -| [google_bigquery_dataset_iam_member.parser_bq](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_dataset_iam_member) | resource | -| [google_bigquery_routine.func_json2array](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_routine) | resource | -| [google_bigquery_routine.func_multiFormatParseTimestamp](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_routine) | resource | -| [google_bigquery_table.events_raw](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_table) | resource | -| [google_bigquery_table.view_changes](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_table) | resource | -| [google_bigquery_table.view_deployments](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_table) | resource | -| [google_bigquery_table.view_incidents](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_table) | resource | -| [google_cloud_run_service.dashboard](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloud_run_service) | resource | -| [google_cloud_run_service.event_handler](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloud_run_service) | resource | -| [google_cloud_run_service_iam_binding.dashboard_noauth](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloud_run_service_iam_binding) | resource | -| [google_cloud_run_service_iam_binding.event_handler_noauth](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloud_run_service_iam_binding) | resource | -| [google_project_iam_member.bigquery_user](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | -| [google_project_iam_member.cloud_run_invoker](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | -| [google_project_iam_member.parser_bq_project_access](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | -| [google_project_iam_member.parser_run_invoker](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | -| [google_project_iam_member.storage_admin](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | -| [google_project_service.fourkeys_services](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_service) | resource | -| [google_secret_manager_secret.event_handler](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | -| [google_secret_manager_secret_iam_member.event_handler](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | -| [google_secret_manager_secret_version.event_handler](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | -| [google_service_account.fourkeys](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account) | resource | -| [random_id.event_handler_random_value](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [time_sleep.wait_for_services](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [bigquery\_region](#input\_bigquery\_region) | Region to deploy BigQuery resources in. | `string` | `"US"` | no | -| [circleci\_parser\_url](#input\_circleci\_parser\_url) | The URL for the CircleCI parser container image. A default value pointing to the project's container registry is defined in under local values of this module. | `string` | `""` | no | -| [cloud\_build\_parser\_url](#input\_cloud\_build\_parser\_url) | The URL for the Cloud Build parser container image. A default value pointing to the project's container registry is defined in under local values of this module. | `string` | `""` | no | -| [dashboard\_container\_url](#input\_dashboard\_container\_url) | The URL for the dashboard container image. A default value pointing to the project's container registry is defined in under local values of this module. | `string` | `""` | no | -| [enable\_apis](#input\_enable\_apis) | Toggle to include required APIs. | `bool` | `false` | no | -| [enable\_dashboard](#input\_enable\_dashboard) | Toggle to enable cloud run service creation. | `bool` | `true` | no | -| [event\_handler\_container\_url](#input\_event\_handler\_container\_url) | The URL for the event\_handler container image. A default value pointing to the project's container registry is defined in under local values of this module. | `string` | `""` | no | -| [github\_parser\_url](#input\_github\_parser\_url) | The URL for the Github parser container image. A default value pointing to the project's container registry is defined in under local values of this module. | `string` | `""` | no | -| [gitlab\_parser\_url](#input\_gitlab\_parser\_url) | The URL for the Gitlab parser container image. A default value pointing to the project's container registry is defined in under local values of this module. | `string` | `""` | no | -| [pagerduty\_parser\_url](#input\_pagerduty\_parser\_url) | The URL for the Pager Duty parser container image. A default value pointing to the project's container registry is defined in under local values of this module. | `string` | `""` | no | -| [parsers](#input\_parsers) | List of data parsers to configure. Acceptable values are: 'github', 'gitlab', 'cloud-build', 'tekton', 'circleci', 'pagerduty' | `list(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | project to deploy four keys resources to | `string` | n/a | yes | -| [region](#input\_region) | Region to deploy fource keys resources in. | `string` | `"us-central1"` | no | -| [tekton\_parser\_url](#input\_tekton\_parser\_url) | The URL for the Tekton parser container image. A default value pointing to the project's container registry is defined in under local values of this module. | `string` | `""` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [dashboard\_endpoint](#output\_dashboard\_endpoint) | n/a | -| [event\_handler\_endpoint](#output\_event\_handler\_endpoint) | n/a | -| [event\_handler\_secret](#output\_event\_handler\_secret) | n/a | -| [fourkeys\_service\_account\_email](#output\_fourkeys\_service\_account\_email) | n/a | - \ No newline at end of file diff --git a/terraform/modules/fourkeys/bigquery.tf b/terraform/modules/fourkeys/bigquery.tf deleted file mode 100644 index 17899c2a..00000000 --- a/terraform/modules/fourkeys/bigquery.tf +++ /dev/null @@ -1,105 +0,0 @@ -resource "google_bigquery_dataset_iam_member" "parser_bq" { - project = var.project_id - dataset_id = google_bigquery_dataset.four_keys.dataset_id - role = "roles/bigquery.dataEditor" - member = "serviceAccount:${google_service_account.fourkeys.email}" -} - -resource "google_bigquery_dataset" "four_keys" { - project = var.project_id - dataset_id = "four_keys" - location = var.bigquery_region - depends_on = [ - google_project_service.fourkeys_services - ] -} - -resource "google_bigquery_table" "events_raw" { - project = var.project_id - dataset_id = google_bigquery_dataset.four_keys.dataset_id - table_id = "events_raw" - schema = file("${path.module}/files/events_raw_schema.json") - deletion_protection = false - depends_on = [ - google_project_service.fourkeys_services - ] -} - -resource "google_bigquery_table" "view_changes" { - project = var.project_id - dataset_id = google_bigquery_dataset.four_keys.dataset_id - table_id = "changes" - view { - query = file("${path.module}/queries/changes.sql") - use_legacy_sql = false - } - deletion_protection = false - depends_on = [ - google_project_service.fourkeys_services, - google_bigquery_table.events_raw - ] -} - -resource "google_bigquery_routine" "func_json2array" { - project = var.project_id - dataset_id = google_bigquery_dataset.four_keys.dataset_id - routine_id = "json2array" - routine_type = "SCALAR_FUNCTION" - return_type = "{\"typeKind\": \"ARRAY\", \"arrayElementType\": {\"typeKind\": \"STRING\"}}" - language = "JAVASCRIPT" - arguments { - name = "json" - data_type = "{\"typeKind\" : \"STRING\"}" - } - definition_body = file("${path.module}/queries/function_json2array.js") - depends_on = [ - google_project_service.fourkeys_services - ] -} - -resource "google_bigquery_routine" "func_multiFormatParseTimestamp" { - project = var.project_id - dataset_id = google_bigquery_dataset.four_keys.dataset_id - routine_id = "multiFormatParseTimestamp" - routine_type = "SCALAR_FUNCTION" - return_type = "{\"typeKind\" : \"TIMESTAMP\"}" - language = "SQL" - arguments { - name = "input" - data_type = "{\"typeKind\" : \"STRING\"}" - } - definition_body = file("${path.module}/queries/function_multiFormatParseTimestamp.sql") -} - -resource "google_bigquery_table" "view_deployments" { - project = var.project_id - dataset_id = google_bigquery_dataset.four_keys.dataset_id - table_id = "deployments" - view { - query = file("${path.module}/queries/deployments.sql") - use_legacy_sql = false - } - deletion_protection = false - depends_on = [ - google_project_service.fourkeys_services, - google_bigquery_table.events_raw, - google_bigquery_routine.func_json2array - ] -} - -resource "google_bigquery_table" "view_incidents" { - project = var.project_id - dataset_id = google_bigquery_dataset.four_keys.dataset_id - table_id = "incidents" - view { - query = file("${path.module}/queries/incidents.sql") - use_legacy_sql = false - } - deletion_protection = false - depends_on = [ - google_project_service.fourkeys_services, - google_bigquery_table.events_raw, - google_bigquery_table.view_deployments, - google_bigquery_routine.func_multiFormatParseTimestamp - ] -} diff --git a/terraform/modules/fourkeys/dashboard.tf b/terraform/modules/fourkeys/dashboard.tf deleted file mode 100644 index 0600bc64..00000000 --- a/terraform/modules/fourkeys/dashboard.tf +++ /dev/null @@ -1,45 +0,0 @@ -resource "google_cloud_run_service" "dashboard" { - count = var.enable_dashboard ? 1 : 0 - name = "fourkeys-grafana-dashboard" - location = var.region - project = var.project_id - template { - spec { - containers { - ports { - name = "http1" - container_port = 3000 - } - image = local.dashboard_container_url - env { - name = "PROJECT_NAME" - value = var.project_id - } - } - service_account_name = google_service_account.fourkeys.email - } - } - - traffic { - percent = 100 - latest_revision = true - } - metadata { - labels = { "created_by" : "fourkeys" } - } - autogenerate_revision_name = true - depends_on = [ - time_sleep.wait_for_services - ] -} - -resource "google_cloud_run_service_iam_binding" "dashboard_noauth" { - count = var.enable_dashboard ? 1 : 0 - location = var.region - project = var.project_id - service = "fourkeys-grafana-dashboard" - - role = "roles/run.invoker" - members = ["allUsers"] - depends_on = [google_cloud_run_service.dashboard] -} diff --git a/terraform/modules/fourkeys/event-handler.tf b/terraform/modules/fourkeys/event-handler.tf deleted file mode 100644 index 74f6a3fb..00000000 --- a/terraform/modules/fourkeys/event-handler.tf +++ /dev/null @@ -1,70 +0,0 @@ -resource "google_cloud_run_service" "event_handler" { - name = "event-handler" - project = var.project_id - location = var.region - - template { - spec { - containers { - image = local.event_handler_container_url - env { - name = "PROJECT_NAME" - value = var.project_id - } - } - service_account_name = google_service_account.fourkeys.email - } - } - - traffic { - percent = 100 - latest_revision = true - } - - autogenerate_revision_name = true - depends_on = [ - time_sleep.wait_for_services - ] -} - -resource "google_cloud_run_service_iam_binding" "event_handler_noauth" { - location = var.region - project = var.project_id - service = google_cloud_run_service.event_handler.name - role = "roles/run.invoker" - members = ["allUsers"] - depends_on = [google_cloud_run_service.event_handler] -} - -resource "google_secret_manager_secret" "event_handler" { - project = var.project_id - secret_id = "event-handler" - replication { - user_managed { - replicas { - location = var.region - } - } - } - depends_on = [ - time_sleep.wait_for_services - ] -} - -resource "random_id" "event_handler_random_value" { - byte_length = "20" -} - -resource "google_secret_manager_secret_version" "event_handler" { - secret = google_secret_manager_secret.event_handler.id - secret_data = random_id.event_handler_random_value.hex - depends_on = [google_secret_manager_secret.event_handler] -} - -resource "google_secret_manager_secret_iam_member" "event_handler" { - project = var.project_id - secret_id = google_secret_manager_secret.event_handler.id - role = "roles/secretmanager.secretAccessor" - member = "serviceAccount:${google_service_account.fourkeys.email}" - depends_on = [google_secret_manager_secret.event_handler, google_secret_manager_secret_version.event_handler] -} diff --git a/terraform/modules/fourkeys/files/events_raw_schema.json b/terraform/modules/fourkeys/files/events_raw_schema.json deleted file mode 120000 index d023e72a..00000000 --- a/terraform/modules/fourkeys/files/events_raw_schema.json +++ /dev/null @@ -1 +0,0 @@ -../../../../setup/events_raw_schema.json \ No newline at end of file diff --git a/terraform/modules/fourkeys/iam.tf b/terraform/modules/fourkeys/iam.tf deleted file mode 100644 index 18de3aa9..00000000 --- a/terraform/modules/fourkeys/iam.tf +++ /dev/null @@ -1,41 +0,0 @@ -resource "google_project_iam_member" "parser_bq_project_access" { - project = var.project_id - role = "roles/bigquery.user" - member = "serviceAccount:${google_service_account.fourkeys.email}" -} - -resource "google_project_iam_member" "parser_run_invoker" { - project = var.project_id - member = "serviceAccount:${google_service_account.fourkeys.email}" - role = "roles/run.invoker" -} - -resource "google_service_account" "fourkeys" { - project = var.project_id - account_id = "fourkeys" - display_name = "Service Account for Four Keys resources" -} - -resource "google_project_iam_member" "storage_admin" { - project = var.project_id - role = "roles/storage.admin" - member = "serviceAccount:${local.cloud_build_service_account}" -} - -resource "google_project_iam_member" "bigquery_user" { - project = var.project_id - role = "roles/bigquery.user" - member = "serviceAccount:${google_service_account.fourkeys.email}" - depends_on = [ - google_service_account.fourkeys - ] -} - -resource "google_project_iam_member" "cloud_run_invoker" { - project = var.project_id - role = "roles/run.invoker" - member = "serviceAccount:${google_service_account.fourkeys.email}" - depends_on = [ - google_service_account.fourkeys - ] -} diff --git a/terraform/modules/fourkeys/locals.tf b/terraform/modules/fourkeys/locals.tf deleted file mode 100644 index a8f92647..00000000 --- a/terraform/modules/fourkeys/locals.tf +++ /dev/null @@ -1,21 +0,0 @@ -data "google_project" "project" { - project_id = var.project_id -} - -locals { - cloud_build_service_account = "${data.google_project.project.number}@cloudbuild.gserviceaccount.com" - event_handler_container_url = var.event_handler_container_url == "" ? format("gcr.io/%s/event-handler", var.project_id) : var.event_handler_container_url - dashboard_container_url = var.dashboard_container_url == "" ? format("gcr.io/%s/fourkeys-grafana-dashboard", var.project_id) : var.dashboard_container_url - github_parser_url = var.github_parser_url == "" ? format("gcr.io/%s/github-parser", var.project_id) : var.github_parser_url - gitlab_parser_url = var.gitlab_parser_url == "" ? format("gcr.io/%s/gitlab-parser", var.project_id) : var.gitlab_parser_url - cloud_build_parser_url = var.cloud_build_parser_url == "" ? format("gcr.io/%s/cloud-build-parser", var.project_id) : var.cloud_build_parser_url - tekton_parser_url = var.tekton_parser_url == "" ? format("gcr.io/%s/tekton-parser", var.project_id) : var.tekton_parser_url - circleci_parser_url = var.circleci_parser_url == "" ? format("gcr.io/%s/circleci-parser", var.project_id) : var.circleci_parser_url - pagerduty_parser_url = var.pagerduty_parser_url == "" ? format("gcr.io/%s/pagerduty-parser", var.project_id) : var.pagerduty_parser_url - services = var.enable_apis ? [ - "bigquery.googleapis.com", - "cloudbuild.googleapis.com", - "run.googleapis.com", - "secretmanager.googleapis.com", - ] : [] -} diff --git a/terraform/modules/fourkeys/outputs.tf b/terraform/modules/fourkeys/outputs.tf deleted file mode 100644 index 5ae57b90..00000000 --- a/terraform/modules/fourkeys/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -output "event_handler_endpoint" { - value = google_cloud_run_service.event_handler.status[0]["url"] -} - -output "event_handler_name" { - value = google_cloud_run_service.event_handler.name -} - -output "event_handler_secret" { - value = google_secret_manager_secret_version.event_handler.secret_data - sensitive = true -} - -output "dashboard_endpoint" { - value = try(google_cloud_run_service.dashboard[0].status[0]["url"], "") -} - -output "fourkeys_service_account_email" { - value = google_service_account.fourkeys.email -} - -output "dataset_id" { - value = google_bigquery_dataset.four_keys.id -} - diff --git a/terraform/modules/fourkeys/parsers.tf b/terraform/modules/fourkeys/parsers.tf deleted file mode 100644 index c7b91b74..00000000 --- a/terraform/modules/fourkeys/parsers.tf +++ /dev/null @@ -1,77 +0,0 @@ -module "circleci_parser" { - source = "../fourkeys-circleci-parser" - count = contains(var.parsers, "circleci") ? 1 : 0 - project_id = var.project_id - parser_container_url = local.circleci_parser_url - region = var.region - fourkeys_service_account_email = google_service_account.fourkeys.email - enable_apis = var.enable_apis - depends_on = [ - time_sleep.wait_for_services - ] -} - -module "github_parser" { - source = "../fourkeys-github-parser" - count = contains(var.parsers, "github") ? 1 : 0 - project_id = var.project_id - parser_container_url = local.github_parser_url - region = var.region - fourkeys_service_account_email = google_service_account.fourkeys.email - enable_apis = var.enable_apis - depends_on = [ - time_sleep.wait_for_services - ] -} - -module "gitlab_parser" { - source = "../fourkeys-gitlab-parser" - count = contains(var.parsers, "gitlab") ? 1 : 0 - project_id = var.project_id - parser_container_url = local.gitlab_parser_url - region = var.region - fourkeys_service_account_email = google_service_account.fourkeys.email - enable_apis = var.enable_apis - depends_on = [ - time_sleep.wait_for_services - ] -} - -module "pagerduty_parser" { - source = "../fourkeys-pagerduty-parser" - count = contains(var.parsers, "pagerduty") ? 1 : 0 - project_id = var.project_id - parser_container_url = local.pagerduty_parser_url - region = var.region - fourkeys_service_account_email = google_service_account.fourkeys.email - enable_apis = var.enable_apis - depends_on = [ - time_sleep.wait_for_services - ] -} - -module "tekton_parser" { - source = "../fourkeys-tekton-parser" - count = contains(var.parsers, "tekton") ? 1 : 0 - project_id = var.project_id - parser_container_url = local.tekton_parser_url - region = var.region - fourkeys_service_account_email = google_service_account.fourkeys.email - enable_apis = var.enable_apis - depends_on = [ - time_sleep.wait_for_services - ] -} - -module "cloud_build_parser" { - source = "../fourkeys-cloud-build-parser" - count = contains(var.parsers, "cloud-build") ? 1 : 0 - project_id = var.project_id - parser_container_url = local.cloud_build_parser_url - region = var.region - fourkeys_service_account_email = google_service_account.fourkeys.email - enable_apis = var.enable_apis - depends_on = [ - time_sleep.wait_for_services - ] -} \ No newline at end of file diff --git a/terraform/modules/fourkeys/provider.tf b/terraform/modules/fourkeys/provider.tf deleted file mode 100644 index 2378736f..00000000 --- a/terraform/modules/fourkeys/provider.tf +++ /dev/null @@ -1,8 +0,0 @@ -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.17.0" - } - } -} \ No newline at end of file diff --git a/terraform/modules/fourkeys/queries/changes.sql b/terraform/modules/fourkeys/queries/changes.sql deleted file mode 120000 index 72299c99..00000000 --- a/terraform/modules/fourkeys/queries/changes.sql +++ /dev/null @@ -1 +0,0 @@ -../../../../queries/changes.sql \ No newline at end of file diff --git a/terraform/modules/fourkeys/queries/deployments.sql b/terraform/modules/fourkeys/queries/deployments.sql deleted file mode 120000 index 427f39dc..00000000 --- a/terraform/modules/fourkeys/queries/deployments.sql +++ /dev/null @@ -1 +0,0 @@ -../../../../queries/deployments.sql \ No newline at end of file diff --git a/terraform/modules/fourkeys/queries/function_json2array.js b/terraform/modules/fourkeys/queries/function_json2array.js deleted file mode 120000 index 5be3f026..00000000 --- a/terraform/modules/fourkeys/queries/function_json2array.js +++ /dev/null @@ -1 +0,0 @@ -../../../../queries/function_json2array.js \ No newline at end of file diff --git a/terraform/modules/fourkeys/queries/function_multiFormatParseTimestamp.sql b/terraform/modules/fourkeys/queries/function_multiFormatParseTimestamp.sql deleted file mode 100644 index f3bb1b33..00000000 --- a/terraform/modules/fourkeys/queries/function_multiFormatParseTimestamp.sql +++ /dev/null @@ -1,20 +0,0 @@ -CASE - --- 2022-01-05 04:36:28 -0800 -or- (...)+0800 -WHEN REGEXP_CONTAINS(input, r"^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [+-][0-9]{4}$") - THEN PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S %z', input) - --- 2022-01-12T09:47:26.948+01:00 -or- (...)-0100 -WHEN REGEXP_CONTAINS(input, r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3}[+-][0-9]{2}:[0-9]{2}$") - THEN PARSE_TIMESTAMP('%Y-%m-%dT%H:%M:%E3S%Ez', input) - --- 2022-01-18 05:35:35.320020 -or- 2022-01-18 05:35:35 -WHEN REGEXP_CONTAINS(input, r"^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.?[0-9]*$") - THEN PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%E*S', input) - -ELSE - -- no regex match; attempt to cast directly to timestamp - -- (if unparseable, this will throw an error) - CAST(input AS TIMESTAMP) - -END \ No newline at end of file diff --git a/terraform/modules/fourkeys/queries/incidents.sql b/terraform/modules/fourkeys/queries/incidents.sql deleted file mode 120000 index a1599d2c..00000000 --- a/terraform/modules/fourkeys/queries/incidents.sql +++ /dev/null @@ -1 +0,0 @@ -../../../../queries/incidents.sql \ No newline at end of file diff --git a/terraform/modules/fourkeys/services.tf b/terraform/modules/fourkeys/services.tf deleted file mode 100644 index 26d4c47e..00000000 --- a/terraform/modules/fourkeys/services.tf +++ /dev/null @@ -1,13 +0,0 @@ - -resource "google_project_service" "fourkeys_services" { - project = var.project_id - for_each = toset(local.services) - service = each.value - disable_on_destroy = false -} - -resource "time_sleep" "wait_for_services" { - depends_on = [google_project_service.fourkeys_services] - - create_duration = "30s" -} \ No newline at end of file diff --git a/terraform/modules/fourkeys/variables.tf b/terraform/modules/fourkeys/variables.tf deleted file mode 100644 index df6504e7..00000000 --- a/terraform/modules/fourkeys/variables.tf +++ /dev/null @@ -1,85 +0,0 @@ -variable "project_id" { - type = string - description = "project to deploy four keys resources to" -} - -variable "region" { - type = string - default = "us-central1" - description = "Region to deploy fource keys resources in." -} - -variable "bigquery_region" { - type = string - default = "US" - description = "Region to deploy BigQuery resources in." - validation { - condition = can(regex("^(US|EU)$", var.bigquery_region)) - error_message = "The value for 'bigquery_region' must be one of: 'US','EU'." - } -} - -variable "parsers" { - type = list(string) - description = "List of data parsers to configure. Acceptable values are: 'github', 'gitlab', 'cloud-build', 'tekton', 'circleci', 'pagerduty'" -} - -variable "enable_apis" { - type = bool - description = "Toggle to include required APIs." - default = false -} - -variable "enable_dashboard" { - type = bool - description = "Toggle to enable cloud run service creation." - default = true -} - -variable "event_handler_container_url" { - type = string - description = "The URL for the event_handler container image. A default value pointing to the project's container registry is defined in under local values of this module." - default = "" -} - -variable "dashboard_container_url" { - type = string - description = "The URL for the dashboard container image. A default value pointing to the project's container registry is defined in under local values of this module." - default = "" -} - -variable "github_parser_url" { - type = string - description = "The URL for the Github parser container image. A default value pointing to the project's container registry is defined in under local values of this module." - default = "" -} - -variable "gitlab_parser_url" { - type = string - description = "The URL for the Gitlab parser container image. A default value pointing to the project's container registry is defined in under local values of this module." - default = "" -} - -variable "cloud_build_parser_url" { - type = string - description = "The URL for the Cloud Build parser container image. A default value pointing to the project's container registry is defined in under local values of this module." - default = "" -} - -variable "tekton_parser_url" { - type = string - description = "The URL for the Tekton parser container image. A default value pointing to the project's container registry is defined in under local values of this module." - default = "" -} - -variable "circleci_parser_url" { - type = string - description = "The URL for the CircleCI parser container image. A default value pointing to the project's container registry is defined in under local values of this module." - default = "" -} - -variable "pagerduty_parser_url" { - type = string - description = "The URL for the Pager Duty parser container image. A default value pointing to the project's container registry is defined in under local values of this module." - default = "" -} From e0f54a03b4fdcd12a1dd8cc38053195bf013153b Mon Sep 17 00:00:00 2001 From: Graham Beckley Date: Mon, 21 Oct 2024 10:09:18 -0400 Subject: [PATCH 2/6] Delete queries and schemas --- queries/changes.sql | 38 ----- queries/deployments.sql | 154 ------------------ queries/events.sql | 12 -- queries/function_json2array.js | 5 - .../function_multiFormatParseTimestamp.sql | 20 --- queries/incidents.sql | 107 ------------ setup/events_enriched_schema.json | 14 -- setup/events_raw_schema.json | 37 ----- 8 files changed, 387 deletions(-) delete mode 100644 queries/changes.sql delete mode 100644 queries/deployments.sql delete mode 100644 queries/events.sql delete mode 100644 queries/function_json2array.js delete mode 100644 queries/function_multiFormatParseTimestamp.sql delete mode 100644 queries/incidents.sql delete mode 100644 setup/events_enriched_schema.json delete mode 100644 setup/events_raw_schema.json diff --git a/queries/changes.sql b/queries/changes.sql deleted file mode 100644 index 9051f03f..00000000 --- a/queries/changes.sql +++ /dev/null @@ -1,38 +0,0 @@ --- Changes View: The `change_id` field is a Git commit's ID -WITH - pushes AS ( - SELECT - source, - event_type, - CASE - WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.repository.full_name') - END - AS metadata_service, - JSON_EXTRACT_SCALAR(commit, '$.id') AS change_id, - TIMESTAMP_TRUNC(TIMESTAMP(JSON_EXTRACT_SCALAR(commit, '$.timestamp')),second) AS time_created - FROM - `four_keys.events_raw` e, - -- Create a row for each element in the array of `commits` from the `metadata` field. - -- The other fields in the row are repeated for each `commit`. - UNNEST(JSON_EXTRACT_ARRAY(e.metadata, '$.commits')) AS commit - WHERE - event_type = "push" - GROUP BY - 1, - 2, - 3, - 4, - 5 - ) -SELECT - pushes.*, - service_catalog.service, -FROM - pushes -LEFT JOIN - `four_keys.services` AS service_catalog -ON - CASE - WHEN pushes.source = "github" THEN pushes.metadata_service = service_catalog.github_repository - ELSE FALSE - END diff --git a/queries/deployments.sql b/queries/deployments.sql deleted file mode 100644 index 2624238c..00000000 --- a/queries/deployments.sql +++ /dev/null @@ -1,154 +0,0 @@ --- Deployments View: For GitHub `deploy_id` is the ID of the Deployment Status. -WITH - github_repositories AS ( - SELECT - github_repository, - COUNT(DISTINCT service) AS count_services, - FROM - `four_keys.services` - GROUP BY - 1 - ), - deploys AS ( -- Cloud Build, GitHub, ArgoCD - SELECT - source, - CASE - WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.repository.full_name') - END - AS metadata_service, - CASE - WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.deployment_status.environment') - END - AS metadata_environment, - id AS deploy_id, - time_created, - CASE - WHEN source = "cloud_build" THEN JSON_EXTRACT_SCALAR(metadata, '$.substitutions.COMMIT_SHA') - WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.deployment.sha') - WHEN source = "argocd" THEN JSON_EXTRACT_SCALAR(metadata, '$.commit_sha') - END - AS main_commit, - CASE - WHEN source LIKE "github%" THEN ARRAY( SELECT JSON_EXTRACT_SCALAR(string_element, '$') FROM UNNEST(JSON_EXTRACT_ARRAY(metadata, '$.deployment.additional_sha')) AS string_element) - ELSE - ARRAY[] - END - AS additional_commits - FROM - `four_keys.events_raw` - WHERE - ( - -- Cloud Build Deployments - (source = "cloud_build" AND JSON_EXTRACT_SCALAR(metadata, '$.status') = "SUCCESS") - -- GitHub Deployments - OR (source LIKE "github%" AND event_type = "deployment_status" AND JSON_EXTRACT_SCALAR(metadata, '$.deployment_status.state') = "success") - -- ArgoCD Deployments - OR (source = "argocd" AND JSON_EXTRACT_SCALAR(metadata, '$.status') = "SUCCESS") - ) - ), - deploys_with_service AS ( - SELECT - deploys.*, - service_catalog.service, - service_catalog.production_env, - service_catalog.staging_env, - FROM - deploys - LEFT JOIN - github_repositories - ON - CASE - WHEN deploys.source = "github" THEN deploys.metadata_service = github_repositories.github_repository - ELSE FALSE - END - LEFT JOIN - `four_keys.services` AS service_catalog - ON - CASE - WHEN - deploys.source = "github" - AND github_repositories.count_services > 1 -- there's more than 1 service in our catalog linked to this GitHub repo. - AND metadata_environment LIKE '%:%' -- the GitHub deployment environment name follows the '%:%' format. - THEN - deploys.metadata_service = service_catalog.github_repository - AND SPLIT(metadata_environment, ':')[OFFSET(0)] = service_catalog.service - WHEN - deploys.source = "github" - THEN - deploys.metadata_service = service_catalog.github_repository - ELSE FALSE - END - ), - changes_raw AS ( - SELECT - source, - id, - metadata AS change_metadata, - CASE - WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.repository.full_name') - END - AS metadata_service - FROM - `four_keys.events_raw` - ), - changes_raw_with_service AS ( - SELECT - changes_raw.*, - service_catalog.service, - FROM - changes_raw - LEFT JOIN - `four_keys.services` AS service_catalog - ON - CASE - WHEN changes_raw.source = "github" THEN changes_raw.metadata_service = service_catalog.github_repository - ELSE FALSE - END - ), - deployment_changes AS ( - SELECT - deploys.source, - deploys.service, - deploys.metadata_service as deploys_service, - changes_raw.metadata_service as changes_service, - CASE - WHEN deploys.metadata_environment = production_env THEN "production" - WHEN deploys.metadata_environment = staging_env THEN "staging" - ELSE deploys.metadata_environment - END - AS environment, - deploy_id, - deploys.time_created time_created, - change_metadata, - four_keys.json2array(JSON_EXTRACT(change_metadata, '$.commits')) AS array_commits, - main_commit - FROM - deploys_with_service as deploys - JOIN - changes_raw_with_service as changes_raw - ON - ( changes_raw.service = deploys.service ) AND ( changes_raw.id = deploys.main_commit OR changes_raw.id IN UNNEST(deploys.additional_commits) ) - ) -SELECT - source, - service, - deploys_service, - changes_service, - environment, - deploy_id, - time_created, - main_commit, - ARRAY_AGG(DISTINCT JSON_EXTRACT_SCALAR(array_commits, '$.id')) AS changes, -FROM - deployment_changes -CROSS JOIN - deployment_changes.array_commits -GROUP BY - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8; diff --git a/queries/events.sql b/queries/events.sql deleted file mode 100644 index 9a9a90f0..00000000 --- a/queries/events.sql +++ /dev/null @@ -1,12 +0,0 @@ -# events table -SELECT raw.id, - raw.event_type, - raw.time_created, - raw.metadata, - enr.enriched_metadata, - raw.signature, - raw.msg_id, - raw.source -FROM four_keys.events_raw raw -JOIN four_keys.events_enriched enr - ON raw.signature = enr.events_raw_signature diff --git a/queries/function_json2array.js b/queries/function_json2array.js deleted file mode 100644 index b4b96a18..00000000 --- a/queries/function_json2array.js +++ /dev/null @@ -1,5 +0,0 @@ -if (json) { - return JSON.parse(json).map(x=>JSON.stringify(x)); -} else { - return []; -} \ No newline at end of file diff --git a/queries/function_multiFormatParseTimestamp.sql b/queries/function_multiFormatParseTimestamp.sql deleted file mode 100644 index f3bb1b33..00000000 --- a/queries/function_multiFormatParseTimestamp.sql +++ /dev/null @@ -1,20 +0,0 @@ -CASE - --- 2022-01-05 04:36:28 -0800 -or- (...)+0800 -WHEN REGEXP_CONTAINS(input, r"^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [+-][0-9]{4}$") - THEN PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S %z', input) - --- 2022-01-12T09:47:26.948+01:00 -or- (...)-0100 -WHEN REGEXP_CONTAINS(input, r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3}[+-][0-9]{2}:[0-9]{2}$") - THEN PARSE_TIMESTAMP('%Y-%m-%dT%H:%M:%E3S%Ez', input) - --- 2022-01-18 05:35:35.320020 -or- 2022-01-18 05:35:35 -WHEN REGEXP_CONTAINS(input, r"^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.?[0-9]*$") - THEN PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%E*S', input) - -ELSE - -- no regex match; attempt to cast directly to timestamp - -- (if unparseable, this will throw an error) - CAST(input AS TIMESTAMP) - -END \ No newline at end of file diff --git a/queries/incidents.sql b/queries/incidents.sql deleted file mode 100644 index 2e1fa6b0..00000000 --- a/queries/incidents.sql +++ /dev/null @@ -1,107 +0,0 @@ --- Incidents View -WITH - github_pagerduty AS ( - SELECT - source, - CASE - WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.repository.full_name') - WHEN source LIKE "pagerduty%" THEN JSON_EXTRACT_SCALAR(metadata, '$.event.data.service.summary') - END - AS metadata_service, - CASE - WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.issue.number') - WHEN source LIKE "pagerduty%" THEN JSON_EXTRACT_SCALAR(metadata, '$.event.data.id') - END - AS incident_id, - CASE - WHEN source LIKE "github%" THEN TIMESTAMP(JSON_EXTRACT_SCALAR(metadata, '$.issue.created_at')) - WHEN source LIKE "pagerduty%" THEN TIMESTAMP(JSON_EXTRACT_SCALAR(metadata, '$.event.occurred_at')) - END - AS time_created, - CASE - WHEN source LIKE "github%" THEN TIMESTAMP(JSON_EXTRACT_SCALAR(metadata, '$.issue.closed_at')) - WHEN source LIKE "pagerduty%" THEN TIMESTAMP(JSON_EXTRACT_SCALAR(metadata, '$.event.occurred_at')) - END - AS time_resolved, - REGEXP_EXTRACT(metadata, r"root cause: ([[:alnum:]]*)") AS root_cause, - CASE - WHEN source LIKE "github%" THEN REGEXP_CONTAINS(JSON_EXTRACT(metadata, '$.issue.labels'), '"name":"Incident"') - WHEN source LIKE "pagerduty%" THEN TRUE # All Pager Duty events are incident-related - END - AS bug, - FROM - four_keys.events_raw - WHERE - event_type LIKE "issue%" - OR event_type LIKE "incident%" - OR (event_type = "note" AND JSON_EXTRACT_SCALAR(metadata, '$.object_attributes.noteable_type') = 'Issue') - ), - issue AS ( - SELECT - *, - NULL as deployment_environment -- Remove this once we collect deployment_environment - FROM - github_pagerduty - UNION ALL - SELECT - source, - github_repo AS metadata_service, - incident_id, - time_created, - time_resolved, - root_cause, - TRUE as bug, - deployment_environment, -- This is expected to always be `production` - FROM - `four_keys.incidents_google_form` - ) -SELECT - source, - metadata_service, - service_catalog.service, - incident_id, - root.environment as deployment_environment, - root.deploy_id, - MIN(IF(root.time_created < issue.time_created, root.time_created, issue.time_created)) AS time_created, - MAX(time_resolved) AS time_resolved, - ARRAY_AGG(root_cause IGNORE NULLS) AS changes, -FROM - issue -LEFT JOIN - `four_keys.services` AS service_catalog -ON - CASE - WHEN issue.source = "pagerduty" THEN issue.metadata_service = service_catalog.pagerduty_service - WHEN issue.source = "github" THEN issue.metadata_service = service_catalog.github_repository - WHEN issue.source = "google_form" THEN issue.metadata_service = service_catalog.github_repository - ELSE FALSE - END -LEFT JOIN ( - SELECT - time_created, - changes, - service, - environment, - deploy_id - FROM - `four_keys.deployments` AS d, - d.changes -) AS root -ON - service_catalog.service = root.service - -- root.changes are full Git Commit SHAs in deployment_status events. - -- root_cause are either full or abbreviated Git Commit SHAs in incident events. - AND STARTS_WITH(root.changes, root_cause) - -- GitHub and PagerDuty incidents don't contain information about the environment. - -- Google Form incidents are always reported for the `production` environment. - -- The following ensures we don't correlate incidents with `staging` deployments. - AND issue.deployment_environment = root.environment -GROUP BY - 1, - 2, - 3, - 4, - 5, - 6 -HAVING - MAX(bug) IS TRUE ; diff --git a/setup/events_enriched_schema.json b/setup/events_enriched_schema.json deleted file mode 100644 index 5d65bf0e..00000000 --- a/setup/events_enriched_schema.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "mode": "REQUIRED", - "name": "events_raw_signature", - "type": "STRING", - "description": "signature of corresponding events_raw row (Foreign Key)" - }, - { - "mode": "NULLABLE", - "name": "enriched_metadata", - "type": "STRING", - "description": "The processed metadata that has been enriched with additional information" - } -] diff --git a/setup/events_raw_schema.json b/setup/events_raw_schema.json deleted file mode 100644 index 35e796eb..00000000 --- a/setup/events_raw_schema.json +++ /dev/null @@ -1,37 +0,0 @@ -[ - { - "mode": "NULLABLE", - "name": "event_type", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "id", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "metadata", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "time_created", - "type": "TIMESTAMP" - }, - { - "mode": "NULLABLE", - "name": "signature", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "msg_id", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "source", - "type": "STRING" - } -] From 7f2f02722043a329a7ba4642c4d578598b0fdf2a Mon Sep 17 00:00:00 2001 From: Graham Beckley Date: Mon, 21 Oct 2024 10:09:52 -0400 Subject: [PATCH 3/6] Delete `deprecated` directory --- setup/deprecated/.gitignore | 3 - setup/deprecated/README.md | 195 -------- setup/deprecated/changes_schema.json | 17 - setup/deprecated/cleanup.sh | 60 --- setup/deprecated/cloudbuild-helloworld.yaml | 19 - setup/deprecated/deployments_schema.json | 17 - setup/deprecated/e2e_tests.cloudbuild.yaml | 100 ----- setup/deprecated/incidents_schema.json | 22 - setup/deprecated/new_source.sh | 96 ---- setup/deprecated/setup.sh | 471 -------------------- 10 files changed, 1000 deletions(-) delete mode 100644 setup/deprecated/.gitignore delete mode 100644 setup/deprecated/README.md delete mode 100644 setup/deprecated/changes_schema.json delete mode 100755 setup/deprecated/cleanup.sh delete mode 100644 setup/deprecated/cloudbuild-helloworld.yaml delete mode 100644 setup/deprecated/deployments_schema.json delete mode 100644 setup/deprecated/e2e_tests.cloudbuild.yaml delete mode 100644 setup/deprecated/incidents_schema.json delete mode 100755 setup/deprecated/new_source.sh delete mode 100755 setup/deprecated/setup.sh diff --git a/setup/deprecated/.gitignore b/setup/deprecated/.gitignore deleted file mode 100644 index a5e10e04..00000000 --- a/setup/deprecated/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -.terraform -env.sh -golang-samples diff --git a/setup/deprecated/README.md b/setup/deprecated/README.md deleted file mode 100644 index 7e8ee312..00000000 --- a/setup/deprecated/README.md +++ /dev/null @@ -1,195 +0,0 @@ -# Installation guide - -This guide describes how to set up Four Keys with your GitHub or GitLab project. The main steps are: - -1. [Running the setup script](#running-the-setup-script) -1. Integrating with your GitHub or Git Lab repo by: - 1. [Collecting changes data](#collecting-changes-data) - 1. [Collecting deployment data](#collecting-deployment-data) - 1. [Collecting incident data](#collecting-incident-data) - -## Before you begin - -1. Install [GCloud SDK](https://cloud.google.com/sdk/install). -1. You must be owner on a Google Cloud project that has billing enabled. You may either use this project to house the architecture for the Four Keys, or you will be given the option to create new projects. If you create new projects, the original Google Cloud project will NOT be altered during set up, but the billing information from this parent project will be applied to any projects created. - -## Running the setup script - -1. Once you have your Google Cloud project, run the following setup script from the top-level directory of this repository: - - ```bash - gcloud config set project $PARENT_PROJECT_ID - cd setup - ./setup.sh 2>&1 | tee setup.log - ``` - -1. Answer the setup script's questions: - - * Would you like to create a new Google Cloud Project for the four key metrics? (y/n) - * If you choose no, you will be asked to input and confirm the ID of the project that you want to use. - * Are you using GitLab? (y/n) - * If you choose yes, the GitLab specific Pub/Sub topic, subscriptions, and worker will be created. - * Are you using GitHub? (y/n) - * If you choose yes, the GitHub specific Pub/Sub topic, subscriptions, and worker will be created. - * BigQuery setup - * If you've never setup BigQuery before, a setup page will open in your browser. - * Would you like to create a separate new project to test deployments for the four key metrics? (y/n) - * You have the option of creating a new Google Cloud project to test out doing deployments and seeing how they are tracked in the dashboard. However, if you already have a project with deployments, you may select no to skip this step. You do not need to select yes to generate mock data. - * Would you like to generate mock data? (y/n) - * If you select yes, a script will run through and send mock GitLab or GitHub events to your event-handler. This will populate your dashboard with mock data. The mock data will include the work "mock" in the source. You can generate mock data without using the setup script. See [Generating mock data](../readme.md). - -### New Google Cloud projects - -If you've chosen to create new Google Cloud projects, after the script finishes you will have an `env.sh` file specifying two new project-id's in the form of `fourkeys-XXXX` and `helloworld-XXXXX`. The `fourkeys-XXXX` project is home to all the services that collect data from your deployments, while `helloworld-XXXX` is the staging and prod deployments for your example application. - -If you ever want to remove the newly created projects and all associated data, you can run `cleanup.sh`. **Only do this when you are done experimenting with Four Keys entirely, or want to start over. Running `cleanup.sh` will remove the projects and all the collected data.** - -If you want to bulk delete many projects that you've created via the setup script, all of which will be named `fourkeys-XXXX` and `helloworld-XXXXX`, pass a flag to the cleanup script: `./cleanup.sh -b` - -### The setup explained - -The setup script does many things to help create the service architecture described in the `README.md`. The script will output the commands you would otherwise need to do manually. - -The steps are: -- Create randomly generated project names -- Creates an `env.sh` and saves the project values in it -- Set up Four Keys project - - Create project - - Link billing to parent project - - Enable APIs - - Add IAM Policy Bindings - - Create Pub/Sub Topics - - Deploy Event Handler - - Deploy BigQuery GitHub and/or GitLab Worker - - Deploy BigQuery Cloud Build Worker - - Create BigQuery Pub/Sub Subscriptions - - Create BigQuery Dataset, Tables, and Scheduled Queries -- Set up Helloworld project - - Create Google Cloud project - - Link billing to parent project - - Enable APIs - - Deploy Helloworld to staging - - Deploy Helloworld to prod -- Generate mock data using the scripts found in the `data-generator/` directory -- Connect to the DataStudio Dashboard template - - Select organization and project - - Click **Create Report** on the next screen with the list of fields - - -## Integrating with a live repo - -The setup script can create mock data, but it cannot integrate automatically with live projects. To measure your team's performance, you need to integrate to your live GitHub or GitLab repo that has ongoing deployments. You can then measure the four key metrics, and experiment with how changes, successful deployments, and failed deployments affect your metrics. - -To integrate Four Keys with a live repo, you need to: - -1. [Collect changes data](#collecting-changes-data) -1. [Collect deployment data](#collecting-deployment-data) -1. [Collect incident data](#collecting-incident-data) - -### Collecting changes data - -#### GitHub instructions - -1. Start with your GitHub repo - * If you're using the `Helloworld` sample, fork the demo by navigating to the [GitHub Repo](https://github.com/knative/docs.git) and clicking **Fork**. -1. Navigate to your repo (or forked repo) and click **Settings**. -1. Select **Webhooks** from the left hand side. -1. Click **Add Webhook**. -1. Get the Event Handler endpoint for your Four Keys service: - ```bash - . ./env.sh - gcloud config set project ${FOURKEYS_PROJECT} - gcloud run services describe event-handler --platform managed --region ${FOURKEYS_REGION} --format=yaml | grep url | head -1 | sed -e 's/ *url: //g' - ``` -1. In the **Add Webhook** interface use the Event Handler endpoint for **Payload URL**. -1. Run the following command to get the secret from Google Secrets Manager: - ```bash - gcloud secrets versions access 1 --secret="event-handler" - ``` -1. Put the secret in the box labelled **Secret**. -1. For **Content Type**, select **application/json**. -1. Select **Send me everything**. -1. Click **Add Webhook**. - -#### GitLab instructions - -1. Navigate to your repo and click **Settings**. -1. Select **Webhooks** from the menu. -1. Get the Event Handler endpoint for your Four Keys service by running the following: - ```bash - . ./env.sh - gcloud config set project ${FOURKEYS_PROJECT} - gcloud run services describe event-handler --platform managed --region ${FOURKEYS_REGION} --format=yaml | grep url | head -1 | sed -e 's/ *url: //g' - ``` -1. For **Payload URL**, use the Event Handler endpoint. -1. Run the following command to get the secret from Google Secrets Manager: - ```bash - gcloud secrets versions access 1 --secret="event-handler" - ``` -1. Put the secret in the box labelled **Secret Token**. -1. Select all the checkboxes. -1. Leave the **Enable SSL verification** selected. -1. Click **Add Webhook**. - -### Collecting deployment data - -#### Configuring Cloud Build to deploy on GitHub Pull Request merges - -1. Go back to your repo's main page. -1. At the top of the GitHub page, click **Marketplace**. -1. Search for **Cloud Build**. -1. Select **Google Cloud Build**. -1. Click **Set Up Plan**. -1. Click **Set up with Google Cloud Build**. -1. Select **Only select repositories**. -1. Fill in your forked repo. -1. Log in to Google Cloud Platform. -1. Add your new Four Keys project named `fourkeys-XXXXX`. -1. Select your repo. -1. Click **Connect repository**. -1. Click **Create push trigger**. - -And now, whenever a pull request is merged into master of your fork, Cloud Build will trigger a deploy into prod and data will flow into your Four Keys project. - -#### Configuring Cloud Build to deploy on GitLab merges - -1. Go to your Four Keys project and [create a service account](https://cloud.google.com/iam/docs/creating-managing-service-accounts#iam-service-accounts-create-console) called `gitlab-deploy`. -1. [Create a JSON service account key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#iam-service-account-keys-create-console) for your `gitlab-deploy` service account. -1. In your GitLab repo, navigate to `Settings` on the left-hand menu and then select `CI/CD`. -1. Save your account key under variables. - 1. In the **key** field, input `SERVICE_ACCOUNT`. - 1. In the **Value** field, input the JSON . - 1. Select **Protect variable**. -1. Save your Google Cloud project-id under variables. - 1. In the **key** field, input `PROJECT_ID`. - 1. In the **value** field, input your `project-id`. -1. Add a `.gitlab-ci.yml` file to your repo. - ``` - image: google/cloud-sdk:alpine - - deploy_production: - stage: deploy - environment: Production - only: - - master - script: - - echo $SERVICE_ACCOUNT > /tmp/$CI_PIPELINE_ID.json - - gcloud auth activate-service-account --key-file /tmp/$CI_PIPELINE_ID.json - - gcloud builds submit . --project $PROJECT_ID - after_script: - - rm /tmp/$CI_PIPELINE_ID.json - ``` - -This setup will trigger a deployment on any `push` to the `master` branch. - -### Collecting incident data - -Four Keys uses GitLab and/or GitHub issues to track incidents. - -#### Creating an incident - -1. Open an issue. -1. Add the tag `Incident`. -1. In the body of the issue, input `root cause: {SHA of the commit}`. - -When the incident is resolved, close the issue. Four Keys will measure the incident from the time of the deployment to when the issue is closed. diff --git a/setup/deprecated/changes_schema.json b/setup/deprecated/changes_schema.json deleted file mode 100644 index 3db2975c..00000000 --- a/setup/deprecated/changes_schema.json +++ /dev/null @@ -1,17 +0,0 @@ -[ - { - "mode": "NULLABLE", - "name": "change_id", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "time_created", - "type": "TIMESTAMP" - }, - { - "mode": "NULLABLE", - "name": "event_type", - "type": "STRING" - } -] diff --git a/setup/deprecated/cleanup.sh b/setup/deprecated/cleanup.sh deleted file mode 100755 index b5157f36..00000000 --- a/setup/deprecated/cleanup.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Bulk delete: if flag `-b` is specified, delete all projects with IDs that -# match patterns: fourkeys_* or helloworld_* - -bulk_delete=0 - -while getopts ":b" opt; do - case ${opt} in - b ) bulk_delete=1 ;; - \? ) echo "Usage: ./cleanup.sh [-b]" ;; - esac -done - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -if [[ ${bulk_delete} -gt 0 ]] -then - # disregard env variables and delete all projects matching "fourkeys-XXXXXX" or "helloworld-XXXXXX" - projects=$(gcloud projects list --filter="projectId~^fourkeys-\d{6}$ OR projectId~^helloworld-\d{6}$" --format="value(projectId)") -else - - [[ -f "$DIR/env.sh" ]] && echo "Importing environment from $DIR/env.sh..." && . $DIR/env.sh - - projects="${FOURKEYS_PROJECT}" - if [[ ! -z "${HELLOWORLD_PROJECT}" && ! -z "$(gcloud projects list --filter="projectId=${HELLOWORLD_PROJECT}" --format="value(projectId)")" ]] - then - projects="${FOURKEYS_PROJECT} ${HELLOWORLD_PROJECT}" - fi -fi - -if [ ! -z "${projects}" ]; then echo "Deleting projects..."; else echo "no projects to delete."; fi - -for project in $projects; do - echo "delete project ${project}..." - gcloud projects delete "${project}" -done - -if [ ! -z "${PARENT_PROJECT}" ] -then - gcloud config set project ${PARENT_PROJECT} -fi - -# purge env.sh file, if it exists -[[ -f "$DIR/env.sh" ]] && rm env.sh - - diff --git a/setup/deprecated/cloudbuild-helloworld.yaml b/setup/deprecated/cloudbuild-helloworld.yaml deleted file mode 100644 index 769b358f..00000000 --- a/setup/deprecated/cloudbuild-helloworld.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -steps: -- name: 'gcr.io/cloud-builders/gcloud' - args: ['app', 'deploy', '--version', '${_VERSION}'] -substitutions: - _VERSION: '(undefined)' diff --git a/setup/deprecated/deployments_schema.json b/setup/deprecated/deployments_schema.json deleted file mode 100644 index 82a40ec3..00000000 --- a/setup/deprecated/deployments_schema.json +++ /dev/null @@ -1,17 +0,0 @@ -[ - { - "mode": "NULLABLE", - "name": "deploy_id", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "time_created", - "type": "TIMESTAMP" - }, - { - "mode": "REPEATED", - "name": "changes", - "type": "STRING" - } -] diff --git a/setup/deprecated/e2e_tests.cloudbuild.yaml b/setup/deprecated/e2e_tests.cloudbuild.yaml deleted file mode 100644 index c01453dd..00000000 --- a/setup/deprecated/e2e_tests.cloudbuild.yaml +++ /dev/null @@ -1,100 +0,0 @@ -steps: - - id: Setup Script - name: 'google/cloud-sdk' - entrypoint: /bin/bash - args: - - '-c' - - | - pip3 install --upgrade pip - cd setup/ - export FOURKEYS_PROJECT=${PROJECT_ID} - - # passing in responses to the prompts - # "n\ny\n2\n1\nn\ny\n" maps to: - ### No, do not make new project - ### Yes, it's the correct project id - ### 2, Using github for version control - ### 1, Using Cloud Build for deployments - ### No, do not create a test project - ### Yes, make mock data - - printf "n\ny\n2\n1\nn\ny\n" | ./setup.sh - - - id: Check Tables are Populated - name: 'google/cloud-sdk' - entrypoint: /bin/bash - args: - - '-c' - - | - result_check="import sys, json - results = int(json.load(sys.stdin)[0]['vol']) - try: - if results < 1: - sys.exit('Table is empty') - except Exception as e: - sys.exit(e)" - - echo "Checking events_raw table" - bq query --format=json \ - 'SELECT count(*) as vol FROM four_keys.events_raw' | \ - python3 -c "$result_check" - - echo "Checking deployments table" - bq query --format=json \ - 'SELECT count(*) as vol FROM four_keys.deployments' | \ - python3 -c "$result_check" - - echo "Checking changes table" - bq query --format=json \ - 'SELECT count(*) as vol FROM four_keys.changes' | \ - python3 -c "$result_check" - - echo "Checking incidents table" - bq query --format=json \ - 'SELECT count(*) as vol FROM four_keys.incidents' | \ - python3 -c "$result_check" - - - id: Tear Down Resources - name: 'google/cloud-sdk' - entrypoint: /bin/bash - args: - - '-c' - - | - echo "Dropping BQ Resources" - set -x - bq rm -r -f -d ${PROJECT_ID}:four_keys - set +x - configs=$(bq ls --transfer_config --transfer_location us --format=json) - - config_ids=$(python3 -c "import json, subprocess - for config in $configs: - print(config['name'])") - - set -x - for config in $config_ids - do - bq rm -f --transfer_config $config - done - - echo "Removing secret" - gcloud beta secrets delete event-handler -q - - echo "Delete Cloud Run services" - gcloud run services delete event-handler --platform managed --region us-central1 -q - gcloud run services delete github-worker --platform managed --region us-central1 -q - gcloud run services delete cloud-build-worker --platform managed --region us-central1 -q - - echo "Delete Pub/Sub topics and subscriptions" - gcloud pubsub topics delete GitHub-Hookshot -q - gcloud pubsub subscriptions delete GithubSubscription -q - gcloud pubsub subscriptions delete CloudBuildSubscription -q - - echo "Turning off APIs" - gcloud services disable compute.googleapis.com - gcloud services disable run.googleapis.com - gcloud services disable bigquery.googleapis.com - gcloud services disable bigquerydatatransfer.googleapis.com - gcloud services disable bigqueryconnection.googleapis.com - gcloud services disable secretmanager.googleapis.com - set +x -timeout: '3600s' \ No newline at end of file diff --git a/setup/deprecated/incidents_schema.json b/setup/deprecated/incidents_schema.json deleted file mode 100644 index 3581a442..00000000 --- a/setup/deprecated/incidents_schema.json +++ /dev/null @@ -1,22 +0,0 @@ -[ - { - "mode": "NULLABLE", - "name": "incident_id", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "time_created", - "type": "TIMESTAMP" - }, - { - "mode": "NULLABLE", - "name": "time_resolved", - "type": "TIMESTAMP" - }, - { - "mode": "REPEATED", - "name": "changes", - "type": "STRING" - } -] diff --git a/setup/deprecated/new_source.sh b/setup/deprecated/new_source.sh deleted file mode 100755 index 934923bb..00000000 --- a/setup/deprecated/new_source.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/bash -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# TODO: update this script for Terraform - -environment(){ - # If env.sh exists, use values in there - DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - [[ -f "$DIR/env.sh" ]] && echo "Importing environment from $DIR/env.sh..." && . $DIR/env.sh - - if [[ ! ${FOURKEYS_PROJECT} ]] - # If env.sh does not exist, use current active project - then FOURKEYS_PROJECT=$(gcloud config get-value project) - fi -} - -project_prompt(){ - # Confirm project is the correct one to use for four-keys - continue=1 - while [[ ${continue} -gt 0 ]] - do - - # Prompt until project-id is correct - read -p "Would you like to use ${FOURKEYS_PROJECT} to deploy a new Cloud Run worker? (y/n) :" yesno - - if [[ ${yesno} == "y" ]] - then continue=0 - else read -p "Please input project_id: " projectid - export FOURKEYS_PROJECT=${projectid} - fi - - done -} - -source_prompt(){ - # Will be used to name the Cloud Run service and pubsub topic. Should be lowercase. - read -p "What is the nickname of your source? Eg github (lowercase): " nickname -} - -build_deploy_cloud_run(){ - # Build and deploy by copying new_source_template - echo "Creating ${nickname}-worker" - cp -R $DIR/../../bq-workers/new-source-template $DIR/../../bq-workers/${nickname}-parser - cd $DIR/../../bq-workers/${nickname}-parser - gcloud builds submit --substitutions _SOURCE=${nickname},_REGION=us-central1 \ - --project ${FOURKEYS_PROJECT} . -} - -set_permissions(){ - gcloud iam service-accounts create cloud-run-pubsub-invoker \ - --display-name "Cloud Run Pub/Sub Invoker" --project ${FOURKEYS_PROJECT} - gcloud run services add-iam-policy-binding ${nickname}-worker \ - --member="serviceAccount:cloud-run-pubsub-invoker@${FOURKEYS_PROJECT}.iam.gserviceaccount.com" \ - --role=roles/run.invoker --project ${FOURKEYS_PROJECT} --platform managed --region us-central1 -} - -setup_pubsub_topic_subscription(){ - # Get push endpoint for new service - export PUSH_ENDPOINT_URL=$(gcloud run services describe ${nickname}-worker \ - --format="value(status.url)" --project ${FOURKEYS_PROJECT} \ - --platform managed --region us-central1) - - # Create topic - echo "Creating event handler Pub/Sub topic..."; set -x - gcloud pubsub topics create ${nickname} --project ${FOURKEYS_PROJECT} - - # configure the subscription push identity - gcloud pubsub subscriptions create ${nickname}Subscription \ - --topic=${nickname} \ - --push-endpoint=${PUSH_ENDPOINT_URL} \ - --push-auth-service-account=cloud-run-pubsub-invoker@${FOURKEYS_PROJECT}.iam.gserviceaccount.com \ - --project ${FOURKEYS_PROJECT} - set +x; echo -} - - -environment -project_prompt -source_prompt -build_deploy_cloud_run -set_permissions -setup_pubsub_topic_subscription - - diff --git a/setup/deprecated/setup.sh b/setup/deprecated/setup.sh deleted file mode 100755 index 62bf748d..00000000 --- a/setup/deprecated/setup.sh +++ /dev/null @@ -1,471 +0,0 @@ -#!/bin/bash -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -environment () { - # Set values that will be overwritten if env.sh exists - RANDOM_IDENTIFIER=$((RANDOM%999999)) - export PARENT_PROJECT=$(gcloud config get-value project) - export FOURKEYS_PROJECT=$(printf "fourkeys-%06d" $RANDOM_IDENTIFIER) - export FOURKEYS_REGION=us-central1 - export HELLOWORLD_PROJECT=$(printf "helloworld-%06d" $RANDOM_IDENTIFIER) - export HELLOWORLD_REGION=us-central - export HELLOWORLD_ZONE=${HELLOWORLD_REGION}1-a - export PARENT_FOLDER=$(gcloud projects describe ${PARENT_PROJECT} --format="value(parent.id)") - export BILLING_ACCOUNT=$(gcloud beta billing projects describe ${PARENT_PROJECT} --format="value(billingAccountName)") - - export PYTHONHTTPSVERIFY=0 - - [[ -f "$DIR/env.sh" ]] && echo "Importing environment from $DIR/env.sh..." && . $DIR/env.sh - echo "Writing $DIR/env.sh..." - cat > $DIR/env.sh << EOF -export FOURKEYS_PROJECT=${FOURKEYS_PROJECT} -export FOURKEYS_REGION=${FOURKEYS_REGION} -export HELLOWORLD_PROJECT=${HELLOWORLD_PROJECT} -export HELLOWORLD_ZONE=${HELLOWORLD_ZONE} -export BILLING_ACCOUNT=${BILLING_ACCOUNT} -export PARENT_PROJECT=${PARENT_PROJECT} -export PARENT_FOLDER=${PARENT_FOLDER} -EOF -} - -create_new_project(){ - echo "Creating new project for Four Keys Dashboard..."; set -x - gcloud projects create ${FOURKEYS_PROJECT} --folder=${PARENT_FOLDER} - gcloud beta billing projects link ${FOURKEYS_PROJECT} --billing-account=${BILLING_ACCOUNT} - - set +x; echo -} - -fourkeys_project_setup () { - # Check that the Four Keys Project has a billing account - export BILLING_ACCOUNT=$(gcloud beta billing projects describe ${FOURKEYS_PROJECT} --format="value(billingAccountName)") - - if [[ ! ${BILLING_ACCOUNT} ]] - then echo "Please enable billing account on ${FOURKEYS_PROJECT}" - exit - fi - - export FOURKEYS_REGION=us-central1 - - echo "Setting up project for Four Keys Dashboard..." - get_project_number - gcloud config set project ${FOURKEYS_PROJECT}; set -x - set +x; echo - - echo "Enabling apis..."; set -x - gcloud services enable compute.googleapis.com - gcloud services enable run.googleapis.com - gcloud services enable cloudbuild.googleapis.com - gcloud services enable pubsub.googleapis.com - gcloud services enable containerregistry.googleapis.com - gcloud services enable bigquery.googleapis.com - gcloud services enable bigquerydatatransfer.googleapis.com - gcloud services enable bigqueryconnection.googleapis.com - gcloud services enable secretmanager.googleapis.com - set +x; echo - - echo "Setting Cloud Run options"; set -x - gcloud config set run/platform managed - gcloud config set run/region ${FOURKEYS_REGION} - set +x; echo - - echo "Setting up service accounts and permissions.."; set -x - gcloud projects add-iam-policy-binding ${FOURKEYS_PROJECT} \ - --member serviceAccount:${FOURKEYS_PROJECTNUM}@cloudbuild.gserviceaccount.com \ - --role roles/run.admin - gcloud projects add-iam-policy-binding ${FOURKEYS_PROJECT} \ - --member serviceAccount:${FOURKEYS_PROJECTNUM}@cloudbuild.gserviceaccount.com \ - --role roles/iam.serviceAccountUser - - echo "Deploying event-handler..."; set -x - cd $DIR/../../event-handler - gcloud builds submit --substitutions _TAG=latest,_REGION=${FOURKEYS_REGION} . - set +x; echo - - echo "Grant Cloud Pub/Sub the permission to create tokens..."; set -x - export PUBSUB_SERVICE_ACCOUNT="service-${FOURKEYS_PROJECTNUM}@gcp-sa-pubsub.iam.gserviceaccount.com" - gcloud projects add-iam-policy-binding ${FOURKEYS_PROJECT} \ - --member="serviceAccount:${PUBSUB_SERVICE_ACCOUNT}"\ - --role='roles/iam.serviceAccountTokenCreator' - - gcloud iam service-accounts create cloud-run-pubsub-invoker \ - --display-name "Cloud Run Pub/Sub Invoker" - set +x; echo - - echo "Creating source pipelines" - if [[ ${git_system} == "1" ]] - then gitlab_pipeline - fi - if [[ ${git_system} == "2" ]] - then github_pipeline - else echo "Please see the documentation to learn how to extend to sources other than GitHub or GitLab" - fi - - if [[ ${cicd_system} == "1" ]] - then cloud_build_pipeline - fi - if [[ ${cicd_system} == "2" ]] - then tekton_pipeline - fi - # Only set up GitLab pipeline if it wasn't selected as the version control system - if [[ ${cicd_system} == "3" && ${git_system} != "1" ]] - then gitlab_pipeline - else echo "Please see the documentation to learn how to extend to sources other than Cloud Build, Tekton, GitLab, or GitHub." - fi - - - echo "Creating BigQuery dataset and tables"; set -x - bq mk \ - --dataset -f \ - ${FOURKEYS_PROJECT}:four_keys - - bq mk \ - --table -f\ - ${FOURKEYS_PROJECT}:four_keys.changes \ - $DIR/changes_schema.json - - bq mk \ - --table -f\ - ${FOURKEYS_PROJECT}:four_keys.deployments \ - $DIR/deployments_schema.json - - bq mk \ - --table -f\ - ${FOURKEYS_PROJECT}:four_keys.events_raw \ - $DIR/../events_raw_schema.json - - bq mk \ - --table -f\ - ${FOURKEYS_PROJECT}:four_keys.incidents \ - $DIR/incidents_schema.json - set +x; echo - - # Create the json2array function - bq query --nouse_legacy_sql $(cat ${DIR}/../../queries/json2array.sql) - - echo "Saving Event Handler Secret in Secret Manager.." - # Set permissions so Cloud Run can access secrets - SERVICE_ACCOUNT="${FOURKEYS_PROJECTNUM}-compute@developer.gserviceaccount.com" - gcloud projects add-iam-policy-binding ${FOURKEYS_PROJECT} \ - --member=serviceAccount:$SERVICE_ACCOUNT \ - --role=roles/secretmanager.secretAccessor - - # Check if event-handler secret already exists - check_secret=$(gcloud secrets versions access "1" --secret="event-handler" 2>/dev/null) - if [[ $check_secret ]] - then - SECRET=$check_secret - else - - # If not, create and save secret - SECRET="$(python3 -c 'import secrets -print(secrets.token_hex(20))' | tr -d '\n')" - echo $SECRET | tr -d '\n' | gcloud beta secrets create event-handler \ - --replication-policy=automatic \ - --data-file=- - fi -} - -helloworld_project_setup () { - echo "Setting up project for Helloworld..."; set -x - gcloud projects create ${HELLOWORLD_PROJECT} --folder=${PARENT_FOLDER} - gcloud beta billing projects link ${HELLOWORLD_PROJECT} --billing-account=${BILLING_ACCOUNT} - gcloud config set project ${HELLOWORLD_PROJECT} - set +x; echo - - echo "Enabling apis..."; set -x - gcloud services enable compute.googleapis.com - gcloud services enable cloudbuild.googleapis.com - gcloud services enable run.googleapis.com - gcloud services enable containerregistry.googleapis.com - set +x; echo - - echo "Cloning Helloworld demo..."; set -x - cd $DIR - git clone https://github.com/knative/docs.git - set +x - - echo "Building default helloworld app..."; set -x - cd ${DIR}/docs/docs/serving/samples/hello-world/helloworld-python - gcloud builds submit --tag gcr.io/${HELLOWORLD_PROJECT}/helloworld . - set +x - - echo "Deploying to staging..."; set -x - gcloud run deploy helloworld-staging --image gcr.io/${HELLOWORLD_PROJECT}/helloworld --allow-unauthenticated - set +x - - echo "Deploying to prod..."; set -x - gcloud run deploy helloworld-prod --image gcr.io/${HELLOWORLD_PROJECT}/helloworld --allow-unauthenticated - set +x - -} - -github_pipeline(){ - echo "Creating Github Data Pipeline..." - - echo "Deploying BQ github worker..."; set -x - cd $DIR/../../bq-workers/github-parser - gcloud builds submit --substitutions _TAG=latest,_REGION=${FOURKEYS_REGION} . - set +x; echo - - echo "Creating Github Pub/Sub Topic & Subscription..." - gcloud pubsub topics create github - - gcloud run services add-iam-policy-binding github-worker \ - --member="serviceAccount:cloud-run-pubsub-invoker@${FOURKEYS_PROJECT}.iam.gserviceaccount.com" \ - --role=roles/run.invoker - - # Get push endpoint for github-worker - export PUSH_ENDPOINT_URL=$(gcloud run services describe github-worker --format="value(status.url)") - # configure the subscription push identity - gcloud pubsub subscriptions create GithubSubscription \ - --topic=github \ - --push-endpoint=${PUSH_ENDPOINT_URL} \ - --push-auth-service-account=cloud-run-pubsub-invoker@${FOURKEYS_PROJECT}.iam.gserviceaccount.com - set +x; echo - cd $DIR -} - -gitlab_pipeline(){ - echo "Creating Gitlab Data Pipeline..." - - echo "Deploying BQ gitlab worker..."; set -x - cd $DIR/../../bq-workers/gitlab-parser - gcloud builds submit --substitutions _TAG=latest,_REGION=${FOURKEYS_REGION} . - set +x; echo - - echo "Creating Github Pub/Sub Topic & Subscription..." - gcloud pubsub topics create gitlab - - gcloud run services add-iam-policy-binding gitlab-worker \ - --member="serviceAccount:cloud-run-pubsub-invoker@${FOURKEYS_PROJECT}.iam.gserviceaccount.com" \ - --role=roles/run.invoker - - # Get push endpoint for gitlab-worker - export PUSH_ENDPOINT_URL=$(gcloud run services describe gitlab-worker --format="value(status.url)") - # configure the subscription push identity - gcloud pubsub subscriptions create GitlabSubscription \ - --topic=gitlab \ - --push-endpoint=${PUSH_ENDPOINT_URL} \ - --push-auth-service-account=cloud-run-pubsub-invoker@${FOURKEYS_PROJECT}.iam.gserviceaccount.com - set +x; echo - cd $DIR -} - - -cloud_build_pipeline(){ - echo "Creating Cloud Build Data Pipeline..." - - echo "Deploying BQ cloud build worker..."; set -x - cd $DIR/../../bq-workers/cloud-build-parser - gcloud builds submit --substitutions _TAG=latest,_REGION=${FOURKEYS_REGION} . - set +x; echo - - echo "Creating cloud-builds topic..."; set -x - gcloud pubsub topics create cloud-builds - set +x; echo - - gcloud run services add-iam-policy-binding cloud-build-worker \ - --member="serviceAccount:cloud-run-pubsub-invoker@${FOURKEYS_PROJECT}.iam.gserviceaccount.com" \ - --role=roles/run.invoker - - # Get push endpoint for cloud-build-worker - export PUSH_ENDPOINT_URL=$(gcloud run services describe cloud-build-worker --format="value(status.url)") - # configure the subscription push identity - gcloud pubsub subscriptions create CloudBuildSubscription \ - --topic=cloud-builds \ - --push-endpoint=${PUSH_ENDPOINT_URL} \ - --push-auth-service-account=cloud-run-pubsub-invoker@${FOURKEYS_PROJECT}.iam.gserviceaccount.com - set +x; echo - cd $DIR -} - - -tekton_pipeline(){ - echo "Creating Tekton Data Pipeline..." - - echo "Deploying BQ tekton worker..."; set -x - cd $DIR/../../bq-workers/tekton-parser - gcloud builds submit --substitutions _TAG=latest,_REGION=${FOURKEYS_REGION} . - set +x; echo - - echo "Creating Tekton Pub/Sub Topic & Subscription..." - gcloud pubsub topics create tekton - - gcloud run services add-iam-policy-binding tekton-worker \ - --member="serviceAccount:cloud-run-pubsub-invoker@${FOURKEYS_PROJECT}.iam.gserviceaccount.com" \ - --role=roles/run.invoker - - # Get push endpoint for tekton-worker - export PUSH_ENDPOINT_URL=$(gcloud run services describe tekton-worker --format="value(status.url)") - # configure the subscription push identity - gcloud pubsub subscriptions create TektonSubscription \ - --topic=tekton \ - --push-endpoint=${PUSH_ENDPOINT_URL} \ - --push-auth-service-account=cloud-run-pubsub-invoker@${FOURKEYS_PROJECT}.iam.gserviceaccount.com - set +x; echo - cd $DIR -} - - -generate_data(){ - gcloud config set project ${FOURKEYS_PROJECT} - echo "Creating mock data..."; - export WEBHOOK=$(gcloud run services describe event-handler --format="value(status.url)") - export SECRET=$SECRET - - # Create an identity token if running in cloudbuild tests - if [[ "$(gcloud config get-value account)" == "${FOURKEYS_PROJECTNUM}@cloudbuild.gserviceaccount.com" ]] - then - export TOKEN=$(curl -X POST -H "content-type: application/json" \ - -H "Authorization: Bearer $(gcloud auth print-access-token)" \ - -d "{\"audience\": \"${WEBHOOK}\"}" \ - "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/cloud-run-pubsub-invoker@${FOURKEYS_PROJECT}.iam.gserviceaccount.com:generateIdToken" | \ - python3 -c "import sys, json; print(json.load(sys.stdin)['token'])") - fi - - if [[ ${git_system} == "1" ]] - then set -x; python3 ${DIR}/../../data-generator/generate_data.py --vc_system=gitlab - set +x - fi - if [[ ${git_system} == "2" ]] - then set -x; python3 ${DIR}/../../data-generator/generate_data.py --vc_system=github - set +x - fi - -} - -schedule_bq_queries(){ - echo "Check BigQueryDataTransfer is enabled" - enabled=$(gcloud services list --enabled --filter name:bigquerydatatransfer.googleapis.com) - - while [[ "${enabled}" != *"bigquerydatatransfer.googleapis.com"* ]] - do gcloud services enable bigquerydatatransfer.googleapis.com - # Keep checking if it's enabled - enabled=$(gcloud services list --enabled --filter name:bigquerydatatransfer.googleapis.com) - done - - echo "Creating BigQuery scheduled queries for derived tables.."; set -x - cd ${DIR}/../../queries/ - - ./schedule.sh --query_file=changes.sql --table=changes --project_id=$FOURKEYS_PROJECT - ./schedule.sh --query_file=deployments.sql --table=deployments --project_id=$FOURKEYS_PROJECT - ./schedule.sh --query_file=incidents.sql --table=incidents --project_id=$FOURKEYS_PROJECT - - set +x; echo - cd ${DIR} -} - -project_prompt(){ - # Confirm project is the correct one to use for four-keys - continue=1 - while [[ ${continue} -gt 0 ]] - do - - # Prompt until project-id is correct - if [[ ${FOURKEYS_PROJECT} ]] - then read -p "Would you like to use ${FOURKEYS_PROJECT} to deploy a new Cloud Run worker? (y/n) :" yesno - fi - - if [[ ${yesno} == "y" ]] - then continue=0 - else read -p "Please input project_id: " projectid - export FOURKEYS_PROJECT=${projectid} - fi - - done -} - -get_project_number(){ - # There is sometimes a delay in the API and the gcloud command - # Run the gcloud command until it returns a value - continue=1 - while [[ ${continue} -gt 0 ]] - do - - export FOURKEYS_PROJECTNUM=$(gcloud projects describe ${FOURKEYS_PROJECT} --format='value(projectNumber)') - if [[ ${FOURKEYS_PROJECTNUM} ]] - then continue=0 - fi - - done -} - -check_bq_status(){ - echo "Waiting for BigQuery jobs to complete..." - continue=1 - while [[ ${continue} -gt 0 ]] - do - - # Wait for BQ jobs to run - jobStatus=$(bq ls -j -a -n 10 ${FOURKEYS_PROJECT}) - if [[ "${jobStatus}" != *"PENDING"* ]] - then continue=0 - echo "BigQuery jobs done!" - fi - - done -} - -# # Main -read -p "Would you like to create a new Google Cloud Project for the four key metrics? (y/n):" new_yesno -if [[ ${new_yesno} == "y" ]] -then echo "Setting up the environment..." -environment -create_new_project -else project_prompt -fi - -# Create workers for the correct sources -read -p "Which version control system are you using? -(1) GitLab -(2) GitHub -(3) Other - -Enter a selection (1 - 3):" git_system - - -read -p "Which CI/CD system are you using? -(1) Cloud Build -(2) Tekton -(3) GitLab -(4) Other - -Enter a selection (1 - 4):" cicd_system - -fourkeys_project_setup - -read -p "Would you like to create a separate new project to test deployments for the four key metrics? (y/n):" test_yesno -if [[ ${test_yesno} == "y" ]] -then environment -helloworld_project_setup -fi - -read -p "Would you like to generate mock data? (y/n):" mock_yesno -if [[ ${mock_yesno} == "y" ]] -then generate_data -fi - -schedule_bq_queries -check_bq_status - -DATASTUDIO_URL="https://datastudio.google.com/datasources/create?connectorId=AKfycbxCOPCqhVOJQlRpOPgJ47dPZNdDu44MXbjsgKw_2-s" -python3 -m webbrowser ${DATASTUDIO_URL} -echo "Please visit $DATASTUDIO_URL to connect your data to the dashboard template." - -echo "\nSetup complete. To integrate with your own repo or other services, please see the README.md" From 0593f887361b8a543845b0b03b3b5fa49b7568a9 Mon Sep 17 00:00:00 2001 From: Graham Beckley Date: Mon, 21 Oct 2024 10:12:05 -0400 Subject: [PATCH 4/6] Delete rest of setup directory --- setup/README.md | 96 ------------------------------- setup/data_parser/main.tf | 56 ------------------ setup/data_parser/outputs.tf | 3 - setup/data_parser/variables.tf | 15 ----- setup/fourkeys-builder/Dockerfile | 8 --- setup/fourkeys-builder/README.md | 9 --- 6 files changed, 187 deletions(-) delete mode 100644 setup/README.md delete mode 100644 setup/data_parser/main.tf delete mode 100644 setup/data_parser/outputs.tf delete mode 100644 setup/data_parser/variables.tf delete mode 100644 setup/fourkeys-builder/Dockerfile delete mode 100644 setup/fourkeys-builder/README.md diff --git a/setup/README.md b/setup/README.md deleted file mode 100644 index a7e0ba7f..00000000 --- a/setup/README.md +++ /dev/null @@ -1,96 +0,0 @@ -# Installation guide -This guide describes how to set up Four Keys with your GitHub or GitLab project. The main steps are: - - -1. Forking or cloning this repository -1. Building required images with Cloud Build -1. Providing values for required Terraform variables -1. Executing Terraform to deploy resources -1. Generating sample data (optional) - -> Alternatively, to deploy Four Keys as a remote Terraform module, see [`terraform/modules/fourkeys/README.md`](../terraform/modules/fourkeys/README.md) - -## Before you begin - -To deploy Four Keys with Terraform, you will first need: - -* A Google Cloud project with billing enabled -* The owner role assigned to you on the project -* The [Google Cloud CLI](https://cloud.google.com/sdk/docs/install) and [Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) installed on your local machine. We recommend deploying from [Cloud Shell](https://shell.cloud.google.com/?show=ide%2Cterminal) on your Google Cloud project. - -## Deploying with Terraform - -1. Set an environment variable indicating your Google Cloud project ID: - ```sh - export PROJECT_ID="YOUR_PROJECT_ID" - ``` - -1. Clone the fourkeys git repository and change into the root directory - ``` - git clone https://github.com/mozilla-services/fourkeys.git && cd fourkeys - ``` - -1. Use Cloud Build to build and push containers to Google Container Registry for the dashboard, event-handler: - ``` - gcloud builds submit dashboard --config=dashboard/cloudbuild.yaml --project $PROJECT_ID && \ - gcloud builds submit event-handler --config=event-handler/cloudbuild.yaml --project $PROJECT_ID - ``` - -1. Use Cloud Build to build and push containers to Google Container Registry for the parsers you plan to use. See the [`bq-workers`](../bq-workers/) for available options. GitHub for example: - ``` - gcloud builds submit bq-workers --config=bq-workers/parsers.cloudbuild.yaml --project $PROJECT_ID --substitutions=_SERVICE=github - ``` - -1. Change your working directory to `terraform/example` and rename `terraform.tfvars.example` to `terraform.tfvars` - ``` - cd terraform/example && mv terraform.tfvars.example terraform.tfvars - ``` - -1. Edit `terraform.tfvars` with values for the required variables. See `variables.tf` for a list of the variables, along with their descriptions and default values. Values not defined in `terraform.tfvars` will use default values defined in `variables.tf` - -1. Run the following commands from the `example` directory: - - `terraform init` to inialize Terraform and download the module - - `terraform plan` to preview changes. - - `terraform apply` to deploy the resources. - -## Generating mock data - -To test your Four Keys deployment, you can generate mock data that simulates events from a GitHub repository. - -1. Export your event handler URL an environment variable. This is the webhook URL that will receive events: - - ```sh - export WEBHOOK=`gcloud run services list --project $PROJECT_ID | grep event-handler | awk '{print $4}'` - ``` - -1. Export your event handler secret to an environment variable. This is the secret used to authenticate events sent to the webhook: - - ```sh - export SECRET=`gcloud secrets versions access 1 --secret=event-handler --project $PROJECT_ID` - ``` - -1. From the root of the fourkeys project run: - - ```sh - python3 data-generator/generate_data.py --vc_system=github - ``` - - You can see these events being run through the pipeline: - * The event handler logs show successful requests - * The Pub/Sub topic show messages posted - * The BigQuery GitHub parser show successful requests - -1. View the generated data in the `events_raw` table in with bq: - - ```sh - bq query --project_id $PROJECT_ID 'SELECT * FROM four_keys.events_raw WHERE source = "githubmock";' - ``` - - Or query the table directly in [BigQuery](https://console.cloud.google.com/bigquery): - - ```sql - SELECT * FROM four_keys.events_raw WHERE source = 'githubmock'; - ``` diff --git a/setup/data_parser/main.tf b/setup/data_parser/main.tf deleted file mode 100644 index 07e90437..00000000 --- a/setup/data_parser/main.tf +++ /dev/null @@ -1,56 +0,0 @@ -resource "google_cloud_run_service" "parser" { - name = "${var.parser_service_name}-parser" - location = var.google_region - - template { - spec { - containers { - image = "gcr.io/${var.google_project_id}/${var.parser_service_name}-parser" - env { - name = "PROJECT_NAME" - value = var.google_project_id - } - } - service_account_name = var.fourkeys_service_account_email - } - } - - traffic { - percent = 100 - latest_revision = true - } - - autogenerate_revision_name = true - - metadata { - labels = { "created_by" : "fourkeys" } - } - -} - -resource "google_pubsub_topic" "parser" { - name = var.parser_service_name - labels = { "created_by" : "fourkeys" } -} - -resource "google_pubsub_topic_iam_member" "event_handler" { - topic = google_pubsub_topic.parser.id - role = "roles/editor" - member = "serviceAccount:${var.fourkeys_service_account_email}" -} - -resource "google_pubsub_subscription" "parser" { - name = "${var.parser_service_name}-subscription" - topic = google_pubsub_topic.parser.id - - push_config { - push_endpoint = google_cloud_run_service.parser.status[0]["url"] - - oidc_token { - service_account_email = var.fourkeys_service_account_email - } - - } - labels = { "created_by" : "fourkeys" } - -} diff --git a/setup/data_parser/outputs.tf b/setup/data_parser/outputs.tf deleted file mode 100644 index 194bfc38..00000000 --- a/setup/data_parser/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "cloud_run_endpoint" { - value = google_cloud_run_service.parser.status[0]["url"] -} \ No newline at end of file diff --git a/setup/data_parser/variables.tf b/setup/data_parser/variables.tf deleted file mode 100644 index 434f8229..00000000 --- a/setup/data_parser/variables.tf +++ /dev/null @@ -1,15 +0,0 @@ -variable "parser_service_name" { - type = string -} - -variable "google_project_id" { - type = string -} - -variable "google_region" { - type = string -} - -variable "fourkeys_service_account_email" { - type = string -} \ No newline at end of file diff --git a/setup/fourkeys-builder/Dockerfile b/setup/fourkeys-builder/Dockerfile deleted file mode 100644 index 7b78a170..00000000 --- a/setup/fourkeys-builder/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM gcr.io/google.com/cloudsdktool/cloud-sdk:slim - -ARG VERSION=1.0.7 - -RUN apt install unzip -y -RUN curl https://releases.hashicorp.com/terraform/${VERSION}/terraform_${VERSION}_linux_amd64.zip -o terraform.zip -RUN unzip terraform.zip && rm terraform.zip -RUN mv terraform /usr/bin \ No newline at end of file diff --git a/setup/fourkeys-builder/README.md b/setup/fourkeys-builder/README.md deleted file mode 100644 index 6b32f474..00000000 --- a/setup/fourkeys-builder/README.md +++ /dev/null @@ -1,9 +0,0 @@ -This folder contains a Dockerfile to make a builder container for use -in Cloud Build... the container has gcloud, python 3, and Terraform installed, -all of which are needed to install and test Four Keys. - -Before using this in Cloud Build, publish the builder to your GCP project, by running the following command in this folder: - -``` -gcloud builds submit -t gcr.io/$(gcloud config list project --format="value(core.project)")/fourkeys-builder -``` \ No newline at end of file From e4e398c873018d29d451e5ac519e44a0f4db2105 Mon Sep 17 00:00:00 2001 From: Graham Beckley Date: Mon, 21 Oct 2024 10:53:54 -0400 Subject: [PATCH 5/6] Remove references to now-obsolete setup instructions in documentation --- README.md | 42 +----------------------------------------- dashboard/README.md | 5 ----- 2 files changed, 1 insertion(+), 46 deletions(-) diff --git a/README.md b/README.md index d713da74..1a6eeda6 100644 --- a/README.md +++ b/README.md @@ -50,36 +50,14 @@ This diagram shows the design of the Four Keys system: * Contains a Python script for generating mock GitHub or Gitlab data. * `event-handler/` * Contains the code for the `event-handler`, which is the public service that accepts incoming webhooks. -* `queries/` - * Contains the SQL queries for creating the derived tables. -* `setup/` - * Contains the code for setting up and tearing down the Four Keys pipeline. Also contains a script for extending the data sources. * `shared/` * Contains a shared module for inserting data into BigQuery, which is used by the `bq-workers` -* `terraform/` - * Contains Terraform modules and submodules, and examples for deploying Four Keys using Terraform. # How to use -## Out of the box - -_The project uses Python 3 and supports data extraction for Cloud Build and GitHub events._ - -1. Fork this project. -1. Run the automation scripts, which does the following (See the [setup README](setup/README.md) for more details): - 1. Create and deploy the Cloud Run webhook target and ETL workers. - 1. Create the Pub/Sub topics and subscriptions. - 1. Enable the Google Secret Manager and create a secret for your GitHub repo. - 1. Create a BigQuery dataset, tables and views. - 1. Output a URL for the newly generated Grafana dashboard. -1. Set up your development environment to send events to the webhook created in the second step. - 1. Add the secret to your GitHub webhook. - _NOTE: Make sure you don't use "Squash Merging" in Git when merging back into trunk. This breaks the link between the commit into trunk and the commits from the branch you developed on and as thus it is not possible to measure "Time to Change" on these commits. It is possible to disable this feature in the settings of your repo_ ## Generating mock data -The setup script includes an option to generate mock data. Generate mock data to play with and test the Four Keys project. - The data generator creates mocked GitHub events, which are ingested into the table with the source “githubmock.” It creates following events: * 5 mock commits with timestamps no earlier than a week ago @@ -88,7 +66,7 @@ The data generator creates mocked GitHub events, which are ingested into the tab * Associated mock incidents * _Note: By default, less than 15% of deployments create a mock incident. This threshold can be adjusted in the script._ -To run outside of the setup script: +To run this data generator: 1. Ensure that you’ve saved your webhook URL and secret in your environment variables: @@ -114,24 +92,6 @@ To run outside of the setup script: SELECT * FROM four_keys.events_raw WHERE source = 'githubmock'; ``` -## Reclassifying events / updating your queries - -The scripts consider some events to be “changes”, “deploys”, and “incidents.” You may want to reclassify one or more of these events, for example, if you want to use a label for your incidents other than “incident.” To reclassify one of the events in the table, no changes are required on the architecture or code of the project. - -1. Update the view in BigQuery for the following tables: - - * `four_keys.changes` - * `four_keys.deployments` - * `four_keys.incidents` - - To update the view, we recommend that you update the `sql` files in the `queries` folder, rather than in the BigQuery UI. - -1. Once you've edited the SQL, run `terraform apply` to update the view that populates the table: - - ```sh - cd ./setup && terraform apply - ``` - Notes: * To feed into the dashboard, the table name should be one of `changes`, `deployments`, `incidents`. diff --git a/dashboard/README.md b/dashboard/README.md index e3dc958a..11e2f7c5 100644 --- a/dashboard/README.md +++ b/dashboard/README.md @@ -29,8 +29,3 @@ The dashboard is running in a transient container. It does not store data. Ther To rebuild and deploy the container, you can run `gcloud builds submit` in this directory. - -## To deploy dashboard -If using [Terraform](https://www.terraform.io), please see the [setup](../setup/) to create the resources. - -Once the resource is created or if you are not using Terraform, feel free to build and deploy outside of Terraform by running `gcloud builds submit` in this directory. From 3d442a523935abc298503bd2304ad180a13a24d5 Mon Sep 17 00:00:00 2001 From: Graham Beckley Date: Mon, 21 Oct 2024 10:54:35 -0400 Subject: [PATCH 6/6] Remove data refresh script Relied on `setup` dir assets, and isn't something we'll be using since we're running this system in production --- data-generator/tools/refresh_data.sh | 72 ---------------------------- 1 file changed, 72 deletions(-) delete mode 100755 data-generator/tools/refresh_data.sh diff --git a/data-generator/tools/refresh_data.sh b/data-generator/tools/refresh_data.sh deleted file mode 100755 index 73f8362b..00000000 --- a/data-generator/tools/refresh_data.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Utility script to add mock data and refresh derived tables - -set -eo pipefail - -# get current execution path -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -if [[ -z "$WEBHOOK" || -z "$SECRET" ]]; then -echo "Unable to proceed. Please ensure the following environment variables \ -are set: WEBHOOK, SECRET" -exit 1 -fi - -read -p "Which version control system are you using? -(1) GitLab -(2) GitHub - -Enter a selection (1 - 2): " git_system - -if [[ ${git_system} != "1" && ${git_system} != "2" ]]; then - echo "Invalid choice for version control system" - exit 1 -fi - -purge_data="n" -read -p "Do you want to delete existing data from the project? (y/n): " purge_data - -if [[ ${purge_data} == "y" ]]; then - - yesno="n" - read -p "Are you sure? This will delete ALL data that has been collected in project ${FOURKEYS_PROJECT} (y/n): " yesno - if [[ ${yesno} != "y" ]]; then - echo "Aborting." - exit 0 - fi - - # drop and recreate the events_raw table and events_enriched table - # (why not delete the data? Because delete may fail due to https://stackoverflow.com/questions/43085896) - bq query --use_legacy_sql=false "DROP TABLE IF EXISTS ${FOURKEYS_PROJECT}.four_keys.events_raw" - bq query --use_legacy_sql=false "DROP TABLE IF EXISTS ${FOURKEYS_PROJECT}.four_keys.events_enriched" - bq mk --table -f ${FOURKEYS_PROJECT}:four_keys.events_raw ${DIR}/../../setup/events_raw_schema.json - bq mk --table -f ${FOURKEYS_PROJECT}:four_keys.events_enriched ${DIR}/../../setup/events_enriched_schema.json -fi - -# insert new data -if [[ ${git_system} == "1" ]]; then - vcs_name="gitlab" -elif [[ ${git_system} == "2" ]]; then - vcs_name="github" -fi -python3 ${DIR}/../generate_data.py --vc_system="$vcs_name" - -# run scheduled queries -for table in changes deployments incidents; do - scheduled_query=$(bq ls --transfer_config --transfer_location=US | grep "four_keys_$table" -m 1 | awk '{print $1;}') - bq mk --transfer_run --run_time "$(date --iso-8601=seconds)" $scheduled_query -done \ No newline at end of file