diff --git a/.github/workflows/precommit.yaml b/.github/workflows/precommit.yaml new file mode 100644 index 00000000..e85ce661 --- /dev/null +++ b/.github/workflows/precommit.yaml @@ -0,0 +1,21 @@ +name: pre-commit + +on: +- pull_request + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - name: Clone Repository + uses: actions/checkout@v2 + - run: python -m pip install pre-commit + shell: bash + - run: python -m pip freeze --local + shell: bash + - uses: actions/cache@v3 + with: + path: ~/.cache/pre-commit + key: pre-commit-3|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} + - run: pre-commit run --show-diff-on-failure --color=always --all-files + shell: bash \ No newline at end of file diff --git a/.gitignore b/.gitignore index 78e0e9ba..6d404065 100644 --- a/.gitignore +++ b/.gitignore @@ -144,8 +144,8 @@ dmypy.json crash.log # Exclude all .tfvars files, which are likely to contain sentitive data, such as -# password, private keys, and other secrets. These should not be part of version -# control as they are data points which are potentially sensitive and subject +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject # to change depending on the environment. # *.tfvars diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..ca37b9ae --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,24 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.1.0 + hooks: + - id: trailing-whitespace + - id: check-added-large-files + - id: check-docstring-first + - id: check-case-conflict + - id: check-merge-conflict + - id: check-yaml + exclude: .*/k8s/.*/templates/.*\.yaml$ + - id: check-json + - id: detect-private-key + - id: pretty-format-json + args: [ + '--autofix', + '--indent', '4', + '--no-sort-keys', + ] + - repo: https://github.com/antonbabenko/pre-commit-terraform + rev: v1.31.0 + hooks: + - id: terraform_fmt + exclude: \.terraform\/.*$ \ No newline at end of file diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index bdd31c33..11f88778 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -69,7 +69,7 @@ dispute. If you are unable to resolve the matter for any reason, or if the behavior is threatening or harassing, report it. We are dedicated to providing an environment where participants feel welcome and safe. -Reports should be directed to Dina Graves Portman dinagraves@google.com and +Reports should be directed to Dina Graves Portman dinagraves@google.com and Don McCasland donmccasland@google.com, the Project Steward(s) for Four Keys. It is the Project Steward’s duty to receive and address reported violations of the code of conduct. They will then diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ecbefde6..314cd76a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,17 +30,17 @@ This project follows ## Discussion -Feel free to join us on the #fourkeys channel on the [Google Cloud Platform Slack](https://goo.gle/gcp-slack)! +Feel free to join us on the #fourkeys channel on the [Google Cloud Platform Slack](https://goo.gle/gcp-slack)! ## Office Hours -We host regular office hours for users needing help with installation, or for general discussion. +We host regular office hours for users needing help with installation, or for general discussion. Book an appointment on our [Four Keys Office Hours appointment calendar](https://bit.ly/fourkeys-officehours) -- feel free to book multiple slots if you have a lot of questions! When booking, please (briefly) list your questions in the "description" area. ## Meetups -We'll be hosting quarterly meetups to discuss proposed changes, integrations, roadmaps, etc. +We'll be hosting quarterly meetups to discuss proposed changes, integrations, roadmaps, etc. Next Meetup TBD diff --git a/METRICS.md b/METRICS.md index 97ac7309..4a3d6203 100644 --- a/METRICS.md +++ b/METRICS.md @@ -4,11 +4,11 @@ This page describes the calculation of each metric that the Four Keys dashboard ![Image of the Four Keys dashboard.](images/dashboard.png) -For each of the metrics, the dashboard shows a running daily calculation, as well as a 3 month bucketed view. The buckets are categorized per the [2019 State of DevOps Report](https://www.devops-research.com/research.html#reports). +For each of the metrics, the dashboard shows a running daily calculation, as well as a 3 month bucketed view. The buckets are categorized per the [2019 State of DevOps Report](https://www.devops-research.com/research.html#reports). ## Deployment Frequency ## -**Definition**: How frequently a team successfully releases to production, e.g., daily, weekly, monthly, yearly. +**Definition**: How frequently a team successfully releases to production, e.g., daily, weekly, monthly, yearly. ### Daily Deployment Volumes ### ![Image of chart from the Four Keys dashboard, showing the daily deployment volume.](images/daily_deployments.png) @@ -25,7 +25,7 @@ four_keys.deployments GROUP BY day; ``` -### Calculating the bucket ### +### Calculating the bucket ### ![Image of chart from the Four Keys dashboard, showing the deployment frequency.](images/deployment_frequency.png) Here we see more complexity. The first thing to consider is that we need rows for the days with no deployments. To achieve this, we unpack a date array to join against our table, which will create Null values for days without deployments. @@ -54,12 +54,12 @@ LEFT JOIN( FROM four_keys.deployments) deployments ON deployments.day = last_three_months.day; ``` -Now we have a full picture of the last three months and will use this to calculate the frequency. To do this we have to decide what each bucket means. +Now we have a full picture of the last three months and will use this to calculate the frequency. To do this we have to decide what each bucket means. - **Daily**: Over the last three months, the median number of days per week with deployments is equal to or greater than three; ie, most working days have deployments. - **Weekly**: Over the last three months, the median number of days per week with deployments is at least 1; ie, most weeks have at least one deployment. - **Monthly**: Over the last three months, the median number of deployments per month is at least 1; ie, most months have at least one deployment. -- **Yearly**: Any frequency slower than Monthly. This is the else statement and will default to Yearly if the above conditions are not met. +- **Yearly**: Any frequency slower than Monthly. This is the else statement and will default to Yearly if the above conditions are not met. ```sql WITH last_three_months AS @@ -75,10 +75,10 @@ WHERE day > (SELECT date(min(time_created)) FROM four_keys.events_raw) ) SELECT -CASE WHEN daily THEN "Daily" - WHEN weekly THEN "Weekly" +CASE WHEN daily THEN "Daily" + WHEN weekly THEN "Weekly" # If at least one per month, then Monthly - WHEN PERCENTILE_CONT(monthly_deploys, 0.5) OVER () >= 1 THEN "Monthly" + WHEN PERCENTILE_CONT(monthly_deploys, 0.5) OVER () >= 1 THEN "Monthly" ELSE "Yearly" END as deployment_frequency FROM ( @@ -88,7 +88,7 @@ FROM ( # If most weeks have a deployment, then Weekly PERCENTILE_CONT(week_deployed, 0.5) OVER() >= 1 AS weekly, - # Count the number of deployments per month. + # Count the number of deployments per month. # Cannot mix aggregate and analytic functions, so calculate the median in the outer select statement SUM(week_deployed) OVER(partition by TIMESTAMP_TRUNC(week, MONTH)) monthly_deploys FROM( @@ -127,7 +127,7 @@ FROM four_keys.deployments d, d.changes LEFT JOIN four_keys.changes c ON changes = c.change_id; ``` -From this base, we want to extract the daily median lead time to change. +From this base, we want to extract the daily median lead time to change. ```sql SELECT @@ -138,7 +138,7 @@ FROM ( day, PERCENTILE_CONT( # Ignore automated changes - IF(time_to_change_minutes > 0, time_to_change_minutes, NULL), + IF(time_to_change_minutes > 0, time_to_change_minutes, NULL), 0.5) # Median OVER (partition by day) AS median_time_to_change FROM ( @@ -175,15 +175,15 @@ FROM ( GROUP BY day ORDER BY day; ``` -Automated changes are excluded from this metric. This is a subject up for debate. Our rationale is that when we merge a Pull Request it creates a Push event in the main branch. This Push event is not its own distinct change, but rather a link in the workflow. If we trigger a deployment off of this push event, this artificially skews the metrics and does not give us a clear picture of developer velocity. +Automated changes are excluded from this metric. This is a subject up for debate. Our rationale is that when we merge a Pull Request it creates a Push event in the main branch. This Push event is not its own distinct change, but rather a link in the workflow. If we trigger a deployment off of this push event, this artificially skews the metrics and does not give us a clear picture of developer velocity. ### Calculating the bucket ### ![Image of chart from the Four Keys dashboard, showing the median Lead Time to Change.](images/lead_time.png) -To get the buckets, rather than aggregating daily, we look at the last 3 months and bucket the results according to the DORA research. +To get the buckets, rather than aggregating daily, we look at the last 3 months and bucket the results according to the DORA research. ```sql -SELECT +SELECT CASE WHEN median_time_to_change < 24 * 60 then "One day" WHEN median_time_to_change < 168 * 60 then "One week" @@ -210,7 +210,7 @@ FROM ( ## Time to Restore Services ## -**Definition**: For a failure, the median amount of time between the deployment which caused the failure and the remediation. The remediation is measured by closing an associated bug / incident report. The timestamps in the `four_keys.incidents` table should align with these events, whenever available. +**Definition**: For a failure, the median amount of time between the deployment which caused the failure and the remediation. The remediation is measured by closing an associated bug / incident report. The timestamps in the `four_keys.incidents` table should align with these events, whenever available. ### Daily Median Time to Restore Services ### ![Image of chart from the Four Keys dashboard, showing the daily MTTR.](images/daily_mttr.png) @@ -286,7 +286,7 @@ SELECT CASE WHEN change_fail_rate <= .15 then "0-15%" WHEN change_fail_rate < .46 then "16-45%" ELSE "46-60%" end as change_fail_rate -FROM +FROM (SELECT SUM(IF(i.incident_id is NULL, 0, 1)) / COUNT(DISTINCT deploy_id) as change_fail_rate FROM four_keys.deployments d, d.changes diff --git a/README.md b/README.md index 74299316..c13d79fb 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ These four key metrics are: Use Four Keys if: -* You want to measure your team's software delivery performance. For example, you may want to track the impact of new tooling or more automated test coverage, or you may want a baseline of your team's performance. +* You want to measure your team's software delivery performance. For example, you may want to track the impact of new tooling or more automated test coverage, or you may want a baseline of your team's performance. * You have a project in GitHub or GitLab. * Your project has deployments. @@ -49,7 +49,7 @@ This diagram shows the design of the Four Keys system: * `data-generator/` * Contains a Python script for generating mock GitHub or Gitlab data. * `event-handler/` - * Contains the code for the `event-handler`, which is the public service that accepts incoming webhooks. + * Contains the code for the `event-handler`, which is the public service that accepts incoming webhooks. * `queries/` * Contains the SQL queries for creating the derived tables. * `setup/` @@ -59,7 +59,7 @@ This diagram shows the design of the Four Keys system: * `terraform/` * Contains Terraform modules and submodules, and examples for deploying Four Keys using Terraform. -# How to use +# How to use ## Out of the box @@ -71,7 +71,7 @@ _The project uses Python 3 and supports data extraction for Cloud Build and GitH 1. Create the Pub/Sub topics and subscriptions. 1. Enable the Google Secret Manager and create a secret for your GitHub repo. 1. Create a BigQuery dataset, tables and views. - 1. Output a URL for the newly generated Grafana dashboard. + 1. Output a URL for the newly generated Grafana dashboard. 1. Set up your development environment to send events to the webhook created in the second step. 1. Add the secret to your GitHub webhook. @@ -80,12 +80,12 @@ _NOTE: Make sure you don't use "Squash Merging" in Git when merging back into tr The setup script includes an option to generate mock data. Generate mock data to play with and test the Four Keys project. -The data generator creates mocked GitHub events, which are ingested into the table with the source “githubmock.” It creates following events: +The data generator creates mocked GitHub events, which are ingested into the table with the source “githubmock.” It creates following events: * 5 mock commits with timestamps no earlier than a week ago * _Note: Number can be adjusted_ * 1 associated deployment -* Associated mock incidents +* Associated mock incidents * _Note: By default, less than 15% of deployments create a mock incident. This threshold can be adjusted in the script._ To run outside of the setup script: @@ -128,13 +128,13 @@ The scripts consider some events to be “changes”, “deploys”, and “inci 1. Once you've edited the SQL, run `terraform apply` to update the view that populates the table: - ```sh + ```sh cd ./setup && terraform apply ``` -Notes: +Notes: -* To feed into the dashboard, the table name should be one of `changes`, `deployments`, `incidents`. +* To feed into the dashboard, the table name should be one of `changes`, `deployments`, `incidents`. ## Extending to other event sources @@ -151,7 +151,7 @@ To add other event sources: ## Running tests -This project uses nox to manage tests. The `noxfile` defines what tests run on the project. It’s set up to run all the `pytest` files in all the directories, as well as run a linter on all directories. +This project uses nox to manage tests. The `noxfile` defines what tests run on the project. It’s set up to run all the `pytest` files in all the directories, as well as run a linter on all directories. To run nox: @@ -180,10 +180,10 @@ python3 -m nox -l Once you have the list of test sessions, you can run a specific session with: ```sh -python3 -m nox -s "{name_of_session}" +python3 -m nox -s "{name_of_session}" ``` -The "name_of_session" will be something like "py-3.6(folder='.....'). +The "name_of_session" will be something like "py-3.6(folder='.....'). # Data schema @@ -243,7 +243,7 @@ The "name_of_session" will be something like "py-3.6(folder='.....'). STRING - Encrypted signature key from the event. This will be the unique key for the table. + Encrypted signature key from the event. This will be the unique key for the table. @@ -258,10 +258,10 @@ The "name_of_session" will be something like "py-3.6(folder='.....'). *indicates that the ID is generated by the original system, such as GitHub. -This table will be used to create the following three derived tables: +This table will be used to create the following three derived tables: -#### `four_keys.deployments` +#### `four_keys.deployments` _Note: Deployments and changes have a many to one relationship. Table only contains successful deployments._ @@ -288,7 +288,7 @@ _Note: Deployments and changes have a many to one relationship. Table only cont array of strings - List of id’s associated with the deployment. Eg: commit_id’s, bug_id’s, etc. + List of id’s associated with the deployment. Eg: commit_id’s, bug_id’s, etc. @@ -388,13 +388,13 @@ _Note: Deployments and changes have a many to one relationship. Table only cont -# Dashboard +# Dashboard ![Image of the Four Keys dashboard.](images/dashboard.png) The dashboard displays all four metrics with daily systems data, as well as a current snapshot of the last 90 days. The key metric definitions and description of the color coding are below. -For a deeper understanding of the metrics and intent of the dashboard, see the [2019 State of DevOps Report](https://www.devops-research.com/research.html#reports). +For a deeper understanding of the metrics and intent of the dashboard, see the [2019 State of DevOps Report](https://www.devops-research.com/research.html#reports). For details about how Four Keys calculates each metric in this dashboard, see the [Four Keys Metrics calculation](METRICS.md) doc. @@ -405,7 +405,7 @@ This Four Keys project defines the key metrics as follows: **Deployment Frequency** -* How frequently a team successfully releases to production, e.g., daily, weekly, monthly, yearly. +* How frequently a team successfully releases to production, e.g., daily, weekly, monthly, yearly. **Lead Time for Changes** @@ -413,7 +413,7 @@ This Four Keys project defines the key metrics as follows: **Time to Restore Services** -* For a failure, the median amount of time between the deployment which caused the failure and the remediation. The remediation is measured by closing an associated bug / incident report. +* For a failure, the median amount of time between the deployment which caused the failure and the remediation. The remediation is measured by closing an associated bug / incident report. **Change Failure Rate** @@ -425,24 +425,24 @@ For more information on the calculation of the metrics, see the [METRICS.md](MET The dashboard has color coding to show the performance of each metric. Green is strong performance, yellow is moderate performance, and red is poor performance. Below is the description of the data that corresponds to the color for each metric. -The data ranges used for this color coding roughly follows the ranges for elite, high, medium, and low performers that are described in the [2019 State of DevOps Report](https://www.devops-research.com/research.html#reports). +The data ranges used for this color coding roughly follows the ranges for elite, high, medium, and low performers that are described in the [2019 State of DevOps Report](https://www.devops-research.com/research.html#reports). **Deployment Frequency** * **Purple:** On-Demand (multiple deploys per day) -* **Green:** Daily, Weekly +* **Green:** Daily, Weekly * **Yellow:** Monthly -* **Red:** Between once per month and once every 6 months. - * This is expressed as “Yearly.” +* **Red:** Between once per month and once every 6 months. + * This is expressed as “Yearly.” **Lead Time to Change** * **Purple:** Less than one day * **Green:** Less than one week * **Yellow:** Between one week and one month -* **Red:** Between one month and 6 months. +* **Red:** Between one month and 6 months. * **Red:** Anything greater than 6 months - * This is expressed as “One year.” + * This is expressed as “One year.” **Time to Restore Service** @@ -450,9 +450,9 @@ The data ranges used for this color coding roughly follows the ranges for elite, * **Green:** Less than one day * **Yellow:** Less than one week * **Red:** Between one week and a month - * This is expressed as “One month” + * This is expressed as “One month” * **Red:** Anything greater than a month - * This is expressed as “One year” + * This is expressed as “One year” **Change Failure Rate** diff --git a/ROADMAP.md b/ROADMAP.md index bea937ed..31cf79ce 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -4,7 +4,7 @@ Four Keys mission: - Be the industry-standard reference implementation for the automated calculation of the [DORA](https://cloud.google.com/blog/products/devops-sre/the-2019-accelerate-state-of-devops-elite-performance-productivity-and-scaling) Four Key metrics. + Be the industry-standard reference implementation for the automated calculation of the [DORA](https://cloud.google.com/blog/products/devops-sre/the-2019-accelerate-state-of-devops-elite-performance-productivity-and-scaling) Four Key metrics. The vision for this is: @@ -16,7 +16,7 @@ The vision for this is: Non-goals: * The DORA research includes predictive analytics, recommendations, and improvement strategies. The Four Keys dashboard will not provide these resources. It will simply be a reflection of an organization’s software delivery performance. -* Four Keys does not intend to be a one-stop shop for all operational performance metrics. The focus of this project should always be on the predictive metrics identified by the DORA research. +* Four Keys does not intend to be a one-stop shop for all operational performance metrics. The focus of this project should always be on the predictive metrics identified by the DORA research. ## Roadmap @@ -24,10 +24,10 @@ Non-goals: * Enriching the dashboard * [More data points](https://github.com/dora-team/fourkeys/issues/77) * New data views for drilling down into the metrics - * More native Terraform installation + * More native Terraform installation * Currently, the setup is a mix of `gcloud` commands and Terraform configuration. We will move away from `gcloud` commands and rely more on the declarative Terraform setup. * Long Term - * CloudEvents migration + * CloudEvents migration * Migrate the `four_keys.events_raw` schema to [CloudEvents](https://github.com/cloudevents/spec) schema * Use the CloudEvents adapters to do the ETL rather than the current [workers](bq-workers/) * New Integrations diff --git a/bq-workers/argocd-parser/cloudbuild.yaml b/bq-workers/argocd-parser/cloudbuild.yaml index 9293dd60..c4692063 100644 --- a/bq-workers/argocd-parser/cloudbuild.yaml +++ b/bq-workers/argocd-parser/cloudbuild.yaml @@ -15,7 +15,7 @@ steps: - # Build argocd-parser image name: gcr.io/cloud-builders/docker:latest - args: ['build', + args: ['build', '--tag=gcr.io/$PROJECT_ID/argocd-parser:${_TAG}', '.'] dir: 'bq-workers/argocd-parser' id: build diff --git a/bq-workers/circleci-parser/cloudbuild.yaml b/bq-workers/circleci-parser/cloudbuild.yaml index b2608b0e..dc9239ef 100644 --- a/bq-workers/circleci-parser/cloudbuild.yaml +++ b/bq-workers/circleci-parser/cloudbuild.yaml @@ -15,7 +15,7 @@ steps: - # Build circleci worker image name: gcr.io/cloud-builders/docker:latest - args: ['build', + args: ['build', '--tag=gcr.io/$PROJECT_ID/circleci-parser', '.'] id: build diff --git a/bq-workers/cloud-build-parser/cloudbuild.yaml b/bq-workers/cloud-build-parser/cloudbuild.yaml index ce2d3475..1688493e 100644 --- a/bq-workers/cloud-build-parser/cloudbuild.yaml +++ b/bq-workers/cloud-build-parser/cloudbuild.yaml @@ -15,7 +15,7 @@ steps: - # Build cloud-build-parser image name: gcr.io/cloud-builders/docker:latest - args: ['build', + args: ['build', '--tag=gcr.io/$PROJECT_ID/cloud-build-parser:${_TAG}', '.'] id: build diff --git a/bq-workers/github-parser/cloudbuild.yaml b/bq-workers/github-parser/cloudbuild.yaml index d134c545..a8ccf7e5 100644 --- a/bq-workers/github-parser/cloudbuild.yaml +++ b/bq-workers/github-parser/cloudbuild.yaml @@ -15,7 +15,7 @@ steps: - # Build github-parser image name: gcr.io/cloud-builders/docker:latest - args: ['build', + args: ['build', '--tag=gcr.io/$PROJECT_ID/github-parser:${_TAG}', '.'] id: build diff --git a/bq-workers/gitlab-parser/cloudbuild.yaml b/bq-workers/gitlab-parser/cloudbuild.yaml index 48530bc5..1b3ddf16 100644 --- a/bq-workers/gitlab-parser/cloudbuild.yaml +++ b/bq-workers/gitlab-parser/cloudbuild.yaml @@ -15,7 +15,7 @@ steps: - # Build new-source worker image name: gcr.io/cloud-builders/docker:latest - args: ['build', + args: ['build', '--tag=gcr.io/$PROJECT_ID/gitlab-parser:${_TAG}', '.'] id: build diff --git a/bq-workers/new-source-template/cloudbuild.yaml b/bq-workers/new-source-template/cloudbuild.yaml index 8210a6b2..436e6cc2 100644 --- a/bq-workers/new-source-template/cloudbuild.yaml +++ b/bq-workers/new-source-template/cloudbuild.yaml @@ -15,7 +15,7 @@ steps: - # Build new-source worker image name: gcr.io/cloud-builders/docker:latest - args: ['build', + args: ['build', '--tag=gcr.io/$PROJECT_ID/${_SOURCE}-parser', '.'] id: build diff --git a/bq-workers/pagerduty-parser/cloudbuild.yaml b/bq-workers/pagerduty-parser/cloudbuild.yaml index 890d853b..09cd4c08 100644 --- a/bq-workers/pagerduty-parser/cloudbuild.yaml +++ b/bq-workers/pagerduty-parser/cloudbuild.yaml @@ -15,7 +15,7 @@ steps: - # Build pagerduty-parser image name: gcr.io/cloud-builders/docker:latest - args: ['build', + args: ['build', '--tag=gcr.io/$PROJECT_ID/pagerduty-parser:${_TAG}', '.'] id: build diff --git a/bq-workers/parsers.cloudbuild.yaml b/bq-workers/parsers.cloudbuild.yaml index b3471f3f..833416c9 100644 --- a/bq-workers/parsers.cloudbuild.yaml +++ b/bq-workers/parsers.cloudbuild.yaml @@ -16,7 +16,7 @@ steps: - # Build parser image name: gcr.io/cloud-builders/docker:latest dir: ${_SERVICE}-parser - args: ['build', + args: ['build', '--tag=gcr.io/$PROJECT_ID/${_SERVICE}-parser:${_TAG}', '.'] id: build diff --git a/bq-workers/tekton-parser/cloudbuild.yaml b/bq-workers/tekton-parser/cloudbuild.yaml index 22ed50b0..73043d91 100644 --- a/bq-workers/tekton-parser/cloudbuild.yaml +++ b/bq-workers/tekton-parser/cloudbuild.yaml @@ -15,7 +15,7 @@ steps: - # Build tekton worker image name: gcr.io/cloud-builders/docker:latest - args: ['build', + args: ['build', '--tag=gcr.io/$PROJECT_ID/tekton-parser', '.'] id: build diff --git a/ci/e2e_terraform_module_test.cloudbuild.yaml b/ci/e2e_terraform_module_test.cloudbuild.yaml index 987c3f02..275cf99d 100644 --- a/ci/e2e_terraform_module_test.cloudbuild.yaml +++ b/ci/e2e_terraform_module_test.cloudbuild.yaml @@ -1,4 +1,4 @@ -### Work in progress! +### Work in progress! # This Cloud Build config will provision 4Keys infra using the modules in /terraform # and then confirm that the dashboard is working correctly @@ -151,8 +151,8 @@ steps: echo "Dashboard URL: $${DASHBOARD_URL}" DASHBOARD_HTTP_RESPONSE_CODE=$(curl -LI $${DASHBOARD_URL} -o /dev/null -w '%{http_code}\n' -s) - if [ $${DASHBOARD_HTTP_RESPONSE_CODE} == "200" ]; then - echo "success" + if [ $${DASHBOARD_HTTP_RESPONSE_CODE} == "200" ]; then + echo "success" else echo "Error validating dashboard URL; received $${DASHBOARD_HTTP_RESPONSE_CODE} from $${DASHBOARD_URL}" exit 1 diff --git a/dashboard/README.md b/dashboard/README.md index e3dc958a..3e2b12cf 100644 --- a/dashboard/README.md +++ b/dashboard/README.md @@ -1,12 +1,12 @@ # Grafana Dashboard -[Grafana](https://grafana.com) is an open source dashboard. The Four Keys project is using it to display the four key metrics. +[Grafana](https://grafana.com) is an open source dashboard. The Four Keys project is using it to display the four key metrics. ## Current configuration -You can see the configuration of the Grafana image in the `Dockerfile`. +You can see the configuration of the Grafana image in the `Dockerfile`. ### Authentication -Auth is currently disabled by default. Learn more about [Grafana User Authentication](https://grafana.com/docs/grafana/latest/auth/). To require a login, delete or comment out these lines: +Auth is currently disabled by default. Learn more about [Grafana User Authentication](https://grafana.com/docs/grafana/latest/auth/). To require a login, delete or comment out these lines: ``` ENV GF_AUTH_DISABLE_LOGIN_FORM "true" @@ -17,20 +17,20 @@ ENV GF_AUTH_ANONYMOUS_ORG_ROLE "Admin" Then update `grafana.ini` with the auth configuration of your choice. ### Provisioning -The datasource and dashboard are configured in `datasource.yaml` and `dashboards.yaml`. Learn more about provisioning [here](https://grafana.com/docs/grafana/latest/administration/provisioning/). +The datasource and dashboard are configured in `datasource.yaml` and `dashboards.yaml`. Learn more about provisioning [here](https://grafana.com/docs/grafana/latest/administration/provisioning/). ## How to update the dashboard -The dashboard is running in a transient container. It does not store data. Therefore, if you want to update the dashboard, you must update the `fourkeys_dashboard.json`. The easiest way to update the JSON is to make changes in the UI and export the JSON. *Any changes in the UI will be temporary and must be saved in the container image.* +The dashboard is running in a transient container. It does not store data. Therefore, if you want to update the dashboard, you must update the `fourkeys_dashboard.json`. The easiest way to update the JSON is to make changes in the UI and export the JSON. *Any changes in the UI will be temporary and must be saved in the container image.* 1. Update the dashboard in the UI 1. Export the JSON 1. Save the JSON in fourkeys_dashboard.json 1. Re-build the image and re-deploy the container -To rebuild and deploy the container, you can run `gcloud builds submit` in this directory. +To rebuild and deploy the container, you can run `gcloud builds submit` in this directory. ## To deploy dashboard -If using [Terraform](https://www.terraform.io), please see the [setup](../setup/) to create the resources. +If using [Terraform](https://www.terraform.io), please see the [setup](../setup/) to create the resources. -Once the resource is created or if you are not using Terraform, feel free to build and deploy outside of Terraform by running `gcloud builds submit` in this directory. +Once the resource is created or if you are not using Terraform, feel free to build and deploy outside of Terraform by running `gcloud builds submit` in this directory. diff --git a/dashboard/cloudbuild.yaml b/dashboard/cloudbuild.yaml index 875e449c..e59d9e3d 100644 --- a/dashboard/cloudbuild.yaml +++ b/dashboard/cloudbuild.yaml @@ -15,7 +15,7 @@ steps: - # Build grafana image name: gcr.io/cloud-builders/docker:latest - args: ['build', '--tag', + args: ['build', '--tag', 'gcr.io/$PROJECT_ID/fourkeys-grafana-dashboard:${_TAG}', '.'] id: build @@ -24,7 +24,7 @@ steps: args: ['push', 'gcr.io/$PROJECT_ID/fourkeys-grafana-dashboard:${_TAG}'] id: push -# Read more about substitutions +# Read more about substitutions # https://cloud.google.com/build/docs/configuring-builds/substitute-variable-values substitutions: _TAG: latest diff --git a/dashboard/fourkeys_dashboard.json b/dashboard/fourkeys_dashboard.json index 995f67c6..5f8a06ce 100644 --- a/dashboard/fourkeys_dashboard.json +++ b/dashboard/fourkeys_dashboard.json @@ -1,935 +1,935 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 2, - "id": 1, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "grafana-bigquery-datasource", - "uid": "P8042CABD7873BED6" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Hours", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 9, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "grafana-bigquery-datasource", - "uid": "P8042CABD7873BED6" - }, - "editorMode": "code", - "format": 1, - "location": "US", - "project": "gglobo-deployment-frq-hdg-prd", - "rawQuery": true, - "rawSql": "SELECT\n day,\n IFNULL(ANY_VALUE(med_time_to_change)/60, 0) AS median_time_to_change, # Hours\nFROM (\n SELECT\n d.deploy_id,\n TIMESTAMP_TRUNC(d.time_created, DAY) AS day,\n PERCENTILE_CONT(\n IF(TIMESTAMP_DIFF(d.time_created, c.time_created, MINUTE) > 0, TIMESTAMP_DIFF(d.time_created, c.time_created, MINUTE), NULL), # Ignore automated pushes\n 0.5) # Median\n OVER (PARTITION BY TIMESTAMP_TRUNC(d.time_created, DAY)) AS med_time_to_change, # Minutes\n FROM four_keys.deployments d, d.changes\n LEFT JOIN four_keys.changes c ON changes = c.change_id\n)\nGROUP BY day\nORDER BY day", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Lead Time for Changes", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-bigquery-datasource", - "uid": "P8042CABD7873BED6" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ + "annotations": { + "list": [ { - "options": { - "One day": { - "color": "purple", - "index": 0 + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" }, - "One month": { - "color": "yellow", - "index": 2 + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" }, - "One week": { - "color": "green", - "index": 1 + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 2, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "grafana-bigquery-datasource", + "uid": "P8042CABD7873BED6" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Hours", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "One year": { - "color": "red", - "index": 4 + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 9, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "Six months": { - "color": "red", - "index": 3 + "tooltip": { + "mode": "single", + "sort": "none" } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 9, - "y": 0 - }, - "id": 5, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^lead_time_to_change$/", - "values": false + }, + "targets": [ + { + "datasource": { + "type": "grafana-bigquery-datasource", + "uid": "P8042CABD7873BED6" + }, + "editorMode": "code", + "format": 1, + "location": "US", + "project": "gglobo-deployment-frq-hdg-prd", + "rawQuery": true, + "rawSql": "SELECT\n day,\n IFNULL(ANY_VALUE(med_time_to_change)/60, 0) AS median_time_to_change, # Hours\nFROM (\n SELECT\n d.deploy_id,\n TIMESTAMP_TRUNC(d.time_created, DAY) AS day,\n PERCENTILE_CONT(\n IF(TIMESTAMP_DIFF(d.time_created, c.time_created, MINUTE) > 0, TIMESTAMP_DIFF(d.time_created, c.time_created, MINUTE), NULL), # Ignore automated pushes\n 0.5) # Median\n OVER (PARTITION BY TIMESTAMP_TRUNC(d.time_created, DAY)) AS med_time_to_change, # Minutes\n FROM four_keys.deployments d, d.changes\n LEFT JOIN four_keys.changes c ON changes = c.change_id\n)\nGROUP BY day\nORDER BY day", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Lead Time for Changes", + "type": "timeseries" }, - "textMode": "auto" - }, - "pluginVersion": "9.3.1", - "targets": [ { - "datasource": { - "type": "grafana-bigquery-datasource", - "uid": "P8042CABD7873BED6" - }, - "editorMode": "code", - "format": 1, - "location": "US", - "project": "gglobo-deployment-frq-hdg-prd", - "rawQuery": true, - "rawSql": "SELECT \n CASE\n WHEN median_time_to_change < 24 * 60 then \"One day\"\n WHEN median_time_to_change < 168 * 60 then \"One week\"\n WHEN median_time_to_change < 730 * 60 then \"One month\"\n WHEN median_time_to_change < 730 * 6 * 60 then \"Six months\"\n ELSE \"One year\"\n END as lead_time_to_change\nFROM (\n SElECT\n IFNULL(ANY_VALUE(med_time_to_change), 0) AS median_time_to_change\n FROM (\n SELECT\n PERCENTILE_CONT(\n IF(TIMESTAMP_DIFF(d.time_created, c.time_created, MINUTE) > 0, TIMESTAMP_DIFF(d.time_created, c.time_created, MINUTE), NULL), # Ignore automated pushes\n 0.5) # Median\n OVER () AS med_time_to_change, # Minutes\n FROM four_keys.deployments d, d.changes\n LEFT JOIN four_keys.changes c ON changes = c.change_id\n WHERE d.time_created > TIMESTAMP(DATE_SUB(CURRENT_DATE(), INTERVAL 3 MONTH))\n )\n)", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" + "datasource": { + "type": "grafana-bigquery-datasource", + "uid": "P8042CABD7873BED6" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "One day": { + "color": "purple", + "index": 0 + }, + "One month": { + "color": "yellow", + "index": 2 + }, + "One week": { + "color": "green", + "index": 1 + }, + "One year": { + "color": "red", + "index": 4 + }, + "Six months": { + "color": "red", + "index": 3 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, - "type": "groupBy" - } + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 9, + "y": 0 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^lead_time_to_change$/", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "grafana-bigquery-datasource", + "uid": "P8042CABD7873BED6" + }, + "editorMode": "code", + "format": 1, + "location": "US", + "project": "gglobo-deployment-frq-hdg-prd", + "rawQuery": true, + "rawSql": "SELECT \n CASE\n WHEN median_time_to_change < 24 * 60 then \"One day\"\n WHEN median_time_to_change < 168 * 60 then \"One week\"\n WHEN median_time_to_change < 730 * 60 then \"One month\"\n WHEN median_time_to_change < 730 * 6 * 60 then \"Six months\"\n ELSE \"One year\"\n END as lead_time_to_change\nFROM (\n SElECT\n IFNULL(ANY_VALUE(med_time_to_change), 0) AS median_time_to_change\n FROM (\n SELECT\n PERCENTILE_CONT(\n IF(TIMESTAMP_DIFF(d.time_created, c.time_created, MINUTE) > 0, TIMESTAMP_DIFF(d.time_created, c.time_created, MINUTE), NULL), # Ignore automated pushes\n 0.5) # Median\n OVER () AS med_time_to_change, # Minutes\n FROM four_keys.deployments d, d.changes\n LEFT JOIN four_keys.changes c ON changes = c.change_id\n WHERE d.time_created > TIMESTAMP(DATE_SUB(CURRENT_DATE(), INTERVAL 3 MONTH))\n )\n)", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } ], - "limit": 50 - } - } - ], - "title": "Lead Time to Change Bucket", - "type": "stat" - }, - { - "datasource": { - "type": "grafana-bigquery-datasource", - "uid": "P8042CABD7873BED6" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 50, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "title": "Lead Time to Change Bucket", + "type": "stat" }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 9, - "x": 13, - "y": 0 - }, - "id": 9, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { - "datasource": { - "type": "grafana-bigquery-datasource", - "uid": "P8042CABD7873BED6" - }, - "editorMode": "code", - "format": 1, - "location": "US", - "project": "gglobo-deployment-frq-hdg-prd", - "rawQuery": true, - "rawSql": "SELECT\nTIMESTAMP_TRUNC(time_created, DAY) AS day,\nCOUNT(distinct deploy_id) AS deployments\nFROM\nfour_keys.deployments\nGROUP BY day\nORDER BY day", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Daily Deployments", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-bigquery-datasource", - "uid": "P8042CABD7873BED6" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "Daily": { - "color": "purple", - "index": 0, - "text": "Elite Performance" - }, - "Monthly": { - "color": "yellow", - "index": 2 + "datasource": { + "type": "grafana-bigquery-datasource", + "uid": "P8042CABD7873BED6" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "Weekly": { - "color": "green", - "index": 1 + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 9, + "x": 13, + "y": 0 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "Yearly": { - "color": "red", - "index": 3 + "tooltip": { + "mode": "single", + "sort": "none" } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 9, - "y": 5 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^deployment_frequency$/", - "values": false + }, + "targets": [ + { + "datasource": { + "type": "grafana-bigquery-datasource", + "uid": "P8042CABD7873BED6" + }, + "editorMode": "code", + "format": 1, + "location": "US", + "project": "gglobo-deployment-frq-hdg-prd", + "rawQuery": true, + "rawSql": "SELECT\nTIMESTAMP_TRUNC(time_created, DAY) AS day,\nCOUNT(distinct deploy_id) AS deployments\nFROM\nfour_keys.deployments\nGROUP BY day\nORDER BY day", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Daily Deployments", + "type": "timeseries" }, - "textMode": "auto" - }, - "pluginVersion": "9.3.1", - "targets": [ { - "datasource": { - "type": "grafana-bigquery-datasource", - "uid": "P8042CABD7873BED6" - }, - "editorMode": "code", - "format": 1, - "location": "US", - "project": "gglobo-deployment-frq-hdg-prd", - "rawQuery": true, - "rawSql": "WITH last_three_months AS\n(SELECT\nTIMESTAMP(day) AS day\nFROM\nUNNEST(\nGENERATE_DATE_ARRAY(\n DATE_SUB(CURRENT_DATE(), INTERVAL 3 MONTH), CURRENT_DATE(),\n INTERVAL 1 DAY)) AS day\n# FROM the start of the data\nWHERE day > (SELECT date(min(time_created)) FROM four_keys.events_raw)\n)\n\nSELECT\nCASE WHEN daily THEN \"Daily\" \n WHEN weekly THEN \"Weekly\" \n # If at least one per month, then Monthly\n WHEN PERCENTILE_CONT(monthly_deploys, 0.5) OVER () >= 1 THEN \"Monthly\" \n ELSE \"Yearly\"\n END as deployment_frequency\nFROM (\n SELECT\n # If the median number of days per week is more than 3, then Daily\n PERCENTILE_CONT(days_deployed, 0.5) OVER() >= 3 AS daily,\n # If most weeks have a deployment, then Weekly\n PERCENTILE_CONT(week_deployed, 0.5) OVER() >= 1 AS weekly,\n\n # Count the number of deployments per month. \n # Cannot mix aggregate and analytic functions, so calculate the median in the outer select statement\n SUM(week_deployed) OVER(partition by TIMESTAMP_TRUNC(week, MONTH)) monthly_deploys\n FROM(\n SELECT\n TIMESTAMP_TRUNC(last_three_months.day, WEEK) as week,\n MAX(if(deployments.day is not null, 1, 0)) as week_deployed,\n COUNT(distinct deployments.day) as days_deployed\n FROM last_three_months\n LEFT JOIN(\n SELECT\n TIMESTAMP_TRUNC(time_created, DAY) AS day,\n deploy_id\n FROM four_keys.deployments) deployments ON deployments.day = last_three_months.day\n GROUP BY week)\n )\nLIMIT 1", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" + "datasource": { + "type": "grafana-bigquery-datasource", + "uid": "P8042CABD7873BED6" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Daily": { + "color": "purple", + "index": 0, + "text": "Elite Performance" + }, + "Monthly": { + "color": "yellow", + "index": 2 + }, + "Weekly": { + "color": "green", + "index": 1 + }, + "Yearly": { + "color": "red", + "index": 3 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 9, + "y": 5 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^deployment_frequency$/", + "values": false }, - "type": "groupBy" - } + "textMode": "auto" + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "grafana-bigquery-datasource", + "uid": "P8042CABD7873BED6" + }, + "editorMode": "code", + "format": 1, + "location": "US", + "project": "gglobo-deployment-frq-hdg-prd", + "rawQuery": true, + "rawSql": "WITH last_three_months AS\n(SELECT\nTIMESTAMP(day) AS day\nFROM\nUNNEST(\nGENERATE_DATE_ARRAY(\n DATE_SUB(CURRENT_DATE(), INTERVAL 3 MONTH), CURRENT_DATE(),\n INTERVAL 1 DAY)) AS day\n# FROM the start of the data\nWHERE day > (SELECT date(min(time_created)) FROM four_keys.events_raw)\n)\n\nSELECT\nCASE WHEN daily THEN \"Daily\" \n WHEN weekly THEN \"Weekly\" \n # If at least one per month, then Monthly\n WHEN PERCENTILE_CONT(monthly_deploys, 0.5) OVER () >= 1 THEN \"Monthly\" \n ELSE \"Yearly\"\n END as deployment_frequency\nFROM (\n SELECT\n # If the median number of days per week is more than 3, then Daily\n PERCENTILE_CONT(days_deployed, 0.5) OVER() >= 3 AS daily,\n # If most weeks have a deployment, then Weekly\n PERCENTILE_CONT(week_deployed, 0.5) OVER() >= 1 AS weekly,\n\n # Count the number of deployments per month. \n # Cannot mix aggregate and analytic functions, so calculate the median in the outer select statement\n SUM(week_deployed) OVER(partition by TIMESTAMP_TRUNC(week, MONTH)) monthly_deploys\n FROM(\n SELECT\n TIMESTAMP_TRUNC(last_three_months.day, WEEK) as week,\n MAX(if(deployments.day is not null, 1, 0)) as week_deployed,\n COUNT(distinct deployments.day) as days_deployed\n FROM last_three_months\n LEFT JOIN(\n SELECT\n TIMESTAMP_TRUNC(time_created, DAY) AS day,\n deploy_id\n FROM four_keys.deployments) deployments ON deployments.day = last_three_months.day\n GROUP BY week)\n )\nLIMIT 1", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } ], - "limit": 50 - } - } - ], - "title": "Deployment Frequency", - "type": "stat" - }, - { - "datasource": { - "type": "grafana-bigquery-datasource", - "uid": "P8042CABD7873BED6" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Hours", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "title": "Deployment Frequency", + "type": "stat" }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 9, - "x": 0, - "y": 10 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { - "datasource": { - "type": "grafana-bigquery-datasource", - "uid": "P8042CABD7873BED6" - }, - "editorMode": "code", - "format": 1, - "location": "US", - "project": "gglobo-deployment-frq-hdg-prd", - "rawQuery": true, - "rawSql": "SELECT\n TIMESTAMP_TRUNC(time_created, DAY) as day,\n #### Median time to resolve\n PERCENTILE_CONT(\n TIMESTAMP_DIFF(time_resolved, time_created, HOUR), 0.5)\n OVER(PARTITION BY TIMESTAMP_TRUNC(time_created, DAY)\n ) as daily_med_time_to_restore,\n FROM four_keys.incidents\nORDER BY day", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Daily Median Time to Restore Services", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-bigquery-datasource", - "uid": "P8042CABD7873BED6" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "One day": { - "color": "green", - "index": 0 - }, - "One month": { - "color": "red", - "index": 2 - }, - "One week": { - "color": "yellow", - "index": 1 + "datasource": { + "type": "grafana-bigquery-datasource", + "uid": "P8042CABD7873BED6" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Hours", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "One year": { - "color": "red", - "index": 4 + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 9, + "x": 0, + "y": 10 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "Six months": { - "color": "red", - "index": 3 + "tooltip": { + "mode": "single", + "sort": "none" } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 9, - "y": 10 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^med_time_to_resolve$/", - "values": false + }, + "targets": [ + { + "datasource": { + "type": "grafana-bigquery-datasource", + "uid": "P8042CABD7873BED6" + }, + "editorMode": "code", + "format": 1, + "location": "US", + "project": "gglobo-deployment-frq-hdg-prd", + "rawQuery": true, + "rawSql": "SELECT\n TIMESTAMP_TRUNC(time_created, DAY) as day,\n #### Median time to resolve\n PERCENTILE_CONT(\n TIMESTAMP_DIFF(time_resolved, time_created, HOUR), 0.5)\n OVER(PARTITION BY TIMESTAMP_TRUNC(time_created, DAY)\n ) as daily_med_time_to_restore,\n FROM four_keys.incidents\nORDER BY day", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Daily Median Time to Restore Services", + "type": "timeseries" }, - "textMode": "auto" - }, - "pluginVersion": "9.3.1", - "targets": [ { - "datasource": { - "type": "grafana-bigquery-datasource", - "uid": "P8042CABD7873BED6" - }, - "editorMode": "code", - "format": 1, - "location": "US", - "project": "gglobo-deployment-frq-hdg-prd", - "rawQuery": true, - "rawSql": "SELECT\nCASE WHEN med_time_to_resolve < 24 then \"One day\"\n WHEN med_time_to_resolve < 168 then \"One week\"\n WHEN med_time_to_resolve < 730 then \"One month\"\n WHEN med_time_to_resolve < 730 * 6 then \"Six months\"\n ELSE \"One year\"\n END as med_time_to_resolve,\nFROM (\n SELECT\n #### Median time to resolve\n PERCENTILE_CONT(\n TIMESTAMP_DIFF(time_resolved, time_created, HOUR), 0.5)\n OVER() as med_time_to_resolve,\n FROM four_keys.incidents\n # Limit to 3 months\n WHERE time_created > TIMESTAMP(DATE_SUB(CURRENT_DATE(), INTERVAL 3 MONTH)))\nLIMIT 1", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" + "datasource": { + "type": "grafana-bigquery-datasource", + "uid": "P8042CABD7873BED6" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "One day": { + "color": "green", + "index": 0 + }, + "One month": { + "color": "red", + "index": 2 + }, + "One week": { + "color": "yellow", + "index": 1 + }, + "One year": { + "color": "red", + "index": 4 + }, + "Six months": { + "color": "red", + "index": 3 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, - "type": "groupBy" - } + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 9, + "y": 10 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^med_time_to_resolve$/", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "grafana-bigquery-datasource", + "uid": "P8042CABD7873BED6" + }, + "editorMode": "code", + "format": 1, + "location": "US", + "project": "gglobo-deployment-frq-hdg-prd", + "rawQuery": true, + "rawSql": "SELECT\nCASE WHEN med_time_to_resolve < 24 then \"One day\"\n WHEN med_time_to_resolve < 168 then \"One week\"\n WHEN med_time_to_resolve < 730 then \"One month\"\n WHEN med_time_to_resolve < 730 * 6 then \"Six months\"\n ELSE \"One year\"\n END as med_time_to_resolve,\nFROM (\n SELECT\n #### Median time to resolve\n PERCENTILE_CONT(\n TIMESTAMP_DIFF(time_resolved, time_created, HOUR), 0.5)\n OVER() as med_time_to_resolve,\n FROM four_keys.incidents\n # Limit to 3 months\n WHERE time_created > TIMESTAMP(DATE_SUB(CURRENT_DATE(), INTERVAL 3 MONTH)))\nLIMIT 1", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } ], - "limit": 50 - } - } - ], - "title": "Median Time to Restore Services", - "type": "stat" - }, - { - "datasource": { - "type": "grafana-bigquery-datasource", - "uid": "P8042CABD7873BED6" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 9, - "x": 13, - "y": 10 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "title": "Median Time to Restore Services", + "type": "stat" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { - "datasource": { - "type": "grafana-bigquery-datasource", - "uid": "P8042CABD7873BED6" - }, - "editorMode": "code", - "format": 1, - "location": "US", - "project": "gglobo-deployment-frq-hdg-prd", - "rawQuery": true, - "rawSql": "SELECT\nTIMESTAMP_TRUNC(d.time_created, DAY) as day,\n IF(COUNT(DISTINCT change_id) = 0,0, SUM(IF(i.incident_id is NULL, 0, 1)) / COUNT(DISTINCT deploy_id)) as change_fail_rate\nFROM four_keys.deployments d, d.changes\nLEFT JOIN four_keys.changes c ON changes = c.change_id\nLEFT JOIN(SELECT\n DISTINCT incident_id,\n change,\n time_resolved\n FROM four_keys.incidents i,\n i.changes change) i ON i.change = changes\nGROUP BY day", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Daily Change Failure Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-bigquery-datasource", - "uid": "P8042CABD7873BED6" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0-15%": { - "color": "green", - "index": 0 + "datasource": { + "type": "grafana-bigquery-datasource", + "uid": "P8042CABD7873BED6" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "16-45%": { - "color": "yellow", - "index": 1 + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 9, + "x": 13, + "y": 10 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "46-60%": { - "color": "red", - "index": 2 + "tooltip": { + "mode": "single", + "sort": "none" } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 9, - "y": 15 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^change_fail_rate$/", - "values": false + }, + "targets": [ + { + "datasource": { + "type": "grafana-bigquery-datasource", + "uid": "P8042CABD7873BED6" + }, + "editorMode": "code", + "format": 1, + "location": "US", + "project": "gglobo-deployment-frq-hdg-prd", + "rawQuery": true, + "rawSql": "SELECT\nTIMESTAMP_TRUNC(d.time_created, DAY) as day,\n IF(COUNT(DISTINCT change_id) = 0,0, SUM(IF(i.incident_id is NULL, 0, 1)) / COUNT(DISTINCT deploy_id)) as change_fail_rate\nFROM four_keys.deployments d, d.changes\nLEFT JOIN four_keys.changes c ON changes = c.change_id\nLEFT JOIN(SELECT\n DISTINCT incident_id,\n change,\n time_resolved\n FROM four_keys.incidents i,\n i.changes change) i ON i.change = changes\nGROUP BY day", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Daily Change Failure Rate", + "type": "timeseries" }, - "textMode": "auto" - }, - "pluginVersion": "9.3.1", - "targets": [ { - "datasource": { - "type": "grafana-bigquery-datasource", - "uid": "P8042CABD7873BED6" - }, - "editorMode": "code", - "format": 1, - "location": "US", - "project": "gglobo-deployment-frq-hdg-prd", - "rawQuery": true, - "rawSql": "SELECT\nCASE WHEN change_fail_rate <= .15 then \"0-15%\"\n WHEN change_fail_rate < .46 then \"16-45%\"\n ELSE \"46-60%\" end as change_fail_rate\nFROM \n (SELECT\n IF(COUNT(DISTINCT change_id) = 0,0, SUM(IF(i.incident_id is NULL, 0, 1)) / COUNT(DISTINCT deploy_id)) as change_fail_rate\n FROM four_keys.deployments d, d.changes\n LEFT JOIN four_keys.changes c ON changes = c.change_id\n LEFT JOIN(SELECT\n incident_id,\n change,\n time_resolved\n FROM four_keys.incidents i,\n i.changes change) i ON i.change = changes\n # Limit to 3 months\n WHERE d.time_created > TIMESTAMP(DATE_SUB(CURRENT_DATE(), INTERVAL 3 MONTH))\n )\nLIMIT 1", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" + "datasource": { + "type": "grafana-bigquery-datasource", + "uid": "P8042CABD7873BED6" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0-15%": { + "color": "green", + "index": 0 + }, + "16-45%": { + "color": "yellow", + "index": 1 + }, + "46-60%": { + "color": "red", + "index": 2 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 9, + "y": 15 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^change_fail_rate$/", + "values": false }, - "type": "groupBy" - } + "textMode": "auto" + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "grafana-bigquery-datasource", + "uid": "P8042CABD7873BED6" + }, + "editorMode": "code", + "format": 1, + "location": "US", + "project": "gglobo-deployment-frq-hdg-prd", + "rawQuery": true, + "rawSql": "SELECT\nCASE WHEN change_fail_rate <= .15 then \"0-15%\"\n WHEN change_fail_rate < .46 then \"16-45%\"\n ELSE \"46-60%\" end as change_fail_rate\nFROM \n (SELECT\n IF(COUNT(DISTINCT change_id) = 0,0, SUM(IF(i.incident_id is NULL, 0, 1)) / COUNT(DISTINCT deploy_id)) as change_fail_rate\n FROM four_keys.deployments d, d.changes\n LEFT JOIN four_keys.changes c ON changes = c.change_id\n LEFT JOIN(SELECT\n incident_id,\n change,\n time_resolved\n FROM four_keys.incidents i,\n i.changes change) i ON i.change = changes\n # Limit to 3 months\n WHERE d.time_created > TIMESTAMP(DATE_SUB(CURRENT_DATE(), INTERVAL 3 MONTH))\n )\nLIMIT 1", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } ], - "limit": 50 - } + "title": "Change Failure Rate", + "type": "stat" } - ], - "title": "Change Failure Rate", - "type": "stat" - } - ], - "refresh": "", - "schemaVersion": 37, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "2022-08-01T03:00:00.000Z", - "to": "now" - }, - "timepicker": { - "hidden": false, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" ], - "type": "timepicker" - }, - "timezone": "utc", - "title": "Four Keys", - "uid": "yVtwoQ4nk", - "version": 2, - "weekStart": "" -} \ No newline at end of file + "refresh": "", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "2022-08-01T03:00:00.000Z", + "to": "now" + }, + "timepicker": { + "hidden": false, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "utc", + "title": "Four Keys", + "uid": "yVtwoQ4nk", + "version": 2, + "weekStart": "" +} diff --git a/data-generator/tools/refresh_data.sh b/data-generator/tools/refresh_data.sh index 73f8362b..89f6cbae 100755 --- a/data-generator/tools/refresh_data.sh +++ b/data-generator/tools/refresh_data.sh @@ -26,7 +26,7 @@ are set: WEBHOOK, SECRET" exit 1 fi -read -p "Which version control system are you using? +read -p "Which version control system are you using? (1) GitLab (2) GitHub diff --git a/queries/changes.sql b/queries/changes.sql index 711e7328..c35df4da 100644 --- a/queries/changes.sql +++ b/queries/changes.sql @@ -1,5 +1,5 @@ # Changes Table -SELECT +SELECT source, event_type, JSON_EXTRACT_SCALAR(commit, '$.id') change_id, diff --git a/queries/deployments.sql b/queries/deployments.sql index 3fabdb04..aeb6f1dd 100644 --- a/queries/deployments.sql +++ b/queries/deployments.sql @@ -1,7 +1,7 @@ # Deployments Table WITH deploys_cloudbuild_github_gitlab AS (# Cloud Build, Github, Gitlab pipelines - SELECT + SELECT source, id as deploy_id, time_created, @@ -20,15 +20,15 @@ WITH deploys_cloudbuild_github_gitlab AS (# Cloud Build, Github, Gitlab pipeline SELECT JSON_EXTRACT_SCALAR(string_element, '$') FROM UNNEST(JSON_EXTRACT_ARRAY(metadata, '$.deployment.additional_sha')) AS string_element) ELSE ARRAY[] end as additional_commits - FROM four_keys.events_raw + FROM four_keys.events_raw WHERE ( # Cloud Build Deployments (source = "cloud_build" AND JSON_EXTRACT_SCALAR(metadata, '$.status') = "SUCCESS") # GitHub Deployments OR (source LIKE "github%" and event_type = "deployment_status" and JSON_EXTRACT_SCALAR(metadata, '$.deployment_status.state') = "success") - # GitLab Pipelines + # GitLab Pipelines OR (source LIKE "gitlab%" AND event_type = "pipeline" AND JSON_EXTRACT_SCALAR(metadata, '$.object_attributes.status') = "success") - # GitLab Deployments + # GitLab Deployments OR (source LIKE "gitlab%" AND event_type = "deployment" AND JSON_EXTRACT_SCALAR(metadata, '$.status') = "success") # ArgoCD Deployments OR (source = "argocd" AND JSON_EXTRACT_SCALAR(metadata, '$.status') = "SUCCESS") @@ -42,13 +42,13 @@ WITH deploys_cloudbuild_github_gitlab AS (# Cloud Build, Github, Gitlab pipeline IF(JSON_EXTRACT_SCALAR(param, '$.name') = "gitrevision", JSON_EXTRACT_SCALAR(param, '$.value'), Null) as main_commit, ARRAY[] AS additional_commits FROM ( - SELECT + SELECT id, TIMESTAMP_TRUNC(time_created, second) as time_created, source, four_keys.json2array(JSON_EXTRACT(metadata, '$.data.pipelineRun.spec.params')) params FROM four_keys.events_raw - WHERE event_type = "dev.tekton.event.pipelinerun.successful.v1" + WHERE event_type = "dev.tekton.event.pipelinerun.successful.v1" AND metadata like "%gitrevision%") e, e.params as param ), deploys_circleci AS (# CircleCI pipelines @@ -91,12 +91,12 @@ WITH deploys_cloudbuild_github_gitlab AS (# Cloud Build, Github, Gitlab pipeline ) ) - SELECT + SELECT source, deploy_id, time_created, - main_commit, - ARRAY_AGG(DISTINCT JSON_EXTRACT_SCALAR(array_commits, '$.id')) changes, + main_commit, + ARRAY_AGG(DISTINCT JSON_EXTRACT_SCALAR(array_commits, '$.id')) changes, FROM deployment_changes CROSS JOIN deployment_changes.array_commits GROUP BY 1,2,3,4; diff --git a/queries/incidents.sql b/queries/incidents.sql index bd80619e..ed057c22 100644 --- a/queries/incidents.sql +++ b/queries/incidents.sql @@ -7,7 +7,7 @@ MAX(time_resolved) as time_resolved, ARRAY_AGG(root_cause IGNORE NULLS) changes, FROM ( -SELECT +SELECT source, CASE WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.issue.number') WHEN source LIKE "gitlab%" AND event_type = "note" THEN JSON_EXTRACT_SCALAR(metadata, '$.object_attributes.noteable_id') @@ -27,7 +27,7 @@ CASE WHEN source LIKE "github%" THEN REGEXP_CONTAINS(JSON_EXTRACT(metadata, '$.i WHEN source LIKE "gitlab%" THEN REGEXP_CONTAINS(JSON_EXTRACT(metadata, '$.object_attributes.labels'), '"title":"Incident"') WHEN source LIKE "pagerduty%" THEN TRUE # All Pager Duty events are incident-related END AS bug, -FROM four_keys.events_raw +FROM four_keys.events_raw WHERE event_type LIKE "issue%" OR event_type LIKE "incident%" OR (event_type = "note" and JSON_EXTRACT_SCALAR(metadata, '$.object_attributes.noteable_type') = 'Issue') ) issue LEFT JOIN (SELECT time_created, changes FROM four_keys.deployments d, d.changes) root on root.changes = root_cause diff --git a/setup/README.md b/setup/README.md index 642439b2..ee952d4d 100644 --- a/setup/README.md +++ b/setup/README.md @@ -58,7 +58,7 @@ To deploy Four Keys with Terraform, you will first need: ## Generating mock data -To test your Four Keys deployment, you can generate mock data that simulates events from a GitHub repository. +To test your Four Keys deployment, you can generate mock data that simulates events from a GitHub repository. 1. Export your event handler URL an environment variable. This is the webhook URL that will receive events: diff --git a/setup/deprecated/README.md b/setup/deprecated/README.md index 7e8ee312..586b250f 100644 --- a/setup/deprecated/README.md +++ b/setup/deprecated/README.md @@ -158,7 +158,7 @@ And now, whenever a pull request is merged into master of your fork, Cloud Build 1. In your GitLab repo, navigate to `Settings` on the left-hand menu and then select `CI/CD`. 1. Save your account key under variables. 1. In the **key** field, input `SERVICE_ACCOUNT`. - 1. In the **Value** field, input the JSON . + 1. In the **Value** field, input the JSON . 1. Select **Protect variable**. 1. Save your Google Cloud project-id under variables. 1. In the **key** field, input `PROJECT_ID`. @@ -184,7 +184,7 @@ This setup will trigger a deployment on any `push` to the `master` branch. ### Collecting incident data -Four Keys uses GitLab and/or GitHub issues to track incidents. +Four Keys uses GitLab and/or GitHub issues to track incidents. #### Creating an incident diff --git a/setup/deprecated/changes_schema.json b/setup/deprecated/changes_schema.json index 3db2975c..58a4c364 100644 --- a/setup/deprecated/changes_schema.json +++ b/setup/deprecated/changes_schema.json @@ -1,17 +1,17 @@ [ - { - "mode": "NULLABLE", - "name": "change_id", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "time_created", - "type": "TIMESTAMP" - }, - { - "mode": "NULLABLE", - "name": "event_type", - "type": "STRING" - } + { + "mode": "NULLABLE", + "name": "change_id", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "time_created", + "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "event_type", + "type": "STRING" + } ] diff --git a/setup/deprecated/deployments_schema.json b/setup/deprecated/deployments_schema.json index 82a40ec3..bcd81718 100644 --- a/setup/deprecated/deployments_schema.json +++ b/setup/deprecated/deployments_schema.json @@ -1,17 +1,17 @@ [ - { - "mode": "NULLABLE", - "name": "deploy_id", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "time_created", - "type": "TIMESTAMP" - }, - { - "mode": "REPEATED", - "name": "changes", - "type": "STRING" - } + { + "mode": "NULLABLE", + "name": "deploy_id", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "time_created", + "type": "TIMESTAMP" + }, + { + "mode": "REPEATED", + "name": "changes", + "type": "STRING" + } ] diff --git a/setup/deprecated/e2e_tests.cloudbuild.yaml b/setup/deprecated/e2e_tests.cloudbuild.yaml index c01453dd..99017a5b 100644 --- a/setup/deprecated/e2e_tests.cloudbuild.yaml +++ b/setup/deprecated/e2e_tests.cloudbuild.yaml @@ -57,17 +57,17 @@ steps: - id: Tear Down Resources name: 'google/cloud-sdk' entrypoint: /bin/bash - args: + args: - '-c' - | echo "Dropping BQ Resources" set -x bq rm -r -f -d ${PROJECT_ID}:four_keys set +x - configs=$(bq ls --transfer_config --transfer_location us --format=json) + configs=$(bq ls --transfer_config --transfer_location us --format=json) config_ids=$(python3 -c "import json, subprocess - for config in $configs: + for config in $configs: print(config['name'])") set -x @@ -90,11 +90,11 @@ steps: gcloud pubsub subscriptions delete CloudBuildSubscription -q echo "Turning off APIs" - gcloud services disable compute.googleapis.com - gcloud services disable run.googleapis.com - gcloud services disable bigquery.googleapis.com - gcloud services disable bigquerydatatransfer.googleapis.com - gcloud services disable bigqueryconnection.googleapis.com - gcloud services disable secretmanager.googleapis.com + gcloud services disable compute.googleapis.com + gcloud services disable run.googleapis.com + gcloud services disable bigquery.googleapis.com + gcloud services disable bigquerydatatransfer.googleapis.com + gcloud services disable bigqueryconnection.googleapis.com + gcloud services disable secretmanager.googleapis.com set +x timeout: '3600s' \ No newline at end of file diff --git a/setup/deprecated/incidents_schema.json b/setup/deprecated/incidents_schema.json index 3581a442..0d466593 100644 --- a/setup/deprecated/incidents_schema.json +++ b/setup/deprecated/incidents_schema.json @@ -1,22 +1,22 @@ [ - { - "mode": "NULLABLE", - "name": "incident_id", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "time_created", - "type": "TIMESTAMP" - }, - { - "mode": "NULLABLE", - "name": "time_resolved", - "type": "TIMESTAMP" - }, - { - "mode": "REPEATED", - "name": "changes", - "type": "STRING" - } + { + "mode": "NULLABLE", + "name": "incident_id", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "time_created", + "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "time_resolved", + "type": "TIMESTAMP" + }, + { + "mode": "REPEATED", + "name": "changes", + "type": "STRING" + } ] diff --git a/setup/deprecated/new_source.sh b/setup/deprecated/new_source.sh index 934923bb..0f42821e 100755 --- a/setup/deprecated/new_source.sh +++ b/setup/deprecated/new_source.sh @@ -22,7 +22,7 @@ environment(){ if [[ ! ${FOURKEYS_PROJECT} ]] # If env.sh does not exist, use current active project - then FOURKEYS_PROJECT=$(gcloud config get-value project) + then FOURKEYS_PROJECT=$(gcloud config get-value project) fi } @@ -55,7 +55,7 @@ build_deploy_cloud_run(){ cp -R $DIR/../../bq-workers/new-source-template $DIR/../../bq-workers/${nickname}-parser cd $DIR/../../bq-workers/${nickname}-parser gcloud builds submit --substitutions _SOURCE=${nickname},_REGION=us-central1 \ - --project ${FOURKEYS_PROJECT} . + --project ${FOURKEYS_PROJECT} . } set_permissions(){ @@ -70,7 +70,7 @@ setup_pubsub_topic_subscription(){ # Get push endpoint for new service export PUSH_ENDPOINT_URL=$(gcloud run services describe ${nickname}-worker \ --format="value(status.url)" --project ${FOURKEYS_PROJECT} \ - --platform managed --region us-central1) + --platform managed --region us-central1) # Create topic echo "Creating event handler Pub/Sub topic..."; set -x diff --git a/setup/deprecated/setup.sh b/setup/deprecated/setup.sh index 62bf748d..563b2ded 100755 --- a/setup/deprecated/setup.sh +++ b/setup/deprecated/setup.sh @@ -56,13 +56,13 @@ fourkeys_project_setup () { export BILLING_ACCOUNT=$(gcloud beta billing projects describe ${FOURKEYS_PROJECT} --format="value(billingAccountName)") if [[ ! ${BILLING_ACCOUNT} ]] - then echo "Please enable billing account on ${FOURKEYS_PROJECT}" + then echo "Please enable billing account on ${FOURKEYS_PROJECT}" exit fi export FOURKEYS_REGION=us-central1 - echo "Setting up project for Four Keys Dashboard..." + echo "Setting up project for Four Keys Dashboard..." get_project_number gcloud config set project ${FOURKEYS_PROJECT}; set -x set +x; echo @@ -123,7 +123,7 @@ fourkeys_project_setup () { then tekton_pipeline fi # Only set up GitLab pipeline if it wasn't selected as the version control system - if [[ ${cicd_system} == "3" && ${git_system} != "1" ]] + if [[ ${cicd_system} == "3" && ${git_system} != "1" ]] then gitlab_pipeline else echo "Please see the documentation to learn how to extend to sources other than Cloud Build, Tekton, GitLab, or GitHub." fi @@ -143,7 +143,7 @@ fourkeys_project_setup () { --table -f\ ${FOURKEYS_PROJECT}:four_keys.deployments \ $DIR/deployments_schema.json - + bq mk \ --table -f\ ${FOURKEYS_PROJECT}:four_keys.events_raw \ @@ -173,7 +173,7 @@ fourkeys_project_setup () { else # If not, create and save secret - SECRET="$(python3 -c 'import secrets + SECRET="$(python3 -c 'import secrets print(secrets.token_hex(20))' | tr -d '\n')" echo $SECRET | tr -d '\n' | gcloud beta secrets create event-handler \ --replication-policy=automatic \ @@ -273,7 +273,7 @@ cloud_build_pipeline(){ echo "Deploying BQ cloud build worker..."; set -x cd $DIR/../../bq-workers/cloud-build-parser - gcloud builds submit --substitutions _TAG=latest,_REGION=${FOURKEYS_REGION} . + gcloud builds submit --substitutions _TAG=latest,_REGION=${FOURKEYS_REGION} . set +x; echo echo "Creating cloud-builds topic..."; set -x @@ -325,7 +325,7 @@ tekton_pipeline(){ generate_data(){ gcloud config set project ${FOURKEYS_PROJECT} - echo "Creating mock data..."; + echo "Creating mock data..."; export WEBHOOK=$(gcloud run services describe event-handler --format="value(status.url)") export SECRET=$SECRET @@ -344,14 +344,14 @@ generate_data(){ set +x fi if [[ ${git_system} == "2" ]] - then set -x; python3 ${DIR}/../../data-generator/generate_data.py --vc_system=github + then set -x; python3 ${DIR}/../../data-generator/generate_data.py --vc_system=github set +x fi - + } schedule_bq_queries(){ - echo "Check BigQueryDataTransfer is enabled" + echo "Check BigQueryDataTransfer is enabled" enabled=$(gcloud services list --enabled --filter name:bigquerydatatransfer.googleapis.com) while [[ "${enabled}" != *"bigquerydatatransfer.googleapis.com"* ]] @@ -366,7 +366,7 @@ schedule_bq_queries(){ ./schedule.sh --query_file=changes.sql --table=changes --project_id=$FOURKEYS_PROJECT ./schedule.sh --query_file=deployments.sql --table=deployments --project_id=$FOURKEYS_PROJECT ./schedule.sh --query_file=incidents.sql --table=incidents --project_id=$FOURKEYS_PROJECT - + set +x; echo cd ${DIR} } @@ -380,7 +380,7 @@ project_prompt(){ # Prompt until project-id is correct if [[ ${FOURKEYS_PROJECT} ]] then read -p "Would you like to use ${FOURKEYS_PROJECT} to deploy a new Cloud Run worker? (y/n) :" yesno - fi + fi if [[ ${yesno} == "y" ]] then continue=0 @@ -407,7 +407,7 @@ get_project_number(){ } check_bq_status(){ - echo "Waiting for BigQuery jobs to complete..." + echo "Waiting for BigQuery jobs to complete..." continue=1 while [[ ${continue} -gt 0 ]] do @@ -432,7 +432,7 @@ else project_prompt fi # Create workers for the correct sources -read -p "Which version control system are you using? +read -p "Which version control system are you using? (1) GitLab (2) GitHub (3) Other @@ -440,7 +440,7 @@ read -p "Which version control system are you using? Enter a selection (1 - 3):" git_system -read -p "Which CI/CD system are you using? +read -p "Which CI/CD system are you using? (1) Cloud Build (2) Tekton (3) GitLab diff --git a/setup/events_enriched_schema.json b/setup/events_enriched_schema.json index 5d65bf0e..fd04dc00 100644 --- a/setup/events_enriched_schema.json +++ b/setup/events_enriched_schema.json @@ -1,14 +1,14 @@ [ - { - "mode": "REQUIRED", - "name": "events_raw_signature", - "type": "STRING", - "description": "signature of corresponding events_raw row (Foreign Key)" - }, - { - "mode": "NULLABLE", - "name": "enriched_metadata", - "type": "STRING", - "description": "The processed metadata that has been enriched with additional information" - } + { + "mode": "REQUIRED", + "name": "events_raw_signature", + "type": "STRING", + "description": "signature of corresponding events_raw row (Foreign Key)" + }, + { + "mode": "NULLABLE", + "name": "enriched_metadata", + "type": "STRING", + "description": "The processed metadata that has been enriched with additional information" + } ] diff --git a/setup/events_raw_schema.json b/setup/events_raw_schema.json index 35e796eb..45f8d510 100644 --- a/setup/events_raw_schema.json +++ b/setup/events_raw_schema.json @@ -1,37 +1,37 @@ [ - { - "mode": "NULLABLE", - "name": "event_type", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "id", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "metadata", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "time_created", - "type": "TIMESTAMP" - }, - { - "mode": "NULLABLE", - "name": "signature", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "msg_id", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "source", - "type": "STRING" - } + { + "mode": "NULLABLE", + "name": "event_type", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "id", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "metadata", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "time_created", + "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "signature", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "msg_id", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "source", + "type": "STRING" + } ] diff --git a/setup/fourkeys-builder/README.md b/setup/fourkeys-builder/README.md index 6b32f474..02ae83cc 100644 --- a/setup/fourkeys-builder/README.md +++ b/setup/fourkeys-builder/README.md @@ -1,4 +1,4 @@ -This folder contains a Dockerfile to make a builder container for use +This folder contains a Dockerfile to make a builder container for use in Cloud Build... the container has gcloud, python 3, and Terraform installed, all of which are needed to install and test Four Keys. diff --git a/terraform/README.md b/terraform/README.md index 60de684d..ea29ee50 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -1,6 +1,6 @@ # Four Keys Terraform -This directory contains modules and examples for deploying Four Keys with Terraform. The primary module `modules/fourkeys` uses the other sub-modules to deploy resources to a provided Google Cloud Project. +This directory contains modules and examples for deploying Four Keys with Terraform. The primary module `modules/fourkeys` uses the other sub-modules to deploy resources to a provided Google Cloud Project. ## Usage @@ -74,7 +74,7 @@ To deploy Four Keys with Terraform, you will first need: ## Generating mock data -To test your Four Keys deployment, you can generate mock data that simulates events from a GitHub repository. +To test your Four Keys deployment, you can generate mock data that simulates events from a GitHub repository. 1. Export your event handler URL an environment variable. This is the webhook URL that will receive events: diff --git a/terraform/example/main.tf b/terraform/example/main.tf index 272d1cb5..900a541b 100644 --- a/terraform/example/main.tf +++ b/terraform/example/main.tf @@ -1,8 +1,8 @@ module "fourkeys" { - source = "../modules/fourkeys" - project_id = var.project_id - enable_apis = var.enable_apis - region = var.region - bigquery_region = var.bigquery_region - parsers = var.parsers + source = "../modules/fourkeys" + project_id = var.project_id + enable_apis = var.enable_apis + region = var.region + bigquery_region = var.bigquery_region + parsers = var.parsers } diff --git a/terraform/example/variables.tf b/terraform/example/variables.tf index 8b8ea2a8..0b3149b1 100644 --- a/terraform/example/variables.tf +++ b/terraform/example/variables.tf @@ -1,6 +1,6 @@ variable "project_id" { - type = string - description = "project to deploy four keys resources to" + type = string + description = "project to deploy four keys resources to" } variable "enable_apis" { @@ -10,9 +10,9 @@ variable "enable_apis" { } variable "region" { - type = string - default = "us-central1" - description = "Region to deploy four keys resources in." + type = string + default = "us-central1" + description = "Region to deploy four keys resources in." } variable "bigquery_region" { diff --git a/terraform/modules/fourkeys-circleci-parser/main.tf b/terraform/modules/fourkeys-circleci-parser/main.tf index 9e2e5951..3f3d2b52 100644 --- a/terraform/modules/fourkeys-circleci-parser/main.tf +++ b/terraform/modules/fourkeys-circleci-parser/main.tf @@ -9,10 +9,10 @@ locals { } resource "google_project_service" "data_source_services" { - project = var.project_id - for_each = toset(local.services) - service = each.value - disable_on_destroy = false + project = var.project_id + for_each = toset(local.services) + service = each.value + disable_on_destroy = false } resource "google_cloud_run_service" "circleci_parser" { diff --git a/terraform/modules/fourkeys-circleci-parser/variables.tf b/terraform/modules/fourkeys-circleci-parser/variables.tf index 34ebfe1e..4dac26b0 100644 --- a/terraform/modules/fourkeys-circleci-parser/variables.tf +++ b/terraform/modules/fourkeys-circleci-parser/variables.tf @@ -1,16 +1,16 @@ variable "project_id" { - type = string + type = string description = "Project ID of the target project." } variable "region" { - type = string + type = string description = "Region to deploy resources." - default = "us-central1" + default = "us-central1" } variable "fourkeys_service_account_email" { - type = string + type = string description = "Service account for fourkeys." } @@ -21,6 +21,6 @@ variable "enable_apis" { } variable "parser_container_url" { - type = string + type = string description = "URL of image to use in Cloud Run service configuration." } \ No newline at end of file diff --git a/terraform/modules/fourkeys-cloud-build-parser/README.md b/terraform/modules/fourkeys-cloud-build-parser/README.md index 216ce62d..3dc92563 100644 --- a/terraform/modules/fourkeys-cloud-build-parser/README.md +++ b/terraform/modules/fourkeys-cloud-build-parser/README.md @@ -1,4 +1,4 @@ -This module creates the pubsub topic and parser for Cloud Build. The name of the pubsub topic is important! +This module creates the pubsub topic and parser for Cloud Build. The name of the pubsub topic is important! Cloud Build is designed so that if a topic exists named `cloud-builds`, build events are automatically written to it. But that topic is not created by default, [it has to be deliberately created](https://cloud.google.com/build/docs/subscribe-build-notifications#receiving_build_notifications). diff --git a/terraform/modules/fourkeys-cloud-build-parser/main.tf b/terraform/modules/fourkeys-cloud-build-parser/main.tf index 8a7515dd..e42a27d5 100644 --- a/terraform/modules/fourkeys-cloud-build-parser/main.tf +++ b/terraform/modules/fourkeys-cloud-build-parser/main.tf @@ -9,10 +9,10 @@ locals { } resource "google_project_service" "data_source_services" { - project = var.project_id - for_each = toset(local.services) - service = each.value - disable_on_destroy = false + project = var.project_id + for_each = toset(local.services) + service = each.value + disable_on_destroy = false } resource "google_cloud_run_service" "cloudbuild_parser" { diff --git a/terraform/modules/fourkeys-cloud-build-parser/variables.tf b/terraform/modules/fourkeys-cloud-build-parser/variables.tf index 34ebfe1e..4dac26b0 100644 --- a/terraform/modules/fourkeys-cloud-build-parser/variables.tf +++ b/terraform/modules/fourkeys-cloud-build-parser/variables.tf @@ -1,16 +1,16 @@ variable "project_id" { - type = string + type = string description = "Project ID of the target project." } variable "region" { - type = string + type = string description = "Region to deploy resources." - default = "us-central1" + default = "us-central1" } variable "fourkeys_service_account_email" { - type = string + type = string description = "Service account for fourkeys." } @@ -21,6 +21,6 @@ variable "enable_apis" { } variable "parser_container_url" { - type = string + type = string description = "URL of image to use in Cloud Run service configuration." } \ No newline at end of file diff --git a/terraform/modules/fourkeys-github-parser/main.tf b/terraform/modules/fourkeys-github-parser/main.tf index 1aaebe9b..368c95ce 100644 --- a/terraform/modules/fourkeys-github-parser/main.tf +++ b/terraform/modules/fourkeys-github-parser/main.tf @@ -9,10 +9,10 @@ locals { } resource "google_project_service" "data_source_services" { - project = var.project_id - for_each = toset(local.services) - service = each.value - disable_on_destroy = false + project = var.project_id + for_each = toset(local.services) + service = each.value + disable_on_destroy = false } resource "google_cloud_run_service" "github_parser" { diff --git a/terraform/modules/fourkeys-github-parser/variables.tf b/terraform/modules/fourkeys-github-parser/variables.tf index 34ebfe1e..4dac26b0 100644 --- a/terraform/modules/fourkeys-github-parser/variables.tf +++ b/terraform/modules/fourkeys-github-parser/variables.tf @@ -1,16 +1,16 @@ variable "project_id" { - type = string + type = string description = "Project ID of the target project." } variable "region" { - type = string + type = string description = "Region to deploy resources." - default = "us-central1" + default = "us-central1" } variable "fourkeys_service_account_email" { - type = string + type = string description = "Service account for fourkeys." } @@ -21,6 +21,6 @@ variable "enable_apis" { } variable "parser_container_url" { - type = string + type = string description = "URL of image to use in Cloud Run service configuration." } \ No newline at end of file diff --git a/terraform/modules/fourkeys-gitlab-parser/main.tf b/terraform/modules/fourkeys-gitlab-parser/main.tf index 1fc2931b..28f99531 100644 --- a/terraform/modules/fourkeys-gitlab-parser/main.tf +++ b/terraform/modules/fourkeys-gitlab-parser/main.tf @@ -9,10 +9,10 @@ locals { } resource "google_project_service" "data_source_services" { - project = var.project_id - for_each = toset(local.services) - service = each.value - disable_on_destroy = false + project = var.project_id + for_each = toset(local.services) + service = each.value + disable_on_destroy = false } resource "google_cloud_run_service" "gitlab_parser" { @@ -72,7 +72,7 @@ resource "google_pubsub_subscription" "gitlab" { project = var.project_id name = "gitlab" topic = google_pubsub_topic.gitlab.id - + expiration_policy { ttl = "" } diff --git a/terraform/modules/fourkeys-gitlab-parser/variables.tf b/terraform/modules/fourkeys-gitlab-parser/variables.tf index 34ebfe1e..4dac26b0 100644 --- a/terraform/modules/fourkeys-gitlab-parser/variables.tf +++ b/terraform/modules/fourkeys-gitlab-parser/variables.tf @@ -1,16 +1,16 @@ variable "project_id" { - type = string + type = string description = "Project ID of the target project." } variable "region" { - type = string + type = string description = "Region to deploy resources." - default = "us-central1" + default = "us-central1" } variable "fourkeys_service_account_email" { - type = string + type = string description = "Service account for fourkeys." } @@ -21,6 +21,6 @@ variable "enable_apis" { } variable "parser_container_url" { - type = string + type = string description = "URL of image to use in Cloud Run service configuration." } \ No newline at end of file diff --git a/terraform/modules/fourkeys-pagerduty-parser/main.tf b/terraform/modules/fourkeys-pagerduty-parser/main.tf index b91df506..521bed56 100644 --- a/terraform/modules/fourkeys-pagerduty-parser/main.tf +++ b/terraform/modules/fourkeys-pagerduty-parser/main.tf @@ -9,10 +9,10 @@ locals { } resource "google_project_service" "data_source_services" { - project = var.project_id - for_each = toset(local.services) - service = each.value - disable_on_destroy = false + project = var.project_id + for_each = toset(local.services) + service = each.value + disable_on_destroy = false } resource "google_cloud_run_service" "pagerduty_parser" { @@ -37,7 +37,7 @@ resource "google_cloud_run_service" "pagerduty_parser" { percent = 100 latest_revision = true } - + metadata { annotations = { "run.googleapis.com/ingress" = "internal" diff --git a/terraform/modules/fourkeys-pagerduty-parser/variables.tf b/terraform/modules/fourkeys-pagerduty-parser/variables.tf index 34ebfe1e..4dac26b0 100644 --- a/terraform/modules/fourkeys-pagerduty-parser/variables.tf +++ b/terraform/modules/fourkeys-pagerduty-parser/variables.tf @@ -1,16 +1,16 @@ variable "project_id" { - type = string + type = string description = "Project ID of the target project." } variable "region" { - type = string + type = string description = "Region to deploy resources." - default = "us-central1" + default = "us-central1" } variable "fourkeys_service_account_email" { - type = string + type = string description = "Service account for fourkeys." } @@ -21,6 +21,6 @@ variable "enable_apis" { } variable "parser_container_url" { - type = string + type = string description = "URL of image to use in Cloud Run service configuration." } \ No newline at end of file diff --git a/terraform/modules/fourkeys-tekton-parser/main.tf b/terraform/modules/fourkeys-tekton-parser/main.tf index dfcd1bd0..92f5b55b 100644 --- a/terraform/modules/fourkeys-tekton-parser/main.tf +++ b/terraform/modules/fourkeys-tekton-parser/main.tf @@ -9,10 +9,10 @@ locals { } resource "google_project_service" "data_source_services" { - project = var.project_id - for_each = toset(local.services) - service = each.value - disable_on_destroy = false + project = var.project_id + for_each = toset(local.services) + service = each.value + disable_on_destroy = false } resource "google_cloud_run_service" "tekton_parser" { diff --git a/terraform/modules/fourkeys-tekton-parser/variables.tf b/terraform/modules/fourkeys-tekton-parser/variables.tf index 34ebfe1e..4dac26b0 100644 --- a/terraform/modules/fourkeys-tekton-parser/variables.tf +++ b/terraform/modules/fourkeys-tekton-parser/variables.tf @@ -1,16 +1,16 @@ variable "project_id" { - type = string + type = string description = "Project ID of the target project." } variable "region" { - type = string + type = string description = "Region to deploy resources." - default = "us-central1" + default = "us-central1" } variable "fourkeys_service_account_email" { - type = string + type = string description = "Service account for fourkeys." } @@ -21,6 +21,6 @@ variable "enable_apis" { } variable "parser_container_url" { - type = string + type = string description = "URL of image to use in Cloud Run service configuration." } \ No newline at end of file diff --git a/terraform/modules/fourkeys/locals.tf b/terraform/modules/fourkeys/locals.tf index a8f92647..e71857e9 100644 --- a/terraform/modules/fourkeys/locals.tf +++ b/terraform/modules/fourkeys/locals.tf @@ -6,12 +6,12 @@ locals { cloud_build_service_account = "${data.google_project.project.number}@cloudbuild.gserviceaccount.com" event_handler_container_url = var.event_handler_container_url == "" ? format("gcr.io/%s/event-handler", var.project_id) : var.event_handler_container_url dashboard_container_url = var.dashboard_container_url == "" ? format("gcr.io/%s/fourkeys-grafana-dashboard", var.project_id) : var.dashboard_container_url - github_parser_url = var.github_parser_url == "" ? format("gcr.io/%s/github-parser", var.project_id) : var.github_parser_url - gitlab_parser_url = var.gitlab_parser_url == "" ? format("gcr.io/%s/gitlab-parser", var.project_id) : var.gitlab_parser_url - cloud_build_parser_url = var.cloud_build_parser_url == "" ? format("gcr.io/%s/cloud-build-parser", var.project_id) : var.cloud_build_parser_url - tekton_parser_url = var.tekton_parser_url == "" ? format("gcr.io/%s/tekton-parser", var.project_id) : var.tekton_parser_url - circleci_parser_url = var.circleci_parser_url == "" ? format("gcr.io/%s/circleci-parser", var.project_id) : var.circleci_parser_url - pagerduty_parser_url = var.pagerduty_parser_url == "" ? format("gcr.io/%s/pagerduty-parser", var.project_id) : var.pagerduty_parser_url + github_parser_url = var.github_parser_url == "" ? format("gcr.io/%s/github-parser", var.project_id) : var.github_parser_url + gitlab_parser_url = var.gitlab_parser_url == "" ? format("gcr.io/%s/gitlab-parser", var.project_id) : var.gitlab_parser_url + cloud_build_parser_url = var.cloud_build_parser_url == "" ? format("gcr.io/%s/cloud-build-parser", var.project_id) : var.cloud_build_parser_url + tekton_parser_url = var.tekton_parser_url == "" ? format("gcr.io/%s/tekton-parser", var.project_id) : var.tekton_parser_url + circleci_parser_url = var.circleci_parser_url == "" ? format("gcr.io/%s/circleci-parser", var.project_id) : var.circleci_parser_url + pagerduty_parser_url = var.pagerduty_parser_url == "" ? format("gcr.io/%s/pagerduty-parser", var.project_id) : var.pagerduty_parser_url services = var.enable_apis ? [ "bigquery.googleapis.com", "cloudbuild.googleapis.com",