From 8c6c01ed7f861faea5086cb4df8924b80425f729 Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Tue, 12 Mar 2024 11:09:30 +0100 Subject: [PATCH 01/12] Reformat SQL in changes.sql --- queries/changes.sql | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/queries/changes.sql b/queries/changes.sql index 711e7328..0f397227 100644 --- a/queries/changes.sql +++ b/queries/changes.sql @@ -1,10 +1,16 @@ -# Changes Table -SELECT -source, -event_type, -JSON_EXTRACT_SCALAR(commit, '$.id') change_id, -TIMESTAMP_TRUNC(TIMESTAMP(JSON_EXTRACT_SCALAR(commit, '$.timestamp')),second) as time_created, -FROM four_keys.events_raw e, -UNNEST(JSON_EXTRACT_ARRAY(e.metadata, '$.commits')) as commit -WHERE event_type = "push" -GROUP BY 1,2,3,4 \ No newline at end of file +-- Changes View: The `change_id` field is a Git commit's ID +SELECT + source, + event_type, + JSON_EXTRACT_SCALAR(commit, '$.id') AS change_id, + TIMESTAMP_TRUNC(TIMESTAMP(JSON_EXTRACT_SCALAR(commit, '$.timestamp')),second) AS time_created, +FROM + four_keys.events_raw e, + UNNEST(JSON_EXTRACT_ARRAY(e.metadata, '$.commits')) AS commit +WHERE + event_type = "push" +GROUP BY + 1, + 2, + 3, + 4 From 841a2db32e87aed9dc10d1d6675ea8519845befe Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Tue, 12 Mar 2024 11:18:51 +0100 Subject: [PATCH 02/12] Add service field for service repo to changes.sql --- queries/changes.sql | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/queries/changes.sql b/queries/changes.sql index 0f397227..559073e4 100644 --- a/queries/changes.sql +++ b/queries/changes.sql @@ -2,10 +2,16 @@ SELECT source, event_type, + CASE + WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.repository.full_name') + END + AS service, JSON_EXTRACT_SCALAR(commit, '$.id') AS change_id, - TIMESTAMP_TRUNC(TIMESTAMP(JSON_EXTRACT_SCALAR(commit, '$.timestamp')),second) AS time_created, + TIMESTAMP_TRUNC(TIMESTAMP(JSON_EXTRACT_SCALAR(commit, '$.timestamp')),second) AS time_created FROM four_keys.events_raw e, + -- Create a row for each element in the array of `commits` from the `metadata` field. + -- The other fields in the fields in the row are repeated for each element. UNNEST(JSON_EXTRACT_ARRAY(e.metadata, '$.commits')) AS commit WHERE event_type = "push" @@ -13,4 +19,5 @@ GROUP BY 1, 2, 3, - 4 + 4, + 5 From 15f40a972917fb44e65c90b043c8704dce68d607 Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Tue, 12 Mar 2024 11:31:22 +0100 Subject: [PATCH 03/12] Delete GitLab query logic from deployments.sql --- queries/deployments.sql | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/queries/deployments.sql b/queries/deployments.sql index 3fabdb04..6e348631 100644 --- a/queries/deployments.sql +++ b/queries/deployments.sql @@ -1,20 +1,12 @@ # Deployments Table -WITH deploys_cloudbuild_github_gitlab AS (# Cloud Build, Github, Gitlab pipelines +WITH deploys_cloudbuild_github AS (# Cloud Build, Github SELECT source, id as deploy_id, time_created, CASE WHEN source = "cloud_build" then JSON_EXTRACT_SCALAR(metadata, '$.substitutions.COMMIT_SHA') WHEN source like "github%" then JSON_EXTRACT_SCALAR(metadata, '$.deployment.sha') - WHEN source like "gitlab%" then COALESCE( - # Data structure from GitLab Pipelines - JSON_EXTRACT_SCALAR(metadata, '$.commit.id'), - # Data structure from GitLab Deployments - # REGEX to get the commit sha from the URL - REGEXP_EXTRACT( - JSON_EXTRACT_SCALAR(metadata, '$.commit_url'), r".*commit\/(.*)") - ) WHEN source = "argocd" then JSON_EXTRACT_SCALAR(metadata, '$.commit_sha') end as main_commit, CASE WHEN source LIKE "github%" THEN ARRAY( SELECT JSON_EXTRACT_SCALAR(string_element, '$') @@ -26,10 +18,6 @@ WITH deploys_cloudbuild_github_gitlab AS (# Cloud Build, Github, Gitlab pipeline (source = "cloud_build" AND JSON_EXTRACT_SCALAR(metadata, '$.status') = "SUCCESS") # GitHub Deployments OR (source LIKE "github%" and event_type = "deployment_status" and JSON_EXTRACT_SCALAR(metadata, '$.deployment_status.state') = "success") - # GitLab Pipelines - OR (source LIKE "gitlab%" AND event_type = "pipeline" AND JSON_EXTRACT_SCALAR(metadata, '$.object_attributes.status') = "success") - # GitLab Deployments - OR (source LIKE "gitlab%" AND event_type = "deployment" AND JSON_EXTRACT_SCALAR(metadata, '$.status') = "success") # ArgoCD Deployments OR (source = "argocd" AND JSON_EXTRACT_SCALAR(metadata, '$.status') = "SUCCESS") ) @@ -63,7 +51,7 @@ WITH deploys_cloudbuild_github_gitlab AS (# Cloud Build, Github, Gitlab pipeline ), deploys AS ( SELECT * FROM - deploys_cloudbuild_github_gitlab + deploys_cloudbuild_github UNION ALL SELECT * FROM deploys_tekton UNION ALL From 8b09cc5e5316e3cc792a5c82c5890ef50c1e827b Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Tue, 12 Mar 2024 11:32:53 +0100 Subject: [PATCH 04/12] Delete Tekton query logic from deployments.sql --- queries/deployments.sql | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/queries/deployments.sql b/queries/deployments.sql index 6e348631..5a082847 100644 --- a/queries/deployments.sql +++ b/queries/deployments.sql @@ -22,23 +22,6 @@ WITH deploys_cloudbuild_github AS (# Cloud Build, Github OR (source = "argocd" AND JSON_EXTRACT_SCALAR(metadata, '$.status') = "SUCCESS") ) ), - deploys_tekton AS (# Tekton Pipelines - SELECT - source, - id as deploy_id, - time_created, - IF(JSON_EXTRACT_SCALAR(param, '$.name') = "gitrevision", JSON_EXTRACT_SCALAR(param, '$.value'), Null) as main_commit, - ARRAY[] AS additional_commits - FROM ( - SELECT - id, - TIMESTAMP_TRUNC(time_created, second) as time_created, - source, - four_keys.json2array(JSON_EXTRACT(metadata, '$.data.pipelineRun.spec.params')) params - FROM four_keys.events_raw - WHERE event_type = "dev.tekton.event.pipelinerun.successful.v1" - AND metadata like "%gitrevision%") e, e.params as param - ), deploys_circleci AS (# CircleCI pipelines SELECT source, @@ -53,8 +36,6 @@ WITH deploys_cloudbuild_github AS (# Cloud Build, Github SELECT * FROM deploys_cloudbuild_github UNION ALL - SELECT * FROM deploys_tekton - UNION ALL SELECT * FROM deploys_circleci ), changes_raw AS ( From c3c7fecce5863a5101e4fb8b3c4508e3a29d04b7 Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Tue, 12 Mar 2024 12:02:41 +0100 Subject: [PATCH 05/12] Reformat SQL in deployments.sql --- queries/deployments.sql | 136 +++++++++++++++++++++++----------------- 1 file changed, 79 insertions(+), 57 deletions(-) diff --git a/queries/deployments.sql b/queries/deployments.sql index 5a082847..79e05eae 100644 --- a/queries/deployments.sql +++ b/queries/deployments.sql @@ -1,71 +1,93 @@ -# Deployments Table - -WITH deploys_cloudbuild_github AS (# Cloud Build, Github - SELECT +-- Deployments View: For GitHub `deploy_id` is the ID of the Deployment Status. +WITH + deploys_cloudbuild_github AS ( -- Cloud Build, GitHub, ArgoCD + SELECT source, - id as deploy_id, + id AS deploy_id, time_created, - CASE WHEN source = "cloud_build" then JSON_EXTRACT_SCALAR(metadata, '$.substitutions.COMMIT_SHA') - WHEN source like "github%" then JSON_EXTRACT_SCALAR(metadata, '$.deployment.sha') - WHEN source = "argocd" then JSON_EXTRACT_SCALAR(metadata, '$.commit_sha') end as main_commit, - CASE WHEN source LIKE "github%" THEN ARRAY( - SELECT JSON_EXTRACT_SCALAR(string_element, '$') - FROM UNNEST(JSON_EXTRACT_ARRAY(metadata, '$.deployment.additional_sha')) AS string_element) - ELSE ARRAY[] end as additional_commits - FROM four_keys.events_raw - WHERE ( - # Cloud Build Deployments - (source = "cloud_build" AND JSON_EXTRACT_SCALAR(metadata, '$.status') = "SUCCESS") - # GitHub Deployments - OR (source LIKE "github%" and event_type = "deployment_status" and JSON_EXTRACT_SCALAR(metadata, '$.deployment_status.state') = "success") - # ArgoCD Deployments - OR (source = "argocd" AND JSON_EXTRACT_SCALAR(metadata, '$.status') = "SUCCESS") - ) - ), - deploys_circleci AS (# CircleCI pipelines - SELECT + CASE + WHEN source = "cloud_build" THEN JSON_EXTRACT_SCALAR(metadata, '$.substitutions.COMMIT_SHA') + WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.deployment.sha') + WHEN source = "argocd" THEN JSON_EXTRACT_SCALAR(metadata, '$.commit_sha') + END + AS main_commit, + CASE + WHEN source LIKE "github%" THEN ARRAY( SELECT JSON_EXTRACT_SCALAR(string_element, '$') FROM UNNEST(JSON_EXTRACT_ARRAY(metadata, '$.deployment.additional_sha')) AS string_element) + ELSE + ARRAY[] + END + AS additional_commits + FROM + four_keys.events_raw + WHERE + ( + -- Cloud Build Deployments + (source = "cloud_build" AND JSON_EXTRACT_SCALAR(metadata, '$.status') = "SUCCESS") + -- GitHub Deployments + OR (source LIKE "github%" AND event_type = "deployment_status" AND JSON_EXTRACT_SCALAR(metadata, '$.deployment_status.state') = "success") + -- ArgoCD Deployments + OR (source = "argocd" AND JSON_EXTRACT_SCALAR(metadata, '$.status') = "SUCCESS") ) + ), + deploys_circleci AS ( -- CircleCI pipelines + SELECT source, id AS deploy_id, time_created, JSON_EXTRACT_SCALAR(metadata, '$.pipeline.vcs.revision') AS main_commit, ARRAY[] AS additional_commits - FROM four_keys.events_raw - WHERE (source = "circleci" AND event_type = "workflow-completed" AND JSON_EXTRACT_SCALAR(metadata, '$.workflow.name') LIKE "%deploy%" AND JSON_EXTRACT_SCALAR(metadata, '$.workflow.status') = "success") - ), - deploys AS ( - SELECT * FROM + FROM + four_keys.events_raw + WHERE + (source = "circleci" + AND event_type = "workflow-completed" + AND JSON_EXTRACT_SCALAR(metadata, '$.workflow.name') LIKE "%deploy%" + AND JSON_EXTRACT_SCALAR(metadata, '$.workflow.status') = "success") + ), + deploys AS ( + SELECT + * + FROM deploys_cloudbuild_github - UNION ALL - SELECT * FROM deploys_circleci - ), - changes_raw AS ( - SELECT + UNION ALL + SELECT + * + FROM + deploys_circleci + ), + changes_raw AS ( + SELECT id, - metadata as change_metadata - FROM four_keys.events_raw - ), - deployment_changes as ( - SELECT + metadata AS change_metadata + FROM + four_keys.events_raw + ), + deployment_changes AS ( + SELECT source, deploy_id, deploys.time_created time_created, change_metadata, - four_keys.json2array(JSON_EXTRACT(change_metadata, '$.commits')) as array_commits, + four_keys.json2array(JSON_EXTRACT(change_metadata, '$.commits')) AS array_commits, main_commit - FROM deploys - JOIN - changes_raw on ( - changes_raw.id = deploys.main_commit - or changes_raw.id in unnest(deploys.additional_commits) - ) - ) - - SELECT - source, - deploy_id, - time_created, - main_commit, - ARRAY_AGG(DISTINCT JSON_EXTRACT_SCALAR(array_commits, '$.id')) changes, - FROM deployment_changes - CROSS JOIN deployment_changes.array_commits - GROUP BY 1,2,3,4; + FROM + deploys + JOIN + changes_raw + ON + ( changes_raw.id = deploys.main_commit OR changes_raw.id IN UNNEST(deploys.additional_commits) ) + ) +SELECT + source, + deploy_id, + time_created, + main_commit, + ARRAY_AGG(DISTINCT JSON_EXTRACT_SCALAR(array_commits, '$.id')) AS changes, +FROM + deployment_changes +CROSS JOIN + deployment_changes.array_commits +GROUP BY + 1, + 2, + 3, + 4; From 196727f6a1909700ce8484f251ff5a14fe6dce37 Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Tue, 12 Mar 2024 12:07:20 +0100 Subject: [PATCH 06/12] Add service field for service repo to deployments.sql --- queries/deployments.sql | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/queries/deployments.sql b/queries/deployments.sql index 79e05eae..d1a24848 100644 --- a/queries/deployments.sql +++ b/queries/deployments.sql @@ -80,6 +80,10 @@ SELECT source, deploy_id, time_created, + CASE + WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(change_metadata, '$.repository.full_name') + END + AS service, main_commit, ARRAY_AGG(DISTINCT JSON_EXTRACT_SCALAR(array_commits, '$.id')) AS changes, FROM @@ -90,4 +94,5 @@ GROUP BY 1, 2, 3, - 4; + 4, + 5; From 7dde90d832dae1bb6ecddba89e0fade2f7979290 Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Tue, 12 Mar 2024 20:22:35 +0100 Subject: [PATCH 07/12] Delete CircleCI query logic from deployments.sql --- queries/deployments.sql | 31 +++---------------------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/queries/deployments.sql b/queries/deployments.sql index d1a24848..874b4e26 100644 --- a/queries/deployments.sql +++ b/queries/deployments.sql @@ -1,6 +1,6 @@ -- Deployments View: For GitHub `deploy_id` is the ID of the Deployment Status. WITH - deploys_cloudbuild_github AS ( -- Cloud Build, GitHub, ArgoCD + deploys AS ( -- Cloud Build, GitHub, ArgoCD SELECT source, id AS deploy_id, @@ -26,33 +26,8 @@ WITH -- GitHub Deployments OR (source LIKE "github%" AND event_type = "deployment_status" AND JSON_EXTRACT_SCALAR(metadata, '$.deployment_status.state') = "success") -- ArgoCD Deployments - OR (source = "argocd" AND JSON_EXTRACT_SCALAR(metadata, '$.status') = "SUCCESS") ) - ), - deploys_circleci AS ( -- CircleCI pipelines - SELECT - source, - id AS deploy_id, - time_created, - JSON_EXTRACT_SCALAR(metadata, '$.pipeline.vcs.revision') AS main_commit, - ARRAY[] AS additional_commits - FROM - four_keys.events_raw - WHERE - (source = "circleci" - AND event_type = "workflow-completed" - AND JSON_EXTRACT_SCALAR(metadata, '$.workflow.name') LIKE "%deploy%" - AND JSON_EXTRACT_SCALAR(metadata, '$.workflow.status') = "success") - ), - deploys AS ( - SELECT - * - FROM - deploys_cloudbuild_github - UNION ALL - SELECT - * - FROM - deploys_circleci + OR (source = "argocd" AND JSON_EXTRACT_SCALAR(metadata, '$.status') = "SUCCESS") + ) ), changes_raw AS ( SELECT From ad67e5c5fd9b5eff5ca03902c2a01f9052f4ea40 Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Tue, 12 Mar 2024 20:46:03 +0100 Subject: [PATCH 08/12] Update deployments.sql to join changes also on service --- queries/deployments.sql | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/queries/deployments.sql b/queries/deployments.sql index 874b4e26..a00d2bc5 100644 --- a/queries/deployments.sql +++ b/queries/deployments.sql @@ -3,6 +3,10 @@ WITH deploys AS ( -- Cloud Build, GitHub, ArgoCD SELECT source, + CASE + WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.repository.full_name') + END + AS service, id AS deploy_id, time_created, CASE @@ -32,13 +36,18 @@ WITH changes_raw AS ( SELECT id, - metadata AS change_metadata + metadata AS change_metadata, + CASE + WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.repository.full_name') + END + AS service FROM four_keys.events_raw ), deployment_changes AS ( SELECT source, + deploys.service, deploy_id, deploys.time_created time_created, change_metadata, @@ -49,16 +58,13 @@ WITH JOIN changes_raw ON - ( changes_raw.id = deploys.main_commit OR changes_raw.id IN UNNEST(deploys.additional_commits) ) + ( changes_raw.service = deploys.service ) AND ( changes_raw.id = deploys.main_commit OR changes_raw.id IN UNNEST(deploys.additional_commits) ) ) SELECT source, + service, deploy_id, time_created, - CASE - WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(change_metadata, '$.repository.full_name') - END - AS service, main_commit, ARRAY_AGG(DISTINCT JSON_EXTRACT_SCALAR(array_commits, '$.id')) AS changes, FROM From 5765a386d8b5a2cce097512b1b442609261db723 Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Tue, 12 Mar 2024 12:15:27 +0100 Subject: [PATCH 09/12] Delete GitLab query logic from incidents.sql --- queries/incidents.sql | 5 ----- 1 file changed, 5 deletions(-) diff --git a/queries/incidents.sql b/queries/incidents.sql index bd80619e..dc8c36fa 100644 --- a/queries/incidents.sql +++ b/queries/incidents.sql @@ -10,21 +10,16 @@ FROM SELECT source, CASE WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.issue.number') - WHEN source LIKE "gitlab%" AND event_type = "note" THEN JSON_EXTRACT_SCALAR(metadata, '$.object_attributes.noteable_id') - WHEN source LIKE "gitlab%" AND event_type = "issue" THEN JSON_EXTRACT_SCALAR(metadata, '$.object_attributes.id') WHEN source LIKE "pagerduty%" THEN JSON_EXTRACT_SCALAR(metadata, '$.event.data.id') END AS incident_id, CASE WHEN source LIKE "github%" THEN TIMESTAMP(JSON_EXTRACT_SCALAR(metadata, '$.issue.created_at')) - WHEN source LIKE "gitlab%" THEN four_keys.multiFormatParseTimestamp(JSON_EXTRACT_SCALAR(metadata, '$.object_attributes.created_at')) WHEN source LIKE "pagerduty%" THEN TIMESTAMP(JSON_EXTRACT_SCALAR(metadata, '$.event.occurred_at')) END AS time_created, CASE WHEN source LIKE "github%" THEN TIMESTAMP(JSON_EXTRACT_SCALAR(metadata, '$.issue.closed_at')) - WHEN source LIKE "gitlab%" THEN four_keys.multiFormatParseTimestamp(JSON_EXTRACT_SCALAR(metadata, '$.object_attributes.closed_at')) WHEN source LIKE "pagerduty%" THEN TIMESTAMP(JSON_EXTRACT_SCALAR(metadata, '$.event.occurred_at')) END AS time_resolved, REGEXP_EXTRACT(metadata, r"root cause: ([[:alnum:]]*)") as root_cause, CASE WHEN source LIKE "github%" THEN REGEXP_CONTAINS(JSON_EXTRACT(metadata, '$.issue.labels'), '"name":"Incident"') - WHEN source LIKE "gitlab%" THEN REGEXP_CONTAINS(JSON_EXTRACT(metadata, '$.object_attributes.labels'), '"title":"Incident"') WHEN source LIKE "pagerduty%" THEN TRUE # All Pager Duty events are incident-related END AS bug, FROM four_keys.events_raw From 73bdc9ede1e3ae6ab13108259f35e684452f2f27 Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Tue, 12 Mar 2024 12:27:02 +0100 Subject: [PATCH 10/12] Reformat SQL in incidents.sql --- queries/incidents.sql | 82 +++++++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 30 deletions(-) diff --git a/queries/incidents.sql b/queries/incidents.sql index dc8c36fa..c790a8a7 100644 --- a/queries/incidents.sql +++ b/queries/incidents.sql @@ -1,31 +1,53 @@ -# Incidents Table +-- Incidents View SELECT -source, -incident_id, -MIN(IF(root.time_created < issue.time_created, root.time_created, issue.time_created)) as time_created, -MAX(time_resolved) as time_resolved, -ARRAY_AGG(root_cause IGNORE NULLS) changes, -FROM -( -SELECT -source, -CASE WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.issue.number') - WHEN source LIKE "pagerduty%" THEN JSON_EXTRACT_SCALAR(metadata, '$.event.data.id') - END AS incident_id, -CASE WHEN source LIKE "github%" THEN TIMESTAMP(JSON_EXTRACT_SCALAR(metadata, '$.issue.created_at')) - WHEN source LIKE "pagerduty%" THEN TIMESTAMP(JSON_EXTRACT_SCALAR(metadata, '$.event.occurred_at')) - END AS time_created, -CASE WHEN source LIKE "github%" THEN TIMESTAMP(JSON_EXTRACT_SCALAR(metadata, '$.issue.closed_at')) - WHEN source LIKE "pagerduty%" THEN TIMESTAMP(JSON_EXTRACT_SCALAR(metadata, '$.event.occurred_at')) - END AS time_resolved, -REGEXP_EXTRACT(metadata, r"root cause: ([[:alnum:]]*)") as root_cause, -CASE WHEN source LIKE "github%" THEN REGEXP_CONTAINS(JSON_EXTRACT(metadata, '$.issue.labels'), '"name":"Incident"') - WHEN source LIKE "pagerduty%" THEN TRUE # All Pager Duty events are incident-related - END AS bug, -FROM four_keys.events_raw -WHERE event_type LIKE "issue%" OR event_type LIKE "incident%" OR (event_type = "note" and JSON_EXTRACT_SCALAR(metadata, '$.object_attributes.noteable_type') = 'Issue') -) issue -LEFT JOIN (SELECT time_created, changes FROM four_keys.deployments d, d.changes) root on root.changes = root_cause -GROUP BY 1,2 -HAVING max(bug) is True -; + source, + incident_id, + MIN(IF(root.time_created < issue.time_created, root.time_created, issue.time_created)) AS time_created, + MAX(time_resolved) AS time_resolved, + ARRAY_AGG(root_cause IGNORE NULLS) AS changes, +FROM ( + SELECT + source, + CASE + WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.issue.number') + WHEN source LIKE "pagerduty%" THEN JSON_EXTRACT_SCALAR(metadata, '$.event.data.id') + END + AS incident_id, + CASE + WHEN source LIKE "github%" THEN TIMESTAMP(JSON_EXTRACT_SCALAR(metadata, '$.issue.created_at')) + WHEN source LIKE "pagerduty%" THEN TIMESTAMP(JSON_EXTRACT_SCALAR(metadata, '$.event.occurred_at')) + END + AS time_created, + CASE + WHEN source LIKE "github%" THEN TIMESTAMP(JSON_EXTRACT_SCALAR(metadata, '$.issue.closed_at')) + WHEN source LIKE "pagerduty%" THEN TIMESTAMP(JSON_EXTRACT_SCALAR(metadata, '$.event.occurred_at')) + END + AS time_resolved, + REGEXP_EXTRACT(metadata, r"root cause: ([[:alnum:]]*)") AS root_cause, + CASE + WHEN source LIKE "github%" THEN REGEXP_CONTAINS(JSON_EXTRACT(metadata, '$.issue.labels'), '"name":"Incident"') + WHEN source LIKE "pagerduty%" THEN TRUE # All Pager Duty events are incident-related + END + AS bug, + FROM + four_keys.events_raw + WHERE + event_type LIKE "issue%" + OR event_type LIKE "incident%" + OR (event_type = "note" AND JSON_EXTRACT_SCALAR(metadata, '$.object_attributes.noteable_type') = 'Issue') +) AS issue +LEFT JOIN ( + SELECT + time_created, + changes + FROM + four_keys.deployments d, + d.changes +) AS root +ON + root.changes = root_cause +GROUP BY + 1, + 2 +HAVING + MAX(bug) IS TRUE ; From da5102fc430a6d996cd66272c513c7c6b3086d43 Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Tue, 12 Mar 2024 12:35:59 +0100 Subject: [PATCH 11/12] Add service field to incidents.sql --- queries/incidents.sql | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/queries/incidents.sql b/queries/incidents.sql index c790a8a7..11f470be 100644 --- a/queries/incidents.sql +++ b/queries/incidents.sql @@ -1,6 +1,7 @@ -- Incidents View SELECT source, + service, incident_id, MIN(IF(root.time_created < issue.time_created, root.time_created, issue.time_created)) AS time_created, MAX(time_resolved) AS time_resolved, @@ -8,6 +9,11 @@ SELECT FROM ( SELECT source, + CASE + WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.repository.full_name') + WHEN source LIKE "pagerduty%" THEN JSON_EXTRACT_SCALAR(metadata, '$.event.data.service.summary') + END + AS service, CASE WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.issue.number') WHEN source LIKE "pagerduty%" THEN JSON_EXTRACT_SCALAR(metadata, '$.event.data.id') @@ -48,6 +54,7 @@ ON root.changes = root_cause GROUP BY 1, - 2 + 2, + 3 HAVING MAX(bug) IS TRUE ; From e7b6bee71692fe917b8aac500066abbc82368009 Mon Sep 17 00:00:00 2001 From: Raphael Pierzina Date: Wed, 13 Mar 2024 20:26:33 +0100 Subject: [PATCH 12/12] Fix comment in changes.sql --- queries/changes.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queries/changes.sql b/queries/changes.sql index 559073e4..4b535bda 100644 --- a/queries/changes.sql +++ b/queries/changes.sql @@ -11,7 +11,7 @@ SELECT FROM four_keys.events_raw e, -- Create a row for each element in the array of `commits` from the `metadata` field. - -- The other fields in the fields in the row are repeated for each element. + -- The other fields in the row are repeated for each `commit`. UNNEST(JSON_EXTRACT_ARRAY(e.metadata, '$.commits')) AS commit WHERE event_type = "push"