From 234c3060d6fb452f53fb2437bc0dd3aaccc345a5 Mon Sep 17 00:00:00 2001 From: Tony Hsu Date: Tue, 14 Jan 2025 09:51:25 +0100 Subject: [PATCH 1/2] Implement polling vaccine --- .gitlab-ci.yml | 99 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 96 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f13052d9266..8664205f615 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -158,9 +158,102 @@ vaccine: needs: [create-multiarch-lib-injection-image] script: | GH_VACCINE_PAT=$(vault kv get -field=vaccine-token kv/k8s/gitlab-runner/dd-trace-rb/github-token) + REPO="TonyCTHsu/vaccine" + POLL_INTERVAL=30 # seconds - curl -X POST \ + # Trigger workflow + echo "Triggering workflow..." + TRIGGER_RESPONSE=$(curl -X POST \ -H "Accept: application/vnd.github.v3+json" \ -H "Authorization: token $GH_VACCINE_PAT" \ - https://api.github.com/repos/TonyCTHsu/vaccine/actions/workflows/vaccine.yml/dispatches \ - -d '{"ref":"master", "inputs": {"commit_sha": "'$CI_COMMIT_SHA'"}}' + -w "\n%{http_code}" \ + "https://api.github.com/repos/$REPO/actions/workflows/vaccine.yml/dispatches" \ + -d '{"ref":"master", "inputs": {"commit_sha": "'$CI_COMMIT_SHA'"}}' 2>&1) + + HTTP_STATUS=$(echo "$TRIGGER_RESPONSE" | tail -n1) + RESPONSE_BODY=$(echo "$TRIGGER_RESPONSE" | sed '$ d') + + if [ "$HTTP_STATUS" -eq 403 ]; then + echo "Error: Workflow trigger failed - Authentication failed" + echo "Response: $RESPONSE_BODY" + exit 1 + elif [ "$HTTP_STATUS" -ne 204 ]; then + echo "Error: Workflow trigger failed with status $HTTP_STATUS" + echo "Response: $RESPONSE_BODY" + exit 1 + fi + + echo "Successfully triggered workflow. Waiting for workflow to start..." + sleep 10 # Give GitHub a moment to create the workflow run + + # Get the most recent workflow run + echo "Fetching most recent workflow run..." + RUNS_RESPONSE=$(curl -s \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: token $GH_VACCINE_PAT" \ + -w "\n%{http_code}" \ + "https://api.github.com/repos/$REPO/actions/runs?event=workflow_dispatch&per_page=1" 2>&1) + + HTTP_STATUS=$(echo "$RUNS_RESPONSE" | tail -n1) + RESPONSE_BODY=$(echo "$RUNS_RESPONSE" | sed '$ d') + + if [ "$HTTP_STATUS" -eq 403 ]; then + echo "Error: Fetching runs failed - Authentication failed" + echo "Response: $RESPONSE_BODY" + exit 1 + elif [ "$HTTP_STATUS" -ne 200 ]; then + echo "Error: Fetching runs failed with status $HTTP_STATUS" + echo "Response: $RESPONSE_BODY" + exit 1 + fi + + echo "Response body: $RESPONSE_BODY" + + # Get the most recent run ID + WORKFLOW_ID=$(echo "$RESPONSE_BODY" | jq -r '.workflow_runs[0].id') + + if [ -z "$WORKFLOW_ID" ] || [ "$WORKFLOW_ID" = "null" ]; then + echo "Error: Could not find recent workflow run" + exit 1 + fi + + echo "Found workflow run ID: $WORKFLOW_ID" + + # Poll workflow status + while true; do + RUN_RESPONSE=$(curl -s \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: token $GH_VACCINE_PAT" \ + -w "\n%{http_code}" \ + "https://api.github.com/repos/$REPO/actions/runs/$WORKFLOW_ID" 2>&1) + + HTTP_STATUS=$(echo "$RUN_RESPONSE" | tail -n1) + RESPONSE_BODY=$(echo "$RUN_RESPONSE" | sed '$ d') + + if [ "$HTTP_STATUS" -eq 403 ]; then + echo "Error: Fetching run status failed - Authentication failed" + echo "Response: $RESPONSE_BODY" + exit 1 + elif [ "$HTTP_STATUS" -ne 200 ]; then + echo "Error: Fetching run status failed with status $HTTP_STATUS" + echo "Response: $RESPONSE_BODY" + exit 1 + fi + + STATUS=$(echo "$RESPONSE_BODY" | jq -r .status) + CONCLUSION=$(echo "$RESPONSE_BODY" | jq -r .conclusion) + + if [ "$STATUS" = "completed" ]; then + if [ "$CONCLUSION" = "success" ]; then + echo "✅ Workflow completed successfully!" + exit 0 + else + echo "❌ Workflow failed with conclusion: $CONCLUSION" + echo "See details: https://github.com/$REPO/actions/runs/$WORKFLOW_ID" + exit 1 + fi + fi + + echo "Current status: $STATUS (Checking again in ${POLL_INTERVAL}s)" + sleep $POLL_INTERVAL + done From 5ce85388e5e74071534db5c7bebc8d34567c3162 Mon Sep 17 00:00:00 2001 From: Tony Hsu Date: Tue, 14 Jan 2025 12:43:35 +0100 Subject: [PATCH 2/2] Extract to .gitlab/scripts --- .gitlab-ci.yml | 112 ++++--------------------------------- .gitlab/scripts/vaccine.sh | 88 +++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 100 deletions(-) create mode 100755 .gitlab/scripts/vaccine.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8664205f615..58294686337 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -151,109 +151,21 @@ deploy_to_reliability_env: needs: - save_versions +# Currently, the job is implemented with polling mechanism. +# +# Due to the constraints of Github workflow dispatch endpoint, it does not return the workflow run id. +# https://docs.github.com/en/rest/actions/workflows?apiVersion=2022-11-28#create-a-workflow-dispatch-event +# +# We fetch the latest workflow run from vaccine after 5 seconds of the dispatch event. +# False positive/negative result can happen when multiple requests are made within the same window. +# +# TODO: +# Replace polling implementation with reporting status to Github with Github App. This will allow us +# to get a deterministic result without mismatched workflow run id. vaccine: image: $DOCKER_REGISTRY/docker:20.10.13 tags: [ "arch:amd64" ] stage: vaccine needs: [create-multiarch-lib-injection-image] script: | - GH_VACCINE_PAT=$(vault kv get -field=vaccine-token kv/k8s/gitlab-runner/dd-trace-rb/github-token) - REPO="TonyCTHsu/vaccine" - POLL_INTERVAL=30 # seconds - - # Trigger workflow - echo "Triggering workflow..." - TRIGGER_RESPONSE=$(curl -X POST \ - -H "Accept: application/vnd.github.v3+json" \ - -H "Authorization: token $GH_VACCINE_PAT" \ - -w "\n%{http_code}" \ - "https://api.github.com/repos/$REPO/actions/workflows/vaccine.yml/dispatches" \ - -d '{"ref":"master", "inputs": {"commit_sha": "'$CI_COMMIT_SHA'"}}' 2>&1) - - HTTP_STATUS=$(echo "$TRIGGER_RESPONSE" | tail -n1) - RESPONSE_BODY=$(echo "$TRIGGER_RESPONSE" | sed '$ d') - - if [ "$HTTP_STATUS" -eq 403 ]; then - echo "Error: Workflow trigger failed - Authentication failed" - echo "Response: $RESPONSE_BODY" - exit 1 - elif [ "$HTTP_STATUS" -ne 204 ]; then - echo "Error: Workflow trigger failed with status $HTTP_STATUS" - echo "Response: $RESPONSE_BODY" - exit 1 - fi - - echo "Successfully triggered workflow. Waiting for workflow to start..." - sleep 10 # Give GitHub a moment to create the workflow run - - # Get the most recent workflow run - echo "Fetching most recent workflow run..." - RUNS_RESPONSE=$(curl -s \ - -H "Accept: application/vnd.github.v3+json" \ - -H "Authorization: token $GH_VACCINE_PAT" \ - -w "\n%{http_code}" \ - "https://api.github.com/repos/$REPO/actions/runs?event=workflow_dispatch&per_page=1" 2>&1) - - HTTP_STATUS=$(echo "$RUNS_RESPONSE" | tail -n1) - RESPONSE_BODY=$(echo "$RUNS_RESPONSE" | sed '$ d') - - if [ "$HTTP_STATUS" -eq 403 ]; then - echo "Error: Fetching runs failed - Authentication failed" - echo "Response: $RESPONSE_BODY" - exit 1 - elif [ "$HTTP_STATUS" -ne 200 ]; then - echo "Error: Fetching runs failed with status $HTTP_STATUS" - echo "Response: $RESPONSE_BODY" - exit 1 - fi - - echo "Response body: $RESPONSE_BODY" - - # Get the most recent run ID - WORKFLOW_ID=$(echo "$RESPONSE_BODY" | jq -r '.workflow_runs[0].id') - - if [ -z "$WORKFLOW_ID" ] || [ "$WORKFLOW_ID" = "null" ]; then - echo "Error: Could not find recent workflow run" - exit 1 - fi - - echo "Found workflow run ID: $WORKFLOW_ID" - - # Poll workflow status - while true; do - RUN_RESPONSE=$(curl -s \ - -H "Accept: application/vnd.github.v3+json" \ - -H "Authorization: token $GH_VACCINE_PAT" \ - -w "\n%{http_code}" \ - "https://api.github.com/repos/$REPO/actions/runs/$WORKFLOW_ID" 2>&1) - - HTTP_STATUS=$(echo "$RUN_RESPONSE" | tail -n1) - RESPONSE_BODY=$(echo "$RUN_RESPONSE" | sed '$ d') - - if [ "$HTTP_STATUS" -eq 403 ]; then - echo "Error: Fetching run status failed - Authentication failed" - echo "Response: $RESPONSE_BODY" - exit 1 - elif [ "$HTTP_STATUS" -ne 200 ]; then - echo "Error: Fetching run status failed with status $HTTP_STATUS" - echo "Response: $RESPONSE_BODY" - exit 1 - fi - - STATUS=$(echo "$RESPONSE_BODY" | jq -r .status) - CONCLUSION=$(echo "$RESPONSE_BODY" | jq -r .conclusion) - - if [ "$STATUS" = "completed" ]; then - if [ "$CONCLUSION" = "success" ]; then - echo "✅ Workflow completed successfully!" - exit 0 - else - echo "❌ Workflow failed with conclusion: $CONCLUSION" - echo "See details: https://github.com/$REPO/actions/runs/$WORKFLOW_ID" - exit 1 - fi - fi - - echo "Current status: $STATUS (Checking again in ${POLL_INTERVAL}s)" - sleep $POLL_INTERVAL - done + .gitlab/scripts/vaccine.sh diff --git a/.gitlab/scripts/vaccine.sh b/.gitlab/scripts/vaccine.sh new file mode 100755 index 00000000000..306e6178b7f --- /dev/null +++ b/.gitlab/scripts/vaccine.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +set -e + +GH_VACCINE_PAT=$(vault kv get -field=vaccine-token kv/k8s/gitlab-runner/dd-trace-rb/github-token) +REPO="TonyCTHsu/vaccine" # To be migrated +POLL_INTERVAL=60 # seconds + +# Trigger workflow +echo "Triggering workflow..." +TRIGGER_RESPONSE=$(curl -X POST \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: token $GH_VACCINE_PAT" \ + -w "\n%{http_code}" \ + "https://api.github.com/repos/$REPO/actions/workflows/vaccine.yml/dispatches" \ + -d '{"ref":"master", "inputs": {"commit_sha": "'$CI_COMMIT_SHA'"}}' 2>&1) + +HTTP_STATUS=$(echo "$TRIGGER_RESPONSE" | tail -n1) +if [ "$HTTP_STATUS" -ne 204 ]; then + echo "Error: Workflow trigger failed with status $HTTP_STATUS" + echo "Response: $(echo "$TRIGGER_RESPONSE" | sed '$ d')" + exit 1 +fi + +echo "Successfully triggered workflow. Waiting for workflow to start..." +sleep 5 # Give GitHub a moment to create the workflow run + +# Get the most recent workflow run +echo "Fetching most recent workflow run..." +RUNS_RESPONSE=$(curl -s \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: token $GH_VACCINE_PAT" \ + -w "\n%{http_code}" \ + "https://api.github.com/repos/$REPO/actions/runs?event=workflow_dispatch&per_page=1" 2>&1) + +HTTP_STATUS=$(echo "$RUNS_RESPONSE" | tail -n1) +RESPONSE_BODY=$(echo "$RUNS_RESPONSE" | sed '$ d') + +if [ "$HTTP_STATUS" -ne 200 ]; then + echo "Error: Fetching runs failed with status $HTTP_STATUS" + echo "Response: $RESPONSE_BODY" + exit 1 +fi + +# Get the most recent run ID +WORKFLOW_ID=$(echo "$RESPONSE_BODY" | jq -r '.workflow_runs[0].id') + +if [ -z "$WORKFLOW_ID" ] || [ "$WORKFLOW_ID" = "null" ]; then + echo "Error: Could not find recent workflow run" + exit 1 +fi + +echo "Found workflow run ID: $WORKFLOW_ID" + +# Poll workflow status +while true; do + RUN_RESPONSE=$(curl -s \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: token $GH_VACCINE_PAT" \ + -w "\n%{http_code}" \ + "https://api.github.com/repos/$REPO/actions/runs/$WORKFLOW_ID" 2>&1) + + HTTP_STATUS=$(echo "$RUN_RESPONSE" | tail -n1) + RESPONSE_BODY=$(echo "$RUN_RESPONSE" | sed '$ d') + + if [ "$HTTP_STATUS" -ne 200 ]; then + echo "Error: Fetching run status failed with status $HTTP_STATUS" + echo "Response: $RESPONSE_BODY" + exit 1 + fi + + STATUS=$(echo "$RESPONSE_BODY" | jq -r .status) + CONCLUSION=$(echo "$RESPONSE_BODY" | jq -r .conclusion) + + if [ "$STATUS" = "completed" ]; then + if [ "$CONCLUSION" = "success" ]; then + echo "✅ Workflow completed successfully!" + exit 0 + else + echo "❌ Workflow failed with conclusion: $CONCLUSION" + echo "See details: https://github.com/$REPO/actions/runs/$WORKFLOW_ID" + exit 1 + fi + fi + + echo "Current status: $STATUS (Checking again in ${POLL_INTERVAL}s)" + sleep $POLL_INTERVAL +done