GitHub - FastTrack Pipeline Validation Workflow #4
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: GitHub - Pipeline Validation Workflow | |
| on: | |
| workflow_dispatch: # Allows manual triggering | |
| pull_request: | |
| branches: | |
| - release # Triggers on PRs to release | |
| - 'feature/*' # Triggers on PRs to any feature branch | |
| schedule: | |
| - cron: "0 2 * * *" # Runs at 3 AM UTC | |
| ### Step 1: Define Repository Secrets ### | |
| ### Environment Secrets - Create Environment Secrets ### | |
| ### Navigate to Settings > Secrets & Variables > Actions | |
| # RGCLONE_API_ENDPOINT: Enter the RGClone API Endpoint Address | |
| # RGCLONE_ACCESS_TOKEN: Enter the Access Token for connecting to the above endpoint | |
| ### Step 2: Define REpository Variables ### | |
| ### Environment Secrets - Create Environment Secrets ### | |
| ### Navigate to Settings > Secrets & Variables > Actions | |
| # RGCLONE_IMAGE_NAME: Enter the name of the RGClone Image to be used | |
| # RGCLONE_CONTAINER_NAME: Enter the name of the container to be created | |
| # RGCLONE_PROXY_PORT: Enter the local port number to be used by the proxy command | |
| ### End of Environment Variables ### | |
| jobs: | |
| setup-validation-environment: | |
| if: github.repository_owner == 'RG-AutoPilot' # Only runs if the repo is in a specific organisation | |
| name: Spin Up RGClone Validation Container | |
| runs-on: "self-hosted" | |
| env: | |
| DATA_IMAGE_NAME: "${{ vars.RGCLONE_IMAGE_NAME }}" | |
| DATA_CONTAINER_NAME: "${{ vars.RGCLONE_CONTAINER_NAME }}-${{ github.run_number }}" | |
| PROXY_PORT: "${{ vars.RGCLONE_PROXY_PORT }}" | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| - name: Install rgclone CLI | |
| id: installCLI | |
| env: | |
| RGCLONE_API_ENDPOINT: ${{ secrets.RGCLONE_API_ENDPOINT }} | |
| RGCLONE_ACCESS_TOKEN: ${{ secrets.RGCLONE_ACCESS_TOKEN }} | |
| run: | | |
| # Check if rgclone is already installed | |
| if command -v rgclone &> /dev/null; then | |
| echo "rgclone is already installed. Skipping installation." | |
| rgclone version | |
| exit 0 | |
| fi | |
| echo "rgclone not found. Installing..." | |
| # Define install location | |
| INSTALL_DIR="/usr/local/bin" | |
| TEMP_DIR=$(mktemp -d) | |
| # Download and extract rgclone | |
| curl $RGCLONE_API_ENDPOINT/cloning-api/download/cli/linux-amd64 | tar xz -C "$TEMP_DIR" | |
| # Move rgclone to the install directory | |
| sudo mv "$TEMP_DIR/rgclone" "$INSTALL_DIR/" | |
| sudo chmod +x "$INSTALL_DIR/rgclone" | |
| # Verify installation | |
| echo "rgclone installed successfully." | |
| rgclone version | |
| - name: Validate Data Image | |
| id: createDataImage | |
| env: | |
| RGCLONE_API_ENDPOINT: ${{ secrets.RGCLONE_API_ENDPOINT }} | |
| RGCLONE_ACCESS_TOKEN: ${{ secrets.RGCLONE_ACCESS_TOKEN }} | |
| run: | | |
| output=$(rgclone get di "$DATA_IMAGE_NAME" --ignore-not-found -o json) | |
| if [[ -z "$output" ]]; then | |
| echo "Image does not exist. Creating image..." | |
| rgclone create data-image -f /home/flywayap/rgclone/yaml/AutoPilot_MSSQL-databases-from-script.yaml | |
| else | |
| echo "Image already exists. Moving on..." | |
| fi | |
| - name: Create data container | |
| id: createDc | |
| env: | |
| RGCLONE_API_ENDPOINT: ${{ secrets.RGCLONE_API_ENDPOINT }} | |
| RGCLONE_ACCESS_TOKEN: ${{ secrets.RGCLONE_ACCESS_TOKEN }} | |
| run: | | |
| # Spin up a Data Container for chosen Image | |
| echo "Check if Data Container Already Exists" | |
| if ! rgclone get data-container "$DATA_CONTAINER_NAME" &> /dev/null ; then | |
| echo "Creating container for $DATA_CONTAINER_NAME" | |
| output=$(rgclone create dc -n "$DATA_CONTAINER_NAME" -i "$DATA_IMAGE_NAME" -t 20m -o json) | |
| fi | |
| # Parse JSON output using jq | |
| dbUser=$(echo "$output" | jq -r '.user') | |
| dbPassword=$(echo "$output" | jq -r '.password') | |
| sqlhost=$(echo "$output" | jq -r '.host') | |
| sqlport=$(echo "$output" | jq -r '.port') | |
| instance="${sqlhost},${sqlport}" | |
| echo "Data container created successfully and available at: $instance" | |
| # Set output values for use in subsequent steps | |
| echo "dbUser=$dbUser" >> "$GITHUB_ENV" | |
| echo "dbPassword=$dbPassword" >> "$GITHUB_ENV" | |
| echo "instance=$instance" >> "$GITHUB_ENV" | |
| - name: Create Local Proxy for Container | |
| id: createProxy | |
| env: | |
| RGCLONE_API_ENDPOINT: ${{ secrets.RGCLONE_API_ENDPOINT }} | |
| RGCLONE_ACCESS_TOKEN: ${{ secrets.RGCLONE_ACCESS_TOKEN }} | |
| run: | | |
| echo "Checking if port $PROXY_PORT is in use..." | |
| # Check if any process is listening on the specified port | |
| if lsof -i :$PROXY_PORT &>/dev/null; then | |
| echo "Port $PROXY_PORT is in use. Killing the active process..." | |
| # Get the PID of the process using the port and kill it | |
| PID=$(lsof -ti :$PROXY_PORT) | |
| kill -9 $PID | |
| echo "Process $PID using port $PROXY_PORT has been terminated." | |
| else | |
| echo "Port $PROXY_PORT is free. No existing process found." | |
| fi | |
| # Start the new RGClone proxy | |
| echo "Starting new RGClone Proxy on port $PROXY_PORT" | |
| RUNNER_TRACKING_ID="" && rgclone proxy dc "$DATA_CONTAINER_NAME" -p "$PROXY_PORT" & # Run the proxy in the background | |
| Validate-Flyway-Pipeline: | |
| if: github.repository_owner == 'RG-AutoPilot' # Only runs if the repo is in a specific organisation | |
| name: Run Flyway Pipeline | |
| needs: setup-validation-environment | |
| uses: ./.github/workflows/GitHub-Flyway-CICD-Pipeline_Linux.yml | |
| secrets: inherit # Inherit existing secrets but allow environment overrides | |
| cleanup: | |
| name: Clean-up Container | |
| runs-on: "self-hosted" | |
| needs: Validate-Flyway-Pipeline # This ensures cleanup runs after the validation pipeline job | |
| if: always() && github.repository_owner == 'RG-AutoPilot' # Ensures this job runs even if Validate-Flyway-Pipeline fails | |
| steps: | |
| - name: Clean-up Containers | |
| run: | | |
| echo "Cleaning up containers" | |
| # Validate if container exists | |
| echo "Check if Data Container Already Exists" | |
| if rgclone get data-container "$DATA_CONTAINER_NAME" &> /dev/null ; then | |
| echo "Deleting container for $DATA_CONTAINER_NAME" | |
| rgclone delete dc -n "$DATA_CONTAINER_NAME" | |
| fi | |
| # This step will check if all jobs have complete and if any fail, the whole workflow will be rerun ONCE. This is required for reasons such as multiple PRs running at the same time. | |
| retry-workflow: | |
| name: Retry Entire Workflow If Any Job Fails | |
| runs-on: "self-hosted" | |
| needs: [setup-validation-environment, Validate-Flyway-Pipeline, cleanup] | |
| if: failure() && github.repository_owner == 'RG-AutoPilot' # Triggers only if any previous job failed | |
| steps: | |
| - name: Trigger Workflow Again | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.PAT }} | |
| run: | | |
| echo "Retrying workflow due to failure..." | |
| gh workflow run "${{ github.workflow }}" -f run_id="${{ github.run_id }}" |