Add manual GHA CI/CD for BQ and SF

.github/workflows/cd.yml

@@ -0,0 +1,55 @@
+name: Run dbt Cloud Deploy to Prod
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  run_snowflake:
+    name: dbt Cloud Deploy Prod Snowflake
+    runs-on: macos-latest
+
+    env:
+      DBT_ACCOUNT_ID: 188483
+      DBT_PROJECT_ID: 283328
+      DBT_PR_JOB_ID: 409009
+      DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }}
+      DBT_JOB_CAUSE: "GitHub Actions Request"
+      DBT_JOB_BRANCH: main
+
+    steps:
+      - uses: "actions/checkout@v4"
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.12"
+      - name: Install uv
+        run: python3 -m pip install uv
+      - name: Install deps
+        run: uv pip install -r requirements.txt --system
+      - name: Run dbt Cloud job
+        run: python3 .github/workflows/scripts/dbt_cloud_run_job.py
+
+  run_bigquery:
+    name: dbt Cloud Deploy Prod BigQuery
+    runs-on: macos-latest
+
+    env:
+      DBT_ACCOUNT_ID: 188483
+      DBT_PROJECT_ID: 275557
+      DBT_PR_JOB_ID: 553247
+      DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }}
+      DBT_JOB_CAUSE: "GitHub Actions Request"
+      DBT_JOB_BRANCH: main
+
+    steps:
+      - uses: "actions/checkout@v4"
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.12"
+      - name: Install uv
+        run: python3 -m pip install uv
+      - name: Install deps
+        run: uv pip install -r requirements.txt --system
+      - name: Run dbt Cloud job
+        run: python3 .github/workflows/scripts/dbt_cloud_run_job.py

.github/workflows/ci.yml

@@ -0,0 +1,57 @@
+name: Run dbt Cloud CI job
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  run_snowflake:
+    name: dbt Cloud PR CI Snowflake
+    runs-on: macos-latest
+
+    env:
+      DBT_ACCOUNT_ID: 188483
+      DBT_PROJECT_ID: 283328
+      DBT_PR_JOB_ID: 552843
+      DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }}
+      DBT_JOB_CAUSE: "GitHub Actions Request"
+      DBT_JOB_BRANCH: ${{ github.head_ref }}
+      DBT_JOB_SCHEMA_OVERRIDE: dbt_jsdx__pr_${{ github.head_ref}}
+
+    steps:
+      - uses: "actions/checkout@v4"
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.12"
+      - name: Install uv
+        run: python3 -m pip install uv
+      - name: Install deps
+        run: uv pip install -r requirements.txt --system
+      - name: Run dbt Cloud job
+        run: python3 .github/workflows/scripts/dbt_cloud_run_job.py
+
+  run_bigquery:
+    name: dbt Cloud PR CI BigQuery
+    runs-on: macos-latest
+
+    env:
+      DBT_ACCOUNT_ID: 188483
+      DBT_PROJECT_ID: 275557
+      DBT_PR_JOB_ID: 561096
+      DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }}
+      DBT_JOB_CAUSE: "GitHub Actions Request"
+      DBT_JOB_BRANCH: ${{ github.head_ref }}
+      DBT_JOB_SCHEMA_OVERRIDE: dbt_jsdx__pr_${{ github.head_ref}}
+
+    steps:
+      - uses: "actions/checkout@v4"
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.12"
+      - name: Install uv
+        run: python3 -m pip install uv
+      - name: Install deps
+        run: uv pip install -r requirements.txt --system
+      - name: Run dbt Cloud job
+        run: python3 .github/workflows/scripts/dbt_cloud_run_job.py

.github/workflows/scripts/dbt_cloud_run_job.py

@@ -0,0 +1,134 @@
+import os
+import time
+import requests
+
+# ------------------------------------------------------------------------------
+# get environment variables
+# ------------------------------------------------------------------------------
+api_base = os.getenv(
+    "DBT_URL", "https://cloud.getdbt.com"
+)  # default to multitenant url
+job_cause = os.getenv(
+    "DBT_JOB_CAUSE", "API-triggered job"
+)  # default to generic message
+git_branch = os.getenv("DBT_JOB_BRANCH", None)  # default to None
+schema_override = os.getenv("DBT_JOB_SCHEMA_OVERRIDE", None)  # default to None
+api_key = os.environ[
+    "DBT_API_KEY"
+]  # no default here, just throw an error here if key not provided
+account_id = os.environ[
+    "DBT_ACCOUNT_ID"
+]  # no default here, just throw an error here if id not provided
+project_id = os.environ[
+    "DBT_PROJECT_ID"
+]  # no default here, just throw an error here if id not provided
+job_id = os.environ[
+    "DBT_PR_JOB_ID"
+]  # no default here, just throw an error here if id not provided
+
+print(f"""
+Configuration:
+api_base: {api_base}
+job_cause: {job_cause}
+git_branch: {git_branch}
+schema_override: {schema_override}
+account_id: {account_id}
+project_id: {project_id}
+job_id: {job_id}
+""")
+
+req_auth_header = {"Authorization": f"Token {api_key}"}
+req_job_url = f"{api_base}/api/v2/accounts/{account_id}/jobs/{job_id}/run/"
+run_status_map = {  # dbt run statuses are encoded as integers. This map provides a human-readable status
+    1: "Queued",
+    2: "Starting",
+    3: "Running",
+    10: "Success",
+    20: "Error",
+    30: "Cancelled",
+}
+
+type AuthHeader = dict[str, str]
+
+
+def run_job(
+    url: str,
+    headers: AuthHeader,
+    cause: str,
+    branch: str | None = None,
+    schema_override: str | None = None,
+) -> int:
+    """
+    Runs a dbt job
+    """
+
+    # build payload
+    req_payload = {"cause": cause}
+    if branch and not branch.startswith(
+        "$("
+    ):  # starts with '$(' indicates a valid branch name was not provided
+        req_payload["git_branch"] = branch.replace("refs/heads/", "")
+    if schema_override:
+        req_payload["schema_override"] = schema_override.replace("-", "_").replace(
+            "/", "_"
+        )
+
+    # trigger job
+    print(f"Triggering job:\n\turl: {url}\n\tpayload: {req_payload}")
+
+    response = requests.post(url, headers=headers, json=req_payload)
+    run_id: int = response.json()["data"]["id"]
+    return run_id
+
+
+def get_run_status(url: str, headers: AuthHeader) -> str:
+    """
+    gets the status of a running dbt job
+    """
+    # get status
+    response = requests.get(url, headers=headers)
+    run_status_code: int = response.json()["data"]["status"]
+    run_status = run_status_map[run_status_code]
+    return run_status
+
+
+def main():
+    print("Beginning request for job run...")
+
+    # run job
+    run_id: int = 0
+    try:
+        run_id = run_job(
+            req_job_url, req_auth_header, job_cause, git_branch, schema_override
+        )
+    except Exception as e:
+        print(f"ERROR! - Could not trigger job:\n {e}")
+        raise
+
+    # build status check url and run status link
+    req_status_url = f"{api_base}/api/v2/accounts/{account_id}/runs/{run_id}/"
+    run_status_link = (
+        f"{api_base}/deploy/{account_id}/projects/{project_id}/runs/{run_id}/"
+    )
+
+    # update user with status link
+    print(f"Job running! See job status at {run_status_link}")
+
+    # check status indefinitely with an initial wait period
+    time.sleep(30)
+    while True:
+        status = get_run_status(req_status_url, req_auth_header)
+        print(f"Run status -> {status}")
+
+        if status in ["Error", "Cancelled"]:
+            raise Exception(f"Run failed or canceled. See why at {run_status_link}")
+
+        if status == "Success":
+            print(f"Job completed successfully! See details at {run_status_link}")
+            return
+
+        time.sleep(10)
+
+
+if __name__ == "__main__":
+    main()

.gitignore

@@ -10,3 +10,4 @@ logs/
 .DS_Store
 
 .user.yml
+*.hurl

.pre-commit-config.yaml

@@ -11,6 +11,7 @@ repos:
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
+      - id: ruff-format
   - repo: https://github.com/sqlfluff/sqlfluff
     rev: "3.0.1"
     hooks:
@@ -26,7 +27,3 @@ repos:
             "dbt-metricflow[duckdb,snowflake,postgres]~=0.6.0",
             "sqlfluff-templater-dbt~=3.0.1",
           ]
-  - repo: https://github.com/psf/black
-    rev: "24.3.0"
-    hooks:
-      - id: black

requirements.in

@@ -1 +1,2 @@
 pre-commit~=3.6.0
+requests~=2.31.0

requirements.txt

@@ -1,21 +1,30 @@
 # This file was autogenerated by uv via the following command:
 #    uv pip compile requirements.in -o requirements.txt
+certifi==2024.2.2
+    # via requests
 cfgv==3.4.0
     # via pre-commit
+charset-normalizer==3.3.2
+    # via requests
 distlib==0.3.8
     # via virtualenv
 filelock==3.13.1
     # via virtualenv
 identify==2.5.35
     # via pre-commit
+idna==3.6
+    # via requests
 nodeenv==1.8.0
     # via pre-commit
 platformdirs==4.2.0
     # via virtualenv
 pre-commit==3.6.2
 pyyaml==6.0.1
     # via pre-commit
+requests==2.31.0
 setuptools==69.2.0
     # via nodeenv
+urllib3==2.2.1
+    # via requests
 virtualenv==20.25.1
     # via pre-commit