Merge branch 'MagicBot/dbt-utils-cross-db-migration' into feature/cre…

…dit-card-payments
fivetran · Oct 14, 2022 · 6c34057 · 6c34057
2 parents 53f7678 + f173c5e
commit 6c34057
Show file tree

Hide file tree

Showing 86 changed files with 1,068 additions and 1,023 deletions.
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+# Export secrets for Docker containers.
+# Restrict exposing secrets only to the steps that need them
+export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917")
+export CI_POSTGRES_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_HOST" --project="dbt-package-testing-363917")
+export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917")
+export CI_POSTGRES_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_PASS" --project="dbt-package-testing-363917")
+export CI_POSTGRES_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_DBNAME" --project="dbt-package-testing-363917")
+export CI_REDSHIFT_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_DBNAME" --project="dbt-package-testing-363917")
+export CI_REDSHIFT_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_HOST" --project="dbt-package-testing-363917")
+export CI_REDSHIFT_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_PASS" --project="dbt-package-testing-363917")
+export CI_REDSHIFT_DBT_USER=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_USER" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_ACCOUNT=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ACCOUNT" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_DATABASE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_DATABASE" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_PASS" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_ROLE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ROLE" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_USER" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -0,0 +1,73 @@
+steps:
+  - label: ":postgres: Run Tests - Postgres"
+    key: "run-dbt-postgres"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_POSTGRES_DBT_DBNAME"
+            - "CI_POSTGRES_DBT_HOST"
+            - "CI_POSTGRES_DBT_PASS"
+            - "CI_POSTGRES_DBT_USER"
+    commands: |
+      bash .buildkite/scripts/run_models.sh postgres 
+
+  - label: ":snowflake-db: Run Tests - Snowflake"
+    key: "run_dbt_snowflake"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_SNOWFLAKE_DBT_ACCOUNT"
+            - "CI_SNOWFLAKE_DBT_DATABASE"
+            - "CI_SNOWFLAKE_DBT_PASS"
+            - "CI_SNOWFLAKE_DBT_ROLE"
+            - "CI_SNOWFLAKE_DBT_USER"
+            - "CI_SNOWFLAKE_DBT_WAREHOUSE"
+    commands: |
+      bash .buildkite/scripts/run_models.sh snowflake
+
+  - label: ":gcloud: Run Tests - BigQuery"
+    key: "run_dbt_bigquery"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "GCLOUD_SERVICE_KEY"
+    commands: |
+      bash .buildkite/scripts/run_models.sh bigquery
+
+  - label: ":amazon-redshift: Run Tests - Redshift"
+    key: "run_dbt_redshift"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_REDSHIFT_DBT_DBNAME"
+            - "CI_REDSHIFT_DBT_HOST"
+            - "CI_REDSHIFT_DBT_PASS"
+            - "CI_REDSHIFT_DBT_USER"
+    commands: |
+      bash .buildkite/scripts/run_models.sh redshift
+
+  - label: ":upside_down_face: Run Tests - Databricks"
+    key: "run_dbt_databricks"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DATABRICKS_DBT_HOST"
+            - "CI_DATABRICKS_DBT_HTTP_PATH"
+            - "CI_DATABRICKS_DBT_TOKEN"
+    commands: |
+      bash .buildkite/scripts/run_models.sh databricks
diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+apt-get update
+apt-get install libsasl2-dev
+
+python3 -m venv venv
+. venv/bin/activate
+pip install --upgrade pip setuptools
+pip install -r integration_tests/requirements.txt
+mkdir -p ~/.dbt
+cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml
+
+db=$1
+echo `pwd`
+cd integration_tests
+dbt deps
+dbt seed --target "$db" --full-refresh
+dbt run --target "$db" --full-refresh
+dbt test --target "$db"
+## UPDATE FOR VARS HERE, IF NO VARS, PLEASE REMOVE
+dbt run --vars '{apple_search_ads__using_search_terms: True}' --target "$db" --full-refresh
+dbt test --target "$db"
+### END VARS CHUNK, REMOVE IF NOT USING
diff --git a/.circleci/config.yml b/.circleci/config.yml
diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,17 +1,13 @@
 config-version: 2
-
 name: 'quickbooks_source'
-version: '0.5.0'
-
-require-dbt-version: [">=1.0.0", "<2.0.0"]
-
+version: '0.6.0'
+require-dbt-version: [">=1.2.0", "<2.0.0"]
 models:
   quickbooks_source:
     +schema: quickbooks_staging
     materialized: table
     tmp:
       materialized: view
-
 vars:
   quickbooks_source:
     account: "{{ source('quickbooks','account') }}"
@@ -54,8 +50,6 @@ vars:
     vendor_credit_line: "{{ source('quickbooks','vendor_credit_line') }}"
     vendor_credit: "{{ source('quickbooks','vendor_credit') }}"
     vendor: "{{ source('quickbooks','vendor') }}"
-
-    #Variables if your company utilizes the below parent and child transactional tables.
     using_address: true
     using_bill: true
     using_credit_memo: true
@@ -71,7 +65,6 @@ vars:
     using_vendor_credit: true
     using_sales_receipt: true
     using_purchase_order: false
-
 clean-targets:
 - target
 - dbt_modules

diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml
@@ -1,13 +1,13 @@
 
-# HEY! This file is used in the QuickBooks integrations tests with CircleCI.
+# HEY! This file is used in the dbt package integrations tests with CircleCI.
 # You should __NEVER__ check credentials into version control. Thanks for reading :)
 
 config:
     send_anonymous_usage_stats: False
     use_colors: True
 
 integration_tests:
-  target: snowflake
+  target: redshift
   outputs:
     redshift:
       type: redshift
@@ -18,22 +18,13 @@ integration_tests:
       port: 5439
       schema: quickbooks_source_integrations_tests
       threads: 8
-    postgres:
-      type: postgres
-      host: "{{ env_var('CI_POSTGRES_DBT_HOST') }}"
-      user: "{{ env_var('CI_POSTGRES_DBT_USER') }}"
-      pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
-      dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
-      port: 5432
-      schema: quickbooks_source_integrations_tests
-      threads: 8
     bigquery:
       type: bigquery
-      method: service-account
-      keyfile: "{{ env_var('GCLOUD_SERVICE_KEY_PATH') }}"
+      method: service-account-json
       project: 'dbt-package-testing'
       schema: quickbooks_source_integrations_tests
       threads: 8
+      keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
       type: snowflake
       account: "{{ env_var('CI_SNOWFLAKE_DBT_ACCOUNT') }}"
@@ -42,5 +33,22 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
+      schema: <PLEASE INSERT INTEGRATION TESTS SCHEMA NAMES>
+      threads: 8
+    postgres:
+      type: postgres
+      host: "{{ env_var('CI_POSTGRES_DBT_HOST') }}"
+      user: "{{ env_var('CI_POSTGRES_DBT_USER') }}"
+      pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
+      dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
+      port: 5432
+      schema: quickbooks_source_integrations_tests
+      threads: 8
+    databricks:
+      catalog: null
+      host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
+      http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
       schema: quickbooks_source_integrations_tests
-      threads: 8
+      threads: 2
+      token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
+      type: databricks
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,14 +1,13 @@
 name: 'quickbooks_source_integration_tests'
-version: '0.5.1'
+version: '0.6.0'
+
 profile: 'integration_tests'
 config-version: 2
-
-models: 
+models:
   quickbooks_source:
     materialized: table
     tmp:
       materialized: view
-
 vars:
   quickbooks_schema: quickbooks_source_integrations_tests
   quickbooks_source:
@@ -56,9 +55,8 @@ vars:
     vendor:                     "{{ ref('vendor_data') }}"
 
 seeds:
-    +quote_columns: "{{ true if target.type == 'redshift' else false }}"
-    quickbooks_source_integration_tests:
-
+  +quote_columns: "{{ true if target.type == 'redshift' else false }}"
+  quickbooks_source_integration_tests:
 clean-targets:
-    - target
-    - dbt_modules
+- target
+- dbt_modules
diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt
@@ -1,6 +1,7 @@
-dbt-snowflake~=1.0.0
-dbt-bigquery~=1.0.0
-dbt-redshift~=1.0.0
-dbt-postgres~=1.0.0
-dbt-spark~=1.0.0
-dbt-spark[PyHive]~=1.0.0
+dbt-snowflake~=1.2.0
+dbt-bigquery~=1.2.0
+dbt-redshift~=1.2.0
+dbt-postgres~=1.2.0
+dbt-spark~=1.2.0
+dbt-spark[PyHive]~=1.2.0
+dbt-databricks~=1.2.0
diff --git a/macros/get_account_columns.sql b/macros/get_account_columns.sql
@@ -1,25 +1,25 @@
 {% macro get_account_columns() %}
 
 {% set columns = [
-    {"name": "_fivetran_synced", "datatype": dbt_utils.type_timestamp()},
-    {"name": "account_number", "datatype": dbt_utils.type_string()},
-    {"name": "account_sub_type", "datatype": dbt_utils.type_string()},
-    {"name": "account_type", "datatype": dbt_utils.type_string()},
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "account_number", "datatype": dbt.type_string()},
+    {"name": "account_sub_type", "datatype": dbt.type_string()},
+    {"name": "account_type", "datatype": dbt.type_string()},
     {"name": "active", "datatype": "boolean"},
-    {"name": "balance", "datatype": dbt_utils.type_float()},
-    {"name": "balance_with_sub_accounts", "datatype": dbt_utils.type_float()},
-    {"name": "classification", "datatype": dbt_utils.type_string()},
-    {"name": "created_at", "datatype": dbt_utils.type_timestamp()},
-    {"name": "currency_id", "datatype": dbt_utils.type_string()},
-    {"name": "description", "datatype": dbt_utils.type_string()},
-    {"name": "fully_qualified_name", "datatype": dbt_utils.type_string()},
-    {"name": "id", "datatype": dbt_utils.type_string()},
-    {"name": "name", "datatype": dbt_utils.type_string()},
-    {"name": "parent_account_id", "datatype": dbt_utils.type_string()},
+    {"name": "balance", "datatype": dbt.type_float()},
+    {"name": "balance_with_sub_accounts", "datatype": dbt.type_float()},
+    {"name": "classification", "datatype": dbt.type_string()},
+    {"name": "created_at", "datatype": dbt.type_timestamp()},
+    {"name": "currency_id", "datatype": dbt.type_string()},
+    {"name": "description", "datatype": dbt.type_string()},
+    {"name": "fully_qualified_name", "datatype": dbt.type_string()},
+    {"name": "id", "datatype": dbt.type_string()},
+    {"name": "name", "datatype": dbt.type_string()},
+    {"name": "parent_account_id", "datatype": dbt.type_string()},
     {"name": "sub_account", "datatype": "boolean"},
-    {"name": "sync_token", "datatype": dbt_utils.type_string()},
-    {"name": "tax_code_id", "datatype": dbt_utils.type_string()},
-    {"name": "updated_at", "datatype": dbt_utils.type_timestamp()}
+    {"name": "sync_token", "datatype": dbt.type_string()},
+    {"name": "tax_code_id", "datatype": dbt.type_string()},
+    {"name": "updated_at", "datatype": dbt.type_timestamp()}
 ] %}
 
 {{ return(columns) }}