Merge pull request #181 from fivetran/release/v0.19.next

[Feature] Brand and organization variable configs

.buildkite/scripts/run_models.sh

@@ -20,7 +20,7 @@ dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
 dbt run --target "$db"
 dbt test --target "$db"
-dbt run --vars '{zendesk__unstructured_enabled: true, using_schedules: false, using_domain_names: false, using_user_tags: false, using_ticket_form_history: false, using_organization_tags: false}' --target "$db" --full-refresh
+dbt run --vars '{zendesk__unstructured_enabled: true, using_schedules: false, using_domain_names: false, using_user_tags: false, using_ticket_form_history: false, using_organization_tags: false, using_organizations: false, using}' --target "$db" --full-refresh
 dbt test --target "$db"
 dbt run --vars '{using_schedule_histories: true, using_holidays: false}' --target "$db" --full-refresh
 dbt test --target "$db"

.quickstart/quickstart.yml

@@ -16,10 +16,15 @@ table_variables:
     - ticket_form_history
   using_organization_tags:
     - organization_tag
+    - organization
   using_schedule_histories:
     - audit_log
   using_holidays:
     - schedule_holiday
+  using_brands:
+    - brand
+  using_organizations:
+    - organization
 
 destination_configurations:
   databricks:

CHANGELOG.md

@@ -1,3 +1,24 @@
+# dbt_zendesk v0.19.2
+[PR #181](https://github.com/fivetran/dbt_zendesk/pull/181) includes the following updates:
+
+## New Features
+- Introduced new config variables `using_brands` and `using_organizations` to allow  customers to enable and disable `brand` and `organization` source data, resulting in potential downstream impacts:
+- End models that could be impacted by these variables include: 
+  - `zendesk__ticket_backlog`: `using_brands` and/or `using_organizations` can be set to `false` to disable `brand` and/or `organization` fields.
+  - `zendesk__ticket_enriched`: `using_organizations` can be set to `false` to remove `organization` fields from the final model.
+- Intermediate models that could be impacted by these variables include:
+  - `int_zendesk__organization_aggregates`: `using_organizations` can be utilized to completely disable the model.
+  - `int_zendesk__ticket_aggregates`: `using_brands` can be set to `false` to remove `brand` fields from the final model.
+  - `int_zendesk__updater_information`: `using_organizations` can be set to `false` to remove `organization` fields from the final model.
+
+## Under the Hood
+- Updated `table_variables` in the `quickstart.yml` with the new `brand` and `organization` tables.
+- Updated our Buildkite model run script to ensure we test for when `using_brands` and `using_organizations` is set to either true or false. 
+
+## Documentation Updates
+- [Updated README](https://github.com/fivetran/dbt_zendesk?tab=readme-ov-file#step-4-enabledisable-models-for-non-existent-sources) with instructions on how to disable `brand` and `organization` sources. 
+
+
 # dbt_zendesk v0.19.1
 [PR #180](https://github.com/fivetran/dbt_zendesk/pull/180) includes the following update:
 

README.md

@@ -138,7 +138,7 @@ vars:
 ### Step 4: Enable/Disable models for non-existent sources
 
 > _This step is optional if you are unioning multiple connectors together in the previous step. The `union_data` macro will create empty staging models for sources that are not found in any of your Zendesk schemas/databases. However, you can still leverage the below variables if you would like to avoid this behavior._
-This package takes into consideration that not every Zendesk Support account utilizes the `schedule`, `schedule_holiday`, `ticket_schedule`, `daylight_time`, `time_zone`, `audit_log`, `domain_name`, `user_tag`, `organization_tag`, or `ticket_form_history` features, and allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true`, except for `using_schedule_histories`. Add variables for only the tables you want to enable/disable:
+This package takes into consideration that not every Zendesk Support account utilizes the `schedule`, `schedule_holiday`, `ticket_schedule`, `daylight_time`, `time_zone`, `audit_log`, `domain_name`, `user_tag`, `brand`,`organization`, `organization_tag`, or `ticket_form_history` features, and allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true`, except for `using_schedule_histories`. Add variables for only the tables you want to enable/disable:
 ```yml
 vars:
     using_schedule_histories:   True          #Enable if you are using audit_logs for schedule histories
@@ -147,6 +147,8 @@ vars:
     using_domain_names:         False         #Disable if you are not using domain names
     using_user_tags:            False         #Disable if you are not using user tags
     using_ticket_form_history:  False         #Disable if you are not using ticket form history
+    using_brands:               False         #Disable if you are not using brands
+    using_organizations:        False         #Disable if you are not using organizations. Setting this to False will also disable organization tags. 
     using_organization_tags:    False         #Disable if you are not using organization tags
 ```
 

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'zendesk'
-version: '0.19.1'
+version: '0.19.2'
 
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: zendesk_integration_tests_60
+      schema: zendesk_integration_tests_61
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: zendesk_integration_tests_60
+      schema: zendesk_integration_tests_61
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: zendesk_integration_tests_60
+      schema: zendesk_integration_tests_61
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: zendesk_integration_tests_60
+      schema: zendesk_integration_tests_61
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: zendesk_integration_tests_60
+      schema: zendesk_integration_tests_61
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,12 +1,12 @@
 config-version: 2
 
 name: 'zendesk_integration_tests'
-version: '0.19.1'
+version: '0.19.2'
 
 profile: 'integration_tests'
 
 vars:
-  zendesk_schema: zendesk_integration_tests_60
+  zendesk_schema: zendesk_integration_tests_61
 
   zendesk_source:
     zendesk_organization_identifier:           "organization_data"
@@ -32,16 +32,15 @@ vars:
   # zendesk__unstructured_enabled: True
   # using_schedule_histories: True
 
-  ## For validation testing. To be commented out before release.
-  # zendesk_schema: zendesk_test_env
+  # For validation testing. To be commented out before release.
   # using_domain_names: false
   # using_user_tags: false
   # using_organization_tags: false
   # using_holidays: false
   # fivetran_integrity_sla_metric_parity_exclusion_tickets: (56,80)
   # fivetran_integrity_sla_first_reply_time_exclusion_tickets: (56,80)
-  consistency_test_exclude_fields: ['source_relation', 'ticket_tags', 'ticket_day_id', 'assignee_ticket_last_update_at', 'assignee_last_login_at', 'requester_created_at', 'requester_updated_at', 'requester_ticket_last_update_at','requester_organization_created_at', 'requester_organization_updated_at', 'requester_last_login_at', 'created_at', 'updated_at'] 
-  # get rid of source_relation and ticket_day_id after v0.19.0
+  # fivetran_integrity_sla_count_match_tickets: (11109)
+  # consistency_test_exclude_fields: ['ticket_tags', 'assignee_ticket_last_update_at', 'assignee_last_login_at', 'requester_created_at', 'requester_updated_at', 'requester_ticket_last_update_at','requester_organization_created_at', 'requester_organization_updated_at', 'requester_last_login_at', 'created_at', 'updated_at'] 
 
 models:
   +schema: "zendesk_{{ var('directed_schema','dev') }}"

models/intermediate/int_zendesk__organization_aggregates.sql

@@ -1,8 +1,10 @@
+{{ config(enabled=var('using_organizations', True)) }}
+
 with organizations as (
     select * 
     from {{ ref('stg_zendesk__organization') }}
 
---If you use organization tags this will be included, if not it will be ignored.
+--If you use organization tags, this will be included, if not it will be ignored.
 {% if var('using_organization_tags', True) %}
 ), organization_tags as (
     select * 
@@ -19,6 +21,7 @@ with organizations as (
         on organizations.organization_id = organization_tags.organization_id 
         and organizations.source_relation = organization_tags.source_relation
 
+
     group by 1, 2
 {% endif %}
 

models/intermediate/int_zendesk__ticket_aggregates.sql

@@ -7,10 +7,13 @@ with tickets as (
   select *
   from {{ ref('stg_zendesk__ticket_tag') }}
 
+--If you use using_brands this will be included, if not it will be ignored.
+{% if var('using_brands', True) %}
 ), brands as (
 
   select *
   from {{ ref('stg_zendesk__brand') }}
+{% endif %}
 
 ), ticket_tag_aggregate as (
   select
@@ -27,17 +30,21 @@ with tickets as (
       then true
       else false
         end as is_incident,
+    {% if var('using_brands', True) %}
     brands.name as ticket_brand_name,
+    {% endif %}
     ticket_tag_aggregate.ticket_tags
   from tickets
 
   left join ticket_tag_aggregate
     on tickets.ticket_id = ticket_tag_aggregate.ticket_id 
     and tickets.source_relation = ticket_tag_aggregate.source_relation
 
+  {% if var('using_brands', True) %}
   left join brands
     on brands.brand_id = tickets.brand_id
     and brands.source_relation = tickets.source_relation
+  {% endif %}    
 )
 
 select *

models/ticket_history/int_zendesk__updater_information.sql

@@ -2,9 +2,12 @@ with users as (
     select *
     from {{ ref('int_zendesk__user_aggregates') }}
 
+--If using organizations, this will be included, if not it will be ignored.
+{% if var('using_organizations', True) %}
 ), organizations as (
     select *
     from {{ ref('int_zendesk__organization_aggregates') }}
+{% endif %}
 
 ), final as (
     select
@@ -24,22 +27,26 @@ with users as (
 
         ,users.last_login_at as updater_last_login_at
         ,users.time_zone as updater_time_zone
+        {% if var('using_organizations', True) %}
         ,organizations.organization_id as updater_organization_id
+        {% endif %}
 
         --If you use using_domain_names tags this will be included, if not it will be ignored.
-        {% if var('using_domain_names', True) %}
+        {% if var('using_domain_names', True) and var('using_organizations', True) %}
         ,organizations.domain_names as updater_organization_domain_names
         {% endif %}
 
-        --If you use organization tags this will be included, if not it will be ignored.
-        {% if var('using_organization_tags', True) %}
+        --If you use organization tags, this will be included, if not it will be ignored.
+        {% if var('using_organization_tags', True) and var('using_organizations', True) %}
         ,organizations.organization_tags as updater_organization_organization_tags
         {% endif %}
     from users
 
+    {% if var('using_organizations', True) %}
     left join organizations
         on users.source_relation = organizations.source_relation
         and users.organization_id = organizations.organization_id
+    {% endif %}
 )
 
 select * 

models/zendesk__ticket_backlog.sql

@@ -17,9 +17,11 @@ with ticket_field_history as (
     select *
     from {{ ref('stg_zendesk__user') }}
 
+{% if var('using_brands', True) %}
 ), brands as (
     select *
     from {{ ref('stg_zendesk__brand') }}
+{% endif %}
 
 --The below model is excluded if the user does not include ticket_form_id in the variable as a low percentage of accounts use ticket forms.
 {% if 'ticket_form_id' in var('ticket_field_history_columns') %}
@@ -28,9 +30,12 @@ with ticket_field_history as (
     from {{ ref('int_zendesk__latest_ticket_form') }}
 {% endif %}
 
+--If using organizations, this will be included, if not it will be ignored.
+{% if var('using_organizations', True) %}
 ), organizations as (
     select *
     from {{ ref('stg_zendesk__organization') }}
+{% endif %}
 
 ), backlog as (
     select
@@ -49,10 +54,10 @@ with ticket_field_history as (
             {% elif col in ['ticket_form_id'] %} --Standard ID field where the name can easily be joined from stg model.
                 ,ticket_forms.name as ticket_form_name
 
-            {% elif col in ['organization_id'] %} --Standard ID field where the name can easily be joined from stg model.
+            {% elif var('using_organizations', True) and col in ['organization_id'] %} --Standard ID field where the name can easily be joined from stg model.
                 ,organizations.name as organization_name
 
-            {% elif col in ['brand_id'] %} --Standard ID field where the name can easily be joined from stg model.
+            {% elif var('using_brands', True) and col in ['brand_id'] %} --Standard ID field where the name can easily be joined from stg model.
                 ,brands.name as brand_name
 
             {% elif col in ['group_id'] %} --Standard ID field where the name can easily be joined from stg model.
@@ -96,13 +101,13 @@ with ticket_field_history as (
         and requester.source_relation = ticket_field_history.source_relation
     {% endif %}
 
-    {% if 'brand_id' in var('ticket_field_history_columns') %} --Join not needed if field is not located in variable, otherwise it is included.
+    {% if var('using_brands', True) and 'brand_id' in var('ticket_field_history_columns') %} --Join not needed if field is not located in variable, otherwise it is included.
     left join brands
         on brands.brand_id = cast(ticket_field_history.brand_id as {{ dbt.type_bigint() }})
         and brands.source_relation = ticket_field_history.source_relation
     {% endif %}
 
-    {% if 'organization_id' in var('ticket_field_history_columns') %} --Join not needed if field is not located in variable, otherwise it is included.
+    {% if var('using_organizations', True) and 'organization_id' in var('ticket_field_history_columns') %} --Join not needed if field is not located in variable, otherwise it is included.
     left join organizations
         on organizations.organization_id = cast(ticket_field_history.organization_id as {{ dbt.type_bigint() }})
         and organizations.source_relation = ticket_field_history.source_relation

models/zendesk__ticket_enriched.sql

@@ -39,10 +39,13 @@ with ticket as (
     select *
     from {{ ref('stg_zendesk__group') }}
 
+--If using organizations, this will be included, if not it will be ignored.
+{% if var('using_organizations', True) %}
 ), organization as (
 
     select *
     from {{ ref('int_zendesk__organization_aggregates') }}
+{% endif %}
 
 ), joined as (
 
@@ -63,9 +66,9 @@ with ticket as (
         latest_satisfaction_ratings.is_good_to_bad_satisfaction_score,
         latest_satisfaction_ratings.is_bad_to_good_satisfaction_score,
 
-        --If you use using_domain_names tags this will be included, if not it will be ignored.
-        {% if var('using_domain_names', True) %}
-        organization.domain_names as ticket_organization_domain_names,
+        --If you use using_domain_names tags, this will be included, if not it will be ignored.
+        {% if var('using_domain_names', True) and var('using_organizations', True) %}
+        organization.domain_names as ticket_organization_domain_names, 
         requester_org.domain_names as requester_organization_domain_names,
         {% endif %}
 
@@ -82,15 +85,20 @@ with ticket as (
         requester_updates.last_updated as requester_ticket_last_update_at,
         requester.last_login_at as requester_last_login_at,
         requester.organization_id as requester_organization_id,
+        {% if var('using_organizations', True) %}
         requester_org.name as requester_organization_name,
+        {% endif %}
 
         --If you use organization tags this will be included, if not it will be ignored.
-        {% if var('using_organization_tags', True) %}
+        {% if var('using_organization_tags', True) and var('using_organizations', True) %}
         requester_org.organization_tags as requester_organization_tags,
         {% endif %}
+        --If you use organizations this will be included, if not it will be ignored.
+        {% if var('using_organizations', True) %}
         requester_org.external_id as requester_organization_external_id,
         requester_org.created_at as requester_organization_created_at,
         requester_org.updated_at as requester_organization_updated_at,
+        {% endif %}
         submitter.external_id as submitter_external_id,
         submitter.role as submitter_role,
         case when submitter.role in ('agent','admin') 
@@ -112,9 +120,11 @@ with ticket as (
         coalesce(assignee_updates.total_updates, 0) as assignee_ticket_update_count,
         assignee_updates.last_updated as assignee_ticket_last_update_at,
         assignee.last_login_at as assignee_last_login_at,
-        ticket_group.name as group_name,
-        organization.name as organization_name
-
+        ticket_group.name as group_name
+        --If you use organizations this will be included, if not it will be ignored.
+        {% if var('using_organizations', True) %}
+        ,organization.name as organization_name
+        {% endif %}
         --If you use using_user_tags this will be included, if not it will be ignored.
         {% if var('using_user_tags', True) %}
         ,requester.user_tags as requester_tag,
@@ -129,10 +139,12 @@ with ticket as (
     join users as requester
         on requester.user_id = ticket.requester_id
         and requester.source_relation = ticket.source_relation
-
+
+    {% if var('using_organizations', True) %}
     left join organization as requester_org
         on requester_org.organization_id = requester.organization_id
         and requester_org.source_relation = requester.source_relation
+    {% endif %}
 
     left join requester_updates
         on requester_updates.ticket_id = ticket.ticket_id
@@ -166,9 +178,11 @@ with ticket as (
         and latest_ticket_form.source_relation = ticket.source_relation
     {% endif %}
 
+    {% if var('using_organizations', True) %}
     left join organization
         on organization.organization_id = ticket.organization_id
         and organization.source_relation = ticket.source_relation
+    {% endif %}
 
     left join latest_satisfaction_ratings
         on latest_satisfaction_ratings.ticket_id = ticket.ticket_id