Merge branch 'bugfix/add_schedule_holiday' into MagicBot/integation-t…

…est-webhooks-14

.buildkite/hooks/pre-command

@@ -21,4 +21,5 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C
 export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")

.buildkite/pipeline.yml

@@ -69,5 +69,6 @@ steps:
             - "CI_DATABRICKS_DBT_HOST"
             - "CI_DATABRICKS_DBT_HTTP_PATH"
             - "CI_DATABRICKS_DBT_TOKEN"
+            - "CI_DATABRICKS_DBT_CATALOG"
     commands: |
       bash .buildkite/scripts/run_models.sh databricks

CHANGELOG.md

@@ -1,9 +1,12 @@
-# dbt_zendesk_source v0.UPDATE.UPDATE
+# dbt_zendesk_source v0.9.0
 
- ## Under the Hood:
+# 🚨 New Schedule Holiday Table 🚨
+- Adding the `schedule_holiday` source table so that downstream models that involve business minutes calculations will accurately take holiday time into account. This staging model may be disabled by setting `using_schedules` to false. ([#92](https://github.com/fivetran/dbt_zendesk/issues/92))
 
+## Under the Hood:
 - Incorporated the new `fivetran_utils.drop_schemas_automation` macro into the end of each Buildkite integration test job.
 - Updated the pull request [templates](/.github).
+
 # dbt_zendesk_source v0.8.1
 ## Bug Fixes
 - Updated the dbt-utils dispatch within the `stg_zendesk__ticket_schedule_tmp` model to properly dispatch `dbt` as opposed to `dbt_utils` for the cross-db-macros. ([#32](https://github.com/fivetran/dbt_zendesk_source/pull/32))

README.md

@@ -44,7 +44,7 @@ Include the following zendesk_source package version in your `packages.yml` file
 ```yaml
 packages:
   - package: fivetran/zendesk_source
-    version: [">=0.8.0", "<0.9.0"]
+    version: [">=0.9.0", "<0.10.0"]
 ```
 ## Step 3: Define database and schema variables
 By default, this package runs using your target database and the `zendesk` schema. If this is not where your Zendesk data is (for example, if your zendesk schema is named `zendesk_fivetran`), add the following configuration to your root `dbt_project.yml` file:

dbt_project.yml

@@ -1,6 +1,6 @@
 config-version: 2
 name: 'zendesk_source'
-version: '0.8.1'
+version: '0.9.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
   zendesk_source:
@@ -16,6 +16,7 @@ vars:
     organization_tag: "{{ source('zendesk', 'organization_tag') }}"
     organization: "{{ source('zendesk', 'organization') }}"
     schedule: "{{ source('zendesk', 'schedule') }}"
+    schedule_holiday: "{{ source('zendesk', 'schedule_holiday') }}"
     satisfaction_rating: "{{ source('zendesk', 'satisfaction_rating') }}"
     ticket: "{{ source('zendesk', 'ticket') }}"
     ticket_form_history: "{{ source('zendesk', 'ticket_form_history') }}"

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: zendesk_source_integrations_tests_1
+      schema: zendesk_source_integrations_tests_2
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: zendesk_source_integrations_tests_1
+      schema: zendesk_source_integrations_tests_2
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: zendesk_source_integrations_tests_1
+      schema: zendesk_source_integrations_tests_2
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: zendesk_source_integrations_tests_1
+      schema: zendesk_source_integrations_tests_2
       threads: 8
     databricks:
-      catalog: null
+      catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: zendesk_source_integrations_tests_1
+      schema: zendesk_source_integrations_tests_2
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,15 +1,16 @@
 config-version: 2
 
 name: 'zendesk_source_integration_tests'
-version: '0.8.1'
+version: '0.9.0'
 
 profile: 'integration_tests'
 
 vars:
-  zendesk_schema: zendesk_source_integrations_tests_1
+  zendesk_schema: zendesk_source_integrations_tests_2
   zendesk_source:
     zendesk_organization_identifier:           "organization_data"
     zendesk_schedule_identifier:               "schedule_data"
+    zendesk_schedule_holiday_identifier:       "schedule_holiday_data"
     zendesk_ticket_identifier:                 "ticket_data"
     zendesk_ticket_comment_identifier:         "ticket_comment_data"
     zendesk_ticket_field_history_identifier:   "ticket_field_history_data"
@@ -54,6 +55,10 @@ seeds:
         end_time_utc: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
         start_time_utc: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
         created_at: timestamp
+    schedule_holiday_data:
+      +column_types:
+        id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        schedule_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
     ticket_comment_data:
       +column_types:
         id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"

integration_tests/seeds/schedule_holiday_data.csv

@@ -0,0 +1,3 @@
+id,schedule_id,_fivetran_deleted,_fivetran_synced,end_date,name,start_date
+11630286047764,11630270684308,FALSE,2023-01-19 21:53:06.281,2022-12-27,Test Holiday,2022-12-26
+11630270740884,11630270684308,FALSE,2023-01-19 21:53:06.281,2022-12-29,Test Holiday 2,2022-12-29

macros/get_schedule_holiday_columns.sql

@@ -0,0 +1,15 @@
+{% macro get_schedule_holiday_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "end_date", "datatype": dbt.type_string()},
+    {"name": "id", "datatype": dbt.type_int()},
+    {"name": "name", "datatype": dbt.type_string()},
+    {"name": "schedule_id", "datatype": dbt.type_int()},
+    {"name": "start_date", "datatype": dbt.type_string()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}

models/src_zendesk.yml

@@ -331,4 +331,22 @@ sources:
           - name: time_zone
             description: Name of the time zone. 
           - name: standard_offset 
-            description: Standard offset of the timezone (non-daylight savings hours). In `+/-hh:mm` format.
+            description: Standard offset of the timezone (non-daylight savings hours). In `+/-hh:mm` format.
+
+      - name: schedule_holiday
+        identifier: "{{ var('zendesk_schedule_holiday_identifier', 'schedule_holiday') }}"
+        description: Information about holidays for each specified schedule.
+        freshness: null
+        meta:
+          is_enabled: "{{ var('using_schedules', true) }}"
+        columns:
+          - name: end_date
+            description: ISO 8601 representation of the holiday end date.
+          - name: id
+            description: The ID of the scheduled holiday.
+          - name: name
+            description: Name of the holiday.
+          - name: schedule_id
+            description: The ID of the schedule.
+          - name: start_date
+            description: ISO 8601 representation of the holiday start date.

models/stg_zendesk.yml

@@ -293,4 +293,21 @@ models:
       - name: standard_offset 
         description: Standard offset of the timezone (non-daylight savings hours). In `+/-hh:mm` format.
       - name: standard_offset_minutes
-        description: Standard offset of the timezone (non-daylight savings hours) in minutes.
+        description: Standard offset of the timezone (non-daylight savings hours) in minutes.
+
+  - name: stg_zendesk__schedule_holiday
+    description: Information about holidays for each specified schedule.
+    columns:
+      - name: end_date_at
+        description: ISO 8601 representation of the holiday end date.
+      - name: holiday_id
+        description: The ID of the scheduled holiday.
+        tests:
+          - unique
+          - not_null
+      - name: holiday_name
+        description: Name of the holiday.
+      - name: schedule_id
+        description: The ID of the schedule.
+      - name: start_date_at
+        description: ISO 8601 representation of the holiday start date.

models/stg_zendesk__schedule_holiday.sql

@@ -0,0 +1,36 @@
+--To disable this model, set the using_schedules variable within your dbt_project.yml file to False.
+{{ config(enabled=var('using_schedules', True)) }}
+
+with base as (
+
+    select * 
+    from {{ ref('stg_zendesk__schedule_holiday_tmp') }}
+),
+
+fields as (
+
+    select
+        {{
+            fivetran_utils.fill_staging_columns(
+                source_columns=adapter.get_columns_in_relation(ref('stg_zendesk__schedule_holiday_tmp')),
+                staging_columns=get_schedule_holiday_columns()
+            )
+        }}
+    from base
+),
+
+final as (
+
+    select
+        _fivetran_deleted,
+        cast(_fivetran_synced as {{ dbt.type_timestamp() }} ) as _fivetran_synced,
+        cast(end_date as {{ dbt.type_timestamp() }} ) as holiday_end_date_at,
+        cast(id as {{ dbt.type_string() }} ) as holiday_id,
+        name as holiday_name,
+        cast(schedule_id as {{ dbt.type_string() }} ) as schedule_id,
+        cast(start_date as {{ dbt.type_timestamp() }} ) as holiday_start_date_at
+    from fields
+)
+
+select *
+from final

models/tmp/stg_zendesk__schedule_holiday_tmp.sql

@@ -0,0 +1,5 @@
+--To disable this model, set the using_schedules variable within your dbt_project.yml file to False.
+{{ config(enabled=var('using_schedules', True)) }}
+
+select * 
+from {{ var('schedule_holiday') }}