snowplow · bill-warner · Dec 17, 2021 · Nov 30, 2021 · Dec 14, 2021 · Dec 14, 2021
@@ -1,9 +1,10 @@
-name: pr_to_main 
+name: pr_tests 
 
 on:
   pull_request:
     branches:
       - main
+      - 'release/**'
 
 env:
   # Set profiles.yml directory
@@ -38,21 +39,18 @@ env:
   POSTGRES_TEST_DBNAME: ${{ secrets.POSTGRES_TEST_DBNAME }} 
 
 jobs:
-  pr_to_main:
-    name: pr_to_main
+  pr_tests:
+    name: pr_tests
     runs-on: ubuntu-latest
     defaults:
       run:
         # Run tests from integration_tests sub dir
         working-directory: ./integration_tests
     strategy:
       matrix:
-        dbt_version: ["0.18.*", "0.19.*"]
+        dbt_version: ["0.20.*", "0.21.*", "1.0.*"]
         warehouse: ["postgres", "bigquery", "snowflake"] # TODO: Add RS self-hosted runner
-        exclude:
-          # Issue w. tmp table name lengths (https://github.com/dbt-labs/dbt-core/pull/2850/files)
-          - warehouse: postgres 
-            dbt_version: "0.18.*"
+
     services:
       postgres:
         image: postgres:latest
@@ -81,6 +79,10 @@ jobs:
         env:
           DBT_VERSION: ${{ matrix.dbt_version }}
 
+      - name: Set DEFAULT_TARGET env
+        run: |
+         echo "DEFAULT_TARGET=${{ matrix.warehouse }}" >> $GITHUB_ENV
+
       - name: Write BigQuery creds to json file
         run: |
          echo "$BIGQUERY_SERVICE_KEYFILE" > ./dbt-service-account.json
@@ -94,14 +96,15 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ matrix.dbt_version }}
+          key: ${{ runner.os }}-pip-${{ matrix.dbt_version }}-${{ matrix.warehouse }}
           restore-keys: |
-            ${{ runner.os }}-pip-${{ matrix.dbt_version }}
+            ${{ runner.os }}-pip-${{ matrix.dbt_version }}-${{ matrix.warehouse }}
 
       # Install latest patch version. Upgrade if cache contains old patch version.
       - name: Install dependencies
         run: |
-          pip install -Iv dbt==${{ matrix.dbt_version }} --upgrade
+          pip install --upgrade pip wheel setuptools
+          pip install -Iv dbt-${{ matrix.warehouse }}==${{ matrix.dbt_version }} --upgrade
           dbt deps
 
       - name: Block concurrent executions tests
@@ -117,8 +120,7 @@ jobs:
           dbt run-operation post_ci_cleanup --target ${{ matrix.warehouse }}
 
       - name: Run tests
-        working-directory: ./integration_tests/.scripts
-        run: ./integration_test.sh -d ${{ matrix.warehouse }}
+        run: ./.scripts/integration_test.sh -d ${{ matrix.warehouse }}
 
       # post_ci_cleanup sits in utils package 
       - name: "Post-test: Drop ci schemas"

@@ -1,4 +1,5 @@
 
 target/
 dbt_modules/
+dbt_packages/
 logs/
@@ -1,3 +1,8 @@
+snowplow-web 0.5.0 (2021-12-17)
+---------------------------------------
+Regenerate doc site (Close #52)
+Add dbt v1 compatibility (Close #38)
+
 snowplow-web 0.4.1 (2021-12-06)
 ---------------------------------------
 Update docs (Close #51)

@@ -15,18 +15,18 @@ Please refer to the [doc site][snowplow-web-docs] for a full breakdown of the pa
 
 ### Adapter Support
 
-The snowplow-web v0.4.1 package currently supports BigQuery, Redshift, Snowflake & Postgres.
+The snowplow-web v0.5.0 package currently supports BigQuery, Redshift, Snowflake & Postgres.
 
-|            Warehouse           |     dbt versions    | snowplow-web version |
-|:------------------------------:|:-------------------:|:--------------------:|
-| BigQuery, Redshift & Snowflake | >=0.18.0 to <0.20.0 |         0.4.1        |
-|            Postgres            | >=0.19.0 to <0.20.0 |         0.4.1        |
+|                 Warehouse                |     dbt versions    | snowplow-web version |
+|:----------------------------------------:|:-------------------:|:--------------------:|
+| BigQuery, Redshift, Snowflake & Postgres | >=0.20.0 to <1.1.0  |         0.5.0        |
+|      BigQuery, Redshift & Snowflake      | >=0.18.0 to <0.20.0 |         0.4.1        |
+|                 Postgres                 | >=0.19.0 to <0.20.0 |         0.4.1        |
 
 ### Requirements
 
 - A dataset of web events from the [Snowplow JavaScript tracker][tracker-docs] must be available in the database.
 - Have the [`webPage` context][webpage-context] enabled.
-- Running dbt versions >=0.18.0 to <0.20.0. dbt v.0.20.0+ support to follow soon.
 
 ### Installation
 

@@ -1,5 +1,5 @@
 name: 'snowplow_custom_example'
-version: '0.4.1'
+version: '0.5.0'
 config-version: 2
 
 

@@ -1,8 +1,8 @@
 name: 'snowplow_web'
-version: '0.4.1'
+version: '0.5.0'
 config-version: 2
 
-require-dbt-version: [">=0.18.0", "<0.20.0"]
+require-dbt-version: [">=0.20.0", "<1.1.0"]
 
 profile: 'default'
 
@@ -16,6 +16,7 @@ target-path: "target"
 clean-targets:
     - "target"
     - "dbt_modules"
+    - "dbt_packages"
 
 vars:
   # Sources
@@ -33,9 +34,12 @@ vars:
   snowplow__heartbeat: 10
   snowplow__backfill_limit_days: 30
   snowplow__app_id: []
-  snowplow__enable_iab: false
-  snowplow__enable_ua: false
-  snowplow__enable_yauaa: false
+  # Setting default values for 3 enrichments below throughout package rather than here
+  # Workaround for a bug (https://github.com/dbt-labs/dbt-core/issues/3698). 
+  # TODO: Return to having default values here once fixed.
+  # snowplow__enable_iab: false
+  # snowplow__enable_ua: false
+  # snowplow__enable_yauaa: false
   snowplow__ua_bot_filter: true
   snowplow__derived_tstamp_partitioned: true
   snowplow__session_stitching: true

@@ -24,7 +24,7 @@ Each module produces a table which acts as the input to the subsequent module (t
 
 ## Adapter Support
 
-The Snowplow Web v0.4.1 package currently supports BigQuery, Redshift, Snowflake & Postgres.
+The Snowplow Web v0.5.0 package currently supports BigQuery, Redshift, Snowflake & Postgres.
 
 ## Installation
 
@@ -559,24 +559,7 @@ All the incremental models in the Snowplow web package have recommended cluster
 
 ### Overriding Macros
 
-Both the cluster key macros (see above) and the `allow_refresh()` macro can be overridden. These are both [dispatched macros](https://docs.getdbt.com/reference/dbt-jinja-functions/dispatch) and can be overridden by creating your own version of the macro with a `default__`, within your projects own macro dir. For example to override `cluster_by_fields_page_views`:
-
-```SQL
-# macros/your_version_of_cluster_by_fields_page_views.sql
-{% default__cluster_by_fields_page_views %}
-  {{ return(["<your_cluster_field>"]) }}
-{% endmacro %}
-```
-
-In addition you will need to add the following to your `project.yml` file:
-
-```yml
-vars:
-  snowplow_web_dispatch_list:
-    - <your_project_name> 
-```
-
-For more details on how this works please [read this](https://github.com/dbt-labs/dbt-utils/tree/0.6.6#dispatch-macros). Note this method of overriding dispatch macros is specific to dbt v0.18-0.19. The methodology has changed in later versions of dbt, so the documentation on dbt's doc site is not applicable here.
+Both the cluster key macros (see above) and the `allow_refresh()` macro can be overridden. These are both [dispatched macros](https://docs.getdbt.com/reference/dbt-jinja-functions/dispatch) and can be overridden by creating your own version of the macro and setting a project level dispatch config. More details can be found in [dbt's docs](https://docs.getdbt.com/reference/dbt-jinja-functions/dispatch#overriding-package-macros)
 
 ## Duplicates
 

@@ -18,3 +18,5 @@ Supported warehouses:
 - redshift
 - bigquery
 - snowflake
+- postgres
+- all (iterates through all supported warehouses)
@@ -7,7 +7,7 @@ config:
     use_colors: True
 
 integration_tests:
-  target: bigquery
+  target: "{{ env_var('DEFAULT_TARGET') }}"
   outputs:
     postgres:
       type: postgres

@@ -1,5 +1,5 @@
 name: 'snowplow_web_integration_tests'
-version: '0.4.1'
+version: '0.5.0'
 config-version: 2
 
 profile: 'integration_tests'
@@ -14,6 +14,7 @@ target-path: "target"
 clean-targets:
     - "target"
     - "dbt_modules"
+    - "dbt_packages"
 
 quoting:
   identifier: false

@@ -1,7 +1,7 @@
 {# Default: Allow refresh in dev, block refresh otherwise. dev defined by snowplow__dev_target_name #}
 
 {% macro allow_refresh() %}
-  {{ return(adapter.dispatch('allow_refresh', snowplow_web._get_namespaces())()) }}
+  {{ return(adapter.dispatch('allow_refresh', 'snowplow_web')()) }}
 {% endmacro %}
 
 {% macro default__allow_refresh() %}

@@ -1,6 +1,6 @@
 {% macro cluster_by_fields_sessions_lifecycle() %}
 
-  {{ return(adapter.dispatch('cluster_by_fields_sessions_lifecycle', snowplow_web._get_namespaces())()) }}
+  {{ return(adapter.dispatch('cluster_by_fields_sessions_lifecycle', 'snowplow_web')()) }}
 
 {% endmacro %}
 
@@ -13,7 +13,7 @@
 
 {% macro cluster_by_fields_page_views() %}
 
-  {{ return(adapter.dispatch('cluster_by_fields_page_views', snowplow_web._get_namespaces())()) }}
+  {{ return(adapter.dispatch('cluster_by_fields_page_views', 'snowplow_web')()) }}
 
 {% endmacro %}
 
@@ -26,7 +26,7 @@
 
 {% macro cluster_by_fields_sessions() %}
 
-  {{ return(adapter.dispatch('cluster_by_fields_sessions', snowplow_web._get_namespaces())()) }}
+  {{ return(adapter.dispatch('cluster_by_fields_sessions', 'snowplow_web')()) }}
 
 {% endmacro %}
 
@@ -39,7 +39,7 @@
 
 {% macro cluster_by_fields_users() %}
 
-  {{ return(adapter.dispatch('cluster_by_fields_users', snowplow_web._get_namespaces())()) }}
+  {{ return(adapter.dispatch('cluster_by_fields_users', 'snowplow_web')()) }}
 
 {% endmacro %}
 

@@ -87,23 +87,23 @@ select
 
   -- iab enrichment fields: set iab variable to true to enable
   {{ snowplow_utils.get_optional_fields(
-        enabled=var('snowplow__enable_iab'),
+        enabled=var('snowplow__enable_iab', false),
         fields=iab_fields(),
         col_prefix='contexts_com_iab_snowplow_spiders_and_robots_1',
         relation=ref('snowplow_web_base_events_this_run'),
         relation_alias='ev') }},
 
   -- ua parser enrichment fields: set ua_parser variable to true to enable
   {{ snowplow_utils.get_optional_fields(
-        enabled=var('snowplow__enable_ua'),
+        enabled=var('snowplow__enable_ua', false),
         fields=ua_fields(),
         col_prefix='contexts_com_snowplowanalytics_snowplow_ua_parser_context_1',
         relation=ref('snowplow_web_base_events_this_run'),
         relation_alias='ev') }},
 
   -- yauaa enrichment fields: set yauaa variable to true to enable
   {{ snowplow_utils.get_optional_fields(
-        enabled=var('snowplow__enable_yauaa'),
+        enabled=var('snowplow__enable_yauaa', false),
         fields=yauaa_fields(),
         col_prefix='contexts_nl_basjes_yauaa_context_1',
         relation=ref('snowplow_web_base_events_this_run'),

@@ -96,7 +96,7 @@ select
   -- optional fields, only populated if enabled.
 
   -- iab enrichment fields: set iab variable to true to enable
-  {% if var('snowplow__enable_iab') %}
+  {% if var('snowplow__enable_iab', false) %}
     iab.category,
     iab.primary_impact,
     iab.reason,
@@ -109,7 +109,7 @@ select
   {% endif %}
 
   -- ua parser enrichment fields: set ua_parser variable to true to enable
-  {% if var('snowplow__enable_ua') %}
+  {% if var('snowplow__enable_ua', false) %}
     ua.useragent_family,
     ua.useragent_major,
     ua.useragent_minor,
@@ -138,7 +138,7 @@ select
   {% endif %}
 
   -- yauaa enrichment fields: set yauaa variable to true to enable
-  {% if var('snowplow__enable_yauaa') %}
+  {% if var('snowplow__enable_yauaa', false) %}
     ya.device_class,
     ya.agent_class,
     ya.agent_name,
@@ -190,21 +190,21 @@ on ev.page_view_id = t.page_view_id
 left join {{ ref('snowplow_web_pv_scroll_depth') }} sd
 on ev.page_view_id = sd.page_view_id
 
-{% if var('snowplow__enable_iab') -%}
+{% if var('snowplow__enable_iab', false) -%}
 
   left join {{ ref('snowplow_web_pv_iab') }} iab
   on ev.page_view_id = iab.page_view_id
 
 {% endif -%}
 
-{% if var('snowplow__enable_ua') -%}
+{% if var('snowplow__enable_ua', false) -%}
 
   left join {{ ref('snowplow_web_pv_ua_parser') }} ua
   on ev.page_view_id = ua.page_view_id
 
 {% endif -%}
 
-{% if var('snowplow__enable_yauaa') -%}
+{% if var('snowplow__enable_yauaa', false) -%}
 
   left join {{ ref('snowplow_web_pv_yauaa') }} ya
   on ev.page_view_id = ya.page_view_id

@@ -2,7 +2,7 @@
   config(
     sort='page_view_id',
     dist='page_view_id',
-    enabled=(var('snowplow__enable_iab') and target.type in ['redshift', 'postgres'] | as_bool())
+    enabled=(var('snowplow__enable_iab', false) and target.type in ['redshift', 'postgres'] | as_bool())
   ) 
 }}
 

@@ -2,7 +2,7 @@
   config(
     sort='page_view_id',
     dist='page_view_id',
-    enabled=(var('snowplow__enable_ua') and target.type in ['redshift', 'postgres'] | as_bool())
+    enabled=(var('snowplow__enable_ua', false) and target.type in ['redshift', 'postgres'] | as_bool())
   ) 
 }}