Merge pull request #48 from fivetran/feature/revamp/daily-shop-model

updates to discounts and adde dthe daily shop model

.buildkite/scripts/run_models.sh

@@ -19,5 +19,5 @@ dbt deps
 dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
-dbt run --vars '{shopify_timezone: "America/New_York", shopify_using_all_metafields: true}' --target "$db" --full-refresh
+dbt run --vars '{shopify_timezone: "America/New_York", shopify_using_fulfillment_event: true, shopify_using_all_metafields: true}' --target "$db" --full-refresh
 dbt test --target "$db"

DECISIONLOG.md

@@ -8,6 +8,10 @@ In validating metrics with the Sales over Time reports in the Shopify UI, you ma
 
 We felt that reporting on the order date made more sense in reality, but, if you feel differently, please reach out and create a Feature Request. To align with the Shopify method yourself, this would most likely involve aggregating `transactions` data (relying on the `kind` column to determine sales vs returns) instead of `orders`.
 
+## Using an Order's `created_timestamp` Instead of `processed_timestamp` 
+
+In a similar vein to the above, in the customer cohort and daily shop models, we aggregate orders on a daily grain. To do so, we truncate the timestamp at which the order was _created_. In contrast, Shopify in-app reports truncate the timestamp at which the order was _processed_. This may also contribute to discrepancies when comparing the package models to in-app reports. We felt that the creation timestamp makes more sense to use in reality, but please reach out if you have other thoughts by opening an [issue](https://github.com/fivetran/dbt_shopify/issues/new?assignees=&labels=enhancement&template=feature-request.yml&title=%5BFeature%5D+%3Ctitle%3E).
+
 ## Creating Empty Tables for Refunds, Order Line Refunds, Order Adjustments, and Discount Codes
 
 Source tables related to `refunds`, `order_line_refunds`, `order_adjustments`, and `discount_codes` are created in the Shopify schema dyanmically. For example, if your shop has not incurred any refunds, you will not have a `refund` table yet until you do refund an order. 

README.md

@@ -21,12 +21,13 @@ The following table provides a detailed list of all models materialized within t
 
 | **model**                 | **description**                                                                                                    |
 | ------------------------- | ------------------------------------------------------------------------------------------------------------------ |
-| [shopify__customer_cohorts](https://github.com/fivetran/dbt_shopify/blob/main/models/shopify__customer_cohorts.sql)  | Each record represents the monthly performance of a customer, including fields for the month of their 'cohort'.    |
-| [shopify__customers](https://github.com/fivetran/dbt_shopify/blob/main/models/shopify__customers.sql)        | Each record represents a customer, with additional dimensions like lifetime value and number of orders.            |
-| [shopify__orders](https://github.com/fivetran/dbt_shopify/blob/main/models/shopify__orders.sql)           | Each record represents an order, with additional dimensions like whether it is a new or repeat purchase.           |
-| [shopify__order_lines](https://github.com/fivetran/dbt_shopify/blob/main/models/shopify__order_lines.sql)     | Each record represents an order line item, with additional dimensions like how many items were refunded.           |
-| [shopify__products](https://github.com/fivetran/dbt_shopify/blob/main/models/shopify__products.sql)         | Each record represents a product, with additional dimensions like most recent order date and order volume.         |
-| [shopify__transactions](https://github.com/fivetran/dbt_shopify/blob/main/models/shopify__transactions.sql)     | Each record represents a transaction with additional calculations to handle exchange rates.                        |
+| [shopify__customer_cohorts](https://fivetran.github.io/dbt_shopify/#!/model/model.shopify.shopify__customer_cohorts.sql)  | Each record represents the monthly performance of a customer, including fields for the month of their 'cohort'.    |
+| [shopify__customers](https://fivetran.github.io/dbt_shopify/#!/model/model.shopify.shopify__customers.sql)        | Each record represents a customer, with additional dimensions like lifetime value and number of orders.            |
+| [shopify__orders](https://fivetran.github.io/dbt_shopify/#!/model/model.shopify.shopify__orders.sql)           | Each record represents an order, with additional dimensions like whether it is a new or repeat purchase.           |
+| [shopify__order_lines](https://fivetran.github.io/dbt_shopify/#!/model/model.shopify.shopify__order_lines.sql)     | Each record represents an order line item, with additional dimensions like how many items were refunded.           |
+| [shopify__products](https://fivetran.github.io/dbt_shopify/#!/model/model.shopify.shopify__products.sql)         | Each record represents a product, with additional dimensions like most recent order date and order volume.         |
+| [shopify__transactions](https://fivetran.github.io/dbt_shopify/#!/model/model.shopify.shopify__transactions)     | Each record represents a transaction with additional calculations to handle exchange rates.                        |
+| [shopify__daily_shop](https://fivetran.github.io/dbt_shopify/#!/model/model.shopify.shopify__daily_shop.sql)     | Each record represents a day of activity for each of your shops, conveyed by a suite of daily metrics.                        |
 
 # 🎯 How do I use the dbt package?
 

dbt_project.yml

@@ -45,4 +45,5 @@ vars:
     shopify_tender_transaction: "{{ ref('stg_shopify__tender_transaction') }}"
     shopify_abandoned_checkout_discount_code: "{{ ref('stg_shopify__abandoned_checkout_discount_code') }}"
     shopify_order_discount_code: "{{ ref('stg_shopify__order_discount_code') }}"
-    shopify_abandoned_checkout_shipping_line: "{{ ref('stg_shopify__abandoned_checkout_shipping_line') }}"
+    shopify_abandoned_checkout_shipping_line: "{{ ref('stg_shopify__abandoned_checkout_shipping_line') }}"
+    shopify_fulfillment_event: "{{ ref('stg_shopify__fulfillment_event') }}"

integration_tests/dbt_project.yml

@@ -22,6 +22,7 @@ vars:
     shopify_customer_tag_identifier: "shopify_customer_tag_data"
     shopify_discount_code_identifier: "shopify_discount_code_data"
     shopify_fulfillment_identifier: "shopify_fulfillment_data"
+    shopify_fulfillment_event_identifier: "shopify_fulfillment_event_data"
     shopify_inventory_item_identifier: "shopify_inventory_item_data"
     shopify_inventory_level_identifier: "shopify_inventory_level_data"
     shopify_location_identifier: "shopify_location_data"

integration_tests/seeds/shopify_fulfillment_event_data.csv

@@ -0,0 +1,6 @@
+id,_fivetran_synced,address_1,city,country,created_at,estimated_delivery_at,fulfillment_id,happened_at,latitude,longitude,message,order_id,province,shop_id,status,updated_at,zip,_fivetran_deleted
+451435,2022-11-18 04:39:07.945000,,,,2022-08-29 20:52:39.000000,,40495,2022-08-29 20:52:39.000000,,,,4502987,,89440612,delivered,2022-08-29 20:52:39.000000,,false
+48779,2022-11-18 05:48:01.773000,,LONDON,GB,2022-09-13 08:07:57.000000,,4064737,2022-08-15 12:41:00.000000,101.349998474121094,-14.0333000011742115,Delay,4588203,,320612,out_for_delivery,2022-09-13 08:07:57.000000,CR0,false
+1481515,2022-11-18 05:41:00.745000,,ECHO PARK,AU,2022-09-14 14:16:52.000000,2022-09-14 08:00:00.000000,4019339,2022-09-14 01:26:00.000000,-3.797698974609375,190.783958203125,Delay,451915,,89320612,out_for_delivery,2022-09-14 14:16:52.000000,2759,false
+558955,2022-11-18 10:51:24.286000,,LAZYTOWN,US,2022-08-13 12:40:26.000000,,402947,2022-03-01 10:36:39.000000,22.337699890136719,-71.731002807617188,Delay,429188587,MA,89420612,in_transit,2022-08-13 12:40:26.000000,01505,false
+6904235,2022-11-18 08:58:00.458000,,LA,US,2022-08-24 06:29:21.000000,2022-08-24 23:59:59.000000,4060491,2022-08-24 05:30:57.000000,12.287498474121094,-21.3573989868164,Delay,4242667,MA,89420612,in_transit,2022-08-24 06:29:21.000000,01760,false

models/intermediate/int_shopify__daily_abandoned_checkouts.sql

@@ -0,0 +1,25 @@
+with abandoned_checkout as (
+
+    select *
+    from {{ var('shopify_abandoned_checkout') }}
+
+    -- "deleted" abandoned checkouts do not appear to have any data tying them to customers,
+    -- discounts, or products (and should therefore not get joined in) but let's filter them out here
+    where not coalesce(is_deleted, false)
+),
+
+abandoned_checkout_aggregates as (
+
+    select
+        source_relation,
+        cast({{ dbt.date_trunc('day','created_at') }} as date) as date_day,
+        count(distinct checkout_id) as count_abandoned_checkouts,
+        count(distinct customer_id) as count_customers_abandoned_checkout,
+        count(distinct email) as count_customer_emails_abandoned_checkout
+
+    from abandoned_checkout
+    group by 1,2
+)
+
+select * 
+from abandoned_checkout_aggregates

models/intermediate/int_shopify__daily_fulfillment.sql

@@ -0,0 +1,25 @@
+{{ config(enabled=var('shopify_using_fulfillment_event', false)) }}
+
+with fulfillment_event as (
+
+    select *
+    from {{ var('shopify_fulfillment_event') }}
+),
+
+fulfillment_aggregates as (
+
+    select 
+        source_relation,
+        cast({{ dbt.date_trunc('day','happened_at') }} as date) as date_day
+
+        {% for status in ['attempted_delivery', 'delivered', 'failure', 'in_transit', 'out_for_delivery', 'ready_for_pickup', 'label_printed', 'label_purchased', 'confirmed']%}
+        , sum(case when lower(status) = '{{ status }}' then 1 else 0 end) as count_fulfillment_{{ status }}
+        {% endfor %}
+
+    from fulfillment_event
+    group by 1,2
+
+)
+
+select *
+from fulfillment_aggregates

models/intermediate/int_shopify__daily_orders.sql

@@ -0,0 +1,96 @@
+with orders as (
+
+    select *
+    from {{ ref('shopify__orders') }}
+
+    where not coalesce(is_deleted, false)
+),
+
+order_lines as(
+
+    select *
+    from {{ ref('shopify__order_lines') }}
+),
+
+order_aggregates as (
+
+    select
+        source_relation,
+        cast({{ dbt.date_trunc('day','created_timestamp') }} as date) as date_day,
+        count(distinct order_id) as count_orders,
+        sum(line_item_count) as count_line_items,
+        avg(line_item_count) as avg_line_item_count,
+        count(distinct customer_id) as count_customers,
+        count(distinct email) as count_customer_emails,
+        sum(order_adjusted_total) as order_adjusted_total,
+        avg(order_adjusted_total) as avg_order_value,
+        sum(shipping_cost) as shipping_cost,
+        sum(order_adjustment_amount) as order_adjustment_amount,
+        sum(order_adjustment_tax_amount) as order_adjustment_tax_amount,
+        sum(refund_subtotal) as refund_subtotal,
+        sum(refund_total_tax) as refund_total_tax,
+        sum(total_discounts) as total_discounts,
+        avg(total_discounts) as avg_discount,
+        sum(shipping_discount_amount) as shipping_discount_amount,
+        avg(shipping_discount_amount) as avg_shipping_discount_amount,
+        sum(percentage_calc_discount_amount) as percentage_calc_discount_amount,
+        avg(percentage_calc_discount_amount) as avg_percentage_calc_discount_amount,
+        sum(fixed_amount_discount_amount) as fixed_amount_discount_amount,
+        avg(fixed_amount_discount_amount) as avg_fixed_amount_discount_amount,
+        sum(count_discount_codes_applied) as count_discount_codes_applied,
+        count(distinct location_id) as count_locations_ordered_from,
+        sum(case when count_discount_codes_applied > 0 then 1 else 0 end) as count_orders_with_discounts,
+        sum(case when refund_subtotal > 0 then 1 else 0 end) as count_orders_with_refunds,
+        min(created_timestamp) as first_order_timestamp,
+        max(created_timestamp) as last_order_timestamp
+
+    from orders
+    group by 1,2
+
+),
+
+order_line_aggregates as (
+
+    select
+        order_lines.source_relation,
+        cast({{ dbt.date_trunc('day','orders.created_timestamp') }} as date) as date_day,
+        sum(order_lines.quantity) as quantity_sold,
+        sum(order_lines.refunded_quantity) as quantity_refunded,
+        sum(order_lines.quantity_net_refunds) as quantity_net,
+        sum(order_lines.quantity) / count(distinct order_lines.order_id) as avg_quantity_sold,
+        sum(order_lines.quantity_net_refunds) / count(distinct order_lines.order_id) as avg_quantity_net,
+        count(distinct order_lines.variant_id) as count_variants_sold, 
+        count(distinct order_lines.product_id) as count_products_sold, 
+        sum(case when order_lines.is_gift_card then order_lines.quantity_net_refunds else 0 end) as quantity_gift_cards_sold,
+        sum(case when order_lines.is_shipping_required then order_lines.quantity_net_refunds else 0 end) as quantity_requiring_shipping
+
+    from order_lines
+    left join orders -- just joining with order to get the created_timestamp
+        on order_lines.order_id = orders.order_id
+        and order_lines.source_relation = orders.source_relation
+
+    group by 1,2
+),
+
+final as (
+
+    select 
+        order_aggregates.*,
+        order_line_aggregates.quantity_sold,
+        order_line_aggregates.quantity_refunded,
+        order_line_aggregates.quantity_net,
+        order_line_aggregates.count_variants_sold,
+        order_line_aggregates.count_products_sold,
+        order_line_aggregates.quantity_gift_cards_sold,
+        order_line_aggregates.quantity_requiring_shipping,
+        order_line_aggregates.avg_quantity_sold,
+        order_line_aggregates.avg_quantity_net
+
+    from order_aggregates
+    left join order_line_aggregates
+        on order_aggregates.date_day = order_line_aggregates.date_day
+        and order_aggregates.source_relation = order_line_aggregates.source_relation
+)
+
+select *
+from final

models/intermediate/int_shopify__discounts__abandoned_checkouts.sql

@@ -31,8 +31,10 @@ abandoned_checkouts_aggregated as (
         abandoned_checkout_discount_code.type,
         abandoned_checkout_discount_code.source_relation,
         sum(abandoned_checkout_discount_code.amount) as total_abandoned_checkout_discount_amount,
-        sum(coalesce(abandoned_checkout.total_line_items_price, 0)) as total_abandoned_checkout_line_items_price,
-        sum(coalesce(abandoned_checkout_shipping_line.price, 0)) as total_abandoned_checkout_shipping_price 
+        sum(coalesce(abandoned_checkout_shipping_line.price, 0)) as total_abandoned_checkout_shipping_price,
+        count(distinct customer_id) as count_abandoned_checkout_customers,
+        count(distinct email) as count_abandoned_checkout_customer_emails,
+        count(distinct abandoned_checkout.checkout_id) as count_abandoned_checkouts
 
     from abandoned_checkout_discount_code
     left join abandoned_checkout

models/intermediate/int_shopify__discounts__order_aggregates.sql

@@ -21,8 +21,9 @@ orders_aggregated as (
         sum(orders.total_line_items_price) as total_order_line_items_price,
         sum(orders.shipping_cost) as total_order_shipping_cost,
         sum(orders.refund_subtotal + orders.refund_total_tax) as total_order_refund_amount,
-        count(distinct customer_id) as count_distinct_customers,
-        count(distinct email) as count_distinct_customer_emails
+        count(distinct customer_id) as count_customers,
+        count(distinct email) as count_customer_emails,
+        count(distinct order_discount_code.order_id) as count_orders
 
     from order_discount_code
     join orders