discounts!

dbt_project.yml

@@ -44,4 +44,5 @@ vars:
     shopify_shop: "{{ ref('stg_shopify__shop') }}"
     shopify_tender_transaction: "{{ ref('stg_shopify__tender_transaction') }}"
     shopify_abandoned_checkout_discount_code: "{{ ref('stg_shopify__abandoned_checkout_discount_code') }}"
-    shopify_order_discount_code: "{{ ref('stg_shopify__order_discount_code') }}"
+    shopify_order_discount_code: "{{ ref('stg_shopify__order_discount_code') }}"
+    shopify_abandoned_checkout_shipping_line: "{{ ref('stg_shopify__abandoned_checkout_shipping_line') }}"

integration_tests/dbt_project.yml

@@ -16,31 +16,30 @@ vars:
     shopify_product_variant_identifier:   "shopify_product_variant_data"
     shopify_refund_identifier:            "shopify_refund_data"
     shopify_transaction_identifier:       "shopify_transaction_data"
-
-  # will change these to identifiers later
-    abandoned_checkout_source: "{{ ref('shopify_abandoned_checkout_data') }}"
-    collection_product_source: "{{ ref('shopify_collection_product_data') }}"
-    collection_source: "{{ ref('shopify_collection_data') }}"
-    customer_tag_source: "{{ ref('shopify_customer_tag_data') }}"
-    discount_code_source: "{{ ref('shopify_discount_code_data') }}"
-    fulfillment_source: "{{ ref('shopify_fulfillment_data') }}"
-    inventory_item_source: "{{ ref('shopify_inventory_item_data') }}"
-    inventory_level_source: "{{ ref('shopify_inventory_level_data') }}"
-    location_source: "{{ ref('shopify_location_data') }}"
-    metafield_source: "{{ ref('shopify_metafield_data') }}"
-    order_note_attribute_source: "{{ ref('shopify_order_note_attribute_data') }}"
-    order_shipping_line_source: "{{ ref('shopify_order_shipping_line_data') }}"
-    order_shipping_tax_line_source: "{{ ref('shopify_order_shipping_tax_line_data') }}"
-    order_tag_source: "{{ ref('shopify_order_tag_data') }}"
-    order_url_tag_source: "{{ ref('shopify_order_url_tag_data') }}"
-    price_rule_source: "{{ ref('shopify_price_rule_data') }}"
-    product_image_source: "{{ ref('shopify_product_image_data') }}"
-    product_tag_source: "{{ ref('shopify_product_tag_data') }}"
-    shop_source: "{{ ref('shopify_shop_data') }}"
-    tender_transaction_source: "{{ ref('shopify_tender_transaction_data') }}"
-    abandoned_checkout_discount_code_source: "{{ ref('shopify_abandoned_checkout_discount_code_data') }}"
-    order_discount_code_source: "{{ ref('shopify_order_discount_code_data') }}"
-
+    shopify_abandoned_checkout_identifier: "shopify_abandoned_checkout_data"
+    shopify_collection_product_identifier: "shopify_collection_product_data"
+    shopify_collection_identifier: "shopify_collection_data"
+    shopify_customer_tag_identifier: "shopify_customer_tag_data"
+    shopify_discount_code_identifier: "shopify_discount_code_data"
+    shopify_fulfillment_identifier: "shopify_fulfillment_data"
+    shopify_inventory_item_identifier: "shopify_inventory_item_data"
+    shopify_inventory_level_identifier: "shopify_inventory_level_data"
+    shopify_location_identifier: "shopify_location_data"
+    shopify_metafield_identifier: "shopify_metafield_data"
+    shopify_order_note_attribute_identifier: "shopify_order_note_attribute_data"
+    shopify_order_shipping_line_identifier: "shopify_order_shipping_line_data"
+    shopify_order_shipping_tax_line_identifier: "shopify_order_shipping_tax_line_data"
+    shopify_order_tag_identifier: "shopify_order_tag_data"
+    shopify_order_url_tag_identifier: "shopify_order_url_tag_data"
+    shopify_price_rule_identifier: "shopify_price_rule_data"
+    shopify_product_image_identifier: "shopify_product_image_data"
+    shopify_product_tag_identifier: "shopify_product_tag_data"
+    shopify_shop_identifier: "shopify_shop_data"
+    shopify_tender_transaction_identifier: "shopify_tender_transaction_data"
+    shopify_abandoned_checkout_discount_code_identifier: "shopify_abandoned_checkout_discount_code_data"
+    shopify_order_discount_code_identifier: "shopify_order_discount_code_data"
+    shopify_abandoned_checkout_shipping_line_identifier: "shopify_abandoned_checkout_shipping_line_data"
+
 dispatch:
   - macro_namespace: dbt_utils
     search_order: ['spark_utils', 'dbt_utils']
@@ -121,6 +120,7 @@ seeds:
         closed_at: timestamp
         created_at: timestamp
         updated_at: timestamp
+        _fivetran_deleted: boolean
     shopify_discount_code_data:
       +column_types:
         usage_count: float
@@ -159,4 +159,10 @@ seeds:
     shopify_inventory_item_data:
       +column_types:
         updated_at: timestamp
-        created_at: timestamp
+        created_at: timestamp
+    shopify_abandoned_checkout_shipping_line_data:
+      +column_types:
+        markup: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
+        price: float
+        original_shop_markup: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
+        original_shop_price: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"

integration_tests/seeds/shopify_abandoned_checkout_shipping_line_data.csv

@@ -0,0 +1,6 @@
+checkout_id,index,_fivetran_synced,api_client_id,carrier_identifier,carrier_service_id,code,delivery_category,discounted_price,id,markup,phone,price,requested_fulfillment_service_id,source,title,validation_context,delivery_expectation_range,delivery_expectation_type,original_shop_markup,original_shop_price,presentment_title,delivery_expectation_range_min,delivery_expectation_range_max
+653675,1,2023-01-09 06:48:18.093000,,,,Standard,,,c3ce0972c2e30eaf7001bea,0.0,,0.0,,shopify,Standard,,,,0.0,0.0,Standard,,
+379,1,2023-01-09 06:48:23.540000,,,,Standard,,,bf7c90953344902c13,0.0,,0.0,,shopify,Standard,,,,0.0,0.0,Standard,,
+635,1,2023-01-09 06:48:24.243000,,,,Standard,,,519ff4275cd972e282db,0.0,,0.0,,shopify,Standard,,,,0.0,0.0,Standard,,
+3211,1,2023-01-09 06:48:18.068000,,,,Standard,,,8d18671d481ad46a,0.0,,0.0,,shopify,Standard,,,,0.0,0.0,Standard,,
+381227,1,2023-01-09 06:48:16.985000,,,,Standard,,,8f2fab1b455ec9e597,0.0,,0.0,,shopify,Standard,,,,0.0,0.0,Standard,,

models/intermediate/int_shopify__customer_email_rollup.sql

@@ -22,20 +22,19 @@ with customers as (
         min(created_timestamp) as first_account_created_at,
         max(created_timestamp) as last_account_created_at,
         max(updated_timestamp) as last_updated_at,
-        max(accepts_marketing_updated_at) as accepts_marketing_last_updated_at,
+        max(marketing_consent_updated_at) as marketing_consent_updated_at,
         max(_fivetran_synced) as last_fivetran_synced,
         sum(orders_count) as orders_count,
         sum(total_spent) as total_spent,
 
         -- take true if ever given for boolean fields
-        {{ fivetran_utils.max_bool("has_accepted_marketing") }} as has_accepted_marketing,
         {{ fivetran_utils.max_bool("case when customer_index = 1 then is_tax_exempt else null end") }} as is_tax_exempt, -- since this changes every year
         {{ fivetran_utils.max_bool("is_verified_email") }} as is_verified_email
 
         -- for all other fields, just take the latest value
         {% set cols = adapter.get_columns_in_relation(ref('stg_shopify__customer')) %}
         {% set except_cols = ['_fivetran_synced', 'email', 'source_relation', 'customer_id', 'phone', 'created_at', 
-                                'updated_at', 'has_accepted_marketing', 'accepts_marketing_updated_at', 'orders_count', 'total_spent',
+                                'updated_at', 'marketing_consent_updated_at', 'orders_count', 'total_spent',
                                 'is_tax_exempt', 'is_verified_email'] %}
         {% for col in cols %}
             {% if col.column|lower not in except_cols %}

models/intermediate/int_shopify__discounts__abandoned_checkouts.sql

@@ -0,0 +1,49 @@
+with abandoned_checkout as (
+
+    select *
+    from {{ var('shopify_abandoned_checkout') }}
+
+    -- "deleted" abandoned checkouts do not appear to have any data tying them to customers,
+    -- discounts, or products (and should therefore not get joined in) but let's filter them out here
+    where not coalesce(is_deleted, false)
+),
+
+abandoned_checkout_discount_code as (
+
+    select *
+    from {{ var('shopify_abandoned_checkout_discount_code') }}
+
+    -- we need the TYPE of discount (shipping, percentage, fixed_amount) to avoid fanning out of joins
+    -- so filter out records that have this
+    where coalesce(type, '') != ''
+),
+
+abandoned_checkout_shipping_line as (
+
+    select *
+    from {{ var('shopify_abandoned_checkout_shipping_line') }}
+),
+
+abandoned_checkouts_aggregated as (
+
+    select 
+        abandoned_checkout_discount_code.code,
+        abandoned_checkout_discount_code.type,
+        abandoned_checkout_discount_code.source_relation,
+        sum(abandoned_checkout_discount_code.amount) as total_abandoned_checkout_discount_amount,
+        sum(coalesce(abandoned_checkout.total_line_items_price, 0)) as total_abandoned_checkout_line_items_price,
+        sum(coalesce(abandoned_checkout_shipping_line.price, 0)) as total_abandoned_checkout_shipping_price 
+
+    from abandoned_checkout_discount_code
+    left join abandoned_checkout
+        on abandoned_checkout_discount_code.checkout_id = abandoned_checkout.checkout_id
+        and abandoned_checkout_discount_code.source_relation = abandoned_checkout.source_relation
+    left join abandoned_checkout_shipping_line
+        on abandoned_checkout_shipping_line.checkout_id = abandoned_checkout_discount_code.checkout_id 
+        and abandoned_checkout_shipping_line.source_relation = abandoned_checkout_discount_code.source_relation
+
+    group by 1,2,3
+)
+
+select *
+from abandoned_checkouts_aggregated

models/intermediate/int_shopify__discounts__order_aggregates.sql

@@ -0,0 +1,35 @@
+with order_discount_code as (
+
+    select *
+    from {{ var('shopify_order_discount_code') }}
+),
+
+orders as (
+
+    select *
+    from {{ ref('shopify__orders') }}
+),
+
+orders_aggregated as (
+
+    select 
+        order_discount_code.code,
+        order_discount_code.type,
+        order_discount_code.source_relation,
+        sum(order_discount_code.amount) as total_order_discount_amount,
+        sum(orders.total_line_items_price) as total_order_line_items_price,
+        sum(orders.shipping_cost) as total_order_shipping_cost,
+        sum(orders.refund_subtotal + orders.refund_total_tax) as total_order_refund_amount,
+        count(distinct customer_id) as count_distinct_customers,
+        count(distinct email) as count_distinct_customer_emails
+
+    from order_discount_code
+    join orders 
+        on order_discount_code.order_id = orders.order_id 
+        and order_discount_code.source_relation = orders.source_relation
+
+    group by 1,2,3
+)
+
+select *
+from orders_aggregated

models/intermediate/intermediate.yml

@@ -26,10 +26,24 @@ models:
           combination_of_columns:
             - email
             - source_relation
-  - name: int_hopify__inventory_level__aggregates
+  - name: int_shopify__inventory_level__aggregates
     tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
             - variant_id
             - location_id
+            - source_relation
+  - name: int_shopify__discounts__order_aggregates
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - code
+            - type
+            - source_relation
+  - name: int_shopify__discounts__abandoned_checkouts
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - code
+            - type
             - source_relation

models/shopify.yml

@@ -721,4 +721,106 @@ models:
           Count of fulfillments for this inventory level that failed.
           "Failure" = The fulfillment request failed.
       - name: source_relation
-        description: The schema or database this record came from, if you are unioning multiple connectors. Null if not.
+        description: The schema or database this record came from, if you are unioning multiple connectors. Null if not.
+
+  - name: shopify__discounts
+    description: >
+      Each record represents a unique discount, enriched with information about its assoicated `price_rule`
+      and metrics regarding orders and abandoned checkouts.
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - discount_code_id
+            - source_relation
+    columns:
+      - name: _fivetran_synced
+        description: "{{ doc('_fivetran_synced') }}"
+      - name: code
+        description: The case-insensitive discount code that customers use at checkout. Shopify recommends this map onto the associated `price_rule.title`.
+      - name: created_at
+        description: The date and time (ISO 8601 format) when the discount code was created.
+      - name: discount_code_id
+        description: The ID for the discount code.
+        tests:
+          - not_null
+      - name: price_rule_id
+        description: The ID for the price rule that this discount code belongs to.
+      - name: updated_at
+        description: The date and time (ISO 8601 format) when the discount code was updated.
+      - name: usage_count
+        description: The number of times that the discount code has been redeemed.
+
+      - name: allocation_limit
+        description: >
+          The number of times the discount can be allocated on the cart - if eligible. For example a Buy 1 hat Get 1 hat for free discount can be applied 3 times on a cart having more than 6 hats, 
+          where maximum of 3 hats get discounted - if the allocation_limit is 3. Empty (null) allocation_limit means unlimited number of allocations.
+      - name: allocation_method
+        description: >
+          The allocation method of the price rule. Valid values include `each` (the discount is applied to each of the entitled items. For example, for a price rule that takes $15 off, each entitled line item in a checkout will be discounted by $15)
+          and `across` (the calculated discount amount will be applied across the entitled items. For example, for a price rule that takes $15 off, the discount will be applied across all the entitled items).
+      - name: price_rule_created_at
+        description: The date and time (ISO 8601 format) when the price rule was created.
+      - name: customer_selection
+        description: >
+          The customer selection for the price rule. Valid values include `all` (the price rule is valid for all customers) and `prerequisite` 
+          (the customer must either belong to one of the customer segments specified by customer_segment_prerequisite_ids, or be one of the customers specified by prerequisite_customer_ids).
+      - name: ends_at
+        description: The date and time (ISO 8601 format) when the price rule ends. Must be after starts_at.
+      - name: price_rule_id
+        description: The ID for the price rule.
+      - name: is_once_per_customer
+        description: Boolean representing whether the generated discount code will be valid only for a single use per customer. This is tracked using customer ID.
+      - name: prereq_min_quantity
+        description: If `customer_selection` is `prerequisite`, the minimum number of items for the price rule to be applicable. The quantity of an entitled cart item must be greater than or equal to this value.
+      - name: prereq_max_shipping_price
+        description: If `customer_selection` is `prerequisite`, the maximum shipping price for the price rule to be applicable. The shipping price must be less than or equal to this value
+      - name: prereq_min_subtotal
+        description: If `customer_selection` is `prerequisite`, the minimum subtotal for the price rule to be applicable. The subtotal of the entitled cart items must be greater than or equal to this value for the discount to apply.
+      - name: prereq_min_purchase_quantity_for_entitlement
+        description: If `customer_selection` is `prerequisite`, the prerequisite purchase for a Buy X Get Y discount. The minimum purchase amount required to be entitled to the discount.
+      - name: prereq_buy_x_get_this
+        description: If `customer_selection` is `prerequisite`, in a Buy/Get ratio for a Buy X Get Y discount, this is the offered 'get' quantity.
+      - name: prereq_buy_this_get_y
+        description: If `customer_selection` is `prerequisite`, in a Buy/Get ratio for a Buy X Get Y discount, this defines the necessary 'buy' quantity.
+      - name: starts_at
+        description: The date and time (ISO 8601 format) when the price rule starts.
+      - name: target_selection
+        description: >
+          The target selection method of the price rule. Valid values include `all` (the price rule applies the discount to all line items in the checkout) and 
+          `entitled` (the price rule applies the discount to selected entitlements only).
+      - name: target_type
+        description: The target type that the price rule applies to. Valid values include `line_item` (the price rule applies to the cart's line items) and `shipping_line` (the price rule applies to the cart's shipping lines).
+      - name: title
+        description: >
+          The title of the price rule. This is used by the Shopify admin search to retrieve discounts. It is also displayed on the Discounts page of the Shopify admin for bulk discounts. 
+          Shopify recommends that this map onto the associated `discount_code.code`.
+      - name: pirce_rule_updated_at
+        description: The date and time (ISO 8601 format) when the price rule was updated.
+      - name: usage_limit
+        description: The maximum number of times the price rule can be used, per discount code.
+      - name: value
+        description: The value of the price rule. If if the value of `target_type` is `shipping_line`, then only -100 is accepted. The value must be negative.
+      - name: value_type
+        description: >
+          The value type of the price rule. Valid values include `fixed_amount` (applies a discount of value as a unit of the store's currency. For example, if value is -30 and the store's currency is USD, then $30 USD is deducted when the discount is applied)
+          and `percentage` (applies a percentage discount of value. For example, if value is -30, then 30% will be deducted when the discount is applied).
+
+          If `target_type` is `shipping_line`, then only `percentage` is accepted.
+      - name: total_order_discount_amount
+        description: Total monetary amount of discounts taken off of orders. 
+      - name: total_abandoned_checkout_discount_amount
+        description: Total monetary amount of discounts taken off abanadoned checkout orders.
+      - name: total_order_line_items_price
+        description: Total monetary amount of line items for orders that have used this discount.
+      - name: total_abandoned_checkout_line_items_price
+        description: Total monetary amount of line items for orders that have used this discount.
+      - name: total_order_shipping_cost
+        description: Total shipping costs for orders that used this discount.
+      - name: total_abandoned_checkout_shipping_price
+        description: Total projected shipping costs for abandoned checkouts that applied this discount first.
+      - name: total_order_refund_amount
+        description: Total refunded amount for orders that used this discount code. 
+      - name: count_distinct_customers
+        description: Count of distinct customers who placed orders using this discount.
+      - name: count_distinct_customer_emails
+        description: Count of distinct customer emails who placed orders using this discount.