diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index ee7e16d..f683882 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -58,7 +58,7 @@ steps: commands: | bash .buildkite/scripts/run_models.sh redshift - - label: ":bricks: Run Tests - Databricks" + - label: ":databricks: Run Tests - Databricks" key: "run_dbt_databricks" plugins: - docker#v3.13.0: diff --git a/dbt_project.yml b/dbt_project.yml index be52c40..c51fe8e 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -42,6 +42,7 @@ vars: shopify_product_image: "{{ ref('stg_shopify__product_image') }}" shopify_product_tag: "{{ ref('stg_shopify__product_tag') }}" shopify_shop: "{{ ref('stg_shopify__shop') }}" + shopify_tax_line: "{{ ref('stg_shopify__tax_line') }}" shopify_tender_transaction: "{{ ref('stg_shopify__tender_transaction') }}" shopify_abandoned_checkout_discount_code: "{{ ref('stg_shopify__abandoned_checkout_discount_code') }}" shopify_order_discount_code: "{{ ref('stg_shopify__order_discount_code') }}" diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index ac65cae..fe377ef 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -16,13 +16,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: shopify_integration_tests_1 + schema: shopify_integration_tests_3 threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: shopify_integration_tests_1 + schema: shopify_integration_tests_3 threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -33,7 +33,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: shopify_integration_tests_1 + schema: shopify_integration_tests_3 threads: 8 postgres: type: postgres @@ -42,13 +42,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: shopify_integration_tests_1 + schema: shopify_integration_tests_3 threads: 8 databricks: catalog: null host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: shopify_integration_tests_1 + schema: shopify_integration_tests_3 threads: 2 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 7caaa05..c6588ee 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -5,12 +5,12 @@ config-version: 2 vars: - shopify_schema: shopify_integration_tests_1 + shopify_schema: shopify_integration_tests_3 shopify_source: shopify_customer_identifier: "shopify_customer_data" shopify_order_line_refund_identifier: "shopify_order_line_refund_data" shopify_order_line_identifier: "shopify_order_line_data" - order_source: "{{ ref('shopify_order_data') }}" + shopify_order_identifier: "shopify_order_data" shopify_product_identifier: "shopify_product_data" shopify_order_adjustment_identifier: "shopify_order_adjustment_data" shopify_product_variant_identifier: "shopify_product_variant_data" @@ -36,6 +36,7 @@ vars: shopify_product_image_identifier: "shopify_product_image_data" shopify_product_tag_identifier: "shopify_product_tag_data" shopify_shop_identifier: "shopify_shop_data" + shopify_tax_line_identifier: "shopify_tax_line_data" shopify_tender_transaction_identifier: "shopify_tender_transaction_data" shopify_abandoned_checkout_discount_code_identifier: "shopify_abandoned_checkout_discount_code_data" shopify_order_discount_code_identifier: "shopify_order_discount_code_data" @@ -55,6 +56,7 @@ seeds: _fivetran_synced: timestamp id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}" default_address_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}" + email: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" shopify_order_data: +column_types: created_at: timestamp @@ -166,4 +168,8 @@ seeds: markup: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" price: float original_shop_markup: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" - original_shop_price: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" \ No newline at end of file + original_shop_price: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" + shopify_tax_line_data: + +column_types: + price: float + rate: float \ No newline at end of file diff --git a/integration_tests/seeds/shopify_tax_line_data.csv b/integration_tests/seeds/shopify_tax_line_data.csv new file mode 100644 index 0000000..cfcbfb2 --- /dev/null +++ b/integration_tests/seeds/shopify_tax_line_data.csv @@ -0,0 +1,6 @@ +index,order_line_id,_fivetran_synced,price,rate,title,price_set +1,29227,2022-11-19 05:30:34.023000,0.0,0.0,VAT,"{""shop_money"":{""amount"":""0.00"",""currency_code"":""USD""},""presentment_money"":{""amount"":""0.00"",""currency_code"":""USD""}}" +1,1839083,2022-11-19 07:14:05.023000,0.0,0.0,VAT,"{""shop_money"":{""amount"":""0.00"",""currency_code"":""USD""},""presentment_money"":{""amount"":""0.00"",""currency_code"":""USD""}}" +1,11995,2022-11-19 05:30:34.023000,0.0,0.0,VAT,"{""shop_money"":{""amount"":""0.00"",""currency_code"":""USD""},""presentment_money"":{""amount"":""0.00"",""currency_code"":""USD""}}" +1,10751,2022-11-19 07:14:05.024000,0.0,0.0,VAT,"{""shop_money"":{""amount"":""0.00"",""currency_code"":""USD""},""presentment_money"":{""amount"":""0.00"",""currency_code"":""USD""}}" +1,194763,2022-11-19 05:30:34.023000,0.0,0.0,VAT,"{""shop_money"":{""amount"":""0.00"",""currency_code"":""USD""},""presentment_money"":{""amount"":""0.00"",""currency_code"":""USD""}}" \ No newline at end of file diff --git a/models/intermediate/int_shopify__customer_email_rollup.sql b/models/intermediate/int_shopify__customer_email_rollup.sql index 60cade2..d0a5955 100644 --- a/models/intermediate/int_shopify__customer_email_rollup.sql +++ b/models/intermediate/int_shopify__customer_email_rollup.sql @@ -7,29 +7,36 @@ with customers as ( from {{ var('shopify_customer') }} where email is not null -- nonsensical to include any null emails here +), customer_tags as ( + + select + * + from {{ var('shopify_customer_tag' )}} + ), rollup_customers as ( select -- fields to group by - lower(email) as email, - source_relation, + lower(customers.email) as email, + customers.source_relation, -- fields to string agg together - {{ fivetran_utils.string_agg("cast(customer_id as " ~ dbt.type_string() ~ ")", "', '") }} as customer_ids, - {{ fivetran_utils.string_agg("distinct cast(phone as " ~ dbt.type_string() ~ ")", "', '") }} as phone_numbers, + {{ fivetran_utils.string_agg("cast(customers.customer_id as " ~ dbt.type_string() ~ ")", "', '") }} as customer_ids, + {{ fivetran_utils.string_agg("distinct cast(customers.phone as " ~ dbt.type_string() ~ ")", "', '") }} as phone_numbers, + {{ fivetran_utils.string_agg("distinct cast(customer_tags.value as " ~ dbt.type_string() ~ ")", "', '") }} as customer_tags, -- fields to take aggregates of - min(created_timestamp) as first_account_created_at, - max(created_timestamp) as last_account_created_at, - max(updated_timestamp) as last_updated_at, - max(marketing_consent_updated_at) as marketing_consent_updated_at, - max(_fivetran_synced) as last_fivetran_synced, - sum(orders_count) as orders_count, - sum(total_spent) as total_spent, + min(customers.created_timestamp) as first_account_created_at, + max(customers.created_timestamp) as last_account_created_at, + max(customers.updated_timestamp) as last_updated_at, + max(customers.marketing_consent_updated_at) as marketing_consent_updated_at, + max(customers._fivetran_synced) as last_fivetran_synced, + sum(customers.orders_count) as orders_count, + sum(customers.total_spent) as total_spent, -- take true if ever given for boolean fields - {{ fivetran_utils.max_bool("case when customer_index = 1 then is_tax_exempt else null end") }} as is_tax_exempt, -- since this changes every year - {{ fivetran_utils.max_bool("is_verified_email") }} as is_verified_email + {{ fivetran_utils.max_bool("case when customers.customer_index = 1 then customers.is_tax_exempt else null end") }} as is_tax_exempt, -- since this changes every year + {{ fivetran_utils.max_bool("customers.is_verified_email") }} as is_verified_email -- for all other fields, just take the latest value {% set cols = adapter.get_columns_in_relation(ref('stg_shopify__customer')) %} @@ -38,15 +45,18 @@ with customers as ( 'is_tax_exempt', 'is_verified_email'] %} {% for col in cols %} {% if col.column|lower not in except_cols %} - , max(case when customer_index = 1 then {{ col.column }} else null end) as {{ col.column }} + , max(case when customers.customer_index = 1 then customers.{{ col.column }} else null end) as {{ col.column }} {% endif %} {% endfor %} from customers + left join customer_tags + on customers.customer_id = customer_tags.customer_id + and customers.source_relation = customer_tags.source_relation group by 1,2 ) select * -from rollup_customers \ No newline at end of file +from rollup_customers diff --git a/models/intermediate/int_shopify__discounts__abandoned_checkouts.sql b/models/intermediate/int_shopify__discounts__abandoned_checkouts.sql index d583bfc..7f199a7 100644 --- a/models/intermediate/int_shopify__discounts__abandoned_checkouts.sql +++ b/models/intermediate/int_shopify__discounts__abandoned_checkouts.sql @@ -24,6 +24,17 @@ abandoned_checkout_shipping_line as ( from {{ var('shopify_abandoned_checkout_shipping_line') }} ), +roll_up_shipping_line as ( + + select + checkout_id, + source_relation, + sum(price) as price + + from abandoned_checkout_shipping_line + group by 1,2 +), + abandoned_checkouts_aggregated as ( select @@ -31,7 +42,8 @@ abandoned_checkouts_aggregated as ( abandoned_checkout_discount_code.type, abandoned_checkout_discount_code.source_relation, sum(abandoned_checkout_discount_code.amount) as total_abandoned_checkout_discount_amount, - sum(coalesce(abandoned_checkout_shipping_line.price, 0)) as total_abandoned_checkout_shipping_price, + sum(coalesce(abandoned_checkout.total_line_items_price, 0)) as total_abandoned_checkout_line_items_price, + sum(coalesce(roll_up_shipping_line.price, 0)) as total_abandoned_checkout_shipping_price, count(distinct customer_id) as count_abandoned_checkout_customers, count(distinct email) as count_abandoned_checkout_customer_emails, count(distinct abandoned_checkout.checkout_id) as count_abandoned_checkouts @@ -40,9 +52,9 @@ abandoned_checkouts_aggregated as ( left join abandoned_checkout on abandoned_checkout_discount_code.checkout_id = abandoned_checkout.checkout_id and abandoned_checkout_discount_code.source_relation = abandoned_checkout.source_relation - left join abandoned_checkout_shipping_line - on abandoned_checkout_shipping_line.checkout_id = abandoned_checkout_discount_code.checkout_id - and abandoned_checkout_shipping_line.source_relation = abandoned_checkout_discount_code.source_relation + left join roll_up_shipping_line + on roll_up_shipping_line.checkout_id = abandoned_checkout_discount_code.checkout_id + and roll_up_shipping_line.source_relation = abandoned_checkout_discount_code.source_relation group by 1,2,3 ) diff --git a/models/intermediate/int_shopify__discounts__order_aggregates.sql b/models/intermediate/int_shopify__discounts__order_aggregates.sql index fbc0521..9ac5922 100644 --- a/models/intermediate/int_shopify__discounts__order_aggregates.sql +++ b/models/intermediate/int_shopify__discounts__order_aggregates.sql @@ -18,9 +18,9 @@ orders_aggregated as ( order_discount_code.source_relation, avg(order_discount_code.amount) as avg_order_discount_amount, sum(order_discount_code.amount) as total_order_discount_amount, - sum(orders.total_line_items_price) as total_order_line_items_price, - sum(orders.shipping_cost) as total_order_shipping_cost, - sum(orders.refund_subtotal + orders.refund_total_tax) as total_order_refund_amount, + max(orders.total_line_items_price) as total_order_line_items_price, -- summing would multiply the total by the # of discount codes applied to an order + max(orders.shipping_cost) as total_order_shipping_cost, -- summing would multiply the total by the # of discount codes applied to an order + max(orders.refund_subtotal + orders.refund_total_tax) as total_order_refund_amount, -- summing would multiply the total by the # of discount codes applied to an order count(distinct customer_id) as count_customers, count(distinct email) as count_customer_emails, count(distinct order_discount_code.order_id) as count_orders diff --git a/models/intermediate/int_shopify__emails__order_aggregates.sql b/models/intermediate/int_shopify__emails__order_aggregates.sql index 93cb6ba..37b622a 100644 --- a/models/intermediate/int_shopify__emails__order_aggregates.sql +++ b/models/intermediate/int_shopify__emails__order_aggregates.sql @@ -4,11 +4,30 @@ with orders as ( from {{ var('shopify_order') }} where email is not null +), order_aggregates as ( + + select * + from {{ ref('shopify__orders__order_line_aggregates') }} + ), transactions as ( select * - from {{ ref('shopify__transactions' )}} + from {{ ref('shopify__transactions')}} + where lower(status) = 'success' + and lower(kind) not in ('authorization', 'void') + and lower(gateway) != 'gift_card' -- redeeming a giftcard does not introduce new revenue + +), transaction_aggregates as ( + -- this is necessary as customers can pay via multiple payment gateways + select + order_id, + source_relation, + kind, + sum(currency_exchange_calculated_amount) as currency_exchange_calculated_amount + + from transactions + {{ dbt_utils.group_by(n=3) }} ), aggregated as ( @@ -17,14 +36,34 @@ with orders as ( orders.source_relation, min(orders.created_timestamp) as first_order_timestamp, max(orders.created_timestamp) as most_recent_order_timestamp, - avg(case when lower(transactions.kind) in ('sale','capture') then transactions.currency_exchange_calculated_amount end) as average_order_value, - sum(case when lower(transactions.kind) in ('sale','capture') then transactions.currency_exchange_calculated_amount end) as lifetime_total_spent, - sum(case when lower(transactions.kind) in ('refund') then transactions.currency_exchange_calculated_amount end) as lifetime_total_refunded, - count(distinct orders.order_id) as lifetime_count_orders + avg(transaction_aggregates.currency_exchange_calculated_amount) as avg_order_value, + sum(transaction_aggregates.currency_exchange_calculated_amount) as lifetime_total_spent, + sum(refunds.currency_exchange_calculated_amount) as lifetime_total_refunded, + count(distinct orders.order_id) as lifetime_count_orders, + avg(order_aggregates.order_total_quantity) as avg_quantity_per_order, + sum(order_aggregates.order_total_tax) as lifetime_total_tax, + avg(order_aggregates.order_total_tax) as avg_tax_per_order, + sum(order_aggregates.order_total_discount) as lifetime_total_discount, + avg(order_aggregates.order_total_discount) as avg_discount_per_order, + sum(order_aggregates.order_total_shipping) as lifetime_total_shipping, + avg(order_aggregates.order_total_shipping) as avg_shipping_per_order, + sum(order_aggregates.order_total_shipping_with_discounts) as lifetime_total_shipping_with_discounts, + avg(order_aggregates.order_total_shipping_with_discounts) as avg_shipping_with_discounts_per_order, + sum(order_aggregates.order_total_shipping_tax) as lifetime_total_shipping_tax, + avg(order_aggregates.order_total_shipping_tax) as avg_shipping_tax_per_order from orders - left join transactions - on orders.order_id = transactions.order_id - and orders.source_relation = transactions.source_relation + left join transaction_aggregates + on orders.order_id = transaction_aggregates.order_id + and orders.source_relation = transaction_aggregates.source_relation + and transaction_aggregates.kind in ('sale','capture') + left join transaction_aggregates as refunds + on orders.order_id = refunds.order_id + and orders.source_relation = refunds.source_relation + and refunds.kind = 'refund' + left join order_aggregates + on orders.order_id = order_aggregates.order_id + and orders.source_relation = order_aggregates.source_relation + group by 1,2 ) diff --git a/models/intermediate/int_shopify__order__shipping_aggregates.sql b/models/intermediate/int_shopify__order__shipping_aggregates.sql new file mode 100644 index 0000000..d67a60b --- /dev/null +++ b/models/intermediate/int_shopify__order__shipping_aggregates.sql @@ -0,0 +1,40 @@ +with order_shipping_line as ( + + select + order_id, + source_relation, + order_shipping_line_id, + sum(price) as shipping_price, + sum(discounted_price) as discounted_shipping_price + + from {{ var('shopify_order_shipping_line') }} + group by 1,2,3 + +), order_shipping_tax_line as ( + + select + order_shipping_line_id, + source_relation, + sum(price) as shipping_tax + + from {{ var('shopify_order_shipping_tax_line') }} + group by 1,2 + +), aggregated as ( + + select + order_shipping_line.order_id, + order_shipping_line.source_relation, + sum(order_shipping_line.shipping_price) as shipping_price, + sum(order_shipping_line.discounted_shipping_price) as discounted_shipping_price, + sum(order_shipping_tax_line.shipping_tax) as shipping_tax + + from order_shipping_line + left join order_shipping_tax_line + on order_shipping_line.order_shipping_line_id = order_shipping_tax_line.order_shipping_line_id + and order_shipping_line.source_relation = order_shipping_tax_line.source_relation + group by 1,2 +) + +select * +from aggregated \ No newline at end of file diff --git a/models/intermediate/int_shopify__product__order_line_aggregates.sql b/models/intermediate/int_shopify__product__order_line_aggregates.sql new file mode 100644 index 0000000..1d874ff --- /dev/null +++ b/models/intermediate/int_shopify__product__order_line_aggregates.sql @@ -0,0 +1,40 @@ +with order_lines as ( + + select * + from {{ ref('shopify__order_lines') }} + +), orders as ( + + select * + from {{ ref('shopify__orders')}} + +), product_aggregated as ( + select + order_lines.product_id, + order_lines.source_relation, + + -- moved over from shopify__products + sum(order_lines.quantity) as quantity_sold, + sum(order_lines.pre_tax_price) as subtotal_sold, + sum(order_lines.quantity_net_refunds) as quantity_sold_net_refunds, + sum(order_lines.subtotal_net_refunds) as subtotal_sold_net_refunds, + min(orders.created_timestamp) as first_order_timestamp, + max(orders.created_timestamp) as most_recent_order_timestamp, + + -- new columns + sum(order_lines.total_discount) as product_total_discount, + sum(order_lines.order_line_tax) as product_total_tax, + avg(order_lines.quantity) as avg_quantity_per_order_line, + avg(order_lines.total_discount) as product_avg_discount_per_order_line, + avg(order_lines.order_line_tax) as product_avg_tax_per_order_line + + from order_lines + left join orders + on order_lines.order_id = orders.order_id + and order_lines.source_relation = orders.source_relation + group by 1,2 + +) + +select * +from product_aggregated \ No newline at end of file diff --git a/models/intermediate/intermediate.yml b/models/intermediate/intermediate.yml index b880351..c64c748 100644 --- a/models/intermediate/intermediate.yml +++ b/models/intermediate/intermediate.yml @@ -46,4 +46,22 @@ models: combination_of_columns: - code - type + - source_relation + - name: int_shopify__order__shipping_aggregates + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - order_id + - source_relation + - name: int_shopify__product__order_line_aggregates + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - product_id + - source_relation + - name: int_shopify__products_with_aggregates + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - product_id - source_relation \ No newline at end of file diff --git a/models/intermediate/shopify__customers__order_aggregates.sql b/models/intermediate/shopify__customers__order_aggregates.sql index de29c65..15b0234 100644 --- a/models/intermediate/shopify__customers__order_aggregates.sql +++ b/models/intermediate/shopify__customers__order_aggregates.sql @@ -2,30 +2,89 @@ with orders as ( select * from {{ var('shopify_order') }} + where customer_id is not null + +), order_aggregates as ( + + select * + from {{ ref('shopify__orders__order_line_aggregates') }} ), transactions as ( select * - from {{ ref('shopify__transactions' )}} + from {{ ref('shopify__transactions')}} + where lower(status) = 'success' + and lower(kind) not in ('authorization', 'void') + and lower(gateway) != 'gift_card' -- redeeming a giftcard does not introduce new revenue + +), transaction_aggregates as ( + -- this is necessary as customers can pay via multiple payment gateways + select + order_id, + source_relation, + kind, + sum(currency_exchange_calculated_amount) as currency_exchange_calculated_amount + + from transactions + {{ dbt_utils.group_by(n=3) }} + +), customer_tags as ( + + select * + from {{ var('shopify_customer_tag' )}} + +), customer_tags_aggregated as ( + + select + customer_id, + source_relation, + {{ fivetran_utils.string_agg("distinct cast(value as " ~ dbt.type_string() ~ ")", "', '") }} as customer_tags + + from customer_tags + group by 1,2 ), aggregated as ( select orders.customer_id, orders.source_relation, + customer_tags_aggregated.customer_tags, min(orders.created_timestamp) as first_order_timestamp, max(orders.created_timestamp) as most_recent_order_timestamp, - avg(case when lower(transactions.kind) in ('sale','capture') then transactions.currency_exchange_calculated_amount end) as average_order_value, - sum(case when lower(transactions.kind) in ('sale','capture') then transactions.currency_exchange_calculated_amount end) as lifetime_total_spent, - sum(case when lower(transactions.kind) in ('refund') then transactions.currency_exchange_calculated_amount end) as lifetime_total_refunded, - count(distinct orders.order_id) as lifetime_count_orders - from orders - left join transactions - using (order_id, source_relation) - where customer_id is not null - group by 1,2 + avg(transaction_aggregates.currency_exchange_calculated_amount) as avg_order_value, + sum(transaction_aggregates.currency_exchange_calculated_amount) as lifetime_total_spent, + sum(refunds.currency_exchange_calculated_amount) as lifetime_total_refunded, + count(distinct orders.order_id) as lifetime_count_orders, + avg(order_aggregates.order_total_quantity) as avg_quantity_per_order, + sum(order_aggregates.order_total_tax) as lifetime_total_tax, + avg(order_aggregates.order_total_tax) as avg_tax_per_order, + sum(order_aggregates.order_total_discount) as lifetime_total_discount, + avg(order_aggregates.order_total_discount) as avg_discount_per_order, + sum(order_aggregates.order_total_shipping) as lifetime_total_shipping, + avg(order_aggregates.order_total_shipping) as avg_shipping_per_order, + sum(order_aggregates.order_total_shipping_with_discounts) as lifetime_total_shipping_with_discounts, + avg(order_aggregates.order_total_shipping_with_discounts) as avg_shipping_with_discounts_per_order, + sum(order_aggregates.order_total_shipping_tax) as lifetime_total_shipping_tax, + avg(order_aggregates.order_total_shipping_tax) as avg_shipping_tax_per_order + from orders + left join transaction_aggregates + on orders.order_id = transaction_aggregates.order_id + and orders.source_relation = transaction_aggregates.source_relation + and transaction_aggregates.kind in ('sale','capture') + left join transaction_aggregates as refunds + on orders.order_id = refunds.order_id + and orders.source_relation = refunds.source_relation + and refunds.kind = 'refund' + left join order_aggregates + on orders.order_id = order_aggregates.order_id + and orders.source_relation = order_aggregates.source_relation + left join customer_tags_aggregated + on orders.customer_id = customer_tags_aggregated.customer_id + and orders.source_relation = customer_tags_aggregated.source_relation + + {{ dbt_utils.group_by(n=3) }} ) select * diff --git a/models/intermediate/shopify__orders__order_line_aggregates.sql b/models/intermediate/shopify__orders__order_line_aggregates.sql index 3e58bdd..ff105c3 100644 --- a/models/intermediate/shopify__orders__order_line_aggregates.sql +++ b/models/intermediate/shopify__orders__order_line_aggregates.sql @@ -3,16 +3,62 @@ with order_line as ( select * from {{ var('shopify_order_line') }} -), aggregated as ( +), tax as ( - select - order_id, + select + * + from {{ var('shopify_tax_line') }} + +), shipping as ( + + select + * + from {{ ref('int_shopify__order__shipping_aggregates')}} + +), tax_aggregates as ( + + select + order_line_id, source_relation, - count(*) as line_item_count + sum(price) price + + from tax + group by 1,2 + +), order_line_aggregates as ( + + select + order_line.order_id, + order_line.source_relation, + count(*) as line_item_count, + sum(order_line.quantity) as order_total_quantity, + sum(tax_aggregates.price) as order_total_tax, + sum(order_line.total_discount) as order_total_discount + from order_line + left join tax_aggregates + on tax_aggregates.order_line_id = order_line.order_line_id + and tax_aggregates.source_relation = order_line.source_relation group by 1,2 +), final as ( + + select + order_line_aggregates.order_id, + order_line_aggregates.source_relation, + order_line_aggregates.line_item_count, + order_line_aggregates.order_total_quantity, + order_line_aggregates.order_total_tax, + order_line_aggregates.order_total_discount, + shipping.shipping_price as order_total_shipping, + shipping.discounted_shipping_price as order_total_shipping_with_discounts, + shipping.shipping_tax as order_total_shipping_tax + + from order_line_aggregates + left join shipping + on shipping.order_id = order_line_aggregates.order_id + and shipping.source_relation = order_line_aggregates.source_relation ) select * -from aggregated \ No newline at end of file +from final \ No newline at end of file diff --git a/models/intermediate/shopify__orders__order_refunds.sql b/models/intermediate/shopify__orders__order_refunds.sql index cd775ee..98808a3 100644 --- a/models/intermediate/shopify__orders__order_refunds.sql +++ b/models/intermediate/shopify__orders__order_refunds.sql @@ -22,6 +22,7 @@ with refunds as ( order_line_refunds.quantity, order_line_refunds.subtotal, order_line_refunds.total_tax + from refunds left join order_line_refunds on refunds.refund_id = order_line_refunds.refund_id diff --git a/models/shopify.yml b/models/shopify.yml index c9a0418..a163679 100644 --- a/models/shopify.yml +++ b/models/shopify.yml @@ -27,6 +27,8 @@ models: description: Number of orders purchased in the `date_month` - name: order_count_lifetime description: Number of orders purchased up until and including this `date_month`. + - name: source_relation + description: The schema or database this record came from, if you are unioning multiple connectors. Null if not. - name: total_price_in_month description: Total amount (in currency) purchased in the `date_month` - name: total_price_lifetime @@ -58,6 +60,8 @@ models: description: Number of orders purchased in the `date_month` - name: order_count_lifetime description: Number of orders purchased up until and including this `date_month`. + - name: source_relation + description: The schema or database this record came from, if you are unioning multiple connectors. Null if not. - name: total_price_in_month description: Total amount (in currency) purchased in the `date_month` - name: total_price_lifetime @@ -181,6 +185,8 @@ models: description: The postal code (zip, postcode, Eircode, …) of the shipping address. - name: source_name description: Where the order originated. Can be set only during order creation, and is not writeable afterwards. + - name: source_relation + description: The schema or database this record came from, if you are unioning multiple connectors. Null if not. - name: subtotal_price description: The price of the order in the shop currency after discounts but before shipping, taxes, and tips. - name: has_taxes_included @@ -224,14 +230,60 @@ models: Order total adjusted for refunds and other adjustments. The calculation used for this field is as follows: total price listed on the original order (including shipping costs and discounts) + adjustments + adjustments tax - total refunds - refunds tax The order_adjusted_total will equate to the total sales - refunds listed within the transactions table for the order (after currency exchange). - - name: index - description: Field representing the index of the order line in relation to the order. - - name: pre_tax_price - description: The pre tax price of the order line. - name: checkout_token description: The checkout token applied to the order. - name: total_shipping_price_set description: The total shipping price set to the order. + - name: order_total_shipping_tax + description: Total shipping tax attributed to the order. + - name: order_tags + description: List of tags associated with the order. + - name: order_url_tags + description: List of url tags associated with the order. + - name: number_of_fulfillments + description: Total fulfillments for the order. + - name: fulfillment_services + description: List of fulfillment services for the order. + - name: tracking_companies + description: List of tracking companies for the order. + - name: tracking_numbers + description: List of tracking numbers for the order. + - name: total_tip_received + description: The sum of all the tips in the order in the shop currency. + - name: checkout_id + description: The ID for the checkout. + - name: client_details_user_agent + description: Details of the browsing client, including software and operating versions. + - name: customer_locale + description: The two or three-letter language code, optionally followed by a region modifier. Example values - en, en-CA. + - name: order_status_url + description: The URL pointing to the order status web page, if applicable. + - name: presentment_currency + description: The three-letter code (ISO 4217 format) of the currency that the customer used at checkout. For the shop's default currency, see `currency`. + - name: is_deleted + description: "{{ doc('_fivetran_deleted') }}" + - name: tracking_numbers + description: List of tracking numbers for the order. + - name: total_discounts_set + description: __FILL_IN_DESCRIPTION + - name: total_line_items_price_set + description: __FILL_IN_DESCRIPTION + - name: total_price_set + description: __FILL_IN_DESCRIPTION + - name: total_price_usd + description: Total price in USD. + - name: total_tax_set + description: __FILL_IN_DESCRIPTION + - name: is_confirmed + description: __FILL_IN_DESCRIPTION + - name: shipping_discount_amount + description: Shipping portion of the discount. + - name: percentage_calc_discount_amount + description: __FILL_IN_DESCRIPTION + - name: fixed_amount_discount_amount + description: __FILL_IN_DESCRIPTION + - name: count_discount_codes_applied + description: __FILL_IN_DESCRIPTION - name: shopify__customers description: Each record represents a customer in Shopify. @@ -243,8 +295,14 @@ models: columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" - - name: has_accepted_marketing - description: Whether the customer has consented to receive marketing material via email. + - name: marketing_consent_state + description: The current email marketing state for the customer. New version of `accepts_marketing` field. + - name: marketing_opt_in_level + description: Deprecated. The package will coalesce with `email_marketing_consent_opt_in_level`. + - name: lifetime_duration + description: The amount of time since the customer was first added to the store. + - name: marketing_consent_updated_at + description: Timestamp when marketing consent was updated. - name: created_timestamp description: The date and time when the customer was created. - name: default_address_id @@ -273,14 +331,46 @@ models: description: The timestamp the customer completed their first order. - name: most_recent_order_timestamp description: The timestamp the customer completed their most recent order. - - name: average_order_value + - name: avg_order_value description: The average order value for the customer. - name: lifetime_total_spent description: The total amount of money that the customer has spent on orders across their order history. - name: lifetime_total_refunded description: The total amount of money that the customer has been refunded on orders across their order history. - - name: lifetime_total_amount + - name: lifetime_total_net description: The total amount of money (minus refunds) that the customer has spent across their order history. + - name: source_relation + description: The schema or database this record came from, if you are unioning multiple connectors. Null if not. + - name: lifetime_abandoned_checkouts + description: Total number of abandoned checkouts abandoned by the customer. + - name: customer_tags + description: A string aggregated list of all tags associated with a customer. + - name: avg_quantity_per_order + description: Average quantity of items per order customer orders. + - name: lifetime_total_tax + description: Total amount of tax attributed to the customer. + - name: avg_tax_per_order + description: Average tax per order attributed to the customer. + - name: lifetime_total_discount + description: Total discounts attributed to the customer. + - name: avg_discount_per_order + description: Average discount per order attributed to the customer. + - name: lifetime_total_shipping + description: Total shipping costs attributed to the customer. + - name: avg_shipping_per_order + description: Average shipping cost per order attributed to the customer. + - name: lifetime_total_shipping_with_discounts + description: Total shipping costs after discounts attributed to the customer. + - name: avg_shipping_with_discounts_per_order + description: Average shipping costs after discounts per order attributed to the customer. + - name: lifetime_total_shipping_tax + description: Total shipping tax attributed to the customer. + - name: avg_shipping_tax_per_order + description: Average shipping tax per order attributed to the customer. + - name: currency + description: The three-letter code (ISO 4217 format) for the currency that the customer used when they paid for their last order. Defaults to the shop currency. Returns the shop currency for test orders. + - name: note + description: __FILL_IN_DESCRIPTION - name: shopify__customer_emails description: Each record represents a customer email in Shopify. @@ -292,8 +382,6 @@ models: columns: - name: last_fivetran_synced description: "{{ doc('_fivetran_synced') }}" - - name: has_accepted_marketing - description: Whether the customer has consented to receive marketing material via email. - name: created_timestamp description: The date and time when the customer was created. - name: default_address_id @@ -322,20 +410,60 @@ models: description: The timestamp the customer completed their first order. - name: most_recent_order_timestamp description: The timestamp the customer completed their most recent order. - - name: average_order_value + - name: avg_order_value description: The average order value for the customer. - name: lifetime_total_spent description: The total amount of money that the customer has spent on orders across their order history. - name: lifetime_total_refunded description: The total amount of money that the customer has been refunded on orders across their order history. - - name: lifetime_total_amount + - name: lifetime_total_net description: The total amount of money (minus refunds) that the customer has spent across their order history. - name: first_account_created_at description: Timestamp of when the first account associated with this email was created. - name: last_account_created_at description: Timestamp of when the last account associated with this email was created. - - name: accepts_marketing_last_updated_at - description: When the customer consented or objected to receiving marketing material by email. + - name: source_relation + description: The schema or database this record came from, if you are unioning multiple connectors. Null if not. + - name: lifetime_abandoned_checkouts + description: Total number of abandoned checkouts abandoned by the customer. + - name: customer_tags + description: A string aggregated list of all tags associated with a customer. + - name: avg_quantity_per_order + description: Average quantity of items per order customer orders. + - name: lifetime_total_tax + description: Total amount of tax attributed to the customer. + - name: avg_tax_per_order + description: Average tax per order attributed to the customer. + - name: lifetime_total_discount + description: Total discounts attributed to the customer. + - name: avg_discount_per_order + description: Average discount per order attributed to the customer. + - name: lifetime_total_shipping + description: Total shipping costs attributed to the customer. + - name: avg_shipping_per_order + description: Average shipping cost per order attributed to the customer. + - name: lifetime_total_shipping_with_discounts + description: Total shipping costs after discounts attributed to the customer. + - name: avg_shipping_with_discounts_per_order + description: Average shipping costs after discounts per order attributed to the customer. + - name: lifetime_total_shipping_tax + description: Total shipping tax attributed to the customer. + - name: avg_shipping_tax_per_order + description: Average shipping tax per order attributed to the customer. + - name: marketing_consent_updated_at + description: Timestamp when marketing consent was updated. + - name: lifetime_duration + description: The amount of time since the customer was first added to the store. + - name: currency + description: The three-letter code (ISO 4217 format) for the currency that the customer used when they paid for their last order. Defaults to the shop currency. Returns the shop currency for test orders. + - name: marketing_consent_state + description: The current email marketing state for the customer. New version of `accepts_marketing` field. + - name: marketing_opt_in_level + description: Deprecated. The package will coalesce with `email_marketing_consent_opt_in_level`. + - name: updated_timestamp + description: The date and time when the customer information was last updated. + - name: note + description: __FILL_IN_DESCRIPTION - name: shopify__products description: Each record represents a product in Shopify. @@ -345,7 +473,7 @@ models: - product_id - source_relation columns: - - name: _fivetran_deleted + - name: is_deleted description: Whether the record has been deleted in the source system. - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -367,7 +495,7 @@ models: description: The date and time when the product was last modified. - name: vendor description: The name of the product's vendor. - - name: quantity_sold + - name: total_quantity_sold description: Quantity of the product sold. - name: subtotal_sold description: Total amount of the product sold. @@ -379,6 +507,28 @@ models: description: The timestamp the product was first ordered. - name: most_recent_order_timestamp description: The timestamp the product was most recently ordered. + - name: source_relation + description: The schema or database this record came from, if you are unioning multiple connectors. Null if not. + - name: avg_quantity_per_order_line + description: Average quantity per order line with this product. + - name: product_total_discount + description: Total discounts associated with the product. + - name: product_avg_discount_per_order_line + description: Average discount per order line with this product. + - name: product_total_tax + description: Total taxes associated with the product. + - name: product_avg_tax_per_order_line + description: Average taxes per order line with this product. + - name: count_variants + description: Count of product variants. + - name: has_product_image + description: If the product has an image. + - name: status + description: __FILL_IN_DESCRIPTION + - name: collections + description: __FILL_IN_DESCRIPTION + - name: tags + description: __FILL_IN_DESCRIPTION - name: shopify__order_lines description: Each record represents a line item of an order in Shopify. @@ -412,7 +562,7 @@ models: description: The ID of the product that the line item belongs to. Can be null if the original product associated with the order is deleted at a later date. - name: quantity description: The number of items that were purchased. - - name: is_requiring_shipping + - name: is_shipping_required description: Whether the item requires shipping. - name: sku description: The item's SKU (stock keeping unit). @@ -440,8 +590,6 @@ models: description: The fulfillment service associated with the product variant. - name: variant_grams description: The weight of the product variant in grams. - - name: variant_image_id - description: The unique numeric identifier for a product's image. The image must be associated to the same product as the variant. - name: inventory_item_id description: The unique identifier for the inventory item, which is used in the Inventory API to query for inventory information. - name: variant_inventory_management @@ -460,12 +608,12 @@ models: description: The order of the product variant in the list of product variants. The first position in the list is 1. The position of variants is indicated by the order in which they are listed. - name: variant_price description: The price of the product variant. - - name: variant_is_requiring_shipping - description: This property is deprecated. Use the `requires_shipping` property on the InventoryItem resource instead. - name: variant_sku description: A unique identifier for the product variant in the shop. Required in order to connect to a FulfillmentService. - name: variant_is_taxable description: Whether a tax is charged when the product variant is sold. + - name: variant_tax_code + description: This parameter applies only to the stores that have the Avalara AvaTax app installed. Specifies the Avalara tax code for the product variant. - name: variant_title description: The title of the product variant. The title field is a concatenation of the option1, option2, and option3 fields. You can only update title indirectly using the option fields. - name: variant_updated_at @@ -480,7 +628,69 @@ models: description: Subtotal of the order line with refunds subtracted. - name: image_id description: Image id of the product variant associated with the order line. - + - name: source_relation + description: The schema or database this record came from, if you are unioning multiple connectors. Null if not. + - name: restock_types + description: List of restock types for the line item. + - name: order_line_tax + description: Total taxes for the line item. + - name: index + description: The index associated with the order. + - name: pre_tax_price + description: The total pre tax price of the order. + - name: tax_code + description: This parameter applies only to the stores that have the Avalara AvaTax app installed. Specifies the Avalara tax code for the product variant. + - name: pre_tax_price_set + description: __FILL_IN_DESCRIPTION + - name: price_set + description: __FILL_IN_DESCRIPTION + - name: property_charge_interval_frequency + description: __FILL_IN_DESCRIPTION + - name: property_for_shipping_jan_3_rd_2020 + description: __FILL_IN_DESCRIPTION + - name: property_shipping_interval_frequency + description: __FILL_IN_DESCRIPTION + - name: property_shipping_interval_unit_type + description: __FILL_IN_DESCRIPTION + - name: property_subscription_id + description: __FILL_IN_DESCRIPTION + - name: total_discount_set + description: __FILL_IN_DESCRIPTION + - name: properties + description: __FILL_IN_DESCRIPTION + - name: destination_location_address_1 + description: Destination location's street address. + - name: destination_location_address_2 + description: The optional second line of the destination location's street address. + - name: destination_location_city + description: The city the destination location is in. + - name: destination_location_country_code + description: The country code of the destination location. + - name: destination_location_id + description: The ID of the destination location. + - name: destination_location_name + description: The name of the destination location. + - name: destination_location_province_code + description: The province, state, or district code (ISO 3166-2 alpha-2 format) of the destination location. + - name: destination_location_zip + description: The zip or postal code. + - name: origin_location_address_1 + description: Origin location's street address. + - name: origin_location_address_2 + description: The optional second line of the origin location's street address. + - name: origin_location_city + description: The city the origin location is in. + - name: origin_location_country_code + description: The country code of the origin location. + - name: origin_location_id + description: The ID of the origin location. + - name: origin_location_name + description: The name of the origin location. + - name: origin_location_province_code + description: The province, state, or district code (ISO 3166-2 alpha-2 format) of the origin location. + - name: origin_location_zip + description: The zip or postal code. + - name: shopify__transactions description: Each record represents a transaction in Shopify. tests: @@ -497,7 +707,7 @@ models: description: The ID associated with a refund in the refund table. - name: amount description: The amount of money included in the transaction. - - name: authorization + - name: authorization_code description: The authorization code associated with the transaction. - name: created_timestamp description: The date and time when the transaction was created. @@ -545,8 +755,6 @@ models: description: A standardized error code, independent of the payment provider. - name: status description: The status of the transaction. - - name: test - description: Whether the transaction is a test transaction. - name: user_id description: The ID for the user who was logged into the Shopify POS device when the order was processed, if applicable. - name: _fivetran_synced @@ -555,6 +763,20 @@ models: description: The exchange rate between the home currency and the currency of sale at the time of the transaction. - name: currency_exchange_calculated_amount description: The total amount of the transaction with the currency exchange rate applied. + - name: source_relation + description: The schema or database this record came from, if you are unioning multiple connectors. Null if not. + - name: payment_method + description: Method of payment. + - name: parent_created_timestamp + description: Created on timestamp of the parent transaction. + - name: parent_kind + description: Kind of the parent transaction. + - name: parent_amount + description: Amount of the parent transaction. + - name: parent_status + description: Status of the parent transaction. + - name: authorization_expires_at + description: Timestamp when the authorization expires. - name: shopify__inventory_levels description: > @@ -801,7 +1023,7 @@ models: description: > The title of the price rule. This is used by the Shopify admin search to retrieve discounts. It is also displayed on the Discounts page of the Shopify admin for bulk discounts. Shopify recommends that this map onto the associated `discount_code.code`. - - name: pirce_rule_updated_at + - name: price_rule_updated_at description: The date and time (ISO 8601 format) when the price rule was updated. - name: usage_limit description: The maximum number of times the price rule can be used, per discount code. @@ -831,6 +1053,8 @@ models: description: Count of distinct customer emails who placed orders using this discount. - name: avg_order_discount_amount description: Average amount of discount taken off orders. + - name: source_relation + description: The schema or database this record came from, if you are unioning multiple connectors. Null if not. - name: shopify__daily_shop description: > @@ -952,4 +1176,4 @@ models: - name: avg_quantity_sold description: Average quantity sold per order on this day. Includes refunds. - name: avg_quantity_net - description: Average net quantity sold per order on this day. Excludes refunds. \ No newline at end of file + description: Average net quantity sold per order on this day. Excludes refunds. diff --git a/models/shopify__customer_emails.sql b/models/shopify__customer_emails.sql index 1b70210..75afad7 100644 --- a/models/shopify__customer_emails.sql +++ b/models/shopify__customer_emails.sql @@ -8,24 +8,49 @@ with customer_emails as ( select * from {{ ref('int_shopify__emails__order_aggregates' )}} + where email is not null + +), abandoned as ( + + select + lower(email) as email, + source_relation, + count(distinct checkout_id) as lifetime_abandoned_checkouts + from {{ var('shopify_abandoned_checkout' )}} + where email is not null + group by 1,2 ), joined as ( select customer_emails.*, + coalesce(abandoned.lifetime_abandoned_checkouts, 0) as lifetime_abandoned_checkouts, orders.first_order_timestamp, orders.most_recent_order_timestamp, - coalesce(orders.average_order_value, 0) as average_order_value, + orders.avg_order_value, coalesce(orders.lifetime_total_spent, 0) as lifetime_total_spent, coalesce(orders.lifetime_total_refunded, 0) as lifetime_total_refunded, - (coalesce(orders.lifetime_total_spent, 0) - coalesce(orders.lifetime_total_refunded, 0)) as lifetime_total_amount, - coalesce(orders.lifetime_count_orders, 0) as lifetime_count_orders + (coalesce(orders.lifetime_total_spent, 0) - coalesce(orders.lifetime_total_refunded, 0)) as lifetime_total_net, + coalesce(orders.lifetime_count_orders, 0) as lifetime_count_orders, + orders.avg_quantity_per_order, + coalesce(orders.lifetime_total_tax, 0) as lifetime_total_tax, + orders.avg_tax_per_order, + coalesce(orders.lifetime_total_discount, 0) as lifetime_total_discount, + orders.avg_discount_per_order, + coalesce(orders.lifetime_total_shipping, 0) as lifetime_total_shipping, + orders.avg_shipping_per_order, + coalesce(orders.lifetime_total_shipping_with_discounts, 0) as lifetime_total_shipping_with_discounts, + orders.avg_shipping_with_discounts_per_order, + coalesce(orders.lifetime_total_shipping_tax, 0) as lifetime_total_shipping_tax, + orders.avg_shipping_tax_per_order from customer_emails left join orders on customer_emails.email = orders.email and customer_emails.source_relation = orders.source_relation - + left join abandoned + on customer_emails.email = abandoned.email + and customer_emails.source_relation = abandoned.source_relation ) select * diff --git a/models/shopify__customers.sql b/models/shopify__customers.sql index f36bc10..7702bcf 100644 --- a/models/shopify__customers.sql +++ b/models/shopify__customers.sql @@ -9,21 +9,48 @@ with customers as ( select * from {{ ref('shopify__customers__order_aggregates' )}} +), abandoned as ( + + select + customer_id, + source_relation, + count(distinct checkout_id) as lifetime_abandoned_checkouts + from {{ var('shopify_abandoned_checkout' )}} + where customer_id is not null + group by 1,2 + ), joined as ( select customers.*, + coalesce(abandoned.lifetime_abandoned_checkouts, 0) as lifetime_abandoned_checkouts, orders.first_order_timestamp, orders.most_recent_order_timestamp, - coalesce(orders.average_order_value, 0) as average_order_value, + orders.customer_tags, + orders.avg_order_value, coalesce(orders.lifetime_total_spent, 0) as lifetime_total_spent, coalesce(orders.lifetime_total_refunded, 0) as lifetime_total_refunded, - (coalesce(orders.lifetime_total_spent, 0) - coalesce(orders.lifetime_total_refunded, 0)) as lifetime_total_amount, - coalesce(orders.lifetime_count_orders, 0) as lifetime_count_orders + (coalesce(orders.lifetime_total_spent, 0) - coalesce(orders.lifetime_total_refunded, 0)) as lifetime_total_net, + coalesce(orders.lifetime_count_orders, 0) as lifetime_count_orders, + orders.avg_quantity_per_order, + coalesce(orders.lifetime_total_tax, 0) as lifetime_total_tax, + orders.avg_tax_per_order, + coalesce(orders.lifetime_total_discount, 0) as lifetime_total_discount, + orders.avg_discount_per_order, + coalesce(orders.lifetime_total_shipping, 0) as lifetime_total_shipping, + orders.avg_shipping_per_order, + coalesce(orders.lifetime_total_shipping_with_discounts, 0) as lifetime_total_shipping_with_discounts, + orders.avg_shipping_with_discounts_per_order, + coalesce(orders.lifetime_total_shipping_tax, 0) as lifetime_total_shipping_tax, + orders.avg_shipping_tax_per_order + from customers left join orders - using (customer_id, source_relation) - + on customers.customer_id = orders.customer_id + and customers.source_relation = orders.source_relation + left join abandoned + on customers.customer_id = abandoned.customer_id + and customers.source_relation = abandoned.source_relation ) select * diff --git a/models/shopify__order_lines.sql b/models/shopify__order_lines.sql index 1b1f8de..a92b752 100644 --- a/models/shopify__order_lines.sql +++ b/models/shopify__order_lines.sql @@ -19,14 +19,32 @@ with order_lines as ( order_line_id, source_relation, sum(quantity) as quantity, - sum(coalesce(subtotal, 0)) as subtotal + sum(coalesce(subtotal, 0)) as subtotal, + {{ fivetran_utils.string_agg("distinct cast(refunds.restock_type as " ~ dbt.type_string() ~ ")", "', '") }} as restock_types from refunds group by 1,2 +), tax_lines as ( + + select * + from {{ var('shopify_tax_line')}} + +), tax_lines_aggregated as ( + + select + tax_lines.order_line_id, + tax_lines.source_relation, + sum(tax_lines.price) as order_line_tax + + from tax_lines + group by 1,2 + ), joined as ( select order_lines.*, + + refunds_aggregated.restock_types, coalesce(refunds_aggregated.quantity,0) as refunded_quantity, coalesce(refunds_aggregated.subtotal,0) as refunded_subtotal, @@ -54,10 +72,10 @@ with order_lines as ( product_variants.option_1 as variant_option_1, product_variants.option_2 as variant_option_2, product_variants.option_3 as variant_option_3, - product_variants.tax_code as variant_tax_code - {# , - use inventoryitem or order line itself -- add later TODO - product_variants.is_requiring_shipping as variant_is_requiring_shipping #} + product_variants.tax_code as variant_tax_code, + + tax_lines_aggregated.order_line_tax + from order_lines left join refunds_aggregated on refunds_aggregated.order_line_id = order_lines.order_line_id @@ -65,6 +83,10 @@ with order_lines as ( left join product_variants on product_variants.variant_id = order_lines.variant_id and product_variants.source_relation = order_lines.source_relation + left join tax_lines_aggregated + on tax_lines_aggregated.order_line_id = order_lines.order_line_id + and tax_lines_aggregated.source_relation = order_lines.source_relation + ) diff --git a/models/shopify__orders.sql b/models/shopify__orders.sql index ddfc668..f44229a 100644 --- a/models/shopify__orders.sql +++ b/models/shopify__orders.sql @@ -54,6 +54,39 @@ with orders as ( from order_discount_code group by 1,2 +), order_tag as ( + + select + order_id, + source_relation, + {{ fivetran_utils.string_agg("distinct cast(value as " ~ dbt.type_string() ~ ")", "', '") }} as order_tags + + from {{ var('shopify_order_tag') }} + group by 1,2 + +), order_url_tag as ( + + select + order_id, + source_relation, + {{ fivetran_utils.string_agg("distinct cast(value as " ~ dbt.type_string() ~ ")", "', '") }} as order_url_tags + + from {{ var('shopify_order_url_tag') }} + group by 1,2 + +), fulfillments as ( + + select + order_id, + source_relation, + count(fulfillment_id) as number_of_fulfillments, + {{ fivetran_utils.string_agg("distinct cast(service as " ~ dbt.type_string() ~ ")", "', '") }} as fulfillment_services, + {{ fivetran_utils.string_agg("distinct cast(tracking_company as " ~ dbt.type_string() ~ ")", "', '") }} as tracking_companies, + {{ fivetran_utils.string_agg("distinct cast(tracking_number as " ~ dbt.type_string() ~ ")", "', '") }} as tracking_numbers + + from {{ var('shopify_fulfillment') }} + group by 1,2 + ), joined as ( select @@ -74,13 +107,20 @@ with orders as ( coalesce(discount_aggregates.shipping_discount_amount, 0) as shipping_discount_amount, coalesce(discount_aggregates.percentage_calc_discount_amount, 0) as percentage_calc_discount_amount, coalesce(discount_aggregates.fixed_amount_discount_amount, 0) as fixed_amount_discount_amount, - coalesce(discount_aggregates.count_discount_codes_applied, 0) as count_discount_codes_applied + coalesce(discount_aggregates.count_discount_codes_applied, 0) as count_discount_codes_applied, + coalesce(order_lines.order_total_shipping_tax, 0) as order_total_shipping_tax, + order_tag.order_tags, + order_url_tag.order_url_tags, + fulfillments.number_of_fulfillments, + fulfillments.fulfillment_services, + fulfillments.tracking_companies, + fulfillments.tracking_numbers + from orders left join order_lines on orders.order_id = order_lines.order_id and orders.source_relation = order_lines.source_relation - left join refund_aggregates on orders.order_id = refund_aggregates.order_id and orders.source_relation = refund_aggregates.source_relation @@ -90,6 +130,15 @@ with orders as ( left join discount_aggregates on orders.order_id = discount_aggregates.order_id and orders.source_relation = discount_aggregates.source_relation + left join order_tag + on orders.order_id = order_tag.order_id + and orders.source_relation = order_tag.source_relation + left join order_url_tag + on orders.order_id = order_url_tag.order_id + and orders.source_relation = order_url_tag.source_relation + left join fulfillments + on orders.order_id = fulfillments.order_id + and orders.source_relation = fulfillments.source_relation ), windows as ( diff --git a/models/shopify__products.sql b/models/shopify__products.sql index 21a8c81..2c4fbe3 100644 --- a/models/shopify__products.sql +++ b/models/shopify__products.sql @@ -2,55 +2,32 @@ with products as ( select * from {{ ref('int_shopify__products_with_aggregates') }} -), -order_lines as ( +), product_order_lines as ( select * - from {{ ref('shopify__order_lines') }} -), + from {{ ref('int_shopify__product__order_line_aggregates')}} -orders as ( - - select * - from {{ ref('shopify__orders')}} -), - -order_lines_aggregated as ( - - select - order_lines.product_id, - order_lines.source_relation, - sum(order_lines.quantity) as quantity_sold, - sum(order_lines.pre_tax_price) as subtotal_sold, - - sum(order_lines.quantity_net_refunds) as quantity_sold_net_refunds, - sum(order_lines.subtotal_net_refunds) as subtotal_sold_net_refunds, - - min(orders.created_timestamp) as first_order_timestamp, - max(orders.created_timestamp) as most_recent_order_timestamp - from order_lines - left join orders - using (order_id, source_relation) - group by 1,2 - -), - -joined as ( +), joined as ( select products.*, - coalesce(order_lines_aggregated.quantity_sold,0) as quantity_sold, - coalesce(order_lines_aggregated.subtotal_sold,0) as subtotal_sold, - coalesce(order_lines_aggregated.quantity_sold_net_refunds,0) as quantity_sold_net_refunds, - coalesce(order_lines_aggregated.subtotal_sold_net_refunds,0) as subtotal_sold_net_refunds, - order_lines_aggregated.first_order_timestamp, - order_lines_aggregated.most_recent_order_timestamp + coalesce(product_order_lines.quantity_sold,0) as total_quantity_sold, + coalesce(product_order_lines.subtotal_sold,0) as subtotal_sold, + coalesce(product_order_lines.quantity_sold_net_refunds,0) as quantity_sold_net_refunds, + coalesce(product_order_lines.subtotal_sold_net_refunds,0) as subtotal_sold_net_refunds, + product_order_lines.first_order_timestamp, + product_order_lines.most_recent_order_timestamp, + product_order_lines.avg_quantity_per_order_line as avg_quantity_per_order_line, + coalesce(product_order_lines.product_total_discount,0) as product_total_discount, + product_order_lines.product_avg_discount_per_order_line as product_avg_discount_per_order_line, + coalesce(product_order_lines.product_total_tax,0) as product_total_tax, + product_order_lines.product_avg_tax_per_order_line as product_avg_tax_per_order_line from products - left join order_lines_aggregated - on products.product_id = order_lines_aggregated.product_id - and products.source_relation = order_lines_aggregated.source_relation + left join product_order_lines + on products.product_id = product_order_lines.product_id + and products.source_relation = product_order_lines.source_relation ) select * diff --git a/models/shopify__transactions.sql b/models/shopify__transactions.sql index cab9d4c..b393bb7 100644 --- a/models/shopify__transactions.sql +++ b/models/shopify__transactions.sql @@ -2,13 +2,34 @@ with transactions as ( select * from {{ var('shopify_transaction') }} +), tender_transactions as ( + + select * + from {{ var('shopify_tender_transaction') }} + +), joined as ( + select + transactions.*, + tender_transactions.payment_method, + parent_transactions.created_timestamp as parent_created_timestamp, + parent_transactions.kind as parent_kind, + parent_transactions.amount as parent_amount, + parent_transactions.status as parent_status + from transactions + left join tender_transactions + on transactions.transaction_id = tender_transactions.transaction_id + and transactions.source_relation = tender_transactions.source_relation + left join transactions as parent_transactions + on transactions.parent_id = parent_transactions.transaction_id + and transactions.source_relation = parent_transactions.source_relation + ), exchange_rate as ( select *, coalesce(cast(nullif({{ fivetran_utils.json_parse("receipt",["charges","data",0,"balance_transaction","exchange_rate"]) }}, '') as {{ dbt.type_numeric() }} ),1) as exchange_rate, coalesce(cast(nullif({{ fivetran_utils.json_parse("receipt",["charges","data",0,"balance_transaction","exchange_rate"]) }}, '') as {{ dbt.type_numeric() }} ),1) * amount as currency_exchange_calculated_amount - from transactions + from joined ) diff --git a/packages.yml b/packages.yml index 33e6487..bd84e8b 100644 --- a/packages.yml +++ b/packages.yml @@ -3,5 +3,5 @@ packages: # version: [">=0.7.0", "<0.8.0"] - git: https://github.com/fivetran/dbt_shopify_source.git - revision: feature/package-revamp + revision: feature/revamp/update-for-snowflake warn-unpinned: false