diff --git a/README.md b/README.md index b9bb865..1ffcca1 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,19 @@ vars: shopify_schema: your_schema_name ``` +If you have multiple Shopify connectors in Fivetran and would like to use this package on all of them simultaneously, we have provided functionality to do so. The package will union all of the data together and pass the unioned table into the transformations. You will be able to see which source it came from in the `source_relation` column of each model. To use this functionality, you will need to set either the `union_schemas` or `union_databases` variables: + +```yml +# dbt_project.yml + +... +config-version: 2 + +vars: + union_schema: ['shopify_usa','shopify_canada'] # use this if the data is in different schemas/datasets of the same database/project + union_databases: ['shopify_usa','shopify_canada'] # use this if the data is in different databases/projects but uses the same schema name +``` + This package includes all source columns defined in the staging_columns.sql macro. To add additional columns to this package, do so using our pass-through column variables. This is extremely useful if you'd like to include custom fields to the package. ```yml diff --git a/dbt_project.yml b/dbt_project.yml index 7ef21e4..7d80a5f 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,6 +1,6 @@ name: 'shopify_source' -version: '0.3.0' +version: '0.3.1' config-version: 2 require-dbt-version: [">=0.18.0", "<0.20.0"] diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index f7107c4..303b2b1 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'shopify_source_integration_tests' -version: '0.3.0' +version: '0.3.1' profile: 'integration_tests' config-version: 2 diff --git a/models/stg_shopify.yml b/models/stg_shopify.yml index b913382..e8c066e 100644 --- a/models/stg_shopify.yml +++ b/models/stg_shopify.yml @@ -3,6 +3,11 @@ version: 2 models: - name: stg_shopify__customer description: Each record represents a customer in Shopify. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - customer_id + - source_relation columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -17,9 +22,6 @@ models: - name: first_name description: The customer's first name. - name: customer_id - tests: - - unique - - not_null description: A unique identifier for the customer. - name: last_name description: The customer's last name. @@ -37,15 +39,18 @@ models: description: The date and time when the customer information was last updated. - name: is_verified_email description: Whether the customer has verified their email address. + - name: stg_shopify__order_line_refund description: Each record represents a line item from an order in Shopify. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - order_line_refund_id + - source_relation columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" - name: order_line_refund_id - tests: - - unique - - not_null description: The unique identifier of the line item in the refund. - name: location_id description: TThe unique identifier of the location where the items will be restockedBD @@ -61,8 +66,14 @@ models: description: Subtotal amount of the order line refund - name: total_tax description: The total tax applied to the refund. + - name: stg_shopify__order_line description: Each record represents a line item from an order in Shopify. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - order_line_id + - source_relation columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -77,9 +88,6 @@ models: - name: grams description: The weight of the item in grams. - name: order_line_id - tests: - - unique - - not_null description: The ID of the line item. - name: name description: The name of the product variant. @@ -105,8 +113,14 @@ models: description: The ID of the product variant. - name: vendor description: The name of the item's supplier. + - name: stg_shopify__order description: Each record represents an order in Shopify. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - order_id + - source_relation columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -167,9 +181,6 @@ models: - name: fulfillment_status description: The order's status in terms of fulfilled line items. - name: order_id - tests: - - unique - - not_null description: The ID of the order, used for API purposes. This is different from the order_number property, which is the ID used by the shop owner and customer. - name: landing_site_base_url description: The URL for the page where the buyer landed when they entered the shop. @@ -249,8 +260,14 @@ models: description: The index associated with the order. - name: pre_tax_price description: The total pre tax price of the order. + - name: stg_shopify__product description: Each record represents a product in Shopify. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - product_id + - source_relation columns: - name: _fivetran_deleted description: Whether the record has been deleted in the source system. @@ -261,9 +278,6 @@ models: - name: handle description: A unique human-friendly string for the product. Automatically generated from the product's title. - name: product_id - tests: - - unique - - not_null description: An unsigned 64-bit integer that's used as a unique identifier for the product. Each id is unique across the Shopify system. No two products will have the same id, even if they're from different shops. - name: product_type description: A categorization for the product used for filtering and searching products. @@ -277,8 +291,14 @@ models: description: The date and time when the product was last modified. - name: vendor description: The name of the product's vendor. + - name: stg_shopify__product_variant description: Each record represents a product variant in Shopify + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - variant_id + - source_relation columns: - name: barcode description: The barcode, UPC, or ISBN number for the product. @@ -292,9 +312,6 @@ models: description: The weight of the product variant in grams. - name: variant_id description: The unique numeric identifier for the product variant. - tests: - - not_null - - unique - name: image_id description: The unique numeric identifier for a product's image. The image must be associated to the same product as the variant. - name: inventory_item_id @@ -337,13 +354,16 @@ models: description: "The unit of measurement that applies to the product variant's weight. If you don't specify a value for weight_unit, then the shop's default unit of measurement is applied. Valid values: g, kg, oz, and lb." - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" + - name: stg_shopify__transaction description: Each record represents a transaction in Shopify. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - transaction_id + - source_relation columns: - name: transaction_id - tests: - - unique - - not_null description: The ID for the transaction. - name: order_id description: The ID for the order that the transaction is associated with. @@ -405,14 +425,17 @@ models: description: The ID for the user who was logged into the Shopify POS device when the order was processed, if applicable. - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" + - name: stg_shopify__refund description: Each record represents a refund within Shopify. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - refund_id + - source_relation columns: - name: refund_id description: The unique numeric identifier for the refund. - tests: - - unique - - not_null - name: created_at description: Timestamp of the date when the refund was created. - name: processed_at @@ -429,14 +452,17 @@ models: description: Record representing total duties set for the refund. - name: order_id description: Reference to the order which the refund is associated. + - name: stg_shopify__order_adjustment description: Each record represents and adjustment to and order within Shopify. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - order_adjustment_id + - source_relation columns: - name: order_adjustment_id description: The unique numeric identifier for the order adjustment. - tests: - - unique - - not_null - name: order_id description: Reference to the order which the adjustment is associated. - name: refund_id diff --git a/models/stg_shopify__customer.sql b/models/stg_shopify__customer.sql index cdb9180..d4cf5d3 100644 --- a/models/stg_shopify__customer.sql +++ b/models/stg_shopify__customer.sql @@ -22,6 +22,8 @@ renamed as ( {% endif %} + {{ fivetran_utils.source_relation() }} + from source ) diff --git a/models/stg_shopify__order.sql b/models/stg_shopify__order.sql index 2fde46d..7215216 100644 --- a/models/stg_shopify__order.sql +++ b/models/stg_shopify__order.sql @@ -22,6 +22,8 @@ renamed as ( {% endif %} + {{ fivetran_utils.source_relation() }} + from source ) diff --git a/models/stg_shopify__order_adjustment.sql b/models/stg_shopify__order_adjustment.sql index 1f363c0..63b90ab 100644 --- a/models/stg_shopify__order_adjustment.sql +++ b/models/stg_shopify__order_adjustment.sql @@ -15,6 +15,8 @@ renamed as ( staging_columns=get_order_adjustment_columns() ) }} + + {{ fivetran_utils.source_relation() }} from source ) diff --git a/models/stg_shopify__order_line.sql b/models/stg_shopify__order_line.sql index ec00c13..1861790 100644 --- a/models/stg_shopify__order_line.sql +++ b/models/stg_shopify__order_line.sql @@ -22,6 +22,8 @@ renamed as ( {% endif %} + {{ fivetran_utils.source_relation() }} + from source ) diff --git a/models/stg_shopify__order_line_refund.sql b/models/stg_shopify__order_line_refund.sql index 043b53a..d7d7d1a 100644 --- a/models/stg_shopify__order_line_refund.sql +++ b/models/stg_shopify__order_line_refund.sql @@ -22,6 +22,8 @@ renamed as ( {% endif %} + {{ fivetran_utils.source_relation() }} + from source ) diff --git a/models/stg_shopify__product.sql b/models/stg_shopify__product.sql index a541453..190efab 100644 --- a/models/stg_shopify__product.sql +++ b/models/stg_shopify__product.sql @@ -22,6 +22,8 @@ renamed as ( {% endif %} + {{ fivetran_utils.source_relation() }} + from source ) diff --git a/models/stg_shopify__product_variant.sql b/models/stg_shopify__product_variant.sql index 232aad8..a39fcd6 100644 --- a/models/stg_shopify__product_variant.sql +++ b/models/stg_shopify__product_variant.sql @@ -22,6 +22,8 @@ renamed as ( {% endif %} + {{ fivetran_utils.source_relation() }} + from source ) diff --git a/models/stg_shopify__refund.sql b/models/stg_shopify__refund.sql index 87e476d..4783213 100644 --- a/models/stg_shopify__refund.sql +++ b/models/stg_shopify__refund.sql @@ -15,6 +15,8 @@ renamed as ( staging_columns=get_refund_columns() ) }} + + {{ fivetran_utils.source_relation() }} from source ) diff --git a/models/stg_shopify__transaction.sql b/models/stg_shopify__transaction.sql index 25e57f3..28c78fe 100644 --- a/models/stg_shopify__transaction.sql +++ b/models/stg_shopify__transaction.sql @@ -22,6 +22,8 @@ renamed as ( {% endif %} + {{ fivetran_utils.source_relation() }} + from source where not test diff --git a/models/tmp/stg_shopify__customer_tmp.sql b/models/tmp/stg_shopify__customer_tmp.sql index 0698d7e..53cb4c5 100644 --- a/models/tmp/stg_shopify__customer_tmp.sql +++ b/models/tmp/stg_shopify__customer_tmp.sql @@ -1,2 +1,10 @@ -select * -from {{ var('customer_source') }} \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='customer', + database_variable='shopify_database', + schema_variable='shopify_schema', + default_database=target.database, + default_schema='shopify', + default_variable='customer_source' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_shopify__order_adjustment_tmp.sql b/models/tmp/stg_shopify__order_adjustment_tmp.sql index 0661b40..0bb9d95 100644 --- a/models/tmp/stg_shopify__order_adjustment_tmp.sql +++ b/models/tmp/stg_shopify__order_adjustment_tmp.sql @@ -1 +1,10 @@ -select * from {{ var('order_adjustment_source') }} +{{ + fivetran_utils.union_data( + table_identifier='order_adjustment', + database_variable='shopify_database', + schema_variable='shopify_schema', + default_database=target.database, + default_schema='shopify', + default_variable='order_adjustment_source' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_shopify__order_line_refund_tmp.sql b/models/tmp/stg_shopify__order_line_refund_tmp.sql index 57fbb3c..0adc837 100644 --- a/models/tmp/stg_shopify__order_line_refund_tmp.sql +++ b/models/tmp/stg_shopify__order_line_refund_tmp.sql @@ -1,2 +1,10 @@ -select * -from {{ var('order_line_refund_source') }} \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='order_line_refund', + database_variable='shopify_database', + schema_variable='shopify_schema', + default_database=target.database, + default_schema='shopify', + default_variable='order_line_refund_source' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_shopify__order_line_tmp.sql b/models/tmp/stg_shopify__order_line_tmp.sql index 7c6b189..1d91d74 100644 --- a/models/tmp/stg_shopify__order_line_tmp.sql +++ b/models/tmp/stg_shopify__order_line_tmp.sql @@ -1,2 +1,10 @@ -select * -from {{ var('order_line_source') }} \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='order_line', + database_variable='shopify_database', + schema_variable='shopify_schema', + default_database=target.database, + default_schema='shopify', + default_variable='order_line_source' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_shopify__order_tmp.sql b/models/tmp/stg_shopify__order_tmp.sql index 49cf535..71723fd 100644 --- a/models/tmp/stg_shopify__order_tmp.sql +++ b/models/tmp/stg_shopify__order_tmp.sql @@ -1,2 +1,10 @@ -select * -from {{ var('order_source') }} \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='order', + database_variable='shopify_database', + schema_variable='shopify_schema', + default_database=target.database, + default_schema='shopify', + default_variable='order_source' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_shopify__product_tmp.sql b/models/tmp/stg_shopify__product_tmp.sql index 2f45797..aaec57d 100644 --- a/models/tmp/stg_shopify__product_tmp.sql +++ b/models/tmp/stg_shopify__product_tmp.sql @@ -1,2 +1,10 @@ -select * -from {{ var('product_source') }} \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='product', + database_variable='shopify_database', + schema_variable='shopify_schema', + default_database=target.database, + default_schema='shopify', + default_variable='product_source' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_shopify__product_variant_tmp.sql b/models/tmp/stg_shopify__product_variant_tmp.sql index 684fa8f..3b1e479 100644 --- a/models/tmp/stg_shopify__product_variant_tmp.sql +++ b/models/tmp/stg_shopify__product_variant_tmp.sql @@ -1,2 +1,10 @@ -select * -from {{ var('product_variant_source') }} \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='product_variant', + database_variable='shopify_database', + schema_variable='shopify_schema', + default_database=target.database, + default_schema='shopify', + default_variable='product_variant_source' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_shopify__refund_tmp.sql b/models/tmp/stg_shopify__refund_tmp.sql index ac2559d..8e7e631 100644 --- a/models/tmp/stg_shopify__refund_tmp.sql +++ b/models/tmp/stg_shopify__refund_tmp.sql @@ -1 +1,10 @@ -select * from {{ var('refund_source') }} +{{ + fivetran_utils.union_data( + table_identifier='refund', + database_variable='shopify_database', + schema_variable='shopify_schema', + default_database=target.database, + default_schema='shopify', + default_variable='refund_source' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_shopify__transaction_tmp.sql b/models/tmp/stg_shopify__transaction_tmp.sql index 3e1abb8..659c81a 100644 --- a/models/tmp/stg_shopify__transaction_tmp.sql +++ b/models/tmp/stg_shopify__transaction_tmp.sql @@ -1,2 +1,10 @@ -select * -from {{ var('transaction_source') }} +{{ + fivetran_utils.union_data( + table_identifier='transaction', + database_variable='shopify_database', + schema_variable='shopify_schema', + default_database=target.database, + default_schema='shopify', + default_variable='transaction_source' + ) +}} \ No newline at end of file