Merge pull request #29 from fivetran/feature/credit-card-payments

Feature/credit card payments

.github/workflows/check_docs.yml

@@ -0,0 +1,35 @@
+name: 'check docs'
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  changed-files:
+    runs-on: ubuntu-latest
+    name: test changed-files
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v29.0.4
+        with:
+          files: docs/**
+
+      - name: Check to see if docs folder hasn't changed
+        if: steps.changed-files.outputs.any_changed == 'false'
+        run: |
+          echo "Docs have not been regenerated."
+          exit 1
+      
+      - name: Check if docs folder has changed
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: |
+          echo "Docs have been regenerated!"
+          exit 0 

.gitignore

@@ -2,5 +2,4 @@
 target/
 dbt_modules/
 logs/
-
 dbt_packages/

CHANGELOG.md

@@ -1,3 +1,10 @@
+# dbt_quickbooks_source v0.5.1
+## Features
+- Addition of the `credit_card_payment_txn` (enabled/disabled using the `using_credit_card_payment_txn` variable) source as well as the accompanying staging models. This source includes all credit card payment transactions and will be used in downstream General Ledger generation to ensure accurate reporting of all transaction types. ([#29](https://github.com/fivetran/dbt_quickbooks_source/pull/29))
+  >**Note**: the `credit_card_payment_txn` source and models are disabled by default. In order to enabled them, you will want to set the `using_credit_card_payment_txn` variable to `true` in your dbt_project.yml.
+
+## Under the Hood
+- A GitHub workflow has been added to ensure the dbt docs are regenerated before each merge to the `main` release branch. 
 # dbt_quickbooks_source v0.5.0
 ## 🚨 Breaking Changes 🚨
 - It was discovered that IDs from the source tables can sometimes be strings. The previous build of the package interpreted all IDs as integers. To ensure the package operates as intended, the package has been updated to cast all IDs to the string datatype. If you were leveraging the end models in downstream analysis, this change could break your join conditions. Be sure to be aware of any join conditions you may have downstream before upgrading your QuickBooks package. ([#36](https://github.com/fivetran/dbt_quickbooks/pull/36)) ([#25](https://github.com/fivetran/dbt_quickbooks_source/pull/25)) ([#24](https://github.com/fivetran/dbt_quickbooks_source/pull/24))

README.md

@@ -56,7 +56,7 @@ models:
 ```
 ### Disabling models
 
-This package takes into consideration that not every QuickBooks account utilizes the same transactional tables, and allows you to disable the corresponding functionality. By default, most variables' values are assumed to be `true` (with exception of purchase orders). Add variables for only the tables you want to disable or enable respectively:
+This package takes into consideration that not every QuickBooks account utilizes the same transactional tables, and allows you to disable the corresponding functionality. By default, most variables' values are assumed to be `true` (with exception of purchase orders and credit card payments). Add variables for only the tables you want to disable or enable respectively:
 
 ```yml
 # dbt_project.yml
@@ -79,6 +79,7 @@ vars:
     using_vendor_credit:  false         #disable if you don't have vendor credits in QuickBooks
     using_sales_receipt:  false         #disable if you don't have sales receipts in QuickBooks
     using_purchase_order: true          #enable if you want to include purchase orders in your staging models
+    using_credit_card_payment_txn: true #enable if you want to include credit card payment transactions in your staging models
 ```
 
 ## Contributions

dbt_project.yml

@@ -19,6 +19,7 @@ vars:
     bill: "{{ source('quickbooks','bill') }}"
     bundle_item: "{{ source('quickbooks','bundle_item') }}"
     bundle: "{{ source('quickbooks','bundle') }}"
+    credit_card_payment_txn: "{{ source('quickbooks','credit_card_payment_txn') }}"
     credit_memo_line: "{{ source('quickbooks','credit_memo_line') }}"
     credit_memo: "{{ source('quickbooks','credit_memo') }}"
     customer: "{{ source('quickbooks','customer') }}"

integration_tests/.gitignore

@@ -1,3 +1,4 @@
 target/
 dbt_modules/
+dbt_packages/
 logs/

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: <PLEASE INSERT INTEGRATION TESTS SCHEMA NAMES>
+      schema: quickbooks_source_integrations_tests
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: <PLEASE INSERT INTEGRATION TESTS SCHEMA NAMES>
+      schema: quickbooks_source_integrations_tests
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: <PLEASE INSERT INTEGRATION TESTS SCHEMA NAMES>
+      schema: quickbooks_source_integrations_tests
       threads: 8
     databricks:
       catalog: null
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: <PLEASE INSERT INTEGRATION TESTS SCHEMA NAMES>
+      schema: quickbooks_source_integrations_tests
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
-      type: databricks
+      type: databricks

integration_tests/dbt_project.yml

@@ -1,5 +1,6 @@
 name: 'quickbooks_source_integration_tests'
 version: '0.6.0'
+
 profile: 'integration_tests'
 config-version: 2
 models:
@@ -8,47 +9,51 @@ models:
     tmp:
       materialized: view
 vars:
+  quickbooks_schema: quickbooks_source_integrations_tests
   quickbooks_source:
-    using_purchase_order: true
-    account: "{{ ref('account_data') }}"
-    address: "{{ ref('address_data') }}"
-    bill_line: "{{ ref('bill_line_data') }}"
-    bill_linked_txn: "{{ ref('bill_linked_txn_data') }}"
-    bill_payment_line: "{{ ref('bill_payment_line_data') }}"
-    bill_payment: "{{ ref('bill_payment_data') }}"
-    bill: "{{ ref('bill_data') }}"
-    bundle_item: "{{ ref('bundle_item_data') }}"
-    bundle: "{{ ref('bundle_data') }}"
-    credit_memo: "{{ ref('credit_memo_data') }}"
-    credit_memo_line: "{{ ref('credit_memo_line_data') }}"
-    customer: "{{ ref('customer_data') }}"
-    department: "{{ ref('department_data') }}"
-    deposit_line: "{{ ref('deposit_line_data') }}"
-    deposit: "{{ ref('deposit_data') }}"
-    estimate: "{{ ref('estimate_data') }}"
-    estimate_line: "{{ ref('estimate_line_data') }}"
-    invoice: "{{ ref('invoice_data') }}"
-    invoice_line: "{{ ref('invoice_line_data') }}"
-    invoice_line_bundle: "{{ ref('invoice_line_bundle_data') }}"
-    invoice_linked_txn: "{{ ref('invoice_linked_txn_data') }}"
-    item: "{{ ref('item_data') }}"
-    journal_entry_line: "{{ ref('journal_entry_line_data') }}"
-    journal_entry: "{{ ref('journal_entry_data') }}"
-    payment: "{{ ref('payment_data') }}"
-    payment_line: "{{ ref('payment_line_data') }}"
-    purchase_line: "{{ ref('purchase_line_data') }}"
-    purchase_order: "{{ ref('purchase_order_data') }}"
-    purchase_order_line: "{{ ref('purchase_order_line_data') }}"
-    purchase_order_linked_txn: "{{ ref('purchase_order_linked_txn_data') }}"
-    purchase: "{{ ref('purchase_data') }}"
-    refund_receipt_line: "{{ ref('refund_receipt_line_data') }}"
-    refund_receipt: "{{ ref('refund_receipt_data') }}"
-    sales_receipt_line: "{{ ref('sales_receipt_line_data') }}"
-    sales_receipt: "{{ ref('sales_receipt_data') }}"
-    transfer: "{{ ref('transfer_data') }}"
-    vendor_credit_line: "{{ ref('vendor_credit_line_data') }}"
-    vendor_credit: "{{ ref('vendor_credit_data') }}"
-    vendor: "{{ ref('vendor_data') }}"
+    using_purchase_order: True
+    using_credit_card_payment_txn: True
+    account:                    "{{ ref('account_data') }}"
+    address:                    "{{ ref('address_data') }}"
+    bill_line:                  "{{ ref('bill_line_data') }}"
+    bill_linked_txn:            "{{ ref('bill_linked_txn_data') }}"
+    bill_payment_line:          "{{ ref('bill_payment_line_data') }}"
+    bill_payment:               "{{ ref('bill_payment_data') }}"
+    bill:                       "{{ ref('bill_data') }}"
+    bundle_item:                "{{ ref('bundle_item_data') }}"
+    bundle:                     "{{ ref('bundle_data') }}"
+    credit_card_payment_txn:    "{{ ref('credit_card_payment_txn_data') }}"
+    credit_memo:                "{{ ref('credit_memo_data') }}"
+    credit_memo_line:           "{{ ref('credit_memo_line_data') }}"
+    customer:                   "{{ ref('customer_data') }}"
+    department:                 "{{ ref('department_data') }}"
+    deposit_line:               "{{ ref('deposit_line_data') }}"
+    deposit:                    "{{ ref('deposit_data') }}"
+    estimate:                   "{{ ref('estimate_data') }}"
+    estimate_line:              "{{ ref('estimate_line_data') }}"
+    invoice:                    "{{ ref('invoice_data') }}"
+    invoice_line:               "{{ ref('invoice_line_data') }}"
+    invoice_line_bundle:        "{{ ref('invoice_line_bundle_data') }}"
+    invoice_linked_txn:         "{{ ref('invoice_linked_txn_data') }}"
+    item:                       "{{ ref('item_data') }}"
+    journal_entry_line:         "{{ ref('journal_entry_line_data') }}"
+    journal_entry:              "{{ ref('journal_entry_data') }}"
+    payment:                    "{{ ref('payment_data') }}"
+    payment_line:               "{{ ref('payment_line_data') }}"
+    purchase_line:              "{{ ref('purchase_line_data') }}"
+    purchase_order:             "{{ ref('purchase_order_data') }}"
+    purchase_order_line:        "{{ ref('purchase_order_line_data') }}"
+    purchase_order_linked_txn:  "{{ ref('purchase_order_linked_txn_data') }}"
+    purchase:                   "{{ ref('purchase_data') }}"
+    refund_receipt_line:        "{{ ref('refund_receipt_line_data') }}"
+    refund_receipt:             "{{ ref('refund_receipt_data') }}"
+    sales_receipt_line:         "{{ ref('sales_receipt_line_data') }}"
+    sales_receipt:              "{{ ref('sales_receipt_data') }}"
+    transfer:                   "{{ ref('transfer_data') }}"    
+    vendor_credit_line:         "{{ ref('vendor_credit_line_data') }}"
+    vendor_credit:              "{{ ref('vendor_credit_data') }}"
+    vendor:                     "{{ ref('vendor_data') }}"
+
 seeds:
   +quote_columns: "{{ true if target.type == 'redshift' else false }}"
   quickbooks_source_integration_tests:

integration_tests/seeds/credit_card_payment_txn_data.csv

@@ -0,0 +1,6 @@
+id,amount,bank_account_id,credit_card_account_id,created_at,updated_at,currency_id,transaction_date,_fivetran_deleted
+1111,5,5,7,2022-07-21 09:57:41,2022-07-21 09:57:41,USD,2022-07-21,FALSE
+2222,10.02,5,7,2020-05-24 19:39:23,2020-05-24 19:39:23,USD,2020-05-19,FALSE
+3333,13.52,5,7,2022-09-06 12:12:07,2022-09-06 12:12:07,USD,2022-09-04,FALSE
+4444,65.22,5,7,2022-01-05 22:22:11,2022-01-05 22:22:11,USD,2022-01-04,FALSE
+5555,94.1,5,7,2022-07-09 21:15:24,2022-07-09 21:15:24,USD,2022-06-18,FALSE

macros/get_credit_card_payment_txn_columns.sql

@@ -0,0 +1,20 @@
+{% macro get_credit_card_payment_txn_columns() %}
+
+{% set columns = [    	
+    {"name": "id", "datatype": dbt_utils.type_string()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
+    {"name": "_fivetran_synced", "datatype": dbt_utils.type_timestamp()},
+    {"name": "amount", "datatype": dbt_utils.type_float()},
+    {"name": "bank_account_id", "datatype": dbt_utils.type_string()},
+    {"name": "created_at", "datatype": dbt_utils.type_timestamp()},
+    {"name": "credit_card_account_id", "datatype": dbt_utils.type_string()},
+    {"name": "currency_id", "datatype": dbt_utils.type_string()},
+    {"name": "sync_token", "datatype": dbt_utils.type_string()},
+    {"name": "transaction_date", "datatype": "date"},
+    {"name": "updated_at", "datatype": dbt_utils.type_timestamp()}
+
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}

models/src_quickbooks.yml

@@ -208,6 +208,28 @@ sources:
           - name: updated_at
             description: Timestamp of the date when the bundle was last updated.
 
+      - name: credit_card_payment_txn
+        description: Table containing all credit card payment transactions.
+        columns:
+          - name: id
+            description: Unique identifier of the credit card payment transaction.
+          - name: amount
+            description: Total amount of money applied to the credit card payment transaction.
+          - name: bank_account_id
+            description: Reference to the bank account which the credit card payment is debited towards.
+          - name: credit_card_account_id
+            description: Reference to the credit card account which the credit card payment is credited towards.
+          - name: created_at
+            description: Timestamp when the credit card payment was created.
+          - name: updated_at
+            description: Timestamp when the credit card payment was last updated.
+          - name: currency_id
+            description: Reference to the currency used for the credit card payment.
+          - name: transaction_date
+            description: Date the credit card payment transaction took place.
+          - name: _fivetran_deleted
+            description: Boolean indicating if the credit card payment was deleted.
+
       - name: credit_memo_line
         description: Table containing individual records of credit memos.
         columns:

models/stg_quickbooks.yml

@@ -226,6 +226,30 @@ models:
       - name: updated_at
         description: Timestamp of the date when the bundle was last updated.
 
+  - name: stg_quickbooks__credit_card_payment_txn
+    description: Table containing all credit card payment transactions.
+    columns:
+      - name: credit_card_payment_id
+        description: Unique identifier of the credit card payment transaction.
+      - name: amount
+        description: Total amount of money applied to the credit card payment transaction.
+      - name: bank_account_id
+        description: Reference to the bank account which the credit card payment is debited towards.
+      - name: credit_card_account_id
+        description: Reference to the credit card account which the credit card payment is credited towards.
+      - name: created_at
+        description: Timestamp when the credit card payment was created.
+      - name: updated_at
+        description: Timestamp when the credit card payment was last updated.
+      - name: currency_id
+        description: Reference to the currency used for the credit card payment.
+      - name: transaction_date
+        description: Date the credit card payment transaction took place.
+      - name: _fivetran_deleted
+        description: Boolean indicating if the credit card payment was deleted.
+      - name: is_most_recent_record
+        description: Boolean indicating if the record is the most recent updated version of the credit card payment transaction.
+
   - name: stg_quickbooks__credit_memo_line
     description: Table containing individual records of credit memos.
     columns:

models/stg_quickbooks__credit_card_payment_txn.sql

@@ -0,0 +1,40 @@
+--To enable this model, set the using_invoice_bundle variable within your dbt_project.yml file to True.
+{{ config(enabled=var('using_credit_card_payment_txn', False)) }}
+
+with base as (
+
+    select * 
+    from {{ ref('stg_quickbooks__credit_card_payment_txn_tmp') }}
+),
+
+fields as (
+
+    select
+        {{
+            fivetran_utils.fill_staging_columns(
+                source_columns=adapter.get_columns_in_relation(ref('stg_quickbooks__credit_card_payment_txn_tmp')),
+                staging_columns=get_credit_card_payment_txn_columns()
+            )
+        }}
+    from base
+),
+
+final as (
+
+    select 
+        cast(id as {{ dbt_utils.type_string() }}) as credit_card_payment_id,
+        amount,
+        bank_account_id,
+        credit_card_account_id,
+        created_at,
+        updated_at,
+        currency_id,
+        transaction_date,
+        _fivetran_deleted,
+        row_number() over (partition by id, updated_at order by updated_at desc) = 1 as is_most_recent_record
+    from fields
+)
+
+select * 
+from final
+where not coalesce(_fivetran_deleted, false)

models/tmp/stg_quickbooks__credit_card_payment_txn_tmp.sql

@@ -0,0 +1,5 @@
+--To enable this model, set the using_credit_card_payment_txn variable within your dbt_project.yml file to True.
+{{ config(enabled=var('using_credit_card_payment_txn', False)) }}
+
+select * 
+from {{ var('credit_card_payment_txn') }}