salesforce dlt-dbt v1.5

.gitignore

@@ -1,2 +1,10 @@
-logs/
-target/
+# ignore secrets, virtual environments and typical python compilation artifacts
+secrets.toml
+# ignore basic python artifacts
+.env
+**/__pycache__/
+**/*.py[cod]
+**/*$py.class
+# ignore duckdb
+*.duckdb
+*.wal

README.md

@@ -108,13 +108,13 @@ for data model which can be further modified as required.
 
 1. The schema of data modelled above using dlt-dbt-generator:
 
-   ![picture](https://storage.googleapis.com/dlt-blog-images/Salesforce%20dlt-dbt%20package.png)
+   ![picture](https://storage.googleapis.com/dlt-blog-images/salesforce_dlt_dbt_dim_model.png)
 
 > ⚠️ **Note:**
 > 
 > Please note that this is a starting template for your data model and is not the final product. It is advised to customize the data model as per your needs.
 
-   Here's the link to the DB diagram: [link](https://dbdiagram.io/d/Salesforce-dlt-dbt-package-6714f07197a66db9a39a7419).
+   Here's the link to the DB diagram: [link](https://dbdiagram.io/d/salesforce_dlt_dbt_v1-5-67486894e9daa85aca057fb1).
 
 #### Optional: Advanced Usage (Generator and Licensing)
 

dbt_project.yml

@@ -1,29 +1,29 @@
-name: 'salesforce'
-
-config-version: 2
-version: '0.1'
-
-profile: 'salesforce'
-
-model-paths: ["models"]
-test-paths: ["tests"]
-analysis-paths: ["analysis"]
-macro-paths: ["macros"]
-
-target-path: "target"
-clean-targets:
-    - "target"
-    - "dbt_modules"
-    - "logs"
-
-require-dbt-version: [">=1.0.0", "<2.0.0"]
-
-models:
-  salesforce_tst:
-    materialized: table
-    staging:
-      materialized: view
-      +docs:
-        node_color: 'silver'
-    +docs:
+name: 'salesforce'
+
+config-version: 2
+version: '0.1'
+
+profile: 'salesforce'
+
+model-paths: ["models"]
+test-paths: ["tests"]
+analysis-paths: ["analysis"]
+macro-paths: ["macros"]
+
+target-path: "target"
+clean-targets:
+    - "target"
+    - "dbt_modules"
+    - "logs"
+
+require-dbt-version: [">=1.0.0", "<2.0.0"]
+
+models:
+  salesforce:
+    materialized: table
+    staging:
+      materialized: view
+      +docs:
+        node_color: 'silver'
+    +docs:
       node_color: 'gold'

models/dlt_active_load_ids.sql

@@ -1,13 +1,13 @@
-/* check if the destination already has a list of processed ids
-if not we will process all loads with status 0 (==success) */
-{% set active_ids_exist = adapter.get_relation(
-        database=this.database ,
-        schema=this.schema,
-        identifier="dlt_processed_load_ids" ) %}
-
-SELECT load_id FROM {{  source('raw_data', '_dlt_loads') }}
-WHERE status = 0
-/* exclude already processed load_ids */
-{% if active_ids_exist and not should_full_refresh() %}
-AND load_id NOT IN (SELECT load_id FROM {{ source('transformed_data', 'dlt_processed_load_ids') }})
+/* check if the destination already has a list of processed ids
+if not we will process all loads with status 0 (==success) */
+{% set active_ids_exist = adapter.get_relation(
+        database=this.database ,
+        schema=this.schema,
+        identifier="dlt_processed_load_ids" ) %}
+
+SELECT load_id FROM {{  source('raw_data', '_dlt_loads') }}
+WHERE status = 0
+/* exclude already processed load_ids */
+{% if active_ids_exist and not should_full_refresh() %}
+AND load_id NOT IN (SELECT load_id FROM {{ source('transformed_data', 'dlt_processed_load_ids') }})
 {% endif %}

models/dlt_processed_load_ids.sql

@@ -1,23 +1,23 @@
-{{
-    config(
-        materialized='incremental'
-    )
-}}
--- depends_on: {{ ref('dim__dlt_loads') }}
--- depends_on: {{ ref('fact_event') }}
--- depends_on: {{ ref('fact_task') }}
--- depends_on: {{ ref('dim_pricebook_entry') }}
--- depends_on: {{ ref('dim_pricebook_2') }}
--- depends_on: {{ ref('dim_product_2') }}
--- depends_on: {{ ref('dim_campaign_member') }}
--- depends_on: {{ ref('dim_campaign') }}
--- depends_on: {{ ref('dim_lead') }}
--- depends_on: {{ ref('dim_contact') }}
--- depends_on: {{ ref('dim_account') }}
--- depends_on: {{ ref('dim_opportunity_contact_role') }}
--- depends_on: {{ ref('dim_opportunity_line_item') }}
--- depends_on: {{ ref('fact_opportunity') }}
--- depends_on: {{ ref('dim_user_role') }}
--- depends_on: {{ ref('dim_sf_user') }}
-/* we save all currently active load ids into the processed ids table */
+{{
+    config(
+        materialized='incremental'
+    )
+}}
+-- depends_on: {{ ref('dim__dlt_loads') }}
+-- depends_on: {{ ref('dim_event') }}
+-- depends_on: {{ ref('dim_task') }}
+-- depends_on: {{ ref('dim_pricebook_entry') }}
+-- depends_on: {{ ref('dim_pricebook_2') }}
+-- depends_on: {{ ref('dim_product_2') }}
+-- depends_on: {{ ref('dim_campaign_member') }}
+-- depends_on: {{ ref('dim_campaign') }}
+-- depends_on: {{ ref('dim_lead') }}
+-- depends_on: {{ ref('dim_contact') }}
+-- depends_on: {{ ref('dim_account') }}
+-- depends_on: {{ ref('dim_opportunity_contact_role') }}
+-- depends_on: {{ ref('dim_opportunity_line_item') }}
+-- depends_on: {{ ref('dim_opportunity') }}
+-- depends_on: {{ ref('dim_user_role') }}
+-- depends_on: {{ ref('dim_sf_user') }}
+/* we save all currently active load ids into the processed ids table */
 select load_id, {{ current_timestamp() }} as inserted_at FROM {{ ref('dlt_active_load_ids') }}

models/marts/dim__dlt_loads.sql

@@ -1,14 +1,14 @@
-/* Table: _dlt_loads */
-{{
-    config(
-        materialized='incremental'
-    )
-}}
-/* Description: Created by DLT. Tracks completed loads */
-SELECT
-    t.load_id,
-    t.schema_name,
-    t.status,
-    t.inserted_at,
-    t.schema_version_hash,
+/* Table: _dlt_loads */
+{{
+    config(
+        materialized='incremental'
+    )
+}}
+/* Description: Created by DLT. Tracks completed loads */
+SELECT
+    t.load_id,
+    t.schema_name,
+    t.status,
+    t.inserted_at,
+    t.schema_version_hash,
 FROM  {{ ref('stg__dlt_loads') }} as t

models/marts/dim_account.sql

@@ -1,56 +1,56 @@
-/* Table: account */
-{{
-    config(
-        materialized='incremental'
-    )
-}}
-SELECT
-    t.id,
-    t.is_deleted,
-    t.name,
-    t.type,
-    t.billing_street,
-    t.billing_city,
-    t.billing_state,
-    t.shipping_street,
-    t.phone,
-    t.fax,
-    t.account_number,
-    t.website,
-    t.photo_url,
-    t.sic,
-    t.industry,
-    t.annual_revenue,
-    t.number_of_employees,
-    t.ownership,
-    t.ticker_symbol,
-    t.description,
-    t.rating,
-    t.owner_id,
-    t.created_date,
-    t.created_by_id,
-    t.last_modified_date,
-    t.last_modified_by_id,
-    t.system_modstamp,
-    t.clean_status,
-    t.customer_priority_c,
-    t.sla_c,
-    t.active_c,
-    t.numberof_locations_c,
-    t.upsell_opportunity_c,
-    t.sla_serial_number_c,
-    t.sla_expiration_date_c,
-    t._dlt_load_id,
-    t._dlt_id,
-    t.billing_postal_code,
-    t.billing_country,
-    t.shipping_city,
-    t.shipping_postal_code,
-    t.shipping_country,
-    t.shipping_state,
-    t.site,
-    t.last_activity_date,
-    t.last_viewed_date,
-    t.last_referenced_date,
-    t.parent_id,
+/* Table: account */
+{{
+    config(
+        materialized='incremental'
+    )
+}}
+SELECT
+    t.id,
+    t.is_deleted,
+    t.name,
+    t.type,
+    t.billing_street,
+    t.billing_city,
+    t.billing_state,
+    t.shipping_street,
+    t.phone,
+    t.fax,
+    t.account_number,
+    t.website,
+    t.photo_url,
+    t.sic,
+    t.industry,
+    t.annual_revenue,
+    t.number_of_employees,
+    t.ownership,
+    t.ticker_symbol,
+    t.description,
+    t.rating,
+    t.owner_id,
+    t.created_date,
+    t.created_by_id,
+    t.last_modified_date,
+    t.last_modified_by_id,
+    t.system_modstamp,
+    t.clean_status,
+    t.customer_priority__c,
+    t.sla__c,
+    t.active__c,
+    t.numberof_locations__c,
+    t.upsell_opportunity__c,
+    t.sla_serial_number__c,
+    t.sla_expiration_date__c,
+    t._dlt_load_id,
+    t._dlt_id,
+    t.billing_postal_code,
+    t.billing_country,
+    t.shipping_city,
+    t.shipping_postal_code,
+    t.shipping_country,
+    t.shipping_state,
+    t.site,
+    t.last_activity_date,
+    t.last_viewed_date,
+    t.last_referenced_date,
+    t.parent_id,
 FROM  {{ ref('stg_account') }} as t

models/marts/dim_campaign.sql

@@ -1,39 +1,39 @@
-/* Table: campaign */
-{{
-    config(
-        materialized='incremental'
-    )
-}}
-SELECT
-    t.id,
-    t.is_deleted,
-    t.name,
-    t.type,
-    t.status,
-    t.start_date,
-    t.end_date,
-    t.expected_revenue,
-    t.budgeted_cost,
-    t.actual_cost,
-    t.expected_response,
-    t.number_sent,
-    t.is_active,
-    t.number_of_leads,
-    t.number_of_converted_leads,
-    t.number_of_contacts,
-    t.number_of_responses,
-    t.number_of_opportunities,
-    t.number_of_won_opportunities,
-    t.amount_all_opportunities,
-    t.amount_won_opportunities,
-    t.owner_id,
-    t.created_date,
-    t.created_by_id,
-    t.last_modified_date,
-    t.last_modified_by_id,
-    t.system_modstamp,
-    t._dlt_load_id,
-    t._dlt_id,
-    t.last_viewed_date,
-    t.last_referenced_date,
+/* Table: campaign */
+{{
+    config(
+        materialized='incremental'
+    )
+}}
+SELECT
+    t.id,
+    t.is_deleted,
+    t.name,
+    t.type,
+    t.status,
+    t.start_date,
+    t.end_date,
+    t.expected_revenue,
+    t.budgeted_cost,
+    t.actual_cost,
+    t.expected_response,
+    t.number_sent,
+    t.is_active,
+    t.number_of_leads,
+    t.number_of_converted_leads,
+    t.number_of_contacts,
+    t.number_of_responses,
+    t.number_of_opportunities,
+    t.number_of_won_opportunities,
+    t.amount_all_opportunities,
+    t.amount_won_opportunities,
+    t.owner_id,
+    t.created_date,
+    t.created_by_id,
+    t.last_modified_date,
+    t.last_modified_by_id,
+    t.system_modstamp,
+    t._dlt_load_id,
+    t._dlt_id,
+    t.last_viewed_date,
+    t.last_referenced_date,
 FROM  {{ ref('stg_campaign') }} as t