Skip to content

Commit

Permalink
Remove cross-db dbt_utils references (#650)
Browse files Browse the repository at this point in the history
* First pass at removing cross-db dbt_utils references

* Update docs

* Update README.md
  • Loading branch information
clausherther authored Aug 28, 2022
1 parent 7351d1e commit fc8b083
Show file tree
Hide file tree
Showing 17 changed files with 341 additions and 168 deletions.
379 changes: 276 additions & 103 deletions README.md

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
{#
This macro is copied and slightly edited from the dbt_expectations package.
At the time of this addition, dbt_expectations couldn't be added because
integration_tests is installing dbt_utils from local without a hard-coded
integration_tests is installing dbt_utils from local without a hard-coded
path. dbt is not able to resolve duplicate dependencies of dbt_utils
due to this.
#}
{%- test expect_table_columns_to_match_set(model, column_list, transform="upper") -%}
{%- if execute -%}
{%- set column_list = column_list | map(transform) | list -%}
{# Replaces dbt_expectations._get_column_list() #}
{%- set relation_column_names = adapter.get_columns_in_relation(model)
| map(attribute="name")
| map(transform)
| list
{# Replaces dbt_expectations._get_column_list() #}
{%- set relation_column_names = adapter.get_columns_in_relation(model)
| map(attribute="name")
| map(transform)
| list
-%}
{# Replaces dbt_expectations._list_intersect() #}
Expand All @@ -28,14 +28,14 @@
with relation_columns as (
{% for col_name in relation_column_names %}
select cast('{{ col_name }}' as {{ dbt_utils.type_string() }}) as relation_column
select cast('{{ col_name }}' as {{ type_string() }}) as relation_column
{% if not loop.last %}union all{% endif %}
{% endfor %}
),
input_columns as (
{% for col_name in column_list %}
select cast('{{ col_name }}' as {{ dbt_utils.type_string() }}) as input_column
select cast('{{ col_name }}' as {{ type_string() }}) as input_column
{% if not loop.last %}union all{% endif %}
{% endfor %}
)
Expand All @@ -51,4 +51,4 @@
i.input_column is null
{%- endif -%}
{%- endtest -%}
{%- endtest -%}
10 changes: 5 additions & 5 deletions macros/cross_db_utils/cast_array_to_string.sql
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,20 @@
{% endmacro %}

{% macro default__cast_array_to_string(array) %}
cast({{ array }} as {{ dbt_utils.type_string() }})
cast({{ array }} as {{ type_string() }})
{% endmacro %}

{# when casting as array to string, postgres uses {} (ex: {1,2,3}) while other dbs use [] (ex: [1,2,3]) #}
{% macro postgres__cast_array_to_string(array) %}
{%- set array_as_string -%}cast({{ array }} as {{ dbt_utils.type_string() }}){%- endset -%}
{{ dbt_utils.replace(dbt_utils.replace(array_as_string,"'}'","']'"),"'{'","'['") }}
{%- set array_as_string -%}cast({{ array }} as {{ type_string() }}){%- endset -%}
{{ replace(replace(array_as_string,"'}'","']'"),"'{'","'['") }}
{% endmacro %}

{# redshift should use default instead of postgres #}
{% macro redshift__cast_array_to_string(array) %}
cast({{ array }} as {{ dbt_utils.type_string() }})
cast({{ array }} as {{ type_string() }})
{% endmacro %}

{% macro bigquery__cast_array_to_string(array) %}
'['||(select string_agg(cast(element as string), ',') from unnest({{ array }}) element)||']'
{% endmacro %}
{% endmacro %}
8 changes: 4 additions & 4 deletions macros/cross_db_utils/current_timestamp.sql
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
{%- endmacro %}

{% macro default__current_timestamp() %}
current_timestamp::{{dbt_utils.type_timestamp()}}
current_timestamp::{{ type_timestamp() }}
{% endmacro %}

{% macro redshift__current_timestamp() %}
Expand All @@ -23,15 +23,15 @@
{%- endmacro %}

{% macro default__current_timestamp_in_utc() %}
{{dbt_utils.current_timestamp()}}
{{ dbt_utils.current_timestamp() }}
{% endmacro %}

{% macro snowflake__current_timestamp_in_utc() %}
convert_timezone('UTC', {{dbt_utils.current_timestamp()}})::{{dbt_utils.type_timestamp()}}
convert_timezone('UTC', {{ dbt_utils.current_timestamp() }})::{{ type_timestamp() }}
{% endmacro %}

{% macro postgres__current_timestamp_in_utc() %}
(current_timestamp at time zone 'utc')::{{dbt_utils.type_timestamp()}}
(current_timestamp at time zone 'utc')::{{ type_timestamp() }}
{% endmacro %}

{# redshift should use default instead of postgres #}
Expand Down
4 changes: 2 additions & 2 deletions macros/generic_tests/cardinality_equality.sql
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,15 @@ group by {{ field }}
except_a as (
select *
from table_a
{{ dbt_utils.except() }}
{{ except() }}
select *
from table_b
),

except_b as (
select *
from table_b
{{ dbt_utils.except() }}
{{ except() }}
select *
from table_a
),
Expand Down
4 changes: 2 additions & 2 deletions macros/generic_tests/equality.sql
Original file line number Diff line number Diff line change
Expand Up @@ -49,15 +49,15 @@ b as (
a_minus_b as (

select {{compare_cols_csv}} from a
{{ dbt_utils.except() }}
{{ except() }}
select {{compare_cols_csv}} from b

),

b_minus_a as (

select {{compare_cols_csv}} from b
{{ dbt_utils.except() }}
{{ except() }}
select {{compare_cols_csv}} from a

),
Expand Down
2 changes: 1 addition & 1 deletion macros/generic_tests/recency.sql
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

{% macro default__test_recency(model, field, datepart, interval) %}

{% set threshold = dbt_utils.dateadd(datepart, interval * -1, dbt_utils.current_timestamp()) %}
{% set threshold = dateadd(datepart, interval * -1, current_timestamp()) %}

with recency as (

Expand Down
2 changes: 1 addition & 1 deletion macros/generic_tests/sequential_values.sql
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ validation_errors as (
*
from windowed
{% if datepart %}
where not(cast({{ column_name }} as {{ dbt_utils.type_timestamp() }})= cast({{ dbt_utils.dateadd(datepart, interval, previous_column_name) }} as {{ dbt_utils.type_timestamp() }}))
where not(cast({{ column_name }} as {{ type_timestamp() }})= cast({{ dateadd(datepart, interval, previous_column_name) }} as {{ type_timestamp() }}))
{% else %}
where not({{ column_name }} = {{ previous_column_name }} + {{ interval }})
{% endif %}
Expand Down
16 changes: 8 additions & 8 deletions macros/materializations/insert_by_period_materialization.sql
Original file line number Diff line number Diff line change
Expand Up @@ -9,20 +9,20 @@
select
coalesce(max("{{timestamp_field}}"), '{{start_date}}')::timestamp as start_timestamp,
coalesce(
{{dbt_utils.dateadd('millisecond',
{{ dateadd('millisecond',
-1,
"nullif('" ~ stop_date ~ "','')::timestamp")}},
{{dbt_utils.current_timestamp()}}
"nullif('" ~ stop_date ~ "','')::timestamp") }},
{{ dbt_utils.current_timestamp() }}
) as stop_timestamp
from "{{target_schema}}"."{{target_table}}"
)

select
start_timestamp,
stop_timestamp,
{{dbt_utils.datediff('start_timestamp',
{{ datediff('start_timestamp',
'stop_timestamp',
period)}} + 1 as num_periods
period) }} + 1 as num_periods
from data
{%- endcall %}

Expand Down Expand Up @@ -55,7 +55,7 @@
{%- set start_date = config.require('start_date') -%}
{%- set stop_date = config.get('stop_date') or '' -%}
{%- set period = config.get('period') or 'week' -%}

{%- set deprecation_warning = "Warning: the `insert_by_period` materialization will be removed from dbt_utils in version 1.0.0. Install from dbt-labs/dbt-labs-experimental-features instead (see https://github.com/dbt-labs/dbt-utils/discussions/487). The " ~ package ~ "." ~ model ~ " model triggered this warning." -%}
{%- do exceptions.warn(deprecation_warning) -%}

Expand Down Expand Up @@ -159,7 +159,7 @@
{% else %} {# older versions #}
{% set rows_inserted = result['status'].split(" ")[2] | int %}
{% endif %}

{%- set sum_rows_inserted = loop_vars['sum_rows_inserted'] + rows_inserted -%}
{%- if loop_vars.update({'sum_rows_inserted': sum_rows_inserted}) %} {% endif -%}

Expand Down Expand Up @@ -187,6 +187,6 @@
{%- endcall %}

-- Return the relations created in this materialization
{{ return({'relations': [target_relation]}) }}
{{ return({'relations': [target_relation]}) }}

{%- endmaterialization %}
4 changes: 2 additions & 2 deletions macros/sql/date_spine.sql
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
{% macro default__get_intervals_between(start_date, end_date, datepart) -%}
{%- call statement('get_intervals_between', fetch_result=True) %}

select {{dbt_utils.datediff(start_date, end_date, datepart)}}
select {{ datediff(start_date, end_date, datepart) }}

{%- endcall -%}

Expand Down Expand Up @@ -51,7 +51,7 @@ all_periods as (

select (
{{
dbt_utils.dateadd(
dateadd(
datepart,
"row_number() over (order by 1) - 1",
start_date
Expand Down
4 changes: 2 additions & 2 deletions macros/sql/surrogate_key.sql
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ deprecated in a future release of dbt-utils. The {}.{} model triggered this warn
{%- for field in field_list_xf -%}

{%- set _ = fields.append(
"coalesce(cast(" ~ field ~ " as " ~ dbt_utils.type_string() ~ "), '')"
"coalesce(cast(" ~ field ~ " as " ~ type_string() ~ "), '')"
) -%}

{%- if not loop.last %}
Expand All @@ -46,6 +46,6 @@ deprecated in a future release of dbt-utils. The {}.{} model triggered this warn

{%- endfor -%}

{{dbt_utils.hash(dbt_utils.concat(fields))}}
{{ hash(concat(fields)) }}

{%- endmacro -%}
2 changes: 1 addition & 1 deletion macros/sql/union.sql
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@
select

{%- if source_column_name != none %}
cast({{ dbt_utils.string_literal(relation) }} as {{ dbt_utils.type_string() }}) as {{ source_column_name }},
cast({{ dbt_utils.string_literal(relation) }} as {{ type_string() }}) as {{ source_column_name }},
{%- endif %}

{% for col_name in ordered_column_names -%}
Expand Down
4 changes: 2 additions & 2 deletions macros/sql/unpivot.sql
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,9 @@ Arguments:
{{ exclude_col }},
{%- endfor %}

cast('{{ col.column }}' as {{ dbt_utils.type_string() }}) as {{ field_name }},
cast('{{ col.column }}' as {{ type_string() }}) as {{ field_name }},
cast( {% if col.data_type == 'boolean' %}
{{ dbt_utils.cast_bool_to_text(col.column) }}
{{ cast_bool_to_text(col.column) }}
{% else %}
{{ col.column }}
{% endif %}
Expand Down
8 changes: 4 additions & 4 deletions macros/sql/width_bucket.sql
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@
case
when
mod(
{{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric() ) }},
{{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric() ) }}
{{ dbt.safe_cast(expr, type_numeric() ) }},
{{ dbt.safe_cast(bin_size, type_numeric() ) }}
) = 0
then 1
else 0
Expand All @@ -38,8 +38,8 @@
-- to break ties when the amount is exactly at the bucket edge
case
when
{{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric() ) }} %
{{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric() ) }}
{{ dbt.safe_cast(expr, type_numeric() ) }} %
{{ dbt.safe_cast(bin_size, type_numeric() ) }}
= 0
then 1
else 0
Expand Down
14 changes: 7 additions & 7 deletions macros/web/get_url_host.sql
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@
{% macro default__get_url_host(field) -%}

{%- set parsed =
dbt_utils.split_part(
dbt_utils.split_part(
dbt_utils.replace(
dbt_utils.replace(
dbt_utils.replace(field, "'android-app://'", "''"
split_part(
split_part(
replace(
replace(
replace(field, "'android-app://'", "''"
), "'http://'", "''"
), "'https://'", "''"
), "'/'", 1
Expand All @@ -19,9 +19,9 @@
-%}


{{ dbt_utils.safe_cast(
{{ dbt.safe_cast(
parsed,
dbt_utils.type_string()
type_string()
)}}

{%- endmacro %}
2 changes: 1 addition & 1 deletion macros/web/get_url_parameter.sql
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

{%- set formatted_url_parameter = "'" + url_parameter + "='" -%}

{%- set split = dbt_utils.split_part(dbt_utils.split_part(field, formatted_url_parameter, 2), "'&'", 1) -%}
{%- set split = split_part(split_part(field, formatted_url_parameter, 2), "'&'", 1) -%}

nullif({{ split }},'')

Expand Down
26 changes: 13 additions & 13 deletions macros/web/get_url_path.sql
Original file line number Diff line number Diff line change
Expand Up @@ -4,31 +4,31 @@

{% macro default__get_url_path(field) -%}

{%- set stripped_url =
dbt_utils.replace(
dbt_utils.replace(field, "'http://'", "''"), "'https://'", "''")
{%- set stripped_url =
replace(
replace(field, "'http://'", "''"), "'https://'", "''")
-%}

{%- set first_slash_pos -%}
coalesce(
nullif({{dbt_utils.position("'/'", stripped_url)}}, 0),
{{dbt_utils.position("'?'", stripped_url)}} - 1
nullif({{ position("'/'", stripped_url) }}, 0),
{{ position("'?'", stripped_url) }} - 1
)
{%- endset -%}

{%- set parsed_path =
dbt_utils.split_part(
dbt_utils.right(
stripped_url,
dbt_utils.length(stripped_url) ~ "-" ~ first_slash_pos
),
split_part(
right(
stripped_url,
length(stripped_url) ~ "-" ~ first_slash_pos
),
"'?'", 1
)
-%}

{{ dbt_utils.safe_cast(
{{ safe_cast(
parsed_path,
dbt_utils.type_string()
type_string()
)}}

{%- endmacro %}

0 comments on commit fc8b083

Please sign in to comment.