Skip to content

Commit

Permalink
🐛 (fix): calc_batch_size maximum number of parameters allowed
Browse files Browse the repository at this point in the history
  • Loading branch information
schlich committed Feb 13, 2024
2 parents 80300df + 0b828b1 commit 06b6edd
Showing 1 changed file with 57 additions and 0 deletions.
57 changes: 57 additions & 0 deletions dbt/include/sqlserver/macros/materializations/seeds/helpers.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
{% macro sqlserver__get_binding_char() %}
{{ return('?') }}
{% endmacro %}

{% macro sqlserver__get_batch_size() %}
{{ return(400) }}
{% endmacro %}

{% macro calc_batch_size(num_columns) %}
{#
SQL Server allows for a max of 2098 parameters in a single statement.
Check if the max_batch_size fits with the number of columns, otherwise
reduce the batch size so it fits.
#}
{% set max_batch_size = get_batch_size() %}
{% set calculated_batch = (2098 / num_columns)|int %}
{% set batch_size = [max_batch_size, calculated_batch] | min %}

{{ return(batch_size) }}
{% endmacro %}

{% macro sqlserver__load_csv_rows(model, agate_table) %}
{% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
{% set batch_size = calc_batch_size(agate_table.column_names|length) %}
{% set bindings = [] %}
{% set statements = [] %}

{{ log("Inserting batches of " ~ batch_size ~ " records") }}

{% for chunk in agate_table.rows | batch(batch_size) %}
{% set bindings = [] %}

{% for row in chunk %}
{% do bindings.extend(row) %}
{% endfor %}

{% set sql %}
insert into {{ this.render() }} ({{ cols_sql }}) values
{% for row in chunk -%}
({%- for column in agate_table.column_names -%}
{{ get_binding_char() }}
{%- if not loop.last%},{%- endif %}
{%- endfor -%})
{%- if not loop.last%},{%- endif %}
{%- endfor %}
{% endset %}

{% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}

{% if loop.index0 == 0 %}
{% do statements.append(sql) %}
{% endif %}
{% endfor %}

{# Return SQL so we can render it out into the compiled files #}
{{ return(statements[0]) }}
{% endmacro %}

0 comments on commit 06b6edd

Please sign in to comment.