diff --git a/dbt/adapters/dremio/api/cursor.py b/dbt/adapters/dremio/api/cursor.py index 72a0d5cd..98a90401 100644 --- a/dbt/adapters/dremio/api/cursor.py +++ b/dbt/adapters/dremio/api/cursor.py @@ -77,7 +77,7 @@ def close(self): self._initialize() self.closed = True - def execute(self, sql, bindings=None, fetch=True): + def execute(self, sql, bindings=None, fetch=False): if self.closed: raise Exception("CursorClosed") if bindings is None: @@ -147,7 +147,7 @@ def _populate_rowcount(self): self._rowcount = rows - def _populate_job_results(self, row_limit=100): + def _populate_job_results(self, row_limit=500): if self._job_results == None: combined_job_results = job_results( self._parameters, @@ -158,6 +158,11 @@ def _populate_job_results(self, row_limit=100): total_row_count = combined_job_results["rowCount"] current_row_count = len(combined_job_results["rows"]) + if total_row_count > 100000: + logger.warning( + "Fetching more than 100000 records. This may result in slower performance." + ) + while current_row_count < total_row_count: combined_job_results["rows"].extend( job_results( diff --git a/dbt/adapters/dremio/connections.py b/dbt/adapters/dremio/connections.py index 0c322ee4..da665c78 100644 --- a/dbt/adapters/dremio/connections.py +++ b/dbt/adapters/dremio/connections.py @@ -130,7 +130,7 @@ def add_commit_query(self): # Auto_begin may not be relevant with the rest_api def add_query( - self, sql, auto_begin=True, bindings=None, abridge_sql_log=False, fetch=True + self, sql, auto_begin=True, bindings=None, abridge_sql_log=False, fetch=False ): connection = self.get_thread_connection() if auto_begin and connection.transaction_open is False: @@ -176,7 +176,6 @@ def execute( sql = self._add_query_comment(sql) _, cursor = self.add_query(sql, auto_begin, fetch=fetch) response = self.get_response(cursor) - # fetch = True if fetch: table = cursor.table else: diff --git a/dbt/include/dremio/macros/materializations/incremental/incremental.sql b/dbt/include/dremio/macros/materializations/incremental/incremental.sql index 80a4748e..4c9e5f5b 100644 --- a/dbt/include/dremio/macros/materializations/incremental/incremental.sql +++ b/dbt/include/dremio/macros/materializations/incremental/incremental.sql @@ -51,7 +51,9 @@ limitations under the License.*/ {% set build_sql = get_create_table_as_sql(False, intermediate_relation, external_query(sql)) %} {% set need_swap = true %} {% else %} - {% do run_query(get_create_table_as_sql(True, temp_relation, external_query(sql))) %} + {% call statement('temp') -%} + {{ create_table_as(True, temp_relation, external_query(sql)) }} + {%- endcall %} {% do to_drop.append(temp_relation) %} {% do adapter.expand_target_column_types( from_relation=temp_relation,