From a7145238fe64df390570e801f014f1fff97b5503 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Mon, 25 Nov 2024 22:50:23 +0100 Subject: [PATCH] dbt: Start using dbt-cratedb2 --- framework/dbt/basic/macros/catalog.sql | 43 ------------------------ framework/dbt/basic/macros/overrides.sql | 33 ++---------------- framework/dbt/basic/macros/relations.sql | 34 ------------------- framework/dbt/basic/profiles.yml | 2 +- framework/dbt/basic/requirements.txt | 3 +- 5 files changed, 4 insertions(+), 111 deletions(-) delete mode 100644 framework/dbt/basic/macros/catalog.sql delete mode 100644 framework/dbt/basic/macros/relations.sql diff --git a/framework/dbt/basic/macros/catalog.sql b/framework/dbt/basic/macros/catalog.sql deleted file mode 100644 index de2b0f80..00000000 --- a/framework/dbt/basic/macros/catalog.sql +++ /dev/null @@ -1,43 +0,0 @@ -{% macro postgres__get_catalog(information_schema, schemas) -%} - - {%- call statement('catalog', fetch_result=True) -%} - {# - Derived from https://github.com/dbt-labs/dbt-core/blob/main/plugins/postgres/dbt/include/postgres/macros/catalog.sql - #} - {% set database = information_schema.database %} - {{ adapter.verify_database(database) }} - select - '{{ database }}' as table_database, - sch.nspname as table_schema, - tbl.relname as table_name, - case tbl.relkind - when 'v' then 'VIEW' - else 'BASE TABLE' - end as table_type, - tbl_desc.description as table_comment, - col.attname as column_name, - col.attnum as column_index, - pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type, - col_desc.description as column_comment, - pg_get_userbyid(tbl.relowner) as table_owner - from pg_catalog.pg_namespace sch - join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid - join pg_catalog.pg_attribute col on col.attrelid = tbl.oid - left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0) - left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum) - where ( - {%- for schema in schemas -%} - upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} - {%- endfor -%} - ) - and tbl.relpersistence in ('p', 'u') -- [p]ermanent table or [u]nlogged table. Exclude [t]emporary tables - and tbl.relkind in ('r', 'v', 'f', 'p') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table, [m]aterialized view - and col.attnum > 0 -- negative numbers are used for system columns such as oid - and not col.attisdropped -- column as not been dropped - order by - sch.nspname, - tbl.relname, - col.attnum - {%- endcall -%} - {{ return(load_result('catalog').table) }} -{%- endmacro %} diff --git a/framework/dbt/basic/macros/overrides.sql b/framework/dbt/basic/macros/overrides.sql index e062636e..96b43a3a 100644 --- a/framework/dbt/basic/macros/overrides.sql +++ b/framework/dbt/basic/macros/overrides.sql @@ -26,15 +26,7 @@ {%- endmacro %} -{% macro postgres__create_schema(relation) -%} - {%- call statement('create_schema') -%} - /* schemas are not created in CrateDB */ - DROP TABLE IF EXISTS thisschemadefinitelydoesnotexits.thiswouldnotexist - /* but we need to run something to not have just EOF */ - {% endcall %} -{% endmacro %} - -{% macro postgres__create_table_as(temporary, relation, sql) -%} +{% macro cratedb__create_table_as(temporary, relation, sql) -%} {%- set unlogged = config.get('unlogged', default=false) -%} {%- set sql_header = config.get('sql_header', none) -%} @@ -46,27 +38,6 @@ ); {%- endmacro %} -{% macro postgres__drop_schema(relation) -%} - {% if relation.database -%} - {{ adapter.verify_database(relation.database) }} - {%- endif -%} - {%- call statement('drop_schema') -%} - /* schemas are not dropped in CrateDB */ - {%- endcall -%} -{% endmacro %} - -{% macro default__drop_relation(relation) -%} - {% call statement('drop_relation', auto_begin=False) -%} - drop {{ relation.type }} if exists "{{ relation.schema }}"."{{ relation.identifier }}" - {%- endcall %} -{% endmacro %} - -{% macro default__drop_schema(relation) -%} - {%- call statement('drop_schema') -%} - /* schemas are not dropped in CrateDB */ - {% endcall %} -{% endmacro %} - {% macro default__create_view_as(relation, sql) -%} {%- set sql_header = config.get('sql_header', none) -%} @@ -76,7 +47,7 @@ ; {%- endmacro %} -{% macro postgres__rename_relation(from_relation, to_relation) -%} +{% macro cratedb__rename_relation(from_relation, to_relation) -%} {% do drop_relation(to_relation) %} {% set schema_query = "SELECT table_type FROM information_schema.tables WHERE table_schema = '{}' AND table_name = '{}'".format(from_relation.schema, from_relation.identifier) %} {% set results = run_query(schema_query) %} diff --git a/framework/dbt/basic/macros/relations.sql b/framework/dbt/basic/macros/relations.sql deleted file mode 100644 index 1f1583e9..00000000 --- a/framework/dbt/basic/macros/relations.sql +++ /dev/null @@ -1,34 +0,0 @@ -{% macro postgres_get_relations () -%} - {%- call statement('relations', fetch_result=True) -%} - select 'mock' as referenced_schema,'referenced ' as referenced_name, - 'mock' as dependent_schema,'dependent' as dependent_name; - {%- endcall -%} - {{ return(load_result('relations').table) }} -{% endmacro %} - -{% macro postgres__list_relations_without_caching(schema_relation) %} - {% call statement('list_relations_without_caching', fetch_result=True) -%} - select - '{{ schema_relation.database }}' as database, - tablename as name, - schemaname as schema, - 'table' as type - from pg_tables - where schemaname ilike '{{ schema_relation.schema }}' - union all - select - '{{ schema_relation.database }}' as database, - viewname as name, - schemaname as schema, - 'view' as type - from pg_views - where schemaname ilike '{{ schema_relation.schema }}' - {% endcall %} - {{ return(load_result('list_relations_without_caching').table) }} -{% endmacro %} - -{% macro default__truncate_relation(relation) -%} - {% call statement('truncate_relation') -%} - delete from {{ relation }} - {%- endcall %} -{% endmacro %} diff --git a/framework/dbt/basic/profiles.yml b/framework/dbt/basic/profiles.yml index a381e50e..caa4a7a4 100644 --- a/framework/dbt/basic/profiles.yml +++ b/framework/dbt/basic/profiles.yml @@ -1,7 +1,7 @@ cratedb_localhost: outputs: dev: - type: postgres + type: cratedb host: localhost user: crate pass: crate diff --git a/framework/dbt/basic/requirements.txt b/framework/dbt/basic/requirements.txt index 67711ec8..2e3f31c0 100644 --- a/framework/dbt/basic/requirements.txt +++ b/framework/dbt/basic/requirements.txt @@ -1,2 +1 @@ -dbt-postgres<1.7 -snowplow-tracker<0.13 +dbt-cratedb2>=0.0.1dev2