From aca00a04f74fff27a5d049b7ff1c340b1408b53f Mon Sep 17 00:00:00 2001 From: Ishankoradia Date: Sun, 17 Mar 2024 12:37:48 +0530 Subject: [PATCH 1/4] minor changes in arithmetic operation --- dbt_automation/assets/operations.template.yml | 18 ++++++++++++------ dbt_automation/operations/arithmetic.py | 12 ++++++------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/dbt_automation/assets/operations.template.yml b/dbt_automation/assets/operations.template.yml index 412e580..0e47824 100644 --- a/dbt_automation/assets/operations.template.yml +++ b/dbt_automation/assets/operations.template.yml @@ -126,9 +126,12 @@ operations: - - ... operands: - - - - - - + - value: + is_col: + - value: + is_col: + - value: + is_col: output_column_name: - type: dropcolumns config: @@ -312,9 +315,12 @@ operations: - - ... operands: - - - - - - + - value: + is_col: + - value: + is_col: + - value: + is_col: output_column_name: - type: coalescecolumns config: diff --git a/dbt_automation/operations/arithmetic.py b/dbt_automation/operations/arithmetic.py index 861dd97..1453bb6 100644 --- a/dbt_automation/operations/arithmetic.py +++ b/dbt_automation/operations/arithmetic.py @@ -5,7 +5,7 @@ from logging import basicConfig, getLogger, INFO from dbt_automation.utils.dbtproject import dbtProject from dbt_automation.utils.interfaces.warehouse_interface import WarehouseInterface -from dbt_automation.utils.columnutils import quote_columnname +from dbt_automation.utils.columnutils import quote_columnname, quote_constvalue from dbt_automation.utils.tableutils import source_or_ref @@ -20,7 +20,7 @@ def arithmetic_dbt_sql(config: dict, warehouse: WarehouseInterface): config["input"] is dict {"source_name": "", "input_name": "", "input_type": ""} """ operator = config["operator"] - operands = config["operands"] + operands = config["operands"] # {"is_col": true, "value": "1"}[] output_col_name = config["output_column_name"] source_columns = config.get("source_columns", []) @@ -43,7 +43,7 @@ def arithmetic_dbt_sql(config: dict, warehouse: WarehouseInterface): dbt_code += "," dbt_code += "{{dbt_utils.safe_add([" for operand in operands: - dbt_code += f"'{quote_columnname(str(operand), warehouse.name)}'," + dbt_code += f"{quote_columnname(str(operand['value']), warehouse.name) if operand['is_col'] else quote_constvalue(str(operand['value']), warehouse.name)}," dbt_code = dbt_code[:-1] dbt_code += "])}}" dbt_code += f" AS {output_col_name} " @@ -51,7 +51,7 @@ def arithmetic_dbt_sql(config: dict, warehouse: WarehouseInterface): if operator == "mul": dbt_code += "," for operand in operands: - dbt_code += f"{operand} * " + dbt_code += f"{quote_columnname(str(operand['value']), warehouse.name) if operand['is_col'] else quote_constvalue(str(operand['value']), warehouse.name)} * " dbt_code = dbt_code[:-2] dbt_code += f" AS {output_col_name} " @@ -59,7 +59,7 @@ def arithmetic_dbt_sql(config: dict, warehouse: WarehouseInterface): dbt_code += "," dbt_code += "{{dbt_utils.safe_subtract([" for operand in operands: - dbt_code += f"'{quote_columnname(str(operand), warehouse.name)}'," + dbt_code += f"{quote_columnname(str(operand['value']), warehouse.name) if operand['is_col'] else quote_constvalue(str(operand['value']), warehouse.name)}," dbt_code = dbt_code[:-1] dbt_code += "])}}" dbt_code += f" AS {output_col_name} " @@ -68,7 +68,7 @@ def arithmetic_dbt_sql(config: dict, warehouse: WarehouseInterface): dbt_code += "," dbt_code += "{{dbt_utils.safe_divide(" for operand in operands: - dbt_code += f"'{quote_columnname(str(operand), warehouse.name)}'," + dbt_code += f"{quote_columnname(str(operand['value']), warehouse.name) if operand['is_col'] else quote_constvalue(str(operand['value']), warehouse.name)}," dbt_code += ")}}" dbt_code += f" AS {output_col_name} " From 569d339b347650b642f3d421313d50cd4ca2a583 Mon Sep 17 00:00:00 2001 From: Ishankoradia Date: Sun, 17 Mar 2024 12:47:33 +0530 Subject: [PATCH 2/4] fixed test cases --- tests/warehouse/test_bigquery_ops.py | 25 ++++++++++++++++++++----- tests/warehouse/test_postgres_ops.py | 25 ++++++++++++++++++++----- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/tests/warehouse/test_bigquery_ops.py b/tests/warehouse/test_bigquery_ops.py index 3a8c0d3..3a0e81a 100644 --- a/tests/warehouse/test_bigquery_ops.py +++ b/tests/warehouse/test_bigquery_ops.py @@ -351,7 +351,10 @@ def test_arithmetic_add(self): "dest_schema": "pytest_intermediate", "output_name": output_name, "operator": "add", - "operands": ["measure1", "measure2"], + "operands": [ + {"value": "measure1", "is_col": True}, + {"value": "measure2", "is_col": True}, + ], "output_column_name": "add_col", "source_columns": ["NGO", "Month", "measure1", "measure2", "Indicator"], } @@ -391,7 +394,10 @@ def test_arithmetic_sub(self): "dest_schema": "pytest_intermediate", "output_name": output_name, "operator": "sub", - "operands": ["measure1", "measure2"], + "operands": [ + {"value": "measure1", "is_col": True}, + {"value": "measure2", "is_col": True}, + ], "output_column_name": "sub_col", "source_columns": ["NGO", "Month", "measure1", "measure2", "Indicator"], } @@ -431,7 +437,10 @@ def test_arithmetic_mul(self): "dest_schema": "pytest_intermediate", "output_name": output_name, "operator": "mul", - "operands": ["measure1", "measure2"], + "operands": [ + {"value": "measure1", "is_col": True}, + {"value": "measure2", "is_col": True}, + ], "output_column_name": "mul_col", "source_columns": ["NGO", "Month", "measure1", "measure2", "Indicator"], } @@ -471,7 +480,10 @@ def test_arithmetic_div(self): "dest_schema": "pytest_intermediate", "output_name": output_name, "operator": "div", - "operands": ["measure1", "measure2"], + "operands": [ + {"value": "measure1", "is_col": True}, + {"value": "measure2", "is_col": True}, + ], "output_column_name": "div_col", "source_columns": ["NGO", "Month", "measure1", "measure2", "Indicator"], } @@ -632,7 +644,10 @@ def test_merge_operation(self): "type": "arithmetic", "config": { "operator": "add", - "operands": ["measure1", "measure2"], + "operands": [ + {"value": "measure1", "is_col": True}, + {"value": "measure2", "is_col": True}, + ], "output_column_name": "add_col", "source_columns": [ "NGO", diff --git a/tests/warehouse/test_postgres_ops.py b/tests/warehouse/test_postgres_ops.py index 900bd38..e6dd9f8 100644 --- a/tests/warehouse/test_postgres_ops.py +++ b/tests/warehouse/test_postgres_ops.py @@ -359,7 +359,10 @@ def test_arithmetic_add(self): "dest_schema": "pytest_intermediate", "output_name": output_name, "operator": "add", - "operands": ["measure1", "measure2"], + "operands": [ + {"value": "measure1", "is_col": True}, + {"value": "measure2", "is_col": True}, + ], "source_columns": ["NGO", "Month", "measure1", "measure2", "Indicator"], "output_column_name": "add_col", } @@ -399,7 +402,10 @@ def test_arithmetic_sub(self): "dest_schema": "pytest_intermediate", "output_name": output_name, "operator": "sub", - "operands": ["measure1", "measure2"], + "operands": [ + {"value": "measure1", "is_col": True}, + {"value": "measure2", "is_col": True}, + ], "source_columns": ["NGO", "Month", "measure1", "measure2", "Indicator"], "output_column_name": "sub_col", } @@ -439,7 +445,10 @@ def test_arithmetic_mul(self): "dest_schema": "pytest_intermediate", "output_name": output_name, "operator": "mul", - "operands": ["measure1", "measure2"], + "operands": [ + {"value": "measure1", "is_col": True}, + {"value": "measure2", "is_col": True}, + ], "source_columns": ["NGO", "Month", "measure1", "measure2", "Indicator"], "output_column_name": "mul_col", } @@ -479,7 +488,10 @@ def test_arithmetic_div(self): "dest_schema": "pytest_intermediate", "output_name": output_name, "operator": "div", - "operands": ["measure1", "measure2"], + "operands": [ + {"value": "measure1", "is_col": True}, + {"value": "measure2", "is_col": True}, + ], "source_columns": ["NGO", "Month", "measure1", "measure2", "Indicator"], "output_column_name": "div_col", } @@ -647,7 +659,10 @@ def test_merge_operation(self): "type": "arithmetic", "config": { "operator": "add", - "operands": ["measure1", "measure2"], + "operands": [ + {"value": "measure1", "is_col": True}, + {"value": "measure2", "is_col": True}, + ], "output_column_name": "add_col", "source_columns": [ "NGO", From f587c8d5547b5fb2e0b0225d11d8a15ad4032177 Mon Sep 17 00:00:00 2001 From: Ishankoradia Date: Sun, 17 Mar 2024 13:04:08 +0530 Subject: [PATCH 3/4] minor change --- dbt_automation/operations/arithmetic.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dbt_automation/operations/arithmetic.py b/dbt_automation/operations/arithmetic.py index 1453bb6..b39bd7e 100644 --- a/dbt_automation/operations/arithmetic.py +++ b/dbt_automation/operations/arithmetic.py @@ -39,11 +39,13 @@ def arithmetic_dbt_sql(config: dict, warehouse: WarehouseInterface): [quote_columnname(col, warehouse.name) for col in source_columns] ) + # dbt_utils function safe_add, safe_subtract, safe_divide take input as single quoted fields for column eg. 'field1' + if operator == "add": dbt_code += "," dbt_code += "{{dbt_utils.safe_add([" for operand in operands: - dbt_code += f"{quote_columnname(str(operand['value']), warehouse.name) if operand['is_col'] else quote_constvalue(str(operand['value']), warehouse.name)}," + dbt_code += f"{quote_constvalue(str(operand['value']), warehouse.name) if operand['is_col'] else quote_constvalue(str(operand['value']), warehouse.name)}," dbt_code = dbt_code[:-1] dbt_code += "])}}" dbt_code += f" AS {output_col_name} " @@ -59,7 +61,7 @@ def arithmetic_dbt_sql(config: dict, warehouse: WarehouseInterface): dbt_code += "," dbt_code += "{{dbt_utils.safe_subtract([" for operand in operands: - dbt_code += f"{quote_columnname(str(operand['value']), warehouse.name) if operand['is_col'] else quote_constvalue(str(operand['value']), warehouse.name)}," + dbt_code += f"{quote_constvalue(str(operand['value']), warehouse.name) if operand['is_col'] else quote_constvalue(str(operand['value']), warehouse.name)}," dbt_code = dbt_code[:-1] dbt_code += "])}}" dbt_code += f" AS {output_col_name} " @@ -68,7 +70,7 @@ def arithmetic_dbt_sql(config: dict, warehouse: WarehouseInterface): dbt_code += "," dbt_code += "{{dbt_utils.safe_divide(" for operand in operands: - dbt_code += f"{quote_columnname(str(operand['value']), warehouse.name) if operand['is_col'] else quote_constvalue(str(operand['value']), warehouse.name)}," + dbt_code += f"{quote_constvalue(str(operand['value']), warehouse.name) if operand['is_col'] else quote_constvalue(str(operand['value']), warehouse.name)}," dbt_code += ")}}" dbt_code += f" AS {output_col_name} " From 7ae61fe99b16a12cbba1c72157a693bfa42fdbd8 Mon Sep 17 00:00:00 2001 From: Ishankoradia Date: Sun, 17 Mar 2024 13:13:21 +0530 Subject: [PATCH 4/4] removing redundant if else --- dbt_automation/operations/arithmetic.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dbt_automation/operations/arithmetic.py b/dbt_automation/operations/arithmetic.py index b39bd7e..df1bf72 100644 --- a/dbt_automation/operations/arithmetic.py +++ b/dbt_automation/operations/arithmetic.py @@ -39,13 +39,14 @@ def arithmetic_dbt_sql(config: dict, warehouse: WarehouseInterface): [quote_columnname(col, warehouse.name) for col in source_columns] ) - # dbt_utils function safe_add, safe_subtract, safe_divide take input as single quoted fields for column eg. 'field1' + # dbt_utils function safe_add, safe_subtract, safe_divide + # takes input as single quoted fields for column eg. 'field1'. regardless of the warehouse if operator == "add": dbt_code += "," dbt_code += "{{dbt_utils.safe_add([" for operand in operands: - dbt_code += f"{quote_constvalue(str(operand['value']), warehouse.name) if operand['is_col'] else quote_constvalue(str(operand['value']), warehouse.name)}," + dbt_code += f"{quote_constvalue(str(operand['value']), warehouse.name)}," dbt_code = dbt_code[:-1] dbt_code += "])}}" dbt_code += f" AS {output_col_name} " @@ -61,7 +62,7 @@ def arithmetic_dbt_sql(config: dict, warehouse: WarehouseInterface): dbt_code += "," dbt_code += "{{dbt_utils.safe_subtract([" for operand in operands: - dbt_code += f"{quote_constvalue(str(operand['value']), warehouse.name) if operand['is_col'] else quote_constvalue(str(operand['value']), warehouse.name)}," + dbt_code += f"{quote_constvalue(str(operand['value']), warehouse.name)}," dbt_code = dbt_code[:-1] dbt_code += "])}}" dbt_code += f" AS {output_col_name} " @@ -70,7 +71,7 @@ def arithmetic_dbt_sql(config: dict, warehouse: WarehouseInterface): dbt_code += "," dbt_code += "{{dbt_utils.safe_divide(" for operand in operands: - dbt_code += f"{quote_constvalue(str(operand['value']), warehouse.name) if operand['is_col'] else quote_constvalue(str(operand['value']), warehouse.name)}," + dbt_code += f"{quote_constvalue(str(operand['value']), warehouse.name)}," dbt_code += ")}}" dbt_code += f" AS {output_col_name} "