Skip to content

Commit

Permalink
refactoring sync sources operation
Browse files Browse the repository at this point in the history
  • Loading branch information
Ishankoradia committed Mar 6, 2024
1 parent 9af8ed3 commit 8d3673f
Showing 1 changed file with 19 additions and 6 deletions.
25 changes: 19 additions & 6 deletions dbt_automation/operations/syncsources.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,15 @@
logger = getLogger()


def sync_sources(config, warehouse: WarehouseInterface, dbtproject: dbtProject):
def generate_source_definitions_yaml(
source_name: str, input_schema: str, tablenames: list, dbtproject: dbtProject
):
"""
reads tables from the input_schema to create a dbt sources.yml
uses the metadata from the existing source definitions, if any
Generate the sources.yml file. Or merge if one already exists
"""
input_schema = config["source_schema"]
source_name = config["source_name"]
source_dir = dbtproject.models_dir(input_schema)
sources_file = dbtproject.sources_filename(input_schema)

tablenames = warehouse.get_tables(input_schema)
dbsourcedefinitions = mksourcedefinition(source_name, input_schema, tablenames)
logger.info("read sources from database schema %s", input_schema)

Expand All @@ -59,6 +57,21 @@ def sync_sources(config, warehouse: WarehouseInterface, dbtproject: dbtProject):
return dbtproject.strip_project_dir(sources_file)


def sync_sources(config, warehouse: WarehouseInterface, dbtproject: dbtProject):
"""
reads tables from the input_schema to create a dbt sources.yml
uses the metadata from the existing source definitions, if any
"""
input_schema = config["source_schema"]
source_name = config["source_name"]

tablenames = warehouse.get_tables(input_schema)

return generate_source_definitions_yaml(
source_name, input_schema, tablenames, dbtproject
)


if __name__ == "__main__":
load_dotenv("dbconnection.env")

Expand Down

0 comments on commit 8d3673f

Please sign in to comment.