Skip to content

Commit

Permalink
update the read sources api to send back relative path of the source
Browse files Browse the repository at this point in the history
  • Loading branch information
Ishankoradia committed Mar 7, 2024
1 parent d8ee14e commit 21501af
Showing 1 changed file with 22 additions and 13 deletions.
35 changes: 22 additions & 13 deletions dbt_automation/utils/dbtsources.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,27 +77,36 @@ def read_sources(project_dir) -> list[dict]:
"""parse all yaml files inside models dir and read sources"""
models_dir = Path(project_dir) / "models"

sources = []
# {"models/example/sources.yml": [{"name": "source1", "tables": [{"identifier": "table1"}]}
sources = {}
for root, dirs, files in os.walk(models_dir):
for file in files:
if file.endswith(".yml") or file.endswith(".yaml"):
file_path = os.path.join(root, file)
temp_sources = []
with open(file_path, "r") as f:
yaml_data = yaml.safe_load(f)
if "sources" in yaml_data:
sources += yaml_data["sources"]
temp_sources = yaml_data["sources"]

if len(temp_sources) > 0:
src_yml_path = Path(file_path).relative_to(project_dir)
sources[str(src_yml_path)] = temp_sources

src_tables = []
for src in sources:
for table in src["tables"]:
# keeping the schema same as input of each operations
src_tables.append(
{
"source_name": src["name"],
"input_name": table["identifier"], # table
"input_type": "source",
"schema": src["schema"],
}
)
for src_yml_rel_path, srcs_yml in sources.items():
# yaml can have more than one source
for src in srcs_yml:
for table in src["tables"]:
# keeping the schema same as input of each operations
src_tables.append(
{
"source_name": src["name"],
"input_name": table["identifier"], # table
"input_type": "source",
"schema": src["schema"],
"sql_path": src_yml_rel_path,
}
)

return src_tables

0 comments on commit 21501af

Please sign in to comment.