From 1b4bde3cecdd917e831da6135764da6c311470e0 Mon Sep 17 00:00:00 2001 From: pankajastro Date: Fri, 15 Nov 2024 23:25:56 +0530 Subject: [PATCH] Add basic docs --- dev/dags/example_clone.py | 51 ++++++++++++++++++++++++++++++ docs/getting_started/operators.rst | 24 ++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 dev/dags/example_clone.py create mode 100644 docs/getting_started/operators.rst diff --git a/dev/dags/example_clone.py b/dev/dags/example_clone.py new file mode 100644 index 000000000..d3d238758 --- /dev/null +++ b/dev/dags/example_clone.py @@ -0,0 +1,51 @@ +from datetime import datetime + +from airflow import DAG + +from cosmos import DbtCloneLocalOperator, DbtRunLocalOperator, DbtSeedLocalOperator, ProfileConfig + +DBT_PROJ_DIR = "/usr/local/airflow/dbt/jaffle_shop" + +profile_config1 = ProfileConfig( + profile_name="bigquery_dev", + target_name="dev", + profiles_yml_filepath="/usr/local/airflow/dbt/jaffle_shop/profiles.yml", +) + +profile_config2 = ProfileConfig( + profile_name="bigquery_clone", + target_name="dev", + profiles_yml_filepath="/usr/local/airflow/dbt/jaffle_shop/profiles.yml", +) + + +with DAG("test-id-1", start_date=datetime(2024, 1, 1), catchup=False) as dag: + seed_operator = DbtSeedLocalOperator( + profile_config=profile_config1, + project_dir=DBT_PROJ_DIR, + task_id="seed", + dbt_cmd_flags=["--select", "raw_customers"], + install_deps=True, + append_env=True, + ) + run_operator = DbtRunLocalOperator( + profile_config=profile_config1, + project_dir=DBT_PROJ_DIR, + task_id="run", + dbt_cmd_flags=["--models", "stg_customers"], + install_deps=True, + append_env=True, + ) + + # [START clone_example] + clone_operator = DbtCloneLocalOperator( + profile_config=profile_config2, + project_dir=DBT_PROJ_DIR, + task_id="clone", + dbt_cmd_flags=["--models", "stg_customers", "--state", "/usr/local/airflow/dbt/jaffle_shop/target"], + install_deps=True, + append_env=True, + ) + # [END clone_example] + + seed_operator >> run_operator >> clone_operator diff --git a/docs/getting_started/operators.rst b/docs/getting_started/operators.rst new file mode 100644 index 000000000..9f955047d --- /dev/null +++ b/docs/getting_started/operators.rst @@ -0,0 +1,24 @@ +.. _operators: + +Operators +========= + +Cosmos exposes individual operators that correspond to specific dbt commands, which can be used just like traditional +`Apache Airflow® `_ operators. Cosmos names these operators using the format ``DbtOperator``. For example, ``DbtBuildLocalOperator``. + +Clone +----- + +Requirement + +* Cosmos >= 1.8.0 +* dbt-core >= 1.6.0 + +The ``DbtCloneLocalOperator`` implement `dbt clone `_ command. + +Example of how to use + +.. literalinclude:: ../../dev/dags/example_clone.py + :language: python + :start-after: [START clone_example] + :end-before: [END clone_example]