From 1d76ff2ca44dd530877836fb2577bc68207674ac Mon Sep 17 00:00:00 2001 From: Baris Terzioglu Date: Mon, 30 Dec 2024 23:04:13 +0100 Subject: [PATCH 1/8] seperate pipelines for integration tests --- integration-tests/integration-test.go | 24 ++--- .../test-pipelines/bruin-pipeline/.gitignore | 1 + .../test-pipelines/bruin-pipeline/README.md | 98 +++++++++++++++++++ .../assets/product_categories.sql | 35 +++++++ .../bruin-pipeline/assets/products.sql | 44 +++++++++ .../bruin-pipeline/pipeline.yml | 3 + .../format-if-changed-pipeline/.gitignore | 1 + .../format-if-changed-pipeline/README.md | 98 +++++++++++++++++++ .../assets/correctly-formatted.sql | 38 +++++++ .../assets/products.sql | 44 +++++++++ .../format-if-changed-pipeline/pipeline.yml | 3 + .../assets/shopify.inventory_items.asset.yml | 0 .../assets/shopify.products.asset.yml | 0 .../push-metadata-pipeline}/pipeline.yml | 0 .../assets/chess_games.asset.yml | 0 .../assets/chess_profiles.asset.yml | 0 .../assets/game_outcome_summary.sql | 0 .../assets/player_profile_summary.sql | 0 .../assets/player_summary.sql | 0 .../pipeline.yml | 0 .../assets/chess_games.asset.yml | 9 ++ .../assets/chess_profiles.asset.yml | 8 ++ .../assets/game_outcome_summary.sql | 31 ++++++ .../assets/player_profile_summary.sql | 30 ++++++ .../assets/player_summary.sql | 62 ++++++++++++ .../run-with-downstream-pipeline/pipeline.yml | 1 + .../assets/chess_games.asset.yml | 9 ++ .../assets/chess_profiles.asset.yml | 8 ++ .../assets/game_outcome_summary.sql | 31 ++++++ .../assets/player_profile_summary.sql | 30 ++++++ .../assets/player_summary.sql | 62 ++++++++++++ .../pipeline.yml | 1 + .../assets/chess_games.asset.yml | 9 ++ .../assets/chess_profiles.asset.yml | 8 ++ .../assets/game_outcome_summary.sql | 31 ++++++ .../assets/player_profile_summary.sql | 30 ++++++ .../assets/player_summary.sql | 62 ++++++++++++ .../run-with-filters-pipeline/pipeline.yml | 1 + .../assets/chess_games.asset.yml | 9 ++ .../assets/chess_profiles.asset.yml | 8 ++ .../assets/game_outcome_summary.sql | 31 ++++++ .../assets/player_profile_summary.sql | 30 ++++++ .../assets/player_summary.sql | 62 ++++++++++++ .../run-with-tags-pipeline/pipeline.yml | 1 + 44 files changed, 941 insertions(+), 12 deletions(-) create mode 100644 integration-tests/test-pipelines/bruin-pipeline/.gitignore create mode 100644 integration-tests/test-pipelines/bruin-pipeline/README.md create mode 100644 integration-tests/test-pipelines/bruin-pipeline/assets/product_categories.sql create mode 100644 integration-tests/test-pipelines/bruin-pipeline/assets/products.sql create mode 100644 integration-tests/test-pipelines/bruin-pipeline/pipeline.yml create mode 100644 integration-tests/test-pipelines/format-if-changed-pipeline/.gitignore create mode 100644 integration-tests/test-pipelines/format-if-changed-pipeline/README.md create mode 100644 integration-tests/test-pipelines/format-if-changed-pipeline/assets/correctly-formatted.sql create mode 100644 integration-tests/test-pipelines/format-if-changed-pipeline/assets/products.sql create mode 100644 integration-tests/test-pipelines/format-if-changed-pipeline/pipeline.yml rename integration-tests/{bigquery-metadata => test-pipelines/push-metadata-pipeline}/assets/shopify.inventory_items.asset.yml (100%) rename integration-tests/{bigquery-metadata => test-pipelines/push-metadata-pipeline}/assets/shopify.products.asset.yml (100%) rename integration-tests/{bigquery-metadata => test-pipelines/push-metadata-pipeline}/pipeline.yml (100%) rename integration-tests/{chess-extended => test-pipelines/run-main-with-downstream-pipeline}/assets/chess_games.asset.yml (100%) rename integration-tests/{chess-extended => test-pipelines/run-main-with-downstream-pipeline}/assets/chess_profiles.asset.yml (100%) rename integration-tests/{chess-extended => test-pipelines/run-main-with-downstream-pipeline}/assets/game_outcome_summary.sql (100%) rename integration-tests/{chess-extended => test-pipelines/run-main-with-downstream-pipeline}/assets/player_profile_summary.sql (100%) rename integration-tests/{chess-extended => test-pipelines/run-main-with-downstream-pipeline}/assets/player_summary.sql (100%) rename integration-tests/{chess-extended => test-pipelines/run-main-with-downstream-pipeline}/pipeline.yml (100%) create mode 100644 integration-tests/test-pipelines/run-with-downstream-pipeline/assets/chess_games.asset.yml create mode 100644 integration-tests/test-pipelines/run-with-downstream-pipeline/assets/chess_profiles.asset.yml create mode 100644 integration-tests/test-pipelines/run-with-downstream-pipeline/assets/game_outcome_summary.sql create mode 100644 integration-tests/test-pipelines/run-with-downstream-pipeline/assets/player_profile_summary.sql create mode 100644 integration-tests/test-pipelines/run-with-downstream-pipeline/assets/player_summary.sql create mode 100644 integration-tests/test-pipelines/run-with-downstream-pipeline/pipeline.yml create mode 100644 integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/chess_games.asset.yml create mode 100644 integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/chess_profiles.asset.yml create mode 100644 integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/game_outcome_summary.sql create mode 100644 integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/player_profile_summary.sql create mode 100644 integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/player_summary.sql create mode 100644 integration-tests/test-pipelines/run-with-filters-case2-pipeline/pipeline.yml create mode 100644 integration-tests/test-pipelines/run-with-filters-pipeline/assets/chess_games.asset.yml create mode 100644 integration-tests/test-pipelines/run-with-filters-pipeline/assets/chess_profiles.asset.yml create mode 100644 integration-tests/test-pipelines/run-with-filters-pipeline/assets/game_outcome_summary.sql create mode 100644 integration-tests/test-pipelines/run-with-filters-pipeline/assets/player_profile_summary.sql create mode 100644 integration-tests/test-pipelines/run-with-filters-pipeline/assets/player_summary.sql create mode 100644 integration-tests/test-pipelines/run-with-filters-pipeline/pipeline.yml create mode 100644 integration-tests/test-pipelines/run-with-tags-pipeline/assets/chess_games.asset.yml create mode 100644 integration-tests/test-pipelines/run-with-tags-pipeline/assets/chess_profiles.asset.yml create mode 100644 integration-tests/test-pipelines/run-with-tags-pipeline/assets/game_outcome_summary.sql create mode 100644 integration-tests/test-pipelines/run-with-tags-pipeline/assets/player_profile_summary.sql create mode 100644 integration-tests/test-pipelines/run-with-tags-pipeline/assets/player_summary.sql create mode 100644 integration-tests/test-pipelines/run-with-tags-pipeline/pipeline.yml diff --git a/integration-tests/integration-test.go b/integration-tests/integration-test.go index d87e5f2d..9f3d6acd 100644 --- a/integration-tests/integration-test.go +++ b/integration-tests/integration-test.go @@ -253,9 +253,9 @@ func getTasks(binary string, currentFolder string) []e2e.Task { }, }, { - Name: "chess-extended", + Name: "run-with-tags", Command: binary, - Args: []string{"run", "--tag", "include", "--exclude-tag", "exclude", filepath.Join(currentFolder, "chess-extended")}, + Args: []string{"run", "--tag", "include", "--exclude-tag", "exclude", filepath.Join(currentFolder, "test-pipelines/run-with-tags-pipeline")}, Env: []string{}, Expected: e2e.Output{ @@ -266,9 +266,9 @@ func getTasks(binary string, currentFolder string) []e2e.Task { }, }, { - Name: "chess-extended-only-checks", + Name: "run-with-filters", Command: binary, - Args: []string{"run", "--tag", "include", "--exclude-tag", "exclude", "--only", "checks", filepath.Join(currentFolder, "chess-extended")}, + Args: []string{"run", "--tag", "include", "--exclude-tag", "exclude", "--only", "checks", filepath.Join(currentFolder, "test-pipelines/run-with-filters-pipeline")}, Env: []string{}, Expected: e2e.Output{ @@ -283,7 +283,7 @@ func getTasks(binary string, currentFolder string) []e2e.Task { { Name: "format-if-fail", Command: binary, - Args: []string{"format", "--fail-if-changed", filepath.Join(currentFolder, "chess-extended/assets/chess_games.asset.yml")}, + Args: []string{"format", "--fail-if-changed", filepath.Join(currentFolder, "test-pipelines/format-if-changed-pipeline/assets/correctly-formatted.sql")}, Env: []string{}, Expected: e2e.Output{ ExitCode: 0, @@ -293,9 +293,9 @@ func getTasks(binary string, currentFolder string) []e2e.Task { }, }, { - Name: "chess-extended-only-main", + Name: "run-with-filters-case2", Command: binary, - Args: []string{"run", "--tag", "include", "--exclude-tag", "exclude", "--only", "main", filepath.Join(currentFolder, "chess-extended")}, + Args: []string{"run", "--tag", "include", "--exclude-tag", "exclude", "--only", "main", filepath.Join(currentFolder, "test-pipelines/run-with-filters-case2-pipeline")}, Env: []string{}, Expected: e2e.Output{ @@ -308,9 +308,9 @@ func getTasks(binary string, currentFolder string) []e2e.Task { }, }, { - Name: "downstream-chess-extended", + Name: "run-with-downstream", Command: binary, - Args: []string{"run", "--downstream", filepath.Join(currentFolder, "chess-extended/assets/game_outcome_summary.sql")}, + Args: []string{"run", "--downstream", filepath.Join(currentFolder, "test-pipelines/run-with-downstream-pipeline/assets/game_outcome_summary.sql")}, Env: []string{}, Expected: e2e.Output{ ExitCode: 0, @@ -322,9 +322,9 @@ func getTasks(binary string, currentFolder string) []e2e.Task { }, }, { - Name: "downstream-only-main-chess-extended", + Name: "run-main-with-downstream", Command: binary, - Args: []string{"run", "--downstream", "--only", "main", filepath.Join(currentFolder, "chess-extended/assets/game_outcome_summary.sql")}, + Args: []string{"run", "--downstream", "--only", "main", filepath.Join(currentFolder, "test-pipelines/run-main-with-downstream-pipeline/assets/game_outcome_summary.sql")}, Env: []string{}, Expected: e2e.Output{ ExitCode: 0, @@ -338,7 +338,7 @@ func getTasks(binary string, currentFolder string) []e2e.Task { { Name: "push-metadata", Command: binary, - Args: []string{"run", "--push-metadata", "--only", "push-metadata", filepath.Join(currentFolder, "bigquery-metadata")}, + Args: []string{"run", "--push-metadata", "--only", "push-metadata", filepath.Join(currentFolder, "test-pipelines/push-metadata-pipeline")}, Env: []string{}, Expected: e2e.Output{ ExitCode: 1, diff --git a/integration-tests/test-pipelines/bruin-pipeline/.gitignore b/integration-tests/test-pipelines/bruin-pipeline/.gitignore new file mode 100644 index 00000000..4bcaecc8 --- /dev/null +++ b/integration-tests/test-pipelines/bruin-pipeline/.gitignore @@ -0,0 +1 @@ +.bruin.yml \ No newline at end of file diff --git a/integration-tests/test-pipelines/bruin-pipeline/README.md b/integration-tests/test-pipelines/bruin-pipeline/README.md new file mode 100644 index 00000000..294fd3bc --- /dev/null +++ b/integration-tests/test-pipelines/bruin-pipeline/README.md @@ -0,0 +1,98 @@ +# Bruin - Sample Pipeline + +This pipeline is a simple example of a Bruin pipeline. It demonstrates how to use the `bruin` CLI to build and run a pipeline. + +The pipeline includes two sample assets already: +- `myschema.example`: A simple SQL asset that creates a table in BigQuery. + - Feel free to change the type from `bq.sql` to anything. +- `myschema.country_list`: A simple Python asset that installs a dependency and runs the logic. + +## Setup +The pipeline already includes an empty `.bruin.yml` file, fill it with your connections and environments. You can read more about connections [here](https://bruin-data.github.io/bruin/connections/overview.html). + +Here's a sample `.bruin.yml` file: + +```yaml +environments: + default: + connections: + google_cloud_platform: + - name: "gcp" + service_account_file: "/path/to/my/key.json" + project_id: "my-project-dev" + snowflake: + - name: "snowflake" + username: "my-user" + password: "my-password" + account: "my-account" + database: "my-database" + warehouse: "my-warehouse" + schema: "my-dev-schema" + generic: + - name: KEY1 + value: value1 + production: + connections: + google_cloud_platform: + - name: "gcp" + service_account_file: "/path/to/my/prod-key.json" + project_id: "my-project-prod" + snowflake: + - name: "snowflake" + username: "my-user" + password: "my-password" + account: "my-account" + database: "my-database" + warehouse: "my-warehouse" + schema: "my-prod-schema" + generic: + - name: KEY1 + value: value1 +``` + +You can simply switch the environment using the `--environment` flag, e.g.: + +```shell +bruin validate --environment production . +``` + +## Running the pipeline + +bruin CLI can run the whole pipeline or any task with the downstreams: + +```shell +bruin run . +``` + +```shell +Starting the pipeline execution... + +[2023-03-16T18:25:14Z] [worker-0] Running: dashboard.bruin-test +[2023-03-16T18:25:16Z] [worker-0] Completed: dashboard.bruin-test (1.681s) +[2023-03-16T18:25:16Z] [worker-4] Running: hello +[2023-03-16T18:25:16Z] [worker-4] [hello] >> Hello, world! +[2023-03-16T18:25:16Z] [worker-4] Completed: hello (116ms) + +Executed 2 tasks in 1.798s +``` + +You can also run a single task: + +```shell +bruin run assets/hello.py +``` + +```shell +Starting the pipeline execution... + +[2023-03-16T18:25:59Z] [worker-0] Running: hello +[2023-03-16T18:26:00Z] [worker-0] [hello] >> Hello, world! +[2023-03-16T18:26:00Z] [worker-0] Completed: hello (103ms) + + +Executed 1 tasks in 103ms +``` + +You can optionally pass a `--downstream` flag to run the task with all of its downstreams. + +That's it, good luck! \ No newline at end of file diff --git a/integration-tests/test-pipelines/bruin-pipeline/assets/product_categories.sql b/integration-tests/test-pipelines/bruin-pipeline/assets/product_categories.sql new file mode 100644 index 00000000..1c24bc27 --- /dev/null +++ b/integration-tests/test-pipelines/bruin-pipeline/assets/product_categories.sql @@ -0,0 +1,35 @@ +/* @bruin +name: product_categories +type: duckdb.sql +materialization: + type: table + +columns: + - name: category_id + type: INTEGER + description: "Unique identifier for the product category" + primary_key: true + checks: + - name: not_null + - name: positive + - name: category_name + type: VARCHAR + description: "Name of the product category" + checks: + - name: not_null + - name: description + type: VARCHAR + description: "Description of the product category" +@bruin */ + +SELECT + 1 AS category_id, 'Electronics' AS category_name, 'Devices like phones, laptops, and monitors' AS description +UNION ALL +SELECT + 2 AS category_id, 'Accessories' AS category_name, 'Complementary items like headphones and chargers' AS description +UNION ALL +SELECT + 3 AS category_id, 'Appliances' AS category_name, 'Household devices like refrigerators and microwaves' AS description +UNION ALL +SELECT + 4 AS category_id, 'Furniture' AS category_name, 'Home and office furniture like desks and chairs' AS description; diff --git a/integration-tests/test-pipelines/bruin-pipeline/assets/products.sql b/integration-tests/test-pipelines/bruin-pipeline/assets/products.sql new file mode 100644 index 00000000..61f4841c --- /dev/null +++ b/integration-tests/test-pipelines/bruin-pipeline/assets/products.sql @@ -0,0 +1,44 @@ +/* @bruin +name: products +type: duckdb.sql + +materialization: + type: table + +columns: + - name: product_id + type: INTEGER + description: "Unique identifier for the product" + primary_key: true + checks: + - name: not_null + - name: positive + - name: product_name + type: VARCHAR + description: "Name of the product" + checks: + - name: not_null + - name: price + type: FLOAT + description: "Price of the product in USD" + checks: + - name: not_null + - name: positive + - name: stock + type: INTEGER + description: "Number of units in stock" + checks: + - name: non_negative +@bruin */ + +SELECT + 1 AS product_id, 'Laptop' AS product_name, 999.99 AS price, 10 AS stock +UNION ALL +SELECT + 2 AS product_id, 'Smartphone' AS product_name, 699.99 AS price, 50 AS stock +UNION ALL +SELECT + 3 AS product_id, 'Headphones' AS product_name, 199.99 AS price, 100 AS stock +UNION ALL +SELECT + 4 AS product_id, 'Monitor' AS product_name, 299.99 AS price, 25 AS stock; diff --git a/integration-tests/test-pipelines/bruin-pipeline/pipeline.yml b/integration-tests/test-pipelines/bruin-pipeline/pipeline.yml new file mode 100644 index 00000000..9bdb0b42 --- /dev/null +++ b/integration-tests/test-pipelines/bruin-pipeline/pipeline.yml @@ -0,0 +1,3 @@ +name: bruin-init + + diff --git a/integration-tests/test-pipelines/format-if-changed-pipeline/.gitignore b/integration-tests/test-pipelines/format-if-changed-pipeline/.gitignore new file mode 100644 index 00000000..4bcaecc8 --- /dev/null +++ b/integration-tests/test-pipelines/format-if-changed-pipeline/.gitignore @@ -0,0 +1 @@ +.bruin.yml \ No newline at end of file diff --git a/integration-tests/test-pipelines/format-if-changed-pipeline/README.md b/integration-tests/test-pipelines/format-if-changed-pipeline/README.md new file mode 100644 index 00000000..294fd3bc --- /dev/null +++ b/integration-tests/test-pipelines/format-if-changed-pipeline/README.md @@ -0,0 +1,98 @@ +# Bruin - Sample Pipeline + +This pipeline is a simple example of a Bruin pipeline. It demonstrates how to use the `bruin` CLI to build and run a pipeline. + +The pipeline includes two sample assets already: +- `myschema.example`: A simple SQL asset that creates a table in BigQuery. + - Feel free to change the type from `bq.sql` to anything. +- `myschema.country_list`: A simple Python asset that installs a dependency and runs the logic. + +## Setup +The pipeline already includes an empty `.bruin.yml` file, fill it with your connections and environments. You can read more about connections [here](https://bruin-data.github.io/bruin/connections/overview.html). + +Here's a sample `.bruin.yml` file: + +```yaml +environments: + default: + connections: + google_cloud_platform: + - name: "gcp" + service_account_file: "/path/to/my/key.json" + project_id: "my-project-dev" + snowflake: + - name: "snowflake" + username: "my-user" + password: "my-password" + account: "my-account" + database: "my-database" + warehouse: "my-warehouse" + schema: "my-dev-schema" + generic: + - name: KEY1 + value: value1 + production: + connections: + google_cloud_platform: + - name: "gcp" + service_account_file: "/path/to/my/prod-key.json" + project_id: "my-project-prod" + snowflake: + - name: "snowflake" + username: "my-user" + password: "my-password" + account: "my-account" + database: "my-database" + warehouse: "my-warehouse" + schema: "my-prod-schema" + generic: + - name: KEY1 + value: value1 +``` + +You can simply switch the environment using the `--environment` flag, e.g.: + +```shell +bruin validate --environment production . +``` + +## Running the pipeline + +bruin CLI can run the whole pipeline or any task with the downstreams: + +```shell +bruin run . +``` + +```shell +Starting the pipeline execution... + +[2023-03-16T18:25:14Z] [worker-0] Running: dashboard.bruin-test +[2023-03-16T18:25:16Z] [worker-0] Completed: dashboard.bruin-test (1.681s) +[2023-03-16T18:25:16Z] [worker-4] Running: hello +[2023-03-16T18:25:16Z] [worker-4] [hello] >> Hello, world! +[2023-03-16T18:25:16Z] [worker-4] Completed: hello (116ms) + +Executed 2 tasks in 1.798s +``` + +You can also run a single task: + +```shell +bruin run assets/hello.py +``` + +```shell +Starting the pipeline execution... + +[2023-03-16T18:25:59Z] [worker-0] Running: hello +[2023-03-16T18:26:00Z] [worker-0] [hello] >> Hello, world! +[2023-03-16T18:26:00Z] [worker-0] Completed: hello (103ms) + + +Executed 1 tasks in 103ms +``` + +You can optionally pass a `--downstream` flag to run the task with all of its downstreams. + +That's it, good luck! \ No newline at end of file diff --git a/integration-tests/test-pipelines/format-if-changed-pipeline/assets/correctly-formatted.sql b/integration-tests/test-pipelines/format-if-changed-pipeline/assets/correctly-formatted.sql new file mode 100644 index 00000000..cbf18e50 --- /dev/null +++ b/integration-tests/test-pipelines/format-if-changed-pipeline/assets/correctly-formatted.sql @@ -0,0 +1,38 @@ +/* @bruin + +name: product_categories +type: duckdb.sql + +materialization: + type: table + +columns: + - name: category_id + type: INTEGER + description: Unique identifier for the product category + primary_key: true + checks: + - name: not_null + - name: positive + - name: category_name + type: VARCHAR + description: Name of the product category + checks: + - name: not_null + - name: description + type: VARCHAR + description: Description of the product category + +@bruin */ + +SELECT + 1 AS category_id, 'Electronics' AS category_name, 'Devices like phones, laptops, and monitors' AS description +UNION ALL +SELECT + 2 AS category_id, 'Accessories' AS category_name, 'Complementary items like headphones and chargers' AS description +UNION ALL +SELECT + 3 AS category_id, 'Appliances' AS category_name, 'Household devices like refrigerators and microwaves' AS description +UNION ALL +SELECT + 4 AS category_id, 'Furniture' AS category_name, 'Home and office furniture like desks and chairs' AS description; diff --git a/integration-tests/test-pipelines/format-if-changed-pipeline/assets/products.sql b/integration-tests/test-pipelines/format-if-changed-pipeline/assets/products.sql new file mode 100644 index 00000000..61f4841c --- /dev/null +++ b/integration-tests/test-pipelines/format-if-changed-pipeline/assets/products.sql @@ -0,0 +1,44 @@ +/* @bruin +name: products +type: duckdb.sql + +materialization: + type: table + +columns: + - name: product_id + type: INTEGER + description: "Unique identifier for the product" + primary_key: true + checks: + - name: not_null + - name: positive + - name: product_name + type: VARCHAR + description: "Name of the product" + checks: + - name: not_null + - name: price + type: FLOAT + description: "Price of the product in USD" + checks: + - name: not_null + - name: positive + - name: stock + type: INTEGER + description: "Number of units in stock" + checks: + - name: non_negative +@bruin */ + +SELECT + 1 AS product_id, 'Laptop' AS product_name, 999.99 AS price, 10 AS stock +UNION ALL +SELECT + 2 AS product_id, 'Smartphone' AS product_name, 699.99 AS price, 50 AS stock +UNION ALL +SELECT + 3 AS product_id, 'Headphones' AS product_name, 199.99 AS price, 100 AS stock +UNION ALL +SELECT + 4 AS product_id, 'Monitor' AS product_name, 299.99 AS price, 25 AS stock; diff --git a/integration-tests/test-pipelines/format-if-changed-pipeline/pipeline.yml b/integration-tests/test-pipelines/format-if-changed-pipeline/pipeline.yml new file mode 100644 index 00000000..9bdb0b42 --- /dev/null +++ b/integration-tests/test-pipelines/format-if-changed-pipeline/pipeline.yml @@ -0,0 +1,3 @@ +name: bruin-init + + diff --git a/integration-tests/bigquery-metadata/assets/shopify.inventory_items.asset.yml b/integration-tests/test-pipelines/push-metadata-pipeline/assets/shopify.inventory_items.asset.yml similarity index 100% rename from integration-tests/bigquery-metadata/assets/shopify.inventory_items.asset.yml rename to integration-tests/test-pipelines/push-metadata-pipeline/assets/shopify.inventory_items.asset.yml diff --git a/integration-tests/bigquery-metadata/assets/shopify.products.asset.yml b/integration-tests/test-pipelines/push-metadata-pipeline/assets/shopify.products.asset.yml similarity index 100% rename from integration-tests/bigquery-metadata/assets/shopify.products.asset.yml rename to integration-tests/test-pipelines/push-metadata-pipeline/assets/shopify.products.asset.yml diff --git a/integration-tests/bigquery-metadata/pipeline.yml b/integration-tests/test-pipelines/push-metadata-pipeline/pipeline.yml similarity index 100% rename from integration-tests/bigquery-metadata/pipeline.yml rename to integration-tests/test-pipelines/push-metadata-pipeline/pipeline.yml diff --git a/integration-tests/chess-extended/assets/chess_games.asset.yml b/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/chess_games.asset.yml similarity index 100% rename from integration-tests/chess-extended/assets/chess_games.asset.yml rename to integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/chess_games.asset.yml diff --git a/integration-tests/chess-extended/assets/chess_profiles.asset.yml b/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/chess_profiles.asset.yml similarity index 100% rename from integration-tests/chess-extended/assets/chess_profiles.asset.yml rename to integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/chess_profiles.asset.yml diff --git a/integration-tests/chess-extended/assets/game_outcome_summary.sql b/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/game_outcome_summary.sql similarity index 100% rename from integration-tests/chess-extended/assets/game_outcome_summary.sql rename to integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/game_outcome_summary.sql diff --git a/integration-tests/chess-extended/assets/player_profile_summary.sql b/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/player_profile_summary.sql similarity index 100% rename from integration-tests/chess-extended/assets/player_profile_summary.sql rename to integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/player_profile_summary.sql diff --git a/integration-tests/chess-extended/assets/player_summary.sql b/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/player_summary.sql similarity index 100% rename from integration-tests/chess-extended/assets/player_summary.sql rename to integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/player_summary.sql diff --git a/integration-tests/chess-extended/pipeline.yml b/integration-tests/test-pipelines/run-main-with-downstream-pipeline/pipeline.yml similarity index 100% rename from integration-tests/chess-extended/pipeline.yml rename to integration-tests/test-pipelines/run-main-with-downstream-pipeline/pipeline.yml diff --git a/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/chess_games.asset.yml b/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/chess_games.asset.yml new file mode 100644 index 00000000..68bc4e41 --- /dev/null +++ b/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/chess_games.asset.yml @@ -0,0 +1,9 @@ +name: chess_playground.games +type: ingestr +tags: + - include + +parameters: + destination: duckdb + source_connection: chess-default + source_table: games diff --git a/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/chess_profiles.asset.yml b/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/chess_profiles.asset.yml new file mode 100644 index 00000000..cd770cdb --- /dev/null +++ b/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/chess_profiles.asset.yml @@ -0,0 +1,8 @@ +name: chess_playground.profiles +type: ingestr +parameters: + source_connection: chess-default + source_table: profiles + destination: duckdb +tags: + - include \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/game_outcome_summary.sql b/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/game_outcome_summary.sql new file mode 100644 index 00000000..65cb4340 --- /dev/null +++ b/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/game_outcome_summary.sql @@ -0,0 +1,31 @@ +/* @bruin + +name: chess_playground.game_outcome_summary +type: duckdb.sql +materialization: + type: table + +depends: + - chess_playground.games + +columns: + - name: result_type + type: string + description: "Type of game result (win, draw, etc.)" + - name: total_games + type: integer + description: "Total number of games with this result" + checks: + - name: positive +tags: + - include + +@bruin */ + +SELECT + g.white->>'result' AS result_type, + COUNT(*) AS total_games +FROM chess_playground.games g +WHERE g.white->>'result' IS NOT NULL +GROUP BY g.white->>'result' +ORDER BY total_games DESC; diff --git a/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/player_profile_summary.sql b/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/player_profile_summary.sql new file mode 100644 index 00000000..4aedab67 --- /dev/null +++ b/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/player_profile_summary.sql @@ -0,0 +1,30 @@ +/* @bruin + +name: chess_playground.player_profile_summary +type: duckdb.sql +materialization: + type: table + +depends: + - chess_playground.profiles +columns: + - name: total_players + type: integer + description: "Total number of players in the profiles table" + checks: + - name: positive + - name: active_players + type: integer + description: "Number of players marked as active" + - name: inactive_players + type: integer + description: "Number of players marked as inactive" +tags: + - exclude +@bruin */ + +SELECT + COUNT(*) AS total_players, + COUNT(CASE WHEN p.status = 'active' THEN 1 END) AS active_players, + COUNT(CASE WHEN p.status = 'inactive' THEN 1 END) AS inactive_players +FROM chess_playground.profiles p; diff --git a/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/player_summary.sql b/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/player_summary.sql new file mode 100644 index 00000000..0adf29fa --- /dev/null +++ b/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/player_summary.sql @@ -0,0 +1,62 @@ +/* @bruin + +name: chess_playground.player_summary +type: duckdb.sql +materialization: + type: table + +depends: + - chess_playground.game_outcome_summary + - chess_playground.player_profile_summary + - chess_playground.games + - chess_playground.profiles + +columns: + - name: username + type: string + description: "Username of the player" + - name: total_games + type: integer + description: "Total games played by the player" + checks: + - name: non_negative + - name: total_wins + type: integer + description: "Total games won by the player" + - name: win_rate + type: float + description: "Win rate of the player" +tags: + - include + - exclude +@bruin */ + +WITH game_results AS ( + SELECT + CASE + WHEN g.white->>'result' = 'win' THEN g.white->>'@id' + WHEN g.black->>'result' = 'win' THEN g.black->>'@id' + ELSE NULL + END AS winner_aid, + g.white->>'@id' AS white_aid, + g.black->>'@id' AS black_aid +FROM chess_playground.games g + ) + +SELECT + p.username, + p.aid, + COUNT(*) AS total_games, + COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) + + COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS total_wins, + ROUND( + (COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) + + COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END)) * 100.0 / + NULLIF(COUNT(*), 0), + 2 + ) AS win_rate +FROM chess_playground.profiles p + LEFT JOIN game_results g + ON p.aid IN (g.white_aid, g.black_aid) +GROUP BY p.username, p.aid +ORDER BY total_games DESC; diff --git a/integration-tests/test-pipelines/run-with-downstream-pipeline/pipeline.yml b/integration-tests/test-pipelines/run-with-downstream-pipeline/pipeline.yml new file mode 100644 index 00000000..9172e408 --- /dev/null +++ b/integration-tests/test-pipelines/run-with-downstream-pipeline/pipeline.yml @@ -0,0 +1 @@ +name: chess_extended_duckdb \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/chess_games.asset.yml b/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/chess_games.asset.yml new file mode 100644 index 00000000..68bc4e41 --- /dev/null +++ b/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/chess_games.asset.yml @@ -0,0 +1,9 @@ +name: chess_playground.games +type: ingestr +tags: + - include + +parameters: + destination: duckdb + source_connection: chess-default + source_table: games diff --git a/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/chess_profiles.asset.yml b/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/chess_profiles.asset.yml new file mode 100644 index 00000000..cd770cdb --- /dev/null +++ b/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/chess_profiles.asset.yml @@ -0,0 +1,8 @@ +name: chess_playground.profiles +type: ingestr +parameters: + source_connection: chess-default + source_table: profiles + destination: duckdb +tags: + - include \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/game_outcome_summary.sql b/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/game_outcome_summary.sql new file mode 100644 index 00000000..65cb4340 --- /dev/null +++ b/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/game_outcome_summary.sql @@ -0,0 +1,31 @@ +/* @bruin + +name: chess_playground.game_outcome_summary +type: duckdb.sql +materialization: + type: table + +depends: + - chess_playground.games + +columns: + - name: result_type + type: string + description: "Type of game result (win, draw, etc.)" + - name: total_games + type: integer + description: "Total number of games with this result" + checks: + - name: positive +tags: + - include + +@bruin */ + +SELECT + g.white->>'result' AS result_type, + COUNT(*) AS total_games +FROM chess_playground.games g +WHERE g.white->>'result' IS NOT NULL +GROUP BY g.white->>'result' +ORDER BY total_games DESC; diff --git a/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/player_profile_summary.sql b/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/player_profile_summary.sql new file mode 100644 index 00000000..4aedab67 --- /dev/null +++ b/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/player_profile_summary.sql @@ -0,0 +1,30 @@ +/* @bruin + +name: chess_playground.player_profile_summary +type: duckdb.sql +materialization: + type: table + +depends: + - chess_playground.profiles +columns: + - name: total_players + type: integer + description: "Total number of players in the profiles table" + checks: + - name: positive + - name: active_players + type: integer + description: "Number of players marked as active" + - name: inactive_players + type: integer + description: "Number of players marked as inactive" +tags: + - exclude +@bruin */ + +SELECT + COUNT(*) AS total_players, + COUNT(CASE WHEN p.status = 'active' THEN 1 END) AS active_players, + COUNT(CASE WHEN p.status = 'inactive' THEN 1 END) AS inactive_players +FROM chess_playground.profiles p; diff --git a/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/player_summary.sql b/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/player_summary.sql new file mode 100644 index 00000000..0adf29fa --- /dev/null +++ b/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/player_summary.sql @@ -0,0 +1,62 @@ +/* @bruin + +name: chess_playground.player_summary +type: duckdb.sql +materialization: + type: table + +depends: + - chess_playground.game_outcome_summary + - chess_playground.player_profile_summary + - chess_playground.games + - chess_playground.profiles + +columns: + - name: username + type: string + description: "Username of the player" + - name: total_games + type: integer + description: "Total games played by the player" + checks: + - name: non_negative + - name: total_wins + type: integer + description: "Total games won by the player" + - name: win_rate + type: float + description: "Win rate of the player" +tags: + - include + - exclude +@bruin */ + +WITH game_results AS ( + SELECT + CASE + WHEN g.white->>'result' = 'win' THEN g.white->>'@id' + WHEN g.black->>'result' = 'win' THEN g.black->>'@id' + ELSE NULL + END AS winner_aid, + g.white->>'@id' AS white_aid, + g.black->>'@id' AS black_aid +FROM chess_playground.games g + ) + +SELECT + p.username, + p.aid, + COUNT(*) AS total_games, + COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) + + COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS total_wins, + ROUND( + (COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) + + COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END)) * 100.0 / + NULLIF(COUNT(*), 0), + 2 + ) AS win_rate +FROM chess_playground.profiles p + LEFT JOIN game_results g + ON p.aid IN (g.white_aid, g.black_aid) +GROUP BY p.username, p.aid +ORDER BY total_games DESC; diff --git a/integration-tests/test-pipelines/run-with-filters-case2-pipeline/pipeline.yml b/integration-tests/test-pipelines/run-with-filters-case2-pipeline/pipeline.yml new file mode 100644 index 00000000..9172e408 --- /dev/null +++ b/integration-tests/test-pipelines/run-with-filters-case2-pipeline/pipeline.yml @@ -0,0 +1 @@ +name: chess_extended_duckdb \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-with-filters-pipeline/assets/chess_games.asset.yml b/integration-tests/test-pipelines/run-with-filters-pipeline/assets/chess_games.asset.yml new file mode 100644 index 00000000..68bc4e41 --- /dev/null +++ b/integration-tests/test-pipelines/run-with-filters-pipeline/assets/chess_games.asset.yml @@ -0,0 +1,9 @@ +name: chess_playground.games +type: ingestr +tags: + - include + +parameters: + destination: duckdb + source_connection: chess-default + source_table: games diff --git a/integration-tests/test-pipelines/run-with-filters-pipeline/assets/chess_profiles.asset.yml b/integration-tests/test-pipelines/run-with-filters-pipeline/assets/chess_profiles.asset.yml new file mode 100644 index 00000000..cd770cdb --- /dev/null +++ b/integration-tests/test-pipelines/run-with-filters-pipeline/assets/chess_profiles.asset.yml @@ -0,0 +1,8 @@ +name: chess_playground.profiles +type: ingestr +parameters: + source_connection: chess-default + source_table: profiles + destination: duckdb +tags: + - include \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-with-filters-pipeline/assets/game_outcome_summary.sql b/integration-tests/test-pipelines/run-with-filters-pipeline/assets/game_outcome_summary.sql new file mode 100644 index 00000000..65cb4340 --- /dev/null +++ b/integration-tests/test-pipelines/run-with-filters-pipeline/assets/game_outcome_summary.sql @@ -0,0 +1,31 @@ +/* @bruin + +name: chess_playground.game_outcome_summary +type: duckdb.sql +materialization: + type: table + +depends: + - chess_playground.games + +columns: + - name: result_type + type: string + description: "Type of game result (win, draw, etc.)" + - name: total_games + type: integer + description: "Total number of games with this result" + checks: + - name: positive +tags: + - include + +@bruin */ + +SELECT + g.white->>'result' AS result_type, + COUNT(*) AS total_games +FROM chess_playground.games g +WHERE g.white->>'result' IS NOT NULL +GROUP BY g.white->>'result' +ORDER BY total_games DESC; diff --git a/integration-tests/test-pipelines/run-with-filters-pipeline/assets/player_profile_summary.sql b/integration-tests/test-pipelines/run-with-filters-pipeline/assets/player_profile_summary.sql new file mode 100644 index 00000000..4aedab67 --- /dev/null +++ b/integration-tests/test-pipelines/run-with-filters-pipeline/assets/player_profile_summary.sql @@ -0,0 +1,30 @@ +/* @bruin + +name: chess_playground.player_profile_summary +type: duckdb.sql +materialization: + type: table + +depends: + - chess_playground.profiles +columns: + - name: total_players + type: integer + description: "Total number of players in the profiles table" + checks: + - name: positive + - name: active_players + type: integer + description: "Number of players marked as active" + - name: inactive_players + type: integer + description: "Number of players marked as inactive" +tags: + - exclude +@bruin */ + +SELECT + COUNT(*) AS total_players, + COUNT(CASE WHEN p.status = 'active' THEN 1 END) AS active_players, + COUNT(CASE WHEN p.status = 'inactive' THEN 1 END) AS inactive_players +FROM chess_playground.profiles p; diff --git a/integration-tests/test-pipelines/run-with-filters-pipeline/assets/player_summary.sql b/integration-tests/test-pipelines/run-with-filters-pipeline/assets/player_summary.sql new file mode 100644 index 00000000..0adf29fa --- /dev/null +++ b/integration-tests/test-pipelines/run-with-filters-pipeline/assets/player_summary.sql @@ -0,0 +1,62 @@ +/* @bruin + +name: chess_playground.player_summary +type: duckdb.sql +materialization: + type: table + +depends: + - chess_playground.game_outcome_summary + - chess_playground.player_profile_summary + - chess_playground.games + - chess_playground.profiles + +columns: + - name: username + type: string + description: "Username of the player" + - name: total_games + type: integer + description: "Total games played by the player" + checks: + - name: non_negative + - name: total_wins + type: integer + description: "Total games won by the player" + - name: win_rate + type: float + description: "Win rate of the player" +tags: + - include + - exclude +@bruin */ + +WITH game_results AS ( + SELECT + CASE + WHEN g.white->>'result' = 'win' THEN g.white->>'@id' + WHEN g.black->>'result' = 'win' THEN g.black->>'@id' + ELSE NULL + END AS winner_aid, + g.white->>'@id' AS white_aid, + g.black->>'@id' AS black_aid +FROM chess_playground.games g + ) + +SELECT + p.username, + p.aid, + COUNT(*) AS total_games, + COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) + + COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS total_wins, + ROUND( + (COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) + + COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END)) * 100.0 / + NULLIF(COUNT(*), 0), + 2 + ) AS win_rate +FROM chess_playground.profiles p + LEFT JOIN game_results g + ON p.aid IN (g.white_aid, g.black_aid) +GROUP BY p.username, p.aid +ORDER BY total_games DESC; diff --git a/integration-tests/test-pipelines/run-with-filters-pipeline/pipeline.yml b/integration-tests/test-pipelines/run-with-filters-pipeline/pipeline.yml new file mode 100644 index 00000000..9172e408 --- /dev/null +++ b/integration-tests/test-pipelines/run-with-filters-pipeline/pipeline.yml @@ -0,0 +1 @@ +name: chess_extended_duckdb \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-with-tags-pipeline/assets/chess_games.asset.yml b/integration-tests/test-pipelines/run-with-tags-pipeline/assets/chess_games.asset.yml new file mode 100644 index 00000000..68bc4e41 --- /dev/null +++ b/integration-tests/test-pipelines/run-with-tags-pipeline/assets/chess_games.asset.yml @@ -0,0 +1,9 @@ +name: chess_playground.games +type: ingestr +tags: + - include + +parameters: + destination: duckdb + source_connection: chess-default + source_table: games diff --git a/integration-tests/test-pipelines/run-with-tags-pipeline/assets/chess_profiles.asset.yml b/integration-tests/test-pipelines/run-with-tags-pipeline/assets/chess_profiles.asset.yml new file mode 100644 index 00000000..cd770cdb --- /dev/null +++ b/integration-tests/test-pipelines/run-with-tags-pipeline/assets/chess_profiles.asset.yml @@ -0,0 +1,8 @@ +name: chess_playground.profiles +type: ingestr +parameters: + source_connection: chess-default + source_table: profiles + destination: duckdb +tags: + - include \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-with-tags-pipeline/assets/game_outcome_summary.sql b/integration-tests/test-pipelines/run-with-tags-pipeline/assets/game_outcome_summary.sql new file mode 100644 index 00000000..65cb4340 --- /dev/null +++ b/integration-tests/test-pipelines/run-with-tags-pipeline/assets/game_outcome_summary.sql @@ -0,0 +1,31 @@ +/* @bruin + +name: chess_playground.game_outcome_summary +type: duckdb.sql +materialization: + type: table + +depends: + - chess_playground.games + +columns: + - name: result_type + type: string + description: "Type of game result (win, draw, etc.)" + - name: total_games + type: integer + description: "Total number of games with this result" + checks: + - name: positive +tags: + - include + +@bruin */ + +SELECT + g.white->>'result' AS result_type, + COUNT(*) AS total_games +FROM chess_playground.games g +WHERE g.white->>'result' IS NOT NULL +GROUP BY g.white->>'result' +ORDER BY total_games DESC; diff --git a/integration-tests/test-pipelines/run-with-tags-pipeline/assets/player_profile_summary.sql b/integration-tests/test-pipelines/run-with-tags-pipeline/assets/player_profile_summary.sql new file mode 100644 index 00000000..4aedab67 --- /dev/null +++ b/integration-tests/test-pipelines/run-with-tags-pipeline/assets/player_profile_summary.sql @@ -0,0 +1,30 @@ +/* @bruin + +name: chess_playground.player_profile_summary +type: duckdb.sql +materialization: + type: table + +depends: + - chess_playground.profiles +columns: + - name: total_players + type: integer + description: "Total number of players in the profiles table" + checks: + - name: positive + - name: active_players + type: integer + description: "Number of players marked as active" + - name: inactive_players + type: integer + description: "Number of players marked as inactive" +tags: + - exclude +@bruin */ + +SELECT + COUNT(*) AS total_players, + COUNT(CASE WHEN p.status = 'active' THEN 1 END) AS active_players, + COUNT(CASE WHEN p.status = 'inactive' THEN 1 END) AS inactive_players +FROM chess_playground.profiles p; diff --git a/integration-tests/test-pipelines/run-with-tags-pipeline/assets/player_summary.sql b/integration-tests/test-pipelines/run-with-tags-pipeline/assets/player_summary.sql new file mode 100644 index 00000000..0adf29fa --- /dev/null +++ b/integration-tests/test-pipelines/run-with-tags-pipeline/assets/player_summary.sql @@ -0,0 +1,62 @@ +/* @bruin + +name: chess_playground.player_summary +type: duckdb.sql +materialization: + type: table + +depends: + - chess_playground.game_outcome_summary + - chess_playground.player_profile_summary + - chess_playground.games + - chess_playground.profiles + +columns: + - name: username + type: string + description: "Username of the player" + - name: total_games + type: integer + description: "Total games played by the player" + checks: + - name: non_negative + - name: total_wins + type: integer + description: "Total games won by the player" + - name: win_rate + type: float + description: "Win rate of the player" +tags: + - include + - exclude +@bruin */ + +WITH game_results AS ( + SELECT + CASE + WHEN g.white->>'result' = 'win' THEN g.white->>'@id' + WHEN g.black->>'result' = 'win' THEN g.black->>'@id' + ELSE NULL + END AS winner_aid, + g.white->>'@id' AS white_aid, + g.black->>'@id' AS black_aid +FROM chess_playground.games g + ) + +SELECT + p.username, + p.aid, + COUNT(*) AS total_games, + COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) + + COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS total_wins, + ROUND( + (COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) + + COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END)) * 100.0 / + NULLIF(COUNT(*), 0), + 2 + ) AS win_rate +FROM chess_playground.profiles p + LEFT JOIN game_results g + ON p.aid IN (g.white_aid, g.black_aid) +GROUP BY p.username, p.aid +ORDER BY total_games DESC; diff --git a/integration-tests/test-pipelines/run-with-tags-pipeline/pipeline.yml b/integration-tests/test-pipelines/run-with-tags-pipeline/pipeline.yml new file mode 100644 index 00000000..9172e408 --- /dev/null +++ b/integration-tests/test-pipelines/run-with-tags-pipeline/pipeline.yml @@ -0,0 +1 @@ +name: chess_extended_duckdb \ No newline at end of file From 8e37aa3593ddd60b17d20e356ea8cb01dc73b66b Mon Sep 17 00:00:00 2001 From: Baris Terzioglu Date: Mon, 30 Dec 2024 23:11:36 +0100 Subject: [PATCH 2/8] more changes --- integration-tests/integration-test.go | 4 +- .../run-use-uv-pipeline/assets/asset.py | 25 ++++++++++ .../assets/chess_games.asset.yml | 6 +++ .../assets/chess_profiles.asset.yml | 6 +++ .../assets/player_summary.sql | 47 +++++++++++++++++++ .../run-use-uv-pipeline/pipeline.yml | 1 + .../run-use-uv-pipeline/requirements.txt | 1 + 7 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 integration-tests/test-pipelines/run-use-uv-pipeline/assets/asset.py create mode 100644 integration-tests/test-pipelines/run-use-uv-pipeline/assets/chess_games.asset.yml create mode 100644 integration-tests/test-pipelines/run-use-uv-pipeline/assets/chess_profiles.asset.yml create mode 100644 integration-tests/test-pipelines/run-use-uv-pipeline/assets/player_summary.sql create mode 100644 integration-tests/test-pipelines/run-use-uv-pipeline/pipeline.yml create mode 100644 integration-tests/test-pipelines/run-use-uv-pipeline/requirements.txt diff --git a/integration-tests/integration-test.go b/integration-tests/integration-test.go index 9f3d6acd..147c7384 100644 --- a/integration-tests/integration-test.go +++ b/integration-tests/integration-test.go @@ -362,9 +362,9 @@ func getTasks(binary string, currentFolder string) []e2e.Task { }, }, { - Name: "run-use-uv-happy-path", + Name: "run-use-uv", Command: binary, - Args: []string{"run", "--use-uv", filepath.Join(currentFolder, "happy-path")}, + Args: []string{"run", "--use-uv", filepath.Join(currentFolder, "test-pipelines/run-use-uv-pipeline")}, Env: []string{}, Expected: e2e.Output{ ExitCode: 0, diff --git a/integration-tests/test-pipelines/run-use-uv-pipeline/assets/asset.py b/integration-tests/test-pipelines/run-use-uv-pipeline/assets/asset.py new file mode 100644 index 00000000..b3dc6f55 --- /dev/null +++ b/integration-tests/test-pipelines/run-use-uv-pipeline/assets/asset.py @@ -0,0 +1,25 @@ +""" @bruin + +name: python_asset +image: python:3.11 +depends: + - chess_playground.player_summary + +secrets: + - key: KEY1 + inject_as: INJECTED1 + +@bruin """ + +import os +import duckdb + +if os.getenv('INJECTED1') != "value1": + raise Exception("KEY1 is not injected correctly") + +con = duckdb.connect(database = "duckdb.db", read_only = False) + +con.execute("SELECT * FROM chess_playground.player_summary") +result = con.fetchall() +if len(result) != 2: + raise Exception("Incorrect number of rows in player_summary") diff --git a/integration-tests/test-pipelines/run-use-uv-pipeline/assets/chess_games.asset.yml b/integration-tests/test-pipelines/run-use-uv-pipeline/assets/chess_games.asset.yml new file mode 100644 index 00000000..caa10252 --- /dev/null +++ b/integration-tests/test-pipelines/run-use-uv-pipeline/assets/chess_games.asset.yml @@ -0,0 +1,6 @@ +name: chess_playground.games +type: ingestr +parameters: + source_connection: chess-default + source_table: games + destination: duckdb \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-use-uv-pipeline/assets/chess_profiles.asset.yml b/integration-tests/test-pipelines/run-use-uv-pipeline/assets/chess_profiles.asset.yml new file mode 100644 index 00000000..d8158e02 --- /dev/null +++ b/integration-tests/test-pipelines/run-use-uv-pipeline/assets/chess_profiles.asset.yml @@ -0,0 +1,6 @@ +name: chess_playground.profiles +type: ingestr +parameters: + source_connection: chess-default + source_table: profiles + destination: duckdb \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-use-uv-pipeline/assets/player_summary.sql b/integration-tests/test-pipelines/run-use-uv-pipeline/assets/player_summary.sql new file mode 100644 index 00000000..b4d4432c --- /dev/null +++ b/integration-tests/test-pipelines/run-use-uv-pipeline/assets/player_summary.sql @@ -0,0 +1,47 @@ +/* @bruin + +name: chess_playground.player_summary +type: duckdb.sql +materialization: + type: table + +depends: + - chess_playground.games + - chess_playground.profiles + +columns: + - name: total_games + type: integer + description: "the games" + checks: + - name: positive + +@bruin */ + +WITH game_results AS ( + SELECT + CASE + WHEN g.white->>'result' = 'win' THEN g.white->>'@id' + WHEN g.black->>'result' = 'win' THEN g.black->>'@id' + ELSE NULL + END AS winner_aid, + g.white->>'@id' AS white_aid, + g.black->>'@id' AS black_aid +FROM chess_playground.games g +) + +SELECT + p.username, + p.aid, + COUNT(*) AS total_games, + COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS white_wins, + COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS black_wins, + COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END) AS white_games, + COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END) AS black_games, + ROUND(COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END), 0), 2) AS white_win_rate, + ROUND(COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END), 0), 2) AS black_win_rate +FROM chess_playground.profiles p +LEFT JOIN game_results g + ON p.aid IN (g.white_aid, g.black_aid) +GROUP BY p.username, p.aid +ORDER BY total_games DESC diff --git a/integration-tests/test-pipelines/run-use-uv-pipeline/pipeline.yml b/integration-tests/test-pipelines/run-use-uv-pipeline/pipeline.yml new file mode 100644 index 00000000..2108b581 --- /dev/null +++ b/integration-tests/test-pipelines/run-use-uv-pipeline/pipeline.yml @@ -0,0 +1 @@ +name: integration_test \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-use-uv-pipeline/requirements.txt b/integration-tests/test-pipelines/run-use-uv-pipeline/requirements.txt new file mode 100644 index 00000000..425ad7ab --- /dev/null +++ b/integration-tests/test-pipelines/run-use-uv-pipeline/requirements.txt @@ -0,0 +1 @@ +duckdb==1.1.3 \ No newline at end of file From 07b705b7f82f7baa2c1a211f41091c02e29f22f1 Mon Sep 17 00:00:00 2001 From: Baris Terzioglu Date: Mon, 30 Dec 2024 23:24:04 +0100 Subject: [PATCH 3/8] more changes --- integration-tests/integration-test.go | 6 +++--- .../missing-upstream-pipeline}/assets/nonexistent.sql | 0 .../expectations/missing_upstream.json | 0 .../missing-upstream-pipeline}/pipeline.yml | 0 .../run-malformed-pipeline}/assets/malformed.sql | 0 .../run-malformed-pipeline}/expectations/malformed.json | 0 .../run-malformed-pipeline}/pipeline.yml | 0 7 files changed, 3 insertions(+), 3 deletions(-) rename integration-tests/{missing-upstream => test-pipelines/missing-upstream-pipeline}/assets/nonexistent.sql (100%) rename integration-tests/{missing-upstream => test-pipelines/missing-upstream-pipeline}/expectations/missing_upstream.json (100%) rename integration-tests/{missing-upstream => test-pipelines/missing-upstream-pipeline}/pipeline.yml (100%) rename integration-tests/{malformed => test-pipelines/run-malformed-pipeline}/assets/malformed.sql (100%) rename integration-tests/{malformed => test-pipelines/run-malformed-pipeline}/expectations/malformed.json (100%) rename integration-tests/{malformed => test-pipelines/run-malformed-pipeline}/pipeline.yml (100%) diff --git a/integration-tests/integration-test.go b/integration-tests/integration-test.go index 147c7384..35546bb6 100644 --- a/integration-tests/integration-test.go +++ b/integration-tests/integration-test.go @@ -451,12 +451,12 @@ func getTasks(binary string, currentFolder string) []e2e.Task { { Name: "validate-missing-upstream", Command: binary, - Args: []string{"validate", "-o", "json", filepath.Join(currentFolder, "missing-upstream/assets/nonexistent.sql")}, + Args: []string{"validate", "-o", "json", filepath.Join(currentFolder, "test-pipelines/missing-upstream-pipeline/assets/nonexistent.sql")}, Env: []string{}, SkipJSONNodes: []string{"\"path\""}, Expected: e2e.Output{ ExitCode: 0, - Output: helpers.ReadFile(filepath.Join(currentFolder, "missing-upstream/expectations/missing_upstream.json")), + Output: helpers.ReadFile(filepath.Join(currentFolder, "test-pipelines/missing-upstream-pipeline/expectations/missing_upstream.json")), }, Asserts: []func(*e2e.Task) error{ e2e.AssertByExitCode, @@ -466,7 +466,7 @@ func getTasks(binary string, currentFolder string) []e2e.Task { { Name: "run-malformed-sql", Command: binary, - Args: []string{"run", filepath.Join(currentFolder, "malformed/assets/malformed.sql")}, + Args: []string{"run", filepath.Join(currentFolder, "test-pipelines/run-malformed-pipeline/assets/malformed.sql")}, Env: []string{}, Expected: e2e.Output{ diff --git a/integration-tests/missing-upstream/assets/nonexistent.sql b/integration-tests/test-pipelines/missing-upstream-pipeline/assets/nonexistent.sql similarity index 100% rename from integration-tests/missing-upstream/assets/nonexistent.sql rename to integration-tests/test-pipelines/missing-upstream-pipeline/assets/nonexistent.sql diff --git a/integration-tests/missing-upstream/expectations/missing_upstream.json b/integration-tests/test-pipelines/missing-upstream-pipeline/expectations/missing_upstream.json similarity index 100% rename from integration-tests/missing-upstream/expectations/missing_upstream.json rename to integration-tests/test-pipelines/missing-upstream-pipeline/expectations/missing_upstream.json diff --git a/integration-tests/missing-upstream/pipeline.yml b/integration-tests/test-pipelines/missing-upstream-pipeline/pipeline.yml similarity index 100% rename from integration-tests/missing-upstream/pipeline.yml rename to integration-tests/test-pipelines/missing-upstream-pipeline/pipeline.yml diff --git a/integration-tests/malformed/assets/malformed.sql b/integration-tests/test-pipelines/run-malformed-pipeline/assets/malformed.sql similarity index 100% rename from integration-tests/malformed/assets/malformed.sql rename to integration-tests/test-pipelines/run-malformed-pipeline/assets/malformed.sql diff --git a/integration-tests/malformed/expectations/malformed.json b/integration-tests/test-pipelines/run-malformed-pipeline/expectations/malformed.json similarity index 100% rename from integration-tests/malformed/expectations/malformed.json rename to integration-tests/test-pipelines/run-malformed-pipeline/expectations/malformed.json diff --git a/integration-tests/malformed/pipeline.yml b/integration-tests/test-pipelines/run-malformed-pipeline/pipeline.yml similarity index 100% rename from integration-tests/malformed/pipeline.yml rename to integration-tests/test-pipelines/run-malformed-pipeline/pipeline.yml From b2178cef45a40a37cc045916646961ec1117590c Mon Sep 17 00:00:00 2001 From: Baris Terzioglu Date: Mon, 30 Dec 2024 23:36:28 +0100 Subject: [PATCH 4/8] more pipelines --- integration-tests/integration-test.go | 14 +- .../faulty-pipeline/assets/error.sql | 0 .../faulty-pipeline/assets/nonexistent.sql | 0 .../faulty-pipeline/pipeline.yml | 0 .../assets/country.sql | 0 .../assets/example.sql | 0 .../assets/people.sql | 0 .../assets/users.sql | 0 .../expectations/lineage-asset.json | 0 .../expectations/lineage.json | 0 .../pipeline.yml | 0 .../parse-lineage-pipeline/assets/country.sql | 11 + .../parse-lineage-pipeline/assets/example.sql | 19 + .../parse-lineage-pipeline/assets/people.sql | 16 + .../parse-lineage-pipeline/assets/users.sql | 27 ++ .../expectations/lineage.json | 455 ++++++++++++++++++ .../parse-lineage-pipeline/pipeline.yml | 2 + 17 files changed, 537 insertions(+), 7 deletions(-) rename integration-tests/{ => test-pipelines}/faulty-pipeline/assets/error.sql (100%) rename integration-tests/{ => test-pipelines}/faulty-pipeline/assets/nonexistent.sql (100%) rename integration-tests/{ => test-pipelines}/faulty-pipeline/pipeline.yml (100%) rename integration-tests/{lineage => test-pipelines/parse-asset-lineage-pipeline}/assets/country.sql (100%) rename integration-tests/{lineage => test-pipelines/parse-asset-lineage-pipeline}/assets/example.sql (100%) rename integration-tests/{lineage => test-pipelines/parse-asset-lineage-pipeline}/assets/people.sql (100%) rename integration-tests/{lineage => test-pipelines/parse-asset-lineage-pipeline}/assets/users.sql (100%) rename integration-tests/{lineage => test-pipelines/parse-asset-lineage-pipeline}/expectations/lineage-asset.json (100%) rename integration-tests/{lineage => test-pipelines/parse-asset-lineage-pipeline}/expectations/lineage.json (100%) rename integration-tests/{lineage => test-pipelines/parse-asset-lineage-pipeline}/pipeline.yml (100%) create mode 100644 integration-tests/test-pipelines/parse-lineage-pipeline/assets/country.sql create mode 100644 integration-tests/test-pipelines/parse-lineage-pipeline/assets/example.sql create mode 100644 integration-tests/test-pipelines/parse-lineage-pipeline/assets/people.sql create mode 100644 integration-tests/test-pipelines/parse-lineage-pipeline/assets/users.sql create mode 100644 integration-tests/test-pipelines/parse-lineage-pipeline/expectations/lineage.json create mode 100644 integration-tests/test-pipelines/parse-lineage-pipeline/pipeline.yml diff --git a/integration-tests/integration-test.go b/integration-tests/integration-test.go index 35546bb6..269fda3d 100644 --- a/integration-tests/integration-test.go +++ b/integration-tests/integration-test.go @@ -436,7 +436,7 @@ func getTasks(binary string, currentFolder string) []e2e.Task { { Name: "parse-asset-faulty-pipeline-error-sql", Command: binary, - Args: []string{"internal", "parse-asset", filepath.Join(currentFolder, "faulty-pipeline/assets/error.sql")}, + Args: []string{"internal", "parse-asset", filepath.Join(currentFolder, "test-pipelines/faulty-pipeline/assets/error.sql")}, Env: []string{}, Expected: e2e.Output{ @@ -509,14 +509,14 @@ func getTasks(binary string, currentFolder string) []e2e.Task { }, }, { - Name: "parse-pipeline-lineage", + Name: "parse-lineage", Command: binary, - Args: []string{"internal", "parse-pipeline", "-c", filepath.Join(currentFolder, "lineage")}, + Args: []string{"internal", "parse-pipeline", "-c", filepath.Join(currentFolder, "test-pipelines/parse-lineage-pipeline")}, Env: []string{}, SkipJSONNodes: []string{"\"path\""}, Expected: e2e.Output{ ExitCode: 0, - Output: helpers.ReadFile(filepath.Join(currentFolder, "lineage/expectations/lineage.json")), + Output: helpers.ReadFile(filepath.Join(currentFolder, "test-pipelines/parse-lineage-pipeline/expectations/lineage.json")), }, Asserts: []func(*e2e.Task) error{ e2e.AssertByExitCode, @@ -524,14 +524,14 @@ func getTasks(binary string, currentFolder string) []e2e.Task { }, }, { - Name: "parse-asset-lineage-example", + Name: "parse-asset-lineage", Command: binary, - Args: []string{"internal", "parse-asset", "-c", filepath.Join(currentFolder, "lineage/assets/example.sql")}, + Args: []string{"internal", "parse-asset", "-c", filepath.Join(currentFolder, "test-pipelines/parse-asset-lineage-pipeline/assets/example.sql")}, Env: []string{}, SkipJSONNodes: []string{"\"path\""}, Expected: e2e.Output{ ExitCode: 0, - Output: helpers.ReadFile(filepath.Join(currentFolder, "lineage/expectations/lineage-asset.json")), + Output: helpers.ReadFile(filepath.Join(currentFolder, "test-pipelines/parse-asset-lineage-pipeline/expectations/lineage-asset.json")), }, Asserts: []func(*e2e.Task) error{ e2e.AssertByExitCode, diff --git a/integration-tests/faulty-pipeline/assets/error.sql b/integration-tests/test-pipelines/faulty-pipeline/assets/error.sql similarity index 100% rename from integration-tests/faulty-pipeline/assets/error.sql rename to integration-tests/test-pipelines/faulty-pipeline/assets/error.sql diff --git a/integration-tests/faulty-pipeline/assets/nonexistent.sql b/integration-tests/test-pipelines/faulty-pipeline/assets/nonexistent.sql similarity index 100% rename from integration-tests/faulty-pipeline/assets/nonexistent.sql rename to integration-tests/test-pipelines/faulty-pipeline/assets/nonexistent.sql diff --git a/integration-tests/faulty-pipeline/pipeline.yml b/integration-tests/test-pipelines/faulty-pipeline/pipeline.yml similarity index 100% rename from integration-tests/faulty-pipeline/pipeline.yml rename to integration-tests/test-pipelines/faulty-pipeline/pipeline.yml diff --git a/integration-tests/lineage/assets/country.sql b/integration-tests/test-pipelines/parse-asset-lineage-pipeline/assets/country.sql similarity index 100% rename from integration-tests/lineage/assets/country.sql rename to integration-tests/test-pipelines/parse-asset-lineage-pipeline/assets/country.sql diff --git a/integration-tests/lineage/assets/example.sql b/integration-tests/test-pipelines/parse-asset-lineage-pipeline/assets/example.sql similarity index 100% rename from integration-tests/lineage/assets/example.sql rename to integration-tests/test-pipelines/parse-asset-lineage-pipeline/assets/example.sql diff --git a/integration-tests/lineage/assets/people.sql b/integration-tests/test-pipelines/parse-asset-lineage-pipeline/assets/people.sql similarity index 100% rename from integration-tests/lineage/assets/people.sql rename to integration-tests/test-pipelines/parse-asset-lineage-pipeline/assets/people.sql diff --git a/integration-tests/lineage/assets/users.sql b/integration-tests/test-pipelines/parse-asset-lineage-pipeline/assets/users.sql similarity index 100% rename from integration-tests/lineage/assets/users.sql rename to integration-tests/test-pipelines/parse-asset-lineage-pipeline/assets/users.sql diff --git a/integration-tests/lineage/expectations/lineage-asset.json b/integration-tests/test-pipelines/parse-asset-lineage-pipeline/expectations/lineage-asset.json similarity index 100% rename from integration-tests/lineage/expectations/lineage-asset.json rename to integration-tests/test-pipelines/parse-asset-lineage-pipeline/expectations/lineage-asset.json diff --git a/integration-tests/lineage/expectations/lineage.json b/integration-tests/test-pipelines/parse-asset-lineage-pipeline/expectations/lineage.json similarity index 100% rename from integration-tests/lineage/expectations/lineage.json rename to integration-tests/test-pipelines/parse-asset-lineage-pipeline/expectations/lineage.json diff --git a/integration-tests/lineage/pipeline.yml b/integration-tests/test-pipelines/parse-asset-lineage-pipeline/pipeline.yml similarity index 100% rename from integration-tests/lineage/pipeline.yml rename to integration-tests/test-pipelines/parse-asset-lineage-pipeline/pipeline.yml diff --git a/integration-tests/test-pipelines/parse-lineage-pipeline/assets/country.sql b/integration-tests/test-pipelines/parse-lineage-pipeline/assets/country.sql new file mode 100644 index 00000000..5e6a4ab5 --- /dev/null +++ b/integration-tests/test-pipelines/parse-lineage-pipeline/assets/country.sql @@ -0,0 +1,11 @@ +/* @bruin +name: country +type: duckdb.sql +materialization: + type: table + +depends: + - users +@bruin */ + +select id, country from users where id > 10000; \ No newline at end of file diff --git a/integration-tests/test-pipelines/parse-lineage-pipeline/assets/example.sql b/integration-tests/test-pipelines/parse-lineage-pipeline/assets/example.sql new file mode 100644 index 00000000..ad67c8b3 --- /dev/null +++ b/integration-tests/test-pipelines/parse-lineage-pipeline/assets/example.sql @@ -0,0 +1,19 @@ +/* @bruin +name: example +type: duckdb.sql +materialization: + type: table + +depends: + - country + - people + +@bruin */ + +select + a.name, + a.last_name, + a.created_at as updated_at, + b.country +from people a +join country b on a.id = b.id; \ No newline at end of file diff --git a/integration-tests/test-pipelines/parse-lineage-pipeline/assets/people.sql b/integration-tests/test-pipelines/parse-lineage-pipeline/assets/people.sql new file mode 100644 index 00000000..59efef3c --- /dev/null +++ b/integration-tests/test-pipelines/parse-lineage-pipeline/assets/people.sql @@ -0,0 +1,16 @@ +/* @bruin +name: people +type: duckdb.sql +materialization: + type: table + +depends: + - users +@bruin */ + +select + id, + name, + last_name, + created_at +from users where country = 'UK'; \ No newline at end of file diff --git a/integration-tests/test-pipelines/parse-lineage-pipeline/assets/users.sql b/integration-tests/test-pipelines/parse-lineage-pipeline/assets/users.sql new file mode 100644 index 00000000..5f1f46ed --- /dev/null +++ b/integration-tests/test-pipelines/parse-lineage-pipeline/assets/users.sql @@ -0,0 +1,27 @@ +/* @bruin +name: users +type: duckdb.sql +materialization: + type: table + + +columns: + - name: id + type: integer + description: "Just a number" + primary_key: true + - name: name + type: varchar + description: "Just a name" + - name: last_name + type: varchar + description: "Just a last name" + - name: country + type: varchar + description: "Just a country" + - name: created_at + type: timestamp + description: "Just a timestamp" +@bruin */ + +select * from user_data; \ No newline at end of file diff --git a/integration-tests/test-pipelines/parse-lineage-pipeline/expectations/lineage.json b/integration-tests/test-pipelines/parse-lineage-pipeline/expectations/lineage.json new file mode 100644 index 00000000..ea3ddf6e --- /dev/null +++ b/integration-tests/test-pipelines/parse-lineage-pipeline/expectations/lineage.json @@ -0,0 +1,455 @@ +{ + "legacy_id": "", + "name": "integration_test", + "schedule": "", + "start_date": "", + "definition_file": { + "name": "pipeline.yml", + "path": "__PIPELINEDIR__pipeline.yml" + }, + "default_parameters": {}, + "default_connections": {}, + "assets": [ + { + "id": "aff64e4fd520bd185cb01adab98d2d20060f621c62d5cad5204712cfa2294ef7", + "uri": "", + "name": "country", + "type": "duckdb.sql", + "description": "", + "connection": "", + "tags": [], + "materialization": { + "type": "table", + "strategy": "", + "partition_by": "", + "cluster_by": null, + "incremental_key": "" + }, + "upstreams": [ + { + "type": "asset", + "value": "users", + "columns": [ + { + "name": "id", + "usage": "" + }, + { + "name": "country", + "usage": "" + } + ] + } + ], + "image": "", + "instance": "", + "owner": "", + "executable_file": { + "name": "country.sql", + "path": "__ASSETSDIR__country.sql", + "content": "" + }, + "definition_file": { + "name": "country.sql", + "path": "__ASSETSDIR__country.sql", + "type": "comment" + }, + "parameters": {}, + "secrets": [], + "columns": [ + { + "entity_attribute": null, + "name": "country", + "type": "varchar", + "description": "", + "primary_key": false, + "update_on_merge": false, + "checks": [], + "upstreams": [ + { + "column": "country", + "table": "users" + } + ] + }, + { + "entity_attribute": null, + "name": "id", + "type": "integer", + "description": "", + "primary_key": false, + "update_on_merge": false, + "checks": [], + "upstreams": [ + { + "column": "id", + "table": "users" + } + ] + } + ], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + }, + { + "id": "50d858e0985ecc7f60418aaf0cc5ab587f42c2570a884095a9e8ccacd0f6545c", + "uri": "", + "name": "example", + "type": "duckdb.sql", + "description": "", + "connection": "", + "tags": [], + "materialization": { + "type": "table", + "strategy": "", + "partition_by": "", + "cluster_by": null, + "incremental_key": "" + }, + "upstreams": [ + { + "type": "asset", + "value": "country", + "columns": [ + { + "name": "id", + "usage": "" + }, + { + "name": "country", + "usage": "" + } + ] + }, + { + "type": "asset", + "value": "people", + "columns": [ + { + "name": "id", + "usage": "" + }, + { + "name": "last_name", + "usage": "" + }, + { + "name": "name", + "usage": "" + }, + { + "name": "updated_at", + "usage": "" + } + ] + } + ], + "image": "", + "instance": "", + "owner": "", + "executable_file": { + "name": "example.sql", + "path": "__ASSETSDIR__example.sql", + "content": "" + }, + "definition_file": { + "name": "example.sql", + "path": "__ASSETSDIR__example.sql", + "type": "comment" + }, + "parameters": {}, + "secrets": [], + "columns": [ + { + "entity_attribute": null, + "name": "country", + "type": "varchar", + "description": "", + "primary_key": false, + "update_on_merge": false, + "checks": [], + "upstreams": [ + { + "column": "country", + "table": "country" + } + ] + }, + { + "entity_attribute": null, + "name": "last_name", + "type": "varchar", + "description": "", + "primary_key": false, + "update_on_merge": false, + "checks": [], + "upstreams": [ + { + "column": "last_name", + "table": "people" + } + ] + }, + { + "entity_attribute": null, + "name": "name", + "type": "varchar", + "description": "", + "primary_key": false, + "update_on_merge": false, + "checks": [], + "upstreams": [ + { + "column": "name", + "table": "people" + } + ] + }, + { + "entity_attribute": null, + "name": "updated_at", + "type": "timestamp", + "description": "", + "primary_key": false, + "update_on_merge": false, + "checks": [], + "upstreams": [ + { + "column": "created_at", + "table": "people" + } + ] + } + ], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + }, + { + "id": "c9022680f888674e2b2274758755bfa07dea729b68d71cde5c521ed70ef261bf", + "uri": "", + "name": "people", + "type": "duckdb.sql", + "description": "", + "connection": "", + "tags": [], + "materialization": { + "type": "table", + "strategy": "", + "partition_by": "", + "cluster_by": null, + "incremental_key": "" + }, + "upstreams": [ + { + "type": "asset", + "value": "users", + "columns": [ + { + "name": "country", + "usage": "" + }, + { + "name": "created_at", + "usage": "" + }, + { + "name": "id", + "usage": "" + }, + { + "name": "last_name", + "usage": "" + }, + { + "name": "name", + "usage": "" + } + ] + } + ], + "image": "", + "instance": "", + "owner": "", + "executable_file": { + "name": "people.sql", + "path": "__ASSETSDIR__people.sql", + "content": "" + }, + "definition_file": { + "name": "people.sql", + "path": "__ASSETSDIR__people.sql", + "type": "comment" + }, + "parameters": {}, + "secrets": [], + "columns": [ + { + "entity_attribute": null, + "name": "created_at", + "type": "timestamp", + "description": "", + "primary_key": false, + "update_on_merge": false, + "checks": [], + "upstreams": [ + { + "column": "created_at", + "table": "users" + } + ] + }, + { + "entity_attribute": null, + "name": "id", + "type": "integer", + "description": "", + "primary_key": false, + "update_on_merge": false, + "checks": [], + "upstreams": [ + { + "column": "id", + "table": "users" + } + ] + }, + { + "entity_attribute": null, + "name": "last_name", + "type": "varchar", + "description": "", + "primary_key": false, + "update_on_merge": false, + "checks": [], + "upstreams": [ + { + "column": "last_name", + "table": "users" + } + ] + }, + { + "entity_attribute": null, + "name": "name", + "type": "varchar", + "description": "", + "primary_key": false, + "update_on_merge": false, + "checks": [], + "upstreams": [ + { + "column": "name", + "table": "users" + } + ] + } + ], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + }, + { + "id": "7dfb4cf67742cb0660305e56ef816c53fcec892cae7f6ee39b75f34e659d672c", + "uri": "", + "name": "users", + "type": "duckdb.sql", + "description": "", + "connection": "", + "tags": [], + "materialization": { + "type": "table", + "strategy": "", + "partition_by": "", + "cluster_by": null, + "incremental_key": "" + }, + "upstreams": [], + "image": "", + "instance": "", + "owner": "", + "executable_file": { + "name": "users.sql", + "path": "__ASSETSDIR__users.sql", + "content": "" + }, + "definition_file": { + "name": "users.sql", + "path": "__ASSETSDIR__users.sql", + "type": "comment" + }, + "parameters": {}, + "secrets": [], + "columns": [ + { + "entity_attribute": null, + "name": "id", + "type": "integer", + "description": "Just a number", + "primary_key": true, + "update_on_merge": false, + "checks": [], + "upstreams": [] + }, + { + "entity_attribute": null, + "name": "name", + "type": "varchar", + "description": "Just a name", + "primary_key": false, + "update_on_merge": false, + "checks": [], + "upstreams": [] + }, + { + "entity_attribute": null, + "name": "last_name", + "type": "varchar", + "description": "Just a last name", + "primary_key": false, + "update_on_merge": false, + "checks": [], + "upstreams": [] + }, + { + "entity_attribute": null, + "name": "country", + "type": "varchar", + "description": "Just a country", + "primary_key": false, + "update_on_merge": false, + "checks": [], + "upstreams": [] + }, + { + "entity_attribute": null, + "name": "created_at", + "type": "timestamp", + "description": "Just a timestamp", + "primary_key": false, + "update_on_merge": false, + "checks": [], + "upstreams": [] + } + ], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + } + ], + "notifications": { + "slack": [], + "ms_teams": [], + "discord": [] + }, + "catchup": false, + "metadata_push": { + "bigquery": false + }, + "retries": 0 +} diff --git a/integration-tests/test-pipelines/parse-lineage-pipeline/pipeline.yml b/integration-tests/test-pipelines/parse-lineage-pipeline/pipeline.yml new file mode 100644 index 00000000..c5361b34 --- /dev/null +++ b/integration-tests/test-pipelines/parse-lineage-pipeline/pipeline.yml @@ -0,0 +1,2 @@ +name: integration_test + From b1760ac58d067db9fc11a2baba32c83613777ffe Mon Sep 17 00:00:00 2001 From: Baris Terzioglu Date: Tue, 31 Dec 2024 00:41:47 +0100 Subject: [PATCH 5/8] more changes --- integration-tests/integration-test.go | 28 +- .../happy-path/assets/asset.py | 0 .../happy-path/assets/chess_games.asset.yml | 0 .../assets/chess_profiles.asset.yml | 0 .../happy-path/assets/player_summary.sql | 0 .../happy-path/expectations/asset.py.json | 0 .../expectations/chess_games.asset.yml.json | 0 .../chess_profiles.asset.yml.json | 0 .../happy-path/expectations/pipeline.yml.json | 0 .../expectations/player_summary.sql.json | 0 .../happy-path/pipeline.yml | 0 .../happy-path/requirements.txt | 0 .../parse-whole-pipeline/assets/asset.py | 25 ++ .../assets/chess_games.asset.yml | 6 + .../assets/chess_profiles.asset.yml | 6 + .../assets/player_summary.sql | 47 +++ .../expectations/asset.py.json | 249 ++++++++++++++++ .../expectations/chess_games.asset.yml.json | 242 ++++++++++++++++ .../chess_profiles.asset.yml.json | 242 ++++++++++++++++ .../expectations/pipeline.yml.json | 202 +++++++++++++ .../expectations/player_summary.sql.json | 273 ++++++++++++++++++ .../parse-whole-pipeline/pipeline.yml | 1 + .../parse-whole-pipeline/requirements.txt | 1 + .../assets/chess_games.asset.yml | 9 + .../assets/chess_profiles.asset.yml | 8 + .../assets/game_outcome_summary.sql | 31 ++ .../assets/player_profile_summary.sql | 30 ++ .../assets/player_summary.sql | 62 ++++ .../pipeline.yml | 1 + 29 files changed, 1449 insertions(+), 14 deletions(-) rename integration-tests/{ => test-pipelines}/happy-path/assets/asset.py (100%) rename integration-tests/{ => test-pipelines}/happy-path/assets/chess_games.asset.yml (100%) rename integration-tests/{ => test-pipelines}/happy-path/assets/chess_profiles.asset.yml (100%) rename integration-tests/{ => test-pipelines}/happy-path/assets/player_summary.sql (100%) rename integration-tests/{ => test-pipelines}/happy-path/expectations/asset.py.json (100%) rename integration-tests/{ => test-pipelines}/happy-path/expectations/chess_games.asset.yml.json (100%) rename integration-tests/{ => test-pipelines}/happy-path/expectations/chess_profiles.asset.yml.json (100%) rename integration-tests/{ => test-pipelines}/happy-path/expectations/pipeline.yml.json (100%) rename integration-tests/{ => test-pipelines}/happy-path/expectations/player_summary.sql.json (100%) rename integration-tests/{ => test-pipelines}/happy-path/pipeline.yml (100%) rename integration-tests/{ => test-pipelines}/happy-path/requirements.txt (100%) create mode 100644 integration-tests/test-pipelines/parse-whole-pipeline/assets/asset.py create mode 100644 integration-tests/test-pipelines/parse-whole-pipeline/assets/chess_games.asset.yml create mode 100644 integration-tests/test-pipelines/parse-whole-pipeline/assets/chess_profiles.asset.yml create mode 100644 integration-tests/test-pipelines/parse-whole-pipeline/assets/player_summary.sql create mode 100644 integration-tests/test-pipelines/parse-whole-pipeline/expectations/asset.py.json create mode 100644 integration-tests/test-pipelines/parse-whole-pipeline/expectations/chess_games.asset.yml.json create mode 100644 integration-tests/test-pipelines/parse-whole-pipeline/expectations/chess_profiles.asset.yml.json create mode 100644 integration-tests/test-pipelines/parse-whole-pipeline/expectations/pipeline.yml.json create mode 100644 integration-tests/test-pipelines/parse-whole-pipeline/expectations/player_summary.sql.json create mode 100644 integration-tests/test-pipelines/parse-whole-pipeline/pipeline.yml create mode 100644 integration-tests/test-pipelines/parse-whole-pipeline/requirements.txt create mode 100644 integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/chess_games.asset.yml create mode 100644 integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/chess_profiles.asset.yml create mode 100644 integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/game_outcome_summary.sql create mode 100644 integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/player_profile_summary.sql create mode 100644 integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/player_summary.sql create mode 100644 integration-tests/test-pipelines/run-main-with-filters-pipeline/pipeline.yml diff --git a/integration-tests/integration-test.go b/integration-tests/integration-test.go index 269fda3d..f5204b00 100644 --- a/integration-tests/integration-test.go +++ b/integration-tests/integration-test.go @@ -238,14 +238,14 @@ func getWorkflow(binary string, currentFolder string, tempfile string) []e2e.Wor func getTasks(binary string, currentFolder string) []e2e.Task { return []e2e.Task{ { - Name: "happy-path", + Name: "parse-whole-pipeline", Command: binary, - Args: []string{"internal", "parse-pipeline", filepath.Join(currentFolder, "happy-path")}, + Args: []string{"internal", "parse-pipeline", filepath.Join(currentFolder, "test-pipelines/parse-whole-pipeline")}, Env: []string{}, SkipJSONNodes: []string{"\"path\""}, Expected: e2e.Output{ ExitCode: 0, - Output: helpers.ReadFile(filepath.Join(currentFolder, "happy-path/expectations/pipeline.yml.json")), + Output: helpers.ReadFile(filepath.Join(currentFolder, "test-pipelines/parse-whole-pipeline/expectations/pipeline.yml.json")), }, Asserts: []func(*e2e.Task) error{ e2e.AssertByExitCode, @@ -293,9 +293,9 @@ func getTasks(binary string, currentFolder string) []e2e.Task { }, }, { - Name: "run-with-filters-case2", + Name: "run-main-with-filters", Command: binary, - Args: []string{"run", "--tag", "include", "--exclude-tag", "exclude", "--only", "main", filepath.Join(currentFolder, "test-pipelines/run-with-filters-case2-pipeline")}, + Args: []string{"run", "--tag", "include", "--exclude-tag", "exclude", "--only", "main", filepath.Join(currentFolder, "test-pipelines/run-main-with-filters-pipeline")}, Env: []string{}, Expected: e2e.Output{ @@ -352,7 +352,7 @@ func getTasks(binary string, currentFolder string) []e2e.Task { { Name: "validate-happy-path", Command: binary, - Args: []string{"validate", filepath.Join(currentFolder, "happy-path")}, + Args: []string{"validate", filepath.Join(currentFolder, "test-pipelines/happy-path")}, Env: []string{}, Expected: e2e.Output{ ExitCode: 0, @@ -376,12 +376,12 @@ func getTasks(binary string, currentFolder string) []e2e.Task { { Name: "parse-asset-happy-path-asset-py", Command: binary, - Args: []string{"internal", "parse-asset", filepath.Join(currentFolder, "happy-path/assets/asset.py")}, + Args: []string{"internal", "parse-asset", filepath.Join(currentFolder, "test-pipelines/happy-path/assets/asset.py")}, Env: []string{}, SkipJSONNodes: []string{"\"path\""}, Expected: e2e.Output{ ExitCode: 0, - Output: helpers.ReadFile(filepath.Join(currentFolder, "happy-path/expectations/asset.py.json")), + Output: helpers.ReadFile(filepath.Join(currentFolder, "test-pipelines/happy-path/expectations/asset.py.json")), }, Asserts: []func(*e2e.Task) error{ e2e.AssertByExitCode, @@ -391,12 +391,12 @@ func getTasks(binary string, currentFolder string) []e2e.Task { { Name: "parse-asset-happy-path-chess-games", Command: binary, - Args: []string{"internal", "parse-asset", filepath.Join(currentFolder, "happy-path/assets/chess_games.asset.yml")}, + Args: []string{"internal", "parse-asset", filepath.Join(currentFolder, "test-pipelines/happy-path/assets/chess_games.asset.yml")}, Env: []string{}, SkipJSONNodes: []string{"\"path\""}, Expected: e2e.Output{ ExitCode: 0, - Output: helpers.ReadFile(filepath.Join(currentFolder, "happy-path/expectations/chess_games.asset.yml.json")), + Output: helpers.ReadFile(filepath.Join(currentFolder, "test-pipelines/happy-path/expectations/chess_games.asset.yml.json")), }, Asserts: []func(*e2e.Task) error{ e2e.AssertByExitCode, @@ -406,12 +406,12 @@ func getTasks(binary string, currentFolder string) []e2e.Task { { Name: "parse-asset-happy-path-chess-profiles", Command: binary, - Args: []string{"internal", "parse-asset", filepath.Join(currentFolder, "happy-path/assets/chess_profiles.asset.yml")}, + Args: []string{"internal", "parse-asset", filepath.Join(currentFolder, "test-pipelines/happy-path/assets/chess_profiles.asset.yml")}, Env: []string{}, SkipJSONNodes: []string{"\"path\""}, Expected: e2e.Output{ ExitCode: 0, - Output: helpers.ReadFile(filepath.Join(currentFolder, "happy-path/expectations/chess_profiles.asset.yml.json")), + Output: helpers.ReadFile(filepath.Join(currentFolder, "test-pipelines/happy-path/expectations/chess_profiles.asset.yml.json")), }, Asserts: []func(*e2e.Task) error{ e2e.AssertByExitCode, @@ -421,12 +421,12 @@ func getTasks(binary string, currentFolder string) []e2e.Task { { Name: "parse-asset-happy-path-player-summary", Command: binary, - Args: []string{"internal", "parse-asset", filepath.Join(currentFolder, "happy-path/assets/player_summary.sql")}, + Args: []string{"internal", "parse-asset", filepath.Join(currentFolder, "test-pipelines/happy-path/assets/player_summary.sql")}, Env: []string{}, SkipJSONNodes: []string{"\"path\""}, Expected: e2e.Output{ ExitCode: 0, - Output: helpers.ReadFile(filepath.Join(currentFolder, "happy-path/expectations/player_summary.sql.json")), + Output: helpers.ReadFile(filepath.Join(currentFolder, "test-pipelines/happy-path/expectations/player_summary.sql.json")), }, Asserts: []func(*e2e.Task) error{ e2e.AssertByExitCode, diff --git a/integration-tests/happy-path/assets/asset.py b/integration-tests/test-pipelines/happy-path/assets/asset.py similarity index 100% rename from integration-tests/happy-path/assets/asset.py rename to integration-tests/test-pipelines/happy-path/assets/asset.py diff --git a/integration-tests/happy-path/assets/chess_games.asset.yml b/integration-tests/test-pipelines/happy-path/assets/chess_games.asset.yml similarity index 100% rename from integration-tests/happy-path/assets/chess_games.asset.yml rename to integration-tests/test-pipelines/happy-path/assets/chess_games.asset.yml diff --git a/integration-tests/happy-path/assets/chess_profiles.asset.yml b/integration-tests/test-pipelines/happy-path/assets/chess_profiles.asset.yml similarity index 100% rename from integration-tests/happy-path/assets/chess_profiles.asset.yml rename to integration-tests/test-pipelines/happy-path/assets/chess_profiles.asset.yml diff --git a/integration-tests/happy-path/assets/player_summary.sql b/integration-tests/test-pipelines/happy-path/assets/player_summary.sql similarity index 100% rename from integration-tests/happy-path/assets/player_summary.sql rename to integration-tests/test-pipelines/happy-path/assets/player_summary.sql diff --git a/integration-tests/happy-path/expectations/asset.py.json b/integration-tests/test-pipelines/happy-path/expectations/asset.py.json similarity index 100% rename from integration-tests/happy-path/expectations/asset.py.json rename to integration-tests/test-pipelines/happy-path/expectations/asset.py.json diff --git a/integration-tests/happy-path/expectations/chess_games.asset.yml.json b/integration-tests/test-pipelines/happy-path/expectations/chess_games.asset.yml.json similarity index 100% rename from integration-tests/happy-path/expectations/chess_games.asset.yml.json rename to integration-tests/test-pipelines/happy-path/expectations/chess_games.asset.yml.json diff --git a/integration-tests/happy-path/expectations/chess_profiles.asset.yml.json b/integration-tests/test-pipelines/happy-path/expectations/chess_profiles.asset.yml.json similarity index 100% rename from integration-tests/happy-path/expectations/chess_profiles.asset.yml.json rename to integration-tests/test-pipelines/happy-path/expectations/chess_profiles.asset.yml.json diff --git a/integration-tests/happy-path/expectations/pipeline.yml.json b/integration-tests/test-pipelines/happy-path/expectations/pipeline.yml.json similarity index 100% rename from integration-tests/happy-path/expectations/pipeline.yml.json rename to integration-tests/test-pipelines/happy-path/expectations/pipeline.yml.json diff --git a/integration-tests/happy-path/expectations/player_summary.sql.json b/integration-tests/test-pipelines/happy-path/expectations/player_summary.sql.json similarity index 100% rename from integration-tests/happy-path/expectations/player_summary.sql.json rename to integration-tests/test-pipelines/happy-path/expectations/player_summary.sql.json diff --git a/integration-tests/happy-path/pipeline.yml b/integration-tests/test-pipelines/happy-path/pipeline.yml similarity index 100% rename from integration-tests/happy-path/pipeline.yml rename to integration-tests/test-pipelines/happy-path/pipeline.yml diff --git a/integration-tests/happy-path/requirements.txt b/integration-tests/test-pipelines/happy-path/requirements.txt similarity index 100% rename from integration-tests/happy-path/requirements.txt rename to integration-tests/test-pipelines/happy-path/requirements.txt diff --git a/integration-tests/test-pipelines/parse-whole-pipeline/assets/asset.py b/integration-tests/test-pipelines/parse-whole-pipeline/assets/asset.py new file mode 100644 index 00000000..b3dc6f55 --- /dev/null +++ b/integration-tests/test-pipelines/parse-whole-pipeline/assets/asset.py @@ -0,0 +1,25 @@ +""" @bruin + +name: python_asset +image: python:3.11 +depends: + - chess_playground.player_summary + +secrets: + - key: KEY1 + inject_as: INJECTED1 + +@bruin """ + +import os +import duckdb + +if os.getenv('INJECTED1') != "value1": + raise Exception("KEY1 is not injected correctly") + +con = duckdb.connect(database = "duckdb.db", read_only = False) + +con.execute("SELECT * FROM chess_playground.player_summary") +result = con.fetchall() +if len(result) != 2: + raise Exception("Incorrect number of rows in player_summary") diff --git a/integration-tests/test-pipelines/parse-whole-pipeline/assets/chess_games.asset.yml b/integration-tests/test-pipelines/parse-whole-pipeline/assets/chess_games.asset.yml new file mode 100644 index 00000000..caa10252 --- /dev/null +++ b/integration-tests/test-pipelines/parse-whole-pipeline/assets/chess_games.asset.yml @@ -0,0 +1,6 @@ +name: chess_playground.games +type: ingestr +parameters: + source_connection: chess-default + source_table: games + destination: duckdb \ No newline at end of file diff --git a/integration-tests/test-pipelines/parse-whole-pipeline/assets/chess_profiles.asset.yml b/integration-tests/test-pipelines/parse-whole-pipeline/assets/chess_profiles.asset.yml new file mode 100644 index 00000000..d8158e02 --- /dev/null +++ b/integration-tests/test-pipelines/parse-whole-pipeline/assets/chess_profiles.asset.yml @@ -0,0 +1,6 @@ +name: chess_playground.profiles +type: ingestr +parameters: + source_connection: chess-default + source_table: profiles + destination: duckdb \ No newline at end of file diff --git a/integration-tests/test-pipelines/parse-whole-pipeline/assets/player_summary.sql b/integration-tests/test-pipelines/parse-whole-pipeline/assets/player_summary.sql new file mode 100644 index 00000000..b4d4432c --- /dev/null +++ b/integration-tests/test-pipelines/parse-whole-pipeline/assets/player_summary.sql @@ -0,0 +1,47 @@ +/* @bruin + +name: chess_playground.player_summary +type: duckdb.sql +materialization: + type: table + +depends: + - chess_playground.games + - chess_playground.profiles + +columns: + - name: total_games + type: integer + description: "the games" + checks: + - name: positive + +@bruin */ + +WITH game_results AS ( + SELECT + CASE + WHEN g.white->>'result' = 'win' THEN g.white->>'@id' + WHEN g.black->>'result' = 'win' THEN g.black->>'@id' + ELSE NULL + END AS winner_aid, + g.white->>'@id' AS white_aid, + g.black->>'@id' AS black_aid +FROM chess_playground.games g +) + +SELECT + p.username, + p.aid, + COUNT(*) AS total_games, + COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS white_wins, + COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS black_wins, + COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END) AS white_games, + COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END) AS black_games, + ROUND(COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END), 0), 2) AS white_win_rate, + ROUND(COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END), 0), 2) AS black_win_rate +FROM chess_playground.profiles p +LEFT JOIN game_results g + ON p.aid IN (g.white_aid, g.black_aid) +GROUP BY p.username, p.aid +ORDER BY total_games DESC diff --git a/integration-tests/test-pipelines/parse-whole-pipeline/expectations/asset.py.json b/integration-tests/test-pipelines/parse-whole-pipeline/expectations/asset.py.json new file mode 100644 index 00000000..7a33d045 --- /dev/null +++ b/integration-tests/test-pipelines/parse-whole-pipeline/expectations/asset.py.json @@ -0,0 +1,249 @@ +{ + "asset": { + "id": "17c2f1111545c0e72ea13a3a07fb3d5a4d96074d128ed10c1c03cc477401d61a", + "uri": "", + "name": "python_asset", + "type": "python", + "description": "", + "connection": "", + "tags": [], + "materialization": null, + "upstreams": [ + { + "type": "asset", + "value": "chess_playground.player_summary", + "columns": [] + } + ], + "image": "python:3.11", + "instance": "", + "owner": "", + "executable_file": { + "name": "asset.py", + "path": "/integration-tests/happy-path/assets/asset.py", + "content": "import os\nimport duckdb\n\nif os.getenv('INJECTED1') != \"value1\":\n raise Exception(\"KEY1 is not injected correctly\")\n\ncon = duckdb.connect(database = \"duckdb.db\", read_only = False)\n\ncon.execute(\"SELECT * FROM chess_playground.player_summary\")\nresult = con.fetchall()\nif len(result) != 2:\n raise Exception(\"Incorrect number of rows in player_summary\")" + }, + "definition_file": { + "name": "asset.py", + "path": "/integration-tests/happy-path/assets/asset.py", + "type": "comment" + }, + "parameters": {}, + "secrets": [ + { + "secret_key": "KEY1", + "injected_key": "INJECTED1" + } + ], + "columns": [], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + }, + "pipeline": { + "legacy_id": "", + "name": "integration_test", + "schedule": "", + "start_date": "", + "definition_file": { + "name": "pipeline.yml", + "path": "/integration-tests/happy-path/pipeline.yml" + }, + "default_parameters": {}, + "default_connections": {}, + "assets": [ + { + "id": "17c2f1111545c0e72ea13a3a07fb3d5a4d96074d128ed10c1c03cc477401d61a", + "uri": "", + "name": "python_asset", + "type": "python", + "description": "", + "connection": "", + "tags": [], + "materialization": null, + "upstreams": [ + { + "type": "asset", + "value": "chess_playground.player_summary", + "columns": [] + } + ], + "image": "python:3.11", + "instance": "", + "owner": "", + "executable_file": { + "name": "asset.py", + "path": "/integration-tests/happy-path/assets/asset.py", + "content": "import os\nimport duckdb\n\nif os.getenv('INJECTED1') != \"value1\":\n raise Exception(\"KEY1 is not injected correctly\")\n\ncon = duckdb.connect(database = \"duckdb.db\", read_only = False)\n\ncon.execute(\"SELECT * FROM chess_playground.player_summary\")\nresult = con.fetchall()\nif len(result) != 2:\n raise Exception(\"Incorrect number of rows in player_summary\")" + }, + "definition_file": { + "name": "asset.py", + "path": "/integration-tests/happy-path/assets/asset.py", + "type": "comment" + }, + "parameters": {}, + "secrets": [ + { + "secret_key": "KEY1", + "injected_key": "INJECTED1" + } + ], + "columns": [], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + }, + { + "id": "209c299a591add072bfa259ad5f311ab7c5aa154960a55b20f3d6de33bb8f21b", + "uri": "", + "name": "chess_playground.games", + "type": "ingestr", + "description": "", + "connection": "", + "tags": [], + "materialization": null, + "upstreams": [], + "image": "", + "instance": "", + "owner": "", + "executable_file": { + "name": "chess_games.asset.yml", + "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", + "content": "name: chess_playground.games\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: games\n destination: duckdb" + }, + "definition_file": { + "name": "chess_games.asset.yml", + "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", + "type": "yaml" + }, + "parameters": { + "destination": "duckdb", + "source_connection": "chess-default", + "source_table": "games" + }, + "secrets": [], + "columns": [], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + }, + { + "id": "c53385eb13eb4d3d102be02b0d3fe4a10661339b3c098de8b226f7317fc47d21", + "uri": "", + "name": "chess_playground.profiles", + "type": "ingestr", + "description": "", + "connection": "", + "tags": [], + "materialization": null, + "upstreams": [], + "image": "", + "instance": "", + "owner": "", + "executable_file": { + "name": "chess_profiles.asset.yml", + "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", + "content": "name: chess_playground.profiles\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: profiles\n destination: duckdb" + }, + "definition_file": { + "name": "chess_profiles.asset.yml", + "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", + "type": "yaml" + }, + "parameters": { + "destination": "duckdb", + "source_connection": "chess-default", + "source_table": "profiles" + }, + "secrets": [], + "columns": [], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + }, + { + "id": "8361c0131fda306b28fd4f3c8f2af121cca5e57baf77a771b4cb218abda4ca5b", + "uri": "", + "name": "chess_playground.player_summary", + "type": "duckdb.sql", + "description": "", + "connection": "", + "tags": [], + "materialization": { + "type": "table", + "strategy": "", + "partition_by": "", + "cluster_by": null, + "incremental_key": "" + }, + "upstreams": [ + { + "type": "asset", + "value": "chess_playground.games", + "columns": [] + }, + { + "type": "asset", + "value": "chess_playground.profiles", + "columns": [] + } + ], + "image": "", + "instance": "", + "owner": "", + "executable_file": { + "name": "player_summary.sql", + "path": "/integration-tests/happy-path/assets/player_summary.sql", + "content": "WITH game_results AS (\n SELECT\n CASE\n WHEN g.white-\u003e\u003e'result' = 'win' THEN g.white-\u003e\u003e'@id'\n WHEN g.black-\u003e\u003e'result' = 'win' THEN g.black-\u003e\u003e'@id'\n ELSE NULL\n END AS winner_aid,\n g.white-\u003e\u003e'@id' AS white_aid,\n g.black-\u003e\u003e'@id' AS black_aid\nFROM chess_playground.games g\n)\n\nSELECT\n p.username,\n p.aid,\n COUNT(*) AS total_games,\n COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS white_wins,\n COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS black_wins,\n COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END) AS white_games,\n COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END) AS black_games,\n ROUND(COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END), 0), 2) AS white_win_rate,\n ROUND(COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END), 0), 2) AS black_win_rate\nFROM chess_playground.profiles p\nLEFT JOIN game_results g\n ON p.aid IN (g.white_aid, g.black_aid)\nGROUP BY p.username, p.aid\nORDER BY total_games DESC" + }, + "definition_file": { + "name": "player_summary.sql", + "path": "/integration-tests/happy-path/assets/player_summary.sql", + "type": "comment" + }, + "parameters": {}, + "secrets": [], + "columns": [ + { + "entity_attribute": null, + "name": "total_games", + "type": "integer", + "description": "the games", + "primary_key": false, + "update_on_merge": false, + "checks": [ + { + "id": "df9255080865c27b164a7de36a0a26bcc00345dddd66849d96aa96a2c68266ea", + "name": "positive", + "value": null, + "blocking": true + } + ], + "upstreams": [] + } + ], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + } + ], + "notifications": { + "slack": [], + "ms_teams": [], + "discord": [] + }, + "catchup": false, + "metadata_push": { + "bigquery": false + }, + "retries": 0 + }, + "repo": { + "path": "/integration-tests" + } +} diff --git a/integration-tests/test-pipelines/parse-whole-pipeline/expectations/chess_games.asset.yml.json b/integration-tests/test-pipelines/parse-whole-pipeline/expectations/chess_games.asset.yml.json new file mode 100644 index 00000000..c3fe78ca --- /dev/null +++ b/integration-tests/test-pipelines/parse-whole-pipeline/expectations/chess_games.asset.yml.json @@ -0,0 +1,242 @@ +{ + "asset": { + "id": "209c299a591add072bfa259ad5f311ab7c5aa154960a55b20f3d6de33bb8f21b", + "uri": "", + "name": "chess_playground.games", + "type": "ingestr", + "description": "", + "connection": "", + "tags": [], + "materialization": null, + "upstreams": [], + "image": "", + "instance": "", + "owner": "", + "executable_file": { + "name": "chess_games.asset.yml", + "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", + "content": "name: chess_playground.games\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: games\n destination: duckdb" + }, + "definition_file": { + "name": "chess_games.asset.yml", + "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", + "type": "yaml" + }, + "parameters": { + "destination": "duckdb", + "source_connection": "chess-default", + "source_table": "games" + }, + "secrets": [], + "columns": [], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + }, + "pipeline": { + "legacy_id": "", + "name": "integration_test", + "schedule": "", + "start_date": "", + "definition_file": { + "name": "pipeline.yml", + "path": "/integration-tests/happy-path/pipeline.yml" + }, + "default_parameters": {}, + "default_connections": {}, + "assets": [ + { + "id": "17c2f1111545c0e72ea13a3a07fb3d5a4d96074d128ed10c1c03cc477401d61a", + "uri": "", + "name": "python_asset", + "type": "python", + "description": "", + "connection": "", + "tags": [], + "materialization": null, + "upstreams": [ + { + "type": "asset", + "value": "chess_playground.player_summary", + "columns": [] + } + ], + "image": "python:3.11", + "instance": "", + "owner": "", + "executable_file": { + "name": "asset.py", + "path": "/integration-tests/happy-path/assets/asset.py", + "content": "import os\nimport duckdb\n\nif os.getenv('INJECTED1') != \"value1\":\n raise Exception(\"KEY1 is not injected correctly\")\n\ncon = duckdb.connect(database = \"duckdb.db\", read_only = False)\n\ncon.execute(\"SELECT * FROM chess_playground.player_summary\")\nresult = con.fetchall()\nif len(result) != 2:\n raise Exception(\"Incorrect number of rows in player_summary\")" + }, + "definition_file": { + "name": "asset.py", + "path": "/integration-tests/happy-path/assets/asset.py", + "type": "comment" + }, + "parameters": {}, + "secrets": [ + { + "secret_key": "KEY1", + "injected_key": "INJECTED1" + } + ], + "columns": [], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + }, + { + "id": "209c299a591add072bfa259ad5f311ab7c5aa154960a55b20f3d6de33bb8f21b", + "uri": "", + "name": "chess_playground.games", + "type": "ingestr", + "description": "", + "connection": "", + "tags": [], + "materialization": null, + "upstreams": [], + "image": "", + "instance": "", + "owner": "", + "executable_file": { + "name": "chess_games.asset.yml", + "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", + "content": "name: chess_playground.games\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: games\n destination: duckdb" + }, + "definition_file": { + "name": "chess_games.asset.yml", + "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", + "type": "yaml" + }, + "parameters": { + "destination": "duckdb", + "source_connection": "chess-default", + "source_table": "games" + }, + "secrets": [], + "columns": [], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + }, + { + "id": "c53385eb13eb4d3d102be02b0d3fe4a10661339b3c098de8b226f7317fc47d21", + "uri": "", + "name": "chess_playground.profiles", + "type": "ingestr", + "description": "", + "connection": "", + "tags": [], + "materialization": null, + "upstreams": [], + "image": "", + "instance": "", + "owner": "", + "executable_file": { + "name": "chess_profiles.asset.yml", + "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", + "content": "name: chess_playground.profiles\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: profiles\n destination: duckdb" + }, + "definition_file": { + "name": "chess_profiles.asset.yml", + "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", + "type": "yaml" + }, + "parameters": { + "destination": "duckdb", + "source_connection": "chess-default", + "source_table": "profiles" + }, + "secrets": [], + "columns": [], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + }, + { + "id": "8361c0131fda306b28fd4f3c8f2af121cca5e57baf77a771b4cb218abda4ca5b", + "uri": "", + "name": "chess_playground.player_summary", + "type": "duckdb.sql", + "description": "", + "connection": "", + "tags": [], + "materialization": { + "type": "table", + "strategy": "", + "partition_by": "", + "cluster_by": null, + "incremental_key": "" + }, + "upstreams": [ + { + "type": "asset", + "value": "chess_playground.games", + "columns": [] + }, + { + "type": "asset", + "value": "chess_playground.profiles", + "columns": [] + } + ], + "image": "", + "instance": "", + "owner": "", + "executable_file": { + "name": "player_summary.sql", + "path": "/integration-tests/happy-path/assets/player_summary.sql", + "content": "WITH game_results AS (\n SELECT\n CASE\n WHEN g.white->>'result' = 'win' THEN g.white->>'@id'\n WHEN g.black->>'result' = 'win' THEN g.black->>'@id'\n ELSE NULL\n END AS winner_aid,\n g.white->>'@id' AS white_aid,\n g.black->>'@id' AS black_aid\nFROM chess_playground.games g\n)\n\nSELECT\n p.username,\n p.aid,\n COUNT(*) AS total_games,\n COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS white_wins,\n COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS black_wins,\n COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END) AS white_games,\n COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END) AS black_games,\n ROUND(COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END), 0), 2) AS white_win_rate,\n ROUND(COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END), 0), 2) AS black_win_rate\nFROM chess_playground.profiles p\nLEFT JOIN game_results g\n ON p.aid IN (g.white_aid, g.black_aid)\nGROUP BY p.username, p.aid\nORDER BY total_games DESC" + }, + "definition_file": { + "name": "player_summary.sql", + "path": "/integration-tests/happy-path/assets/player_summary.sql", + "type": "comment" + }, + "parameters": {}, + "secrets": [], + "columns": [ + { + "entity_attribute": null, + "name": "total_games", + "type": "integer", + "description": "the games", + "primary_key": false, + "update_on_merge": false, + "checks": [ + { + "id": "df9255080865c27b164a7de36a0a26bcc00345dddd66849d96aa96a2c68266ea", + "name": "positive", + "value": null, + "blocking": true + } + ], + "upstreams": [] + } + ], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + } + ], + "notifications": { + "slack": [], + "ms_teams": [], + "discord": [] + }, + "catchup": false, + "metadata_push": { + "bigquery": false + }, + "retries": 0 + }, + "repo": { + "path": "/integration-tests" + } +} diff --git a/integration-tests/test-pipelines/parse-whole-pipeline/expectations/chess_profiles.asset.yml.json b/integration-tests/test-pipelines/parse-whole-pipeline/expectations/chess_profiles.asset.yml.json new file mode 100644 index 00000000..2538260b --- /dev/null +++ b/integration-tests/test-pipelines/parse-whole-pipeline/expectations/chess_profiles.asset.yml.json @@ -0,0 +1,242 @@ +{ + "asset": { + "id": "c53385eb13eb4d3d102be02b0d3fe4a10661339b3c098de8b226f7317fc47d21", + "uri": "", + "name": "chess_playground.profiles", + "type": "ingestr", + "description": "", + "connection": "", + "tags": [], + "materialization": null, + "upstreams": [], + "image": "", + "instance": "", + "owner": "", + "executable_file": { + "name": "chess_profiles.asset.yml", + "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", + "content": "name: chess_playground.profiles\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: profiles\n destination: duckdb" + }, + "definition_file": { + "name": "chess_profiles.asset.yml", + "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", + "type": "yaml" + }, + "parameters": { + "destination": "duckdb", + "source_connection": "chess-default", + "source_table": "profiles" + }, + "secrets": [], + "columns": [], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + }, + "pipeline": { + "legacy_id": "", + "name": "integration_test", + "schedule": "", + "start_date": "", + "definition_file": { + "name": "pipeline.yml", + "path": "/integration-tests/happy-path/pipeline.yml" + }, + "default_parameters": {}, + "default_connections": {}, + "assets": [ + { + "id": "17c2f1111545c0e72ea13a3a07fb3d5a4d96074d128ed10c1c03cc477401d61a", + "uri": "", + "name": "python_asset", + "type": "python", + "description": "", + "connection": "", + "tags": [], + "materialization": null, + "upstreams": [ + { + "type": "asset", + "value": "chess_playground.player_summary", + "columns": [] + } + ], + "image": "python:3.11", + "instance": "", + "owner": "", + "executable_file": { + "name": "asset.py", + "path": "/integration-tests/happy-path/assets/asset.py", + "content": "import os\nimport duckdb\n\nif os.getenv('INJECTED1') != \"value1\":\n raise Exception(\"KEY1 is not injected correctly\")\n\ncon = duckdb.connect(database = \"duckdb.db\", read_only = False)\n\ncon.execute(\"SELECT * FROM chess_playground.player_summary\")\nresult = con.fetchall()\nif len(result) != 2:\n raise Exception(\"Incorrect number of rows in player_summary\")" + }, + "definition_file": { + "name": "asset.py", + "path": "/integration-tests/happy-path/assets/asset.py", + "type": "comment" + }, + "parameters": {}, + "secrets": [ + { + "secret_key": "KEY1", + "injected_key": "INJECTED1" + } + ], + "columns": [], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + }, + { + "id": "209c299a591add072bfa259ad5f311ab7c5aa154960a55b20f3d6de33bb8f21b", + "uri": "", + "name": "chess_playground.games", + "type": "ingestr", + "description": "", + "connection": "", + "tags": [], + "materialization": null, + "upstreams": [], + "image": "", + "instance": "", + "owner": "", + "executable_file": { + "name": "chess_games.asset.yml", + "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", + "content": "name: chess_playground.games\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: games\n destination: duckdb" + }, + "definition_file": { + "name": "chess_games.asset.yml", + "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", + "type": "yaml" + }, + "parameters": { + "destination": "duckdb", + "source_connection": "chess-default", + "source_table": "games" + }, + "secrets": [], + "columns": [], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + }, + { + "id": "c53385eb13eb4d3d102be02b0d3fe4a10661339b3c098de8b226f7317fc47d21", + "uri": "", + "name": "chess_playground.profiles", + "type": "ingestr", + "description": "", + "connection": "", + "tags": [], + "materialization": null, + "upstreams": [], + "image": "", + "instance": "", + "owner": "", + "executable_file": { + "name": "chess_profiles.asset.yml", + "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", + "content": "name: chess_playground.profiles\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: profiles\n destination: duckdb" + }, + "definition_file": { + "name": "chess_profiles.asset.yml", + "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", + "type": "yaml" + }, + "parameters": { + "destination": "duckdb", + "source_connection": "chess-default", + "source_table": "profiles" + }, + "secrets": [], + "columns": [], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + }, + { + "id": "8361c0131fda306b28fd4f3c8f2af121cca5e57baf77a771b4cb218abda4ca5b", + "uri": "", + "name": "chess_playground.player_summary", + "type": "duckdb.sql", + "description": "", + "connection": "", + "tags": [], + "materialization": { + "type": "table", + "strategy": "", + "partition_by": "", + "cluster_by": null, + "incremental_key": "" + }, + "upstreams": [ + { + "type": "asset", + "value": "chess_playground.games", + "columns": [] + }, + { + "type": "asset", + "value": "chess_playground.profiles", + "columns": [] + } + ], + "image": "", + "instance": "", + "owner": "", + "executable_file": { + "name": "player_summary.sql", + "path": "/integration-tests/happy-path/assets/player_summary.sql", + "content": "WITH game_results AS (\n SELECT\n CASE\n WHEN g.white-\u003e\u003e'result' = 'win' THEN g.white-\u003e\u003e'@id'\n WHEN g.black-\u003e\u003e'result' = 'win' THEN g.black-\u003e\u003e'@id'\n ELSE NULL\n END AS winner_aid,\n g.white-\u003e\u003e'@id' AS white_aid,\n g.black-\u003e\u003e'@id' AS black_aid\nFROM chess_playground.games g\n)\n\nSELECT\n p.username,\n p.aid,\n COUNT(*) AS total_games,\n COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS white_wins,\n COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS black_wins,\n COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END) AS white_games,\n COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END) AS black_games,\n ROUND(COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END), 0), 2) AS white_win_rate,\n ROUND(COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END), 0), 2) AS black_win_rate\nFROM chess_playground.profiles p\nLEFT JOIN game_results g\n ON p.aid IN (g.white_aid, g.black_aid)\nGROUP BY p.username, p.aid\nORDER BY total_games DESC" + }, + "definition_file": { + "name": "player_summary.sql", + "path": "/integration-tests/happy-path/assets/player_summary.sql", + "type": "comment" + }, + "parameters": {}, + "secrets": [], + "columns": [ + { + "entity_attribute": null, + "name": "total_games", + "type": "integer", + "description": "the games", + "primary_key": false, + "update_on_merge": false, + "checks": [ + { + "id": "df9255080865c27b164a7de36a0a26bcc00345dddd66849d96aa96a2c68266ea", + "name": "positive", + "value": null, + "blocking": true + } + ], + "upstreams": [] + } + ], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + } + ], + "notifications": { + "slack": [], + "ms_teams": [], + "discord": [] + }, + "catchup": false, + "metadata_push": { + "bigquery": false + }, + "retries": 0 + }, + "repo": { + "path": "/integration-tests" + } +} diff --git a/integration-tests/test-pipelines/parse-whole-pipeline/expectations/pipeline.yml.json b/integration-tests/test-pipelines/parse-whole-pipeline/expectations/pipeline.yml.json new file mode 100644 index 00000000..d6c9d256 --- /dev/null +++ b/integration-tests/test-pipelines/parse-whole-pipeline/expectations/pipeline.yml.json @@ -0,0 +1,202 @@ +{ + "legacy_id": "", + "name": "integration_test", + "schedule": "", + "start_date": "", + "definition_file": { + "name": "pipeline.yml", + "path": "__PIPELINEDIR__pipeline.yml" + }, + "default_parameters": {}, + "default_connections": {}, + "assets": [ + { + "id": "17c2f1111545c0e72ea13a3a07fb3d5a4d96074d128ed10c1c03cc477401d61a", + "uri": "", + "name": "python_asset", + "type": "python", + "description": "", + "connection": "", + "tags": [], + "materialization": null, + "upstreams": [ + { + "type": "asset", + "value": "chess_playground.player_summary", + "columns": [] + } + ], + "image": "python:3.11", + "instance": "", + "owner": "", + "executable_file": { + "name": "asset.py", + "path": "__ASSETSDIR__asset.py", + "content": "" + }, + "definition_file": { + "name": "asset.py", + "path": "__ASSETSDIR__asset.py", + "type": "comment" + }, + "parameters": {}, + "secrets": [ + { + "secret_key": "KEY1", + "injected_key": "INJECTED1" + } + ], + "columns": [], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + }, + { + "id": "209c299a591add072bfa259ad5f311ab7c5aa154960a55b20f3d6de33bb8f21b", + "uri": "", + "name": "chess_playground.games", + "type": "ingestr", + "description": "", + "connection": "", + "tags": [], + "materialization": null, + "upstreams": [], + "image": "", + "instance": "", + "owner": "", + "executable_file": { + "name": "chess_games.asset.yml", + "path": "__ASSETSDIR__chess_games.asset.yml", + "content": "" + }, + "definition_file": { + "name": "chess_games.asset.yml", + "path": "__ASSETSDIR__chess_games.asset.yml", + "type": "yaml" + }, + "parameters": { + "destination": "duckdb", + "source_connection": "chess-default", + "source_table": "games" + }, + "secrets": [], + "columns": [], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + }, + { + "id": "c53385eb13eb4d3d102be02b0d3fe4a10661339b3c098de8b226f7317fc47d21", + "uri": "", + "name": "chess_playground.profiles", + "type": "ingestr", + "description": "", + "connection": "", + "tags": [], + "materialization": null, + "upstreams": [], + "image": "", + "instance": "", + "owner": "", + "executable_file": { + "name": "chess_profiles.asset.yml", + "path": "__ASSETSDIR__chess_profiles.asset.yml", + "content": "" + }, + "definition_file": { + "name": "chess_profiles.asset.yml", + "path": "__ASSETSDIR__chess_profiles.asset.yml", + "type": "yaml" + }, + "parameters": { + "destination": "duckdb", + "source_connection": "chess-default", + "source_table": "profiles" + }, + "secrets": [], + "columns": [], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + }, + { + "id": "8361c0131fda306b28fd4f3c8f2af121cca5e57baf77a771b4cb218abda4ca5b", + "uri": "", + "name": "chess_playground.player_summary", + "type": "duckdb.sql", + "description": "", + "connection": "", + "tags": [], + "materialization": { + "type": "table", + "strategy": "", + "partition_by": "", + "cluster_by": null, + "incremental_key": "" + }, + "upstreams": [ + { + "type": "asset", + "value": "chess_playground.games", + "columns": [] + }, + { + "type": "asset", + "value": "chess_playground.profiles", + "columns": [] + } + ], + "image": "", + "instance": "", + "owner": "", + "executable_file": { + "name": "player_summary.sql", + "path": "__ASSETSDIR__player_summary.sql", + "content": "" + }, + "definition_file": { + "name": "player_summary.sql", + "path": "__ASSETSDIR__player_summary.sql", + "type": "comment" + }, + "parameters": {}, + "secrets": [], + "columns": [ + { + "entity_attribute": null, + "name": "total_games", + "type": "integer", + "description": "the games", + "primary_key": false, + "update_on_merge": false, + "checks": [ + { + "id": "df9255080865c27b164a7de36a0a26bcc00345dddd66849d96aa96a2c68266ea", + "name": "positive", + "value": null, + "blocking": true + } + ], + "upstreams": [] + } + ], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + } + ], + "notifications": { + "slack": [], + "ms_teams": [], + "discord": [] + }, + "catchup": false, + "metadata_push": { + "bigquery": false + }, + "retries": 0 +} \ No newline at end of file diff --git a/integration-tests/test-pipelines/parse-whole-pipeline/expectations/player_summary.sql.json b/integration-tests/test-pipelines/parse-whole-pipeline/expectations/player_summary.sql.json new file mode 100644 index 00000000..cfe05f4e --- /dev/null +++ b/integration-tests/test-pipelines/parse-whole-pipeline/expectations/player_summary.sql.json @@ -0,0 +1,273 @@ +{ + "asset": { + "id": "8361c0131fda306b28fd4f3c8f2af121cca5e57baf77a771b4cb218abda4ca5b", + "uri": "", + "name": "chess_playground.player_summary", + "type": "duckdb.sql", + "description": "", + "connection": "", + "tags": [], + "materialization": { + "type": "table", + "strategy": "", + "partition_by": "", + "cluster_by": null, + "incremental_key": "" + }, + "upstreams": [ + { + "type": "asset", + "value": "chess_playground.games", + "columns": [] + }, + { + "type": "asset", + "value": "chess_playground.profiles", + "columns": [] + } + ], + "image": "", + "instance": "", + "owner": "", + "executable_file": { + "name": "player_summary.sql", + "path": "/integration-tests/happy-path/assets/player_summary.sql", + "content": "WITH game_results AS (\n SELECT\n CASE\n WHEN g.white-\u003e\u003e'result' = 'win' THEN g.white-\u003e\u003e'@id'\n WHEN g.black-\u003e\u003e'result' = 'win' THEN g.black-\u003e\u003e'@id'\n ELSE NULL\n END AS winner_aid,\n g.white-\u003e\u003e'@id' AS white_aid,\n g.black-\u003e\u003e'@id' AS black_aid\nFROM chess_playground.games g\n)\n\nSELECT\n p.username,\n p.aid,\n COUNT(*) AS total_games,\n COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS white_wins,\n COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS black_wins,\n COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END) AS white_games,\n COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END) AS black_games,\n ROUND(COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END), 0), 2) AS white_win_rate,\n ROUND(COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END), 0), 2) AS black_win_rate\nFROM chess_playground.profiles p\nLEFT JOIN game_results g\n ON p.aid IN (g.white_aid, g.black_aid)\nGROUP BY p.username, p.aid\nORDER BY total_games DESC" + }, + "definition_file": { + "name": "player_summary.sql", + "path": "/integration-tests/happy-path/assets/player_summary.sql", + "type": "comment" + }, + "parameters": {}, + "secrets": [], + "columns": [ + { + "entity_attribute": null, + "name": "total_games", + "type": "integer", + "description": "the games", + "primary_key": false, + "update_on_merge": false, + "checks": [ + { + "id": "df9255080865c27b164a7de36a0a26bcc00345dddd66849d96aa96a2c68266ea", + "name": "positive", + "value": null, + "blocking": true + } + ], + "upstreams": [] + } + ], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + }, + "pipeline": { + "legacy_id": "", + "name": "integration_test", + "schedule": "", + "start_date": "", + "definition_file": { + "name": "pipeline.yml", + "path": "/integration-tests/happy-path/pipeline.yml" + }, + "default_parameters": {}, + "default_connections": {}, + "assets": [ + { + "id": "17c2f1111545c0e72ea13a3a07fb3d5a4d96074d128ed10c1c03cc477401d61a", + "uri": "", + "name": "python_asset", + "type": "python", + "description": "", + "connection": "", + "tags": [], + "materialization": null, + "upstreams": [ + { + "type": "asset", + "value": "chess_playground.player_summary", + "columns": [] + } + ], + "image": "python:3.11", + "instance": "", + "owner": "", + "executable_file": { + "name": "asset.py", + "path": "/integration-tests/happy-path/assets/asset.py", + "content": "import os\nimport duckdb\n\nif os.getenv('INJECTED1') != \"value1\":\n raise Exception(\"KEY1 is not injected correctly\")\n\ncon = duckdb.connect(database = \"duckdb.db\", read_only = False)\n\ncon.execute(\"SELECT * FROM chess_playground.player_summary\")\nresult = con.fetchall()\nif len(result) != 2:\n raise Exception(\"Incorrect number of rows in player_summary\")" + }, + "definition_file": { + "name": "asset.py", + "path": "/integration-tests/happy-path/assets/asset.py", + "type": "comment" + }, + "parameters": {}, + "secrets": [ + { + "secret_key": "KEY1", + "injected_key": "INJECTED1" + } + ], + "columns": [], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + }, + { + "id": "209c299a591add072bfa259ad5f311ab7c5aa154960a55b20f3d6de33bb8f21b", + "uri": "", + "name": "chess_playground.games", + "type": "ingestr", + "description": "", + "connection": "", + "tags": [], + "materialization": null, + "upstreams": [], + "image": "", + "instance": "", + "owner": "", + "executable_file": { + "name": "chess_games.asset.yml", + "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", + "content": "name: chess_playground.games\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: games\n destination: duckdb" + }, + "definition_file": { + "name": "chess_games.asset.yml", + "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", + "type": "yaml" + }, + "parameters": { + "destination": "duckdb", + "source_connection": "chess-default", + "source_table": "games" + }, + "secrets": [], + "columns": [], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + }, + { + "id": "c53385eb13eb4d3d102be02b0d3fe4a10661339b3c098de8b226f7317fc47d21", + "uri": "", + "name": "chess_playground.profiles", + "type": "ingestr", + "description": "", + "connection": "", + "tags": [], + "materialization": null, + "upstreams": [], + "image": "", + "instance": "", + "owner": "", + "executable_file": { + "name": "chess_profiles.asset.yml", + "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", + "content": "name: chess_playground.profiles\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: profiles\n destination: duckdb" + }, + "definition_file": { + "name": "chess_profiles.asset.yml", + "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", + "type": "yaml" + }, + "parameters": { + "destination": "duckdb", + "source_connection": "chess-default", + "source_table": "profiles" + }, + "secrets": [], + "columns": [], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + }, + { + "id": "8361c0131fda306b28fd4f3c8f2af121cca5e57baf77a771b4cb218abda4ca5b", + "uri": "", + "name": "chess_playground.player_summary", + "type": "duckdb.sql", + "description": "", + "connection": "", + "tags": [], + "materialization": { + "type": "table", + "strategy": "", + "partition_by": "", + "cluster_by": null, + "incremental_key": "" + }, + "upstreams": [ + { + "type": "asset", + "value": "chess_playground.games", + "columns": [] + }, + { + "type": "asset", + "value": "chess_playground.profiles", + "columns": [] + } + ], + "image": "", + "instance": "", + "owner": "", + "executable_file": { + "name": "player_summary.sql", + "path": "/integration-tests/happy-path/assets/player_summary.sql", + "content": "WITH game_results AS (\n SELECT\n CASE\n WHEN g.white-\u003e\u003e'result' = 'win' THEN g.white-\u003e\u003e'@id'\n WHEN g.black-\u003e\u003e'result' = 'win' THEN g.black-\u003e\u003e'@id'\n ELSE NULL\n END AS winner_aid,\n g.white-\u003e\u003e'@id' AS white_aid,\n g.black-\u003e\u003e'@id' AS black_aid\nFROM chess_playground.games g\n)\n\nSELECT\n p.username,\n p.aid,\n COUNT(*) AS total_games,\n COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS white_wins,\n COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS black_wins,\n COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END) AS white_games,\n COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END) AS black_games,\n ROUND(COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END), 0), 2) AS white_win_rate,\n ROUND(COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END), 0), 2) AS black_win_rate\nFROM chess_playground.profiles p\nLEFT JOIN game_results g\n ON p.aid IN (g.white_aid, g.black_aid)\nGROUP BY p.username, p.aid\nORDER BY total_games DESC" + }, + "definition_file": { + "name": "player_summary.sql", + "path": "/integration-tests/happy-path/assets/player_summary.sql", + "type": "comment" + }, + "parameters": {}, + "secrets": [], + "columns": [ + { + "entity_attribute": null, + "name": "total_games", + "type": "integer", + "description": "the games", + "primary_key": false, + "update_on_merge": false, + "checks": [ + { + "id": "df9255080865c27b164a7de36a0a26bcc00345dddd66849d96aa96a2c68266ea", + "name": "positive", + "value": null, + "blocking": true + } + ], + "upstreams": [] + } + ], + "custom_checks": [], + "metadata": {}, + "snowflake": null, + "athena": null + } + ], + "notifications": { + "slack": [], + "ms_teams": [], + "discord": [] + }, + "catchup": false, + "metadata_push": { + "bigquery": false + }, + "retries": 0 + }, + "repo": { + "path": "/integration-tests" + } +} diff --git a/integration-tests/test-pipelines/parse-whole-pipeline/pipeline.yml b/integration-tests/test-pipelines/parse-whole-pipeline/pipeline.yml new file mode 100644 index 00000000..2108b581 --- /dev/null +++ b/integration-tests/test-pipelines/parse-whole-pipeline/pipeline.yml @@ -0,0 +1 @@ +name: integration_test \ No newline at end of file diff --git a/integration-tests/test-pipelines/parse-whole-pipeline/requirements.txt b/integration-tests/test-pipelines/parse-whole-pipeline/requirements.txt new file mode 100644 index 00000000..425ad7ab --- /dev/null +++ b/integration-tests/test-pipelines/parse-whole-pipeline/requirements.txt @@ -0,0 +1 @@ +duckdb==1.1.3 \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/chess_games.asset.yml b/integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/chess_games.asset.yml new file mode 100644 index 00000000..68bc4e41 --- /dev/null +++ b/integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/chess_games.asset.yml @@ -0,0 +1,9 @@ +name: chess_playground.games +type: ingestr +tags: + - include + +parameters: + destination: duckdb + source_connection: chess-default + source_table: games diff --git a/integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/chess_profiles.asset.yml b/integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/chess_profiles.asset.yml new file mode 100644 index 00000000..cd770cdb --- /dev/null +++ b/integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/chess_profiles.asset.yml @@ -0,0 +1,8 @@ +name: chess_playground.profiles +type: ingestr +parameters: + source_connection: chess-default + source_table: profiles + destination: duckdb +tags: + - include \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/game_outcome_summary.sql b/integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/game_outcome_summary.sql new file mode 100644 index 00000000..65cb4340 --- /dev/null +++ b/integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/game_outcome_summary.sql @@ -0,0 +1,31 @@ +/* @bruin + +name: chess_playground.game_outcome_summary +type: duckdb.sql +materialization: + type: table + +depends: + - chess_playground.games + +columns: + - name: result_type + type: string + description: "Type of game result (win, draw, etc.)" + - name: total_games + type: integer + description: "Total number of games with this result" + checks: + - name: positive +tags: + - include + +@bruin */ + +SELECT + g.white->>'result' AS result_type, + COUNT(*) AS total_games +FROM chess_playground.games g +WHERE g.white->>'result' IS NOT NULL +GROUP BY g.white->>'result' +ORDER BY total_games DESC; diff --git a/integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/player_profile_summary.sql b/integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/player_profile_summary.sql new file mode 100644 index 00000000..4aedab67 --- /dev/null +++ b/integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/player_profile_summary.sql @@ -0,0 +1,30 @@ +/* @bruin + +name: chess_playground.player_profile_summary +type: duckdb.sql +materialization: + type: table + +depends: + - chess_playground.profiles +columns: + - name: total_players + type: integer + description: "Total number of players in the profiles table" + checks: + - name: positive + - name: active_players + type: integer + description: "Number of players marked as active" + - name: inactive_players + type: integer + description: "Number of players marked as inactive" +tags: + - exclude +@bruin */ + +SELECT + COUNT(*) AS total_players, + COUNT(CASE WHEN p.status = 'active' THEN 1 END) AS active_players, + COUNT(CASE WHEN p.status = 'inactive' THEN 1 END) AS inactive_players +FROM chess_playground.profiles p; diff --git a/integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/player_summary.sql b/integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/player_summary.sql new file mode 100644 index 00000000..0adf29fa --- /dev/null +++ b/integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/player_summary.sql @@ -0,0 +1,62 @@ +/* @bruin + +name: chess_playground.player_summary +type: duckdb.sql +materialization: + type: table + +depends: + - chess_playground.game_outcome_summary + - chess_playground.player_profile_summary + - chess_playground.games + - chess_playground.profiles + +columns: + - name: username + type: string + description: "Username of the player" + - name: total_games + type: integer + description: "Total games played by the player" + checks: + - name: non_negative + - name: total_wins + type: integer + description: "Total games won by the player" + - name: win_rate + type: float + description: "Win rate of the player" +tags: + - include + - exclude +@bruin */ + +WITH game_results AS ( + SELECT + CASE + WHEN g.white->>'result' = 'win' THEN g.white->>'@id' + WHEN g.black->>'result' = 'win' THEN g.black->>'@id' + ELSE NULL + END AS winner_aid, + g.white->>'@id' AS white_aid, + g.black->>'@id' AS black_aid +FROM chess_playground.games g + ) + +SELECT + p.username, + p.aid, + COUNT(*) AS total_games, + COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) + + COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS total_wins, + ROUND( + (COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) + + COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END)) * 100.0 / + NULLIF(COUNT(*), 0), + 2 + ) AS win_rate +FROM chess_playground.profiles p + LEFT JOIN game_results g + ON p.aid IN (g.white_aid, g.black_aid) +GROUP BY p.username, p.aid +ORDER BY total_games DESC; diff --git a/integration-tests/test-pipelines/run-main-with-filters-pipeline/pipeline.yml b/integration-tests/test-pipelines/run-main-with-filters-pipeline/pipeline.yml new file mode 100644 index 00000000..9172e408 --- /dev/null +++ b/integration-tests/test-pipelines/run-main-with-filters-pipeline/pipeline.yml @@ -0,0 +1 @@ +name: chess_extended_duckdb \ No newline at end of file From 63a26d93292645f00a9767b8d57f394f6e783917 Mon Sep 17 00:00:00 2001 From: Baris Terzioglu Date: Tue, 31 Dec 2024 00:48:52 +0100 Subject: [PATCH 6/8] delete unnecessary files --- .../expectations/asset.py.json | 249 ---------------- .../expectations/chess_games.asset.yml.json | 242 ---------------- .../chess_profiles.asset.yml.json | 242 ---------------- .../expectations/player_summary.sql.json | 273 ------------------ 4 files changed, 1006 deletions(-) delete mode 100644 integration-tests/test-pipelines/parse-whole-pipeline/expectations/asset.py.json delete mode 100644 integration-tests/test-pipelines/parse-whole-pipeline/expectations/chess_games.asset.yml.json delete mode 100644 integration-tests/test-pipelines/parse-whole-pipeline/expectations/chess_profiles.asset.yml.json delete mode 100644 integration-tests/test-pipelines/parse-whole-pipeline/expectations/player_summary.sql.json diff --git a/integration-tests/test-pipelines/parse-whole-pipeline/expectations/asset.py.json b/integration-tests/test-pipelines/parse-whole-pipeline/expectations/asset.py.json deleted file mode 100644 index 7a33d045..00000000 --- a/integration-tests/test-pipelines/parse-whole-pipeline/expectations/asset.py.json +++ /dev/null @@ -1,249 +0,0 @@ -{ - "asset": { - "id": "17c2f1111545c0e72ea13a3a07fb3d5a4d96074d128ed10c1c03cc477401d61a", - "uri": "", - "name": "python_asset", - "type": "python", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [ - { - "type": "asset", - "value": "chess_playground.player_summary", - "columns": [] - } - ], - "image": "python:3.11", - "instance": "", - "owner": "", - "executable_file": { - "name": "asset.py", - "path": "/integration-tests/happy-path/assets/asset.py", - "content": "import os\nimport duckdb\n\nif os.getenv('INJECTED1') != \"value1\":\n raise Exception(\"KEY1 is not injected correctly\")\n\ncon = duckdb.connect(database = \"duckdb.db\", read_only = False)\n\ncon.execute(\"SELECT * FROM chess_playground.player_summary\")\nresult = con.fetchall()\nif len(result) != 2:\n raise Exception(\"Incorrect number of rows in player_summary\")" - }, - "definition_file": { - "name": "asset.py", - "path": "/integration-tests/happy-path/assets/asset.py", - "type": "comment" - }, - "parameters": {}, - "secrets": [ - { - "secret_key": "KEY1", - "injected_key": "INJECTED1" - } - ], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - "pipeline": { - "legacy_id": "", - "name": "integration_test", - "schedule": "", - "start_date": "", - "definition_file": { - "name": "pipeline.yml", - "path": "/integration-tests/happy-path/pipeline.yml" - }, - "default_parameters": {}, - "default_connections": {}, - "assets": [ - { - "id": "17c2f1111545c0e72ea13a3a07fb3d5a4d96074d128ed10c1c03cc477401d61a", - "uri": "", - "name": "python_asset", - "type": "python", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [ - { - "type": "asset", - "value": "chess_playground.player_summary", - "columns": [] - } - ], - "image": "python:3.11", - "instance": "", - "owner": "", - "executable_file": { - "name": "asset.py", - "path": "/integration-tests/happy-path/assets/asset.py", - "content": "import os\nimport duckdb\n\nif os.getenv('INJECTED1') != \"value1\":\n raise Exception(\"KEY1 is not injected correctly\")\n\ncon = duckdb.connect(database = \"duckdb.db\", read_only = False)\n\ncon.execute(\"SELECT * FROM chess_playground.player_summary\")\nresult = con.fetchall()\nif len(result) != 2:\n raise Exception(\"Incorrect number of rows in player_summary\")" - }, - "definition_file": { - "name": "asset.py", - "path": "/integration-tests/happy-path/assets/asset.py", - "type": "comment" - }, - "parameters": {}, - "secrets": [ - { - "secret_key": "KEY1", - "injected_key": "INJECTED1" - } - ], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "209c299a591add072bfa259ad5f311ab7c5aa154960a55b20f3d6de33bb8f21b", - "uri": "", - "name": "chess_playground.games", - "type": "ingestr", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "chess_games.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", - "content": "name: chess_playground.games\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: games\n destination: duckdb" - }, - "definition_file": { - "name": "chess_games.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", - "type": "yaml" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-default", - "source_table": "games" - }, - "secrets": [], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "c53385eb13eb4d3d102be02b0d3fe4a10661339b3c098de8b226f7317fc47d21", - "uri": "", - "name": "chess_playground.profiles", - "type": "ingestr", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "chess_profiles.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", - "content": "name: chess_playground.profiles\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: profiles\n destination: duckdb" - }, - "definition_file": { - "name": "chess_profiles.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", - "type": "yaml" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-default", - "source_table": "profiles" - }, - "secrets": [], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "8361c0131fda306b28fd4f3c8f2af121cca5e57baf77a771b4cb218abda4ca5b", - "uri": "", - "name": "chess_playground.player_summary", - "type": "duckdb.sql", - "description": "", - "connection": "", - "tags": [], - "materialization": { - "type": "table", - "strategy": "", - "partition_by": "", - "cluster_by": null, - "incremental_key": "" - }, - "upstreams": [ - { - "type": "asset", - "value": "chess_playground.games", - "columns": [] - }, - { - "type": "asset", - "value": "chess_playground.profiles", - "columns": [] - } - ], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "player_summary.sql", - "path": "/integration-tests/happy-path/assets/player_summary.sql", - "content": "WITH game_results AS (\n SELECT\n CASE\n WHEN g.white-\u003e\u003e'result' = 'win' THEN g.white-\u003e\u003e'@id'\n WHEN g.black-\u003e\u003e'result' = 'win' THEN g.black-\u003e\u003e'@id'\n ELSE NULL\n END AS winner_aid,\n g.white-\u003e\u003e'@id' AS white_aid,\n g.black-\u003e\u003e'@id' AS black_aid\nFROM chess_playground.games g\n)\n\nSELECT\n p.username,\n p.aid,\n COUNT(*) AS total_games,\n COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS white_wins,\n COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS black_wins,\n COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END) AS white_games,\n COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END) AS black_games,\n ROUND(COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END), 0), 2) AS white_win_rate,\n ROUND(COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END), 0), 2) AS black_win_rate\nFROM chess_playground.profiles p\nLEFT JOIN game_results g\n ON p.aid IN (g.white_aid, g.black_aid)\nGROUP BY p.username, p.aid\nORDER BY total_games DESC" - }, - "definition_file": { - "name": "player_summary.sql", - "path": "/integration-tests/happy-path/assets/player_summary.sql", - "type": "comment" - }, - "parameters": {}, - "secrets": [], - "columns": [ - { - "entity_attribute": null, - "name": "total_games", - "type": "integer", - "description": "the games", - "primary_key": false, - "update_on_merge": false, - "checks": [ - { - "id": "df9255080865c27b164a7de36a0a26bcc00345dddd66849d96aa96a2c68266ea", - "name": "positive", - "value": null, - "blocking": true - } - ], - "upstreams": [] - } - ], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - } - ], - "notifications": { - "slack": [], - "ms_teams": [], - "discord": [] - }, - "catchup": false, - "metadata_push": { - "bigquery": false - }, - "retries": 0 - }, - "repo": { - "path": "/integration-tests" - } -} diff --git a/integration-tests/test-pipelines/parse-whole-pipeline/expectations/chess_games.asset.yml.json b/integration-tests/test-pipelines/parse-whole-pipeline/expectations/chess_games.asset.yml.json deleted file mode 100644 index c3fe78ca..00000000 --- a/integration-tests/test-pipelines/parse-whole-pipeline/expectations/chess_games.asset.yml.json +++ /dev/null @@ -1,242 +0,0 @@ -{ - "asset": { - "id": "209c299a591add072bfa259ad5f311ab7c5aa154960a55b20f3d6de33bb8f21b", - "uri": "", - "name": "chess_playground.games", - "type": "ingestr", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "chess_games.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", - "content": "name: chess_playground.games\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: games\n destination: duckdb" - }, - "definition_file": { - "name": "chess_games.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", - "type": "yaml" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-default", - "source_table": "games" - }, - "secrets": [], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - "pipeline": { - "legacy_id": "", - "name": "integration_test", - "schedule": "", - "start_date": "", - "definition_file": { - "name": "pipeline.yml", - "path": "/integration-tests/happy-path/pipeline.yml" - }, - "default_parameters": {}, - "default_connections": {}, - "assets": [ - { - "id": "17c2f1111545c0e72ea13a3a07fb3d5a4d96074d128ed10c1c03cc477401d61a", - "uri": "", - "name": "python_asset", - "type": "python", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [ - { - "type": "asset", - "value": "chess_playground.player_summary", - "columns": [] - } - ], - "image": "python:3.11", - "instance": "", - "owner": "", - "executable_file": { - "name": "asset.py", - "path": "/integration-tests/happy-path/assets/asset.py", - "content": "import os\nimport duckdb\n\nif os.getenv('INJECTED1') != \"value1\":\n raise Exception(\"KEY1 is not injected correctly\")\n\ncon = duckdb.connect(database = \"duckdb.db\", read_only = False)\n\ncon.execute(\"SELECT * FROM chess_playground.player_summary\")\nresult = con.fetchall()\nif len(result) != 2:\n raise Exception(\"Incorrect number of rows in player_summary\")" - }, - "definition_file": { - "name": "asset.py", - "path": "/integration-tests/happy-path/assets/asset.py", - "type": "comment" - }, - "parameters": {}, - "secrets": [ - { - "secret_key": "KEY1", - "injected_key": "INJECTED1" - } - ], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "209c299a591add072bfa259ad5f311ab7c5aa154960a55b20f3d6de33bb8f21b", - "uri": "", - "name": "chess_playground.games", - "type": "ingestr", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "chess_games.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", - "content": "name: chess_playground.games\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: games\n destination: duckdb" - }, - "definition_file": { - "name": "chess_games.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", - "type": "yaml" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-default", - "source_table": "games" - }, - "secrets": [], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "c53385eb13eb4d3d102be02b0d3fe4a10661339b3c098de8b226f7317fc47d21", - "uri": "", - "name": "chess_playground.profiles", - "type": "ingestr", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "chess_profiles.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", - "content": "name: chess_playground.profiles\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: profiles\n destination: duckdb" - }, - "definition_file": { - "name": "chess_profiles.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", - "type": "yaml" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-default", - "source_table": "profiles" - }, - "secrets": [], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "8361c0131fda306b28fd4f3c8f2af121cca5e57baf77a771b4cb218abda4ca5b", - "uri": "", - "name": "chess_playground.player_summary", - "type": "duckdb.sql", - "description": "", - "connection": "", - "tags": [], - "materialization": { - "type": "table", - "strategy": "", - "partition_by": "", - "cluster_by": null, - "incremental_key": "" - }, - "upstreams": [ - { - "type": "asset", - "value": "chess_playground.games", - "columns": [] - }, - { - "type": "asset", - "value": "chess_playground.profiles", - "columns": [] - } - ], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "player_summary.sql", - "path": "/integration-tests/happy-path/assets/player_summary.sql", - "content": "WITH game_results AS (\n SELECT\n CASE\n WHEN g.white->>'result' = 'win' THEN g.white->>'@id'\n WHEN g.black->>'result' = 'win' THEN g.black->>'@id'\n ELSE NULL\n END AS winner_aid,\n g.white->>'@id' AS white_aid,\n g.black->>'@id' AS black_aid\nFROM chess_playground.games g\n)\n\nSELECT\n p.username,\n p.aid,\n COUNT(*) AS total_games,\n COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS white_wins,\n COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS black_wins,\n COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END) AS white_games,\n COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END) AS black_games,\n ROUND(COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END), 0), 2) AS white_win_rate,\n ROUND(COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END), 0), 2) AS black_win_rate\nFROM chess_playground.profiles p\nLEFT JOIN game_results g\n ON p.aid IN (g.white_aid, g.black_aid)\nGROUP BY p.username, p.aid\nORDER BY total_games DESC" - }, - "definition_file": { - "name": "player_summary.sql", - "path": "/integration-tests/happy-path/assets/player_summary.sql", - "type": "comment" - }, - "parameters": {}, - "secrets": [], - "columns": [ - { - "entity_attribute": null, - "name": "total_games", - "type": "integer", - "description": "the games", - "primary_key": false, - "update_on_merge": false, - "checks": [ - { - "id": "df9255080865c27b164a7de36a0a26bcc00345dddd66849d96aa96a2c68266ea", - "name": "positive", - "value": null, - "blocking": true - } - ], - "upstreams": [] - } - ], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - } - ], - "notifications": { - "slack": [], - "ms_teams": [], - "discord": [] - }, - "catchup": false, - "metadata_push": { - "bigquery": false - }, - "retries": 0 - }, - "repo": { - "path": "/integration-tests" - } -} diff --git a/integration-tests/test-pipelines/parse-whole-pipeline/expectations/chess_profiles.asset.yml.json b/integration-tests/test-pipelines/parse-whole-pipeline/expectations/chess_profiles.asset.yml.json deleted file mode 100644 index 2538260b..00000000 --- a/integration-tests/test-pipelines/parse-whole-pipeline/expectations/chess_profiles.asset.yml.json +++ /dev/null @@ -1,242 +0,0 @@ -{ - "asset": { - "id": "c53385eb13eb4d3d102be02b0d3fe4a10661339b3c098de8b226f7317fc47d21", - "uri": "", - "name": "chess_playground.profiles", - "type": "ingestr", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "chess_profiles.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", - "content": "name: chess_playground.profiles\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: profiles\n destination: duckdb" - }, - "definition_file": { - "name": "chess_profiles.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", - "type": "yaml" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-default", - "source_table": "profiles" - }, - "secrets": [], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - "pipeline": { - "legacy_id": "", - "name": "integration_test", - "schedule": "", - "start_date": "", - "definition_file": { - "name": "pipeline.yml", - "path": "/integration-tests/happy-path/pipeline.yml" - }, - "default_parameters": {}, - "default_connections": {}, - "assets": [ - { - "id": "17c2f1111545c0e72ea13a3a07fb3d5a4d96074d128ed10c1c03cc477401d61a", - "uri": "", - "name": "python_asset", - "type": "python", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [ - { - "type": "asset", - "value": "chess_playground.player_summary", - "columns": [] - } - ], - "image": "python:3.11", - "instance": "", - "owner": "", - "executable_file": { - "name": "asset.py", - "path": "/integration-tests/happy-path/assets/asset.py", - "content": "import os\nimport duckdb\n\nif os.getenv('INJECTED1') != \"value1\":\n raise Exception(\"KEY1 is not injected correctly\")\n\ncon = duckdb.connect(database = \"duckdb.db\", read_only = False)\n\ncon.execute(\"SELECT * FROM chess_playground.player_summary\")\nresult = con.fetchall()\nif len(result) != 2:\n raise Exception(\"Incorrect number of rows in player_summary\")" - }, - "definition_file": { - "name": "asset.py", - "path": "/integration-tests/happy-path/assets/asset.py", - "type": "comment" - }, - "parameters": {}, - "secrets": [ - { - "secret_key": "KEY1", - "injected_key": "INJECTED1" - } - ], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "209c299a591add072bfa259ad5f311ab7c5aa154960a55b20f3d6de33bb8f21b", - "uri": "", - "name": "chess_playground.games", - "type": "ingestr", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "chess_games.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", - "content": "name: chess_playground.games\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: games\n destination: duckdb" - }, - "definition_file": { - "name": "chess_games.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", - "type": "yaml" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-default", - "source_table": "games" - }, - "secrets": [], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "c53385eb13eb4d3d102be02b0d3fe4a10661339b3c098de8b226f7317fc47d21", - "uri": "", - "name": "chess_playground.profiles", - "type": "ingestr", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "chess_profiles.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", - "content": "name: chess_playground.profiles\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: profiles\n destination: duckdb" - }, - "definition_file": { - "name": "chess_profiles.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", - "type": "yaml" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-default", - "source_table": "profiles" - }, - "secrets": [], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "8361c0131fda306b28fd4f3c8f2af121cca5e57baf77a771b4cb218abda4ca5b", - "uri": "", - "name": "chess_playground.player_summary", - "type": "duckdb.sql", - "description": "", - "connection": "", - "tags": [], - "materialization": { - "type": "table", - "strategy": "", - "partition_by": "", - "cluster_by": null, - "incremental_key": "" - }, - "upstreams": [ - { - "type": "asset", - "value": "chess_playground.games", - "columns": [] - }, - { - "type": "asset", - "value": "chess_playground.profiles", - "columns": [] - } - ], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "player_summary.sql", - "path": "/integration-tests/happy-path/assets/player_summary.sql", - "content": "WITH game_results AS (\n SELECT\n CASE\n WHEN g.white-\u003e\u003e'result' = 'win' THEN g.white-\u003e\u003e'@id'\n WHEN g.black-\u003e\u003e'result' = 'win' THEN g.black-\u003e\u003e'@id'\n ELSE NULL\n END AS winner_aid,\n g.white-\u003e\u003e'@id' AS white_aid,\n g.black-\u003e\u003e'@id' AS black_aid\nFROM chess_playground.games g\n)\n\nSELECT\n p.username,\n p.aid,\n COUNT(*) AS total_games,\n COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS white_wins,\n COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS black_wins,\n COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END) AS white_games,\n COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END) AS black_games,\n ROUND(COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END), 0), 2) AS white_win_rate,\n ROUND(COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END), 0), 2) AS black_win_rate\nFROM chess_playground.profiles p\nLEFT JOIN game_results g\n ON p.aid IN (g.white_aid, g.black_aid)\nGROUP BY p.username, p.aid\nORDER BY total_games DESC" - }, - "definition_file": { - "name": "player_summary.sql", - "path": "/integration-tests/happy-path/assets/player_summary.sql", - "type": "comment" - }, - "parameters": {}, - "secrets": [], - "columns": [ - { - "entity_attribute": null, - "name": "total_games", - "type": "integer", - "description": "the games", - "primary_key": false, - "update_on_merge": false, - "checks": [ - { - "id": "df9255080865c27b164a7de36a0a26bcc00345dddd66849d96aa96a2c68266ea", - "name": "positive", - "value": null, - "blocking": true - } - ], - "upstreams": [] - } - ], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - } - ], - "notifications": { - "slack": [], - "ms_teams": [], - "discord": [] - }, - "catchup": false, - "metadata_push": { - "bigquery": false - }, - "retries": 0 - }, - "repo": { - "path": "/integration-tests" - } -} diff --git a/integration-tests/test-pipelines/parse-whole-pipeline/expectations/player_summary.sql.json b/integration-tests/test-pipelines/parse-whole-pipeline/expectations/player_summary.sql.json deleted file mode 100644 index cfe05f4e..00000000 --- a/integration-tests/test-pipelines/parse-whole-pipeline/expectations/player_summary.sql.json +++ /dev/null @@ -1,273 +0,0 @@ -{ - "asset": { - "id": "8361c0131fda306b28fd4f3c8f2af121cca5e57baf77a771b4cb218abda4ca5b", - "uri": "", - "name": "chess_playground.player_summary", - "type": "duckdb.sql", - "description": "", - "connection": "", - "tags": [], - "materialization": { - "type": "table", - "strategy": "", - "partition_by": "", - "cluster_by": null, - "incremental_key": "" - }, - "upstreams": [ - { - "type": "asset", - "value": "chess_playground.games", - "columns": [] - }, - { - "type": "asset", - "value": "chess_playground.profiles", - "columns": [] - } - ], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "player_summary.sql", - "path": "/integration-tests/happy-path/assets/player_summary.sql", - "content": "WITH game_results AS (\n SELECT\n CASE\n WHEN g.white-\u003e\u003e'result' = 'win' THEN g.white-\u003e\u003e'@id'\n WHEN g.black-\u003e\u003e'result' = 'win' THEN g.black-\u003e\u003e'@id'\n ELSE NULL\n END AS winner_aid,\n g.white-\u003e\u003e'@id' AS white_aid,\n g.black-\u003e\u003e'@id' AS black_aid\nFROM chess_playground.games g\n)\n\nSELECT\n p.username,\n p.aid,\n COUNT(*) AS total_games,\n COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS white_wins,\n COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS black_wins,\n COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END) AS white_games,\n COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END) AS black_games,\n ROUND(COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END), 0), 2) AS white_win_rate,\n ROUND(COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END), 0), 2) AS black_win_rate\nFROM chess_playground.profiles p\nLEFT JOIN game_results g\n ON p.aid IN (g.white_aid, g.black_aid)\nGROUP BY p.username, p.aid\nORDER BY total_games DESC" - }, - "definition_file": { - "name": "player_summary.sql", - "path": "/integration-tests/happy-path/assets/player_summary.sql", - "type": "comment" - }, - "parameters": {}, - "secrets": [], - "columns": [ - { - "entity_attribute": null, - "name": "total_games", - "type": "integer", - "description": "the games", - "primary_key": false, - "update_on_merge": false, - "checks": [ - { - "id": "df9255080865c27b164a7de36a0a26bcc00345dddd66849d96aa96a2c68266ea", - "name": "positive", - "value": null, - "blocking": true - } - ], - "upstreams": [] - } - ], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - "pipeline": { - "legacy_id": "", - "name": "integration_test", - "schedule": "", - "start_date": "", - "definition_file": { - "name": "pipeline.yml", - "path": "/integration-tests/happy-path/pipeline.yml" - }, - "default_parameters": {}, - "default_connections": {}, - "assets": [ - { - "id": "17c2f1111545c0e72ea13a3a07fb3d5a4d96074d128ed10c1c03cc477401d61a", - "uri": "", - "name": "python_asset", - "type": "python", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [ - { - "type": "asset", - "value": "chess_playground.player_summary", - "columns": [] - } - ], - "image": "python:3.11", - "instance": "", - "owner": "", - "executable_file": { - "name": "asset.py", - "path": "/integration-tests/happy-path/assets/asset.py", - "content": "import os\nimport duckdb\n\nif os.getenv('INJECTED1') != \"value1\":\n raise Exception(\"KEY1 is not injected correctly\")\n\ncon = duckdb.connect(database = \"duckdb.db\", read_only = False)\n\ncon.execute(\"SELECT * FROM chess_playground.player_summary\")\nresult = con.fetchall()\nif len(result) != 2:\n raise Exception(\"Incorrect number of rows in player_summary\")" - }, - "definition_file": { - "name": "asset.py", - "path": "/integration-tests/happy-path/assets/asset.py", - "type": "comment" - }, - "parameters": {}, - "secrets": [ - { - "secret_key": "KEY1", - "injected_key": "INJECTED1" - } - ], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "209c299a591add072bfa259ad5f311ab7c5aa154960a55b20f3d6de33bb8f21b", - "uri": "", - "name": "chess_playground.games", - "type": "ingestr", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "chess_games.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", - "content": "name: chess_playground.games\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: games\n destination: duckdb" - }, - "definition_file": { - "name": "chess_games.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_games.asset.yml", - "type": "yaml" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-default", - "source_table": "games" - }, - "secrets": [], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "c53385eb13eb4d3d102be02b0d3fe4a10661339b3c098de8b226f7317fc47d21", - "uri": "", - "name": "chess_playground.profiles", - "type": "ingestr", - "description": "", - "connection": "", - "tags": [], - "materialization": null, - "upstreams": [], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "chess_profiles.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", - "content": "name: chess_playground.profiles\ntype: ingestr\nparameters:\n source_connection: chess-default\n source_table: profiles\n destination: duckdb" - }, - "definition_file": { - "name": "chess_profiles.asset.yml", - "path": "/integration-tests/happy-path/assets/chess_profiles.asset.yml", - "type": "yaml" - }, - "parameters": { - "destination": "duckdb", - "source_connection": "chess-default", - "source_table": "profiles" - }, - "secrets": [], - "columns": [], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - }, - { - "id": "8361c0131fda306b28fd4f3c8f2af121cca5e57baf77a771b4cb218abda4ca5b", - "uri": "", - "name": "chess_playground.player_summary", - "type": "duckdb.sql", - "description": "", - "connection": "", - "tags": [], - "materialization": { - "type": "table", - "strategy": "", - "partition_by": "", - "cluster_by": null, - "incremental_key": "" - }, - "upstreams": [ - { - "type": "asset", - "value": "chess_playground.games", - "columns": [] - }, - { - "type": "asset", - "value": "chess_playground.profiles", - "columns": [] - } - ], - "image": "", - "instance": "", - "owner": "", - "executable_file": { - "name": "player_summary.sql", - "path": "/integration-tests/happy-path/assets/player_summary.sql", - "content": "WITH game_results AS (\n SELECT\n CASE\n WHEN g.white-\u003e\u003e'result' = 'win' THEN g.white-\u003e\u003e'@id'\n WHEN g.black-\u003e\u003e'result' = 'win' THEN g.black-\u003e\u003e'@id'\n ELSE NULL\n END AS winner_aid,\n g.white-\u003e\u003e'@id' AS white_aid,\n g.black-\u003e\u003e'@id' AS black_aid\nFROM chess_playground.games g\n)\n\nSELECT\n p.username,\n p.aid,\n COUNT(*) AS total_games,\n COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS white_wins,\n COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS black_wins,\n COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END) AS white_games,\n COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END) AS black_games,\n ROUND(COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.white_aid = p.aid THEN 1 END), 0), 2) AS white_win_rate,\n ROUND(COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) * 100.0 / NULLIF(COUNT(CASE WHEN g.black_aid = p.aid THEN 1 END), 0), 2) AS black_win_rate\nFROM chess_playground.profiles p\nLEFT JOIN game_results g\n ON p.aid IN (g.white_aid, g.black_aid)\nGROUP BY p.username, p.aid\nORDER BY total_games DESC" - }, - "definition_file": { - "name": "player_summary.sql", - "path": "/integration-tests/happy-path/assets/player_summary.sql", - "type": "comment" - }, - "parameters": {}, - "secrets": [], - "columns": [ - { - "entity_attribute": null, - "name": "total_games", - "type": "integer", - "description": "the games", - "primary_key": false, - "update_on_merge": false, - "checks": [ - { - "id": "df9255080865c27b164a7de36a0a26bcc00345dddd66849d96aa96a2c68266ea", - "name": "positive", - "value": null, - "blocking": true - } - ], - "upstreams": [] - } - ], - "custom_checks": [], - "metadata": {}, - "snowflake": null, - "athena": null - } - ], - "notifications": { - "slack": [], - "ms_teams": [], - "discord": [] - }, - "catchup": false, - "metadata_push": { - "bigquery": false - }, - "retries": 0 - }, - "repo": { - "path": "/integration-tests" - } -} From e9c1180352fc846ab4b33098455a96a87591847b Mon Sep 17 00:00:00 2001 From: Baris Terzioglu Date: Tue, 31 Dec 2024 00:55:30 +0100 Subject: [PATCH 7/8] delete more files --- .../assets/chess_games.asset.yml | 9 --- .../assets/chess_profiles.asset.yml | 8 --- .../assets/game_outcome_summary.sql | 31 ---------- .../assets/player_profile_summary.sql | 30 --------- .../assets/player_summary.sql | 62 ------------------- .../pipeline.yml | 1 - 6 files changed, 141 deletions(-) delete mode 100644 integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/chess_games.asset.yml delete mode 100644 integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/chess_profiles.asset.yml delete mode 100644 integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/game_outcome_summary.sql delete mode 100644 integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/player_profile_summary.sql delete mode 100644 integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/player_summary.sql delete mode 100644 integration-tests/test-pipelines/run-with-filters-case2-pipeline/pipeline.yml diff --git a/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/chess_games.asset.yml b/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/chess_games.asset.yml deleted file mode 100644 index 68bc4e41..00000000 --- a/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/chess_games.asset.yml +++ /dev/null @@ -1,9 +0,0 @@ -name: chess_playground.games -type: ingestr -tags: - - include - -parameters: - destination: duckdb - source_connection: chess-default - source_table: games diff --git a/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/chess_profiles.asset.yml b/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/chess_profiles.asset.yml deleted file mode 100644 index cd770cdb..00000000 --- a/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/chess_profiles.asset.yml +++ /dev/null @@ -1,8 +0,0 @@ -name: chess_playground.profiles -type: ingestr -parameters: - source_connection: chess-default - source_table: profiles - destination: duckdb -tags: - - include \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/game_outcome_summary.sql b/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/game_outcome_summary.sql deleted file mode 100644 index 65cb4340..00000000 --- a/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/game_outcome_summary.sql +++ /dev/null @@ -1,31 +0,0 @@ -/* @bruin - -name: chess_playground.game_outcome_summary -type: duckdb.sql -materialization: - type: table - -depends: - - chess_playground.games - -columns: - - name: result_type - type: string - description: "Type of game result (win, draw, etc.)" - - name: total_games - type: integer - description: "Total number of games with this result" - checks: - - name: positive -tags: - - include - -@bruin */ - -SELECT - g.white->>'result' AS result_type, - COUNT(*) AS total_games -FROM chess_playground.games g -WHERE g.white->>'result' IS NOT NULL -GROUP BY g.white->>'result' -ORDER BY total_games DESC; diff --git a/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/player_profile_summary.sql b/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/player_profile_summary.sql deleted file mode 100644 index 4aedab67..00000000 --- a/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/player_profile_summary.sql +++ /dev/null @@ -1,30 +0,0 @@ -/* @bruin - -name: chess_playground.player_profile_summary -type: duckdb.sql -materialization: - type: table - -depends: - - chess_playground.profiles -columns: - - name: total_players - type: integer - description: "Total number of players in the profiles table" - checks: - - name: positive - - name: active_players - type: integer - description: "Number of players marked as active" - - name: inactive_players - type: integer - description: "Number of players marked as inactive" -tags: - - exclude -@bruin */ - -SELECT - COUNT(*) AS total_players, - COUNT(CASE WHEN p.status = 'active' THEN 1 END) AS active_players, - COUNT(CASE WHEN p.status = 'inactive' THEN 1 END) AS inactive_players -FROM chess_playground.profiles p; diff --git a/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/player_summary.sql b/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/player_summary.sql deleted file mode 100644 index 0adf29fa..00000000 --- a/integration-tests/test-pipelines/run-with-filters-case2-pipeline/assets/player_summary.sql +++ /dev/null @@ -1,62 +0,0 @@ -/* @bruin - -name: chess_playground.player_summary -type: duckdb.sql -materialization: - type: table - -depends: - - chess_playground.game_outcome_summary - - chess_playground.player_profile_summary - - chess_playground.games - - chess_playground.profiles - -columns: - - name: username - type: string - description: "Username of the player" - - name: total_games - type: integer - description: "Total games played by the player" - checks: - - name: non_negative - - name: total_wins - type: integer - description: "Total games won by the player" - - name: win_rate - type: float - description: "Win rate of the player" -tags: - - include - - exclude -@bruin */ - -WITH game_results AS ( - SELECT - CASE - WHEN g.white->>'result' = 'win' THEN g.white->>'@id' - WHEN g.black->>'result' = 'win' THEN g.black->>'@id' - ELSE NULL - END AS winner_aid, - g.white->>'@id' AS white_aid, - g.black->>'@id' AS black_aid -FROM chess_playground.games g - ) - -SELECT - p.username, - p.aid, - COUNT(*) AS total_games, - COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) + - COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS total_wins, - ROUND( - (COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) + - COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END)) * 100.0 / - NULLIF(COUNT(*), 0), - 2 - ) AS win_rate -FROM chess_playground.profiles p - LEFT JOIN game_results g - ON p.aid IN (g.white_aid, g.black_aid) -GROUP BY p.username, p.aid -ORDER BY total_games DESC; diff --git a/integration-tests/test-pipelines/run-with-filters-case2-pipeline/pipeline.yml b/integration-tests/test-pipelines/run-with-filters-case2-pipeline/pipeline.yml deleted file mode 100644 index 9172e408..00000000 --- a/integration-tests/test-pipelines/run-with-filters-case2-pipeline/pipeline.yml +++ /dev/null @@ -1 +0,0 @@ -name: chess_extended_duckdb \ No newline at end of file From 4c6e6fcd59034772cab686a77a0bbb080aa9922a Mon Sep 17 00:00:00 2001 From: Burak Karakan Date: Tue, 31 Dec 2024 11:51:56 +0300 Subject: [PATCH 8/8] add logs/runs to the gitignore as well --- integration-tests/.gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration-tests/.gitignore b/integration-tests/.gitignore index 458e6812..b79f8ef5 100644 --- a/integration-tests/.gitignore +++ b/integration-tests/.gitignore @@ -1,2 +1,3 @@ logs/*.log -.bruin.yml \ No newline at end of file +.bruin.yml +logs/runs \ No newline at end of file