diff --git a/.gitignore b/.gitignore index ea7064ee..2af1a944 100644 --- a/.gitignore +++ b/.gitignore @@ -140,6 +140,7 @@ glossary.yaml # pixi dev env .pixi +integration-tests/duckdb-files integration-tests/duckdb.db integration-tests/logs integration-tests/bruin diff --git a/Makefile b/Makefile index 94701619..c3319963 100644 --- a/Makefile +++ b/Makefile @@ -23,12 +23,12 @@ build: deps @CGO_ENABLED=1 go build -v -tags="no_duckdb_arrow" -ldflags="-s -w -X main.Version=$(or $(tag), dev-$(shell git describe --tags --abbrev=0)) -X main.TelemetryKey=$(TELEMETRY_KEY)" -o "$(BUILD_DIR)/$(NAME)" "$(BUILD_SRC)" integration-test: build + @rm -rf integration-tests/duckdb-files # Clean up the directory if it exists + @mkdir -p integration-tests/duckdb-files # Recreate the directory @touch integration-tests/.git @touch integration-tests/bruin - @touch integration-tests/duckdb.db @rm -rf integration-tests/.git - @rm integration-tests/bruin - @rm integration-tests/duckdb.db + @rm integration-tests/bruin @echo "$(OK_COLOR)==> Running integration tests...$(NO_COLOR)" @cd integration-tests && git init @go run integration-tests/integration-test.go diff --git a/integration-tests/.bruin.yml b/integration-tests/.bruin.yml index a3fa2c93..b1395279 100644 --- a/integration-tests/.bruin.yml +++ b/integration-tests/.bruin.yml @@ -4,12 +4,110 @@ environments: connections: duckdb: - name: "duckdb-default" - path: "duckdb.db" + path: "duckdb-files/duckdb.db" chess: - name: "chess-default" players: - "MagnusCarlsen" - "Hikaru" + generic: + - name: KEY1 + value: value1 + + env-run-with-filters: + connections: + duckdb: + - name: "duckdb-run-with-filters" + path: "duckdb-files/run-with-filters.db" + chess: + - name: "chess-run-with-filters" + players: + - "MagnusCarlsen" + - "Hikaru" + generic: + - name: KEY1 + value: value1 + env-run-with-tags: + connections: + duckdb: + - name: "duckdb-run-with-tags" + path: "duckdb-files/run-with-tags.db" + chess: + - name: "chess-run-with-tags" + players: + - "MagnusCarlsen" + - "Hikaru" + generic: + - name: KEY1 + value: value1 + env-run-main-with-filters: + connections: + duckdb: + - name: "duckdb-run-main-with-filters" + path: "duckdb-files/run-main-with-filters.db" + chess: + - name: "chess-run-main-with-filters" + players: + - "MagnusCarlsen" + - "Hikaru" + generic: + - name: KEY1 + value: value1 + + env-run-with-downstream: + connections: + duckdb: + - name: "duckdb-run-with-downstream" + path: "duckdb-files/run-with-downstream.db" + chess: + - name: "chess-run-with-downstream" + players: + - "MagnusCarlsen" + - "Hikaru" + generic: + - name: KEY1 + value: value1 + env-run-main-with-downstream: + connections: + duckdb: + - name: "duckdb-run-main-with-downstream" + path: "duckdb-files/run-main-with-downstream.db" + chess: + - name: "chess-run-main-with-downstream" + players: + - "MagnusCarlsen" + - "Hikaru" + generic: + - name: KEY1 + value: value1 + env-push-metadata: + connections: + duckdb: + - name: "duckdb-push-metadata" + path: "duckdb-files/push-metadata.db" + env-run-use-uv: + connections: + duckdb: + - name: "duckdb-env-run-use-uv" + path: "duckdb-files/env-run-use-uv.db" + chess: + - name: "chess-run-use-uv" + players: + - "MagnusCarlsen" + - "Hikaru" + generic: + - name: KEY1 + value: value1 + env-run-malformed-sql: + connections: + duckdb: + - name: "duckdb-run-malformed-sql" + path: "duckdb-files/run-malformed-sql.db" + chess: + - name: "chess-run-malformed-sql" + players: + - "MagnusCarlsen" + - "Hikaru" generic: - name: KEY1 value: value1 \ No newline at end of file diff --git a/integration-tests/expected_connections.json b/integration-tests/expected_connections.json index 08902683..847094a5 100644 --- a/integration-tests/expected_connections.json +++ b/integration-tests/expected_connections.json @@ -12,7 +12,7 @@ "duckdb": [ { "name": "duckdb-default", - "path": "duckdb.db" + "path": "duckdb-files/duckdb.db" } ], "chess": [ @@ -38,7 +38,7 @@ "duckdb": [ { "name": "duckdb-default", - "path": "duckdb.db" + "path": "duckdb-files/duckdb.db" } ], "chess": [ @@ -51,6 +51,191 @@ } ] } + }, + "env-run-with-filters": { + "connections": { + "generic": [ + { + "name": "KEY1", + "value": "value1" + } + ], + "duckdb": [ + { + "name": "duckdb-run-with-filters", + "path": "duckdb-files/run-with-filters.db" + } + ], + "chess": [ + { + "name": "chess-run-with-filters", + "players": [ + "MagnusCarlsen", + "Hikaru" + ] + } + ] + } + }, + "env-run-with-tags": { + "connections": { + "generic": [ + { + "name": "KEY1", + "value": "value1" + } + ], + "duckdb": [ + { + "name": "duckdb-run-with-tags", + "path": "duckdb-files/run-with-tags.db" + } + ], + "chess": [ + { + "name": "chess-run-with-tags", + "players": [ + "MagnusCarlsen", + "Hikaru" + ] + } + ] + } + }, + "env-run-main-with-filters": { + "connections": { + "generic": [ + { + "name": "KEY1", + "value": "value1" + } + ], + "duckdb": [ + { + "name": "duckdb-run-main-with-filters", + "path": "duckdb-files/run-main-with-filters.db" + } + ], + "chess": [ + { + "name": "chess-run-main-with-filters", + "players": [ + "MagnusCarlsen", + "Hikaru" + ] + } + ] + } + }, + "env-run-with-downstream": { + "connections": { + "generic": [ + { + "name": "KEY1", + "value": "value1" + } + ], + "duckdb": [ + { + "name": "duckdb-run-with-downstream", + "path": "duckdb-files/run-with-downstream.db" + } + ], + "chess": [ + { + "name": "chess-run-with-downstream", + "players": [ + "MagnusCarlsen", + "Hikaru" + ] + } + ] + } + }, + "env-run-main-with-downstream": { + "connections": { + "generic": [ + { + "name": "KEY1", + "value": "value1" + } + ], + "duckdb": [ + { + "name": "duckdb-run-main-with-downstream", + "path": "duckdb-files/run-main-with-downstream.db" + } + ], + "chess": [ + { + "name": "chess-run-main-with-downstream", + "players": [ + "MagnusCarlsen", + "Hikaru" + ] + } + ] + } + }, + "env-push-metadata": { + "connections": { + "duckdb": [ + { + "name": "duckdb-push-metadata", + "path": "duckdb-files/push-metadata.db" + } + ] + } + }, + "env-run-use-uv": { + "connections": { + "generic": [ + { + "name": "KEY1", + "value": "value1" + } + ], + "duckdb": [ + { + "name": "duckdb-env-run-use-uv", + "path": "duckdb-files/env-run-use-uv.db" + } + ], + "chess": [ + { + "name": "chess-run-use-uv", + "players": [ + "MagnusCarlsen", + "Hikaru" + ] + } + ] + } + }, + "env-run-malformed-sql": { + "connections": { + "generic": [ + { + "name": "KEY1", + "value": "value1" + } + ], + "duckdb": [ + { + "name": "duckdb-run-malformed-sql", + "path": "duckdb-files/run-malformed-sql.db" + } + ], + "chess": [ + { + "name": "chess-run-malformed-sql", + "players": [ + "MagnusCarlsen", + "Hikaru" + ] + } + ] + } } } -} \ No newline at end of file +} diff --git a/integration-tests/integration-test.go b/integration-tests/integration-test.go index f5204b00..3ada12b2 100644 --- a/integration-tests/integration-test.go +++ b/integration-tests/integration-test.go @@ -255,7 +255,7 @@ func getTasks(binary string, currentFolder string) []e2e.Task { { Name: "run-with-tags", Command: binary, - Args: []string{"run", "--tag", "include", "--exclude-tag", "exclude", filepath.Join(currentFolder, "test-pipelines/run-with-tags-pipeline")}, + Args: []string{"run", "--env", "env-run-with-tags", "--tag", "include", "--exclude-tag", "exclude", filepath.Join(currentFolder, "test-pipelines/run-with-tags-pipeline")}, Env: []string{}, Expected: e2e.Output{ @@ -268,12 +268,12 @@ func getTasks(binary string, currentFolder string) []e2e.Task { { Name: "run-with-filters", Command: binary, - Args: []string{"run", "--tag", "include", "--exclude-tag", "exclude", "--only", "checks", filepath.Join(currentFolder, "test-pipelines/run-with-filters-pipeline")}, + Args: []string{"run", "-env", "env-run-with-filters", "--tag", "include", "--exclude-tag", "exclude", filepath.Join(currentFolder, "test-pipelines/run-with-filters-pipeline")}, Env: []string{}, Expected: e2e.Output{ ExitCode: 0, - Contains: []string{"Executed 1 tasks", "total_games:positive"}, + Contains: []string{"Executed 4 tasks", "Finished: chess_playground.games", " Finished: chess_playground.game_outcome_summary", "Finished: chess_playground.game_outcome_summary:total_games:positive", "Finished: chess_playground.profiles"}, }, Asserts: []func(*e2e.Task) error{ e2e.AssertByExitCode, @@ -295,7 +295,7 @@ func getTasks(binary string, currentFolder string) []e2e.Task { { Name: "run-main-with-filters", Command: binary, - Args: []string{"run", "--tag", "include", "--exclude-tag", "exclude", "--only", "main", filepath.Join(currentFolder, "test-pipelines/run-main-with-filters-pipeline")}, + Args: []string{"run", "--env", "env-run-main-with-filters", "--tag", "include", "--exclude-tag", "exclude", "--only", "main", filepath.Join(currentFolder, "test-pipelines/run-main-with-filters-pipeline")}, Env: []string{}, Expected: e2e.Output{ @@ -310,11 +310,11 @@ func getTasks(binary string, currentFolder string) []e2e.Task { { Name: "run-with-downstream", Command: binary, - Args: []string{"run", "--downstream", filepath.Join(currentFolder, "test-pipelines/run-with-downstream-pipeline/assets/game_outcome_summary.sql")}, + Args: []string{"run", "--env", "env-run-with-downstream", "--downstream", filepath.Join(currentFolder, "test-pipelines/run-with-downstream-pipeline/assets/products.sql")}, Env: []string{}, Expected: e2e.Output{ ExitCode: 0, - Contains: []string{"Executed 4 tasks", " Finished: chess_playground.game_outcome_summary", "Finished: chess_playground.game_outcome_summary:total_games:positive", "Finished: chess_playground.player_summary", " Finished: chess_playground.player_summary:total_games:non_negative"}, + Contains: []string{"Executed 5 tasks", "Finished: products", "Finished: products:price:positive", "Finished: product_price_summary", "Finished: product_price_summary:product_count:non_negative", "Finished: product_price_summary:total_stock:non_negative"}, }, Asserts: []func(*e2e.Task) error{ e2e.AssertByExitCode, @@ -324,11 +324,11 @@ func getTasks(binary string, currentFolder string) []e2e.Task { { Name: "run-main-with-downstream", Command: binary, - Args: []string{"run", "--downstream", "--only", "main", filepath.Join(currentFolder, "test-pipelines/run-main-with-downstream-pipeline/assets/game_outcome_summary.sql")}, + Args: []string{"run", "--env", "env-run-main-with-downstream", "--downstream", "--only", "main", filepath.Join(currentFolder, "test-pipelines/run-main-with-downstream-pipeline/assets/products.sql")}, Env: []string{}, Expected: e2e.Output{ ExitCode: 0, - Contains: []string{"Executed 2 tasks", " Finished: chess_playground.game_outcome_summary", "Finished: chess_playground.player_summary"}, + Contains: []string{"Executed 2 tasks", "Finished: products", "Finished: product_price_summary"}, }, Asserts: []func(*e2e.Task) error{ e2e.AssertByExitCode, @@ -338,7 +338,7 @@ func getTasks(binary string, currentFolder string) []e2e.Task { { Name: "push-metadata", Command: binary, - Args: []string{"run", "--push-metadata", "--only", "push-metadata", filepath.Join(currentFolder, "test-pipelines/push-metadata-pipeline")}, + Args: []string{"run", "--env", "env-push-metadata", "--push-metadata", "--only", "push-metadata", filepath.Join(currentFolder, "test-pipelines/push-metadata-pipeline")}, Env: []string{}, Expected: e2e.Output{ ExitCode: 1, @@ -364,7 +364,7 @@ func getTasks(binary string, currentFolder string) []e2e.Task { { Name: "run-use-uv", Command: binary, - Args: []string{"run", "--use-uv", filepath.Join(currentFolder, "test-pipelines/run-use-uv-pipeline")}, + Args: []string{"run", "--env", "env-run-use-uv", "--use-uv", filepath.Join(currentFolder, "test-pipelines/run-use-uv-pipeline")}, Env: []string{}, Expected: e2e.Output{ ExitCode: 0, @@ -466,7 +466,7 @@ func getTasks(binary string, currentFolder string) []e2e.Task { { Name: "run-malformed-sql", Command: binary, - Args: []string{"run", filepath.Join(currentFolder, "test-pipelines/run-malformed-pipeline/assets/malformed.sql")}, + Args: []string{"run", "--env", "env-run-malformed-sql", filepath.Join(currentFolder, "test-pipelines/run-malformed-pipeline/assets/malformed.sql")}, Env: []string{}, Expected: e2e.Output{ diff --git a/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/chess_games.asset.yml b/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/chess_games.asset.yml deleted file mode 100644 index 68bc4e41..00000000 --- a/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/chess_games.asset.yml +++ /dev/null @@ -1,9 +0,0 @@ -name: chess_playground.games -type: ingestr -tags: - - include - -parameters: - destination: duckdb - source_connection: chess-default - source_table: games diff --git a/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/chess_profiles.asset.yml b/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/chess_profiles.asset.yml deleted file mode 100644 index cd770cdb..00000000 --- a/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/chess_profiles.asset.yml +++ /dev/null @@ -1,8 +0,0 @@ -name: chess_playground.profiles -type: ingestr -parameters: - source_connection: chess-default - source_table: profiles - destination: duckdb -tags: - - include \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/game_outcome_summary.sql b/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/game_outcome_summary.sql deleted file mode 100644 index 65cb4340..00000000 --- a/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/game_outcome_summary.sql +++ /dev/null @@ -1,31 +0,0 @@ -/* @bruin - -name: chess_playground.game_outcome_summary -type: duckdb.sql -materialization: - type: table - -depends: - - chess_playground.games - -columns: - - name: result_type - type: string - description: "Type of game result (win, draw, etc.)" - - name: total_games - type: integer - description: "Total number of games with this result" - checks: - - name: positive -tags: - - include - -@bruin */ - -SELECT - g.white->>'result' AS result_type, - COUNT(*) AS total_games -FROM chess_playground.games g -WHERE g.white->>'result' IS NOT NULL -GROUP BY g.white->>'result' -ORDER BY total_games DESC; diff --git a/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/player_profile_summary.sql b/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/player_profile_summary.sql deleted file mode 100644 index 4aedab67..00000000 --- a/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/player_profile_summary.sql +++ /dev/null @@ -1,30 +0,0 @@ -/* @bruin - -name: chess_playground.player_profile_summary -type: duckdb.sql -materialization: - type: table - -depends: - - chess_playground.profiles -columns: - - name: total_players - type: integer - description: "Total number of players in the profiles table" - checks: - - name: positive - - name: active_players - type: integer - description: "Number of players marked as active" - - name: inactive_players - type: integer - description: "Number of players marked as inactive" -tags: - - exclude -@bruin */ - -SELECT - COUNT(*) AS total_players, - COUNT(CASE WHEN p.status = 'active' THEN 1 END) AS active_players, - COUNT(CASE WHEN p.status = 'inactive' THEN 1 END) AS inactive_players -FROM chess_playground.profiles p; diff --git a/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/player_summary.sql b/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/player_summary.sql deleted file mode 100644 index 0adf29fa..00000000 --- a/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/player_summary.sql +++ /dev/null @@ -1,62 +0,0 @@ -/* @bruin - -name: chess_playground.player_summary -type: duckdb.sql -materialization: - type: table - -depends: - - chess_playground.game_outcome_summary - - chess_playground.player_profile_summary - - chess_playground.games - - chess_playground.profiles - -columns: - - name: username - type: string - description: "Username of the player" - - name: total_games - type: integer - description: "Total games played by the player" - checks: - - name: non_negative - - name: total_wins - type: integer - description: "Total games won by the player" - - name: win_rate - type: float - description: "Win rate of the player" -tags: - - include - - exclude -@bruin */ - -WITH game_results AS ( - SELECT - CASE - WHEN g.white->>'result' = 'win' THEN g.white->>'@id' - WHEN g.black->>'result' = 'win' THEN g.black->>'@id' - ELSE NULL - END AS winner_aid, - g.white->>'@id' AS white_aid, - g.black->>'@id' AS black_aid -FROM chess_playground.games g - ) - -SELECT - p.username, - p.aid, - COUNT(*) AS total_games, - COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) + - COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS total_wins, - ROUND( - (COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) + - COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END)) * 100.0 / - NULLIF(COUNT(*), 0), - 2 - ) AS win_rate -FROM chess_playground.profiles p - LEFT JOIN game_results g - ON p.aid IN (g.white_aid, g.black_aid) -GROUP BY p.username, p.aid -ORDER BY total_games DESC; diff --git a/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/product_categories.sql b/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/product_categories.sql new file mode 100644 index 00000000..1c24bc27 --- /dev/null +++ b/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/product_categories.sql @@ -0,0 +1,35 @@ +/* @bruin +name: product_categories +type: duckdb.sql +materialization: + type: table + +columns: + - name: category_id + type: INTEGER + description: "Unique identifier for the product category" + primary_key: true + checks: + - name: not_null + - name: positive + - name: category_name + type: VARCHAR + description: "Name of the product category" + checks: + - name: not_null + - name: description + type: VARCHAR + description: "Description of the product category" +@bruin */ + +SELECT + 1 AS category_id, 'Electronics' AS category_name, 'Devices like phones, laptops, and monitors' AS description +UNION ALL +SELECT + 2 AS category_id, 'Accessories' AS category_name, 'Complementary items like headphones and chargers' AS description +UNION ALL +SELECT + 3 AS category_id, 'Appliances' AS category_name, 'Household devices like refrigerators and microwaves' AS description +UNION ALL +SELECT + 4 AS category_id, 'Furniture' AS category_name, 'Home and office furniture like desks and chairs' AS description; diff --git a/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/product_price_summary.sql b/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/product_price_summary.sql new file mode 100644 index 00000000..6bd88706 --- /dev/null +++ b/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/product_price_summary.sql @@ -0,0 +1,45 @@ +/* @bruin + +name: product_price_summary +type: duckdb.sql +materialization: + type: table + +depends: + - products + +columns: + - name: price_range + type: VARCHAR + description: "Range of product prices" + - name: total_stock + type: INTEGER + description: "Total stock available in the price range" + checks: + - name: non_negative + - name: product_count + type: INTEGER + description: "Number of products in the price range" + checks: + - name: non_negative +@bruin */ + +WITH price_buckets AS ( + SELECT + CASE + WHEN price < 200 THEN 'Below $200' + WHEN price BETWEEN 200 AND 500 THEN '$200 - $500' + WHEN price BETWEEN 501 AND 1000 THEN '$501 - $1000' + ELSE 'Above $1000' + END AS price_range, + stock + FROM products +) + +SELECT + price_range, + SUM(stock) AS total_stock, + COUNT(*) AS product_count +FROM price_buckets +GROUP BY price_range +ORDER BY price_range; diff --git a/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/products.sql b/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/products.sql new file mode 100644 index 00000000..9ca75304 --- /dev/null +++ b/integration-tests/test-pipelines/run-main-with-downstream-pipeline/assets/products.sql @@ -0,0 +1,36 @@ +/* @bruin +name: products +type: duckdb.sql + +materialization: + type: table + +columns: + - name: product_id + type: INTEGER + description: "Unique identifier for the product" + primary_key: true + - name: product_name + type: VARCHAR + description: "Name of the product" + - name: price + type: FLOAT + description: "Price of the product in USD" + checks: + - name: positive + - name: stock + type: INTEGER + description: "Number of units in stock" +@bruin */ + +SELECT + 1 AS product_id, 'Laptop' AS product_name, 999.99 AS price, 10 AS stock +UNION ALL +SELECT + 2 AS product_id, 'Smartphone' AS product_name, 699.99 AS price, 50 AS stock +UNION ALL +SELECT + 3 AS product_id, 'Headphones' AS product_name, 199.99 AS price, 100 AS stock +UNION ALL +SELECT + 4 AS product_id, 'Monitor' AS product_name, 299.99 AS price, 25 AS stock; diff --git a/integration-tests/test-pipelines/run-main-with-downstream-pipeline/pipeline.yml b/integration-tests/test-pipelines/run-main-with-downstream-pipeline/pipeline.yml index 9172e408..5e17193c 100644 --- a/integration-tests/test-pipelines/run-main-with-downstream-pipeline/pipeline.yml +++ b/integration-tests/test-pipelines/run-main-with-downstream-pipeline/pipeline.yml @@ -1 +1,5 @@ -name: chess_extended_duckdb \ No newline at end of file +name: run-main-with-downstream-pipeline + +default_connections: + duckdb: "duckdb-run-main-with-downstream" + chess: "chess-run-main-with-downstream" \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/chess_games.asset.yml b/integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/chess_games.asset.yml index 68bc4e41..705fffcf 100644 --- a/integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/chess_games.asset.yml +++ b/integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/chess_games.asset.yml @@ -5,5 +5,5 @@ tags: parameters: destination: duckdb - source_connection: chess-default + source_connection: chess-run-main-with-filters source_table: games diff --git a/integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/chess_profiles.asset.yml b/integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/chess_profiles.asset.yml index cd770cdb..aa70c117 100644 --- a/integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/chess_profiles.asset.yml +++ b/integration-tests/test-pipelines/run-main-with-filters-pipeline/assets/chess_profiles.asset.yml @@ -1,8 +1,8 @@ name: chess_playground.profiles type: ingestr parameters: - source_connection: chess-default - source_table: profiles destination: duckdb + source_connection: chess-run-main-with-filters + source_table: profiles tags: - include \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-main-with-filters-pipeline/pipeline.yml b/integration-tests/test-pipelines/run-main-with-filters-pipeline/pipeline.yml index 9172e408..137693af 100644 --- a/integration-tests/test-pipelines/run-main-with-filters-pipeline/pipeline.yml +++ b/integration-tests/test-pipelines/run-main-with-filters-pipeline/pipeline.yml @@ -1 +1,4 @@ -name: chess_extended_duckdb \ No newline at end of file +name: run-main-with-filters-pipeline +default_connections: + duckdb: "duckdb-run-main-with-filters" + chess: "chess-run-main-with-filters" \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-malformed-pipeline/pipeline.yml b/integration-tests/test-pipelines/run-malformed-pipeline/pipeline.yml index c688837a..01687eb0 100644 --- a/integration-tests/test-pipelines/run-malformed-pipeline/pipeline.yml +++ b/integration-tests/test-pipelines/run-malformed-pipeline/pipeline.yml @@ -1 +1,4 @@ -name: integration_test_malformed \ No newline at end of file +name: integration_test_malformed + +default_connections: + duckdb: "duckdb-run-malformed-sql" \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-use-uv-pipeline/assets/chess_games.asset.yml b/integration-tests/test-pipelines/run-use-uv-pipeline/assets/chess_games.asset.yml index caa10252..b3bafc77 100644 --- a/integration-tests/test-pipelines/run-use-uv-pipeline/assets/chess_games.asset.yml +++ b/integration-tests/test-pipelines/run-use-uv-pipeline/assets/chess_games.asset.yml @@ -1,6 +1,6 @@ name: chess_playground.games type: ingestr parameters: - source_connection: chess-default + source_connection: chess-run-use-uv source_table: games destination: duckdb \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-use-uv-pipeline/assets/chess_profiles.asset.yml b/integration-tests/test-pipelines/run-use-uv-pipeline/assets/chess_profiles.asset.yml index d8158e02..4034f4d6 100644 --- a/integration-tests/test-pipelines/run-use-uv-pipeline/assets/chess_profiles.asset.yml +++ b/integration-tests/test-pipelines/run-use-uv-pipeline/assets/chess_profiles.asset.yml @@ -1,6 +1,6 @@ name: chess_playground.profiles type: ingestr parameters: - source_connection: chess-default + source_connection: chess-run-use-uv source_table: profiles destination: duckdb \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-use-uv-pipeline/pipeline.yml b/integration-tests/test-pipelines/run-use-uv-pipeline/pipeline.yml index 2108b581..f49b7360 100644 --- a/integration-tests/test-pipelines/run-use-uv-pipeline/pipeline.yml +++ b/integration-tests/test-pipelines/run-use-uv-pipeline/pipeline.yml @@ -1 +1,4 @@ -name: integration_test \ No newline at end of file +name: run-use-uv-pipeline +default_connections: + duckdb: "duckdb-run-use-uv" + chess: "chess-run-use-uv" \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/chess_games.asset.yml b/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/chess_games.asset.yml deleted file mode 100644 index 68bc4e41..00000000 --- a/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/chess_games.asset.yml +++ /dev/null @@ -1,9 +0,0 @@ -name: chess_playground.games -type: ingestr -tags: - - include - -parameters: - destination: duckdb - source_connection: chess-default - source_table: games diff --git a/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/chess_profiles.asset.yml b/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/chess_profiles.asset.yml deleted file mode 100644 index cd770cdb..00000000 --- a/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/chess_profiles.asset.yml +++ /dev/null @@ -1,8 +0,0 @@ -name: chess_playground.profiles -type: ingestr -parameters: - source_connection: chess-default - source_table: profiles - destination: duckdb -tags: - - include \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/game_outcome_summary.sql b/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/game_outcome_summary.sql deleted file mode 100644 index 65cb4340..00000000 --- a/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/game_outcome_summary.sql +++ /dev/null @@ -1,31 +0,0 @@ -/* @bruin - -name: chess_playground.game_outcome_summary -type: duckdb.sql -materialization: - type: table - -depends: - - chess_playground.games - -columns: - - name: result_type - type: string - description: "Type of game result (win, draw, etc.)" - - name: total_games - type: integer - description: "Total number of games with this result" - checks: - - name: positive -tags: - - include - -@bruin */ - -SELECT - g.white->>'result' AS result_type, - COUNT(*) AS total_games -FROM chess_playground.games g -WHERE g.white->>'result' IS NOT NULL -GROUP BY g.white->>'result' -ORDER BY total_games DESC; diff --git a/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/player_profile_summary.sql b/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/player_profile_summary.sql deleted file mode 100644 index 4aedab67..00000000 --- a/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/player_profile_summary.sql +++ /dev/null @@ -1,30 +0,0 @@ -/* @bruin - -name: chess_playground.player_profile_summary -type: duckdb.sql -materialization: - type: table - -depends: - - chess_playground.profiles -columns: - - name: total_players - type: integer - description: "Total number of players in the profiles table" - checks: - - name: positive - - name: active_players - type: integer - description: "Number of players marked as active" - - name: inactive_players - type: integer - description: "Number of players marked as inactive" -tags: - - exclude -@bruin */ - -SELECT - COUNT(*) AS total_players, - COUNT(CASE WHEN p.status = 'active' THEN 1 END) AS active_players, - COUNT(CASE WHEN p.status = 'inactive' THEN 1 END) AS inactive_players -FROM chess_playground.profiles p; diff --git a/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/player_summary.sql b/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/player_summary.sql deleted file mode 100644 index 0adf29fa..00000000 --- a/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/player_summary.sql +++ /dev/null @@ -1,62 +0,0 @@ -/* @bruin - -name: chess_playground.player_summary -type: duckdb.sql -materialization: - type: table - -depends: - - chess_playground.game_outcome_summary - - chess_playground.player_profile_summary - - chess_playground.games - - chess_playground.profiles - -columns: - - name: username - type: string - description: "Username of the player" - - name: total_games - type: integer - description: "Total games played by the player" - checks: - - name: non_negative - - name: total_wins - type: integer - description: "Total games won by the player" - - name: win_rate - type: float - description: "Win rate of the player" -tags: - - include - - exclude -@bruin */ - -WITH game_results AS ( - SELECT - CASE - WHEN g.white->>'result' = 'win' THEN g.white->>'@id' - WHEN g.black->>'result' = 'win' THEN g.black->>'@id' - ELSE NULL - END AS winner_aid, - g.white->>'@id' AS white_aid, - g.black->>'@id' AS black_aid -FROM chess_playground.games g - ) - -SELECT - p.username, - p.aid, - COUNT(*) AS total_games, - COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) + - COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) AS total_wins, - ROUND( - (COUNT(CASE WHEN g.white_aid = p.aid AND g.winner_aid = p.aid THEN 1 END) + - COUNT(CASE WHEN g.black_aid = p.aid AND g.winner_aid = p.aid THEN 1 END)) * 100.0 / - NULLIF(COUNT(*), 0), - 2 - ) AS win_rate -FROM chess_playground.profiles p - LEFT JOIN game_results g - ON p.aid IN (g.white_aid, g.black_aid) -GROUP BY p.username, p.aid -ORDER BY total_games DESC; diff --git a/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/product_categories.sql b/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/product_categories.sql new file mode 100644 index 00000000..1c24bc27 --- /dev/null +++ b/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/product_categories.sql @@ -0,0 +1,35 @@ +/* @bruin +name: product_categories +type: duckdb.sql +materialization: + type: table + +columns: + - name: category_id + type: INTEGER + description: "Unique identifier for the product category" + primary_key: true + checks: + - name: not_null + - name: positive + - name: category_name + type: VARCHAR + description: "Name of the product category" + checks: + - name: not_null + - name: description + type: VARCHAR + description: "Description of the product category" +@bruin */ + +SELECT + 1 AS category_id, 'Electronics' AS category_name, 'Devices like phones, laptops, and monitors' AS description +UNION ALL +SELECT + 2 AS category_id, 'Accessories' AS category_name, 'Complementary items like headphones and chargers' AS description +UNION ALL +SELECT + 3 AS category_id, 'Appliances' AS category_name, 'Household devices like refrigerators and microwaves' AS description +UNION ALL +SELECT + 4 AS category_id, 'Furniture' AS category_name, 'Home and office furniture like desks and chairs' AS description; diff --git a/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/product_price_summary.sql b/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/product_price_summary.sql new file mode 100644 index 00000000..6bd88706 --- /dev/null +++ b/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/product_price_summary.sql @@ -0,0 +1,45 @@ +/* @bruin + +name: product_price_summary +type: duckdb.sql +materialization: + type: table + +depends: + - products + +columns: + - name: price_range + type: VARCHAR + description: "Range of product prices" + - name: total_stock + type: INTEGER + description: "Total stock available in the price range" + checks: + - name: non_negative + - name: product_count + type: INTEGER + description: "Number of products in the price range" + checks: + - name: non_negative +@bruin */ + +WITH price_buckets AS ( + SELECT + CASE + WHEN price < 200 THEN 'Below $200' + WHEN price BETWEEN 200 AND 500 THEN '$200 - $500' + WHEN price BETWEEN 501 AND 1000 THEN '$501 - $1000' + ELSE 'Above $1000' + END AS price_range, + stock + FROM products +) + +SELECT + price_range, + SUM(stock) AS total_stock, + COUNT(*) AS product_count +FROM price_buckets +GROUP BY price_range +ORDER BY price_range; diff --git a/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/products.sql b/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/products.sql new file mode 100644 index 00000000..9ca75304 --- /dev/null +++ b/integration-tests/test-pipelines/run-with-downstream-pipeline/assets/products.sql @@ -0,0 +1,36 @@ +/* @bruin +name: products +type: duckdb.sql + +materialization: + type: table + +columns: + - name: product_id + type: INTEGER + description: "Unique identifier for the product" + primary_key: true + - name: product_name + type: VARCHAR + description: "Name of the product" + - name: price + type: FLOAT + description: "Price of the product in USD" + checks: + - name: positive + - name: stock + type: INTEGER + description: "Number of units in stock" +@bruin */ + +SELECT + 1 AS product_id, 'Laptop' AS product_name, 999.99 AS price, 10 AS stock +UNION ALL +SELECT + 2 AS product_id, 'Smartphone' AS product_name, 699.99 AS price, 50 AS stock +UNION ALL +SELECT + 3 AS product_id, 'Headphones' AS product_name, 199.99 AS price, 100 AS stock +UNION ALL +SELECT + 4 AS product_id, 'Monitor' AS product_name, 299.99 AS price, 25 AS stock; diff --git a/integration-tests/test-pipelines/run-with-downstream-pipeline/pipeline.yml b/integration-tests/test-pipelines/run-with-downstream-pipeline/pipeline.yml index 9172e408..0fed75f2 100644 --- a/integration-tests/test-pipelines/run-with-downstream-pipeline/pipeline.yml +++ b/integration-tests/test-pipelines/run-with-downstream-pipeline/pipeline.yml @@ -1 +1,3 @@ -name: chess_extended_duckdb \ No newline at end of file +name: run-with-downstream-pipeline +default_connections: + duckdb: "duckdb-run-with-downstream" diff --git a/integration-tests/test-pipelines/run-with-filters-pipeline/assets/chess_games.asset.yml b/integration-tests/test-pipelines/run-with-filters-pipeline/assets/chess_games.asset.yml index 68bc4e41..a2fb34ee 100644 --- a/integration-tests/test-pipelines/run-with-filters-pipeline/assets/chess_games.asset.yml +++ b/integration-tests/test-pipelines/run-with-filters-pipeline/assets/chess_games.asset.yml @@ -5,5 +5,5 @@ tags: parameters: destination: duckdb - source_connection: chess-default + source_connection: chess-run-with-filters source_table: games diff --git a/integration-tests/test-pipelines/run-with-filters-pipeline/assets/chess_profiles.asset.yml b/integration-tests/test-pipelines/run-with-filters-pipeline/assets/chess_profiles.asset.yml index cd770cdb..e9a17816 100644 --- a/integration-tests/test-pipelines/run-with-filters-pipeline/assets/chess_profiles.asset.yml +++ b/integration-tests/test-pipelines/run-with-filters-pipeline/assets/chess_profiles.asset.yml @@ -1,8 +1,8 @@ name: chess_playground.profiles type: ingestr parameters: - source_connection: chess-default source_table: profiles destination: duckdb + source_connection: chess-run-with-filters tags: - include \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-with-filters-pipeline/pipeline.yml b/integration-tests/test-pipelines/run-with-filters-pipeline/pipeline.yml index 9172e408..550f2db5 100644 --- a/integration-tests/test-pipelines/run-with-filters-pipeline/pipeline.yml +++ b/integration-tests/test-pipelines/run-with-filters-pipeline/pipeline.yml @@ -1 +1,5 @@ -name: chess_extended_duckdb \ No newline at end of file +name: run-with-filters-pipeline + +default_connections: + duckdb: "duckdb-run-with-filters" + chess: "chess-run-with-filters" \ No newline at end of file diff --git a/integration-tests/test-pipelines/run-with-tags-pipeline/assets/chess_games.asset.yml b/integration-tests/test-pipelines/run-with-tags-pipeline/assets/chess_games.asset.yml index 68bc4e41..9a7b5b97 100644 --- a/integration-tests/test-pipelines/run-with-tags-pipeline/assets/chess_games.asset.yml +++ b/integration-tests/test-pipelines/run-with-tags-pipeline/assets/chess_games.asset.yml @@ -5,5 +5,5 @@ tags: parameters: destination: duckdb - source_connection: chess-default + source_connection: chess-run-with-tags source_table: games diff --git a/integration-tests/test-pipelines/run-with-tags-pipeline/assets/chess_profiles.asset.yml b/integration-tests/test-pipelines/run-with-tags-pipeline/assets/chess_profiles.asset.yml index cd770cdb..90a2e782 100644 --- a/integration-tests/test-pipelines/run-with-tags-pipeline/assets/chess_profiles.asset.yml +++ b/integration-tests/test-pipelines/run-with-tags-pipeline/assets/chess_profiles.asset.yml @@ -1,7 +1,7 @@ name: chess_playground.profiles type: ingestr parameters: - source_connection: chess-default + source_connection: chess-run-with-tags source_table: profiles destination: duckdb tags: diff --git a/integration-tests/test-pipelines/run-with-tags-pipeline/pipeline.yml b/integration-tests/test-pipelines/run-with-tags-pipeline/pipeline.yml index 9172e408..e340507d 100644 --- a/integration-tests/test-pipelines/run-with-tags-pipeline/pipeline.yml +++ b/integration-tests/test-pipelines/run-with-tags-pipeline/pipeline.yml @@ -1 +1,5 @@ -name: chess_extended_duckdb \ No newline at end of file +name: run_with_tags + +default_connections: + duckdb: "duckdb-run-with-tags" + chess: "chess-run-with-tags" \ No newline at end of file