Add queries generator

ydb/library/benchmarks/gen_queries/gen_queries.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+
+import os
+import argparse
+from dataclasses import dataclass
+from ydb.library.benchmarks.template import Builder
+
+
+@dataclass
+class Profile:
+    syntax: str
+    profile: str
+    pragmas: str
+    tables: str
+
+
+def main():
+    profiles = [
+        Profile("yql", "dqrun", "pragmas_scalar.yql", "tables_bindings.jinja"),
+        Profile("yql", "dqrun_block", "pragmas_block.yql", "tables_bindings.jinja"),
+        Profile("pg", "dqrun", "pragmas_scalar_pg.yql", "tables_bindings.jinja"),
+        Profile("pg", "postgres", None, "tables_postgres.jinja"),
+    ]
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--syntax', default='yql', help='syntax "pg" or "yql"')
+    parser.add_argument('--profile', default='dqrun', help='profile "dqrun" "dqrun_block" or "postgres"')
+    parser.add_argument('--variant', default="h", help='variant "h" or "ds"')
+    parser.add_argument('--output', default='q', help='output directory')
+    args = parser.parse_args()
+    profile = None
+    for p in profiles:
+        if p.syntax == args.syntax and p.profile == args.profile:
+            profile = p
+            break
+    if profile is None:
+        print("Cannot find syntax/profile pair")
+        print("Awailable variants:")
+        for p in profiles:
+            print(f"  {p.syntax}/{p.profile}")
+        return
+
+    path = f"{args.output}/{args.variant}"
+    if not os.path.exists(path):
+        os.makedirs(path, exist_ok=True)
+    b = Builder()
+    if p.pragmas:
+        b.add_link("pragmas.sql", p.pragmas)
+    else:
+        b.add("pragmas.sql", "")
+    b.add_link("tables.jinja", p.tables)
+    queries = None
+    if args.variant == "h":
+        queries = range(1, 23)
+    else:
+        queries = range(1, 100)
+
+    for q in queries:
+        with open(f"{path}/q{q}.sql", "w") as f:
+            print(f"Generating {args.variant}/{p.syntax}/q{q}.sql")
+            sql = b.build(f"{args.variant}/{p.syntax}/q{q}.sql", True)
+            f.write(sql)
+
+
+if __name__ == "__main__":
+    main()

ydb/library/benchmarks/gen_queries/header.sql.jinja

@@ -0,0 +1,2 @@
+{% include 'pragmas.sql' %}
+{% include 'tables.jinja' %}

ydb/library/benchmarks/gen_queries/pragmas_block.yql

@@ -0,0 +1,5 @@
+PRAGMA dq.EnableDqReplicate = "1";
+PRAGMA DisableSimpleColumns;
+PRAGMA UseBlocks;
+PRAGMA EmitAggApply;
+pragma OrderedColumns="true";

ydb/library/benchmarks/gen_queries/pragmas_scalar.yql

@@ -0,0 +1,16 @@
+PRAGMA dq.EnableDqReplicate = "1";
+PRAGMA DisableSimpleColumns;
+
+PRAGMA dq.MaxTasksPerStage='200';
+PRAGMA dq.MaxTasksPerOperation='1700';
+PRAGMA dq.HashJoinMode = "grace";
+PRAGMA dq.HashShuffleTasksRatio="1.0";
+PRAGMA dq.HashShuffleMaxTasks="200";
+
+-- pragma dq.AggregateStatsByStage="false";
+pragma dq.ComputeActorType="sync";
+pragma dq.OptLLVM="off";
+
+PRAGMA s3.UseBlocksSource="true";
+pragma OrderedColumns="true";
+

ydb/library/benchmarks/gen_queries/pragmas_scalar_pg.yql

@@ -0,0 +1,2 @@
+--!syntax_pg
+set dq.EnableDqReplicate="1";

ydb/library/benchmarks/gen_queries/tables.jinja

@@ -0,0 +1,32 @@
+{% set customer = "bindings.customer" %}
+{% set lineitem = "bindings.lineitem" %}
+{% set nation = "bindings.nation" %}
+{% set orders = "bindings.orders" %}
+{% set part = "bindings.part" %}
+{% set partsupp = "bindings.partsupp" %}
+{% set region = "bindings.region" %}
+{% set supplier = "bindings.supplier" %}
+{% set call_center = "bindings.call_center" %}
+{% set catalog_page = "bindings.catalog_page" %}
+{% set catalog_returns = "bindings.catalog_returns" %}
+{% set catalog_sales = "bindings.catalog_sales" %}
+{% set customer = "bindings.customer" %}
+{% set customer_address = "bindings.customer_address" %}
+{% set customer_demographics = "bindings.customer_demographics" %}
+{% set date_dim = "bindings.date_dim" %}
+{% set household_demographics = "bindings.household_demographics" %}
+{% set income_band = "bindings.income_band" %}
+{% set inventory = "bindings.inventory" %}
+{% set item = "bindings.item" %}
+{% set promotion = "bindings.promotion" %}
+{% set reason = "bindings.reason" %}
+{% set ship_mode = "bindings.ship_mode" %}
+{% set store = "bindings.store" %}
+{% set store_returns = "bindings.store_returns" %}
+{% set store_sales = "bindings.store_sales" %}
+{% set time_dim = "bindings.time_dim" %}
+{% set warehouse = "bindings.warehouse" %}
+{% set web_page = "bindings.web_page" %}
+{% set web_returns = "bindings.web_returns" %}
+{% set web_sales = "bindings.web_sales" %}
+{% set web_site = "bindings.web_site" %}

ydb/library/benchmarks/gen_queries/tables_postgres.jinja

@@ -0,0 +1,32 @@
+{% set customer = "customer" %}
+{% set lineitem = "lineitem" %}
+{% set nation = "nation" %}
+{% set orders = "orders" %}
+{% set part = "part" %}
+{% set partsupp = "partsupp" %}
+{% set region = "region" %}
+{% set supplier = "supplier" %}
+{% set call_center = "call_center" %}
+{% set catalog_page = "catalog_page" %}
+{% set catalog_returns = "catalog_returns" %}
+{% set catalog_sales = "catalog_sales" %}
+{% set customer = "customer" %}
+{% set customer_address = "customer_address" %}
+{% set customer_demographics = "customer_demographics" %}
+{% set date_dim = "date_dim" %}
+{% set household_demographics = "household_demographics" %}
+{% set income_band = "income_band" %}
+{% set inventory = "inventory" %}
+{% set item = "item" %}
+{% set promotion = "promotion" %}
+{% set reason = "reason" %}
+{% set ship_mode = "ship_mode" %}
+{% set store = "store" %}
+{% set store_returns = "store_returns" %}
+{% set store_sales = "store_sales" %}
+{% set time_dim = "time_dim" %}
+{% set warehouse = "warehouse" %}
+{% set web_page = "web_page" %}
+{% set web_returns = "web_returns" %}
+{% set web_sales = "web_sales" %}
+{% set web_site = "web_site" %}