Merge branch 'main' into refactor-counters-clean-up-total-stats

.github/config/muted_ya.txt

@@ -10,7 +10,6 @@ ydb/core/keyvalue/ut_trace TKeyValueTracingTest.*
 ydb/core/kqp/provider/ut KikimrIcGateway.TestLoadBasicSecretValueFromExternalDataSourceMetadata
 ydb/core/kqp/ut/olap KqpOlapBlobsSharing.*
 ydb/core/kqp/ut/olap KqpOlapStatistics.StatsUsageWithTTL
-ydb/core/kqp/ut/olap KqpOlapAggregations.Aggregation_ResultCountAll_FilterL
 ydb/core/kqp/ut/pg KqpPg.CreateIndex
 ydb/core/kqp/ut/query KqpLimits.QueryReplySize
 ydb/core/kqp/ut/query KqpQuery.QueryTimeout
@@ -88,10 +87,6 @@ ydb/tests/fq/plans test_stats_mode.py.TestStatsMode.test_mode[v1-STATS_MODE_FULL
 ydb/tests/fq/yds *
 ydb/tests/fq/control_plane_storage *
 ydb/tests/functional/audit *
-ydb/tests/functional/clickbench test.py.test_run_determentistic[column]
-ydb/tests/functional/clickbench test.py.test_run_benchmark[scan-column]
-ydb/tests/functional/clickbench test.py.test_run_benchmark[generic-column]
-ydb/tests/functional/clickbench test.py.test_plans[column]
 ydb/tests/functional/blobstorage test_replication.py.TestReplicationAfterNodesRestart.test_replication*
 ydb/tests/functional/kqp/kqp_indexes ConsistentIndexRead.InteractiveTx
 ydb/tests/functional/kqp/kqp_query_session KqpQuerySession.NoLocalAttach

.github/scripts/tests/generate-summary.py

@@ -2,10 +2,11 @@
 import argparse
 import dataclasses
 import datetime
+import json
 import os
 import re
-import json
 import sys
+import traceback
 from github import Github, Auth as GithubAuth
 from github.PullRequest import PullRequest
 from enum import Enum
@@ -246,29 +247,38 @@ def render_testlist_html(rows, fn, build_preset):
     # remove status group without tests
     status_order = [s for s in status_order if s in status_test]
 
-    # get failed tests
-    failed_tests_array = []
-    history={}
-    for test in status_test.get(TestStatus.FAIL, []):
-        failed_tests_array.append(test.full_name)
-
-    if failed_tests_array:
-        try:
-            history = get_test_history(failed_tests_array, last_n_runs, build_preset)
-        except Exception as e:
-            print(f'Error:{e}')
-
-    # sorting, at first show tests with passed resuts in history
-
-    if TestStatus.FAIL in status_test:
-        for test in status_test.get(TestStatus.FAIL, []):
-            if test.full_name in history:
-                test.count_of_passed = history[test.full_name][
-                    next(iter(history[test.full_name]))
-                ]["count_of_passed"]
-            else:
-                test.count_of_passed = 0
-        status_test[TestStatus.FAIL].sort(key=lambda val: (val.count_of_passed, val.full_name), reverse=True)
+    # statuses for history
+    status_for_history = [TestStatus.FAIL, TestStatus.MUTE]
+    status_for_history = [s for s in status_for_history if s in status_test]
+
+    tests_names_for_history = []
+    history= {}
+    tests_in_statuses = [test for status in status_for_history for test in status_test.get(status)]
+
+    # get tests for history
+    for test in tests_in_statuses:
+        tests_names_for_history.append(test.full_name)
+
+    try:
+        history = get_test_history(tests_names_for_history, last_n_runs, build_preset)
+    except Exception:
+        print(traceback.format_exc())
+
+    #geting count of passed tests in history for sorting
+    for test in tests_in_statuses:
+        if test.full_name in history:
+            test.count_of_passed = len(
+                [
+                    history[test.full_name][x]
+                    for x in history[test.full_name]
+                    if history[test.full_name][x]["status"] == "passed"
+                ]
+            )
+    # sorting, 
+    # at first - show tests with passed resuts in history
+    # at second - sorted by test name
+    for current_status in status_for_history:
+        status_test.get(current_status,[]).sort(key=lambda val: (-val.count_of_passed, val.full_name))
 
     content = env.get_template("summary.html").render(
         status_order=status_order,

.github/scripts/tests/get_test_history.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 
 import configparser
+import datetime
 import os
 import ydb
-import datetime
 
 
 dir = os.path.dirname(__file__)
@@ -14,7 +14,6 @@
 DATABASE_ENDPOINT = config["QA_DB"]["DATABASE_ENDPOINT"]
 DATABASE_PATH = config["QA_DB"]["DATABASE_PATH"]
 
-
 def get_test_history(test_names_array, last_n_runs_of_test_amount, build_type):
     if "CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" not in os.environ:
         print(
@@ -46,8 +45,7 @@ def get_test_history(test_names_array, last_n_runs_of_test_amount, build_type):
             and status != 'skipped'
         );
 
-        select full_name,test_name,build_type, commit, branch, run_timestamp, status, status_description,rn, 
-        COUNT_IF(status = 'passed') over (PARTITION BY test_name) as count_of_passed
+        select full_name,test_name,build_type, commit, branch, run_timestamp, status, status_description,rn
         from  $tests
         WHERE rn <= $rn_max
         ORDER BY test_name, run_timestamp;  
@@ -65,27 +63,32 @@ def get_test_history(test_names_array, last_n_runs_of_test_amount, build_type):
 
         with session.transaction() as transaction:
             prepared_query = session.prepare(query)
-            query_params = {
-                "$test_names": test_names_array,
-                "$rn_max": last_n_runs_of_test_amount,
-                "$build_type": build_type,
-            }
-
-            result_set = session.transaction(ydb.SerializableReadWrite()).execute(
-                prepared_query, parameters=query_params, commit_tx=True
-            )
 
             results = {}
-            for row in result_set[0].rows:
-                if not row["full_name"].decode("utf-8") in results:
-                    results[row["full_name"].decode("utf-8")] = {}
-
-                results[row["full_name"].decode("utf-8")][row["run_timestamp"]] = {
-                    "status": row["status"],
-                    "commit": row["commit"],
-                    "datetime": datetime.datetime.fromtimestamp(int(row["run_timestamp"] / 1000000)).strftime("%H:%m %B %d %Y"),
-                    "count_of_passed": row["count_of_passed"],
+            batch_size = 100
+            for start in range(0, len(test_names_array), batch_size):
+                test_names_batch = test_names_array[start:start + batch_size]
+
+                query_params = {
+                    "$test_names": test_names_batch,
+                    "$rn_max": last_n_runs_of_test_amount,
+                    "$build_type": build_type,
                 }
+
+                result_set = session.transaction(ydb.SerializableReadWrite()).execute(
+                    prepared_query, parameters=query_params, commit_tx=True
+                )
+
+                for row in result_set[0].rows:
+                    if not row["full_name"].decode("utf-8") in results:
+                        results[row["full_name"].decode("utf-8")] = {}
+
+                    results[row["full_name"].decode("utf-8")][row["run_timestamp"]] = {
+                        "status": row["status"],
+                        "commit": row["commit"],
+                        "datetime": datetime.datetime.fromtimestamp(int(row["run_timestamp"] / 1000000)).strftime("%H:%m %B %d %Y"),
+                        "status_description": row["status_description"],
+                    }
             return results
 
 

.github/scripts/tests/templates/summary.html

@@ -2,7 +2,7 @@
 <head>
     <style>
         body {
-            font-family: Arial, sans-serif;
+            font-family: -apple-system,BlinkMacSystemFont,"Segoe UI","Noto Sans",Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji";
             margin: 0;
             padding: 0;
             box-sizing: border-box;
@@ -69,8 +69,10 @@
 
         .svg-icon {
             float: left;
-            width: 16px;
-            height: 16px;
+            width: 14px;
+            height: 14px;
+            padding-left: 1px;
+            padding-right: 1px;
             margin-right: 0px;
             border-radius: 50%;
             position: relative; /* Essential for tooltips */
@@ -79,7 +81,9 @@
 
         .tooltip {
             visibility: hidden;
-            width: 180px;
+            max-width: 340px;
+            min-width: 260px;
+            word-break: break-word;
             background-color: #7d7d92;
             color: #fff;
             text-align: left;
@@ -277,7 +281,7 @@ <h1 id="{{ status.name}}" class="collapsible-header">{{ status.name }} ({{ tests
     <thead>
     <tr>
         <th>test name</th>
-    {% if status.name == 'FAIL' and history%}
+    {% if status.is_error  and history%}
         <th>history<br>
             old->new
         </th>
@@ -292,7 +296,7 @@ <h1 id="{{ status.name}}" class="collapsible-header">{{ status.name }} ({{ tests
     <tbody>
     {% for t in tests[status] %}
     <tr>
-        {% if status.name == 'FAIL' %} 
+        {% if status.is_error %} 
         <td class="test_name with_history">
         {% else %} 
         <td class="test_name">
@@ -308,7 +312,7 @@ <h1 id="{{ status.name}}" class="collapsible-header">{{ status.name }} ({{ tests
             </button>
             {% endif %}
         </td>
-        {% if (status.name == 'FAIL' and t.full_name in history) %}
+        {% if (status.is_error and t.full_name in history) %}
         <td>
             {% for h in history[t.full_name] %}
                 <span class="svg-icon">
@@ -321,22 +325,25 @@ <h1 id="{{ status.name}}" class="collapsible-header">{{ status.name }} ({{ tests
                         <path fill-rule="evenodd" d="M13.5 8a5.5 5.5 0 1 1-11 0 5.5 5.5 0 0 1 11 0M15 8A7 7 0 1 1 1 8a7 7 0 0 1 14 0m-3.9-1.55a.75.75 0 1 0-1.2-.9L7.419 8.858 6.03 7.47a.75.75 0 0 0-1.06 1.06l2 2a.75.75 0 0 0 1.13-.08z" clip-rule="evenodd"></path>
                     </svg>
                     {% elif history[t.full_name][h].status == 'mute' %}
-                    <svg class="svg_mute" viewBox="0 0 16 16" >
+                    <svg class="svg_mute" viewBox="0 0 17 16" >
                         <path  fill-rule="evenodd" d="M5.06 9.94A1.5 1.5 0 0 0 4 9.5H2a.5.5 0 0 1-.5-.5V7a.5.5 0 0 1 .5-.5h2a1.5 1.5 0 0 0 1.06-.44l2.483-2.482a.268.268 0 0 1 .457.19v8.464a.268.268 0 0 1-.457.19zM2 5h2l2.482-2.482A1.768 1.768 0 0 1 9.5 3.768v8.464a1.768 1.768 0 0 1-3.018 1.25L4 11H2a2 2 0 0 1-2-2V7a2 2 0 0 1 2-2m10.28.72a.75.75 0 1 0-1.06 1.06L12.44 8l-1.22 1.22a.75.75 0 1 0 1.06 1.06l1.22-1.22 1.22 1.22a.75.75 0 1 0 1.06-1.06L14.56 8l1.22-1.22a.75.75 0 0 0-1.06-1.06L13.5 6.94z" clip-rule="evenodd"></path>                    </svg>
                     </svg>
                     {% endif %}
                     <span class="tooltip">
                         Status: {{history[t.full_name][h].status}}<br>
                         Date: {{ history[t.full_name][h].datetime }}<br>
-                        SHA: <a href="https://github.com/ydb-platform/ydb//commit/{{ history[t.full_name][h].commit }}" style="color: #00f;" target="_blank">{{history[t.full_name][h].commit}}</a>
+                        {% if history[t.full_name][h].status_description != "" %}
+                        Info: {{ history[t.full_name][h].status_description.split(';')[0][0:100] }}<br>
+                        {% endif %}
+                        SHA: <a href="https://github.com/ydb-platform/ydb//commit/{{ history[t.full_name][h].commit }}" style="color: #00f;" target="_blank">{{history[t.full_name][h].commit[0:8]}}</a>
                     </span>
 
                 </span>
             {% endfor %}
         </td> 
-        {% elif (status.name == 'FAIL' and  history) %}
+        {% elif (status.is_error and  history) %}
         <td></td>
-        {% elif (status.name == 'FAIL') %}
+        {% elif status.is_error %}
         {% endif %}
         <td><span title="{{ t.elapsed }}s">{{ t.elapsed_display }}</span></td>
         <td>

.github/workflows/pr_check.yml

@@ -245,9 +245,9 @@ jobs:
         run: |
           successbuilds=$(curl -L -X GET -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \
             https://api.github.com/repos/${{github.repository}}/commits/${{github.event.pull_request.head.sha}}/status | \
-            jq -cr '.statuses | .[] | select(.state=="success") | select(.context | startswith("build_")) | .context' | \
+            jq -cr '.statuses | .[] | select(.state=="success") | select(.context | (startswith("build_") or startswith("test_relwithdebinfo")) ) | .context' | \
             wc -l )
-          if [[ $successbuilds == "3" ]];then
+          if [[ $successbuilds == "4" ]];then
             integrated_status="success"
           else
             integrated_status="failure"

build/internal/ya.conf

@@ -22,3 +22,5 @@ USE_ICONV = "static"
 USE_IDN = "static"
 APPLE_SDK_LOCAL = "yes"
 CFLAGS = "-fno-omit-frame-pointer"
+# TODO: uncomment after warnings fix
+FIX_UNUSED_PARAMETR_PLS = "1"  # 2 aug 2024. Transitional flag for protobuf fix, DTCC-2275

ydb/ci/build_bloat/html/index.html

@@ -12,9 +12,9 @@
             <div class="note">Click on a box to zoom in.  Click on the outermost box to zoom out.</div>
             <legend>
                 <div>Legend:</div>
-                <div class="webtreemap-node webtreemap-type-dir">Dir</div>
-                <div class="webtreemap-node webtreemap-type-cpp">Cpp</div>
-                <div class="webtreemap-node webtreemap-type-h">Header</div>
+                {% for id_, name, color in types %}
+                <div class="webtreemap-node webtreemap-type-{{ id_ }}">{{ name }}</div>
+                {% endfor %}
             </legend>
         </div>
         <div id='map'></div>

ydb/ci/build_bloat/html/webtreemap.css

@@ -18,15 +18,11 @@
 }
 
 /* Optional: Different background colors depending on type. */
-.webtreemap-type-h {
-  background: #66C2A5;
-}
-.webtreemap-type-cpp {
-  background: #FC8D62;
-}
-.webtreemap-type-dir {
-  background: #8DA0CB;
+{% for id_, name, color in types %}
+.webtreemap-type-{{ id_ }} {
+  background: {{ color }};
 }
+{% endfor %}
 
 #legend > * {
   border: solid 1px #444;