Implemented IdWrapper for TGroupId

.github/CODEOWNERS

@@ -15,3 +15,9 @@
 /ydb/library/yql/yt @Krock21 @Krisha11 @zlobober @gritukan
 
 /ydb/services/fq/ @ydb-platform/fq
+
+/ydb/core/kafka_proxy @ydb-platform/Topics
+/ydb/core/persqueue @ydb-platform/Topics
+/ydb/services/datastreams @ydb-platform/Topics
+/ydb/services/deprecated/persqueue_v0 @ydb-platform/Topics
+/ydb/services/persqueue_v1 @ydb-platform/Topics

.github/actions/build_analytics/action.yml

@@ -0,0 +1,54 @@
+name: build_analytics
+description: Gather and send build analytics 
+inputs:
+  build_preset:
+    type: string
+    default: ""
+  build_target:
+    type: string
+    default: ""
+  secs:
+    type: string
+    default: ""
+  vars:
+    type: string
+    default: ""
+runs:
+  using: "composite"
+  steps:
+    - name: Prepare s3cmd
+      uses: ./.github/actions/s3cmd
+      with:
+        s3_bucket: ${{ fromJSON( inputs.vars ).AWS_BUCKET }}
+        s3_endpoint: ${{ fromJSON( inputs.vars ).AWS_ENDPOINT }}
+        s3_key_id: ${{ fromJSON( inputs.secs ).AWS_KEY_ID }}
+        s3_key_secret: ${{ fromJSON( inputs.secs ).AWS_KEY_VALUE }}
+        folder_prefix: ya-
+        build_preset: ${{ inputs.build_preset }}
+
+    - name: Run analytics tools
+      shell: bash
+      run: |
+        set -ex
+        # FIXME: target name may be not the same as dir name
+        export TARGET_NAME=`basename ${{ inputs.build_target }}`
+        export TARGET_DIR=${{ inputs.build_target }}
+        export build_preset="${{ inputs.build_preset }}" 
+        export build_target="${{ inputs.build_target }}" 
+        ./ya tool bloat --linker-map $TARGET_DIR/$TARGET_NAME.map.lld --input $TARGET_DIR/$TARGET_NAME --save-html ya_bloat_html
+        ./ydb/ci/build_bloat/main.py --build-dir . --html-dir-cpp html_cpp_impact --html-dir-headers html_headers_impact
+        ./ydb/ci/build_bloat/ydb_upload.py --html-dir-cpp html_cpp_impact --html-dir-headers html_headers_impact
+
+    - name: Upload results
+      shell: bash
+      run: |
+        set -ex
+        s3cmd sync -r --acl-public --stats --no-progress --no-mime-magic --guess-mime-type --no-check-md5 "ya_bloat_html/" "$S3_BUCKET_PATH/ya_bloat_html/"
+        echo "[ya bloat tool]($S3_URL_PREFIX/ya_bloat_html/index.html) (better use Safari, because it is too large for Chrome)" >> $GITHUB_STEP_SUMMARY
+        
+        s3cmd sync -r --acl-public --stats --no-progress --no-mime-magic --guess-mime-type --no-check-md5 "html_cpp_impact/" "$S3_BUCKET_PATH/html_cpp_impact/"
+        echo "[cpp compilation time]($S3_URL_PREFIX/html_cpp_impact/index.html)" >> $GITHUB_STEP_SUMMARY
+        
+        s3cmd sync -r --acl-public --stats --no-progress --no-mime-magic --guess-mime-type --no-check-md5 "html_headers_impact/" "$S3_BUCKET_PATH/html_headers_impact/"
+        echo "[headers impact]($S3_URL_PREFIX/html_headers_impact/index.html)" >> $GITHUB_STEP_SUMMARY
+

.github/actions/build_and_test_ya/action.yml

@@ -42,6 +42,9 @@ inputs:
   put_build_results_to_cache:
     type: boolean
     default: true
+  additional_ya_make_args:
+    type: string
+    default: ""
   secs:
     type: string
     default: ""
@@ -66,7 +69,17 @@ runs:
         check_url=$(curl -s $jobs_url | jq --arg n "$BUILD_PRESET" -r '.jobs[] | select(.name | contains($n)) | .html_url')
         
         echo "Pre-commit [check]($check_url) for $(git rev-parse HEAD) has started." | .github/scripts/tests/comment-pr.py --rewrite
-    
+
+        curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \
+          https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \
+          -d '{"state":"pending","description":"The check has been started","context":"build_${{inputs.build_preset}}"}'
+
+        if [[ "${{inputs.run_tests}}" == "true" ]];then
+          curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \
+            https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \
+            -d '{"state":"pending","description":"The check has been started","context":"test_${{inputs.build_preset}}"}'
+        fi
+   
     - name: Prepare s3cmd
       uses: ./.github/actions/s3cmd
       with:
@@ -77,6 +90,20 @@ runs:
         folder_prefix: ya-
         build_preset: ${{ inputs.build_preset }}
 
+    - name: Generate ya.make with affected test suites and modules list
+      if: inputs.increment == 'true'
+      uses: ./.github/actions/graph_compare
+
+    - name: Change target in case of incremental (tests)
+      id: test_run_choice
+      shell: bash
+      run: |
+        if [ "${{ inputs.increment }}" == "true" ]; then
+          echo "target=." >> $GITHUB_OUTPUT
+        else
+          echo "target=${{ inputs.build_target }}" >> $GITHUB_OUTPUT
+        fi
+
     - name: Build
       uses: ./.github/actions/build_ya
       id: build
@@ -85,25 +112,11 @@ runs:
         build_target: ${{ inputs.build_target }}
         build_preset: ${{ inputs.build_preset }}
         bazel_remote_uri: ${{  fromJSON( inputs.vars ).REMOTE_CACHE_URL || '' }}
-        bazel_remote_username: ${{ inputs.put_build_results_to_cache &&  fromJSON( inputs.secs ).REMOTE_CACHE_USERNAME || '' }}
-        bazel_remote_password: ${{ inputs.put_build_results_to_cache &&  fromJSON( inputs.secs ).REMOTE_CACHE_PASSWORD || '' }}
+        bazel_remote_username: ${{ fromJSON( inputs.secs ).REMOTE_CACHE_USERNAME || '' }}
+        bazel_remote_password: ${{ fromJSON( inputs.secs ).REMOTE_CACHE_PASSWORD || '' }}
+        put_build_results_to_cache: ${{ inputs.put_build_results_to_cache }}
         link_threads: ${{ inputs.link_threads }}
-
-    - name: Generate ya.make with affected test suites list
-      if: inputs.run_tests == 'true' && inputs.increment == 'true'
-      uses: ./.github/actions/graph_compare
-
-    - name: Check if there's a list of tests to run
-      id: test_run_choice
-      shell: bash
-      run: |
-        if [ -f ya.make ];then
-          echo "target='.'" >> $GITHUB_OUTPUT
-          echo "Listed test targets: "
-          cat ya.make
-        else
-          echo "target=${{ inputs.build_target }}" >> $GITHUB_OUTPUT
-        fi
+        additional_ya_make_args: ${{ inputs.additional_ya_make_args }}
 
     - name: Run tests
       uses: ./.github/actions/test_ya
@@ -117,6 +130,7 @@ runs:
         testman_url: ${{  fromJSON( inputs.vars ).TESTMO_URL }}
         testman_project_id: ${{  fromJSON( inputs.vars ).TESTMO_PROJECT_ID }}
         link_threads: ${{ inputs.link_threads }}
+        additional_ya_make_args: ${{ inputs.additional_ya_make_args }}
         test_threads: ${{ inputs.test_threads }}
 
     - name: Notify about failed build

.github/actions/build_ya/action.yml

@@ -8,6 +8,9 @@ inputs:
     required: true
     default: "relwithdebinfo"
     description: "debug, relwithdebinfo, release-asan, release-tsan, release, release-cmake14"
+  put_build_results_to_cache:
+    required: false
+    default: "true"
   bazel_remote_uri:
     required: false
     description: "bazel-remote endpoint"
@@ -21,6 +24,9 @@ inputs:
     required: false
     default: "8"
     description: "link threads count"
+  additional_ya_make_args:
+    type: string
+    default: ""
 outputs:
   success:
     value: ${{ steps.build.outputs.status }}
@@ -65,7 +71,7 @@ runs:
           extra_params+=(--bazel-remote-base-uri "${{ inputs.bazel_remote_uri }}")
         fi
         
-        if [ ! -z "${{ inputs.bazel_remote_username }}" ]; then
+        if [ "${{ inputs.put_build_results_to_cache }}" = "true" ]; then
           extra_params+=(--bazel-remote-username "${{ inputs.bazel_remote_username }}")
           extra_params+=(--bazel-remote-password "${{ inputs.bazel_remote_password }}")
           extra_params+=(--bazel-remote-put --dist-cache-evict-bins --add-result .o --add-result .a)
@@ -107,6 +113,10 @@ runs:
             ;;
         esac
         
+        if [ ! -z "${{ inputs.additional_ya_make_args }}" ]; then
+          extra_params+=(${{ inputs.additional_ya_make_args }})
+        fi
+
         echo "::debug::get version"
         ./ya --version
         
@@ -140,8 +150,14 @@ runs:
       run: |
         set -x
         if [ "${{ steps.build.outputs.status }}" == "failed" ]; then
+          curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \
+            https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \
+            -d '{"state":"failure","description":"The check has been failed","context":"build_${{inputs.build_preset}}"}'
           echo "Build failed. see the [build logs]($LOG_URL)." | .github/scripts/tests/comment-pr.py --fail
         else
+          curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \
+            https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \
+            -d '{"state":"success","description":"The check has been completed successfully","context":"build_${{inputs.build_preset}}"}'
           echo "Build successful." | .github/scripts/tests/comment-pr.py --ok
         fi
 

.github/actions/test_ya/action.yml

@@ -24,6 +24,9 @@ inputs:
     required: false
     default: "12"
     description: "link threads count"
+  additional_ya_make_args:
+    type: string
+    default: ""
   testman_token:
     required: false
     description: "test manager auth token"
@@ -41,8 +44,10 @@ runs:
       shell: bash
       run: |
         export TMP_DIR=$(pwd)/tmp
+        export LOG_DIR=$TMP_DIR/logs
+        
         echo "TMP_DIR=$TMP_DIR" >> $GITHUB_ENV
-        echo "LOG_DIR=$TMP_DIR/logs" >> $GITHUB_ENV
+        echo "LOG_DIR=$LOG_DIR" >> $GITHUB_ENV
         echo "OUT_DIR=$TMP_DIR/out" >> $GITHUB_ENV
         echo "ARTIFACTS_DIR=$TMP_DIR/artifacts" >> $GITHUB_ENV
         echo "TEST_ARTIFACTS_DIR=$TMP_DIR/test_artifacts" >> $GITHUB_ENV
@@ -54,6 +59,11 @@ runs:
         echo "SUMMARY_LINKS=$(mktemp)" >> $GITHUB_ENV
         echo "GITHUB_TOKEN=${{ github.token }}" >> $GITHUB_ENV
         echo "BUILD_PRESET=${{ inputs.build_preset }}" >> $GITHUB_ENV
+        
+        echo "YA_TEST_LOG_FILENAME=$LOG_DIR/ya_test.log" >> $GITHUB_ENV
+        echo "YA_TEST_LOG_URL=$S3_URL_PREFIX/test_logs/ya_test.log" >> $GITHUB_ENV
+        echo "S3_LOG_BUCKET_PATH=$S3_BUCKET_PATH/test_logs" >> $GITHUB_ENV
+        
     
     - name: prepare
       shell: bash
@@ -122,7 +132,7 @@ runs:
             EXTRA_TAG=""
             ;;
         esac
-        
+
         testmo automation:resources:add-link --name build --url "$RUN_URL" --resources testmo.json
         testmo automation:resources:add-field --name git-sha --type string --value "${GITHUB_SHA:0:7}" --resources testmo.json
         RUN_ID=$(
@@ -191,17 +201,26 @@ runs:
             ;;
         esac
         
+        if [ ! -z "${{ inputs.additional_ya_make_args }}" ]; then
+          params+=(${{ inputs.additional_ya_make_args }})
+        fi
+
+        # Also build targets which are not in tests' dependencies
+        params+=(--build-all)
+
         echo "::debug::get version"
         ./ya --version
         
         echo "Tests are running..." | .github/scripts/tests/comment-pr.py
 
         ./ya test ${{ inputs.build_target }} "${params[@]}" \
-          --stat --log-file "$LOG_DIR/ya_log.txt" --evlog-file "$LOG_DIR/ya_evlog.jsonl" -DCONSISTENT_DEBUG \
+          --stat --log-file "$YA_TEST_LOG_FILENAME" -DCONSISTENT_DEBUG \
           --no-dir-outputs \
           --junit "$JUNIT_REPORT_XML" --output "$OUT_DIR" || (
             RC=$?
             if [ $RC -ne 0 ]; then
+              echo "10 [Test run log]($YA_TEST_LOG_URL)" >> $SUMMARY_LINKS
+
               echo "ya test returned $RC, check existence $JUNIT_REPORT_XML"
               if [ -s "$JUNIT_REPORT_XML" ]; then
                 echo "$JUNIT_REPORT_XML exists"
@@ -271,6 +290,15 @@ runs:
       run: |
         testmo automation:run:complete --instance "$TESTMO_URL" --run-id ${{ steps.th.outputs.runid }}
 
+    - name: sync logs results to s3
+      if: always()
+      shell: bash
+      run: |
+        set -x
+        echo "::group::s3-sync"
+        s3cmd sync --follow-symlinks --acl-public --no-progress --stats --no-check-md5 "$LOG_DIR/" "$S3_LOG_BUCKET_PATH/"
+        echo "::endgroup::"
+    
     - name: write tests summary
       shell: bash
       if: always()
@@ -279,14 +307,26 @@ runs:
         mkdir $ARTIFACTS_DIR/summary/
         
         cat $SUMMARY_LINKS | python3 -c 'import sys; print(" | ".join([v for _, v in sorted([l.strip().split(" ", 1) for l in sys.stdin], key=lambda a: (int(a[0]), a))]))' >> $GITHUB_STEP_SUMMARY
-        
+
         .github/scripts/tests/generate-summary.py \
           --summary-out-path $ARTIFACTS_DIR/summary/ \
           --summary-url-prefix $S3_URL_PREFIX/summary/ \
           --test-history-url $TEST_HISTORY_URL \
+          --test-log-url="$YA_TEST_LOG_URL" \
           --build-preset "$BUILD_PRESET" \
+          --status-report-file statusrep.txt \
           "Tests" ya-test.html "$JUNIT_REPORT_XML"
 
+        teststatus=$(cat statusrep.txt)
+        if [[ $teststatus == "success" ]];then
+          testmessage="The check has been completed successfully"
+        else
+          testmessage="The check has been failed"
+        fi
+        curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \
+            https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \
+            -d '{"state":"'$teststatus'","description":"'"$testmessage"'","context":"test_${{inputs.build_preset}}"}'
+
     - name: sync test results to s3
       if: always()
       shell: bash
@@ -297,15 +337,6 @@ runs:
         s3cmd sync -r --follow-symlinks --acl-public --no-progress --stats --no-check-md5 "$TEST_ARTIFACTS_DIR/" "$S3_TEST_ARTIFACTS_BUCKET_PATH/"
         echo "::endgroup::"
     
-    - name: sync logs results to s3
-      if: always()
-      shell: bash
-      run: |
-        set -x
-        echo "::group::s3-sync"
-        s3cmd sync --follow-symlinks --acl-private --no-progress --stats --no-check-md5 "$LOG_DIR/" "$S3_BUCKET_PATH/test_logs/"
-        echo "::endgroup::"
-        
     - name: check test results
       shell: bash
       run: |

.github/config/muted_ya.txt

@@ -4,6 +4,7 @@ ydb/core/client/ut TClientTest.PromoteFollower
 ydb/core/client/ut TClientTest.ReadFromFollower
 ydb/core/client/ut TFlatTest.AutoSplitMergeQueue
 ydb/core/cms/ut_sentinel TSentinelTests.BSControllerCantChangeStatus
+ydb/core/external_sources *
 ydb/core/quoter/ut QuoterWithKesusTest.PrefetchCoefficient
 ydb/core/kqp/provider/ut KikimrIcGateway.TestLoadBasicSecretValueFromExternalDataSourceMetadata
 ydb/core/kqp/ut/olap KqpOlapIndexes.IndexesActualization
@@ -21,14 +22,17 @@ ydb/core/kqp/ut/scheme KqpOlap.OlapRead_GenericQuerys
 ydb/core/kqp/ut/scheme KqpOlap.OlapRead_StreamGenericQuery
 ydb/core/kqp/ut/scheme KqpOlap.OlapRead_UsesGenericQueryOnJoinWithDataShardTable
 ydb/core/kqp/ut/scheme KqpQueryService.TableSink_ReplaceFromSelectLargeOlap
+ydb/core/kqp/ut/scheme KqpScheme.AlterAsyncReplication
 ydb/core/kqp/ut/scheme KqpScheme.QueryWithAlter
+ydb/core/kqp/ut/scheme [15/50]*
 ydb/core/kqp/ut/scheme [44/50]*
 ydb/core/kqp/ut/service KqpQueryService.ExecuteQueryPgTableSelect
 ydb/core/kqp/ut/service KqpQueryService.QueryOnClosedSession
 ydb/core/kqp/ut/service KqpService.CloseSessionsWithLoad
 ydb/core/kqp/ut/service [38/50]*
 ydb/core/persqueue/ut TPQTest.*DirectRead*
 ydb/core/persqueue/ut TopicAutoscaling.PartitionSplit_ManySession_NewSDK
+ydb/core/persqueue/ut TopicAutoscaling.PartitionSplit_AutosplitByLoad
 ydb/core/tx/coordinator/ut Coordinator.RestoreTenantConfiguration
 ydb/core/tx/datashard/ut_change_exchange Cdc.InitialScanDebezium
 ydb/core/tx/schemeshard/ut_restore TImportTests.ShouldSucceedOnManyTables
@@ -42,21 +46,18 @@ ydb/library/actors/http/ut HttpProxy.TooLongHeader
 ydb/library/actors/http/ut sole*
 ydb/library/yql/providers/generic/connector/tests/datasource/ydb* *
 ydb/public/lib/ydb_cli/topic/ut TTopicReaderTests.TestRun_ReadOneMessage
-ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut RetryPolicy.RetryWithBatching
 ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut RetryPolicy.TWriteSession_TestBrokenPolicy
 ydb/public/sdk/cpp/client/ydb_topic/ut BasicUsage.ConflictingWrites
 ydb/public/sdk/cpp/client/ydb_topic/ut BasicUsage.WriteRead
 ydb/public/sdk/cpp/client/ydb_topic/ut TSettingsValidation.TestDifferentDedupParams
 ydb/public/sdk/cpp/client/ydb_topic/ut [0/10]*
 ydb/public/sdk/cpp/client/ydb_topic/ut [6/10]*
-ydb/public/sdk/cpp/client/ydb_topic/ut TxUsage::WriteToTopic_Demo_*
 ydb/services/datastreams/ut DataStreams.TestGetRecordsStreamWithSingleShard
 ydb/services/datastreams/ut DataStreams.TestPutRecordsWithRead
 ydb/services/datastreams/ut DataStreams.TestReservedConsumersMetering
 ydb/services/datastreams/ut DataStreams.TestReservedStorageMetering
 ydb/services/fq/ut_integration Yq_1.*
-ydb/services/keyvalue/ut KeyValueGRPCService.SimpleWriteRead
-ydb/services/keyvalue/ut KeyValueGRPCService.SimpleCopyUnexistedKey
+ydb/services/keyvalue/ut KeyValueGRPCService.*
 ydb/services/keyvalue/ut sole*
 ydb/services/persqueue_v1/ut TPersQueueTest.CheckACLForGrpcWrite
 ydb/services/persqueue_v1/ut TPersQueueTest.SetupLockSession