Skip to content

Commit

Permalink
Merge branch 'main' into node-name
Browse files Browse the repository at this point in the history
  • Loading branch information
stanislav-shchetinin authored Oct 17, 2024
2 parents 18dbe1c + e8cfb02 commit 28a4dce
Show file tree
Hide file tree
Showing 18,686 changed files with 4,890,639 additions and 14,446 deletions.
The diff you're trying to view is too large. We only load the first 3000 changed files.
7 changes: 7 additions & 0 deletions .github/actions/s3cmd/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,13 @@ runs:
host_base = storage.yandexcloud.net
host_bucket = %(bucket)s.storage.yandexcloud.net
EOF
# specify mime type for correct content-type in s3cmd sync
# we want to .out/.err files treat as text
MIME_TYPES_LINE="text/plain out err log"
MIME_TYPES_FILE='/etc/mime.types'
# add if not exists
sudo bash -c "grep -qF -- '$MIME_TYPES_LINE' '$MIME_TYPES_FILE' || echo '$MIME_TYPES_LINE' >> '$MIME_TYPES_FILE'"
env:
s3_key_id: ${{ inputs.s3_key_id }}
s3_secret_access_key: ${{ inputs.s3_key_secret }}
Expand Down
8 changes: 7 additions & 1 deletion .github/actions/test_ya/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,13 @@ runs:
echo "ya make returned $RC, build failed"
echo "status=failed" >> $GITHUB_OUTPUT
BUILD_FAILED=1
echo "Build failed, see the [logs]($YA_MAKE_OUTPUT_URL)." | GITHUB_TOKEN="${{ github.token }}" .github/scripts/tests/comment-pr.py --color red
# sed is to remove richness (tags like '[[rst]]')
(( \
cat $CURRENT_REPORT \
| jq -r '.results[] | select((.status == "FAILED") and (.error_type == "REGULAR") and (.type = "build")) | "path: " + .path + "\n\n" + ."rich-snippet" + "\n\n\n"' \
| sed 's/\[\[[^]]*]\]//g' \
) || true) > $CURRENT_PUBLIC_DIR/fail_summary.txt
echo "Build failed, see the [logs]($YA_MAKE_OUTPUT_URL). Also see [fail summary]($CURRENT_PUBLIC_DIR_URL/fail_summary.txt)" | GITHUB_TOKEN="${{ github.token }}" .github/scripts/tests/comment-pr.py --color red
break
fi

Expand Down
4 changes: 1 addition & 3 deletions .github/config/muted_ya.txt
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ ydb/core/kqp/ut/service [*/*]*
ydb/core/kqp/ut/service KqpQueryService.ExecuteQueryPgTableSelect
ydb/core/kqp/ut/service KqpQueryService.QueryOnClosedSession
ydb/core/kqp/ut/service KqpService.CloseSessionsWithLoad
ydb/core/mind/hive/ut THiveTest.DrainWithHiveRestart
ydb/core/persqueue/ut [*/*]*
ydb/core/persqueue/ut TPQTest.*DirectRead*
ydb/core/quoter/ut QuoterWithKesusTest.PrefetchCoefficient
Expand Down Expand Up @@ -64,9 +65,7 @@ ydb/services/fq/ut_integration Yq_1.*
ydb/services/keyvalue/ut KeyValueGRPCService.*
ydb/services/keyvalue/ut sole*
ydb/services/persqueue_v1/ut [*/*]*
ydb/services/persqueue_v1/ut TPersQueueTest.CheckACLForGrpcWrite
ydb/services/persqueue_v1/ut TPersQueueTest.DirectRead*Cache*
ydb/services/persqueue_v1/ut TPQCompatTest.BadTopics
ydb/services/ydb/sdk_sessions_pool_ut YdbSdkSessionsPool.StressTestSync*
ydb/services/ydb/sdk_sessions_ut [*/*]*
ydb/services/ydb/sdk_sessions_ut YdbSdkSessions.TestActiveSessionCountAfterBadSession
Expand Down Expand Up @@ -182,7 +181,6 @@ ydb/library/actors/interconnect/ut_huge_cluster HugeCluster.AllToAll
ydb/tests/functional/hive test_drain.py.TestHive.test_drain_on_stop
ydb/library/yql/tests/sql/hybrid_file/part1 test.py.test[in-in_noansi_join--Debug]
ydb/core/kqp/ut/query KqpLimits.QueryExecTimeoutCancel
ydb/library/yql/providers/generic/connector/tests/datasource/* *
ydb/library/yql/providers/generic/connector/tests/join test.py.test_join[join_ch_ch-dqrun]
ydb/tests/fq/generic/streaming test_join.py.TestJoinStreaming.test_streamlookup[v1-1-True-3-fq_client0-mvp_external_ydb_endpoint0]
ydb/tests/fq/generic/streaming test_join.py.TestJoinStreaming.test_streamlookup[v1-2-True-3-fq_client0-mvp_external_ydb_endpoint0]
Expand Down
25 changes: 25 additions & 0 deletions .github/scripts/find_unreachable_targets_from_root.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#!/bin/bash

# Used to find targets without RECURSE/RECURCE_FOR_TESTS/.. paths from ydb/ya.make
# TODO: run this check in CI

set -e

# find all modules+tests from ydb/ya.make
./ya make -Gj0 -ttt ydb --build release -k --cache-tests --build-all > output_dump
cat output_dump | jq '.graph[]' | jq 'select( ."node-type"=="test")' | jq -r ".kv.path" | sort | uniq | sed -E 's/\/[^/]*$//g;/^null$/d' | sort | uniq | { grep "^ydb" || true; } > tests.txt
cat output_dump | jq '.graph[]' | jq -r 'select( ."target_properties"."module_type" != null) | select( ( ."target_properties"."module_tag" // "-" | strings | contains("proto") ) | not ) | .target_properties.module_dir' | sort | uniq | { grep "^ydb" || true; } > modules.txt
cat modules.txt tests.txt | (echo 'RECURSE(';cat;echo ')') > ya.make


# find all modules+tests from generated ya.make, which contains all paths as RECURSE from previous step
./ya make -Gj0 -ttt . --build release -k --cache-tests --build-all > output_dump2
cat output_dump2 | jq '.graph[]' | jq 'select( ."node-type"=="test")' | jq -r ".kv.path" | sort | uniq | sed -E 's/\/[^/]*$//g;/^null$/d' | sort | uniq | { grep "^ydb" || true; } > tests2.txt
cat output_dump2 | jq '.graph[]' | jq -r 'select( ."target_properties"."module_type" != null) | select( ( ."target_properties"."module_tag" // "-" | strings | contains("proto") ) | not ) | .target_properties.module_dir' | sort | uniq | { grep "^ydb" || true; } > modules2.txt

# put all targets together
cat modules.txt tests.txt | sort | uniq > targets.txt
cat modules2.txt tests2.txt | sort | uniq > targets2.txt

# print targets which need to be fixes
comm -13 targets.txt targets2.txt
4 changes: 2 additions & 2 deletions .github/scripts/tests/templates/summary.html
Original file line number Diff line number Diff line change
Expand Up @@ -357,11 +357,11 @@
let [path, testName] = [pieces[1], pieces[2]];

if (success_count + fail_count != 0){
let url = "https://github.com/ydb-platform/ydb/issues/new?title=Mute "+ encodeURIComponent(path)+"/"+ encodeURIComponent(testName) + "&body=" + encodeURIComponent(path)+"/"+ encodeURIComponent(testName) +"%0A%0A**Add%20line%20to%20[muted_ya.txt](https://github.com/ydb-platform/ydb/blob/main/.github/config/muted_ya.txt):**%0A%60" + encodeURIComponent(path)+"/"+ encodeURIComponent(testName)+"%60%0A%0A%20Owner:%20[TEAM:@ydb-platform/"+owner+"](https://github.com/orgs/ydb-platform/teams/"+owner+")%0A%0A**Read%20more%20in%20[mute_rules.md](https://github.com/ydb-platform/ydb/blob/main/.github/config/mute_rules.md)**%20%20%0A%0A**Summary%20history:**%0A%20Success%20rate%20**"+(success_count/(success_count+fail_count)*100)+"%25**%0APass:"+success_count+"%20Fail:"+fail_count+"%20%0A%0A**Test%20run%20history:**%20[link](https://datalens.yandex/34xnbsom67hcq?full_name=" +path+ "/"+ testName+ ")%0A%0AMore%20info%20in%20[dashboard](https://datalens.yandex/4un3zdm0zcnyr)&labels=mute"
let url = "https://github.com/ydb-platform/ydb/issues/new?title=Mute "+ encodeURIComponent(path)+"/"+ encodeURIComponent(testName) + "&body=" + encodeURIComponent(path)+"/"+ encodeURIComponent(testName) +"%0A%0A**Add%20line%20to%20[muted_ya.txt](https://github.com/ydb-platform/ydb/blob/main/.github/config/muted_ya.txt):**%0A%60" + encodeURIComponent(path)+" "+ encodeURIComponent(testName)+"%60%0A%0A%20Owner:%20[TEAM:@ydb-platform/"+owner+"](https://github.com/orgs/ydb-platform/teams/"+owner+")%0A%0A**Read%20more%20in%20[mute_rules.md](https://github.com/ydb-platform/ydb/blob/main/.github/config/mute_rules.md)**%20%20%0A%0A**Summary%20history:**%0A%20Success%20rate%20**"+(success_count/(success_count+fail_count)*100)+"%25**%0APass:"+success_count+"%20Fail:"+fail_count+"%20%0A%0A**Test%20run%20history:**%20[link](https://datalens.yandex/34xnbsom67hcq?full_name=" +path+ "/"+ testName+ ")%0A%0AMore%20info%20in%20[dashboard](https://datalens.yandex/4un3zdm0zcnyr)&labels=mute"
window.open(url, '_blank');
}
else {
let url = "https://github.com/ydb-platform/ydb/issues/new?title=Mute "+ encodeURIComponent(path)+"/"+ encodeURIComponent(testName) + "&body=" + encodeURIComponent(path)+"/"+ encodeURIComponent(testName) +"%0A%0A**Add%20line%20to%20[muted_ya.txt](https://github.com/ydb-platform/ydb/blob/main/.github/config/muted_ya.txt):**%0A%60" + encodeURIComponent(path)+"/"+ encodeURIComponent(testName)+"%60%0A%0A%20Owner:%20[TEAM:@ydb-platform/"+owner+"](https://github.com/orgs/ydb-platform/teams/"+owner+")%0A%0A**Read%20more%20in%20[mute_rules.md](https://github.com/ydb-platform/ydb/blob/main/.github/config/mute_rules.md)**%20%20%0A%0A**Summary%20history:**%0APass:"+success_count+"%20Fail:"+fail_count+"%20%0A%0A**Test%20run%20history:**%20[link](https://datalens.yandex/34xnbsom67hcq?full_name=" +path+ "/"+ testName+ ")%0A%0AMore%20info%20in%20[dashboard](https://datalens.yandex/4un3zdm0zcnyr)&labels=mute"
let url = "https://github.com/ydb-platform/ydb/issues/new?title=Mute "+ encodeURIComponent(path)+"/"+ encodeURIComponent(testName) + "&body=" + encodeURIComponent(path)+"/"+ encodeURIComponent(testName) +"%0A%0A**Add%20line%20to%20[muted_ya.txt](https://github.com/ydb-platform/ydb/blob/main/.github/config/muted_ya.txt):**%0A%60" + encodeURIComponent(path)+" "+ encodeURIComponent(testName)+"%60%0A%0A%20Owner:%20[TEAM:@ydb-platform/"+owner+"](https://github.com/orgs/ydb-platform/teams/"+owner+")%0A%0A**Read%20more%20in%20[mute_rules.md](https://github.com/ydb-platform/ydb/blob/main/.github/config/mute_rules.md)**%20%20%0A%0A**Summary%20history:**%0APass:"+success_count+"%20Fail:"+fail_count+"%20%0A%0A**Test%20run%20history:**%20[link](https://datalens.yandex/34xnbsom67hcq?full_name=" +path+ "/"+ testName+ ")%0A%0AMore%20info%20in%20[dashboard](https://datalens.yandex/4un3zdm0zcnyr)&labels=mute"
window.open(url, '_blank');
}

Expand Down
22 changes: 22 additions & 0 deletions .github/workflows/label_external_issues.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
name: Label issue from external users
on:
issues:
types: [opened]

jobs:
label-external-issues:
name: Label issue from external user
runs-on: ubuntu-latest
# https://docs.github.com/en/graphql/reference/enums#commentauthorassociation
if: ${{ !contains(fromJson('["MEMBER", "OWNER", "COLLABORATOR"]'), github.event.issue.author_association) }}
steps:
- name: add external label
uses: actions/github-script@v7
with:
script: |
github.rest.issues.addLabels({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
labels: ['external']
})
2 changes: 1 addition & 1 deletion .github/workflows/postcommit_cmakebuild.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,6 @@ jobs:
name: Build and test cmake
uses: ./.github/workflows/build_and_test_provisioned.yml
with:
runner_label: auto-provisioned, postcommit
runner_label: auto-provisioned
secrets: inherit

44 changes: 44 additions & 0 deletions .github/workflows/update_dstool_package.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
name: Update dstool package

on:
workflow_dispatch:
inputs:
commit_sha:
type: string
default: ""

jobs:
update-dstool:
name: Update dstool package
runs-on: ubuntu-latest
steps:
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.7.13'

- name: Install dependencies
run: |
set -ex
python -m pip install --upgrade pip
pip install build twine grpcio-tools setuptools
- name: Checkout
uses: actions/checkout@v4
with:
ref: ${{ inputs.commit_sha }}

- name: Compile proto files and build package
run: |
set -ex
${{ github.workspace }}/ydb/apps/dstool/compile_protos.py --ydb-root ${{ github.workspace }}
mv ${{ github.workspace }}/ydb/apps/dstool/setup.py ${{ github.workspace }}
python -m build
- name: Upload package to PyPI
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
run: |
set -ex
python -m twine upload dist/*
3 changes: 0 additions & 3 deletions build/conf/linkers/ld.conf
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
LDFLAGS+=$USER_LDFLAGS $_LD_FLAGS
LDFLAGS_GLOBAL+=

_START_GROUP=
_END_GROUP=
Expand Down Expand Up @@ -115,8 +114,6 @@ when ($OS_EMSCRIPTEN == "yes") {
}
C_SYSTEM_LIBRARIES=$_C_SYSTEM_LIBRARIES

OBJADDE=

LD_EXPORT_ALL_DYNAMIC_SYMBOLS_FLAG=
when ($OS_LINUX == "yes" || $OS_ANDROID == "yes") {
LD_EXPORT_ALL_DYNAMIC_SYMBOLS_FLAG=-rdynamic
Expand Down
6 changes: 0 additions & 6 deletions build/conf/linkers/msvc_linker.conf
Original file line number Diff line number Diff line change
Expand Up @@ -99,12 +99,6 @@ LINK_STDLIBS=\
winspool.lib \
ws2_32.lib

LDFLAGS_GLOBAL=
LDFLAGS=
OBJADDE=
OBJADDE_LIB=
OBJADDE_LIB_GLOBAL=

LINK_EXE_FLAGS=$LINK_EXE_FLAGS_PER_TYPE
LINK_EXE_VFS_FLAGS=
when ($_UNDER_WINE_LINK != "yes" && $USE_MSVC_LINKER != "yes") {
Expand Down
3 changes: 3 additions & 0 deletions build/ymake.core.conf
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ SFLAGS=
SRCFLAGS=
FORCE_COVERAGE_ENABLED=
FORCE_COVERAGE_DISABLED=
LDFLAGS=
LDFLAGS_GLOBAL=
OBJADDE=
OBJADDE_LIB=
OBJADDE_LIB_GLOBAL=
TRUE=yes
Expand Down
27 changes: 27 additions & 0 deletions conanfile.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
from conan import ConanFile


class App(ConanFile):

settings = "os", "compiler", "build_type", "arch"

default_options = {"libiconv:shared": "True"}

def requirements(self):
self.requires("libiconv/1.15")

def build_requirements(self):
self.tool_requires("bison/3.8.2")
self.tool_requires("m4/1.4.19")
self.tool_requires("ragel/6.10")
self.tool_requires("yasm/1.3.0")

generators = "cmake_find_package", "cmake_paths"

def imports(self):
self.copy(pattern="*yasm*", src="bin", dst="./bin")
self.copy(pattern="bison*", src="bin", dst="./bin/bison/bin")
self.copy(pattern="m4*", src="bin", dst="./bin/m4/bin")
self.copy(pattern="ragel*", src="bin", dst="./bin")
self.copy(pattern="ytasm*", src="bin", dst="./bin")
self.copy(pattern="*", src="res", dst="./bin/bison/res")
23 changes: 0 additions & 23 deletions conanfile.txt

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
exclude:
- backports/__init__.py # Для избежания коллизий с другими backports пакетами
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
exclude:
- backports/__init__.py # Для избежания коллизий с другими backports пакетами
- shutil_backports/* # В Аркадии нет потребителей для этого пакета
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
--- contrib/deprecated/python/backports.shutil-get-terminal-size/backports/shutil_get_terminal_size/get_terminal_size.py (index)
+++ contrib/deprecated/python/backports.shutil-get-terminal-size/backports/shutil_get_terminal_size/get_terminal_size.py (working tree)
@@ -41,7 +41,7 @@ try:

return terminal_size(columns, lines)

-except ImportError:
+except (ImportError, OSError):
import fcntl
import termios

Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
exclude:
- backports/__init__.py # Для избежания коллизий с другими backports пакетами
3 changes: 3 additions & 0 deletions contrib/deprecated/python/enum34/.yandex_meta/yamaker.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
exclude_from_macros:
- enum/LICENSE
- enum/README
82 changes: 82 additions & 0 deletions contrib/deprecated/python/enum34/patches/01-unknown.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
--- contrib/deprecated/python/enum34/enum/__init__.py (index)
+++ contrib/deprecated/python/enum34/enum/__init__.py (working tree)
@@ -8,6 +8,8 @@ version = 1, 1, 10

pyver = float('%s.%s' % _sys.version_info[:2])

+ALLOW_SYNONYMS = '__allow_synonyms__'
+
try:
any
except NameError:
@@ -161,6 +163,7 @@ class EnumMeta(type):
for k, v in original_dict.items():
classdict[k] = v

+ allow_synonyms = classdict.get(ALLOW_SYNONYMS, True)
member_type, first_enum = metacls._get_mixins_(bases)
__new__, save_new, use_args = metacls._find_new_(classdict, member_type,
first_enum)
@@ -215,8 +218,18 @@ class EnumMeta(type):
# auto-numbering ;)
if __new__ is None:
__new__ = enum_class.__new__
+
+ val2name = {}
for member_name in _order_:
value = members[member_name]
+ if not allow_synonyms:
+ if value in val2name:
+ raise ValueError(
+ 'allow_synonyms=False forbids multiple names of the same value; '
+ 'Members {!r} and {!r} break this'.format(val2name[value], member_name)
+ )
+ val2name[value] = member_name
+
if not isinstance(value, tuple):
args = (value, )
else:
@@ -237,7 +250,7 @@ class EnumMeta(type):
enum_member.__init__(*args)
# If another member with the same value was already defined, the
# new member becomes an alias to the existing one.
- for name, canonical_member in enum_class._member_map_.items():
+ for name, canonical_member in (enum_class._member_map_.items() if allow_synonyms else ()):
if canonical_member.value == enum_member._value_:
enum_member = canonical_member
break
@@ -328,7 +341,7 @@ class EnumMeta(type):
"""
return True

- def __call__(cls, value, names=None, module=None, type=None, start=1):
+ def __call__(cls, value, names=None, module=None, type=None, start=1, allow_synonyms=True):
"""Either returns an existing member, or creates a new enum class.

This method is used both when an enum class is given a value to match
@@ -347,7 +360,7 @@ class EnumMeta(type):
if names is None: # simple value lookup
return cls.__new__(cls, value)
# otherwise, functional API: we're creating a new Enum type
- return cls._create_(value, names, module=module, type=type, start=start)
+ return cls._create_(value, names, module=module, type=type, start=start, allow_synonyms=allow_synonyms)

def __contains__(cls, member):
return isinstance(member, cls) and member.name in cls._member_map_
@@ -420,7 +433,7 @@ class EnumMeta(type):
raise AttributeError('Cannot reassign members.')
super(EnumMeta, cls).__setattr__(name, value)

- def _create_(cls, class_name, names=None, module=None, type=None, start=1):
+ def _create_(cls, class_name, names=None, module=None, type=None, start=1, allow_synonyms=True):
"""Convenience method to create a new Enum class.

`names` can be:
@@ -465,6 +478,7 @@ class EnumMeta(type):
# only set _order_ in classdict if name/value was not from a mapping
if not isinstance(item, basestring):
classdict['_order_'] = _order_
+ classdict[ALLOW_SYNONYMS] = getattr(cls, ALLOW_SYNONYMS, allow_synonyms)
enum_class = metacls.__new__(metacls, class_name, bases, classdict)

# TODO: replace the frame hack if a blessed way to know the calling
Loading

0 comments on commit 28a4dce

Please sign in to comment.