Skip to content

Commit

Permalink
[yql] gateway extension contains only additional settings (ydb-platfo…
Browse files Browse the repository at this point in the history 
…rm#700)
  • Loading branch information
@rvu1024 @adameat
rvu1024 authored and adameat committed Dec 29, 2023
1 parent 30560ea commit 9109d3f
Show file tree
Hide file tree
Showing 4 changed files with 19 additions and 239 deletions.
Empty file added ydb/library/yql/cfg/tests/gateways-experimental.conf
View file
Empty file.
229 changes: 0 additions & 229 deletions ydb/library/yql/cfg/tests/gateways-forceblocks.conf
View file
Original file line number Diff line number Diff line change
@@ -1,234 +1,5 @@
Yt {
GatewayThreads: 0
YtLogLevel: YL_INFO

DefaultSettings {
Name: "InferSchemaMode"
Value: "rpc"
}

DefaultSettings {
Name: "DQRPCReaderInflight"
Value: "10"
}

DefaultSettings {
Name: "UseRPCReaderInDq"
Value: "true"
}

DefaultSettings {
Name: "DefaultMaxJobFails"
Value: "1"
}

DefaultSettings {
Name: "ReleaseTempData"
Value: "immediate"
}

DefaultSettings {
Name: "NativeYtTypeCompatibility"
Value: "all"
}

DefaultSettings {
Name: "UseYqlRowSpecCompactForm"
Value: "true"
}

DefaultSettings {
Name: "UseNewPredicateExtraction"
Value: "true"
}

DefaultSettings {
Name: "PruneKeyFilterLambda"
Value: "true"
}

DefaultSettings {
Name: "_UseKeyBoundApi"
Value: "true"
}

DefaultSettings {
Name: "JoinCommonUseMapMultiOut"
Value: "true"
}

DefaultSettings {
Name: "UseAggPhases"
Value: "true"
}

DefaultSettings {
Name: "_EnableWriteReorder"
Value: "true"
}

DefaultSettings {
Name: "_EnableYtPartitioning"
Value: "true"
}

DefaultSettings {
Name: "TableContentLocalExecution"
Value: "true"
}

RemoteFilePatterns {
Pattern: "yt://([a-zA-Z0-9\\-_]+)/([^&@?]+)$"
Cluster: "$1"
Path: "$2"
}

RemoteFilePatterns {
Pattern: "https?://yt\\.yandex(\\.net|-team\\.ru)/([a-zA-Z0-9\\-_]+)/#page=navigation&path=//([^&]+)$"
Cluster: "$2"
Path: "$3"
}

RemoteFilePatterns {
Pattern: "https?://yt\\.yandex(\\.net|-team\\.ru)/([a-zA-Z0-9\\-_]+)/navigation\\?path=//([^&]+)$"
Cluster: "$2"
Path: "$3"
}

RemoteFilePatterns {
Pattern: "https?://([a-zA-Z0-9\\-_]+)\\.yt\\.yandex(\\.net|-team\\.ru)/api/v(2/download|3/read_file)\\?(|.+&)path=//([^&]+)($|&.*)"
Cluster: "$1"
Path: "$5"
}
}

SqlCore {
TranslationFlags: ["FlexibleTypes", "DisableAnsiOptionalAs", "EmitAggApply"]
}

YqlCore {
Flags {
Name: "DisableYsonCastToString"
}
Flags {
Name: "_EnableMatchRecognize"
}
Flags {
Name: "UseBlocks"
}
}

Dq {
DefaultSettings {
Name: "EnableLLVM"
Value: "true"
}

DefaultSettings {
Name: "EnableInsert"
Value: "true"
}

DefaultSettings {
Name: "EnableFullResultWrite"
Value: "true"
}

DefaultSettings {
Name: "_EnablePrecompute"
Value: "true"
}

DefaultSettings {
Name: "_LiteralTimeout"
Value: "3600000"
}

DefaultSettings {
Name: "_TableTimeout"
Value: "3600000"
}
DefaultSettings {
Name: "EnableComputeActor"
Value: "1"
}
DefaultSettings {
Name: "ComputeActorType"
Value: "async"
}

DefaultSettings {
Name: "UseAggPhases"
Value: "true"
}

DefaultSettings {
Name: "HashJoinMode"
Value: "grace"
}

DefaultSettings {
Name: "UseFastPickleTransport"
Value: "true"
}

DefaultSettings {
Name: "UseOOBTransport"
Value: "true"
}

DefaultSettings {
Name: "UseFinalizeByKey"
Value: "true"
}

DefaultSettings {
Name: "EnableDqReplicate"
Value: "true"
}

DefaultSettings {
Name: "UseWideChannels"
Value: "true"
}
}

Fs {
CustomSchemes {
Pattern: "sbr:(?://)?(\\d+)"
TargetUrl: "https://proxy.sandbox.yandex-team.ru/$1"
}
CustomSchemes {
# backward compatibility rule
Pattern: "yt://([a-zA-Z0-9\\-_]+)/(.+)@t=([^&=]+)"
TargetUrl: "yt://$1/$2?transaction_id=$3"
}
CustomSchemes {
Pattern: "https?://yt\\.yandex(?:\\.net|-team\\.ru)/([a-zA-Z0-9\\-_]+)/#page=navigation&(.+)"
TargetUrl: "yt://$1/?$2"
}
CustomSchemes {
Pattern: "https?://yt\\.yandex(?:\\.net|-team\\.ru)/([a-zA-Z0-9\\-_]+)/navigation\\?(.+)"
TargetUrl: "yt://$1/?$2"
}
CustomSchemes {
Pattern: "https?://([a-zA-Z0-9\\-_]+)\\.yt\\.yandex(?:\\.net|-team\\.ru)/api/v3/read_file\\?(.+)"
TargetUrl: "yt://$1/?$2"
}
CustomSchemes {
Pattern: "https?://a\\.yandex-team\\.ru/(?:arc|svn)/(.+)/arcadia/(.+)\\?rev=r?(\\d+)"
TargetUrl: "arc:/$2?rev=$3&branch=$1"
}
CustomSchemes {
Pattern: "https?://a(?:rcanum)?\\.yandex-team\\.ru/arcadia/(.+)\\?rev=r(\\d+)"
TargetUrl: "arc:/$1?rev=$2"
}
CustomSchemes {
Pattern: "https?://a(?:rcanum)?\\.yandex-team\\.ru/arcadia/(.+)\\?rev=([^r].*)"
TargetUrl: "arc:/$1?hash=$2"
}
CustomSchemes {
Pattern: "arc://(.*)"
TargetUrl: "arc:/$1"
}
}
22 changes: 17 additions & 5 deletions ydb/library/yql/tests/common/test_framework/yql_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import tempfile
import shutil

from google.protobuf import text_format
from collections import namedtuple, defaultdict, OrderedDict
from functools import partial
import codecs
Expand Down Expand Up @@ -37,11 +38,6 @@ def get_param(name, default=None):
return yatest.common.get_param(name, os.environ.get(name) or default)


def get_gateway_cfg_suffix():
default_suffix = None
return get_param('gateway_config_suffix', default_suffix) or ''


def do_custom_query_check(res, sql_query):
custom_check = re.search(r"/\* custom check:(.*)\*/", sql_query)
if not custom_check:
Expand All @@ -54,6 +50,11 @@ def do_custom_query_check(res, sql_query):
return True


def get_gateway_cfg_suffix():
default_suffix = None
return get_param('gateway_config_suffix', default_suffix) or ''


def get_gateway_cfg_filename():
suffix = get_gateway_cfg_suffix()
if suffix == '':
Expand All @@ -62,6 +63,17 @@ def get_gateway_cfg_filename():
return 'gateways-' + suffix + '.conf'


def merge_default_gateway_cfg(cfg_dir, gateway_config):

with open(yql_source_path(os.path.join(cfg_dir, 'gateways.conf'))) as f:
text_format.Merge(f.read(), gateway_config)

suffix = get_gateway_cfg_suffix()
if suffix:
with open(yql_source_path(os.path.join(cfg_dir, 'gateways-' + suffix + '.conf'))) as f:
text_format.Merge(f.read(), gateway_config)


def find_file(path):
arcadia_root = '.'
while '.arcadia.root' not in os.listdir(arcadia_root):
Expand Down
7 changes: 2 additions & 5 deletions ydb/library/yql/tests/common/test_framework/yqlrun.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,14 +61,11 @@ def __init__(self, udfs_dir=None, prov='yt', use_sql2yql=False, keep_temp=True,
if gateway_config is not None:
text_format.Merge(gateway_config, self.gateway_config)

if cfg_dir is None:
cfg_dir = 'ydb/library/yql/cfg/tests'
with open(yql_utils.yql_source_path(cfg_dir + '/' + yql_utils.get_gateway_cfg_filename())) as f:
text_format.Merge(f.read(), self.gateway_config)
yql_utils.merge_default_gateway_cfg(cfg_dir or 'ydb/library/yql/cfg/tests', self.gateway_config)

self.fs_config = file_storage_pb2.TFileStorageConfig()

with open(yql_utils.yql_source_path(cfg_dir + '/fs.conf')) as f:
with open(yql_utils.yql_source_path(os.path.join(cfg_dir or 'ydb/library/yql/cfg/tests', 'fs.conf'))) as f:
text_format.Merge(f.read(), self.fs_config)

if fs_config is not None:
Expand Down

0 comments on commit 9109d3f

Please sign in to comment.