Merge pull request #9353 from uzhastik/24_3_merge

24 3 merge v2
ydb-platform · Sep 17, 2024 · c0e2d53 · c0e2d53
2 parents 9883bf0 + 7a2500b
commit c0e2d53
Show file tree

Hide file tree

Showing 174 changed files with 3,349 additions and 988 deletions.
diff --git a/ydb/core/external_sources/external_data_source.cpp b/ydb/core/external_sources/external_data_source.cpp
@@ -36,6 +36,10 @@ struct TExternalDataSource : public IExternalSource {
         ythrow TExternalSourceException() << "Only external table supports parameters";
     }
 
+    bool DataSourceMustHaveDataBaseName(const TProtoStringType& sourceType) const {
+        return IsIn({"Greenplum", "PostgreSQL", "MySQL", "MsSQLServer", "ClickHouse"}, sourceType);
+    }
+
     virtual void ValidateExternalDataSource(const TString& externalDataSourceDescription) const override {
         NKikimrSchemeOp::TExternalDataSourceDescription proto;
         if (!proto.ParseFromString(externalDataSourceDescription)) {
@@ -49,6 +53,10 @@ struct TExternalDataSource : public IExternalSource {
             ythrow TExternalSourceException() << "Unsupported property: " << key;
         }
 
+        if (DataSourceMustHaveDataBaseName(proto.GetSourceType()) && !proto.GetProperties().GetProperties().contains("database_name")) {
+            ythrow TExternalSourceException() << proto.GetSourceType() << " source must provide database_name";
+        }
+
         ValidateHostname(HostnamePatterns, proto.GetLocation());
     }
 

diff --git a/ydb/core/external_sources/external_source_factory.cpp b/ydb/core/external_sources/external_source_factory.cpp
@@ -36,12 +36,13 @@ IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector<TStri
                                                          NActors::TActorSystem* actorSystem,
                                                          size_t pathsLimit,
                                                          std::shared_ptr<NYql::ISecuredServiceAccountCredentialsFactory> credentialsFactory,
-                                                         bool enableInfer) {
+                                                         bool enableInfer,
+                                                         bool allowLocalFiles) {
     std::vector<TRegExMatch> hostnamePatternsRegEx(hostnamePatterns.begin(), hostnamePatterns.end());
     return MakeIntrusive<TExternalSourceFactory>(TMap<TString, IExternalSource::TPtr>{
         {
             ToString(NYql::EDatabaseType::ObjectStorage),
-            CreateObjectStorageExternalSource(hostnamePatternsRegEx, actorSystem, pathsLimit, std::move(credentialsFactory), enableInfer)
+            CreateObjectStorageExternalSource(hostnamePatternsRegEx, actorSystem, pathsLimit, std::move(credentialsFactory), enableInfer, allowLocalFiles)
         },
         {
             ToString(NYql::EDatabaseType::ClickHouse),

diff --git a/ydb/core/external_sources/external_source_factory.h b/ydb/core/external_sources/external_source_factory.h
@@ -15,6 +15,7 @@ IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector<TStri
                                                          NActors::TActorSystem* actorSystem = nullptr,
                                                          size_t pathsLimit = 50000,
                                                          std::shared_ptr<NYql::ISecuredServiceAccountCredentialsFactory> credentialsFactory = nullptr,
-                                                         bool enableInfer = false);
+                                                         bool enableInfer = false,
+                                                         bool allowLocalFiles = false);
 
 }
diff --git a/ydb/core/external_sources/hive_metastore/ut/common.cpp b/ydb/core/external_sources/hive_metastore/ut/common.cpp
@@ -25,7 +25,7 @@ TString Exec(const TString& cmd) {
 
 TString GetExternalPort(const TString& service, const TString& port) {
     auto dockerComposeBin = BinaryPath("library/recipes/docker_compose/bin/docker-compose");
-    auto composeFileYml = ArcadiaSourceRoot() + "/ydb/core/external_sources/hive_metastore/ut/docker-compose.yml";
+    auto composeFileYml = ArcadiaFromCurrentLocation(__SOURCE_FILE__, "docker-compose.yml");
     auto result = StringSplitter(Exec(dockerComposeBin + " -f " + composeFileYml + " port " + service + " " + port)).Split(':').ToList<TString>();
     return result ? Strip(result.back()) : TString{};
 }

diff --git a/ydb/core/external_sources/hive_metastore/ut/ya.make b/ydb/core/external_sources/hive_metastore/ut/ya.make
@@ -29,13 +29,19 @@ IF (AUTOCHECK)
     )
 ENDIF()
 
+ENV(COMPOSE_HTTP_TIMEOUT=1200)  # during parallel tests execution there could be huge disk io, which triggers timeouts in docker-compose 
 INCLUDE(${ARCADIA_ROOT}/library/recipes/docker_compose/recipe.inc)
 
 IF (OPENSOURCE)
-    # Including of docker_compose/recipe.inc automatically converts these tests into LARGE, 
-    # which makes it impossible to run them during precommit checks on Github CI. 
-    # Next several lines forces these tests to be MEDIUM. To see discussion, visit YDBOPS-8928.
-    SIZE(MEDIUM)
+    IF (SANITIZER_TYPE)
+        # Too huge for precommit check with sanitizers
+        SIZE(LARGE)
+    ELSE()
+        # Including of docker_compose/recipe.inc automatically converts these tests into LARGE, 
+        # which makes it impossible to run them during precommit checks on Github CI. 
+        # Next several lines forces these tests to be MEDIUM. To see discussion, visit YDBOPS-8928.
+        SIZE(MEDIUM)
+    ENDIF()
     SET(TEST_TAGS_VALUE)
     SET(TEST_REQUIREMENTS_VALUE)