From 335747a032300d0ff873468b050ab5d5f205e885 Mon Sep 17 00:00:00 2001 From: daidai <2017501503@qq.com> Date: Sat, 17 Aug 2024 23:56:05 +0800 Subject: [PATCH] fix comment --- .../doris/datasource/ExternalCatalog.java | 10 +++++----- .../doris/datasource/ExternalDatabase.java | 6 ++++-- .../datasource/hive/HMSExternalCatalog.java | 17 ++++++++++++----- .../datasource/hive/event/IgnoredEvent.java | 2 +- .../datasource/hive/event/MetastoreEvent.java | 12 ++++++++++++ .../hive/event/MetastoreEventsProcessor.java | 6 ++---- .../hive/test_hms_event_notification.groovy | 7 ++++--- ..._hms_event_notification_multi_catalog.groovy | 7 +++++-- 8 files changed, 45 insertions(+), 22 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index d99ac76c7b9f138..a8170e837770e73 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -291,11 +291,11 @@ public void checkProperties() throws DdlException { } } - if (properties.getOrDefault(ExternalCatalog.USE_META_CACHE, "true").equals("false")) { - LOG.warn("force to set use_meta_cache to true for catalog: {} when creating", name); - getCatalogProperty().addProperty(ExternalCatalog.USE_META_CACHE, "true"); - useMetaCache = Optional.of(true); - } + // if (properties.getOrDefault(ExternalCatalog.USE_META_CACHE, "true").equals("false")) { + // LOG.warn("force to set use_meta_cache to true for catalog: {} when creating", name); + // getCatalogProperty().addProperty(ExternalCatalog.USE_META_CACHE, "true"); + // useMetaCache = Optional.of(true); + // } } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index ab5921aac0759d3..d653a5a178e484a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -492,8 +492,10 @@ public boolean registerTable(TableIf tableIf) { metaCache.updateCache(tableName, (T) tableIf, Util.genIdByName(getQualifiedName(tableName))); } } else { - tableNameToId.put(tableName, tableId); - idToTbl.put(tableId, buildTableForInit(tableName, tableId, extCatalog)); + if (!tableNameToId.containsKey(tableName)) { + tableNameToId.put(tableName, tableId); + idToTbl.put(tableId, buildTableForInit(tableName, tableId, extCatalog)); + } } setLastUpdateTime(System.currentTimeMillis()); return true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index e30ab1443a916cd..5faf1f2bb6e7238 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -105,12 +105,19 @@ public void checkProperties() throws DdlException { throw new DdlException( "The parameter " + FILE_META_CACHE_TTL_SECOND + " is wrong, value is " + fileMetaCacheTtlSecond); } + Map properties = catalogProperty.getProperties(); + if (properties.containsKey(HMSProperties.ENABLE_HMS_EVENTS_INCREMENTAL_SYNC)) { + enableHmsEventsIncrementalSync = + properties.get(HMSProperties.ENABLE_HMS_EVENTS_INCREMENTAL_SYNC).equals("true"); + } else { + enableHmsEventsIncrementalSync = Config.enable_hms_events_incremental_sync; + } - enableHmsEventsIncrementalSync = - catalogProperty.getOrDefault(HMSProperties.ENABLE_HMS_EVENTS_INCREMENTAL_SYNC, "false").equals("true"); - - hmsEventsBatchSizePerRpc = - Integer.valueOf(catalogProperty.getOrDefault(HMSProperties.HMS_EVENTIS_BATCH_SIZE_PER_RPC, "-1")); + if (properties.containsKey(HMSProperties.HMS_EVENTIS_BATCH_SIZE_PER_RPC)) { + hmsEventsBatchSizePerRpc = Integer.valueOf(properties.get(HMSProperties.HMS_EVENTIS_BATCH_SIZE_PER_RPC)); + } else { + hmsEventsBatchSizePerRpc = Config.hms_events_batch_size_per_rpc; + } // check the dfs.ha properties // 'dfs.nameservices'='your-nameservice', diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/IgnoredEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/IgnoredEvent.java index d504c2917f9d020..e7e6643e647b987 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/IgnoredEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/IgnoredEvent.java @@ -28,7 +28,7 @@ */ public class IgnoredEvent extends MetastoreEvent { private IgnoredEvent(NotificationEvent event, String catalogName) { - super(event, catalogName); + super(event); } protected static List getEvents(NotificationEvent event, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEvent.java index 04b0ccab799b813..695dd57b2150722 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEvent.java @@ -78,6 +78,18 @@ protected MetastoreEvent(long eventId, String catalogName, String dbName, this.event = null; } + // for IgnoredEvent + protected MetastoreEvent(NotificationEvent event) { + this.event = event; + this.metastoreNotificationEvent = event; + this.eventId = -1; + this.eventTime = -1L; + this.catalogName = null; + this.dbName = null; + this.tblName = null; + this.eventType = null; + } + protected MetastoreEvent(NotificationEvent event, String catalogName) { this.event = event; // Some events that we don't care about, dbName may be empty diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java index cffb3982fb14e52..cbd0bfb5fa6fb58 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEventsProcessor.java @@ -128,7 +128,7 @@ private void realRun() { } catch (MetastoreNotificationFetchException e) { LOG.warn("Failed to fetch hms events on {}. msg: ", hmsExternalCatalog.getName(), e); } catch (Exception ex) { - hmsExternalCatalog.onRefresh(true); + hmsExternalCatalog.onRefreshCache(true); updateLastSyncedEventId(hmsExternalCatalog, -1); LOG.warn("Failed to process hive metastore [{}] events .", hmsExternalCatalog.getName(), ex); @@ -212,9 +212,7 @@ private NotificationEventResponse getNextEventResponseForMaster(HMSExternalCatal return null; } - int batchSize = hmsExternalCatalog.getHmsEventsBatchSizePerRpc() == -1 - ? Config.hms_events_batch_size_per_rpc - : hmsExternalCatalog.getHmsEventsBatchSizePerRpc(); + int batchSize = hmsExternalCatalog.getHmsEventsBatchSizePerRpc(); try { NotificationEventResponse notificationEventResponse = hmsExternalCatalog.getClient().getNextNotification(lastSyncedEventId, batchSize, null); diff --git a/regression-test/suites/external_table_p0/hive/test_hms_event_notification.groovy b/regression-test/suites/external_table_p0/hive/test_hms_event_notification.groovy index fed2205db89036c..c3019455bab951c 100644 --- a/regression-test/suites/external_table_p0/hive/test_hms_event_notification.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hms_event_notification.groovy @@ -21,7 +21,7 @@ suite("test_hms_event_notification", "p0,external,hive,external_docker,external_ logger.info("diable Hive test.") return; } - + for (String useMetaCache : ["true","false"] ) { for (String hivePrefix : [ "hive2","hive3"]) { try { setHivePrefix(hivePrefix) @@ -38,7 +38,7 @@ suite("test_hms_event_notification", "p0,external,hive,external_docker,external_ sql """create catalog if not exists ${catalog_name} properties ( "type"="hms", 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}', - "use_meta_cache" = "false", + "use_meta_cache" = "${useMetaCache}", "enable_hms_events_incremental_sync" ="true", "hms_events_batch_size_per_rpc" = "1000" );""" @@ -46,7 +46,7 @@ suite("test_hms_event_notification", "p0,external,hive,external_docker,external_ sql """create catalog if not exists ${catalog_name_2} properties ( "type"="hms", 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}', - "use_meta_cache" = "false", + "use_meta_cache" = "${useMetaCache}", "enable_hms_events_incremental_sync" ="true" );""" @@ -383,6 +383,7 @@ suite("test_hms_event_notification", "p0,external,hive,external_docker,external_ sql """drop catalog if exists ${catalog_name}""" } finally { } + } } } diff --git a/regression-test/suites/external_table_p0/hive/test_hms_event_notification_multi_catalog.groovy b/regression-test/suites/external_table_p0/hive/test_hms_event_notification_multi_catalog.groovy index c9ef63be5502059..fa83ab24b2ab82b 100644 --- a/regression-test/suites/external_table_p0/hive/test_hms_event_notification_multi_catalog.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hms_event_notification_multi_catalog.groovy @@ -22,6 +22,8 @@ suite("test_hms_event_notification_multi_catalog", "p0,external,hive,external_do return; } + for (String useMetaCache : ["true","false"] ) { + for (String hivePrefix : [ "hive2","hive3"]) { try { setHivePrefix(hivePrefix) @@ -38,7 +40,7 @@ suite("test_hms_event_notification_multi_catalog", "p0,external,hive,external_do sql """create catalog if not exists ${catalog_name} properties ( "type"="hms", 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}', - "use_meta_cache" = "false", + "use_meta_cache" = "${useMetaCache}", "enable_hms_events_incremental_sync" ="true", "hms_events_batch_size_per_rpc" = "10000" );""" @@ -47,7 +49,7 @@ suite("test_hms_event_notification_multi_catalog", "p0,external,hive,external_do sql """create catalog if not exists ${catalog_name_2} properties ( "type"="hms", 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}', - "use_meta_cache" = "false", + "use_meta_cache" = "${useMetaCache}", "enable_hms_events_incremental_sync" ="true", "hms_events_batch_size_per_rpc" = "100000" );""" @@ -666,6 +668,7 @@ suite("test_hms_event_notification_multi_catalog", "p0,external,hive,external_do } finally { } } + } }