Skip to content

Commit

Permalink
[iceberg] Disallow in-memory hive metastore caching
Browse files Browse the repository at this point in the history
  • Loading branch information
imjalpreet authored and ZacBlanco committed Jan 9, 2025
1 parent b4fcb0b commit 02a3a20
Show file tree
Hide file tree
Showing 3 changed files with 51 additions and 13 deletions.
14 changes: 1 addition & 13 deletions presto-docs/src/main/sphinx/connector/iceberg.rst
Original file line number Diff line number Diff line change
Expand Up @@ -515,19 +515,7 @@ JMX queries to get the metrics and verify the cache usage::
Metastore Cache
^^^^^^^^^^^^^^^

Metastore Cache only caches the schema, table, and table statistics. The table object cached in the `tableCache`
is only used for reading the table metadata location and table properties and, the rest of the table metadata
is fetched from the filesystem/object storage metadata location.

.. note::

Metastore Cache would be applicable only for Hive Catalog in the Presto Iceberg connector.

.. code-block:: none

hive.metastore-cache-ttl=2d
hive.metastore-refresh-interval=3d
hive.metastore-cache-maximum-size=10000000
Iceberg Connector does not support Metastore Caching.

Extra Hidden Metadata Columns
-----------------------------
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import java.util.Optional;

import static com.facebook.airlift.configuration.ConfigBinder.configBinder;
import static com.google.common.base.Preconditions.checkArgument;
import static org.weakref.jmx.ObjectNames.generatedNameOf;
import static org.weakref.jmx.guice.ExportBinder.newExporter;

Expand All @@ -52,6 +53,10 @@ public void setup(Binder binder)
configBinder(binder).bindConfig(IcebergHiveTableOperationsConfig.class);

configBinder(binder).bindConfig(MetastoreClientConfig.class);

long metastoreCacheTtl = buildConfigObject(MetastoreClientConfig.class).getMetastoreCacheTtl().toMillis();
checkArgument(metastoreCacheTtl == 0, "In-memory hive metastore caching must not be enabled for Iceberg");

binder.bind(PartitionMutator.class).to(HivePartitionMutator.class).in(Scopes.SINGLETON);

binder.bind(MetastoreCacheStats.class).to(HiveMetastoreCacheStats.class).in(Scopes.SINGLETON);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.iceberg;

import com.facebook.presto.spi.connector.ConnectorFactory;
import com.facebook.presto.testing.TestingConnectorContext;
import com.google.common.collect.ImmutableMap;
import org.testng.annotations.Test;

import java.util.Map;

import static org.assertj.core.api.Assertions.assertThatThrownBy;

public class TestIcebergConnectorFactory
{
@Test
public void testCachingHiveMetastore()
{
Map<String, String> config = ImmutableMap.<String, String>builder()
.put("hive.metastore.uri", "thrift://localhost:9083")
.put("hive.metastore-cache-ttl", "10m")
.buildOrThrow();

assertThatThrownBy(() -> createConnector(config))
.hasMessageContaining("In-memory hive metastore caching must not be enabled for Iceberg");
}

private static void createConnector(Map<String, String> config)
{
ConnectorFactory factory = new IcebergConnectorFactory();
factory.create("iceberg-test", config, new TestingConnectorContext())
.shutdown();
}
}

0 comments on commit 02a3a20

Please sign in to comment.