From e9e913c16c2b0f9fa8ad6058fb9f2a47acc8fa13 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Wed, 2 Jan 2019 10:10:32 +0100 Subject: [PATCH] Add hybridfs store type With this commit we introduce a new store type `hybridfs` that is a hybrid between `mmapfs` and `niofs`. This store type chooses different strategies to read Lucene files based on the read access pattern (random or linear) in order to optimize performance. This store type has been available in earlier versions of Elasticsearch as `default_fs`. We have chosen a different name now in order to convey the intent of the store type instead of tying it to the fact whether it is the default choice. Relates #36668 --- docs/reference/index-modules/store.asciidoc | 20 +++++-- .../org/elasticsearch/index/IndexModule.java | 3 +- .../index/store/FsDirectoryService.java | 55 ++++++++++++------- .../index/store/IndexStoreTests.java | 9 +++ 4 files changed, 62 insertions(+), 25 deletions(-) diff --git a/docs/reference/index-modules/store.asciidoc b/docs/reference/index-modules/store.asciidoc index c2b3d700e9b7c..c354f551ad524 100644 --- a/docs/reference/index-modules/store.asciidoc +++ b/docs/reference/index-modules/store.asciidoc @@ -67,12 +67,22 @@ process equal to the size of the file being mapped. Before using this class, be sure you have allowed plenty of <>. +[[hybridfs]]`hybridfs`:: + +The `hybridfs` type is a hybrid of `niofs` and `mmapfs`, which chooses the best +file system type for each type of file based on the read access pattern. +Currently only the Lucene term dictionary, norms and doc values files are +memory mapped. All other files are opened using Lucene `NIOFSDirectory`. +Similarly to `mmapfs` be sure you have allowed plenty of +<>. + [[allow-mmapfs]] -You can restrict the use of the `mmapfs` store type via the setting -`node.store.allow_mmapfs`. This is a boolean setting indicating whether or not -`mmapfs` is allowed. The default is to allow `mmapfs`. This setting is useful, -for example, if you are in an environment where you can not control the ability -to create a lot of memory maps so you need disable the ability to use `mmapfs`. +You can restrict the use of the `mmapfs` and the related `hybridfs` store type +via the setting `node.store.allow_mmapfs`. This is a boolean setting indicating +whether or not memory-mapping is allowed. The default is to allow it. This +setting is useful, for example, if you are in an environment where you can not +control the ability to create a lot of memory maps so you need disable the +ability to use memory-mapping. === Pre-loading data into the file system cache diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java index 7f2eae492fd56..091e78051b08d 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java @@ -302,6 +302,7 @@ public static boolean isBuiltinType(String storeType) { public enum Type { + HYBRIDFS("hybridfs"), NIOFS("niofs"), MMAPFS("mmapfs"), SIMPLEFS("simplefs"), @@ -330,7 +331,7 @@ public String getSettingsKey() { public static Type fromSettingsKey(final String key) { final Type type = TYPES.get(key); if (type == null) { - throw new IllegalArgumentException("no matching type for [" + key + "]"); + throw new IllegalArgumentException("no matching store type for [" + key + "]"); } return type; } diff --git a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java index f95cdb3a9f692..345ef1eef41f6 100644 --- a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java +++ b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.store; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.FileSwitchDirectory; import org.apache.lucene.store.LockFactory; import org.apache.lucene.store.MMapDirectory; @@ -30,6 +31,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardPath; @@ -37,10 +39,16 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Collections; import java.util.HashSet; import java.util.Set; public class FsDirectoryService extends DirectoryService { + /* + * We are mmapping norms, docvalues as well as term dictionaries, all other files are served through NIOFS + * this provides good random access performance and does not lead to page cache thrashing. + */ + private static final Set PRIMARY_EXTENSIONS = Collections.unmodifiableSet(Sets.newHashSet("nvd", "dvd", "tim")); protected final IndexStore indexStore; public static final Setting INDEX_LOCK_FACTOR_SETTING = new Setting<>("index.store.fs.fs_lock", "native", (s) -> { @@ -78,27 +86,36 @@ public Directory newDirectory() throws IOException { protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException { final String storeType = indexSettings.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.FS.getSettingsKey()); + IndexModule.Type type; if (IndexModule.Type.FS.match(storeType)) { - final IndexModule.Type type = - IndexModule.defaultStoreType(IndexModule.NODE_STORE_ALLOW_MMAPFS.get(indexSettings.getNodeSettings())); - switch (type) { - case MMAPFS: - return new MMapDirectory(location, lockFactory); - case SIMPLEFS: - return new SimpleFSDirectory(location, lockFactory); - case NIOFS: - return new NIOFSDirectory(location, lockFactory); - default: - throw new AssertionError("unexpected built-in store type [" + type + "]"); - } - } else if (IndexModule.Type.SIMPLEFS.match(storeType)) { - return new SimpleFSDirectory(location, lockFactory); - } else if (IndexModule.Type.NIOFS.match(storeType)) { - return new NIOFSDirectory(location, lockFactory); - } else if (IndexModule.Type.MMAPFS.match(storeType)) { - return new MMapDirectory(location, lockFactory); + type = IndexModule.defaultStoreType(IndexModule.NODE_STORE_ALLOW_MMAPFS.get(indexSettings.getNodeSettings())); + } else { + type = IndexModule.Type.fromSettingsKey(storeType); + } + switch (type) { + case HYBRIDFS: + // Use Lucene defaults + final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory); + if (primaryDirectory instanceof MMapDirectory) { + return new FileSwitchDirectory(PRIMARY_EXTENSIONS, primaryDirectory, new NIOFSDirectory(location, lockFactory), true) { + @Override + public String[] listAll() throws IOException { + // Avoid doing listAll twice: + return primaryDirectory.listAll(); + } + }; + } else { + return primaryDirectory; + } + case MMAPFS: + return new MMapDirectory(location, lockFactory); + case SIMPLEFS: + return new SimpleFSDirectory(location, lockFactory); + case NIOFS: + return new NIOFSDirectory(location, lockFactory); + default: + throw new AssertionError("unexpected built-in store type [" + type + "]"); } - throw new IllegalArgumentException("No directory found for type [" + storeType + "]"); } private static Directory setPreload(Directory directory, Path location, LockFactory lockFactory, diff --git a/server/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java b/server/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java index 155473c83cf30..3fb4aea9b6f89 100644 --- a/server/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java +++ b/server/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.store; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FileSwitchDirectory; import org.apache.lucene.store.MMapDirectory; import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.store.NoLockFactory; @@ -64,6 +65,9 @@ private void doTestStoreDirectory(Index index, Path tempDir, String typeSettingV new ShardPath(false, tempDir, tempDir, new ShardId(index, 0))); try (Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) { switch (type) { + case HYBRIDFS: + assertHybridDirectory(directory); + break; case NIOFS: assertTrue(type + " " + directory.toString(), directory instanceof NIOFSDirectory); break; @@ -88,4 +92,9 @@ private void doTestStoreDirectory(Index index, Path tempDir, String typeSettingV } } + private void assertHybridDirectory(Directory directory) { + assertTrue(directory.toString(), directory instanceof FileSwitchDirectory); + Directory primaryDirectory = ((FileSwitchDirectory) directory).getPrimaryDir(); + assertTrue("primary directory " + primaryDirectory.toString(), primaryDirectory instanceof MMapDirectory); + } }