[Searchable Snapshots] Add integration test to validate cache files a…

…re closed after use (opensearch-project#6215) * [Searchable Snapshots] Add integration test to validate cache files are closed after use Signed-off-by: Tianli Feng <ftianli@amazon.com> * Add changelog Signed-off-by: Tianli Feng <ftianli@amazon.com> * Change the outdated File API to Path Signed-off-by: Tianli Feng <ftianli@amazon.com> * Reduce a for loop of the sharditerator Signed-off-by: Tianli Feng <ftianli@amazon.com> * Revert changes in changelog file Signed-off-by: Tianli Feng <ftianli@amazon.com> * Get file cache path directly Signed-off-by: Tianli Feng <ftianli@amazon.com> * Correct a log message Signed-off-by: Tianli Feng <ftianli@amazon.com> --------- Signed-off-by: Tianli Feng <ftianli@amazon.com> Signed-off-by: Mingshi Liu <mingshl@amazon.com>
mingshl · Mar 24, 2023 · f1b9b88 · f1b9b88
1 parent 558a516
commit f1b9b88
Showing 1 changed file with 51 additions and 0 deletions.
diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java
@@ -15,6 +15,7 @@
 import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
 import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder;
 import org.opensearch.action.index.IndexRequestBuilder;
+import org.opensearch.action.search.SearchResponse;
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.client.Client;
 import org.opensearch.cluster.ClusterState;
@@ -28,16 +29,22 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.ByteSizeUnit;
 import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.env.NodeEnvironment;
 import org.opensearch.index.Index;
 import org.opensearch.index.IndexNotFoundException;
+import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.index.store.remote.file.CleanerDaemonThreadLeakFilter;
 import org.opensearch.index.store.remote.filecache.FileCacheStats;
 import org.opensearch.monitor.fs.FsInfo;
 import org.opensearch.repositories.fs.FsRepository;
 
+import java.io.IOException;
 import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 import java.util.List;
+import java.util.stream.Stream;
 
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
@@ -46,6 +53,7 @@
 import static org.hamcrest.Matchers.notNullValue;
 import static org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest.Metric.FS;
 import static org.opensearch.common.util.CollectionUtils.iterableAsArrayList;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
 
 @ThreadLeakFilters(filters = CleanerDaemonThreadLeakFilter.class)
 public final class SearchableSnapshotIT extends AbstractSnapshotIntegTestCase {
@@ -489,4 +497,47 @@ public void testPruneFileCacheOnIndexDeletion() throws Exception {
         deleteIndicesAndEnsureGreen(client, restoredIndexName1);
         assertAllNodesFileCacheEmpty();
     }
+
+    public void testCacheFilesAreClosedAfterUse() throws Exception {
+        final int numReplicasIndex = randomIntBetween(1, 4);
+        final String indexName = "test-idx";
+        final String restoredIndexName = indexName + "-copy";
+        final String repoName = "test-repo";
+        final String snapshotName = "test-snap";
+        final String id = randomAlphaOfLength(5);
+        final Client client = client();
+
+        internalCluster().ensureAtLeastNumSearchAndDataNodes(numReplicasIndex + 1);
+        createIndex(indexName);
+        client().prepareIndex(indexName).setId(id).setSource("field", "test").get();
+        ensureGreen();
+        createRepositoryWithSettings(null, repoName);
+        takeSnapshot(client, snapshotName, repoName, indexName);
+        restoreSnapshotAndEnsureGreen(client, snapshotName, repoName);
+
+        // Search document to make the index fetch data from the remote snapshot to local storage
+        SearchResponse searchResponse = client().prepareSearch(restoredIndexName).setQuery(QueryBuilders.termQuery("field", "test")).get();
+        assertHitCount(searchResponse, 1);
+
+        // The local cache files should be closed by deleting the restored index
+        deleteIndicesAndEnsureGreen(client, restoredIndexName);
+
+        logger.info("--> validate all the cache files are closed");
+        // Get path of cache files
+        final NodeEnvironment nodeEnv = internalCluster().getInstance(NodeEnvironment.class);
+        Path fileCachePath = nodeEnv.fileCacheNodePath().fileCachePath;
+        // Find all the files in the path
+        try (Stream<Path> paths = Files.walk(fileCachePath)) {
+            paths.filter(Files::isRegularFile).forEach(path -> {
+                // Testing moving the file to check the file is closed or not.
+                try {
+                    Files.move(path, path, StandardCopyOption.REPLACE_EXISTING);
+                } catch (IOException e) {
+                    fail("No exception is expected. The file can't be moved, so it may not be closed.");
+                }
+            });
+        } catch (NoSuchFileException e) {
+            logger.debug("--> the path for the cache files doesn't exist");
+        }
+    }
 }