Fix DedicatedClusterSnapshotRestoreIT testSnapshotWithStuckNode (#43537…

…) (#44082)

* Fix DedicatedClusterSnapshotRestoreIT testSnapshotWithStuckNode

* See comment in the test: The problem is that when the snapshot delete works out partially on master failover and the retry fails on `SnapshotMissingException` no repository cleanup is run => we still failed even with repo cleanup logic in the delete path now
   * Fixed the test by rerunning a create snapshot and delete loop to clean up the repo before verifying file counts
* Closes #39852

server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java

@@ -420,7 +420,6 @@ public void testSnapshotDuringNodeShutdown() throws Exception {
         logger.info("--> done");
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/39852")
     public void testSnapshotWithStuckNode() throws Exception {
         logger.info("--> start 2 nodes");
         ArrayList<String> nodes = new ArrayList<>();
@@ -484,11 +483,21 @@ public void testSnapshotWithStuckNode() throws Exception {
         }
 
         logger.info("--> making sure that snapshot no longer exists");
-        assertThrows(client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").execute(),
-            SnapshotMissingException.class);
+        expectThrows(SnapshotMissingException.class,
+            () -> client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap")
+                .execute().actionGet());
+
+        // TODO: Replace this by repository cleanup endpoint call once that's available
+        logger.info("--> Go through a loop of creating and deleting a snapshot to trigger repository cleanup");
+        client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-tmp")
+            .setWaitForCompletion(true)
+            .setIndices("test-idx")
+            .get();
+        client().admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap-tmp").get();
+
         // Subtract four files that will remain in the repository:
-        //   (1) index-1
-        //   (2) index-0 (because we keep the previous version) and
+        //   (1) index-(N+1)
+        //   (2) index-N (because we keep the previous version) and
         //   (3) index-latest
         //   (4) incompatible-snapshots
         assertFileCount(repo, 4);