From bba591bea0b1c4b718559f14cba9b5af412662a6 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 9 Jan 2018 14:01:57 +0100 Subject: [PATCH 1/4] Consistent updates of IndexShardSnapshotStatus (#28130) This commit changes IndexShardSnapshotStatus so that the Stage is updated coherently with any required information. It also provides a asCopy() method that returns the status of a IndexShardSnapshotStatus at a given point in time, ensuring that all information are coherent. Closes #26480 --- .../status/SnapshotIndexShardStatus.java | 15 +- .../snapshots/status/SnapshotStats.java | 21 +- .../status/TransportNodesSnapshotsStatus.java | 16 +- .../TransportSnapshotsStatusAction.java | 3 +- .../snapshots/IndexShardSnapshotStatus.java | 323 +++++++++--------- .../repositories/Repository.java | 2 +- .../blobstore/BlobStoreRepository.java | 125 ++++--- .../snapshots/SnapshotShardsService.java | 66 ++-- .../snapshots/SnapshotsService.java | 9 +- .../snapshots/SnapshotShardsServiceIT.java | 2 +- .../index/shard/IndexShardTestCase.java | 10 +- 11 files changed, 288 insertions(+), 304 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java index 324cb3712adf1..1b7ead5b96510 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java @@ -22,7 +22,6 @@ import org.elasticsearch.action.support.broadcast.BroadcastShardResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.shard.ShardId; @@ -49,13 +48,13 @@ private SnapshotIndexShardStatus() { this.stats = new SnapshotStats(); } - SnapshotIndexShardStatus(ShardId shardId, IndexShardSnapshotStatus indexShardStatus) { + SnapshotIndexShardStatus(ShardId shardId, IndexShardSnapshotStatus.Copy indexShardStatus) { this(shardId, indexShardStatus, null); } - SnapshotIndexShardStatus(ShardId shardId, IndexShardSnapshotStatus indexShardStatus, String nodeId) { + SnapshotIndexShardStatus(ShardId shardId, IndexShardSnapshotStatus.Copy indexShardStatus, String nodeId) { super(shardId); - switch (indexShardStatus.stage()) { + switch (indexShardStatus.getStage()) { case INIT: stage = SnapshotIndexShardStage.INIT; break; @@ -72,10 +71,12 @@ private SnapshotIndexShardStatus() { stage = SnapshotIndexShardStage.FAILURE; break; default: - throw new IllegalArgumentException("Unknown stage type " + indexShardStatus.stage()); + throw new IllegalArgumentException("Unknown stage type " + indexShardStatus.getStage()); } - stats = new SnapshotStats(indexShardStatus); - failure = indexShardStatus.failure(); + this.stats = new SnapshotStats(indexShardStatus.getStartTime(), indexShardStatus.getTotalTime(), + indexShardStatus.getNumberOfFiles(), indexShardStatus.getProcessedFiles(), + indexShardStatus.getTotalSize(), indexShardStatus.getProcessedSize()); + this.failure = indexShardStatus.getFailure(); this.nodeId = nodeId; } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java index ba11e51d56f87..5b2bdd7c614c6 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java @@ -25,33 +25,28 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; import java.io.IOException; public class SnapshotStats implements Streamable, ToXContentFragment { - private long startTime; + private long startTime; private long time; - private int numberOfFiles; - private int processedFiles; - private long totalSize; - private long processedSize; SnapshotStats() { } - SnapshotStats(IndexShardSnapshotStatus indexShardStatus) { - startTime = indexShardStatus.startTime(); - time = indexShardStatus.time(); - numberOfFiles = indexShardStatus.numberOfFiles(); - processedFiles = indexShardStatus.processedFiles(); - totalSize = indexShardStatus.totalSize(); - processedSize = indexShardStatus.processedSize(); + SnapshotStats(long startTime, long time, int numberOfFiles, int processedFiles, long totalSize, long processedSize) { + this.startTime = startTime; + this.time = time; + this.numberOfFiles = numberOfFiles; + this.processedFiles = processedFiles; + this.totalSize = totalSize; + this.processedSize = processedSize; } /** diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java index 872793f6ef21a..77578546b9585 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java @@ -96,7 +96,7 @@ protected NodesSnapshotStatus newResponse(Request request, List> snapshotMapBuilder = new HashMap<>(); try { - String nodeId = clusterService.localNode().getId(); + final String nodeId = clusterService.localNode().getId(); for (Snapshot snapshot : request.snapshots) { Map shardsStatus = snapshotShardsService.currentSnapshotShards(snapshot); if (shardsStatus == null) { @@ -104,15 +104,17 @@ protected NodeSnapshotStatus nodeOperation(NodeRequest request) { } Map shardMapBuilder = new HashMap<>(); for (Map.Entry shardEntry : shardsStatus.entrySet()) { - SnapshotIndexShardStatus shardStatus; - IndexShardSnapshotStatus.Stage stage = shardEntry.getValue().stage(); + final ShardId shardId = shardEntry.getKey(); + + final IndexShardSnapshotStatus.Copy lastSnapshotStatus = shardEntry.getValue().asCopy(); + final IndexShardSnapshotStatus.Stage stage = lastSnapshotStatus.getStage(); + + String shardNodeId = null; if (stage != IndexShardSnapshotStatus.Stage.DONE && stage != IndexShardSnapshotStatus.Stage.FAILURE) { // Store node id for the snapshots that are currently running. - shardStatus = new SnapshotIndexShardStatus(shardEntry.getKey(), shardEntry.getValue(), nodeId); - } else { - shardStatus = new SnapshotIndexShardStatus(shardEntry.getKey(), shardEntry.getValue()); + shardNodeId = nodeId; } - shardMapBuilder.put(shardEntry.getKey(), shardStatus); + shardMapBuilder.put(shardEntry.getKey(), new SnapshotIndexShardStatus(shardId, lastSnapshotStatus, shardNodeId)); } snapshotMapBuilder.put(snapshot, unmodifiableMap(shardMapBuilder)); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java index 71bb1995dd57e..dc13c8dab5188 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java @@ -233,7 +233,8 @@ private SnapshotsStatusResponse buildResponse(SnapshotsStatusRequest request, Li Map shardStatues = snapshotsService.snapshotShards(request.repository(), snapshotInfo); for (Map.Entry shardStatus : shardStatues.entrySet()) { - shardStatusBuilder.add(new SnapshotIndexShardStatus(shardStatus.getKey(), shardStatus.getValue())); + IndexShardSnapshotStatus.Copy lastSnapshotStatus = shardStatus.getValue().asCopy(); + shardStatusBuilder.add(new SnapshotIndexShardStatus(shardStatus.getKey(), lastSnapshotStatus)); } final SnapshotsInProgress.State state; switch (snapshotInfo.state()) { diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java b/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java index 644caa7520be5..f1c247a41bb6d 100644 --- a/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java +++ b/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java @@ -19,6 +19,9 @@ package org.elasticsearch.index.snapshots; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; + /** * Represent shard snapshot status */ @@ -47,119 +50,85 @@ public enum Stage { /** * Snapshot failed */ - FAILURE + FAILURE, + /** + * Snapshot aborted + */ + ABORTED } - private Stage stage = Stage.INIT; - + private final AtomicReference stage; private long startTime; - - private long time; - + private long totalTime; private int numberOfFiles; - - private volatile int processedFiles; - + private int processedFiles; private long totalSize; - - private volatile long processedSize; - + private long processedSize; private long indexVersion; - - private volatile boolean aborted; - private String failure; - /** - * Returns current snapshot stage - * - * @return current snapshot stage - */ - public Stage stage() { - return this.stage; - } - - /** - * Sets new snapshot stage - * - * @param stage new snapshot stage - */ - public void updateStage(Stage stage) { - this.stage = stage; - } - - /** - * Returns snapshot start time - * - * @return snapshot start time - */ - public long startTime() { - return this.startTime; - } - - /** - * Sets snapshot start time - * - * @param startTime snapshot start time - */ - public void startTime(long startTime) { + private IndexShardSnapshotStatus(final Stage stage, final long startTime, final long totalTime, + final int numberOfFiles, final int processedFiles, final long totalSize, final long processedSize, + final long indexVersion, final String failure) { + this.stage = new AtomicReference<>(Objects.requireNonNull(stage)); this.startTime = startTime; + this.totalTime = totalTime; + this.numberOfFiles = numberOfFiles; + this.processedFiles = processedFiles; + this.totalSize = totalSize; + this.processedSize = processedSize; + this.indexVersion = indexVersion; + this.failure = failure; } - /** - * Returns snapshot processing time - * - * @return processing time - */ - public long time() { - return this.time; + public synchronized Copy moveToStarted(final long startTime, final int numberOfFiles, final long totalSize) { + if (stage.compareAndSet(Stage.INIT, Stage.STARTED)) { + this.startTime = startTime; + this.numberOfFiles = numberOfFiles; + this.totalSize = totalSize; + } else { + throw new IllegalStateException("Unable to move the shard snapshot status to [STARTED]: " + + "expecting [INIT] but got [" + stage.get() + "]"); + } + return asCopy(); } - /** - * Sets snapshot processing time - * - * @param time snapshot processing time - */ - public void time(long time) { - this.time = time; + public synchronized Copy moveToFinalize(final long indexVersion) { + if (stage.compareAndSet(Stage.STARTED, Stage.FINALIZE)) { + this.indexVersion = indexVersion; + } else { + throw new IllegalStateException("Unable to move the shard snapshot status to [FINALIZE]: " + + "expecting [STARTED] but got [" + stage.get() + "]"); + } + return asCopy(); } - /** - * Returns true if snapshot process was aborted - * - * @return true if snapshot process was aborted - */ - public boolean aborted() { - return this.aborted; + public synchronized Copy moveToDone(final long endTime) { + if (stage.compareAndSet(Stage.FINALIZE, Stage.DONE)) { + this.totalTime = Math.max(0L, endTime - startTime); + } else { + throw new IllegalStateException("Unable to move the shard snapshot status to [DONE]: " + + "expecting [FINALIZE] but got [" + stage.get() + "]"); + } + return asCopy(); } - /** - * Marks snapshot as aborted - */ - public void abort() { - this.aborted = true; + public synchronized Copy abortIfNotCompleted(final String failure) { + if (stage.compareAndSet(Stage.INIT, Stage.ABORTED) || stage.compareAndSet(Stage.STARTED, Stage.ABORTED)) { + this.failure = failure; + } + return asCopy(); } - /** - * Sets files stats - * - * @param numberOfFiles number of files in this snapshot - * @param totalSize total size of files in this snapshot - */ - public void files(int numberOfFiles, long totalSize) { - this.numberOfFiles = numberOfFiles; - this.totalSize = totalSize; + public synchronized void moveToFailed(final long endTime, final String failure) { + if (stage.getAndSet(Stage.FAILURE) != Stage.FAILURE) { + this.totalTime = Math.max(0L, endTime - startTime); + this.failure = failure; + } } - /** - * Sets processed files stats - * - * @param numberOfFiles number of files in this snapshot - * @param totalSize total size of files in this snapshot - */ - public synchronized void processedFiles(int numberOfFiles, long totalSize) { - processedFiles = numberOfFiles; - processedSize = totalSize; + public boolean isAborted() { + return stage.get() == Stage.ABORTED; } /** @@ -171,71 +140,111 @@ public synchronized void addProcessedFile(long size) { } /** - * Number of files - * - * @return number of files - */ - public int numberOfFiles() { - return numberOfFiles; - } - - /** - * Total snapshot size - * - * @return snapshot size - */ - public long totalSize() { - return totalSize; - } - - /** - * Number of processed files - * - * @return number of processed files - */ - public int processedFiles() { - return processedFiles; - } - - /** - * Size of processed files + * Returns a copy of the current {@link IndexShardSnapshotStatus}. This method is + * intended to be used when a coherent state of {@link IndexShardSnapshotStatus} is needed. * - * @return size of processed files - */ - public long processedSize() { - return processedSize; - } - - - /** - * Sets index version - * - * @param indexVersion index version - */ - public void indexVersion(long indexVersion) { - this.indexVersion = indexVersion; - } - - /** - * Returns index version - * - * @return index version - */ - public long indexVersion() { - return indexVersion; - } - - /** - * Sets the reason for the failure if the snapshot is in the {@link IndexShardSnapshotStatus.Stage#FAILURE} state - */ - public void failure(String failure) { - this.failure = failure; - } - - /** - * Returns the reason for the failure if the snapshot is in the {@link IndexShardSnapshotStatus.Stage#FAILURE} state - */ - public String failure() { - return failure; + * @return a {@link IndexShardSnapshotStatus.Copy} + */ + public synchronized IndexShardSnapshotStatus.Copy asCopy() { + return new IndexShardSnapshotStatus.Copy(stage.get(), startTime, totalTime, numberOfFiles, processedFiles, totalSize, processedSize, + indexVersion, failure); + } + + public static IndexShardSnapshotStatus newInitializing() { + return new IndexShardSnapshotStatus(Stage.INIT, 0L, 0L, 0, 0, 0, 0, 0, null); + } + + public static IndexShardSnapshotStatus newFailed(final String failure) { + assert failure != null : "expecting non null failure for a failed IndexShardSnapshotStatus"; + if (failure == null) { + throw new IllegalArgumentException("A failure description is required for a failed IndexShardSnapshotStatus"); + } + return new IndexShardSnapshotStatus(Stage.FAILURE, 0L, 0L, 0, 0, 0, 0, 0, failure); + } + + public static IndexShardSnapshotStatus newDone(final long startTime, final long totalTime, final int files, final long size) { + // The snapshot is done which means the number of processed files is the same as total + return new IndexShardSnapshotStatus(Stage.DONE, startTime, totalTime, files, files, size, size, 0, null); + } + + /** + * Returns an immutable state of {@link IndexShardSnapshotStatus} at a given point in time. + */ + public static class Copy { + + private final Stage stage; + private final long startTime; + private final long totalTime; + private final int numberOfFiles; + private final int processedFiles; + private final long totalSize; + private final long processedSize; + private final long indexVersion; + private final String failure; + + public Copy(final Stage stage, final long startTime, final long totalTime, + final int numberOfFiles, final int processedFiles, final long totalSize, final long processedSize, + final long indexVersion, final String failure) { + this.stage = stage; + this.startTime = startTime; + this.totalTime = totalTime; + this.numberOfFiles = numberOfFiles; + this.processedFiles = processedFiles; + this.totalSize = totalSize; + this.processedSize = processedSize; + this.indexVersion = indexVersion; + this.failure = failure; + } + + public Stage getStage() { + return stage; + } + + public long getStartTime() { + return startTime; + } + + public long getTotalTime() { + return totalTime; + } + + public int getNumberOfFiles() { + return numberOfFiles; + } + + public int getProcessedFiles() { + return processedFiles; + } + + public long getTotalSize() { + return totalSize; + } + + public long getProcessedSize() { + return processedSize; + } + + public long getIndexVersion() { + return indexVersion; + } + + public String getFailure() { + return failure; + } + + @Override + public String toString() { + return "index shard snapshot status (" + + "stage=" + stage + + ", startTime=" + startTime + + ", totalTime=" + totalTime + + ", numberOfFiles=" + numberOfFiles + + ", processedFiles=" + processedFiles + + ", totalSize=" + totalSize + + ", processedSize=" + processedSize + + ", indexVersion=" + indexVersion + + ", failure='" + failure + '\'' + + ')'; + } } } diff --git a/core/src/main/java/org/elasticsearch/repositories/Repository.java b/core/src/main/java/org/elasticsearch/repositories/Repository.java index f711a72b67757..4c3d58e67ff72 100644 --- a/core/src/main/java/org/elasticsearch/repositories/Repository.java +++ b/core/src/main/java/org/elasticsearch/repositories/Repository.java @@ -180,7 +180,7 @@ SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, List indices, long * Repository implementations shouldn't release the snapshot index commit point. It is done by the method caller. *

* As snapshot process progresses, implementation of this method should update {@link IndexShardSnapshotStatus} object and check - * {@link IndexShardSnapshotStatus#aborted()} to see if the snapshot process should be aborted. + * {@link IndexShardSnapshotStatus#isAborted()} to see if the snapshot process should be aborted. * * @param shard shard to be snapshotted * @param snapshotId snapshot id diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 06812be5aab2c..9068c6ff39743 100644 --- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -805,17 +805,11 @@ private void writeAtomic(final String blobName, final BytesReference bytesRef) t @Override public void snapshotShard(IndexShard shard, SnapshotId snapshotId, IndexId indexId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus) { - SnapshotContext snapshotContext = new SnapshotContext(shard, snapshotId, indexId, snapshotStatus); - snapshotStatus.startTime(System.currentTimeMillis()); - + SnapshotContext snapshotContext = new SnapshotContext(shard, snapshotId, indexId, snapshotStatus, System.currentTimeMillis()); try { snapshotContext.snapshot(snapshotIndexCommit); - snapshotStatus.time(System.currentTimeMillis() - snapshotStatus.startTime()); - snapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.DONE); } catch (Exception e) { - snapshotStatus.time(System.currentTimeMillis() - snapshotStatus.startTime()); - snapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.FAILURE); - snapshotStatus.failure(ExceptionsHelper.detailedMessage(e)); + snapshotStatus.moveToFailed(System.currentTimeMillis(), ExceptionsHelper.detailedMessage(e)); if (e instanceof IndexShardSnapshotFailedException) { throw (IndexShardSnapshotFailedException) e; } else { @@ -838,14 +832,7 @@ public void restoreShard(IndexShard shard, SnapshotId snapshotId, Version versio public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, Version version, IndexId indexId, ShardId shardId) { Context context = new Context(snapshotId, version, indexId, shardId); BlobStoreIndexShardSnapshot snapshot = context.loadSnapshot(); - IndexShardSnapshotStatus status = new IndexShardSnapshotStatus(); - status.updateStage(IndexShardSnapshotStatus.Stage.DONE); - status.startTime(snapshot.startTime()); - status.files(snapshot.numberOfFiles(), snapshot.totalSize()); - // The snapshot is done which means the number of processed files is the same as total - status.processedFiles(snapshot.numberOfFiles(), snapshot.totalSize()); - status.time(snapshot.time()); - return status; + return IndexShardSnapshotStatus.newDone(snapshot.startTime(), snapshot.time(), snapshot.numberOfFiles(), snapshot.totalSize()); } @Override @@ -1103,8 +1090,8 @@ protected Tuple buildBlobStoreIndexShardS private class SnapshotContext extends Context { private final Store store; - private final IndexShardSnapshotStatus snapshotStatus; + private final long startTime; /** * Constructs new context @@ -1114,10 +1101,11 @@ private class SnapshotContext extends Context { * @param indexId the id of the index being snapshotted * @param snapshotStatus snapshot status to report progress */ - SnapshotContext(IndexShard shard, SnapshotId snapshotId, IndexId indexId, IndexShardSnapshotStatus snapshotStatus) { + SnapshotContext(IndexShard shard, SnapshotId snapshotId, IndexId indexId, IndexShardSnapshotStatus snapshotStatus, long startTime) { super(snapshotId, Version.CURRENT, indexId, shard.shardId()); this.snapshotStatus = snapshotStatus; this.store = shard.store(); + this.startTime = startTime; } /** @@ -1125,24 +1113,25 @@ private class SnapshotContext extends Context { * * @param snapshotIndexCommit snapshot commit point */ - public void snapshot(IndexCommit snapshotIndexCommit) { + public void snapshot(final IndexCommit snapshotIndexCommit) { logger.debug("[{}] [{}] snapshot to [{}] ...", shardId, snapshotId, metadata.name()); - store.incRef(); + + final Map blobs; try { - final Map blobs; - try { - blobs = blobContainer.listBlobs(); - } catch (IOException e) { - throw new IndexShardSnapshotFailedException(shardId, "failed to list blobs", e); - } + blobs = blobContainer.listBlobs(); + } catch (IOException e) { + throw new IndexShardSnapshotFailedException(shardId, "failed to list blobs", e); + } - long generation = findLatestFileNameGeneration(blobs); - Tuple tuple = buildBlobStoreIndexShardSnapshots(blobs); - BlobStoreIndexShardSnapshots snapshots = tuple.v1(); - int fileListGeneration = tuple.v2(); + long generation = findLatestFileNameGeneration(blobs); + Tuple tuple = buildBlobStoreIndexShardSnapshots(blobs); + BlobStoreIndexShardSnapshots snapshots = tuple.v1(); + int fileListGeneration = tuple.v2(); - final List indexCommitPointFiles = new ArrayList<>(); + final List indexCommitPointFiles = new ArrayList<>(); + store.incRef(); + try { int indexNumberOfFiles = 0; long indexTotalFilesSize = 0; ArrayList filesToSnapshot = new ArrayList<>(); @@ -1156,10 +1145,11 @@ public void snapshot(IndexCommit snapshotIndexCommit) { throw new IndexShardSnapshotFailedException(shardId, "Failed to get store file metadata", e); } for (String fileName : fileNames) { - if (snapshotStatus.aborted()) { + if (snapshotStatus.isAborted()) { logger.debug("[{}] [{}] Aborted on the file [{}], exiting", shardId, snapshotId, fileName); throw new IndexShardSnapshotFailedException(shardId, "Aborted"); } + logger.trace("[{}] [{}] Processing [{}]", shardId, snapshotId, fileName); final StoreFileMetaData md = metadata.get(fileName); BlobStoreIndexShardSnapshot.FileInfo existingFileInfo = null; @@ -1195,14 +1185,7 @@ public void snapshot(IndexCommit snapshotIndexCommit) { } } - snapshotStatus.files(indexNumberOfFiles, indexTotalFilesSize); - - if (snapshotStatus.aborted()) { - logger.debug("[{}] [{}] Aborted during initialization", shardId, snapshotId); - throw new IndexShardSnapshotFailedException(shardId, "Aborted"); - } - - snapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.STARTED); + snapshotStatus.moveToStarted(startTime, indexNumberOfFiles, indexTotalFilesSize); for (BlobStoreIndexShardSnapshot.FileInfo snapshotFileInfo : filesToSnapshot) { try { @@ -1211,36 +1194,42 @@ public void snapshot(IndexCommit snapshotIndexCommit) { throw new IndexShardSnapshotFailedException(shardId, "Failed to perform snapshot (index files)", e); } } - - snapshotStatus.indexVersion(snapshotIndexCommit.getGeneration()); - // now create and write the commit point - snapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.FINALIZE); - - BlobStoreIndexShardSnapshot snapshot = new BlobStoreIndexShardSnapshot(snapshotId.getName(), - snapshotIndexCommit.getGeneration(), indexCommitPointFiles, snapshotStatus.startTime(), - // snapshotStatus.startTime() is assigned on the same machine, so it's safe to use with VLong - System.currentTimeMillis() - snapshotStatus.startTime(), indexNumberOfFiles, indexTotalFilesSize); - //TODO: The time stored in snapshot doesn't include cleanup time. - logger.trace("[{}] [{}] writing shard snapshot file", shardId, snapshotId); - try { - indexShardSnapshotFormat.write(snapshot, blobContainer, snapshotId.getUUID()); - } catch (IOException e) { - throw new IndexShardSnapshotFailedException(shardId, "Failed to write commit point", e); - } - - // delete all files that are not referenced by any commit point - // build a new BlobStoreIndexShardSnapshot, that includes this one and all the saved ones - List newSnapshotsList = new ArrayList<>(); - newSnapshotsList.add(new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles())); - for (SnapshotFiles point : snapshots) { - newSnapshotsList.add(point); - } - // finalize the snapshot and rewrite the snapshot index with the next sequential snapshot index - finalize(newSnapshotsList, fileListGeneration + 1, blobs); - snapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.DONE); } finally { store.decRef(); } + + final IndexShardSnapshotStatus.Copy lastSnapshotStatus = snapshotStatus.moveToFinalize(snapshotIndexCommit.getGeneration()); + + // now create and write the commit point + final BlobStoreIndexShardSnapshot snapshot = new BlobStoreIndexShardSnapshot(snapshotId.getName(), + lastSnapshotStatus.getIndexVersion(), + indexCommitPointFiles, + lastSnapshotStatus.getStartTime(), + // snapshotStatus.startTime() is assigned on the same machine, + // so it's safe to use with VLong + System.currentTimeMillis() - lastSnapshotStatus.getStartTime(), + lastSnapshotStatus.getNumberOfFiles(), + lastSnapshotStatus.getTotalSize()); + + //TODO: The time stored in snapshot doesn't include cleanup time. + logger.trace("[{}] [{}] writing shard snapshot file", shardId, snapshotId); + try { + indexShardSnapshotFormat.write(snapshot, blobContainer, snapshotId.getUUID()); + } catch (IOException e) { + throw new IndexShardSnapshotFailedException(shardId, "Failed to write commit point", e); + } + + // delete all files that are not referenced by any commit point + // build a new BlobStoreIndexShardSnapshot, that includes this one and all the saved ones + List newSnapshotsList = new ArrayList<>(); + newSnapshotsList.add(new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles())); + for (SnapshotFiles point : snapshots) { + newSnapshotsList.add(point); + } + // finalize the snapshot and rewrite the snapshot index with the next sequential snapshot index + finalize(newSnapshotsList, fileListGeneration + 1, blobs); + snapshotStatus.moveToDone(System.currentTimeMillis()); + } /** @@ -1335,7 +1324,7 @@ public int read(byte[] b, int off, int len) throws IOException { } private void checkAborted() { - if (snapshotStatus.aborted()) { + if (snapshotStatus.isAborted()) { logger.debug("[{}] [{}] Aborted on the file [{}], exiting", shardId, snapshotId, fileName); throw new IndexShardSnapshotFailedException(shardId, "Aborted"); } diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 248f9a555a3d6..35e0b10fd8769 100644 --- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -51,7 +51,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.index.engine.Engine; @@ -188,7 +187,7 @@ public void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexSh Map shards = snapshotShards.getValue().shards; if (shards.containsKey(shardId)) { logger.debug("[{}] shard closing, abort snapshotting for snapshot [{}]", shardId, snapshotShards.getKey().getSnapshotId()); - shards.get(shardId).abort(); + shards.get(shardId).abortIfNotCompleted("shard is closing, aborting"); } } } @@ -230,9 +229,7 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) { // running shards is missed, then the snapshot is removed is a subsequent cluster // state update, which is being processed here for (IndexShardSnapshotStatus snapshotStatus : entry.getValue().shards.values()) { - if (snapshotStatus.stage() == Stage.INIT || snapshotStatus.stage() == Stage.STARTED) { - snapshotStatus.abort(); - } + snapshotStatus.abortIfNotCompleted("snapshot has been removed in cluster state, aborting"); } } } @@ -255,7 +252,7 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) { if (localNodeId.equals(shard.value.nodeId())) { if (shard.value.state() == State.INIT && (snapshotShards == null || !snapshotShards.shards.containsKey(shard.key))) { logger.trace("[{}] - Adding shard to the queue", shard.key); - startedShards.put(shard.key, new IndexShardSnapshotStatus()); + startedShards.put(shard.key, IndexShardSnapshotStatus.newInitializing()); } } } @@ -278,30 +275,26 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) { // Abort all running shards for this snapshot SnapshotShards snapshotShards = shardSnapshots.get(entry.snapshot()); if (snapshotShards != null) { + final String failure = "snapshot has been aborted"; for (ObjectObjectCursor shard : entry.shards()) { - IndexShardSnapshotStatus snapshotStatus = snapshotShards.shards.get(shard.key); + + final IndexShardSnapshotStatus snapshotStatus = snapshotShards.shards.get(shard.key); if (snapshotStatus != null) { - switch (snapshotStatus.stage()) { - case INIT: - case STARTED: - snapshotStatus.abort(); - break; - case FINALIZE: - logger.debug("[{}] trying to cancel snapshot on shard [{}] that is finalizing, " + - "letting it finish", entry.snapshot(), shard.key); - break; - case DONE: - logger.debug("[{}] trying to cancel snapshot on the shard [{}] that is already done, " + - "updating status on the master", entry.snapshot(), shard.key); - notifySuccessfulSnapshotShard(entry.snapshot(), shard.key, localNodeId); - break; - case FAILURE: - logger.debug("[{}] trying to cancel snapshot on the shard [{}] that has already failed, " + - "updating status on the master", entry.snapshot(), shard.key); - notifyFailedSnapshotShard(entry.snapshot(), shard.key, localNodeId, snapshotStatus.failure()); - break; - default: - throw new IllegalStateException("Unknown snapshot shard stage " + snapshotStatus.stage()); + final IndexShardSnapshotStatus.Copy lastSnapshotStatus = snapshotStatus.abortIfNotCompleted(failure); + final Stage stage = lastSnapshotStatus.getStage(); + if (stage == Stage.FINALIZE) { + logger.debug("[{}] trying to cancel snapshot on shard [{}] that is finalizing, " + + "letting it finish", entry.snapshot(), shard.key); + + } else if (stage == Stage.DONE) { + logger.debug("[{}] trying to cancel snapshot on the shard [{}] that is already done, " + + "updating status on the master", entry.snapshot(), shard.key); + notifySuccessfulSnapshotShard(entry.snapshot(), shard.key, localNodeId); + + } else if (stage == Stage.FAILURE) { + logger.debug("[{}] trying to cancel snapshot on the shard [{}] that has already failed, " + + "updating status on the master", entry.snapshot(), shard.key); + notifyFailedSnapshotShard(entry.snapshot(), shard.key, localNodeId, lastSnapshotStatus.getFailure()); } } } @@ -400,12 +393,8 @@ private void snapshot(final IndexShard indexShard, final Snapshot snapshot, fina try (Engine.IndexCommitRef snapshotRef = indexShard.acquireIndexCommit(true)) { repository.snapshotShard(indexShard, snapshot.getSnapshotId(), indexId, snapshotRef.getIndexCommit(), snapshotStatus); if (logger.isDebugEnabled()) { - StringBuilder details = new StringBuilder(); - details.append(" index : version [").append(snapshotStatus.indexVersion()); - details.append("], number_of_files [").append(snapshotStatus.numberOfFiles()); - details.append("] with total_size [").append(new ByteSizeValue(snapshotStatus.totalSize())).append("]\n"); - logger.debug("snapshot ({}) completed to {}, took [{}]\n{}", snapshot, repository, - TimeValue.timeValueMillis(snapshotStatus.time()), details); + final IndexShardSnapshotStatus.Copy lastSnapshotStatus = snapshotStatus.asCopy(); + logger.debug("snapshot ({}) completed to {} with {}", snapshot, repository, lastSnapshotStatus); } } } catch (SnapshotFailedEngineException | IndexShardSnapshotFailedException e) { @@ -432,21 +421,22 @@ private void syncShardStatsOnNewMaster(ClusterChangedEvent event) { ImmutableOpenMap masterShards = snapshot.shards(); for(Map.Entry localShard : localShards.entrySet()) { ShardId shardId = localShard.getKey(); - IndexShardSnapshotStatus localShardStatus = localShard.getValue(); ShardSnapshotStatus masterShard = masterShards.get(shardId); if (masterShard != null && masterShard.state().completed() == false) { + final IndexShardSnapshotStatus.Copy indexShardSnapshotStatus = localShard.getValue().asCopy(); + final Stage stage = indexShardSnapshotStatus.getStage(); // Master knows about the shard and thinks it has not completed - if (localShardStatus.stage() == Stage.DONE) { + if (stage == Stage.DONE) { // but we think the shard is done - we need to make new master know that the shard is done logger.debug("[{}] new master thinks the shard [{}] is not completed but the shard is done locally, " + "updating status on the master", snapshot.snapshot(), shardId); notifySuccessfulSnapshotShard(snapshot.snapshot(), shardId, localNodeId); - } else if (localShard.getValue().stage() == Stage.FAILURE) { + } else if (stage == Stage.FAILURE) { // but we think the shard failed - we need to make new master know that the shard failed logger.debug("[{}] new master thinks the shard [{}] is not completed but the shard failed locally, " + "updating status on master", snapshot.snapshot(), shardId); - notifyFailedSnapshotShard(snapshot.snapshot(), shardId, localNodeId, localShardStatus.failure()); + notifyFailedSnapshotShard(snapshot.snapshot(), shardId, localNodeId, indexShardSnapshotStatus.getFailure()); } } } diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index ec63b7d228ec5..bf8edcf576704 100644 --- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -607,10 +607,7 @@ public Map snapshotShards(final String reposi ShardId shardId = new ShardId(indexMetaData.getIndex(), i); SnapshotShardFailure shardFailure = findShardFailure(snapshotInfo.shardFailures(), shardId); if (shardFailure != null) { - IndexShardSnapshotStatus shardSnapshotStatus = new IndexShardSnapshotStatus(); - shardSnapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.FAILURE); - shardSnapshotStatus.failure(shardFailure.reason()); - shardStatus.put(shardId, shardSnapshotStatus); + shardStatus.put(shardId, IndexShardSnapshotStatus.newFailed(shardFailure.reason())); } else { final IndexShardSnapshotStatus shardSnapshotStatus; if (snapshotInfo.state() == SnapshotState.FAILED) { @@ -621,9 +618,7 @@ public Map snapshotShards(final String reposi // snapshot status will throw an exception. Instead, we create // a status for the shard to indicate that the shard snapshot // could not be taken due to partial being set to false. - shardSnapshotStatus = new IndexShardSnapshotStatus(); - shardSnapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.FAILURE); - shardSnapshotStatus.failure("skipped"); + shardSnapshotStatus = IndexShardSnapshotStatus.newFailed("skipped"); } else { shardSnapshotStatus = repository.getShardSnapshotStatus( snapshotInfo.snapshotId(), diff --git a/core/src/test/java/org/elasticsearch/snapshots/SnapshotShardsServiceIT.java b/core/src/test/java/org/elasticsearch/snapshots/SnapshotShardsServiceIT.java index 651cd96776e75..8431c8fa69f54 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/SnapshotShardsServiceIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/SnapshotShardsServiceIT.java @@ -93,7 +93,7 @@ public void testRetryPostingSnapshotStatusMessages() throws Exception { assertBusy(() -> { final Snapshot snapshot = new Snapshot("test-repo", snapshotId); List stages = snapshotShardsService.currentSnapshotShards(snapshot) - .values().stream().map(IndexShardSnapshotStatus::stage).collect(Collectors.toList()); + .values().stream().map(status -> status.asCopy().getStage()).collect(Collectors.toList()); assertThat(stages, hasSize(shards)); assertThat(stages, everyItem(equalTo(IndexShardSnapshotStatus.Stage.DONE))); }); diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index c06b4a433cb0a..4737befa30e45 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -619,16 +619,18 @@ protected void recoverShardFromSnapshot(final IndexShard shard, protected void snapshotShard(final IndexShard shard, final Snapshot snapshot, final Repository repository) throws IOException { - final IndexShardSnapshotStatus snapshotStatus = new IndexShardSnapshotStatus(); + final IndexShardSnapshotStatus snapshotStatus = IndexShardSnapshotStatus.newInitializing(); try (Engine.IndexCommitRef indexCommitRef = shard.acquireIndexCommit(true)) { Index index = shard.shardId().getIndex(); IndexId indexId = new IndexId(index.getName(), index.getUUID()); repository.snapshotShard(shard, snapshot.getSnapshotId(), indexId, indexCommitRef.getIndexCommit(), snapshotStatus); } - assertEquals(IndexShardSnapshotStatus.Stage.DONE, snapshotStatus.stage()); - assertEquals(shard.snapshotStoreMetadata().size(), snapshotStatus.numberOfFiles()); - assertNull(snapshotStatus.failure()); + + final IndexShardSnapshotStatus.Copy lastSnapshotStatus = snapshotStatus.asCopy(); + assertEquals(IndexShardSnapshotStatus.Stage.DONE, lastSnapshotStatus.getStage()); + assertEquals(shard.snapshotStoreMetadata().size(), lastSnapshotStatus.getNumberOfFiles()); + assertNull(lastSnapshotStatus.getFailure()); } /** From 79e8ef0305893da8a8961a1bec9d53e41a7c19ce Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 9 Jan 2018 00:40:59 -0500 Subject: [PATCH 2/4] Declare empty package dirs as output dirs Otherwise newer versions of Gradle will see the outputs as stale and remove the directory between having created the directory and copying files into the directory (leading to the directory being created again, this time missing some sub-directories). --- distribution/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/distribution/build.gradle b/distribution/build.gradle index 01ed3102dcf6b..542a34bb4f52f 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -335,6 +335,7 @@ configure(distributions.findAll { ['deb', 'rpm'].contains(it.name) }) { task createEtc(type: EmptyDirTask) { dir "${packagingFiles}/etc/elasticsearch" dirMode 0750 + outputs.dir dir } task fillEtc(type: Copy) { From 36729d1c46bbf1b56a7dacc4b079d708830ebbba Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Tue, 9 Jan 2018 18:28:43 +0100 Subject: [PATCH 3/4] Add the ability to bundle multiple plugins into a meta plugin (#28022) This commit adds the ability to package multiple plugins in a single zip. The zip file for a meta plugin must contains the following structure: |____elasticsearch/ | |____ <-- The plugin files for plugin1 (the content of the elastisearch directory) | |____ <-- The plugin files for plugin2 | |____ meta-plugin-descriptor.properties <-- example contents below The meta plugin properties descriptor is mandatory and must contain the following properties: description: simple summary of the meta plugin. name: the meta plugin name The installation process installs each plugin in a sub-folder inside the meta plugin directory. The example above would create the following structure in the plugins directory: |_____ plugins | |____ | | |____ meta-plugin-descriptor.properties | | |____ | | |____ If the sub plugins contain a config or a bin directory, they are copied in a sub folder inside the meta plugin config/bin directory. |_____ config | |____ | | |____ | | |____ |_____ bin | |____ | | |____ | | |____ The sub-plugins are loaded at startup like normal plugins with the same restrictions; they have a separate class loader and a sub-plugin cannot have the same name than another plugin (or a sub-plugin inside another meta plugin). It is also not possible to remove a sub-plugin inside a meta plugin, only full removal of the meta plugin is allowed. Closes #27316 --- .../meta-plugin-descriptor.properties | 21 ++ .../cluster/node/info/PluginsAndModules.java | 9 +- .../org/elasticsearch/bootstrap/Security.java | 12 +- .../org/elasticsearch/bootstrap/Spawner.java | 38 ++- .../elasticsearch/plugins/MetaPluginInfo.java | 149 ++++++++++ .../org/elasticsearch/plugins/PluginInfo.java | 69 ++++- .../elasticsearch/plugins/PluginsService.java | 34 +-- .../plugins/MetaPluginInfoTests.java | 120 ++++++++ .../plugins/PluginInfoTests.java | 34 +-- .../plugins/PluginsServiceTests.java | 6 +- .../plugins/InstallPluginCommand.java | 210 ++++++++++---- .../plugins/ListPluginsCommand.java | 44 ++- .../plugins/InstallPluginCommandTests.java | 260 ++++++++++++++++-- .../plugins/ListPluginsCommandTests.java | 122 +++++++- .../plugins/RemovePluginCommandTests.java | 31 ++- docs/plugins/authors.asciidoc | 33 ++- plugins/examples/meta-plugin/build.gradle | 56 ++++ .../meta-plugin/dummy-plugin1/build.gradle | 29 ++ .../elasticsearch/example/DummyPlugin1.java | 29 ++ .../meta-plugin/dummy-plugin2/build.gradle | 29 ++ .../elasticsearch/example/DummyPlugin2.java | 29 ++ .../meta-plugin-descriptor.properties | 4 + ...SmokeTestPluginsClientYamlTestSuiteIT.java | 39 +++ .../test/smoke_test_plugins/10_basic.yml | 14 + .../bootstrap/SpawnerNoBootstrapTests.java | 86 +++++- settings.gradle | 25 +- .../elasticsearch/plugins/PluginTestUtil.java | 14 +- 27 files changed, 1331 insertions(+), 215 deletions(-) create mode 100644 buildSrc/src/main/resources/meta-plugin-descriptor.properties create mode 100644 core/src/main/java/org/elasticsearch/plugins/MetaPluginInfo.java create mode 100644 core/src/test/java/org/elasticsearch/plugins/MetaPluginInfoTests.java create mode 100644 plugins/examples/meta-plugin/build.gradle create mode 100644 plugins/examples/meta-plugin/dummy-plugin1/build.gradle create mode 100644 plugins/examples/meta-plugin/dummy-plugin1/src/main/java/org/elasticsearch/example/DummyPlugin1.java create mode 100644 plugins/examples/meta-plugin/dummy-plugin2/build.gradle create mode 100644 plugins/examples/meta-plugin/dummy-plugin2/src/main/java/org/elasticsearch/example/DummyPlugin2.java create mode 100644 plugins/examples/meta-plugin/src/main/resources/meta-plugin-descriptor.properties create mode 100644 plugins/examples/meta-plugin/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java create mode 100644 plugins/examples/meta-plugin/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yml diff --git a/buildSrc/src/main/resources/meta-plugin-descriptor.properties b/buildSrc/src/main/resources/meta-plugin-descriptor.properties new file mode 100644 index 0000000000000..16dbe4c38a55b --- /dev/null +++ b/buildSrc/src/main/resources/meta-plugin-descriptor.properties @@ -0,0 +1,21 @@ +# Elasticsearch meta plugin descriptor file +# This file must exist as 'meta-plugin-descriptor.properties' in a folder named `elasticsearch`. +# +### example meta plugin for "meta-foo" +# +# meta-foo.zip <-- zip file for the meta plugin, with this structure: +#|____elasticsearch/ +#| |____ <-- The plugin files for bundled_plugin_1 (the content of the elastisearch directory) +#| |____ <-- The plugin files for bundled_plugin_2 +#| |____ meta-plugin-descriptor.properties <-- example contents below: +# +# description=My meta plugin +# name=meta-foo +# +### mandatory elements for all meta plugins: +# +# 'description': simple summary of the meta plugin +description=${description} +# +# 'name': the meta plugin name +name=${name} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsAndModules.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsAndModules.java index e562adf2602c1..67ee575678988 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsAndModules.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsAndModules.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.List; /** @@ -60,23 +61,23 @@ public void writeTo(StreamOutput out) throws IOException { */ public List getPluginInfos() { List plugins = new ArrayList<>(this.plugins); - Collections.sort(plugins, (p1, p2) -> p1.getName().compareTo(p2.getName())); + Collections.sort(plugins, Comparator.comparing(PluginInfo::getName)); return plugins; } - + /** * Returns an ordered list based on modules name */ public List getModuleInfos() { List modules = new ArrayList<>(this.modules); - Collections.sort(modules, (p1, p2) -> p1.getName().compareTo(p2.getName())); + Collections.sort(modules, Comparator.comparing(PluginInfo::getName)); return modules; } public void addPlugin(PluginInfo info) { plugins.add(info); } - + public void addModule(PluginInfo info) { modules.add(info); } diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Security.java b/core/src/main/java/org/elasticsearch/bootstrap/Security.java index 89a1f794e89f8..e3de41c09c1c2 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/Security.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/Security.java @@ -163,16 +163,8 @@ static Map getCodebaseJarMap(Set urls) { static Map getPluginPermissions(Environment environment) throws IOException, NoSuchAlgorithmException { Map map = new HashMap<>(); // collect up set of plugins and modules by listing directories. - Set pluginsAndModules = new LinkedHashSet<>(); // order is already lost, but some filesystems have it - if (Files.exists(environment.pluginsFile())) { - try (DirectoryStream stream = Files.newDirectoryStream(environment.pluginsFile())) { - for (Path plugin : stream) { - if (pluginsAndModules.add(plugin) == false) { - throw new IllegalStateException("duplicate plugin: " + plugin); - } - } - } - } + Set pluginsAndModules = new LinkedHashSet<>(PluginInfo.extractAllPlugins(environment.pluginsFile())); + if (Files.exists(environment.modulesFile())) { try (DirectoryStream stream = Files.newDirectoryStream(environment.modulesFile())) { for (Path module : stream) { diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Spawner.java b/core/src/main/java/org/elasticsearch/bootstrap/Spawner.java index 0b9913f7f06a4..d6d66e1828361 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/Spawner.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/Spawner.java @@ -21,14 +21,12 @@ import org.apache.lucene.util.Constants; import org.apache.lucene.util.IOUtils; -import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.Platforms; import org.elasticsearch.plugins.PluginInfo; import java.io.Closeable; import java.io.IOException; -import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -72,27 +70,23 @@ void spawnNativePluginControllers(final Environment environment) throws IOExcept * For each plugin, attempt to spawn the controller daemon. Silently ignore any plugin that * don't include a controller for the correct platform. */ - try (DirectoryStream stream = Files.newDirectoryStream(pluginsFile)) { - for (final Path plugin : stream) { - if (FileSystemUtils.isDesktopServicesStore(plugin)) { - continue; - } - final PluginInfo info = PluginInfo.readFromProperties(plugin); - final Path spawnPath = Platforms.nativeControllerPath(plugin); - if (!Files.isRegularFile(spawnPath)) { - continue; - } - if (!info.hasNativeController()) { - final String message = String.format( - Locale.ROOT, - "plugin [%s] does not have permission to fork native controller", - plugin.getFileName()); - throw new IllegalArgumentException(message); - } - final Process process = - spawnNativePluginController(spawnPath, environment.tmpFile()); - processes.add(process); + List paths = PluginInfo.extractAllPlugins(pluginsFile); + for (Path plugin : paths) { + final PluginInfo info = PluginInfo.readFromProperties(plugin); + final Path spawnPath = Platforms.nativeControllerPath(plugin); + if (!Files.isRegularFile(spawnPath)) { + continue; } + if (!info.hasNativeController()) { + final String message = String.format( + Locale.ROOT, + "plugin [%s] does not have permission to fork native controller", + plugin.getFileName()); + throw new IllegalArgumentException(message); + } + final Process process = + spawnNativePluginController(spawnPath, environment.tmpFile()); + processes.add(process); } } diff --git a/core/src/main/java/org/elasticsearch/plugins/MetaPluginInfo.java b/core/src/main/java/org/elasticsearch/plugins/MetaPluginInfo.java new file mode 100644 index 0000000000000..d8bb176273ce2 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/plugins/MetaPluginInfo.java @@ -0,0 +1,149 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plugins; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Map; +import java.util.Properties; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * An in-memory representation of the meta plugin descriptor. + */ +public class MetaPluginInfo { + static final String ES_META_PLUGIN_PROPERTIES = "meta-plugin-descriptor.properties"; + + private final String name; + private final String description; + + /** + * Construct plugin info. + * + * @param name the name of the plugin + * @param description a description of the plugin + */ + private MetaPluginInfo(String name, String description) { + this.name = name; + this.description = description; + } + + /** + * @return Whether the provided {@code path} is a meta plugin. + */ + public static boolean isMetaPlugin(final Path path) { + return Files.exists(path.resolve(ES_META_PLUGIN_PROPERTIES)); + } + + /** + * @return Whether the provided {@code path} is a meta properties file. + */ + public static boolean isPropertiesFile(final Path path) { + return ES_META_PLUGIN_PROPERTIES.equals(path.getFileName().toString()); + } + + /** reads (and validates) meta plugin metadata descriptor file */ + + /** + * Reads and validates the meta plugin descriptor file. + * + * @param path the path to the root directory for the meta plugin + * @return the meta plugin info + * @throws IOException if an I/O exception occurred reading the meta plugin descriptor + */ + public static MetaPluginInfo readFromProperties(final Path path) throws IOException { + final Path descriptor = path.resolve(ES_META_PLUGIN_PROPERTIES); + + final Map propsMap; + { + final Properties props = new Properties(); + try (InputStream stream = Files.newInputStream(descriptor)) { + props.load(stream); + } + propsMap = props.stringPropertyNames().stream().collect(Collectors.toMap(Function.identity(), props::getProperty)); + } + + final String name = propsMap.remove("name"); + if (name == null || name.isEmpty()) { + throw new IllegalArgumentException( + "property [name] is missing for meta plugin in [" + descriptor + "]"); + } + final String description = propsMap.remove("description"); + if (description == null) { + throw new IllegalArgumentException( + "property [description] is missing for meta plugin [" + name + "]"); + } + + if (propsMap.isEmpty() == false) { + throw new IllegalArgumentException("Unknown properties in meta plugin descriptor: " + propsMap.keySet()); + } + + return new MetaPluginInfo(name, description); + } + + /** + * The name of the meta plugin. + * + * @return the meta plugin name + */ + public String getName() { + return name; + } + + /** + * The description of the meta plugin. + * + * @return the meta plugin description + */ + public String getDescription() { + return description; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + MetaPluginInfo that = (MetaPluginInfo) o; + + if (!name.equals(that.name)) return false; + + return true; + } + + @Override + public int hashCode() { + return name.hashCode(); + } + + @Override + public String toString() { + final StringBuilder information = new StringBuilder() + .append("- Plugin information:\n") + .append("Name: ").append(name).append("\n") + .append("Description: ").append(description); + return information.toString(); + } + +} diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginInfo.java b/core/src/main/java/org/elasticsearch/plugins/PluginInfo.java index 01cc7ea65908b..42c9df6d3dd3e 100644 --- a/core/src/main/java/org/elasticsearch/plugins/PluginInfo.java +++ b/core/src/main/java/org/elasticsearch/plugins/PluginInfo.java @@ -22,7 +22,9 @@ import org.elasticsearch.Version; import org.elasticsearch.bootstrap.JarHell; import org.elasticsearch.common.Booleans; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -31,14 +33,19 @@ import java.io.IOException; import java.io.InputStream; +import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -125,7 +132,46 @@ public void writeTo(final StreamOutput out) throws IOException { } } - /** reads (and validates) plugin metadata descriptor file */ + /** + * Extracts all {@link PluginInfo} from the provided {@code rootPath} expanding meta plugins if needed. + * @param rootPath the path where the plugins are installed + * @return A list of all plugin paths installed in the {@code rootPath} + * @throws IOException if an I/O exception occurred reading the plugin descriptors + */ + public static List extractAllPlugins(final Path rootPath) throws IOException { + final List plugins = new LinkedList<>(); // order is already lost, but some filesystems have it + final Set seen = new HashSet<>(); + if (Files.exists(rootPath)) { + try (DirectoryStream stream = Files.newDirectoryStream(rootPath)) { + for (Path plugin : stream) { + if (FileSystemUtils.isDesktopServicesStore(plugin) || + plugin.getFileName().toString().startsWith(".removing-")) { + continue; + } + if (seen.add(plugin.getFileName().toString()) == false) { + throw new IllegalStateException("duplicate plugin: " + plugin); + } + if (MetaPluginInfo.isMetaPlugin(plugin)) { + try (DirectoryStream subStream = Files.newDirectoryStream(plugin)) { + for (Path subPlugin : subStream) { + if (MetaPluginInfo.isPropertiesFile(subPlugin) || + FileSystemUtils.isDesktopServicesStore(subPlugin)) { + continue; + } + if (seen.add(subPlugin.getFileName().toString()) == false) { + throw new IllegalStateException("duplicate plugin: " + subPlugin); + } + plugins.add(subPlugin); + } + } + } else { + plugins.add(plugin); + } + } + } + } + return plugins; + } /** * Reads and validates the plugin descriptor file. @@ -341,16 +387,19 @@ public int hashCode() { @Override public String toString() { + return toString(""); + } + + public String toString(String prefix) { final StringBuilder information = new StringBuilder() - .append("- Plugin information:\n") - .append("Name: ").append(name).append("\n") - .append("Description: ").append(description).append("\n") - .append("Version: ").append(version).append("\n") - .append("Native Controller: ").append(hasNativeController).append("\n") - .append("Requires Keystore: ").append(requiresKeystore).append("\n") - .append("Extended Plugins: ").append(extendedPlugins).append("\n") - .append(" * Classname: ").append(classname); + .append(prefix).append("- Plugin information:\n") + .append(prefix).append("Name: ").append(name).append("\n") + .append(prefix).append("Description: ").append(description).append("\n") + .append(prefix).append("Version: ").append(version).append("\n") + .append(prefix).append("Native Controller: ").append(hasNativeController).append("\n") + .append(prefix).append("Requires Keystore: ").append(requiresKeystore).append("\n") + .append(prefix).append("Extended Plugins: ").append(extendedPlugins).append("\n") + .append(prefix).append(" * Classname: ").append(classname); return information.toString(); } - } diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java index 445edec1c19c4..d60d01273bbba 100644 --- a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -34,7 +34,6 @@ import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.inject.Module; -import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; @@ -322,29 +321,20 @@ static Set getPluginBundles(Path pluginsDirectory) throws IOException { Logger logger = Loggers.getLogger(PluginsService.class); Set bundles = new LinkedHashSet<>(); - try (DirectoryStream stream = Files.newDirectoryStream(pluginsDirectory)) { - for (Path plugin : stream) { - if (FileSystemUtils.isDesktopServicesStore(plugin)) { - continue; - } - if (plugin.getFileName().toString().startsWith(".removing-")) { - continue; - } - logger.trace("--- adding plugin [{}]", plugin.toAbsolutePath()); - final PluginInfo info; - try { - info = PluginInfo.readFromProperties(plugin); - } catch (IOException e) { - throw new IllegalStateException("Could not load plugin descriptor for existing plugin [" - + plugin.getFileName() + "]. Was the plugin built before 2.0?", e); - } - - if (bundles.add(new Bundle(info, plugin)) == false) { - throw new IllegalStateException("duplicate plugin: " + info); - } + List infos = PluginInfo.extractAllPlugins(pluginsDirectory); + for (Path plugin : infos) { + logger.trace("--- adding plugin [{}]", plugin.toAbsolutePath()); + final PluginInfo info; + try { + info = PluginInfo.readFromProperties(plugin); + } catch (IOException e) { + throw new IllegalStateException("Could not load plugin descriptor for existing plugin [" + + plugin.getFileName() + "]. Was the plugin built before 2.0?", e); + } + if (bundles.add(new Bundle(info, plugin)) == false) { + throw new IllegalStateException("duplicate plugin: " + info); } } - return bundles; } diff --git a/core/src/test/java/org/elasticsearch/plugins/MetaPluginInfoTests.java b/core/src/test/java/org/elasticsearch/plugins/MetaPluginInfoTests.java new file mode 100644 index 0000000000000..2b7f50056a9c8 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/plugins/MetaPluginInfoTests.java @@ -0,0 +1,120 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plugins; + +import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.Version; +import org.elasticsearch.test.ESTestCase; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.endsWith; + +@LuceneTestCase.SuppressFileSystems(value = "ExtrasFS") +public class MetaPluginInfoTests extends ESTestCase { + + public void testReadFromProperties() throws Exception { + Path pluginDir = createTempDir().resolve("fake-meta-plugin"); + PluginTestUtil.writeMetaPluginProperties(pluginDir, + "description", "fake desc", + "name", "my_meta_plugin"); + MetaPluginInfo info = MetaPluginInfo.readFromProperties(pluginDir); + assertEquals("my_meta_plugin", info.getName()); + assertEquals("fake desc", info.getDescription()); + } + + public void testReadFromPropertiesNameMissing() throws Exception { + Path pluginDir = createTempDir().resolve("fake-meta-plugin"); + PluginTestUtil.writeMetaPluginProperties(pluginDir); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> MetaPluginInfo.readFromProperties(pluginDir)); + assertThat(e.getMessage(), containsString("property [name] is missing for")); + + PluginTestUtil.writeMetaPluginProperties(pluginDir, "name", ""); + e = expectThrows(IllegalArgumentException.class, () -> MetaPluginInfo.readFromProperties(pluginDir)); + assertThat(e.getMessage(), containsString("property [name] is missing for")); + } + + public void testReadFromPropertiesDescriptionMissing() throws Exception { + Path pluginDir = createTempDir().resolve("fake-meta-plugin"); + PluginTestUtil.writeMetaPluginProperties(pluginDir, "name", "fake-meta-plugin"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> MetaPluginInfo.readFromProperties(pluginDir)); + assertThat(e.getMessage(), containsString("[description] is missing")); + } + + public void testUnknownProperties() throws Exception { + Path pluginDir = createTempDir().resolve("fake-meta-plugin"); + PluginTestUtil.writeMetaPluginProperties(pluginDir, + "extra", "property", + "unknown", "property", + "description", "fake desc", + "name", "my_meta_plugin"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> MetaPluginInfo.readFromProperties(pluginDir)); + assertThat(e.getMessage(), containsString("Unknown properties in meta plugin descriptor")); + } + + public void testExtractAllPluginsWithDuplicates() throws Exception { + Path pluginDir = createTempDir().resolve("plugins"); + // Simple plugin + Path plugin1 = pluginDir.resolve("plugin1"); + Files.createDirectories(plugin1); + PluginTestUtil.writePluginProperties(plugin1, + "description", "fake desc", + "name", "plugin1", + "version", "1.0", + "elasticsearch.version", Version.CURRENT.toString(), + "java.version", System.getProperty("java.specification.version"), + "classname", "FakePlugin"); + + // Meta plugin + Path metaPlugin = pluginDir.resolve("meta_plugin"); + Files.createDirectory(metaPlugin); + PluginTestUtil.writeMetaPluginProperties(metaPlugin, + "description", "fake desc", + "name", "meta_plugin"); + Path plugin2 = metaPlugin.resolve("plugin1"); + Files.createDirectory(plugin2); + PluginTestUtil.writePluginProperties(plugin2, + "description", "fake desc", + "name", "plugin1", + "version", "1.0", + "elasticsearch.version", Version.CURRENT.toString(), + "java.version", System.getProperty("java.specification.version"), + "classname", "FakePlugin"); + Path plugin3 = metaPlugin.resolve("plugin2"); + Files.createDirectory(plugin3); + PluginTestUtil.writePluginProperties(plugin3, + "description", "fake desc", + "name", "plugin2", + "version", "1.0", + "elasticsearch.version", Version.CURRENT.toString(), + "java.version", System.getProperty("java.specification.version"), + "classname", "FakePlugin"); + + IllegalStateException exc = + expectThrows(IllegalStateException.class, () -> PluginInfo.extractAllPlugins(pluginDir)); + assertThat(exc.getMessage(), containsString("duplicate plugin")); + assertThat(exc.getMessage(), endsWith("plugin1")); + } +} diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java b/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java index 6d2b09f87eb70..a767dad204efc 100644 --- a/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java +++ b/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java @@ -41,7 +41,7 @@ public class PluginInfoTests extends ESTestCase { public void testReadFromProperties() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - PluginTestUtil.writeProperties(pluginDir, + PluginTestUtil.writePluginProperties(pluginDir, "description", "fake desc", "name", "my_plugin", "version", "1.0", @@ -58,25 +58,25 @@ public void testReadFromProperties() throws Exception { public void testReadFromPropertiesNameMissing() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - PluginTestUtil.writeProperties(pluginDir); + PluginTestUtil.writePluginProperties(pluginDir); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> PluginInfo.readFromProperties(pluginDir)); assertThat(e.getMessage(), containsString("property [name] is missing in")); - PluginTestUtil.writeProperties(pluginDir, "name", ""); + PluginTestUtil.writePluginProperties(pluginDir, "name", ""); e = expectThrows(IllegalArgumentException.class, () -> PluginInfo.readFromProperties(pluginDir)); assertThat(e.getMessage(), containsString("property [name] is missing in")); } public void testReadFromPropertiesDescriptionMissing() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - PluginTestUtil.writeProperties(pluginDir, "name", "fake-plugin"); + PluginTestUtil.writePluginProperties(pluginDir, "name", "fake-plugin"); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> PluginInfo.readFromProperties(pluginDir)); assertThat(e.getMessage(), containsString("[description] is missing")); } public void testReadFromPropertiesVersionMissing() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - PluginTestUtil.writeProperties( + PluginTestUtil.writePluginProperties( pluginDir, "description", "fake desc", "name", "fake-plugin"); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> PluginInfo.readFromProperties(pluginDir)); assertThat(e.getMessage(), containsString("[version] is missing")); @@ -84,7 +84,7 @@ public void testReadFromPropertiesVersionMissing() throws Exception { public void testReadFromPropertiesElasticsearchVersionMissing() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - PluginTestUtil.writeProperties(pluginDir, + PluginTestUtil.writePluginProperties(pluginDir, "description", "fake desc", "name", "my_plugin", "version", "1.0"); @@ -94,7 +94,7 @@ public void testReadFromPropertiesElasticsearchVersionMissing() throws Exception public void testReadFromPropertiesJavaVersionMissing() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - PluginTestUtil.writeProperties(pluginDir, + PluginTestUtil.writePluginProperties(pluginDir, "description", "fake desc", "name", "my_plugin", "elasticsearch.version", Version.CURRENT.toString(), @@ -106,7 +106,7 @@ public void testReadFromPropertiesJavaVersionMissing() throws Exception { public void testReadFromPropertiesJavaVersionIncompatible() throws Exception { String pluginName = "fake-plugin"; Path pluginDir = createTempDir().resolve(pluginName); - PluginTestUtil.writeProperties(pluginDir, + PluginTestUtil.writePluginProperties(pluginDir, "description", "fake desc", "name", pluginName, "elasticsearch.version", Version.CURRENT.toString(), @@ -120,7 +120,7 @@ public void testReadFromPropertiesJavaVersionIncompatible() throws Exception { public void testReadFromPropertiesBadJavaVersionFormat() throws Exception { String pluginName = "fake-plugin"; Path pluginDir = createTempDir().resolve(pluginName); - PluginTestUtil.writeProperties(pluginDir, + PluginTestUtil.writePluginProperties(pluginDir, "description", "fake desc", "name", pluginName, "elasticsearch.version", Version.CURRENT.toString(), @@ -134,7 +134,7 @@ public void testReadFromPropertiesBadJavaVersionFormat() throws Exception { public void testReadFromPropertiesBogusElasticsearchVersion() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - PluginTestUtil.writeProperties(pluginDir, + PluginTestUtil.writePluginProperties(pluginDir, "description", "fake desc", "version", "1.0", "name", "my_plugin", @@ -145,7 +145,7 @@ public void testReadFromPropertiesBogusElasticsearchVersion() throws Exception { public void testReadFromPropertiesOldElasticsearchVersion() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - PluginTestUtil.writeProperties(pluginDir, + PluginTestUtil.writePluginProperties(pluginDir, "description", "fake desc", "name", "my_plugin", "version", "1.0", @@ -156,7 +156,7 @@ public void testReadFromPropertiesOldElasticsearchVersion() throws Exception { public void testReadFromPropertiesJvmMissingClassname() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - PluginTestUtil.writeProperties(pluginDir, + PluginTestUtil.writePluginProperties(pluginDir, "description", "fake desc", "name", "my_plugin", "version", "1.0", @@ -168,7 +168,7 @@ public void testReadFromPropertiesJvmMissingClassname() throws Exception { public void testExtendedPluginsSingleExtension() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - PluginTestUtil.writeProperties(pluginDir, + PluginTestUtil.writePluginProperties(pluginDir, "description", "fake desc", "name", "my_plugin", "version", "1.0", @@ -182,7 +182,7 @@ public void testExtendedPluginsSingleExtension() throws Exception { public void testExtendedPluginsMultipleExtensions() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - PluginTestUtil.writeProperties(pluginDir, + PluginTestUtil.writePluginProperties(pluginDir, "description", "fake desc", "name", "my_plugin", "version", "1.0", @@ -196,7 +196,7 @@ public void testExtendedPluginsMultipleExtensions() throws Exception { public void testExtendedPluginsEmpty() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - PluginTestUtil.writeProperties(pluginDir, + PluginTestUtil.writePluginProperties(pluginDir, "description", "fake desc", "name", "my_plugin", "version", "1.0", @@ -224,7 +224,7 @@ public void testPluginListSorted() { List plugins = new ArrayList<>(); plugins.add(new PluginInfo("c", "foo", "dummy", "dummyclass", Collections.emptyList(), randomBoolean(), randomBoolean())); plugins.add(new PluginInfo("b", "foo", "dummy", "dummyclass", Collections.emptyList(),randomBoolean(), randomBoolean())); - plugins.add(new PluginInfo("e", "foo", "dummy", "dummyclass", Collections.emptyList(),randomBoolean(), randomBoolean())); + plugins.add(new PluginInfo( "e", "foo", "dummy", "dummyclass", Collections.emptyList(),randomBoolean(), randomBoolean())); plugins.add(new PluginInfo("a", "foo", "dummy", "dummyclass", Collections.emptyList(),randomBoolean(), randomBoolean())); plugins.add(new PluginInfo("d", "foo", "dummy", "dummyclass", Collections.emptyList(),randomBoolean(), randomBoolean())); PluginsAndModules pluginsInfo = new PluginsAndModules(plugins, Collections.emptyList()); @@ -236,7 +236,7 @@ public void testPluginListSorted() { public void testUnknownProperties() throws Exception { Path pluginDir = createTempDir().resolve("fake-plugin"); - PluginTestUtil.writeProperties(pluginDir, + PluginTestUtil.writePluginProperties(pluginDir, "extra", "property", "unknown", "property", "description", "fake desc", diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java b/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java index 16c3eb34b0e63..1bb9d675988d8 100644 --- a/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java +++ b/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java @@ -180,7 +180,7 @@ public void testStartupWithRemovingMarker() throws IOException { Files.createFile(fake.resolve("plugin.jar")); final Path removing = home.resolve("plugins").resolve(".removing-fake"); Files.createFile(removing); - PluginTestUtil.writeProperties( + PluginTestUtil.writePluginProperties( fake, "description", "fake", "name", "fake", @@ -541,7 +541,7 @@ public void testNonExtensibleDep() throws Exception { Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), homeDir).build(); Path pluginsDir = homeDir.resolve("plugins"); Path mypluginDir = pluginsDir.resolve("myplugin"); - PluginTestUtil.writeProperties( + PluginTestUtil.writePluginProperties( mypluginDir, "description", "whatever", "name", "myplugin", @@ -554,7 +554,7 @@ public void testNonExtensibleDep() throws Exception { Files.copy(jar, mypluginDir.resolve("plugin.jar")); } Path nonextensibleDir = pluginsDir.resolve("nonextensible"); - PluginTestUtil.writeProperties( + PluginTestUtil.writePluginProperties( nonextensibleDir, "description", "whatever", "name", "nonextensible", diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java index 5fedb050ff2e1..216eb46411ac2 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java @@ -30,6 +30,7 @@ import org.elasticsearch.cli.ExitCodes; import org.elasticsearch.cli.Terminal; import org.elasticsearch.cli.UserException; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.hash.MessageDigests; @@ -86,8 +87,8 @@ *

  • A URL to a plugin zip
    • * * - * Plugins are packaged as zip files. Each packaged plugin must contain a - * plugin properties file. See {@link PluginInfo}. + * Plugins are packaged as zip files. Each packaged plugin must contain a plugin properties file + * or a meta plugin properties file. See {@link PluginInfo} and {@link MetaPluginInfo}, respectively. *

    * The installation process first extracts the plugin files into a temporary * directory in order to verify the plugin satisfies the following requirements: @@ -105,6 +106,11 @@ * files specific to the plugin. The config files be installed into a subdirectory of the * elasticsearch config directory, using the name of the plugin. If any files to be installed * already exist, they will be skipped. + *

    + * If the plugin is a meta plugin, the installation process installs each plugin separately + * inside the meta plugin directory. The {@code bin} and {@code config} directory are also moved + * inside the meta plugin directory. + *

    */ class InstallPluginCommand extends EnvironmentAwareCommand { @@ -527,22 +533,44 @@ private Path stagingDirectoryWithoutPosixPermissions(Path pluginsDir) throws IOE return Files.createTempDirectory(pluginsDir, ".installing-"); } - /** Load information about the plugin, and verify it can be installed with no errors. */ - private PluginInfo verify(Terminal terminal, Path pluginRoot, boolean isBatch, Environment env) throws Exception { - // read and validate the plugin descriptor - PluginInfo info = PluginInfo.readFromProperties(pluginRoot); - - // checking for existing version of the plugin - final Path destination = env.pluginsFile().resolve(info.getName()); + // checking for existing version of the plugin + private void verifyPluginName(Path pluginPath, String pluginName, Path candidateDir) throws UserException, IOException { + final Path destination = pluginPath.resolve(pluginName); if (Files.exists(destination)) { final String message = String.format( Locale.ROOT, "plugin directory [%s] already exists; if you need to update the plugin, " + "uninstall it first using command 'remove %s'", destination.toAbsolutePath(), - info.getName()); + pluginName); throw new UserException(PLUGIN_EXISTS, message); } + // checks meta plugins too + try (DirectoryStream stream = Files.newDirectoryStream(pluginPath)) { + for (Path plugin : stream) { + if (candidateDir.equals(plugin.resolve(pluginName))) { + continue; + } + if (MetaPluginInfo.isMetaPlugin(plugin) && Files.exists(plugin.resolve(pluginName))) { + final MetaPluginInfo info = MetaPluginInfo.readFromProperties(plugin); + final String message = String.format( + Locale.ROOT, + "plugin name [%s] already exists in a meta plugin; if you need to update the meta plugin, " + + "uninstall it first using command 'remove %s'", + plugin.resolve(pluginName).toAbsolutePath(), + info.getName()); + throw new UserException(PLUGIN_EXISTS, message); + } + } + } + } + + /** Load information about the plugin, and verify it can be installed with no errors. */ + private PluginInfo verify(Terminal terminal, Path pluginRoot, boolean isBatch, Environment env) throws Exception { + final PluginInfo info = PluginInfo.readFromProperties(pluginRoot); + + // checking for existing version of the plugin + verifyPluginName(env.pluginsFile(), info.getName(), pluginRoot); PluginsService.checkForFailedPluginRemovals(env.pluginsFile()); @@ -569,14 +597,15 @@ private PluginInfo verify(Terminal terminal, Path pluginRoot, boolean isBatch, E } /** check a candidate plugin for jar hell before installing it */ - void jarHellCheck(PluginInfo info, Path candidate, Path pluginsDir, Path modulesDir) throws Exception { + void jarHellCheck(PluginInfo candidateInfo, Path candidateDir, Path pluginsDir, Path modulesDir) throws Exception { // create list of current jars in classpath final Set jars = new HashSet<>(JarHell.parseClassPath()); + // read existing bundles. this does some checks on the installation too. Set bundles = new HashSet<>(PluginsService.getPluginBundles(pluginsDir)); bundles.addAll(PluginsService.getModuleBundles(modulesDir)); - bundles.add(new PluginsService.Bundle(info, candidate)); + bundles.add(new PluginsService.Bundle(candidateInfo, candidateDir)); List sortedBundles = PluginsService.sortBundles(bundles); // check jarhell of all plugins so we know this plugin and anything depending on it are ok together @@ -590,78 +619,138 @@ void jarHellCheck(PluginInfo info, Path candidate, Path pluginsDir, Path modules // TODO: verify the classname exists in one of the jars! } - /** - * Installs the plugin from {@code tmpRoot} into the plugins dir. - * If the plugin has a bin dir and/or a config dir, those are copied. - */ private void install(Terminal terminal, boolean isBatch, Path tmpRoot, Environment env) throws Exception { List deleteOnFailure = new ArrayList<>(); deleteOnFailure.add(tmpRoot); - try { - PluginInfo info = verify(terminal, tmpRoot, isBatch, env); - final Path destination = env.pluginsFile().resolve(info.getName()); + if (MetaPluginInfo.isMetaPlugin(tmpRoot)) { + installMetaPlugin(terminal, isBatch, tmpRoot, env, deleteOnFailure); + } else { + installPlugin(terminal, isBatch, tmpRoot, env, deleteOnFailure); + } + } catch (Exception installProblem) { + try { + IOUtils.rm(deleteOnFailure.toArray(new Path[0])); + } catch (IOException exceptionWhileRemovingFiles) { + installProblem.addSuppressed(exceptionWhileRemovingFiles); + } + throw installProblem; + } + } - Path tmpBinDir = tmpRoot.resolve("bin"); + /** + * Installs the meta plugin and all the bundled plugins from {@code tmpRoot} into the plugins dir. + * If a bundled plugin has a bin dir and/or a config dir, those are copied. + */ + private void installMetaPlugin(Terminal terminal, boolean isBatch, Path tmpRoot, + Environment env, List deleteOnFailure) throws Exception { + final MetaPluginInfo metaInfo = MetaPluginInfo.readFromProperties(tmpRoot); + verifyPluginName(env.pluginsFile(), metaInfo.getName(), tmpRoot); + final Path destination = env.pluginsFile().resolve(metaInfo.getName()); + deleteOnFailure.add(destination); + terminal.println(VERBOSE, metaInfo.toString()); + final List pluginPaths = new ArrayList<>(); + try (DirectoryStream paths = Files.newDirectoryStream(tmpRoot)) { + // Extract bundled plugins path and validate plugin names + for (Path plugin : paths) { + if (MetaPluginInfo.isPropertiesFile(plugin)) { + continue; + } + final PluginInfo info = PluginInfo.readFromProperties(plugin); + verifyPluginName(env.pluginsFile(), info.getName(), plugin); + pluginPaths.add(plugin); + } + } + final List pluginInfos = new ArrayList<>(); + for (Path plugin : pluginPaths) { + final PluginInfo info = verify(terminal, plugin, isBatch, env); + pluginInfos.add(info); + Path tmpBinDir = plugin.resolve("bin"); if (Files.exists(tmpBinDir)) { - Path destBinDir = env.binFile().resolve(info.getName()); + Path destBinDir = env.binFile().resolve(metaInfo.getName()).resolve(info.getName()); deleteOnFailure.add(destBinDir); installBin(info, tmpBinDir, destBinDir); } - Path tmpConfigDir = tmpRoot.resolve("config"); + Path tmpConfigDir = plugin.resolve("config"); if (Files.exists(tmpConfigDir)) { // some files may already exist, and we don't remove plugin config files on plugin removal, // so any installed config files are left on failure too - installConfig(info, tmpConfigDir, env.configFile().resolve(info.getName())); + Path destConfigDir = env.configFile().resolve(metaInfo.getName()).resolve(info.getName()); + installConfig(info, tmpConfigDir, destConfigDir); } + } + movePlugin(tmpRoot, destination); + for (PluginInfo info : pluginInfos) { + if (info.requiresKeystore()) { + createKeystoreIfNeeded(terminal, env, info); + break; + } + } + String[] plugins = pluginInfos.stream().map(PluginInfo::getName).toArray(String[]::new); + terminal.println("-> Installed " + metaInfo.getName() + " with: " + Strings.arrayToCommaDelimitedString(plugins)); + } - Files.move(tmpRoot, destination, StandardCopyOption.ATOMIC_MOVE); - Files.walkFileTree(destination, new SimpleFileVisitor() { - @Override - public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { - if ("bin".equals(file.getParent().getFileName().toString())) { - setFileAttributes(file, BIN_FILES_PERMS); - } else { - setFileAttributes(file, PLUGIN_FILES_PERMS); - } - return FileVisitResult.CONTINUE; - } + /** + * Installs the plugin from {@code tmpRoot} into the plugins dir. + * If the plugin has a bin dir and/or a config dir, those are copied. + */ + private void installPlugin(Terminal terminal, boolean isBatch, Path tmpRoot, + Environment env, List deleteOnFailure) throws Exception { + final PluginInfo info = verify(terminal, tmpRoot, isBatch, env); + final Path destination = env.pluginsFile().resolve(info.getName()); + deleteOnFailure.add(destination); - @Override - public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) throws IOException { - setFileAttributes(dir, PLUGIN_DIR_PERMS); - return FileVisitResult.CONTINUE; - } - }); + Path tmpBinDir = tmpRoot.resolve("bin"); + if (Files.exists(tmpBinDir)) { + Path destBinDir = env.binFile().resolve(info.getName()); + deleteOnFailure.add(destBinDir); + installBin(info, tmpBinDir, destBinDir); + } - if (info.requiresKeystore()) { - KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configFile()); - if (keystore == null) { - terminal.println("Elasticsearch keystore is required by plugin [" + info.getName() + "], creating..."); - keystore = KeyStoreWrapper.create(new char[0]); - keystore.save(env.configFile()); + Path tmpConfigDir = tmpRoot.resolve("config"); + if (Files.exists(tmpConfigDir)) { + // some files may already exist, and we don't remove plugin config files on plugin removal, + // so any installed config files are left on failure too + Path destConfigDir = env.configFile().resolve(info.getName()); + installConfig(info, tmpConfigDir, destConfigDir); + } + movePlugin(tmpRoot, destination); + if (info.requiresKeystore()) { + createKeystoreIfNeeded(terminal, env, info); + } + terminal.println("-> Installed " + info.getName()); + } + + /** Moves the plugin directory into its final destination. **/ + private void movePlugin(Path tmpRoot, Path destination) throws IOException { + Files.move(tmpRoot, destination, StandardCopyOption.ATOMIC_MOVE); + Files.walkFileTree(destination, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { + if ("bin".equals(file.getParent().getFileName().toString())) { + setFileAttributes(file, BIN_FILES_PERMS); + } else { + setFileAttributes(file, PLUGIN_FILES_PERMS); } + return FileVisitResult.CONTINUE; } - terminal.println("-> Installed " + info.getName()); - - } catch (Exception installProblem) { - try { - IOUtils.rm(deleteOnFailure.toArray(new Path[0])); - } catch (IOException exceptionWhileRemovingFiles) { - installProblem.addSuppressed(exceptionWhileRemovingFiles); + @Override + public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) throws IOException { + setFileAttributes(dir, PLUGIN_DIR_PERMS); + return FileVisitResult.CONTINUE; } - throw installProblem; - } + }); } + /** Copies the files from {@code tmpBinDir} into {@code destBinDir}, along with permissions from dest dirs parent. */ private void installBin(PluginInfo info, Path tmpBinDir, Path destBinDir) throws Exception { if (Files.isDirectory(tmpBinDir) == false) { throw new UserException(PLUGIN_MALFORMED, "bin in plugin " + info.getName() + " is not a directory"); } - Files.createDirectory(destBinDir); + Files.createDirectories(destBinDir); setFileAttributes(destBinDir, BIN_DIR_PERMS); try (DirectoryStream stream = Files.newDirectoryStream(tmpBinDir)) { @@ -719,6 +808,15 @@ private void installConfig(PluginInfo info, Path tmpConfigDir, Path destConfigDi IOUtils.rm(tmpConfigDir); // clean up what we just copied } + private void createKeystoreIfNeeded(Terminal terminal, Environment env, PluginInfo info) throws Exception { + KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configFile()); + if (keystore == null) { + terminal.println("Elasticsearch keystore is required by plugin [" + info.getName() + "], creating..."); + keystore = KeyStoreWrapper.create(new char[0]); + keystore.save(env.configFile()); + } + } + private static void setOwnerGroup(final Path path, final PosixFileAttributes attributes) throws IOException { Objects.requireNonNull(attributes); PosixFileAttributeView fileAttributeView = Files.getFileAttributeView(path, PosixFileAttributeView.class); diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java index c2b5ce34b5469..70acf62bd8e1c 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java @@ -22,6 +22,7 @@ import joptsimple.OptionSet; import org.elasticsearch.cli.EnvironmentAwareCommand; import org.elasticsearch.cli.Terminal; +import org.elasticsearch.common.Nullable; import org.elasticsearch.env.Environment; import java.io.IOException; @@ -29,8 +30,11 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; /** * A command for the plugin cli to list plugins installed in elasticsearch. @@ -56,16 +60,38 @@ protected void execute(Terminal terminal, OptionSet options, Environment env) th } Collections.sort(plugins); for (final Path plugin : plugins) { - terminal.println(Terminal.Verbosity.SILENT, plugin.getFileName().toString()); - try { - PluginInfo info = PluginInfo.readFromProperties(env.pluginsFile().resolve(plugin.toAbsolutePath())); - terminal.println(Terminal.Verbosity.VERBOSE, info.toString()); - } catch (IllegalArgumentException e) { - if (e.getMessage().contains("incompatible with version")) { - terminal.println("WARNING: " + e.getMessage()); - } else { - throw e; + if (MetaPluginInfo.isMetaPlugin(plugin)) { + MetaPluginInfo metaInfo = MetaPluginInfo.readFromProperties(plugin); + List subPluginPaths = new ArrayList<>(); + try (DirectoryStream subPaths = Files.newDirectoryStream(plugin)) { + for (Path subPlugin : subPaths) { + if (MetaPluginInfo.isPropertiesFile(subPlugin)) { + continue; + } + subPluginPaths.add(subPlugin); + } } + Collections.sort(subPluginPaths); + terminal.println(Terminal.Verbosity.SILENT, metaInfo.getName()); + for (Path subPlugin : subPluginPaths) { + printPlugin(env, terminal, subPlugin, "\t"); + } + } else { + printPlugin(env, terminal, plugin, ""); + } + } + } + + private void printPlugin(Environment env, Terminal terminal, Path plugin, String prefix) throws IOException { + terminal.println(Terminal.Verbosity.SILENT, prefix + plugin.getFileName().toString()); + try { + PluginInfo info = PluginInfo.readFromProperties(env.pluginsFile().resolve(plugin.toAbsolutePath())); + terminal.println(Terminal.Verbosity.VERBOSE, info.toString(prefix)); + } catch (IllegalArgumentException e) { + if (e.getMessage().contains("incompatible with version")) { + terminal.println("WARNING: " + e.getMessage()); + } else { + throw e; } } } diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java index e545609ccbff1..9d05fe1e76654 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java @@ -115,7 +115,7 @@ public void setUp() throws Exception { super.setUp(); skipJarHellCommand = new InstallPluginCommand() { @Override - void jarHellCheck(PluginInfo info, Path candidate, Path pluginsDir, Path modulesDir) throws Exception { + void jarHellCheck(PluginInfo candidateInfo, Path candidate, Path pluginsDir, Path modulesDir) throws Exception { // no jarhell check } }; @@ -214,7 +214,19 @@ static String createPluginUrl(String name, Path structure, String... additionalP return createPlugin(name, structure, false, additionalProps).toUri().toURL().toString(); } - static Path createPlugin(String name, Path structure, boolean createSecurityPolicyFile, String... additionalProps) throws IOException { + /** creates an meta plugin .zip and returns the url for testing */ + static String createMetaPluginUrl(String name, Path structure) throws IOException { + return createMetaPlugin(name, structure).toUri().toURL().toString(); + } + + static void writeMetaPlugin(String name, Path structure) throws IOException { + PluginTestUtil.writeMetaPluginProperties(structure, + "description", "fake desc", + "name", name + ); + } + + static void writePlugin(String name, Path structure, boolean createSecurityPolicyFile, String... additionalProps) throws IOException { String[] properties = Stream.concat(Stream.of( "description", "fake desc", "name", name, @@ -223,12 +235,22 @@ static Path createPlugin(String name, Path structure, boolean createSecurityPoli "java.version", System.getProperty("java.specification.version"), "classname", "FakePlugin" ), Arrays.stream(additionalProps)).toArray(String[]::new); - PluginTestUtil.writeProperties(structure, properties); + PluginTestUtil.writePluginProperties(structure, properties); if (createSecurityPolicyFile) { String securityPolicyContent = "grant {\n permission java.lang.RuntimePermission \"setFactory\";\n};\n"; Files.write(structure.resolve("plugin-security.policy"), securityPolicyContent.getBytes(StandardCharsets.UTF_8)); } - writeJar(structure.resolve("plugin.jar"), "FakePlugin"); + String className = name.substring(0, 1).toUpperCase(Locale.ENGLISH) + name.substring(1) + "Plugin"; + writeJar(structure.resolve("plugin.jar"), className); + } + + static Path createPlugin(String name, Path structure, boolean createSecurityPolicyFile, String... additionalProps) throws IOException { + writePlugin(name, structure, createSecurityPolicyFile, additionalProps); + return writeZip(structure, "elasticsearch"); + } + + static Path createMetaPlugin(String name, Path structure) throws IOException { + writeMetaPlugin(name, structure); return writeZip(structure, "elasticsearch"); } @@ -243,8 +265,20 @@ MockTerminal installPlugin(String pluginUrl, Path home, InstallPluginCommand com return terminal; } + void assertMetaPlugin(String metaPlugin, String name, Path original, Environment env) throws IOException { + assertPluginInternal(name, original, env, + env.pluginsFile().resolve(metaPlugin), env.configFile().resolve(metaPlugin), env.binFile().resolve(metaPlugin)); + } + + void assertPlugin(String name, Path original, Environment env) throws IOException { - Path got = env.pluginsFile().resolve(name); + assertPluginInternal(name, original, env, + env.pluginsFile(), env.configFile(), env.binFile()); + } + + void assertPluginInternal(String name, Path original, Environment env, + Path pluginsFile, Path configFile, Path binFile) throws IOException { + Path got = pluginsFile.resolve(name); assertTrue("dir " + name + " exists", Files.exists(got)); if (isPosix) { @@ -265,12 +299,12 @@ void assertPlugin(String name, Path original, Environment env) throws IOExceptio assertFalse("bin was not copied", Files.exists(got.resolve("bin"))); assertFalse("config was not copied", Files.exists(got.resolve("config"))); if (Files.exists(original.resolve("bin"))) { - Path binDir = env.binFile().resolve(name); + Path binDir = binFile.resolve(name); assertTrue("bin dir exists", Files.exists(binDir)); assertTrue("bin is a dir", Files.isDirectory(binDir)); PosixFileAttributes binAttributes = null; if (isPosix) { - binAttributes = Files.readAttributes(env.binFile(), PosixFileAttributes.class); + binAttributes = Files.readAttributes(binFile, PosixFileAttributes.class); } try (DirectoryStream stream = Files.newDirectoryStream(binDir)) { for (Path file : stream) { @@ -283,7 +317,7 @@ void assertPlugin(String name, Path original, Environment env) throws IOExceptio } } if (Files.exists(original.resolve("config"))) { - Path configDir = env.configFile().resolve(name); + Path configDir = configFile.resolve(name); assertTrue("config dir exists", Files.exists(configDir)); assertTrue("config is a dir", Files.isDirectory(configDir)); @@ -292,7 +326,7 @@ void assertPlugin(String name, Path original, Environment env) throws IOExceptio if (isPosix) { PosixFileAttributes configAttributes = - Files.getFileAttributeView(env.configFile(), PosixFileAttributeView.class).readAttributes(); + Files.getFileAttributeView(configFile, PosixFileAttributeView.class).readAttributes(); user = configAttributes.owner(); group = configAttributes.group(); @@ -344,9 +378,23 @@ public void testSomethingWorks() throws Exception { assertPlugin("fake", pluginDir, env.v2()); } - public void testInstallFailsIfPreviouslyRemovedPluginFailed() throws Exception { + public void testWithMetaPlugin() throws Exception { Tuple env = createEnv(fs, temp); Path pluginDir = createPluginDir(temp); + Files.createDirectory(pluginDir.resolve("fake1")); + writePlugin("fake1", pluginDir.resolve("fake1"), false); + Files.createDirectory(pluginDir.resolve("fake2")); + writePlugin("fake2", pluginDir.resolve("fake2"), false); + String pluginZip = createMetaPluginUrl("my_plugins", pluginDir); + installPlugin(pluginZip, env.v1()); + assertMetaPlugin("my_plugins", "fake1", pluginDir, env.v2()); + assertMetaPlugin("my_plugins", "fake2", pluginDir, env.v2()); + } + + public void testInstallFailsIfPreviouslyRemovedPluginFailed() throws Exception { + Tuple env = createEnv(fs, temp); + Path metaDir = createPluginDir(temp); + Path pluginDir = metaDir.resolve("fake"); String pluginZip = createPluginUrl("fake", pluginDir); final Path removing = env.v2().pluginsFile().resolve(".removing-failed"); Files.createDirectory(removing); @@ -356,6 +404,11 @@ public void testInstallFailsIfPreviouslyRemovedPluginFailed() throws Exception { "found file [%s] from a failed attempt to remove the plugin [failed]; execute [elasticsearch-plugin remove failed]", removing); assertThat(e, hasToString(containsString(expected))); + + // test with meta plugin + String metaZip = createMetaPluginUrl("my_plugins", metaDir); + final IllegalStateException e1 = expectThrows(IllegalStateException.class, () -> installPlugin(metaZip, env.v1())); + assertThat(e1, hasToString(containsString(expected))); } public void testSpaceInUrl() throws Exception { @@ -418,6 +471,23 @@ public void testJarHell() throws Exception { assertInstallCleaned(environment.v2()); } + public void testJarHellInMetaPlugin() throws Exception { + // jar hell test needs a real filesystem + assumeTrue("real filesystem", isReal); + Tuple environment = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); + Files.createDirectory(pluginDir.resolve("fake1")); + writePlugin("fake1", pluginDir.resolve("fake1"), false); + Files.createDirectory(pluginDir.resolve("fake2")); + writePlugin("fake2", pluginDir.resolve("fake2"), false); // adds plugin.jar with Fake2Plugin + writeJar(pluginDir.resolve("fake2").resolve("other.jar"), "Fake2Plugin"); + String pluginZip = createMetaPluginUrl("my_plugins", pluginDir); + IllegalStateException e = expectThrows(IllegalStateException.class, + () -> installPlugin(pluginZip, environment.v1(), defaultCommand)); + assertTrue(e.getMessage(), e.getMessage().contains("jar hell")); + assertInstallCleaned(environment.v2()); + } + public void testIsolatedPlugins() throws Exception { Tuple env = createEnv(fs, temp); // these both share the same FakePlugin class @@ -441,6 +511,23 @@ public void testExistingPlugin() throws Exception { assertInstallCleaned(env.v2()); } + public void testExistingMetaPlugin() throws Exception { + Tuple env = createEnv(fs, temp); + Path metaZip = createPluginDir(temp); + Path pluginDir = metaZip.resolve("fake"); + Files.createDirectory(pluginDir); + String pluginZip = createPluginUrl("fake", pluginDir); + installPlugin(pluginZip, env.v1()); + UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1())); + assertTrue(e.getMessage(), e.getMessage().contains("already exists")); + assertInstallCleaned(env.v2()); + + String anotherZip = createMetaPluginUrl("another_plugins", metaZip); + e = expectThrows(UserException.class, () -> installPlugin(anotherZip, env.v1())); + assertTrue(e.getMessage(), e.getMessage().contains("already exists")); + assertInstallCleaned(env.v2()); + } + public void testBin() throws Exception { Tuple env = createEnv(fs, temp); Path pluginDir = createPluginDir(temp); @@ -452,20 +539,43 @@ public void testBin() throws Exception { assertPlugin("fake", pluginDir, env.v2()); } + public void testMetaBin() throws Exception { + Tuple env = createEnv(fs, temp); + Path metaDir = createPluginDir(temp); + Path pluginDir = metaDir.resolve("fake"); + Files.createDirectory(pluginDir); + writePlugin("fake", pluginDir, false); + Path binDir = pluginDir.resolve("bin"); + Files.createDirectory(binDir); + Files.createFile(binDir.resolve("somescript")); + String pluginZip = createMetaPluginUrl("my_plugins", metaDir); + installPlugin(pluginZip, env.v1()); + assertMetaPlugin("my_plugins","fake", pluginDir, env.v2()); + } + public void testBinNotDir() throws Exception { Tuple env = createEnv(fs, temp); - Path pluginDir = createPluginDir(temp); + Path metaDir = createPluginDir(temp); + Path pluginDir = metaDir.resolve("fake"); + Files.createDirectory(pluginDir); Path binDir = pluginDir.resolve("bin"); Files.createFile(binDir); String pluginZip = createPluginUrl("fake", pluginDir); UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1())); assertTrue(e.getMessage(), e.getMessage().contains("not a directory")); assertInstallCleaned(env.v2()); + + String metaZip = createMetaPluginUrl("my_plugins", metaDir); + e = expectThrows(UserException.class, () -> installPlugin(metaZip, env.v1())); + assertTrue(e.getMessage(), e.getMessage().contains("not a directory")); + assertInstallCleaned(env.v2()); } public void testBinContainsDir() throws Exception { Tuple env = createEnv(fs, temp); - Path pluginDir = createPluginDir(temp); + Path metaDir = createPluginDir(temp); + Path pluginDir = metaDir.resolve("fake"); + Files.createDirectory(pluginDir); Path dirInBinDir = pluginDir.resolve("bin").resolve("foo"); Files.createDirectories(dirInBinDir); Files.createFile(dirInBinDir.resolve("somescript")); @@ -473,11 +583,16 @@ public void testBinContainsDir() throws Exception { UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1())); assertTrue(e.getMessage(), e.getMessage().contains("Directories not allowed in bin dir for plugin")); assertInstallCleaned(env.v2()); + + String metaZip = createMetaPluginUrl("my_plugins", metaDir); + e = expectThrows(UserException.class, () -> installPlugin(metaZip, env.v1())); + assertTrue(e.getMessage(), e.getMessage().contains("Directories not allowed in bin dir for plugin")); + assertInstallCleaned(env.v2()); } public void testBinConflict() throws Exception { Tuple env = createEnv(fs, temp); - Path pluginDir = createPluginDir(temp); + Path pluginDir = createPluginDir(temp); Path binDir = pluginDir.resolve("bin"); Files.createDirectory(binDir); Files.createFile(binDir.resolve("somescript")); @@ -505,6 +620,27 @@ public void testBinPermissions() throws Exception { } } + public void testMetaBinPermissions() throws Exception { + assumeTrue("posix filesystem", isPosix); + Tuple env = createEnv(fs, temp); + Path metaDir = createPluginDir(temp); + Path pluginDir = metaDir.resolve("fake"); + Files.createDirectory(pluginDir); + writePlugin("fake", pluginDir, false); + Path binDir = pluginDir.resolve("bin"); + Files.createDirectory(binDir); + Files.createFile(binDir.resolve("somescript")); + String pluginZip = createMetaPluginUrl("my_plugins", metaDir); + try (PosixPermissionsResetter binAttrs = new PosixPermissionsResetter(env.v2().binFile())) { + Set perms = binAttrs.getCopyPermissions(); + // make sure at least one execute perm is missing, so we know we forced it during installation + perms.remove(PosixFilePermission.GROUP_EXECUTE); + binAttrs.setPermissions(perms); + installPlugin(pluginZip, env.v1()); + assertMetaPlugin("my_plugins", "fake", pluginDir, env.v2()); + } + } + public void testPluginPermissions() throws Exception { assumeTrue("posix filesystem", isPosix); @@ -596,15 +732,44 @@ public void testExistingConfig() throws Exception { assertTrue(Files.exists(envConfigDir.resolve("other.yml"))); } + public void testExistingMetaConfig() throws Exception { + Tuple env = createEnv(fs, temp); + Path envConfigDir = env.v2().configFile().resolve("my_plugins").resolve("fake"); + Files.createDirectories(envConfigDir); + Files.write(envConfigDir.resolve("custom.yml"), "existing config".getBytes(StandardCharsets.UTF_8)); + Path metaDir = createPluginDir(temp); + Path pluginDir = metaDir.resolve("fake"); + Files.createDirectory(pluginDir); + writePlugin("fake", pluginDir, false); + Path configDir = pluginDir.resolve("config"); + Files.createDirectory(configDir); + Files.write(configDir.resolve("custom.yml"), "new config".getBytes(StandardCharsets.UTF_8)); + Files.createFile(configDir.resolve("other.yml")); + String pluginZip = createMetaPluginUrl("my_plugins", metaDir); + installPlugin(pluginZip, env.v1()); + assertMetaPlugin("my_plugins", "fake", pluginDir, env.v2()); + List configLines = Files.readAllLines(envConfigDir.resolve("custom.yml"), StandardCharsets.UTF_8); + assertEquals(1, configLines.size()); + assertEquals("existing config", configLines.get(0)); + assertTrue(Files.exists(envConfigDir.resolve("other.yml"))); + } + public void testConfigNotDir() throws Exception { Tuple env = createEnv(fs, temp); - Path pluginDir = createPluginDir(temp); + Path metaDir = createPluginDir(temp); + Path pluginDir = metaDir.resolve("fake"); + Files.createDirectories(pluginDir); Path configDir = pluginDir.resolve("config"); Files.createFile(configDir); String pluginZip = createPluginUrl("fake", pluginDir); UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1())); assertTrue(e.getMessage(), e.getMessage().contains("not a directory")); assertInstallCleaned(env.v2()); + + String metaZip = createMetaPluginUrl("my_plugins", metaDir); + e = expectThrows(UserException.class, () -> installPlugin(metaZip, env.v1())); + assertTrue(e.getMessage(), e.getMessage().contains("not a directory")); + assertInstallCleaned(env.v2()); } public void testConfigContainsDir() throws Exception { @@ -619,26 +784,21 @@ public void testConfigContainsDir() throws Exception { assertInstallCleaned(env.v2()); } - public void testConfigConflict() throws Exception { - Tuple env = createEnv(fs, temp); - Path pluginDir = createPluginDir(temp); - Path configDir = pluginDir.resolve("config"); - Files.createDirectory(configDir); - Files.createFile(configDir.resolve("myconfig.yml")); - String pluginZip = createPluginUrl("elasticsearch.yml", pluginDir); - FileAlreadyExistsException e = expectThrows(FileAlreadyExistsException.class, () -> installPlugin(pluginZip, env.v1())); - assertTrue(e.getMessage(), e.getMessage().contains(env.v2().configFile().resolve("elasticsearch.yml").toString())); - assertInstallCleaned(env.v2()); - } - public void testMissingDescriptor() throws Exception { Tuple env = createEnv(fs, temp); - Path pluginDir = createPluginDir(temp); + Path metaDir = createPluginDir(temp); + Path pluginDir = metaDir.resolve("fake"); + Files.createDirectory(pluginDir); Files.createFile(pluginDir.resolve("fake.yml")); String pluginZip = writeZip(pluginDir, "elasticsearch").toUri().toURL().toString(); NoSuchFileException e = expectThrows(NoSuchFileException.class, () -> installPlugin(pluginZip, env.v1())); assertTrue(e.getMessage(), e.getMessage().contains("plugin-descriptor.properties")); assertInstallCleaned(env.v2()); + + String metaZip = createMetaPluginUrl("my_plugins", metaDir); + e = expectThrows(NoSuchFileException.class, () -> installPlugin(metaZip, env.v1())); + assertTrue(e.getMessage(), e.getMessage().contains("plugin-descriptor.properties")); + assertInstallCleaned(env.v2()); } public void testMissingDirectory() throws Exception { @@ -651,6 +811,16 @@ public void testMissingDirectory() throws Exception { assertInstallCleaned(env.v2()); } + public void testMissingDirectoryMeta() throws Exception { + Tuple env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); + Files.createFile(pluginDir.resolve(MetaPluginInfo.ES_META_PLUGIN_PROPERTIES)); + String pluginZip = writeZip(pluginDir, null).toUri().toURL().toString(); + UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1())); + assertTrue(e.getMessage(), e.getMessage().contains("`elasticsearch` directory is missing in the plugin zip")); + assertInstallCleaned(env.v2()); + } + public void testZipRelativeOutsideEntryName() throws Exception { Tuple env = createEnv(fs, temp); Path zip = createTempDir().resolve("broken.zip"); @@ -748,6 +918,29 @@ public void testPluginAlreadyInstalled() throws Exception { "if you need to update the plugin, uninstall it first using command 'remove fake'")); } + public void testMetaPluginAlreadyInstalled() throws Exception { + Tuple env = createEnv(fs, temp); + { + // install fake plugin + Path pluginDir = createPluginDir(temp); + String pluginZip = createPluginUrl("fake", pluginDir); + installPlugin(pluginZip, env.v1()); + } + + Path pluginDir = createPluginDir(temp); + Files.createDirectory(pluginDir.resolve("fake")); + writePlugin("fake", pluginDir.resolve("fake"), false); + Files.createDirectory(pluginDir.resolve("other")); + writePlugin("other", pluginDir.resolve("other"), false); + String metaZip = createMetaPluginUrl("meta", pluginDir); + final UserException e = expectThrows(UserException.class, + () -> installPlugin(metaZip, env.v1(), randomFrom(skipJarHellCommand, defaultCommand))); + assertThat( + e.getMessage(), + equalTo("plugin directory [" + env.v2().pluginsFile().resolve("fake") + "] already exists; " + + "if you need to update the plugin, uninstall it first using command 'remove fake'")); + } + private void installPlugin(MockTerminal terminal, boolean isBatch) throws Exception { Tuple env = createEnv(fs, temp); Path pluginDir = createPluginDir(temp); @@ -791,7 +984,7 @@ String getStagingHash() { return stagingHash; } @Override - void jarHellCheck(PluginInfo info, Path candidate, Path pluginsDir, Path modulesDir) throws Exception { + void jarHellCheck(PluginInfo candidateInfo, Path candidate, Path pluginsDir, Path modulesDir) throws Exception { // no jarhell check } }; @@ -951,6 +1144,17 @@ public void testKeystoreRequiredCreated() throws Exception { assertTrue(Files.exists(KeyStoreWrapper.keystorePath(env.v2().configFile()))); } + public void testKeystoreRequiredCreatedWithMetaPlugin() throws Exception { + Tuple env = createEnv(fs, temp); + Path metaDir = createPluginDir(temp); + Path pluginDir = metaDir.resolve("fake"); + Files.createDirectory(pluginDir); + writePlugin("fake", pluginDir, false, "requires.keystore", "true"); + String metaZip = createMetaPluginUrl("my_plugins", metaDir); + MockTerminal terminal = installPlugin(metaZip, env.v1()); + assertTrue(Files.exists(KeyStoreWrapper.keystorePath(env.v2().configFile()))); + } + private Function checksum(final MessageDigest digest) { return checksumAndString(digest, ""); } diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java index 4a243daf2ba70..372a4cae8f263 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java @@ -33,7 +33,6 @@ import org.elasticsearch.Version; import org.elasticsearch.cli.ExitCodes; import org.elasticsearch.cli.MockTerminal; -import org.elasticsearch.cli.Terminal; import org.elasticsearch.cli.UserException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; @@ -94,18 +93,39 @@ private static void buildFakePlugin( final String description, final String name, final String classname) throws IOException { - buildFakePlugin(env, description, name, classname, false, false); + buildFakePlugin(env, null, description, name, classname, false, false); } private static void buildFakePlugin( final Environment env, + final String metaPlugin, + final String description, + final String name, + final String classname) throws IOException { + buildFakePlugin(env, metaPlugin, description, name, classname, false, false); + } + + private static void buildFakePlugin( + final Environment env, + final String description, + final String name, + final String classname, + final boolean hasNativeController, + final boolean requiresKeystore) throws IOException { + buildFakePlugin(env, null, description, name, classname, hasNativeController, requiresKeystore); + } + + private static void buildFakePlugin( + final Environment env, + final String metaPlugin, final String description, final String name, final String classname, final boolean hasNativeController, final boolean requiresKeystore) throws IOException { - PluginTestUtil.writeProperties( - env.pluginsFile().resolve(name), + Path dest = metaPlugin != null ? env.pluginsFile().resolve(metaPlugin) : env.pluginsFile(); + PluginTestUtil.writePluginProperties( + dest.resolve(name), "description", description, "name", name, "version", "1.0", @@ -116,6 +136,16 @@ private static void buildFakePlugin( "requires.keystore", Boolean.toString(requiresKeystore)); } + private static void buildFakeMetaPlugin( + final Environment env, + final String description, + final String name) throws IOException { + PluginTestUtil.writeMetaPluginProperties( + env.pluginsFile().resolve(name), + "description", description, + "name", name); + } + public void testPluginsDirMissing() throws Exception { Files.delete(env.pluginsFile()); IOException e = expectThrows(IOException.class, () -> listPlugins(home)); @@ -140,6 +170,16 @@ public void testTwoPlugins() throws Exception { assertEquals(buildMultiline("fake1", "fake2"), terminal.getOutput()); } + public void testMetaPlugin() throws Exception { + buildFakeMetaPlugin(env, "fake meta desc", "meta_plugin"); + buildFakePlugin(env, "meta_plugin", "fake desc", "fake1", "org.fake1"); + buildFakePlugin(env, "meta_plugin", "fake desc 2", "fake2", "org.fake2"); + buildFakePlugin(env, "fake desc 3", "fake3", "org.fake3"); + buildFakePlugin(env, "fake desc 4", "fake4", "org.fake4"); + MockTerminal terminal = listPlugins(home); + assertEquals(buildMultiline("fake3", "fake4", "meta_plugin", "\tfake1", "\tfake2"), terminal.getOutput()); + } + public void testPluginWithVerbose() throws Exception { buildFakePlugin(env, "fake desc", "fake_plugin", "org.fake"); String[] params = { "-v" }; @@ -226,6 +266,37 @@ public void testPluginWithVerboseMultiplePlugins() throws Exception { terminal.getOutput()); } + public void testPluginWithVerboseMetaPlugins() throws Exception { + buildFakeMetaPlugin(env, "fake meta desc", "meta_plugin"); + buildFakePlugin(env, "meta_plugin", "fake desc 1", "fake_plugin1", "org.fake"); + buildFakePlugin(env, "meta_plugin", "fake desc 2", "fake_plugin2", "org.fake2"); + String[] params = { "-v" }; + MockTerminal terminal = listPlugins(home, params); + assertEquals( + buildMultiline( + "Plugins directory: " + env.pluginsFile(), + "meta_plugin", + "\tfake_plugin1", + "\t- Plugin information:", + "\tName: fake_plugin1", + "\tDescription: fake desc 1", + "\tVersion: 1.0", + "\tNative Controller: false", + "\tRequires Keystore: false", + "\tExtended Plugins: []", + "\t * Classname: org.fake", + "\tfake_plugin2", + "\t- Plugin information:", + "\tName: fake_plugin2", + "\tDescription: fake desc 2", + "\tVersion: 1.0", + "\tNative Controller: false", + "\tRequires Keystore: false", + "\tExtended Plugins: []", + "\t * Classname: org.fake2"), + terminal.getOutput()); + } + public void testPluginWithoutVerboseMultiplePlugins() throws Exception { buildFakePlugin(env, "fake desc 1", "fake_plugin1", "org.fake"); buildFakePlugin(env, "fake desc 2", "fake_plugin2", "org.fake2"); @@ -243,7 +314,7 @@ public void testPluginWithoutDescriptorFile() throws Exception{ public void testPluginWithWrongDescriptorFile() throws Exception{ final Path pluginDir = env.pluginsFile().resolve("fake1"); - PluginTestUtil.writeProperties(pluginDir, "description", "fake desc"); + PluginTestUtil.writePluginProperties(pluginDir, "description", "fake desc"); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, () -> listPlugins(home)); @@ -253,8 +324,21 @@ public void testPluginWithWrongDescriptorFile() throws Exception{ e.getMessage()); } + public void testMetaPluginWithWrongDescriptorFile() throws Exception{ + buildFakeMetaPlugin(env, "fake meta desc", "meta_plugin"); + final Path pluginDir = env.pluginsFile().resolve("meta_plugin").resolve("fake_plugin1"); + PluginTestUtil.writePluginProperties(pluginDir, "description", "fake desc"); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> listPlugins(home)); + final Path descriptorPath = pluginDir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES); + assertEquals( + "property [name] is missing in [" + descriptorPath.toString() + "]", + e.getMessage()); + } + public void testExistingIncompatiblePlugin() throws Exception { - PluginTestUtil.writeProperties(env.pluginsFile().resolve("fake_plugin1"), + PluginTestUtil.writePluginProperties(env.pluginsFile().resolve("fake_plugin1"), "description", "fake desc 1", "name", "fake_plugin1", "version", "1.0", @@ -278,4 +362,30 @@ public void testExistingIncompatiblePlugin() throws Exception { assertEquals("fake_plugin1\nfake_plugin2\n", terminal.getOutput()); } + public void testExistingIncompatibleMetaPlugin() throws Exception { + buildFakeMetaPlugin(env, "fake meta desc", "meta_plugin"); + PluginTestUtil.writePluginProperties(env.pluginsFile().resolve("meta_plugin").resolve("fake_plugin1"), + "description", "fake desc 1", + "name", "fake_plugin1", + "version", "1.0", + "elasticsearch.version", Version.fromString("1.0.0").toString(), + "java.version", System.getProperty("java.specification.version"), + "classname", "org.fake1"); + buildFakePlugin(env, "fake desc 2", "fake_plugin2", "org.fake2"); + + MockTerminal terminal = listPlugins(home); + final String message = String.format(Locale.ROOT, + "plugin [%s] is incompatible with version [%s]; was designed for version [%s]", + "fake_plugin1", + Version.CURRENT.toString(), + "1.0.0"); + assertEquals( + "fake_plugin2\nmeta_plugin\n\tfake_plugin1\n" + "WARNING: " + message + "\n", + terminal.getOutput()); + + String[] params = {"-s"}; + terminal = listPlugins(home, params); + assertEquals("fake_plugin2\nmeta_plugin\n\tfake_plugin1\n", terminal.getOutput()); + } + } diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java index c128a245cd2ec..d15e0e642c836 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java @@ -79,8 +79,12 @@ public void setUp() throws Exception { } void createPlugin(String name) throws Exception { - PluginTestUtil.writeProperties( - env.pluginsFile().resolve(name), + createPlugin(env.pluginsFile(), name); + } + + void createPlugin(Path path, String name) throws Exception { + PluginTestUtil.writePluginProperties( + path.resolve(name), "description", "dummy", "name", name, "version", "1.0", @@ -89,6 +93,16 @@ void createPlugin(String name) throws Exception { "classname", "SomeClass"); } + void createMetaPlugin(String name, String... plugins) throws Exception { + PluginTestUtil.writeMetaPluginProperties( + env.pluginsFile().resolve(name), + "description", "dummy", + "name", name); + for (String plugin : plugins) { + createPlugin(env.pluginsFile().resolve(name), plugin); + } + } + static MockTerminal removePlugin(String name, Path home, boolean purge) throws Exception { Environment env = TestEnvironment.newEnvironment(Settings.builder().put("path.home", home).build()); MockTerminal terminal = new MockTerminal(); @@ -123,6 +137,19 @@ public void testBasic() throws Exception { assertRemoveCleaned(env); } + public void testBasicMeta() throws Exception { + createMetaPlugin("meta", "fake1"); + createPlugin("other"); + removePlugin("meta", home, randomBoolean()); + assertFalse(Files.exists(env.pluginsFile().resolve("meta"))); + assertTrue(Files.exists(env.pluginsFile().resolve("other"))); + assertRemoveCleaned(env); + + UserException exc = + expectThrows(UserException.class, () -> removePlugin("fake1", home, randomBoolean())); + assertThat(exc.getMessage(), containsString("plugin [fake1] not found")); + } + public void testBin() throws Exception { createPlugin("fake"); Path binDir = env.binFile().resolve("fake"); diff --git a/docs/plugins/authors.asciidoc b/docs/plugins/authors.asciidoc index 8dc06d1433ad1..fd408e358fe74 100644 --- a/docs/plugins/authors.asciidoc +++ b/docs/plugins/authors.asciidoc @@ -1,10 +1,18 @@ [[plugin-authors]] == Help for plugin authors +:plugin-properties-files: {docdir}/../../buildSrc/src/main/resources + The Elasticsearch repository contains examples of: * a https://github.com/elastic/elasticsearch/tree/master/plugins/jvm-example[Java plugin] which contains Java code. +* a https://github.com/elastic/elasticsearch/tree/master/plugins/examples/rescore[Java plugin] + which contains a rescore plugin. +* a https://github.com/elastic/elasticsearch/tree/master/plugins/examples/script-expert-scoring[Java plugin] + which contains a script plugin. +* a https://github.com/elastic/elasticsearch/tree/master/plugins/examples/meta-plugin[Java plugin] + which contains a meta plugin. These examples provide the bare bones needed to get started. For more information about how to write a plugin, we recommend looking at the plugins @@ -18,10 +26,13 @@ All plugin files must be contained in a directory called `elasticsearch`. [float] === Plugin descriptor file -All plugins must contain a file called `plugin-descriptor.properties` in the folder named `elasticsearch`. The format -for this file is described in detail here: +All plugins must contain a file called `plugin-descriptor.properties` in the folder named `elasticsearch`. +The format for this file is described in detail in this example: -https://github.com/elastic/elasticsearch/blob/master/buildSrc/src/main/resources/plugin-descriptor.properties[`/buildSrc/src/main/resources/plugin-descriptor.properties`]. +["source","properties",subs="attributes"] +-------------------------------------------------- +include-tagged::{plugin-properties-files}/plugin-descriptor.properties[plugin-descriptor.properties] +-------------------------------------------------- Either fill in this template yourself or, if you are using Elasticsearch's Gradle build system, you can fill in the necessary values in the `build.gradle` file for your plugin. @@ -112,3 +123,19 @@ AccessController.doPrivileged( See http://www.oracle.com/technetwork/java/seccodeguide-139067.html[Secure Coding Guidelines for Java SE] for more information. + +[float] +=== Meta Plugin + +It is also possible to bundle multiple plugins into a meta plugin. +A directory for each sub-plugin must be contained in a directory called `elasticsearch. +The meta plugin must also contain a file called `meta-plugin-descriptor.properties` in the directory named +`elasticsearch`. +The format for this file is described in detail in this example: + +["source","properties",subs="attributes"] +-------------------------------------------------- +include-tagged::{plugin-properties-files}/meta-plugin-descriptor.properties[meta-plugin-descriptor.properties] +-------------------------------------------------- + +A meta plugin can be installed/removed like a normal plugin with the `bin/elasticsearch-plugin` command. diff --git a/plugins/examples/meta-plugin/build.gradle b/plugins/examples/meta-plugin/build.gradle new file mode 100644 index 0000000000000..3674837b0b2f9 --- /dev/null +++ b/plugins/examples/meta-plugin/build.gradle @@ -0,0 +1,56 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// A meta plugin packaging example that bundles multiple plugins in a single zip. +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' + +File plugins = new File(buildDir, 'plugins-unzip') +subprojects { + // unzip the subproject plugins + task unzip(type:Copy, dependsOn: "${project.path}:bundlePlugin") { + File dest = new File(plugins, project.name) + from { zipTree(project(project.path).bundlePlugin.outputs.files.singleFile) } + eachFile { f -> f.path = f.path.replaceFirst('elasticsearch', '') } + into dest + } +} + +// Build the meta plugin zip from the subproject plugins (unzipped) +task buildZip(type:Zip) { + subprojects.each { dependsOn("${it.name}:unzip") } + from plugins + from 'src/main/resources/meta-plugin-descriptor.properties' + into 'elasticsearch' + includeEmptyDirs false +} + +integTestCluster { + dependsOn buildZip + + // This is important, so that all the modules are available too. + // There are index templates that use token filters that are in analysis-module and + // processors are being used that are in ingest-common module. + distribution = 'zip' + + // Install the meta plugin before start. + setupCommand 'installMetaPlugin', + 'bin/elasticsearch-plugin', 'install', 'file:' + buildZip.archivePath +} +check.dependsOn integTest diff --git a/plugins/examples/meta-plugin/dummy-plugin1/build.gradle b/plugins/examples/meta-plugin/dummy-plugin1/build.gradle new file mode 100644 index 0000000000000..5a02e993f8c25 --- /dev/null +++ b/plugins/examples/meta-plugin/dummy-plugin1/build.gradle @@ -0,0 +1,29 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +apply plugin: 'elasticsearch.esplugin' + +esplugin { + name 'dummy-plugin1' + description 'A dummy plugin' + classname 'org.elasticsearch.example.DummyPlugin1' +} + +test.enabled = false +integTestRunner.enabled = false \ No newline at end of file diff --git a/plugins/examples/meta-plugin/dummy-plugin1/src/main/java/org/elasticsearch/example/DummyPlugin1.java b/plugins/examples/meta-plugin/dummy-plugin1/src/main/java/org/elasticsearch/example/DummyPlugin1.java new file mode 100644 index 0000000000000..65102dbc2e337 --- /dev/null +++ b/plugins/examples/meta-plugin/dummy-plugin1/src/main/java/org/elasticsearch/example/DummyPlugin1.java @@ -0,0 +1,29 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.example; + +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SearchPlugin; + +import java.util.List; + +import static java.util.Collections.singletonList; + +public class DummyPlugin1 extends Plugin {} diff --git a/plugins/examples/meta-plugin/dummy-plugin2/build.gradle b/plugins/examples/meta-plugin/dummy-plugin2/build.gradle new file mode 100644 index 0000000000000..d90983adfed0c --- /dev/null +++ b/plugins/examples/meta-plugin/dummy-plugin2/build.gradle @@ -0,0 +1,29 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +apply plugin: 'elasticsearch.esplugin' + +esplugin { + name 'dummy-plugin2' + description 'Another dummy plugin' + classname 'org.elasticsearch.example.DummyPlugin2' +} + +test.enabled = false +integTestRunner.enabled = false \ No newline at end of file diff --git a/plugins/examples/meta-plugin/dummy-plugin2/src/main/java/org/elasticsearch/example/DummyPlugin2.java b/plugins/examples/meta-plugin/dummy-plugin2/src/main/java/org/elasticsearch/example/DummyPlugin2.java new file mode 100644 index 0000000000000..2d74d7603d15f --- /dev/null +++ b/plugins/examples/meta-plugin/dummy-plugin2/src/main/java/org/elasticsearch/example/DummyPlugin2.java @@ -0,0 +1,29 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.example; + +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SearchPlugin; + +import java.util.List; + +import static java.util.Collections.singletonList; + +public class DummyPlugin2 extends Plugin {} diff --git a/plugins/examples/meta-plugin/src/main/resources/meta-plugin-descriptor.properties b/plugins/examples/meta-plugin/src/main/resources/meta-plugin-descriptor.properties new file mode 100644 index 0000000000000..1fd5a86b95a54 --- /dev/null +++ b/plugins/examples/meta-plugin/src/main/resources/meta-plugin-descriptor.properties @@ -0,0 +1,4 @@ +# The name of the meta plugin +name=my_meta_plugin +# The description of the meta plugin +description=A meta plugin example \ No newline at end of file diff --git a/plugins/examples/meta-plugin/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java b/plugins/examples/meta-plugin/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java new file mode 100644 index 0000000000000..d1f9e6b73703e --- /dev/null +++ b/plugins/examples/meta-plugin/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.smoketest; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; + +public class SmokeTestPluginsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + public SmokeTestPluginsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } +} + diff --git a/plugins/examples/meta-plugin/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yml b/plugins/examples/meta-plugin/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yml new file mode 100644 index 0000000000000..011a278ed8949 --- /dev/null +++ b/plugins/examples/meta-plugin/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_basic.yml @@ -0,0 +1,14 @@ +# Integration tests for testing meta plugins +# +"Check meta plugin install": + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + - do: + nodes.info: {} + + - match: { nodes.$master.plugins.0.name: dummy-plugin1 } + - match: { nodes.$master.plugins.1.name: dummy-plugin2 } diff --git a/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java b/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java index d9d4ab5c3aca9..e4e603dff9503 100644 --- a/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java +++ b/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java @@ -78,7 +78,7 @@ public void testNoControllerSpawn() throws IOException, InterruptedException { // This plugin will NOT have a controller daemon Path plugin = environment.pluginsFile().resolve("a_plugin"); Files.createDirectories(plugin); - PluginTestUtil.writeProperties( + PluginTestUtil.writePluginProperties( plugin, "description", "a_plugin", "version", Version.CURRENT.toString(), @@ -114,7 +114,7 @@ public void testControllerSpawn() throws IOException, InterruptedException { // this plugin will have a controller daemon Path plugin = environment.pluginsFile().resolve("test_plugin"); Files.createDirectories(plugin); - PluginTestUtil.writeProperties( + PluginTestUtil.writePluginProperties( plugin, "description", "test_plugin", "version", Version.CURRENT.toString(), @@ -129,7 +129,7 @@ public void testControllerSpawn() throws IOException, InterruptedException { // this plugin will not have a controller daemon Path otherPlugin = environment.pluginsFile().resolve("other_plugin"); Files.createDirectories(otherPlugin); - PluginTestUtil.writeProperties( + PluginTestUtil.writePluginProperties( otherPlugin, "description", "other_plugin", "version", Version.CURRENT.toString(), @@ -163,6 +163,84 @@ public void testControllerSpawn() throws IOException, InterruptedException { } } + /** + * Two plugins in a meta plugin - one with a controller daemon and one without. + */ + public void testControllerSpawnMetaPlugin() throws IOException, InterruptedException { + /* + * On Windows you can not directly run a batch file - you have to run cmd.exe with the batch + * file as an argument and that's out of the remit of the controller daemon process spawner. + */ + assumeFalse("This test does not work on Windows", Constants.WINDOWS); + + Path esHome = createTempDir().resolve("esHome"); + Settings.Builder settingsBuilder = Settings.builder(); + settingsBuilder.put(Environment.PATH_HOME_SETTING.getKey(), esHome.toString()); + Settings settings = settingsBuilder.build(); + + Environment environment = TestEnvironment.newEnvironment(settings); + + Path metaPlugin = environment.pluginsFile().resolve("meta_plugin"); + Files.createDirectories(metaPlugin); + PluginTestUtil.writeMetaPluginProperties( + metaPlugin, + "description", "test_plugin", + "name", "meta_plugin", + "plugins", "test_plugin,other_plugin"); + + // this plugin will have a controller daemon + Path plugin = metaPlugin.resolve("test_plugin"); + + Files.createDirectories(plugin); + PluginTestUtil.writePluginProperties( + plugin, + "description", "test_plugin", + "version", Version.CURRENT.toString(), + "elasticsearch.version", Version.CURRENT.toString(), + "name", "test_plugin", + "java.version", "1.8", + "classname", "TestPlugin", + "has.native.controller", "true"); + Path controllerProgram = Platforms.nativeControllerPath(plugin); + createControllerProgram(controllerProgram); + + // this plugin will not have a controller daemon + Path otherPlugin = metaPlugin.resolve("other_plugin"); + Files.createDirectories(otherPlugin); + PluginTestUtil.writePluginProperties( + otherPlugin, + "description", "other_plugin", + "version", Version.CURRENT.toString(), + "elasticsearch.version", Version.CURRENT.toString(), + "name", "other_plugin", + "java.version", "1.8", + "classname", "OtherPlugin", + "has.native.controller", "false"); + + Spawner spawner = new Spawner(); + spawner.spawnNativePluginControllers(environment); + + List processes = spawner.getProcesses(); + /* + * As there should only be a reference in the list for the plugin that had the controller + * daemon, we expect one here. + */ + assertThat(processes, hasSize(1)); + Process process = processes.get(0); + final InputStreamReader in = + new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8); + try (BufferedReader stdoutReader = new BufferedReader(in)) { + String line = stdoutReader.readLine(); + assertEquals("I am alive", line); + spawner.close(); + /* + * Fail if the process does not die within one second; usually it will be even quicker + * but it depends on OS scheduling. + */ + assertTrue(process.waitFor(1, TimeUnit.SECONDS)); + } + } + public void testControllerSpawnWithIncorrectDescriptor() throws IOException { // this plugin will have a controller daemon Path esHome = createTempDir().resolve("esHome"); @@ -174,7 +252,7 @@ public void testControllerSpawnWithIncorrectDescriptor() throws IOException { Path plugin = environment.pluginsFile().resolve("test_plugin"); Files.createDirectories(plugin); - PluginTestUtil.writeProperties( + PluginTestUtil.writePluginProperties( plugin, "description", "test_plugin", "version", Version.CURRENT.toString(), diff --git a/settings.gradle b/settings.gradle index cd6d2976e0272..d4b280c346b3b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -90,15 +90,6 @@ List projects = [ 'qa:query-builder-bwc' ] -File examplePluginsDir = new File(rootProject.projectDir, 'plugins/examples') -List examplePlugins = [] -for (File example : examplePluginsDir.listFiles()) { - if (example.isDirectory() == false) continue; - if (example.name.startsWith('build') || example.name.startsWith('.')) continue; - projects.add("example-plugins:${example.name}".toString()) - examplePlugins.add(example.name) -} - projects.add("libs") File libsDir = new File(rootProject.projectDir, 'libs') for (File libDir : new File(rootProject.projectDir, 'libs').listFiles()) { @@ -124,11 +115,6 @@ if (isEclipse) { include projects.toArray(new String[0]) project(':build-tools').projectDir = new File(rootProject.projectDir, 'buildSrc') -project(':example-plugins').projectDir = new File(rootProject.projectDir, 'plugins/examples') - -for (String example : examplePlugins) { - project(":example-plugins:${example}").projectDir = new File(rootProject.projectDir, "plugins/examples/${example}") -} /* The BWC snapshot projects share the same build directory and build file, * but apply to different backwards compatibility branches. */ @@ -170,7 +156,7 @@ void addSubProjects(String path, File dir, List projects, List b } // TODO do we want to assert that there's nothing else in the bwc directory? } else { - if (path.isEmpty()) { + if (path.isEmpty() || path.startsWith(':example-plugins')) { project(projectName).projectDir = dir } for (File subdir : dir.listFiles()) { @@ -179,6 +165,15 @@ void addSubProjects(String path, File dir, List projects, List b } } +// include example plugins +File examplePluginsDir = new File(rootProject.projectDir, 'plugins/examples') +for (File example : examplePluginsDir.listFiles()) { + if (example.isDirectory() == false) continue; + if (example.name.startsWith('build') || example.name.startsWith('.')) continue; + addSubProjects(':example-plugins', example, projects, []) +} +project(':example-plugins').projectDir = new File(rootProject.projectDir, 'plugins/examples') + // look for extra plugins for elasticsearch File extraProjects = new File(rootProject.projectDir.parentFile, "${dirName}-extra") if (extraProjects.exists()) { diff --git a/test/framework/src/main/java/org/elasticsearch/plugins/PluginTestUtil.java b/test/framework/src/main/java/org/elasticsearch/plugins/PluginTestUtil.java index 10f4de2482a94..5a92c99d61870 100644 --- a/test/framework/src/main/java/org/elasticsearch/plugins/PluginTestUtil.java +++ b/test/framework/src/main/java/org/elasticsearch/plugins/PluginTestUtil.java @@ -27,12 +27,18 @@ /** Utility methods for testing plugins */ public class PluginTestUtil { - + public static void writeMetaPluginProperties(Path pluginDir, String... stringProps) throws IOException { + writeProperties(pluginDir.resolve(MetaPluginInfo.ES_META_PLUGIN_PROPERTIES), stringProps); + } + + public static void writePluginProperties(Path pluginDir, String... stringProps) throws IOException { + writeProperties(pluginDir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES), stringProps); + } + /** convenience method to write a plugin properties file */ - public static void writeProperties(Path pluginDir, String... stringProps) throws IOException { + private static void writeProperties(Path propertiesFile, String... stringProps) throws IOException { assert stringProps.length % 2 == 0; - Files.createDirectories(pluginDir); - Path propertiesFile = pluginDir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES); + Files.createDirectories(propertiesFile.getParent()); Properties properties = new Properties(); for (int i = 0; i < stringProps.length; i += 2) { properties.put(stringProps[i], stringProps[i + 1]); From 2c24ac742634e8c0af849f74d885e0efd52f3c0f Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 9 Jan 2018 12:51:50 -0500 Subject: [PATCH 4/4] Set watermarks in single-node test cases We set the watermarks to low values in other test cases to prevent test failures on nodes with low disk space (if the disk space is too low, the test will fail anyway but we should not prematurely fail). This commit sets the watermarks in the single-node test cases to avoid test failures in such situations. Relates #28134 --- .../java/org/elasticsearch/test/ESSingleNodeTestCase.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java index 0363a938dd18f..d6c4942ab6084 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings; import org.elasticsearch.common.Priority; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; @@ -176,6 +177,10 @@ private Node newNode() { .put("transport.type", getTestTransportType()) .put(Node.NODE_DATA_SETTING.getKey(), true) .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), random().nextLong()) + // default the watermarks low values to prevent tests from failing on nodes without enough disk space + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1b") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1b") + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "1b") .put(nodeSettings()) // allow test cases to provide their own settings or override these .build(); Collection> plugins = getPlugins();