diff --git a/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/configuration/AppProperties.java b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/configuration/AppProperties.java index 0fb2b34a..f473f284 100644 --- a/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/configuration/AppProperties.java +++ b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/configuration/AppProperties.java @@ -40,6 +40,9 @@ public class AppProperties { @Value("${app.uploadPath}") private String uploadPath; + @Value("${app.cachePath}") + private String cachePath; + @Value("${app.key}") private String appKey; @@ -82,6 +85,10 @@ public String getUploadPath() { return uploadPath; } + public String getCachePath() { + return cachePath; + } + public String getAppKey() { return appKey; } @@ -123,6 +130,7 @@ public String toString() { return "AppProperties{" + "name='" + name + '\'' + ", uploadPath='" + uploadPath + '\'' + + ", cachePath='" + cachePath + '\'' + ", appKey='XXXXXX'" + ", maxConcurrentWebSocketConsumers=" + maxConcurrentWebSocketConsumers + ", consumerIdPrefix='" + consumerIdPrefix + '\'' diff --git a/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/configuration/PluginConfig.java b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/configuration/PluginConfig.java index 2c4cdf48..5bf3179e 100644 --- a/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/configuration/PluginConfig.java +++ b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/configuration/PluginConfig.java @@ -31,6 +31,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sourcelab.kafka.webview.ui.manager.encryption.SecretManager; +import org.sourcelab.kafka.webview.ui.manager.file.FileManager; +import org.sourcelab.kafka.webview.ui.manager.file.FileStorageService; +import org.sourcelab.kafka.webview.ui.manager.file.FileType; +import org.sourcelab.kafka.webview.ui.manager.file.LocalDiskStorage; import org.sourcelab.kafka.webview.ui.manager.kafka.KafkaAdminFactory; import org.sourcelab.kafka.webview.ui.manager.kafka.KafkaClientConfigUtil; import org.sourcelab.kafka.webview.ui.manager.kafka.KafkaConsumerFactory; @@ -57,34 +61,32 @@ public class PluginConfig { /** * Upload manager, for handling uploads of Plugins and Keystores. - * @param appProperties Definition of app properties. + * @param fileManager for managing file persistence. * @return UploadManager for Plugins */ @Bean - public UploadManager getPluginUploadManager(final AppProperties appProperties) { - return new UploadManager(appProperties.getUploadPath()); + public UploadManager getPluginUploadManager(final FileManager fileManager) { + return new UploadManager(fileManager); } /** * PluginFactory for creating instances of Deserializers. - * @param appProperties Definition of app properties. + * @param fileManager for managing file persistence. * @return PluginFactory for Deserializers. */ - @Bean - public PluginFactory getDeserializerPluginFactory(final AppProperties appProperties) { - final String jarDirectory = appProperties.getUploadPath() + "/deserializers"; - return new PluginFactory<>(jarDirectory, Deserializer.class); + @Bean("PluginFactoryForDeserializer") + public PluginFactory getDeserializerPluginFactory(final FileManager fileManager) { + return new PluginFactory<>(FileType.DESERIALIZER, Deserializer.class, fileManager); } /** * PluginFactory for creating instances of Record Filters. - * @param appProperties Definition of app properties. + * @param fileManager for managing file persistence. * @return PluginFactory for Record Filters. */ - @Bean - public PluginFactory getRecordFilterPluginFactory(final AppProperties appProperties) { - final String jarDirectory = appProperties.getUploadPath() + "/filters"; - return new PluginFactory<>(jarDirectory, RecordFilter.class); + @Bean("PluginFactoryForRecordFilter") + public PluginFactory getRecordFilterPluginFactory(final FileManager fileManager) { + return new PluginFactory<>(FileType.FILTER, RecordFilter.class, fileManager); } /** @@ -103,7 +105,13 @@ public SecretManager getSecretManager(final AppProperties appProperties) { * @return Web Kafka Consumer Factory instance. */ @Bean - public WebKafkaConsumerFactory getWebKafkaConsumerFactory(final AppProperties appProperties, final KafkaClientConfigUtil configUtil) { + public WebKafkaConsumerFactory getWebKafkaConsumerFactory( + final AppProperties appProperties, + final KafkaClientConfigUtil configUtil, + final PluginFactory deserializerPluginFactory, + final PluginFactory recordFilterPluginFactory, + final SecretManager secretManager + ) { final ExecutorService executorService; // If we have multi-threaded consumer option enabled @@ -123,9 +131,9 @@ public WebKafkaConsumerFactory getWebKafkaConsumerFactory(final AppProperties ap } return new WebKafkaConsumerFactory( - getDeserializerPluginFactory(appProperties), - getRecordFilterPluginFactory(appProperties), - getSecretManager(appProperties), + deserializerPluginFactory, + recordFilterPluginFactory, + secretManager, getKafkaConsumerFactory(configUtil), executorService ); @@ -133,14 +141,16 @@ public WebKafkaConsumerFactory getWebKafkaConsumerFactory(final AppProperties ap /** * For creating Kafka operational consumers. - * @param appProperties Definition of app properties. * @param configUtil Utility for configuring kafka clients. * @return Web Kafka Operations Client Factory instance. */ @Bean - public KafkaOperationsFactory getKafkaOperationsFactory(final AppProperties appProperties, final KafkaClientConfigUtil configUtil) { + public KafkaOperationsFactory getKafkaOperationsFactory( + final KafkaClientConfigUtil configUtil, + final SecretManager secretManager + ) { return new KafkaOperationsFactory( - getSecretManager(appProperties), + secretManager, getKafkaAdminFactory(configUtil) ); } @@ -202,4 +212,21 @@ public KafkaClientConfigUtil getKafkaClientConfigUtil(final AppProperties appPro public SaslUtility getSaslUtility(final SecretManager secretManager) { return new SaslUtility(secretManager); } + + @Bean + public FileStorageService fileStorageService(final AppProperties appProperties) { + return new LocalDiskStorage(appProperties.getUploadPath()); + } + + /** + * Utility for managing file storage operations. + * @param fileStorageService Where to back filestorage. + * @param appProperties Definition of app properties. + * @return FileManager instance. + */ + @Bean + public FileManager fileManager(final FileStorageService fileStorageService, final AppProperties appProperties) { + final LocalDiskStorage localCacheStorage = new LocalDiskStorage(appProperties.getCachePath()); + return new FileManager(fileStorageService, localCacheStorage); + } } diff --git a/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/controller/configuration/cluster/ClusterConfigController.java b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/controller/configuration/cluster/ClusterConfigController.java index 2db235fe..5d01e1b7 100644 --- a/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/controller/configuration/cluster/ClusterConfigController.java +++ b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/controller/configuration/cluster/ClusterConfigController.java @@ -238,7 +238,12 @@ public String clusterUpdate( if (!clusterForm.exists() || (clusterForm.getTrustStoreFile() != null && !clusterForm.getTrustStoreFile().isEmpty())) { // Delete previous trust store if updating if (cluster.getTrustStoreFile() != null) { - uploadManager.deleteKeyStore(cluster.getTrustStoreFile()); + try { + uploadManager.deleteKeyStore(cluster.getTrustStoreFile()); + } catch (final IOException exception) { + // TODO handle + throw new RuntimeException(exception.getMessage(), exception); + } cluster.setTrustStoreFile(null); cluster.setTrustStorePassword(null); } @@ -257,7 +262,7 @@ public String clusterUpdate( // Persist in model. cluster.setTrustStoreFile(filename); cluster.setTrustStorePassword(encrypted); - } catch (IOException exception) { + } catch (final IOException exception) { // TODO handle throw new RuntimeException(exception.getMessage(), exception); } @@ -266,7 +271,12 @@ public String clusterUpdate( if (!clusterForm.exists() || (clusterForm.getKeyStoreFile() != null && !clusterForm.getKeyStoreFile().isEmpty())) { // Delete previous key store if updating, or if SASL is enabled. if (clusterForm.getSasl() || cluster.getKeyStoreFile() != null) { - uploadManager.deleteKeyStore(cluster.getKeyStoreFile()); + try { + uploadManager.deleteKeyStore(cluster.getKeyStoreFile()); + } catch (final IOException exception) { + // TODO handle + throw new RuntimeException(exception.getMessage(), exception); + } cluster.setKeyStoreFile(null); cluster.setKeyStorePassword(null); } @@ -298,8 +308,13 @@ public String clusterUpdate( cluster.setSslEnabled(false); // Remove from disk - uploadManager.deleteKeyStore(cluster.getKeyStoreFile()); - uploadManager.deleteKeyStore(cluster.getTrustStoreFile()); + try { + uploadManager.deleteKeyStore(cluster.getKeyStoreFile()); + uploadManager.deleteKeyStore(cluster.getTrustStoreFile()); + } catch (final IOException exception) { + // TODO handle + throw new RuntimeException(exception.getMessage(), exception); + } // Null out fields cluster.setKeyStoreFile(null); @@ -376,11 +391,16 @@ public String deleteCluster(@PathVariable final Long id, final RedirectAttribute final Cluster cluster = clusterOptional.get(); // Delete KeyStores - if (cluster.getTrustStoreFile() != null) { - uploadManager.deleteKeyStore(cluster.getTrustStoreFile()); - } - if (cluster.getKeyStoreFile() != null) { - uploadManager.deleteKeyStore(cluster.getKeyStoreFile()); + try { + if (cluster.getTrustStoreFile() != null) { + uploadManager.deleteKeyStore(cluster.getTrustStoreFile()); + } + if (cluster.getKeyStoreFile() != null) { + uploadManager.deleteKeyStore(cluster.getKeyStoreFile()); + } + } catch (final IOException exception) { + // TODO handle + throw new RuntimeException(exception.getMessage(), exception); } // Delete it diff --git a/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/controller/configuration/filter/FilterConfigController.java b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/controller/configuration/filter/FilterConfigController.java index bac7be16..75f08ae5 100644 --- a/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/controller/configuration/filter/FilterConfigController.java +++ b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/controller/configuration/filter/FilterConfigController.java @@ -26,6 +26,8 @@ import org.sourcelab.kafka.webview.ui.controller.BaseController; import org.sourcelab.kafka.webview.ui.controller.configuration.filter.forms.FilterForm; +import org.sourcelab.kafka.webview.ui.manager.file.FileManager; +import org.sourcelab.kafka.webview.ui.manager.file.FileType; import org.sourcelab.kafka.webview.ui.manager.plugin.PluginFactory; import org.sourcelab.kafka.webview.ui.manager.plugin.UploadManager; import org.sourcelab.kafka.webview.ui.manager.plugin.exception.LoaderException; @@ -69,6 +71,9 @@ public class FilterConfigController extends BaseController { @Autowired private UploadManager uploadManager; + @Autowired + private FileManager fileManager; + @Autowired private PluginFactory recordFilterPluginFactory; @@ -218,7 +223,6 @@ public String update( // Persist jar on filesystem into temp location final String tmpJarLocation = uploadManager.handleFilterUpload(file, tmpFilename); - final String finalJarLocation = tmpJarLocation.substring(0, tmpJarLocation.lastIndexOf(".tmp")); // Attempt to load jar? final String filterOptionNames; @@ -230,7 +234,7 @@ public String update( filterOptionNames = filterOptions.stream().collect(Collectors.joining(",")); } catch (final LoaderException exception) { // Remove jar - Files.delete(new File(tmpJarLocation).toPath()); + fileManager.deleteFile(tmpJarLocation, FileType.FILTER); bindingResult.addError(new FieldError( "filterForm", "file", "", true, null, null, exception.getMessage()) @@ -239,10 +243,7 @@ public String update( } // If successful overwrite original jar - final Path tmpJarPath = new File(tmpJarLocation).toPath(); - final Path finalJarPath = new File(finalJarLocation).toPath(); - Files.deleteIfExists(finalJarPath); - Files.move(tmpJarPath, finalJarPath); + fileManager.moveFile(tmpFilename, filename, FileType.FILTER); // Set properties filter.setClasspath(filterForm.getClasspath()); @@ -289,7 +290,7 @@ public String delete(@PathVariable final Long id, final RedirectAttributes redir filterRepository.deleteById(id); // Delete jar from disk - Files.delete(recordFilterPluginFactory.getPathForJar(filter.getJar())); + fileManager.deleteFile(filter.getJar(), FileType.FILTER); redirectAttributes.addFlashAttribute("FlashMessage", FlashMessage.newSuccess("Deleted filter!")); } catch (IOException e) { e.printStackTrace(); diff --git a/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/controller/configuration/messageformat/MessageFormatController.java b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/controller/configuration/messageformat/MessageFormatController.java index 376bec2e..c0866de4 100644 --- a/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/controller/configuration/messageformat/MessageFormatController.java +++ b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/controller/configuration/messageformat/MessageFormatController.java @@ -29,6 +29,8 @@ import org.apache.kafka.common.serialization.Deserializer; import org.sourcelab.kafka.webview.ui.controller.BaseController; import org.sourcelab.kafka.webview.ui.controller.configuration.messageformat.forms.MessageFormatForm; +import org.sourcelab.kafka.webview.ui.manager.file.FileManager; +import org.sourcelab.kafka.webview.ui.manager.file.FileType; import org.sourcelab.kafka.webview.ui.manager.plugin.PluginFactory; import org.sourcelab.kafka.webview.ui.manager.plugin.UploadManager; import org.sourcelab.kafka.webview.ui.manager.plugin.exception.LoaderException; @@ -71,6 +73,9 @@ public class MessageFormatController extends BaseController { @Autowired private UploadManager uploadManager; + @Autowired + private FileManager fileManager; + @Autowired private PluginFactory deserializerLoader; @@ -229,7 +234,7 @@ public String create( deserializerLoader.checkPlugin(tempFilename, messageFormatForm.getClasspath()); } catch (final LoaderException exception) { // If we had issues, remove the temp location - Files.delete(Paths.get(jarPath)); + fileManager.deleteFile(jarPath, FileType.DESERIALIZER); // Add an error bindingResult.addError(new FieldError( @@ -242,13 +247,11 @@ public String create( // 1 - remove pre-existing jar if it exists if (messageFormat.getJar() != null && !messageFormat.getJar().isEmpty()) { // Delete pre-existing jar. - Files.deleteIfExists(deserializerLoader.getPathForJar(messageFormat.getJar())); + fileManager.deleteFile(messageFormat.getJar(), FileType.DESERIALIZER); } // 2 - move tempFilename => filename. - // Lets just delete the temp path and re-handle the upload. - Files.deleteIfExists(Paths.get(jarPath)); - uploadManager.handleDeserializerUpload(file, newFilename); + fileManager.moveFile(tempFilename, newFilename, FileType.DESERIALIZER); // 3 - Update the jar and class path properties. messageFormat.setJar(newFilename); @@ -336,7 +339,7 @@ public String deleteCluster(@PathVariable final Long id, final RedirectAttribute // Delete jar from disk try { - Files.deleteIfExists(deserializerLoader.getPathForJar(messageFormat.getJar())); + fileManager.deleteFile(messageFormat.getJar(), FileType.DESERIALIZER); } catch (final NoSuchFileException exception) { // swallow. } diff --git a/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/manager/file/FileManager.java b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/manager/file/FileManager.java new file mode 100644 index 00000000..f0b19e1d --- /dev/null +++ b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/manager/file/FileManager.java @@ -0,0 +1,128 @@ +/** + * MIT License + * + * Copyright (c) 2017, 2018, 2019 SourceLab.org (https://github.com/SourceLabOrg/kafka-webview/) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.sourcelab.kafka.webview.ui.manager.file; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.Objects; + +/** + * Manages loading files from FileStorageServices. This class manages a local read/write thru cache of the files + * on local disk so they can be loaded. + * + * In situations where local disk is always maintained (IE installed on dedicated hardware) then using the default + * LocalFileStorageService should transparent. You upload a file, it gets saved to local disk, things just work. + * + * In situations where local disk may not always be maintained (IE deployed on a containerization/virtualization service + * where disk space may be ethereal) this class attempts to provide the same mechnisms as if the files were stored on + * local disk, even if behind the scenes it's retrieving from some external data storage service. + */ +public class FileManager { + private static final Logger logger = LoggerFactory.getLogger(FileManager.class); + private final FileStorageService fileStorageService; + private final LocalDiskStorage localCacheStorage; + + public FileManager(final FileStorageService fileStorageService, final LocalDiskStorage localCacheStorage) { + this.fileStorageService = Objects.requireNonNull(fileStorageService); + this.localCacheStorage = Objects.requireNonNull(localCacheStorage); + } + + public synchronized Path getFile(final String filename, final FileType fileType) throws IOException { + // If the file is just stored locally, + if (fileStorageService instanceof LocalFileStorageService) { + // No need to deal with pull through cache. Just retrieve path directly. + return ((LocalFileStorageService) fileStorageService).getLocalPathToFile(filename, fileType); + } + + // If the file exists within our local disk cache + if (localCacheStorage.doesFileExist(filename, fileType)) { + // Return path to the file on local disk. + localCacheStorage.getLocalPathToFile(filename, fileType); + } + + // If not in the cache, we'll retrieve the file from the external source (S3, Database, etc..) + // and save it to the local disk as a cache. + final InputStream inputStream = fileStorageService.getFile(filename, fileType); + + // Save to local disk cache + localCacheStorage.saveFile(inputStream, filename, fileType); + + // Return local path + return localCacheStorage.getLocalPathToFile(filename, fileType); + } + + public synchronized boolean deleteFile(final String filename, final FileType fileType) throws IOException { + + // Delete from local cache + localCacheStorage.deleteFile(filename, fileType); + + // Delete from storage service. + return fileStorageService.deleteFile(filename, fileType); + } + + public synchronized boolean moveFile(final String originalFilename, final String newFileName, final FileType fileType) throws IOException { + // If the storage engine is just a local file on disk. + if (fileStorageService instanceof LocalFileStorageService) { + // Just save it and we're done. + return fileStorageService.moveFile(originalFilename, newFileName, fileType); + } + + // Update in local cache + if (localCacheStorage.doesFileExist(originalFilename, fileType)) { + return localCacheStorage.moveFile(originalFilename, newFileName, fileType); + } + + return true; + } + + public synchronized boolean doesFileExist(final String filename, final FileType fileType) throws IOException { + return fileStorageService.doesFileExist(filename, fileType); + } + + public synchronized boolean putFile(final InputStream inputStream, final String filename, final FileType fileType) throws IOException { + // If the storage engine is just a local file on disk. + if (fileStorageService instanceof LocalFileStorageService) { + // Just save it and we're done. + return fileStorageService.saveFile(inputStream, filename, fileType); + } + + // Otherwise remove from local cache + if (localCacheStorage.doesFileExist(filename, fileType)) { + localCacheStorage.deleteFile(filename, fileType); + } + + // Write to local cache + inputStream.reset(); + localCacheStorage.saveFile(inputStream, filename, fileType); + + // Write to storage service + inputStream.reset(); + return fileStorageService.saveFile(inputStream, filename, fileType); + } +} diff --git a/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/manager/file/FileStorageService.java b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/manager/file/FileStorageService.java new file mode 100644 index 00000000..e2d3134e --- /dev/null +++ b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/manager/file/FileStorageService.java @@ -0,0 +1,44 @@ +/** + * MIT License + * + * Copyright (c) 2017, 2018, 2019 SourceLab.org (https://github.com/SourceLabOrg/kafka-webview/) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.sourcelab.kafka.webview.ui.manager.file; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Defines interface for storing files. + */ +public interface FileStorageService { + + boolean saveFile(final InputStream fileInputStream, final String filename, final FileType type) throws IOException; + + boolean doesFileExist(final String filename, final FileType type) throws IOException; + + boolean deleteFile(final String filename, final FileType type) throws IOException; + + boolean moveFile(final String originalFilename, final String newFileName, final FileType type) throws IOException; + + InputStream getFile(final String filename, final FileType type) throws IOException; +} diff --git a/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/manager/file/FileType.java b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/manager/file/FileType.java new file mode 100644 index 00000000..175d3f01 --- /dev/null +++ b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/manager/file/FileType.java @@ -0,0 +1,31 @@ +/** + * MIT License + * + * Copyright (c) 2017, 2018, 2019 SourceLab.org (https://github.com/SourceLabOrg/kafka-webview/) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.sourcelab.kafka.webview.ui.manager.file; + +public enum FileType { + DESERIALIZER, + FILTER, + KEYSTORE; +} diff --git a/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/manager/file/LocalDiskStorage.java b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/manager/file/LocalDiskStorage.java new file mode 100644 index 00000000..11232a32 --- /dev/null +++ b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/manager/file/LocalDiskStorage.java @@ -0,0 +1,185 @@ +/** + * MIT License + * + * Copyright (c) 2017, 2018, 2019 SourceLab.org (https://github.com/SourceLabOrg/kafka-webview/) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.sourcelab.kafka.webview.ui.manager.file; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Objects; + +/** + * Implementation that stores files on local disk. + */ +public class LocalDiskStorage implements FileStorageService, LocalFileStorageService { + private static final Logger logger = LoggerFactory.getLogger(LocalDiskStorage.class); + + /** + * Root upload path. + */ + private final String uploadPath; + + /** + * Constructor. + * @param uploadPath Parent upload directory. + */ + public LocalDiskStorage(final String uploadPath) { + this.uploadPath = Objects.requireNonNull(uploadPath); + + try { + final Path rootPath = Paths.get(uploadPath).toAbsolutePath(); + if (!Files.isDirectory(rootPath)) { + Files.createDirectory(rootPath); + } + + // Ensure all of our directories exist + for (final FileType fileType : FileType.values()) { + final Path typeDirectory = getPathForType(fileType); + if (Files.isDirectory(typeDirectory)) { + continue; + } + Files.createDirectory(typeDirectory); + } + } catch (final IOException exception) { + throw new RuntimeException("Unable to create directory: " + exception.getMessage(), exception); + } + } + + @Override + public boolean saveFile(final InputStream fileInputStream, final String filename, final FileType type) throws IOException { + final Path rootPath = getPathForType(type); + if (!Files.exists(rootPath)) { + Files.createDirectory(rootPath); + } + + // Create final output file name + final Path fullOutputPath = getFilePath(filename, type); + if (fullOutputPath.toFile().exists()) { + throw new IOException("Output file already exists with filename: " + fullOutputPath.toString()); + } + + // Get the file and save it somewhere + Files.copy(fileInputStream, fullOutputPath); + + return true; + } + + @Override + public boolean doesFileExist(final String filename, final FileType type) throws IOException { + // Create final output file name + final Path fullOutputPath = getFilePath(filename, type); + return fullOutputPath.toFile().exists(); + } + + @Override + public boolean deleteFile(final String filename, final FileType type) throws IOException { + // Handle nulls gracefully. + if (filename == null || filename.trim().isEmpty()) { + return true; + } + + // Create final output file name + final Path fullOutputPath = getFilePath(filename, type); + if (!fullOutputPath.toFile().exists()) { + return true; + } + + // Only remove files + if (!fullOutputPath.toFile().isFile()) { + return false; + } + + try { + Files.delete(fullOutputPath); + } catch (final IOException ex) { + logger.error("Failed to remove file {} - {}", fullOutputPath, ex.getMessage(), ex); + return false; + } + return true; + } + + @Override + public boolean moveFile(final String originalFilename, final String newFileName, final FileType type) throws IOException { + if (!doesFileExist(originalFilename, type)) { + throw new IOException("Unable to find original file name: " + originalFilename); + } + + // Delete destination + Files.deleteIfExists(getFilePath(newFileName, type)); + + // Move original file to destination file + Files.move( + getFilePath(originalFilename, type), + getFilePath(newFileName, type) + ); + + return true; + } + + @Override + public InputStream getFile(final String filename, final FileType type) throws IOException { + if (!doesFileExist(filename, type)) { + // error? + throw new IOException("File does not exist at " + getFilePath(filename, type)); + } + final Path fullOutputPath = getFilePath(filename, type); + return Files.newInputStream(fullOutputPath); + } + + private Path getPathForType(final FileType type) { + switch (type) { + // For backwards compat. + case DESERIALIZER: + return Paths.get(uploadPath, "deserializers").toAbsolutePath(); + case FILTER: + return Paths.get(uploadPath, "filters").toAbsolutePath(); + case KEYSTORE: + return Paths.get(uploadPath, "keyStores").toAbsolutePath(); + + // Any future ones just use the enum type. + default: + return Paths.get(uploadPath, type.name()).toAbsolutePath(); + } + } + + private Path getFilePath(final String filename, final FileType type) { + final Path rootPath = getPathForType(type); + + // Create final output file name + return rootPath.resolve(filename).toAbsolutePath(); + } + + @Override + public Path getLocalPathToFile(final String filename, final FileType fileType) { + return getFilePath(filename, fileType); + } +} diff --git a/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/manager/file/LocalFileStorageService.java b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/manager/file/LocalFileStorageService.java new file mode 100644 index 00000000..1d0d85bd --- /dev/null +++ b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/manager/file/LocalFileStorageService.java @@ -0,0 +1,31 @@ +/** + * MIT License + * + * Copyright (c) 2017, 2018, 2019 SourceLab.org (https://github.com/SourceLabOrg/kafka-webview/) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.sourcelab.kafka.webview.ui.manager.file; + +import java.nio.file.Path; + +public interface LocalFileStorageService { + Path getLocalPathToFile(final String filename, final FileType fileType); +} diff --git a/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/manager/plugin/PluginFactory.java b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/manager/plugin/PluginFactory.java index 2c924f22..1890bb48 100644 --- a/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/manager/plugin/PluginFactory.java +++ b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/manager/plugin/PluginFactory.java @@ -24,6 +24,10 @@ package org.sourcelab.kafka.webview.ui.manager.plugin; +import org.sourcelab.kafka.webview.ui.manager.file.FileManager; +import org.sourcelab.kafka.webview.ui.manager.file.FileStorageService; +import org.sourcelab.kafka.webview.ui.manager.file.FileType; +import org.sourcelab.kafka.webview.ui.manager.file.LocalDiskStorage; import org.sourcelab.kafka.webview.ui.manager.plugin.exception.LoaderException; import org.sourcelab.kafka.webview.ui.manager.plugin.exception.UnableToFindClassException; import org.sourcelab.kafka.webview.ui.manager.plugin.exception.WrongImplementationException; @@ -35,6 +39,7 @@ import java.net.URLClassLoader; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Objects; /** * A factory class for creating instances of uploaded plugins. @@ -44,23 +49,28 @@ */ public class PluginFactory { /** - * Directory where JARs can be loaded from. + * Manages access to files. */ - private final String jarDirectory; + private final FileManager fileManager; /** * Type/Interface of class we want to create instances of. */ private final Class typeParameterClass; + /** + * Type of file. + */ + private final FileType fileType; + /** * Constructor. - * @param jarDirectory Where we can load JARs from. * @param typeParameterClass The type/interface of classes we can create instances of. */ - public PluginFactory(final String jarDirectory, final Class typeParameterClass) { - this.jarDirectory = jarDirectory; - this.typeParameterClass = typeParameterClass; + public PluginFactory(final FileType fileType, final Class typeParameterClass, final FileManager fileManager) { + this.fileType = Objects.requireNonNull(fileType); + this.typeParameterClass = Objects.requireNonNull(typeParameterClass); + this.fileManager = Objects.requireNonNull(fileManager); } /** @@ -90,7 +100,7 @@ public Class getPluginClass(final String jarName, final String clas final URL jarUrl = absolutePath.toUri().toURL(); final ClassLoader pluginClassLoader = new PluginClassLoader(jarUrl, getClass().getClassLoader()); return getPluginClass(pluginClassLoader, classpath); - } catch (MalformedURLException exception) { + } catch (final IOException exception) { throw new LoaderException("Unable to load jar " + jarName, exception); } } @@ -185,7 +195,7 @@ public boolean checkPlugin(final String jarName, final String classpath) throws * Get the full path on disk to the given Jar file. * @param jarName Jar to lookup full path to. */ - public Path getPathForJar(final String jarName) { - return Paths.get(jarDirectory, jarName).toAbsolutePath(); + public Path getPathForJar(final String jarName) throws IOException { + return fileManager.getFile(jarName, fileType); } } diff --git a/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/manager/plugin/UploadManager.java b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/manager/plugin/UploadManager.java index 789e53ff..b19f722c 100644 --- a/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/manager/plugin/UploadManager.java +++ b/kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/manager/plugin/UploadManager.java @@ -24,58 +24,31 @@ package org.sourcelab.kafka.webview.ui.manager.plugin; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.sourcelab.kafka.webview.ui.manager.file.FileManager; +import org.sourcelab.kafka.webview.ui.manager.file.FileStorageService; +import org.sourcelab.kafka.webview.ui.manager.file.FileType; +import org.sourcelab.kafka.webview.ui.manager.file.LocalDiskStorage; import org.springframework.web.multipart.MultipartFile; import java.io.BufferedInputStream; -import java.io.File; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.util.Objects; /** * Handles uploading jars from the frontend UI and placing them into the expected locations on disk. */ public class UploadManager { - private static final Logger logger = LoggerFactory.getLogger(UploadManager.class); - - /** - * Where to upload JARs associated with a deserializer. - */ - private final String deserializerUploadPath; - - /** - * Where to upload JARs associated with filters. - */ - private final String filterUploadPath; - /** - * Where to upload SSL JKS key stores. + * Underlying Storage Mechanism. */ - private final String keyStoreUploadPath; + private final FileManager fileManager; /** * Constructor. - * @param uploadPath Parent upload directory. + * @param fileManager manages file storage. */ - public UploadManager(final String uploadPath) { - this.deserializerUploadPath = uploadPath + "/deserializers"; - this.filterUploadPath = uploadPath + "/filters"; - this.keyStoreUploadPath = uploadPath + "/keyStores"; - } - - String getDeserializerUploadPath() { - return deserializerUploadPath; - } - - String getFilterUploadPath() { - return filterUploadPath; - } - - String getKeyStoreUploadPath() { - return keyStoreUploadPath; + public UploadManager(final FileManager fileManager) { + this.fileManager = Objects.requireNonNull(fileManager); } /** @@ -85,7 +58,7 @@ String getKeyStoreUploadPath() { * @return Path to uploaded file. */ public String handleDeserializerUpload(final MultipartFile file, final String outFileName) throws IOException { - return handleFileUpload(file, outFileName, getDeserializerUploadPath()); + return handleFileUpload(file, outFileName, FileType.DESERIALIZER); } /** @@ -95,7 +68,7 @@ public String handleDeserializerUpload(final MultipartFile file, final String ou * @return Path to uploaded file. */ public String handleFilterUpload(final MultipartFile file, final String outFileName) throws IOException { - return handleFileUpload(file, outFileName, getFilterUploadPath()); + return handleFileUpload(file, outFileName, FileType.FILTER); } /** @@ -105,7 +78,7 @@ public String handleFilterUpload(final MultipartFile file, final String outFileN * @return Path to uploaded file. */ public String handleKeystoreUpload(final MultipartFile file, final String outFileName) throws IOException { - return handleFileUpload(file, outFileName, getKeyStoreUploadPath()); + return handleFileUpload(file, outFileName, FileType.KEYSTORE); } /** @@ -113,54 +86,19 @@ public String handleKeystoreUpload(final MultipartFile file, final String outFil * @param keyStoreFile Filename of keystore file to be removed. * @return True if successful, false if not. */ - public boolean deleteKeyStore(final String keyStoreFile) { - return deleteFile(keyStoreFile, keyStoreUploadPath); + public boolean deleteKeyStore(final String keyStoreFile) throws IOException { + return fileManager.deleteFile(keyStoreFile, FileType.KEYSTORE); } - private boolean deleteFile(final String filename, final String rootPath) { - // Handle nulls gracefully. - if (filename == null || filename.trim().isEmpty()) { - return true; - } - - // Create final output file name - final Path fullOutputPath = Paths.get(rootPath, filename).toAbsolutePath(); - - if (!fullOutputPath.toFile().exists()) { - return true; - } - - // Only remove files - if (!fullOutputPath.toFile().isFile()) { - return false; - } - - try { - Files.delete(fullOutputPath); - } catch (final IOException ex) { - logger.error("Failed to remove file {} - {}", fullOutputPath, ex.getMessage(), ex); - return false; - } - return true; - } - - private String handleFileUpload(final MultipartFile file, final String outFileName, final String rootPath) throws IOException { - final File parentDir = new File(rootPath); - if (!parentDir.exists() && !parentDir.mkdirs()) { - throw new IOException("Failed to createConsumer directory: " + rootPath); - } - - // Create final output file name - final Path fullOutputPath = Paths.get(rootPath, outFileName); - if (fullOutputPath.toFile().exists()) { - throw new IOException("Output file already exists"); - } - // Get the file and save it somewhere - try (BufferedInputStream in = new BufferedInputStream(file.getInputStream())) { - Files.copy(in, fullOutputPath); + private String handleFileUpload(final MultipartFile file, final String outFileName, final FileType fileType) throws IOException { + // Check if file exists + if (fileManager.doesFileExist(outFileName, fileType)) { + throw new IOException("Output file of type " + fileType.name() + " already exists with name " + outFileName); } - return fullOutputPath.toString(); + // Store it + fileManager.putFile(file.getInputStream(), outFileName, fileType); + return outFileName; } } diff --git a/kafka-webview-ui/src/main/resources/config/base.yml b/kafka-webview-ui/src/main/resources/config/base.yml index 5b91a6b1..8db0025f 100644 --- a/kafka-webview-ui/src/main/resources/config/base.yml +++ b/kafka-webview-ui/src/main/resources/config/base.yml @@ -73,6 +73,7 @@ info: app: name: Kafka Web View uploadPath: "./data/uploads" + cachePath: "./data/cache" key: "SuperSecretKey" multiThreadedConsumer: true maxConcurrentWebConsumers: 32 diff --git a/kafka-webview-ui/src/test/java/org/sourcelab/kafka/webview/ui/manager/kafka/WebKafkaConsumerFactoryTest.java b/kafka-webview-ui/src/test/java/org/sourcelab/kafka/webview/ui/manager/kafka/WebKafkaConsumerFactoryTest.java index cede4dfd..0591f853 100644 --- a/kafka-webview-ui/src/test/java/org/sourcelab/kafka/webview/ui/manager/kafka/WebKafkaConsumerFactoryTest.java +++ b/kafka-webview-ui/src/test/java/org/sourcelab/kafka/webview/ui/manager/kafka/WebKafkaConsumerFactoryTest.java @@ -36,6 +36,9 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.sourcelab.kafka.webview.ui.manager.encryption.SecretManager; +import org.sourcelab.kafka.webview.ui.manager.file.FileManager; +import org.sourcelab.kafka.webview.ui.manager.file.FileType; +import org.sourcelab.kafka.webview.ui.manager.file.LocalDiskStorage; import org.sourcelab.kafka.webview.ui.manager.kafka.dto.KafkaResult; import org.sourcelab.kafka.webview.ui.manager.kafka.dto.KafkaResults; import org.sourcelab.kafka.webview.ui.manager.plugin.PluginFactory; @@ -228,8 +231,11 @@ private List consumeAllResults( } private WebKafkaConsumerFactory createDefaultFactory() { - final PluginFactory deserializerPluginFactory = new PluginFactory<>("not/used", Deserializer.class); - final PluginFactory filterPluginFactoryPluginFactory = new PluginFactory<>("not/used", RecordFilter.class); + final FileManager fileManager = new FileManager(new LocalDiskStorage("/tmp"), new LocalDiskStorage("/tmp")); + final PluginFactory deserializerPluginFactory + = new PluginFactory<>(FileType.DESERIALIZER, Deserializer.class, fileManager); + final PluginFactory filterPluginFactoryPluginFactory + = new PluginFactory<>(FileType.FILTER, RecordFilter.class, fileManager); final SecretManager secretManager = new SecretManager("Passphrase"); final KafkaConsumerFactory kafkaConsumerFactory = new KafkaConsumerFactory( new KafkaClientConfigUtil("not/used", "MyPrefix") @@ -245,8 +251,12 @@ private WebKafkaConsumerFactory createDefaultFactory() { } private WebKafkaConsumerFactory createMultiThreadedFactory() { - final PluginFactory deserializerPluginFactory = new PluginFactory<>("not/used", Deserializer.class); - final PluginFactory filterPluginFactoryPluginFactory = new PluginFactory<>("not/used", RecordFilter.class); + final FileManager fileManager = new FileManager(new LocalDiskStorage("/tmp"), new LocalDiskStorage("/tmp")); + final PluginFactory deserializerPluginFactory + = new PluginFactory<>(FileType.DESERIALIZER, Deserializer.class, fileManager); + final PluginFactory filterPluginFactoryPluginFactory + = new PluginFactory<>(FileType.FILTER, RecordFilter.class, fileManager); + final SecretManager secretManager = new SecretManager("Passphrase"); final KafkaConsumerFactory kafkaConsumerFactory = new KafkaConsumerFactory( new KafkaClientConfigUtil("not/used", "MyPrefix") diff --git a/kafka-webview-ui/src/test/java/org/sourcelab/kafka/webview/ui/manager/plugin/PluginFactoryTest.java b/kafka-webview-ui/src/test/java/org/sourcelab/kafka/webview/ui/manager/plugin/PluginFactoryTest.java index aa95f4f4..04a9f398 100644 --- a/kafka-webview-ui/src/test/java/org/sourcelab/kafka/webview/ui/manager/plugin/PluginFactoryTest.java +++ b/kafka-webview-ui/src/test/java/org/sourcelab/kafka/webview/ui/manager/plugin/PluginFactoryTest.java @@ -26,11 +26,17 @@ import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.StringDeserializer; +import org.junit.Before; import org.junit.Test; +import org.sourcelab.kafka.webview.ui.manager.file.FileManager; +import org.sourcelab.kafka.webview.ui.manager.file.FileStorageService; +import org.sourcelab.kafka.webview.ui.manager.file.FileType; +import org.sourcelab.kafka.webview.ui.manager.file.LocalDiskStorage; import org.sourcelab.kafka.webview.ui.manager.plugin.exception.LoaderException; import org.sourcelab.kafka.webview.ui.plugin.filter.RecordFilter; import java.io.File; +import java.io.IOException; import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Path; @@ -41,20 +47,39 @@ public class PluginFactoryTest { + private PluginFactory deserializerPluginFactory; + private PluginFactory recordFilterPluginFactory; + + private FileManager fileManager; + + @Before + public void setup() { + // Determine root path to test resources testFiles/ directory. + final URL testFilesDirectory = getClass().getClassLoader().getResource("testFiles"); + final String uploadPath = testFilesDirectory.getPath(); + + final LocalDiskStorage fileStorageService = new LocalDiskStorage(uploadPath); + + // Create file Manager + fileManager = new FileManager(fileStorageService, fileStorageService); + + deserializerPluginFactory = new PluginFactory<>(FileType.DESERIALIZER, Deserializer.class, fileManager); + recordFilterPluginFactory = new PluginFactory<>(FileType.FILTER, RecordFilter.class, fileManager); + } + /** * Test creating a RecordFilter. */ @Test - public void testWithRecordFilter() throws LoaderException { + public void testWithRecordFilter() throws LoaderException, IOException { final String jarFilename = "testPlugins.jar"; final String classPath = "examples.filter.LowOffsetFilter"; // Find jar on filesystem. - final URL jar = getClass().getClassLoader().getResource("testDeserializer/" + jarFilename); - final String jarPath = new File(jar.getFile()).getParent(); + final URL jar = getClass().getClassLoader().getResource("testFiles/filters/" + jarFilename); // Create factory - final PluginFactory factory = new PluginFactory<>(jarPath, RecordFilter.class); + final PluginFactory factory = recordFilterPluginFactory; final Path pathForJar = factory.getPathForJar(jarFilename); // Validate path is correct @@ -81,16 +106,15 @@ public void testWithRecordFilter() throws LoaderException { * Test checking a RecordFilter. */ @Test - public void testCheckPlugin_WithRecordFilter() throws LoaderException { + public void testCheckPlugin_WithRecordFilter() throws LoaderException, IOException { final String jarFilename = "testPlugins.jar"; final String classPath = "examples.filter.LowOffsetFilter"; // Find jar on filesystem. - final URL jar = getClass().getClassLoader().getResource("testDeserializer/" + jarFilename); - final String jarPath = new File(jar.getFile()).getParent(); + final URL jar = getClass().getClassLoader().getResource("testFiles/filters/" + jarFilename); // Create factory - final PluginFactory factory = new PluginFactory<>(jarPath, RecordFilter.class); + final PluginFactory factory = recordFilterPluginFactory; final Path pathForJar = factory.getPathForJar(jarFilename); // Validate path is correct @@ -113,16 +137,15 @@ public void testCheckPlugin_WithRecordFilter() throws LoaderException { * Test creating a Deserializer. */ @Test - public void testWithDeserializer() throws LoaderException { + public void testWithDeserializer() throws LoaderException, IOException { final String jarFilename = "testPlugins.jar"; final String classPath = "examples.deserializer.ExampleDeserializer"; // Find jar on filesystem. - final URL jar = getClass().getClassLoader().getResource("testDeserializer/" + jarFilename); - final String jarPath = new File(jar.getFile()).getParent(); + final URL jar = getClass().getClassLoader().getResource("testFiles/deserializers/" + jarFilename); // Create factory - final PluginFactory factory = new PluginFactory<>(jarPath, Deserializer.class); + final PluginFactory factory = deserializerPluginFactory; final Path pathForJar = factory.getPathForJar(jarFilename); // Validate path is correct @@ -144,22 +167,23 @@ public void testWithDeserializer() throws LoaderException { // Call method on interface final String value = "MyValue"; final String result = (String) deserializer.deserialize("MyTopic", value.getBytes(StandardCharsets.UTF_8)); + assertEquals("Prefixed Value: " + value, result); } /** * Test checking a Deserializer. */ @Test - public void testCheckPlugin_WithDeserializer() throws LoaderException { + public void testCheckPlugin_WithDeserializer() throws LoaderException, IOException { final String jarFilename = "testPlugins.jar"; final String classPath = "examples.deserializer.ExampleDeserializer"; // Find jar on filesystem. - final URL jar = getClass().getClassLoader().getResource("testDeserializer/" + jarFilename); + final URL jar = getClass().getClassLoader().getResource("testFiles/deserializers/" + jarFilename); final String jarPath = new File(jar.getFile()).getParent(); // Create factory - final PluginFactory factory = new PluginFactory<>(jarPath, Deserializer.class); + final PluginFactory factory = deserializerPluginFactory; final Path pathForJar = factory.getPathForJar(jarFilename); // Validate path is correct @@ -186,7 +210,7 @@ public void testLoadingDefaultDeserializer() throws LoaderException { final String classPath = StringDeserializer.class.getName(); // Create factory - final PluginFactory factory = new PluginFactory<>("/tmp", Deserializer.class); + final PluginFactory factory = new PluginFactory<>(FileType.DESERIALIZER, Deserializer.class, fileManager); // Get class instance final Class pluginFilterClass = factory.getPluginClass(classPath); diff --git a/kafka-webview-ui/src/test/java/org/sourcelab/kafka/webview/ui/manager/plugin/UploadManagerTest.java b/kafka-webview-ui/src/test/java/org/sourcelab/kafka/webview/ui/manager/plugin/UploadManagerTest.java deleted file mode 100644 index 92fcdabd..00000000 --- a/kafka-webview-ui/src/test/java/org/sourcelab/kafka/webview/ui/manager/plugin/UploadManagerTest.java +++ /dev/null @@ -1,222 +0,0 @@ -/** - * MIT License - * - * Copyright (c) 2017, 2018, 2019 SourceLab.org (https://github.com/SourceLabOrg/kafka-webview/) - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.sourcelab.kafka.webview.ui.manager.plugin; - -import org.junit.Test; -import org.springframework.mock.web.MockMultipartFile; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; - -import static junit.framework.TestCase.assertFalse; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class UploadManagerTest { - - /** - * Test constructor works about as we expect. - */ - @Test - public void testConstructor() { - final String parentUploadDir = "/tmp/uploads"; - final String expectedDeserializerPath = parentUploadDir + "/deserializers"; - final String expectedFilterPath = parentUploadDir + "/filters"; - final String expectedKeyStorePath = parentUploadDir + "/keyStores"; - - // Create manager - final UploadManager uploadManager = new UploadManager(parentUploadDir); - - // Validate - assertEquals("Has expected deserializer path", expectedDeserializerPath, uploadManager.getDeserializerUploadPath()); - assertEquals("Has expected filter path", expectedFilterPath, uploadManager.getFilterUploadPath()); - assertEquals("Has expected keystore path", expectedKeyStorePath, uploadManager.getKeyStoreUploadPath()); - } - - /** - * Tests uploading a Deserializer file. - */ - @Test - public void testHandleDeserializerUpload() throws IOException { - // Make a temp directory - final Path tempDirectory = Files.createTempDirectory(null); - - // Create a "multi-part" file - final String mockContent = "test content"; - final MockMultipartFile myFile = new MockMultipartFile( - "data", - "filename.txt", - "text/plain", - mockContent.getBytes(StandardCharsets.UTF_8) - ); - - final String outputFilename = "MyUpload.jar"; - final String expectedUploadedPath = tempDirectory.toString() + "/deserializers/" + outputFilename; - - // Create manager - final UploadManager uploadManager = new UploadManager(tempDirectory.toString()); - - // Handle the "upload" - final String result = uploadManager.handleDeserializerUpload(myFile, outputFilename); - - // Validate - assertEquals("Has expected result filename", expectedUploadedPath, result); - - // Validate contents - final byte[] contentBytes = Files.readAllBytes(new File(result).toPath()); - final String contentString = new String(contentBytes, StandardCharsets.UTF_8); - assertEquals("Contents are expected", mockContent, contentString); - } - - /** - * Tests uploading a Filter file. - */ - @Test - public void testHandleFilterUpload() throws IOException { - // Make a temp directory - final Path tempDirectory = Files.createTempDirectory(null); - - // Create a "multi-part" file - final String mockContent = "test content"; - final MockMultipartFile myFile = new MockMultipartFile( - "data", - "filename.txt", - "text/plain", - mockContent.getBytes(StandardCharsets.UTF_8) - ); - - final String outputFilename = "MyUpload.jar"; - final String expectedUploadedPath = tempDirectory.toString() + "/filters/" + outputFilename; - - // Create manager - final UploadManager uploadManager = new UploadManager(tempDirectory.toString()); - - // Handle the "upload" - final String result = uploadManager.handleFilterUpload(myFile, outputFilename); - - // Validate - assertEquals("Has expected result filename", expectedUploadedPath, result); - - // Validate contents - final byte[] contentBytes = Files.readAllBytes(new File(result).toPath()); - final String contentString = new String(contentBytes, StandardCharsets.UTF_8); - assertEquals("Contents are expected", mockContent, contentString); - } - - /** - * Tests uploading a Deserializer file. - */ - @Test - public void testHandleKeyStoreUpload() throws IOException { - // Make a temp directory - final Path tempDirectory = Files.createTempDirectory(null); - - // Create a "multi-part" file - final String mockContent = "test content"; - final MockMultipartFile myFile = new MockMultipartFile( - "data", - "filename.txt", - "text/plain", - mockContent.getBytes(StandardCharsets.UTF_8) - ); - - final String outputFilename = "MyUpload.jar"; - final String expectedUploadedPath = tempDirectory.toString() + "/keyStores/" + outputFilename; - - // Create manager - final UploadManager uploadManager = new UploadManager(tempDirectory.toString()); - - // Handle the "upload" - final String result = uploadManager.handleKeystoreUpload(myFile, outputFilename); - - // Validate - assertEquals("Has expected result filename", expectedUploadedPath, result); - - // Validate contents - final Path filePath = new File(result).toPath(); - final byte[] contentBytes = Files.readAllBytes(filePath); - final String contentString = new String(contentBytes, StandardCharsets.UTF_8); - assertEquals("Contents are expected", mockContent, contentString); - - // Now test deleting a keystore - final boolean deleteResult = uploadManager.deleteKeyStore(outputFilename); - assertEquals("Should be true", true, deleteResult); - assertFalse("File no longer exists", Files.exists(filePath)); - } - - /** - * Test UploadManager gracefully handles deleting files that don't exist. - */ - @Test - public void testDeleteNonExistantFile() throws IOException { - // Make a temp directory - final Path tempDirectory = Files.createTempDirectory(null); - - // Create manager - final UploadManager uploadManager = new UploadManager(tempDirectory.toString()); - - final boolean result = uploadManager.deleteKeyStore("This-File-Does-not-exist"); - assertTrue("Gracefully returns true", result); - } - - /** - * Test UploadManager gracefully handles deleting empty string filenames that don't exist. - */ - @Test - public void testDeleteEmptyFile() throws IOException { - // Make a temp directory - final Path tempDirectory = Files.createTempDirectory(null); - - // Create manager - final UploadManager uploadManager = new UploadManager(tempDirectory.toString()); - - final boolean result = uploadManager.deleteKeyStore(""); - assertTrue("Gracefully returns true", result); - - // Sanity test - assertTrue("Temp dir still exists", tempDirectory.toFile().exists()); - } - - /** - * Test UploadManager gracefully handles deleting null string filenames. - */ - @Test - public void testDeleteNullFile() throws IOException { - // Make a temp directory - final Path tempDirectory = Files.createTempDirectory(null); - - // Create manager - final UploadManager uploadManager = new UploadManager(tempDirectory.toString()); - - final boolean result = uploadManager.deleteKeyStore(null); - assertTrue("Gracefully returns true", result); - - // Sanity test - assertTrue("Temp dir still exists", tempDirectory.toFile().exists()); - } -} \ No newline at end of file diff --git a/kafka-webview-ui/src/test/java/org/sourcelab/kafka/webview/ui/manager/plugin/UploadManager_LocalFileStorageTest.java b/kafka-webview-ui/src/test/java/org/sourcelab/kafka/webview/ui/manager/plugin/UploadManager_LocalFileStorageTest.java new file mode 100644 index 00000000..0ffac7d0 --- /dev/null +++ b/kafka-webview-ui/src/test/java/org/sourcelab/kafka/webview/ui/manager/plugin/UploadManager_LocalFileStorageTest.java @@ -0,0 +1,206 @@ +/** + * MIT License + * + * Copyright (c) 2017, 2018, 2019 SourceLab.org (https://github.com/SourceLabOrg/kafka-webview/) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.sourcelab.kafka.webview.ui.manager.plugin; + +import org.junit.Test; +import org.springframework.mock.web.MockMultipartFile; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static junit.framework.TestCase.assertFalse; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Integration test of UploadManager using LocalFileStorage implementation. + */ +public class UploadManager_LocalFileStorageTest { + +// /** +// * Tests uploading a Deserializer file. +// */ +// @Test +// public void testHandleDeserializerUpload() throws IOException { +// // Make a temp directory +// final Path tempDirectory = Files.createTempDirectory(null); +// +// // Create a "multi-part" file +// final String mockContent = "test content"; +// final MockMultipartFile myFile = new MockMultipartFile( +// "data", +// "filename.txt", +// "text/plain", +// mockContent.getBytes(StandardCharsets.UTF_8) +// ); +// +// final String outputFilename = "MyUpload.jar"; +// final String expectedUploadedPath = tempDirectory.toString() + "/deserializers/" + outputFilename; +// +// // Create manager +// final UploadManager uploadManager = new UploadManager(tempDirectory.toString()); +// +// // Handle the "upload" +// final String result = uploadManager.handleDeserializerUpload(myFile, outputFilename); +// +// // Validate +// assertEquals("Has expected result filename", outputFilename, result); +// +// // Validate contents +// final byte[] contentBytes = Files.readAllBytes(new File(expectedUploadedPath).toPath()); +// final String contentString = new String(contentBytes, StandardCharsets.UTF_8); +// assertEquals("Contents are expected", mockContent, contentString); +// } +// +// /** +// * Tests uploading a Filter file. +// */ +// @Test +// public void testHandleFilterUpload() throws IOException { +// // Make a temp directory +// final Path tempDirectory = Files.createTempDirectory(null); +// +// // Create a "multi-part" file +// final String mockContent = "test content"; +// final MockMultipartFile myFile = new MockMultipartFile( +// "data", +// "filename.txt", +// "text/plain", +// mockContent.getBytes(StandardCharsets.UTF_8) +// ); +// +// final String outputFilename = "MyUpload.jar"; +// final String expectedUploadedPath = tempDirectory.toString() + "/filters/" + outputFilename; +// +// // Create manager +// final UploadManager uploadManager = new UploadManager(tempDirectory.toString()); +// +// // Handle the "upload" +// final String result = uploadManager.handleFilterUpload(myFile, outputFilename); +// +// // Validate +// assertEquals("Has expected result filename", outputFilename, result); +// +// // Validate contents +// final byte[] contentBytes = Files.readAllBytes(new File(expectedUploadedPath).toPath()); +// final String contentString = new String(contentBytes, StandardCharsets.UTF_8); +// assertEquals("Contents are expected", mockContent, contentString); +// } +// +// /** +// * Tests uploading a Deserializer file. +// */ +// @Test +// public void testHandleKeyStoreUpload() throws IOException { +// // Make a temp directory +// final Path tempDirectory = Files.createTempDirectory(null); +// +// // Create a "multi-part" file +// final String mockContent = "test content"; +// final MockMultipartFile myFile = new MockMultipartFile( +// "data", +// "filename.txt", +// "text/plain", +// mockContent.getBytes(StandardCharsets.UTF_8) +// ); +// +// final String outputFilename = "MyUpload.jar"; +// final String expectedUploadedPath = tempDirectory.toString() + "/keyStores/" + outputFilename; +// +// // Create manager +// final UploadManager uploadManager = new UploadManager(tempDirectory.toString()); +// +// // Handle the "upload" +// final String result = uploadManager.handleKeystoreUpload(myFile, outputFilename); +// +// // Validate +// assertEquals("Has expected result filename", outputFilename, result); +// +// // Validate contents +// final Path filePath = new File(expectedUploadedPath).toPath(); +// final byte[] contentBytes = Files.readAllBytes(filePath); +// final String contentString = new String(contentBytes, StandardCharsets.UTF_8); +// assertEquals("Contents are expected", mockContent, contentString); +// +// // Now test deleting a keystore +// final boolean deleteResult = uploadManager.deleteKeyStore(outputFilename); +// assertEquals("Should be true", true, deleteResult); +// assertFalse("File no longer exists", Files.exists(filePath)); +// } +// +// /** +// * Test UploadManager gracefully handles deleting files that don't exist. +// */ +// @Test +// public void testDeleteNonExistantFile() throws IOException { +// // Make a temp directory +// final Path tempDirectory = Files.createTempDirectory(null); +// +// // Create manager +// final UploadManager uploadManager = new UploadManager(tempDirectory.toString()); +// +// final boolean result = uploadManager.deleteKeyStore("This-File-Does-not-exist"); +// assertTrue("Gracefully returns true", result); +// } +// +// /** +// * Test UploadManager gracefully handles deleting empty string filenames that don't exist. +// */ +// @Test +// public void testDeleteEmptyFile() throws IOException { +// // Make a temp directory +// final Path tempDirectory = Files.createTempDirectory(null); +// +// // Create manager +// final UploadManager uploadManager = new UploadManager(tempDirectory.toString()); +// +// final boolean result = uploadManager.deleteKeyStore(""); +// assertTrue("Gracefully returns true", result); +// +// // Sanity test +// assertTrue("Temp dir still exists", tempDirectory.toFile().exists()); +// } +// +// /** +// * Test UploadManager gracefully handles deleting null string filenames. +// */ +// @Test +// public void testDeleteNullFile() throws IOException { +// // Make a temp directory +// final Path tempDirectory = Files.createTempDirectory(null); +// +// // Create manager +// final UploadManager uploadManager = new UploadManager(tempDirectory.toString()); +// +// final boolean result = uploadManager.deleteKeyStore(null); +// assertTrue("Gracefully returns true", result); +// +// // Sanity test +// assertTrue("Temp dir still exists", tempDirectory.toFile().exists()); +// } +} \ No newline at end of file diff --git a/kafka-webview-ui/src/test/resources/application.yml b/kafka-webview-ui/src/test/resources/application.yml index 827cdc27..04052916 100644 --- a/kafka-webview-ui/src/test/resources/application.yml +++ b/kafka-webview-ui/src/test/resources/application.yml @@ -41,6 +41,7 @@ spring: app: name: Kafka Web View uploadPath: "./data/uploads" + cachePath: "./data/cache" key: "SuperSecretKey" maxConcurrentWebSocketConsumers: 64 consumerIdPrefix: "KafkaWebViewConsumer" diff --git a/kafka-webview-ui/src/test/resources/testFiles/deserializers/testPlugins.jar b/kafka-webview-ui/src/test/resources/testFiles/deserializers/testPlugins.jar new file mode 100644 index 00000000..61d84231 Binary files /dev/null and b/kafka-webview-ui/src/test/resources/testFiles/deserializers/testPlugins.jar differ diff --git a/kafka-webview-ui/src/test/resources/testFiles/filters/testPlugins.jar b/kafka-webview-ui/src/test/resources/testFiles/filters/testPlugins.jar new file mode 100644 index 00000000..61d84231 Binary files /dev/null and b/kafka-webview-ui/src/test/resources/testFiles/filters/testPlugins.jar differ