From 5dadb06ecbbc8ae5f8fc9f674ab0871c448c1b1c Mon Sep 17 00:00:00 2001 From: uuqjz Date: Wed, 26 Jun 2024 15:54:56 +0200 Subject: [PATCH 1/4] Initial tuhhpipeline --- .../.gitignore | 4 +- .../converter/tests/TUHHPipeline.java | 199 ++++++++++++++++++ 2 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 tests/org.dataflowanalysis.converter.tests/src/org/dataflowanalysis/converter/tests/TUHHPipeline.java diff --git a/tests/org.dataflowanalysis.converter.tests/.gitignore b/tests/org.dataflowanalysis.converter.tests/.gitignore index 94a2dd1..61441f9 100644 --- a/tests/org.dataflowanalysis.converter.tests/.gitignore +++ b/tests/org.dataflowanalysis.converter.tests/.gitignore @@ -1 +1,3 @@ -*.json \ No newline at end of file +*.json +TUHH-Models/ +microSecEnD-main/ \ No newline at end of file diff --git a/tests/org.dataflowanalysis.converter.tests/src/org/dataflowanalysis/converter/tests/TUHHPipeline.java b/tests/org.dataflowanalysis.converter.tests/src/org/dataflowanalysis/converter/tests/TUHHPipeline.java new file mode 100644 index 0000000..351a54b --- /dev/null +++ b/tests/org.dataflowanalysis.converter.tests/src/org/dataflowanalysis/converter/tests/TUHHPipeline.java @@ -0,0 +1,199 @@ +package org.dataflowanalysis.converter.tests; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import org.dataflowanalysis.converter.MicroSecEndConverter; + +public class TUHHPipeline { + + public static String TUHH_REPO = "microSecEnD-main"; + public static Path converter; + + @Disabled + @Test + public void runPipeline() throws IOException { + assertTrue(Files.isDirectory(Paths.get(TUHH_REPO))); + + var replacerRepo=Paths.get(TUHH_REPO,"dataset","error_replacer.py"); + var converterRepo=Paths.get(TUHH_REPO,"convert_model.py"); + assertTrue(Files.isReadable(replacerRepo)); + assertTrue(Files.isReadable(converterRepo)); + + Path datasetRepo = Paths.get(TUHH_REPO,"dataset"); + List datasetsRepo = new ArrayList<>(); + Files.list(datasetRepo).forEach(path -> { + if (Files.isDirectory(path)) { + datasetsRepo.add(path); + }}); + assertEquals(datasetsRepo.size(),17); + + var tuhh = Paths.get("TUHH-Models"); + removeAndCreateDir(tuhh); + + converter=tuhh.resolve(converterRepo.getFileName()); + var replacer=tuhh.resolve(replacerRepo.getFileName()); + Files.copy(converterRepo, converter, StandardCopyOption.REPLACE_EXISTING); + Files.copy(replacerRepo, replacer, StandardCopyOption.REPLACE_EXISTING); + + String oldRoot="root_dir = Path(r'put\\your\\root\\dir\\here') #set root dir"; + String newRoot="root_dir = \".\""; + replaceStringInFile(replacer, oldRoot, newRoot); + + List datasets = new ArrayList<>(); + for (var dataset:datasetsRepo) { + var dirName=dataset.getFileName().toString(); + int underscoreIndex = dirName.indexOf('_'); + if (dirName.contains("kafka")) { + dirName = dirName.substring(0, underscoreIndex) + "-kafka"; + } + else { + dirName = dirName.substring(0, underscoreIndex); + } + var newDataset=tuhh.resolve(dirName); + copyDir(dataset.toString(),newDataset.toString()); + datasets.add(newDataset); + } + + for(var dataset:datasets) { + cleanTopLevelOfDataset(dataset); + renameTxtVariants(dataset.resolve("model_variants")); + moveTxtVariantsUp(dataset.resolve("model_variants")); + convertTxtToJson(dataset); + convertJsonToDFD(dataset); + } + + Files.delete(converter); + Files.delete(replacer); + } + + private void convertJsonToDFD(Path dataset) throws IOException { + var microConverter = new MicroSecEndConverter(); + Files.list(dataset).forEach(path -> { + if (Files.isRegularFile(path) && path.toString().endsWith(".json")) { + System.out.println(path); + var complete = microConverter.microToDfd(path.toString()); + microConverter.storeDFD(complete, path.toString()); + }}); + } + + private void convertTxtToJson(Path dataset) throws IOException { + Files.list(dataset).forEach(path -> { + if (Files.isRegularFile(path)) { + if(path.toString().endsWith(".txt")) { + System.out.println(path); + try { + runPythonScript(converter.toString(), path.toString(), "json", path.toString().replace(".txt", ".json")); + Files.delete(path); + } catch (InterruptedException | IOException e) {} + } + }}); + } + + private void moveTxtVariantsUp (Path dir) throws IOException { + Path parentDir = dir.getParent(); + + Files.list(dir).forEach(path -> { + if (Files.isRegularFile(path)) { + Path targetPath = parentDir.resolve(path.getFileName()); + try { + Files.move(path, targetPath, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) {} + }}); + + Files.delete(dir); + } + + private void cleanTopLevelOfDataset(Path dataset) throws IOException { + Files.list(dataset).forEach(path -> { + if (Files.isRegularFile(path)) { + if(!path.toString().endsWith(".json") || path.toString().endsWith("traceability.json")) { + try { + Files.delete(path); + } catch (IOException e) {} + } + else { + var renamedBaseModel = dataset.resolve(dataset.getFileName()+"_0.json"); + try { + Files.move(path, renamedBaseModel, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) {} + } + }}); + } + + private void removeAndCreateDir(Path dir) throws IOException { + if (Files.exists(dir)) { + Files.walk(dir) + .sorted(Comparator.reverseOrder()) + .forEach(path -> { + try { + Files.delete(path); + } catch (IOException e) {} + }); + } + Files.createDirectories(dir); + } + + private void replaceStringInFile(Path file, String oldString, String newString) throws IOException { + List lines = Files.readAllLines(file); + + for (int i = 0; i < lines.size(); i++) { + String line = lines.get(i); + if (line.contains(oldString)) { + lines.set(i, line.replace(oldString, newString)); + } + } + + Files.write(file, lines); + } + + private void copyDir(String sourceDirectoryLocation, String destinationDirectoryLocation) + throws IOException { + Files.walk(Paths.get(sourceDirectoryLocation)) + .forEach(source -> { + Path destination = Paths.get(destinationDirectoryLocation, source.toString() + .substring(sourceDirectoryLocation.length())); + try { + Files.copy(source, destination); + } catch (IOException e) { + e.printStackTrace(); + } + }); + } + + private void renameTxtVariants(Path variants) throws IOException { + var modelName=variants.getParent().getFileName(); + Files.list(variants).forEach(path -> { + if (Files.isRegularFile(path)) { + var renamedModel = variants.resolve(modelName+"_"+path.getFileName()); + try { + Files.move(path, renamedModel, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) {} + }}); + } + + private int runPythonScript(String script, String in, String format, String out) throws InterruptedException, IOException { + String[] command = {"python3", script, in, format, "-op", out}; + + ProcessBuilder processBuilder = new ProcessBuilder(command); + Process process; + + process = processBuilder.start(); + return process.waitFor(); + + } +} From bf633d6443bcb1680af0484dab82e9fb179d58d7 Mon Sep 17 00:00:00 2001 From: uuqjz Date: Wed, 26 Jun 2024 15:58:05 +0200 Subject: [PATCH 2/4] Removed replacer --- .../converter/tests/TUHHPipeline.java | 24 +------------------ 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/tests/org.dataflowanalysis.converter.tests/src/org/dataflowanalysis/converter/tests/TUHHPipeline.java b/tests/org.dataflowanalysis.converter.tests/src/org/dataflowanalysis/converter/tests/TUHHPipeline.java index 351a54b..052ffff 100644 --- a/tests/org.dataflowanalysis.converter.tests/src/org/dataflowanalysis/converter/tests/TUHHPipeline.java +++ b/tests/org.dataflowanalysis.converter.tests/src/org/dataflowanalysis/converter/tests/TUHHPipeline.java @@ -28,10 +28,8 @@ public class TUHHPipeline { @Test public void runPipeline() throws IOException { assertTrue(Files.isDirectory(Paths.get(TUHH_REPO))); - - var replacerRepo=Paths.get(TUHH_REPO,"dataset","error_replacer.py"); + var converterRepo=Paths.get(TUHH_REPO,"convert_model.py"); - assertTrue(Files.isReadable(replacerRepo)); assertTrue(Files.isReadable(converterRepo)); Path datasetRepo = Paths.get(TUHH_REPO,"dataset"); @@ -46,13 +44,7 @@ public void runPipeline() throws IOException { removeAndCreateDir(tuhh); converter=tuhh.resolve(converterRepo.getFileName()); - var replacer=tuhh.resolve(replacerRepo.getFileName()); Files.copy(converterRepo, converter, StandardCopyOption.REPLACE_EXISTING); - Files.copy(replacerRepo, replacer, StandardCopyOption.REPLACE_EXISTING); - - String oldRoot="root_dir = Path(r'put\\your\\root\\dir\\here') #set root dir"; - String newRoot="root_dir = \".\""; - replaceStringInFile(replacer, oldRoot, newRoot); List datasets = new ArrayList<>(); for (var dataset:datasetsRepo) { @@ -78,7 +70,6 @@ public void runPipeline() throws IOException { } Files.delete(converter); - Files.delete(replacer); } private void convertJsonToDFD(Path dataset) throws IOException { @@ -147,19 +138,6 @@ private void removeAndCreateDir(Path dir) throws IOException { } Files.createDirectories(dir); } - - private void replaceStringInFile(Path file, String oldString, String newString) throws IOException { - List lines = Files.readAllLines(file); - - for (int i = 0; i < lines.size(); i++) { - String line = lines.get(i); - if (line.contains(oldString)) { - lines.set(i, line.replace(oldString, newString)); - } - } - - Files.write(file, lines); - } private void copyDir(String sourceDirectoryLocation, String destinationDirectoryLocation) throws IOException { From 9abbc4df75c4029606b68da490a822cb363bc9cf Mon Sep 17 00:00:00 2001 From: uuqjz Date: Wed, 26 Jun 2024 16:14:10 +0200 Subject: [PATCH 3/4] Pretty pipeline --- .../converter/tests/TUHHPipeline.java | 212 ++++++++++-------- 1 file changed, 117 insertions(+), 95 deletions(-) diff --git a/tests/org.dataflowanalysis.converter.tests/src/org/dataflowanalysis/converter/tests/TUHHPipeline.java b/tests/org.dataflowanalysis.converter.tests/src/org/dataflowanalysis/converter/tests/TUHHPipeline.java index 052ffff..7294dab 100644 --- a/tests/org.dataflowanalysis.converter.tests/src/org/dataflowanalysis/converter/tests/TUHHPipeline.java +++ b/tests/org.dataflowanalysis.converter.tests/src/org/dataflowanalysis/converter/tests/TUHHPipeline.java @@ -20,131 +20,149 @@ import org.dataflowanalysis.converter.MicroSecEndConverter; public class TUHHPipeline { - - public static String TUHH_REPO = "microSecEnD-main"; + public static Path converter; - + @Disabled @Test public void runPipeline() throws IOException { - assertTrue(Files.isDirectory(Paths.get(TUHH_REPO))); - - var converterRepo=Paths.get(TUHH_REPO,"convert_model.py"); - assertTrue(Files.isReadable(converterRepo)); - - Path datasetRepo = Paths.get(TUHH_REPO,"dataset"); + var tuhhRepo = "microSecEnD-main"; + assertTrue(Files.isDirectory(Paths.get(tuhhRepo))); + + var converterRepo = Paths.get(tuhhRepo, "convert_model.py"); + assertTrue(Files.isRegularFile(converterRepo)); + + Path datasetFolderRepo = Paths.get(tuhhRepo, "dataset"); List datasetsRepo = new ArrayList<>(); - Files.list(datasetRepo).forEach(path -> { - if (Files.isDirectory(path)) { - datasetsRepo.add(path); - }}); - assertEquals(datasetsRepo.size(),17); - + Files.list(datasetFolderRepo) + .forEach(path -> { + if (Files.isDirectory(path)) { + datasetsRepo.add(path); + } + }); + assertEquals(datasetsRepo.size(), 17); + var tuhh = Paths.get("TUHH-Models"); removeAndCreateDir(tuhh); - - converter=tuhh.resolve(converterRepo.getFileName()); + + converter = tuhh.resolve(converterRepo.getFileName()); Files.copy(converterRepo, converter, StandardCopyOption.REPLACE_EXISTING); - + List datasets = new ArrayList<>(); - for (var dataset:datasetsRepo) { - var dirName=dataset.getFileName().toString(); - int underscoreIndex = dirName.indexOf('_'); - if (dirName.contains("kafka")) { - dirName = dirName.substring(0, underscoreIndex) + "-kafka"; - } - else { - dirName = dirName.substring(0, underscoreIndex); + for (var dataset : datasetsRepo) { + var datasetName = dataset.getFileName() + .toString(); + int underscoreIndex = datasetName.indexOf('_'); + if (datasetName.contains("kafka")) { + datasetName = datasetName.substring(0, underscoreIndex) + "-kafka"; + } else { + datasetName = datasetName.substring(0, underscoreIndex); } - var newDataset=tuhh.resolve(dirName); - copyDir(dataset.toString(),newDataset.toString()); + var newDataset = tuhh.resolve(datasetName); + copyDir(dataset.toString(), newDataset.toString()); datasets.add(newDataset); } - for(var dataset:datasets) { + for (var dataset : datasets) { cleanTopLevelOfDataset(dataset); renameTxtVariants(dataset.resolve("model_variants")); moveTxtVariantsUp(dataset.resolve("model_variants")); convertTxtToJson(dataset); convertJsonToDFD(dataset); } - + Files.delete(converter); } private void convertJsonToDFD(Path dataset) throws IOException { var microConverter = new MicroSecEndConverter(); - Files.list(dataset).forEach(path -> { - if (Files.isRegularFile(path) && path.toString().endsWith(".json")) { - System.out.println(path); - var complete = microConverter.microToDfd(path.toString()); - microConverter.storeDFD(complete, path.toString()); - }}); + Files.list(dataset) + .forEach(path -> { + if (Files.isRegularFile(path) && path.toString() + .endsWith(".json")) { + System.out.println(path); + var complete = microConverter.microToDfd(path.toString()); + microConverter.storeDFD(complete, path.toString()); + } + }); } private void convertTxtToJson(Path dataset) throws IOException { - Files.list(dataset).forEach(path -> { - if (Files.isRegularFile(path)) { - if(path.toString().endsWith(".txt")) { - System.out.println(path); - try { - runPythonScript(converter.toString(), path.toString(), "json", path.toString().replace(".txt", ".json")); - Files.delete(path); - } catch (InterruptedException | IOException e) {} - } - }}); + Files.list(dataset) + .forEach(path -> { + if (Files.isRegularFile(path)) { + if (path.toString() + .endsWith(".txt")) { + System.out.println(path); + try { + runPythonScript(converter.toString(), path.toString(), "json", path.toString() + .replace(".txt", ".json")); + Files.delete(path); + } catch (InterruptedException | IOException e) { + } + } + } + }); } - private void moveTxtVariantsUp (Path dir) throws IOException { + private void moveTxtVariantsUp(Path dir) throws IOException { Path parentDir = dir.getParent(); - - Files.list(dir).forEach(path -> { - if (Files.isRegularFile(path)) { - Path targetPath = parentDir.resolve(path.getFileName()); - try { - Files.move(path, targetPath, StandardCopyOption.REPLACE_EXISTING); - } catch (IOException e) {} - }}); - + + Files.list(dir) + .forEach(path -> { + if (Files.isRegularFile(path)) { + Path targetPath = parentDir.resolve(path.getFileName()); + try { + Files.move(path, targetPath, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + } + } + }); + Files.delete(dir); } private void cleanTopLevelOfDataset(Path dataset) throws IOException { - Files.list(dataset).forEach(path -> { - if (Files.isRegularFile(path)) { - if(!path.toString().endsWith(".json") || path.toString().endsWith("traceability.json")) { - try { - Files.delete(path); - } catch (IOException e) {} - } - else { - var renamedBaseModel = dataset.resolve(dataset.getFileName()+"_0.json"); - try { - Files.move(path, renamedBaseModel, StandardCopyOption.REPLACE_EXISTING); - } catch (IOException e) {} - } - }}); + Files.list(dataset) + .forEach(path -> { + if (Files.isRegularFile(path)) { + if (!path.toString() + .endsWith(".json") || path.toString() + .endsWith("traceability.json")) { + try { + Files.delete(path); + } catch (IOException e) { + } + } else { + var renamedBaseModel = dataset.resolve(dataset.getFileName() + "_0.json"); + try { + Files.move(path, renamedBaseModel, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + } + } + } + }); } private void removeAndCreateDir(Path dir) throws IOException { - if (Files.exists(dir)) { + if (Files.exists(dir)) { Files.walk(dir) - .sorted(Comparator.reverseOrder()) - .forEach(path -> { - try { - Files.delete(path); - } catch (IOException e) {} - }); - } + .sorted(Comparator.reverseOrder()) + .forEach(path -> { + try { + Files.delete(path); + } catch (IOException e) { + } + }); + } Files.createDirectories(dir); } - - private void copyDir(String sourceDirectoryLocation, String destinationDirectoryLocation) - throws IOException { - Files.walk(Paths.get(sourceDirectoryLocation)) + + private void copyDir(String sourceDirectoryLocation, String destinationDirectoryLocation) throws IOException { + Files.walk(Paths.get(sourceDirectoryLocation)) .forEach(source -> { Path destination = Paths.get(destinationDirectoryLocation, source.toString() - .substring(sourceDirectoryLocation.length())); + .substring(sourceDirectoryLocation.length())); try { Files.copy(source, destination); } catch (IOException e) { @@ -152,26 +170,30 @@ private void copyDir(String sourceDirectoryLocation, String destinationDirectory } }); } - + private void renameTxtVariants(Path variants) throws IOException { - var modelName=variants.getParent().getFileName(); - Files.list(variants).forEach(path -> { - if (Files.isRegularFile(path)) { - var renamedModel = variants.resolve(modelName+"_"+path.getFileName()); - try { - Files.move(path, renamedModel, StandardCopyOption.REPLACE_EXISTING); - } catch (IOException e) {} - }}); + var modelName = variants.getParent() + .getFileName(); + Files.list(variants) + .forEach(path -> { + if (Files.isRegularFile(path)) { + var renamedModel = variants.resolve(modelName + "_" + path.getFileName()); + try { + Files.move(path, renamedModel, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + } + } + }); } - + private int runPythonScript(String script, String in, String format, String out) throws InterruptedException, IOException { String[] command = {"python3", script, in, format, "-op", out}; ProcessBuilder processBuilder = new ProcessBuilder(command); Process process; - + process = processBuilder.start(); return process.waitFor(); - + } } From 31a8b62ed2f5774d7f4548052151b25dbd59da7e Mon Sep 17 00:00:00 2001 From: uuqjz Date: Thu, 27 Jun 2024 08:58:34 +0200 Subject: [PATCH 4/4] Logger --- .../dataflowanalysis/converter/tests/TUHHPipeline.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/org.dataflowanalysis.converter.tests/src/org/dataflowanalysis/converter/tests/TUHHPipeline.java b/tests/org.dataflowanalysis.converter.tests/src/org/dataflowanalysis/converter/tests/TUHHPipeline.java index 7294dab..483a9c2 100644 --- a/tests/org.dataflowanalysis.converter.tests/src/org/dataflowanalysis/converter/tests/TUHHPipeline.java +++ b/tests/org.dataflowanalysis.converter.tests/src/org/dataflowanalysis/converter/tests/TUHHPipeline.java @@ -16,12 +16,13 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; - +import org.apache.log4j.Logger; import org.dataflowanalysis.converter.MicroSecEndConverter; public class TUHHPipeline { public static Path converter; + private final Logger logger = Logger.getLogger(TUHHPipeline.class); @Disabled @Test @@ -64,6 +65,7 @@ public void runPipeline() throws IOException { } for (var dataset : datasets) { + assertTrue(Files.isDirectory(dataset.resolve("model_variants"))); cleanTopLevelOfDataset(dataset); renameTxtVariants(dataset.resolve("model_variants")); moveTxtVariantsUp(dataset.resolve("model_variants")); @@ -80,7 +82,7 @@ private void convertJsonToDFD(Path dataset) throws IOException { .forEach(path -> { if (Files.isRegularFile(path) && path.toString() .endsWith(".json")) { - System.out.println(path); + logger.info(path); var complete = microConverter.microToDfd(path.toString()); microConverter.storeDFD(complete, path.toString()); } @@ -93,7 +95,7 @@ private void convertTxtToJson(Path dataset) throws IOException { if (Files.isRegularFile(path)) { if (path.toString() .endsWith(".txt")) { - System.out.println(path); + logger.info(path); try { runPythonScript(converter.toString(), path.toString(), "json", path.toString() .replace(".txt", ".json"));