adds support for multiple file inputs as a list. Invalid.txt has the …

…input file as a reference
gregv12 · Dec 21, 2023 · f61493d · f61493d
1 parent f88402c
commit f61493d
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 15 deletions.
diff --git a/server/src/main/java/com/fluxtion/extension/csvcompiler/Main.java b/server/src/main/java/com/fluxtion/extension/csvcompiler/Main.java
@@ -17,10 +17,10 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.HashMap;
+import java.util.List;
 import java.util.concurrent.atomic.LongAdder;
 
 import static com.fluxtion.extension.csvcompiler.Version.VERSION;
-// some exports omitted for the sake of brevity
 
 @Command(name = "csvCheck", version = VERSION, mixinStandardHelpOptions = true)
 public class Main implements Runnable {
@@ -224,9 +224,9 @@ public class Main implements Runnable {
     @Parameters(paramLabel = "<check config>", defaultValue = "processConfig.yaml", index = "0",
             description = "Configuration of csv check logic")
     private File configFile;
-    @Parameters(paramLabel = "<csv data>", defaultValue = "data.csv", index = "1",
-            description = "csv data file")
-    private File dataFile;
+    @Parameters(paramLabel = "<input data>", defaultValue = "data.csv", description = "input data file list", index = "1..*")
+    private List<File> dataFiles;
+    private String currentDataFile = "";
 
     public static void main(String[] args) {
         int exitCode = new CommandLine(new Main()).execute(args);
@@ -250,7 +250,6 @@ public void run() {
     @SneakyThrows
     private void process() {
         System.out.println("config: " + configFile.getAbsolutePath());
-        System.out.println("data  : " + dataFile.getAbsolutePath());
 
         Path resultsDir = Paths.get("results");
         if (!Files.exists(resultsDir)) {
@@ -265,7 +264,6 @@ private void process() {
         LongAdder invalidCount = new LongAdder();
         LongAdder validCount = new LongAdder();
         var metaMap = new HashMap<String, String>();
-        metaMap.put("dataFile", dataFile.getName());
         metaMap.put("configFile", configFile.getName());
 
         RowMarshaller<FieldAccessor> rowMarshaller = CsvChecker.fromYaml(new FileReader(configFile));
@@ -276,16 +274,30 @@ private void process() {
                     @Override
                     public void logFatal(CsvProcessingException csvProcessingException) {
                         invalidCount.increment();
-                        Main.this.write(writerInvalid, csvProcessingException.getMessage());
+                        Main.this.write(writerInvalid,csvProcessingException.getMessage());
                     }
 
                     @Override
                     public void logWarning(CsvProcessingException csvProcessingException) {
                         invalidCount.increment();
-                        Main.this.write(writerInvalid, csvProcessingException.getMessage());
+                        Main.this.write(writerInvalid,csvProcessingException.getMessage());
                     }
-                })
-                .stream(new FileReader(dataFile))
+                });
+        for (File file : dataFiles) {
+            metaMap.put("dataFile", file.getName());
+            processFile(rowMarshaller, file, writer, validCount);
+        }
+        writer.flush();
+        writerInvalid.flush();
+        System.out.println("Valid count  : " + validCount.intValue());
+        System.out.println("Invalid count: " + invalidCount.intValue());
+    }
+
+    @SneakyThrows
+    private void processFile(RowMarshaller<FieldAccessor> rowMarshaller, File dataFile, Writer writer, LongAdder validCount ){
+        currentDataFile = dataFile.getAbsolutePath();
+        System.out.println("data  : " + currentDataFile);
+        rowMarshaller.stream(new FileReader(dataFile))
                 .forEach(r -> {
                     try {
                         validCount.increment();
@@ -294,16 +306,14 @@ public void logWarning(CsvProcessingException csvProcessingException) {
                         throw new RuntimeException(e);
                     }
                 });
-        writer.flush();
-        writerInvalid.flush();
-        System.out.println("Valid count  : " + validCount.intValue());
-        System.out.println("Invalid count: " + invalidCount.intValue());
     }
 
     @SneakyThrows
     private void write(Writer writer, String message) {
         writer.write(message);
         writer.write('\n');
+        writer.write("file:" + currentDataFile);
+        writer.write('\n');
     }
 
     private void printSample() {

diff --git a/server/src/test/sample/processConfig.yaml b/server/src/test/sample/processConfig.yaml
@@ -2,6 +2,7 @@ name: Royalty
 trim: true
 
 columns:
+#  dataFile: {type: String, lookupTable: meta, defaultValue: dataFile, optional: true}
   ageInYears: {type: int, csvColumnName: 'latest age', optional: true, defaultValue: 50, validationFunction: checkAge}
   name:
     defaultValue: testing
@@ -12,7 +13,6 @@ columns:
   registered: {type: int, lookupTable: registeredId, defaultValue: unknown, validationFunction: checkRegistered}
   resident: {type: boolean}
   town: {type: string, converterFunction: toLowerCase}
-  dataFile: {type: String, lookupTable: meta, defaultValue: dataFile, optional: true}
 
 derivedColumns:
   nameAndTown: