Merge pull request #62 from profeg/60-ParseCRLF

#60 issue resolving
orientechnologies · Apr 23, 2015 · 960f5dd · 960f5dd
2 parents a345513 + 110a89e
commit 960f5dd
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 12 deletions.
diff --git a/src/main/java/com/orientechnologies/orient/etl/extractor/ORowExtractor.java b/src/main/java/com/orientechnologies/orient/etl/extractor/ORowExtractor.java
@@ -94,11 +94,24 @@ protected OExtractedItem fetchNext() throws IOException {
     if (!bReader.ready())
       return null;
 
-    final String line = bReader.readLine();
+      final String line = getCheckedString();
 
     if( line == null || line.isEmpty())
       return null;
 
     return new OExtractedItem(current++, line);
   }
+
+    protected String getCheckedString() throws IOException {
+        StringBuilder sbLine = new StringBuilder();
+        boolean isOpenQuote = false;
+        do {
+            if (isOpenQuote) sbLine.append("\r\n");
+            isOpenQuote = false;
+            sbLine.append(bReader.readLine());
+            if ("null".equals(sbLine.toString())) return null;
+            for (char c : sbLine.toString().toCharArray()) if ('"' == c) isOpenQuote = !isOpenQuote;
+        } while (isOpenQuote);
+        return sbLine.toString();
+    }
 }
diff --git a/src/test/java/com/orientechnologies/orient/etl/transformer/OCSVTransformerTest.java b/src/test/java/com/orientechnologies/orient/etl/transformer/OCSVTransformerTest.java
@@ -312,17 +312,16 @@ public void testEndingSpaceInFieldName() {
     assertEquals(new Integer(1), (Integer) doc.field("num "));
   }
 
-  // @Test
-  // Temporary disabled on April 13th 2015 by Luca
-//  public void testCRLFIWithinQuotes() {
-//    String cfgJson = "{source: { content: { value: 'id ,text ,num \r\n1,\"my test\r\n text\",1\r\n'} }, extractor : { row : {} }, transformers : [{ csv : {} }], loader : { test: {} } }";
-//    process(cfgJson);
-//    List<ODocument> res = getResult();
-//    ODocument doc = res.get(0);
-//    assertEquals(new Integer(1), (Integer) doc.field("id "));
-//    assertEquals("my test\r\n text", (String) doc.field("text "));
-//    assertEquals(new Integer(1), (Integer) doc.field("num "));
-//  }
+    @Test
+    public void testCRLFIWithinQuotes() {
+        String cfgJson = "{source: { content: { value: 'id ,text ,num \r\n1,\"my test\r\n text\",1\r\n'} }, extractor : { row : {} }, transformers : [{ csv : {} }], loader : { test: {} } }";
+        process(cfgJson);
+        List<ODocument> res = getResult();
+        ODocument doc = res.get(0);
+        assertEquals(new Integer(1), (Integer) doc.field("id "));
+        assertEquals("my test\r\n text", (String) doc.field("text "));
+        assertEquals(new Integer(1), (Integer) doc.field("num "));
+    }
 
   @Test
   public void testEscapingDoubleQuotes() {