Added missing defaults in Tokenizer

JohnSnowLabs · Aug 18, 2019 · 76eaf77 · 76eaf77
1 parent 51f1f19
commit 76eaf77
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/src/main/scala/com/johnsnowlabs/nlp/annotators/Tokenizer.scala b/src/main/scala/com/johnsnowlabs/nlp/annotators/Tokenizer.scala
@@ -96,6 +96,12 @@ class Tokenizer(override val uid: String) extends AnnotatorApproach[TokenizerMod
     $(splitChars)
   }
 
+  setDefault(
+    targetPattern -> "\\S+",
+    contextChars -> Array(".", ",", ";", ":", "!", "?", "*", "-", "(", ")", "\"", "'"),
+    caseSensitiveExceptions -> true
+  )
+
   def buildRuleFactory: RuleFactory = {
     val rules = ArrayBuffer.empty[String]