Merge pull request #63 from JohnSnowLabs/remove_regex_ner

Remove Regex NER

python/sparknlp/annotator.py

@@ -234,36 +234,6 @@ class PerceptronModel(JavaModel, JavaMLWritable, JavaMLReadable, AnnotatorProper
     name = "PerceptronModel"
 
 
-class NERRegexApproach(JavaEstimator, JavaMLWritable, JavaMLReadable, AnnotatorProperties):
-    corpusPath = Param(Params._dummy(),
-                       "corpusPath",
-                       "corpus path",
-                       typeConverter=TypeConverters.toString)
-
-    @keyword_only
-    def __init__(self):
-        super(NERRegexApproach, self).__init__()
-        self._java_obj = self._new_java_obj("com.johnsnowlabs.nlp.annotators.ner.regex.NERRegexApproach", self.uid)
-        kwargs = self._input_kwargs
-        self._setDefault(corpusPath="__default")
-        self.setParams(**kwargs)
-
-    def setParams(self, corpusPath="__default"):
-        kwargs = self._input_kwargs
-        return self._set(**kwargs)
-
-    def setCorpusPath(self, value):
-        self._set(corpusPath=value)
-        return self
-
-    def _create_model(self, java_model):
-        return NERRegexModel(java_model)
-
-
-class NERRegexModel(JavaModel, JavaMLWritable, JavaMLReadable, AnnotatorProperties):
-    name = "NERRegexModel"
-
-
 class SentenceDetectorModel(AnnotatorTransformer):
 
     useAbbreviations = Param(Params._dummy(),

python/test/annotators.py

@@ -135,32 +135,6 @@ def runTest(self):
         pos_tagger.transform(tokenized).show()
 
 
-class RegexNERApproachTestSpec(unittest.TestCase):
-
-    def setUp(self):
-        self.data = SparkContextForNER.data
-
-    def runTest(self):
-        document_assembler = DocumentAssembler() \
-            .setInputCol("_c0") \
-            .setOutputCol("document")
-        sentence_detector = SentenceDetectorModel() \
-            .setInputCols(["document"]) \
-            .setOutputCol("sentence")
-        tokenizer = RegexTokenizer() \
-            .setInputCols(["sentence"]) \
-            .setOutputCol("token")
-        ner_tagger = NERRegexApproach() \
-            .setInputCols(["sentence"]) \
-            .setOutputCol("NER") \
-            .fit(self.data)
-        assembled = document_assembler.transform(self.data)
-        sentenced = sentence_detector.transform(assembled)
-        tokenized = tokenizer.transform(sentenced)
-        result = ner_tagger.transform(tokenized)
-        result.select("NER").take(10)
-
-
 class PragmaticSBDTestSpec(unittest.TestCase):
 
     def setUp(self):

src/test/scala/com/johnsnowlabs/nlp/AnnotatorBuilder.scala

@@ -2,7 +2,6 @@ package com.johnsnowlabs.nlp
 
 import com.johnsnowlabs.nlp.annotators._
 import com.johnsnowlabs.nlp.annotators.ner.crf.{NerCrfApproach, NerCrfModel}
-import com.johnsnowlabs.nlp.annotators.ner.regex.NERRegexApproach
 import com.johnsnowlabs.nlp.annotators.parser.dep.DependencyParser
 import com.johnsnowlabs.nlp.annotators.pos.perceptron.PerceptronApproach
 import com.johnsnowlabs.nlp.annotators.sbd.pragmatic.SentenceDetectorModel
@@ -124,16 +123,6 @@ object AnnotatorBuilder extends FlatSpec { this: Suite =>
     spellChecker.fit(withFullNormalizer(dataset)).transform(withFullNormalizer(dataset))
   }
 
-  def withNERTagger(dataset: Dataset[Row]): Dataset[Row] = {
-    val nerTagger = new NERRegexApproach()
-      .setInputCols(Array("sentence"))
-      .setOutputCol("ner")
-      .setCorpusPath("/ner-corpus/dict.txt")
-    nerTagger
-      .fit(withFullPragmaticSentenceDetector(dataset))
-      .transform(withFullPragmaticSentenceDetector(dataset))
-  }
-
   def withDependencyParser(dataset: Dataset[Row]): Dataset[Row] = {
     val df = withFullPOSTagger(withTokenizer(dataset))
     new DependencyParser()