chore(nlp): stemmer and normalizer tests implemented

hubtype · Jan 15, 2021 · faa1fb3 · faa1fb3
1 parent 00a87db
commit faa1fb3
Show file tree

Hide file tree

Showing 5 changed files with 24 additions and 4 deletions.
diff --git a/packages/botonic-plugin-contentful/package-lock.json b/packages/botonic-plugin-contentful/package-lock.json
diff --git a/packages/botonic-plugin-contentful/src/nlp/stemmer.ts b/packages/botonic-plugin-contentful/src/nlp/stemmer.ts
@@ -96,8 +96,8 @@ const stemmers = new SingletonMap<Stemmer>({
     return new StemmerSl()
   },
   [locales.HUNGARIAN]: () => {
-    // eslint-disable-next-line @typescript-eslint/no-var-requires,node/no-missing-require
-    const { StemmerHu } = require('@nlpjs/lang-hu/src/stemmer-hu')
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const StemmerHu = require('@nlpjs/lang-hu/src/stemmer-hu')
     return new StemmerHu()
   },
 })

diff --git a/packages/botonic-plugin-contentful/src/nlp/tokens.ts b/packages/botonic-plugin-contentful/src/nlp/tokens.ts
@@ -164,8 +164,8 @@ const lazyTokenizers = new SingletonMap<Tokenizer>({
     return new TokenizerSl()
   },
   [locales.HUNGARIAN]: () => {
-    // eslint-disable-next-line @typescript-eslint/no-var-requires,node/no-missing-require
-    const { TokenizerHu } = require('@nlpjs/lang-hu/src/tokenizer-hu')
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const TokenizerHu = require('@nlpjs/lang-hu/src/tokenizer-hu')
     return new TokenizerHu()
   },
 })

diff --git a/packages/botonic-plugin-contentful/tests/nlp/normalizer.test.ts b/packages/botonic-plugin-contentful/tests/nlp/normalizer.test.ts
@@ -128,6 +128,16 @@ test.each<any>([
       new Word('narediti', 'nared'),
     ],
   ],
+  [
+    'hu',
+    'ne csinálj semmi hülyeséget',
+    [
+      Word.StopWord('ne'),
+      new Word('csinalj', 'csinalj'),
+      Word.StopWord('semmi'),
+      new Word('hulyeseget', 'hulyeseg'),
+    ],
+  ],
 ])(
   'TEST: stemmer removes stopwords: lang=%s input="%j"',
   (locale: string, raw: string, words: Word[]) => {

diff --git a/packages/botonic-plugin-contentful/tests/nlp/stemmer.test.ts b/packages/botonic-plugin-contentful/tests/nlp/stemmer.test.ts
@@ -35,6 +35,8 @@ test.each<any>([
   ['sk', 'európskej', ['europsk']],
   ['sl', 'Evropski', ['evrop']],
   ['sl', 'hoditi', ['hodit']],
+  ['hu', 'Európai', ['europ']],
+  ['hu', 'érvelni', ['erveln']],
 ])(
   'TEST: stemmer removes final letters(%s) =>%j',
   (locale: string, raw: string, expected: string) => {