post: apply string normalization in deduplication script

pelias · Jan 10, 2022 · 94a7f30 · 94a7f30
1 parent 63ce210
commit 94a7f30
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 6 deletions.
diff --git a/post/deduplication.js b/post/deduplication.js
@@ -4,16 +4,24 @@
  */
 
 const _ = require('lodash');
-const prefixes = [ 'name', 'phrase', 'address_parts' ];
+const prefixes = ['name', 'phrase', 'address_parts'];
+const punctuation = /[\.]+/g;
+const normalize = (v) => _.isString(v) ? _.replace(v.toLowerCase(), punctuation, '') : v;
 
-function deduplication( doc ){
+// if values are strings then apply a string
+// normalization function to both strings.
+const comparitor = (value, other) => {
+  return _.isEqual(normalize(value), normalize(other));
+};
+
+function deduplication(doc) {
   prefixes.forEach(prefix => {
     let index = doc[prefix];
-    if ( _.isPlainObject( index ) ){
-      for( let field in index ){
+    if (_.isPlainObject(index)) {
+      for (let field in index) {
         let values = index[field];
-        if( _.isArray( values ) && values.length > 1 ){
-          index[field] = _.uniq(values);
+        if (_.isArray(values) && values.length > 1) {
+          index[field] = _.uniqWith(values, comparitor);
         }
       }
     }

diff --git a/test/post/deduplication.js b/test/post/deduplication.js
@@ -13,6 +13,7 @@ module.exports.tests.dedupe = function (test) {
     doc.setNameAlias('default', 'test');
     doc.setNameAlias('default', 'test 2');
     doc.setNameAlias('default', 'test');
+    doc.setNameAlias('default', '.Test.');
 
     deduplication(doc);
     t.deepEquals(doc.name.default, ['test', 'test 2']);
@@ -27,6 +28,7 @@ module.exports.tests.dedupe = function (test) {
     doc.setAddressAlias('street', 'test');
     doc.setAddressAlias('street', 'test 2');
     doc.setAddressAlias('street', 'test');
+    doc.setAddressAlias('street', '..Test..');
 
     deduplication(doc);
     t.deepEquals(doc.address_parts.street, ['test', 'test 2']);
@@ -42,6 +44,7 @@ module.exports.tests.dedupe = function (test) {
     doc.setNameAlias('default', 'test');
     doc.setNameAlias('default', 'test 2');
     doc.setNameAlias('default', 'test');
+    doc.setNameAlias('default', '...Te...st...');
 
     deduplication(doc);
     t.deepEquals(doc.phrase.default, ['test', 'test 2']);