Add User Dictionary Rules to NoriTokenizer (#3634)

Relates #3615, elastic/elasticsearch#3620
elastic · May 8, 2019 · 8ace22d · 8ace22d
1 parent 74efe05
commit 8ace22d
Show file tree

Hide file tree

Showing 2 changed files with 45 additions and 1 deletion.
diff --git a/src/Nest/Analysis/Tokenizers/NoriTokenizer.cs b/src/Nest/Analysis/Tokenizers/NoriTokenizer.cs
@@ -1,4 +1,5 @@
-using System.Runtime.Serialization;
+using System.Collections.Generic;
+using System.Runtime.Serialization;
 using Elasticsearch.Net;
 
 namespace Nest
@@ -36,6 +37,16 @@ public interface INoriTokenizer : ITokenizer
 		/// </summary>
 		[DataMember(Name = "user_dictionary")]
 		string UserDictionary { get; set; }
+
+		/// <summary>
+		/// The Nori tokenizer uses the mecab-ko-dic dictionary by default. A user_dictionary with custom nouns (NNG)
+		/// can be specified inline with this property
+		/// </summary>
+		/// <remarks>
+		/// Valid for Elasticsearch 6.6.0+
+		/// </remarks>
+		[DataMember(Name = "user_dictionary_rules")]
+		IEnumerable<string> UserDictionaryRules { get; set; }
 	}
 
 	/// <inheritdoc cref="INoriTokenizer" />
@@ -48,6 +59,9 @@ public class NoriTokenizer : TokenizerBase, INoriTokenizer
 
 		/// <inheritdoc cref="INoriTokenizer.UserDictionary" />
 		public string UserDictionary { get; set; }
+
+		/// <inheritdoc cref="INoriTokenizer.UserDictionaryRules" />
+		public IEnumerable<string> UserDictionaryRules { get; set; }
 	}
 
 	/// <inheritdoc cref="INoriTokenizer" />
@@ -58,11 +72,18 @@ public class NoriTokenizerDescriptor
 
 		NoriDecompoundMode? INoriTokenizer.DecompoundMode { get; set; }
 		string INoriTokenizer.UserDictionary { get; set; }
+		IEnumerable<string> INoriTokenizer.UserDictionaryRules { get; set; }
 
 		/// <inheritdoc cref="INoriTokenizer.DecompoundMode" />
 		public NoriTokenizerDescriptor DecompoundMode(NoriDecompoundMode? mode) => Assign(mode, (a, v) => a.DecompoundMode = v);
 
 		/// <inheritdoc cref="INoriTokenizer.UserDictionary" />
 		public NoriTokenizerDescriptor UserDictionary(string path) => Assign(path, (a, v) => a.UserDictionary = v);
+
+		/// <inheritdoc cref="INoriTokenizer.UserDictionaryRules" />
+		public NoriTokenizerDescriptor UserDictionaryRules(params string[] rules) => Assign(rules, (a, v) => a.UserDictionaryRules = v);
+
+		/// <inheritdoc cref="INoriTokenizer.UserDictionaryRules" />
+		public NoriTokenizerDescriptor UserDictionaryRules(IEnumerable<string> rules) => Assign(rules, (a, v) => a.UserDictionaryRules = v);
 	}
 }
diff --git a/src/Tests/Tests/Analysis/Tokenizers/TokenizerTests.cs b/src/Tests/Tests/Analysis/Tokenizers/TokenizerTests.cs
@@ -222,6 +222,29 @@ public class NoriTests : TokenizerAssertionBase<NoriTests>
 			public override string Name => "nori";
 		}
 
+		[SkipVersion("<6.6.0", "inline user dictionary rules introduced in 6.6.0")]
+		public class NoriWithUserDictionaryTests : TokenizerAssertionBase<NoriWithUserDictionaryTests>
+		{
+			public override FuncTokenizer Fluent => (n, t) => t.Nori(n, e => e
+				.DecompoundMode(NoriDecompoundMode.Mixed)
+				.UserDictionaryRules("c++", "C샤프", "세종", "세종시 세종 시")
+			);
+
+			public override ITokenizer Initializer => new NoriTokenizer
+			{
+				DecompoundMode = NoriDecompoundMode.Mixed,
+				UserDictionaryRules = new [] { "c++", "C샤프", "세종", "세종시 세종 시" }
+			};
+
+			public override object Json => new
+			{
+				type = "nori_tokenizer",
+				decompound_mode = "mixed",
+				user_dictionary_rules = new [] { "c++", "C샤프", "세종", "세종시 세종 시" }
+			};
+			public override string Name => "nori_userdictionary";
+		}
+
 		[SkipVersion("<6.4.0", "char_group introduced in 6.4.0")]
 		public class CharGroupTests : TokenizerAssertionBase<CharGroupTests>
 		{