LUCENE-10312: MIGRATE entry and small follow-ups (#908)

apache · May 21, 2022 · 71a9acb · 71a9acb
1 parent 63b66e0
commit 71a9acb
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 3 deletions.
diff --git a/lucene/MIGRATE.md b/lucene/MIGRATE.md
@@ -19,6 +19,11 @@
 
 ## Migration from Lucene 9.x to Lucene 10.0
 
+### PersianStemFilter is added to PersianAnalyzer (LUCENE-10312)
+
+PersianAnalyzer now includes PersianStemFilter, that would change analysis results. If you need the exactly same analysis
+behaviour as 9.x, clone `PersianAnalyzer` in 9.x or create custom analyzer by using `CustomAnalyzer` on your own. 
+
 ### AutomatonQuery/CompiledAutomaton/RunAutomaton/RegExp no longer determinize (LUCENE-10010)
 
 These classes no longer take a `determinizeWorkLimit` and no longer determinize

diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java
@@ -121,8 +121,8 @@ public PersianAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
    *
    * @return {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} built from a {@link
    *     StandardTokenizer} filtered with {@link LowerCaseFilter}, {@link DecimalDigitFilter},
-   *     {@link ArabicNormalizationFilter}, {@link PersianNormalizationFilter} and Persian Stop
-   *     words
+   *     {@link ArabicNormalizationFilter}, {@link PersianNormalizationFilter}, Persian Stop words,
+   *     and {@link PersianStemFilter}.
    */
   @Override
   protected TokenStreamComponents createComponents(String fieldName) {

diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianStemFilter.java
@@ -32,7 +32,14 @@ public class TestPersianStemFilter extends BaseTokenStreamTestCase {
   @Override
   public void setUp() throws Exception {
     super.setUp();
-    a = new PersianAnalyzer();
+    a =
+        new Analyzer() {
+          @Override
+          protected TokenStreamComponents createComponents(String fieldName) {
+            final Tokenizer source = new MockTokenizer();
+            return new TokenStreamComponents(source, new PersianStemFilter(source));
+          }
+        };
   }
 
   @Override