Fix duplicate values in org.apache.lucene.analysis.ko.dict.UserDictio…

…nary (#13427)

Remove incorrect assertion in org.apache.lucene.analysis.ko.dict.UserDictionary, and replace with array copy if duplicate values are passed.

lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java

@@ -23,6 +23,7 @@
 import java.util.Comparator;
 import java.util.List;
 import org.apache.lucene.analysis.ko.POS;
+import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.IntsRefBuilder;
 import org.apache.lucene.util.fst.FST;
 import org.apache.lucene.util.fst.FSTCompiler;
@@ -138,10 +139,12 @@ private UserDictionary(List<String> entries) throws IOException {
       lastToken = token;
       ord++;
     }
+    if (entryIndex < rightIds.length) {
+      rightIds = ArrayUtil.copyOfSubArray(rightIds, 0, entryIndex);
+    }
     this.fst =
         new TokenInfoFST(FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader()));
     this.segmentations = segmentations.toArray(new int[segmentations.size()][]);
-    assert entryIndex == rightIds.length;
     this.rightIds = rightIds;
   }
 

lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/TestKoreanTokenizer.java

@@ -20,6 +20,7 @@
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.Reader;
+import java.io.StringReader;
 import java.nio.charset.StandardCharsets;
 import java.util.Random;
 import org.apache.lucene.analysis.Analyzer;
@@ -593,6 +594,22 @@ public void testCombining() throws IOException {
         new POS.Tag[] {POS.Tag.SL});
   }
 
+  public void testDuplicate() throws IOException {
+    String s = "c++\nC쁠쁠\n세종\n세종\n세종시 세종 시";
+    try (Reader rulesReader = new StringReader(s)) {
+      var dict = UserDictionary.open(rulesReader);
+      assertTrue(dict.getRightId(3) != 0);
+      assertThrows(ArrayIndexOutOfBoundsException.class, () -> dict.getRightId(4));
+    }
+
+    String dupdup = "c++\nC쁠쁠\n세종\n세종\n세종시 세종 시\n세종시 세종 시";
+    try (Reader rulesReader = new StringReader(dupdup)) {
+      var dict = UserDictionary.open(rulesReader);
+      assertTrue(dict.getRightId(3) != 0);
+      assertThrows(ArrayIndexOutOfBoundsException.class, () -> dict.getRightId(4));
+    }
+  }
+
   private void assertReadings(Analyzer analyzer, String input, String... readings)
       throws IOException {
     try (TokenStream ts = analyzer.tokenStream("ignored", input)) {