feat(tokenizer): ensure distinct tokenization results

Updated the tokenizer methods in TokenizerProcessor.kt to return distinct results. This change ensures that the tokenization process does not produce duplicate tokens. The distinct operation was removed from the RegexpTokenizer.kt as it is now handled in the main tokenizer methods.
phodal · Oct 4, 2024 · 09746ca · 09746ca
1 parent 59c1f25
commit 09746ca
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 6 deletions.
diff --git a/core/src/main/kotlin/com/phodal/shirecore/search/tokenizer/RegexpTokenizer.kt b/core/src/main/kotlin/com/phodal/shirecore/search/tokenizer/RegexpTokenizer.kt
@@ -38,7 +38,7 @@ open class RegexpTokenizer(opts: RegexTokenizerOptions? = null) : Tokenizer {
             results.ifEmpty { emptyList() }
         }
 
-        return output.distinct()
+        return output
     }
 
     fun without(arr: List<String>, vararg values: String): List<String> {

diff --git a/shirelang/src/main/kotlin/com/phodal/shirelang/compiler/hobbit/execute/TokenizerProcessor.kt b/shirelang/src/main/kotlin/com/phodal/shirelang/compiler/hobbit/execute/TokenizerProcessor.kt
@@ -19,22 +19,22 @@ class TokenizerProcessor {
                     }
                 )
 
-                return tokenizer.tokenize(action.text)
+                return tokenizer.tokenize(action.text).distinct()
             }
 
             when (action.tokType) {
                 "word" -> {
                     val tokenizer = WordTokenizer()
-                    return tokenizer.tokenize(action.text)
+                    return tokenizer.tokenize(action.text).distinct()
                 }
 
                 "naming" -> {
                     val tokenizer = CodeNamingTokenizer()
-                    return tokenizer.tokenize(action.text)
+                    return tokenizer.tokenize(action.text).distinct()
                 }
 
                 "stopwords" -> {
-                    return StopwordsBasedTokenizer.instance().tokenize(action.text)
+                    return StopwordsBasedTokenizer.instance().tokenize(action.text).distinct()
                 }
 
                 "jieba" -> {
@@ -48,7 +48,7 @@ class TokenizerProcessor {
 
                 else -> {
                     val tokenizer = WordTokenizer()
-                    return tokenizer.tokenize(action.text)
+                    return tokenizer.tokenize(action.text).distinct()
                 }
             }
         }