varNew = new ArrayList();
+ String prevPart = str.substring(position, substPos);
+ for (int ii = 0; ii < variations.size(); ii++) {
+ String tmp = variations.get(ii);
+ varNew.add(tmp.concat(prevPart + getReplacements()[i]));
+ variations.set(ii, variations.get(ii) + prevPart + getPatterns()[i]);
+ }
+ variations.addAll(varNew);
+ position = substPos + getPatterns()[i].length();
+ } else {
+ for (int ii = 0; ii < variations.size(); ii++) {
+ variations.set(ii, variations.get(ii) + str.substring(position, str.length()));
+ }
+ position = str.length();
+ }
+ }
+ return variations;
+ }
+
+ private String substitute(String str) {
+ String s = expandUmlauts(str.toUpperCase(Locale.GERMAN));
+ s = removeSequences(s);
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < s.length(); i++) {
+ char current = s.charAt(i);
+ char next = i + 1 < s.length() ? s.charAt(i + 1) : '_';
+ char prev = i > 0 ? s.charAt(i - 1) : '_';
+ switch (current) {
+ case 'A':
+ case 'E':
+ case 'I':
+ case 'J':
+ case 'Y':
+ case 'O':
+ case 'U':
+ if (i == 0 || ((i == 1) && prev == 'H')) {
+ sb.append(getCode());
+ }
+ break;
+ case 'P':
+ sb.append(next == 'H' ? "33" : '1');
+ break;
+ case 'B':
+ sb.append('1');
+ break;
+ case 'D':
+ case 'T':
+ sb.append(csz.contains(next) ? '8' : '2');
+ break;
+ case 'F':
+ case 'V':
+ case 'W':
+ sb.append('3');
+ break;
+ case 'G':
+ case 'K':
+ case 'Q':
+ sb.append('4');
+ break;
+ case 'C':
+ if (i == 0) {
+ sb.append(ahkloqrux.contains(next) ? '4' : '8');
+ } else {
+ sb.append(aouhkxq.contains(next) ? '4' : '8');
+ }
+ if (sb.length() >= 2 && sb.charAt(sb.length() - 2) == '8') {
+ sb.setCharAt(sb.length() - 1, '8');
+ }
+ break;
+ case 'X':
+ sb.append(i < 1 || !ckq.contains(prev) ? "48" : '8');
+ break;
+ case 'L':
+ sb.append('5');
+ break;
+ case 'M':
+ case 'N':
+ sb.append('6');
+ break;
+ case 'R':
+ sb.append('7');
+ break;
+ case 'S':
+ case 'Z':
+ sb.append('8');
+ break;
+ case 'H':
+ break;
+ }
+ }
+ s = sb.toString();
+ s = removeSequences(s);
+ return s;
+ }
+
+ /**
+ *
+ * @param str
+ * @return
+ */
+ private String expandUmlauts(String str) {
+ return str.replaceAll("\u00C4", "AE").replaceAll("\u00D6", "OE").replaceAll("\u00DC", "UE");
+ }
+
+ /**
+ *
+ * @param str
+ * @return
+ */
+ private String removeSequences(String str) {
+ if (str == null || str.length() == 0) {
+ return "";
+ }
+ int i = 0, j = 0;
+ StringBuilder sb = new StringBuilder().append(str.charAt(i++));
+ char c;
+ while (i < str.length()) {
+ c = str.charAt(i);
+ if (c != sb.charAt(j)) {
+ sb.append(c);
+ j++;
+ }
+ i++;
+ }
+ return sb.toString();
+ }
+}
diff --git a/src/main/java/org/elasticsearch/index/analysis/phonetic/Nysiis.java b/src/main/java/org/elasticsearch/index/analysis/phonetic/Nysiis.java
new file mode 100644
index 0000000000000..6275b84677ace
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/analysis/phonetic/Nysiis.java
@@ -0,0 +1,325 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.elasticsearch.index.analysis.phonetic;
+
+import java.util.regex.Pattern;
+import org.apache.commons.codec.EncoderException;
+import org.apache.commons.codec.StringEncoder;
+
+/**
+ *
+ * Taken from commons-codec trunk (unreleased yet)
+ *
+ * Encodes a string into a NYSIIS value. NYSIIS is an encoding used to relate
+ * similar names, but can also be used as a general purpose scheme to find word
+ * with similar phonemes.
+ *
+ * NYSIIS features an accuracy increase of 2.7% over the traditional Soundex
+ * algorithm.
+ *
+ * Algorithm description:
+ *
+ * 1. Transcode first characters of name
+ * 1a. MAC -> MCC
+ * 1b. KN -> NN
+ * 1c. K -> C
+ * 1d. PH -> FF
+ * 1e. PF -> FF
+ * 1f. SCH -> SSS
+ * 2. Transcode last characters of name
+ * 2a. EE, IE -> Y
+ * 2b. DT,RT,RD,NT,ND -> D
+ * 3. First character of key = first character of name
+ * 4. Transcode remaining characters by following these rules, incrementing by one character each time
+ * 4a. EV -> AF else A,E,I,O,U -> A
+ * 4b. Q -> G
+ * 4c. Z -> S
+ * 4d. M -> N
+ * 4e. KN -> N else K -> C
+ * 4f. SCH -> SSS
+ * 4g. PH -> FF
+ * 4h. H -> If previous or next is nonvowel, previous
+ * 4i. W -> If previous is vowel, previous
+ * 4j. Add current to key if current != last key character
+ * 5. If last character is S, remove it
+ * 6. If last characters are AY, replace with Y
+ * 7. If last character is A, remove it
+ * 8. Collapse all strings of repeated characters
+ * 9. Add original first character of name as first character of key
+ *
+ *
+ * @see NYSIIS on Wikipedia
+ * @see NYSIIS on dropby.com
+ *
+ */
+public class Nysiis implements StringEncoder {
+
+ private static final char[] CHARS_A = new char[]{'A'};
+ private static final char[] CHARS_AF = new char[]{'A', 'F'};
+ private static final char[] CHARS_C = new char[]{'C'};
+ private static final char[] CHARS_FF = new char[]{'F', 'F'};
+ private static final char[] CHARS_G = new char[]{'G'};
+ private static final char[] CHARS_N = new char[]{'N'};
+ private static final char[] CHARS_NN = new char[]{'N', 'N'};
+ private static final char[] CHARS_S = new char[]{'S'};
+ private static final char[] CHARS_SSS = new char[]{'S', 'S', 'S'};
+ private static final Pattern PAT_MAC = Pattern.compile("^MAC");
+ private static final Pattern PAT_KN = Pattern.compile("^KN");
+ private static final Pattern PAT_K = Pattern.compile("^K");
+ private static final Pattern PAT_PH_PF = Pattern.compile("^(PH|PF)");
+ private static final Pattern PAT_SCH = Pattern.compile("^SCH");
+ private static final Pattern PAT_EE_IE = Pattern.compile("(EE|IE)$");
+ private static final Pattern PAT_DT_ETC = Pattern.compile("(DT|RT|RD|NT|ND)$");
+ private static final char SPACE = ' ';
+ private static final int TRUE_LENGTH = 6;
+
+ /**
+ * Tests if the given character is a vowel.
+ *
+ * @param c the character to test
+ * @return {@code true} if the character is a vowel, {@code false} otherwise
+ */
+ private static boolean isVowel(final char c) {
+ return c == 'A' || c == 'E' || c == 'I' || c == 'O' || c == 'U';
+ }
+
+ /**
+ * Transcodes the remaining parts of the String. The method operates on a
+ * sliding window, looking at 4 characters at a time: [i-1, i, i+1, i+2].
+ *
+ * @param prev the previous character
+ * @param curr the current character
+ * @param next the next character
+ * @param aNext the after next character
+ * @return a transcoded array of characters, starting from the current
+ * position
+ */
+ private static char[] transcodeRemaining(final char prev, final char curr, final char next, final char aNext) {
+ // 1. EV -> AF
+ if (curr == 'E' && next == 'V') {
+ return CHARS_AF;
+ }
+
+ // A, E, I, O, U -> A
+ if (isVowel(curr)) {
+ return CHARS_A;
+ }
+
+ // 2. Q -> G, Z -> S, M -> N
+ if (curr == 'Q') {
+ return CHARS_G;
+ } else if (curr == 'Z') {
+ return CHARS_S;
+ } else if (curr == 'M') {
+ return CHARS_N;
+ }
+
+ // 3. KN -> NN else K -> C
+ if (curr == 'K') {
+ if (next == 'N') {
+ return CHARS_NN;
+ } else {
+ return CHARS_C;
+ }
+ }
+
+ // 4. SCH -> SSS
+ if (curr == 'S' && next == 'C' && aNext == 'H') {
+ return CHARS_SSS;
+ }
+
+ // PH -> FF
+ if (curr == 'P' && next == 'H') {
+ return CHARS_FF;
+ }
+
+ // 5. H -> If previous or next is a non vowel, previous.
+ if (curr == 'H' && (!isVowel(prev) || !isVowel(next))) {
+ return new char[]{prev};
+ }
+
+ // 6. W -> If previous is vowel, previous.
+ if (curr == 'W' && isVowel(prev)) {
+ return new char[]{prev};
+ }
+
+ return new char[]{curr};
+ }
+ /**
+ * Indicates the strict mode.
+ */
+ private final boolean strict;
+
+ /**
+ * Creates an instance of the {@link Nysiis} encoder with strict mode
+ * (original form), i.e. encoded strings have a maximum length of 6.
+ */
+ public Nysiis() {
+ this(true);
+ }
+
+ /**
+ * Create an instance of the {@link Nysiis} encoder with the specified
+ * strict mode:
+ *
+ * - {@code true}: encoded strings have a maximum length of 6
- {@code false}:
+ * encoded strings may have arbitrary length
+ *
+ * @param strict the strict mode
+ */
+ public Nysiis(final boolean strict) {
+ this.strict = strict;
+ }
+
+ /**
+ * Encodes an Object using the NYSIIS algorithm. This method is provided in
+ * order to satisfy the requirements of the Encoder interface, and will
+ * throw an {@link EncoderException} if the supplied object is not of type
+ * {@link String}.
+ *
+ * @param obj Object to encode
+ * @return An object (or a {@link String}) containing the NYSIIS code which
+ * corresponds to the given String.
+ * @throws EncoderException if the parameter supplied is not of a {@link String}
+ * @throws IllegalArgumentException if a character is not mapped
+ */
+ @Override
+ public Object encode(Object obj) throws EncoderException {
+ if (!(obj instanceof String)) {
+ throw new EncoderException("Parameter supplied to Nysiis encode is not of type java.lang.String");
+ }
+ return this.nysiis((String) obj);
+ }
+
+ /**
+ * Encodes a String using the NYSIIS algorithm.
+ *
+ * @param str A String object to encode
+ * @return A Nysiis code corresponding to the String supplied
+ * @throws IllegalArgumentException if a character is not mapped
+ */
+ @Override
+ public String encode(String str) {
+ return this.nysiis(str);
+ }
+
+ /**
+ * Indicates the strict mode for this {@link Nysiis} encoder.
+ *
+ * @return {@code true} if the encoder is configured for strict mode, {@code false}
+ * otherwise
+ */
+ public boolean isStrict() {
+ return this.strict;
+ }
+
+ /**
+ * Retrieves the NYSIIS code for a given String object.
+ *
+ * @param str String to encode using the NYSIIS algorithm
+ * @return A NYSIIS code for the String supplied
+ */
+ public String nysiis(String str) {
+ if (str == null) {
+ return null;
+ }
+
+ // Use the same clean rules as Soundex
+ str = clean(str);
+
+ if (str.length() == 0) {
+ return str;
+ }
+
+ // Translate first characters of name:
+ // MAC -> MCC, KN -> NN, K -> C, PH | PF -> FF, SCH -> SSS
+ str = PAT_MAC.matcher(str).replaceFirst("MCC");
+ str = PAT_KN.matcher(str).replaceFirst("NN");
+ str = PAT_K.matcher(str).replaceFirst("C");
+ str = PAT_PH_PF.matcher(str).replaceFirst("FF");
+ str = PAT_SCH.matcher(str).replaceFirst("SSS");
+
+ // Translate last characters of name:
+ // EE -> Y, IE -> Y, DT | RT | RD | NT | ND -> D
+ str = PAT_EE_IE.matcher(str).replaceFirst("Y");
+ str = PAT_DT_ETC.matcher(str).replaceFirst("D");
+
+ // First character of key = first character of name.
+ StringBuffer key = new StringBuffer(str.length());
+ key.append(str.charAt(0));
+
+ // Transcode remaining characters, incrementing by one character each time
+ final char[] chars = str.toCharArray();
+ final int len = chars.length;
+
+ for (int i = 1; i < len; i++) {
+ final char next = i < len - 1 ? chars[i + 1] : SPACE;
+ final char aNext = i < len - 2 ? chars[i + 2] : SPACE;
+ final char[] transcoded = transcodeRemaining(chars[i - 1], chars[i], next, aNext);
+ System.arraycopy(transcoded, 0, chars, i, transcoded.length);
+
+ // only append the current char to the key if it is different from the last one
+ if (chars[i] != chars[i - 1]) {
+ key.append(chars[i]);
+ }
+ }
+
+ if (key.length() > 1) {
+ char lastChar = key.charAt(key.length() - 1);
+
+ // If last character is S, remove it.
+ if (lastChar == 'S') {
+ key.deleteCharAt(key.length() - 1);
+ lastChar = key.charAt(key.length() - 1);
+ }
+
+ if (key.length() > 2) {
+ final char last2Char = key.charAt(key.length() - 2);
+ // If last characters are AY, replace with Y.
+ if (last2Char == 'A' && lastChar == 'Y') {
+ key.deleteCharAt(key.length() - 2);
+ }
+ }
+
+ // If last character is A, remove it.
+ if (lastChar == 'A') {
+ key.deleteCharAt(key.length() - 1);
+ }
+ }
+
+ final String string = key.toString();
+ return this.isStrict() ? string.substring(0, Math.min(TRUE_LENGTH, string.length())) : string;
+ }
+
+ static String clean(String str) {
+ if (str == null || str.length() == 0) {
+ return str;
+ }
+ int len = str.length();
+ char[] chars = new char[len];
+ int count = 0;
+ for (int i = 0; i < len; i++) {
+ if (Character.isLetter(str.charAt(i))) {
+ chars[count++] = str.charAt(i);
+ }
+ }
+ if (count == len) {
+ return str.toUpperCase(java.util.Locale.ENGLISH);
+ }
+ return new String(chars, 0, count).toUpperCase(java.util.Locale.ENGLISH);
+ }
+}
\ No newline at end of file
diff --git a/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java b/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java
index 11f0799781ba3..a5aef2de90cef 100644
--- a/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java
+++ b/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java
@@ -2,6 +2,8 @@
import org.elasticsearch.common.inject.Injector;
import org.elasticsearch.common.inject.ModulesBuilder;
+import static org.elasticsearch.common.settings.ImmutableSettings.*;
+import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsModule;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.EnvironmentModule;
@@ -10,29 +12,38 @@
import org.elasticsearch.index.settings.IndexSettingsModule;
import org.elasticsearch.indices.analysis.IndicesAnalysisModule;
import org.elasticsearch.indices.analysis.IndicesAnalysisService;
+import org.hamcrest.MatcherAssert;
+import static org.hamcrest.Matchers.*;
import org.testng.annotations.Test;
-import static org.elasticsearch.common.settings.ImmutableSettings.Builder.EMPTY_SETTINGS;
-
/**
*/
public class SimplePhoneticAnalysisTests {
@Test
- public void testDefaultsIcuAnalysis() {
+ public void testPhoneticTokenFilterFactory() {
+ Settings settings = settingsBuilder().loadFromClasspath("org/elasticsearch/index/analysis/phonetic-1.yml").build();
+ AnalysisService analysisService = testSimpleConfiguration(settings);
+ TokenFilterFactory standardfilterFactory = analysisService.tokenFilter("standard");
+ System.err.println("standard filterfactory = " + standardfilterFactory);
+ TokenFilterFactory filterFactory = analysisService.tokenFilter("phonetic");
+ System.err.println("filterfactory = " + filterFactory);
+ MatcherAssert.assertThat(filterFactory, instanceOf(PhoneticTokenFilterFactory.class));
+ }
+
+ private AnalysisService testSimpleConfiguration(Settings settings) {
Index index = new Index("test");
- Injector parentInjector = new ModulesBuilder().add(new SettingsModule(EMPTY_SETTINGS), new EnvironmentModule(new Environment(EMPTY_SETTINGS)), new IndicesAnalysisModule()).createInjector();
+ Injector parentInjector = new ModulesBuilder().add(new SettingsModule(settings),
+ new EnvironmentModule(new Environment(settings)),
+ new IndicesAnalysisModule()).createInjector();
Injector injector = new ModulesBuilder().add(
- new IndexSettingsModule(index, EMPTY_SETTINGS),
+ new IndexSettingsModule(index, settings),
new IndexNameModule(index),
- new AnalysisModule(EMPTY_SETTINGS, parentInjector.getInstance(IndicesAnalysisService.class)).addProcessor(new PhoneticAnalysisBinderProcessor()))
- .createChildInjector(parentInjector);
+ new AnalysisModule(settings, parentInjector.getInstance(IndicesAnalysisService.class))
+ .addProcessor(new PhoneticAnalysisBinderProcessor())).createChildInjector(parentInjector);
AnalysisService analysisService = injector.getInstance(AnalysisService.class);
-
- // need to create one with encoder...
- //TokenFilterFactory tokenFilterFactory = analysisService.tokenFilter("phonetic");
- //MatcherAssert.assertThat(tokenFilterFactory, Matchers.instanceOf(PhoneticTokenFilterFactory.class));
+ return analysisService;
}
}
diff --git a/src/test/resources/org/elasticsearch/index/analysis/phonetic-1.yml b/src/test/resources/org/elasticsearch/index/analysis/phonetic-1.yml
new file mode 100644
index 0000000000000..41a4e3fc59fdc
--- /dev/null
+++ b/src/test/resources/org/elasticsearch/index/analysis/phonetic-1.yml
@@ -0,0 +1,30 @@
+index:
+ analysis:
+ filter:
+ doublemetaphonefilter:
+ type: phonetic
+ encoder: doublemetaphone
+ metaphonefilter:
+ type: phonetic
+ encoder: metaphone
+ soundexfilter:
+ type: phonetic
+ encoder: soundex
+ refinedsoundexfilter:
+ type: phonetic
+ encoder: refinedsoundex
+ caverphonefilter:
+ type: phonetic
+ encoder: caverphone
+ beidermorsefilter:
+ type: phonetic
+ encoder: beidermorse
+ koelnerphonetikfilter:
+ type: phonetic
+ encoder: koelnerphonetik
+ haasephonetikfilter:
+ type: phonetic
+ encoder: haasephonetik
+ nysiisfilter:
+ type: phonetic
+ encoder: nysiis