Improve Fuzzy Search Algorithm And Fix Undo Bug

- Added utility functions to ConferenceUtils for acronym candidate
  generation and string normalization
- Updated fuzzy search algorithm in ConferenceRepository to include
  Longest Common Substring similarity combined with Levenshtein
  similarity
- Fixed bug in ICORERankingEditor which was causing Undo to not work
- Updated ICORERankingEditorViewModel to use the leaner
  ConferenceRepository API
- Added LCSSimilarity method to StringSimilarity to compute Longest
  Common Substring similarity rating
- Added Javadoc to various non-trivial methods
- Add tests to cover the updated functionality

Part of #13476

Author: TheYorouzoya

jabgui/src/main/java/org/jabref/gui/fieldeditors/ICORERankingEditor.java

@@ -65,8 +65,6 @@ public ICORERankingEditor(Field field,
                 new Tooltip(Localization.lang("Visit ICORE conference page"))
         );
         visitICOREConferencePageButton.disableProperty().bind(textField.textProperty().isEmpty());
-
-        new EditorValidator(preferences).configureValidation(viewModel.getFieldValidator().getValidationStatus(), textField);
     }
 
     @Override

jabgui/src/main/java/org/jabref/gui/fieldeditors/ICORERankingEditorViewModel.java

@@ -9,7 +9,6 @@
 import org.jabref.gui.autocompleter.SuggestionProvider;
 import org.jabref.gui.desktop.os.NativeDesktop;
 import org.jabref.gui.preferences.GuiPreferences;
-import org.jabref.logic.icore.ConferenceAcronymExtractor;
 import org.jabref.logic.icore.ConferenceRepository;
 import org.jabref.logic.integrity.FieldCheckers;
 import org.jabref.logic.l10n.Localization;
@@ -56,18 +55,7 @@ public void lookupIdentifier(BibEntry bibEntry) {
             return;
         }
 
-        Optional<ConferenceEntry> conference;
-        Optional<String> acronym = ConferenceAcronymExtractor.extract(bookTitle.get());
-        if (acronym.isPresent()) {
-            conference = repo.getConferenceFromAcronym(acronym.get());
-            if (conference.isPresent()) {
-                entry.setField(field, conference.get().rank());
-                matchedConference = conference.get();
-                return;
-            }
-        }
-
-        conference = repo.getConferenceFromBookTitle(bookTitle.get());
+        Optional<ConferenceEntry> conference = repo.getConferenceFromBookTitle(bookTitle.get());
         if (conference.isPresent()) {
             entry.setField(field, conference.get().rank());
             matchedConference = conference.get();

jabgui/src/main/java/org/jabref/migrations/PreferencesMigrations.java

@@ -566,9 +566,10 @@ static void moveApiKeysToKeyring(JabRefCliPreferences preferences) {
     /**
      * Updates the default preferences for the editor fields under the "General" tab to include the ICORE Ranking Field
      * if it is missing.
-     *<p>
+     * <p>
      * The function first ensures that the current preferences match the previous default (before the ICORE field was added)
      * and only then does the update.
+     * </p>
      *
      * @implNote The default fields for the "General" tab are defined by {@link FieldFactory#getDefaultGeneralFields()}.
      * @param preferences the user's current preferences

jablib/src/main/java/org/jabref/logic/icore/ConferenceAcronymExtractor.java

@@ -7,13 +7,15 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 
 import org.jabref.logic.JabRefException;
 import org.jabref.logic.util.strings.StringSimilarity;
 import org.jabref.model.icore.ConferenceEntry;
 
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVRecord;
+import org.jspecify.annotations.NonNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -25,15 +27,20 @@
  * Since the website does not expose an API endpoint to fetch this data programmatically, it must be manually exported
  * from the website and stored as a resource. This means that when new ranking data is released, the old data must be
  * replaced and the <code>ICORE_RANK_DATA_FILE</code> variable must be modified to point to the new data file.
+ * </p>
  */
 public class ConferenceRepository {
     private static final Logger LOGGER = LoggerFactory.getLogger(ConferenceRepository.class);
     private static final String ICORE_RANK_DATA_FILE = "/icore/ICORE2023.csv";
-    private static final double FUZZY_SEARCH_THRESHOLD = 0.9;
-    private static final StringSimilarity MATCHER = new StringSimilarity();
+    private static final double LEVENSHTEIN_THRESHOLD = 0.9;
+    private static final double COMBINED_LCS_LEV_THRESHOLD = 0.75;
+    private static final double EPSILON = 1e-6;
+    private static final StringSimilarity LEVENSHTEIN_MATCHER = new StringSimilarity();
 
     private final Map<String, ConferenceEntry> acronymToConference = new HashMap<>();
     private final Map<String, ConferenceEntry> titleToConference = new HashMap<>();
+    private final Map<String, ConferenceEntry> normalizedTitleToConference = new HashMap<>();
+    private int maxAcronymLength = 0;
 
     public ConferenceRepository() throws JabRefException {
         InputStream inputStream = getClass().getResourceAsStream(ICORE_RANK_DATA_FILE);
@@ -63,74 +70,158 @@ private void loadConferenceDataFromInputStream(InputStream inputStream) throws J
             for (CSVRecord record : records) {
                 String id = record.get("Id").strip();
                 String title = record.get("Title").strip().toLowerCase();
-                String acronym = record.get("Acronym").strip().toUpperCase();
+                String acronym = record.get("Acronym").strip().toLowerCase();
                 String rank = record.get("Rank").strip();
 
                 if (id.isEmpty() || title.isEmpty() || acronym.isEmpty() || rank.isEmpty()) {
                     LOGGER.warn("Missing fields in row in ICORE rank data: {}", record);
                     continue;
                 }
 
+                if (title.indexOf('(') >= 0) {
+                    // remove any extra alias strings in parentheses
+                    title = ConferenceUtils.removeAllParenthesesWithContent(title);
+                }
                 ConferenceEntry conferenceEntry = new ConferenceEntry(id, title, acronym, rank);
                 acronymToConference.put(acronym, conferenceEntry);
                 titleToConference.put(title, conferenceEntry);
+                normalizedTitleToConference.put(ConferenceUtils.normalize(title), conferenceEntry);
+                if (acronym.length() >= maxAcronymLength) {
+                    maxAcronymLength = acronym.length();
+                }
             }
         } catch (IOException e) {
             throw new JabRefException("I/O Error while reading ICORE data from resource", e);
         }
+        LOGGER.debug("Max acronym length seen in data: {}", maxAcronymLength);
     }
 
-    public Optional<ConferenceEntry> getConferenceFromAcronym(String acronym) {
-        String query = acronym.strip().toUpperCase();
-
-        ConferenceEntry conference = acronymToConference.get(query);
+    /**
+     * Searches the given query against the ICORE conference ranking data and returns a match, if found.
+     * <p>
+     * While searching, we first look for a conference acronym present inside parentheses, like <code>(ICSE)</code>
+     * or <code>(ICSE 2022)</code>. If acronym lookup fails, the query is processed further and matched against the list
+     * of conference titles.
+     * </p>
+     *
+     * @implNote see {@link ConferenceRepository#fuzzySearchConferenceTitles} for more details on matching
+     * @param bookTitle the string to search, must not be {@code null}
+     * @return an {@code Optional} conference entry, if found
+     *         or {@code Optional.empty()} if no conference entry is found
+     */
+    public Optional<ConferenceEntry> getConferenceFromBookTitle(@NonNull String bookTitle) {
+        String query = bookTitle.strip().toLowerCase();
+        ConferenceEntry conference;
 
-        if (conference == null) {
-            return Optional.empty();
+        conference = acronymToConference.get(query);
+        if (conference != null) {
+            return Optional.of(conference);
         }
 
-        return Optional.of(conference);
-    }
-
-    public Optional<ConferenceEntry> getConferenceFromBookTitle(String bookTitle) {
-        String query = bookTitle.strip().toLowerCase();
-
-        ConferenceEntry conference = titleToConference.get(query);
+        conference = titleToConference.get(query);
         if (conference != null) {
             return Optional.of(conference);
         }
 
-        String bestMatch = fuzzySearchConferenceTitles(query);
-        if (bestMatch.isEmpty()) {
-            return Optional.empty();
+        Optional<ConferenceEntry> acronymConference = getConferenceFromAcronym(query);
+
+        if (acronymConference.isPresent()) {
+            return acronymConference;
         }
 
-        conference = titleToConference.get(bestMatch);
+        return fuzzySearchConferenceTitles(query);
+    }
 
-        return Optional.of(conference);
+    private Optional<ConferenceEntry> getConferenceFromAcronym(String query) {
+        Optional<String> acronym = ConferenceUtils.extractStringFromParentheses(query);
+
+        if (acronym.isPresent()) {
+            ConferenceEntry conference;
+            Set<String> acronymCandidates = ConferenceUtils.generateAcronymCandidates(acronym.get(), maxAcronymLength);
+            LOGGER.debug("Extracted acronym string: {}, Acronym candidates: {}", acronym.get(), acronymCandidates);
+            for (String candidate : acronymCandidates) {
+                conference = acronymToConference.get(candidate);
+                if (conference != null) {
+                    return Optional.of(conference);
+                }
+            }
+        }
+        return Optional.empty();
     }
 
     /**
-     * Searches all conference titles for the given query string using {@link StringSimilarity#similarity} as a MATCHER.
+     * Searches the conference data for the given query string using a combination of Levenshtein similarity
+     * {@link StringSimilarity#similarity} and Longest Common Substring (LCS) similarity {@link StringSimilarity#LCSSimilarity}.
+     * <p>
+     * The input query is first fed through the normalizer at {@link ConferenceUtils#normalize} which strips away much of the
+     * noise.
+     * </p>
      * <p>
-     * The threshold for matching is set at 0.9. This function will always return the conference title with the highest
-     * similarity rating.
+     * While searching, the function computes Levenshtein similarity and LCS similarity between the query and the current conference
+     * title (also normalized) and prioritizes them in the following order:
+     * <ol>
+     *     <li>Whenever LCS similarity returns <code>1.0</code>, i.e., a conference title is found entirely as a substring in the query.</li>
+     *     <li>Whenever Levenshtein similarity exceeds the threshold defined by the <code>LEVENSHTEIN_THRESHOLD</code> constant.</li>
+     *     <li>The combined weighted score of both LCS and Levenshtein similarities exceeds the <code>COMBINED_LCS_LEV_THRESHOLD</code>.</li>
+     * </ol>
+     *
+     * The combined score is calculated as follows:
+     * <code>(0.6 * Levenshtein similarity) + (0.4 * LCS similarity)</code>
+     * </p>
      *
      * @param query The query string to be searched
-     * @return The conference title, if found. Otherwise, an empty string is returned.
+     * @return an {@code Optional} conference entry, if found
+     *         or {@code Optional.empty()} if no conference entry is found
      */
-    private String fuzzySearchConferenceTitles(String query) {
+    private Optional<ConferenceEntry> fuzzySearchConferenceTitles(String query) {
         String bestMatch = "";
-        double bestSimilarity = 0.0;
+        double bestScore = 0.0;
+        String normalizedQuery = ConferenceUtils.normalize(query);
+
+        if (normalizedQuery.isEmpty()) {
+            return Optional.empty();
+        }
+
+        ConferenceEntry acronymConference = acronymToConference.get(normalizedQuery);
+        if (acronymConference != null) {
+            return Optional.of(acronymConference);
+        }
 
-        for (String conferenceTitle : titleToConference.keySet()) {
-            double similarity = MATCHER.similarity(query, conferenceTitle);
-            if (similarity >= FUZZY_SEARCH_THRESHOLD && similarity > bestSimilarity) {
-                bestMatch = conferenceTitle;
-                bestSimilarity = similarity;
+        acronymConference = normalizedTitleToConference.get(normalizedQuery);
+        if (acronymConference != null) {
+            return Optional.of(acronymConference);
+        }
+
+        for (String conferenceTitle : normalizedTitleToConference.keySet()) {
+            // only match for queries longer than the current conference title
+            // this will safeguard against overfitting common prefixes
+            if (normalizedQuery.length() >= conferenceTitle.length()) {
+                double levSimilarity = LEVENSHTEIN_MATCHER.similarity(normalizedQuery, conferenceTitle);
+                double LCSSimilarity = StringSimilarity.LCSSimilarity(normalizedQuery, conferenceTitle);
+                double combinedScore = levSimilarity * 0.6 + LCSSimilarity * 0.4;
+                boolean exactSubstringMatch = (Math.abs(LCSSimilarity - 1.0) <= EPSILON);
+
+                if (exactSubstringMatch) {
+                    return Optional.of(normalizedTitleToConference.get(conferenceTitle));
+                }
+
+                if (levSimilarity >= LEVENSHTEIN_THRESHOLD) {
+                    return Optional.of(normalizedTitleToConference.get(conferenceTitle));
+                }
+
+                if (combinedScore >= COMBINED_LCS_LEV_THRESHOLD && combinedScore >= bestScore) {
+                    bestMatch = conferenceTitle;
+                    bestScore = combinedScore;
+                    LOGGER.debug("Matched query: {} with title: {} with combinedScore: {} and LEV: {} and LCS: {}",
+                            normalizedQuery, conferenceTitle, combinedScore, levSimilarity, LCSSimilarity);
+                }
             }
         }
 
-        return bestMatch;
+        if (!bestMatch.isEmpty()) {
+            return Optional.of(normalizedTitleToConference.get(bestMatch));
+        }
+
+        return Optional.empty();
     }
 }