Add support for transliterated citation keys (#13893)

* feat(citation-key-generator): add support for transliterating fields in citation key

* feat(citation-key-generator): add transliteration action + refactor to class

* chore(test): better format comments in TransliterationTest

* chore(changelog): add changelog entry for transliteration

* refactor(transliteration): refactor code after reviewing PR changes

* refactor(transliteration): address review comments

* chore(transliteration): reformat files

* refactor(citationkey): append "for citation key" to "Transliterate fields" + change tests

* chore(changelog): update CHANGELOG.md

* Update CHANGELOG.md

* fix(citationkey): address review comments

* Fix duplicate entry in CHANGELOG.md

* chore(submodules): fix submodules

* refactor(citationkey): change true to false

* chore(submodules): fix submodules

---------

Co-authored-by: Subhramit Basu <subhramit.bb@live.in>
Co-authored-by: Christoph <siedlerkiller@gmail.com>

Author: 3 people

CHANGELOG.md

@@ -14,6 +14,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv
 - We added "IEEE" as another option for parsing plain text citations. [#14233](github.com/JabRef/jabref/pull/14233)
 - We added automatic date-based groups that create year/month/day subgroups from an entry’s date fields. [#10822](https://github.com/JabRef/jabref/issues/10822)
 - We added `doi-to-bibtex` to `JabKit`. [#14244](https://github.com/JabRef/jabref/pull/14244)
+- We added support for transliteration of fields to English and automatic transliteration of generated citation key. [#11377](https://github.com/JabRef/jabref/issues/11377)
 
 ### Changed
 

jabgui/src/main/java/org/jabref/gui/preferences/citationkeypattern/CitationKeyPatternTab.java

@@ -16,6 +16,7 @@
 
 public class CitationKeyPatternTab extends AbstractPreferenceTabView<CitationKeyPatternTabViewModel> implements PreferencesTab {
 
+    @FXML private CheckBox transliterateFieldsForCitationKey;
     @FXML private CheckBox overwriteAllow;
     @FXML private CheckBox overwriteWarning;
     @FXML private CheckBox generateOnSave;
@@ -42,6 +43,7 @@ public String getTabName() {
     public void initialize() {
         this.viewModel = new CitationKeyPatternTabViewModel(preferences.getCitationKeyPatternPreferences(), preferences.getImporterPreferences());
 
+        transliterateFieldsForCitationKey.selectedProperty().bindBidirectional(viewModel.transliterateFieldsForCitationKeyProperty());
         overwriteAllow.selectedProperty().bindBidirectional(viewModel.overwriteAllowProperty());
         overwriteWarning.selectedProperty().bindBidirectional(viewModel.overwriteWarningProperty());
         generateOnSave.selectedProperty().bindBidirectional(viewModel.generateOnSaveProperty());

jabgui/src/main/java/org/jabref/gui/preferences/citationkeypattern/CitationKeyPatternTabViewModel.java

@@ -19,6 +19,7 @@
 
 public class CitationKeyPatternTabViewModel implements PreferenceTabViewModel {
 
+    private final BooleanProperty transliterateFieldsForCitationKeyProperty = new SimpleBooleanProperty();
     private final BooleanProperty overwriteAllowProperty = new SimpleBooleanProperty();
     private final BooleanProperty overwriteWarningProperty = new SimpleBooleanProperty();
     private final BooleanProperty generateOnSaveProperty = new SimpleBooleanProperty();
@@ -53,6 +54,7 @@ public CitationKeyPatternTabViewModel(CitationKeyPatternPreferences keyPatternPr
 
     @Override
     public void setValues() {
+        transliterateFieldsForCitationKeyProperty.setValue(keyPatternPreferences.shouldTransliterateFieldsForCitationKey());
         overwriteAllowProperty.setValue(!keyPatternPreferences.shouldAvoidOverwriteCiteKey());
         overwriteWarningProperty.setValue(keyPatternPreferences.shouldWarnBeforeOverwriteCiteKey());
         generateOnSaveProperty.setValue(keyPatternPreferences.shouldGenerateCiteKeysBeforeSaving());
@@ -106,6 +108,7 @@ public void storeSettings() {
             keySuffix = CitationKeyPatternPreferences.KeySuffix.SECOND_WITH_B;
         }
 
+        keyPatternPreferences.setShouldTransliterateFieldsForCitationKey(transliterateFieldsForCitationKeyProperty.getValue());
         keyPatternPreferences.setAvoidOverwriteCiteKey(!overwriteAllowProperty.getValue());
         keyPatternPreferences.setWarnBeforeOverwriteCiteKey(overwriteWarningProperty.getValue());
         keyPatternPreferences.setGenerateCiteKeysBeforeSaving(generateOnSaveProperty.getValue());
@@ -117,6 +120,10 @@ public void storeSettings() {
         keyPatternPreferences.setKeyPatterns(newKeyPattern);
     }
 
+    public BooleanProperty transliterateFieldsForCitationKeyProperty() {
+        return transliterateFieldsForCitationKeyProperty;
+    }
+
     public BooleanProperty overwriteAllowProperty() {
         return overwriteAllowProperty;
     }

jabgui/src/main/resources/org/jabref/gui/preferences/citationkeypattern/CitationKeyPatternTab.fxml

@@ -74,6 +74,9 @@
                GridPane.columnIndex="0" GridPane.rowIndex="8"/>
         <TextField fx:id="unwantedCharacters"
                 GridPane.columnIndex="1" GridPane.rowIndex="8"/>
+
+        <CheckBox fx:id="transliterateFieldsForCitationKey" text="%Transliterate fields for citation key"
+                GridPane.columnIndex="0" GridPane.rowIndex="9"/>
     </GridPane>
 
     <HBox spacing="8.0">
@@ -90,4 +93,4 @@
         <Button text="%Reset All" onAction="#resetAllKeyPatterns"
                 AnchorPane.rightAnchor="0.0" AnchorPane.topAnchor="0.0"/>
     </AnchorPane>
-</fx:root>
+</fx:root>

jabgui/src/test/java/org/jabref/gui/externalfiles/AutoRenameFileOnEntryChangeTest.java

@@ -44,6 +44,7 @@ void setUp(@TempDir Path tempDir) {
         GuiPreferences guiPreferences = mock(GuiPreferences.class);
         filePreferences = mock(FilePreferences.class);
         CitationKeyPatternPreferences patternPreferences = new CitationKeyPatternPreferences(
+                false,
                 false,
                 true,
                 false,

jablib/src/main/java/org/jabref/logic/citationkeypattern/CitationKeyGenerator.java

@@ -8,6 +8,7 @@
 import java.util.regex.PatternSyntaxException;
 
 import org.jabref.logic.util.strings.StringUtil;
+import org.jabref.logic.util.strings.Transliteration;
 import org.jabref.model.FieldChange;
 import org.jabref.model.database.BibDatabase;
 import org.jabref.model.database.BibDatabaseContext;
@@ -37,7 +38,7 @@ public class CitationKeyGenerator extends BracketedPattern {
 
     /// Source of disallowed characters: <https://tex.stackexchange.com/a/408548/9075>
     /// These characters are disallowed in BibTeX keys.
-    private static final List<Character> DISALLOWED_CHARACTERS = Arrays.asList('{', '}', '(', ')', ',', '=', '\\', '"', '#', '%', '~', '\'');
+    public static final List<Character> DISALLOWED_CHARACTERS = Arrays.asList('{', '}', '(', ')', ',', '=', '\\', '"', '#', '%', '~', '\'');
 
     private static final Logger LOGGER = LoggerFactory.getLogger(CitationKeyGenerator.class);
 
@@ -109,7 +110,8 @@ public String generateKey(@NonNull BibEntry entry) {
         String newKey = createCitationKeyFromPattern(entry);
         newKey = replaceWithRegex(newKey);
         newKey = appendLettersToKey(newKey, currentKey);
-        return cleanKey(newKey, unwantedCharacters);
+        newKey = cleanKey(newKey, unwantedCharacters);
+        return transliterateIfNeeded(newKey);
     }
 
     /**
@@ -155,6 +157,15 @@ private String appendLettersToKey(String key, String oldKey) {
         return key;
     }
 
+    public String transliterateIfNeeded(String key) {
+        if (!citationKeyPatternPreferences.shouldTransliterateFieldsForCitationKey()) {
+            return key;
+        }
+
+        String result = Transliteration.transliterate(key);
+        return result.replace(" ", "");
+    }
+
     /**
      * Using preferences, replace matches to the provided regex with a string.
      *

jablib/src/main/java/org/jabref/logic/citationkeypattern/CitationKeyGeneratorTestUtils.java

@@ -0,0 +1,22 @@
+package org.jabref.logic.citationkeypattern;
+
+import static org.jabref.logic.citationkeypattern.CitationKeyGenerator.DEFAULT_UNWANTED_CHARACTERS;
+
+public class CitationKeyGeneratorTestUtils {
+
+    public static CitationKeyPatternPreferences getInstanceForTesting() {
+        return new CitationKeyPatternPreferences(
+                true,
+                false,
+                false,
+                false,
+                CitationKeyPatternPreferences.KeySuffix.SECOND_WITH_A,
+                "",
+                "",
+                DEFAULT_UNWANTED_CHARACTERS,
+                GlobalCitationKeyPatterns.fromPattern("[auth][year]"),
+                "",
+                ','
+        );
+    }
+}

jablib/src/main/java/org/jabref/logic/citationkeypattern/CitationKeyPatternPreferences.java

@@ -18,6 +18,7 @@ public enum KeySuffix {
         SECOND_WITH_B   // CiteKey, CiteKeyB, CiteKeyC ...
     }
 
+    private final BooleanProperty shouldTransliterateFieldsForCitationKey = new SimpleBooleanProperty();
     private final BooleanProperty shouldAvoidOverwriteCiteKey = new SimpleBooleanProperty();
     private final BooleanProperty shouldWarnBeforeOverwriteCiteKey = new SimpleBooleanProperty();
     private final BooleanProperty shouldGenerateCiteKeysBeforeSaving = new SimpleBooleanProperty();
@@ -29,7 +30,8 @@ public enum KeySuffix {
     private final String defaultPattern;
     private final ReadOnlyObjectProperty<Character> keywordDelimiter;
 
-    public CitationKeyPatternPreferences(boolean shouldAvoidOverwriteCiteKey,
+    public CitationKeyPatternPreferences(boolean shouldTransliterateFieldsForCitationKey,
+                                         boolean shouldAvoidOverwriteCiteKey,
                                          boolean shouldWarnBeforeOverwriteCiteKey,
                                          boolean shouldGenerateCiteKeysBeforeSaving,
                                          KeySuffix keySuffix,
@@ -40,6 +42,7 @@ public CitationKeyPatternPreferences(boolean shouldAvoidOverwriteCiteKey,
                                          String defaultPattern,
                                          ReadOnlyObjectProperty<Character> keywordDelimiter) {
 
+        this.shouldTransliterateFieldsForCitationKey.set(shouldTransliterateFieldsForCitationKey);
         this.shouldAvoidOverwriteCiteKey.set(shouldAvoidOverwriteCiteKey);
         this.shouldWarnBeforeOverwriteCiteKey.set(shouldWarnBeforeOverwriteCiteKey);
         this.shouldGenerateCiteKeysBeforeSaving.set(shouldGenerateCiteKeysBeforeSaving);
@@ -54,7 +57,8 @@ public CitationKeyPatternPreferences(boolean shouldAvoidOverwriteCiteKey,
     }
 
     @VisibleForTesting
-    public CitationKeyPatternPreferences(boolean shouldAvoidOverwriteCiteKey,
+    public CitationKeyPatternPreferences(boolean shouldTransliterateFieldsForCitationKey,
+                                         boolean shouldAvoidOverwriteCiteKey,
                                          boolean shouldWarnBeforeOverwriteCiteKey,
                                          boolean shouldGenerateCiteKeysBeforeSaving,
                                          KeySuffix keySuffix,
@@ -65,7 +69,8 @@ public CitationKeyPatternPreferences(boolean shouldAvoidOverwriteCiteKey,
                                          String defaultPattern,
                                          Character keywordDelimiter) {
 
-        this(shouldAvoidOverwriteCiteKey,
+        this(shouldTransliterateFieldsForCitationKey,
+                shouldAvoidOverwriteCiteKey,
                 shouldWarnBeforeOverwriteCiteKey,
                 shouldGenerateCiteKeysBeforeSaving,
                 keySuffix,
@@ -77,6 +82,18 @@ public CitationKeyPatternPreferences(boolean shouldAvoidOverwriteCiteKey,
                 new SimpleObjectProperty<>(keywordDelimiter));
     }
 
+    public boolean shouldTransliterateFieldsForCitationKey() {
+        return shouldTransliterateFieldsForCitationKey.get();
+    }
+
+    public BooleanProperty shouldTransliterateFieldsForCitationKeyProperty() {
+        return shouldTransliterateFieldsForCitationKey;
+    }
+
+    public void setShouldTransliterateFieldsForCitationKey(boolean shouldTransliterateFieldsForCitationKey) {
+        this.shouldTransliterateFieldsForCitationKey.set(shouldTransliterateFieldsForCitationKey);
+    }
+
     public boolean shouldAvoidOverwriteCiteKey() {
         return shouldAvoidOverwriteCiteKey.get();
     }

jablib/src/main/java/org/jabref/logic/formatter/Formatters.java

@@ -29,6 +29,7 @@
 import org.jabref.logic.formatter.bibtexfields.RemoveEnclosingBracesFormatter;
 import org.jabref.logic.formatter.bibtexfields.RemoveWordEnclosingAndOuterEnclosingBracesFormatter;
 import org.jabref.logic.formatter.bibtexfields.ShortenDOIFormatter;
+import org.jabref.logic.formatter.bibtexfields.TransliterateFormatter;
 import org.jabref.logic.formatter.bibtexfields.UnicodeToLatexFormatter;
 import org.jabref.logic.formatter.bibtexfields.UnitsToLatexFormatter;
 import org.jabref.logic.formatter.casechanger.CamelFormatter;
@@ -66,7 +67,8 @@ public static List<Formatter> getConverters() {
                 new HtmlToUnicodeFormatter(),
                 new LatexToUnicodeFormatter(),
                 new UnicodeToLatexFormatter(),
-                new ConvertMSCCodesFormatter()
+                new ConvertMSCCodesFormatter(),
+                new TransliterateFormatter()
         );
     }
 

jablib/src/main/java/org/jabref/logic/formatter/bibtexfields/TransliterateFormatter.java

@@ -0,0 +1,32 @@
+package org.jabref.logic.formatter.bibtexfields;
+
+import org.jabref.logic.cleanup.Formatter;
+import org.jabref.logic.l10n.Localization;
+import org.jabref.logic.util.strings.Transliteration;
+
+public class TransliterateFormatter extends Formatter {
+    @Override
+    public String getName() {
+        return Localization.lang("Transliterate");
+    }
+
+    @Override
+    public String getKey() {
+        return "transliterate";
+    }
+
+    @Override
+    public String format(String value) {
+        return Transliteration.transliterate(value);
+    }
+
+    @Override
+    public String getDescription() {
+        return Localization.lang("Converts non-Latin characters to their Latin equivalents.");
+    }
+
+    @Override
+    public String getExampleInput() {
+        return "Карпенко Надежда";
+    }
+}