Fix for issue 5850: Journal abbreviations in UTF-8 not recognized

CHANGELOG.md

@@ -76,6 +76,7 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve
 - We fixed an issue where opening BibTex file (doubleclick) from Folder with spaces not working. [#6487](https://github.com/JabRef/jabref/issues/6487)
 - We fixed an issue with saving large `.bib` files [#7265](https://github.com/JabRef/jabref/issues/7265)
 - We fixed an issue with very large page numbers [#7590](https://github.com/JabRef/jabref/issues/7590)
+- We fixed an issue where journal abbreviations in UTF-8 were not recognized [#5850](https://github.com/JabRef/jabref/issues/5850)
 
 ### Removed
 

src/main/java/org/jabref/logic/integrity/IntegrityCheck.java

@@ -38,9 +38,11 @@ public IntegrityCheck(BibDatabaseContext bibDatabaseContext,
                 new CitationKeyDeviationChecker(bibDatabaseContext, citationKeyPatternPreferences),
                 new CitationKeyDuplicationChecker(bibDatabaseContext.getDatabase())
         ));
-
         if (bibDatabaseContext.isBiblatexMode()) {
-            entryCheckers.add(new JournalInAbbreviationListChecker(StandardField.JOURNALTITLE, journalAbbreviationRepository));
+            entryCheckers.addAll(List.of(
+                    new JournalInAbbreviationListChecker(StandardField.JOURNALTITLE, journalAbbreviationRepository),
+                    new UTF8Checker())
+            );
         } else {
             entryCheckers.addAll(List.of(
                     new JournalInAbbreviationListChecker(StandardField.JOURNAL, journalAbbreviationRepository),
@@ -59,7 +61,6 @@ List<IntegrityMessage> check() {
         for (BibEntry entry : database.getEntries()) {
             result.addAll(checkEntry(entry));
         }
-
         result.addAll(checkDatabase(database));
 
         return result;

src/main/java/org/jabref/logic/integrity/UTF8Checker.java

@@ -0,0 +1,53 @@
+package org.jabref.logic.integrity;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.jabref.logic.l10n.Localization;
+import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.field.Field;
+
+public class UTF8Checker implements EntryChecker {
+
+    /**
+     * Detect any non UTF-8 encoded field
+     * @param entry the BibEntry of BibLatex or Bibtex.
+     * @return return the warning of UTF-8 check for BibLatex.
+     */
+    @Override
+    public List<IntegrityMessage> check(BibEntry entry) {
+        List<IntegrityMessage> results = new ArrayList<>();
+        Charset charset = Charset.forName(System.getProperty("file.encoding"));
+        for (Map.Entry<Field, String> field : entry.getFieldMap().entrySet()) {
+            boolean utfOnly = UTF8EncodingChecker(field.getValue().getBytes(charset));
+            if (!utfOnly) {
+                results.add(new IntegrityMessage(Localization.lang("Non-UTF-8 encoded field found"), entry,
+                        field.getKey()));
+            }
+        }
+        return results;
+    }
+
+    /**
+     * Check whether a byte array is encoded in UTF-8 charset
+     *
+     * Use java api decoder and try&catch block to check the charset.
+     * @param data the byte array used to check the encoding charset
+     * @return true if is encoded in UTF-8 & false is not encoded in UTF-8
+     */
+    public static boolean UTF8EncodingChecker(byte[] data) {
+        CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder();
+        try {
+            decoder.decode(ByteBuffer.wrap(data));
+        } catch (CharacterCodingException ex) {
+            return false;
+        }
+        return true;
+    }
+}

src/main/resources/l10n/JabRef_en.properties

@@ -1629,6 +1629,7 @@ Style\ file=Style file
 Open\ OpenOffice/LibreOffice\ connection=Open OpenOffice/LibreOffice connection
 You\ must\ enter\ at\ least\ one\ field\ name=You must enter at least one field name
 Non-ASCII\ encoded\ character\ found=Non-ASCII encoded character found
+Non-UTF-8\ encoded\ field\ found=Non-UTF-8 encoded field found
 Toggle\ web\ search\ interface=Toggle web search interface
 
 Migration\ help\ information=Migration help information

src/test/java/org/jabref/logic/integrity/UTF8CheckerTest.java

@@ -0,0 +1,40 @@
+package org.jabref.logic.integrity;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.field.StandardField;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class UTF8CheckerTest {
+
+    private final BibEntry entry = new BibEntry();
+
+    @Test
+    void fieldAcceptsUTF8() {
+        UTF8Checker checker = new UTF8Checker();
+        entry.setField(StandardField.TITLE, "Only ascii characters!'@12");
+        assertEquals(Collections.emptyList(), checker.check(entry));
+    }
+
+    @Test
+    void fieldDoesNotAcceptUmlauts() {
+        String defaultCharset = System.getProperty("file.encoding");
+        System.getProperties().put("file.encoding", "GBK");
+        UTF8Checker checker = new UTF8Checker();
+        String NonUTF8 = "";
+        try {
+            NonUTF8 = new String("你好，这条语句使用GBK字符集".getBytes(), "GBK");
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        entry.setField(StandardField.MONTH, NonUTF8);
+        assertEquals(List.of(new IntegrityMessage("Non-UTF-8 encoded field found", entry, StandardField.MONTH)), checker.check(entry));
+        System.getProperties().put("file.encoding", defaultCharset);
+    }
+
+}