Convert CiteSeerX fetcher to new infrastructure

src/main/java/org/jabref/gui/importer/fetcher/EntryFetchers.java

@@ -12,9 +12,7 @@ public class EntryFetchers {
 
     private final List<EntryFetcher> entryFetchers = new LinkedList<>();
 
-    public EntryFetchers(JournalAbbreviationLoader abbreviationLoader) {
-        entryFetchers.add(new CiteSeerXFetcher());
-
+   public EntryFetchers(JournalAbbreviationLoader abbreviationLoader) {
         WebFetchers.getSearchBasedFetchers(Globals.prefs.getImportFormatPreferences()).stream()
                 .map(SearchBasedEntryFetcher::new)
                 .forEach(entryFetchers::add);

src/main/java/org/jabref/logic/importer/Parser.java

@@ -1,5 +1,6 @@
 package org.jabref.logic.importer;
 
+import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.util.List;
 
@@ -11,4 +12,8 @@
 public interface Parser {
 
     List<BibEntry> parseEntries(InputStream inputStream) throws ParseException;
+
+    default List<BibEntry> parseEntries(String dataString) throws ParseException {
+        return parseEntries(new ByteArrayInputStream(dataString.getBytes()));
+    }
 }

src/main/java/org/jabref/logic/importer/WebFetchers.java

@@ -9,6 +9,7 @@
 import org.jabref.logic.importer.fetcher.ACS;
 import org.jabref.logic.importer.fetcher.ArXiv;
 import org.jabref.logic.importer.fetcher.AstrophysicsDataSystem;
+import org.jabref.logic.importer.fetcher.CiteSeer;
 import org.jabref.logic.importer.fetcher.CrossRef;
 import org.jabref.logic.importer.fetcher.DBLPFetcher;
 import org.jabref.logic.importer.fetcher.DOAJFetcher;
@@ -89,6 +90,7 @@ public static List<SearchBasedFetcher> getSearchBasedFetchers(ImportFormatPrefer
         list.add(new DBLPFetcher(importFormatPreferences));
         list.add(new SpringerFetcher());
         list.add(new CrossRef());
+        list.add(new CiteSeer());
         list.add(new DOAJFetcher(importFormatPreferences));
         list.add(new IEEE(importFormatPreferences));
         list.sort(Comparator.comparing(WebFetcher::getName));

src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java

@@ -0,0 +1,94 @@
+package org.jabref.logic.importer.fetcher;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import org.jabref.logic.formatter.bibtexfields.HtmlToUnicodeFormatter;
+import org.jabref.logic.formatter.casechanger.TitleCaseFormatter;
+import org.jabref.logic.help.HelpFile;
+import org.jabref.logic.importer.FetcherException;
+import org.jabref.logic.importer.Parser;
+import org.jabref.logic.importer.SearchBasedParserFetcher;
+import org.jabref.logic.importer.fileformat.CoinsParser;
+import org.jabref.logic.util.OS;
+import org.jabref.model.cleanup.FieldFormatterCleanup;
+import org.jabref.model.cleanup.Formatter;
+import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.FieldName;
+
+import org.apache.http.client.utils.URIBuilder;
+
+public class CiteSeer implements SearchBasedParserFetcher {
+
+    public CiteSeer() {
+    }
+
+    @Override
+    public String getName() {
+        return "CiteSeerX";
+    }
+
+    @Override
+    public HelpFile getHelpPage() {
+        return HelpFile.FETCHER_CITESEERX;
+    }
+
+    @Override
+    public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException {
+        URIBuilder uriBuilder = new URIBuilder("https://citeseer.ist.psu.edu/search");
+        uriBuilder.addParameter("sort", "rlv"); // Sort by relevance
+        uriBuilder.addParameter("q", query); // Query
+        uriBuilder.addParameter("t", "doc"); // Type: documents
+        //uriBuilder.addParameter("start", "0"); // Start index (not supported at the moment)
+        return uriBuilder.build().toURL();
+    }
+
+    @Override
+    public Parser getParser() {
+        // MathSciNet returns COinS result embedded in HTML
+        // So we extract the data string from the <span class="Z3988" title="<data>"></span> tags and pass the content to the COinS parser
+        return inputStream -> {
+            String response = new BufferedReader(new InputStreamReader(inputStream)).lines().collect(Collectors.joining(OS.NEWLINE));
+
+            List<BibEntry> entries = new ArrayList<>();
+            CoinsParser parser = new CoinsParser();
+            Pattern pattern = Pattern.compile("<span class=\"Z3988\" title=\"(.*)\"></span>");
+            Matcher matcher = pattern.matcher(response);
+            while (matcher.find()) {
+                String encodedDataString = matcher.group(1);
+                entries.addAll(parser.parseEntries(encodedDataString));
+            }
+            return entries;
+        };
+    }
+
+    @Override
+    public void doPostCleanup(BibEntry entry) {
+        // CiteSeer escapes some characters in a way that is not recognized by the normal html to unicode formatter
+        // We, of course, also want to convert these special characters
+        Formatter extendedHtmlFormatter = new HtmlToUnicodeFormatter() {
+            @Override
+            public String format(String fieldText) {
+                String formatted = super.format(fieldText);
+                formatted = formatted.replaceAll("%3A", ":");
+                formatted = formatted.replaceAll("%3Cem%3", "");
+                formatted = formatted.replaceAll("%3C%2Fem%3E", "");
+                formatted = formatted.replaceAll("%2C\\+", " ");
+                formatted = formatted.replaceAll("\\+", " ");
+                return formatted;
+            }
+        };
+        new FieldFormatterCleanup(FieldName.INTERNAL_ALL_FIELD, extendedHtmlFormatter).cleanup(entry);
+
+        // Many titles in the CiteSeer database have all-capital titles, for convenience we convert them to title case
+        new FieldFormatterCleanup(FieldName.TITLE, new TitleCaseFormatter()).cleanup(entry);
+    }
+}

src/main/java/org/jabref/logic/importer/fileformat/BibtexParser.java

@@ -6,7 +6,6 @@
 import java.io.InputStreamReader;
 import java.io.PushbackReader;
 import java.io.Reader;
-import java.io.StringReader;
 import java.nio.charset.StandardCharsets;
 import java.util.Collection;
 import java.util.Deque;
@@ -110,10 +109,6 @@ public List<BibEntry> parseEntries(Reader reader) throws ParseException {
         }
     }
 
-    public List<BibEntry> parseEntries(String bibtexString) throws ParseException {
-        return parseEntries(new StringReader(bibtexString));
-    }
-
     public Optional<BibEntry> parseSingleEntry(String bibtexString) throws ParseException {
         return parseEntries(bibtexString).stream().findFirst();
     }

src/main/java/org/jabref/logic/importer/fileformat/CoinsParser.java

@@ -0,0 +1,78 @@
+package org.jabref.logic.importer.fileformat;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import org.jabref.logic.importer.ParseException;
+import org.jabref.logic.importer.Parser;
+import org.jabref.logic.util.OS;
+import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.BiblatexEntryTypes;
+import org.jabref.model.entry.FieldName;
+
+/**
+ * @implNote implemented by reverse-engineering <a href="https://github.com/SeerLabs/CiteSeerX/blob/4df28a98083be2829ec4c56ebbac09eb7772d379/src/java/edu/psu/citeseerx/domain/BiblioTransformer.java#L155-L249">the implementation by CiteSeerX</a>
+ */
+public class CoinsParser implements Parser {
+
+    private final Pattern DOI = Pattern.compile("%3Fdoi%3D([^&]+)");
+    private final Pattern TITLE = Pattern.compile("&amp;rft.atitle=([^&]+)");
+    private final Pattern JOURNAL = Pattern.compile("&amp;rft.jtitle=([^&]+)");
+    private final Pattern YEAR = Pattern.compile("&amp;rft.date=([^&]+)");
+    private final Pattern VOLUME = Pattern.compile("&amp;rft.volume=([^&]+)");
+    private final Pattern PAGES = Pattern.compile("&amp;rft.pages=([^&]+)");
+    private final Pattern ISSUE = Pattern.compile("&amp;rft.issue=([^&]+)");
+    private final Pattern TYPE = Pattern.compile("&amp;rft.genre=([^&]+)");
+    private final Pattern AUTHOR = Pattern.compile("&amp;rft.au=([^&]+)");
+
+    @Override
+    public List<BibEntry> parseEntries(InputStream inputStream) throws ParseException {
+        String data = new BufferedReader(new InputStreamReader(inputStream)).lines().collect(Collectors.joining(OS.NEWLINE));
+        BibEntry entry = new BibEntry();
+
+        appendData(data, entry, DOI, FieldName.DOI);
+        appendData(data, entry, TITLE, FieldName.TITLE);
+        appendData(data, entry, JOURNAL, FieldName.JOURNALTITLE);
+        appendData(data, entry, YEAR, FieldName.YEAR);
+        appendData(data, entry, VOLUME, FieldName.VOLUME);
+        appendData(data, entry, PAGES, FieldName.PAGES);
+        appendData(data, entry, ISSUE, FieldName.ISSUE);
+
+        Matcher matcherType = TYPE.matcher(data);
+        if (matcherType.find()) {
+            switch (matcherType.group(1)) {
+                case "article":
+                    entry.setType(BiblatexEntryTypes.ARTICLE);
+                    break;
+                case "unknown":
+                default:
+                    entry.setType(BiblatexEntryTypes.MISC);
+                    break;
+            }
+        }
+
+        List<String> authors = new ArrayList<>();
+        Matcher matcherAuthors = AUTHOR.matcher(data);
+        while (matcherAuthors.find()) {
+            String author = matcherAuthors.group(1);
+            authors.add(author);
+        }
+        entry.setField(FieldName.AUTHOR, authors.stream().collect(Collectors.joining(" and ")));
+
+        return Collections.singletonList(entry);
+    }
+
+    private void appendData(String data, BibEntry entry, Pattern pattern, String fieldName) {
+        Matcher matcher = pattern.matcher(data);
+        if (matcher.find()) {
+            entry.setField(fieldName, matcher.group(1));
+        }
+    }
+}