Fetcher for IACR eprints

CHANGELOG.md

@@ -35,9 +35,9 @@ We refer to [GitHub issues](https://github.com/JabRef/jabref/issues) by using `#
 - We added an option to mass append to fields via the Quality -> set/clear/append/rename fields dialog. [#2721](https://github.com/JabRef/jabref/issues/2721)
 - We added a check on startup to ensure JabRef is run with an adequate Java version. [3310](https://github.com/JabRef/jabref/issues/3310)
 - In the preference, all installed java Look and Feels are now listed and selectable
+- We added an ID fetcher for [IACR eprints](https://eprint.iacr.org/). [#3473](https://github.com/JabRef/jabref/pull/3473)
 - We added a clear option to the right-click menu of the text field in the entry editor. [koppor#198](https://github.com/koppor/jabref/issues/198)
 
-
 ### Fixed
  - We fixed the translation of \textendash and \textquotesingle in the entry preview [#3307](https://github.com/JabRef/jabref/issues/3307)
  - We fixed an issue where JabRef would not terminated after asking to collect anonymous statistics [#2955 comment](https://github.com/JabRef/jabref/issues/2955#issuecomment-334591123)

src/main/java/org/jabref/logic/importer/WebFetchers.java

@@ -13,6 +13,7 @@
 import org.jabref.logic.importer.fetcher.DoiFetcher;
 import org.jabref.logic.importer.fetcher.GoogleScholar;
 import org.jabref.logic.importer.fetcher.GvkFetcher;
+import org.jabref.logic.importer.fetcher.IacrEprintFetcher;
 import org.jabref.logic.importer.fetcher.IsbnFetcher;
 import org.jabref.logic.importer.fetcher.LibraryOfCongress;
 import org.jabref.logic.importer.fetcher.MathSciNet;
@@ -90,6 +91,7 @@ public static List<IdBasedFetcher> getIdBasedFetchers(ImportFormatPreferences im
         list.add(new MathSciNet(importFormatPreferences));
         list.add(new CrossRef());
         list.add(new LibraryOfCongress());
+        list.add(new IacrEprintFetcher(importFormatPreferences));
         list.sort(Comparator.comparing(WebFetcher::getName));
         return list;
     }

src/main/java/org/jabref/logic/importer/fetcher/IacrEprintFetcher.java

@@ -0,0 +1,201 @@
+package org.jabref.logic.importer.fetcher;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.time.DateTimeException;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+import java.time.temporal.TemporalAccessor;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
+import java.util.function.Predicate;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.jabref.logic.importer.FetcherException;
+import org.jabref.logic.importer.IdBasedFetcher;
+import org.jabref.logic.importer.ImportFormatPreferences;
+import org.jabref.logic.importer.ParseException;
+import org.jabref.logic.importer.fileformat.BibtexParser;
+import org.jabref.logic.l10n.Localization;
+import org.jabref.logic.net.URLDownload;
+import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.FieldName;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class IacrEprintFetcher implements IdBasedFetcher {
+
+    public static final String NAME = "IACR eprints";
+
+    private static final Log LOGGER = LogFactory.getLog(IacrEprintFetcher.class);
+    private static final Pattern DATE_FROM_WEBSITE_AFTER_2000_PATTERN = Pattern.compile("[a-z ]+(\\d{1,2} [A-Za-z][a-z]{2} \\d{4})");
+    private static final DateTimeFormatter DATE_FORMAT_WEBSITE_AFTER_2000 = DateTimeFormatter.ofPattern("d MMM yyyy", Locale.US);
+    private static final Pattern DATE_FROM_WEBSITE_BEFORE_2000_PATTERN = Pattern.compile("[A-Za-z ]+? ([A-Za-z][a-z]{2,10} \\d{1,2}(th|st|nd|rd)?, \\d{4})\\.?");
+    private static final DateTimeFormatter DATE_FORMAT_WEBSITE_BEFORE_2000_LONG_MONTHS = DateTimeFormatter.ofPattern("MMMM d['th']['st']['nd']['rd'] yyyy", Locale.US);
+    private static final DateTimeFormatter DATE_FORMAT_WEBSITE_BEFORE_2000_SHORT_MONTHS = DateTimeFormatter.ofPattern("MMM d['th']['st']['nd']['rd'] yyyy", Locale.US);
+    private static final DateTimeFormatter DATE_FORMAT_BIBTEX = DateTimeFormatter.ISO_LOCAL_DATE;
+    private static final Predicate<String> IDENTIFIER_PREDICATE = Pattern.compile("\\d{4}/\\d{3,5}").asPredicate();
+    private static final String CITATION_URL_PREFIX = "https://eprint.iacr.org/eprint-bin/cite.pl?entry=";
+    private static final String DESCRIPTION_URL_PREFIX = "https://eprint.iacr.org/";
+    private static final Charset WEBSITE_CHARSET = StandardCharsets.ISO_8859_1;
+
+    private final ImportFormatPreferences prefs;
+
+    public IacrEprintFetcher(ImportFormatPreferences prefs) {
+        this.prefs = prefs;
+    }
+
+    @Override
+    public Optional<BibEntry> performSearchById(String identifier) throws FetcherException {
+        String identifierWithoutLettersAndSpaces = identifier.replaceAll("[^0-9/]", " ").trim();
+
+        if (!IDENTIFIER_PREDICATE.test(identifierWithoutLettersAndSpaces)) {
+            throw new FetcherException(Localization.lang("Invalid identifier: '%0'.", identifier));
+        }
+
+        Optional<BibEntry> entry = createEntryFromIacrCitation(identifierWithoutLettersAndSpaces);
+
+        if (entry.isPresent()) {
+            setAdditionalFields(entry.get(), identifierWithoutLettersAndSpaces);
+        }
+
+        return entry;
+    }
+
+    private Optional<BibEntry> createEntryFromIacrCitation(String validIdentifier) throws FetcherException {
+        String bibtexCitationHtml = getHtml(CITATION_URL_PREFIX + validIdentifier);
+        if (bibtexCitationHtml.contains("No such report found")) {
+            throw new FetcherException(Localization.lang("No results found."));
+        }
+        String actualEntry = getRequiredValueBetween("<PRE>", "</PRE>", bibtexCitationHtml);
+
+        try {
+            return BibtexParser.singleFromString(actualEntry, prefs);
+        } catch (ParseException e) {
+            throw new FetcherException(Localization.lang("Entry from %0 could not be parsed.", "IACR"), e);
+        }
+    }
+
+    private void setAdditionalFields(BibEntry entry, String identifier) throws FetcherException {
+        String entryUrl = DESCRIPTION_URL_PREFIX + identifier;
+        String descriptiveHtml = getHtml(entryUrl);
+        entry.setField(FieldName.ABSTRACT, getAbstract(descriptiveHtml));
+        String dateStringAsInHtml = getRequiredValueBetween("<b>Date: </b>", "<p />", descriptiveHtml);
+        entry.setField(FieldName.DATE, getLatestDate(dateStringAsInHtml));
+
+        if (isFromOrAfterYear2000(entry)) {
+            String version = getVersion(identifier, descriptiveHtml);
+            entry.setField(FieldName.VERSION, version);
+            entry.setField(FieldName.URL, entryUrl + "/" + version);
+        } else {
+            // No version information for entries before year 2000
+            entry.setField(FieldName.URL, entryUrl);
+        }
+    }
+
+    private String getVersion(String identifier, String descriptiveHtml) throws FetcherException {
+        String startOfVersionString = "<b>Version: </b><a href=\"/" + identifier + "/";
+        String version = getRequiredValueBetween(startOfVersionString, "\"", descriptiveHtml);
+        return version;
+    }
+
+    private String getAbstract(String descriptiveHtml) throws FetcherException {
+        String abstractText = getRequiredValueBetween("<b>Abstract: </b>", "<p />", descriptiveHtml);
+        // for some reason, all spaces are doubled...
+        abstractText = abstractText.replaceAll("\\s(\\s)", "$1");
+        return abstractText;
+    }
+
+    private String getLatestDate(String dateStringAsInHtml) throws FetcherException {
+        if (dateStringAsInHtml.contains("withdrawn")) {
+            throw new FetcherException(Localization.lang("This paper has been withdrawn."));
+        }
+        String[] rawDates = dateStringAsInHtml.split(", \\D");
+        List<String> formattedDates = new ArrayList<>();
+        for (String rawDate : rawDates) {
+            TemporalAccessor date = parseSingleDateFromWebsite(rawDate);
+            if (date != null) {
+                formattedDates.add(DATE_FORMAT_BIBTEX.format(date));
+            }
+        }
+
+        if (formattedDates.isEmpty()) {
+            throw new FetcherException(Localization.lang("Entry from %0 could not be parsed.", "IACR"));
+        }
+
+        Collections.sort(formattedDates, Collections.reverseOrder());
+        return formattedDates.get(0);
+    }
+
+    private TemporalAccessor parseSingleDateFromWebsite(String dateStringFromWebsite) {
+        TemporalAccessor date = null;
+        // Some entries contain double spaces in the date string (which would break our regexs below)
+        String dateStringWithoutDoubleSpaces = dateStringFromWebsite.replaceAll("\\s\\s+", " ");
+
+        Matcher dateMatcherAfter2000 = DATE_FROM_WEBSITE_AFTER_2000_PATTERN.matcher(dateStringWithoutDoubleSpaces.trim());
+        if (dateMatcherAfter2000.find()) {
+            try {
+                date = DATE_FORMAT_WEBSITE_AFTER_2000.parse(dateMatcherAfter2000.group(1));
+            } catch (DateTimeParseException e) {
+                LOGGER.warn("Date from IACR could not be parsed", e);
+            }
+        }
+
+        // Entries before year 2000 use a variety of date formats - fortunately, we can match them with only two different
+        // date formats (each of which differ from the unified format of post-2000 entries).
+        Matcher dateMatcherBefore2000 = DATE_FROM_WEBSITE_BEFORE_2000_PATTERN.matcher(dateStringWithoutDoubleSpaces.trim());
+        if (dateMatcherBefore2000.find()) {
+            String dateWithoutComma = dateMatcherBefore2000.group(1).replace(",", "");
+            try {
+                date = DATE_FORMAT_WEBSITE_BEFORE_2000_LONG_MONTHS.parse(dateWithoutComma);
+            } catch (DateTimeParseException e) {
+                try {
+                    date = DATE_FORMAT_WEBSITE_BEFORE_2000_SHORT_MONTHS.parse(dateWithoutComma);
+                } catch (DateTimeException e1) {
+                    LOGGER.warn("Date from IACR could not be parsed", e);
+                    LOGGER.warn("Date from IACR could not be parsed", e1);
+                }
+            }
+        }
+
+        return date;
+    }
+
+    private String getHtml(String url) throws FetcherException {
+        try {
+            URLDownload download = new URLDownload(url);
+            return download.asString(WEBSITE_CHARSET);
+        } catch (IOException e) {
+            throw new FetcherException(Localization.lang("Could not retrieve entry data from '%0'.", url), e);
+        }
+    }
+
+    private String getRequiredValueBetween(String from, String to, String haystack) throws FetcherException {
+        String value = StringUtils.substringBetween(haystack, from, to);
+        if (value == null) {
+            throw new FetcherException(Localization.lang("Entry from %0 could not be parsed.", "IACR"));
+        } else {
+            return value;
+        }
+    }
+
+    private boolean isFromOrAfterYear2000(BibEntry entry) throws FetcherException {
+        Optional<String> yearField = entry.getField(FieldName.YEAR);
+        if (yearField.isPresent()) {
+            return Integer.parseInt(yearField.get()) > 2000;
+        }
+        throw new FetcherException(Localization.lang("Entry from %0 could not be parsed.", "IACR"));
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+}

src/main/resources/l10n/JabRef_da.properties

@@ -1,4 +1,3 @@
-#X-Generator: crowdin.com
 %0\ contains\ the\ regular\ expression\ <b>%1</b>=%0 indeholder regulærudtrykket <b>%1</b>
 
 %0\ contains\ the\ term\ <b>%1</b>=%0 indeholder udtrykket <b>%1</b>
@@ -592,7 +591,7 @@ Underline=Underline
 Empty\ Highlight=Empty Highlight
 Empty\ Marking=Empty Marking
 Empty\ Underline=Empty Underline
-The\ marked\ area\ does\ not\ contain\ any\ legible\ text\!=The marked area does not contain any legible text\!
+The\ marked\ area\ does\ not\ contain\ any\ legible\ text!=
 
 Hint\:\ To\ search\ specific\ fields\ only,\ enter\ for\ example\:<p><tt>author\=smith\ and\ title\=electrical</tt>=Hint\: For kun at søge i specifikke felter, skriv f.eks.\:<p><tt>author\=smith and title\=electrical</tt>
 
@@ -2093,8 +2092,8 @@ Open\ console=Open console
 Use\ default\ terminal\ emulator=Use default terminal emulator
 Execute\ command=Execute command
 Note\:\ Use\ the\ placeholder\ %0\ for\ the\ location\ of\ the\ opened\ library\ file.=Note\: Use the placeholder %0 for the location of the opened library file.
-Executing\ command\ "%0"...=Executing command "%0"...
-Error\ occured\ while\ executing\ the\ command\ "%0".=Error occured while executing the command "%0".
+Executing\ command\ \"%0\"...=
+Error\ occured\ while\ executing\ the\ command\ \"%0\".=
 Reformat\ ISSN=Reformat ISSN
 
 Countries\ and\ territories\ in\ English=Countries and territories in English
@@ -2142,7 +2141,7 @@ JabRef\ built\ in\ list=JabRef built in list
 IEEE\ built\ in\ list=IEEE built in list
 
 Event\ log=Event log
-We\ now\ give\ you\ insight\ into\ the\ inner\ workings\ of\ JabRef's\ internals.\ This\ information\ might\ be\ helpful\ to\ diagnose\ the\ root\ cause\ of\ a\ problem.\ Please\ feel\ free\ to\ inform\ the\ developers\ about\ an\ issue.=We now give you insight into the inner workings of JabRef's internals. This information might be helpful to diagnose the root cause of a problem. Please feel free to inform the developers about an issue.
+We\ now\ give\ you\ insight\ into\ the\ inner\ workings\ of\ JabRef\'s\ internals.\ This\ information\ might\ be\ helpful\ to\ diagnose\ the\ root\ cause\ of\ a\ problem.\ Please\ feel\ free\ to\ inform\ the\ developers\ about\ an\ issue.=
 Log\ copied\ to\ clipboard.=Log copied to clipboard.
 Copy\ Log=Copy Log
 Clear\ Log=Clear Log
@@ -2232,7 +2231,7 @@ shared=shared
 should\ contain\ an\ integer\ or\ a\ literal=should contain an integer or a literal
 should\ have\ the\ first\ letter\ capitalized=should have the first letter capitalized
 Tools=Tools
-What's\ new\ in\ this\ version?=What's new in this version?
+What\'s\ new\ in\ this\ version?=
 Want\ to\ help?=Want to help?
 Make\ a\ donation=Make a donation
 get\ involved=get involved
@@ -2352,3 +2351,7 @@ Removes\ all\ hyphenated\ line\ breaks\ in\ the\ field\ content.=Removes all hyp
 Note\ that\ currently,\ JabRef\ does\ not\ run\ with\ Java\ 9.=Note that currently, JabRef does not run with Java 9.
 Your\ current\ Java\ version\ (%0)\ is\ not\ supported.\ Please\ install\ version\ %1\ or\ higher.=Your current Java version (%0) is not supported. Please install version %1 or higher.
 
+Could\ not\ retrieve\ entry\ data\ from\ '%0'.=
+Entry\ from\ %0\ could\ not\ be\ parsed.=
+Invalid\ identifier\:\ '%0'.=
+This\ paper\ has\ been\ withdrawn.=

src/main/resources/l10n/JabRef_de.properties

@@ -1,4 +1,3 @@
-#X-Generator: crowdin.com
 %0\ contains\ the\ regular\ expression\ <b>%1</b>=%0 den regulären Ausdruck <b>%1</b> enthält
 
 %0\ contains\ the\ term\ <b>%1</b>=%0 den Ausdruck <b>%1</b> enthält
@@ -592,7 +591,7 @@ Underline=Unterstreichung
 Empty\ Highlight=Leere Hervorhebung
 Empty\ Marking=Leere Markierung
 Empty\ Underline=Leere Unterstreichung
-The\ marked\ area\ does\ not\ contain\ any\ legible\ text\!=Der markierte Bereich enthält keinen lesbaren Text\!
+The\ marked\ area\ does\ not\ contain\ any\ legible\ text!=
 
 Hint\:\ To\ search\ specific\ fields\ only,\ enter\ for\ example\:<p><tt>author\=smith\ and\ title\=electrical</tt>=Hinweis\: Um ausschließlich bestimmte Felder zu durchsuchen, geben Sie z.B. ein\:<p><tt>author\=smith and title\=electrical</tt>
 
@@ -2093,8 +2092,8 @@ Open\ console=Terminal öffnen
 Use\ default\ terminal\ emulator=Standard Terminal-Emulator verwenden
 Execute\ command=Befehl ausführen
 Note\:\ Use\ the\ placeholder\ %0\ for\ the\ location\ of\ the\ opened\ library\ file.=Hinweis\: %0 als Platzhalter für den Speicherort der Bibliothek benutzen.
-Executing\ command\ "%0"...=Ausführung des Kommandos "%0"...
-Error\ occured\ while\ executing\ the\ command\ "%0".=Während der Ausführung des Befehls "%0" ist ein Fehler aufgetreten.
+Executing\ command\ \"%0\"...=
+Error\ occured\ while\ executing\ the\ command\ \"%0\".=
 Reformat\ ISSN=Formatiere ISSN
 
 Countries\ and\ territories\ in\ English=Länder und Territorien in Englisch
@@ -2142,7 +2141,7 @@ JabRef\ built\ in\ list=Integrierte JabRef-Liste
 IEEE\ built\ in\ list=Integrierte IEEE-Liste
 
 Event\ log=Ereignisprotokoll
-We\ now\ give\ you\ insight\ into\ the\ inner\ workings\ of\ JabRef's\ internals.\ This\ information\ might\ be\ helpful\ to\ diagnose\ the\ root\ cause\ of\ a\ problem.\ Please\ feel\ free\ to\ inform\ the\ developers\ about\ an\ issue.=Sie erhalten nun Einsicht in die Interna von JabRef. Diese Informationen kann dabei helfen die eigentliche Ursache eines Problems zu ergründen.
+We\ now\ give\ you\ insight\ into\ the\ inner\ workings\ of\ JabRef\'s\ internals.\ This\ information\ might\ be\ helpful\ to\ diagnose\ the\ root\ cause\ of\ a\ problem.\ Please\ feel\ free\ to\ inform\ the\ developers\ about\ an\ issue.=
 Log\ copied\ to\ clipboard.=In die Zwischenablage kopiert
 Copy\ Log=Log kopieren
 Clear\ Log=Log löschen
@@ -2232,7 +2231,7 @@ shared=geteilt
 should\ contain\ an\ integer\ or\ a\ literal=Sollte einen Integer oder einen Literal enthalten
 should\ have\ the\ first\ letter\ capitalized=Sollte den ersten Buchstaben großgeschrieben haben
 Tools=Werkzeuge
-What's\ new\ in\ this\ version?=Neuerungen in dieser Version
+What\'s\ new\ in\ this\ version?=
 Want\ to\ help?=Wollen Sie helfen?
 Make\ a\ donation=Spenden
 get\ involved=Engagieren Sie sich
@@ -2352,3 +2351,7 @@ Removes\ all\ hyphenated\ line\ breaks\ in\ the\ field\ content.=Entfernen aller
 Note\ that\ currently,\ JabRef\ does\ not\ run\ with\ Java\ 9.=JabRef kann nicht mit Java 9 verwendet werden.
 Your\ current\ Java\ version\ (%0)\ is\ not\ supported.\ Please\ install\ version\ %1\ or\ higher.=Die verwendete Java Installation (%0) wird nicht unterstützt. Bitte installieren Sie Version %1 oder neuer.
 
+Could\ not\ retrieve\ entry\ data\ from\ '%0'.=
+Entry\ from\ %0\ could\ not\ be\ parsed.=
+Invalid\ identifier\:\ '%0'.=
+This\ paper\ has\ been\ withdrawn.=