Add Worldcat fetcher

CHANGELOG.md

@@ -50,8 +50,10 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve
 - We reintroduced the index column. [#5844](https://github.com/JabRef/jabref/pull/5844)
 - Filenames of external files can no longer contain curly braces. [#5926](https://github.com/JabRef/jabref/pull/5926)
 - We made the filters more easily accessible in the integrity check dialog. [#5955](https://github.com/JabRef/jabref/pull/5955)
+- We added a WorldCcat fetcher. [#1065](https://github.com/JabRef/jabref/issues/1065) [#2581](https://github.com/JabRef/jabref/issues/2581). [WorldCat](http://www.worldcat.org/webservices/catalog/)
 - We reimplemented and improved the dialog "Customize entry types". [#4719](https://github.com/JabRef/jabref/issues/4719)
 - We added an [American Physical Society](https://journals.aps.org/) fetcher. [#818](https://github.com/JabRef/jabref/issues/818)
+- We added an Worldcat fetcher. [#1065](https://github.com/JabRef/jabref/issues/1065) [#2581](https://github.com/JabRef/jabref/issues/2581). [Worldcat](http://www.worldcat.org/webservices/catalog/)
 - We added possibility to enable/disable items quantity in groups. [#6042](https://github.com/JabRef/jabref/issues/6042)
 
 ### Fixed

src/main/java/org/jabref/gui/entryeditor/EntryEditor.java

@@ -335,7 +335,7 @@ private void setupToolBar() {
 
  // Add menu for fetching bibliographic information
  ContextMenu fetcherMenu = new ContextMenu();
- for (EntryBasedFetcher fetcher : WebFetchers.getEntryBasedFetchers(entryEditorPreferences.getImportFormatPreferences())) {
+ for (EntryBasedFetcher fetcher : WebFetchers.getEntryBasedFetchers(entryEditorPreferences.getImportFormatPreferences(), entryEditorPreferences.getAPIApiKeyPreferences())) {
  MenuItem fetcherMenuItem = new MenuItem(fetcher.getName());
  fetcherMenuItem.setOnAction(event -> fetchAndMerge(fetcher));
  fetcherMenu.getItems().add(fetcherMenuItem);

src/main/java/org/jabref/gui/entryeditor/EntryEditorPreferences.java

@@ -7,6 +7,7 @@
 import org.jabref.gui.keyboard.KeyBindingRepository;
 import org.jabref.logic.bibtex.FieldWriterPreferences;
 import org.jabref.logic.bibtexkeypattern.BibtexKeyPatternPreferences;
+import org.jabref.logic.importer.APIKeyPreferences;
 import org.jabref.logic.importer.ImportFormatPreferences;
 import org.jabref.model.entry.field.Field;
 
@@ -16,6 +17,7 @@ public class EntryEditorPreferences {
  private final FieldWriterPreferences fieldWriterPreferences;
  private final ImportFormatPreferences importFormatPreferences;
  private final BibtexKeyPatternPreferences bibtexKeyPatternPreferences;
+ private final APIKeyPreferences apiKeyPreferences;
  private final List<Field> customTabFieldNames;
  private final boolean shouldShowRecommendationsTab;
  private final boolean isMrdlibAccepted;
@@ -24,7 +26,7 @@ public class EntryEditorPreferences {
  private boolean avoidOverwritingCiteKey;
  private final boolean shouldShowLatexCitationsTab;
 
- public EntryEditorPreferences(Map<String, Set<Field>> entryEditorTabList, FieldWriterPreferences fieldWriterPreferences, ImportFormatPreferences importFormatPreferences, List<Field> customTabFieldNames, boolean shouldShowRecommendationsTab, boolean isMrdlibAccepted, boolean shouldShowLatexCitationsTab, boolean showSourceTabByDefault, BibtexKeyPatternPreferences bibtexKeyPatternPreferences, KeyBindingRepository keyBindings, boolean avoidOverwritingCiteKey) {
+ public EntryEditorPreferences(Map<String, Set<Field>> entryEditorTabList, FieldWriterPreferences fieldWriterPreferences, ImportFormatPreferences importFormatPreferences, List<Field> customTabFieldNames, boolean shouldShowRecommendationsTab, boolean isMrdlibAccepted, boolean shouldShowLatexCitationsTab, boolean showSourceTabByDefault, BibtexKeyPatternPreferences bibtexKeyPatternPreferences, KeyBindingRepository keyBindings, boolean avoidOverwritingCiteKey, APIKeyPreferences apiKeyPreferences) {
  this.entryEditorTabList = entryEditorTabList;
  this.fieldWriterPreferences = fieldWriterPreferences;
  this.importFormatPreferences = importFormatPreferences;
@@ -36,6 +38,7 @@ public EntryEditorPreferences(Map<String, Set<Field>> entryEditorTabList, FieldW
  this.keyBindings = keyBindings;
  this.avoidOverwritingCiteKey = avoidOverwritingCiteKey;
  this.shouldShowLatexCitationsTab = shouldShowLatexCitationsTab;
+ this.apiKeyPreferences = apiKeyPreferences;
  }
 
  public Map<String, Set<Field>> getEntryEditorTabList() {
@@ -49,6 +52,10 @@ public FieldWriterPreferences getFieldWriterPreferences() {
  public ImportFormatPreferences getImportFormatPreferences() {
  return importFormatPreferences;
  }
+
+ public APIKeyPreferences getAPIApiKeyPreferences() {
+ return apiKeyPreferences;
+ }
 
  public List<Field> getCustomTabFieldNames() {
  return customTabFieldNames;

src/main/java/org/jabref/gui/preferences/ImportTab.fxml

@@ -31,4 +31,20 @@
  <TextField fx:id="fileDirPattern" GridPane.columnIndex="1" GridPane.rowIndex="1"
  prefWidth="300" minWidth="300" maxWidth="300"/>
  </GridPane>
+
+ <Label styleClass="sectionHeader" text="%API keys"/>
+ <GridPane hgap="4.0" vgap="4.0">
+ <columnConstraints>
+ <ColumnConstraints hgrow="SOMETIMES"/>
+ <ColumnConstraints hgrow="SOMETIMES"/>
+ </columnConstraints>
+ <rowConstraints>
+ <RowConstraints vgrow="SOMETIMES"/>
+ <RowConstraints vgrow="SOMETIMES"/>
+ </rowConstraints>
+ <Label text="%Worldcat Fetcher Key" GridPane.rowIndex="1"/>
+ <TextField fx:id="worldcatKey" GridPane.columnIndex="1" GridPane.rowIndex="1"
+ prefWidth="300" minWidth="300" maxWidth="300"/>
+ </GridPane>
+
 </fx:root>

src/main/java/org/jabref/gui/preferences/ImportTabView.java

@@ -13,6 +13,8 @@ public class ImportTabView extends AbstractPreferenceTabView<ImportTabViewModel>
 
  @FXML private ComboBox<String> fileNamePattern;
  @FXML private TextField fileDirPattern;
+
+ @FXML private TextField worldcatKey;
 
  public ImportTabView(JabRefPreferences preferences) {
  this.preferences = preferences;
@@ -28,6 +30,8 @@ public void initialize () {
  fileNamePattern.valueProperty().bindBidirectional(viewModel.fileNamePatternProperty());
  fileNamePattern.itemsProperty().bind(viewModel.defaultFileNamePatternsProperty());
  fileDirPattern.textProperty().bindBidirectional(viewModel.fileDirPatternProperty());
+
+ worldcatKey.textProperty().bindBidirectional(viewModel.apiKeyPatternProperty());
  }
 
  @Override

src/main/java/org/jabref/gui/preferences/ImportTabViewModel.java

@@ -20,6 +20,8 @@ public class ImportTabViewModel implements PreferenceTabViewModel {
  private final StringProperty fileNamePatternProperty = new SimpleStringProperty();
  private final StringProperty fileDirPatternProperty = new SimpleStringProperty();
 
+ private final StringProperty apiKeyPatternProperty = new SimpleStringProperty();
+
  private final DialogService dialogService;
  private final JabRefPreferences preferences;
 
@@ -32,12 +34,14 @@ public ImportTabViewModel(DialogService dialogService, JabRefPreferences prefere
  public void setValues() {
  fileNamePatternProperty.setValue(preferences.get(JabRefPreferences.IMPORT_FILENAMEPATTERN));
  fileDirPatternProperty.setValue(preferences.get(JabRefPreferences.IMPORT_FILEDIRPATTERN));
+ apiKeyPatternProperty.setValue(preferences.get(JabRefPreferences.IMPORT_API_WORLDCAT));
  }
 
  @Override
  public void storeSettings() {
  preferences.put(JabRefPreferences.IMPORT_FILENAMEPATTERN, fileNamePatternProperty.getValue());
  preferences.put(JabRefPreferences.IMPORT_FILEDIRPATTERN, fileDirPatternProperty.getValue());
+ preferences.put(JabRefPreferences.IMPORT_API_WORLDCAT, apiKeyPatternProperty.getValue());
  }
 
  @Override
@@ -55,4 +59,6 @@ public List<String> getRestartWarnings() {
  public StringProperty fileNamePatternProperty() { return fileNamePatternProperty; }
 
  public StringProperty fileDirPatternProperty() { return fileDirPatternProperty; }
+
+ public StringProperty apiKeyPatternProperty() { return apiKeyPatternProperty; }
 }

src/main/java/org/jabref/logic/importer/APIKeyPreferences.java

@@ -0,0 +1,11 @@
+package org.jabref.logic.importer;
+
+public class APIKeyPreferences {
+ private final String worldcatKey;
+
+ public APIKeyPreferences(String worldcatKey) {
+ this.worldcatKey = worldcatKey;
+ }
+
+ public String getWorldcatKey() { return worldcatKey; }
+}

src/main/java/org/jabref/logic/importer/WebFetchers.java

@@ -34,6 +34,7 @@
 import org.jabref.logic.importer.fetcher.SpringerFetcher;
 import org.jabref.logic.importer.fetcher.SpringerLink;
 import org.jabref.logic.importer.fetcher.TitleFetcher;
+import org.jabref.logic.importer.fetcher.WorldcatFetcher;
 import org.jabref.logic.importer.fetcher.ZbMATH;
 import org.jabref.model.entry.field.Field;
 import org.jabref.model.entry.field.StandardField;
@@ -127,13 +128,14 @@ public static SortedSet<IdBasedFetcher> getIdBasedFetchers(ImportFormatPreferenc
  /**
  * @return sorted set containing entry based fetchers
  */
- public static SortedSet<EntryBasedFetcher> getEntryBasedFetchers(ImportFormatPreferences importFormatPreferences) {
+ public static SortedSet<EntryBasedFetcher> getEntryBasedFetchers(ImportFormatPreferences importFormatPreferences, APIKeyPreferences apiKeyPreferences) {
  SortedSet<EntryBasedFetcher> set = new TreeSet<>(Comparator.comparing(WebFetcher::getName));
  set.add(new AstrophysicsDataSystem(importFormatPreferences));
  set.add(new DoiFetcher(importFormatPreferences));
  set.add(new IsbnFetcher(importFormatPreferences));
  set.add(new MathSciNet(importFormatPreferences));
  set.add(new CrossRef());
+ set.add(new WorldcatFetcher(apiKeyPreferences.getWorldcatKey()));
  return set;
  }
 

src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java

@@ -0,0 +1,216 @@
+package org.jabref.logic.importer.fetcher;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.jabref.logic.importer.EntryBasedFetcher;
+import org.jabref.logic.importer.FetcherException;
+import org.jabref.logic.importer.ParserResult;
+import org.jabref.logic.importer.fileformat.WorldcatImporter;
+import org.jabref.logic.net.URLDownload;
+import org.jabref.model.database.BibDatabase;
+import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.field.StandardField;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+
+/**
+ * EntryBasedFetcher that searches the Worldcat database
+ * 
+ * @see https://www.oclc.org/developer/develop/web-services/worldcat-search-api/bibliographic-resource.en.html
+ */
+public class WorldcatFetcher implements EntryBasedFetcher {
+
+ private String WORLDCAT_OPEN_SEARCH_URL = "http://www.worldcat.org/webservices/catalog/search/opensearch?wskey=";
+ private String WORLDCAT_READ_URL = "http://www.worldcat.org/webservices/catalog/content/{OCLC-NUMBER}?recordSchema=info%3Asrw%2Fschema%2F1%2Fdc&wskey=";
+
+
+ public WorldcatFetcher (String worldcatKey) {
+ WORLDCAT_OPEN_SEARCH_URL += worldcatKey;
+ WORLDCAT_READ_URL += worldcatKey;
+ }
+
+ @Override
+ public String getName () {
+ return "Worldcat Fetcher";
+ }
+
+ /**
+ * Create a open search query with specified title
+ * @param title the title to include in the query
+ * @return the earch query for the api
+ */
+ private String getOpenSearchURL (String title) throws MalformedURLException { 
+ String query = "&q=srw.ti+all+\"" + title + "\"";
+ URL url = new URL (WORLDCAT_OPEN_SEARCH_URL + query);
+ return url.toString ();
+ }
+
+ /**
+ * Make request to open search API of Worldcat, with specified title
+ * @param title the title of the search
+ * @return the body of the HTTP response
+ */
+ private String makeOpenSearchRequest (String title) throws FetcherException { 
+ try {
+ URLDownload urlDownload = new URLDownload (getOpenSearchURL (title));
+ URLDownload.bypassSSLVerification ();
+ String resp = urlDownload.asString ();
+
+ return resp;
+ } catch (MalformedURLException e) {
+ throw new FetcherException ("Bad url", e);
+ } catch (IOException e) {
+ throw new FetcherException ("Error with Open Search Request (Worldcat)", e);
+ } 
+ }
+
+ /**
+ * Get more information about a article through its OCLC id. Picks the first 
+ * element with this tag
+ * @param id the oclc id
+ * @return the XML element that contains all tags
+ */
+ private Element getSpecificInfoOnOCLC (String id) throws IOException {
+ URLDownload urlDownload = new URLDownload (WORLDCAT_READ_URL.replace ("{OCLC-NUMBER}", id));
+ URLDownload.bypassSSLVerification ();
+ String resp = urlDownload.asString (); 
+
+ Document mainDoc = parse (resp);
+ NodeList parentElemOfTags = mainDoc.getElementsByTagName ("oclcdcs");
+
+ return (Element) parentElemOfTags.item (0);
+ }
+
+ /**
+ * Parse a string to an xml document
+ * @param s the string to be parsed
+ * @return XML document representing the content of s
+ * @throws IllegalArgumentException if s is badly formated or other exception occurs during parsing
+ */
+ private Document parse (String s) {
+ try { 
+ BufferedReader r = new BufferedReader (new StringReader(s));
+ DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance ();
+ DocumentBuilder builder = factory.newDocumentBuilder ();
+
+ return builder.parse (new InputSource (r));
+ } catch (ParserConfigurationException e) {
+ throw new IllegalArgumentException ("Parser Config Exception: " + e.getMessage (), e);
+ } catch (SAXException e) {
+ throw new IllegalArgumentException ("SAX Exception: " + e.getMessage (), e);
+ } catch (IOException e) {
+ throw new IllegalArgumentException ("IO Exception: " + e.getMessage (), e);
+ }
+ }
+
+ /**
+ * Parse OpenSearch XML to retrieve OCLC-ids and fetch details about those. 
+ * Returns new XML doc in the form
+ * <pre>
+ * {@literal <entries>}
+ * {@literal <entry>}
+ * OCLC DETAIL
+ * {@literal </entry>}
+ * ...
+ * {@literal </entries>}
+ * </pre>
+ * @param doc the open search result
+ * @return the new xml document
+ * @throws FetcherException
+ */
+ private Document parseOpenSearchXML(Document doc) throws FetcherException {
+ try {
+ Element feed = (Element) doc.getElementsByTagName ("feed").item (0);
+ NodeList entryXMLList = feed.getElementsByTagName ("entry");
+
+ DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance ();
+ DocumentBuilder docBuilder = docFactory.newDocumentBuilder ();
+
+ Document newDoc = docBuilder.newDocument ();
+
+ Element root = newDoc.createElement ("entries");
+ newDoc.appendChild (root);
+ for (int i = 0; i < entryXMLList.getLength (); i++) {
+ Element xmlEntry = (Element) entryXMLList.item (i);
+
+ String oclc = xmlEntry.getElementsByTagName ("oclcterms:recordIdentifier").item (0).getTextContent ();
+ Element detailedInfo = getSpecificInfoOnOCLC (oclc);
+
+ Element newEntry = newDoc.createElement ("entry");
+ newEntry.appendChild (detailedInfo);
+
+ root.appendChild (newEntry);
+ }
+ return newDoc;
+ } catch (ParserConfigurationException e) {
+ throw new FetcherException("Error with XML creation (Worldcat fetcher)", e);
+ } catch (IOException e) {
+ throw new FetcherException("Error with OCLC parsing (Worldcat fetcher)", e);
+ }
+ }
+
+ @Override
+ public List<BibEntry> performSearch (BibEntry entry) throws FetcherException {
+ Optional<String> entryTitle = entry.getLatexFreeField (StandardField.TITLE);
+ if (entryTitle.isPresent ()) {
+ String openSearchXMLResponse = makeOpenSearchRequest (entryTitle.get ());
+ Document openSearchDocument = parse (openSearchXMLResponse);
+
+ Document detailedXML = parseOpenSearchXML (openSearchDocument);
+ String detailedXMLString;
+ try {
+ //Transform XML to String
+ TransformerFactory tf = TransformerFactory.newInstance ();
+ Transformer t = tf.newTransformer ();
+ StringWriter sw = new StringWriter ();
+ t.transform(new DOMSource (detailedXML), new StreamResult (sw));
+ detailedXMLString = sw.toString ();
+ } catch (TransformerException e) {
+ throw new FetcherException ("Could not transform XML", e);
+ }
+
+ //TODO: - Send xml as string to Importer
+ // - Parse the new kind of XML in importer
+
+ WorldcatImporter importer = new WorldcatImporter (); 
+ ParserResult parserResult;
+ try { 
+ if (importer.isRecognizedFormat (detailedXMLString)) {
+ parserResult = importer.importDatabase (detailedXMLString);
+ } else { 
+ // For displaying An ErrorMessage
+ BibDatabase errorBibDataBase = new BibDatabase ();
+ parserResult = new ParserResult (errorBibDataBase);
+ }
+ return parserResult.getDatabase ().getEntries ();
+ }
+ catch (IOException e) {
+ throw new FetcherException ("Could not perform search (Worldcat) ", e);
+ }
+ } else {
+ return new ArrayList<>();
+ }
+ }
+
+}