From 7d72fd8f4d3b3f7ec4afa59484b0db8d01a02499 Mon Sep 17 00:00:00 2001 From: Glenn Olsson Date: Mon, 24 Feb 2020 20:50:08 +0100 Subject: [PATCH 01/48] Add performSearch test case Add more tests --- .../importer/fetcher/WorldcatFetcher.java | 23 +++++++++ .../importer/fetcher/WorldcatFetcherTest.java | 51 +++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java create mode 100644 src/test/java/org/jabref/logic/importer/fetcher/WorldcatFetcherTest.java diff --git a/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java new file mode 100644 index 00000000000..3968ca0c84f --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java @@ -0,0 +1,23 @@ +package org.jabref.logic.importer.fetcher; + +import java.util.List; + +import org.jabref.logic.importer.EntryBasedFetcher; +import org.jabref.logic.importer.FetcherException; +import org.jabref.model.entry.BibEntry; + +public class WorldcatFetcher implements EntryBasedFetcher { + + @Override + public String getName() { + // TODO Auto-generated method stub + return null; + } + + @Override + public List performSearch(BibEntry entry) throws FetcherException { + // TODO Auto-generated method stub + return null; + } + +} \ No newline at end of file diff --git a/src/test/java/org/jabref/logic/importer/fetcher/WorldcatFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/WorldcatFetcherTest.java new file mode 100644 index 00000000000..6f093214e69 --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/WorldcatFetcherTest.java @@ -0,0 +1,51 @@ +package org.jabref.logic.importer.fetcher; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; + +import org.jabref.logic.importer.FetcherException; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.StandardField; +import org.jabref.testutils.category.FetcherTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.BeforeEach; + +@FetcherTest +public class WorldcatFetcherTest{ + + private WorldcatFetcher fetcher; + + @BeforeEach + public void setUp() { + fetcher = new WorldcatFetcher(); + } + + @Test + public void testPerformSearchForBadTitle() throws FetcherException{ + BibEntry entry = new BibEntry(); + //Mashing keyboard. Verified on https://platform.worldcat.org/api-explorer/apis/wcapi/Bib/OpenSearch + entry.setField(StandardField.TITLE, "ASDhbsd fnm"); + List list = fetcher.performSearch(entry); + assertTrue(list.isEmpty()); + } + + @Test + public void testPerformSearchForExistingTitle() throws FetcherException{ + BibEntry entry = new BibEntry(); + //Example "The very best of Glenn Miller". Verified on same link as above + entry.setField(StandardField.TITLE, "The very best of Glenn"); + List list = fetcher.performSearch(entry); + assertFalse(list.isEmpty()); + } + + @Test + public void testPerformSearchForNullTitle() throws FetcherException{ + BibEntry entry = new BibEntry(); + entry.setField(StandardField.TITLE, null); + List list = fetcher.performSearch(entry); + assertTrue(list.isEmpty()); + } + +} \ No newline at end of file From ee96fdc356a5c9073ff1c838c549644b3f235a1d Mon Sep 17 00:00:00 2001 From: Glenn Olsson Date: Tue, 25 Feb 2020 13:59:31 +0100 Subject: [PATCH 02/48] Add basics of fetcher --- .../importer/fetcher/WorldcatFetcher.java | 49 +++++++++++++++++-- .../logic/importer/WebFetchersTest.java | 5 +- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java index 3968ca0c84f..60f1ed6f6bd 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java @@ -1,23 +1,64 @@ package org.jabref.logic.importer.fetcher; +import java.io.IOException; +import java.util.ArrayList; import java.util.List; +import java.util.Optional; import org.jabref.logic.importer.EntryBasedFetcher; import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.net.URLDownload; import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.StandardField; public class WorldcatFetcher implements EntryBasedFetcher { + private final static String NAME = "Worldcat Fetcher"; + private final static String API_KEY = "API-KEY"; + private final static String WORLDCAT_URL = "http://www.worldcat.org/webservices/catalog/search/worldcat/opensearch?wskey=" + API_KEY + "&"; + @Override public String getName() { - // TODO Auto-generated method stub - return null; + return NAME; + } + + private String getURL(String title){ + String query = "q=srw.ti+all+\"" + title.replaceAll(" ", "%20") + "\""; + return WORLDCAT_URL + query.replace("\"", "%22"); + } + + private String makeRequest(String title) throws FetcherException{ + try { + URLDownload urlDownload = new URLDownload(getURL(title)); + URLDownload.bypassSSLVerification(); + String resp = urlDownload.asString(); + + return resp; + } catch (IOException e) { + e.printStackTrace(); + throw new FetcherException("IO Exception", e); + } } + //Example http://www.worldcat.org/webservices/catalog/search/worldcat/opensearch?q=srw.ti+all+"The very best of Glenn"&wskey={built-in-api-key} @Override public List performSearch(BibEntry entry) throws FetcherException { - // TODO Auto-generated method stub - return null; + Optional entryTitle = entry.getLatexFreeField(StandardField.TITLE); + if(entryTitle.isPresent()){ + String xmlResponse = makeRequest(entryTitle.get()); + /* + - Create importer (implements importer) + - Check if recognized format + - yes + - set parserResult to importer.importDatabase + - return pR.getDbase.getEntries() + - no + - return empty bibdatabase's entries + */ + return null; + } else { + return new ArrayList<>(); + } } } \ No newline at end of file diff --git a/src/test/java/org/jabref/logic/importer/WebFetchersTest.java b/src/test/java/org/jabref/logic/importer/WebFetchersTest.java index 904f8198716..44382d727fb 100644 --- a/src/test/java/org/jabref/logic/importer/WebFetchersTest.java +++ b/src/test/java/org/jabref/logic/importer/WebFetchersTest.java @@ -9,6 +9,7 @@ import org.jabref.logic.importer.fetcher.IsbnViaEbookDeFetcher; import org.jabref.logic.importer.fetcher.IsbnViaOttoBibFetcher; import org.jabref.logic.importer.fetcher.MrDLibFetcher; +import org.jabref.logic.importer.fetcher.WorldcatFetcher; import io.github.classgraph.ClassGraph; import io.github.classgraph.ClassInfoList; @@ -61,7 +62,9 @@ void getEntryBasedFetchersReturnsAllFetcherDerivingFromEntryBasedFetcher() throw Set> expected = controlClasses.loadClasses().stream().collect(Collectors.toSet()); expected.remove(EntryBasedParserFetcher.class); - expected.remove(MrDLibFetcher.class); + expected.remove(MrDLibFetcher.class); + expected.remove(WorldcatFetcher.class); + assertEquals(expected, getClasses(idFetchers)); } } From 61a45fe9e225f956d1f339612c8a25217fda0b54 Mon Sep 17 00:00:00 2001 From: Glenn Olsson Date: Tue, 25 Feb 2020 14:30:18 +0100 Subject: [PATCH 03/48] Start work on importer --- .../importer/fileformat/WorldcatImporter.java | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 src/main/java/org/jabref/logic/importer/fileformat/WorldcatImporter.java diff --git a/src/main/java/org/jabref/logic/importer/fileformat/WorldcatImporter.java b/src/main/java/org/jabref/logic/importer/fileformat/WorldcatImporter.java new file mode 100644 index 00000000000..50c927e76f6 --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fileformat/WorldcatImporter.java @@ -0,0 +1,88 @@ +package org.jabref.logic.importer.fileformat; + +import java.io.BufferedReader; +import java.io.IOException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +import org.jabref.logic.importer.Importer; +import org.jabref.logic.importer.ParserResult; +import org.jabref.logic.util.FileType; +import org.jabref.logic.util.StandardFileType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.xml.sax.SAXException; + +public class WorldcatImporter extends Importer { + + private static final Logger LOGGER = LoggerFactory.getLogger(WorldcatImporter.class); + + private final static String NAME = "WorldcatImporter"; + private final static String DESCRIPTION = "Takes valid XML from Worldcat Open Search and parses them to BibEntry"; + + + private String stringifyReader(BufferedReader bf){ + StringBuilder sb = new StringBuilder(); + String ln; + try { + while((ln = bf.readLine()) != null){ + sb.append(ln); + } + } catch (IOException e) { + LOGGER.error(e.getMessage(), e); + throw new IllegalArgumentException("Bad argument"); + } + } + + private Document parse(String s){ + try{ + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + + return builder.parse(s); + } catch(ParserConfigurationException e){ + throw new IllegalArgumentException("Parser Config Exception: " + e.getMessage(), e); + } catch(SAXException e){ + throw new IllegalArgumentException("SAX Exception: " + e.getMessage(), e); + } catch(IOException e){ + throw new IllegalArgumentException("IO Exception: " + e.getMessage(), e); + } + } + + @Override + public boolean isRecognizedFormat(BufferedReader input) throws IOException { + try { + String strInput = stringifyReader(input); + Document doc = parse(strInput); + return doc.getElementsByTagName("feed") != null; + } catch (IllegalArgumentException e) { + + return false; + } + } + + @Override + public ParserResult importDatabase(BufferedReader input) throws IOException { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getName() { + return NAME; + } + + @Override + public FileType getFileType() { + return StandardFileType.XML; + } + + @Override + public String getDescription() { + return DESCRIPTION; + } + +} \ No newline at end of file From b2570859289e73c91f1e55f17ed558f69326e67e Mon Sep 17 00:00:00 2001 From: Glenn Olsson Date: Wed, 26 Feb 2020 10:56:56 +0100 Subject: [PATCH 04/48] Create worldcat importer test Start parsing XML Finish importer, still no key though --- .../importer/fetcher/WorldcatFetcher.java | 45 +++++---- .../importer/fileformat/WorldcatImporter.java | 99 +++++++++++++++---- .../logic/importer/WorldcatImporterTest.java | 54 ++++++++++ 3 files changed, 161 insertions(+), 37 deletions(-) create mode 100644 src/test/java/org/jabref/logic/importer/WorldcatImporterTest.java diff --git a/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java index 60f1ed6f6bd..83166676e5b 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java @@ -7,29 +7,33 @@ import org.jabref.logic.importer.EntryBasedFetcher; import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.importer.ParserResult; +import org.jabref.logic.importer.fileformat.WorldcatImporter; import org.jabref.logic.net.URLDownload; +import org.jabref.model.database.BibDatabase; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; public class WorldcatFetcher implements EntryBasedFetcher { private final static String NAME = "Worldcat Fetcher"; - private final static String API_KEY = "API-KEY"; - private final static String WORLDCAT_URL = "http://www.worldcat.org/webservices/catalog/search/worldcat/opensearch?wskey=" + API_KEY + "&"; + public final static String API_KEY = "API-KEY"; + + private final static String WORLDCAT_OPEN_SEARCH_URL = "http://www.worldcat.org/webservices/catalog/search/opensearch?wskey=" + API_KEY; @Override public String getName() { return NAME; } - private String getURL(String title){ - String query = "q=srw.ti+all+\"" + title.replaceAll(" ", "%20") + "\""; - return WORLDCAT_URL + query.replace("\"", "%22"); + private String getOpenSearchURL(String title){ + String query = "&q=srw.ti+all+\"" + title.replaceAll(" ", "%20") + "\""; + return WORLDCAT_OPEN_SEARCH_URL + query.replace("\"", "%22"); } - private String makeRequest(String title) throws FetcherException{ + private String makeOpenSearchRequest(String title) throws FetcherException{ try { - URLDownload urlDownload = new URLDownload(getURL(title)); + URLDownload urlDownload = new URLDownload(getOpenSearchURL(title)); URLDownload.bypassSSLVerification(); String resp = urlDownload.asString(); @@ -45,17 +49,22 @@ private String makeRequest(String title) throws FetcherException{ public List performSearch(BibEntry entry) throws FetcherException { Optional entryTitle = entry.getLatexFreeField(StandardField.TITLE); if(entryTitle.isPresent()){ - String xmlResponse = makeRequest(entryTitle.get()); - /* - - Create importer (implements importer) - - Check if recognized format - - yes - - set parserResult to importer.importDatabase - - return pR.getDbase.getEntries() - - no - - return empty bibdatabase's entries - */ - return null; + String xmlResponse = makeOpenSearchRequest(entryTitle.get()); + WorldcatImporter importer = new WorldcatImporter(); + ParserResult parserResult; + try{ + if(importer.isRecognizedFormat(xmlResponse)){ + parserResult = importer.importDatabase(xmlResponse); + } else{ + // For displaying An ErrorMessage + BibDatabase errorBibDataBase = new BibDatabase(); + parserResult = new ParserResult(errorBibDataBase); + } + return parserResult.getDatabase().getEntries(); + } + catch(IOException e){ + throw new FetcherException("IO Exception " + e.getMessage(), e); + } } else { return new ArrayList<>(); } diff --git a/src/main/java/org/jabref/logic/importer/fileformat/WorldcatImporter.java b/src/main/java/org/jabref/logic/importer/fileformat/WorldcatImporter.java index 50c927e76f6..464fed7d615 100644 --- a/src/main/java/org/jabref/logic/importer/fileformat/WorldcatImporter.java +++ b/src/main/java/org/jabref/logic/importer/fileformat/WorldcatImporter.java @@ -2,6 +2,9 @@ import java.io.BufferedReader; import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; @@ -9,40 +12,39 @@ import org.jabref.logic.importer.Importer; import org.jabref.logic.importer.ParserResult; +import org.jabref.logic.importer.fetcher.WorldcatFetcher; +import org.jabref.logic.net.URLDownload; import org.jabref.logic.util.FileType; import org.jabref.logic.util.StandardFileType; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.StandardField; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; import org.xml.sax.SAXException; public class WorldcatImporter extends Importer { + private String WORLDCAT_READ_URL; + private static final Logger LOGGER = LoggerFactory.getLogger(WorldcatImporter.class); private final static String NAME = "WorldcatImporter"; private final static String DESCRIPTION = "Takes valid XML from Worldcat Open Search and parses them to BibEntry"; - - private String stringifyReader(BufferedReader bf){ - StringBuilder sb = new StringBuilder(); - String ln; - try { - while((ln = bf.readLine()) != null){ - sb.append(ln); - } - } catch (IOException e) { - LOGGER.error(e.getMessage(), e); - throw new IllegalArgumentException("Bad argument"); - } + public WorldcatImporter(){ + this.WORLDCAT_READ_URL = "http://www.worldcat.org/webservices/catalog/content/{OCLC-NUMBER}?recordSchema=info%3Asrw%2Fschema%2F1%2Fdc&wskey=" + WorldcatFetcher.API_KEY; } - private Document parse(String s){ + private Document parse(BufferedReader s){ try{ DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); DocumentBuilder builder = factory.newDocumentBuilder(); - return builder.parse(s); + return builder.parse(new InputSource(s)); } catch(ParserConfigurationException e){ throw new IllegalArgumentException("Parser Config Exception: " + e.getMessage(), e); } catch(SAXException e){ @@ -52,22 +54,81 @@ private Document parse(String s){ } } + private Element getSpecificInfoOnOCLC(String id) throws IOException { + URLDownload urlDownload = new URLDownload(WORLDCAT_READ_URL.replace("{OCLC-NUMBER}", id)); + URLDownload.bypassSSLVerification(); + String resp = urlDownload.asString(); + + //Used when no API key is present. Comment out lines above as well + // String resp = " Miller, Glenn. 2010 Remastered versions of Glenn Miller's original recordings which bring a freshness to his beloved classics. The album also features a version of 'In the mood' with Jodie Prenger, winner of the BBC's 'I'll do anything'. In the mood -- Moonlight serenade -- Don't sit under the apple tree (with anyone else but me) -- Tuxedo junction -- A string of pearls -- Pennsylvania 6-5000 -- Chattanooga choo-choo -- American patrol -- (I've got a gal in) Kalamazoo -- On a little street in Singapore -- The St. Louis blues march -- A nightingale sang in Berkeley Square -- Star dust -- Little brown jug -- When you wish upon a star -- The woodpecker song -- G.I. jive -- Fools rush in -- Over there -- Blueberry hill -- Over the rainbow -- Serenade in blue -- When Johnny comes marching home -- In the mood (feat. Jodie Prenger). 1 CD (72 min., 18 sec.) ; 4 3/4 in. eng Sony Music Entertainment Dance orchestra music. Big band music. Popular music--1931-1940. Popular music--1941-1950. Compact discs. The very best of Glenn Miller Sound 101104 754508587 "; + + Document mainDoc = parse(new BufferedReader(new StringReader(resp))); + NodeList parentElemOfTags = mainDoc.getElementsByTagName("oclcdcs"); + + return (Element) parentElemOfTags.item(0); + } + + @Override public boolean isRecognizedFormat(BufferedReader input) throws IOException { try { - String strInput = stringifyReader(input); - Document doc = parse(strInput); + Document doc = parse(input); return doc.getElementsByTagName("feed") != null; } catch (IllegalArgumentException e) { - return false; } } + private Element getElementByTag(Element xml, String tag){ + NodeList nl = xml.getElementsByTagName(tag); + return (Element) nl.item(0); + } + + private BibEntry xmlEntryToBibEntry(Element xmlEntry) throws IOException{ + Element authorsElem = getElementByTag(xmlEntry, "author"); + String authors = getElementByTag(authorsElem, "name").getTextContent(); + + String title = getElementByTag(xmlEntry, "title").getTextContent(); + + String url = getElementByTag(xmlEntry, "link").getAttribute("href"); + + String oclc = xmlEntry.getElementsByTagName("oclcterms:recordIdentifier").item(0).getTextContent(); + System.out.println(oclc + " , " + title); + Element detailedInfo = getSpecificInfoOnOCLC(oclc); + + String date = detailedInfo.getElementsByTagName("dc:date").item(0).getTextContent(); + + String publisher = detailedInfo.getElementsByTagName("dc:publisher").item(0).getTextContent(); + + BibEntry entry = new BibEntry(); + + entry.setField(StandardField.AUTHOR, authors); + entry.setField(StandardField.TITLE, title); + entry.setField(StandardField.URL, url); + entry.setField(StandardField.YEAR, date); + entry.setField(StandardField.JOURNAL, publisher); + + return entry; + } + + private ParserResult docToParserRes(Document doc) throws IOException{ + Element feed = (Element) doc.getElementsByTagName("feed").item(0); + NodeList entryXMLList = feed.getElementsByTagName("entry"); + + List bibList = new ArrayList<>(entryXMLList.getLength()); + for(int i = 0; i < entryXMLList.getLength(); i++){ + Element xmlEntry = (Element) entryXMLList.item(i); + BibEntry bibEntry = xmlEntryToBibEntry(xmlEntry); + bibList.add(bibEntry); + } + + return new ParserResult(bibList); + } + @Override public ParserResult importDatabase(BufferedReader input) throws IOException { - // TODO Auto-generated method stub - return null; + Document parsedDoc = parse(input); + return docToParserRes(parsedDoc); } @Override diff --git a/src/test/java/org/jabref/logic/importer/WorldcatImporterTest.java b/src/test/java/org/jabref/logic/importer/WorldcatImporterTest.java new file mode 100644 index 00000000000..6fc59a4647f --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/WorldcatImporterTest.java @@ -0,0 +1,54 @@ +package org.jabref.logic.importer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; + +import org.jabref.logic.importer.fileformat.WorldcatImporter; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class WorldcatImporterTest { + private final String XML_WITH_RESULT = "OCLC Worldcat Search: srw.ti all \"The very best of Glenn\"http://worldcat.org/webservices/catalog/search/worldcat/opensearch?q=srw.ti+all+%22The+very+best+of+Glenn%22&start=1&count=10&format=atom&wskey={built-in-api-key}2020-02-25T06:16:25-05:00Search results for srw.ti all \"The very best of Glenn\" at http://worldcat.org/webservices/catalog89110Miller, Glenn.The very best of Glenn Millerhttp://worldcat.org/oclc/7545085872017-06-29T19:46:28ZRemastered versions of Glenn Miller's original recordings which bring a freshness to his beloved classics. The album also features a version of 'In the mood' with Jodie Prenger, winner of the BBC's 'I'll do anything'.754508587Brennan, Walter, 1894-1974.The very best of Walter Brennan.http://worldcat.org/oclc/170097872018-07-02T10:57:48Zurn:LCCN:9578975117009787Frey, Glenn, composer, performer.Above the clouds : the very best of Glenn Freyhttp://worldcat.org/oclc/10409906042019-05-06T02:09:58Z1040990604Miller, Glenn, 1954- performer.The very best of Glenn Miller.http://worldcat.org/oclc/11404164032020-02-13T17:01:04Z1140416403Miller, Glenn.Very Best of Glenn Miller.http://worldcat.org/oclc/6305478162018-11-28T18:44:17Z630547816Yarbrough, Glenn.Glenn yarbrough - his very besthttp://worldcat.org/oclc/10983940252019-04-25T11:56:52Z1098394025Miller, Glenn.The very best of Glenn Miller : hits & raritieshttp://worldcat.org/oclc/8094628162020-01-23T10:02:43Z809462816Miller, Glenn, 1904-1944, performer.The very best of Glenn Miller & his orchestra.http://worldcat.org/oclc/11232150702019-11-07T01:19:22Z1123215070Miller, Glenn, 1904-1944.The very best of Glenn Miller : in the mood.http://worldcat.org/oclc/556888332016-11-17T00:02:35Z55688833Miller, Glenn.Glenn Miller story : the very best ofhttp://worldcat.org/oclc/4217187472018-10-08T09:50:51Z421718747"; + private final String XML_WITHOUT_RESULT = ""; + + WorldcatImporter importer; + + @BeforeEach + public void setUp(){ + importer = new WorldcatImporter(); + } + + @Test + public void withResultIsRecognizedFormat() throws IOException{ + boolean isReq = importer.isRecognizedFormat(XML_WITH_RESULT); + assertTrue(isReq); + } + + @Test + public void withoutResultIsRecognizedFormat() throws IOException{ + boolean isReq = importer.isRecognizedFormat(XML_WITHOUT_RESULT); + assertTrue(isReq); + } + + @Test + public void badXMLIsNotRecognizedFormat() throws IOException{ + boolean isReq = importer.isRecognizedFormat("Nah bruh"); + assertFalse(isReq); + } + + @Test + public void withResultReturnsNonEmptyResult() throws IOException{ + ParserResult res = importer.importDatabase(XML_WITH_RESULT); + assertTrue(res.getDatabase().getEntries().size() > 0); + } + + @Test + public void withoutResultReturnsEmptyResult() throws IOException{ + ParserResult res = importer.importDatabase(XML_WITHOUT_RESULT); + assertEquals(0, res.getDatabase().getEntries().size()); + } + +} \ No newline at end of file From 27069f4770c8f7472e955e5ce20e9a52c9b4b904 Mon Sep 17 00:00:00 2001 From: Glenn Olsson Date: Thu, 27 Feb 2020 10:57:36 +0100 Subject: [PATCH 05/48] Clean up and document worldcat fetcher and importer --- .../importer/fetcher/WorldcatFetcher.java | 13 ++++++- .../importer/fileformat/WorldcatImporter.java | 39 ++++++++++++++++--- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java index 83166676e5b..fe7aeeef929 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java @@ -17,7 +17,7 @@ public class WorldcatFetcher implements EntryBasedFetcher { private final static String NAME = "Worldcat Fetcher"; - public final static String API_KEY = "API-KEY"; + public final static String API_KEY = "API-KEY-GOES-HERE"; private final static String WORLDCAT_OPEN_SEARCH_URL = "http://www.worldcat.org/webservices/catalog/search/opensearch?wskey=" + API_KEY; @@ -26,11 +26,21 @@ public String getName() { return NAME; } + /** + * Create a open search query with specified title + * @param title the title to include in the query + * @return the earch query for the api + */ private String getOpenSearchURL(String title){ String query = "&q=srw.ti+all+\"" + title.replaceAll(" ", "%20") + "\""; return WORLDCAT_OPEN_SEARCH_URL + query.replace("\"", "%22"); } + /** + * Make request to open search API of Worldcat, with specified title + * @param title the title of the search + * @return the body of the HTTP response + */ private String makeOpenSearchRequest(String title) throws FetcherException{ try { URLDownload urlDownload = new URLDownload(getOpenSearchURL(title)); @@ -44,7 +54,6 @@ private String makeOpenSearchRequest(String title) throws FetcherException{ } } - //Example http://www.worldcat.org/webservices/catalog/search/worldcat/opensearch?q=srw.ti+all+"The very best of Glenn"&wskey={built-in-api-key} @Override public List performSearch(BibEntry entry) throws FetcherException { Optional entryTitle = entry.getLatexFreeField(StandardField.TITLE); diff --git a/src/main/java/org/jabref/logic/importer/fileformat/WorldcatImporter.java b/src/main/java/org/jabref/logic/importer/fileformat/WorldcatImporter.java index 464fed7d615..6f458ef93da 100644 --- a/src/main/java/org/jabref/logic/importer/fileformat/WorldcatImporter.java +++ b/src/main/java/org/jabref/logic/importer/fileformat/WorldcatImporter.java @@ -30,15 +30,20 @@ public class WorldcatImporter extends Importer { private String WORLDCAT_READ_URL; - private static final Logger LOGGER = LoggerFactory.getLogger(WorldcatImporter.class); - private final static String NAME = "WorldcatImporter"; private final static String DESCRIPTION = "Takes valid XML from Worldcat Open Search and parses them to BibEntry"; public WorldcatImporter(){ + //Used the same key as Worldcat fether this.WORLDCAT_READ_URL = "http://www.worldcat.org/webservices/catalog/content/{OCLC-NUMBER}?recordSchema=info%3Asrw%2Fschema%2F1%2Fdc&wskey=" + WorldcatFetcher.API_KEY; } + /** + * Parse the reader to an xml document + * @param s the reader to be parsed + * @return XML document representing the content of s + * @throws IllegalArgumentException if s is badly formated or other exception occurs during parsing + */ private Document parse(BufferedReader s){ try{ DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); @@ -54,14 +59,16 @@ private Document parse(BufferedReader s){ } } + /** + * Get more information about a article through its OCLC id + * @param id the oclc id + * @return the XML element that contains all tags + */ private Element getSpecificInfoOnOCLC(String id) throws IOException { URLDownload urlDownload = new URLDownload(WORLDCAT_READ_URL.replace("{OCLC-NUMBER}", id)); URLDownload.bypassSSLVerification(); String resp = urlDownload.asString(); - //Used when no API key is present. Comment out lines above as well - // String resp = " Miller, Glenn. 2010 Remastered versions of Glenn Miller's original recordings which bring a freshness to his beloved classics. The album also features a version of 'In the mood' with Jodie Prenger, winner of the BBC's 'I'll do anything'. In the mood -- Moonlight serenade -- Don't sit under the apple tree (with anyone else but me) -- Tuxedo junction -- A string of pearls -- Pennsylvania 6-5000 -- Chattanooga choo-choo -- American patrol -- (I've got a gal in) Kalamazoo -- On a little street in Singapore -- The St. Louis blues march -- A nightingale sang in Berkeley Square -- Star dust -- Little brown jug -- When you wish upon a star -- The woodpecker song -- G.I. jive -- Fools rush in -- Over there -- Blueberry hill -- Over the rainbow -- Serenade in blue -- When Johnny comes marching home -- In the mood (feat. Jodie Prenger). 1 CD (72 min., 18 sec.) ; 4 3/4 in. eng Sony Music Entertainment Dance orchestra music. Big band music. Popular music--1931-1940. Popular music--1941-1950. Compact discs. The very best of Glenn Miller Sound 101104 754508587 "; - Document mainDoc = parse(new BufferedReader(new StringReader(resp))); NodeList parentElemOfTags = mainDoc.getElementsByTagName("oclcdcs"); @@ -79,11 +86,24 @@ public boolean isRecognizedFormat(BufferedReader input) throws IOException { } } - private Element getElementByTag(Element xml, String tag){ + /** + * Get the element of a tag in an XML element + * @param xml the element do search trough + * @param tag the tag to find + * @return the tag element + * @throws NullPointerException if there is no element by this tag + */ + private Element getElementByTag(Element xml, String tag) throws NullPointerException{ NodeList nl = xml.getElementsByTagName(tag); return (Element) nl.item(0); } + /** + * Parse the xml entry to a bib entry + * @param xmlEntry the XML element from open search + * @return the correspoinding bibentry + * @throws IOException if we cannot search Worldcat with the OCLC of the entry + */ private BibEntry xmlEntryToBibEntry(Element xmlEntry) throws IOException{ Element authorsElem = getElementByTag(xmlEntry, "author"); String authors = getElementByTag(authorsElem, "name").getTextContent(); @@ -111,6 +131,13 @@ private BibEntry xmlEntryToBibEntry(Element xmlEntry) throws IOException{ return entry; } + /** + * Parse an XML documents with open search entries to a parserResult of + * the bibentries + * @param doc the main XML document from open search + * @return the ParserResult containing the BibEntries collection + * @throws IOException if {@link xmlEntryToBibEntry} throws + */ private ParserResult docToParserRes(Document doc) throws IOException{ Element feed = (Element) doc.getElementsByTagName("feed").item(0); NodeList entryXMLList = feed.getElementsByTagName("entry"); From caa70765bdb438b01c04e91fe031d36a1924e699 Mon Sep 17 00:00:00 2001 From: Glenn Olsson Date: Thu, 27 Feb 2020 11:08:04 +0100 Subject: [PATCH 06/48] Add worldcat fetcher to webFetchers Add Changelog --- CHANGELOG.md | 2 ++ src/main/java/org/jabref/logic/importer/WebFetchers.java | 4 +++- .../jabref/logic/importer/fetcher/WorldcatFetcher.java | 5 +++++ .../logic/importer/fileformat/WorldcatImporter.java | 9 ++++----- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 149ea8d8bc7..c5bcdb72f62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,8 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve - We made the filters more easily accessible in the integrity check dialog. [#5955](https://github.com/JabRef/jabref/pull/5955) - We reimplemented and improved the dialog "Customize entry types". [#4719](https://github.com/JabRef/jabref/issues/4719) - We added an [American Physical Society](https://journals.aps.org/) fetcher. [#818](https://github.com/JabRef/jabref/issues/818) +- We reimplemented and improved the dialog "Customize entry types" [#4719](https://github.com/JabRef/jabref/issues/4719) +- We added an Worldcat fetcher. [#1065](https://github.com/JabRef/jabref/issues/1065) [#2581](https://github.com/JabRef/jabref/issues/2581). [Worldcat](http://www.worldcat.org/webservices/catalog/) ### Fixed diff --git a/src/main/java/org/jabref/logic/importer/WebFetchers.java b/src/main/java/org/jabref/logic/importer/WebFetchers.java index 72c3ace87ab..c3a1c8371e8 100644 --- a/src/main/java/org/jabref/logic/importer/WebFetchers.java +++ b/src/main/java/org/jabref/logic/importer/WebFetchers.java @@ -34,6 +34,7 @@ import org.jabref.logic.importer.fetcher.SpringerFetcher; import org.jabref.logic.importer.fetcher.SpringerLink; import org.jabref.logic.importer.fetcher.TitleFetcher; +import org.jabref.logic.importer.fetcher.WorldcatFetcher; import org.jabref.logic.importer.fetcher.ZbMATH; import org.jabref.model.entry.field.Field; import org.jabref.model.entry.field.StandardField; @@ -133,7 +134,8 @@ public static SortedSet getEntryBasedFetchers(ImportFormatPre set.add(new DoiFetcher(importFormatPreferences)); set.add(new IsbnFetcher(importFormatPreferences)); set.add(new MathSciNet(importFormatPreferences)); - set.add(new CrossRef()); + set.add(new CrossRef()); + set.add(new WorldcatFetcher()); return set; } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java index fe7aeeef929..65194a3db53 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java @@ -14,6 +14,11 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; +/** + * EntryBasedFetcher that searches the Worldcat database + * + * @see https://www.oclc.org/developer/develop/web-services/worldcat-search-api/bibliographic-resource.en.html + */ public class WorldcatFetcher implements EntryBasedFetcher { private final static String NAME = "Worldcat Fetcher"; diff --git a/src/main/java/org/jabref/logic/importer/fileformat/WorldcatImporter.java b/src/main/java/org/jabref/logic/importer/fileformat/WorldcatImporter.java index 6f458ef93da..e162a8379bc 100644 --- a/src/main/java/org/jabref/logic/importer/fileformat/WorldcatImporter.java +++ b/src/main/java/org/jabref/logic/importer/fileformat/WorldcatImporter.java @@ -18,8 +18,6 @@ import org.jabref.logic.util.StandardFileType; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NodeList; @@ -60,7 +58,8 @@ private Document parse(BufferedReader s){ } /** - * Get more information about a article through its OCLC id + * Get more information about a article through its OCLC id. Picks the first + * element with this tag * @param id the oclc id * @return the XML element that contains all tags */ @@ -87,7 +86,8 @@ public boolean isRecognizedFormat(BufferedReader input) throws IOException { } /** - * Get the element of a tag in an XML element + * Get the element of a tag in an XML element. Picks the first element + * with this tag * @param xml the element do search trough * @param tag the tag to find * @return the tag element @@ -113,7 +113,6 @@ private BibEntry xmlEntryToBibEntry(Element xmlEntry) throws IOException{ String url = getElementByTag(xmlEntry, "link").getAttribute("href"); String oclc = xmlEntry.getElementsByTagName("oclcterms:recordIdentifier").item(0).getTextContent(); - System.out.println(oclc + " , " + title); Element detailedInfo = getSpecificInfoOnOCLC(oclc); String date = detailedInfo.getElementsByTagName("dc:date").item(0).getTextContent(); From d303999f7da86b30d16970ac1cae9f1dacaaebd2 Mon Sep 17 00:00:00 2001 From: Glenn Olsson Date: Thu, 27 Feb 2020 15:10:56 +0100 Subject: [PATCH 07/48] Fix tests --- src/test/java/org/jabref/logic/importer/WebFetchersTest.java | 1 - .../java/org/jabref/logic/importer/WorldcatImporterTest.java | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/jabref/logic/importer/WebFetchersTest.java b/src/test/java/org/jabref/logic/importer/WebFetchersTest.java index 44382d727fb..dcd7fc97775 100644 --- a/src/test/java/org/jabref/logic/importer/WebFetchersTest.java +++ b/src/test/java/org/jabref/logic/importer/WebFetchersTest.java @@ -63,7 +63,6 @@ void getEntryBasedFetchersReturnsAllFetcherDerivingFromEntryBasedFetcher() throw expected.remove(EntryBasedParserFetcher.class); expected.remove(MrDLibFetcher.class); - expected.remove(WorldcatFetcher.class); assertEquals(expected, getClasses(idFetchers)); } diff --git a/src/test/java/org/jabref/logic/importer/WorldcatImporterTest.java b/src/test/java/org/jabref/logic/importer/WorldcatImporterTest.java index 6fc59a4647f..7442318be9a 100644 --- a/src/test/java/org/jabref/logic/importer/WorldcatImporterTest.java +++ b/src/test/java/org/jabref/logic/importer/WorldcatImporterTest.java @@ -8,6 +8,7 @@ import org.jabref.logic.importer.fileformat.WorldcatImporter; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class WorldcatImporterTest { @@ -39,6 +40,7 @@ public void badXMLIsNotRecognizedFormat() throws IOException{ assertFalse(isReq); } + @Disabled("Will not work without API key") @Test public void withResultReturnsNonEmptyResult() throws IOException{ ParserResult res = importer.importDatabase(XML_WITH_RESULT); From e0ab5374cf0b6da7518135dfb91a22b7bb29e627 Mon Sep 17 00:00:00 2001 From: Glenn Olsson Date: Sun, 29 Mar 2020 19:12:16 +0200 Subject: [PATCH 08/48] Change to java.net.URL and fix formatting --- src/main/java/org/jabref/logic/importer/WebFetchers.java | 4 ++-- .../org/jabref/logic/importer/fetcher/WorldcatFetcher.java | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/WebFetchers.java b/src/main/java/org/jabref/logic/importer/WebFetchers.java index c3a1c8371e8..342a9d56770 100644 --- a/src/main/java/org/jabref/logic/importer/WebFetchers.java +++ b/src/main/java/org/jabref/logic/importer/WebFetchers.java @@ -134,8 +134,8 @@ public static SortedSet getEntryBasedFetchers(ImportFormatPre set.add(new DoiFetcher(importFormatPreferences)); set.add(new IsbnFetcher(importFormatPreferences)); set.add(new MathSciNet(importFormatPreferences)); - set.add(new CrossRef()); - set.add(new WorldcatFetcher()); + set.add(new CrossRef()); + set.add(new WorldcatFetcher()); return set; } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java index 65194a3db53..c796b201af6 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java @@ -14,6 +14,8 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; +import java.net.URL; + /** * EntryBasedFetcher that searches the Worldcat database * @@ -37,8 +39,9 @@ public String getName() { * @return the earch query for the api */ private String getOpenSearchURL(String title){ - String query = "&q=srw.ti+all+\"" + title.replaceAll(" ", "%20") + "\""; - return WORLDCAT_OPEN_SEARCH_URL + query.replace("\"", "%22"); + String query = "&q=srw.ti+all+\"" + title + "\""; + URL url = new URL(WORLDCAT_OPEN_SEARCH_URL + query); + return url.toString(); } /** From 8503720ed69fd32605d3f8c9a4049c9a887f792a Mon Sep 17 00:00:00 2001 From: Glenn Olsson Date: Sun, 29 Mar 2020 20:21:05 +0200 Subject: [PATCH 09/48] Move XML tests to own files --- .../importer/fetcher/WorldcatFetcher.java | 12 +- .../logic/importer/WorldcatImporterTest.java | 56 --------- .../fileformat/WorldcatImporterTest.java | 93 ++++++++++++++ .../WorldcatImporterTestWithResult.xml | 119 ++++++++++++++++++ .../WorldcatImporterTestWithoutResult.xml | 2 + 5 files changed, 221 insertions(+), 61 deletions(-) delete mode 100644 src/test/java/org/jabref/logic/importer/WorldcatImporterTest.java create mode 100644 src/test/java/org/jabref/logic/importer/fileformat/WorldcatImporterTest.java create mode 100644 src/test/resources/org/jabref/logic/importer/fileformat/WorldcatImporterTestWithResult.xml create mode 100644 src/test/resources/org/jabref/logic/importer/fileformat/WorldcatImporterTestWithoutResult.xml diff --git a/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java index c796b201af6..a82ff0d1c75 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java @@ -15,6 +15,7 @@ import org.jabref.model.entry.field.StandardField; import java.net.URL; +import java.net.MalformedURLException; /** * EntryBasedFetcher that searches the Worldcat database @@ -38,7 +39,7 @@ public String getName() { * @param title the title to include in the query * @return the earch query for the api */ - private String getOpenSearchURL(String title){ + private String getOpenSearchURL(String title) throws MalformedURLException{ String query = "&q=srw.ti+all+\"" + title + "\""; URL url = new URL(WORLDCAT_OPEN_SEARCH_URL + query); return url.toString(); @@ -56,10 +57,11 @@ private String makeOpenSearchRequest(String title) throws FetcherException{ String resp = urlDownload.asString(); return resp; + } catch (MalformedURLException e) { + throw new FetcherException("Bad url", e); } catch (IOException e) { - e.printStackTrace(); - throw new FetcherException("IO Exception", e); - } + throw new FetcherException("Error with Open Search Request (Worldcat)", e); + } } @Override @@ -80,7 +82,7 @@ public List performSearch(BibEntry entry) throws FetcherException { return parserResult.getDatabase().getEntries(); } catch(IOException e){ - throw new FetcherException("IO Exception " + e.getMessage(), e); + throw new FetcherException("Could not perform search (Worldcat) ", e); } } else { return new ArrayList<>(); diff --git a/src/test/java/org/jabref/logic/importer/WorldcatImporterTest.java b/src/test/java/org/jabref/logic/importer/WorldcatImporterTest.java deleted file mode 100644 index 7442318be9a..00000000000 --- a/src/test/java/org/jabref/logic/importer/WorldcatImporterTest.java +++ /dev/null @@ -1,56 +0,0 @@ -package org.jabref.logic.importer; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.io.IOException; - -import org.jabref.logic.importer.fileformat.WorldcatImporter; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -public class WorldcatImporterTest { - private final String XML_WITH_RESULT = "OCLC Worldcat Search: srw.ti all \"The very best of Glenn\"http://worldcat.org/webservices/catalog/search/worldcat/opensearch?q=srw.ti+all+%22The+very+best+of+Glenn%22&start=1&count=10&format=atom&wskey={built-in-api-key}2020-02-25T06:16:25-05:00Search results for srw.ti all \"The very best of Glenn\" at http://worldcat.org/webservices/catalog89110Miller, Glenn.The very best of Glenn Millerhttp://worldcat.org/oclc/7545085872017-06-29T19:46:28ZRemastered versions of Glenn Miller's original recordings which bring a freshness to his beloved classics. The album also features a version of 'In the mood' with Jodie Prenger, winner of the BBC's 'I'll do anything'.754508587Brennan, Walter, 1894-1974.The very best of Walter Brennan.http://worldcat.org/oclc/170097872018-07-02T10:57:48Zurn:LCCN:9578975117009787Frey, Glenn, composer, performer.Above the clouds : the very best of Glenn Freyhttp://worldcat.org/oclc/10409906042019-05-06T02:09:58Z1040990604Miller, Glenn, 1954- performer.The very best of Glenn Miller.http://worldcat.org/oclc/11404164032020-02-13T17:01:04Z1140416403Miller, Glenn.Very Best of Glenn Miller.http://worldcat.org/oclc/6305478162018-11-28T18:44:17Z630547816Yarbrough, Glenn.Glenn yarbrough - his very besthttp://worldcat.org/oclc/10983940252019-04-25T11:56:52Z1098394025Miller, Glenn.The very best of Glenn Miller : hits & raritieshttp://worldcat.org/oclc/8094628162020-01-23T10:02:43Z809462816Miller, Glenn, 1904-1944, performer.The very best of Glenn Miller & his orchestra.http://worldcat.org/oclc/11232150702019-11-07T01:19:22Z1123215070Miller, Glenn, 1904-1944.The very best of Glenn Miller : in the mood.http://worldcat.org/oclc/556888332016-11-17T00:02:35Z55688833Miller, Glenn.Glenn Miller story : the very best ofhttp://worldcat.org/oclc/4217187472018-10-08T09:50:51Z421718747"; - private final String XML_WITHOUT_RESULT = ""; - - WorldcatImporter importer; - - @BeforeEach - public void setUp(){ - importer = new WorldcatImporter(); - } - - @Test - public void withResultIsRecognizedFormat() throws IOException{ - boolean isReq = importer.isRecognizedFormat(XML_WITH_RESULT); - assertTrue(isReq); - } - - @Test - public void withoutResultIsRecognizedFormat() throws IOException{ - boolean isReq = importer.isRecognizedFormat(XML_WITHOUT_RESULT); - assertTrue(isReq); - } - - @Test - public void badXMLIsNotRecognizedFormat() throws IOException{ - boolean isReq = importer.isRecognizedFormat("Nah bruh"); - assertFalse(isReq); - } - - @Disabled("Will not work without API key") - @Test - public void withResultReturnsNonEmptyResult() throws IOException{ - ParserResult res = importer.importDatabase(XML_WITH_RESULT); - assertTrue(res.getDatabase().getEntries().size() > 0); - } - - @Test - public void withoutResultReturnsEmptyResult() throws IOException{ - ParserResult res = importer.importDatabase(XML_WITHOUT_RESULT); - assertEquals(0, res.getDatabase().getEntries().size()); - } - -} \ No newline at end of file diff --git a/src/test/java/org/jabref/logic/importer/fileformat/WorldcatImporterTest.java b/src/test/java/org/jabref/logic/importer/fileformat/WorldcatImporterTest.java new file mode 100644 index 00000000000..1049e7f666f --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fileformat/WorldcatImporterTest.java @@ -0,0 +1,93 @@ +package org.jabref.logic.importer.fileformat; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.net.URISyntaxException; + +import java.util.function.Predicate; +import java.util.Collection; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.jabref.logic.importer.fileformat.WorldcatImporter; +import org.jabref.logic.importer.ParserResult; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +public class WorldcatImporterTest { + private final String XML_WITH_RESULT_PATH = ""; + private final String XML_WITHOUT_RESULT_PATH = ""; + + WorldcatImporter importer; + + private String getFilePath(String filename) throws IOException{ + Predicate filePredicate = name -> { + return name.startsWith(filename) && name.endsWith(".xml"); + }; + Collection paths = ImporterTestEngine.getTestFiles(filePredicate); + if(paths.size() > 1 || paths.size() == 0){ + throw new IllegalArgumentException("Filename returned 0 or more than 1 result: " + filename); + } + return paths.iterator().next(); + } + + private String getFileContent(String filename) throws IOException{ + String path = getFilePath(filename); + String content = Files.readString(getPath(path)); + return content; + } + + private static Path getPath(String fileName) throws IOException { + try { + return Paths.get(ImporterTestEngine.class.getResource(fileName).toURI()); + } catch (URISyntaxException e) { + throw new IOException(e); + } + } + + @BeforeEach + public void setUp(){ + importer = new WorldcatImporter(); + + } + + @Test + public void withResultIsRecognizedFormat() throws IOException{ + ImporterTestEngine.testIsRecognizedFormat(new WorldcatImporter(), getFilePath("WorldcatImporterTestWithResult")); + } + + @Test + public void withoutResultIsRecognizedFormat() throws IOException{ + ImporterTestEngine.testIsRecognizedFormat(new WorldcatImporter(), getFilePath("WorldcatImporterTestWithoutResult")); + } + + @Test + public void badXMLIsNotRecognizedFormat() throws IOException{ + boolean isReq = importer.isRecognizedFormat("Nah bruh"); + assertFalse(isReq); + } + + @Disabled("Will not work without API key") + @Test + public void withResultReturnsNonEmptyResult() throws IOException{ + String withResultXML = getFileContent("WorldcatImporterTestWithResult"); + ParserResult res = importer.importDatabase(withResultXML); + assertTrue(res.getDatabase().getEntries().size() > 0); + } + + @Disabled("Will not work without API key") + @Test + public void withoutResultReturnsEmptyResult() throws IOException{ + String withoutResultXML = getFileContent("WorldcatImporterTestWithResult"); + ParserResult res = importer.importDatabase(withoutResultXML); + assertEquals(0, res.getDatabase().getEntries().size()); + } + +} \ No newline at end of file diff --git a/src/test/resources/org/jabref/logic/importer/fileformat/WorldcatImporterTestWithResult.xml b/src/test/resources/org/jabref/logic/importer/fileformat/WorldcatImporterTestWithResult.xml new file mode 100644 index 00000000000..d9a3f926db5 --- /dev/null +++ b/src/test/resources/org/jabref/logic/importer/fileformat/WorldcatImporterTestWithResult.xml @@ -0,0 +1,119 @@ + + + OCLC Worldcat Search: srw.ti all "The very best of Glenn" + http://worldcat.org/webservices/catalog/search/worldcat/opensearch?q=srw.ti+all+%22The+very+best+of+Glenn%22&start=1&count=10&format=atom&wskey={built-in-api-key} + 2020-02-25T06:16:25-05:00 + Search results for srw.ti all "The very best of Glenn" at http://worldcat.org/webservices/catalog + 89 + 1 + 10 + + + + + + + + + + Miller, Glenn. + + The very best of Glenn Miller + + http://worldcat.org/oclc/754508587 + 2017-06-29T19:46:28Z + Remastered versions of Glenn Miller's original recordings which bring a freshness to his beloved classics. The album also features a version of 'In the mood' with Jodie Prenger, winner of the BBC's 'I'll do anything'. + 754508587 + + + + Brennan, Walter, 1894-1974. + + The very best of Walter Brennan. + + http://worldcat.org/oclc/17009787 + 2018-07-02T10:57:48Z + urn:LCCN:95789751 + 17009787 + + + + Frey, Glenn, composer, performer. + + Above the clouds : the very best of Glenn Frey + + http://worldcat.org/oclc/1040990604 + 2019-05-06T02:09:58Z + 1040990604 + + + + Miller, Glenn, 1954- performer. + + The very best of Glenn Miller. + + http://worldcat.org/oclc/1140416403 + 2020-02-13T17:01:04Z + 1140416403 + + + + Miller, Glenn. + + Very Best of Glenn Miller. + + http://worldcat.org/oclc/630547816 + 2018-11-28T18:44:17Z + 630547816 + + + + Yarbrough, Glenn. + + Glenn yarbrough - his very best + + http://worldcat.org/oclc/1098394025 + 2019-04-25T11:56:52Z + 1098394025 + + + + Miller, Glenn. + + The very best of Glenn Miller : hits & rarities + + http://worldcat.org/oclc/809462816 + 2020-01-23T10:02:43Z + 809462816 + + + + Miller, Glenn, 1904-1944, performer. + + The very best of Glenn Miller & his orchestra. + + http://worldcat.org/oclc/1123215070 + 2019-11-07T01:19:22Z + 1123215070 + + + + Miller, Glenn, 1904-1944. + + The very best of Glenn Miller : in the mood. + + http://worldcat.org/oclc/55688833 + 2016-11-17T00:02:35Z + 55688833 + + + + Miller, Glenn. + + Glenn Miller story : the very best of + + http://worldcat.org/oclc/421718747 + 2018-10-08T09:50:51Z + 421718747 + + \ No newline at end of file diff --git a/src/test/resources/org/jabref/logic/importer/fileformat/WorldcatImporterTestWithoutResult.xml b/src/test/resources/org/jabref/logic/importer/fileformat/WorldcatImporterTestWithoutResult.xml new file mode 100644 index 00000000000..56106ec2148 --- /dev/null +++ b/src/test/resources/org/jabref/logic/importer/fileformat/WorldcatImporterTestWithoutResult.xml @@ -0,0 +1,2 @@ + + \ No newline at end of file From c6b3e262d5691085f99605503365673cc3eceaab Mon Sep 17 00:00:00 2001 From: Glenn Olsson Date: Sun, 29 Mar 2020 21:39:54 +0200 Subject: [PATCH 10/48] Add preferences field --- .../jabref/gui/entryeditor/EntryEditor.java | 4 ++-- .../entryeditor/EntryEditorPreferences.java | 15 +++++++++---- .../org/jabref/gui/preferences/ImportTab.fxml | 16 ++++++++++++++ .../jabref/gui/preferences/ImportTabView.java | 8 +++++-- .../gui/preferences/ImportTabViewModel.java | 10 +++++++-- .../logic/importer/APIKeyPreferences.java | 11 ++++++++++ .../jabref/logic/importer/WebFetchers.java | 4 ++-- .../importer/fetcher/WorldcatFetcher.java | 8 ++++++- .../jabref/preferences/JabRefPreferences.java | 21 +++++++++++++------ .../fileformat/WorldcatImporterTest.java | 2 -- 10 files changed, 78 insertions(+), 21 deletions(-) create mode 100644 src/main/java/org/jabref/logic/importer/APIKeyPreferences.java diff --git a/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java b/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java index f6cbfb7b996..e98943a7f6a 100644 --- a/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java +++ b/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java @@ -97,7 +97,7 @@ public EntryEditor(BasePanel panel, ExternalFileTypes externalFileTypes) { .root(this) .load(); - this.entryEditorPreferences = preferencesService.getEntryEditorPreferences(); + this.entryEditorPreferences = preferencesService.getEntryEditorPreferences(); this.fileLinker = new ExternalFilesEntryLinker(externalFileTypes, preferencesService.getFilePreferences(), databaseContext); @@ -335,7 +335,7 @@ private void setupToolBar() { // Add menu for fetching bibliographic information ContextMenu fetcherMenu = new ContextMenu(); - for (EntryBasedFetcher fetcher : WebFetchers.getEntryBasedFetchers(entryEditorPreferences.getImportFormatPreferences())) { + for (EntryBasedFetcher fetcher : WebFetchers.getEntryBasedFetchers(entryEditorPreferences.getImportFormatPreferences(), entryEditorPreferences.getAPIApiKeyPreferences())) { MenuItem fetcherMenuItem = new MenuItem(fetcher.getName()); fetcherMenuItem.setOnAction(event -> fetchAndMerge(fetcher)); fetcherMenu.getItems().add(fetcherMenuItem); diff --git a/src/main/java/org/jabref/gui/entryeditor/EntryEditorPreferences.java b/src/main/java/org/jabref/gui/entryeditor/EntryEditorPreferences.java index 26c4b48ef2a..fa040184937 100644 --- a/src/main/java/org/jabref/gui/entryeditor/EntryEditorPreferences.java +++ b/src/main/java/org/jabref/gui/entryeditor/EntryEditorPreferences.java @@ -7,6 +7,7 @@ import org.jabref.gui.keyboard.KeyBindingRepository; import org.jabref.logic.bibtex.FieldWriterPreferences; import org.jabref.logic.bibtexkeypattern.BibtexKeyPatternPreferences; +import org.jabref.logic.importer.APIKeyPreferences; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.model.entry.field.Field; @@ -15,7 +16,8 @@ public class EntryEditorPreferences { private final Map> entryEditorTabList; private final FieldWriterPreferences fieldWriterPreferences; private final ImportFormatPreferences importFormatPreferences; - private final BibtexKeyPatternPreferences bibtexKeyPatternPreferences; + private final BibtexKeyPatternPreferences bibtexKeyPatternPreferences; + private final APIKeyPreferences apiKeyPreferences; private final List customTabFieldNames; private final boolean shouldShowRecommendationsTab; private final boolean isMrdlibAccepted; @@ -24,7 +26,7 @@ public class EntryEditorPreferences { private boolean avoidOverwritingCiteKey; private final boolean shouldShowLatexCitationsTab; - public EntryEditorPreferences(Map> entryEditorTabList, FieldWriterPreferences fieldWriterPreferences, ImportFormatPreferences importFormatPreferences, List customTabFieldNames, boolean shouldShowRecommendationsTab, boolean isMrdlibAccepted, boolean shouldShowLatexCitationsTab, boolean showSourceTabByDefault, BibtexKeyPatternPreferences bibtexKeyPatternPreferences, KeyBindingRepository keyBindings, boolean avoidOverwritingCiteKey) { + public EntryEditorPreferences(Map> entryEditorTabList, FieldWriterPreferences fieldWriterPreferences, ImportFormatPreferences importFormatPreferences, List customTabFieldNames, boolean shouldShowRecommendationsTab, boolean isMrdlibAccepted, boolean shouldShowLatexCitationsTab, boolean showSourceTabByDefault, BibtexKeyPatternPreferences bibtexKeyPatternPreferences, KeyBindingRepository keyBindings, boolean avoidOverwritingCiteKey, APIKeyPreferences apiKeyPreferences) { this.entryEditorTabList = entryEditorTabList; this.fieldWriterPreferences = fieldWriterPreferences; this.importFormatPreferences = importFormatPreferences; @@ -35,7 +37,8 @@ public EntryEditorPreferences(Map> entryEditorTabList, FieldW this.bibtexKeyPatternPreferences = bibtexKeyPatternPreferences; this.keyBindings = keyBindings; this.avoidOverwritingCiteKey = avoidOverwritingCiteKey; - this.shouldShowLatexCitationsTab = shouldShowLatexCitationsTab; + this.shouldShowLatexCitationsTab = shouldShowLatexCitationsTab; + this.apiKeyPreferences = apiKeyPreferences; } public Map> getEntryEditorTabList() { @@ -48,7 +51,11 @@ public FieldWriterPreferences getFieldWriterPreferences() { public ImportFormatPreferences getImportFormatPreferences() { return importFormatPreferences; - } + } + + public APIKeyPreferences getAPIApiKeyPreferences() { + return apiKeyPreferences; + } public List getCustomTabFieldNames() { return customTabFieldNames; diff --git a/src/main/java/org/jabref/gui/preferences/ImportTab.fxml b/src/main/java/org/jabref/gui/preferences/ImportTab.fxml index 8cb14943e5d..ee7523edf09 100644 --- a/src/main/java/org/jabref/gui/preferences/ImportTab.fxml +++ b/src/main/java/org/jabref/gui/preferences/ImportTab.fxml @@ -31,4 +31,20 @@ + +