diff --git a/src/main/java/org/jabref/gui/slr/ManageStudyDefinitionViewModel.java b/src/main/java/org/jabref/gui/slr/ManageStudyDefinitionViewModel.java index c7f47a0ca56..2be80d56ef6 100644 --- a/src/main/java/org/jabref/gui/slr/ManageStudyDefinitionViewModel.java +++ b/src/main/java/org/jabref/gui/slr/ManageStudyDefinitionViewModel.java @@ -49,7 +49,7 @@ public ManageStudyDefinitionViewModel(Study study, Path studyDirectory, ImportFo title.setValue(study.getTitle()); authors.addAll(study.getAuthors()); researchQuestions.addAll(study.getResearchQuestions()); - queries.addAll(study.getQueries().stream().map(StudyQuery::getQuery).collect(Collectors.toList())); + queries.addAll(study.getQueries().stream().map(StudyQuery::getBaseQuery).collect(Collectors.toList())); databases.addAll(study.getDatabases() .stream() .map(studyDatabase -> new StudyDatabaseItem(studyDatabase.getName(), studyDatabase.isEnabled())) diff --git a/src/main/java/org/jabref/logic/crawler/Crawler.java b/src/main/java/org/jabref/logic/crawler/Crawler.java index b54f371f4f2..894ac494be0 100644 --- a/src/main/java/org/jabref/logic/crawler/Crawler.java +++ b/src/main/java/org/jabref/logic/crawler/Crawler.java @@ -35,7 +35,7 @@ public class Crawler { public Crawler(Path studyRepositoryRoot, SlrGitHandler gitHandler, ImportFormatPreferences importFormatPreferences, SavePreferences savePreferences, TimestampPreferences timestampPreferences, BibEntryTypesManager bibEntryTypesManager, FileUpdateMonitor fileUpdateMonitor) throws IllegalArgumentException, IOException, ParseException { studyRepository = new StudyRepository(studyRepositoryRoot, gitHandler, importFormatPreferences, fileUpdateMonitor, savePreferences, bibEntryTypesManager); StudyDatabaseToFetcherConverter studyDatabaseToFetcherConverter = new StudyDatabaseToFetcherConverter(studyRepository.getActiveLibraryEntries(), importFormatPreferences); - this.studyFetcher = new StudyFetcher(studyDatabaseToFetcherConverter.getActiveFetchers(), studyRepository.getSearchQueryStrings()); + this.studyFetcher = new StudyFetcher(studyDatabaseToFetcherConverter.getActiveFetchers(), studyRepository.getSearchQueries()); } /** diff --git a/src/main/java/org/jabref/logic/crawler/StudyDatabaseToFetcherConverter.java b/src/main/java/org/jabref/logic/crawler/StudyDatabaseToFetcherConverter.java index a5b51854cd8..b38a21f836c 100644 --- a/src/main/java/org/jabref/logic/crawler/StudyDatabaseToFetcherConverter.java +++ b/src/main/java/org/jabref/logic/crawler/StudyDatabaseToFetcherConverter.java @@ -56,7 +56,7 @@ private SearchBasedFetcher createFetcherFromLibraryEntry(StudyDatabase studyData Set searchBasedFetchers = WebFetchers.getSearchBasedFetchers(importFormatPreferences); String libraryNameFromFetcher = studyDatabase.getName(); return searchBasedFetchers.stream() - .filter(searchBasedFetcher -> searchBasedFetcher.getName().toLowerCase().equals(libraryNameFromFetcher.toLowerCase())) + .filter(searchBasedFetcher -> searchBasedFetcher.getName().equalsIgnoreCase(libraryNameFromFetcher)) .findAny() .orElse(null); } diff --git a/src/main/java/org/jabref/logic/crawler/StudyFetcher.java b/src/main/java/org/jabref/logic/crawler/StudyFetcher.java index c39ba7efe52..03a47bd1912 100644 --- a/src/main/java/org/jabref/logic/crawler/StudyFetcher.java +++ b/src/main/java/org/jabref/logic/crawler/StudyFetcher.java @@ -12,6 +12,7 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.study.FetchResult; import org.jabref.model.study.QueryResult; +import org.jabref.model.study.StudyQuery; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,9 +26,9 @@ class StudyFetcher { private static final int MAX_AMOUNT_OF_RESULTS_PER_FETCHER = 100; private final List activeFetchers; - private final List searchQueries; + private final List searchQueries; - StudyFetcher(List activeFetchers, List searchQueries) throws IllegalArgumentException { + StudyFetcher(List activeFetchers, List searchQueries) throws IllegalArgumentException { this.searchQueries = searchQueries; this.activeFetchers = activeFetchers; } @@ -43,8 +44,9 @@ public List crawl() { .collect(Collectors.toList()); } - private QueryResult getQueryResult(String searchQuery) { - return new QueryResult(searchQuery, performSearchOnQuery(searchQuery)); + private QueryResult getQueryResult(StudyQuery searchQuery) { + // Results for all query refinements are stored within the directory of the base query + return new QueryResult(searchQuery.getBaseQuery(), performSearchOnQuery(searchQuery)); } /** @@ -53,9 +55,9 @@ private QueryResult getQueryResult(String searchQuery) { * @param searchQuery The query the search is performed for. * @return Mapping of each fetcher by name and all their retrieved publications as a BibDatabase */ - private List performSearchOnQuery(String searchQuery) { + private List performSearchOnQuery(StudyQuery searchQuery) { return activeFetchers.parallelStream() - .map(fetcher -> performSearchOnQueryForFetcher(searchQuery, fetcher)) + .map(fetcher -> performSearchOnQueryForFetcher(searchQuery.getLibrarySpecificQueryOrDefault(fetcher.getName()), fetcher)) .filter(Objects::nonNull) .collect(Collectors.toList()); } diff --git a/src/main/java/org/jabref/logic/crawler/StudyRepository.java b/src/main/java/org/jabref/logic/crawler/StudyRepository.java index cab44b804da..509f2239560 100644 --- a/src/main/java/org/jabref/logic/crawler/StudyRepository.java +++ b/src/main/java/org/jabref/logic/crawler/StudyRepository.java @@ -6,7 +6,6 @@ import java.nio.charset.UnsupportedCharsetException; import java.nio.file.Files; import java.nio.file.Path; -import java.time.LocalDate; import java.time.LocalDateTime; import java.time.temporal.ChronoUnit; import java.util.List; @@ -22,8 +21,6 @@ import org.jabref.logic.git.SlrGitHandler; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.OpenDatabase; -import org.jabref.logic.importer.ParseException; -import org.jabref.logic.importer.SearchBasedFetcher; import org.jabref.logic.l10n.Localization; import org.jabref.model.database.BibDatabase; import org.jabref.model.database.BibDatabaseContext; @@ -53,7 +50,6 @@ class StudyRepository { private static final Pattern MATCHCOLON = Pattern.compile(":"); private static final Pattern MATCHILLEGALCHARACTERS = Pattern.compile("[^A-Za-z0-9_.\\s=-]"); // Currently we make assumptions about the configuration: the remotes, work and search branch names - private static final String REMOTE = "origin"; private static final String WORK_BRANCH = "work"; private static final String SEARCH_BRANCH = "search"; @@ -75,14 +71,13 @@ class StudyRepository { * contain the study definition file. * @throws IOException Thrown if the given repository does not exists, or the study definition file * does not exist - * @throws ParseException Problem parsing the study definition file. */ public StudyRepository(Path pathToRepository, SlrGitHandler gitHandler, ImportFormatPreferences importFormatPreferences, FileUpdateMonitor fileUpdateMonitor, SavePreferences savePreferences, - BibEntryTypesManager bibEntryTypesManager) throws IOException, ParseException { + BibEntryTypesManager bibEntryTypesManager) throws IOException { this.repositoryPath = pathToRepository; this.gitHandler = gitHandler; this.importFormatPreferences = importFormatPreferences; @@ -170,12 +165,11 @@ private Study parseStudyFile() throws IOException { /** * Returns all query strings of the study definition * - * @return List of all queries as Strings. + * @return List of all queries. */ - public List getSearchQueryStrings() { + public List getSearchQueries() { return study.getQueries() .parallelStream() - .map(StudyQuery::getQuery) .collect(Collectors.toList()); } @@ -210,12 +204,10 @@ public Study getStudy() { */ public void persist(List crawlResults) throws IOException, GitAPIException, SaveException { updateWorkAndSearchBranch(); - study.setLastSearchDate(LocalDate.now()); persistStudy(); - gitHandler.createCommitOnCurrentBranch("Update search date", true); + gitHandler.createCommitOnCurrentBranch("Write config file", true); gitHandler.checkoutBranch(SEARCH_BRANCH); persistResults(crawlResults); - study.setLastSearchDate(LocalDate.now()); persistStudy(); try { // First commit changes to search branch branch and update remote @@ -269,12 +261,10 @@ private void persistStudy() throws IOException { */ private void setUpRepositoryStructure() throws IOException { // Cannot use stream here since IOException has to be thrown - StudyDatabaseToFetcherConverter converter = new StudyDatabaseToFetcherConverter(this.getActiveLibraryEntries(), importFormatPreferences); - for (String query : this.getSearchQueryStrings()) { - createQueryResultFolder(query); - converter.getActiveFetchers() - .forEach(searchBasedFetcher -> createFetcherResultFile(query, searchBasedFetcher)); - createQueryResultFile(query); + for (StudyQuery query : this.getSearchQueries()) { + createQueryResultFolder(query.getBaseQuery()); + getActiveLibraryEntries().forEach(library -> createFetcherResultFile(query.getBaseQuery(), library.getName())); + createQueryResultFile(query.getBaseQuery()); } createStudyResultFile(); } @@ -296,8 +286,7 @@ private void createFolder(Path folder) throws IOException { } } - private void createFetcherResultFile(String query, SearchBasedFetcher searchBasedFetcher) { - String fetcherName = searchBasedFetcher.getName(); + private void createFetcherResultFile(String query, String fetcherName) { Path fetcherResultFile = getPathToFetcherResultFile(query, fetcherName); createBibFile(fetcherResultFile); } diff --git a/src/main/java/org/jabref/logic/crawler/StudyYamlParser.java b/src/main/java/org/jabref/logic/crawler/StudyYamlParser.java index df2138434d2..db1bb1b9a9f 100644 --- a/src/main/java/org/jabref/logic/crawler/StudyYamlParser.java +++ b/src/main/java/org/jabref/logic/crawler/StudyYamlParser.java @@ -8,10 +8,8 @@ import org.jabref.model.study.Study; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; public class StudyYamlParser { @@ -20,7 +18,6 @@ public class StudyYamlParser { */ public Study parseStudyYamlFile(Path studyYamlFile) throws IOException { ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory()); - yamlMapper.registerModule(new JavaTimeModule()); try (InputStream fileInputStream = new FileInputStream(studyYamlFile.toFile())) { return yamlMapper.readValue(fileInputStream, Study.class); } @@ -32,8 +29,6 @@ public Study parseStudyYamlFile(Path studyYamlFile) throws IOException { public void writeStudyYamlFile(Study study, Path studyYamlFile) throws IOException { ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory().disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER) .enable(YAMLGenerator.Feature.MINIMIZE_QUOTES)); - yamlMapper.registerModule(new JavaTimeModule()); - yamlMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); yamlMapper.writeValue(studyYamlFile.toFile(), study); } } diff --git a/src/main/java/org/jabref/logic/git/GitHandler.java b/src/main/java/org/jabref/logic/git/GitHandler.java index f486ae52878..45bca133568 100644 --- a/src/main/java/org/jabref/logic/git/GitHandler.java +++ b/src/main/java/org/jabref/logic/git/GitHandler.java @@ -136,6 +136,8 @@ public boolean createCommitOnCurrentBranch(String commitMessage, boolean amend) .setMessage(commitMessage) .call(); } + } catch (GitAPIException e) { + LOGGER.error("Could not create commit on branch", e); } return commitCreated; } diff --git a/src/main/java/org/jabref/logic/importer/FetcherDelegator.java b/src/main/java/org/jabref/logic/importer/FetcherDelegator.java new file mode 100644 index 00000000000..dda9dcca661 --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/FetcherDelegator.java @@ -0,0 +1,97 @@ +package org.jabref.logic.importer; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import org.jabref.logic.importer.fetcher.transformers.AbstractQueryTransformer; +import org.jabref.logic.importer.fetcher.transformers.ArXivQueryTransformer; +import org.jabref.logic.importer.fetcher.transformers.CollectionOfComputerScienceBibliographiesQueryTransformer; +import org.jabref.logic.importer.fetcher.transformers.DBLPQueryTransformer; +import org.jabref.logic.importer.fetcher.transformers.DefaultQueryTransformer; +import org.jabref.logic.importer.fetcher.transformers.GVKQueryTransformer; +import org.jabref.logic.importer.fetcher.transformers.IEEEQueryTransformer; +import org.jabref.logic.importer.fetcher.transformers.JstorQueryTransformer; +import org.jabref.logic.importer.fetcher.transformers.ScholarQueryTransformer; +import org.jabref.logic.importer.fetcher.transformers.SpringerQueryTransformer; +import org.jabref.logic.importer.fetcher.transformers.ZbMathQueryTransformer; +import org.jabref.model.entry.BibEntry; + +import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.core.parser.SyntaxParser; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; + +import static org.jabref.logic.importer.fetcher.transformers.AbstractQueryTransformer.NO_EXPLICIT_FIELD; + +/** + * Manages how a query is executed (either raw or transformed) + */ +public class FetcherDelegator { + + public FetcherDelegator() { + } + + /** + * Looks for hits which are matched by the given free-text query. + * + * @param searchQuery query string that can be parsed into a lucene query + * @return a list of {@link BibEntry}, which are matched by the query (may be empty) + */ + List performSearch(String searchQuery, SearchBasedFetcher fetcher) throws FetcherException { + if (searchQuery.isBlank()) { + return Collections.emptyList(); + } + // TODO: The delegator furthermore should detect whether the proposed query is a specialized query (e.g. doi and use the appropriate fetcher, instead of the provided one + + SyntaxParser parser = new StandardSyntaxParser(); + QueryNode queryNode; + try { + queryNode = parser.parse(searchQuery, NO_EXPLICIT_FIELD); + } catch (QueryNodeParseException e) { + throw new FetcherException("An error occurred when parsing the query"); + } + + AbstractQueryTransformer transformer = getTransformer(fetcher); + Optional transformedQuery = transformer.transformLuceneQuery(queryNode); + return postFilterResult(fetcher.performSearch(transformedQuery.orElse(searchQuery)), transformer); + } + + private AbstractQueryTransformer getTransformer(SearchBasedFetcher fetcher) { + String lowerCaseFetcherName = fetcher.getName().toLowerCase(); + return switch (lowerCaseFetcherName) { + case "arxiv" -> new ArXivQueryTransformer(); + case "ieeexplore" -> new IEEEQueryTransformer(); + case "gvk" -> new GVKQueryTransformer(); + case "springer" -> new SpringerQueryTransformer(); + case "google scholar" -> new ScholarQueryTransformer(); + case "dblp" -> new DBLPQueryTransformer(); + case "collection of computer science bibliographies" -> new CollectionOfComputerScienceBibliographiesQueryTransformer(); + case "zbmath" -> new ZbMathQueryTransformer(); + case "jstor" -> new JstorQueryTransformer(); + default -> new DefaultQueryTransformer(); + }; + } + + /** + * Depending on the type of query transformer the result has to be filtered + * + * @param result the result + * @param transformer transformer used for the query, depending on the type, post filtering is required + * @return a filtered result list + */ + private List postFilterResult(List result, AbstractQueryTransformer transformer) { + // TODO + return result; + } + + /** + * Looks for hits which are matched by the given free-text query. + * + * @param searchQuery query string + * @return a list of {@link BibEntry}, which are matched by the query (may be empty) + */ + List performSearchRaw(String searchQuery, SearchBasedFetcher fetcher) throws FetcherException { + return fetcher.performSearch(searchQuery); + } +} diff --git a/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java index a6b1cd4aea1..a6df69f202b 100644 --- a/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java +++ b/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java @@ -7,37 +7,14 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.paging.Page; -import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.core.parser.SyntaxParser; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; - public interface PagedSearchBasedFetcher extends SearchBasedFetcher { - /** - * @param luceneQuery the root node of the lucene query - * @param pageNumber requested site number indexed from 0 - * @return Page with search results - */ - Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException; - /** * @param searchQuery query string that can be parsed into a lucene query * @param pageNumber requested site number indexed from 0 * @return Page with search results */ - default Page performSearchPaged(String searchQuery, int pageNumber) throws FetcherException { - if (searchQuery.isBlank()) { - return new Page<>(searchQuery, pageNumber, Collections.emptyList()); - } - SyntaxParser parser = new StandardSyntaxParser(); - final String NO_EXPLICIT_FIELD = "default"; - try { - return this.performSearchPaged(parser.parse(searchQuery, NO_EXPLICIT_FIELD), pageNumber); - } catch (QueryNodeParseException e) { - throw new FetcherException("An error occurred during parsing of the query."); - } - } + Page performSearchPaged(String searchQuery, int pageNumber) throws FetcherException; /** * @return default pageSize @@ -49,11 +26,11 @@ default int getPageSize() { /** * This method is used to send complex queries using fielded search. * - * @param luceneQuery the root node of the lucene query + * @param query the query string used to identify relevant documents * @return a list of {@link BibEntry}, which are matched by the query (may be empty) */ - default List performSearch(QueryNode luceneQuery) throws FetcherException { - return new ArrayList<>(performSearchPaged(luceneQuery, 0).getContent()); + default List performSearch(String query) throws FetcherException { + return new ArrayList<>(performSearchPaged(query, 0).getContent()); } } diff --git a/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java index bbbc848cc99..d837e9c56e8 100644 --- a/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java @@ -15,15 +15,15 @@ public interface PagedSearchBasedParserFetcher extends SearchBasedParserFetcher, PagedSearchBasedFetcher { @Override - default Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException { + default Page performSearchPaged(String query, int pageNumber) throws FetcherException { // ADR-0014 URL urlForQuery; try { - urlForQuery = getURLForQuery(luceneQuery, pageNumber); + urlForQuery = getURLForQuery(query, pageNumber); } catch (URISyntaxException | MalformedURLException e) { throw new FetcherException("Search URI crafted from complex search query is malformed", e); } - return new Page<>(luceneQuery.toString(), pageNumber, getBibEntries(urlForQuery)); + return new Page<>(query, pageNumber, getBibEntries(urlForQuery)); } private List getBibEntries(URL urlForQuery) throws FetcherException { @@ -40,18 +40,18 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { /** * Constructs a URL based on the query, size and page number. - * @param luceneQuery the search query + * @param query the query string used to identify relevant documents * @param pageNumber the number of the page indexed from 0 */ - URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException; + URL getURLForQuery(String query, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException; @Override - default URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { - return getURLForQuery(luceneQuery, 0); + default URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + getURLForQuery(query, 0); } @Override - default List performSearch(QueryNode luceneQuery) throws FetcherException { - return SearchBasedParserFetcher.super.performSearch(luceneQuery); + default List performSearch(String query) throws FetcherException { + return SearchBasedParserFetcher.super.performSearch(query); } } diff --git a/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java index 8e3e3d75533..844cf435e5a 100644 --- a/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java +++ b/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java @@ -15,36 +15,18 @@ /** * Searches web resources for bibliographic information based on a free-text query. * May return multiple search hits. + * + * Note: All queries should go through the Search Delegator, as this allows the use of the query transformer/or explicit use of the raw query + * The delegator additionally performs post filtering of results where possible (for transformed queries) */ public interface SearchBasedFetcher extends WebFetcher { - /** - * This method is used to send complex queries using fielded search. - * - * @param luceneQuery the root node of the lucene query - * @return a list of {@link BibEntry}, which are matched by the query (may be empty) - */ - List performSearch(QueryNode luceneQuery) throws FetcherException; - /** * Looks for hits which are matched by the given free-text query. * - * @param searchQuery query string that can be parsed into a lucene query + * @param searchQuery query the query string used to identify relevant documents * @return a list of {@link BibEntry}, which are matched by the query (may be empty) */ - default List performSearch(String searchQuery) throws FetcherException { - if (searchQuery.isBlank()) { - return Collections.emptyList(); - } - - SyntaxParser parser = new StandardSyntaxParser(); - QueryNode queryNode; - try { - queryNode = parser.parse(searchQuery, NO_EXPLICIT_FIELD); - } catch (QueryNodeParseException e) { - throw new FetcherException("An error occurred when parsing the query"); - } + List performSearch(String searchQuery) throws FetcherException; - return this.performSearch(queryNode); - } } diff --git a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java index 9aadba697b9..ec7d63d2c74 100644 --- a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java @@ -27,14 +27,14 @@ public interface SearchBasedParserFetcher extends SearchBasedFetcher { * This method is necessary as the performSearch method does not support certain URL parameters that are used for * fielded search, such as a title, author, or year parameter. * - * @param luceneQuery the root node of the lucene query + * @param query the query string used to identify relevant documents */ @Override - default List performSearch(QueryNode luceneQuery) throws FetcherException { + default List performSearch(String query) throws FetcherException { // ADR-0014 URL urlForQuery; try { - urlForQuery = getURLForQuery(luceneQuery); + urlForQuery = getURLForQuery(query); } catch (URISyntaxException | MalformedURLException | FetcherException e) { throw new FetcherException("Search URI crafted from complex search query is malformed", e); } @@ -61,9 +61,9 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { /** * Constructs a URL based on the lucene query. * - * @param luceneQuery the root node of the lucene query + * @param query the query string used to identify relevant documents */ - URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException; + URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException; /** * Performs a cleanup of the fetched entry. diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java index a981dd835bc..54fb7990a80 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java @@ -36,10 +36,6 @@ public Optional getHelpPage() { return Optional.of(HelpFile.FETCHER_ACM); } - private static String createQueryString(QueryNode query) { - return new DefaultQueryTransformer().transformLuceneQuery(query).orElse(""); - } - /** * Constructing the url for the searchpage. * @@ -47,9 +43,9 @@ private static String createQueryString(QueryNode query) { * @return query URL */ @Override - public URL getURLForQuery(QueryNode query) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); - uriBuilder.addParameter("AllField", createQueryString(query)); + uriBuilder.addParameter("AllField", query); return uriBuilder.build().toURL(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java b/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java index cf8c8aec2ec..721636f3698 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java @@ -253,17 +253,16 @@ public Optional getHelpPage() { /** * Constructs a complex query string using the field prefixes specified at https://arxiv.org/help/api/user-manual * - * @param luceneQuery the root node of the lucene query + * @param query the query string used to identify relevant documents * @return A list of entries matching the complex query */ @Override - public Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException { + public Page performSearchPaged(String query, int pageNumber) throws FetcherException { ArXivQueryTransformer transformer = new ArXivQueryTransformer(); - String transformedQuery = transformer.transformLuceneQuery(luceneQuery).orElse(""); - List searchResult = searchForEntries(transformedQuery, pageNumber).stream() + List searchResult = searchForEntries(query, pageNumber).stream() .map((arXivEntry) -> arXivEntry.toBibEntry(importFormatPreferences.getKeywordSeparator())) .collect(Collectors.toList()); - return new Page<>(transformedQuery, pageNumber, filterYears(searchResult, transformer)); + return new Page<>(query, pageNumber, filterYears(searchResult, transformer)); } private List filterYears(List searchResult, ArXivQueryTransformer transformer) { diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java index c3dc605e7ef..7a38f395896 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java @@ -81,13 +81,13 @@ public String getName() { } /** - * @param luceneQuery query string, matching the apache solr format + * @param query the query string used to identify relevant documents * @return URL which points to a search request for given query */ @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String query, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder builder = new URIBuilder(API_SEARCH_URL); - builder.addParameter("q", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); + builder.addParameter("q", query); builder.addParameter("fl", "bibcode"); builder.addParameter("rows", String.valueOf(getPageSize())); builder.addParameter("start", String.valueOf(getPageSize() * pageNumber)); @@ -273,10 +273,10 @@ private List performSearchByIds(Collection identifiers) throws } @Override - public List performSearch(QueryNode luceneQuery) throws FetcherException { + public List performSearch(String query) throws FetcherException { URL urlForQuery; try { - urlForQuery = getURLForQuery(luceneQuery); + urlForQuery = getURLForQuery(query); } catch (URISyntaxException e) { throw new FetcherException("Search URI is malformed", e); } catch (IOException e) { @@ -288,10 +288,10 @@ public List performSearch(QueryNode luceneQuery) throws FetcherExcepti } @Override - public Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException { + public Page performSearchPaged(String query, int pageNumber) throws FetcherException { URL urlForQuery; try { - urlForQuery = getURLForQuery(luceneQuery, pageNumber); + urlForQuery = getURLForQuery(query, pageNumber); } catch (URISyntaxException e) { throw new FetcherException("Search URI is malformed", e); } catch (IOException e) { @@ -300,7 +300,7 @@ public Page performSearchPaged(QueryNode luceneQuery, int pageNumber) // This is currently just interpreting the complex query as a default string query List bibCodes = fetchBibcodes(urlForQuery); Collection results = performSearchByIds(bibCodes); - return new Page<>(luceneQuery.toString(), pageNumber, results); + return new Page<>(query, pageNumber, results); } @Override diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java b/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java index 4c72717d730..fca434d099a 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java @@ -46,10 +46,10 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder("https://citeseer.ist.psu.edu/search"); uriBuilder.addParameter("sort", "rlv"); // Sort by relevance - uriBuilder.addParameter("q", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // Query + uriBuilder.addParameter("q", query); // Query uriBuilder.addParameter("t", "doc"); // Type: documents // uriBuilder.addParameter("start", "0"); // Start index (not supported at the moment) return uriBuilder.build().toURL(); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java index 71bf613c79d..b592acf171d 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java @@ -35,9 +35,9 @@ public CollectionOfComputerScienceBibliographiesFetcher(ImportFormatPreferences } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { return new URIBuilder(BASIC_SEARCH_URL) - .addParameter("query", new CollectionOfComputerScienceBibliographiesQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")) + .addParameter("query", query) .addParameter("sort", "score") .build() .toURL(); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java index 97506c284a6..a436532b2dd 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java @@ -45,12 +45,12 @@ public Optional getHelpPage() { } @Override - public List performSearch(QueryNode luceneQuery) throws FetcherException { + public List performSearch(String query) throws FetcherException { // All entries have to be converted into one format, this is necessary for the format conversion return fetchers.parallelStream() .flatMap(searchBasedFetcher -> { try { - return searchBasedFetcher.performSearch(luceneQuery).stream(); + return searchBasedFetcher.performSearch(query).stream(); } catch (FetcherException e) { LOGGER.warn(String.format("%s API request failed", searchBasedFetcher.getName()), e); return Stream.empty(); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java b/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java index 9c201e1e7c1..38a3587ae6a 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java @@ -67,9 +67,9 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(API_URL); - uriBuilder.addParameter("query", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); + uriBuilder.addParameter("query", query); return uriBuilder.build().toURL(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java index d961ce0b3bc..61bb9db5cd8 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java @@ -44,9 +44,9 @@ public DBLPFetcher(ImportFormatPreferences importFormatPreferences) { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(BASIC_SEARCH_URL); - uriBuilder.addParameter("q", new DBLPQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); + uriBuilder.addParameter("q", query); uriBuilder.addParameter("h", String.valueOf(100)); // number of hits uriBuilder.addParameter("c", String.valueOf(0)); // no need for auto-completion uriBuilder.addParameter("f", String.valueOf(0)); // "from", index of first hit to download diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java index a8656936077..0da3961508f 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java @@ -190,9 +190,9 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); - DOAJFetcher.addPath(uriBuilder, new DefaultLuceneQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); + DOAJFetcher.addPath(uriBuilder, query); // Number of results uriBuilder.addParameter("pageSize", "30"); // Page (not needed so far) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java b/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java index fc72717086f..aae818cf5ac 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java @@ -177,18 +177,18 @@ private void obtainAndModifyCookie() throws FetcherException { } @Override - public Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException { - ScholarQueryTransformer queryTransformer = new ScholarQueryTransformer(); - String transformedQuery = queryTransformer.transformLuceneQuery(luceneQuery).orElse(""); + public Page performSearchPaged(String query, int pageNumber) throws FetcherException { try { obtainAndModifyCookie(); List foundEntries = new ArrayList<>(10); URIBuilder uriBuilder = new URIBuilder(BASIC_SEARCH_URL); uriBuilder.addParameter("hl", "en"); uriBuilder.addParameter("btnG", "Search"); - uriBuilder.addParameter("q", transformedQuery); + uriBuilder.addParameter("q", query); uriBuilder.addParameter("start", String.valueOf(pageNumber * getPageSize())); uriBuilder.addParameter("num", String.valueOf(getPageSize())); + // TODO: How do we handle this now? -> Check whether you can add it to the query string! and implement it that way, otherwise maybe just add the transformers to the ones that do not support it + // -> We can add it to the string just as a year and/or do post processing.... uriBuilder.addParameter("as_ylo", String.valueOf(queryTransformer.getStartYear())); uriBuilder.addParameter("as_yhi", String.valueOf(queryTransformer.getEndYear())); @@ -211,7 +211,7 @@ public Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throw new FetcherException("Error while fetching from " + getName(), e); } } - return new Page<>(transformedQuery, pageNumber, foundEntries); + return new Page<>(query, pageNumber, foundEntries); } catch (URISyntaxException e) { throw new FetcherException("Error while fetching from " + getName(), e); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java index 4dccde35980..7ec21569163 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java @@ -77,11 +77,4 @@ public List performSearch(String searchQuery) throws FetcherException return collect; } - /** - * Not used - */ - @Override - public List performSearch(QueryNode luceneQuery) throws FetcherException { - return Collections.emptyList(); - } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java index 23dd4e8080e..81f4900c39b 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java @@ -38,11 +38,11 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(URL_PATTERN); uriBuilder.addParameter("version", "1.1"); uriBuilder.addParameter("operation", "searchRetrieve"); - uriBuilder.addParameter("query", new GVKQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); + uriBuilder.addParameter("query", query); uriBuilder.addParameter("maximumRecords", "50"); uriBuilder.addParameter("recordSchema", "picaxml"); uriBuilder.addParameter("sortKeys", "Year,,1"); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java index 77b3c626cd0..46e726b5e7a 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java @@ -58,8 +58,6 @@ public class IEEE implements FulltextFetcher, PagedSearchBasedParserFetcher { private final ImportFormatPreferences preferences; - private IEEEQueryTransformer transformer; - public IEEE(ImportFormatPreferences preferences) { this.preferences = Objects.requireNonNull(preferences); } @@ -243,18 +241,16 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String query, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { // transformer is stored globally, because we need to filter out the bib entries by the year manually // the transformer stores the min and max year - transformer = new IEEEQueryTransformer(); - String transformedQuery = transformer.transformLuceneQuery(luceneQuery).orElse(""); URIBuilder uriBuilder = new URIBuilder("https://ieeexploreapi.ieee.org/api/v1/search/articles"); uriBuilder.addParameter("apikey", API_KEY); - if (!transformedQuery.isBlank()) { - uriBuilder.addParameter("querytext", transformedQuery); + if (!query.isBlank()) { + uriBuilder.addParameter("querytext", query); } uriBuilder.addParameter("max_records", String.valueOf(getPageSize())); - // Currently not working as part of the query string + // Currently not working as part of the query string TODO: This should work now, add it to the transformer if (transformer.getJournal().isPresent()) { uriBuilder.addParameter("publication_title", transformer.getJournal().get()); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java index 553114e46c6..dc0dbf2128c 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java @@ -50,9 +50,9 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(INSPIRE_HOST); - uriBuilder.addParameter("q", new DefaultLuceneQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); + uriBuilder.addParameter("q", query); return uriBuilder.build().toURL(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java index 9e78db86377..24b5bbdfc50 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java @@ -49,9 +49,9 @@ public JstorFetcher(ImportFormatPreferences importFormatPreferences) { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(SEARCH_HOST); - uriBuilder.addParameter("Query", new JstorQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); + uriBuilder.addParameter("Query", query); return uriBuilder.build().toURL(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java b/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java index f8282fce74d..24ad2b33075 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java @@ -74,10 +74,10 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder("https://mathscinet.ams.org/mathscinet/search/publications.html"); uriBuilder.addParameter("pg7", "ALLF"); // search all fields - uriBuilder.addParameter("s7", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // query + uriBuilder.addParameter("s7", query); // query uriBuilder.addParameter("r", "1"); // start index uriBuilder.addParameter("extend", "1"); // should return up to 100 items (instead of default 10) uriBuilder.addParameter("fmt", "bibtex"); // BibTeX format diff --git a/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java index 42731cc1ae5..a9e1b2ea2fe 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java @@ -198,16 +198,15 @@ private List fetchMedline(List ids) throws FetcherException { } @Override - public List performSearch(QueryNode luceneQuery) throws FetcherException { + public List performSearch(String query) throws FetcherException { List entryList; DefaultQueryTransformer transformer = new DefaultQueryTransformer(); - Optional transformedQuery = transformer.transformLuceneQuery(luceneQuery); - if (transformedQuery.isEmpty() || transformedQuery.get().isBlank()) { + if (query.isBlank()) { return Collections.emptyList(); } else { // searching for pubmed ids matching the query - List idList = getPubMedIdsFromQuery(transformedQuery.get()); + List idList = getPubMedIdsFromQuery(query); if (idList.isEmpty()) { LOGGER.info("No results found."); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java index efd8f4b325b..fb3fb29ce26 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java @@ -161,9 +161,9 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String query, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(API_URL); - uriBuilder.addParameter("q", new SpringerQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // Search query + uriBuilder.addParameter("q", query); // Search query uriBuilder.addParameter("api_key", API_KEY); // API key uriBuilder.addParameter("s", String.valueOf(getPageSize() * pageNumber + 1)); // Start entry, starts indexing at 1 uriBuilder.addParameter("p", String.valueOf(getPageSize())); // Page size diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java b/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java index 592688412ce..7ae8daa0f41 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java @@ -105,9 +105,9 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder("https://zbmath.org/bibtexoutput/"); - uriBuilder.addParameter("q", new ZbMathQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // search all fields + uriBuilder.addParameter("q", query); // search all fields uriBuilder.addParameter("start", "0"); // start index uriBuilder.addParameter("count", "200"); // should return up to 200 items (instead of default 100) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java index 21d45a13952..581af965ac5 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java @@ -165,6 +165,8 @@ protected void parseYearRange(String yearRange) { /** * Return a string representation of the year-range fielded term * Should follow the structure yyyy-yyyy + * Default implementation emulates year range query using year field and OR concatenation + * Should be overwritten if year range is supported natively by the API * * Example: 2015-2021 */ diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformer.java index a5015a82fab..5112ceb70c7 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformer.java @@ -48,12 +48,12 @@ protected String handleTitle(String title) { @Override protected String handleJournal(String journal) { - this.journal = journal; return StringUtil.quoteStringIfSpaceIsContained(journal); } @Override protected String handleYear(String year) { + // TODO: This is possible in the query string, implement here! startYear = Math.min(startYear, Integer.parseInt(year)); endYear = Math.max(endYear, Integer.parseInt(year)); return ""; @@ -62,6 +62,7 @@ protected String handleYear(String year) { @Override protected Optional handleOtherField(String fieldAsString, String term) { return switch (fieldAsString) { + // TODO: Add here article field handling case "article_number" -> handleArticleNumber(term); default -> super.handleOtherField(fieldAsString, term); }; @@ -80,10 +81,6 @@ private Optional handleArticleNumber(String term) { return Optional.empty(); } - public Optional getJournal() { - return Objects.isNull(journal) ? Optional.empty() : Optional.of(journal); - } - public Optional getArticleNumber() { return Objects.isNull(articleNumber) ? Optional.empty() : Optional.of(articleNumber); } diff --git a/src/main/java/org/jabref/model/entry/types/EntryTypeFactory.java b/src/main/java/org/jabref/model/entry/types/EntryTypeFactory.java index 29422891f4e..1ecf238382f 100644 --- a/src/main/java/org/jabref/model/entry/types/EntryTypeFactory.java +++ b/src/main/java/org/jabref/model/entry/types/EntryTypeFactory.java @@ -50,7 +50,6 @@ public static EntryType parse(String typeName) { List types = new ArrayList<>(Arrays.asList(StandardEntryType.values())); types.addAll(Arrays.asList(IEEETranEntryType.values())); - types.addAll(Arrays.asList(SystematicLiteratureReviewStudyEntryType.values())); return types.stream().filter(type -> type.getName().equals(typeName.toLowerCase(Locale.ENGLISH))).findFirst().orElse(new UnknownEntryType(typeName)); } diff --git a/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryType.java b/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryType.java deleted file mode 100644 index 1d9bd4be112..00000000000 --- a/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryType.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.jabref.model.entry.types; - -import java.util.Arrays; -import java.util.Locale; -import java.util.Optional; - -public enum SystematicLiteratureReviewStudyEntryType implements EntryType { - STUDY_ENTRY("Study"), - SEARCH_QUERY_ENTRY("SearchQuery"), - LIBRARY_ENTRY("Library"); - - private final String displayName; - - SystematicLiteratureReviewStudyEntryType(String displayName) { - this.displayName = displayName; - } - - public static Optional fromName(String name) { - return Arrays.stream(SystematicLiteratureReviewStudyEntryType.values()) - .filter(field -> field.getName().equalsIgnoreCase(name)) - .findAny(); - } - - @Override - public String getName() { - return displayName.toLowerCase(Locale.ENGLISH); - } - - @Override - public String getDisplayName() { - return displayName; - } -} diff --git a/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryTypeDefinitions.java b/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryTypeDefinitions.java deleted file mode 100644 index 5d1bf665bfe..00000000000 --- a/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryTypeDefinitions.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.jabref.model.entry.types; - -import java.util.Arrays; -import java.util.List; - -import org.jabref.model.entry.BibEntryType; -import org.jabref.model.entry.BibEntryTypeBuilder; -import org.jabref.model.entry.field.StandardField; -import org.jabref.model.entry.field.UnknownField; - -/** - * This class represents all supported entry types used in a study definition file - */ -public class SystematicLiteratureReviewStudyEntryTypeDefinitions { - - /** - * Entry type used for study meta data within a study definition file - * - *
    - *
  • Required fields: author, lastsearchdate, name, enabled
    • - *
  • Optional fields:
    • - *
- */ - private static final BibEntryType STUDY_ENTRY = new BibEntryTypeBuilder() - .withType(SystematicLiteratureReviewStudyEntryType.STUDY_ENTRY) - .withRequiredFields(StandardField.AUTHOR, new UnknownField("lastsearchdate"), new UnknownField("name"), new UnknownField("researchquestions")) - .build(); - - /** - * Entry type for the queries within the study definition file - * - *
    - *
  • Required fields: query
    • - *
  • Optional fields:
    • - *
- */ - private static final BibEntryType SEARCH_QUERY_ENTRY = new BibEntryTypeBuilder() - .withType(SystematicLiteratureReviewStudyEntryType.SEARCH_QUERY_ENTRY) - .withRequiredFields(new UnknownField("query")) - .build(); - - /** - * Entry type for the targeted libraries within a study definition file - * - *
    - *
  • Required fields: name, enabled
    • - *
  • Optional fields: comment
    • - *
- */ - private static final BibEntryType LIBRARY_ENTRY = new BibEntryTypeBuilder() - .withType(SystematicLiteratureReviewStudyEntryType.STUDY_ENTRY) - .withRequiredFields(new UnknownField("name"), new UnknownField("enabled")) - .withImportantFields(StandardField.COMMENT) - .build(); - - public static final List ALL = Arrays.asList(STUDY_ENTRY, SEARCH_QUERY_ENTRY, LIBRARY_ENTRY); - - private SystematicLiteratureReviewStudyEntryTypeDefinitions() { - } -} diff --git a/src/main/java/org/jabref/model/study/FetchResult.java b/src/main/java/org/jabref/model/study/FetchResult.java index 80637feb4ab..6a4af66f1ff 100644 --- a/src/main/java/org/jabref/model/study/FetchResult.java +++ b/src/main/java/org/jabref/model/study/FetchResult.java @@ -10,7 +10,7 @@ public class FetchResult { private final BibDatabase fetchResult; public FetchResult(String fetcherName, BibDatabase fetcherResult) { - this.fetcherName = fetcherName; + this.fetcherName = fetcherName.toLowerCase(); this.fetchResult = fetcherResult; } diff --git a/src/main/java/org/jabref/model/study/Study.java b/src/main/java/org/jabref/model/study/Study.java index cc117f0160e..80e46239354 100644 --- a/src/main/java/org/jabref/model/study/Study.java +++ b/src/main/java/org/jabref/model/study/Study.java @@ -13,12 +13,10 @@ * This class defines all aspects of a scientific study relevant to the application. It is a proxy for the file based study definition. */ -@JsonPropertyOrder({"authors", "title", "last-search-date", "research-questions", "queries", "databases"}) +@JsonPropertyOrder({"authors", "title", "research-questions", "queries", "databases"}) public class Study { private List authors; private String title; - @JsonProperty("last-search-date") - private LocalDate lastSearchDate; @JsonProperty("research-questions") private List researchQuestions; private List queries; @@ -54,14 +52,6 @@ public void setQueries(List queries) { this.queries = queries; } - public LocalDate getLastSearchDate() { - return lastSearchDate; - } - - public void setLastSearchDate(LocalDate date) { - lastSearchDate = date; - } - public List getDatabases() { return databases; } @@ -91,7 +81,6 @@ public String toString() { return "Study{" + "authors=" + authors + ", studyName='" + title + '\'' + - ", lastSearchDate=" + lastSearchDate + ", researchQuestions=" + researchQuestions + ", queries=" + queries + ", libraries=" + databases + @@ -115,9 +104,6 @@ public boolean equals(Object o) { if (getTitle() != null ? !getTitle().equals(study.getTitle()) : study.getTitle() != null) { return false; } - if (getLastSearchDate() != null ? !getLastSearchDate().equals(study.getLastSearchDate()) : study.getLastSearchDate() != null) { - return false; - } if (getResearchQuestions() != null ? !getResearchQuestions().equals(study.getResearchQuestions()) : study.getResearchQuestions() != null) { return false; } diff --git a/src/main/java/org/jabref/model/study/StudyDatabase.java b/src/main/java/org/jabref/model/study/StudyDatabase.java index ac71a7160c2..2b222cdac0f 100644 --- a/src/main/java/org/jabref/model/study/StudyDatabase.java +++ b/src/main/java/org/jabref/model/study/StudyDatabase.java @@ -5,7 +5,7 @@ public class StudyDatabase { private boolean enabled; public StudyDatabase(String name, boolean enabled) { - this.name = name; + this.name = name.toLowerCase(); this.enabled = enabled; } @@ -22,7 +22,7 @@ public String getName() { } public void setName(String name) { - this.name = name; + this.name = name.toLowerCase(); } public boolean isEnabled() { diff --git a/src/main/java/org/jabref/model/study/StudyQuery.java b/src/main/java/org/jabref/model/study/StudyQuery.java index ae5a42b0783..1273fb23aa9 100644 --- a/src/main/java/org/jabref/model/study/StudyQuery.java +++ b/src/main/java/org/jabref/model/study/StudyQuery.java @@ -1,7 +1,16 @@ package org.jabref.model.study; +import java.util.HashMap; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonGetter; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.annotation.JsonSetter; + +@JsonPropertyOrder({"query", "refinements"}) public class StudyQuery { private String query; + private Map queryRefinements = new HashMap<>(); public StudyQuery(String query) { this.query = query; @@ -14,14 +23,32 @@ public StudyQuery() { } - public String getQuery() { + // Serialization as query + @JsonGetter("query") + public String getBaseQuery() { return query; } - public void setQuery(String query) { + @JsonGetter("refinements") + public Map getQueryRefinements() { + return queryRefinements; + } + + public String getLibrarySpecificQueryOrDefault(String libraryName) { + return queryRefinements.getOrDefault(libraryName, query); + } + + // Deserialization from query + @JsonSetter("query") + public void setBaseQuery(String query) { this.query = query; } + @JsonSetter("refinements") + public void setQueryRefinements(Map queryRefinements) { + this.queryRefinements = queryRefinements; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -33,18 +60,23 @@ public boolean equals(Object o) { StudyQuery that = (StudyQuery) o; - return getQuery() != null ? getQuery().equals(that.getQuery()) : that.getQuery() == null; + if (!queryRefinements.equals(that.getQueryRefinements())) { + return false; + } + + return getBaseQuery() != null ? getBaseQuery().equals(that.getBaseQuery()) : that.getBaseQuery() == null; } @Override public int hashCode() { - return getQuery() != null ? getQuery().hashCode() : 0; + return getBaseQuery() != null ? getBaseQuery().hashCode() : 0; } @Override public String toString() { return "QueryEntry{" + - "query='" + query + '\'' + + "query='" + query + '\'' + ", " + + queryRefinements + '}'; } } diff --git a/src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java b/src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java index cc483f897c5..0589474f1d2 100644 --- a/src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java +++ b/src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java @@ -103,7 +103,6 @@ void providePathToNonExistentRepositoryThrowsException() { */ @Test void repositoryStructureCorrectlyCreated() throws Exception { - // When repository is instantiated the directory structure is created assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), hashCodeQuantum + " - Quantum"))); assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), hashCodeCloudComputing + " - Cloud Computing"))); @@ -155,15 +154,6 @@ void mergedResultsPersistedCorrectly() throws Exception { assertEquals(getSpringerCloudComputingMockResults(), getTestStudyRepository().getQueryResultEntries("Cloud Computing").getEntries()); } - @Test - void setsLastSearchDatePersistedCorrectly() throws Exception { - List mockResults = getMockResults(); - - studyRepository.persist(mockResults); - - assertEquals(LocalDate.now(), getTestStudyRepository().getStudy().getLastSearchDate()); - } - @Test void studyResultsPersistedCorrectly() throws Exception { List mockResults = getMockResults(); diff --git a/src/test/java/org/jabref/logic/crawler/StudyYamlParserTest.java b/src/test/java/org/jabref/logic/crawler/StudyYamlParserTest.java index 8fbd419f01f..5ca71cdc05b 100644 --- a/src/test/java/org/jabref/logic/crawler/StudyYamlParserTest.java +++ b/src/test/java/org/jabref/logic/crawler/StudyYamlParserTest.java @@ -2,8 +2,9 @@ import java.net.URL; import java.nio.file.Path; -import java.time.LocalDate; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.jabref.logic.util.io.FileUtil; import org.jabref.model.study.Study; @@ -31,10 +32,12 @@ void setupStudy() throws Exception { String studyName = "TestStudyName"; List researchQuestions = List.of("Question1", "Question2"); List queryEntries = List.of(new StudyQuery("Quantum"), new StudyQuery("Cloud Computing"), new StudyQuery("\"Software Engineering\"")); + Map librarySpecificQuery = new HashMap<>(); + librarySpecificQuery.put("ArXiv", "quantum arxiv query"); + queryEntries.get(0).setQueryRefinements(librarySpecificQuery); List libraryEntries = List.of(new StudyDatabase("Springer", true), new StudyDatabase("ArXiv", true), new StudyDatabase("IEEEXplore", false)); expectedStudy = new Study(authors, studyName, researchQuestions, queryEntries, libraryEntries); - expectedStudy.setLastSearchDate(LocalDate.parse("2020-11-26")); } @Test diff --git a/src/test/resources/org/jabref/logic/crawler/study.yml b/src/test/resources/org/jabref/logic/crawler/study.yml index 620edaf0eab..a97c0e12032 100644 --- a/src/test/resources/org/jabref/logic/crawler/study.yml +++ b/src/test/resources/org/jabref/logic/crawler/study.yml @@ -1,12 +1,13 @@ authors: - Jab Ref title: TestStudyName -last-search-date: 2020-11-26 research-questions: - Question1 - Question2 queries: - query: Quantum + refinements: + ArXiv: "quantum arxiv query" - query: Cloud Computing - query: '"Software Engineering"' databases: