From fac0a231da663386026d9bc51e5b8503845438b1 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Fri, 22 Jan 2021 20:41:08 +0100 Subject: [PATCH 01/25] Add pressing enter when the search field is focused as a way to trigger search (#7377) --- .../org/jabref/gui/importer/fetcher/WebSearchPane.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPane.java b/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPane.java index 23d009133aa..080614374a0 100644 --- a/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPane.java +++ b/src/main/java/org/jabref/gui/importer/fetcher/WebSearchPane.java @@ -5,6 +5,7 @@ import javafx.scene.control.Button; import javafx.scene.control.ComboBox; import javafx.scene.control.TextField; +import javafx.scene.input.KeyCode; import javafx.scene.layout.HBox; import javafx.scene.layout.Priority; import javafx.scene.layout.StackPane; @@ -83,6 +84,13 @@ protected Node createContentPane() { viewModel.queryProperty().bind(query.textProperty()); + // Allows to trigger search on pressing enter + query.setOnKeyPressed(event -> { + if (event.getCode() == KeyCode.ENTER) { + viewModel.search(); + } + }); + // Create button that triggers search Button search = new Button(Localization.lang("Search")); search.setDefaultButton(false); From 86d52cd6468335d9e460aaacc870aa5088c62431 Mon Sep 17 00:00:00 2001 From: Christoph Date: Sun, 24 Jan 2021 18:56:08 +0100 Subject: [PATCH 02/25] Refactor unlinked files (#7209) * [WIP] Refactor unlinked files create dialog in fxml and use a background task for import * add controller * inject stuff * empty line * add progreess indicator copy over some methods * finish export implementation TODO: Import Button does not yet work, always null * prepare background task for import * prepare eception handling * add further logging * add progrees indicator linkage TODO: Progress not yet shown * Fix threading issues, report progress * remove useless undo stuff * wire buttons to the viewModel adjust dialog * show import results dialog view * better error messages * Rename files, fix cancel, fix gui * Cleanup checkstyle * checkstyle * fix checkstyle in md * Make table columns more wider disable import button * preapre localization * fix typo * fix md errors * fix l10n key * add l10n * further l10n fixes * further l10n fixs to reuse * remove one dot * idea extend filenode wrapper * Remove extra dialog * fix progressIndicator still visible * replace with spaces * fix checkstyle * add titled pane * fix checkstyle * fix duplicate method * align browse button * adjust combobox display * Fixed whitespaces, fxml and refactored for some readability * Fixed accordion and l10n * Add changelog * fix link in changelog * fix changelog * fix wrong loop var fix dnd * refactor only call pdf import when xmp did not find a result * create viewModel for filter model * wip refactor like in parse latex * fix view model stuff adapt to check View model * add validator * fix selection and export move file node view model * fix bug using wrong parameter remove l10n keys * Refactored some style issues and a minor suggestions of IntelliJ * l10n * only show results after import * Add custom skin for putting arrow to the right * add checkstyle exception * only change import order * checkstyle * load custom skin only on accordion * Add arrow rotation hack * Fix merge conflict * Set disable instead of visible * Fixed jumping arrow * Refactored for mvvm pattern and optics * Remove obsolete language key * refactor * cleanup * fix checkstyle and l10n * move vars down to background task add explaination to checkstyle * Made treeRootProperty a property of Optional * l10n Co-authored-by: Carl Christian Snethlage --- CHANGELOG.md | 3 + config/checkstyle/suppressions.xml | 2 + docs/getting-into-the-code/code-howtos.md | 4 +- src/main/java/org/jabref/gui/Base.css | 25 ++ src/main/java/org/jabref/gui/JabRefFrame.java | 2 +- .../BibtexExtractorViewModel.java | 9 +- .../externalfiles/FileExtensionViewModel.java | 40 ++ .../FindUnlinkedFilesAction.java | 13 +- .../FindUnlinkedFilesDialog.java | 402 ------------------ .../ImportFilesResultItemViewModel.java | 46 ++ .../gui/externalfiles/ImportHandler.java | 148 +++++-- .../externalfiles/UnlinkedFilesCrawler.java | 102 +++++ .../externalfiles/UnlinkedFilesDialog.fxml | 111 +++++ .../UnlinkedFilesDialogView.java | 243 +++++++++++ .../UnlinkedFilesDialogViewModel.java | 260 +++++++++++ .../externalfiles/UnlinkedPDFFileFilter.java | 36 ++ .../gui/importer/ImportEntriesViewModel.java | 1 - .../gui/importer/UnlinkedFilesCrawler.java | 107 ----- .../gui/importer/UnlinkedPDFFileFilter.java | 35 -- .../org/jabref/gui/maintable/MainTable.java | 11 +- .../gui/texparser/ParseLatexDialogView.java | 1 + .../texparser/ParseLatexDialogViewModel.java | 5 +- .../org/jabref/gui/util/BackgroundTask.java | 10 +- .../jabref/gui/util/CustomTitledPaneSkin.java | 210 +++++++++ .../jabref/gui/util/FileFilterConverter.java | 31 +- .../FileNodeViewModel.java | 2 +- .../ExternalFilesContentImporter.java | 16 +- .../fileformat/PdfContentImporter.java | 4 +- .../org/jabref/logic/l10n/Localization.java | 7 +- .../jabref/logic/util/StandardFileType.java | 3 +- .../org/jabref/logic/util/io/FileUtil.java | 10 + src/main/resources/l10n/JabRef_en.properties | 20 +- 32 files changed, 1268 insertions(+), 651 deletions(-) create mode 100644 src/main/java/org/jabref/gui/externalfiles/FileExtensionViewModel.java delete mode 100644 src/main/java/org/jabref/gui/externalfiles/FindUnlinkedFilesDialog.java create mode 100644 src/main/java/org/jabref/gui/externalfiles/ImportFilesResultItemViewModel.java create mode 100644 src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesCrawler.java create mode 100644 src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesDialog.fxml create mode 100644 src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesDialogView.java create mode 100644 src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesDialogViewModel.java create mode 100644 src/main/java/org/jabref/gui/externalfiles/UnlinkedPDFFileFilter.java delete mode 100644 src/main/java/org/jabref/gui/importer/UnlinkedFilesCrawler.java delete mode 100644 src/main/java/org/jabref/gui/importer/UnlinkedPDFFileFilter.java create mode 100644 src/main/java/org/jabref/gui/util/CustomTitledPaneSkin.java rename src/main/java/org/jabref/gui/{texparser => util}/FileNodeViewModel.java (97%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93d09b0cbbc..2316c209889 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve ### Changed +- We improved the "Find unlinked files" dialog to show import results for each file. [#7209](https://github.com/JabRef/jabref/pull/7209) - The content of the field `timestamp` is migrated to `creationdate`. In case one configured "udpate timestampe", it is migrated to `modificationdate`. [koppor#130](https://github.com/koppor/jabref/issues/130) - The JabRef specific meta-data content in the main field such as priorities (prio1, prio2, ...) are migrated to their respective fields. They are removed from the keywords. [#6840](https://github.com/jabref/jabref/issues/6840) - We fixed an issue where groups generated from authors' last names did not include all entries of the authors' [#5833](https://github.com/JabRef/jabref/issues/5833) @@ -30,6 +31,8 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve - We fixed an issue with the style of highlighted check boxes while searching in preferences. [#7226](https://github.com/JabRef/jabref/issues/7226) - We fixed an issue where the option "Move file to file directory" was disabled in the entry editor for all files [#7194](https://github.com/JabRef/jabref/issues/7194) - We fixed an issue where application dialogs were opening in the wrong display when using multiple screens [#7273](https://github.com/JabRef/jabref/pull/7273) +- We fixed an issue where the "Find unlinked files" dialog would freeze JabRef on importing. [#7205](https://github.com/JabRef/jabref/issues/7205) +- We fixed an issue where the "Find unlinked files" would stop importing when importing a single file failed. [#7206](https://github.com/JabRef/jabref/issues/7206) - We fixed an issue where an exception would be displayed for previewing and preferences when a custom theme has been configured but is missing [#7177](https://github.com/JabRef/jabref/issues/7177) - We fixed an issue where the Harvard RTF exporter used the wrong default file extension. [4508](https://github.com/JabRef/jabref/issues/4508) - We fixed an issue where the Harvard RTF exporter did not use the new authors formatter and therefore did not export "organization" authors correctly. [4508](https://github.com/JabRef/jabref/issues/4508) diff --git a/config/checkstyle/suppressions.xml b/config/checkstyle/suppressions.xml index fa0bef88c8b..2169d4246f3 100644 --- a/config/checkstyle/suppressions.xml +++ b/config/checkstyle/suppressions.xml @@ -6,4 +6,6 @@ + + diff --git a/docs/getting-into-the-code/code-howtos.md b/docs/getting-into-the-code/code-howtos.md index b4b188dcb38..45b72a2957b 100644 --- a/docs/getting-into-the-code/code-howtos.md +++ b/docs/getting-into-the-code/code-howtos.md @@ -61,7 +61,7 @@ Many times there is a need to provide an object on many locations simultaneously ### Register to the `EventBus` -Any listening method has to be annotated with `@Subscribe` keyword and must have only one accepting parameter. Furthermore the object which contains such listening method\(s\) has to be registered using the `register(Object)` method provided by `EventBus`. The listening methods can be overloaded by using different parameter types. +Any listening method has to be annotated with `@Subscribe` keyword and must have only one accepting parameter. Furthermore, the object which contains such listening method\(s\) has to be registered using the `register(Object)` method provided by `EventBus`. The listening methods can be overloaded by using different parameter types. ### Posting an object @@ -190,7 +190,7 @@ If the language is a variant of a language `zh_CN` or `pt_BR` it is necessary to ## Cleanup and Formatters -We try to build a cleanup mechanism based on formatters. The idea is that we can register these actions in arbitrary places, e.g., onSave, onImport, onExport, cleanup, etc. and apply them to different fields. The formatters themself are independent of any logic and therefore easy to test. +We try to build a cleanup mechanism based on formatters. The idea is that we can register these actions in arbitrary places, e.g., onSave, onImport, onExport, cleanup, etc. and apply them to different fields. The formatters themselves are independent of any logic and therefore easy to test. Example: [NormalizePagesFormatter](https://github.com/JabRef/jabref/blob/master/src/main/java/org/jabref/logic/formatter/bibtexfields/NormalizePagesFormatter.java) diff --git a/src/main/java/org/jabref/gui/Base.css b/src/main/java/org/jabref/gui/Base.css index 9c1635a129b..bf6f8fb5612 100644 --- a/src/main/java/org/jabref/gui/Base.css +++ b/src/main/java/org/jabref/gui/Base.css @@ -690,6 +690,31 @@ -fx-padding: 0; } +.accordion .titled-pane { + -fx-skin: "org.jabref.gui.util.CustomTitledPaneSkin"; + -fx-arrow-side: right; +} + +.accordion .titled-pane .title { + -fx-background-color: transparent; + -fx-border-color: transparent; + -fx-background-insets: 5 5 5 5; +} + +.accordion .titled-pane > *.content { + -fx-background-color: transparent; + -fx-border-color: transparent; +} + +/* + * The arrow button has some right padding that's added + * by "modena.css". This simply puts the padding on the + * left since the arrow is positioned on the right. + */ +.titled-pane > .title > .arrow-button { + -fx-padding: 0.0em 0.0em 0.0em 0.583em; +} + .text-input { -fx-background-color: -fx-outer-border, -fx-control-inner-background; -fx-background-insets: 0, 1; diff --git a/src/main/java/org/jabref/gui/JabRefFrame.java b/src/main/java/org/jabref/gui/JabRefFrame.java index 27f5263ee17..154efbc6aa4 100644 --- a/src/main/java/org/jabref/gui/JabRefFrame.java +++ b/src/main/java/org/jabref/gui/JabRefFrame.java @@ -797,7 +797,7 @@ private MenuBar createMenu() { new SeparatorMenuItem(), - factory.createMenuItem(StandardActions.FIND_UNLINKED_FILES, new FindUnlinkedFilesAction(dialogService, prefs, undoManager, stateManager)) + factory.createMenuItem(StandardActions.FIND_UNLINKED_FILES, new FindUnlinkedFilesAction(dialogService, stateManager)) ); // PushToApplication diff --git a/src/main/java/org/jabref/gui/bibtexextractor/BibtexExtractorViewModel.java b/src/main/java/org/jabref/gui/bibtexextractor/BibtexExtractorViewModel.java index a7c4d76b8b8..eb020c0be0c 100644 --- a/src/main/java/org/jabref/gui/bibtexextractor/BibtexExtractorViewModel.java +++ b/src/main/java/org/jabref/gui/bibtexextractor/BibtexExtractorViewModel.java @@ -31,10 +31,10 @@ public class BibtexExtractorViewModel { private static final Logger LOGGER = LoggerFactory.getLogger(BibtexExtractorViewModel.class); private final StringProperty inputTextProperty = new SimpleStringProperty(""); - private DialogService dialogService; - private GrobidCitationFetcher currentCitationfetcher; - private TaskExecutor taskExecutor; - private ImportHandler importHandler; + private final DialogService dialogService; + private final GrobidCitationFetcher currentCitationfetcher; + private final TaskExecutor taskExecutor; + private final ImportHandler importHandler; public BibtexExtractorViewModel(BibDatabaseContext bibdatabaseContext, DialogService dialogService, @@ -48,7 +48,6 @@ public BibtexExtractorViewModel(BibDatabaseContext bibdatabaseContext, currentCitationfetcher = new GrobidCitationFetcher(preferencesService.getImportFormatPreferences()); this.taskExecutor = taskExecutor; this.importHandler = new ImportHandler( - dialogService, bibdatabaseContext, ExternalFileTypes.getInstance(), preferencesService, diff --git a/src/main/java/org/jabref/gui/externalfiles/FileExtensionViewModel.java b/src/main/java/org/jabref/gui/externalfiles/FileExtensionViewModel.java new file mode 100644 index 00000000000..f1f51862c30 --- /dev/null +++ b/src/main/java/org/jabref/gui/externalfiles/FileExtensionViewModel.java @@ -0,0 +1,40 @@ +package org.jabref.gui.externalfiles; + +import java.nio.file.DirectoryStream.Filter; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; + +import org.jabref.gui.externalfiletype.ExternalFileType; +import org.jabref.gui.externalfiletype.ExternalFileTypes; +import org.jabref.gui.icon.JabRefIcon; +import org.jabref.gui.util.FileFilterConverter; +import org.jabref.logic.l10n.Localization; +import org.jabref.logic.util.FileType; + +public class FileExtensionViewModel { + + private final String description; + private final List extensions; + private final ExternalFileTypes externalFileTypes; + + FileExtensionViewModel(FileType fileType, ExternalFileTypes externalFileTypes) { + this.description = Localization.lang("%0 file", fileType.toString()); + this.extensions = fileType.getExtensionsWithDot(); + this.externalFileTypes = externalFileTypes; + } + + public String getDescription() { + return this.description + extensions.stream().collect(Collectors.joining(", ", " (", ")")); + } + + public JabRefIcon getIcon() { + return externalFileTypes.getExternalFileTypeByExt(extensions.get(0)) + .map(ExternalFileType::getIcon) + .orElse(null); + } + + public Filter dirFilter() { + return FileFilterConverter.toDirFilter(extensions); + } +} diff --git a/src/main/java/org/jabref/gui/externalfiles/FindUnlinkedFilesAction.java b/src/main/java/org/jabref/gui/externalfiles/FindUnlinkedFilesAction.java index 2f23e54be25..7c5cb7bf644 100644 --- a/src/main/java/org/jabref/gui/externalfiles/FindUnlinkedFilesAction.java +++ b/src/main/java/org/jabref/gui/externalfiles/FindUnlinkedFilesAction.java @@ -1,26 +1,18 @@ package org.jabref.gui.externalfiles; -import javax.swing.undo.UndoManager; - import org.jabref.gui.DialogService; import org.jabref.gui.StateManager; import org.jabref.gui.actions.SimpleCommand; -import org.jabref.model.database.BibDatabaseContext; -import org.jabref.preferences.PreferencesService; import static org.jabref.gui.actions.ActionHelper.needsDatabase; public class FindUnlinkedFilesAction extends SimpleCommand { private final DialogService dialogService; - private final PreferencesService preferencesService; - private final UndoManager undoManager; private final StateManager stateManager; - public FindUnlinkedFilesAction(DialogService dialogService, PreferencesService preferencesService, UndoManager undoManager, StateManager stateManager) { + public FindUnlinkedFilesAction(DialogService dialogService, StateManager stateManager) { this.dialogService = dialogService; - this.preferencesService = preferencesService; - this.undoManager = undoManager; this.stateManager = stateManager; this.executable.bind(needsDatabase(this.stateManager)); @@ -28,7 +20,6 @@ public FindUnlinkedFilesAction(DialogService dialogService, PreferencesService p @Override public void execute() { - BibDatabaseContext database = stateManager.getActiveDatabase().orElseThrow(() -> new NullPointerException("Database null")); - dialogService.showCustomDialogAndWait(new FindUnlinkedFilesDialog(database, dialogService, preferencesService, undoManager)); + dialogService.showCustomDialogAndWait(new UnlinkedFilesDialogView()); } } diff --git a/src/main/java/org/jabref/gui/externalfiles/FindUnlinkedFilesDialog.java b/src/main/java/org/jabref/gui/externalfiles/FindUnlinkedFilesDialog.java deleted file mode 100644 index 0f2236352b0..00000000000 --- a/src/main/java/org/jabref/gui/externalfiles/FindUnlinkedFilesDialog.java +++ /dev/null @@ -1,402 +0,0 @@ -package org.jabref.gui.externalfiles; - -import java.io.BufferedWriter; -import java.io.FileFilter; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; - -import javax.swing.undo.UndoManager; - -import javafx.collections.FXCollections; -import javafx.geometry.Insets; -import javafx.geometry.Pos; -import javafx.scene.control.Button; -import javafx.scene.control.ButtonBar; -import javafx.scene.control.ButtonType; -import javafx.scene.control.CheckBoxTreeItem; -import javafx.scene.control.ComboBox; -import javafx.scene.control.Label; -import javafx.scene.control.ProgressIndicator; -import javafx.scene.control.ScrollPane; -import javafx.scene.control.TextField; -import javafx.scene.control.Tooltip; -import javafx.scene.control.TreeItem; -import javafx.scene.control.TreeView; -import javafx.scene.layout.BorderPane; -import javafx.scene.layout.HBox; -import javafx.scene.layout.Priority; -import javafx.scene.layout.StackPane; -import javafx.scene.layout.VBox; -import javafx.stage.FileChooser; - -import org.jabref.gui.DialogService; -import org.jabref.gui.Globals; -import org.jabref.gui.externalfiletype.ExternalFileType; -import org.jabref.gui.externalfiletype.ExternalFileTypes; -import org.jabref.gui.importer.UnlinkedFilesCrawler; -import org.jabref.gui.util.BackgroundTask; -import org.jabref.gui.util.BaseDialog; -import org.jabref.gui.util.DirectoryDialogConfiguration; -import org.jabref.gui.util.FileDialogConfiguration; -import org.jabref.gui.util.FileFilterConverter; -import org.jabref.gui.util.ViewModelListCellFactory; -import org.jabref.gui.util.ViewModelTreeCellFactory; -import org.jabref.logic.l10n.Localization; -import org.jabref.logic.util.StandardFileType; -import org.jabref.model.database.BibDatabaseContext; -import org.jabref.preferences.PreferencesService; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * GUI Dialog for the feature "Search for unlinked local files". - */ -public class FindUnlinkedFilesDialog extends BaseDialog { - - private static final Logger LOGGER = LoggerFactory.getLogger(FindUnlinkedFilesDialog.class); - private final BibDatabaseContext databaseContext; - private final ImportHandler importHandler; - private final PreferencesService preferences; - private final DialogService dialogService; - private Button buttonScan; - private Button buttonExport; - private Button buttonApply; - private TextField textfieldDirectoryPath; - private TreeView tree; - private ComboBox comboBoxFileTypeSelection; - private VBox panelSearchProgress; - private BackgroundTask findUnlinkedFilesTask; - - public FindUnlinkedFilesDialog(BibDatabaseContext database, DialogService dialogService, PreferencesService preferencesService, UndoManager undoManager) { - super(); - this.setTitle(Localization.lang("Search for unlinked local files")); - this.dialogService = dialogService; - this.preferences = preferencesService; - - databaseContext = database; - importHandler = new ImportHandler( - dialogService, - databaseContext, - ExternalFileTypes.getInstance(), - preferences, - Globals.getFileUpdateMonitor(), - undoManager, - Globals.stateManager); - - initialize(); - } - - /** - * Initializes the components, the layout, the data structure and the actions in this dialog. - */ - private void initialize() { - Button buttonBrowse = new Button(Localization.lang("Browse")); - buttonBrowse.setTooltip(new Tooltip(Localization.lang("Opens the file browser."))); - buttonBrowse.getStyleClass().add("text-button"); - buttonBrowse.setOnAction(e -> { - DirectoryDialogConfiguration directoryDialogConfiguration = new DirectoryDialogConfiguration.Builder() - .withInitialDirectory(preferences.getWorkingDir()).build(); - dialogService.showDirectorySelectionDialog(directoryDialogConfiguration) - .ifPresent(selectedDirectory -> { - textfieldDirectoryPath.setText(selectedDirectory.toAbsolutePath().toString()); - preferences.setWorkingDirectory(selectedDirectory.toAbsolutePath()); - }); - }); - - buttonScan = new Button(Localization.lang("Scan directory")); - buttonScan.setTooltip(new Tooltip((Localization.lang("Searches the selected directory for unlinked files.")))); - buttonScan.setOnAction(e -> startSearch()); - buttonScan.setDefaultButton(true); - buttonScan.setPadding(new Insets(5, 0, 0, 0)); - - buttonExport = new Button(Localization.lang("Export selected entries")); - buttonExport.setTooltip(new Tooltip(Localization.lang("Export to text file."))); - buttonExport.getStyleClass().add("text-button"); - buttonExport.setDisable(true); - buttonExport.setOnAction(e -> startExport()); - - ButtonType buttonTypeImport = new ButtonType(Localization.lang("Import"), ButtonBar.ButtonData.OK_DONE); - getDialogPane().getButtonTypes().setAll( - buttonTypeImport, - ButtonType.CANCEL - ); - buttonApply = (Button) getDialogPane().lookupButton(buttonTypeImport); - buttonApply.setTooltip(new Tooltip((Localization.lang("Starts the import of BibTeX entries.")))); - buttonApply.setDisable(true); - - /* Actions for the TreeView */ - Button buttonOptionSelectAll = new Button(); - buttonOptionSelectAll.setText(Localization.lang("Select all")); - buttonOptionSelectAll.getStyleClass().add("text-button"); - buttonOptionSelectAll.setOnAction(event -> { - CheckBoxTreeItem root = (CheckBoxTreeItem) tree.getRoot(); - // Need to toggle a twice to make sure everything is selected - root.setSelected(true); - root.setSelected(false); - root.setSelected(true); - }); - Button buttonOptionDeselectAll = new Button(); - buttonOptionDeselectAll.setText(Localization.lang("Unselect all")); - buttonOptionDeselectAll.getStyleClass().add("text-button"); - buttonOptionDeselectAll.setOnAction(event -> { - CheckBoxTreeItem root = (CheckBoxTreeItem) tree.getRoot(); - // Need to toggle a twice to make sure nothing is selected - root.setSelected(false); - root.setSelected(true); - root.setSelected(false); - }); - Button buttonOptionExpandAll = new Button(); - buttonOptionExpandAll.setText(Localization.lang("Expand all")); - buttonOptionExpandAll.getStyleClass().add("text-button"); - buttonOptionExpandAll.setOnAction(event -> { - CheckBoxTreeItem root = (CheckBoxTreeItem) tree.getRoot(); - expandTree(root, true); - }); - Button buttonOptionCollapseAll = new Button(); - buttonOptionCollapseAll.setText(Localization.lang("Collapse all")); - buttonOptionCollapseAll.getStyleClass().add("text-button"); - buttonOptionCollapseAll.setOnAction(event -> { - CheckBoxTreeItem root = (CheckBoxTreeItem) tree.getRoot(); - expandTree(root, false); - root.setExpanded(false); - }); - - textfieldDirectoryPath = new TextField(); - Path initialPath = databaseContext.getFirstExistingFileDir(preferences.getFilePreferences()) - .orElse(preferences.getWorkingDir()); - textfieldDirectoryPath.setText(initialPath.toAbsolutePath().toString()); - - Label labelDirectoryDescription = new Label(Localization.lang("Select a directory where the search shall start.")); - Label labelFileTypesDescription = new Label(Localization.lang("Select file type:")); - Label labelFilesDescription = new Label(Localization.lang("These files are not linked in the active library.")); - Label labelSearchingDirectoryInfo = new Label(Localization.lang("Searching file system...")); - - tree = new TreeView<>(); - tree.setPrefWidth(Double.POSITIVE_INFINITY); - - ScrollPane scrollPaneTree = new ScrollPane(tree); - scrollPaneTree.setFitToWidth(true); - - ProgressIndicator progressBarSearching = new ProgressIndicator(); - progressBarSearching.setMaxSize(50, 50); - - setResultConverter(buttonPressed -> { - if (buttonPressed == buttonTypeImport) { - startImport(); - } else { - if (findUnlinkedFilesTask != null) { - findUnlinkedFilesTask.cancel(); - } - } - return false; - }); - - new ViewModelTreeCellFactory() - .withText(node -> { - if (Files.isRegularFile(node.path)) { - // File - return node.path.getFileName().toString(); - } else { - // Directory - return node.path.getFileName() + " (" + node.fileCount + " file" + (node.fileCount > 1 ? "s" : "") + ")"; - } - }) - .install(tree); - List fileFilterList = Arrays.asList( - FileFilterConverter.ANY_FILE, - FileFilterConverter.toExtensionFilter(StandardFileType.PDF), - FileFilterConverter.toExtensionFilter(StandardFileType.BIBTEX_DB) - ); - - comboBoxFileTypeSelection = new ComboBox<>(FXCollections.observableArrayList(fileFilterList)); - comboBoxFileTypeSelection.getSelectionModel().selectFirst(); - new ViewModelListCellFactory() - .withText(fileFilter -> fileFilter.getDescription() + fileFilter.getExtensions().stream().collect(Collectors.joining(", ", " (", ")"))) - .withIcon(fileFilter -> ExternalFileTypes.getInstance().getExternalFileTypeByExt(fileFilter.getExtensions().get(0)) - .map(ExternalFileType::getIcon) - .orElse(null)) - .install(comboBoxFileTypeSelection); - - panelSearchProgress = new VBox(5, labelSearchingDirectoryInfo, progressBarSearching); - panelSearchProgress.toFront(); - panelSearchProgress.setVisible(false); - - VBox panelDirectory = new VBox(5); - panelDirectory.getChildren().setAll( - labelDirectoryDescription, - new HBox(10, textfieldDirectoryPath, buttonBrowse), - new HBox(15, labelFileTypesDescription, comboBoxFileTypeSelection), - buttonScan - ); - HBox.setHgrow(textfieldDirectoryPath, Priority.ALWAYS); - - StackPane stackPaneTree = new StackPane(scrollPaneTree, panelSearchProgress); - StackPane.setAlignment(panelSearchProgress, Pos.CENTER); - BorderPane panelFiles = new BorderPane(); - panelFiles.setTop(labelFilesDescription); - panelFiles.setCenter(stackPaneTree); - panelFiles.setBottom(new HBox(5, buttonOptionSelectAll, buttonOptionDeselectAll, buttonOptionExpandAll, buttonOptionCollapseAll, buttonExport)); - - VBox container = new VBox(20); - container.getChildren().addAll( - panelDirectory, - panelFiles - ); - container.setPrefWidth(600); - getDialogPane().setContent(container); - } - - /** - * Expands or collapses the specified tree according to the expand-parameter. - */ - private void expandTree(TreeItem item, boolean expand) { - if (item != null && !item.isLeaf()) { - item.setExpanded(expand); - for (TreeItem child : item.getChildren()) { - expandTree(child, expand); - } - } - } - - /** - * Starts the search of unlinked files according chosen directory and the file type selection. The search will - * process in a separate thread and a progress indicator will be displayed. - */ - private void startSearch() { - Path directory = getSearchDirectory(); - FileFilter selectedFileFilter = FileFilterConverter.toFileFilter(comboBoxFileTypeSelection.getValue()); - - findUnlinkedFilesTask = new UnlinkedFilesCrawler(directory, selectedFileFilter, databaseContext) - .onRunning(() -> { - panelSearchProgress.setVisible(true); - buttonScan.setDisable(true); - tree.setRoot(null); - }) - .onFinished(() -> { - panelSearchProgress.setVisible(false); - buttonScan.setDisable(false); - }) - .onSuccess(root -> { - tree.setRoot(root); - root.setSelected(true); - root.setExpanded(true); - - buttonApply.setDisable(false); - buttonExport.setDisable(false); - buttonScan.setDefaultButton(false); - }); - findUnlinkedFilesTask.executeWith(Globals.TASK_EXECUTOR); - } - - private Path getSearchDirectory() { - Path directory = Path.of(textfieldDirectoryPath.getText()); - if (Files.notExists(directory)) { - directory = Path.of(System.getProperty("user.dir")); - textfieldDirectoryPath.setText(directory.toAbsolutePath().toString()); - } - if (!Files.isDirectory(directory)) { - directory = directory.getParent(); - textfieldDirectoryPath.setText(directory.toAbsolutePath().toString()); - } - return directory; - } - - /** - * This will start the import of all file of all selected nodes in the file tree view. - */ - private void startImport() { - CheckBoxTreeItem root = (CheckBoxTreeItem) tree.getRoot(); - final List fileList = getFileListFromNode(root); - - if (fileList.isEmpty()) { - return; - } - - importHandler.importAsNewEntries(fileList); - } - - /** - * This starts the export of all files of all selected nodes in the file tree view. - */ - private void startExport() { - CheckBoxTreeItem root = (CheckBoxTreeItem) tree.getRoot(); - - final List fileList = getFileListFromNode(root); - if (fileList.isEmpty()) { - return; - } - - buttonExport.setVisible(false); - buttonApply.setVisible(false); - - FileDialogConfiguration fileDialogConfiguration = new FileDialogConfiguration.Builder() - .withInitialDirectory(preferences.getWorkingDir()).build(); - Optional exportPath = dialogService.showFileSaveDialog(fileDialogConfiguration); - - if (exportPath.isEmpty()) { - buttonExport.setVisible(true); - buttonApply.setVisible(true); - return; - } - - try (BufferedWriter writer = - Files.newBufferedWriter(exportPath.get(), StandardCharsets.UTF_8, - StandardOpenOption.CREATE)) { - for (Path file : fileList) { - writer.write(file.toString() + "\n"); - } - } catch (IOException e) { - LOGGER.warn("IO Error.", e); - } - - buttonExport.setVisible(true); - buttonApply.setVisible(true); - } - - /** - * Creates a list of all files (leaf nodes in the tree structure), which have been selected. - * - * @param node The root node representing a tree structure. - */ - private List getFileListFromNode(CheckBoxTreeItem node) { - List filesList = new ArrayList<>(); - for (TreeItem childNode : node.getChildren()) { - CheckBoxTreeItem child = (CheckBoxTreeItem) childNode; - if (child.isLeaf()) { - if (child.isSelected()) { - Path nodeFile = child.getValue().path; - if ((nodeFile != null) && Files.isRegularFile(nodeFile)) { - filesList.add(nodeFile); - } - } - } else { - filesList.addAll(getFileListFromNode(child)); - } - } - return filesList; - } - - public static class FileNodeWrapper { - - public final Path path; - public final int fileCount; - - public FileNodeWrapper(Path path) { - this(path, 0); - } - - public FileNodeWrapper(Path path, int fileCount) { - this.path = path; - this.fileCount = fileCount; - } - } -} diff --git a/src/main/java/org/jabref/gui/externalfiles/ImportFilesResultItemViewModel.java b/src/main/java/org/jabref/gui/externalfiles/ImportFilesResultItemViewModel.java new file mode 100644 index 00000000000..3677107b427 --- /dev/null +++ b/src/main/java/org/jabref/gui/externalfiles/ImportFilesResultItemViewModel.java @@ -0,0 +1,46 @@ +package org.jabref.gui.externalfiles; + +import java.nio.file.Path; + +import javafx.beans.property.ObjectProperty; +import javafx.beans.property.SimpleObjectProperty; +import javafx.beans.property.SimpleStringProperty; +import javafx.beans.property.StringProperty; +import javafx.scene.paint.Color; + +import org.jabref.gui.icon.IconTheme; +import org.jabref.gui.icon.JabRefIcon; + +public class ImportFilesResultItemViewModel { + + private final StringProperty file = new SimpleStringProperty(""); + private final ObjectProperty icon = new SimpleObjectProperty<>(IconTheme.JabRefIcons.WARNING); + private final StringProperty message = new SimpleStringProperty(""); + + public ImportFilesResultItemViewModel(Path file, boolean success, String message) { + this.file.setValue(file.toString()); + this.message.setValue(message); + if (success) { + this.icon.setValue(IconTheme.JabRefIcons.CHECK.withColor(Color.GREEN)); + } else { + this.icon.setValue(IconTheme.JabRefIcons.WARNING.withColor(Color.RED)); + } + } + + public ObjectProperty icon() { + return this.icon; + } + + public StringProperty file() { + return this.file; + } + + public StringProperty message() { + return this.message; + } + + @Override + public String toString() { + return "ImportFilesResultItemViewModel [file=" + file.get() + ", message=" + message.get() + "]"; + } +} diff --git a/src/main/java/org/jabref/gui/externalfiles/ImportHandler.java b/src/main/java/org/jabref/gui/externalfiles/ImportHandler.java index 4b8e3309946..eddef6b4caf 100644 --- a/src/main/java/org/jabref/gui/externalfiles/ImportHandler.java +++ b/src/main/java/org/jabref/gui/externalfiles/ImportHandler.java @@ -1,6 +1,8 @@ package org.jabref.gui.externalfiles; +import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -8,14 +10,15 @@ import javax.swing.undo.CompoundEdit; import javax.swing.undo.UndoManager; -import org.jabref.gui.DialogService; -import org.jabref.gui.Globals; import org.jabref.gui.StateManager; import org.jabref.gui.externalfiletype.ExternalFileTypes; import org.jabref.gui.undo.UndoableInsertEntries; +import org.jabref.gui.util.BackgroundTask; +import org.jabref.gui.util.DefaultTaskExecutor; import org.jabref.logic.citationkeypattern.CitationKeyGenerator; import org.jabref.logic.externalfiles.ExternalFilesContentImporter; import org.jabref.logic.importer.ImportCleanup; +import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.UpdateField; import org.jabref.logic.util.io.FileUtil; import org.jabref.model.FieldChange; @@ -27,28 +30,28 @@ import org.jabref.model.util.FileUpdateMonitor; import org.jabref.preferences.PreferencesService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class ImportHandler { - private final BibDatabaseContext database; + private static final Logger LOGGER = LoggerFactory.getLogger(ImportHandler.class); + private final BibDatabaseContext bibdatabase; private final PreferencesService preferencesService; - private final DialogService dialogService; private final FileUpdateMonitor fileUpdateMonitor; private final ExternalFilesEntryLinker linker; private final ExternalFilesContentImporter contentImporter; private final UndoManager undoManager; private final StateManager stateManager; - public ImportHandler(DialogService dialogService, - BibDatabaseContext database, + public ImportHandler(BibDatabaseContext database, ExternalFileTypes externalFileTypes, PreferencesService preferencesService, FileUpdateMonitor fileupdateMonitor, UndoManager undoManager, StateManager stateManager) { - this.dialogService = dialogService; - this.database = database; - + this.bibdatabase = database; this.preferencesService = preferencesService; this.fileUpdateMonitor = fileupdateMonitor; this.stateManager = stateManager; @@ -62,40 +65,94 @@ public ExternalFilesEntryLinker getLinker() { return linker; } - public void importAsNewEntries(List files) { - CompoundEdit ce = new CompoundEdit(); - for (Path file : files) { - List entriesToAdd; - if (FileUtil.getFileExtension(file).filter("pdf"::equals).isPresent()) { - List pdfResult = contentImporter.importPDFContent(file); - List xmpEntriesInFile = contentImporter.importXMPContent(file); - - // First try xmp import, if empty try pdf import, otherwise create empty entry - if (!xmpEntriesInFile.isEmpty()) { - if (!pdfResult.isEmpty()) { - // FIXME: Show merge dialog? - entriesToAdd = xmpEntriesInFile; - } else { - entriesToAdd = xmpEntriesInFile; + public BackgroundTask> importFilesInBackground(List files) { + return new BackgroundTask<>() { + private int counter; + private List entriesToAdd; + private final List results = new ArrayList<>(); + + @Override + protected List call() { + counter = 1; + CompoundEdit ce = new CompoundEdit(); + for (Path file: files) { + entriesToAdd = Collections.emptyList(); + + if (isCanceled()) { + break; } - } else { - if (!pdfResult.isEmpty()) { - entriesToAdd = pdfResult; - } else { - entriesToAdd = Collections.singletonList(createEmptyEntryWithLink(file)); + + DefaultTaskExecutor.runInJavaFXThread(() -> { + updateMessage(Localization.lang("Processing file %0", file.getFileName())); + updateProgress(counter, files.size() - 1); + }); + + try { + if (FileUtil.isPDFFile(file)) { + + var xmpParserResult = contentImporter.importXMPContent(file); + List xmpEntriesInFile = xmpParserResult.getDatabase().getEntries(); + + if (xmpParserResult.hasWarnings()) { + addResultToList(file, false, Localization.lang("Error reading XMP content: %0", xmpParserResult.getErrorMessage())); + } + + // First try xmp import, if empty try pdf import, otherwise create empty entry + if (!xmpEntriesInFile.isEmpty()) { + entriesToAdd = xmpEntriesInFile; + addResultToList(file, true, Localization.lang("Importing using XMP data...")); + } else { + var pdfImporterResult = contentImporter.importPDFContent(file); + List pdfEntriesInFile = pdfImporterResult.getDatabase().getEntries(); + + if (pdfImporterResult.hasWarnings()) { + addResultToList(file, false, Localization.lang("Error reading PDF content: %0", pdfImporterResult.getErrorMessage())); + } + + if (!pdfEntriesInFile.isEmpty()) { + entriesToAdd = pdfEntriesInFile; + addResultToList(file, true, Localization.lang("Importing using extracted PDF data")); + } else { + entriesToAdd = Collections.singletonList(createEmptyEntryWithLink(file)); + addResultToList(file, false, Localization.lang("No metadata found. Creating empty entry with file link")); + } + } + } else if (FileUtil.isBibFile(file)) { + var bibtexParserResult = contentImporter.importFromBibFile(file, fileUpdateMonitor); + if (bibtexParserResult.hasWarnings()) { + addResultToList(file, false, bibtexParserResult.getErrorMessage()); + } + + entriesToAdd = bibtexParserResult.getDatabaseContext().getEntries(); + addResultToList(file, false, Localization.lang("Importing bib entry")); + } else { + entriesToAdd = Collections.singletonList(createEmptyEntryWithLink(file)); + addResultToList(file, false, Localization.lang("No BibTeX data found. Creating empty entry with file link")); + } + } catch (IOException ex) { + LOGGER.error("Error importing", ex); + addResultToList(file, false, Localization.lang("Error from import: %0", ex.getLocalizedMessage())); + + DefaultTaskExecutor.runInJavaFXThread(() -> updateMessage(Localization.lang("Error"))); } + + // We need to run the actual import on the FX Thread, otherwise we will get some deadlocks with the UIThreadList + DefaultTaskExecutor.runInJavaFXThread(() -> importEntries(entriesToAdd)); + + ce.addEdit(new UndoableInsertEntries(bibdatabase.getDatabase(), entriesToAdd)); + ce.end(); + undoManager.addEdit(ce); + + counter++; } - } else if (FileUtil.isBibFile(file)) { - entriesToAdd = contentImporter.importFromBibFile(file, fileUpdateMonitor); - } else { - entriesToAdd = Collections.singletonList(createEmptyEntryWithLink(file)); + return results; } - importEntries(entriesToAdd); - ce.addEdit(new UndoableInsertEntries(database.getDatabase(), entriesToAdd)); - } - ce.end(); - undoManager.addEdit(ce); + private void addResultToList(Path newFile, boolean success, String logMessage) { + var result = new ImportFilesResultItemViewModel(newFile, success, logMessage); + results.add(result); + } + }; } private BibEntry createEmptyEntryWithLink(Path file) { @@ -106,11 +163,9 @@ private BibEntry createEmptyEntryWithLink(Path file) { } public void importEntries(List entries) { - // TODO: Add undo/redo - // undoManager.addEdit(new UndoableInsertEntries(panel.getDatabase(), entries)); - ImportCleanup cleanup = new ImportCleanup(database.getMode()); + ImportCleanup cleanup = new ImportCleanup(bibdatabase.getMode()); cleanup.doPostCleanup(entries); - database.getDatabase().insertEntries(entries); + bibdatabase.getDatabase().insertEntries(entries); // Set owner/timestamp UpdateField.setAutomaticFields(entries, @@ -121,7 +176,7 @@ public void importEntries(List entries) { generateKeys(entries); // Add to group - addToGroups(entries, stateManager.getSelectedGroup(database)); + addToGroups(entries, stateManager.getSelectedGroup(bibdatabase)); } private void addToGroups(List entries, Collection groups) { @@ -145,9 +200,10 @@ private void addToGroups(List entries, Collection group */ private void generateKeys(List entries) { CitationKeyGenerator keyGenerator = new CitationKeyGenerator( - database.getMetaData().getCiteKeyPattern(Globals.prefs.getCitationKeyPatternPreferences().getKeyPattern()), - database.getDatabase(), - Globals.prefs.getCitationKeyPatternPreferences()); + bibdatabase.getMetaData().getCiteKeyPattern(preferencesService.getCitationKeyPatternPreferences() + .getKeyPattern()), + bibdatabase.getDatabase(), + preferencesService.getCitationKeyPatternPreferences()); for (BibEntry entry : entries) { keyGenerator.generateAndSetKey(entry); diff --git a/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesCrawler.java b/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesCrawler.java new file mode 100644 index 00000000000..60d89246e1b --- /dev/null +++ b/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesCrawler.java @@ -0,0 +1,102 @@ +package org.jabref.gui.externalfiles; + +import java.io.File; +import java.io.IOException; +import java.nio.file.DirectoryStream.Filter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +import javafx.scene.control.CheckBoxTreeItem; + +import org.jabref.gui.util.BackgroundTask; +import org.jabref.gui.util.FileNodeViewModel; +import org.jabref.model.database.BibDatabase; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.preferences.FilePreferences; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Util class for searching files on the file system which are not linked to a provided {@link BibDatabase}. + */ +public class UnlinkedFilesCrawler extends BackgroundTask { + + private static final Logger LOGGER = LoggerFactory.getLogger(UnlinkedFilesCrawler.class); + + private final Path directory; + private final Filter fileFilter; + private final BibDatabaseContext databaseContext; + private final FilePreferences filePreferences; + + public UnlinkedFilesCrawler(Path directory, Filter fileFilter, BibDatabaseContext databaseContext, FilePreferences filePreferences) { + this.directory = directory; + this.fileFilter = fileFilter; + this.databaseContext = databaseContext; + this.filePreferences = filePreferences; + } + + @Override + protected FileNodeViewModel call() throws IOException { + UnlinkedPDFFileFilter unlinkedPDFFileFilter = new UnlinkedPDFFileFilter(fileFilter, databaseContext, filePreferences); + return searchDirectory(directory, unlinkedPDFFileFilter); + } + + /** + * Searches recursively all files in the specified directory.
+ *
+ * All files matched by the given {@link UnlinkedPDFFileFilter} are taken into the resulting tree.
+ *
+ * The result will be a tree structure of nodes of the type {@link CheckBoxTreeItem}.
+ *
+ * The user objects that are attached to the nodes is the {@link FileNodeViewModel}, which wraps the {@link + * File}-Object.
+ *
+ * For ensuring the capability to cancel the work of this recursive method, the first position in the integer array + * 'state' must be set to 1, to keep the recursion running. When the states value changes, the method will resolve + * its recursion and return what it has saved so far. + * + * @throws IOException if directory is not a directory or empty + */ + private FileNodeViewModel searchDirectory(Path directory, UnlinkedPDFFileFilter fileFilter) throws IOException { + // Return null if the directory is not valid. + if ((directory == null) || !Files.isDirectory(directory)) { + throw new IOException(String.format("Invalid directory for searching: %s", directory)); + } + + FileNodeViewModel parent = new FileNodeViewModel(directory); + Map> fileListPartition; + + try (Stream filesStream = StreamSupport.stream(Files.newDirectoryStream(directory, fileFilter).spliterator(), false)) { + fileListPartition = filesStream.collect(Collectors.partitioningBy(path -> path.toFile().isDirectory())); + } catch (IOException e) { + LOGGER.error(String.format("%s while searching files: %s", e.getClass().getName(), e.getMessage())); + return parent; + } + + List subDirectories = fileListPartition.get(true); + List files = new ArrayList<>(fileListPartition.get(false)); + int fileCount = 0; + + for (Path subDirectory : subDirectories) { + FileNodeViewModel subRoot = searchDirectory(subDirectory, fileFilter); + + if (!subRoot.getChildren().isEmpty()) { + fileCount += subRoot.getFileCount(); + parent.getChildren().add(subRoot); + } + } + + parent.setFileCount(files.size() + fileCount); + parent.getChildren().addAll(files.stream() + .map(FileNodeViewModel::new) + .collect(Collectors.toList())); + return parent; + } +} diff --git a/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesDialog.fxml b/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesDialog.fxml new file mode 100644 index 00000000000..0d3246da39d --- /dev/null +++ b/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesDialog.fxml @@ -0,0 +1,111 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesDialogView.java b/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesDialogView.java new file mode 100644 index 00000000000..bcbcbf0c07a --- /dev/null +++ b/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesDialogView.java @@ -0,0 +1,243 @@ +package org.jabref.gui.externalfiles; + +import javax.inject.Inject; +import javax.swing.undo.UndoManager; + +import javafx.application.Platform; +import javafx.beans.InvalidationListener; +import javafx.beans.binding.Bindings; +import javafx.beans.binding.BooleanBinding; +import javafx.collections.FXCollections; +import javafx.fxml.FXML; +import javafx.scene.control.Accordion; +import javafx.scene.control.Button; +import javafx.scene.control.ButtonType; +import javafx.scene.control.CheckBoxTreeItem; +import javafx.scene.control.ComboBox; +import javafx.scene.control.Control; +import javafx.scene.control.Label; +import javafx.scene.control.ProgressIndicator; +import javafx.scene.control.SelectionMode; +import javafx.scene.control.TableColumn; +import javafx.scene.control.TableView; +import javafx.scene.control.TextField; +import javafx.scene.control.TitledPane; +import javafx.scene.control.TreeItem; +import javafx.scene.layout.VBox; + +import org.jabref.gui.DialogService; +import org.jabref.gui.StateManager; +import org.jabref.gui.externalfiletype.ExternalFileTypes; +import org.jabref.gui.icon.JabRefIcon; +import org.jabref.gui.util.BaseDialog; +import org.jabref.gui.util.FileNodeViewModel; +import org.jabref.gui.util.IconValidationDecorator; +import org.jabref.gui.util.RecursiveTreeItem; +import org.jabref.gui.util.TaskExecutor; +import org.jabref.gui.util.ValueTableCellFactory; +import org.jabref.gui.util.ViewModelListCellFactory; +import org.jabref.gui.util.ViewModelTreeCellFactory; +import org.jabref.logic.l10n.Localization; +import org.jabref.model.util.FileUpdateMonitor; +import org.jabref.preferences.PreferencesService; + +import com.airhacks.afterburner.views.ViewLoader; +import com.tobiasdiez.easybind.EasyBind; +import de.saxsys.mvvmfx.utils.validation.visualization.ControlsFxVisualizer; +import org.controlsfx.control.CheckTreeView; + +public class UnlinkedFilesDialogView extends BaseDialog { + + @FXML private TextField directoryPathField; + @FXML private ComboBox fileTypeCombo; + @FXML private CheckTreeView unlinkedFilesList; + @FXML private Button scanButton; + @FXML private Button exportButton; + @FXML private Button importButton; + @FXML private Label progressText; + @FXML private Accordion accordion; + @FXML private ProgressIndicator progressDisplay; + @FXML private VBox progressPane; + + @FXML private TableView importResultTable; + @FXML private TableColumn colStatus; + @FXML private TableColumn colMessage; + @FXML private TableColumn colFile; + + @FXML private TitledPane filePane; + @FXML private TitledPane resultPane; + + @Inject private PreferencesService preferencesService; + @Inject private DialogService dialogService; + @Inject private StateManager stateManager; + @Inject private UndoManager undoManager; + @Inject private TaskExecutor taskExecutor; + @Inject private FileUpdateMonitor fileUpdateMonitor; + + private final ControlsFxVisualizer validationVisualizer; + private UnlinkedFilesDialogViewModel viewModel; + + public UnlinkedFilesDialogView() { + this.validationVisualizer = new ControlsFxVisualizer(); + + this.setTitle(Localization.lang("Search for unlinked local files")); + + ViewLoader.view(this) + .load() + .setAsDialogPane(this); + + setResultConverter(button -> { + if (button == ButtonType.CANCEL) { + viewModel.cancelTasks(); + } + return null; + }); + } + + @FXML + private void initialize() { + viewModel = new UnlinkedFilesDialogViewModel(dialogService, ExternalFileTypes.getInstance(), undoManager, fileUpdateMonitor, preferencesService, stateManager, taskExecutor); + + progressDisplay.progressProperty().bind(viewModel.progressValueProperty()); + progressText.textProperty().bind(viewModel.progressTextProperty()); + progressPane.managedProperty().bind(viewModel.taskActiveProperty()); + progressPane.visibleProperty().bind(viewModel.taskActiveProperty()); + accordion.disableProperty().bind(viewModel.taskActiveProperty()); + + viewModel.treeRootProperty().addListener(observable -> { + scanButton.setDefaultButton(false); + importButton.setDefaultButton(true); + scanButton.setDefaultButton(false); + filePane.setExpanded(true); + resultPane.setExpanded(false); + }); + + viewModel.resultTableItems().addListener((InvalidationListener) observable -> { + filePane.setExpanded(false); + resultPane.setExpanded(true); + resultPane.setDisable(false); + }); + + initDirectorySelection(); + initUnlinkedFilesList(); + initResultTable(); + initButtons(); + } + + private void initDirectorySelection() { + validationVisualizer.setDecoration(new IconValidationDecorator()); + + directoryPathField.textProperty().bindBidirectional(viewModel.directoryPathProperty()); + Platform.runLater(() -> validationVisualizer.initVisualization(viewModel.directoryPathValidationStatus(), directoryPathField)); + + new ViewModelListCellFactory() + .withText(FileExtensionViewModel::getDescription) + .withIcon(FileExtensionViewModel::getIcon) + .install(fileTypeCombo); + fileTypeCombo.setItems(viewModel.getFileFilters()); + fileTypeCombo.valueProperty().bindBidirectional(viewModel.selectedExtensionProperty()); + fileTypeCombo.getSelectionModel().selectFirst(); + } + + private void initUnlinkedFilesList() { + new ViewModelTreeCellFactory() + .withText(FileNodeViewModel::getDisplayText) + .install(unlinkedFilesList); + + unlinkedFilesList.maxHeightProperty().bind(((Control) filePane.contentProperty().get()).heightProperty()); + unlinkedFilesList.getSelectionModel().setSelectionMode(SelectionMode.MULTIPLE); + unlinkedFilesList.rootProperty().bind(EasyBind.map(viewModel.treeRootProperty(), + fileNode -> fileNode.map(fileNodeViewModel -> new RecursiveTreeItem<>(fileNodeViewModel, FileNodeViewModel::getChildren)) + .orElse(null))); + + EasyBind.subscribe(unlinkedFilesList.rootProperty(), root -> { + if (root != null) { + ((CheckBoxTreeItem) root).setSelected(true); + root.setExpanded(true); + EasyBind.bindContent(viewModel.checkedFileListProperty(), unlinkedFilesList.getCheckModel().getCheckedItems()); + } else { + EasyBind.bindContent(viewModel.checkedFileListProperty(), FXCollections.observableArrayList()); + } + }); + } + + private void initResultTable() { + colFile.setCellValueFactory(cellData -> cellData.getValue().file()); + new ValueTableCellFactory() + .withText(item -> item).withTooltip(item -> item) + .install(colFile); + + colMessage.setCellValueFactory(cellData -> cellData.getValue().message()); + new ValueTableCellFactory() + .withText(item -> item).withTooltip(item -> item) + .install(colMessage); + + colStatus.setCellValueFactory(cellData -> cellData.getValue().icon()); + colStatus.setCellFactory(new ValueTableCellFactory().withGraphic(JabRefIcon::getGraphicNode)); + importResultTable.setColumnResizePolicy((param) -> true); + + importResultTable.setItems(viewModel.resultTableItems()); + } + + private void initButtons() { + BooleanBinding noItemsChecked = Bindings.isNull(unlinkedFilesList.rootProperty()) + .or(Bindings.isEmpty(viewModel.checkedFileListProperty())); + exportButton.disableProperty().bind(noItemsChecked); + importButton.disableProperty().bind(noItemsChecked); + + scanButton.setDefaultButton(true); + scanButton.disableProperty().bind(viewModel.taskActiveProperty().or(viewModel.directoryPathValidationStatus().validProperty().not())); + } + + @FXML + void browseFileDirectory() { + viewModel.browseFileDirectory(); + } + + @FXML + void collapseAll() { + expandTree(unlinkedFilesList.getRoot(), false); + } + + @FXML + void expandAll() { + expandTree(unlinkedFilesList.getRoot(), true); + } + + @FXML + void scanFiles() { + viewModel.startSearch(); + } + + @FXML + void startImport() { + viewModel.startImport(); + } + + @FXML + void selectAll() { + unlinkedFilesList.getCheckModel().checkAll(); + } + + @FXML + void unselectAll() { + unlinkedFilesList.getCheckModel().clearChecks(); + } + + @FXML + void exportSelected() { + viewModel.startExport(); + } + + /** + * Expands or collapses the specified tree according to the expand-parameter. + */ + private void expandTree(TreeItem item, boolean expand) { + if ((item != null) && !item.isLeaf()) { + item.setExpanded(expand); + for (TreeItem child : item.getChildren()) { + expandTree(child, expand); + } + } + } +} diff --git a/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesDialogViewModel.java b/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesDialogViewModel.java new file mode 100644 index 00000000000..8a766657dd8 --- /dev/null +++ b/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesDialogViewModel.java @@ -0,0 +1,260 @@ +package org.jabref.gui.externalfiles; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream.Filter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.List; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import javax.swing.undo.UndoManager; + +import javafx.beans.property.BooleanProperty; +import javafx.beans.property.DoubleProperty; +import javafx.beans.property.ObjectProperty; +import javafx.beans.property.SimpleBooleanProperty; +import javafx.beans.property.SimpleDoubleProperty; +import javafx.beans.property.SimpleListProperty; +import javafx.beans.property.SimpleObjectProperty; +import javafx.beans.property.SimpleStringProperty; +import javafx.beans.property.StringProperty; +import javafx.collections.FXCollections; +import javafx.collections.ObservableList; +import javafx.scene.control.ProgressIndicator; +import javafx.scene.control.TreeItem; + +import org.jabref.gui.DialogService; +import org.jabref.gui.StateManager; +import org.jabref.gui.externalfiletype.ExternalFileTypes; +import org.jabref.gui.util.BackgroundTask; +import org.jabref.gui.util.DirectoryDialogConfiguration; +import org.jabref.gui.util.FileDialogConfiguration; +import org.jabref.gui.util.FileNodeViewModel; +import org.jabref.gui.util.TaskExecutor; +import org.jabref.logic.l10n.Localization; +import org.jabref.logic.util.StandardFileType; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.util.FileUpdateMonitor; +import org.jabref.preferences.PreferencesService; + +import de.saxsys.mvvmfx.utils.validation.FunctionBasedValidator; +import de.saxsys.mvvmfx.utils.validation.ValidationMessage; +import de.saxsys.mvvmfx.utils.validation.ValidationStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class UnlinkedFilesDialogViewModel { + + private static final Logger LOGGER = LoggerFactory.getLogger(UnlinkedFilesDialogViewModel.class); + + private final ImportHandler importHandler; + private final StringProperty directoryPath = new SimpleStringProperty(""); + private final ObjectProperty selectedExtension = new SimpleObjectProperty<>(); + + private final ObjectProperty> treeRootProperty = new SimpleObjectProperty<>(); + private final SimpleListProperty> checkedFileListProperty = new SimpleListProperty<>(FXCollections.observableArrayList()); + + private final BooleanProperty taskActiveProperty = new SimpleBooleanProperty(false); + private final DoubleProperty progressValueProperty = new SimpleDoubleProperty(0); + private final StringProperty progressTextProperty = new SimpleStringProperty(); + + private final ObservableList resultList = FXCollections.observableArrayList(); + private final ObservableList fileFilterList; + private final DialogService dialogService; + private final PreferencesService preferences; + private BackgroundTask findUnlinkedFilesTask; + private BackgroundTask> importFilesBackgroundTask; + + private final BibDatabaseContext bibDatabase; + private final TaskExecutor taskExecutor; + + private final FunctionBasedValidator scanDirectoryValidator; + + public UnlinkedFilesDialogViewModel(DialogService dialogService, ExternalFileTypes externalFileTypes, UndoManager undoManager, + FileUpdateMonitor fileUpdateMonitor, PreferencesService preferences, StateManager stateManager, TaskExecutor taskExecutor) { + this.preferences = preferences; + this.dialogService = dialogService; + this.taskExecutor = taskExecutor; + this.bibDatabase = stateManager.getActiveDatabase().orElseThrow(() -> new NullPointerException("Database null")); + importHandler = new ImportHandler( + bibDatabase, + externalFileTypes, + preferences, + fileUpdateMonitor, + undoManager, + stateManager); + + this.fileFilterList = FXCollections.observableArrayList( + new FileExtensionViewModel(StandardFileType.ANY_FILE, externalFileTypes), + new FileExtensionViewModel(StandardFileType.BIBTEX_DB, externalFileTypes), + new FileExtensionViewModel(StandardFileType.PDF, externalFileTypes)); + + Predicate isDirectory = path -> Files.isDirectory(Path.of(path)); + scanDirectoryValidator = new FunctionBasedValidator<>(directoryPath, isDirectory, + ValidationMessage.error(Localization.lang("Please enter a valid file path."))); + + treeRootProperty.setValue(Optional.empty()); + } + + public void startSearch() { + Path directory = this.getSearchDirectory(); + Filter selectedFileFilter = selectedExtension.getValue().dirFilter(); + + progressValueProperty.unbind(); + progressTextProperty.unbind(); + + findUnlinkedFilesTask = new UnlinkedFilesCrawler(directory, selectedFileFilter, bibDatabase, preferences.getFilePreferences()) + .onRunning(() -> { + progressValueProperty.set(ProgressIndicator.INDETERMINATE_PROGRESS); + progressTextProperty.setValue(Localization.lang("Searching file system...")); + progressTextProperty.bind(findUnlinkedFilesTask.messageProperty()); + taskActiveProperty.setValue(true); + treeRootProperty.setValue(Optional.empty()); + }) + .onFinished(() -> { + progressValueProperty.set(0); + taskActiveProperty.setValue(false); + }) + .onSuccess(treeRoot -> treeRootProperty.setValue(Optional.of(treeRoot))); + + findUnlinkedFilesTask.executeWith(taskExecutor); + } + + public void startImport() { + List fileList = checkedFileListProperty.stream() + .map(item -> item.getValue().getPath()) + .filter(path -> path.toFile().isFile()) + .collect(Collectors.toList()); + if (fileList.isEmpty()) { + LOGGER.warn("There are no valid files checked"); + return; + } + resultList.clear(); + + importFilesBackgroundTask = importHandler.importFilesInBackground(fileList) + .onRunning(() -> { + progressValueProperty.bind(importFilesBackgroundTask.workDonePercentageProperty()); + progressTextProperty.bind(importFilesBackgroundTask.messageProperty()); + taskActiveProperty.setValue(true); + }) + .onFinished(() -> { + progressValueProperty.unbind(); + progressTextProperty.unbind(); + taskActiveProperty.setValue(false); + }) + .onSuccess(resultList::addAll); + importFilesBackgroundTask.executeWith(taskExecutor); + } + + /** + * This starts the export of all files of all selected nodes in the file tree view. + */ + public void startExport() { + List fileList = checkedFileListProperty.stream() + .map(item -> item.getValue().getPath()) + .filter(path -> path.toFile().isFile()) + .collect(Collectors.toList()); + if (fileList.isEmpty()) { + LOGGER.warn("There are no valid files checked"); + return; + } + + FileDialogConfiguration fileDialogConfiguration = new FileDialogConfiguration.Builder() + .withInitialDirectory(preferences.getWorkingDir()) + .addExtensionFilter(StandardFileType.TXT) + .withDefaultExtension(StandardFileType.TXT) + .build(); + Optional exportPath = dialogService.showFileSaveDialog(fileDialogConfiguration); + + if (exportPath.isEmpty()) { + return; + } + + try (BufferedWriter writer = Files.newBufferedWriter(exportPath.get(), StandardCharsets.UTF_8, + StandardOpenOption.CREATE)) { + for (Path file : fileList) { + writer.write(file.toString() + "\n"); + } + } catch (IOException e) { + LOGGER.error("Error exporting", e); + } + } + + public ObservableList getFileFilters() { + return this.fileFilterList; + } + + public void cancelTasks() { + if (findUnlinkedFilesTask != null) { + findUnlinkedFilesTask.cancel(); + } + if (importFilesBackgroundTask != null) { + importFilesBackgroundTask.cancel(); + } + } + + public void browseFileDirectory() { + DirectoryDialogConfiguration directoryDialogConfiguration = new DirectoryDialogConfiguration.Builder() + .withInitialDirectory(preferences.getWorkingDir()).build(); + + dialogService.showDirectorySelectionDialog(directoryDialogConfiguration) + .ifPresent(selectedDirectory -> { + directoryPath.setValue(selectedDirectory.toAbsolutePath().toString()); + preferences.setWorkingDirectory(selectedDirectory.toAbsolutePath()); + }); + } + + private Path getSearchDirectory() { + Path directory = Path.of(directoryPath.getValue()); + if (Files.notExists(directory)) { + directory = Path.of(System.getProperty("user.dir")); + directoryPath.setValue(directory.toAbsolutePath().toString()); + } + if (!Files.isDirectory(directory)) { + directory = directory.getParent(); + directoryPath.setValue(directory.toAbsolutePath().toString()); + } + return directory; + } + + public ObservableList resultTableItems() { + return this.resultList; + } + + public ObjectProperty> treeRootProperty() { + return this.treeRootProperty; + } + + public ObjectProperty selectedExtensionProperty() { + return this.selectedExtension; + } + + public StringProperty directoryPathProperty() { + return this.directoryPath; + } + + public ValidationStatus directoryPathValidationStatus() { + return this.scanDirectoryValidator.getValidationStatus(); + } + + public DoubleProperty progressValueProperty() { + return this.progressValueProperty; + } + + public StringProperty progressTextProperty() { + return this.progressTextProperty; + } + + public BooleanProperty taskActiveProperty() { + return this.taskActiveProperty; + } + + public SimpleListProperty> checkedFileListProperty() { + return checkedFileListProperty; + } +} diff --git a/src/main/java/org/jabref/gui/externalfiles/UnlinkedPDFFileFilter.java b/src/main/java/org/jabref/gui/externalfiles/UnlinkedPDFFileFilter.java new file mode 100644 index 00000000000..1d235d7fbf3 --- /dev/null +++ b/src/main/java/org/jabref/gui/externalfiles/UnlinkedPDFFileFilter.java @@ -0,0 +1,36 @@ +package org.jabref.gui.externalfiles; + +import java.io.FileFilter; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.DirectoryStream.Filter; +import java.nio.file.Path; + +import org.jabref.logic.util.io.DatabaseFileLookup; +import org.jabref.model.database.BibDatabase; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; +import org.jabref.preferences.FilePreferences; + +/** + * {@link FileFilter} implementation, that allows only files which are not linked in any of the {@link BibEntry}s of the + * specified {@link BibDatabase}. + *

+ * This {@link FileFilter} sits on top of another {@link FileFilter} -implementation, which it first consults. Only if + * this major filefilter has accepted a file, this implementation will verify on that file. + */ +public class UnlinkedPDFFileFilter implements DirectoryStream.Filter { + + private final DatabaseFileLookup lookup; + private final Filter fileFilter; + + public UnlinkedPDFFileFilter(DirectoryStream.Filter fileFilter, BibDatabaseContext databaseContext, FilePreferences filePreferences) { + this.fileFilter = fileFilter; + this.lookup = new DatabaseFileLookup(databaseContext, filePreferences); + } + + @Override + public boolean accept(Path pathname) throws IOException { + return fileFilter.accept(pathname) && !lookup.lookupDatabase(pathname.toFile()); + } +} diff --git a/src/main/java/org/jabref/gui/importer/ImportEntriesViewModel.java b/src/main/java/org/jabref/gui/importer/ImportEntriesViewModel.java index c77950c5d49..5402e77a4e4 100644 --- a/src/main/java/org/jabref/gui/importer/ImportEntriesViewModel.java +++ b/src/main/java/org/jabref/gui/importer/ImportEntriesViewModel.java @@ -171,7 +171,6 @@ public void importEntries(List entriesToImport, boolean shouldDownload private void buildImportHandlerThenImportEntries(List entriesToImport) { ImportHandler importHandler = new ImportHandler( - dialogService, databaseContext, ExternalFileTypes.getInstance(), preferences, diff --git a/src/main/java/org/jabref/gui/importer/UnlinkedFilesCrawler.java b/src/main/java/org/jabref/gui/importer/UnlinkedFilesCrawler.java deleted file mode 100644 index 492656bb540..00000000000 --- a/src/main/java/org/jabref/gui/importer/UnlinkedFilesCrawler.java +++ /dev/null @@ -1,107 +0,0 @@ -package org.jabref.gui.importer; - -import java.io.File; -import java.io.FileFilter; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -import javafx.scene.control.CheckBoxTreeItem; - -import org.jabref.gui.externalfiles.FindUnlinkedFilesDialog.FileNodeWrapper; -import org.jabref.gui.util.BackgroundTask; -import org.jabref.logic.l10n.Localization; -import org.jabref.model.database.BibDatabase; -import org.jabref.model.database.BibDatabaseContext; - -/** - * Util class for searching files on the file system which are not linked to a provided {@link BibDatabase}. - */ -public class UnlinkedFilesCrawler extends BackgroundTask> { - - private final Path directory; - private final FileFilter fileFilter; - private int counter; - private final BibDatabaseContext databaseContext; - - public UnlinkedFilesCrawler(Path directory, FileFilter fileFilter, BibDatabaseContext databaseContext) { - this.directory = directory; - this.fileFilter = fileFilter; - this.databaseContext = databaseContext; - } - - @Override - protected CheckBoxTreeItem call() { - UnlinkedPDFFileFilter unlinkedPDFFileFilter = new UnlinkedPDFFileFilter(fileFilter, databaseContext); - return searchDirectory(directory.toFile(), unlinkedPDFFileFilter); - } - - /** - * Searches recursively all files in the specified directory.
- *
- * All files matched by the given {@link UnlinkedPDFFileFilter} are taken into the resulting tree.
- *
- * The result will be a tree structure of nodes of the type - * {@link CheckBoxTreeItem}.
- *
- * The user objects that are attached to the nodes is the - * {@link FileNodeWrapper}, which wraps the {@link File}-Object.
- *
- * For ensuring the capability to cancel the work of this recursive method, - * the first position in the integer array 'state' must be set to 1, to keep - * the recursion running. When the states value changes, the method will - * resolve its recursion and return what it has saved so far. - */ - private CheckBoxTreeItem searchDirectory(File directory, UnlinkedPDFFileFilter ff) { - // Return null if the directory is not valid. - if ((directory == null) || !directory.exists() || !directory.isDirectory()) { - return null; - } - - File[] filesArray = directory.listFiles(ff); - List files; - if (filesArray == null) { - files = Collections.emptyList(); - } else { - files = Arrays.asList(filesArray); - } - CheckBoxTreeItem root = new CheckBoxTreeItem<>(new FileNodeWrapper(directory.toPath(), 0)); - - int filesCount = 0; - - filesArray = directory.listFiles(pathname -> (pathname != null) && pathname.isDirectory()); - List subDirectories; - if (filesArray == null) { - subDirectories = Collections.emptyList(); - } else { - subDirectories = Arrays.asList(filesArray); - } - for (File subDirectory : subDirectories) { - if (isCanceled()) { - return root; - } - - CheckBoxTreeItem subRoot = searchDirectory(subDirectory, ff); - if ((subRoot != null) && (!subRoot.getChildren().isEmpty())) { - filesCount += subRoot.getValue().fileCount; - root.getChildren().add(subRoot); - } - } - - root.setValue(new FileNodeWrapper(directory.toPath(), files.size() + filesCount)); - - for (File file : files) { - root.getChildren().add(new CheckBoxTreeItem<>(new FileNodeWrapper(file.toPath()))); - - counter++; - if (counter == 1) { - updateMessage(Localization.lang("One file found")); - } else { - updateMessage(Localization.lang("%0 files found", Integer.toString(counter))); - } - } - - return root; - } -} diff --git a/src/main/java/org/jabref/gui/importer/UnlinkedPDFFileFilter.java b/src/main/java/org/jabref/gui/importer/UnlinkedPDFFileFilter.java deleted file mode 100644 index f374fe5fd1a..00000000000 --- a/src/main/java/org/jabref/gui/importer/UnlinkedPDFFileFilter.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.jabref.gui.importer; - -import java.io.File; -import java.io.FileFilter; - -import org.jabref.gui.Globals; -import org.jabref.logic.util.io.DatabaseFileLookup; -import org.jabref.model.database.BibDatabase; -import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.entry.BibEntry; - -/** - * {@link FileFilter} implementation, that allows only files which are not - * linked in any of the {@link BibEntry}s of the specified - * {@link BibDatabase}.
- *
- * This {@link FileFilter} sits on top of another {@link FileFilter} - * -implementation, which it first consults. Only if this major filefilter - * has accepted a file, this implementation will verify on that file. - */ -public class UnlinkedPDFFileFilter implements FileFilter { - - private final DatabaseFileLookup lookup; - private final FileFilter fileFilter; - - public UnlinkedPDFFileFilter(FileFilter fileFilter, BibDatabaseContext databaseContext) { - this.fileFilter = fileFilter; - this.lookup = new DatabaseFileLookup(databaseContext, Globals.prefs.getFilePreferences()); - } - - @Override - public boolean accept(File pathname) { - return fileFilter.accept(pathname) && !lookup.lookupDatabase(pathname); - } -} diff --git a/src/main/java/org/jabref/gui/maintable/MainTable.java b/src/main/java/org/jabref/gui/maintable/MainTable.java index fa236101928..ee11465c2c6 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTable.java +++ b/src/main/java/org/jabref/gui/maintable/MainTable.java @@ -86,7 +86,7 @@ public MainTable(MainTableDataModel model, MainTablePreferences mainTablePreferences = preferencesService.getMainTablePreferences(); importHandler = new ImportHandler( - dialogService, database, externalFileTypes, + database, externalFileTypes, preferencesService, Globals.getFileUpdateMonitor(), undoManager, @@ -137,7 +137,8 @@ public MainTable(MainTableDataModel model, } } */ - mainTablePreferences.getColumnPreferences().getColumnSortOrder().forEach(columnModel -> + + mainTablePreferences.getColumnPreferences().getColumnSortOrder().forEach(columnModel -> this.getColumns().stream() .map(column -> (MainTableColumn) column) .filter(column -> column.getModel().equals(columnModel)) @@ -173,8 +174,8 @@ public MainTable(MainTableDataModel model, } /** - * This is called, if a user starts typing some characters into the keyboard with focus on main table. - * The {@link MainTable} will scroll to the cell with the same starting column value and typed string + * This is called, if a user starts typing some characters into the keyboard with focus on main table. The {@link + * MainTable} will scroll to the cell with the same starting column value and typed string * * @param sortedColumn The sorted column in {@link MainTable} * @param keyEvent The pressed character @@ -354,7 +355,7 @@ private void handleOnDragDropped(TableRow row, BibEntryT // Center -> link files to entry // Depending on the pressed modifier, move/copy/link files to drop target switch (ControlHelper.getDroppingMouseLocation(row, event)) { - case TOP, BOTTOM -> importHandler.importAsNewEntries(files); + case TOP, BOTTOM -> importHandler.importFilesInBackground(files).executeWith(Globals.TASK_EXECUTOR); case CENTER -> { BibEntry entry = target.getEntry(); switch (event.getTransferMode()) { diff --git a/src/main/java/org/jabref/gui/texparser/ParseLatexDialogView.java b/src/main/java/org/jabref/gui/texparser/ParseLatexDialogView.java index ffa941c3bbc..da4cc7f830d 100644 --- a/src/main/java/org/jabref/gui/texparser/ParseLatexDialogView.java +++ b/src/main/java/org/jabref/gui/texparser/ParseLatexDialogView.java @@ -14,6 +14,7 @@ import org.jabref.gui.DialogService; import org.jabref.gui.util.BaseDialog; import org.jabref.gui.util.ControlHelper; +import org.jabref.gui.util.FileNodeViewModel; import org.jabref.gui.util.IconValidationDecorator; import org.jabref.gui.util.RecursiveTreeItem; import org.jabref.gui.util.TaskExecutor; diff --git a/src/main/java/org/jabref/gui/texparser/ParseLatexDialogViewModel.java b/src/main/java/org/jabref/gui/texparser/ParseLatexDialogViewModel.java index acd279de509..50806523d94 100644 --- a/src/main/java/org/jabref/gui/texparser/ParseLatexDialogViewModel.java +++ b/src/main/java/org/jabref/gui/texparser/ParseLatexDialogViewModel.java @@ -25,6 +25,7 @@ import org.jabref.gui.DialogService; import org.jabref.gui.util.BackgroundTask; import org.jabref.gui.util.DirectoryDialogConfiguration; +import org.jabref.gui.util.FileNodeViewModel; import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.l10n.Localization; import org.jabref.logic.texparser.DefaultLatexParser; @@ -139,7 +140,7 @@ public void searchButtonClicked() { } private void handleFailure(Exception exception) { - final boolean permissionProblem = exception instanceof IOException && exception.getCause() instanceof FileSystemException && exception.getCause().getMessage().endsWith("Operation not permitted"); + final boolean permissionProblem = (exception instanceof IOException) && (exception.getCause() instanceof FileSystemException) && exception.getCause().getMessage().endsWith("Operation not permitted"); if (permissionProblem) { dialogService.showErrorDialogAndWait(String.format(Localization.lang("JabRef does not have permission to access %s"), exception.getCause().getMessage())); } else { @@ -148,7 +149,7 @@ private void handleFailure(Exception exception) { } private FileNodeViewModel searchDirectory(Path directory) throws IOException { - if (directory == null || !directory.toFile().isDirectory()) { + if ((directory == null) || !directory.toFile().isDirectory()) { throw new IOException(String.format("Invalid directory for searching: %s", directory)); } diff --git a/src/main/java/org/jabref/gui/util/BackgroundTask.java b/src/main/java/org/jabref/gui/util/BackgroundTask.java index 3209fd7f0bd..9c9626354d1 100644 --- a/src/main/java/org/jabref/gui/util/BackgroundTask.java +++ b/src/main/java/org/jabref/gui/util/BackgroundTask.java @@ -53,7 +53,7 @@ public BackgroundTask() { } public static BackgroundTask wrap(Callable callable) { - return new BackgroundTask() { + return new BackgroundTask<>() { @Override protected V call() throws Exception { return callable.call(); @@ -62,7 +62,7 @@ protected V call() throws Exception { } public static BackgroundTask wrap(Runnable runnable) { - return new BackgroundTask() { + return new BackgroundTask<>() { @Override protected Void call() throws Exception { runnable.run(); @@ -194,7 +194,7 @@ public BackgroundTask onFinished(Runnable onFinished) { * @param type of the return value of the second task */ public BackgroundTask then(Function> nextTaskFactory) { - return new BackgroundTask() { + return new BackgroundTask<>() { @Override protected T call() throws Exception { V result = BackgroundTask.this.call(); @@ -212,7 +212,7 @@ protected T call() throws Exception { * @param type of the return value of the second task */ public BackgroundTask thenRun(Function nextOperation) { - return new BackgroundTask() { + return new BackgroundTask<>() { @Override protected T call() throws Exception { V result = BackgroundTask.this.call(); @@ -229,7 +229,7 @@ protected T call() throws Exception { * @param nextOperation the function that performs the next operation */ public BackgroundTask thenRun(Consumer nextOperation) { - return new BackgroundTask() { + return new BackgroundTask<>() { @Override protected Void call() throws Exception { V result = BackgroundTask.this.call(); diff --git a/src/main/java/org/jabref/gui/util/CustomTitledPaneSkin.java b/src/main/java/org/jabref/gui/util/CustomTitledPaneSkin.java new file mode 100644 index 00000000000..c0ea9bdce4a --- /dev/null +++ b/src/main/java/org/jabref/gui/util/CustomTitledPaneSkin.java @@ -0,0 +1,210 @@ +package org.jabref.gui.util; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import javafx.beans.binding.Bindings; +import javafx.beans.binding.DoubleBinding; +import javafx.beans.property.ObjectProperty; +import javafx.beans.property.Property; +import javafx.css.CssMetaData; +import javafx.css.SimpleStyleableObjectProperty; +import javafx.css.StyleableObjectProperty; +import javafx.css.StyleableProperty; +import javafx.scene.Node; +import javafx.scene.control.Skin; +import javafx.scene.control.TitledPane; +import javafx.scene.control.skin.TitledPaneSkin; +import javafx.scene.layout.Region; +import javafx.scene.text.Text; +import javafx.scene.transform.Rotate; + +import static javafx.css.StyleConverter.getEnumConverter; + +/** + * + * CustomTitledPaneSkin with option to move arrow to the right + * https://stackoverflow.com/a/55085777/3450689s + */ +public class CustomTitledPaneSkin extends TitledPaneSkin { + + public enum ArrowSide { + LEFT, RIGHT + } + + /* ******************************************************** + * * + * Properties * + * * + **********************************************************/ + + private final StyleableObjectProperty arrowSide = new SimpleStyleableObjectProperty<>(StyleableProperties.ARROW_SIDE, this, "arrowSide", ArrowSide.LEFT) { + + @Override + protected void invalidated() { + adjustTitleLayout(); + } + }; + + public final void setArrowSide(ArrowSide arrowSide) { + this.arrowSide.set(arrowSide); + } + + public final ArrowSide getArrowSide() { + return arrowSide.get(); + } + + public final ObjectProperty arrowSideProperty() { + return arrowSide; + } + + /* ******************************************************** + * * + * Instance Fields * + * * + **********************************************************/ + + private final Region title; + private final Region arrowButton; + private final Region arrow; + private final Text text; + + private DoubleBinding arrowTranslateBinding; + private DoubleBinding textGraphicTranslateBinding; + private Node graphic; + + /* ******************************************************** + * * + * Constructors * + * * + **********************************************************/ + + public CustomTitledPaneSkin(TitledPane control) { + super(control); + title = (Region) Objects.requireNonNull(control.lookup(".title")); + arrowButton = (Region) Objects.requireNonNull(title.lookup(".arrow-button")); + arrow = (Region) Objects.requireNonNull(arrowButton.lookup(".arrow")); + text = (Text) Objects.requireNonNull(title.lookup(".text")); + + // based on https://stackoverflow.com/a/55156460/3450689 + Rotate rotate = new Rotate(); + rotate.pivotXProperty().bind(arrow.widthProperty().divide(2.0)); + rotate.pivotYProperty().bind(arrow.heightProperty().divide(2.0)); + rotate.angleProperty().bind( + Bindings.when(control.expandedProperty()) + .then(-180.0) + .otherwise(90.0)); + + arrow.getTransforms().add(rotate); + + registerChangeListener(control.graphicProperty(), ov -> adjustTitleLayout()); + } + + /* ******************************************************** + * * + * Skin Stuff * + * * + **********************************************************/ + + private void adjustTitleLayout() { + clearBindings(); + if (getArrowSide() != ArrowSide.RIGHT) { + // if arrow is on the left we don't need to translate anything + return; + } + + arrowTranslateBinding = Bindings.createDoubleBinding(() -> { + double rightInset = title.getPadding().getRight(); + return title.getWidth() - arrowButton.getLayoutX() - arrowButton.getWidth() - rightInset; + }, title.paddingProperty(), title.widthProperty(), arrowButton.widthProperty(), arrowButton.layoutXProperty()); + arrowButton.translateXProperty().bind(arrowTranslateBinding); + + textGraphicTranslateBinding = Bindings.createDoubleBinding( + () -> switch (getSkinnable().getAlignment()) { + case TOP_CENTER, CENTER, BOTTOM_CENTER, BASELINE_CENTER -> 0.0; + default -> -(arrowButton.getWidth()); + }, getSkinnable().alignmentProperty(), arrowButton.widthProperty()); + text.translateXProperty().bind(textGraphicTranslateBinding); + + graphic = getSkinnable().getGraphic(); + if (graphic != null) { + graphic.translateXProperty().bind(textGraphicTranslateBinding); + } + } + + private void clearBindings() { + if (arrowTranslateBinding != null) { + arrowButton.translateXProperty().unbind(); + arrowButton.setTranslateX(0); + arrowTranslateBinding.dispose(); + arrowTranslateBinding = null; + } + if (textGraphicTranslateBinding != null) { + text.translateXProperty().unbind(); + text.setTranslateX(0); + if (graphic != null) { + graphic.translateXProperty().unbind(); + graphic.setTranslateX(0); + graphic = null; + } + textGraphicTranslateBinding.dispose(); + textGraphicTranslateBinding = null; + } + } + + @Override + public void dispose() { + clearBindings(); + unregisterChangeListeners(getSkinnable().graphicProperty()); + super.dispose(); + } + + /* ******************************************************** + * * + * Stylesheet Handling * + * * + **********************************************************/ + + public static List> getClassCssMetaData() { + return StyleableProperties.CSS_META_DATA; + } + + @Override + public List> getCssMetaData() { + return getClassCssMetaData(); + } + + private static class StyleableProperties { + + private static final CssMetaData ARROW_SIDE = new CssMetaData<>("-fx-arrow-side", getEnumConverter(ArrowSide.class), ArrowSide.LEFT) { + + @Override + public boolean isSettable(TitledPane styleable) { + Property prop = (Property) getStyleableProperty(styleable); + return (prop != null) && !prop.isBound(); + } + + @Override + public StyleableProperty getStyleableProperty(TitledPane styleable) { + Skin skin = styleable.getSkin(); + if (skin instanceof CustomTitledPaneSkin) { + return ((CustomTitledPaneSkin) skin).arrowSide; + } + return null; + } + + }; + + private static final List> CSS_META_DATA; + + static { + List> list = new ArrayList<>(TitledPane.getClassCssMetaData().size() + 1); + list.addAll(TitledPaneSkin.getClassCssMetaData()); + list.add(ARROW_SIDE); + CSS_META_DATA = Collections.unmodifiableList(list); + } + + } +} diff --git a/src/main/java/org/jabref/gui/util/FileFilterConverter.java b/src/main/java/org/jabref/gui/util/FileFilterConverter.java index c42999e4a9e..623fcb54cb2 100644 --- a/src/main/java/org/jabref/gui/util/FileFilterConverter.java +++ b/src/main/java/org/jabref/gui/util/FileFilterConverter.java @@ -1,6 +1,9 @@ package org.jabref.gui.util; import java.io.FileFilter; +import java.io.IOException; +import java.nio.file.DirectoryStream.Filter; +import java.nio.file.Path; import java.util.Collection; import java.util.List; import java.util.Optional; @@ -62,16 +65,30 @@ public static List exporterToExtensionFilter(Collec } public static FileFilter toFileFilter(FileChooser.ExtensionFilter extensionFilter) { - List extensionsCleaned = extensionFilter.getExtensions() - .stream() - .map(extension -> extension.replace(".", "").replace("*", "")) - .filter(StringUtil::isNotBlank) - .collect(Collectors.toList()); + return toFileFilter(extensionFilter.getExtensions()); + } + + public static FileFilter toFileFilter(List extensions) { + var filter = toDirFilter(extensions); + return file -> { + try { + return filter.accept(file.toPath()); + } catch (IOException e) { + return false; + } + }; + } + + public static Filter toDirFilter(List extensions) { + List extensionsCleaned = extensions.stream() + .map(extension -> extension.replace(".", "").replace("*", "")) + .filter(StringUtil::isNotBlank) + .collect(Collectors.toList()); if (extensionsCleaned.isEmpty()) { // Except every file - return pathname -> true; + return path -> true; } else { - return pathname -> FileUtil.getFileExtension(pathname.toPath()) + return path -> FileUtil.getFileExtension(path) .map(extensionsCleaned::contains) .orElse(false); } diff --git a/src/main/java/org/jabref/gui/texparser/FileNodeViewModel.java b/src/main/java/org/jabref/gui/util/FileNodeViewModel.java similarity index 97% rename from src/main/java/org/jabref/gui/texparser/FileNodeViewModel.java rename to src/main/java/org/jabref/gui/util/FileNodeViewModel.java index 2ce06b0948b..5a000996ba4 100644 --- a/src/main/java/org/jabref/gui/texparser/FileNodeViewModel.java +++ b/src/main/java/org/jabref/gui/util/FileNodeViewModel.java @@ -1,4 +1,4 @@ -package org.jabref.gui.texparser; +package org.jabref.gui.util; import java.nio.file.Path; diff --git a/src/main/java/org/jabref/logic/externalfiles/ExternalFilesContentImporter.java b/src/main/java/org/jabref/logic/externalfiles/ExternalFilesContentImporter.java index fcafa9f1e64..da9a88d7c62 100644 --- a/src/main/java/org/jabref/logic/externalfiles/ExternalFilesContentImporter.java +++ b/src/main/java/org/jabref/logic/externalfiles/ExternalFilesContentImporter.java @@ -1,8 +1,8 @@ package org.jabref.logic.externalfiles; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Path; -import java.util.List; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.OpenDatabase; @@ -10,7 +10,6 @@ import org.jabref.logic.importer.fileformat.PdfContentImporter; import org.jabref.logic.importer.fileformat.PdfXmpImporter; import org.jabref.logic.preferences.TimestampPreferences; -import org.jabref.model.entry.BibEntry; import org.jabref.model.util.FileUpdateMonitor; public class ExternalFilesContentImporter { @@ -23,16 +22,15 @@ public ExternalFilesContentImporter(ImportFormatPreferences importFormatPreferen this.timestampPreferences = timestampPreferences; } - public List importPDFContent(Path file) { - return new PdfContentImporter(importFormatPreferences).importDatabase(file, StandardCharsets.UTF_8).getDatabase().getEntries(); + public ParserResult importPDFContent(Path file) { + return new PdfContentImporter(importFormatPreferences).importDatabase(file, StandardCharsets.UTF_8); } - public List importXMPContent(Path file) { - return new PdfXmpImporter(importFormatPreferences.getXmpPreferences()).importDatabase(file, StandardCharsets.UTF_8).getDatabase().getEntries(); + public ParserResult importXMPContent(Path file) { + return new PdfXmpImporter(importFormatPreferences.getXmpPreferences()).importDatabase(file, StandardCharsets.UTF_8); } - public List importFromBibFile(Path bibFile, FileUpdateMonitor fileUpdateMonitor) { - ParserResult parserResult = OpenDatabase.loadDatabase(bibFile.toString(), importFormatPreferences, timestampPreferences, fileUpdateMonitor); - return parserResult.getDatabaseContext().getEntries(); + public ParserResult importFromBibFile(Path bibFile, FileUpdateMonitor fileUpdateMonitor) throws IOException { + return OpenDatabase.loadDatabase(bibFile, importFormatPreferences, timestampPreferences, fileUpdateMonitor); } } diff --git a/src/main/java/org/jabref/logic/importer/fileformat/PdfContentImporter.java b/src/main/java/org/jabref/logic/importer/fileformat/PdfContentImporter.java index a9d62a59181..c7589cce571 100644 --- a/src/main/java/org/jabref/logic/importer/fileformat/PdfContentImporter.java +++ b/src/main/java/org/jabref/logic/importer/fileformat/PdfContentImporter.java @@ -220,10 +220,8 @@ public ParserResult importDatabase(Path filePath, Charset defaultEncoding) { entry.ifPresent(result::add); } catch (EncryptedPdfsNotSupportedException e) { return ParserResult.fromErrorMessage(Localization.lang("Decryption not supported.")); - } catch (IOException exception) { + } catch (IOException | FetcherException exception) { return ParserResult.fromError(exception); - } catch (FetcherException e) { - return ParserResult.fromErrorMessage(e.getMessage()); } result.forEach(entry -> entry.addFile(new LinkedFile("", filePath.toAbsolutePath(), "PDF"))); diff --git a/src/main/java/org/jabref/logic/l10n/Localization.java b/src/main/java/org/jabref/logic/l10n/Localization.java index dcc612a9e20..ec66463e0ef 100644 --- a/src/main/java/org/jabref/logic/l10n/Localization.java +++ b/src/main/java/org/jabref/logic/l10n/Localization.java @@ -1,6 +1,7 @@ package org.jabref.logic.l10n; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; @@ -48,13 +49,15 @@ private Localization() { * @param params Replacement strings for parameters %0, %1, etc. * @return The message with replaced parameters */ - public static String lang(String key, String... params) { + public static String lang(String key, Object... params) { if (localizedMessages == null) { // I'm logging this because it should never happen LOGGER.error("Messages are not initialized before accessing key: {}", key); setLanguage(Language.ENGLISH); } - return lookup(localizedMessages, key, params); + var stringParams = Arrays.stream(params).map(Object::toString).toArray(String[]::new); + + return lookup(localizedMessages, key, stringParams); } /** diff --git a/src/main/java/org/jabref/logic/util/StandardFileType.java b/src/main/java/org/jabref/logic/util/StandardFileType.java index 64cbff44d41..3ad3d1d7e3b 100644 --- a/src/main/java/org/jabref/logic/util/StandardFileType.java +++ b/src/main/java/org/jabref/logic/util/StandardFileType.java @@ -42,7 +42,8 @@ public enum StandardFileType implements FileType { XMP("xmp"), ZIP("zip"), CSS("css"), - YAML("yaml"); + YAML("yaml"), + ANY_FILE("*"); private final List extensions; diff --git a/src/main/java/org/jabref/logic/util/io/FileUtil.java b/src/main/java/org/jabref/logic/util/io/FileUtil.java index d31752d3203..92695f06fe7 100644 --- a/src/main/java/org/jabref/logic/util/io/FileUtil.java +++ b/src/main/java/org/jabref/logic/util/io/FileUtil.java @@ -345,4 +345,14 @@ public static String toPortableString(Path path) { public static boolean isBibFile(Path file) { return getFileExtension(file).filter(type -> "bib".equals(type)).isPresent(); } + + /** + * Test if the file is a bib file by simply checking the extension to be ".bib" + * + * @param file The file to check + * @return True if file extension is ".bib", false otherwise + */ + public static boolean isPDFFile(Path file) { + return getFileExtension(file).filter(type -> "pdf".equals(type)).isPresent(); + } } diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 442885150c7..8cfb645b5fb 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -1107,13 +1107,12 @@ Unable\ to\ clear\ preferences.=Unable to clear preferences. Unselect\ all=Unselect all Expand\ all=Expand all Collapse\ all=Collapse all -Opens\ the\ file\ browser.=Opens the file browser. Scan\ directory=Scan directory Searches\ the\ selected\ directory\ for\ unlinked\ files.=Searches the selected directory for unlinked files. Starts\ the\ import\ of\ BibTeX\ entries.=Starts the import of BibTeX entries. -Select\ a\ directory\ where\ the\ search\ shall\ start.=Select a directory where the search shall start. -Select\ file\ type\:=Select file type: -These\ files\ are\ not\ linked\ in\ the\ active\ library.=These files are not linked in the active library. +Start\ directory\:=Start directory: +Currently\ unlinked\ files=Currently unlinked files +Import\ result=Import result Searching\ file\ system...=Searching file system... Citation\ key\ patterns=Citation key patterns Clear\ priority=Clear priority @@ -1641,8 +1640,6 @@ Open\ OpenOffice/LibreOffice\ connection=Open OpenOffice/LibreOffice connection You\ must\ enter\ at\ least\ one\ field\ name=You must enter at least one field name Non-ASCII\ encoded\ character\ found=Non-ASCII encoded character found Toggle\ web\ search\ interface=Toggle web search interface -%0\ files\ found=%0 files found -One\ file\ found=One file found Migration\ help\ information=Migration help information Entered\ database\ has\ obsolete\ structure\ and\ is\ no\ longer\ supported.=Entered database has obsolete structure and is no longer supported. @@ -2280,6 +2277,17 @@ Regular\ expression=Regular expression Error\ importing.\ See\ the\ error\ log\ for\ details.=Error importing. See the error log for details. +Error\ from\ import\:\ %0=Error from import\: %0 +Error\ reading\ PDF\ content\:\ %0=Error reading PDF content\: %0 +Error\ reading\ XMP\ content\:\ %0=Error reading XMP content\: %0 +Importing\ bib\ entry=Importing bib entry +Importing\ using\ XMP\ data...=Importing using XMP data... +Importing\ using\ extracted\ PDF\ data=Importing using extracted PDF data +No\ BibTeX\ data\ found.\ Creating\ empty\ entry\ with\ file\ link=No BibTeX data found. Creating empty entry with file link +No\ metadata\ found.\ Creating\ empty\ entry\ with\ file\ link=No metadata found. Creating empty entry with file link +Processing\ file\ %0=Processing file %0 +Export\ selected=Export selected + Unprotect\ terms=Unprotect terms Error\ connecting\ to\ Writer\ document=Error connecting to Writer document You\ need\ to\ open\ Writer\ with\ a\ document\ before\ connecting=You need to open Writer with a document before connecting From 7672b2de95aea95b92d47cde3215ffb9f96aadcb Mon Sep 17 00:00:00 2001 From: Jonatan Asketorp <2598631+k3KAW8Pnf7mkmdSMPHz27@users.noreply.github.com> Date: Sun, 24 Jan 2021 14:53:44 -0500 Subject: [PATCH 03/25] Fix expansion of bracketed expressions in RegExpBasedFileFinder (#7338) --- CHANGELOG.md | 1 + .../citationkeypattern/BracketedPattern.java | 2 +- .../logic/util/io/RegExpBasedFileFinder.java | 78 ++++----- .../BracketedPatternTest.java | 40 +++++ .../util/io/RegExpBasedFileFinderTests.java | 148 +++++++++--------- .../subdirectory/2003_Hippel_209.pdf | Bin .../2017_Gra\305\276ulis_726.pdf" | Bin .../subdirectory/pdfInSubdirectory.pdf | Bin 5 -> 0 bytes 8 files changed, 157 insertions(+), 112 deletions(-) delete mode 100644 src/test/resources/org/jabref/logic/importer/unlinkedFilesTestFolder/directory/subdirectory/2003_Hippel_209.pdf delete mode 100644 "src/test/resources/org/jabref/logic/importer/unlinkedFilesTestFolder/directory/subdirectory/2017_Gra\305\276ulis_726.pdf" delete mode 100644 src/test/resources/org/jabref/logic/importer/unlinkedFilesTestFolder/directory/subdirectory/pdfInSubdirectory.pdf diff --git a/CHANGELOG.md b/CHANGELOG.md index 2316c209889..3196eb56b50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve - We fixed an issue where the "Find unlinked files" dialog would freeze JabRef on importing. [#7205](https://github.com/JabRef/jabref/issues/7205) - We fixed an issue where the "Find unlinked files" would stop importing when importing a single file failed. [#7206](https://github.com/JabRef/jabref/issues/7206) - We fixed an issue where an exception would be displayed for previewing and preferences when a custom theme has been configured but is missing [#7177](https://github.com/JabRef/jabref/issues/7177) +- We fixed an issue where the regex based file search miss-interpreted specific symbols [#4342](https://github.com/JabRef/jabref/issues/4342) - We fixed an issue where the Harvard RTF exporter used the wrong default file extension. [4508](https://github.com/JabRef/jabref/issues/4508) - We fixed an issue where the Harvard RTF exporter did not use the new authors formatter and therefore did not export "organization" authors correctly. [4508](https://github.com/JabRef/jabref/issues/4508) - We fixed an issue where the field `urldate` was not exported to the corresponding fields `YearAccessed`, `MonthAccessed`, `DayAccessed` in MS Office XML [#7354](https://github.com/JabRef/jabref/issues/7354) diff --git a/src/main/java/org/jabref/logic/citationkeypattern/BracketedPattern.java b/src/main/java/org/jabref/logic/citationkeypattern/BracketedPattern.java index 0e5a7d4a518..b4d4c203be1 100644 --- a/src/main/java/org/jabref/logic/citationkeypattern/BracketedPattern.java +++ b/src/main/java/org/jabref/logic/citationkeypattern/BracketedPattern.java @@ -187,7 +187,7 @@ public static String expandBrackets(String pattern, Character keywordDelimiter, * @param database The {@link BibDatabase} for field resolving. May be null. * @return a function accepting a bracketed expression and returning the result of expanding it */ - private static Function expandBracketContent(Character keywordDelimiter, BibEntry entry, BibDatabase database) { + public static Function expandBracketContent(Character keywordDelimiter, BibEntry entry, BibDatabase database) { return (String bracket) -> { String expandedPattern; List fieldParts = parseFieldAndModifiers(bracket); diff --git a/src/main/java/org/jabref/logic/util/io/RegExpBasedFileFinder.java b/src/main/java/org/jabref/logic/util/io/RegExpBasedFileFinder.java index cdb2d9a03f9..01bcbbf0ba8 100644 --- a/src/main/java/org/jabref/logic/util/io/RegExpBasedFileFinder.java +++ b/src/main/java/org/jabref/logic/util/io/RegExpBasedFileFinder.java @@ -8,9 +8,9 @@ import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.function.BiPredicate; +import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; @@ -18,7 +18,6 @@ import java.util.stream.Stream; import org.jabref.logic.citationkeypattern.BracketedPattern; -import org.jabref.model.database.BibDatabase; import org.jabref.model.entry.BibEntry; import org.jabref.model.strings.StringUtil; @@ -28,7 +27,6 @@ class RegExpBasedFileFinder implements FileFinder { private static final Pattern ESCAPE_PATTERN = Pattern.compile("([^\\\\])\\\\([^\\\\])"); - private static final Pattern SQUARE_BRACKETS_PATTERN = Pattern.compile("\\[.*?\\]"); private final String regExp; private final Character keywordDelimiter; @@ -41,21 +39,41 @@ class RegExpBasedFileFinder implements FileFinder { } /** - * Takes a string that contains bracketed expression and expands each of these using getFieldAndFormat. - *

- * Unknown Bracket expressions are silently dropped. + * Creates a Pattern that matches the file name corresponding to the last element of {@code fileParts} with any bracketed patterns expanded. + * + * @throws IOException throws an IOException if a PatternSyntaxException occurs */ - public static String expandBrackets(String bracketString, BibEntry entry, BibDatabase database, - Character keywordDelimiter) { - Matcher matcher = SQUARE_BRACKETS_PATTERN.matcher(bracketString); - StringBuilder expandedStringBuffer = new StringBuilder(); - while (matcher.find()) { - String replacement = BracketedPattern.expandBrackets(matcher.group(), keywordDelimiter, entry, database); - matcher.appendReplacement(expandedStringBuffer, replacement); + private Pattern createFileNamePattern(String[] fileParts, String extensionRegExp, BibEntry entry) throws IOException { + // Protect the extension marker so that it isn't treated as a bracketed pattern + String filePart = fileParts[fileParts.length - 1].replace("[extension]", EXT_MARKER); + + // We need to supply a custom function to deal with the content of a bracketed expression and expandBracketContent is the default function + Function expandBracket = BracketedPattern.expandBracketContent(keywordDelimiter, entry, null); + // but, we want to post-process the expanded content so that it can be used as a regex for finding a file name + Function bracketToFileNameRegex = expandBracket.andThen(RegExpBasedFileFinder::toFileNameRegex); + + String expandedBracketAsFileNameRegex = BracketedPattern.expandBrackets(filePart, bracketToFileNameRegex); + + String fileNamePattern = expandedBracketAsFileNameRegex + .replaceAll(EXT_MARKER, extensionRegExp) // Replace the extension marker + .replaceAll("\\\\\\\\", "\\\\"); + try { + return Pattern.compile('^' + fileNamePattern + '$', Pattern.CASE_INSENSITIVE); + } catch (PatternSyntaxException e) { + throw new IOException(String.format("There is a syntax error in the regular expression %s used to search for files", fileNamePattern), e); } - matcher.appendTail(expandedStringBuffer); + } - return expandedStringBuffer.toString(); + /** + * Helper method for both exact matching (if the file name were not created by JabRef) and cleaned file name matching. + * + * @param expandedContent the expanded content of a bracketed expression + * @return a String representation of a regex matching the expanded content and the expanded content cleaned for file name use + */ + private static String toFileNameRegex(String expandedContent) { + var cleanedContent = FileNameCleaner.cleanFileName(expandedContent); + return expandedContent.equals(cleanedContent) ? Pattern.quote(expandedContent) : + "(" + Pattern.quote(expandedContent) + ")|(" + Pattern.quote(cleanedContent) + ")"; } /** @@ -142,9 +160,7 @@ private List findFile(final BibEntry entry, final Path directory, final St } for (int index = 0; index < (fileParts.length - 1); index++) { - String dirToProcess = fileParts[index]; - dirToProcess = expandBrackets(dirToProcess, entry, null, keywordDelimiter); if (dirToProcess.matches("^.:$")) { // Windows Drive Letter actualDirectory = Path.of(dirToProcess + '/'); @@ -179,33 +195,21 @@ private List findFile(final BibEntry entry, final Path directory, final St resultFiles.addAll(findFile(entry, path, restOfFileString, extensionRegExp)); } } catch (UncheckedIOException ioe) { - throw new IOException(ioe); + throw ioe.getCause(); } } // End process directory information } // Last step: check if the given file can be found in this directory - String filePart = fileParts[fileParts.length - 1].replace("[extension]", EXT_MARKER); - String filenameToLookFor = expandBrackets(filePart, entry, null, keywordDelimiter).replaceAll(EXT_MARKER, extensionRegExp); - - try { - final Pattern toMatch = Pattern.compile('^' + filenameToLookFor.replaceAll("\\\\\\\\", "\\\\") + '$', - Pattern.CASE_INSENSITIVE); - BiPredicate matcher = (path, attributes) -> toMatch.matcher(path.getFileName().toString()).matches(); - resultFiles.addAll(collectFilesWithMatcher(actualDirectory, matcher)); - } catch (UncheckedIOException | PatternSyntaxException e) { - throw new IOException("Could not look for " + filenameToLookFor, e); - } - - return resultFiles; - } - - private List collectFilesWithMatcher(Path actualDirectory, BiPredicate matcher) { + Pattern toMatch = createFileNamePattern(fileParts, extensionRegExp, entry); + BiPredicate matcher = (path, attributes) -> toMatch.matcher(path.getFileName().toString()).matches(); try (Stream pathStream = Files.find(actualDirectory, 1, matcher, FileVisitOption.FOLLOW_LINKS)) { - return pathStream.collect(Collectors.toList()); - } catch (UncheckedIOException | IOException ioe) { - return Collections.emptyList(); + resultFiles.addAll(pathStream.collect(Collectors.toList())); + } catch (UncheckedIOException uncheckedIOException) { + // Previously, an empty list were returned here on both IOException and UncheckedIOException + throw uncheckedIOException.getCause(); } + return resultFiles; } private boolean isSubDirectory(Path rootDirectory, Path path) { diff --git a/src/test/java/org/jabref/logic/citationkeypattern/BracketedPatternTest.java b/src/test/java/org/jabref/logic/citationkeypattern/BracketedPatternTest.java index bbc0e6eb3ea..16b0ebe59f8 100644 --- a/src/test/java/org/jabref/logic/citationkeypattern/BracketedPatternTest.java +++ b/src/test/java/org/jabref/logic/citationkeypattern/BracketedPatternTest.java @@ -324,4 +324,44 @@ void expandBracketsLastNameWithChineseCharacters() { assertEquals("æ¨ç§€ç¾¤", BracketedPattern.expandBrackets("[auth]", null, bibEntry, null)); } + + @Test + void expandBracketsWithTestCasesFromRegExpBasedFileFinder() { + BibEntry entry = new BibEntry(StandardEntryType.Article).withCitationKey("HipKro03"); + entry.setField(StandardField.AUTHOR, "Eric von Hippel and Georg von Krogh"); + entry.setField(StandardField.TITLE, "Open Source Software and the \"Private-Collective\" Innovation Model: Issues for Organization Science"); + entry.setField(StandardField.JOURNAL, "Organization Science"); + entry.setField(StandardField.YEAR, "2003"); + entry.setField(StandardField.VOLUME, "14"); + entry.setField(StandardField.PAGES, "209--223"); + entry.setField(StandardField.NUMBER, "2"); + entry.setField(StandardField.ADDRESS, "Institute for Operations Research and the Management Sciences (INFORMS), Linthicum, Maryland, USA"); + entry.setField(StandardField.DOI, "http://dx.doi.org/10.1287/orsc.14.2.209.14992"); + entry.setField(StandardField.ISSN, "1526-5455"); + entry.setField(StandardField.PUBLISHER, "INFORMS"); + + BibDatabase database = new BibDatabase(); + database.insertEntry(entry); + + assertEquals("", BracketedPattern.expandBrackets("", ',', entry, database)); + + assertEquals("dropped", BracketedPattern.expandBrackets("drop[unknownkey]ped", ',', entry, database)); + + assertEquals("Eric von Hippel and Georg von Krogh", + BracketedPattern.expandBrackets("[author]", ',', entry, database)); + + assertEquals("Eric von Hippel and Georg von Krogh are two famous authors.", + BracketedPattern.expandBrackets("[author] are two famous authors.", ',', entry, database)); + + assertEquals("Eric von Hippel and Georg von Krogh are two famous authors.", + BracketedPattern.expandBrackets("[author] are two famous authors.", ',', entry, database)); + + assertEquals( + "Eric von Hippel and Georg von Krogh have published Open Source Software and the \"Private-Collective\" Innovation Model: Issues for Organization Science in Organization Science.", + BracketedPattern.expandBrackets("[author] have published [fulltitle] in [journal].", ',', entry, database)); + + assertEquals( + "Eric von Hippel and Georg von Krogh have published Open Source Software and the \"Private Collective\" Innovation Model: Issues for Organization Science in Organization Science.", + BracketedPattern.expandBrackets("[author] have published [title] in [journal].", ',', entry, database)); + } } diff --git a/src/test/java/org/jabref/logic/util/io/RegExpBasedFileFinderTests.java b/src/test/java/org/jabref/logic/util/io/RegExpBasedFileFinderTests.java index 0cc53fdbc37..0c776e52fdd 100644 --- a/src/test/java/org/jabref/logic/util/io/RegExpBasedFileFinderTests.java +++ b/src/test/java/org/jabref/logic/util/io/RegExpBasedFileFinderTests.java @@ -5,26 +5,33 @@ import java.util.Collections; import java.util.List; -import org.jabref.model.database.BibDatabase; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; class RegExpBasedFileFinderTests { - - private static final String FILES_DIRECTORY = "src/test/resources/org/jabref/logic/importer/unlinkedFilesTestFolder"; - private BibDatabase database; + private static final List PDF_EXTENSION = Collections.singletonList("pdf"); + private static final List FILE_NAMES = List.of( + "ACM_IEEE-CS.pdf", + "pdfInDatabase.pdf", + "Regexp from [A-Z].pdf", + "directory/subdirectory/2003_Hippel_209.pdf", + "directory/subdirectory/2017_Gražulis_726.pdf", + "directory/subdirectory/pdfInSubdirectory.pdf", + "directory/subdirectory/GUO ea - INORG CHEM COMMUN 2010 - Ferroelectric Metal Organic Framework (MOF).pdf" + ); + private Path directory; private BibEntry entry; @BeforeEach - void setUp() { - + void setUp(@TempDir Path tempDir) throws Exception { entry = new BibEntry(); entry.setType(StandardEntryType.Article); entry.setCitationKey("HipKro03"); @@ -40,69 +47,98 @@ void setUp() { entry.setField(StandardField.ISSN, "1526-5455"); entry.setField(StandardField.PUBLISHER, "INFORMS"); - database = new BibDatabase(); - database.insertEntry(entry); + // Create default directories and files + directory = tempDir; + Files.createDirectories(directory.resolve("directory/subdirectory")); + for (String fileName : FILE_NAMES) { + Files.createFile(directory.resolve(fileName)); + } } @Test void testFindFiles() throws Exception { // given - BibEntry localEntry = new BibEntry(StandardEntryType.Article); - localEntry.setCitationKey("pdfInDatabase"); - localEntry.setField(StandardField.YEAR, "2001"); - - List extensions = Collections.singletonList("pdf"); + BibEntry localEntry = new BibEntry(StandardEntryType.Article).withCitationKey("pdfInDatabase"); - List dirs = Collections.singletonList(Path.of(FILES_DIRECTORY)); RegExpBasedFileFinder fileFinder = new RegExpBasedFileFinder("**/[citationkey].*\\\\.[extension]", ','); // when - List result = fileFinder.findAssociatedFiles(localEntry, dirs, extensions); + List result = fileFinder.findAssociatedFiles(localEntry, List.of(directory), PDF_EXTENSION); + List expected = List.of(directory.resolve("pdfInDatabase.pdf")); // then - assertEquals(Collections.singletonList(Path.of("src/test/resources/org/jabref/logic/importer/unlinkedFilesTestFolder/pdfInDatabase.pdf")), - result); + assertEquals(expected, result); } @Test void testYearAuthFirstPageFindFiles() throws Exception { // given - List extensions = Collections.singletonList("pdf"); - - List dirs = Collections.singletonList(Path.of(FILES_DIRECTORY)); RegExpBasedFileFinder fileFinder = new RegExpBasedFileFinder("**/[year]_[auth]_[firstpage].*\\\\.[extension]", ','); // when - List result = fileFinder.findAssociatedFiles(entry, dirs, extensions); + List result = fileFinder.findAssociatedFiles(entry, List.of(directory), PDF_EXTENSION); + List expected = List.of(directory.resolve("directory/subdirectory/2003_Hippel_209.pdf")); // then - assertEquals(Collections.singletonList(Path.of("src/test/resources/org/jabref/logic/importer/unlinkedFilesTestFolder/directory/subdirectory/2003_Hippel_209.pdf")), - result); + assertEquals(expected, result); + } + + @Test + void findAssociatedFilesFindFileContainingBracketsFromBracketedExpression() throws Exception { + var bibEntry = new BibEntry().withField(StandardField.TITLE, "Regexp from [A-Z]"); + + RegExpBasedFileFinder fileFinder = new RegExpBasedFileFinder("[TITLE]\\\\.[extension]", ','); + + List result = fileFinder.findAssociatedFiles(bibEntry, List.of(directory), PDF_EXTENSION); + List pdfFile = List.of(directory.resolve("Regexp from [A-Z].pdf")); + + assertEquals(pdfFile, result); + } + + @Test + void findAssociatedFilesFindCleanedFileFromBracketedExpression() throws Exception { + var bibEntry = new BibEntry().withField(StandardField.JOURNAL, "ACM/IEEE-CS"); + + RegExpBasedFileFinder fileFinder = new RegExpBasedFileFinder("[JOURNAL]\\\\.[extension]", ','); + + List result = fileFinder.findAssociatedFiles(bibEntry, List.of(directory), PDF_EXTENSION); + List pdfFile = List.of(directory.resolve("ACM_IEEE-CS.pdf")); + + assertEquals(pdfFile, result); + } + + @Test + void findAssociatedFilesFindFileContainingParenthesizesFromBracketedExpression() throws Exception { + var bibEntry = new BibEntry().withCitationKey("Guo_ICC_2010") + .withField(StandardField.TITLE, "Ferroelectric Metal Organic Framework (MOF)") + .withField(StandardField.AUTHOR, "Guo, M. and Cai, H.-L. and Xiong, R.-G.") + .withField(StandardField.JOURNAL, "Inorganic Chemistry Communications") + .withField(StandardField.YEAR, "2010"); + + RegExpBasedFileFinder fileFinder = new RegExpBasedFileFinder("**/.*[TITLE].*\\\\.[extension]", ','); + + List result = fileFinder.findAssociatedFiles(bibEntry, List.of(directory), PDF_EXTENSION); + List pdfFile = List.of(directory.resolve("directory/subdirectory/GUO ea - INORG CHEM COMMUN 2010 - Ferroelectric Metal Organic Framework (MOF).pdf")); + + assertEquals(pdfFile, result); } @Test void testAuthorWithDiacritics() throws Exception { // given - BibEntry localEntry = new BibEntry(StandardEntryType.Article); - localEntry.setCitationKey("Grazulis2017"); + BibEntry localEntry = new BibEntry(StandardEntryType.Article).withCitationKey("Grazulis2017"); localEntry.setField(StandardField.YEAR, "2017"); localEntry.setField(StandardField.AUTHOR, "Gražulis, Saulius and O. Kitsune"); localEntry.setField(StandardField.PAGES, "726--729"); - List extensions = Collections.singletonList("pdf"); - - List dirs = Collections.singletonList(Path.of(FILES_DIRECTORY)); RegExpBasedFileFinder fileFinder = new RegExpBasedFileFinder("**/[year]_[auth]_[firstpage]\\\\.[extension]", ','); // when - List result = fileFinder.findAssociatedFiles(localEntry, dirs, extensions); - List expected = Collections.singletonList(Path.of("src/test/resources/org/jabref/logic/importer/unlinkedFilesTestFolder/directory/subdirectory/2017_Gražulis_726.pdf")); + List result = fileFinder.findAssociatedFiles(localEntry, List.of(directory), PDF_EXTENSION); + List expected = List.of(directory.resolve("directory/subdirectory/2017_Gražulis_726.pdf")); // then - assertEquals(expected.size(), result.size()); - for (int i = 0; i < expected.size(); i++) { - assertTrue(Files.isSameFile(expected.get(i), result.get(i))); - } + assertEquals(expected, result); } @Test @@ -112,17 +148,14 @@ void testFindFileInSubdirectory() throws Exception { localEntry.setCitationKey("pdfInSubdirectory"); localEntry.setField(StandardField.YEAR, "2017"); - List extensions = Collections.singletonList("pdf"); - - List dirs = Collections.singletonList(Path.of(FILES_DIRECTORY)); RegExpBasedFileFinder fileFinder = new RegExpBasedFileFinder("**/[citationkey].*\\\\.[extension]", ','); // when - List result = fileFinder.findAssociatedFiles(localEntry, dirs, extensions); + List result = fileFinder.findAssociatedFiles(localEntry, List.of(directory), PDF_EXTENSION); + List expected = List.of(directory.resolve("directory/subdirectory/pdfInSubdirectory.pdf")); // then - assertEquals(Collections.singletonList(Path.of("src/test/resources/org/jabref/logic/importer/unlinkedFilesTestFolder/directory/subdirectory/pdfInSubdirectory.pdf")), - result); + assertEquals(expected, result); } @Test @@ -132,45 +165,12 @@ void testFindFileNonRecursive() throws Exception { localEntry.setCitationKey("pdfInSubdirectory"); localEntry.setField(StandardField.YEAR, "2017"); - List extensions = Collections.singletonList("pdf"); - - List dirs = Collections.singletonList(Path.of(FILES_DIRECTORY)); RegExpBasedFileFinder fileFinder = new RegExpBasedFileFinder("*/[citationkey].*\\\\.[extension]", ','); // when - List result = fileFinder.findAssociatedFiles(localEntry, dirs, extensions); + List result = fileFinder.findAssociatedFiles(localEntry, List.of(directory), PDF_EXTENSION); // then assertTrue(result.isEmpty()); } - - @Test - void testExpandBrackets() { - - assertEquals("", RegExpBasedFileFinder.expandBrackets("", entry, database, ',')); - - assertEquals("dropped", RegExpBasedFileFinder.expandBrackets("drop[unknownkey]ped", entry, database, - ',')); - - assertEquals("Eric von Hippel and Georg von Krogh", - RegExpBasedFileFinder.expandBrackets("[author]", entry, database, ',')); - - assertEquals("Eric von Hippel and Georg von Krogh are two famous authors.", - RegExpBasedFileFinder.expandBrackets("[author] are two famous authors.", entry, database, - ',')); - - assertEquals("Eric von Hippel and Georg von Krogh are two famous authors.", - RegExpBasedFileFinder.expandBrackets("[author] are two famous authors.", entry, database, - ',')); - - assertEquals( - "Eric von Hippel and Georg von Krogh have published Open Source Software and the \"Private-Collective\" Innovation Model: Issues for Organization Science in Organization Science.", - RegExpBasedFileFinder.expandBrackets("[author] have published [fulltitle] in [journal].", entry, database, - ',')); - - assertEquals( - "Eric von Hippel and Georg von Krogh have published Open Source Software and the \"Private Collective\" Innovation Model: Issues for Organization Science in Organization Science.", - RegExpBasedFileFinder.expandBrackets("[author] have published [title] in [journal].", entry, database, - ',')); - } } diff --git a/src/test/resources/org/jabref/logic/importer/unlinkedFilesTestFolder/directory/subdirectory/2003_Hippel_209.pdf b/src/test/resources/org/jabref/logic/importer/unlinkedFilesTestFolder/directory/subdirectory/2003_Hippel_209.pdf deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git "a/src/test/resources/org/jabref/logic/importer/unlinkedFilesTestFolder/directory/subdirectory/2017_Gra\305\276ulis_726.pdf" "b/src/test/resources/org/jabref/logic/importer/unlinkedFilesTestFolder/directory/subdirectory/2017_Gra\305\276ulis_726.pdf" deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/src/test/resources/org/jabref/logic/importer/unlinkedFilesTestFolder/directory/subdirectory/pdfInSubdirectory.pdf b/src/test/resources/org/jabref/logic/importer/unlinkedFilesTestFolder/directory/subdirectory/pdfInSubdirectory.pdf deleted file mode 100644 index 3ac0b7d0dd9994c7ec1c3211b7f4d39211d84753..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5 McmXp_sw_zb00m9~>Hq)$ From ae43548c16ca35b9a83a2d34864557cf503c0df4 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 25 Jan 2021 00:26:28 +0100 Subject: [PATCH 04/25] Fix handling of URL in file field (#7347) * Add workaround of test not working on Windows Co-authored-by: Dominik Voigt * Change input of throws Co-authored-by: Dominik Voigt * Add link as String as constructor parameter * Add CHANGELOG entry * Move tests for FieldFieldParser to FileFieldParserTest (and fix handling of //) * Fix checkstyle Co-authored-by: Dominik Voigt Co-authored-by: Carl Christian Snethlage <50491877+calixtus@users.noreply.github.com> --- CHANGELOG.md | 3 +- .../logic/importer/util/FileFieldParser.java | 53 +++++-- .../org/jabref/model/entry/LinkedFile.java | 17 +- .../logic/bibtex/FileFieldWriterTest.java | 129 +-------------- .../importer/util/FileFieldParserTest.java | 148 ++++++++++++++++++ 5 files changed, 205 insertions(+), 145 deletions(-) create mode 100644 src/test/java/org/jabref/logic/importer/util/FileFieldParserTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 3196eb56b50..826575581e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,8 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve - We fixed an issue where the "Find unlinked files" dialog would freeze JabRef on importing. [#7205](https://github.com/JabRef/jabref/issues/7205) - We fixed an issue where the "Find unlinked files" would stop importing when importing a single file failed. [#7206](https://github.com/JabRef/jabref/issues/7206) - We fixed an issue where an exception would be displayed for previewing and preferences when a custom theme has been configured but is missing [#7177](https://github.com/JabRef/jabref/issues/7177) -- We fixed an issue where the regex based file search miss-interpreted specific symbols [#4342](https://github.com/JabRef/jabref/issues/4342) +- We fixed an issue where URLs in `file` fields could not be handled on Windows. [#7359](https://github.com/JabRef/jabref/issues/7359) +- We fixed an issue where the regex based file search miss-interpreted specific symbols. [#4342](https://github.com/JabRef/jabref/issues/4342) - We fixed an issue where the Harvard RTF exporter used the wrong default file extension. [4508](https://github.com/JabRef/jabref/issues/4508) - We fixed an issue where the Harvard RTF exporter did not use the new authors formatter and therefore did not export "organization" authors correctly. [4508](https://github.com/JabRef/jabref/issues/4508) - We fixed an issue where the field `urldate` was not exported to the corresponding fields `YearAccessed`, `MonthAccessed`, `DayAccessed` in MS Office XML [#7354](https://github.com/JabRef/jabref/issues/7354) diff --git a/src/main/java/org/jabref/logic/importer/util/FileFieldParser.java b/src/main/java/org/jabref/logic/importer/util/FileFieldParser.java index 9c51c7468fc..02632488eed 100644 --- a/src/main/java/org/jabref/logic/importer/util/FileFieldParser.java +++ b/src/main/java/org/jabref/logic/importer/util/FileFieldParser.java @@ -2,6 +2,7 @@ import java.net.MalformedURLException; import java.net.URL; +import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -17,7 +18,7 @@ public static List parse(String value) { return files; } - List entry = new ArrayList<>(); + List linkedFileData = new ArrayList<>(); StringBuilder sb = new StringBuilder(); boolean inXmlChar = false; boolean escaped = false; @@ -39,30 +40,38 @@ public static List parse(String value) { sb.append(c); inXmlChar = false; } else if (!escaped && (c == ':')) { - entry.add(sb.toString()); + // We are in the next LinkedFile data element + linkedFileData.add(sb.toString()); sb = new StringBuilder(); } else if (!escaped && (c == ';') && !inXmlChar) { - entry.add(sb.toString()); - sb = new StringBuilder(); + linkedFileData.add(sb.toString()); + files.add(convert(linkedFileData)); - files.add(convert(entry)); + // next iteration + sb = new StringBuilder(); } else { sb.append(c); } escaped = false; } if (sb.length() > 0) { - entry.add(sb.toString()); + linkedFileData.add(sb.toString()); } - - if (!entry.isEmpty()) { - files.add(convert(entry)); + if (!linkedFileData.isEmpty()) { + files.add(convert(linkedFileData)); } - return files; } - private static LinkedFile convert(List entry) { + /** + * Converts the given textual representation of a LinkedFile object + * + * SIDE EFFECT: The given entry list is cleared upon completion + * + * @param entry the list of elements in the linked file textual representation + * @return a LinkedFile object + */ + static LinkedFile convert(List entry) { // ensure list has at least 3 fields while (entry.size() < 3) { entry.add(""); @@ -71,17 +80,31 @@ private static LinkedFile convert(List entry) { LinkedFile field = null; if (LinkedFile.isOnlineLink(entry.get(1))) { try { - field = new LinkedFile(new URL(entry.get(1)), entry.get(2)); + field = new LinkedFile(entry.get(0), new URL(entry.get(1)), entry.get(2)); } catch (MalformedURLException ignored) { - // ignored + // in case the URL is malformed, store it nevertheless + field = new LinkedFile(entry.get(0), entry.get(1), entry.get(2)); } } if (field == null) { - field = new LinkedFile(entry.get(0), Path.of(entry.get(1)), entry.get(2)); + String pathStr = entry.get(1); + if (pathStr.contains("//")) { + // In case the path contains //, we assume it is a malformed URL, not a malformed path. + // On linux, the double slash would be converted to a single slash. + field = new LinkedFile(entry.get(0), pathStr, entry.get(2)); + } else { + try { + // there is no Path.isValidPath(String) method + Path path = Path.of(pathStr); + field = new LinkedFile(entry.get(0), path, entry.get(2)); + } catch (InvalidPathException e) { + field = new LinkedFile(entry.get(0), pathStr, entry.get(2)); + } + } } - // link is only mandatory field + // link is the only mandatory field if (field.getDescription().isEmpty() && field.getLink().isEmpty() && !field.getFileType().isEmpty()) { field = new LinkedFile("", Path.of(field.getFileType()), ""); } else if (!field.getDescription().isEmpty() && field.getLink().isEmpty() && field.getFileType().isEmpty()) { diff --git a/src/main/java/org/jabref/model/entry/LinkedFile.java b/src/main/java/org/jabref/model/entry/LinkedFile.java index ba98a8ba14e..3e893d79b1a 100644 --- a/src/main/java/org/jabref/model/entry/LinkedFile.java +++ b/src/main/java/org/jabref/model/entry/LinkedFile.java @@ -33,15 +33,24 @@ public class LinkedFile implements Serializable { private transient StringProperty fileType = new SimpleStringProperty(); public LinkedFile(String description, Path link, String fileType) { + this(Objects.requireNonNull(description), Objects.requireNonNull(link).toString(), Objects.requireNonNull(fileType)); + } + + /** + * Constructor for non-valid paths. We need to parse them, because the GUI needs to render it. + */ + public LinkedFile(String description, String link, String fileType) { this.description.setValue(Objects.requireNonNull(description)); - setLink(Objects.requireNonNull(link).toString()); + setLink(link); this.fileType.setValue(Objects.requireNonNull(fileType)); } public LinkedFile(URL link, String fileType) { - this.description.setValue(""); - setLink(Objects.requireNonNull(link).toString()); - this.fileType.setValue(Objects.requireNonNull(fileType)); + this("", Objects.requireNonNull(link).toString(), Objects.requireNonNull(fileType)); + } + + public LinkedFile(String description, URL link, String fileType) { + this(description, Objects.requireNonNull(link).toString(), Objects.requireNonNull(fileType)); } public StringProperty descriptionProperty() { diff --git a/src/test/java/org/jabref/logic/bibtex/FileFieldWriterTest.java b/src/test/java/org/jabref/logic/bibtex/FileFieldWriterTest.java index 7305ebbcfb3..a9c722a3815 100644 --- a/src/test/java/org/jabref/logic/bibtex/FileFieldWriterTest.java +++ b/src/test/java/org/jabref/logic/bibtex/FileFieldWriterTest.java @@ -1,13 +1,7 @@ package org.jabref.logic.bibtex; -import java.net.MalformedURLException; -import java.net.URL; import java.nio.file.Path; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import org.jabref.logic.importer.util.FileFieldParser; import org.jabref.model.entry.LinkedFile; import org.junit.jupiter.api.Test; @@ -17,121 +11,6 @@ public class FileFieldWriterTest { - @Test - public void emptyListForEmptyInput() { - String emptyInput = ""; - - assertEquals(Collections.emptyList(), FileFieldParser.parse(emptyInput)); - assertEquals(Collections.emptyList(), FileFieldParser.parse(null)); - } - - @Test - public void parseCorrectInput() { - String input = "Desc:File.PDF:PDF"; - - assertEquals( - Collections.singletonList(new LinkedFile("Desc", Path.of("File.PDF"), "PDF")), - FileFieldParser.parse(input)); - } - - @Test - public void parseCorrectOnlineInput() throws MalformedURLException { - String input = ":http\\://arxiv.org/pdf/2010.08497v1:PDF"; - String inputURL = "http://arxiv.org/pdf/2010.08497v1"; - List expected = Collections.singletonList(new LinkedFile(new URL(inputURL), "PDF")); - - assertEquals(expected, FileFieldParser.parse(input)); - } - - @Test - public void parseFaultyOnlineInput() { - String input = ":htt\\://arxiv.org/pdf/2010.08497v1:PDF"; - String inputURL = "htt://arxiv.org/pdf/2010.08497v1"; - List expected = Collections.singletonList(new LinkedFile("", Path.of(inputURL), "PDF")); - - assertEquals(expected, FileFieldParser.parse(input)); - } - - @Test - public void ingoreMissingDescription() { - String input = ":wei2005ahp.pdf:PDF"; - - assertEquals( - Collections.singletonList(new LinkedFile("", Path.of("wei2005ahp.pdf"), "PDF")), - FileFieldParser.parse(input)); - } - - @Test - public void interpreteLinkAsOnlyMandatoryField() { - String single = "wei2005ahp.pdf"; - String multiple = "wei2005ahp.pdf;other.pdf"; - - assertEquals( - Collections.singletonList(new LinkedFile("", Path.of("wei2005ahp.pdf"), "")), - FileFieldParser.parse(single)); - - assertEquals( - Arrays.asList( - new LinkedFile("", Path.of("wei2005ahp.pdf"), ""), - new LinkedFile("", Path.of("other.pdf"), "")), - FileFieldParser.parse(multiple)); - } - - @Test - public void escapedCharactersInDescription() { - String input = "test\\:\\;:wei2005ahp.pdf:PDF"; - - assertEquals( - Collections.singletonList(new LinkedFile("test:;", Path.of("wei2005ahp.pdf"), "PDF")), - FileFieldParser.parse(input)); - } - - @Test - public void handleXmlCharacters() { - String input = "test,\\;st\\:\\;:wei2005ahp.pdf:PDF"; - - assertEquals( - Collections.singletonList(new LinkedFile("test,st:;", Path.of("wei2005ahp.pdf"), "PDF")), - FileFieldParser.parse(input)); - } - - @Test - public void handleEscapedFilePath() { - String input = "desc:C\\:\\\\test.pdf:PDF"; - - assertEquals( - Collections.singletonList(new LinkedFile("desc", Path.of("C:\\test.pdf"), "PDF")), - FileFieldParser.parse(input)); - } - - @Test - public void subsetOfFieldsResultsInFileLink() { - String descOnly = "file.pdf::"; - String fileOnly = ":file.pdf"; - String typeOnly = "::file.pdf"; - - assertEquals( - Collections.singletonList(new LinkedFile("", Path.of("file.pdf"), "")), - FileFieldParser.parse(descOnly)); - - assertEquals( - Collections.singletonList(new LinkedFile("", Path.of("file.pdf"), "")), - FileFieldParser.parse(fileOnly)); - - assertEquals( - Collections.singletonList(new LinkedFile("", Path.of("file.pdf"), "")), - FileFieldParser.parse(typeOnly)); - } - - @Test - public void tooManySeparators() { - String input = "desc:file.pdf:PDF:asdf"; - - assertEquals( - Collections.singletonList(new LinkedFile("desc", Path.of("file.pdf"), "PDF")), - FileFieldParser.parse(input)); - } - @Test public void testQuoteStandard() { assertEquals("a", FileFieldWriter.quote("a")); @@ -154,10 +33,10 @@ public void testQuoteNull() { @Test public void testEncodeStringArray() { - assertEquals("a:b;c:d", FileFieldWriter.encodeStringArray(new String[][]{{"a", "b"}, {"c", "d"}})); - assertEquals("a:;c:d", FileFieldWriter.encodeStringArray(new String[][]{{"a", ""}, {"c", "d"}})); - assertEquals("a:" + null + ";c:d", FileFieldWriter.encodeStringArray(new String[][]{{"a", null}, {"c", "d"}})); - assertEquals("a:\\:b;c\\;:d", FileFieldWriter.encodeStringArray(new String[][]{{"a", ":b"}, {"c;", "d"}})); + assertEquals("a:b;c:d", FileFieldWriter.encodeStringArray(new String[][] {{"a", "b"}, {"c", "d"}})); + assertEquals("a:;c:d", FileFieldWriter.encodeStringArray(new String[][] {{"a", ""}, {"c", "d"}})); + assertEquals("a:" + null + ";c:d", FileFieldWriter.encodeStringArray(new String[][] {{"a", null}, {"c", "d"}})); + assertEquals("a:\\:b;c\\;:d", FileFieldWriter.encodeStringArray(new String[][] {{"a", ":b"}, {"c;", "d"}})); } @Test diff --git a/src/test/java/org/jabref/logic/importer/util/FileFieldParserTest.java b/src/test/java/org/jabref/logic/importer/util/FileFieldParserTest.java new file mode 100644 index 00000000000..11c74196d5c --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/util/FileFieldParserTest.java @@ -0,0 +1,148 @@ +package org.jabref.logic.importer.util; + +import java.net.URL; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; + +import org.jabref.model.entry.LinkedFile; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class FileFieldParserTest { + + private static Stream testData() { + return Stream.of( + Arguments.of( + new LinkedFile("arXiv Fulltext PDF", "https://arxiv.org/pdf/1109.0517.pdf", "application/pdf"), + List.of("arXiv Fulltext PDF", "https://arxiv.org/pdf/1109.0517.pdf", "application/pdf") + ), + Arguments.of( + new LinkedFile("arXiv Fulltext PDF", "https/://arxiv.org/pdf/1109.0517.pdf", "application/pdf"), + List.of("arXiv Fulltext PDF", "https\\://arxiv.org/pdf/1109.0517.pdf", "application/pdf") + ) + ); + } + + @ParameterizedTest + @MethodSource("testData") + public void check(LinkedFile expected, List input) { + // we need to convert the unmodifiable list to a modifiable because of the side effect of "convert" + assertEquals(expected, FileFieldParser.convert(new ArrayList<>(input))); + } + + private static Stream stringsToParseTestData() throws Exception { + return Stream.of( + // null string + Arguments.of( + Collections.emptyList(), + null + ), + + // empty string + Arguments.of( + Collections.emptyList(), + "" + ), + + // correct input + Arguments.of( + Collections.singletonList(new LinkedFile("Desc", Path.of("File.PDF"), "PDF")), + "Desc:File.PDF:PDF" + ), + + // parseCorrectOnlineInput + Arguments.of( + Collections.singletonList(new LinkedFile(new URL("http://arxiv.org/pdf/2010.08497v1"), "PDF")), + ":http\\://arxiv.org/pdf/2010.08497v1:PDF" + ), + + // parseFaultyOnlineInput + Arguments.of( + Collections.singletonList(new LinkedFile("", "htt://arxiv.org/pdf/2010.08497v1", "PDF")), + ":htt\\://arxiv.org/pdf/2010.08497v1:PDF" + ), + + // parseFaultyArxivOnlineInput + Arguments.of( + Collections.singletonList(new LinkedFile("arXiv Fulltext PDF", "https://arxiv.org/pdf/1109.0517.pdf", "application/pdf")), + "arXiv Fulltext PDF:https\\://arxiv.org/pdf/1109.0517.pdf:application/pdf" + ), + + // ignoreMissingDescription + Arguments.of( + Collections.singletonList(new LinkedFile("", Path.of("wei2005ahp.pdf"), "PDF")), + ":wei2005ahp.pdf:PDF" + ), + + // interpretLinkAsOnlyMandatoryField: single + Arguments.of( + Collections.singletonList(new LinkedFile("", Path.of("wei2005ahp.pdf"), "")), + "wei2005ahp.pdf" + ), + + // interpretLinkAsOnlyMandatoryField: multiple + Arguments.of( + List.of( + new LinkedFile("", Path.of("wei2005ahp.pdf"), ""), + new LinkedFile("", Path.of("other.pdf"), "") + ), + "wei2005ahp.pdf;other.pdf" + ), + + // escapedCharactersInDescription + Arguments.of( + Collections.singletonList(new LinkedFile("test:;", Path.of("wei2005ahp.pdf"), "PDF")), + "test\\:\\;:wei2005ahp.pdf:PDF" + ), + + // handleXmlCharacters + Arguments.of( + Collections.singletonList(new LinkedFile("test,st:;", Path.of("wei2005ahp.pdf"), "PDF")), + "test,\\;st\\:\\;:wei2005ahp.pdf:PDF" + ), + + // handleEscapedFilePath + Arguments.of( + Collections.singletonList(new LinkedFile("desc", Path.of("C:\\test.pdf"), "PDF")), + "desc:C\\:\\\\test.pdf:PDF" + ), + + // subsetOfFieldsResultsInFileLink: description only + Arguments.of( + Collections.singletonList(new LinkedFile("", Path.of("file.pdf"), "")), + "file.pdf::" + ), + + // subsetOfFieldsResultsInFileLink: file only + Arguments.of( + Collections.singletonList(new LinkedFile("", Path.of("file.pdf"), "")), + ":file.pdf" + ), + + // subsetOfFieldsResultsInFileLink: type only + Arguments.of( + Collections.singletonList(new LinkedFile("", Path.of("file.pdf"), "")), + "::file.pdf" + ), + + // tooManySeparators + Arguments.of( + Collections.singletonList(new LinkedFile("desc", Path.of("file.pdf"), "PDF")), + "desc:file.pdf:PDF:asdf" + ) + ); + } + + @ParameterizedTest + @MethodSource("stringsToParseTestData") + public void testParse(List expected, String input) { + assertEquals(expected, FileFieldParser.parse(input)); + } +} From 7117b6197aa6f69f6b8b25f9452e875c9051046b Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Tue, 26 Jan 2021 22:40:44 +0100 Subject: [PATCH 05/25] Change format for study definition to yaml (#7126) --- build.gradle | 4 + .../0018-use-Jackson-to-parse-study-yml.md | 56 ++++++ docs/adr/0019-keep-study-as-a-dto.md | 30 +++ src/main/java/module-info.java | 3 + .../org/jabref/logic/crawler/Crawler.java | 6 +- ...a => StudyDatabaseToFetcherConverter.java} | 20 +- .../jabref/logic/crawler/StudyRepository.java | 107 +++++------ .../jabref/logic/crawler/StudyYamlParser.java | 39 ++++ .../java/org/jabref/model/study/Study.java | 173 +++++++++++------- .../org/jabref/model/study/StudyDatabase.java | 67 +++++++ .../model/study/StudyMetaDataField.java | 24 --- .../org/jabref/model/study/StudyQuery.java | 50 +++++ .../org/jabref/logic/crawler/CrawlerTest.java | 29 ++- ... StudyDatabaseToFetcherConverterTest.java} | 11 +- .../logic/crawler/StudyRepositoryTest.java | 91 +++------ .../logic/crawler/StudyYamlParserTest.java | 55 ++++++ .../org/jabref/model/study/StudyTest.java | 94 ---------- .../org/jabref/logic/crawler/study.bib | 37 ---- .../org/jabref/logic/crawler/study.yml | 16 ++ 19 files changed, 533 insertions(+), 379 deletions(-) create mode 100644 docs/adr/0018-use-Jackson-to-parse-study-yml.md create mode 100644 docs/adr/0019-keep-study-as-a-dto.md rename src/main/java/org/jabref/logic/crawler/{LibraryEntryToFetcherConverter.java => StudyDatabaseToFetcherConverter.java} (73%) create mode 100644 src/main/java/org/jabref/logic/crawler/StudyYamlParser.java create mode 100644 src/main/java/org/jabref/model/study/StudyDatabase.java delete mode 100644 src/main/java/org/jabref/model/study/StudyMetaDataField.java create mode 100644 src/main/java/org/jabref/model/study/StudyQuery.java rename src/test/java/org/jabref/logic/crawler/{LibraryEntryToFetcherConverterTest.java => StudyDatabaseToFetcherConverterTest.java} (86%) create mode 100644 src/test/java/org/jabref/logic/crawler/StudyYamlParserTest.java delete mode 100644 src/test/java/org/jabref/model/study/StudyTest.java delete mode 100644 src/test/resources/org/jabref/logic/crawler/study.bib create mode 100644 src/test/resources/org/jabref/logic/crawler/study.yml diff --git a/build.gradle b/build.gradle index cd75b24eef4..2273d0a127a 100644 --- a/build.gradle +++ b/build.gradle @@ -42,6 +42,7 @@ java { application { mainClassName = "org.jabref.gui.JabRefLauncher" + mainModule = 'org.jabref' } // TODO: Ugly workaround to temporarily ignore build errors to dependencies of latex2unicode @@ -132,6 +133,9 @@ dependencies { implementation group: 'org.eclipse.jgit', name: 'org.eclipse.jgit', version: '5.10.0.202012080955-r' + implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: '2.12.0-rc2' + implementation group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310', version: '2.12.0-rc2' + implementation group: 'org.mariadb.jdbc', name: 'mariadb-java-client', version: '2.7.1' implementation 'org.postgresql:postgresql:42.2.18' diff --git a/docs/adr/0018-use-Jackson-to-parse-study-yml.md b/docs/adr/0018-use-Jackson-to-parse-study-yml.md new file mode 100644 index 00000000000..70903544d58 --- /dev/null +++ b/docs/adr/0018-use-Jackson-to-parse-study-yml.md @@ -0,0 +1,56 @@ +# Use Jackson to parse study.yml + +## Context and Problem Statement + +The study definition file is formulated as a YAML document. +To accessed the definition within JabRef this document has to be parsed. +What parser should be used to parse YAML files? + +## Considered Options + +* [Jackson](https://github.com/FasterXML/jackson-dataformat-yaml) +* [SnakeYAML Engine](https://bitbucket.org/asomov/snakeyaml) +* [yamlbeans](https://github.com/EsotericSoftware/yamlbeans) +* [eo-yaml](https://github.com/decorators-squad/eo-yaml) +* Self-written parser + +## Decision Outcome + +Chosen option: Jackson, because as it is a dedicated library for parsing YAML. yamlbeans also seem to be viable. They all offer similar functionality + +## Pros and Cons of the Options + +### Jackson + +* Good, because established YAML parser library +* Good, because supports YAML 1.2 +* Good, because it can parse LocalDate + +### SnakeYAML Engine + +* Good, because established YAML parser library +* Good, because supports YAML 1.2 +* Bad, because cannot parse YAML into Java DTOs, only into [basic Java structures](https://bitbucket.org/asomov/snakeyaml-engine/src/master/), this then has to be assembled into DTOs + +### yamlbeans + +* Good, because established YAML parser library +* Good, because [nice getting started page](https://github.com/EsotericSoftware/yamlbeans) +* Bad, because objects need to be annotated in the yaml file to be parsed into Java objects + +### eo-yaml + +* Good, because established YAML parser library +* Good, because supports YAML 1.2 +* Bad, because cannot parse YAML into Java DTOs + +### Own parser + +* Good, because easily customizable +* Bad, because high effort +* Bad, because has to be tested extensively + +## Links + +* [Winery's ADR-0009](https://github.com/eclipse/winery/blob/master/docs/adr/0009-manual-tosca-yaml-serialisation.md) +* [Winery's ADR-0010](https://github.com/eclipse/winery/blob/master/docs/adr/0010-tosca-yaml-deserialisation-using-snakeyaml.md) diff --git a/docs/adr/0019-keep-study-as-a-dto.md b/docs/adr/0019-keep-study-as-a-dto.md new file mode 100644 index 00000000000..bc99d879e31 --- /dev/null +++ b/docs/adr/0019-keep-study-as-a-dto.md @@ -0,0 +1,30 @@ +# Keep study as a DTO + +## Context and Problem Statement + +The study holds query and library entries that could be replaced respectively with complex query and fetcher instances. +This poses the question: should the study remain a pure DTO object or should it contain direct object instances? + +## Considered Options + +* Keep study as DTO and use transformers +* Replace entries with instances + +## Decision Outcome + +Chosen option: "Keep study as DTO and use transformators", because comes out best (see below). + +## Pros and Cons of the Options + +### Keep study as DTO and use transformators + +* Good, because no need for custom serialization +* Good, because deactivated fetchers can be documented (important for traceable Searching (SLRs)) +* Bad, because Entries for databases and queries needed + +### Replace entries with instances + +* Good, because no need for database and query entries +* Bad, because custom de-/serializers for fetchers and complex queries needed +* Bad, because harder to maintain than using "vanilla" jackson de-/serialization +* … diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index 0bee3c9dfd2..3619fe63b0c 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -88,4 +88,7 @@ requires lucene.queryparser; requires lucene.core; requires org.eclipse.jgit; + requires com.fasterxml.jackson.databind; + requires com.fasterxml.jackson.dataformat.yaml; + requires com.fasterxml.jackson.datatype.jsr310; } diff --git a/src/main/java/org/jabref/logic/crawler/Crawler.java b/src/main/java/org/jabref/logic/crawler/Crawler.java index 50feab3cdab..898af4ffbdb 100644 --- a/src/main/java/org/jabref/logic/crawler/Crawler.java +++ b/src/main/java/org/jabref/logic/crawler/Crawler.java @@ -11,7 +11,6 @@ import org.jabref.logic.preferences.TimestampPreferences; import org.jabref.model.entry.BibEntryTypesManager; import org.jabref.model.study.QueryResult; -import org.jabref.model.study.Study; import org.jabref.model.util.FileUpdateMonitor; import org.eclipse.jgit.api.errors.GitAPIException; @@ -36,9 +35,8 @@ public class Crawler { public Crawler(Path studyDefinitionFile, GitHandler gitHandler, FileUpdateMonitor fileUpdateMonitor, ImportFormatPreferences importFormatPreferences, SavePreferences savePreferences, TimestampPreferences timestampPreferences, BibEntryTypesManager bibEntryTypesManager) throws IllegalArgumentException, IOException, ParseException, GitAPIException { Path studyRepositoryRoot = studyDefinitionFile.getParent(); studyRepository = new StudyRepository(studyRepositoryRoot, gitHandler, importFormatPreferences, fileUpdateMonitor, savePreferences, timestampPreferences, bibEntryTypesManager); - Study study = studyRepository.getStudy(); - LibraryEntryToFetcherConverter libraryEntryToFetcherConverter = new LibraryEntryToFetcherConverter(study.getActiveLibraryEntries(), importFormatPreferences); - this.studyFetcher = new StudyFetcher(libraryEntryToFetcherConverter.getActiveFetchers(), study.getSearchQueryStrings()); + StudyDatabaseToFetcherConverter studyDatabaseToFetcherConverter = new StudyDatabaseToFetcherConverter(studyRepository.getActiveLibraryEntries(), importFormatPreferences); + this.studyFetcher = new StudyFetcher(studyDatabaseToFetcherConverter.getActiveFetchers(), studyRepository.getSearchQueryStrings()); } /** diff --git a/src/main/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverter.java b/src/main/java/org/jabref/logic/crawler/StudyDatabaseToFetcherConverter.java similarity index 73% rename from src/main/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverter.java rename to src/main/java/org/jabref/logic/crawler/StudyDatabaseToFetcherConverter.java index cadf5b2978e..a5b51854cd8 100644 --- a/src/main/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverter.java +++ b/src/main/java/org/jabref/logic/crawler/StudyDatabaseToFetcherConverter.java @@ -8,19 +8,16 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.SearchBasedFetcher; import org.jabref.logic.importer.WebFetchers; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.field.UnknownField; - -import static org.jabref.model.entry.types.SystematicLiteratureReviewStudyEntryType.LIBRARY_ENTRY; +import org.jabref.model.study.StudyDatabase; /** * Converts library entries from the given study into their corresponding fetchers. */ -class LibraryEntryToFetcherConverter { - private final List libraryEntries; +class StudyDatabaseToFetcherConverter { + private final List libraryEntries; private final ImportFormatPreferences importFormatPreferences; - public LibraryEntryToFetcherConverter(List libraryEntries, ImportFormatPreferences importFormatPreferences) { + public StudyDatabaseToFetcherConverter(List libraryEntries, ImportFormatPreferences importFormatPreferences) { this.libraryEntries = libraryEntries; this.importFormatPreferences = importFormatPreferences; } @@ -42,9 +39,8 @@ public List getActiveFetchers() { * @param libraryEntries List of entries * @return List of fetcher instances */ - private List getFetchersFromLibraryEntries(List libraryEntries) { + private List getFetchersFromLibraryEntries(List libraryEntries) { return libraryEntries.parallelStream() - .filter(bibEntry -> bibEntry.getType().getName().equals(LIBRARY_ENTRY.getName())) .map(this::createFetcherFromLibraryEntry) .filter(Objects::nonNull) .collect(Collectors.toList()); @@ -53,12 +49,12 @@ private List getFetchersFromLibraryEntries(List li /** * Transforms a library entry into a SearchBasedFetcher instance. This only works if the library entry specifies a supported fetcher. * - * @param libraryEntry the entry that will be converted + * @param studyDatabase the entry that will be converted * @return An instance of the fetcher defined by the library entry. */ - private SearchBasedFetcher createFetcherFromLibraryEntry(BibEntry libraryEntry) { + private SearchBasedFetcher createFetcherFromLibraryEntry(StudyDatabase studyDatabase) { Set searchBasedFetchers = WebFetchers.getSearchBasedFetchers(importFormatPreferences); - String libraryNameFromFetcher = libraryEntry.getField(new UnknownField("name")).orElse(""); + String libraryNameFromFetcher = studyDatabase.getName(); return searchBasedFetchers.stream() .filter(searchBasedFetcher -> searchBasedFetcher.getName().toLowerCase().equals(libraryNameFromFetcher.toLowerCase())) .findAny() diff --git a/src/main/java/org/jabref/logic/crawler/StudyRepository.java b/src/main/java/org/jabref/logic/crawler/StudyRepository.java index 9971ab9dfb1..99aa896cfd5 100644 --- a/src/main/java/org/jabref/logic/crawler/StudyRepository.java +++ b/src/main/java/org/jabref/logic/crawler/StudyRepository.java @@ -2,14 +2,11 @@ import java.io.FileWriter; import java.io.IOException; -import java.io.InputStream; import java.io.Writer; import java.nio.file.Files; import java.nio.file.Path; import java.time.LocalDate; -import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -22,17 +19,15 @@ import org.jabref.logic.importer.OpenDatabase; import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.SearchBasedFetcher; -import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.preferences.TimestampPreferences; import org.jabref.model.database.BibDatabase; import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.BibEntryTypesManager; -import org.jabref.model.entry.field.UnknownField; -import org.jabref.model.entry.types.SystematicLiteratureReviewStudyEntryType; import org.jabref.model.study.FetchResult; import org.jabref.model.study.QueryResult; import org.jabref.model.study.Study; +import org.jabref.model.study.StudyDatabase; +import org.jabref.model.study.StudyQuery; import org.jabref.model.util.FileUpdateMonitor; import org.eclipse.jgit.api.errors.GitAPIException; @@ -48,13 +43,13 @@ */ class StudyRepository { // Tests work with study.bib - private static final String STUDY_DEFINITION_FILE_NAME = "study.bib"; + private static final String STUDY_DEFINITION_FILE_NAME = "study.yml"; private static final Logger LOGGER = LoggerFactory.getLogger(StudyRepository.class); private static final Pattern MATCHCOLON = Pattern.compile(":"); private static final Pattern MATCHILLEGALCHARACTERS = Pattern.compile("[^A-Za-z0-9_.\\s=-]"); private final Path repositoryPath; - private final Path studyDefinitionBib; + private final Path studyDefinitionFile; private final GitHandler gitHandler; private final Study study; private final ImportFormatPreferences importFormatPreferences; @@ -90,14 +85,14 @@ public StudyRepository(Path pathToRepository, } this.importFormatPreferences = importFormatPreferences; this.fileUpdateMonitor = fileUpdateMonitor; - this.studyDefinitionBib = Path.of(repositoryPath.toString(), STUDY_DEFINITION_FILE_NAME); + this.studyDefinitionFile = Path.of(repositoryPath.toString(), STUDY_DEFINITION_FILE_NAME); this.savePreferences = savePreferences; this.timestampPreferences = timestampPreferences; this.bibEntryTypesManager = bibEntryTypesManager; if (Files.notExists(repositoryPath)) { throw new IOException("The given repository does not exists."); - } else if (Files.notExists(studyDefinitionBib)) { + } else if (Files.notExists(studyDefinitionFile)) { throw new IOException("The study definition file does not exist in the given repository."); } study = parseStudyFile(); @@ -126,30 +121,39 @@ public BibDatabaseContext getStudyResultEntries() throws IOException { } /** - * The study definition file contains all the definitions of a study. This method extracts the BibEntries from the study BiB file. + * The study definition file contains all the definitions of a study. This method extracts this study from the yaml study definition file * * @return Returns the BibEntries parsed from the study definition file. * @throws IOException Problem opening the input stream. * @throws ParseException Problem parsing the study definition file. */ - private Study parseStudyFile() throws IOException, ParseException { - BibtexParser parser = new BibtexParser(importFormatPreferences, fileUpdateMonitor); - List parsedEntries = new ArrayList<>(); - try (InputStream inputStream = Files.newInputStream(studyDefinitionBib)) { - parsedEntries.addAll(parser.parseEntries(inputStream)); - } + private Study parseStudyFile() throws IOException { + return new StudyYamlParser().parseStudyYamlFile(studyDefinitionFile); + } + + /** + * Returns all query strings of the study definition + * + * @return List of all queries as Strings. + */ + public List getSearchQueryStrings() { + return study.getQueries() + .parallelStream() + .map(StudyQuery::getQuery) + .collect(Collectors.toList()); + } - BibEntry studyEntry = parsedEntries.parallelStream() - .filter(bibEntry -> bibEntry.getType().equals(SystematicLiteratureReviewStudyEntryType.STUDY_ENTRY)).findAny() - .orElseThrow(() -> new ParseException("Study definition file does not contain a study entry")); - List queryEntries = parsedEntries.parallelStream() - .filter(bibEntry -> bibEntry.getType().equals(SystematicLiteratureReviewStudyEntryType.SEARCH_QUERY_ENTRY)) - .collect(Collectors.toList()); - List libraryEntries = parsedEntries.parallelStream() - .filter(bibEntry -> bibEntry.getType().equals(SystematicLiteratureReviewStudyEntryType.LIBRARY_ENTRY)) - .collect(Collectors.toList()); - - return new Study(studyEntry, queryEntries, libraryEntries); + /** + * Extracts all active fetchers from the library entries. + * + * @return List of BibEntries of type Library + * @throws IllegalArgumentException If a transformation from Library entry to LibraryDefinition fails + */ + public List getActiveLibraryEntries() throws IllegalArgumentException { + return study.getDatabases() + .parallelStream() + .filter(StudyDatabase::isEnabled) + .collect(Collectors.toList()); } public Study getStudy() { @@ -173,7 +177,7 @@ public void persist(List crawlResults) throws IOException, GitAPIEx } private void persistStudy() throws IOException { - writeResultToFile(studyDefinitionBib, new BibDatabase(study.getAllEntries())); + new StudyYamlParser().writeStudyYamlFile(study, studyDefinitionFile); } /** @@ -181,8 +185,8 @@ private void persistStudy() throws IOException { */ private void setUpRepositoryStructure() throws IOException { // Cannot use stream here since IOException has to be thrown - LibraryEntryToFetcherConverter converter = new LibraryEntryToFetcherConverter(study.getActiveLibraryEntries(), importFormatPreferences); - for (String query : study.getSearchQueryStrings()) { + StudyDatabaseToFetcherConverter converter = new StudyDatabaseToFetcherConverter(this.getActiveLibraryEntries(), importFormatPreferences); + for (String query : this.getSearchQueryStrings()) { createQueryResultFolder(query); converter.getActiveFetchers() .forEach(searchBasedFetcher -> createFetcherResultFile(query, searchBasedFetcher)); @@ -239,14 +243,14 @@ private void createBibFile(Path file) { * Structure: ID-trimmed query * * Examples: - * Input: '(title: test-title AND abstract: Test)' as a query entry with id 1 - * Output: '1 - title= test-title AND abstract= Test' + * Input: '(title: test-title AND abstract: Test)' as a query entry with id 12345678 + * Output: '12345678 - title= test-title AND abstract= Test' * - * Input: 'abstract: Test*' as a query entry with id 1 - * Output: '1 - abstract= Test' + * Input: 'abstract: Test*' as a query entry with id 87654321 + * Output: '87654321 - abstract= Test' * - * Input: '"test driven"' as a query entry with id 1 - * Output: '1 - test driven' + * Input: '"test driven"' as a query entry with id 12348765 + * Output: '12348765 - test driven' * * @param query that is trimmed and combined with its query id * @return a unique folder name for any query. @@ -255,31 +259,20 @@ private String trimNameAndAddID(String query) { // Replace all field: with field= for folder name String trimmedNamed = MATCHCOLON.matcher(query).replaceAll("="); trimmedNamed = MATCHILLEGALCHARACTERS.matcher(trimmedNamed).replaceAll(""); - if (query.length() > 240) { - trimmedNamed = query.substring(0, 240); + String id = computeIDForQuery(query); + // Whole path has to be shorter than 260 + int remainingPathLength = 220 - studyDefinitionFile.toString().length() - id.length(); + if (query.length() > remainingPathLength) { + trimmedNamed = query.substring(0, remainingPathLength); } - String id = findQueryIDByQueryString(query); return id + " - " + trimmedNamed; } /** - * Helper to find the query id for folder name creation. - * Returns the id of the first SearchQuery BibEntry with a query field that matches the given query. - * - * @param query The query whose ID is searched - * @return ID of the query defined in the study definition. + * Helper to compute the query id for folder name creation. */ - private String findQueryIDByQueryString(String query) { - String queryField = "query"; - return study.getSearchQueryEntries() - .parallelStream() - .filter(bibEntry -> bibEntry.getField(new UnknownField(queryField)).orElse("").equals(query)) - .map(BibEntry::getCitationKey) - .filter(Optional::isPresent) - .map(Optional::get) - .findFirst() - .orElseThrow() - .replaceFirst(queryField, ""); + private String computeIDForQuery(String query) { + return String.valueOf(query.hashCode()); } /** diff --git a/src/main/java/org/jabref/logic/crawler/StudyYamlParser.java b/src/main/java/org/jabref/logic/crawler/StudyYamlParser.java new file mode 100644 index 00000000000..df2138434d2 --- /dev/null +++ b/src/main/java/org/jabref/logic/crawler/StudyYamlParser.java @@ -0,0 +1,39 @@ +package org.jabref.logic.crawler; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; + +import org.jabref.model.study.Study; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; + +public class StudyYamlParser { + + /** + * Parses the given yaml study definition file into a study instance + */ + public Study parseStudyYamlFile(Path studyYamlFile) throws IOException { + ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory()); + yamlMapper.registerModule(new JavaTimeModule()); + try (InputStream fileInputStream = new FileInputStream(studyYamlFile.toFile())) { + return yamlMapper.readValue(fileInputStream, Study.class); + } + } + + /** + * Writes the given study instance into a yaml file to the given path + */ + public void writeStudyYamlFile(Study study, Path studyYamlFile) throws IOException { + ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory().disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER) + .enable(YAMLGenerator.Feature.MINIMIZE_QUOTES)); + yamlMapper.registerModule(new JavaTimeModule()); + yamlMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + yamlMapper.writeValue(studyYamlFile.toFile(), study); + } +} diff --git a/src/main/java/org/jabref/model/study/Study.java b/src/main/java/org/jabref/model/study/Study.java index 37ed6e2328a..382268ce39a 100644 --- a/src/main/java/org/jabref/model/study/Study.java +++ b/src/main/java/org/jabref/model/study/Study.java @@ -1,98 +1,135 @@ package org.jabref.model.study; import java.time.LocalDate; -import java.util.ArrayList; import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; +import java.util.Objects; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.field.UnknownField; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; /** * This class represents a scientific study. * * This class defines all aspects of a scientific study relevant to the application. It is a proxy for the file based study definition. */ + +@JsonPropertyOrder({"authors", "title", "last-search-date", "research-questions", "queries", "databases"}) public class Study { - private static final String SEARCH_QUERY_FIELD_NAME = "query"; + private List authors; + private String title; + @JsonProperty("last-search-date") + private LocalDate lastSearchDate; + @JsonProperty("research-questions") + private List researchQuestions; + private List queries; + private List databases; - private final BibEntry studyEntry; - private final List queryEntries; - private final List libraryEntries; + public Study(List authors, String title, List researchQuestions, List queryEntries, List databases) { + this.authors = authors; + this.title = title; + this.researchQuestions = researchQuestions; + this.queries = queryEntries; + this.databases = databases; + } - public Study(BibEntry studyEntry, List queryEntries, List libraryEntries) { - this.studyEntry = studyEntry; - this.queryEntries = queryEntries; - this.libraryEntries = libraryEntries; + /** + * Used for Jackson deserialization + */ + public Study() { } - public List getAllEntries() { - List allEntries = new ArrayList<>(); - allEntries.add(studyEntry); - allEntries.addAll(queryEntries); - allEntries.addAll(libraryEntries); - return allEntries; + public List getAuthors() { + return authors; } - /** - * Returns all query strings - * - * @return List of all queries as Strings. - */ - public List getSearchQueryStrings() { - return queryEntries.parallelStream() - .map(bibEntry -> bibEntry.getField(new UnknownField(SEARCH_QUERY_FIELD_NAME))) - .filter(Optional::isPresent) - .map(Optional::get) - .collect(Collectors.toList()); + public void setAuthors(List authors) { + this.authors = authors; } - /** - * This method returns the SearchQuery entries. - * This is required when the BibKey of the search term entry is required in combination with the search query (e.g. - * for the creation of the study repository structure). - */ - public List getSearchQueryEntries() { - return queryEntries; + public List getQueries() { + return queries; } - /** - * Returns a meta data entry of the first study entry found in the study definition file of the provided type. - * - * @param metaDataField The type of requested meta-data - * @return returns the requested meta data type of the first found study entry - * @throws IllegalArgumentException If the study file does not contain a study entry. - */ - public Optional getStudyMetaDataField(StudyMetaDataField metaDataField) throws IllegalArgumentException { - return studyEntry.getField(metaDataField.toField()); + public void setQueries(List queries) { + this.queries = queries; + } + + public LocalDate getLastSearchDate() { + return lastSearchDate; } - /** - * Sets the lastSearchDate field of the study entry - * - * @param date date the last time a search was conducted - */ public void setLastSearchDate(LocalDate date) { - studyEntry.setField(StudyMetaDataField.STUDY_LAST_SEARCH.toField(), date.toString()); + lastSearchDate = date; } - /** - * Extracts all active LibraryEntries from the BibEntries. - * - * @return List of BibEntries of type Library - * @throws IllegalArgumentException If a transformation from Library entry to LibraryDefinition fails - */ - public List getActiveLibraryEntries() throws IllegalArgumentException { - return libraryEntries - .parallelStream() - .filter(bibEntry -> { - // If enabled is not defined, the fetcher is active. - return bibEntry.getField(new UnknownField("enabled")) - .map(enabled -> enabled.equals("true")) - .orElse(true); - }) - .collect(Collectors.toList()); + public List getDatabases() { + return databases; + } + + public void setDatabases(List databases) { + this.databases = databases; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public List getResearchQuestions() { + return researchQuestions; + } + + public void setResearchQuestions(List researchQuestions) { + this.researchQuestions = researchQuestions; + } + + @Override + public String toString() { + return "Study{" + + "authors=" + authors + + ", studyName='" + title + '\'' + + ", lastSearchDate=" + lastSearchDate + + ", researchQuestions=" + researchQuestions + + ", queries=" + queries + + ", libraries=" + databases + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + Study study = (Study) o; + + if (getAuthors() != null ? !getAuthors().equals(study.getAuthors()) : study.getAuthors() != null) { + return false; + } + if (getTitle() != null ? !getTitle().equals(study.getTitle()) : study.getTitle() != null) { + return false; + } + if (getLastSearchDate() != null ? !getLastSearchDate().equals(study.getLastSearchDate()) : study.getLastSearchDate() != null) { + return false; + } + if (getResearchQuestions() != null ? !getResearchQuestions().equals(study.getResearchQuestions()) : study.getResearchQuestions() != null) { + return false; + } + if (getQueries() != null ? !getQueries().equals(study.getQueries()) : study.getQueries() != null) { + return false; + } + return getDatabases() != null ? getDatabases().equals(study.getDatabases()) : study.getDatabases() == null; + } + + @Override + public int hashCode() { + return Objects.hashCode(this); } } diff --git a/src/main/java/org/jabref/model/study/StudyDatabase.java b/src/main/java/org/jabref/model/study/StudyDatabase.java new file mode 100644 index 00000000000..ac71a7160c2 --- /dev/null +++ b/src/main/java/org/jabref/model/study/StudyDatabase.java @@ -0,0 +1,67 @@ +package org.jabref.model.study; + +public class StudyDatabase { + private String name; + private boolean enabled; + + public StudyDatabase(String name, boolean enabled) { + this.name = name; + this.enabled = enabled; + } + + /** + * Used for Jackson deserialization + */ + public StudyDatabase() { + // Per default fetcher is activated + this.enabled = true; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + StudyDatabase that = (StudyDatabase) o; + + if (isEnabled() != that.isEnabled()) { + return false; + } + return getName() != null ? getName().equals(that.getName()) : that.getName() == null; + } + + @Override + public int hashCode() { + int result = getName() != null ? getName().hashCode() : 0; + result = 31 * result + (isEnabled() ? 1 : 0); + return result; + } + + @Override + public String toString() { + return "LibraryEntry{" + + "name='" + name + '\'' + + ", enabled=" + enabled + + '}'; + } +} diff --git a/src/main/java/org/jabref/model/study/StudyMetaDataField.java b/src/main/java/org/jabref/model/study/StudyMetaDataField.java deleted file mode 100644 index 6dbea2a2dc8..00000000000 --- a/src/main/java/org/jabref/model/study/StudyMetaDataField.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.jabref.model.study; - -import org.jabref.model.entry.field.Field; -import org.jabref.model.entry.field.StandardField; -import org.jabref.model.entry.field.UnknownField; - -/** - * This enum represents the different fields in the study entry - */ -public enum StudyMetaDataField { - STUDY_NAME(new UnknownField("name")), STUDY_RESEARCH_QUESTIONS(new UnknownField("researchQuestions")), - STUDY_AUTHORS(StandardField.AUTHOR), STUDY_GIT_REPOSITORY(new UnknownField("gitRepositoryURL")), - STUDY_LAST_SEARCH(new UnknownField("lastSearchDate")); - - private final Field field; - - StudyMetaDataField(Field field) { - this.field = field; - } - - public Field toField() { - return this.field; - } -} diff --git a/src/main/java/org/jabref/model/study/StudyQuery.java b/src/main/java/org/jabref/model/study/StudyQuery.java new file mode 100644 index 00000000000..ae5a42b0783 --- /dev/null +++ b/src/main/java/org/jabref/model/study/StudyQuery.java @@ -0,0 +1,50 @@ +package org.jabref.model.study; + +public class StudyQuery { + private String query; + + public StudyQuery(String query) { + this.query = query; + } + + /** + * Used for Jackson deserialization + */ + public StudyQuery() { + + } + + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + StudyQuery that = (StudyQuery) o; + + return getQuery() != null ? getQuery().equals(that.getQuery()) : that.getQuery() == null; + } + + @Override + public int hashCode() { + return getQuery() != null ? getQuery().hashCode() : 0; + } + + @Override + public String toString() { + return "QueryEntry{" + + "query='" + query + '\'' + + '}'; + } +} diff --git a/src/test/java/org/jabref/logic/crawler/CrawlerTest.java b/src/test/java/org/jabref/logic/crawler/CrawlerTest.java index dfa6e11ba9c..b07dc48c59f 100644 --- a/src/test/java/org/jabref/logic/crawler/CrawlerTest.java +++ b/src/test/java/org/jabref/logic/crawler/CrawlerTest.java @@ -38,6 +38,9 @@ class CrawlerTest { TimestampPreferences timestampPreferences; BibEntryTypesManager entryTypesManager; GitHandler gitHandler = mock(GitHandler.class, Answers.RETURNS_DEFAULTS); + String hashCodeQuantum = String.valueOf("Quantum".hashCode()); + String hashCodeCloudComputing = String.valueOf("Cloud Computing".hashCode()); + String hashCodeSoftwareEngineering = String.valueOf("\"Software Engineering\"".hashCode()); @Test public void testWhetherAllFilesAreCreated() throws Exception { @@ -53,26 +56,22 @@ public void testWhetherAllFilesAreCreated() throws Exception { testCrawler.performCrawl(); - assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum"))); - assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing"))); - assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), hashCodeQuantum + " - Quantum"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), hashCodeCloudComputing + " - Cloud Computing"))); - assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "ArXiv.bib"))); - assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "ArXiv.bib"))); - assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "ArXiv.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), hashCodeQuantum + " - Quantum", "ArXiv.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), hashCodeCloudComputing + " - Cloud Computing", "ArXiv.bib"))); - assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "Springer.bib"))); - assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "Springer.bib"))); - assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "Springer.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), hashCodeQuantum + " - Quantum", "Springer.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), hashCodeCloudComputing + " - Cloud Computing", "Springer.bib"))); - assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "result.bib"))); - assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "result.bib"))); - assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "result.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), hashCodeQuantum + " - Quantum", "result.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), hashCodeCloudComputing + " - Cloud Computing", "result.bib"))); assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "studyResult.bib"))); } private Path getPathToStudyDefinitionFile() { - return tempRepositoryDirectory.resolve("study.bib"); + return tempRepositoryDirectory.resolve("study.yml"); } /** @@ -121,8 +120,8 @@ private void setUpRepository() throws Exception { } private void setUpTestStudyDefinitionFile() throws Exception { - Path destination = tempRepositoryDirectory.resolve("study.bib"); - URL studyDefinition = this.getClass().getResource("study.bib"); + Path destination = tempRepositoryDirectory.resolve("study.yml"); + URL studyDefinition = this.getClass().getResource("study.yml"); FileUtil.copyFile(Path.of(studyDefinition.toURI()), destination, false); } } diff --git a/src/test/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverterTest.java b/src/test/java/org/jabref/logic/crawler/StudyDatabaseToFetcherConverterTest.java similarity index 86% rename from src/test/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverterTest.java rename to src/test/java/org/jabref/logic/crawler/StudyDatabaseToFetcherConverterTest.java index 013a9c118a7..f850658fefb 100644 --- a/src/test/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverterTest.java +++ b/src/test/java/org/jabref/logic/crawler/StudyDatabaseToFetcherConverterTest.java @@ -14,7 +14,6 @@ import org.jabref.logic.util.io.FileUtil; import org.jabref.model.entry.BibEntryTypesManager; import org.jabref.model.metadata.SaveOrderConfig; -import org.jabref.model.study.Study; import org.jabref.model.util.DummyFileUpdateMonitor; import org.junit.jupiter.api.Assertions; @@ -26,7 +25,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -class LibraryEntryToFetcherConverterTest { +class StudyDatabaseToFetcherConverterTest { ImportFormatPreferences importFormatPreferences; SavePreferences savePreferences; TimestampPreferences timestampPreferences; @@ -53,11 +52,11 @@ void setUpMocks() { @Test public void getActiveFetcherInstances() throws Exception { - Path studyDefinition = tempRepositoryDirectory.resolve("study.bib"); + Path studyDefinition = tempRepositoryDirectory.resolve("study.yml"); copyTestStudyDefinitionFileIntoDirectory(studyDefinition); - Study study = new StudyRepository(tempRepositoryDirectory, gitHandler, importFormatPreferences, new DummyFileUpdateMonitor(), savePreferences, timestampPreferences, entryTypesManager).getStudy(); - LibraryEntryToFetcherConverter converter = new LibraryEntryToFetcherConverter(study.getActiveLibraryEntries(), importFormatPreferences); + StudyRepository studyRepository = new StudyRepository(tempRepositoryDirectory, gitHandler, importFormatPreferences, new DummyFileUpdateMonitor(), savePreferences, timestampPreferences, entryTypesManager); + StudyDatabaseToFetcherConverter converter = new StudyDatabaseToFetcherConverter(studyRepository.getActiveLibraryEntries(), importFormatPreferences); List result = converter.getActiveFetchers(); Assertions.assertEquals(2, result.size()); @@ -66,7 +65,7 @@ public void getActiveFetcherInstances() throws Exception { } private void copyTestStudyDefinitionFileIntoDirectory(Path destination) throws Exception { - URL studyDefinition = this.getClass().getResource("study.bib"); + URL studyDefinition = this.getClass().getResource("study.yml"); FileUtil.copyFile(Path.of(studyDefinition.toURI()), destination, false); } } diff --git a/src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java b/src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java index 171fbb70168..8bba87c2a38 100644 --- a/src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java +++ b/src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java @@ -9,8 +9,6 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; import org.jabref.logic.bibtex.FieldContentFormatterPreferences; import org.jabref.logic.citationkeypattern.CitationKeyGenerator; @@ -27,13 +25,10 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.BibEntryTypesManager; import org.jabref.model.entry.field.StandardField; -import org.jabref.model.entry.field.UnknownField; import org.jabref.model.entry.types.StandardEntryType; import org.jabref.model.metadata.SaveOrderConfig; import org.jabref.model.study.FetchResult; import org.jabref.model.study.QueryResult; -import org.jabref.model.study.Study; -import org.jabref.model.study.StudyMetaDataField; import org.jabref.model.util.DummyFileUpdateMonitor; import org.junit.jupiter.api.BeforeEach; @@ -59,12 +54,15 @@ class StudyRepositoryTest { Path tempRepositoryDirectory; StudyRepository studyRepository; GitHandler gitHandler = mock(GitHandler.class, Answers.RETURNS_DEFAULTS); + String hashCodeQuantum = String.valueOf("Quantum".hashCode()); + String hashCodeCloudComputing = String.valueOf("Cloud Computing".hashCode()); + String hashCodeSoftwareEngineering = String.valueOf("\"Software Engineering\"".hashCode()); /** * Set up mocks */ @BeforeEach - public void setUpMocks() { + public void setUpMocks() throws Exception { savePreferences = mock(SavePreferences.class, Answers.RETURNS_DEEP_STUBS); importFormatPreferences = mock(ImportFormatPreferences.class, Answers.RETURNS_DEEP_STUBS); timestampPreferences = mock(TimestampPreferences.class); @@ -88,6 +86,7 @@ public void setUpMocks() { when(importFormatPreferences.getEncoding()).thenReturn(StandardCharsets.UTF_8); when(timestampPreferences.getTimestampField()).then(invocation -> StandardField.TIMESTAMP); entryTypesManager = new BibEntryTypesManager(); + getTestStudyRepository(); } @Test @@ -97,54 +96,25 @@ void providePathToNonExistentRepositoryThrowsException() { assertThrows(IOException.class, () -> new StudyRepository(nonExistingRepositoryDirectory, gitHandler, importFormatPreferences, new DummyFileUpdateMonitor(), savePreferences, timestampPreferences, entryTypesManager)); } - @Test - void providePathToExistentRepositoryWithOutStudyDefinitionFileThrowsException() { - assertThrows(IOException.class, () -> new StudyRepository(tempRepositoryDirectory, gitHandler, importFormatPreferences, new DummyFileUpdateMonitor(), savePreferences, timestampPreferences, entryTypesManager)); - } - - /** - * Tests whether the StudyRepository correctly imports the study file. - */ - @Test - void studyFileCorrectlyImported() throws Exception { - setUpTestStudyDefinitionFile(); - List expectedSearchterms = List.of("Quantum", "Cloud Computing", "TestSearchQuery3"); - List expectedActiveFetchersByName = List.of("Springer", "ArXiv"); - - Study study = new StudyRepository(tempRepositoryDirectory, gitHandler, importFormatPreferences, new DummyFileUpdateMonitor(), savePreferences, timestampPreferences, entryTypesManager).getStudy(); - - assertEquals(expectedSearchterms, study.getSearchQueryStrings()); - assertEquals("TestStudyName", study.getStudyMetaDataField(StudyMetaDataField.STUDY_NAME).get()); - assertEquals("Jab Ref", study.getStudyMetaDataField(StudyMetaDataField.STUDY_AUTHORS).get()); - assertEquals("Question1; Question2", study.getStudyMetaDataField(StudyMetaDataField.STUDY_RESEARCH_QUESTIONS).get()); - assertEquals(expectedActiveFetchersByName, study.getActiveLibraryEntries() - .stream() - .filter(bibEntry -> bibEntry.getType().getName().equals("library")) - .map(bibEntry -> bibEntry.getField(new UnknownField("name")).orElse("")) - .collect(Collectors.toList()) - ); - } - /** * Tests whether the file structure of the repository is created correctly from the study definitions file. */ @Test void repositoryStructureCorrectlyCreated() throws Exception { - // When repository is instantiated the directory structure is created - getTestStudyRepository(); - assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum"))); - assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing"))); - assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3"))); - assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "ArXiv.bib"))); - assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "ArXiv.bib"))); - assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "ArXiv.bib"))); - assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "Springer.bib"))); - assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "Springer.bib"))); - assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "Springer.bib"))); - assertTrue(Files.notExists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "IEEEXplore.bib"))); - assertTrue(Files.notExists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "IEEEXplore.bib"))); - assertTrue(Files.notExists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "IEEEXplore.bib"))); + // When repository is instantiated the directory structure is created + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), hashCodeQuantum + " - Quantum"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), hashCodeCloudComputing + " - Cloud Computing"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), hashCodeSoftwareEngineering + " - Software Engineering"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), hashCodeQuantum + " - Quantum", "ArXiv.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), hashCodeCloudComputing + " - Cloud Computing", "ArXiv.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), hashCodeSoftwareEngineering + " - Software Engineering", "ArXiv.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), hashCodeQuantum + " - Quantum", "Springer.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), hashCodeCloudComputing + " - Cloud Computing", "Springer.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), hashCodeSoftwareEngineering + " - Software Engineering", "Springer.bib"))); + assertTrue(Files.notExists(Path.of(tempRepositoryDirectory.toString(), hashCodeQuantum + " - Quantum", "IEEEXplore.bib"))); + assertTrue(Files.notExists(Path.of(tempRepositoryDirectory.toString(), hashCodeCloudComputing + " - Cloud Computing", "IEEEXplore.bib"))); + assertTrue(Files.notExists(Path.of(tempRepositoryDirectory.toString(), hashCodeSoftwareEngineering + " - Software Engineering", "IEEEXplore.bib"))); } /** @@ -152,9 +122,8 @@ void repositoryStructureCorrectlyCreated() throws Exception { */ @Test void bibEntriesCorrectlyStored() throws Exception { - StudyRepository repository = getTestStudyRepository(); setUpTestResultFile(); - List result = repository.getFetcherResultEntries("Quantum", "ArXiv").getEntries(); + List result = studyRepository.getFetcherResultEntries("Quantum", "ArXiv").getEntries(); assertEquals(getArXivQuantumMockResults(), result); } @@ -162,7 +131,7 @@ void bibEntriesCorrectlyStored() throws Exception { void fetcherResultsPersistedCorrectly() throws Exception { List mockResults = getMockResults(); - getTestStudyRepository().persist(mockResults); + studyRepository.persist(mockResults); assertEquals(getArXivQuantumMockResults(), getTestStudyRepository().getFetcherResultEntries("Quantum", "ArXiv").getEntries()); assertEquals(getSpringerQuantumMockResults(), getTestStudyRepository().getFetcherResultEntries("Quantum", "Springer").getEntries()); @@ -177,7 +146,7 @@ void mergedResultsPersistedCorrectly() throws Exception { expected.add(getSpringerQuantumMockResults().get(1)); expected.add(getSpringerQuantumMockResults().get(2)); - getTestStudyRepository().persist(mockResults); + studyRepository.persist(mockResults); // All Springer results are duplicates for "Quantum" assertEquals(expected, getTestStudyRepository().getQueryResultEntries("Quantum").getEntries()); @@ -188,25 +157,23 @@ void mergedResultsPersistedCorrectly() throws Exception { void setsLastSearchDatePersistedCorrectly() throws Exception { List mockResults = getMockResults(); - getTestStudyRepository().persist(mockResults); + studyRepository.persist(mockResults); - assertEquals(LocalDate.now().toString(), getTestStudyRepository().getStudy().getStudyMetaDataField(StudyMetaDataField.STUDY_LAST_SEARCH).get()); + assertEquals(LocalDate.now(), getTestStudyRepository().getStudy().getLastSearchDate()); } @Test void studyResultsPersistedCorrectly() throws Exception { List mockResults = getMockResults(); - getTestStudyRepository().persist(mockResults); + studyRepository.persist(mockResults); assertEquals(new HashSet<>(getNonDuplicateBibEntryResult().getEntries()), new HashSet<>(getTestStudyRepository().getStudyResultEntries().getEntries())); } private StudyRepository getTestStudyRepository() throws Exception { - if (Objects.isNull(studyRepository)) { - setUpTestStudyDefinitionFile(); - studyRepository = new StudyRepository(tempRepositoryDirectory, gitHandler, importFormatPreferences, new DummyFileUpdateMonitor(), savePreferences, timestampPreferences, entryTypesManager); - } + setUpTestStudyDefinitionFile(); + studyRepository = new StudyRepository(tempRepositoryDirectory, gitHandler, importFormatPreferences, new DummyFileUpdateMonitor(), savePreferences, timestampPreferences, entryTypesManager); return studyRepository; } @@ -214,8 +181,8 @@ private StudyRepository getTestStudyRepository() throws Exception { * Copies the study definition file into the test repository */ private void setUpTestStudyDefinitionFile() throws Exception { - Path destination = tempRepositoryDirectory.resolve("study.bib"); - URL studyDefinition = this.getClass().getResource("study.bib"); + Path destination = tempRepositoryDirectory.resolve("study.yml"); + URL studyDefinition = this.getClass().getResource("study.yml"); FileUtil.copyFile(Path.of(studyDefinition.toURI()), destination, false); } @@ -224,7 +191,7 @@ private void setUpTestStudyDefinitionFile() throws Exception { * The repository has to exist before this method is called. */ private void setUpTestResultFile() throws Exception { - Path queryDirectory = Path.of(tempRepositoryDirectory.toString(), "1 - Quantum"); + Path queryDirectory = Path.of(tempRepositoryDirectory.toString(), hashCodeQuantum + " - Quantum"); Path resultFileLocation = Path.of(queryDirectory.toString(), "ArXiv" + ".bib"); URL resultFile = this.getClass().getResource("ArXivQuantumMock.bib"); FileUtil.copyFile(Path.of(resultFile.toURI()), resultFileLocation, true); diff --git a/src/test/java/org/jabref/logic/crawler/StudyYamlParserTest.java b/src/test/java/org/jabref/logic/crawler/StudyYamlParserTest.java new file mode 100644 index 00000000000..8fbd419f01f --- /dev/null +++ b/src/test/java/org/jabref/logic/crawler/StudyYamlParserTest.java @@ -0,0 +1,55 @@ +package org.jabref.logic.crawler; + +import java.net.URL; +import java.nio.file.Path; +import java.time.LocalDate; +import java.util.List; + +import org.jabref.logic.util.io.FileUtil; +import org.jabref.model.study.Study; +import org.jabref.model.study.StudyDatabase; +import org.jabref.model.study.StudyQuery; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class StudyYamlParserTest { + @TempDir + static Path testDirectory; + Study expectedStudy; + + @BeforeEach + void setupStudy() throws Exception { + Path destination = testDirectory.resolve("study.yml"); + URL studyDefinition = StudyYamlParser.class.getResource("study.yml"); + FileUtil.copyFile(Path.of(studyDefinition.toURI()), destination, true); + + List authors = List.of("Jab Ref"); + String studyName = "TestStudyName"; + List researchQuestions = List.of("Question1", "Question2"); + List queryEntries = List.of(new StudyQuery("Quantum"), new StudyQuery("Cloud Computing"), new StudyQuery("\"Software Engineering\"")); + List libraryEntries = List.of(new StudyDatabase("Springer", true), new StudyDatabase("ArXiv", true), new StudyDatabase("IEEEXplore", false)); + + expectedStudy = new Study(authors, studyName, researchQuestions, queryEntries, libraryEntries); + expectedStudy.setLastSearchDate(LocalDate.parse("2020-11-26")); + } + + @Test + public void parseStudyFileSuccessfully() throws Exception { + Study study = new StudyYamlParser().parseStudyYamlFile(testDirectory.resolve("study.yml")); + + assertEquals(expectedStudy, study); + } + + @Test + public void writeStudyFileSuccessfully() throws Exception { + new StudyYamlParser().writeStudyYamlFile(expectedStudy, testDirectory.resolve("study.yml")); + + Study study = new StudyYamlParser().parseStudyYamlFile(testDirectory.resolve("study.yml")); + + assertEquals(expectedStudy, study); + } +} diff --git a/src/test/java/org/jabref/model/study/StudyTest.java b/src/test/java/org/jabref/model/study/StudyTest.java deleted file mode 100644 index 9ab34fcd55e..00000000000 --- a/src/test/java/org/jabref/model/study/StudyTest.java +++ /dev/null @@ -1,94 +0,0 @@ -package org.jabref.model.study; - -import java.time.LocalDate; -import java.util.List; - -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.field.StandardField; -import org.jabref.model.entry.field.UnknownField; -import org.jabref.model.entry.types.SystematicLiteratureReviewStudyEntryType; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class StudyTest { - Study testStudy; - - @BeforeEach - public void setUpTestStudy() { - BibEntry studyEntry = new BibEntry() - .withField(new UnknownField("name"), "TestStudyName") - .withField(StandardField.AUTHOR, "Jab Ref") - .withField(new UnknownField("researchQuestions"), "Question1; Question2") - .withField(new UnknownField("gitRepositoryURL"), "https://github.com/eclipse/jgit.git"); - studyEntry.setType(SystematicLiteratureReviewStudyEntryType.STUDY_ENTRY); - - // Create three SearchTerm entries. - BibEntry searchQuery1 = new BibEntry() - .withField(new UnknownField("query"), "TestSearchQuery1"); - searchQuery1.setType(SystematicLiteratureReviewStudyEntryType.SEARCH_QUERY_ENTRY); - searchQuery1.setCitationKey("query1"); - - BibEntry searchQuery2 = new BibEntry() - .withField(new UnknownField("query"), "TestSearchQuery2"); - searchQuery2.setType(SystematicLiteratureReviewStudyEntryType.SEARCH_QUERY_ENTRY); - searchQuery2.setCitationKey("query2"); - - BibEntry searchQuery3 = new BibEntry() - .withField(new UnknownField("query"), "TestSearchQuery3"); - searchQuery3.setType(SystematicLiteratureReviewStudyEntryType.SEARCH_QUERY_ENTRY); - searchQuery3.setCitationKey("query3"); - - // Create two Library entries - BibEntry library1 = new BibEntry() - .withField(new UnknownField("name"), "acm") - .withField(new UnknownField("enabled"), "false") - .withField(new UnknownField("comment"), "disabled, because no good results"); - library1.setType(SystematicLiteratureReviewStudyEntryType.LIBRARY_ENTRY); - library1.setCitationKey("library1"); - - BibEntry library2 = new BibEntry() - .withField(new UnknownField("name"), "arxiv") - .withField(new UnknownField("enabled"), "true") - .withField(new UnknownField("Comment"), ""); - library2.setType(SystematicLiteratureReviewStudyEntryType.LIBRARY_ENTRY); - library2.setCitationKey("library2"); - - testStudy = new Study(studyEntry, List.of(searchQuery1, searchQuery2, searchQuery3), List.of(library1, library2)); - } - - @Test - void getSearchTermsAsStrings() { - List expectedSearchTerms = List.of("TestSearchQuery1", "TestSearchQuery2", "TestSearchQuery3"); - assertEquals(expectedSearchTerms, testStudy.getSearchQueryStrings()); - } - - @Test - void setLastSearchTime() { - LocalDate date = LocalDate.now(); - testStudy.setLastSearchDate(date); - assertEquals(date.toString(), testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_LAST_SEARCH).get()); - } - - @Test - void getStudyName() { - assertEquals("TestStudyName", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_NAME).get()); - } - - @Test - void getStudyAuthor() { - assertEquals("Jab Ref", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_AUTHORS).get()); - } - - @Test - void getResearchQuestions() { - assertEquals("Question1; Question2", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_RESEARCH_QUESTIONS).get()); - } - - @Test - void getGitRepositoryURL() { - assertEquals("https://github.com/eclipse/jgit.git", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_GIT_REPOSITORY).get()); - } -} diff --git a/src/test/resources/org/jabref/logic/crawler/study.bib b/src/test/resources/org/jabref/logic/crawler/study.bib deleted file mode 100644 index 3f9809a82e5..00000000000 --- a/src/test/resources/org/jabref/logic/crawler/study.bib +++ /dev/null @@ -1,37 +0,0 @@ -% Encoding: UTF-8 - -@Study{v10, - name={TestStudyName}, - author={Jab Ref}, - researchQuestions={Question1; Question2}, -} - -@SearchQuery{query1, - query={Quantum}, -} - -@SearchQuery{query2, - query={Cloud Computing}, -} - -@SearchQuery{query3, - query={TestSearchQuery3}, -} - -@Library{library1, - name = {Springer}, - enabled = {true}, - comment = {}, -} - -@Library{library2, - name = {ArXiv}, - enabled = {true}, - comment = {}, -} - -@Library{library3, - name = {IEEEXplore}, - enabled = {false}, - comment = {}, -} diff --git a/src/test/resources/org/jabref/logic/crawler/study.yml b/src/test/resources/org/jabref/logic/crawler/study.yml new file mode 100644 index 00000000000..620edaf0eab --- /dev/null +++ b/src/test/resources/org/jabref/logic/crawler/study.yml @@ -0,0 +1,16 @@ +authors: + - Jab Ref +title: TestStudyName +last-search-date: 2020-11-26 +research-questions: + - Question1 + - Question2 +queries: + - query: Quantum + - query: Cloud Computing + - query: '"Software Engineering"' +databases: + - name: Springer + - name: ArXiv + - name: IEEEXplore + enabled: false From 034cf8c975f2ba77ea2a283cb2301d2ef40cde3a Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Tue, 26 Jan 2021 23:43:31 +0100 Subject: [PATCH 06/25] Feature/implement complex queries (#7350) --- .../importer/PagedSearchBasedFetcher.java | 49 +++-- .../PagedSearchBasedParserFetcher.java | 37 +--- .../logic/importer/SearchBasedFetcher.java | 30 ++- .../importer/SearchBasedParserFetcher.java | 20 +- .../importer/fetcher/ACMPortalFetcher.java | 11 +- .../jabref/logic/importer/fetcher/ArXiv.java | 35 +-- .../fetcher/AstrophysicsDataSystem.java | 14 +- .../logic/importer/fetcher/CiteSeer.java | 6 +- ...fComputerScienceBibliographiesFetcher.java | 6 +- .../fetcher/CompositeSearchBasedFetcher.java | 25 +-- .../logic/importer/fetcher/CrossRef.java | 6 +- .../logic/importer/fetcher/DBLPFetcher.java | 6 +- .../logic/importer/fetcher/DOAJFetcher.java | 6 +- .../logic/importer/fetcher/GoogleScholar.java | 30 +-- .../fetcher/GrobidCitationFetcher.java | 45 ++-- .../logic/importer/fetcher/GvkFetcher.java | 41 +--- .../jabref/logic/importer/fetcher/IEEE.java | 54 ++--- .../importer/fetcher/INSPIREFetcher.java | 6 +- .../logic/importer/fetcher/JstorFetcher.java | 41 +--- .../logic/importer/fetcher/MathSciNet.java | 6 +- .../importer/fetcher/MedlineFetcher.java | 57 ++--- .../importer/fetcher/SpringerFetcher.java | 12 +- .../jabref/logic/importer/fetcher/ZbMATH.java | 6 +- .../AbstractQueryTransformer.java | 199 ++++++++++++++++++ .../transformators/ArXivQueryTransformer.java | 77 +++++++ .../transformators/DBLPQueryTransformer.java | 66 ++++++ .../DefaultQueryTransformer.java | 49 +++++ .../transformators/GVKQueryTransformer.java | 67 ++++++ .../transformators/IEEEQueryTransformer.java | 96 +++++++++ .../transformators/JstorQueryTransformer.java | 49 +++++ .../ScholarQueryTransformer.java | 65 ++++++ .../SpringerQueryTransformer.java | 62 ++++++ .../ZbMathQueryTransformer.java | 49 +++++ .../logic/importer/QueryParserTest.java | 2 +- .../logic/importer/fetcher/ArXivTest.java | 4 +- ...puterScienceBibliographiesFetcherTest.java | 9 +- .../CompositeSearchBasedFetcherTest.java | 4 +- .../importer/fetcher/DBLPFetcherTest.java | 3 +- .../fetcher/GrobidCitationFetcherTest.java | 3 +- .../importer/fetcher/GvkFetcherTest.java | 35 ++- .../logic/importer/fetcher/IEEETest.java | 29 +-- .../importer/fetcher/INSPIREFetcherTest.java | 2 +- .../SearchBasedFetcherCapabilityTest.java | 22 +- .../importer/fetcher/SpringerFetcherTest.java | 4 +- .../ArXivQueryTransformerTest.java | 60 ++++++ .../DBLPQueryTransformerTest.java | 54 +++++ .../GVKQueryTransformerTest.java | 54 +++++ .../IEEEQueryTransformerTest.java | 70 ++++++ .../transformators/InfixTransformerTest.java | 97 +++++++++ .../JstorQueryTransformerTest.java | 39 ++++ .../ScholarQueryTransformerTest.java | 59 ++++++ .../SpringerQueryTransformerTest.java | 56 +++++ .../ZbMathQueryTransformerTest.java | 54 +++++ 53 files changed, 1617 insertions(+), 371 deletions(-) create mode 100644 src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java create mode 100644 src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java create mode 100644 src/main/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformer.java create mode 100644 src/main/java/org/jabref/logic/importer/fetcher/transformators/DefaultQueryTransformer.java create mode 100644 src/main/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformer.java create mode 100644 src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java create mode 100644 src/main/java/org/jabref/logic/importer/fetcher/transformators/JstorQueryTransformer.java create mode 100644 src/main/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformer.java create mode 100644 src/main/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformer.java create mode 100644 src/main/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformer.java create mode 100644 src/test/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformerTest.java create mode 100644 src/test/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformerTest.java create mode 100644 src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java create mode 100644 src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java create mode 100644 src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java create mode 100644 src/test/java/org/jabref/logic/importer/fetcher/transformators/JstorQueryTransformerTest.java create mode 100644 src/test/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformerTest.java create mode 100644 src/test/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformerTest.java create mode 100644 src/test/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformerTest.java diff --git a/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java index ea547522049..a6b1cd4aea1 100644 --- a/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java +++ b/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java @@ -3,34 +3,40 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Optional; -import org.jabref.logic.importer.fetcher.ComplexSearchQuery; import org.jabref.model.entry.BibEntry; import org.jabref.model.paging.Page; +import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.core.parser.SyntaxParser; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; + public interface PagedSearchBasedFetcher extends SearchBasedFetcher { /** - * @param complexSearchQuery the complex query defining all fielded search parameters - * @param pageNumber requested site number indexed from 0 + * @param luceneQuery the root node of the lucene query + * @param pageNumber requested site number indexed from 0 * @return Page with search results */ - Page performSearchPaged(ComplexSearchQuery complexSearchQuery, int pageNumber) throws FetcherException; + Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException; /** - * @param complexSearchQuery query string that can be parsed into a complex search query - * @param pageNumber requested site number indexed from 0 + * @param searchQuery query string that can be parsed into a lucene query + * @param pageNumber requested site number indexed from 0 * @return Page with search results */ - default Page performSearchPaged(String complexSearchQuery, int pageNumber) throws FetcherException { - if (complexSearchQuery.isBlank()) { - return new Page<>(complexSearchQuery, pageNumber, Collections.emptyList()); + default Page performSearchPaged(String searchQuery, int pageNumber) throws FetcherException { + if (searchQuery.isBlank()) { + return new Page<>(searchQuery, pageNumber, Collections.emptyList()); + } + SyntaxParser parser = new StandardSyntaxParser(); + final String NO_EXPLICIT_FIELD = "default"; + try { + return this.performSearchPaged(parser.parse(searchQuery, NO_EXPLICIT_FIELD), pageNumber); + } catch (QueryNodeParseException e) { + throw new FetcherException("An error occurred during parsing of the query."); } - QueryParser queryParser = new QueryParser(); - Optional generatedQuery = queryParser.parseQueryStringIntoComplexQuery(complexSearchQuery); - // Otherwise just use query as a default term - return this.performSearchPaged(generatedQuery.orElse(ComplexSearchQuery.builder().defaultFieldPhrase(complexSearchQuery).build()), pageNumber); } /** @@ -40,13 +46,14 @@ default int getPageSize() { return 20; } - @Override - default List performSearch(ComplexSearchQuery complexSearchQuery) throws FetcherException { - return new ArrayList<>(performSearchPaged(complexSearchQuery, 0).getContent()); + /** + * This method is used to send complex queries using fielded search. + * + * @param luceneQuery the root node of the lucene query + * @return a list of {@link BibEntry}, which are matched by the query (may be empty) + */ + default List performSearch(QueryNode luceneQuery) throws FetcherException { + return new ArrayList<>(performSearchPaged(luceneQuery, 0).getContent()); } - @Override - default List performSearch(String complexSearchQuery) throws FetcherException { - return new ArrayList<>(performSearchPaged(complexSearchQuery, 0).getContent()); - } } diff --git a/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java index 7f7b3380f0b..bbbc848cc99 100644 --- a/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java @@ -7,22 +7,23 @@ import java.net.URL; import java.util.List; -import org.jabref.logic.importer.fetcher.ComplexSearchQuery; import org.jabref.model.entry.BibEntry; import org.jabref.model.paging.Page; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; + public interface PagedSearchBasedParserFetcher extends SearchBasedParserFetcher, PagedSearchBasedFetcher { @Override - default Page performSearchPaged(ComplexSearchQuery complexSearchQuery, int pageNumber) throws FetcherException { + default Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException { // ADR-0014 URL urlForQuery; try { - urlForQuery = getComplexQueryURL(complexSearchQuery, pageNumber); + urlForQuery = getURLForQuery(luceneQuery, pageNumber); } catch (URISyntaxException | MalformedURLException e) { throw new FetcherException("Search URI crafted from complex search query is malformed", e); } - return new Page<>(complexSearchQuery.toString(), pageNumber, getBibEntries(urlForQuery)); + return new Page<>(luceneQuery.toString(), pageNumber, getBibEntries(urlForQuery)); } private List getBibEntries(URL urlForQuery) throws FetcherException { @@ -39,34 +40,18 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { /** * Constructs a URL based on the query, size and page number. - * - * @param query the search query + * @param luceneQuery the search query * @param pageNumber the number of the page indexed from 0 */ - URL getURLForQuery(String query, int pageNumber) throws URISyntaxException, MalformedURLException; - - /** - * Constructs a URL based on the query, size and page number. - * - * @param complexSearchQuery the search query - * @param pageNumber the number of the page indexed from 0 - */ - default URL getComplexQueryURL(ComplexSearchQuery complexSearchQuery, int pageNumber) throws URISyntaxException, MalformedURLException { - return getURLForQuery(complexSearchQuery.toString(), pageNumber); - } - - @Override - default List performSearch(ComplexSearchQuery complexSearchQuery) throws FetcherException { - return SearchBasedParserFetcher.super.performSearch(complexSearchQuery); - } + URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException; @Override - default URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { - return getURLForQuery(query, 0); + default URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + return getURLForQuery(luceneQuery, 0); } @Override - default URL getURLForQuery(ComplexSearchQuery query) throws URISyntaxException, MalformedURLException, FetcherException { - return getComplexQueryURL(query, 0); + default List performSearch(QueryNode luceneQuery) throws FetcherException { + return SearchBasedParserFetcher.super.performSearch(luceneQuery); } } diff --git a/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java index faeb4ffa6f6..8b860c3eeeb 100644 --- a/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java +++ b/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java @@ -2,11 +2,16 @@ import java.util.Collections; import java.util.List; -import java.util.Optional; -import org.jabref.logic.importer.fetcher.ComplexSearchQuery; import org.jabref.model.entry.BibEntry; +import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.core.parser.SyntaxParser; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; + +import static org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer.NO_EXPLICIT_FIELD; + /** * Searches web resources for bibliographic information based on a free-text query. * May return multiple search hits. @@ -16,24 +21,27 @@ public interface SearchBasedFetcher extends WebFetcher { /** * This method is used to send complex queries using fielded search. * - * @param complexSearchQuery the complex search query defining all fielded search parameters + * @param luceneQuery the root node of the lucene query * @return a list of {@link BibEntry}, which are matched by the query (may be empty) */ - List performSearch(ComplexSearchQuery complexSearchQuery) throws FetcherException; + List performSearch(QueryNode luceneQuery) throws FetcherException; /** * Looks for hits which are matched by the given free-text query. * - * @param complexSearchQuery query string that can be parsed into a complex search query + * @param searchQuery query string that can be parsed into a lucene query * @return a list of {@link BibEntry}, which are matched by the query (may be empty) */ - default List performSearch(String complexSearchQuery) throws FetcherException { - if (complexSearchQuery.isBlank()) { + default List performSearch(String searchQuery) throws FetcherException { + if (searchQuery.isBlank()) { return Collections.emptyList(); } - QueryParser queryParser = new QueryParser(); - Optional generatedQuery = queryParser.parseQueryStringIntoComplexQuery(complexSearchQuery); - // Otherwise just use query as a default term - return this.performSearch(generatedQuery.orElse(ComplexSearchQuery.builder().defaultFieldPhrase(complexSearchQuery).build())); + SyntaxParser parser = new StandardSyntaxParser(); + + try { + return this.performSearch(parser.parse(searchQuery, NO_EXPLICIT_FIELD)); + } catch (QueryNodeParseException e) { + throw new FetcherException("An error occured when parsing the query"); + } } } diff --git a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java index d0817ade6a0..9aadba697b9 100644 --- a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java @@ -8,9 +8,10 @@ import java.util.List; import org.jabref.logic.cleanup.Formatter; -import org.jabref.logic.importer.fetcher.ComplexSearchQuery; import org.jabref.model.entry.BibEntry; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; + /** * Provides a convenient interface for search-based fetcher, which follow the usual three-step procedure: *

    @@ -26,14 +27,14 @@ public interface SearchBasedParserFetcher extends SearchBasedFetcher { * This method is necessary as the performSearch method does not support certain URL parameters that are used for * fielded search, such as a title, author, or year parameter. * - * @param complexSearchQuery the search query defining all fielded search parameters + * @param luceneQuery the root node of the lucene query */ @Override - default List performSearch(ComplexSearchQuery complexSearchQuery) throws FetcherException { + default List performSearch(QueryNode luceneQuery) throws FetcherException { // ADR-0014 URL urlForQuery; try { - urlForQuery = getURLForQuery(complexSearchQuery); + urlForQuery = getURLForQuery(luceneQuery); } catch (URISyntaxException | MalformedURLException | FetcherException e) { throw new FetcherException("Search URI crafted from complex search query is malformed", e); } @@ -52,22 +53,17 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { } } - default URL getURLForQuery(ComplexSearchQuery query) throws URISyntaxException, MalformedURLException, FetcherException { - // Default implementation behaves as getURLForQuery treating complex query as plain string query - return this.getURLForQuery(query.toString()); - } - /** * Returns the parser used to convert the response to a list of {@link BibEntry}. */ Parser getParser(); /** - * Constructs a URL based on the query. + * Constructs a URL based on the lucene query. * - * @param query the search query + * @param luceneQuery the root node of the lucene query */ - URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException; + URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException; /** * Performs a cleanup of the fetched entry. diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java index e36f730366b..d15e555e461 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java @@ -11,10 +11,12 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; public class ACMPortalFetcher implements SearchBasedParserFetcher { @@ -36,15 +38,16 @@ public Optional getHelpPage() { return Optional.of(HelpFile.FETCHER_ACM); } - private static String createQueryString(String query) { + private static String createQueryString(QueryNode query) throws FetcherException { + String queryString = new DefaultQueryTransformer().transformLuceneQuery(query).orElse(""); // Query syntax to search for an entry that matches "one" and "two" in any field is: (+one +two) - return "(%252B" + query.trim().replaceAll("\\s+", "%20%252B") + ")"; + return "(%252B" + queryString.trim().replaceAll("\\s+", "%20%252B") + ")"; } @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); - uriBuilder.addParameter("query", createQueryString(query)); // Search all fields + uriBuilder.addParameter("query", createQueryString(luceneQuery)); // Search all fields uriBuilder.addParameter("within", "owners.owner=GUIDE"); // Search within the ACM Guide to Computing Literature (encompasses the ACM Full-Text Collection) uriBuilder.addParameter("expformat", "bibtex"); // BibTeX format return uriBuilder.build().toURL(); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java b/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java index 9b640cdd7f0..e0569146721 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java @@ -23,6 +23,7 @@ import org.jabref.logic.importer.IdFetcher; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.PagedSearchBasedFetcher; +import org.jabref.logic.importer.fetcher.transformators.ArXivQueryTransformer; import org.jabref.logic.util.io.XMLUtil; import org.jabref.logic.util.strings.StringSimilarity; import org.jabref.model.entry.BibEntry; @@ -36,6 +37,7 @@ import org.jabref.model.util.OptionalUtil; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; @@ -252,25 +254,26 @@ public Optional getHelpPage() { /** * Constructs a complex query string using the field prefixes specified at https://arxiv.org/help/api/user-manual * - * @param complexSearchQuery the search query defining all fielded search parameters + * @param luceneQuery the root node of the lucene query * @return A list of entries matching the complex query */ @Override - public Page performSearchPaged(ComplexSearchQuery complexSearchQuery, int pageNumber) throws FetcherException { - List searchTerms = new ArrayList<>(); - complexSearchQuery.getAuthors().forEach(author -> searchTerms.add("au:" + author)); - complexSearchQuery.getTitlePhrases().forEach(title -> searchTerms.add("ti:" + title)); - complexSearchQuery.getAbstractPhrases().forEach(abstr -> searchTerms.add("abs:" + abstr)); - complexSearchQuery.getJournal().ifPresent(journal -> searchTerms.add("jr:" + journal)); - // Since ArXiv API does not support year search, we ignore the year related terms - complexSearchQuery.getToYear().ifPresent(year -> searchTerms.add(year.toString())); - searchTerms.addAll(complexSearchQuery.getDefaultFieldPhrases()); - String complexQueryString = String.join(" AND ", searchTerms); - - List searchResult = searchForEntries(complexQueryString, pageNumber).stream() - .map((arXivEntry) -> arXivEntry.toBibEntry(importFormatPreferences.getKeywordSeparator())) - .collect(Collectors.toList()); - return new Page<>(complexQueryString, pageNumber, searchResult); + public Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException { + ArXivQueryTransformer transformer = new ArXivQueryTransformer(); + String transformedQuery = transformer.transformLuceneQuery(luceneQuery).orElse(""); + List searchResult = searchForEntries(transformedQuery, pageNumber).stream() + .map((arXivEntry) -> arXivEntry.toBibEntry(importFormatPreferences.getKeywordSeparator())) + .collect(Collectors.toList()); + return new Page<>(transformedQuery, pageNumber, filterYears(searchResult, transformer)); + } + + private List filterYears(List searchResult, ArXivQueryTransformer transformer) { + return searchResult.stream() + .filter(entry -> entry.getField(StandardField.DATE).isPresent()) + // Filter the date field for year only + .filter(entry -> transformer.getEndYear().isEmpty() || Integer.parseInt(entry.getField(StandardField.DATE).get().substring(0, 4)) <= transformer.getEndYear().get()) + .filter(entry -> transformer.getStartYear().isEmpty() || Integer.parseInt(entry.getField(StandardField.DATE).get().substring(0, 4)) >= transformer.getStartYear().get()) + .collect(Collectors.toList()); } @Override diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java index a266418b351..6da8f0920d3 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java @@ -27,6 +27,7 @@ import org.jabref.logic.importer.PagedSearchBasedParserFetcher; import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; +import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.net.URLDownload; import org.jabref.logic.util.BuildInfo; @@ -41,6 +42,7 @@ import kong.unirest.json.JSONException; import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; /** * Fetches data from the SAO/NASA Astrophysics Data System (https://ui.adsabs.harvard.edu/) @@ -79,13 +81,13 @@ public String getName() { } /** - * @param query query string, matching the apache solr format + * @param luceneQuery query string, matching the apache solr format * @return URL which points to a search request for given query */ @Override - public URL getURLForQuery(String query, int pageNumber) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder builder = new URIBuilder(API_SEARCH_URL); - builder.addParameter("q", query); + builder.addParameter("q", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); builder.addParameter("fl", "bibcode"); builder.addParameter("rows", String.valueOf(getPageSize())); builder.addParameter("start", String.valueOf(getPageSize() * pageNumber)); @@ -274,12 +276,12 @@ private List performSearchByIds(Collection identifiers) throws } @Override - public Page performSearchPaged(ComplexSearchQuery complexSearchQuery, int pageNumber) throws FetcherException { + public Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException { try { // This is currently just interpreting the complex query as a default string query - List bibcodes = fetchBibcodes(getComplexQueryURL(complexSearchQuery, pageNumber)); + List bibcodes = fetchBibcodes(getURLForQuery(luceneQuery, pageNumber)); Collection results = performSearchByIds(bibcodes); - return new Page<>(complexSearchQuery.toString(), pageNumber, results); + return new Page<>(luceneQuery.toString(), pageNumber, results); } catch (URISyntaxException e) { throw new FetcherException("Search URI is malformed", e); } catch (IOException e) { diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java b/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java index 1b51cf04b00..c6e4c2b6ba5 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java @@ -20,6 +20,7 @@ import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.logic.importer.fileformat.CoinsParser; import org.jabref.logic.util.OS; import org.jabref.model.entry.BibEntry; @@ -27,6 +28,7 @@ import org.jabref.model.entry.field.StandardField; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; public class CiteSeer implements SearchBasedParserFetcher { @@ -44,10 +46,10 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder("https://citeseer.ist.psu.edu/search"); uriBuilder.addParameter("sort", "rlv"); // Sort by relevance - uriBuilder.addParameter("q", query); // Query + uriBuilder.addParameter("q", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // Query uriBuilder.addParameter("t", "doc"); // Type: documents // uriBuilder.addParameter("start", "0"); // Start index (not supported at the moment) return uriBuilder.build().toURL(); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java index 3bcbd26a2fb..b6556e0930f 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java @@ -14,6 +14,7 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; import org.jabref.model.entry.field.FieldFactory; @@ -21,6 +22,7 @@ import org.jabref.model.entry.field.UnknownField; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; public class CollectionOfComputerScienceBibliographiesFetcher implements SearchBasedParserFetcher { @@ -33,9 +35,9 @@ public CollectionOfComputerScienceBibliographiesFetcher(ImportFormatPreferences } @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { return new URIBuilder(BASIC_SEARCH_URL) - .addParameter("query", query) + .addParameter("query", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")) .addParameter("sort", "score") .build() .toURL(); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java index 3ce32ebeee6..c616b018cb1 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java @@ -13,6 +13,7 @@ import org.jabref.model.database.BibDatabaseMode; import org.jabref.model.entry.BibEntry; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,13 +37,23 @@ public CompositeSearchBasedFetcher(Set searchBasedFetchers, } @Override - public List performSearch(ComplexSearchQuery complexSearchQuery) { + public String getName() { + return "SearchAll"; + } + + @Override + public Optional getHelpPage() { + return Optional.empty(); + } + + @Override + public List performSearch(QueryNode luceneQuery) throws FetcherException { ImportCleanup cleanup = new ImportCleanup(BibDatabaseMode.BIBTEX); // All entries have to be converted into one format, this is necessary for the format conversion return fetchers.parallelStream() .flatMap(searchBasedFetcher -> { try { - return searchBasedFetcher.performSearch(complexSearchQuery).stream(); + return searchBasedFetcher.performSearch(luceneQuery).stream(); } catch (FetcherException e) { LOGGER.warn(String.format("%s API request failed", searchBasedFetcher.getName()), e); return Stream.empty(); @@ -52,14 +63,4 @@ public List performSearch(ComplexSearchQuery complexSearchQuery) { .map(cleanup::doPostCleanup) .collect(Collectors.toList()); } - - @Override - public String getName() { - return "SearchAll"; - } - - @Override - public Optional getHelpPage() { - return Optional.empty(); - } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java b/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java index fc7603ecd5f..b644c56659d 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java @@ -18,6 +18,7 @@ import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.logic.importer.util.JsonReader; import org.jabref.logic.util.strings.StringSimilarity; import org.jabref.model.entry.AuthorList; @@ -32,6 +33,7 @@ import kong.unirest.json.JSONException; import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; /** * A class for fetching DOIs from CrossRef @@ -63,9 +65,9 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(API_URL); - uriBuilder.addParameter("query", query); + uriBuilder.addParameter("query", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); return uriBuilder.build().toURL(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java index cb223ce061d..2d12d9820f5 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java @@ -16,6 +16,7 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fetcher.transformators.DBLPQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.layout.LayoutFormatterBasedFormatter; import org.jabref.logic.layout.format.RemoveLatexCommandsFormatter; @@ -24,6 +25,7 @@ import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; /** * Fetches BibTeX data from DBLP (dblp.org) @@ -42,9 +44,9 @@ public DBLPFetcher(ImportFormatPreferences importFormatPreferences) { } @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(BASIC_SEARCH_URL); - uriBuilder.addParameter("q", query); + uriBuilder.addParameter("q", new DBLPQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); uriBuilder.addParameter("h", String.valueOf(100)); // number of hits uriBuilder.addParameter("c", String.valueOf(0)); // no need for auto-completion uriBuilder.addParameter("f", String.valueOf(0)); // "from", index of first hit to download diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java index 39146f3f120..5f62756cdf7 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java @@ -16,6 +16,7 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.logic.util.OS; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; @@ -26,6 +27,7 @@ import kong.unirest.json.JSONArray; import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -188,9 +190,9 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); - DOAJFetcher.addPath(uriBuilder, query); + DOAJFetcher.addPath(uriBuilder, new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // Number of results uriBuilder.addParameter("pageSize", "30"); // Page (not needed so far) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java b/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java index 58d6c8546b7..ffd4f9a55e0 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java @@ -19,6 +19,7 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.PagedSearchBasedFetcher; import org.jabref.logic.importer.ParserResult; +import org.jabref.logic.importer.fetcher.transformators.ScholarQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.l10n.Localization; import org.jabref.logic.net.URLDownload; @@ -28,6 +29,7 @@ import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.select.Elements; @@ -127,16 +129,6 @@ public Optional getHelpPage() { return Optional.of(HelpFile.FETCHER_GOOGLE_SCHOLAR); } - private String constructComplexQueryString(ComplexSearchQuery complexSearchQuery) { - List searchTerms = new ArrayList<>(); - searchTerms.addAll(complexSearchQuery.getDefaultFieldPhrases()); - complexSearchQuery.getAuthors().forEach(author -> searchTerms.add("author:" + author)); - searchTerms.add("allintitle:" + String.join(" ", complexSearchQuery.getTitlePhrases())); - complexSearchQuery.getJournal().ifPresent(journal -> searchTerms.add("source:" + journal)); - // API automatically ANDs the terms - return String.join(" ", searchTerms); - } - private void addHitsFromQuery(List entryList, String queryURL) throws IOException, FetcherException { String content = new URLDownload(queryURL).asString(); @@ -185,24 +177,20 @@ private void obtainAndModifyCookie() throws FetcherException { } @Override - public Page performSearchPaged(ComplexSearchQuery complexSearchQuery, int pageNumber) throws FetcherException { + public Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException { + ScholarQueryTransformer queryTransformer = new ScholarQueryTransformer(); + String transformedQuery = queryTransformer.transformLuceneQuery(luceneQuery).orElse(""); try { obtainAndModifyCookie(); List foundEntries = new ArrayList<>(10); - - String complexQueryString = constructComplexQueryString(complexSearchQuery); URIBuilder uriBuilder = new URIBuilder(BASIC_SEARCH_URL); uriBuilder.addParameter("hl", "en"); uriBuilder.addParameter("btnG", "Search"); - uriBuilder.addParameter("q", complexQueryString); + uriBuilder.addParameter("q", transformedQuery); uriBuilder.addParameter("start", String.valueOf(pageNumber * getPageSize())); uriBuilder.addParameter("num", String.valueOf(getPageSize())); - complexSearchQuery.getFromYear().ifPresent(year -> uriBuilder.addParameter("as_ylo", year.toString())); - complexSearchQuery.getToYear().ifPresent(year -> uriBuilder.addParameter("as_yhi", year.toString())); - complexSearchQuery.getSingleYear().ifPresent(year -> { - uriBuilder.addParameter("as_ylo", year.toString()); - uriBuilder.addParameter("as_yhi", year.toString()); - }); + uriBuilder.addParameter("as_ylo", String.valueOf(queryTransformer.getStartYear())); + uriBuilder.addParameter("as_yhi", String.valueOf(queryTransformer.getEndYear())); try { addHitsFromQuery(foundEntries, uriBuilder.toString()); @@ -223,7 +211,7 @@ public Page performSearchPaged(ComplexSearchQuery complexSearchQuery, throw new FetcherException("Error while fetching from " + getName(), e); } } - return new Page<>(complexQueryString, pageNumber, foundEntries); + return new Page<>(transformedQuery, pageNumber, foundEntries); } catch (URISyntaxException e) { throw new FetcherException("Error while fetching from " + getName(), e); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java index 61218575668..bf16d71570d 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.net.SocketTimeoutException; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -16,6 +17,7 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.util.DummyFileUpdateMonitor; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,7 +44,7 @@ public GrobidCitationFetcher(ImportFormatPreferences importFormatPreferences) { * * @return A BibTeX string if extraction is successful */ - private Optional parseUsingGrobid(String plainText) { + private Optional parseUsingGrobid(String plainText) throws RuntimeException { try { return Optional.of(grobidService.processCitation(plainText, GrobidService.ConsolidateCitations.WITH_METADATA)); } catch (SocketTimeoutException e) { @@ -52,7 +54,7 @@ private Optional parseUsingGrobid(String plainText) { } catch (IOException e) { String msg = "Could not process citation. " + e.getMessage(); LOGGER.debug(msg, e); - throw new RuntimeException(msg, e); + return Optional.empty(); } } @@ -66,30 +68,33 @@ private Optional parseBibToBibEntry(String bibtexString) { } @Override - public List performSearch(ComplexSearchQuery complexSearchQuery) throws FetcherException { - List bibEntries = null; - // This just treats the complex query like a normal string query until it it implemented correctly - String query = complexSearchQuery.toString(); + public String getName() { + return "GROBID"; + } + + @Override + public List performSearch(String searchQuery) throws FetcherException { + List collect; try { - bibEntries = Arrays - .stream(query.split("\\r\\r+|\\n\\n+|\\r\\n(\\r\\n)+")) - .map(String::trim) - .filter(str -> !str.isBlank()) - .map(this::parseUsingGrobid) - .flatMap(Optional::stream) - .map(this::parseBibToBibEntry) - .flatMap(Optional::stream) - .collect(Collectors.toList()); + collect = Arrays.stream(searchQuery.split("\\r\\r+|\\n\\n+|\\r\\n(\\r\\n)+")) + .map(String::trim) + .filter(str -> !str.isBlank()) + .map(this::parseUsingGrobid) + .flatMap(Optional::stream) + .map(this::parseBibToBibEntry) + .flatMap(Optional::stream) + .collect(Collectors.toList()); } catch (RuntimeException e) { - // un-wrap the wrapped exceptions throw new FetcherException(e.getMessage(), e.getCause()); } - return bibEntries; + return collect; } + /** + * Not used + */ @Override - public String getName() { - return "GROBID"; + public List performSearch(QueryNode luceneQuery) throws FetcherException { + return Collections.emptyList(); } - } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java index 20b4fe34630..db6afbc6606 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java @@ -5,19 +5,17 @@ import java.net.URL; import java.util.Arrays; import java.util.Collection; -import java.util.LinkedList; -import java.util.List; -import java.util.Objects; import java.util.Optional; -import java.util.stream.Collectors; import org.jabref.logic.help.HelpFile; import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fetcher.transformators.GVKQueryTransformer; import org.jabref.logic.importer.fileformat.GvkParser; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; public class GvkFetcher implements SearchBasedParserFetcher { @@ -39,43 +37,12 @@ public Optional getHelpPage() { return Optional.of(HelpFile.FETCHER_GVK); } - private String getSearchQueryStringForComplexQuery(List queryList) { - String query = ""; - boolean lastWasNoKey = false; - - for (String key : queryList) { - if (searchKeys.contains(key)) { - if (lastWasNoKey) { - query = query + "and "; - } - query = query + "pica." + key + "="; - } else { - query = query + key + " "; - lastWasNoKey = true; - } - } - return query.trim(); - } - - protected String getSearchQueryString(String query) { - Objects.requireNonNull(query); - LinkedList queryList = new LinkedList<>(Arrays.asList(query.split("\\s"))); - - if (searchKeys.contains(queryList.get(0))) { - return getSearchQueryStringForComplexQuery(queryList); - } else { - // query as pica.all - return queryList.stream().collect(Collectors.joining(" ", "pica.all=", "")); - } - } - @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { - String gvkQuery = getSearchQueryString(query); + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(URL_PATTERN); uriBuilder.addParameter("version", "1.1"); uriBuilder.addParameter("operation", "searchRetrieve"); - uriBuilder.addParameter("query", gvkQuery); + uriBuilder.addParameter("query", new GVKQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); uriBuilder.addParameter("maximumRecords", "50"); uriBuilder.addParameter("recordSchema", "picaxml"); uriBuilder.addParameter("sortKeys", "Year,,1"); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java index d89223908a0..9cd5341d357 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java @@ -15,10 +15,12 @@ import java.util.stream.Collectors; import org.jabref.logic.help.HelpFile; +import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.FulltextFetcher; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.PagedSearchBasedParserFetcher; import org.jabref.logic.importer.Parser; +import org.jabref.logic.importer.fetcher.transformators.IEEEQueryTransformer; import org.jabref.logic.net.URLDownload; import org.jabref.logic.util.BuildInfo; import org.jabref.logic.util.OS; @@ -31,6 +33,7 @@ import kong.unirest.json.JSONArray; import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -222,48 +225,33 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String query, int pageNumber) throws URISyntaxException, MalformedURLException { - URIBuilder uriBuilder = new URIBuilder("https://ieeexploreapi.ieee.org/api/v1/search/articles"); - uriBuilder.addParameter("apikey", API_KEY); - uriBuilder.addParameter("querytext", query); - uriBuilder.addParameter("max_records", String.valueOf(getPageSize())); - // Starts to index at 1 for the first entry - uriBuilder.addParameter("start_record", String.valueOf(getPageSize() * pageNumber) + 1); - - URLDownload.bypassSSLVerification(); - - return uriBuilder.build().toURL(); - } - - @Override - public URL getComplexQueryURL(ComplexSearchQuery complexSearchQuery, int pageNumber) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { + IEEEQueryTransformer transformer = new IEEEQueryTransformer(); + String transformedQuery = transformer.transformLuceneQuery(luceneQuery).orElse(""); URIBuilder uriBuilder = new URIBuilder("https://ieeexploreapi.ieee.org/api/v1/search/articles"); uriBuilder.addParameter("apikey", API_KEY); + if (!transformedQuery.isBlank()) { + uriBuilder.addParameter("querytext", transformedQuery); + } uriBuilder.addParameter("max_records", String.valueOf(getPageSize())); - // Starts to index at 1 for the first entry - uriBuilder.addParameter("start_record", String.valueOf(getPageSize() * pageNumber) + 1); - - if (!complexSearchQuery.getDefaultFieldPhrases().isEmpty()) { - uriBuilder.addParameter("querytext", String.join(" AND ", complexSearchQuery.getDefaultFieldPhrases())); + // Currently not working as part of the query string + if (transformer.getJournal().isPresent()) { + uriBuilder.addParameter("publication_title", transformer.getJournal().get()); } - if (!complexSearchQuery.getAuthors().isEmpty()) { - uriBuilder.addParameter("author", String.join(" AND ", complexSearchQuery.getAuthors())); + if (transformer.getStartYear().isPresent()) { + uriBuilder.addParameter("start_year", String.valueOf(transformer.getStartYear().get())); } - if (!complexSearchQuery.getAbstractPhrases().isEmpty()) { - uriBuilder.addParameter("abstract", String.join(" AND ", complexSearchQuery.getAbstractPhrases())); + if (transformer.getEndYear().isPresent()) { + uriBuilder.addParameter("end_year", String.valueOf(transformer.getEndYear().get())); } - if (!complexSearchQuery.getTitlePhrases().isEmpty()) { - uriBuilder.addParameter("article_title", String.join(" AND ", complexSearchQuery.getTitlePhrases())); + if (transformer.getArticleNumber().isPresent()) { + uriBuilder.addParameter("article_number", transformer.getArticleNumber().get()); } - complexSearchQuery.getJournal().ifPresent(journalTitle -> uriBuilder.addParameter("publication_title", journalTitle)); - complexSearchQuery.getFromYear().map(String::valueOf).ifPresent(year -> uriBuilder.addParameter("start_year", year)); - complexSearchQuery.getToYear().map(String::valueOf).ifPresent(year -> uriBuilder.addParameter("end_year", year)); - complexSearchQuery.getSingleYear().map(String::valueOf).ifPresent(year -> { - uriBuilder.addParameter("start_year", year); - uriBuilder.addParameter("end_year", year); - }); + // Starts to index at 1 for the first entry + uriBuilder.addParameter("start_record", String.valueOf(getPageSize() * pageNumber) + 1); URLDownload.bypassSSLVerification(); + return uriBuilder.build().toURL(); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java index d7239dee240..c594ebb5efc 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java @@ -13,6 +13,7 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.importer.util.MediaTypes; import org.jabref.logic.layout.format.LatexToUnicodeFormatter; @@ -23,6 +24,7 @@ import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; /** * Fetches data from the INSPIRE database. @@ -48,9 +50,9 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(INSPIRE_HOST); - uriBuilder.addParameter("q", query); // Query + uriBuilder.addParameter("q", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // Query return uriBuilder.build().toURL(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java index 6315e708c52..4c2f49a16d5 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java @@ -19,6 +19,7 @@ import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.net.URLDownload; import org.jabref.model.entry.BibEntry; @@ -26,6 +27,7 @@ import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -47,44 +49,9 @@ public JstorFetcher(ImportFormatPreferences importFormatPreferences) { } @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(SEARCH_HOST); - uriBuilder.addParameter("Query", query); - return uriBuilder.build().toURL(); - } - - @Override - public URL getURLForQuery(ComplexSearchQuery query) throws URISyntaxException, MalformedURLException, FetcherException { - URIBuilder uriBuilder = new URIBuilder(SEARCH_HOST); - StringBuilder stringBuilder = new StringBuilder(); - if (!query.getDefaultFieldPhrases().isEmpty()) { - stringBuilder.append(query.getDefaultFieldPhrases()); - } - if (!query.getAuthors().isEmpty()) { - for (String author : query.getAuthors()) { - stringBuilder.append("au:").append(author); - } - } - if (!query.getTitlePhrases().isEmpty()) { - for (String title : query.getTitlePhrases()) { - stringBuilder.append("ti:").append(title); - } - } - if (query.getJournal().isPresent()) { - stringBuilder.append("pt:").append(query.getJournal().get()); - } - if (query.getSingleYear().isPresent()) { - uriBuilder.addParameter("sd", String.valueOf(query.getSingleYear().get())); - uriBuilder.addParameter("ed", String.valueOf(query.getSingleYear().get())); - } - if (query.getFromYear().isPresent()) { - uriBuilder.addParameter("sd", String.valueOf(query.getFromYear().get())); - } - if (query.getToYear().isPresent()) { - uriBuilder.addParameter("ed", String.valueOf(query.getToYear().get())); - } - - uriBuilder.addParameter("Query", stringBuilder.toString()); + uriBuilder.addParameter("Query", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); return uriBuilder.build().toURL(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java b/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java index 3ff0bd243b8..bfc2ca3ab8c 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java @@ -23,6 +23,7 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.util.OS; import org.jabref.model.entry.BibEntry; @@ -31,6 +32,7 @@ import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; /** * Fetches data from the MathSciNet (http://www.ams.org/mathscinet) @@ -72,10 +74,10 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder("https://mathscinet.ams.org/mathscinet/search/publications.html"); uriBuilder.addParameter("pg7", "ALLF"); // search all fields - uriBuilder.addParameter("s7", query); // query + uriBuilder.addParameter("s7", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // query uriBuilder.addParameter("r", "1"); // start index uriBuilder.addParameter("extend", "1"); // should return up to 100 items (instead of default 10) uriBuilder.addParameter("fmt", "bibtex"); // BibTeX format diff --git a/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java index a4553bc876b..304b76da38a 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java @@ -28,6 +28,7 @@ import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.ParserResult; import org.jabref.logic.importer.SearchBasedFetcher; +import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.logic.importer.fileformat.MedlineImporter; import org.jabref.logic.l10n.Localization; import org.jabref.model.entry.BibEntry; @@ -35,6 +36,7 @@ import org.jabref.model.entry.field.UnknownField; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -157,33 +159,6 @@ public void doPostCleanup(BibEntry entry) { new FieldFormatterCleanup(StandardField.AUTHOR, new NormalizeNamesFormatter()).cleanup(entry); } - @Override - public List performSearch(ComplexSearchQuery complexSearchQuery) throws FetcherException { - List entryList; - String query = complexSearchQuery.toString(); - - if (query.isBlank()) { - return Collections.emptyList(); - } else { - // searching for pubmed ids matching the query - List idList = getPubMedIdsFromQuery(query); - - if (idList.isEmpty()) { - LOGGER.info("No results found."); - return Collections.emptyList(); - } - if (numberOfResultsFound > NUMBER_TO_FETCH) { - LOGGER.info( - numberOfResultsFound + " results found. Only 50 relevant results will be fetched by default."); - } - - // pass the list of ids to fetchMedline to download them. like a id fetcher for mutliple ids - entryList = fetchMedline(idList); - - return entryList; - } - } - private URL createSearchUrl(String query) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); uriBuilder.addParameter("db", "pubmed"); @@ -221,4 +196,32 @@ private List fetchMedline(List ids) throws FetcherException { Localization.lang("Error while fetching from %0", "Medline"), e); } } + + @Override + public List performSearch(QueryNode luceneQuery) throws FetcherException { + List entryList; + DefaultQueryTransformer transformer = new DefaultQueryTransformer(); + Optional transformedQuery = transformer.transformLuceneQuery(luceneQuery); + + if (transformedQuery.isEmpty() || transformedQuery.get().isBlank()) { + return Collections.emptyList(); + } else { + // searching for pubmed ids matching the query + List idList = getPubMedIdsFromQuery(transformedQuery.get()); + + if (idList.isEmpty()) { + LOGGER.info("No results found."); + return Collections.emptyList(); + } + if (numberOfResultsFound > NUMBER_TO_FETCH) { + LOGGER.info( + numberOfResultsFound + " results found. Only 50 relevant results will be fetched by default."); + } + + // pass the list of ids to fetchMedline to download them. like a id fetcher for mutliple ids + entryList = fetchMedline(idList); + + return entryList; + } + } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java index 1064a7f272e..5c3b34b6d01 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java @@ -11,8 +11,10 @@ import java.util.stream.Collectors; import org.jabref.logic.help.HelpFile; +import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.PagedSearchBasedParserFetcher; import org.jabref.logic.importer.Parser; +import org.jabref.logic.importer.fetcher.transformators.SpringerQueryTransformer; import org.jabref.logic.util.BuildInfo; import org.jabref.logic.util.OS; import org.jabref.model.entry.BibEntry; @@ -25,6 +27,7 @@ import kong.unirest.json.JSONArray; import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -158,20 +161,15 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String query, int pageNumber) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(API_URL); - uriBuilder.addParameter("q", query); // Search query + uriBuilder.addParameter("q", new SpringerQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // Search query uriBuilder.addParameter("api_key", API_KEY); // API key uriBuilder.addParameter("s", String.valueOf(getPageSize() * pageNumber + 1)); // Start entry, starts indexing at 1 uriBuilder.addParameter("p", String.valueOf(getPageSize())); // Page size return uriBuilder.build().toURL(); } - @Override - public URL getComplexQueryURL(ComplexSearchQuery complexSearchQuery, int pageNumber) throws URISyntaxException, MalformedURLException { - return getURLForQuery(constructComplexQueryString(complexSearchQuery), pageNumber); - } - private String constructComplexQueryString(ComplexSearchQuery complexSearchQuery) { List searchTerms = new ArrayList<>(); complexSearchQuery.getAuthors().forEach(author -> searchTerms.add("name:" + author)); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java b/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java index 31105ede0ab..69ea0c81b2a 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java @@ -13,6 +13,7 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fetcher.transformators.ZbMathQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.net.URLDownload; import org.jabref.model.entry.BibEntry; @@ -21,6 +22,7 @@ import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; /** * Fetches data from the Zentralblatt Math (https://www.zbmath.org/) @@ -50,9 +52,9 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } */ @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder("https://zbmath.org/bibtexoutput/"); - uriBuilder.addParameter("q", query); // search all fields + uriBuilder.addParameter("q", new ZbMathQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // search all fields uriBuilder.addParameter("start", "0"); // start index uriBuilder.addParameter("count", "200"); // should return up to 200 items (instead of default 100) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java new file mode 100644 index 00000000000..32c7a6b8d18 --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java @@ -0,0 +1,199 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import java.util.Optional; +import java.util.stream.Collectors; + +import org.apache.lucene.queryparser.flexible.core.nodes.BooleanQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.FieldQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.GroupQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.ModifierQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.OrQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * In case the transformator contains state for a query transformation (such as the {@link IEEEQueryTransformer}), it has to be noted at the JavaDoc. + * Otherwise, a single instance QueryTransformer can be used. + */ +public abstract class AbstractQueryTransformer { + public static final String NO_EXPLICIT_FIELD = "default"; + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractQueryTransformer.class); + + /** + * Transforms a and b and c to (a AND b AND c), where + * a, b, and c can be complex expressions. + */ + private Optional transform(BooleanQueryNode query) { + String delimiter; + if (query instanceof OrQueryNode) { + delimiter = getLogicalOrOperator(); + } else { + // We define the logical AND operator as the default implementation + delimiter = getLogicalAndOperator(); + } + + String result = query.getChildren().stream() + .map(this::transform) + .flatMap(Optional::stream) + .collect(Collectors.joining(delimiter, "(", ")")); + if (result.equals("()")) { + return Optional.empty(); + } + return Optional.of(result); + } + + /** + * Returns the logical AND operator used by the library + * Note: whitespaces have to be included around the operator + * + * Example: " AND " + */ + protected abstract String getLogicalAndOperator(); + + /** + * Returns the logical OR operator used by the library + * Note: whitespaces have to be included around the operator + * + * Example: " OR " + */ + protected abstract String getLogicalOrOperator(); + + /** + * Returns the logical NOT operator used by the library + * + * Example: "!" + */ + protected abstract String getLogicalNotOperator(); + + private Optional transform(FieldQueryNode query) { + String term = query.getTextAsString(); + switch (query.getFieldAsString()) { + case "author" -> { + return Optional.of(handleAuthor(term)); + } + case "title" -> { + return Optional.of(handleTitle(term)); + } + case "journal" -> { + return Optional.of(handleJournal(term)); + } + case "year" -> { + String s = handleYear(term); + return s.isEmpty() ? Optional.empty() : Optional.of(s); + } + case "year-range" -> { + String s = handleYearRange(term); + return s.isEmpty() ? Optional.empty() : Optional.of(s); + } + case "doi" -> { + String s = handleDoi(term); + return s.isEmpty() ? Optional.empty() : Optional.of(s); + } + case NO_EXPLICIT_FIELD -> { + return Optional.of(handleUnFieldedTerm(term)); + } + default -> { + // Just add unknown fields as default + return handleOtherField(query.getFieldAsString(), term); + } + } + } + + protected String handleDoi(String term) { + return "doi:" + term; + } + + /** + * Handles the not modifier, all other cases are silently ignored + */ + private Optional transform(ModifierQueryNode query) { + ModifierQueryNode.Modifier modifier = query.getModifier(); + if (modifier == ModifierQueryNode.Modifier.MOD_NOT) { + return transform(query.getChild()).map(s -> getLogicalNotOperator() + s); + } else { + return transform(query.getChild()); + } + } + + /** + * Return a string representation of the author fielded term + */ + protected abstract String handleAuthor(String author); + + /** + * Return a string representation of the title fielded term + */ + protected abstract String handleTitle(String title); + + /** + * Return a string representation of the journal fielded term + */ + protected abstract String handleJournal(String journalTitle); + + /** + * Return a string representation of the year fielded term + */ + protected abstract String handleYear(String year); + + /** + * Return a string representation of the year-range fielded term + * Should follow the structure yyyy-yyyy + * + * Example: 2015-2021 + */ + protected abstract String handleYearRange(String yearRange); + + /** + * Return a string representation of the un-fielded (default fielded) term + */ + protected abstract String handleUnFieldedTerm(String term); + + /** + * Return a string representation of the provided field + * If it is not supported return an empty optional. + */ + protected Optional handleOtherField(String fieldAsString, String term) { + return Optional.of(String.format("%s:\"%s\"", fieldAsString, term)); + } + + private Optional transform(QueryNode query) { + if (query instanceof BooleanQueryNode) { + return transform((BooleanQueryNode) query); + } else if (query instanceof FieldQueryNode) { + return transform((FieldQueryNode) query); + } else if (query instanceof GroupQueryNode) { + return transform(((GroupQueryNode) query).getChild()); + } else if (query instanceof ModifierQueryNode) { + return transform((ModifierQueryNode) query); + } else { + LOGGER.error("Unsupported case when transforming the query:\n {}", query); + return Optional.empty(); + } + } + + /** + * Parses the given query string into a complex query using lucene. + * Note: For unique fields, the alphabetically and numerically first instance in the query string is used in the complex query. + * + * @param luceneQuery The lucene query tp transform + * @return A query string containing all fields that are contained in the original lucene query and + * that are expressible in the library specific query language, other information either is discarded or + * stored as part of the state of the transformer if it can be used e.g. as a URL parameter for the query. + */ + public Optional transformLuceneQuery(QueryNode luceneQuery) { + Optional transformedQuery = transform(luceneQuery); + transformedQuery = transformedQuery.map(this::removeOuterBraces); + return transformedQuery; + } + + /** + * Removes the outer braces as they are unnecessary + */ + private String removeOuterBraces(String query) { + if (query.startsWith("(") && query.endsWith(")")) { + return query.substring(1, query.length() - 1); + } + return query; + } +} diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java new file mode 100644 index 00000000000..dd94eab746b --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java @@ -0,0 +1,77 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import java.util.Optional; + +public class ArXivQueryTransformer extends AbstractQueryTransformer { + // These can be used for filtering in post processing + private int startYear = Integer.MAX_VALUE; + private int endYear = Integer.MIN_VALUE; + + @Override + protected String getLogicalAndOperator() { + return " AND "; + } + + @Override + protected String getLogicalOrOperator() { + return " OR "; + } + + /** + * Check whether this works as an unary operator + * @return + */ + @Override + protected String getLogicalNotOperator() { + return " ANDNOT "; + } + + @Override + protected String handleAuthor(String author) { + return String.format("au:\"%s\"", author); + } + + @Override + protected String handleTitle(String title) { + return String.format("ti:\"%s\"", title); + } + + @Override + protected String handleJournal(String journalTitle) { + return String.format("jr:\"%s\"", journalTitle); + } + + /** + * Manual testing shows that this works if added as an unfielded term, might lead to false positives + */ + @Override + protected String handleYear(String year) { + startYear = Math.min(startYear, Integer.parseInt(year)); + endYear = Math.max(endYear, Integer.parseInt(year)); + return year; + } + + /** + * Currently not supported + */ + @Override + protected String handleYearRange(String yearRange) { + String[] split = yearRange.split("-"); + startYear = Math.min(startYear, Integer.parseInt(split[0])); + endYear = Math.max(endYear, Integer.parseInt(split[1])); + return ""; + } + + @Override + protected String handleUnFieldedTerm(String term) { + return String.format("all:\"%s\"", term); + } + + public Optional getStartYear() { + return startYear == Integer.MAX_VALUE ? Optional.empty() : Optional.of(startYear); + } + + public Optional getEndYear() { + return endYear == Integer.MIN_VALUE ? Optional.empty() : Optional.of(endYear); + } +} diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformer.java new file mode 100644 index 00000000000..df943509320 --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformer.java @@ -0,0 +1,66 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import java.util.StringJoiner; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DBLPQueryTransformer extends AbstractQueryTransformer { + private static final Logger LOGGER = LoggerFactory.getLogger(DBLPQueryTransformer.class); + + @Override + protected String getLogicalAndOperator() { + return " "; + } + + @Override + protected String getLogicalOrOperator() { + return "|"; + } + + @Override + protected String getLogicalNotOperator() { + LOGGER.warn("DBLP does not support Boolean NOT operator."); + return ""; + } + + @Override + protected String handleAuthor(String author) { + // DBLP does not support explicit author field search + return String.format("\"%s\"", author); + } + + @Override + protected String handleTitle(String title) { + // DBLP does not support explicit title field search + return String.format("\"%s\"", title); + } + + @Override + protected String handleJournal(String journalTitle) { + // DBLP does not support explicit journal field search + return String.format("\"%s\"", journalTitle); + } + + @Override + protected String handleYear(String year) { + // DBLP does not support explicit year field search + return year; + } + + @Override + protected String handleYearRange(String yearRange) { + // DBLP does not support explicit year range search + String[] split = yearRange.split("-"); + StringJoiner resultBuilder = new StringJoiner(getLogicalOrOperator()); + for (int i = Integer.parseInt(split[0]); i <= Integer.parseInt(split[1]); i++) { + resultBuilder.add(String.valueOf(i)); + } + return resultBuilder.toString(); + } + + @Override + protected String handleUnFieldedTerm(String term) { + return String.format("\"%s\"", term); + } +} diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/DefaultQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/DefaultQueryTransformer.java new file mode 100644 index 00000000000..b99e327269c --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/DefaultQueryTransformer.java @@ -0,0 +1,49 @@ +package org.jabref.logic.importer.fetcher.transformators; + +public class DefaultQueryTransformer extends AbstractQueryTransformer { + + @Override + protected String getLogicalAndOperator() { + return " "; + } + + @Override + protected String getLogicalOrOperator() { + return " "; + } + + @Override + protected String getLogicalNotOperator() { + return ""; + } + + @Override + protected String handleAuthor(String author) { + return author; + } + + @Override + protected String handleTitle(String title) { + return title; + } + + @Override + protected String handleJournal(String journalTitle) { + return journalTitle; + } + + @Override + protected String handleYear(String year) { + return year; + } + + @Override + protected String handleYearRange(String yearRange) { + return yearRange; + } + + @Override + protected String handleUnFieldedTerm(String term) { + return term; + } +} diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformer.java new file mode 100644 index 00000000000..402e68b09c0 --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformer.java @@ -0,0 +1,67 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GVKQueryTransformer extends AbstractQueryTransformer { + private static final Logger LOGGER = LoggerFactory.getLogger(GVKQueryTransformer.class); + + @Override + protected String getLogicalAndOperator() { + return " and "; + } + + @Override + protected String getLogicalOrOperator() { + LOGGER.warn("GVK does not support Boolean OR operator"); + return ""; + } + + @Override + protected String getLogicalNotOperator() { + LOGGER.warn("GVK does not support Boolean NOT operator"); + return ""; + } + + @Override + protected String handleAuthor(String author) { + return String.format("pica.per=\"%s\"", author); + } + + @Override + protected String handleTitle(String title) { + return String.format("pica.tit=\"%s\"", title); + } + + @Override + protected String handleJournal(String journalTitle) { + // zti means "Zeitschrift", does not search for conferences (kon:) + return String.format("pica.zti=\"%s\"", journalTitle); + } + + @Override + protected String handleYear(String year) { + // ver means Veröffentlichungsangaben + return "pica.ver=" + year; + } + + @Override + protected String handleYearRange(String yearRange) { + // Returns empty string as otherwise leads to no results + return ""; + } + + @Override + protected String handleUnFieldedTerm(String term) { + // all does not search in full-text + // Other option is txt: but this does not search in meta data + return String.format("pica.all=\"%s\"", term); + } + + @Override + protected Optional handleOtherField(String fieldAsString, String term) { + return Optional.of("pica." + fieldAsString + "=\"" + term + "\""); + } +} diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java new file mode 100644 index 00000000000..a6bb4cd49c4 --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java @@ -0,0 +1,96 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import java.util.Objects; +import java.util.Optional; + +/** + * Needs to be instantiated for each new query + */ +public class IEEEQueryTransformer extends AbstractQueryTransformer { + // These have to be integrated into the IEEE query URL as these are just supported as query parameters + // Journal is wrapped in quotes by the transformer + private String journal; + private String articleNumber; + private int startYear = Integer.MAX_VALUE; + private int endYear = Integer.MIN_VALUE; + + @Override + protected String getLogicalAndOperator() { + return " AND "; + } + + @Override + protected String getLogicalOrOperator() { + return " OR "; + } + + @Override + protected String getLogicalNotOperator() { + return "NOT "; + } + + @Override + protected String handleAuthor(String author) { + return String.format("author:\"%s\"", author); + } + + @Override + protected String handleTitle(String title) { + return String.format("article_title:\"%s\"", title); + } + + @Override + protected String handleJournal(String journalTitle) { + journal = String.format("\"%s\"", journalTitle); + return ""; + } + + @Override + protected String handleYear(String year) { + startYear = Math.min(startYear, Integer.parseInt(year)); + endYear = Math.max(endYear, Integer.parseInt(year)); + return ""; + } + + @Override + protected String handleYearRange(String yearRange) { + String[] split = yearRange.split("-"); + startYear = Math.min(startYear, Integer.parseInt(split[0])); + endYear = Math.max(endYear, Integer.parseInt(split[1])); + return ""; + } + + @Override + protected String handleUnFieldedTerm(String term) { + return String.format("\"%s\"", term); + } + + @Override + protected Optional handleOtherField(String fieldAsString, String term) { + return switch (fieldAsString) { + case "article_number" -> handleArticleNumber(term); + default -> super.handleOtherField(fieldAsString, term); + }; + } + + private Optional handleArticleNumber(String term) { + articleNumber = term; + return Optional.empty(); + } + + public Optional getStartYear() { + return startYear == Integer.MAX_VALUE ? Optional.empty() : Optional.of(startYear); + } + + public Optional getEndYear() { + return endYear == Integer.MIN_VALUE ? Optional.empty() : Optional.of(endYear); + } + + public Optional getJournal() { + return Objects.isNull(journal) ? Optional.empty() : Optional.of(journal); + } + + public Optional getArticleNumber() { + return Objects.isNull(articleNumber) ? Optional.empty() : Optional.of(articleNumber); + } +} diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/JstorQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/JstorQueryTransformer.java new file mode 100644 index 00000000000..1c7be5db728 --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/JstorQueryTransformer.java @@ -0,0 +1,49 @@ +package org.jabref.logic.importer.fetcher.transformators; + +public class JstorQueryTransformer extends AbstractQueryTransformer { + @Override + protected String getLogicalAndOperator() { + return " AND "; + } + + @Override + protected String getLogicalOrOperator() { + return " OR "; + } + + @Override + protected String getLogicalNotOperator() { + return "NOT "; + } + + @Override + protected String handleAuthor(String author) { + return String.format("au:\"%s\"", author); + } + + @Override + protected String handleTitle(String title) { + return String.format("ti:\"%s\"", title); + } + + @Override + protected String handleJournal(String journalTitle) { + return String.format("pt:\"%s\"", journalTitle); + } + + @Override + protected String handleYear(String year) { + return "sd:" + year + getLogicalAndOperator() + "ed: " + year; + } + + @Override + protected String handleYearRange(String yearRange) { + String[] split = yearRange.split("-"); + return "sd:" + split[0] + getLogicalAndOperator() + "ed:" + split[1]; + } + + @Override + protected String handleUnFieldedTerm(String term) { + return String.format("\"%s\"", term); + } +} diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformer.java new file mode 100644 index 00000000000..a83db9364b1 --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformer.java @@ -0,0 +1,65 @@ +package org.jabref.logic.importer.fetcher.transformators; + +public class ScholarQueryTransformer extends AbstractQueryTransformer { + // These have to be integrated into the Google Scholar query URL as these are just supported as query parameters + private int startYear = Integer.MAX_VALUE; + private int endYear = Integer.MIN_VALUE; + + @Override + protected String getLogicalAndOperator() { + return " AND "; + } + + @Override + protected String getLogicalOrOperator() { + return " OR "; + } + + @Override + protected String getLogicalNotOperator() { + return "-"; + } + + @Override + protected String handleAuthor(String author) { + return String.format("author:\"%s\"", author); + } + + @Override + protected String handleTitle(String title) { + return String.format("allintitle:\"%s\"", title); + } + + @Override + protected String handleJournal(String journalTitle) { + return String.format("source:\"%s\"", journalTitle); + } + + @Override + protected String handleYear(String year) { + startYear = Math.min(startYear, Integer.parseInt(year)); + endYear = Math.max(endYear, Integer.parseInt(year)); + return ""; + } + + @Override + protected String handleYearRange(String yearRange) { + String[] split = yearRange.split("-"); + startYear = Math.min(startYear, Integer.parseInt(split[0])); + endYear = Math.max(endYear, Integer.parseInt(split[1])); + return ""; + } + + @Override + protected String handleUnFieldedTerm(String term) { + return String.format("\"%s\"", term); + } + + public int getStartYear() { + return startYear == Integer.MAX_VALUE ? Integer.MIN_VALUE : startYear; + } + + public int getEndYear() { + return endYear == Integer.MIN_VALUE ? Integer.MAX_VALUE : endYear; + } +} diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformer.java new file mode 100644 index 00000000000..3907caef708 --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformer.java @@ -0,0 +1,62 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import java.util.StringJoiner; + +/** + * This class converts a query string written in lucene syntax into a complex query. + * + * For simplicity this is currently limited to fielded data and the boolean AND operator. + */ +public class SpringerQueryTransformer extends AbstractQueryTransformer { + + @Override + public String getLogicalAndOperator() { + return " AND "; + } + + @Override + public String getLogicalOrOperator() { + return " OR "; + } + + @Override + protected String getLogicalNotOperator() { + return "-"; + } + + @Override + protected String handleAuthor(String author) { + return String.format("name:\"%s\"", author); + } + + @Override + protected String handleTitle(String title) { + return String.format("title:\"%s\"", title); + } + + @Override + protected String handleJournal(String journalTitle) { + return String.format("journal:\"%s\"", journalTitle); + + } + + @Override + protected String handleYear(String year) { + return String.format("date:%s*", year); + } + + @Override + protected String handleYearRange(String yearRange) { + String[] split = yearRange.split("-"); + StringJoiner resultBuilder = new StringJoiner("*" + getLogicalOrOperator() + "date:", "(date:", "*)"); + for (int i = Integer.parseInt(split[0]); i <= Integer.parseInt(split[1]); i++) { + resultBuilder.add(String.valueOf(i)); + } + return resultBuilder.toString(); + } + + @Override + protected String handleUnFieldedTerm(String term) { + return "\"" + term + "\""; + } +} diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformer.java new file mode 100644 index 00000000000..b2a466c20a1 --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformer.java @@ -0,0 +1,49 @@ +package org.jabref.logic.importer.fetcher.transformators; + +public class ZbMathQueryTransformer extends AbstractQueryTransformer { + + @Override + protected String getLogicalAndOperator() { + return " & "; + } + + @Override + protected String getLogicalOrOperator() { + return " | "; + } + + @Override + protected String getLogicalNotOperator() { + return "!"; + } + + @Override + protected String handleAuthor(String author) { + return String.format("au:\"%s\"", author); + } + + @Override + protected String handleTitle(String title) { + return String.format("ti:\"%s\"", title); + } + + @Override + protected String handleJournal(String journalTitle) { + return String.format("so:\"%s\"", journalTitle); + } + + @Override + protected String handleYear(String year) { + return "py:" + year; + } + + @Override + protected String handleYearRange(String yearRange) { + return "py:" + yearRange; + } + + @Override + protected String handleUnFieldedTerm(String term) { + return String.format("any:\"%s\"", term); + } +} diff --git a/src/test/java/org/jabref/logic/importer/QueryParserTest.java b/src/test/java/org/jabref/logic/importer/QueryParserTest.java index 970788e86c3..acb8c41a40c 100644 --- a/src/test/java/org/jabref/logic/importer/QueryParserTest.java +++ b/src/test/java/org/jabref/logic/importer/QueryParserTest.java @@ -6,7 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -class QueryParserTest { +public class QueryParserTest { QueryParser parser = new QueryParser(); @Test diff --git a/src/test/java/org/jabref/logic/importer/fetcher/ArXivTest.java b/src/test/java/org/jabref/logic/importer/fetcher/ArXivTest.java index 9fdf8b15ee8..2009f12e798 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/ArXivTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/ArXivTest.java @@ -229,7 +229,7 @@ public SearchBasedFetcher getFetcher() { @Override public List getTestAuthors() { - return List.of("\"Tobias Diez\""); + return List.of("Tobias Diez"); } @Disabled("Is not supported by the current API") @@ -247,7 +247,7 @@ public void supportsYearRangeSearch() throws Exception { @Override public String getTestJournal() { - return "\"Journal of Geometry and Physics (2013)\""; + return "Journal of Geometry and Physics (2013)"; } /** diff --git a/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java index 577f4624196..539decbf37c 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java @@ -10,12 +10,15 @@ import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UnknownField; import org.jabref.model.entry.types.StandardEntryType; import org.jabref.testutils.category.FetcherTest; +import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Answers; @@ -41,9 +44,9 @@ public void getNameReturnsCorrectName() { } @Test - public void getUrlForQueryReturnsCorrectUrl() throws MalformedURLException, URISyntaxException, FetcherException { + public void getUrlForQueryReturnsCorrectUrl() throws MalformedURLException, URISyntaxException, FetcherException, QueryNodeParseException { String query = "java jdk"; - URL url = fetcher.getURLForQuery(query); + URL url = fetcher.getURLForQuery(new StandardSyntaxParser().parse(query, AbstractQueryTransformer.NO_EXPLICIT_FIELD)); assertEquals("http://liinwww.ira.uka.de/bibliography/rss?query=java+jdk&sort=score", url.toString()); } @@ -63,7 +66,7 @@ public void performSearchReturnsMatchingMultipleEntries() throws FetcherExceptio .withField(StandardField.YEAR, "2017") .withField(StandardField.BOOKTITLE, "11th European Conference on Software Architecture, ECSA 2017, Companion Proceedings, Canterbury, United Kingdom, September 11-15, 2017") .withField(new UnknownField("bibsource"), "DBLP, http://dblp.uni-trier.de/https://doi.org/10.1145/3129790.3129810; DBLP, http://dblp.uni-trier.de/db/conf/ecsa/ecsa2017c.html#OlssonEW17") - .withField(new UnknownField("bibdate"), "2018-11-06"); + .withField(new UnknownField("bibdate"), "2020-10-25"); BibEntry secondBibEntry = new BibEntry(StandardEntryType.Article) .withCitationKey("oai:DiVA.org:lnu-68408") diff --git a/src/test/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcherTest.java index b639c35cf5b..1c329d419a8 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcherTest.java @@ -114,8 +114,8 @@ static Stream performSearchParameters() { // list.add(new MedlineFetcher()); // Create different sized sets of fetchers to use in the composite fetcher. - // Selected 273 to have differencing sets - for (int i = 1; i < Math.pow(2, list.size()); i += 273) { + // Selected 1173 to have differencing sets + for (int i = 1; i < Math.pow(2, list.size()); i += 1173) { Set fetchers = new HashSet<>(); // Only shift i at maximum to its MSB to the right for (int j = 0; Math.pow(2, j) <= i; j++) { diff --git a/src/test/java/org/jabref/logic/importer/fetcher/DBLPFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/DBLPFetcherTest.java index 83414d8666d..001d5071fa5 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/DBLPFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/DBLPFetcherTest.java @@ -50,7 +50,8 @@ public void setUp() { @Test public void findSingleEntry() throws FetcherException { - String query = "Process Engine Benchmarking with Betsy in the Context of {ISO/IEC} Quality Standards"; + // In Lucene curly brackets are used for range queries, therefore they have to be escaped using "". See https://lucene.apache.org/core/5_4_0/queryparser/org/apache/lucene/queryparser/classic/package-summary.html + String query = "Process Engine Benchmarking with Betsy in the Context of \"{ISO/IEC}\" Quality Standards"; List result = dblpFetcher.performSearch(query); assertEquals(Collections.singletonList(entry), result); diff --git a/src/test/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcherTest.java index 329748dcd56..f970273c34b 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcherTest.java @@ -1,6 +1,7 @@ package org.jabref.logic.importer.fetcher; import java.io.IOException; +import java.net.SocketTimeoutException; import java.util.Collections; import java.util.List; import java.util.stream.Stream; @@ -110,7 +111,7 @@ public void grobidPerformSearchWithInvalidDataTest(String invalidInput) throws F @Test public void performSearchThrowsExceptionInCaseOfConnectionIssues() throws IOException { GrobidService grobidServiceMock = mock(GrobidService.class); - when(grobidServiceMock.processCitation(anyString(), any())).thenThrow(new IOException("Any IO Exception")); + when(grobidServiceMock.processCitation(anyString(), any())).thenThrow(new SocketTimeoutException("Timeout")); grobidCitationFetcher = new GrobidCitationFetcher(importFormatPreferences, grobidServiceMock); assertThrows(FetcherException.class, () -> { diff --git a/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java index d1af17c5be0..88cd4b91ef0 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java @@ -1,18 +1,19 @@ package org.jabref.logic.importer.fetcher; -import java.net.MalformedURLException; -import java.net.URISyntaxException; import java.net.URL; import java.util.Collections; import java.util.List; import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UnknownField; import org.jabref.model.entry.types.StandardEntryType; import org.jabref.testutils.category.FetcherTest; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -64,31 +65,19 @@ public void testGetName() { } @Test - public void simpleSearchQueryStringCorrect() { + public void simpleSearchQueryURLCorrect() throws Exception { String query = "java jdk"; - String result = fetcher.getSearchQueryString(query); - assertEquals("pica.all=java jdk", result); + QueryNode luceneQuery = new StandardSyntaxParser().parse(query, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + URL url = fetcher.getURLForQuery(luceneQuery); + assertEquals("http://sru.gbv.de/gvk?version=1.1&operation=searchRetrieve&query=pica.all%3D%22java%22+and+pica.all%3D%22jdk%22&maximumRecords=50&recordSchema=picaxml&sortKeys=Year%2C%2C1", url.toString()); } @Test - public void simpleSearchQueryURLCorrect() throws MalformedURLException, URISyntaxException, FetcherException { - String query = "java jdk"; - URL url = fetcher.getURLForQuery(query); - assertEquals("http://sru.gbv.de/gvk?version=1.1&operation=searchRetrieve&query=pica.all%3Djava+jdk&maximumRecords=50&recordSchema=picaxml&sortKeys=Year%2C%2C1", url.toString()); - } - - @Test - public void complexSearchQueryStringCorrect() { - String query = "kon java tit jdk"; - String result = fetcher.getSearchQueryString(query); - assertEquals("pica.kon=java and pica.tit=jdk", result); - } - - @Test - public void complexSearchQueryURLCorrect() throws MalformedURLException, URISyntaxException, FetcherException { - String query = "kon java tit jdk"; - URL url = fetcher.getURLForQuery(query); - assertEquals("http://sru.gbv.de/gvk?version=1.1&operation=searchRetrieve&query=pica.kon%3Djava+and+pica.tit%3Djdk&maximumRecords=50&recordSchema=picaxml&sortKeys=Year%2C%2C1", url.toString()); + public void complexSearchQueryURLCorrect() throws Exception { + String query = "kon:java tit:jdk"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(query, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + URL url = fetcher.getURLForQuery(luceneQuery); + assertEquals("http://sru.gbv.de/gvk?version=1.1&operation=searchRetrieve&query=pica.kon%3D%22java%22+and+pica.tit%3D%22jdk%22&maximumRecords=50&recordSchema=picaxml&sortKeys=Year%2C%2C1", url.toString()); } @Test diff --git a/src/test/java/org/jabref/logic/importer/fetcher/IEEETest.java b/src/test/java/org/jabref/logic/importer/fetcher/IEEETest.java index a063519bb8d..93224f5e8b0 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/IEEETest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/IEEETest.java @@ -85,20 +85,21 @@ void notFoundByDOI() throws Exception { @Test void searchResultHasNoKeywordTerms() throws Exception { BibEntry expected = new BibEntry(StandardEntryType.Article) - .withField(StandardField.AUTHOR, "Shatakshi Jha and Ikhlaq Hussain and Bhim Singh and Sukumar Mishra") - .withField(StandardField.DATE, "25 2 2019") - .withField(StandardField.YEAR, "2019") - .withField(StandardField.DOI, "10.1049/iet-rpg.2018.5648") - .withField(StandardField.FILE, ":https\\://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=8636659:PDF") - .withField(StandardField.ISSUE, "3") - .withField(StandardField.ISSN, "1752-1424") - .withField(StandardField.JOURNALTITLE, "IET Renewable Power Generation") - .withField(StandardField.PAGES, "418--426") - .withField(StandardField.PUBLISHER, "IET") - .withField(StandardField.TITLE, "Optimal operation of PV-DG-battery based microgrid with power quality conditioner") - .withField(StandardField.VOLUME, "13"); - - List fetchedEntries = fetcher.performSearch("8636659"); // article number + .withField(StandardField.AUTHOR, "Shatakshi Sharma and Bhim Singh and Sukumar Mishra") + .withField(StandardField.DATE, "April 2020") + .withField(StandardField.YEAR, "2020") + .withField(StandardField.DOI, "10.1109/TII.2019.2935531") + .withField(StandardField.FILE, ":https\\://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=8801912:PDF") + .withField(StandardField.ISSUE, "4") + .withField(StandardField.ISSN, "1941-0050") + .withField(StandardField.JOURNALTITLE, "IEEE Transactions on Industrial Informatics") + .withField(StandardField.PAGES, "2346--2356") + .withField(StandardField.PUBLISHER, "IEEE") + .withField(StandardField.TITLE, "Economic Operation and Quality Control in PV-BES-DG-Based Autonomous System") + .withField(StandardField.VOLUME, "16") + .withField(StandardField.KEYWORDS, "Batteries, Generators, Economics, Power quality, State of charge, Harmonic analysis, Control systems, Battery, diesel generator (DG), distributed generation, power quality, photovoltaic (PV), voltage source converter (VSC)"); + + List fetchedEntries = fetcher.performSearch("article_number:8801912"); // article number fetchedEntries.forEach(entry -> entry.clearField(StandardField.ABSTRACT)); // Remove abstract due to copyright); assertEquals(Collections.singletonList(expected), fetchedEntries); } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/INSPIREFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/INSPIREFetcherTest.java index ff8c597705b..43d5377fa64 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/INSPIREFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/INSPIREFetcherTest.java @@ -59,7 +59,7 @@ public void searchByIdentifierFindsEntry() throws Exception { .withField(StandardField.EPRINT, "hep-ph/9802379") .withField(StandardField.ARCHIVEPREFIX, "arXiv") .withField(new UnknownField("reportnumber"), "BUDKER-INP-1998-7, TTP-98-10"); - List fetchedEntries = fetcher.performSearch("hep-ph/9802379"); + List fetchedEntries = fetcher.performSearch("\"hep-ph/9802379\""); assertEquals(Collections.singletonList(article), fetchedEntries); } } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java b/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java index c31bd348b0c..a5bd223cdf2 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java @@ -3,6 +3,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.StringJoiner; import java.util.stream.Collectors; import org.jabref.logic.importer.ImportCleanup; @@ -31,10 +32,10 @@ interface SearchBasedFetcherCapabilityTest { */ @Test default void supportsAuthorSearch() throws Exception { - ComplexSearchQuery.ComplexSearchQueryBuilder builder = ComplexSearchQuery.builder(); - getTestAuthors().forEach(builder::author); + StringJoiner queryBuilder = new StringJoiner("\" AND author:\"", "author:\"", "\""); + getTestAuthors().forEach(queryBuilder::add); - List result = getFetcher().performSearch(builder.build()); + List result = getFetcher().performSearch(queryBuilder.toString()); new ImportCleanup(BibDatabaseMode.BIBTEX).doPostCleanup(result); assertFalse(result.isEmpty()); @@ -51,12 +52,7 @@ default void supportsAuthorSearch() throws Exception { */ @Test default void supportsYearSearch() throws Exception { - ComplexSearchQuery complexSearchQuery = ComplexSearchQuery - .builder() - .singleYear(getTestYear()) - .build(); - - List result = getFetcher().performSearch(complexSearchQuery); + List result = getFetcher().performSearch("year:" + getTestYear()); new ImportCleanup(BibDatabaseMode.BIBTEX).doPostCleanup(result); List differentYearsInResult = result.stream() .map(bibEntry -> bibEntry.getField(StandardField.YEAR)) @@ -73,11 +69,9 @@ default void supportsYearSearch() throws Exception { */ @Test default void supportsYearRangeSearch() throws Exception { - ComplexSearchQuery.ComplexSearchQueryBuilder builder = ComplexSearchQuery.builder(); List yearsInYearRange = List.of("2018", "2019", "2020"); - builder.fromYearAndToYear(2018, 2020); - List result = getFetcher().performSearch(builder.build()); + List result = getFetcher().performSearch("year-range:2018-2020"); new ImportCleanup(BibDatabaseMode.BIBTEX).doPostCleanup(result); List differentYearsInResult = result.stream() .map(bibEntry -> bibEntry.getField(StandardField.YEAR)) @@ -94,9 +88,7 @@ default void supportsYearRangeSearch() throws Exception { */ @Test default void supportsJournalSearch() throws Exception { - ComplexSearchQuery.ComplexSearchQueryBuilder builder = ComplexSearchQuery.builder(); - builder.journal(getTestJournal()); - List result = getFetcher().performSearch(builder.build()); + List result = getFetcher().performSearch("journal:\"" + getTestJournal() + "\""); new ImportCleanup(BibDatabaseMode.BIBTEX).doPostCleanup(result); assertFalse(result.isEmpty()); diff --git a/src/test/java/org/jabref/logic/importer/fetcher/SpringerFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/SpringerFetcherTest.java index 74e595c777a..4774e1250e2 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/SpringerFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/SpringerFetcherTest.java @@ -140,12 +140,12 @@ public SearchBasedFetcher getFetcher() { @Override public List getTestAuthors() { - return List.of("\"Steinmacher, Igor\"", "\"Gerosa, Marco\"", "\"Conte, Tayana U.\""); + return List.of("Steinmacher, Igor", "Gerosa, Marco", "Conte, Tayana U."); } @Override public String getTestJournal() { - return "\"Clinical Research in Cardiology\""; + return "Clinical Research in Cardiology"; } @Override diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformerTest.java new file mode 100644 index 00000000000..ac5933ae7bb --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformerTest.java @@ -0,0 +1,60 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import java.util.Optional; + +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class ArXivQueryTransformerTest implements InfixTransformerTest { + + @Override + public AbstractQueryTransformer getTransformator() { + return new ArXivQueryTransformer(); + } + + @Override + public String getAuthorPrefix() { + return "au:"; + } + + @Override + public String getUnFieldedPrefix() { + return "all:"; + } + + @Override + public String getJournalPrefix() { + return "jr:"; + } + + @Override + public String getTitlePrefix() { + return "ti:"; + } + + @Override + public void convertYearField() throws Exception { + ArXivQueryTransformer transformer = ((ArXivQueryTransformer) getTransformator()); + String queryString = "2018"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional query = transformer.transformLuceneQuery(luceneQuery); + Optional expected = Optional.of(queryString); + assertEquals(expected, query); + assertEquals(2018, transformer.getStartYear()); + assertEquals(2018, transformer.getEndYear()); + } + + @Override + public void convertYearRangeField() throws Exception { + ArXivQueryTransformer transformer = ((ArXivQueryTransformer) getTransformator()); + + String queryString = "year-range:2018-2021"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + transformer.transformLuceneQuery(luceneQuery); + + assertEquals(2018, transformer.getStartYear()); + assertEquals(2021, transformer.getEndYear()); + } +} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformerTest.java new file mode 100644 index 00000000000..d531f19da9d --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformerTest.java @@ -0,0 +1,54 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import java.util.Optional; + +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class DBLPQueryTransformerTest implements InfixTransformerTest { + + @Override + public AbstractQueryTransformer getTransformator() { + return new DBLPQueryTransformer(); + } + + @Override + public String getAuthorPrefix() { + return ""; + } + + @Override + public String getUnFieldedPrefix() { + return ""; + } + + @Override + public String getJournalPrefix() { + return ""; + } + + @Override + public String getTitlePrefix() { + return ""; + } + + @Override + public void convertYearField() throws Exception { + String queryString = "year:2015"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); + Optional expected = Optional.of("2015"); + assertEquals(expected, searchQuery); + } + + @Override + public void convertYearRangeField() throws Exception { + String queryString = "year-range:2012-2015"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); + Optional expected = Optional.of("2012|2013|2014|2015"); + assertEquals(expected, searchQuery); + } +} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java new file mode 100644 index 00000000000..9dab863dc10 --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java @@ -0,0 +1,54 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import java.util.Optional; + +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; +import org.junit.jupiter.api.Disabled; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class GVKQueryTransformerTest implements InfixTransformerTest { + + @Override + public AbstractQueryTransformer getTransformator() { + return new GVKQueryTransformer(); + } + + @Override + public String getAuthorPrefix() { + return "pica.per="; + } + + @Override + public String getUnFieldedPrefix() { + return "pica.all="; + } + + @Override + public String getJournalPrefix() { + return "pica.zti="; + } + + @Override + public String getTitlePrefix() { + return "pica.tit="; + } + + @Override + public void convertYearField() throws Exception { + + String queryString = "year:2018"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional query = getTransformator().transformLuceneQuery(luceneQuery); + + Optional expected = Optional.of("ver:2018"); + assertEquals(expected, query); + } + + @Disabled("Not supported by GVK") + @Override + public void convertYearRangeField() throws Exception { + + } +} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java new file mode 100644 index 00000000000..7d2042a34e3 --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java @@ -0,0 +1,70 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class IEEEQueryTransformerTest implements InfixTransformerTest { + + @Override + public AbstractQueryTransformer getTransformator() { + return new IEEEQueryTransformer(); + } + + @Override + public String getAuthorPrefix() { + return "author:"; + } + + @Override + public String getUnFieldedPrefix() { + return ""; + } + + @Override + public String getJournalPrefix() { + return "publication_title:"; + } + + @Override + public String getTitlePrefix() { + return "article_title:"; + } + + @Override + public void convertJournalField() throws Exception { + IEEEQueryTransformer transformer = ((IEEEQueryTransformer) getTransformator()); + + String queryString = "journal:Nature"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + transformer.transformLuceneQuery(luceneQuery); + + assertEquals("\"Nature\"", transformer.getJournal().get()); + } + + @Override + public void convertYearField() throws Exception { + IEEEQueryTransformer transformer = ((IEEEQueryTransformer) getTransformator()); + + String queryString = "year:2021"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + transformer.transformLuceneQuery(luceneQuery); + + assertEquals(2021, transformer.getStartYear()); + assertEquals(2021, transformer.getEndYear()); + } + + @Override + public void convertYearRangeField() throws Exception { + + IEEEQueryTransformer transformer = ((IEEEQueryTransformer) getTransformator()); + + String queryString = "year-range:2018-2021"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + transformer.transformLuceneQuery(luceneQuery); + + assertEquals(2018, transformer.getStartYear()); + assertEquals(2021, transformer.getEndYear()); + } +} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java new file mode 100644 index 00000000000..f986a04c7e0 --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java @@ -0,0 +1,97 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import java.util.Optional; + +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Test Interface for all transformers that use infix notation for their logical binary operators + */ +public interface InfixTransformerTest { + + AbstractQueryTransformer getTransformator(); + + /* All prefixes have to include the used separator + * Example in the case of ':': "author:" + */ + String getAuthorPrefix(); + + String getUnFieldedPrefix(); + + String getJournalPrefix(); + + String getTitlePrefix(); + + @Test + default void convertAuthorField() throws Exception { + String queryString = "author:\"Igor Steinmacher\""; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); + Optional expected = Optional.of(getAuthorPrefix() + "\"Igor Steinmacher\""); + assertEquals(expected, searchQuery); + } + + @Test + default void convertUnFieldedTerm() throws Exception { + String queryString = "\"default value\""; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); + Optional expected = Optional.of(getUnFieldedPrefix() + queryString); + assertEquals(expected, searchQuery); + } + + @Test + default void convertExplicitUnFieldedTerm() throws Exception { + String queryString = "default:\"default value\""; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); + Optional expected = Optional.of(getUnFieldedPrefix() + "\"default value\""); + assertEquals(expected, searchQuery); + } + + @Test + default void convertJournalField() throws Exception { + String queryString = "journal:Nature"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); + Optional expected = Optional.of(getJournalPrefix() + "\"Nature\""); + assertEquals(expected, searchQuery); + } + + @Test + void convertYearField() throws Exception; + + @Test + void convertYearRangeField() throws Exception; + + @Test + default void convertMultipleValuesWithTheSameField() throws Exception { + String queryString = "author:\"Igor Steinmacher\" author:\"Christoph Treude\""; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); + Optional expected = Optional.of(getAuthorPrefix() + "\"Igor Steinmacher\"" + getTransformator().getLogicalAndOperator() + getAuthorPrefix() + "\"Christoph Treude\""); + assertEquals(expected, searchQuery); + } + + @Test + default void groupedOperations() throws Exception { + String queryString = "(author:\"Igor Steinmacher\" OR author:\"Christoph Treude\" AND author:\"Christoph Freunde\") AND title:test"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); + Optional expected = Optional.of("(" + getAuthorPrefix() + "\"Igor Steinmacher\"" + getTransformator().getLogicalOrOperator() + "(" + getAuthorPrefix() + "\"Christoph Treude\"" + getTransformator().getLogicalAndOperator() + getAuthorPrefix() + "\"Christoph Freunde\"))" + getTransformator().getLogicalAndOperator() + getTitlePrefix() + "\"test\""); + assertEquals(expected, searchQuery); + } + + @Test + default void notOperator() throws Exception { + String queryString = "!(author:\"Igor Steinmacher\" OR author:\"Christoph Treude\")"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); + Optional expected = Optional.of(getTransformator().getLogicalNotOperator() + "(" + getAuthorPrefix() + "\"Igor Steinmacher\"" + getTransformator().getLogicalOrOperator() + getAuthorPrefix() + "\"Christoph Treude\")"); + assertEquals(expected, searchQuery); + } +} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/JstorQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/JstorQueryTransformerTest.java new file mode 100644 index 00000000000..e8a5a6014b9 --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/JstorQueryTransformerTest.java @@ -0,0 +1,39 @@ +package org.jabref.logic.importer.fetcher.transformators; + +class JstorQueryTransformerTest implements InfixTransformerTest { + + @Override + public AbstractQueryTransformer getTransformator() { + return new JstorQueryTransformer(); + } + + @Override + public String getAuthorPrefix() { + return "au:"; + } + + @Override + public String getUnFieldedPrefix() { + return ""; + } + + @Override + public String getJournalPrefix() { + return "pt:"; + } + + @Override + public String getTitlePrefix() { + return "ti:"; + } + + @Override + public void convertYearField() throws Exception { + + } + + @Override + public void convertYearRangeField() throws Exception { + + } +} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformerTest.java new file mode 100644 index 00000000000..e83329b893a --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformerTest.java @@ -0,0 +1,59 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class ScholarQueryTransformerTest implements InfixTransformerTest { + + @Override + public AbstractQueryTransformer getTransformator() { + return new ScholarQueryTransformer(); + } + + @Override + public String getAuthorPrefix() { + return "author:"; + } + + @Override + public String getUnFieldedPrefix() { + return ""; + } + + @Override + public String getJournalPrefix() { + return "source:"; + } + + @Override + public String getTitlePrefix() { + return "allintitle:"; + } + + @Override + public void convertYearField() throws Exception { + ScholarQueryTransformer transformer = ((ScholarQueryTransformer) getTransformator()); + + String queryString = "year:2021"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + transformer.transformLuceneQuery(luceneQuery); + + assertEquals(2021, transformer.getStartYear()); + assertEquals(2021, transformer.getEndYear()); + } + + @Override + public void convertYearRangeField() throws Exception { + + ScholarQueryTransformer transformer = ((ScholarQueryTransformer) getTransformator()); + + String queryString = "year-range:2018-2021"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + transformer.transformLuceneQuery(luceneQuery); + + assertEquals(2018, transformer.getStartYear()); + assertEquals(2021, transformer.getEndYear()); + } +} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformerTest.java new file mode 100644 index 00000000000..c0fbd484251 --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformerTest.java @@ -0,0 +1,56 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import java.util.Optional; + +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class SpringerQueryTransformerTest implements InfixTransformerTest { + + @Override + public String getAuthorPrefix() { + return "name:"; + } + + @Override + public AbstractQueryTransformer getTransformator() { + return new SpringerQueryTransformer(); + } + + @Override + public String getUnFieldedPrefix() { + return ""; + } + + @Override + public String getJournalPrefix() { + return "journal:"; + } + + @Override + public String getTitlePrefix() { + return "title:"; + } + + @Override + public void convertYearField() throws Exception { + String queryString = "year:2015"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); + + Optional expected = Optional.of("date:2015*"); + assertEquals(expected, searchQuery); + } + + @Override + public void convertYearRangeField() throws Exception { + String queryString = "year-range:2012-2015"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); + + Optional expected = Optional.of("date:2012* OR date:2013* OR date:2014* OR date:2015*"); + assertEquals(expected, searchQuery); + } +} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformerTest.java new file mode 100644 index 00000000000..73737455534 --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformerTest.java @@ -0,0 +1,54 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import java.util.Optional; + +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class ZbMathQueryTransformerTest implements InfixTransformerTest { + + @Override + public AbstractQueryTransformer getTransformator() { + return new ZbMathQueryTransformer(); + } + + @Override + public String getAuthorPrefix() { + return "au:"; + } + + @Override + public String getUnFieldedPrefix() { + return "any:"; + } + + @Override + public String getJournalPrefix() { + return "so:"; + } + + @Override + public String getTitlePrefix() { + return "ti:"; + } + + @Override + public void convertYearField() throws Exception { + String queryString = "year:2015"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); + Optional expected = Optional.of("py:2015"); + assertEquals(expected, searchQuery); + } + + @Override + public void convertYearRangeField() throws Exception { + String queryString = "year-range:2012-2015"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); + Optional expected = Optional.of("py:2012-2015"); + assertEquals(expected, searchQuery); + } +} From ec88998eb8456295d5360964876b192d01002ec3 Mon Sep 17 00:00:00 2001 From: Christoph Date: Fri, 29 Jan 2021 22:11:29 +0100 Subject: [PATCH 07/25] Fix File Filter and some layout issues (#7385) * Fix File Filter and some layout issues Fixes part of #7383 * better min width * Any file instead of all * remove style class * l10n Co-authored-by: Carl Christian Snethlage <50491877+calixtus@users.noreply.github.com> --- src/main/java/org/jabref/gui/Base.css | 2 +- .../externalfiles/FileExtensionViewModel.java | 2 +- .../externalfiles/UnlinkedFilesCrawler.java | 2 +- .../externalfiles/UnlinkedFilesDialog.fxml | 14 ++-- .../externalfiles/UnlinkedPDFFileFilter.java | 8 +- .../java/org/jabref/logic/util/FileType.java | 2 + .../jabref/logic/util/StandardFileType.java | 78 ++++++++++--------- .../jabref/logic/util/UnknownFileType.java | 5 ++ .../logic/util/io/DatabaseFileLookup.java | 6 +- src/main/resources/l10n/JabRef_en.properties | 2 +- 10 files changed, 70 insertions(+), 51 deletions(-) diff --git a/src/main/java/org/jabref/gui/Base.css b/src/main/java/org/jabref/gui/Base.css index bf6f8fb5612..ffe7cd3f04f 100644 --- a/src/main/java/org/jabref/gui/Base.css +++ b/src/main/java/org/jabref/gui/Base.css @@ -1217,6 +1217,6 @@ TextFlow * { } -.mainTable-header{ +.mainTable-header { -fx-fill: -fx-mid-text-color; } diff --git a/src/main/java/org/jabref/gui/externalfiles/FileExtensionViewModel.java b/src/main/java/org/jabref/gui/externalfiles/FileExtensionViewModel.java index f1f51862c30..121517bf5bc 100644 --- a/src/main/java/org/jabref/gui/externalfiles/FileExtensionViewModel.java +++ b/src/main/java/org/jabref/gui/externalfiles/FileExtensionViewModel.java @@ -19,7 +19,7 @@ public class FileExtensionViewModel { private final ExternalFileTypes externalFileTypes; FileExtensionViewModel(FileType fileType, ExternalFileTypes externalFileTypes) { - this.description = Localization.lang("%0 file", fileType.toString()); + this.description = Localization.lang("%0 file", fileType.getName()); this.extensions = fileType.getExtensionsWithDot(); this.externalFileTypes = externalFileTypes; } diff --git a/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesCrawler.java b/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesCrawler.java index 60d89246e1b..4b29a57204f 100644 --- a/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesCrawler.java +++ b/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesCrawler.java @@ -74,7 +74,7 @@ private FileNodeViewModel searchDirectory(Path directory, UnlinkedPDFFileFilter Map> fileListPartition; try (Stream filesStream = StreamSupport.stream(Files.newDirectoryStream(directory, fileFilter).spliterator(), false)) { - fileListPartition = filesStream.collect(Collectors.partitioningBy(path -> path.toFile().isDirectory())); + fileListPartition = filesStream.collect(Collectors.partitioningBy(Files::isDirectory)); } catch (IOException e) { LOGGER.error(String.format("%s while searching files: %s", e.getClass().getName(), e.getMessage())); return parent; diff --git a/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesDialog.fxml b/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesDialog.fxml index 0d3246da39d..bea10fcc9c3 100644 --- a/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesDialog.fxml +++ b/src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesDialog.fxml @@ -34,14 +34,14 @@ - - + +