Merge remote-tracking branch 'upstream/master' into exportPDF

* upstream/master: (39 commits)
  Fix fetcher test
  Allow failures for fetcher test (#2730)
  Use JabRefExecutor service
  Move DOI fetching to separate thread #2682
  Remove gui dependency in logic (#2726)
  Fixed freeze on Mac OS X when creating/editing groups (#2727)
  Only ask once if telemetry data should be collected
  Update wiremock from 2.5.1 to 2.6.0
  Update mockito-core from 2.7.21 to 2.7.22
  Update log4j to latest version
  Azure test (#2724)
  Fix build
  Move expand filename to FileUtil
  Unicode conversion bibtexkey (#2720)
  Add sorting of all groups and subgroups, recursively (#2666)
  Only check capitalization of note and howpublished fields if they start with a word character
  Remove overhauled @author tag
  Implement #1359: collect telemetry (#2283)
  Add licenses of new dependencies
  Fix cssStyleHelper warnings
  ...

.travis.yml

@@ -19,6 +19,11 @@ env:
  - TEST_SUITE=databaseTest
  - TEST_SUITE=guiTest
 
+matrix:
+ fast_finish: true
+ allow_failures:
+ - env: TEST_SUITE=fetcherTest
+
 # JavaFX localization tests need a running X environment
 before_install:
  - "export DISPLAY=:99.0"

CHANGELOG.md

@@ -35,6 +35,7 @@ We refer to [GitHub issues](https://github.com/JabRef/jabref/issues) by using `#
  - Redesigned error console.
  - All file dialogs now use the native file selector of the OS. [#1711](https://github.com/JabRef/jabref/issues/1711)
 - Switch to the [latex2unicode library](https://github.com/tomtung/latex2unicode) for converting LaTeX to unicode
+- We added a document viewer which allows you to have a glance at your PDF documents directly from within JabRef.
 - The MS Office XML export now exports the field `volumes` and `pubstate`.
 - The integrity checker reports now if a journal is not found in the abbreviation list
 - Comments in PDF files can now be displayed inside JabRef in a separate tab
@@ -50,7 +51,7 @@ We refer to [GitHub issues](https://github.com/JabRef/jabref/issues) by using `#
 - We added a few properties to a group:
  - Icon (with customizable color) that is shown in the groups panel (implements a [feature request in the forum](http://discourse.jabref.org/t/assign-colors-to-groups/321)).
  - Description text that is shown on mouse hover (implements old feature requests [489](https://sourceforge.net/p/jabref/feature-requests/489/) and [818](https://sourceforge.net/p/jabref/feature-requests/818/)
-- We introduced "automatic groups" that automatically create subgroups based on a certain criteria (e.g. a subgroup for every author or keyword). Implements [91](https://sourceforge.net/p/jabref/feature-requests/91/), [398](https://sourceforge.net/p/jabref/feature-requests/398/) and [#1173](https://github.com/JabRef/jabref/issues/1173).
+- We introduced "automatic groups" that automatically create subgroups based on a certain criteria (e.g. a subgroup for every author or keyword) and supports hierarchies. Implements [91](https://sourceforge.net/p/jabref/feature-requests/91/), [398](https://sourceforge.net/p/jabref/feature-requests/398/) and [#1173](https://github.com/JabRef/jabref/issues/1173) and [#628](https://github.com/JabRef/jabref/issues/628).
 - Expansion status of groups are saved across sessions. [#1428](https://github.com/JabRef/jabref/issues/1428)
 - We removed the ordinals-to-superscript formatter from the recommendations for biblatex save actions [#2596](https://github.com/JabRef/jabref/issues/2596)
 - The `Move linked files to default file directory`-Cleanup operation respects the `File directory pattern` setting 
@@ -64,6 +65,8 @@ We refer to [GitHub issues](https://github.com/JabRef/jabref/issues) by using `#
 - Entries with a single corporate author are now correclty exported to the corresponding `corporate` author field in MS-Office XML. [#1497](https://github.com/JabRef/jabref/issues/1497)
 - Improved author handling in MS-Office Import/Export
 - The `day` part of the biblatex `date` field is now exported to the corresponding `day` field in MS-Office XML. [#2691](https://github.com/JabRef/jabref/issues/2691)
+- Single underscores are not converted during the LaTeX to unicode conversion, which does not follow the rules of LaTeX, but is what users require. [#2664](https://github.com/JabRef/jabref/issues/2664)
+- The bibtexkey field is not converted to unicode 
 
 ### Fixed
  - We fixed an issue of duplicate keys after using a fetcher, e.g., DOI or ISBN [#2867](https://github.com/JabRef/jabref/issues/2687)

build.gradle

@@ -96,8 +96,8 @@ dependencies {
  antlr3 'org.antlr:antlr:3.5.2'
  compile 'org.antlr:antlr-runtime:3.5.2'
 
- antlr4 'org.antlr:antlr4:4.6'
- compile 'org.antlr:antlr4-runtime:4.6'
+ antlr4 'org.antlr:antlr4:4.7'
+ compile 'org.antlr:antlr4-runtime:4.7'
 
  // VersionEye states that 6.0.5 is the most recent version, but http://dev.mysql.com/downloads/connector/j/ shows that as "Development Release"
  compile 'mysql:mysql-connector-java:5.1.40'
@@ -114,6 +114,8 @@ dependencies {
  compile 'de.codecentric.centerdevice:javafxsvg:1.2.1'
  compile 'org.controlsfx:controlsfx:8.40.12'
  compile 'org.fxmisc.easybind:easybind:1.0.3'
+ compile 'org.fxmisc.flowless:flowless:0.5.2'
+ compile 'de.jensd:fontawesomefx-materialdesignfont:1.7.22-4'
 
 
  compile 'commons-logging:commons-logging:1.2'
@@ -124,9 +126,9 @@ dependencies {
  compile 'com.mashape.unirest:unirest-java:1.4.9'
  compile 'info.debatty:java-string-similarity:0.23'
 
- compile 'org.apache.logging.log4j:log4j-jcl:2.8.1'
- compile 'org.apache.logging.log4j:log4j-api:2.8.1'
- compile 'org.apache.logging.log4j:log4j-core:2.8.1'
+ compile 'org.apache.logging.log4j:log4j-jcl:2.8.2'
+ compile 'org.apache.logging.log4j:log4j-api:2.8.2'
+ compile 'org.apache.logging.log4j:log4j-core:2.8.2'
  compile 'org.xmlunit:xmlunit-core:2.3.0'
  compile 'org.xmlunit:xmlunit-matchers:2.3.0'
 
@@ -139,9 +141,12 @@ dependencies {
 
  compile 'com.github.tomtung:latex2unicode_2.12:0.2'
 
+ compile group: 'com.microsoft.azure', name: 'applicationinsights-core', version: '1.0.+'
+ compile group: 'com.microsoft.azure', name: 'applicationinsights-logging-log4j2', version: '1.0.+'
+
  testCompile 'junit:junit:4.12'
- testCompile 'org.mockito:mockito-core:2.7.21'
- testCompile 'com.github.tomakehurst:wiremock:2.5.1'
+ testCompile 'org.mockito:mockito-core:2.7.22'
+ testCompile 'com.github.tomakehurst:wiremock:2.6.0'
  testCompile 'org.assertj:assertj-swing-junit:3.5.0'
  testCompile 'org.reflections:reflections:0.9.11'
 }
@@ -163,7 +168,8 @@ processResources {
  expand(version: project.version,
  "year": String.valueOf(Calendar.getInstance().get(Calendar.YEAR)),
  "authors": new File('AUTHORS').readLines().findAll { !it.startsWith("#") }.join(", "),
- "developers": new File('DEVELOPERS').readLines().findAll { !it.startsWith("#") }.join(", "))
+ "developers": new File('DEVELOPERS').readLines().findAll { !it.startsWith("#") }.join(", "),
+ "azureInstrumentationKey": System.getenv('AzureInstrumentationKey'))
  filteringCharset = 'UTF-8'
  }
  filteringCharset = 'UTF-8'

external-libraries.txt

@@ -135,6 +135,16 @@ Projekt: EasyBind
 URL: https://github.com/TomasMikula/EasyBind
 License: BSD-2-Clause
 
+Id: org.fxmisc.flowless:flowless
+Projekt: Flowless
+URL: https://github.com/TomasMikula/Flowless
+License: BSD-2-Clause
+
+Id: de.jensd:fontawesomefx-materialdesignfont
+Projekt: FontAwesomeFX
+URL: https://bitbucket.org/Jerady/fontawesomefx
+License: Apache-2.0
+
 Id: org.jsoup:jsoup
 Project: jsoup
 URL: https://github.com/jhy/jsoup/

src/main/java/org/jabref/Globals.java

@@ -1,5 +1,8 @@
 package org.jabref;
 
+import java.awt.Toolkit;
+import java.util.UUID;
+
 import org.jabref.collab.FileUpdateMonitor;
 import org.jabref.gui.GlobalFocusListener;
 import org.jabref.gui.StateManager;
@@ -13,6 +16,11 @@
 import org.jabref.logic.util.BuildInfo;
 import org.jabref.preferences.JabRefPreferences;
 
+import com.microsoft.applicationinsights.TelemetryClient;
+import com.microsoft.applicationinsights.TelemetryConfiguration;
+import com.microsoft.applicationinsights.telemetry.SessionState;
+import org.apache.commons.lang3.SystemUtils;
+
 public class Globals {
 
  // JabRef version info
@@ -44,6 +52,7 @@ public class Globals {
  // Background tasks
  private static GlobalFocusListener focusListener;
  private static FileUpdateMonitor fileUpdateMonitor;
+ private static TelemetryClient telemetryClient;
 
  private Globals() {
  }
@@ -63,6 +72,30 @@ public static void startBackgroundTasks() {
 
  Globals.fileUpdateMonitor = new FileUpdateMonitor();
  JabRefExecutorService.INSTANCE.executeInterruptableTask(Globals.fileUpdateMonitor, "FileUpdateMonitor");
+
+ startTelemetryClient();
+ }
+
+ private static void stopTelemetryClient() {
+ telemetryClient.trackSessionState(SessionState.End);
+ telemetryClient.flush();
+ }
+
+ private static void startTelemetryClient() {
+ TelemetryConfiguration telemetryConfiguration = TelemetryConfiguration.getActive();
+ telemetryConfiguration.setInstrumentationKey(Globals.BUILD_INFO.getAzureInstrumentationKey());
+ telemetryConfiguration.setTrackingIsDisabled(!Globals.prefs.shouldCollectTelemetry());
+ telemetryClient = new TelemetryClient(telemetryConfiguration);
+ telemetryClient.getContext().getProperties().put("JabRef version", Globals.BUILD_INFO.getVersion().toString());
+ telemetryClient.getContext().getProperties().put("Java version", SystemUtils.JAVA_RUNTIME_VERSION);
+ telemetryClient.getContext().getUser().setId(Globals.prefs.getOrCreateUserId());
+ telemetryClient.getContext().getSession().setId(UUID.randomUUID().toString());
+ telemetryClient.getContext().getDevice().setOperatingSystem(SystemUtils.OS_NAME);
+ telemetryClient.getContext().getDevice().setOperatingSystemVersion(SystemUtils.OS_VERSION);
+ telemetryClient.getContext().getDevice().setScreenResolution(
+ Toolkit.getDefaultToolkit().getScreenSize().toString());
+
+ telemetryClient.trackSessionState(SessionState.Start);
  }
 
  public static GlobalFocusListener getFocusListener() {
@@ -77,4 +110,12 @@ public static void shutdownThreadPools() {
  taskExecutor.shutdown();
  JabRefExecutorService.INSTANCE.shutdownEverything();
  }
+
+ public static void stopBackgroundTasks() {
+ stopTelemetryClient();
+ }
+
+ public static TelemetryClient getTelemetryClient() {
+ return telemetryClient;
+ }
 }

src/main/java/org/jabref/JabRefMain.java

@@ -64,8 +64,8 @@ private static void start(String[] args) {
  Authenticator.setDefault(new ProxyAuthenticator());
  }
 
- Globals.startBackgroundTasks();
  Globals.prefs = preferences;
+ Globals.startBackgroundTasks();
  Localization.setLanguage(preferences.get(JabRefPreferences.LANGUAGE));
  Globals.prefs.setLanguageDependentDefaultValues();
 

src/main/java/org/jabref/collab/ChangeDisplayDialog.java

@@ -9,7 +9,6 @@
 import javax.swing.JButton;
 import javax.swing.JCheckBox;
 import javax.swing.JComponent;
-import javax.swing.JDialog;
 import javax.swing.JFrame;
 import javax.swing.JLabel;
 import javax.swing.JPanel;
@@ -21,11 +20,12 @@
 import javax.swing.tree.DefaultMutableTreeNode;
 
 import org.jabref.gui.BasePanel;
+import org.jabref.gui.JabRefDialog;
 import org.jabref.gui.undo.NamedCompound;
 import org.jabref.logic.l10n.Localization;
 import org.jabref.model.database.BibDatabase;
 
-class ChangeDisplayDialog extends JDialog implements TreeSelectionListener {
+class ChangeDisplayDialog extends JabRefDialog implements TreeSelectionListener {
 
  private final JTree tree;
  private final JPanel infoPanel = new JPanel();
@@ -38,7 +38,7 @@ class ChangeDisplayDialog extends JDialog implements TreeSelectionListener {
 
  public ChangeDisplayDialog(JFrame owner, final BasePanel panel,
  BibDatabase secondary, final DefaultMutableTreeNode root) {
- super(owner, Localization.lang("External changes"), true);
+ super(owner, Localization.lang("External changes"), true, ChangeDisplayDialog.class);
  BibDatabase localSecondary;
 
  // Just to be sure, put in an empty secondary base if none is given:

src/main/java/org/jabref/gui/AbstractView.java

@@ -22,7 +22,7 @@ public Parent getView() {
  Parent view = super.getView();
 
  // Add our base css file
- view.getStylesheets().add(AbstractDialogView.class.getResource("Main.css").toExternalForm());
+ view.getStylesheets().add(0, AbstractDialogView.class.getResource("Main.css").toExternalForm());
 
  // Notify controller about the stage, where it is displayed
  view.sceneProperty().addListener((observable, oldValue, newValue) -> {

src/main/java/org/jabref/gui/BasePanel.java

@@ -41,6 +41,8 @@
 import javax.swing.undo.CannotRedoException;
 import javax.swing.undo.CannotUndoException;
 
+import javafx.application.Platform;
+
 import org.jabref.Globals;
 import org.jabref.JabRefExecutorService;
 import org.jabref.collab.ChangeScanner;
@@ -237,6 +239,26 @@ public BasePanel(JabRefFrame frame, BibDatabaseContext bibDatabaseContext) {
  }
  }
 
+ public static void runWorker(AbstractWorker worker) throws Exception {
+ // This part uses Spin's features:
+ Runnable wrk = worker.getWorker();
+ // The Worker returned by getWorker() has been wrapped
+ // by Spin.off(), which makes its methods be run in
+ // a different thread from the EDT.
+ CallBack clb = worker.getCallBack();
+
+ worker.init(); // This method runs in this same thread, the EDT.
+ // Useful for initial GUI actions, like printing a message.
+
+ // The CallBack returned by getCallBack() has been wrapped
+ // by Spin.over(), which makes its methods be run on
+ // the EDT.
+ wrk.run(); // Runs the potentially time-consuming action
+ // without freezing the GUI. The magic is that THIS line
+ // of execution will not continue until run() is finished.
+ clb.update(); // Runs the update() method on the EDT.
+ }
+
  // Returns a collection of AutoCompleters, which are populated from the current library
  public ContentAutoCompleters getAutoCompleters() {
  return autoCompleters;
@@ -999,26 +1021,6 @@ public void runCommand(final String _command) {
  }
  }
 
- public static void runWorker(AbstractWorker worker) throws Exception {
- // This part uses Spin's features:
- Runnable wrk = worker.getWorker();
- // The Worker returned by getWorker() has been wrapped
- // by Spin.off(), which makes its methods be run in
- // a different thread from the EDT.
- CallBack clb = worker.getCallBack();
-
- worker.init(); // This method runs in this same thread, the EDT.
- // Useful for initial GUI actions, like printing a message.
-
- // The CallBack returned by getCallBack() has been wrapped
- // by Spin.over(), which makes its methods be run on
- // the EDT.
- wrk.run(); // Runs the potentially time-consuming action
- // without freezing the GUI. The magic is that THIS line
- // of execution will not continue until run() is finished.
- clb.update(); // Runs the update() method on the EDT.
- }
-
  private boolean saveDatabase(File file, boolean selectedOnly, Charset enc,
  SavePreferences.DatabaseSaveType saveType) throws SaveException {
  SaveSession session;
@@ -1230,7 +1232,8 @@ private void createMainTable() {
  mainTable.addFocusListener(selectionListener);
 
  // Add the listener that binds selection to state manager (TODO: should be replaced by proper JavaFX binding as soon as table is implemented in JavaFX)
- mainTable.addSelectionListener(listEvent -> Globals.stateManager.setSelectedEntries(mainTable.getSelectedEntries()));
+ mainTable.addSelectionListener(listEvent ->
+ Platform.runLater(() -> Globals.stateManager.setSelectedEntries(mainTable.getSelectedEntries())));
 
  String clearSearch = "clearSearch";
  mainTable.getInputMap().put(Globals.getKeyPrefs().getKey(KeyBinding.CLEAR_SEARCH), clearSearch);

src/main/java/org/jabref/gui/DialogService.java

@@ -64,7 +64,7 @@ default void showErrorDialogAndWait(Exception exception) {
  /**
  * This will create and display a new confirmation dialog.
  * It will include a blue question icon on the left and
- * a OK and Cancel Button. To create a confirmation dialog with custom
+ * a OK and Cancel button. To create a confirmation dialog with custom
  * buttons see also {@link #showCustomButtonDialogAndWait(Alert.AlertType, String, String, ButtonType...)}
  *
  * @return true if the use clicked "OK" otherwise false
@@ -74,13 +74,23 @@ default void showErrorDialogAndWait(Exception exception) {
  /**
  * Create and display a new confirmation dialog.
  * It will include a blue question icon on the left and
- * a OK (with given label) and Cancel Button. To create a confirmation dialog with custom
+ * a OK (with given label) and Cancel button. To create a confirmation dialog with custom
  * buttons see also {@link #showCustomButtonDialogAndWait(Alert.AlertType, String, String, ButtonType...)}
  *
  * @return true if the use clicked "OK" otherwise false
  */
  boolean showConfirmationDialogAndWait(String title, String content, String okButtonLabel);
 
+ /**
+ * Create and display a new confirmation dialog.
+ * It will include a blue question icon on the left and
+ * a OK (with given label) and Cancel (also with given label) button. To create a confirmation dialog with custom
+ * buttons see also {@link #showCustomButtonDialogAndWait(Alert.AlertType, String, String, ButtonType...)}
+ *
+ * @return true if the use clicked "OK" otherwise false
+ */
+ boolean showConfirmationDialogAndWait(String title, String content, String okButtonLabel, String cancelButtonLabel);
+
  /**
  * This will create and display a new dialog of the specified
  * {@link Alert.AlertType} but with user defined buttons as optional

src/main/java/org/jabref/gui/DuplicateResolverDialog.java

@@ -6,7 +6,6 @@
 
 import javax.swing.Box;
 import javax.swing.JButton;
-import javax.swing.JDialog;
 import javax.swing.JPanel;
 
 import org.jabref.gui.help.HelpAction;
@@ -18,7 +17,7 @@
 import org.jabref.model.entry.BibEntry;
 import org.jabref.preferences.JabRefPreferences;
 
-public class DuplicateResolverDialog extends JDialog {
+public class DuplicateResolverDialog extends JabRefDialog {
 
  public enum DuplicateResolverType {
  DUPLICATE_SEARCH,
@@ -46,14 +45,14 @@ public enum DuplicateResolverResult {
  private MergeEntries me;
 
  public DuplicateResolverDialog(JabRefFrame frame, BibEntry one, BibEntry two, DuplicateResolverType type) {
- super(frame, Localization.lang("Possible duplicate entries"), true);
+ super(frame, Localization.lang("Possible duplicate entries"), true, DuplicateResolverDialog.class);
  this.frame = frame;
  init(one, two, type);
  }
 
  public DuplicateResolverDialog(ImportInspectionDialog dialog, BibEntry one, BibEntry two,
  DuplicateResolverType type) {
- super(dialog, Localization.lang("Possible duplicate entries"), true);
+ super(dialog, Localization.lang("Possible duplicate entries"), true, DuplicateResolverDialog.class);
  this.frame = dialog.getFrame();
  init(one, two, type);
  }