From 7f4c36c03629498db8e56281906241a82846e1e3 Mon Sep 17 00:00:00 2001 From: Johannes Theiner Date: Thu, 22 Oct 2020 03:13:11 +0200 Subject: [PATCH] Support for exporting to YAML format (#7007) --- CHANGELOG.md | 2 + .../logic/exporter/BlankLineBehaviour.java | 9 ++ .../logic/exporter/ExporterFactory.java | 3 +- .../logic/exporter/TemplateExporter.java | 75 ++++++------ .../org/jabref/logic/layout/LayoutEntry.java | 3 + .../jabref/logic/layout/format/CSLType.java | 21 ++++ .../jabref/logic/util/StandardFileType.java | 3 +- .../resource/layout/yaml.begin.layout | 2 + .../resources/resource/layout/yaml.end.layout | 1 + .../resources/resource/layout/yaml.layout | 14 +++ .../logic/exporter/YamlExporterTest.java | 108 ++++++++++++++++++ 11 files changed, 201 insertions(+), 40 deletions(-) create mode 100644 src/main/java/org/jabref/logic/exporter/BlankLineBehaviour.java create mode 100644 src/main/java/org/jabref/logic/layout/format/CSLType.java create mode 100644 src/main/resources/resource/layout/yaml.begin.layout create mode 100644 src/main/resources/resource/layout/yaml.end.layout create mode 100644 src/main/resources/resource/layout/yaml.layout create mode 100644 src/test/java/org/jabref/logic/exporter/YamlExporterTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index cffb38db9c2..e09adc83e2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve - We added a query parser and mapping layer to enable conversion of queries formulated in simplified lucene syntax by the user into api queries. [#6799](https://github.com/JabRef/jabref/pull/6799) - We added some basic functionality to customise the look of JabRef by importing a css theme file. [#5790](https://github.com/JabRef/jabref/issues/5790) - We added connection check function in network preference setting [#6560](https://github.com/JabRef/jabref/issues/6560) +- We added support for exporting to YAML. [#6974](https://github.com/JabRef/jabref/issues/6974) - We added a DOI format and organization check to detect [American Physical Society](https://journals.aps.org/) journals to copy the article ID to the page field for cases where the page numbers are missing. [#7019](https://github.com/JabRef/jabref/issues/7019) - We added a new fetcher to enable users to search jstor.org [#6627](https://github.com/JabRef/jabref/issues/6627) @@ -57,6 +58,7 @@ inserting new citations in a OpenOffic/LibreOffice document. [#6957](https://git - We fixed an issue with the python script used by browser plugins that failed to locate JabRef if not installed in its default location. [#6963](https://github.com/JabRef/jabref/pull/6963/files) - We fixed an issue where spaces and newlines in an isbn would generate an exception. [#6456](https://github.com/JabRef/jabref/issues/6456) - We fixed an issue where identity column header had incorrect foreground color in the Dark theme. [#6796](https://github.com/JabRef/jabref/issues/6796) +- We fixed an issue where the RIS exporter added extra blank lines.[#7007](https://github.com/JabRef/jabref/pull/7007/files) - We fixed an issue where clicking on Collapse All button in the Search for Unlinked Local Files expanded the directory structure erroneously [#6848](https://github.com/JabRef/jabref/issues/6848) - We fixed an issue, when pulling changes from shared database via shortcut caused creation a new new tech report [6867](https://github.com/JabRef/jabref/issues/6867) - We fixed an issue where the JabRef GUI does not highlight the "All entries" group on start-up [#6691](https://github.com/JabRef/jabref/issues/6691) diff --git a/src/main/java/org/jabref/logic/exporter/BlankLineBehaviour.java b/src/main/java/org/jabref/logic/exporter/BlankLineBehaviour.java new file mode 100644 index 00000000000..96957869a27 --- /dev/null +++ b/src/main/java/org/jabref/logic/exporter/BlankLineBehaviour.java @@ -0,0 +1,9 @@ +package org.jabref.logic.exporter; + +/** + * This enum represents the behaviour for blank lines in {@link TemplateExporter} + */ +public enum BlankLineBehaviour { + KEEP_BLANKS, + DELETE_BLANKS +} diff --git a/src/main/java/org/jabref/logic/exporter/ExporterFactory.java b/src/main/java/org/jabref/logic/exporter/ExporterFactory.java index fbba86a6aa4..082e284a245 100644 --- a/src/main/java/org/jabref/logic/exporter/ExporterFactory.java +++ b/src/main/java/org/jabref/logic/exporter/ExporterFactory.java @@ -49,8 +49,9 @@ public static ExporterFactory create(List customFormats, exporters.add(new TemplateExporter("ISO 690", "iso690txt", "iso690", "iso690txt", StandardFileType.TXT, layoutPreferences, savePreferences)); exporters.add(new TemplateExporter("Endnote", "endnote", "EndNote", "endnote", StandardFileType.TXT, layoutPreferences, savePreferences)); exporters.add(new TemplateExporter("OpenOffice/LibreOffice CSV", "oocsv", "openoffice-csv", "openoffice", StandardFileType.CSV, layoutPreferences, savePreferences)); - exporters.add(new TemplateExporter("RIS", "ris", "ris", "ris", StandardFileType.RIS, layoutPreferences, savePreferences).withEncoding(StandardCharsets.UTF_8)); + exporters.add(new TemplateExporter("RIS", "ris", "ris", "ris", StandardFileType.RIS, layoutPreferences, savePreferences, BlankLineBehaviour.DELETE_BLANKS).withEncoding(StandardCharsets.UTF_8)); exporters.add(new TemplateExporter("MIS Quarterly", "misq", "misq", "misq", StandardFileType.RTF, layoutPreferences, savePreferences)); + exporters.add(new TemplateExporter("CSL YAML", "yaml", "yaml", null, StandardFileType.YAML, layoutPreferences, savePreferences, BlankLineBehaviour.DELETE_BLANKS)); exporters.add(new BibTeXMLExporter()); exporters.add(new OpenOfficeDocumentCreator()); exporters.add(new OpenDocumentSpreadsheetCreator()); diff --git a/src/main/java/org/jabref/logic/exporter/TemplateExporter.java b/src/main/java/org/jabref/logic/exporter/TemplateExporter.java index af075471e3e..4e3230e509d 100644 --- a/src/main/java/org/jabref/logic/exporter/TemplateExporter.java +++ b/src/main/java/org/jabref/logic/exporter/TemplateExporter.java @@ -14,12 +14,12 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.regex.Pattern; import org.jabref.logic.layout.Layout; import org.jabref.logic.layout.LayoutFormatterPreferences; import org.jabref.logic.layout.LayoutHelper; import org.jabref.logic.util.FileType; +import org.jabref.logic.util.OS; import org.jabref.logic.util.StandardFileType; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -33,15 +33,8 @@ */ public class TemplateExporter extends Exporter { + private static final String BLANK_LINE_PATTERN = "\\r\\n|\\n"; private static final String LAYOUT_PREFIX = "/resource/layout/"; - - /** - * A regular expression that matches blank lines - * - * ?m activates "multimode", which makes ^ match line starts/ends. - * \\s simply marks any whitespace character - */ - private static final Pattern BLANK_LINE_MATCHER = Pattern.compile("(?m)^\\s"); private static final String LAYOUT_EXTENSION = ".layout"; private static final String FORMATTERS_EXTENSION = ".formatters"; private static final String BEGIN_INFIX = ".begin"; @@ -55,11 +48,10 @@ public class TemplateExporter extends Exporter { private final SavePreferences savePreferences; private Charset encoding; // If this value is set, it will be used to override the default encoding for the getCurrentBasePanel. private boolean customExport; - private boolean deleteBlankLines; + private BlankLineBehaviour blankLineBehaviour; /** - * Initialize another export format based on templates stored in dir with - * layoutFile lfFilename. + * Initialize another export format based on templates stored in dir with layoutFile lfFilename. * * @param displayName Name to display to the user. * @param consoleName Name to call this format in the console. @@ -72,8 +64,7 @@ public TemplateExporter(String displayName, String consoleName, String lfFileNam } /** - * Initialize another export format based on templates stored in dir with - * layoutFile lfFilename. + * Initialize another export format based on templates stored in dir with layoutFile lfFilename. * * @param name to display to the user and to call this format in the console. * @param lfFileName Name of the main layout file. @@ -87,8 +78,7 @@ public TemplateExporter(String name, String lfFileName, String extension, Layout } /** - * Initialize another export format based on templates stored in dir with - * layoutFile lfFilename. + * Initialize another export format based on templates stored in dir with layoutFile lfFilename. * * @param displayName Name to display to the user. * @param consoleName Name to call this format in the console. @@ -112,27 +102,36 @@ public TemplateExporter(String displayName, String consoleName, String lfFileNam } /** - * Initialize another export format based on templates stored in dir with - * layoutFile lfFilename. - * The display name is automatically derived from the FileType + * Initialize another export format based on templates stored in dir with layoutFile lfFilename. * + * @param displayName Name to display to the user. * @param consoleName Name to call this format in the console. * @param lfFileName Name of the main layout file. * @param directory Directory in which to find the layout file. * @param extension Should contain the . (for instance .txt). * @param layoutPreferences Preferences for layout * @param savePreferences Preferences for saving - * @param deleteBlankLines If blank lines should be remove (default: false) + * @param blankLineBehaviour how to behave regarding blank lines. */ - public TemplateExporter(String consoleName, String lfFileName, String directory, StandardFileType extension, LayoutFormatterPreferences layoutPreferences, SavePreferences savePreferences, boolean deleteBlankLines) { - this(consoleName, consoleName, lfFileName, directory, extension, layoutPreferences, savePreferences); - this.deleteBlankLines = deleteBlankLines; + public TemplateExporter(String displayName, String consoleName, String lfFileName, String directory, FileType extension, + LayoutFormatterPreferences layoutPreferences, SavePreferences savePreferences, + BlankLineBehaviour blankLineBehaviour) { + super(consoleName, displayName, extension); + if (Objects.requireNonNull(lfFileName).endsWith(LAYOUT_EXTENSION)) { + this.lfFileName = lfFileName.substring(0, lfFileName.length() - LAYOUT_EXTENSION.length()); + } else { + this.lfFileName = lfFileName; + } + this.directory = directory; + this.layoutPreferences = layoutPreferences; + this.savePreferences = savePreferences; + this.blankLineBehaviour = blankLineBehaviour; } /** - * Indicate whether this is a custom export. A custom export looks for its - * layout files using a normal file path, while a built-in export looks in - * the classpath. + * Indicate whether this is a custom export. + * A custom export looks for its layout files using a normal file path, + * while a built-in export looks in the classpath. * * @param custom true to indicate a custom export format. */ @@ -141,8 +140,7 @@ public void setCustomExport(boolean custom) { } /** - * Set an encoding which will be used in preference to the default value - * obtained from the basepanel. + * Set an encoding which will be used in preference to the default value obtained from the basepanel. * * @param encoding The name of the encoding to use. */ @@ -152,11 +150,9 @@ public TemplateExporter withEncoding(Charset encoding) { } /** - * This method should return a reader from which the given layout file can - * be read. + * This method should return a reader from which the given layout file can be read. *

- * Subclasses of TemplateExporter are free to override and provide their own - * implementation. + * Subclasses of TemplateExporter are free to override and provide their own implementation. * * @param filename the filename * @return a newly created reader @@ -277,9 +273,13 @@ public void export(final BibDatabaseContext databaseContext, final Path file, // Write the entry if (layout != null) { - if (deleteBlankLines) { - String withoutBlankLines = BLANK_LINE_MATCHER.matcher(layout.doLayout(entry, databaseContext.getDatabase())).replaceAll(""); - ps.write(withoutBlankLines); + if (blankLineBehaviour == BlankLineBehaviour.DELETE_BLANKS) { + String[] lines = layout.doLayout(entry, databaseContext.getDatabase()).split(BLANK_LINE_PATTERN); + for (String line : lines) { + if (!line.isBlank() && !line.isEmpty()) { + ps.write(line + OS.NEWLINE); + } + } } else { ps.write(layout.doLayout(entry, databaseContext.getDatabase())); } @@ -316,9 +316,8 @@ public void export(final BibDatabaseContext databaseContext, final Path file, } /** - * See if there is a name formatter file bundled with this export format. If so, read - * all the name formatters so they can be used by the filter layouts. - * + * See if there is a name formatter file bundled with this export format. + * If so, read all the name formatters so they can be used by the filter layouts. */ private void readFormatterFile() { File formatterFile = new File(lfFileName + FORMATTERS_EXTENSION); diff --git a/src/main/java/org/jabref/logic/layout/LayoutEntry.java b/src/main/java/org/jabref/logic/layout/LayoutEntry.java index fe5632f9f5b..8a1f6e7349f 100644 --- a/src/main/java/org/jabref/logic/layout/LayoutEntry.java +++ b/src/main/java/org/jabref/logic/layout/LayoutEntry.java @@ -32,6 +32,7 @@ import org.jabref.logic.layout.format.AuthorNatBib; import org.jabref.logic.layout.format.AuthorOrgSci; import org.jabref.logic.layout.format.Authors; +import org.jabref.logic.layout.format.CSLType; import org.jabref.logic.layout.format.CompositeFormat; import org.jabref.logic.layout.format.CreateBibORDFAuthors; import org.jabref.logic.layout.format.CreateDocBook4Authors; @@ -539,6 +540,8 @@ private LayoutFormatter getLayoutFormatterByName(String name) { return new WrapFileLinks(prefs.getFileLinkPreferences()); case "Markdown": return new MarkdownFormatter(); + case "CSLType": + return new CSLType(); default: return null; } diff --git a/src/main/java/org/jabref/logic/layout/format/CSLType.java b/src/main/java/org/jabref/logic/layout/format/CSLType.java new file mode 100644 index 00000000000..aac863567fe --- /dev/null +++ b/src/main/java/org/jabref/logic/layout/format/CSLType.java @@ -0,0 +1,21 @@ +package org.jabref.logic.layout.format; + +import org.jabref.logic.layout.LayoutFormatter; +import org.jabref.model.entry.types.StandardEntryType; + +public class CSLType implements LayoutFormatter { + + @Override + public String format(String value) { + return switch (StandardEntryType.valueOf(value)) { + case Article -> "article"; + case Book -> "book"; + case Conference -> "paper-conference"; + case Report, TechReport -> "report"; + case Thesis, MastersThesis, PhdThesis -> "thesis"; + case WWW, Online -> "webpage"; + + default -> "no-type"; + }; + } +} diff --git a/src/main/java/org/jabref/logic/util/StandardFileType.java b/src/main/java/org/jabref/logic/util/StandardFileType.java index 45c48b6cf76..64cbff44d41 100644 --- a/src/main/java/org/jabref/logic/util/StandardFileType.java +++ b/src/main/java/org/jabref/logic/util/StandardFileType.java @@ -41,7 +41,8 @@ public enum StandardFileType implements FileType { JSON("json"), XMP("xmp"), ZIP("zip"), - CSS("css"); + CSS("css"), + YAML("yaml"); private final List extensions; diff --git a/src/main/resources/resource/layout/yaml.begin.layout b/src/main/resources/resource/layout/yaml.begin.layout new file mode 100644 index 00000000000..3d4a00f98c8 --- /dev/null +++ b/src/main/resources/resource/layout/yaml.begin.layout @@ -0,0 +1,2 @@ +--- +references: diff --git a/src/main/resources/resource/layout/yaml.end.layout b/src/main/resources/resource/layout/yaml.end.layout new file mode 100644 index 00000000000..ed97d539c09 --- /dev/null +++ b/src/main/resources/resource/layout/yaml.end.layout @@ -0,0 +1 @@ +--- diff --git a/src/main/resources/resource/layout/yaml.layout b/src/main/resources/resource/layout/yaml.layout new file mode 100644 index 00000000000..94e4f02a9b9 --- /dev/null +++ b/src/main/resources/resource/layout/yaml.layout @@ -0,0 +1,14 @@ +- id: \citationkey +\begin{entrytype} type: \format[CSLType]{\entrytype}\end{entrytype} +\begin{author} + author: + - literal: "\author" +\end{author} +\begin{title} title: "\title"\end{title} +\begin{shorttitle} title-short: "\shorttitle"\end{shorttitle} +\begin{date} issued: \date\end{date} +\begin{url} url: \url\end{url} +\begin{doi} doi: \doi\end{doi} +\begin{volume} volume: \volume\end{volume} +\begin{number} number: \number\end{number} +\begin{urldate} accessed: \urldate\end{urldate} diff --git a/src/test/java/org/jabref/logic/exporter/YamlExporterTest.java b/src/test/java/org/jabref/logic/exporter/YamlExporterTest.java new file mode 100644 index 00000000000..cb5268cf005 --- /dev/null +++ b/src/test/java/org/jabref/logic/exporter/YamlExporterTest.java @@ -0,0 +1,108 @@ +package org.jabref.logic.exporter; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.jabref.logic.layout.LayoutFormatterPreferences; +import org.jabref.logic.xmp.XmpPreferences; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.types.StandardEntryType; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Answers; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; + +public class YamlExporterTest { + + private static Charset charset; + private static Exporter yamlExporter; + private static BibDatabaseContext databaseContext; + + @BeforeAll + static void setUp() { + List customFormats = new ArrayList<>(); + LayoutFormatterPreferences layoutPreferences = mock(LayoutFormatterPreferences.class, Answers.RETURNS_DEEP_STUBS); + SavePreferences savePreferences = mock(SavePreferences.class); + XmpPreferences xmpPreferences = mock(XmpPreferences.class); + ExporterFactory exporterFactory = ExporterFactory.create(customFormats, layoutPreferences, savePreferences, xmpPreferences); + + databaseContext = new BibDatabaseContext(); + charset = StandardCharsets.UTF_8; + yamlExporter = exporterFactory.getExporterByName("yaml").get(); + } + + @Test + public final void exportForNoEntriesWritesNothing(@TempDir Path tempFile) throws Exception { + Path file = tempFile.resolve("ThisIsARandomlyNamedFile"); + Files.createFile(file); + yamlExporter.export(databaseContext, tempFile, charset, Collections.emptyList()); + assertEquals(Collections.emptyList(), Files.readAllLines(file)); + } + + @Test + public final void exportsCorrectContent(@TempDir Path tempFile) throws Exception { + BibEntry entry = new BibEntry(StandardEntryType.Article) + .withCitationKey("test") + .withField(StandardField.AUTHOR, "Test Author") + .withField(StandardField.TITLE, "Test Title") + .withField(StandardField.URL, "http://example.com") + .withField(StandardField.DATE, "2020-10-14"); + + Path file = tempFile.resolve("RandomFileName"); + Files.createFile(file); + yamlExporter.export(databaseContext, file, charset, Collections.singletonList(entry)); + + List expected = List.of( + "---", + "references:", + "- id: test", + " type: article", + " author:", + " - literal: \"Test Author\"", + " title: \"Test Title\"", + " issued: 2020-10-14", + " url: http://example.com", + "---"); + + assertEquals(expected, Files.readAllLines(file)); + } + + @Test + public final void formatsContentCorrect(@TempDir Path tempFile) throws Exception { + BibEntry entry = new BibEntry(StandardEntryType.Misc) + .withCitationKey("test") + .withField(StandardField.AUTHOR, "Test Author") + .withField(StandardField.TITLE, "Test Title") + .withField(StandardField.URL, "http://example.com") + .withField(StandardField.DATE, "2020-10-14"); + + Path file = tempFile.resolve("RandomFileName"); + Files.createFile(file); + yamlExporter.export(databaseContext, file, charset, Collections.singletonList(entry)); + + List expected = List.of( + "---", + "references:", + "- id: test", + " type: no-type", + " author:", + " - literal: \"Test Author\"", + " title: \"Test Title\"", + " issued: 2020-10-14", + " url: http://example.com", + "---"); + + assertEquals(expected, Files.readAllLines(file)); + } +}