Skip to content

Commit

Permalink
Support for exporting to YAML format (#7007)
Browse files Browse the repository at this point in the history
  • Loading branch information
joethei committed Oct 22, 2020
1 parent 4cf2fb0 commit 7f4c36c
Show file tree
Hide file tree
Showing 11 changed files with 201 additions and 40 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve
- We added a query parser and mapping layer to enable conversion of queries formulated in simplified lucene syntax by the user into api queries. [#6799](https://github.com/JabRef/jabref/pull/6799)
- We added some basic functionality to customise the look of JabRef by importing a css theme file. [#5790](https://github.com/JabRef/jabref/issues/5790)
- We added connection check function in network preference setting [#6560](https://github.com/JabRef/jabref/issues/6560)
- We added support for exporting to YAML. [#6974](https://github.com/JabRef/jabref/issues/6974)
- We added a DOI format and organization check to detect [American Physical Society](https://journals.aps.org/) journals to copy the article ID
to the page field for cases where the page numbers are missing. [#7019](https://github.com/JabRef/jabref/issues/7019)
- We added a new fetcher to enable users to search jstor.org [#6627](https://github.com/JabRef/jabref/issues/6627)
Expand Down Expand Up @@ -57,6 +58,7 @@ inserting new citations in a OpenOffic/LibreOffice document. [#6957](https://git
- We fixed an issue with the python script used by browser plugins that failed to locate JabRef if not installed in its default location. [#6963](https://github.com/JabRef/jabref/pull/6963/files)
- We fixed an issue where spaces and newlines in an isbn would generate an exception. [#6456](https://github.com/JabRef/jabref/issues/6456)
- We fixed an issue where identity column header had incorrect foreground color in the Dark theme. [#6796](https://github.com/JabRef/jabref/issues/6796)
- We fixed an issue where the RIS exporter added extra blank lines.[#7007](https://github.com/JabRef/jabref/pull/7007/files)
- We fixed an issue where clicking on Collapse All button in the Search for Unlinked Local Files expanded the directory structure erroneously [#6848](https://github.com/JabRef/jabref/issues/6848)
- We fixed an issue, when pulling changes from shared database via shortcut caused creation a new new tech report [6867](https://github.com/JabRef/jabref/issues/6867)
- We fixed an issue where the JabRef GUI does not highlight the "All entries" group on start-up [#6691](https://github.com/JabRef/jabref/issues/6691)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package org.jabref.logic.exporter;

/**
* This enum represents the behaviour for blank lines in {@link TemplateExporter}
*/
public enum BlankLineBehaviour {
KEEP_BLANKS,
DELETE_BLANKS
}
3 changes: 2 additions & 1 deletion src/main/java/org/jabref/logic/exporter/ExporterFactory.java
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,9 @@ public static ExporterFactory create(List<TemplateExporter> customFormats,
exporters.add(new TemplateExporter("ISO 690", "iso690txt", "iso690", "iso690txt", StandardFileType.TXT, layoutPreferences, savePreferences));
exporters.add(new TemplateExporter("Endnote", "endnote", "EndNote", "endnote", StandardFileType.TXT, layoutPreferences, savePreferences));
exporters.add(new TemplateExporter("OpenOffice/LibreOffice CSV", "oocsv", "openoffice-csv", "openoffice", StandardFileType.CSV, layoutPreferences, savePreferences));
exporters.add(new TemplateExporter("RIS", "ris", "ris", "ris", StandardFileType.RIS, layoutPreferences, savePreferences).withEncoding(StandardCharsets.UTF_8));
exporters.add(new TemplateExporter("RIS", "ris", "ris", "ris", StandardFileType.RIS, layoutPreferences, savePreferences, BlankLineBehaviour.DELETE_BLANKS).withEncoding(StandardCharsets.UTF_8));
exporters.add(new TemplateExporter("MIS Quarterly", "misq", "misq", "misq", StandardFileType.RTF, layoutPreferences, savePreferences));
exporters.add(new TemplateExporter("CSL YAML", "yaml", "yaml", null, StandardFileType.YAML, layoutPreferences, savePreferences, BlankLineBehaviour.DELETE_BLANKS));
exporters.add(new BibTeXMLExporter());
exporters.add(new OpenOfficeDocumentCreator());
exporters.add(new OpenDocumentSpreadsheetCreator());
Expand Down
75 changes: 37 additions & 38 deletions src/main/java/org/jabref/logic/exporter/TemplateExporter.java
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,12 @@
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.regex.Pattern;

import org.jabref.logic.layout.Layout;
import org.jabref.logic.layout.LayoutFormatterPreferences;
import org.jabref.logic.layout.LayoutHelper;
import org.jabref.logic.util.FileType;
import org.jabref.logic.util.OS;
import org.jabref.logic.util.StandardFileType;
import org.jabref.model.database.BibDatabaseContext;
import org.jabref.model.entry.BibEntry;
Expand All @@ -33,15 +33,8 @@
*/
public class TemplateExporter extends Exporter {

private static final String BLANK_LINE_PATTERN = "\\r\\n|\\n";
private static final String LAYOUT_PREFIX = "/resource/layout/";

/**
* A regular expression that matches blank lines
*
* ?m activates "multimode", which makes ^ match line starts/ends.
* \\s simply marks any whitespace character
*/
private static final Pattern BLANK_LINE_MATCHER = Pattern.compile("(?m)^\\s");
private static final String LAYOUT_EXTENSION = ".layout";
private static final String FORMATTERS_EXTENSION = ".formatters";
private static final String BEGIN_INFIX = ".begin";
Expand All @@ -55,11 +48,10 @@ public class TemplateExporter extends Exporter {
private final SavePreferences savePreferences;
private Charset encoding; // If this value is set, it will be used to override the default encoding for the getCurrentBasePanel.
private boolean customExport;
private boolean deleteBlankLines;
private BlankLineBehaviour blankLineBehaviour;

/**
* Initialize another export format based on templates stored in dir with
* layoutFile lfFilename.
* Initialize another export format based on templates stored in dir with layoutFile lfFilename.
*
* @param displayName Name to display to the user.
* @param consoleName Name to call this format in the console.
Expand All @@ -72,8 +64,7 @@ public TemplateExporter(String displayName, String consoleName, String lfFileNam
}

/**
* Initialize another export format based on templates stored in dir with
* layoutFile lfFilename.
* Initialize another export format based on templates stored in dir with layoutFile lfFilename.
*
* @param name to display to the user and to call this format in the console.
* @param lfFileName Name of the main layout file.
Expand All @@ -87,8 +78,7 @@ public TemplateExporter(String name, String lfFileName, String extension, Layout
}

/**
* Initialize another export format based on templates stored in dir with
* layoutFile lfFilename.
* Initialize another export format based on templates stored in dir with layoutFile lfFilename.
*
* @param displayName Name to display to the user.
* @param consoleName Name to call this format in the console.
Expand All @@ -112,27 +102,36 @@ public TemplateExporter(String displayName, String consoleName, String lfFileNam
}

/**
* Initialize another export format based on templates stored in dir with
* layoutFile lfFilename.
* The display name is automatically derived from the FileType
* Initialize another export format based on templates stored in dir with layoutFile lfFilename.
*
* @param displayName Name to display to the user.
* @param consoleName Name to call this format in the console.
* @param lfFileName Name of the main layout file.
* @param directory Directory in which to find the layout file.
* @param extension Should contain the . (for instance .txt).
* @param layoutPreferences Preferences for layout
* @param savePreferences Preferences for saving
* @param deleteBlankLines If blank lines should be remove (default: false)
* @param blankLineBehaviour how to behave regarding blank lines.
*/
public TemplateExporter(String consoleName, String lfFileName, String directory, StandardFileType extension, LayoutFormatterPreferences layoutPreferences, SavePreferences savePreferences, boolean deleteBlankLines) {
this(consoleName, consoleName, lfFileName, directory, extension, layoutPreferences, savePreferences);
this.deleteBlankLines = deleteBlankLines;
public TemplateExporter(String displayName, String consoleName, String lfFileName, String directory, FileType extension,
LayoutFormatterPreferences layoutPreferences, SavePreferences savePreferences,
BlankLineBehaviour blankLineBehaviour) {
super(consoleName, displayName, extension);
if (Objects.requireNonNull(lfFileName).endsWith(LAYOUT_EXTENSION)) {
this.lfFileName = lfFileName.substring(0, lfFileName.length() - LAYOUT_EXTENSION.length());
} else {
this.lfFileName = lfFileName;
}
this.directory = directory;
this.layoutPreferences = layoutPreferences;
this.savePreferences = savePreferences;
this.blankLineBehaviour = blankLineBehaviour;
}

/**
* Indicate whether this is a custom export. A custom export looks for its
* layout files using a normal file path, while a built-in export looks in
* the classpath.
* Indicate whether this is a custom export.
* A custom export looks for its layout files using a normal file path,
* while a built-in export looks in the classpath.
*
* @param custom true to indicate a custom export format.
*/
Expand All @@ -141,8 +140,7 @@ public void setCustomExport(boolean custom) {
}

/**
* Set an encoding which will be used in preference to the default value
* obtained from the basepanel.
* Set an encoding which will be used in preference to the default value obtained from the basepanel.
*
* @param encoding The name of the encoding to use.
*/
Expand All @@ -152,11 +150,9 @@ public TemplateExporter withEncoding(Charset encoding) {
}

/**
* This method should return a reader from which the given layout file can
* be read.
* This method should return a reader from which the given layout file can be read.
* <p>
* Subclasses of TemplateExporter are free to override and provide their own
* implementation.
* Subclasses of TemplateExporter are free to override and provide their own implementation.
*
* @param filename the filename
* @return a newly created reader
Expand Down Expand Up @@ -277,9 +273,13 @@ public void export(final BibDatabaseContext databaseContext, final Path file,

// Write the entry
if (layout != null) {
if (deleteBlankLines) {
String withoutBlankLines = BLANK_LINE_MATCHER.matcher(layout.doLayout(entry, databaseContext.getDatabase())).replaceAll("");
ps.write(withoutBlankLines);
if (blankLineBehaviour == BlankLineBehaviour.DELETE_BLANKS) {
String[] lines = layout.doLayout(entry, databaseContext.getDatabase()).split(BLANK_LINE_PATTERN);
for (String line : lines) {
if (!line.isBlank() && !line.isEmpty()) {
ps.write(line + OS.NEWLINE);
}
}
} else {
ps.write(layout.doLayout(entry, databaseContext.getDatabase()));
}
Expand Down Expand Up @@ -316,9 +316,8 @@ public void export(final BibDatabaseContext databaseContext, final Path file,
}

/**
* See if there is a name formatter file bundled with this export format. If so, read
* all the name formatters so they can be used by the filter layouts.
*
* See if there is a name formatter file bundled with this export format.
* If so, read all the name formatters so they can be used by the filter layouts.
*/
private void readFormatterFile() {
File formatterFile = new File(lfFileName + FORMATTERS_EXTENSION);
Expand Down
3 changes: 3 additions & 0 deletions src/main/java/org/jabref/logic/layout/LayoutEntry.java
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.jabref.logic.layout.format.AuthorNatBib;
import org.jabref.logic.layout.format.AuthorOrgSci;
import org.jabref.logic.layout.format.Authors;
import org.jabref.logic.layout.format.CSLType;
import org.jabref.logic.layout.format.CompositeFormat;
import org.jabref.logic.layout.format.CreateBibORDFAuthors;
import org.jabref.logic.layout.format.CreateDocBook4Authors;
Expand Down Expand Up @@ -539,6 +540,8 @@ private LayoutFormatter getLayoutFormatterByName(String name) {
return new WrapFileLinks(prefs.getFileLinkPreferences());
case "Markdown":
return new MarkdownFormatter();
case "CSLType":
return new CSLType();
default:
return null;
}
Expand Down
21 changes: 21 additions & 0 deletions src/main/java/org/jabref/logic/layout/format/CSLType.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package org.jabref.logic.layout.format;

import org.jabref.logic.layout.LayoutFormatter;
import org.jabref.model.entry.types.StandardEntryType;

public class CSLType implements LayoutFormatter {

@Override
public String format(String value) {
return switch (StandardEntryType.valueOf(value)) {
case Article -> "article";
case Book -> "book";
case Conference -> "paper-conference";
case Report, TechReport -> "report";
case Thesis, MastersThesis, PhdThesis -> "thesis";
case WWW, Online -> "webpage";

default -> "no-type";
};
}
}
3 changes: 2 additions & 1 deletion src/main/java/org/jabref/logic/util/StandardFileType.java
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ public enum StandardFileType implements FileType {
JSON("json"),
XMP("xmp"),
ZIP("zip"),
CSS("css");
CSS("css"),
YAML("yaml");

private final List<String> extensions;

Expand Down
2 changes: 2 additions & 0 deletions src/main/resources/resource/layout/yaml.begin.layout
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
---
references:
1 change: 1 addition & 0 deletions src/main/resources/resource/layout/yaml.end.layout
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
---
14 changes: 14 additions & 0 deletions src/main/resources/resource/layout/yaml.layout
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
- id: \citationkey
\begin{entrytype} type: \format[CSLType]{\entrytype}\end{entrytype}
\begin{author}
author:
- literal: "\author"
\end{author}
\begin{title} title: "\title"\end{title}
\begin{shorttitle} title-short: "\shorttitle"\end{shorttitle}
\begin{date} issued: \date\end{date}
\begin{url} url: \url\end{url}
\begin{doi} doi: \doi\end{doi}
\begin{volume} volume: \volume\end{volume}
\begin{number} number: \number\end{number}
\begin{urldate} accessed: \urldate\end{urldate}
108 changes: 108 additions & 0 deletions src/test/java/org/jabref/logic/exporter/YamlExporterTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
package org.jabref.logic.exporter;

import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

import org.jabref.logic.layout.LayoutFormatterPreferences;
import org.jabref.logic.xmp.XmpPreferences;
import org.jabref.model.database.BibDatabaseContext;
import org.jabref.model.entry.BibEntry;
import org.jabref.model.entry.field.StandardField;
import org.jabref.model.entry.types.StandardEntryType;

import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.Answers;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.mock;

public class YamlExporterTest {

private static Charset charset;
private static Exporter yamlExporter;
private static BibDatabaseContext databaseContext;

@BeforeAll
static void setUp() {
List<TemplateExporter> customFormats = new ArrayList<>();
LayoutFormatterPreferences layoutPreferences = mock(LayoutFormatterPreferences.class, Answers.RETURNS_DEEP_STUBS);
SavePreferences savePreferences = mock(SavePreferences.class);
XmpPreferences xmpPreferences = mock(XmpPreferences.class);
ExporterFactory exporterFactory = ExporterFactory.create(customFormats, layoutPreferences, savePreferences, xmpPreferences);

databaseContext = new BibDatabaseContext();
charset = StandardCharsets.UTF_8;
yamlExporter = exporterFactory.getExporterByName("yaml").get();
}

@Test
public final void exportForNoEntriesWritesNothing(@TempDir Path tempFile) throws Exception {
Path file = tempFile.resolve("ThisIsARandomlyNamedFile");
Files.createFile(file);
yamlExporter.export(databaseContext, tempFile, charset, Collections.emptyList());
assertEquals(Collections.emptyList(), Files.readAllLines(file));
}

@Test
public final void exportsCorrectContent(@TempDir Path tempFile) throws Exception {
BibEntry entry = new BibEntry(StandardEntryType.Article)
.withCitationKey("test")
.withField(StandardField.AUTHOR, "Test Author")
.withField(StandardField.TITLE, "Test Title")
.withField(StandardField.URL, "http://example.com")
.withField(StandardField.DATE, "2020-10-14");

Path file = tempFile.resolve("RandomFileName");
Files.createFile(file);
yamlExporter.export(databaseContext, file, charset, Collections.singletonList(entry));

List<String> expected = List.of(
"---",
"references:",
"- id: test",
" type: article",
" author:",
" - literal: \"Test Author\"",
" title: \"Test Title\"",
" issued: 2020-10-14",
" url: http://example.com",
"---");

assertEquals(expected, Files.readAllLines(file));
}

@Test
public final void formatsContentCorrect(@TempDir Path tempFile) throws Exception {
BibEntry entry = new BibEntry(StandardEntryType.Misc)
.withCitationKey("test")
.withField(StandardField.AUTHOR, "Test Author")
.withField(StandardField.TITLE, "Test Title")
.withField(StandardField.URL, "http://example.com")
.withField(StandardField.DATE, "2020-10-14");

Path file = tempFile.resolve("RandomFileName");
Files.createFile(file);
yamlExporter.export(databaseContext, file, charset, Collections.singletonList(entry));

List<String> expected = List.of(
"---",
"references:",
"- id: test",
" type: no-type",
" author:",
" - literal: \"Test Author\"",
" title: \"Test Title\"",
" issued: 2020-10-14",
" url: http://example.com",
"---");

assertEquals(expected, Files.readAllLines(file));
}
}

0 comments on commit 7f4c36c

Please sign in to comment.