Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow pre-defined empty column types #899

Closed
wants to merge 1 commit into from
Closed

Allow pre-defined empty column types #899

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file added data/columns-with-empty-column.xlsx
View file
Open in desktop
Binary file not shown.
9 changes: 9 additions & 0 deletions excel/src/main/java/tech/tablesaw/io/xlsx/XlsxReadOptions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,22 @@
import java.io.Reader;
import java.net.URL;
import java.time.format.DateTimeFormatter;
import java.util.HashMap;
import java.util.Locale;
import tech.tablesaw.api.ColumnType;
import tech.tablesaw.io.ReadOptions;
import tech.tablesaw.io.Source;

public class XlsxReadOptions extends ReadOptions {

/** The sheet to read. Null means no specific index was set. First sheet has index 0. */
protected Integer sheetIndex;
protected HashMap<String, ColumnType> preserveColumnType = new HashMap<>();

protected XlsxReadOptions(Builder builder) {
super(builder);
sheetIndex = builder.sheetIndex;
preserveColumnType= builder.preserveColumnType;
}

public static Builder builder(Source source) {
Expand Down Expand Up @@ -51,6 +55,7 @@ public Integer sheetIndex() {
public static class Builder extends ReadOptions.Builder {

protected Integer sheetIndex;
protected HashMap<String, ColumnType> preserveColumnType = new HashMap<>();

protected Builder(Source source) {
super(source);
Expand Down Expand Up @@ -158,5 +163,9 @@ public Builder sheetIndex(int sheetIndex) {
this.sheetIndex = sheetIndex;
return this;
}

public void preserveEmptyColumnType(String columnName, ColumnType columnType) {
preserveColumnType.put(columnName, columnType);
}
}
}
13 changes: 13 additions & 0 deletions excel/src/main/java/tech/tablesaw/io/xlsx/XlsxReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,15 @@ private Table createTable(Sheet sheet, TableRange tableArea, XlsxReadOptions opt
column = altColumn;
columns.set(colNum, column);
}
} else {
if (column == null && rowNum == tableArea.endRow && options.preserveColumnType.containsKey(headerNames.get(colNum))) {
ColumnType type = options.preserveColumnType.get(headerNames.get(colNum));
column = createColumn(headerNames.get(colNum), type);
columns.set(colNum, column);
while (column.size() < rowNum - tableArea.startRow) {
column.appendMissing();
}
}
}
if (column != null) {
while (column.size() <= rowNum - tableArea.startRow) {
Expand Down Expand Up @@ -357,6 +366,10 @@ private Column<?> createColumn(String name, Cell cell) {
return column;
}

private Column<?> createColumn(String name, ColumnType columnType) {
return columnType.create(name);
}

@Override
public Table read(Source source) throws IOException {
return read(XlsxReadOptions.builder(source).build());
Expand Down
79 changes: 74 additions & 5 deletions excel/src/test/java/tech/tablesaw/io/xlsx/XlsxReaderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.util.List;

import static org.junit.jupiter.api.Assertions.*;
import static tech.tablesaw.api.ColumnType.*;

public class XlsxReaderTest {

Expand Down Expand Up @@ -111,11 +112,9 @@ public void testColumnsWithMissingValues() {
"booleancol",
"datecol",
"formulacol");
// stringcol shortcol intcol longcol doublecol booleancol
// datecol
// Hallvard 12345678 12345678900 TRUE 22/02/2019
// 20:54:09
// 124 12345679 13,35
// stringcol shortcol intcol longcol doublecol booleancol datecol formulacol
// Hallvard 12345678 12345678900 TRUE 22/02/2019 20:54:09
// 124 12345679 13.35 137.35
assertColumnValues(table.stringColumn("stringcol"), "Hallvard", null);
assertColumnValues(table.intColumn("shortcol"), null, 124);
assertColumnValues(table.intColumn("intcol"), 12345678, 12345679);
Expand All @@ -127,6 +126,76 @@ public void testColumnsWithMissingValues() {

}

@Test
public void testColumnsWithEmptyColumn() {
Table table =
read1(
"columns-with-empty-column",
2,
"stringcol",
"intcol",
"longcol",
"doublecol",
"booleancol",
"datecol",
"formulacol");
// stringcol shortcol intcol longcol doublecol booleancol datecol formulacol
// Hallvard 12345678 12345678900 TRUE 22/02/2019 20:54:09
// 12345679 13.35 13.35
assertColumnValues(table.stringColumn("stringcol"), "Hallvard", null);
assertColumnValues(table.intColumn("intcol"), 12345678, 12345679);
assertColumnValues(table.longColumn("longcol"), 12345678900L, null);
assertColumnValues(table.doubleColumn("doublecol"), null, 13.35);
assertColumnValues(table.booleanColumn("booleancol"), true, null);
assertColumnValues(table.dateTimeColumn("datecol"), LocalDateTime.of(2019, 2, 22, 20, 54, 9), null);
assertColumnValues(table.doubleColumn("formulacol"), null, 13.35);
}

@Test
public void testColumnsWithEmptyColumnAndPreserve() {
String[] columnNames = new String[]{"stringcol",
"shortcol",
"intcol",
"longcol",
"doublecol",
"booleancol",
"datecol",
"formulacol"};
// stringcol shortcol intcol longcol doublecol booleancol datecol formulacol
// Hallvard 12345678 12345678900 TRUE 22/02/2019 20:54:09
// 12345679 13.35 13.35

Table table = null;
try {
String fileName = "columns-with-empty-column.xlsx";
XlsxReadOptions.Builder builder = XlsxReadOptions.builder("../data/" + fileName);
builder.preserveEmptyColumnType("shortcol", SHORT);
List<Table> tables =
new XlsxReader().readMultiple(builder.build());
assertNotNull(tables, "No tables read from " + fileName);
assertEquals(1, tables.size(), "Wrong number of tables in " + fileName);
table = tables.get(0);
} catch (final IOException e) {
fail(e.getMessage());
}

int colNum = 0, size = 2;
for (final Column<?> column : table.columns()) {
assertEquals(columnNames[colNum], column.name(), "Wrong column name");
assertEquals(size, column.size(), "Wrong size for column " + columnNames[colNum]);
colNum++;
}

assertColumnValues(table.stringColumn("stringcol"), "Hallvard", null);
assertColumnValues(table.shortColumn("shortcol"), null, null);
assertColumnValues(table.intColumn("intcol"), 12345678, 12345679);
assertColumnValues(table.longColumn("longcol"), 12345678900L, null);
assertColumnValues(table.doubleColumn("doublecol"), null, 13.35);
assertColumnValues(table.booleanColumn("booleancol"), true, null);
assertColumnValues(table.dateTimeColumn("datecol"), LocalDateTime.of(2019, 2, 22, 20, 54, 9), null);
assertColumnValues(table.doubleColumn("formulacol"), null, 13.35);
}

@Test
public void testSheetIndex() throws IOException {
Table table =
Expand Down