Skip to content

Commit

Permalink
Fix #469: Add a way to distinguish between null and empty (#471)
Browse files Browse the repository at this point in the history
  • Loading branch information
PujolDavid authored and cowtowncoder committed Apr 10, 2024
1 parent 029030b commit 163849b
Show file tree
Hide file tree
Showing 3 changed files with 194 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -148,11 +148,28 @@ public enum Feature
INSERT_NULLS_FOR_MISSING_COLUMNS(false),

/**
* Feature that enables coercing an empty {@link String} to `null`
* Feature that enables coercing an empty {@link String} to `null`.
*<p>
* Note that if this setting is enabled, {@link #EMPTY_UNQUOTED_STRING_AS_NULL}
* has no effect.
*
* Feature is disabled by default
* Feature is disabled by default for backwards compatibility.
*/
EMPTY_STRING_AS_NULL(false)
EMPTY_STRING_AS_NULL(false),

/**
* Feature that enables coercing an empty un-quoted {@link String} to `null`.
* This feature allow differentiating between an empty quoted {@link String} and an empty un-quoted {@link String}.
*<p>
* Note that this feature is only considered if
* {@link #EMPTY_STRING_AS_NULL}
* is disabled.
*<p>
* Feature is disabled by default for backwards compatibility.
*
* @since 2.18
*/
EMPTY_UNQUOTED_STRING_AS_NULL(false),
;

final boolean _defaultState;
Expand Down Expand Up @@ -326,6 +343,11 @@ private Feature(boolean defaultState) {
*/
protected boolean _cfgEmptyStringAsNull;

/**
* @since 2.18
*/
protected boolean _cfgEmptyUnquotedStringAsNull;

/*
/**********************************************************************
/* State
Expand Down Expand Up @@ -426,6 +448,7 @@ public CsvParser(IOContext ctxt, int stdFeatures, int csvFeatures,
_reader = new CsvDecoder(this, ctxt, reader, _schema, _textBuffer,
stdFeatures, csvFeatures);
_cfgEmptyStringAsNull = CsvParser.Feature.EMPTY_STRING_AS_NULL.enabledIn(csvFeatures);
_cfgEmptyUnquotedStringAsNull = Feature.EMPTY_UNQUOTED_STRING_AS_NULL.enabledIn(csvFeatures);
}

@Override
Expand Down Expand Up @@ -537,6 +560,7 @@ public JsonParser overrideFormatFeatures(int values, int mask) {
_formatFeatures = newF;
_reader.overrideFormatFeatures(newF);
_cfgEmptyStringAsNull = CsvParser.Feature.EMPTY_STRING_AS_NULL.enabledIn(_formatFeatures);
_cfgEmptyUnquotedStringAsNull = Feature.EMPTY_UNQUOTED_STRING_AS_NULL.enabledIn(_formatFeatures);
}
return this;
}
Expand All @@ -555,6 +579,7 @@ public JsonParser enable(Feature f)
{
_formatFeatures |= f.getMask();
_cfgEmptyStringAsNull = CsvParser.Feature.EMPTY_STRING_AS_NULL.enabledIn(_formatFeatures);
_cfgEmptyUnquotedStringAsNull = Feature.EMPTY_UNQUOTED_STRING_AS_NULL.enabledIn(_formatFeatures);
return this;
}

Expand All @@ -566,6 +591,7 @@ public JsonParser disable(Feature f)
{
_formatFeatures &= ~f.getMask();
_cfgEmptyStringAsNull = CsvParser.Feature.EMPTY_STRING_AS_NULL.enabledIn(_formatFeatures);
_cfgEmptyUnquotedStringAsNull = Feature.EMPTY_UNQUOTED_STRING_AS_NULL.enabledIn(_formatFeatures);
return this;
}

Expand Down Expand Up @@ -1434,7 +1460,6 @@ protected void _startArray(CsvSchema.Column column)
_arraySeparator = sep;
}


/**
* Helper method called to check whether specified String value should be considered
* "null" value, if so configured.
Expand All @@ -1450,6 +1475,9 @@ protected boolean _isNullValue(String value) {
if (_cfgEmptyStringAsNull && value.isEmpty()) {
return true;
}
if (_cfgEmptyUnquotedStringAsNull && value.isEmpty() && !_reader.isCurrentTokenQuoted()) {
return true;
}
return false;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,13 @@ public class CsvDecoder
*/
protected int _currInputRowStart = 0;

/**
* Flag that indicates whether the current token has been quoted or not.
*
* @since 2.18
*/
protected boolean _currInputQuoted = false;

// // // Location info at point when current token was started

/**
Expand Down Expand Up @@ -405,6 +412,16 @@ public final int getCurrentColumn() {
}
return ptr - _currInputRowStart + 1; // 1-based
}

/**
* Tell if the current token has been quoted or not.
* @return True if the current token has been quoted, false otherwise
*
* @since 2.18
*/
public final boolean isCurrentTokenQuoted() {
return _currInputQuoted;
}

/*
/**********************************************************************
Expand Down Expand Up @@ -673,7 +690,8 @@ public String nextString() throws IOException
return "";
}
// two modes: quoted, unquoted
if (i == _quoteChar) { // offline quoted case (longer)
_currInputQuoted = i == _quoteChar; // Keep track of quoting
if (_currInputQuoted) { // offline quoted case (longer)
return _nextQuotedString();
}
if (i == _separatorChar) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
package com.fasterxml.jackson.dataformat.csv.deser;

import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import com.fasterxml.jackson.databind.MappingIterator;
import com.fasterxml.jackson.databind.ObjectReader;
import com.fasterxml.jackson.dataformat.csv.CsvMapper;
import com.fasterxml.jackson.dataformat.csv.CsvParser;
import com.fasterxml.jackson.dataformat.csv.ModuleTestBase;

import java.io.IOException;

/**
* Tests for {@code CsvParser.Feature.EMPTY_UNQUOTED_STRING_AS_NULL}
*/
public class EmptyUnquotedStringAsNullTest
extends ModuleTestBase
{
@JsonPropertyOrder({"firstName", "middleName", "lastName"})
static class TestUser {
public String firstName, middleName, lastName;
}

/*
/**********************************************************
/* Test methods
/**********************************************************
*/

private final CsvMapper MAPPER = mapperForCsv();

public void testDefaultParseAsEmptyString() throws IOException {
// setup test data
TestUser expectedTestUser = new TestUser();
expectedTestUser.firstName = "Grace";
expectedTestUser.middleName = "";
expectedTestUser.lastName = "Hopper";
ObjectReader objectReader = MAPPER.readerFor(TestUser.class).with(MAPPER.schemaFor(TestUser.class));
String csv = "Grace,,Hopper";

// execute
TestUser actualTestUser = objectReader.readValue(csv);

// test
assertNotNull(actualTestUser);
assertEquals(expectedTestUser.firstName, actualTestUser.firstName);
assertEquals(expectedTestUser.middleName, actualTestUser.middleName);
assertEquals(expectedTestUser.lastName, actualTestUser.lastName);
}

public void testSimpleParseEmptyUnquotedStringAsNull() throws IOException {
// setup test data
TestUser expectedTestUser = new TestUser();
expectedTestUser.firstName = "Grace";
expectedTestUser.lastName = "Hopper";

ObjectReader objectReader = MAPPER
.readerFor(TestUser.class)
.with(MAPPER.schemaFor(TestUser.class))
.with(CsvParser.Feature.EMPTY_UNQUOTED_STRING_AS_NULL);
String csv = "Grace,,Hopper";

// execute
TestUser actualTestUser = objectReader.readValue(csv);

// test
assertNotNull(actualTestUser);
assertEquals(expectedTestUser.firstName, actualTestUser.firstName);
assertNull("The column that contains an empty String should be deserialized as null ", actualTestUser.middleName);
assertEquals(expectedTestUser.lastName, actualTestUser.lastName);
}

public void testSimpleParseEmptyQuotedStringAsNonNull() throws IOException {
// setup test data
TestUser expectedTestUser = new TestUser();
expectedTestUser.firstName = "Grace";
expectedTestUser.middleName = "";
expectedTestUser.lastName = "Hopper";

ObjectReader objectReader = MAPPER
.readerFor(TestUser.class)
.with(MAPPER.schemaFor(TestUser.class))
.with(CsvParser.Feature.EMPTY_UNQUOTED_STRING_AS_NULL);
String csv = "Grace,\"\",Hopper";

// execute
TestUser actualTestUser = objectReader.readValue(csv);

// test
assertNotNull(actualTestUser);
assertEquals(expectedTestUser.firstName, actualTestUser.firstName);
assertEquals(expectedTestUser.middleName, actualTestUser.middleName);
assertEquals(expectedTestUser.lastName, actualTestUser.lastName);
}

// [dataformats-text#222]
public void testEmptyUnquotedStringAsNullNonPojo() throws Exception
{
String csv = "Grace,,Hopper";

ObjectReader r = MAPPER.reader()
.with(CsvParser.Feature.EMPTY_UNQUOTED_STRING_AS_NULL)
.with(CsvParser.Feature.WRAP_AS_ARRAY);

try (MappingIterator<Object[]> it1 = r.forType(Object[].class).readValues(csv)) {
Object[] array1 = it1.next();
assertEquals(3, array1.length);
assertEquals("Grace", array1[0]);
assertNull(array1[1]);
assertEquals("Hopper", array1[2]);
}
try (MappingIterator<String[]> it2 = r.forType(String[].class).readValues(csv)) {
String[] array2 = it2.next();
assertEquals(3, array2.length);
assertEquals("Grace", array2[0]);
assertNull(array2[1]);
assertEquals("Hopper", array2[2]);
}
}

public void testEmptyQuotedStringAsNonNullNonPojo() throws Exception
{
String csv = "Grace,\"\",Hopper";

ObjectReader r = MAPPER.reader()
.with(CsvParser.Feature.EMPTY_UNQUOTED_STRING_AS_NULL)
.with(CsvParser.Feature.WRAP_AS_ARRAY);

try (MappingIterator<Object[]> it1 = r.forType(Object[].class).readValues(csv)) {
Object[] array1 = it1.next();
assertEquals(3, array1.length);
assertEquals("Grace", array1[0]);
assertEquals("", array1[1]);
assertEquals("Hopper", array1[2]);
}
try (MappingIterator<String[]> it2 = r.forType(String[].class).readValues(csv)) {
String[] array2 = it2.next();
assertEquals(3, array2.length);
assertEquals("Grace", array2[0]);
assertEquals("", array2[1]);
assertEquals("Hopper", array2[2]);
}
}
}

0 comments on commit 163849b

Please sign in to comment.