Skip to content

Commit

Permalink
ISSUE: MCZ Redmine 866 tdwg/bdq#86 PURPOSE: Updating tests to current…
Browse files Browse the repository at this point in the history 
… specifications. DESCRIPTION: Updating implementation of AMENDMENT_EVENTDATE_FROM_VERBATIM to fit current (2022-03-08) specification, making method name consistent, and deprecating old method.
  • Loading branch information
@chicoreus
chicoreus committed Mar 9, 2022
1 parent 4cfb972 commit 29a4551
Show file tree
Hide file tree
Showing 3 changed files with 108 additions and 49 deletions.
87 changes: 50 additions & 37 deletions src/main/java/org/filteredpush/qc/date/DwCEventDQ.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
*
* MEASURE_EVENTDATE_PRECISIONINSECONDS 56b6c695-adf1-418e-95d2-da04cad7be53
*
* Provides support for the following TDWG DQIG TG2 validations and amendments
* Provides support for the following TDWG DQIG TG2 validations
*
* VALIDATION_EVENTDATE_EMPTY f51e15a6-a67d-4729-9c28-3766299d2985
* VALIDATION_YEAR_EMPTY c09ecbf9-34e3-4f3e-b74a-8796af15e59f
Expand All @@ -51,12 +51,12 @@
* TODO: Update:
* VALIDATION_DAY_OUTOFRANGE 5618f083-d55a-4ac2-92b5-b9fb227b832f ** Differs
*
* Provides support for the following TDWG DQIG TG2 amendments
*
* AMENDMENT_EVENTDATE_FROM_VERBATIM 6d0a0c10-5e4a-4759-b448-88932f399812
*
* Provides support for the following draft TDWG DQIG TG2 validations and amendments.
*
* TG2-VALIDATION_EVENTDATE_EMPTY f51e15a6-a67d-4729-9c28-3766299d2985
* TG2-VALIDATION_YEAR_EMPTY c09ecbf9-34e3-4f3e-b74a-8796af15e59f
* TG2-AMENDMENT_EVENTDATE_FROM_VERBATIM
* (Old list, to update)
* ?? TG2-VALIDATION_EVENTDATE_NOTSTANDARD 4f2bf8fd-fc5c-493f-a44c-e7b16153c803
* TG2-AMENDMENT_EVENTDATE_STANDARDIZED 718dfc3c-cb52-4fca-b8e2-0e722f375da7
* TG2-VALIDATION_MONTH_OUTOFRANGE isMonthInRange(@ActedUpon(value = "dwc:month") String month)
Expand Down Expand Up @@ -380,56 +380,69 @@ public static DQResponse<ComplianceValue> isEventDateConsistentWithVerbatim(
}

/**
* Deprecated, use: amendmentEventdateFromVerbatim
*/
@Deprecated
public static DQResponse<AmendmentValue> extractDateFromVerbatim(
@ActedUpon(value = "dwc:eventDate") String eventDate,
@Consulted(value = "dwc:verbatimEventDate") String verbatimEventDate) {
return amendmentEventdateFromVerbatim(eventDate, verbatimEventDate);
}
/**
* #86 Amendment SingleRecord Completeness: eventdate from verbatim
*
* Provides: AMENDMENT_EVENTDATE_FROM_VERBATIM
*
* If a dwc:eventDate is empty and the verbatimEventDate is not empty, try to populate the
* eventDate from the verbatim value.
*
* TG2-AMENDMENT_EVENTDATE_FROM_VERBATIM
*
* Run in order: extractDateFromVerbatim, then eventDateFromYearStartEndDay, then eventDateFromYearMonthDay
*
* @see eventDateFromYearStartEndDay
* @see eventDateFromYearMonthDay
*
* @param eventDate to check for emptyness
* @param verbatimEventDate to try to replace a non-empty event date.
* @return an implementation of DQAmendmentResponse, with a value containing a key for dwc:eventDate and a
* resultState is AMENDED if a new value is proposed.
*
*
* @param eventDate the provided dwc:eventDate to try to populate if empty
* @param verbatimEventDate the provided dwc:verbatimEventDate from which to try to parse an eventDate
* @return DQResponse the response of type AmendmentValue to return, with a value containing
* a key for dwc:eventDate and a resultState is AMENDED if a new value is proposed.
*/
//@Provides(value = "EVENTDATE_FILLED_IN_FROM_VERBATIM")
@Provides(value="urn:uuid:6d0a0c10-5e4a-4759-b448-88932f399812")
@Amendment( label = "AMENDMENT_EVENTDATE_FROM_VERBATIM", description="The value of dwc:eventDate was interpreted from dwc:verbatimEventDate")
@Specification(value="The value of dwc:eventDate was interpreted from dwc:verbatimEventDate The field dwc:eventDate is EMPTY and the field dwc:verbatimEventDate is not EMPTY and is interpretable as an ISO 8601:2004(E) date")
//@Specification(value = "If a dwc:eventDate is empty and the verbatimEventDate is not empty fill in dwc:eventDate " +
// "based on value from dwc:verbatimEventDate")
public static DQResponse<AmendmentValue> extractDateFromVerbatim(@ActedUpon(value = "dwc:eventDate") String eventDate, @Consulted(value = "dwc:verbatimEventDate") String verbatimEventDate) {
DQResponse<AmendmentValue> result = new DQResponse<>();

@Provides("6d0a0c10-5e4a-4759-b448-88932f399812")
public static DQResponse<AmendmentValue> amendmentEventdateFromVerbatim(
@ActedUpon("dwc:eventDate") String eventDate,
@Consulted("dwc:verbatimEventDate") String verbatimEventDate) {
DQResponse<AmendmentValue> result = new DQResponse<AmendmentValue>();

// Specification
// INTERNAL_PREREQUISITES_NOT_MET if dwc:eventDate is not EMPTY
// or the value of dwc:verbatimEventDate is EMPTY or not unambiguously
// interpretable as an ISO 8601-1:2019 date; AMENDED if the
// value of dwc:eventDate was unambiguously interpreted from
// dwc:verbatimEventDate; otherwise NOT_AMENDED

if (DateUtils.isEmpty(eventDate)) {
if (!DateUtils.isEmpty(verbatimEventDate)) {
EventResult extractResponse = DateUtils.extractDateFromVerbatimER(verbatimEventDate);
if (!extractResponse.getResultState().equals(EventResult.EventQCResultState.NOT_RUN) &&
(extractResponse.getResultState().equals(EventResult.EventQCResultState.RANGE) ||
extractResponse.getResultState().equals(EventResult.EventQCResultState.DATE) ||
extractResponse.getResultState().equals(EventResult.EventQCResultState.AMBIGUOUS) ||
extractResponse.getResultState().equals(EventResult.EventQCResultState.SUSPECT)
)
)
extractResponse.getResultState().equals(EventResult.EventQCResultState.DATE)) )
{
Map<String, String> extractedValues = new HashMap<>();
extractedValues.put("dwc:eventDate", extractResponse.getResult());

result.setValue(new AmendmentValue(extractedValues));

result.setResultState(ResultState.AMENDED);
} else if (!extractResponse.getResultState().equals(EventResult.EventQCResultState.NOT_RUN) &&
( extractResponse.getResultState().equals(EventResult.EventQCResultState.AMBIGUOUS) ||
extractResponse.getResultState().equals(EventResult.EventQCResultState.SUSPECT)))
{
result.setResultState(ResultState.NOT_AMENDED);
if (extractResponse.getResultState().equals(EventResult.EventQCResultState.AMBIGUOUS)) {
result.setResultState(ResultState.AMBIGUOUS);
//TODO: Implement specification
// INTERNAL_PREREQUISITES_NOT_MET if dwc:eventDate is not EMPTY
// or the value of dwc:verbatimEventDate is EMPTY or not unambiguously
// interpretable as an ISO 8601-1:2019 date; AMENDED if the
// value of dwc:eventDate was unambiguously interpreted from
//dwc:verbatimEventDate; otherwise NOT_AMENDED result.addComment("Supplied verbatimEventDate [" + verbatimEventDate + "] is ambiguous.");
result.addComment(extractResponse.getComment());
} else {
if (extractResponse.getResultState().equals(EventResult.EventQCResultState.SUSPECT)) {
result.addComment("Interpretation of verbatimEventDate [" + verbatimEventDate + "] is suspect.");
result.addComment(extractResponse.getComment());
}
result.setResultState(ResultState.AMENDED);
}
} else {
result.setResultState(ResultState.INTERNAL_PREREQUISITES_NOT_MET);
Expand All @@ -440,7 +453,7 @@ public static DQResponse<AmendmentValue> extractDateFromVerbatim(@ActedUpon(val
result.addComment("verbatimEventDate does not contains a value.");
}
} else {
result.setResultState(ResultState.NOT_AMENDED);
result.setResultState(ResultState.INTERNAL_PREREQUISITES_NOT_MET);
result.addComment("eventDate contains a value, not changing.");
}

Expand Down
42 changes: 41 additions & 1 deletion src/test/java/org/filteredpush/qc/date/DwCEventDQTestDefinitions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.datakurator.ffdq.api.DQResponse;
import org.datakurator.ffdq.api.result.AmendmentValue;
import org.datakurator.ffdq.api.result.ComplianceValue;
import org.datakurator.ffdq.api.result.NumericalValue;
import org.datakurator.ffdq.model.ResultState;
Expand Down Expand Up @@ -161,7 +162,46 @@ public void testValidationYearOutofrange() {
*/
@Test
public void testAmendmentEventdateFromVerbatim() {
fail("Not yet implemented");

//TODO: Implement specification
// INTERNAL_PREREQUISITES_NOT_MET if dwc:eventDate is not EMPTY
// or the value of dwc:verbatimEventDate is EMPTY or not unambiguously
// interpretable as an ISO 8601-1:2019 date; AMENDED if the
// value of dwc:eventDate was unambiguously interpreted from
//dwc:verbatimEventDate; otherwise NOT_AMENDED

String eventDate = "1900";
String verbatimEventDate = "";
DQResponse<AmendmentValue> response = DwCEventDQ.amendmentEventdateFromVerbatim(eventDate, verbatimEventDate);
assertEquals(ResultState.INTERNAL_PREREQUISITES_NOT_MET.getLabel(), response.getResultState().getLabel());
assertEquals(0, response.getValue().getObject().size());
logger.debug(response.getComment());
eventDate = "";
verbatimEventDate = "";
response = DwCEventDQ.amendmentEventdateFromVerbatim(eventDate, verbatimEventDate);
assertEquals(ResultState.INTERNAL_PREREQUISITES_NOT_MET.getLabel(), response.getResultState().getLabel());
assertEquals(0, response.getValue().getObject().size());
logger.debug(response.getComment());
eventDate = "";
verbatimEventDate = "foo";
response = DwCEventDQ.amendmentEventdateFromVerbatim(eventDate, verbatimEventDate);
assertEquals(ResultState.INTERNAL_PREREQUISITES_NOT_MET.getLabel(), response.getResultState().getLabel());
assertEquals(0, response.getValue().getObject().size());
logger.debug(response.getComment());
eventDate = "";
verbatimEventDate = "1932/11/23";
response = DwCEventDQ.amendmentEventdateFromVerbatim(eventDate, verbatimEventDate);
assertEquals(ResultState.AMENDED.getLabel(), response.getResultState().getLabel());
assertEquals(1, response.getValue().getObject().size());
assertEquals("{dwc:eventDate=1932-11-23}", response.getValue().getObject().toString());
logger.debug(response.getComment());
eventDate = "";
verbatimEventDate = "2/3 1932";
response = DwCEventDQ.amendmentEventdateFromVerbatim(eventDate, verbatimEventDate);
assertEquals(ResultState.NOT_AMENDED.getLabel(), response.getResultState().getLabel());
assertEquals(0, response.getValue().getObject().size());
logger.debug(response.getComment());

}

/**
Expand Down
28 changes: 17 additions & 11 deletions src/test/java/org/filteredpush/qc/date/DwcEventDQTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -380,47 +380,53 @@ public void testDayMonthTransposition() {
}

@Test
public void testExtractDateFromVerbatim() {
public void testAmendmentEventdateFromVerbatim() {

// INTERNAL_PREREQUISITES_NOT_MET if dwc:eventDate is not EMPTY
// or the value of dwc:verbatimEventDate is EMPTY or not unambiguously
// interpretable as an ISO 8601-1:2019 date; AMENDED if the
// value of dwc:eventDate was unambiguously interpreted from
// dwc:verbatimEventDate; otherwise NOT_AMENDED

String eventDate = "";
String verbatimEventDate = "Jan 1884";
DQResponse<AmendmentValue> result = DwCEventDQ.extractDateFromVerbatim(eventDate,verbatimEventDate);
DQResponse<AmendmentValue> result = DwCEventDQ.amendmentEventdateFromVerbatim(eventDate,verbatimEventDate);
assertEquals(ResultState.AMENDED, result.getResultState());
assertEquals("1884-01",result.getValue().getObject().get("dwc:eventDate"));
assertEquals(1,result.getValue().getObject().size());

verbatimEventDate = "1 Mar 1884";
result = DwCEventDQ.extractDateFromVerbatim(eventDate,verbatimEventDate);
result = DwCEventDQ.amendmentEventdateFromVerbatim(eventDate,verbatimEventDate);
assertEquals(ResultState.AMENDED, result.getResultState());
assertEquals("1884-03-01",result.getValue().getObject().get("dwc:eventDate"));
assertEquals(1,result.getValue().getObject().size());

eventDate = "1884";
verbatimEventDate = "1 Mar 1884";
result = DwCEventDQ.extractDateFromVerbatim(eventDate,verbatimEventDate);
result = DwCEventDQ.amendmentEventdateFromVerbatim(eventDate,verbatimEventDate);
assertEquals(ResultState.NOT_AMENDED, result.getResultState());
assertEquals(0,result.getValue().getObject().size());
eventDate = "1884-03-01";
verbatimEventDate = "1 Mar 1884";
result = DwCEventDQ.extractDateFromVerbatim(eventDate,verbatimEventDate);
result = DwCEventDQ.amendmentEventdateFromVerbatim(eventDate,verbatimEventDate);
assertEquals(ResultState.NOT_AMENDED, result.getResultState());
assertEquals(0,result.getValue().getObject().size());

eventDate = null;
verbatimEventDate = "";
result = DwCEventDQ.extractDateFromVerbatim(eventDate,verbatimEventDate);
result = DwCEventDQ.amendmentEventdateFromVerbatim(eventDate,verbatimEventDate);
assertEquals(ResultState.INTERNAL_PREREQUISITES_NOT_MET, result.getResultState());
assertEquals(0,result.getValue().getObject().size());

eventDate = null;
verbatimEventDate = "5-8-1884";
result = DwCEventDQ.extractDateFromVerbatim(eventDate,verbatimEventDate);
assertEquals(ResultState.AMBIGUOUS, result.getResultState());
assertEquals("1884-05-08/1884-08-05",result.getValue().getObject().get("dwc:eventDate"));
assertEquals(1,result.getValue().getObject().size());
result = DwCEventDQ.amendmentEventdateFromVerbatim(eventDate,verbatimEventDate);
assertEquals(ResultState.NOT_AMENDED, result.getResultState());
assertNull(result.getValue());

eventDate = null;
verbatimEventDate = "2001/Feb/29";
result = DwCEventDQ.extractDateFromVerbatim(eventDate,verbatimEventDate);
result = DwCEventDQ.amendmentEventdateFromVerbatim(eventDate,verbatimEventDate);
assertEquals(ResultState.INTERNAL_PREREQUISITES_NOT_MET, result.getResultState());
assertEquals(0,result.getValue().getObject().size());

Expand Down

0 comments on commit 29a4551

Please sign in to comment.