Skip to content

Commit

Permalink
Issue #1732 - Support token modifier ':text'
Browse files Browse the repository at this point in the history 
Signed-off-by: Troy Biesterfeld <tbieste@us.ibm.com>
  • Loading branch information
@tbieste
tbieste committed Apr 5, 2021
1 parent 4db3565 commit 9394776
Show file tree
Hide file tree
Showing 11 changed files with 144 additions and 28 deletions.
6 changes: 3 additions & 3 deletions docs/src/pages/Conformance.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
layout: post
title: Conformance
description: Notes on the Conformance of the IBM FHIR Server
date: 2021-03-30 12:00:00 -0400
date: 2021-04-05 12:00:00 -0400
permalink: /conformance/
---

Expand Down Expand Up @@ -163,7 +163,7 @@ FHIR search modifiers are described at https://www.hl7.org/fhir/R4/search.html#m
|String |`:exact`,`:contains`,`:missing` |"starts with" search that is case-insensitive and accent-insensitive|
|Reference |`:[type]`,`:missing`,`:identifier` |exact match search and targets are implicitly added|
|URI |`:below`,`:above`,`:missing` |exact match search|
|Token |`:missing`,`:not`,`:of-type`,`:in`,`:not-in` |exact match search|
|Token |`:missing`,`:not`,`:of-type`,`:in`,`:not-in`,`:text`|exact match search|
|Number |`:missing` |implicit range search (see http://hl7.org/fhir/R4/search.html#number)|
|Date |`:missing` |implicit range search (see https://www.hl7.org/fhir/search.html#date)|
|Quantity |`:missing` |implicit range search (see http://hl7.org/fhir/R4/search.html#quantity)|
Expand All @@ -172,7 +172,7 @@ FHIR search modifiers are described at https://www.hl7.org/fhir/R4/search.html#m

Due to performance implications, the `:exact` modifier should be used for String search parameters when possible.

The `:text`, `:above`, and `:below` modifiers for Token search parameters are not yet supported by the IBM FHIR server. Use of these modifiers will result in an HTTP 400 error with an OperationOutcome that describes the failure.
The `:above`, and `:below` modifiers for Token search parameters are not yet supported by the IBM FHIR server. Use of these modifiers will result in an HTTP 400 error with an OperationOutcome that describes the failure.

The `:in` and `:not-in` modifiers for Token search parameters are supported, with the following restrictions:
* The search parameter value must be an absolute URI that identifies a value set.
Expand Down
3 changes: 2 additions & 1 deletion fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/JDBCConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ public class JDBCConstants {
supportedModifiersMap.put(Type.STRING, Arrays.asList(Modifier.EXACT, Modifier.CONTAINS, Modifier.MISSING));
supportedModifiersMap.put(Type.REFERENCE, Arrays.asList(Modifier.TYPE, Modifier.MISSING, Modifier.IDENTIFIER));
supportedModifiersMap.put(Type.URI, Arrays.asList(Modifier.BELOW, Modifier.ABOVE, Modifier.MISSING));
supportedModifiersMap.put(Type.TOKEN, Arrays.asList(Modifier.MISSING, Modifier.NOT, Modifier.OF_TYPE, Modifier.IN, Modifier.NOT_IN));
supportedModifiersMap.put(Type.TOKEN, Arrays.asList(Modifier.MISSING, Modifier.NOT, Modifier.OF_TYPE, Modifier.IN, Modifier.NOT_IN, Modifier.TEXT));
supportedModifiersMap.put(Type.NUMBER, Arrays.asList(Modifier.MISSING));
supportedModifiersMap.put(Type.DATE, Arrays.asList(Modifier.MISSING));
supportedModifiersMap.put(Type.QUANTITY, Arrays.asList(Modifier.MISSING));
Expand All @@ -150,6 +150,7 @@ public class JDBCConstants {
modifierOperatorMap.put(Modifier.CONTAINS, LIKE);
modifierOperatorMap.put(Modifier.EXACT, EQ);
modifierOperatorMap.put(Modifier.NOT, EQ); // EQ since it will be within a "WHERE NOT EXISTS" subquery
modifierOperatorMap.put(Modifier.TEXT, LIKE);
}

private JDBCConstants() {
Expand Down
5 changes: 3 additions & 2 deletions ...e-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import com.ibm.fhir.schema.control.FhirSchemaConstants;
import com.ibm.fhir.search.util.ReferenceValue;
import com.ibm.fhir.search.util.ReferenceValue.ReferenceType;
import com.ibm.fhir.search.util.SearchUtil;

/**
* Batch insert into the parameter values tables. Avoids having to create one stored procedure
Expand Down Expand Up @@ -210,7 +211,7 @@ public void visit(StringParmVal param) throws FHIRPersistenceException {
systemStrings.setInt(1, parameterNameId);
if (value != null) {
systemStrings.setString(2, value);
systemStrings.setString(3, value.toLowerCase());
systemStrings.setString(3, SearchUtil.normalizeForSearch(value));
}
else {
systemStrings.setNull(2, Types.VARCHAR);
Expand Down Expand Up @@ -246,7 +247,7 @@ private void setStringParms(PreparedStatement insert, int parameterNameId, Strin
insert.setInt(1, parameterNameId);
if (value != null) {
insert.setString(2, value);
insert.setString(3, value.toLowerCase());
insert.setString(3, SearchUtil.normalizeForSearch(value));
} else {
insert.setNull(2, Types.VARCHAR);
insert.setNull(3, Types.VARCHAR);
Expand Down
16 changes: 16 additions & 0 deletions ...e-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCParameterBuildingVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -401,6 +401,14 @@ public boolean visit(java.lang.String elementName, int elementIndex, CodeableCon
for (Coding coding : codeableConcept.getCoding()) {
visit(elementName, elementIndex, coding);
}
if (codeableConcept.getText() != null && codeableConcept.getText().hasValue()) {
// Extract as token as normalized string since :text modifier is simple string search
TokenParmVal p = new TokenParmVal();
p.setResourceType(resourceType);
p.setName(searchParamCode + SearchConstants.TEXT_MODIFIER_SUFFIX);
p.setValueCode(SearchUtil.normalizeForSearch(codeableConcept.getText().getValue()));
result.add(p);
}
return false;
}

Expand All @@ -418,6 +426,14 @@ public boolean visit(java.lang.String elementName, int elementIndex, Coding codi
p.setValueSystem(coding.getSystem().getValue());
}
result.add(p);
if (coding.getDisplay() != null && coding.getDisplay().hasValue()) {
// Extract as token as normalized string since :text modifier is simple string search
p = new TokenParmVal();
p.setResourceType(resourceType);
p.setName(searchParamCode + SearchConstants.TEXT_MODIFIER_SUFFIX);
p.setValueCode(SearchUtil.normalizeForSearch(coding.getDisplay().getValue()));
result.add(p);
}
}
return false;
}
Expand Down
40 changes: 31 additions & 9 deletions fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCQueryBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1194,12 +1194,18 @@ private SqlQueryData processTokenParm(Class<?> resourceType, QueryParameter quer
StringBuilder whereClauseSegment = new StringBuilder();
String operator = this.getOperator(queryParm, EQ);
boolean parmValueProcessed = false;
boolean appendEscape;
SqlQueryData queryData;
List<Object> bindVariables = new ArrayList<>();
String tableAlias = paramTableAlias;
String queryParmCode = queryParm.getCode();

String code = queryParm.getCode();
if (!QuerySegmentAggregator.ID.equals(code)) {
if (!QuerySegmentAggregator.ID.equals(queryParmCode)) {

// Append the suffix for :text modifier
if (Modifier.TEXT.equals(queryParm.getModifier())) {
queryParmCode += SearchConstants.TEXT_MODIFIER_SUFFIX;
}

// Only generate NOT EXISTS subquery if :not modifier is within chained query;
// when :not modifier is within non-chained query QuerySegmentAggregator.buildWhereClause generates the NOT EXISTS subquery
Expand All @@ -1215,23 +1221,34 @@ private SqlQueryData processTokenParm(Class<?> resourceType, QueryParameter quer

// Build this piece of the segment:
// (P1.PARAMETER_NAME_ID = x AND
this.populateNameIdSubSegment(whereClauseSegment, queryParm.getCode(), tableAlias);
this.populateNameIdSubSegment(whereClauseSegment, queryParmCode, tableAlias);

whereClauseSegment.append(AND).append(LEFT_PAREN);
for (QueryParameterValue value : queryParm.getValues()) {
appendEscape = false;

// If multiple values are present, we need to OR them together.
if (parmValueProcessed) {
whereClauseSegment.append(OR);
}

whereClauseSegment.append(LEFT_PAREN);

if (Modifier.IN.equals(queryParm.getModifier()) || Modifier.NOT_IN.equals(queryParm.getModifier())) {
populateValueSetCodesSubSegment(whereClauseSegment, value.getValueCode(), tableAlias);
} else {
// Include code
whereClauseSegment.append(tableAlias + DOT).append(TOKEN_VALUE).append(operator).append(BIND_VAR);
bindVariables.add(SqlParameterEncoder.encode(value.getValueCode()));
if (LIKE.equals(operator)) {
// Must escape special wildcard characters _ and % in the parameter value string.
String textSearchString = SqlParameterEncoder.encode(value.getValueCode())
.replace(PERCENT_WILDCARD, ESCAPE_PERCENT)
.replace(UNDERSCORE_WILDCARD, ESCAPE_UNDERSCORE) + PERCENT_WILDCARD;
bindVariables.add(SearchUtil.normalizeForSearch(textSearchString));
appendEscape = true;
} else {
bindVariables.add(SqlParameterEncoder.encode(value.getValueCode()));
}

// Include system if present.
if (value.getValueSystem() != null && !value.getValueSystem().isEmpty()) {
Expand Down Expand Up @@ -1260,7 +1277,12 @@ private SqlQueryData processTokenParm(Class<?> resourceType, QueryParameter quer
}
}
}


// Build this piece: ESCAPE '+'
if (appendEscape) {
whereClauseSegment.append(ESCAPE_EXPR);
}

whereClauseSegment.append(RIGHT_PAREN);
parmValueProcessed = true;
}
Expand Down Expand Up @@ -1980,9 +2002,9 @@ private void populateValueSetCodesSubSegment(StringBuilder whereClauseSegment, S
if (codeSystemProcessed) {
whereClauseSegment.append(OR);
}

// TODO: investigate if we can use COMMON_TOKEN_VALUES support

// <parameterTableAlias>.TOKEN_VALUE IN (...)
whereClauseSegment.append(tokenValuePredicateString)
.append("'").append(String.join("','", codes)).append("'")
Expand All @@ -1991,7 +2013,7 @@ private void populateValueSetCodesSubSegment(StringBuilder whereClauseSegment, S
// AND <parameterTableAlias>.CODE_SYSTEM_ID = {n}
whereClauseSegment.append(AND).append(codeSystemIdPredicateString)
.append(nullCheck(identityCache.getCodeSystemId(codeSetUrl)));

codeSystemProcessed = true;
}
}
Expand Down
34 changes: 27 additions & 7 deletions ...e-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterExtractionTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@
*/
public class ParameterExtractionTest {
private static final String SAMPLE_STRING = "test";
private static final String SAMPLE_NON_NORMALIZED_TEXT_STRING = "Text String";
private static final String SAMPLE_NORMALIZED_TEXT_STRING = "text string";
private static final String SAMPLE_URI = "http://example.com";
private static final String SAMPLE_UNIT = "s";
private static final String SAMPLE_REF_RESOURCE_TYPE = "Patient";
Expand Down Expand Up @@ -370,19 +372,32 @@ public void testAge_null() throws FHIRPersistenceProcessorException {
public void testCodeableConcept() throws FHIRPersistenceProcessorException {
JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(SAMPLE_REF_RESOURCE_TYPE, tokenSearchParam);
CodeableConcept.builder()
.coding(Coding.builder().code(Code.of("a")).system(Uri.of(SAMPLE_URI)).build())
.coding(Coding.builder().code(Code.of("a")).system(Uri.of(SAMPLE_URI)).display(string(SAMPLE_NON_NORMALIZED_TEXT_STRING + "a")).build())
.coding(Coding.builder().code(Code.of("b")).system(Uri.of(SAMPLE_URI)).build())
.coding(Coding.builder().code(Code.of("c")).system(Uri.of(SAMPLE_URI)).build())
.coding(Coding.builder().code(Code.of("c")).system(Uri.of(SAMPLE_URI)).display(string(SAMPLE_NON_NORMALIZED_TEXT_STRING + "c")).build())
.text(string(SAMPLE_NON_NORMALIZED_TEXT_STRING))
.build()
.accept(parameterBuilder);
List<ExtractedParameterValue> params = parameterBuilder.getResult();
assertEquals(params.size(), 3, "Number of extracted parameters");
assertEquals(params.size(), 6, "Number of extracted parameters");
assertEquals(((TokenParmVal) params.get(0)).getName(), SEARCH_PARAM_CODE_VALUE);
assertEquals(((TokenParmVal) params.get(0)).getValueCode(), "a");
assertEquals(((TokenParmVal) params.get(0)).getValueSystem(), SAMPLE_URI);
assertEquals(((TokenParmVal) params.get(1)).getValueCode(), "b");
assertEquals(((TokenParmVal) params.get(1)).getValueSystem(), SAMPLE_URI);
assertEquals(((TokenParmVal) params.get(2)).getValueCode(), "c");
assertEquals(((TokenParmVal) params.get(1)).getName(), SEARCH_PARAM_CODE_VALUE + SearchConstants.TEXT_MODIFIER_SUFFIX);
assertEquals(((TokenParmVal) params.get(1)).getValueCode(), SAMPLE_NORMALIZED_TEXT_STRING + "a");
assertEquals(((TokenParmVal) params.get(1)).getValueSystem(), JDBCConstants.DEFAULT_TOKEN_SYSTEM);
assertEquals(((TokenParmVal) params.get(2)).getName(), SEARCH_PARAM_CODE_VALUE);
assertEquals(((TokenParmVal) params.get(2)).getValueCode(), "b");
assertEquals(((TokenParmVal) params.get(2)).getValueSystem(), SAMPLE_URI);
assertEquals(((TokenParmVal) params.get(3)).getName(), SEARCH_PARAM_CODE_VALUE);
assertEquals(((TokenParmVal) params.get(3)).getValueCode(), "c");
assertEquals(((TokenParmVal) params.get(3)).getValueSystem(), SAMPLE_URI);
assertEquals(((TokenParmVal) params.get(4)).getName(), SEARCH_PARAM_CODE_VALUE + SearchConstants.TEXT_MODIFIER_SUFFIX);
assertEquals(((TokenParmVal) params.get(4)).getValueCode(), SAMPLE_NORMALIZED_TEXT_STRING + "c");
assertEquals(((TokenParmVal) params.get(4)).getValueSystem(), JDBCConstants.DEFAULT_TOKEN_SYSTEM);
assertEquals(((TokenParmVal) params.get(5)).getName(), SEARCH_PARAM_CODE_VALUE + SearchConstants.TEXT_MODIFIER_SUFFIX);
assertEquals(((TokenParmVal) params.get(5)).getValueCode(), SAMPLE_NORMALIZED_TEXT_STRING);
assertEquals(((TokenParmVal) params.get(5)).getValueSystem(), JDBCConstants.DEFAULT_TOKEN_SYSTEM);
}

@Test
Expand All @@ -396,12 +411,17 @@ public void testCoding() throws FHIRPersistenceProcessorException {
Coding.builder()
.code(Code.of(SAMPLE_STRING))
.system(Uri.of(SAMPLE_URI))
.display(string(SAMPLE_NON_NORMALIZED_TEXT_STRING))
.build()
.accept(parameterBuilder);
List<ExtractedParameterValue> params = parameterBuilder.getResult();
assertEquals(params.size(), 1, "Number of extracted parameters");
assertEquals(params.size(), 2, "Number of extracted parameters");
assertEquals(((TokenParmVal) params.get(0)).getName(), SEARCH_PARAM_CODE_VALUE);
assertEquals(((TokenParmVal) params.get(0)).getValueCode(), SAMPLE_STRING);
assertEquals(((TokenParmVal) params.get(0)).getValueSystem(), SAMPLE_URI);
assertEquals(((TokenParmVal) params.get(1)).getName(), SEARCH_PARAM_CODE_VALUE + SearchConstants.TEXT_MODIFIER_SUFFIX);
assertEquals(((TokenParmVal) params.get(1)).getValueCode(), SAMPLE_NORMALIZED_TEXT_STRING);
assertEquals(((TokenParmVal) params.get(1)).getValueSystem(), JDBCConstants.DEFAULT_TOKEN_SYSTEM);
}

@Test
Expand Down
3 changes: 3 additions & 0 deletions fhir-search/src/main/java/com/ibm/fhir/search/SearchConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,9 @@ private SearchConstants() {
public static final String OF_TYPE_MODIFIER_COMPONENT_TYPE = "type";
public static final String OF_TYPE_MODIFIER_COMPONENT_VALUE = "value";

// Extracted search parameter suffix for :text modifier
public static final String TEXT_MODIFIER_SUFFIX = ":text";

// set as unmodifiable
public static final Set<String> SEARCH_RESULT_PARAMETER_NAMES =
Collections.unmodifiableSet(new HashSet<>(Arrays.asList(SORT, COUNT, PAGE, INCLUDE, REVINCLUDE, ELEMENTS, SUMMARY, TOTAL)));
Expand Down
13 changes: 8 additions & 5 deletions fhir-search/src/main/java/com/ibm/fhir/search/util/SearchUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1283,6 +1283,7 @@ private static List<QueryParameterValue> parseQueryParameterValuesString(SearchP
// token
// [parameter]=[system]|[code]
// [parameter]:of-type=[system|code|value]
// [parameter]:text=code
/*
* TODO: start enforcing this:
* "For token parameters on elements of type ContactPoint, uri, or boolean,
Expand Down Expand Up @@ -1335,6 +1336,8 @@ private static List<QueryParameterValue> parseQueryParameterValuesString(SearchP
throw SearchExceptionUtil.buildNewInvalidSearchException(msg);
}
parameterValue.setValueCode(unescapeSearchParm(v));
} else if (Modifier.TEXT.equals(modifier)) {
parameterValue.setValueCode(unescapeSearchParm(v));
} else if (parts.length == 2) {
parameterValue.setValueSystem(unescapeSearchParm(parts[0]));
parameterValue.setValueCode(unescapeSearchParm(parts[1]));
Expand Down Expand Up @@ -2102,18 +2105,18 @@ public static QueryParameter parseChainedInclusionCriteria(QueryParameter inclus

/**
* Normalizes a string to be used as a search parameter value. All accents and
* diacritics are removed. And then the
* string is transformed to lower case.
* diacritics are removed. Consecutive whitespace characters are replaced with
* a single space. And then the string is transformed to lower case.
*
* @param value
* @return
* @param value the string to normalize
* @return the normalized string
*/
public static String normalizeForSearch(String value) {

String normalizedValue = null;
if (value != null) {
normalizedValue = Normalizer.normalize(value, Form.NFD).replaceAll("\\p{InCombiningDiacriticalMarks}+", "");
normalizedValue = normalizedValue.toLowerCase();
normalizedValue = normalizedValue.replaceAll("\\s+", " ").toLowerCase();
}

return normalizedValue;
Expand Down
Loading

0 comments on commit 9394776

Please sign in to comment.