diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/ReindexResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/ReindexResourceDAO.java index 0fdcd950b4b..eb125822003 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/ReindexResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/ReindexResourceDAO.java @@ -51,7 +51,7 @@ public class ReindexResourceDAO extends ResourceDAOImpl { // the is_deleted flag. Until it does, the queries will return deleted // resources, which can be skipped for reindex. (issue-2055) private static final String PICK_SINGLE_RESOURCE = "" - + " SELECT lr.logical_resource_id, lr.resource_type_id, lr.logical_id, lr.reindex_txid " + + " SELECT lr.logical_resource_id, lr.resource_type_id, lr.logical_id, lr.reindex_txid, lr.parameter_hash " + " FROM logical_resources lr " + " WHERE lr.resource_type_id = ? " + " AND lr.logical_id = ? " @@ -59,7 +59,7 @@ public class ReindexResourceDAO extends ResourceDAOImpl { ; private static final String PICK_SINGLE_RESOURCE_TYPE = "" - + " SELECT lr.logical_resource_id, lr.resource_type_id, lr.logical_id, lr.reindex_txid " + + " SELECT lr.logical_resource_id, lr.resource_type_id, lr.logical_id, lr.reindex_txid, lr.parameter_hash " + " FROM logical_resources lr " + " WHERE lr.resource_type_id = ? " + " AND lr.reindex_tstamp < ? " @@ -67,7 +67,7 @@ public class ReindexResourceDAO extends ResourceDAOImpl { ; private static final String PICK_ANY_RESOURCE = "" - + " SELECT lr.logical_resource_id, lr.resource_type_id, lr.logical_id, lr.reindex_txid " + + " SELECT lr.logical_resource_id, lr.resource_type_id, lr.logical_id, lr.reindex_txid, lr.parameter_hash " + " FROM logical_resources lr " + " WHERE lr.reindex_tstamp < ? " + "OFFSET ? ROWS FETCH FIRST 1 ROWS ONLY " @@ -111,6 +111,7 @@ public ReindexResourceDAO(Connection connection, IDatabaseTranslator translator, * Getter for the translator currently held by this DAO * @return */ + @Override protected IDatabaseTranslator getTranslator() { return this.translator; } @@ -169,7 +170,7 @@ protected ResourceIndexRecord getNextResource(SecureRandom random, Instant reind } ResultSet rs = stmt.executeQuery(); if (rs.next()) { - result = new ResourceIndexRecord(rs.getLong(1), rs.getInt(2), rs.getString(3), rs.getLong(4)); + result = new ResourceIndexRecord(rs.getLong(1), rs.getInt(2), rs.getString(3), rs.getLong(4), rs.getString(5)); } } catch (SQLException x) { logger.log(Level.SEVERE, select, x); @@ -265,12 +266,13 @@ public ResourceIndexRecord getResourceToReindex(Instant reindexTstamp, Integer r /** * Reindex the resource by deleting existing parameters and replacing them with those passed in. * @param tablePrefix the table prefix - * @param parameters the extracted parameters + * @param parameters A collection of search parameters to be persisted along with the passed Resource + * @param parameterHashB64 Base64 encoded SHA-256 hash of parameters * @param logicalId the logical id * @param logicalResourceId the logical resource id * @throws Exception */ - public void updateParameters(String tablePrefix, List parameters, String logicalId, long logicalResourceId) throws Exception { + public void updateParameters(String tablePrefix, List parameters, String parameterHashB64, String logicalId, long logicalResourceId) throws Exception { final String METHODNAME = "updateParameters() for " + tablePrefix + "/" + logicalId; logger.entering(CLASSNAME, METHODNAME); @@ -295,7 +297,6 @@ public void updateParameters(String tablePrefix, List p deleteFromParameterTable(connection, "str_values", logicalResourceId); deleteFromParameterTable(connection, "date_values", logicalResourceId); - if (parameters != null && !parameters.isEmpty()) { JDBCIdentityCache identityCache = new JDBCIdentityCacheImpl(getCache(), this, parameterDao, getResourceReferenceDAO()); // Check if this is multitenant @@ -310,6 +311,10 @@ identityCache, getResourceReferenceDAO(), getTransactionData())) { throw translator.translate(x); } } + + // Update the "parameters hash" in the LOGICAL_RESOURCES table + updateParametersHash(connection, logicalResourceId, parameterHashB64); + logger.exiting(CLASSNAME, METHODNAME); } @@ -335,4 +340,27 @@ protected void deleteFromParameterTable(Connection conn, String tableName, long throw translator.translate(x); } } + + /** + * Updates the parameters hash in the LOGICAL_RESOURCES table. + * @param conn the connection + * @param logicalResourceId the logical resource ID + * @param hash the parameters hash + * @throws SQLException + */ + protected void updateParametersHash(Connection conn, long logicalResourceId, String parameterHashB64) throws SQLException { + final String SQL = "UPDATE logical_resources SET parameter_hash = ? WHERE logical_resource_id = ?"; + try (PreparedStatement stmt = conn.prepareStatement(SQL)) { + // bind parameters + stmt.setString(1, parameterHashB64); + stmt.setLong(2, logicalResourceId); + stmt.executeUpdate(); + if (logger.isLoggable(Level.FINEST)) { + logger.finest("Update parameter_hash [" + parameterHashB64 + "] for logicalResourceId [" + logicalResourceId + "]"); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, SQL, x); + throw translator.translate(x); + } + } } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceIndexRecord.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceIndexRecord.java index 5b54d7dc95a..0f61707e38d 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceIndexRecord.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceIndexRecord.java @@ -30,11 +30,15 @@ public class ResourceIndexRecord { // Deletion flag for the resource. Set when we read the resource private boolean deleted; - public ResourceIndexRecord(long logicalResourceId, int resourceTypeId, String logicalId, long transactionId) { + // Base64-encoded SHA-256 hash of the search parameters + private String parameterHash; + + public ResourceIndexRecord(long logicalResourceId, int resourceTypeId, String logicalId, long transactionId, String parameterHash) { this.logicalResourceId = logicalResourceId; this.resourceTypeId = resourceTypeId; this.logicalId = logicalId; this.transactionId = transactionId; + this.parameterHash = parameterHash; } /** @@ -92,4 +96,20 @@ public boolean isDeleted() { public void setDeleted(boolean deleted) { this.deleted = deleted; } + + /** + * Gets the Base64-encoded SHA-256 hash of the search parameters. + * @return the Base64-encoded SHA-256 hash + */ + public String getParameterHash() { + return parameterHash; + } + + /** + * Gets the Base64-encoded SHA-256 hash of the search parameters. + * @param parameterHash the Base64-encoded SHA-256 hash + */ + public void setParameterHash(String parameterHash) { + this.parameterHash = parameterHash; + } } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/CompositeParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/CompositeParmVal.java index 90c4522977a..21557b9bf47 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/CompositeParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/CompositeParmVal.java @@ -10,6 +10,7 @@ import java.util.List; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; /** * This class defines the Data Transfer Object representing a row in the X_DATE_VALUES tables. @@ -30,10 +31,16 @@ public CompositeParmVal() { /** * We know our type, so we can call the correct method on the visitor */ + @Override public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { visitor.visit(this); } + @Override + public String getHash(ParameterHashUtil parameterHashUtil) { + return parameterHashUtil.getNameValueHash(getHashHeader(), parameterHashUtil.getParametersHash(component)); + } + /** * @return get the list of components in this composite parameter */ diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/DateParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/DateParmVal.java index 227993960db..d014c37c9b7 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/DateParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/DateParmVal.java @@ -7,8 +7,10 @@ package com.ibm.fhir.persistence.jdbc.dto; import java.sql.Timestamp; +import java.util.Objects; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; /** * This class defines the Data Transfer Object representing a row in the X_DATE_VALUES tables. @@ -18,9 +20,6 @@ public class DateParmVal extends ExtractedParameterValue { private Timestamp valueDateStart; private Timestamp valueDateEnd; - // The SearchParameter base type. If "Resource", then this is a Resource-level attribute - private String base; - public enum TimeType { YEAR, YEAR_MONTH, @@ -53,13 +52,22 @@ public void setValueDateEnd(Timestamp valueDateEnd) { /** * We know our type, so we can call the correct method on the visitor */ + @Override public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { visitor.visit(this); } + @Override + public String getHash(ParameterHashUtil parameterHashUtil) { + StringBuilder sb = new StringBuilder(); + sb.append(Objects.toString(valueDateStart, "")); + sb.append("|").append(Objects.toString(valueDateEnd, "")); + return parameterHashUtil.getNameValueHash(getHashHeader(), sb.toString()); + } + @Override public String toString() { return "DateParmVal [resourceType=" + getResourceType() + ", name=" + getName() - + ", valueDateStart=" + valueDateStart + ", valueDateEnd=" + valueDateEnd + ", base=" + base + "]"; + + ", valueDateStart=" + valueDateStart + ", valueDateEnd=" + valueDateEnd + "]"; } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ExtractedParameterValue.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ExtractedParameterValue.java index 597d44dad43..7316ac4a861 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ExtractedParameterValue.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ExtractedParameterValue.java @@ -6,7 +6,11 @@ package com.ibm.fhir.persistence.jdbc.dto; +import java.util.Objects; + import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; +import com.ibm.fhir.schema.control.FhirSchemaVersion; /** * A search parameter value extracted from a resource and ready to store / index for search @@ -25,6 +29,10 @@ public abstract class ExtractedParameterValue { // The bass resource name private String base; + // URL and version of search parameter + private String url; + private String version; + /** * Protected constructor */ @@ -93,4 +101,53 @@ public String getName() { public void setName(String name) { this.name = name; } + + /** + * @return the url + */ + public String getUrl() { + return url; + } + + /** + * @param url the url to set + */ + public void setUrl(String url) { + this.url = url; + } + + /** + * @return the version + */ + public String getVersion() { + return version; + } + + /** + * @param version the version to set + */ + public void setVersion(String version) { + this.version = version; + } + + /** + * Gets the hash header. + * @return the hash header + */ + protected String getHashHeader() { + StringBuilder sb = new StringBuilder(name); + sb.append(Objects.toString(FhirSchemaVersion.getLatestParameterStorageUpdate(), "")).append("|"); + sb.append(Objects.toString(name, "")).append("|"); + sb.append(Objects.toString(url, "")).append("|"); + sb.append(Objects.toString(version, "")); + return sb.toString(); + } + + /** + * Gets the hash representation of the parameter. + * This should be generated from the search parameter (schemaVersion, code, url, version) and the extracted value. + * @param the parameter hash utility to use for generating hashes + * @return the hash + */ + public abstract String getHash(ParameterHashUtil parameterHashUtil); } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/LocationParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/LocationParmVal.java index da67655aedc..2c2bed459fa 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/LocationParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/LocationParmVal.java @@ -6,7 +6,10 @@ package com.ibm.fhir.persistence.jdbc.dto; +import java.util.Objects; + import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; /** * This class defines the Data Transfer Object representing a row in the X_LATLNG_VALUES tables. @@ -42,7 +45,16 @@ public void setValueLatitude(Double valueLatitude) { /** * We know our type, so we can call the correct method on the visitor */ + @Override public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { visitor.visit(this); } + + @Override + public String getHash(ParameterHashUtil parameterHashUtil) { + StringBuilder sb = new StringBuilder(); + sb.append(Objects.toString(valueLongitude, "")); + sb.append("|").append(Objects.toString(valueLatitude, "")); + return parameterHashUtil.getNameValueHash(getHashHeader(), sb.toString()); + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/NumberParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/NumberParmVal.java index 4897bb48f3e..58293eeb798 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/NumberParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/NumberParmVal.java @@ -7,8 +7,10 @@ package com.ibm.fhir.persistence.jdbc.dto; import java.math.BigDecimal; +import java.util.Objects; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; /** * This class defines the Data Transfer Object representing a row in the X_NUMBER_VALUES tables. @@ -19,15 +21,13 @@ public class NumberParmVal extends ExtractedParameterValue { private BigDecimal valueNumberLow; private BigDecimal valueNumberHigh; - // The SearchParameter base type. If "Resource", then this is a Resource-level attribute - private String base; - /** * Public constructor */ public NumberParmVal() { super(); } + public BigDecimal getValueNumber() { return valueNumber; } @@ -55,7 +55,17 @@ public void setValueNumberHigh(BigDecimal valueNumberHigh) { /** * We know our type, so we can call the correct method on the visitor */ + @Override public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { visitor.visit(this); } + + @Override + public String getHash(ParameterHashUtil parameterHashUtil) { + StringBuilder sb = new StringBuilder(); + sb.append(Objects.toString(valueNumber, "")); + sb.append("|").append(Objects.toString(valueNumberLow, "")); + sb.append("|").append(Objects.toString(valueNumberHigh, "")); + return parameterHashUtil.getNameValueHash(getHashHeader(), sb.toString()); + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/QuantityParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/QuantityParmVal.java index 8098760e231..092bf0d9e31 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/QuantityParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/QuantityParmVal.java @@ -7,9 +7,11 @@ package com.ibm.fhir.persistence.jdbc.dto; import java.math.BigDecimal; +import java.util.Objects; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.JDBCConstants; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; /** * This class defines the Data Transfer Object representing a row in the X_QUANTITY_VALUES tables. @@ -83,4 +85,15 @@ public void setValueNumberHigh(BigDecimal valueNumberHigh) { public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { visitor.visit(this); } + + @Override + public String getHash(ParameterHashUtil parameterHashUtil) { + StringBuilder sb = new StringBuilder(); + sb.append(Objects.toString(valueNumber, "")); + sb.append("|").append(Objects.toString(valueNumberLow, "")); + sb.append("|").append(Objects.toString(valueNumberHigh, "")); + sb.append("|").append(getValueSystem()); + sb.append("|").append(getValueCode()); + return parameterHashUtil.getNameValueHash(getHashHeader(), sb.toString()); + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ReferenceParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ReferenceParmVal.java index 41a2d7a3e70..97754b18e57 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ReferenceParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ReferenceParmVal.java @@ -6,7 +6,10 @@ package com.ibm.fhir.persistence.jdbc.dto; +import java.util.Objects; + import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; import com.ibm.fhir.search.SearchConstants.Type; import com.ibm.fhir.search.util.ReferenceValue; @@ -48,7 +51,17 @@ public Type getType() { /** * We know our type, so we can call the correct method on the visitor */ + @Override public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { visitor.visit(this); } + + @Override + public String getHash(ParameterHashUtil parameterHashUtil) { + StringBuilder sb = new StringBuilder(Objects.toString(refValue.getTargetResourceType(), "")); + sb.append("|").append(Objects.toString(refValue.getValue(), "")); + sb.append("|").append(Objects.toString(refValue.getType(), "")); + sb.append("|").append(Objects.toString(refValue.getVersion(), "")); + return parameterHashUtil.getNameValueHash(getHashHeader(), sb.toString()); + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/StringParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/StringParmVal.java index 71ea95a89e7..576b9010a30 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/StringParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/StringParmVal.java @@ -6,7 +6,10 @@ package com.ibm.fhir.persistence.jdbc.dto; +import java.util.Objects; + import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; /** * This class defines the Data Transfer Object representing a row in the X_STR_VALUES tables. @@ -16,9 +19,6 @@ public class StringParmVal extends ExtractedParameterValue { // The string value of this extracted parameter private String valueString; - // The SearchParameter base type. If "Resource", then this is a Resource-level attribute - private String base; - /** * Public constructor */ @@ -37,21 +37,13 @@ public void setValueString(String valueString) { /** * We know our type, so we can call the correct method on the visitor */ + @Override public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { visitor.visit(this); } - /** - * @return the base - */ - public String getBase() { - return base; - } - - /** - * @param base the base to set - */ - public void setBase(String base) { - this.base = base; + @Override + public String getHash(ParameterHashUtil parameterHashUtil) { + return parameterHashUtil.getNameValueHash(getHashHeader(), Objects.toString(valueString, "")); } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/TokenParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/TokenParmVal.java index 6090f67aaa7..0cc2baf35af 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/TokenParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/TokenParmVal.java @@ -6,8 +6,11 @@ package com.ibm.fhir.persistence.jdbc.dto; +import java.util.Objects; + import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.JDBCConstants; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; /** * This class defines the Data Transfer Object representing a row in the X_TOKEN_VALUES tables. @@ -52,7 +55,15 @@ public void setValueCode(String valueCode) { /** * We know our type, so we can call the correct method on the visitor */ + @Override public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { visitor.visit(this); } + + @Override + public String getHash(ParameterHashUtil parameterHashUtil) { + StringBuilder sb = new StringBuilder(getValueSystem()); + sb.append("|").append(Objects.toString(valueCode, "")); + return parameterHashUtil.getNameValueHash(getHashHeader(), sb.toString()); + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/UriParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/UriParmVal.java index adda8f18f3f..cce65a83ce2 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/UriParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/UriParmVal.java @@ -1,12 +1,15 @@ /* - * (C) Copyright IBM Corp. 2017,2019 + * (C) Copyright IBM Corp. 2017, 2021 * * SPDX-License-Identifier: Apache-2.0 */ package com.ibm.fhir.persistence.jdbc.dto; +import java.util.Objects; + import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; /** * Note used @@ -34,7 +37,13 @@ public void setValueString(String valueString) { /** * We know our type, so we can call the correct method on the visitor */ + @Override public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { // visitor.visit(this); } + + @Override + public String getHash(ParameterHashUtil parameterHashUtil) { + return parameterHashUtil.getNameValueHash(getHashHeader(), Objects.toString(valueString, "")); + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java index 0f668b4ad53..5aae00caf97 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java @@ -141,9 +141,11 @@ import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceFKVException; import com.ibm.fhir.persistence.jdbc.util.CodeSystemsCache; +import com.ibm.fhir.persistence.jdbc.util.ExtractedSearchParameters; import com.ibm.fhir.persistence.jdbc.util.JDBCParameterBuildingVisitor; import com.ibm.fhir.persistence.jdbc.util.JDBCQueryBuilder; import com.ibm.fhir.persistence.jdbc.util.NewQueryBuilder; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; import com.ibm.fhir.persistence.jdbc.util.ParameterNamesCache; import com.ibm.fhir.persistence.jdbc.util.ResourceTypesCache; import com.ibm.fhir.persistence.jdbc.util.SqlQueryData; @@ -215,6 +217,8 @@ public class FHIRPersistenceJDBCImpl implements FHIRPersistence, SchemaNameSuppl // Use the optimized query builder when supported for the search request private final boolean optQueryBuilderEnabled; + private final ParameterHashUtil parameterHashUtil; + /** * Constructor for use when running as web application in WLP. * @throws Exception @@ -265,6 +269,8 @@ public FHIRPersistenceJDBCImpl(FHIRPersistenceJDBCCache cache) throws Exception this.transactionAdapter = new FHIRUserTransactionAdapter(userTransaction, trxSynchRegistry, cache, TXN_DATA_KEY); + this.parameterHashUtil = new ParameterHashUtil(); + log.exiting(CLASSNAME, METHODNAME); } @@ -326,6 +332,9 @@ public FHIRPersistenceJDBCImpl(Properties configProps, IConnectionProvider cp, F // Always want to be testing with the new query builder this.optQueryBuilderEnabled = true; + // Utility for generating hash of search parameters + this.parameterHashUtil = new ParameterHashUtil(); + log.exiting(CLASSNAME, METHODNAME); } @@ -410,9 +419,9 @@ public SingleResourceResult create(FHIRPersistenceContex ParameterDAO parameterDao = makeParameterDAO(connection); // Persist the Resource DTO. - final String parameterHashB64 = ""; resourceDao.setPersistenceContext(context); - resourceDao.insert(resourceDTO, this.extractSearchParameters(updatedResource, resourceDTO), parameterHashB64, parameterDao); + ExtractedSearchParameters searchParameters = this.extractSearchParameters(updatedResource, resourceDTO); + resourceDao.insert(resourceDTO, searchParameters.getParameters(), searchParameters.getHash(), parameterDao); if (log.isLoggable(Level.FINE)) { log.fine("Persisted FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' id=" + resourceDTO.getId() + ", version=" + resourceDTO.getVersionId()); @@ -597,9 +606,9 @@ public SingleResourceResult update(FHIRPersistenceContex zipStream.close(); // Persist the Resource DTO. - final String parameterHashB64 = ""; resourceDao.setPersistenceContext(context); - resourceDao.insert(resourceDTO, this.extractSearchParameters(updatedResource, resourceDTO), parameterHashB64, parameterDao); + ExtractedSearchParameters searchParameters = this.extractSearchParameters(updatedResource, resourceDTO); + resourceDao.insert(resourceDTO, searchParameters.getParameters(), searchParameters.getHash(), parameterDao); if (log.isLoggable(Level.FINE)) { log.fine("Persisted FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' id=" + resourceDTO.getId() + ", version=" + resourceDTO.getVersionId()); @@ -1420,9 +1429,8 @@ public SingleResourceResult delete(FHIRPersistenceContex resourceDTO.setDeleted(true); // Persist the logically deleted Resource DTO. - final String parameterHashB64 = ""; resourceDao.setPersistenceContext(context); - resourceDao.insert(resourceDTO, null, parameterHashB64, null); + resourceDao.insert(resourceDTO, null, null, null); if (log.isLoggable(Level.FINE)) { log.fine("Persisted FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' id=" + resourceDTO.getId() @@ -1881,136 +1889,199 @@ private TransactionSynchronizationRegistry getTrxSynchRegistry() throws FHIRPers * @return list of extracted search parameters * @throws Exception */ - private List extractSearchParameters(Resource fhirResource, com.ibm.fhir.persistence.jdbc.dto.Resource resourceDTOx) + private ExtractedSearchParameters extractSearchParameters(Resource fhirResource, com.ibm.fhir.persistence.jdbc.dto.Resource resourceDTOx) throws Exception { final String METHODNAME = "extractSearchParameters"; log.entering(CLASSNAME, METHODNAME); Map> map; String code; + String url; + String version; String type; String expression; - List allParameters = new ArrayList<>(); + ExtractedSearchParameters extractedParameters = new ExtractedSearchParameters(); try { - if (fhirResource == null) { - return allParameters; - } - map = SearchUtil.extractParameterValues(fhirResource); + if (fhirResource != null) { - for (Entry> entry : map.entrySet()) { - SearchParameter sp = entry.getKey(); - code = sp.getCode().getValue(); - final boolean wholeSystemParam = isWholeSystem(sp); + map = SearchUtil.extractParameterValues(fhirResource); - // As not to inject any other special handling logic, this is a simple inline check to see if - // _id or _lastUpdated are used, and ignore those extracted values. - if (SPECIAL_HANDLING.contains(code)) { - continue; - } - type = sp.getType().getValue(); - expression = sp.getExpression().getValue(); + for (Entry> entry : map.entrySet()) { + SearchParameter sp = entry.getKey(); + code = sp.getCode().getValue(); + url = sp.getUrl().getValue(); + version = sp.getVersion() != null ? sp.getVersion().getValue(): null; + final boolean wholeSystemParam = isWholeSystem(sp); - if (log.isLoggable(Level.FINE)) { - log.fine("Processing SearchParameter resource: " + fhirResource.getClass().getSimpleName() + ", code: " + code + ", type: " + type + ", expression: " + expression); - } + // As not to inject any other special handling logic, this is a simple inline check to see if + // _id or _lastUpdated are used, and ignore those extracted values. + if (SPECIAL_HANDLING.contains(code)) { + continue; + } + type = sp.getType().getValue(); + expression = sp.getExpression().getValue(); - List values = entry.getValue(); - - if (SearchParamType.COMPOSITE.equals(sp.getType())) { - List components = sp.getComponent(); - FHIRPathEvaluator evaluator = FHIRPathEvaluator.evaluator(); - - for (FHIRPathNode value : values) { - Visitable fhirNode; - EvaluationContext context; - if (value.isResourceNode()) { - fhirNode = value.asResourceNode().resource(); - context = new EvaluationContext((Resource) fhirNode); - } else if (value.isElementNode()) { - fhirNode = value.asElementNode().element(); - context = new EvaluationContext((Element) fhirNode); - } else { - throw new IllegalStateException("Composite parameter expression must select one or more FHIR elements"); - } + if (log.isLoggable(Level.FINE)) { + log.fine("Processing SearchParameter resource: " + fhirResource.getClass().getSimpleName() + + ", code: " + code + ", url: " + url + ", version: " + version + ", type: " + type + ", expression: " + expression); + } - CompositeParmVal p = new CompositeParmVal(); - p.setName(code); - p.setResourceType(fhirResource.getClass().getSimpleName()); - - for (int i = 0; i < components.size(); i++) { - Component component = components.get(i); - Collection nodes = evaluator.evaluate(context, component.getExpression().getValue()); - if (nodes.isEmpty()){ - if (log.isLoggable(Level.FINER)) { - log.finer("Component expression '" + component.getExpression().getValue() + "' resulted in 0 nodes; " - + "skipping composite parameter '" + code + "'."); - } - continue; + List values = entry.getValue(); + + if (SearchParamType.COMPOSITE.equals(sp.getType())) { + List components = sp.getComponent(); + FHIRPathEvaluator evaluator = FHIRPathEvaluator.evaluator(); + + for (FHIRPathNode value : values) { + Visitable fhirNode; + EvaluationContext context; + if (value.isResourceNode()) { + fhirNode = value.asResourceNode().resource(); + context = new EvaluationContext((Resource) fhirNode); + } else if (value.isElementNode()) { + fhirNode = value.asElementNode().element(); + context = new EvaluationContext((Element) fhirNode); + } else { + throw new IllegalStateException("Composite parameter expression must select one or more FHIR elements"); } - // Alternative: consider pulling the search parameter from the FHIRRegistry instead so we can use versioned references. - // Of course, that would require adding extension-search-params to the Registry which would require the Registry to be tenant-aware. -// SearchParameter compSP = FHIRRegistry.getInstance().getResource(component.getDefinition().getValue(), SearchParameter.class); - SearchParameter compSP = SearchUtil.getSearchParameter(p.getResourceType(), component.getDefinition()); - JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(p.getResourceType(), compSP); - FHIRPathNode node = nodes.iterator().next(); - if (nodes.size() > 1 ) { - // TODO: support component expressions that result in multiple nodes - // On the current schema, this means creating a different CompositeParmValue for each ordered set of component values. - // For example, if a composite has two components and each one's expression results in two nodes - // ([Code1,Code2] and [Quantity1,Quantity2]) and each node results in a single ExtractedParameterValue, - // then we need to generate CompositeParmVal objects for [Code1,Quantity1], [Code1,Quantity2], - // [Code2,Quantity1], and [Code2,Quantity2]. - // Assumption: this should be rare. - if (log.isLoggable(Level.FINE)) { - log.fine("Component expression '" + component.getExpression().getValue() + "' resulted in multiple nodes; " - + "proceeding with randomly chosen node '" + node.path() + "' for search parameter '" + code + "'."); + CompositeParmVal p = new CompositeParmVal(); + p.setName(code); + p.setUrl(url); + p.setVersion(version); + p.setResourceType(fhirResource.getClass().getSimpleName()); + + for (int i = 0; i < components.size(); i++) { + Component component = components.get(i); + Collection nodes = evaluator.evaluate(context, component.getExpression().getValue()); + if (nodes.isEmpty()){ + if (log.isLoggable(Level.FINER)) { + log.finer("Component expression '" + component.getExpression().getValue() + "' resulted in 0 nodes; " + + "skipping composite parameter '" + code + "'."); + } + continue; } - } - try { - if (node.isElementNode()) { - // parameterBuilder aggregates the results for later retrieval - node.asElementNode().element().accept(parameterBuilder); - // retrieve the list of parameters built from all the FHIRPathElementNode values - List parameters = parameterBuilder.getResult(); - if (parameters.isEmpty()){ - if (log.isLoggable(Level.FINE)) { - log.fine("Selected element '" + node.path() + "' resulted in 0 extracted parameter values; " - + "skipping composite parameter '" + code + "'."); - } - continue; + // Alternative: consider pulling the search parameter from the FHIRRegistry instead so we can use versioned references. + // Of course, that would require adding extension-search-params to the Registry which would require the Registry to be tenant-aware. + // SearchParameter compSP = FHIRRegistry.getInstance().getResource(component.getDefinition().getValue(), SearchParameter.class); + SearchParameter compSP = SearchUtil.getSearchParameter(p.getResourceType(), component.getDefinition()); + JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(p.getResourceType(), compSP); + FHIRPathNode node = nodes.iterator().next(); + if (nodes.size() > 1 ) { + // TODO: support component expressions that result in multiple nodes + // On the current schema, this means creating a different CompositeParmValue for each ordered set of component values. + // For example, if a composite has two components and each one's expression results in two nodes + // ([Code1,Code2] and [Quantity1,Quantity2]) and each node results in a single ExtractedParameterValue, + // then we need to generate CompositeParmVal objects for [Code1,Quantity1], [Code1,Quantity2], + // [Code2,Quantity1], and [Code2,Quantity2]. + // Assumption: this should be rare. + if (log.isLoggable(Level.FINE)) { + log.fine("Component expression '" + component.getExpression().getValue() + "' resulted in multiple nodes; " + + "proceeding with randomly chosen node '" + node.path() + "' for search parameter '" + code + "'."); } + } + + try { + if (node.isElementNode()) { + // parameterBuilder aggregates the results for later retrieval + node.asElementNode().element().accept(parameterBuilder); + // retrieve the list of parameters built from all the FHIRPathElementNode values + List parameters = parameterBuilder.getResult(); + if (parameters.isEmpty()){ + if (log.isLoggable(Level.FINE)) { + log.fine("Selected element '" + node.path() + "' resulted in 0 extracted parameter values; " + + "skipping composite parameter '" + code + "'."); + } + continue; + } + + if (parameters.size() > 1) { + // TODO: support component searchParms that lead to multiple ExtractedParameterValues + // On the current schema, this means creating a different CompositeParmValue for each ordered set of component values. + // For example: + // If a composite has two components and each results in two extracted parameters ([A,B] and [1,2] respectively) + // then we need to generate CompositeParmVal objects for [A,1], [A,2], [B,1], and [B,2] + // Assumption: this should only be common for Quantity search parameters with both a coded unit and a display unit, + // and in these cases, the coded unit is almost always the preferred value for search. + if (log.isLoggable(Level.FINE)) { + log.fine("Selected element '" + node.path() + "' resulted in multiple extracted parameter values; " + + "proceeding with the first extracted value for composite parameter '" + code + "'."); + } + } + ExtractedParameterValue componentParam = parameters.get(0); + // override the component parameter name with the composite parameter name + componentParam.setName(SearchUtil.makeCompositeSubCode(code, componentParam.getName())); + componentParam.setUrl(url); + componentParam.setVersion(version); + // componentParam.setBase(p.getBase()); TODO not needed? + p.addComponent(componentParam); + } else if (node.isSystemValue()){ + ExtractedParameterValue primitiveParam = processPrimitiveValue(node.asSystemValue()); + primitiveParam.setName(code); + primitiveParam.setUrl(url); + primitiveParam.setVersion(version); + primitiveParam.setResourceType(fhirResource.getClass().getSimpleName()); - if (parameters.size() > 1) { - // TODO: support component searchParms that lead to multiple ExtractedParameterValues - // On the current schema, this means creating a different CompositeParmValue for each ordered set of component values. - // For example: - // If a composite has two components and each results in two extracted parameters ([A,B] and [1,2] respectively) - // then we need to generate CompositeParmVal objects for [A,1], [A,2], [B,1], and [B,2] - // Assumption: this should only be common for Quantity search parameters with both a coded unit and a display unit, - // and in these cases, the coded unit is almost always the preferred value for search. if (log.isLoggable(Level.FINE)) { - log.fine("Selected element '" + node.path() + "' resulted in multiple extracted parameter values; " - + "proceeding with the first extracted value for composite parameter '" + code + "'."); + log.fine("Extracted Parameter '" + p.getName() + "' from Resource."); } + p.addComponent(primitiveParam); + } else { + // log and continue + String msg = "Unable to extract value from '" + value.path() + + "'; search parameter value extraction can only be performed on Elements and primitive values."; + if (log.isLoggable(Level.FINE)) { + log.fine(msg); + } + addWarning(IssueType.INVALID, msg); + continue; } - ExtractedParameterValue componentParam = parameters.get(0); - // override the component parameter name with the composite parameter name - componentParam.setName(SearchUtil.makeCompositeSubCode(code, componentParam.getName())); - // componentParam.setBase(p.getBase()); TODO not needed? - p.addComponent(componentParam); - } else if (node.isSystemValue()){ - ExtractedParameterValue primitiveParam = processPrimitiveValue(node.asSystemValue()); - primitiveParam.setName(code); - primitiveParam.setResourceType(fhirResource.getClass().getSimpleName()); + } catch (IllegalArgumentException e) { + // log and continue with the other parameters + StringBuilder msg = new StringBuilder("Skipped search parameter '" + code + "'"); + if (sp.getId() != null) { + msg.append(" with id '" + sp.getId() + "'"); + } + msg.append(" for resource type " + fhirResource.getClass().getSimpleName()); + // just use the message...no need for the whole stack trace + msg.append(" due to \n" + e.getMessage()); + if (log.isLoggable(Level.FINE)) { + log.fine(msg.toString()); + } + addWarning(IssueType.INVALID, msg.toString()); + } + } + if (components.size() == p.getComponent().size()) { + // only add the parameter if all of the components are present and accounted for + extractedParameters.getParameters().add(p); + } + } + } else { // ! SearchParamType.COMPOSITE.equals(sp.getType()) + JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(fhirResource.getClass().getSimpleName(), sp); + + for (FHIRPathNode value : values) { + try { + if (value.isElementNode()) { + // parameterBuilder aggregates the results for later retrieval + value.asElementNode().element().accept(parameterBuilder); + } else if (value.isSystemValue()){ + ExtractedParameterValue p = processPrimitiveValue(value.asSystemValue()); + p.setName(code); + p.setUrl(url); + p.setVersion(version); + p.setResourceType(fhirResource.getClass().getSimpleName()); + + if (wholeSystemParam) { + p.setWholeSystem(true); + } + extractedParameters.getParameters().add(p); if (log.isLoggable(Level.FINE)) { log.fine("Extracted Parameter '" + p.getName() + "' from Resource."); } - p.addComponent(primitiveParam); } else { // log and continue String msg = "Unable to extract value from '" + value.path() + @@ -2023,7 +2094,7 @@ private List extractSearchParameters(Resource fhirResou } } catch (IllegalArgumentException e) { // log and continue with the other parameters - StringBuilder msg = new StringBuilder("Skipped search parameter '" + code + "'"); + StringBuilder msg = new StringBuilder("Skipping search parameter '" + code + "'"); if (sp.getId() != null) { msg.append(" with id '" + sp.getId() + "'"); } @@ -2036,79 +2107,35 @@ private List extractSearchParameters(Resource fhirResou addWarning(IssueType.INVALID, msg.toString()); } } - if (components.size() == p.getComponent().size()) { - // only add the parameter if all of the components are present and accounted for - allParameters.add(p); - } - } - } else { // ! SearchParamType.COMPOSITE.equals(sp.getType()) - JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(fhirResource.getClass().getSimpleName(), sp); - - for (FHIRPathNode value : values) { - - try { - if (value.isElementNode()) { - // parameterBuilder aggregates the results for later retrieval - value.asElementNode().element().accept(parameterBuilder); - } else if (value.isSystemValue()){ - ExtractedParameterValue p = processPrimitiveValue(value.asSystemValue()); - p.setName(code); - p.setResourceType(fhirResource.getClass().getSimpleName()); - - if (wholeSystemParam) { - p.setWholeSystem(true); - } - allParameters.add(p); - if (log.isLoggable(Level.FINE)) { - log.fine("Extracted Parameter '" + p.getName() + "' from Resource."); - } - } else { - // log and continue - String msg = "Unable to extract value from '" + value.path() + - "'; search parameter value extraction can only be performed on Elements and primitive values."; - if (log.isLoggable(Level.FINE)) { - log.fine(msg); - } - addWarning(IssueType.INVALID, msg); - continue; - } - } catch (IllegalArgumentException e) { - // log and continue with the other parameters - StringBuilder msg = new StringBuilder("Skipping search parameter '" + code + "'"); - if (sp.getId() != null) { - msg.append(" with id '" + sp.getId() + "'"); + // retrieve the list of parameters built from all the FHIRPathElementNode values + List parameters = parameterBuilder.getResult(); + for (ExtractedParameterValue p : parameters) { + if (wholeSystemParam) { + p.setWholeSystem(true); } - msg.append(" for resource type " + fhirResource.getClass().getSimpleName()); - // just use the message...no need for the whole stack trace - msg.append(" due to \n" + e.getMessage()); + extractedParameters.getParameters().add(p); if (log.isLoggable(Level.FINE)) { - log.fine(msg.toString()); + log.fine("Extracted Parameter '" + p.getName() + "' from Resource."); } - addWarning(IssueType.INVALID, msg.toString()); - } - } - // retrieve the list of parameters built from all the FHIRPathElementNode values - List parameters = parameterBuilder.getResult(); - for (ExtractedParameterValue p : parameters) { - if (wholeSystemParam) { - p.setWholeSystem(true); - } - allParameters.add(p); - if (log.isLoggable(Level.FINE)) { - log.fine("Extracted Parameter '" + p.getName() + "' from Resource."); } } } + + // Augment the extracted parameter list with special values we use to represent compartment relationships. + // These references are stored as tokens and are used by the search query builder + // for compartment-based searches + addCompartmentParams(extractedParameters.getParameters(), fhirResource); } - // Augment the extracted parameter list with special values we use to represent compartment relationships. - // These references are stored as tokens and are used by the search query builder - // for compartment-based searches - addCompartmentParams(allParameters, fhirResource); + // Generate the hash which is used to quickly determine whether the extracted parameters + // are different than the extracted parameters that currently exist in the database + extractedParameters.generateHash(parameterHashUtil); + } finally { log.exiting(CLASSNAME, METHODNAME); } - return allParameters; + + return extractedParameters; } /** @@ -2534,7 +2561,7 @@ public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder oper } else { // Skip this particular resource because it has been deleted if (log.isLoggable(Level.FINE)) { - log.info("Skipping reindex for deleted FHIR Resource '" + rir.getResourceType() + "/" + rir.getLogicalId() + "'"); + log.fine("Skipping reindex for deleted FHIR Resource '" + rir.getResourceType() + "/" + rir.getLogicalId() + "'"); } rir.setDeleted(true); } @@ -2576,11 +2603,11 @@ public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder oper /** * Update the parameters for the resource described by the given DTO * @param - * @param rir - * @param resourceTypeClass - * @param existingResourceDTO - * @param reindexDAO - * @param operationOutcomeResult + * @param rir the resource index record + * @param resourceTypeClass the resource type class + * @param existingResourceDTO the existing resource DTO + * @param reindexDAO the reindex resource DAO + * @param operationOutcomeResult the operation outcome result * @throws Exception */ public void updateParameters(ResourceIndexRecord rir, Class resourceTypeClass, com.ibm.fhir.persistence.jdbc.dto.Resource existingResourceDTO, @@ -2588,9 +2615,17 @@ public void updateParameters(ResourceIndexRecord rir, Class if (existingResourceDTO != null && !existingResourceDTO.isDeleted()) { T existingResource = this.convertResourceDTO(existingResourceDTO, resourceTypeClass, null); - // Extract parameters from the resource payload we just read and store them, replacing - // the existing set - reindexDAO.updateParameters(rir.getResourceType(), this.extractSearchParameters(existingResource, existingResourceDTO), rir.getLogicalId(), rir.getLogicalResourceId()); + // Extract parameters from the resource payload. + ExtractedSearchParameters searchParameters = this.extractSearchParameters(existingResource, existingResourceDTO); + + // Compare the hash of the extracted parameters with the hash in the index record. + // If hash in the index record is not null and it matches the hash of the extracted parameters, then no need to replace the + // extracted search parameters in the database tables for this resource, which helps with performance during reindex. + if (rir.getParameterHash() == null || !rir.getParameterHash().equals(searchParameters.getHash())) { + reindexDAO.updateParameters(rir.getResourceType(), searchParameters.getParameters(), searchParameters.getHash(), rir.getLogicalId(), rir.getLogicalResourceId()); + } else { + log.fine(() -> "Skipping update of unchanged parameters for FHIR Resource '" + rir.getResourceType() + "/" + rir.getLogicalId() + "'"); + } // Use an OperationOutcome Issue to let the caller know that some work was performed final String diag = "Processed " + rir.getResourceType() + "/" + rir.getLogicalId(); @@ -2600,7 +2635,6 @@ public void updateParameters(ResourceIndexRecord rir, Class final String diag = "Failed to read resource: " + rir.getResourceType() + "/" + rir.getLogicalId(); operationOutcomeResult.issue(Issue.builder().code(IssueType.NOT_FOUND).severity(IssueSeverity.WARNING).diagnostics(string(diag)).build()); } - } @Override diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresReindexResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresReindexResourceDAO.java index 2a6bcaa962a..42908f1afa5 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresReindexResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresReindexResourceDAO.java @@ -48,7 +48,7 @@ public class PostgresReindexResourceDAO extends ReindexResourceDAO { + " AND lr.reindex_tstamp < ? " + " ORDER BY lr.reindex_tstamp " + " FOR UPDATE SKIP LOCKED LIMIT 1) " - + "RETURNING logical_resource_id, resource_type_id, logical_id, reindex_txid " + + "RETURNING logical_resource_id, resource_type_id, logical_id, reindex_txid, parameter_hash " ; private static final String PICK_SINGLE_RESOURCE_TYPE = "" @@ -62,7 +62,7 @@ public class PostgresReindexResourceDAO extends ReindexResourceDAO { + " AND lr.reindex_tstamp < ? " + " ORDER BY lr.reindex_tstamp " + " FOR UPDATE SKIP LOCKED LIMIT 1) " - + "RETURNING logical_resource_id, resource_type_id, logical_id, reindex_txid " + + "RETURNING logical_resource_id, resource_type_id, logical_id, reindex_txid, parameter_hash " ; private static final String PICK_ANY_RESOURCE = "" @@ -75,7 +75,7 @@ public class PostgresReindexResourceDAO extends ReindexResourceDAO { + " WHERE lr.reindex_tstamp < ? " + " ORDER BY lr.reindex_tstamp " + " FOR UPDATE SKIP LOCKED LIMIT 1) " - + "RETURNING logical_resource_id, resource_type_id, logical_id, reindex_txid " + + "RETURNING logical_resource_id, resource_type_id, logical_id, reindex_txid, parameter_hash " ; /** @@ -157,7 +157,7 @@ public ResourceIndexRecord getNextResource(SecureRandom random, Instant reindexT stmt.execute(); ResultSet rs = stmt.getResultSet(); if (rs.next()) { - result = new ResourceIndexRecord(rs.getLong(1), rs.getInt(2), rs.getString(3), rs.getLong(4)); + result = new ResourceIndexRecord(rs.getLong(1), rs.getInt(2), rs.getString(3), rs.getLong(4), rs.getString(5)); } } catch (SQLException x) { logger.log(Level.SEVERE, update, x); diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ExtractedSearchParameters.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ExtractedSearchParameters.java new file mode 100644 index 00000000000..a372cf83f76 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ExtractedSearchParameters.java @@ -0,0 +1,46 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.util; + + +import java.util.ArrayList; +import java.util.List; + +import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValue; + +/** + * Contains a list of extracted search parameters and a Base64-encoded SHA-256 hash. + */ +public class ExtractedSearchParameters { + + private List parameters = new ArrayList<>(); + private String hashB64 = null; + + /** + * Gets the parameters. + * @return the parameters + */ + public List getParameters() { + return parameters; + } + + /** + * Generates the Base64-encoded SHA-256 hash of the parameters. + * @param the parameter hash utility to use for generating the hash + */ + public void generateHash(ParameterHashUtil parameterHashUtil) { + hashB64 = parameterHashUtil.getParametersHash(parameters); + } + + /** + * Gets the already-generated Base64-encoded SHA-256 hash of the parameters. + * @return the Base64 encoded SHA-256 hash + */ + public String getHash() { + return hashB64; + } +} diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCParameterBuildingVisitor.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCParameterBuildingVisitor.java index 9e4f1658f32..6a1eecf0397 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCParameterBuildingVisitor.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCParameterBuildingVisitor.java @@ -94,9 +94,11 @@ public class JDBCParameterBuildingVisitor extends DefaultVisitor { ZonedDateTime.parse("9999-12-31T23:59:59.999999Z").toInstant()); private final String resourceType; - // We only need the SearchParameter type and code, so just store those directly as members + // We only need the SearchParameter code, type, url, and version, so just store those directly as members private final String searchParamCode; private final SearchParamType searchParamType; + private final String searchParamUrl; + private final String searchParamVersion; /** * The result of the visit(s) @@ -105,15 +107,17 @@ public class JDBCParameterBuildingVisitor extends DefaultVisitor { /** * Public constructor - * @param resourceType - * @param searchParameter + * @param resourceType the resource type + * @param searchParameter the search parameter */ public JDBCParameterBuildingVisitor(String resourceType, SearchParameter searchParameter) { super(false); this.resourceType = resourceType; this.searchParamCode = searchParameter.getCode().getValue(); this.searchParamType = searchParameter.getType(); - this.result = new ArrayList<>(); + this.searchParamUrl = searchParameter.getUrl().getValue(); + this.searchParamVersion = searchParameter.getVersion() != null ? searchParameter.getVersion().getValue(): null; + this.result = new ArrayList<>(); } /** @@ -146,6 +150,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, com.ibm.fhi TokenParmVal p = new TokenParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueSystem("http://terminology.hl7.org/CodeSystem/special-values"); if (_boolean.getValue()) { p.setValueCode("true"); @@ -166,6 +172,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, com.ibm.fhi StringParmVal p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(canonical.getValue()); result.add(p); } @@ -181,6 +189,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, com.ibm.fhi TokenParmVal p = new TokenParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); String system = ModelSupport.getSystem(code); setTokenValues(p, system != null ? Uri.of(system) : null, code.getValue()); result.add(p); @@ -197,6 +207,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, com.ibm.fhi DateParmVal p = new DateParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); setDateValues(p, date); result.add(p); } @@ -212,6 +224,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, com.ibm.fhi DateParmVal p = new DateParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); setDateValues(p, dateTime); result.add(p); } @@ -227,6 +241,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, com.ibm.fhi NumberParmVal p = new NumberParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); BigDecimal value = decimal.getValue(); p.setValueNumber(value); p.setValueNumberLow(NumberParmBehaviorUtil.generateLowerBound(value)); @@ -245,6 +261,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, com.ibm.fhi TokenParmVal p = new TokenParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueCode(id.getValue()); result.add(p); } @@ -260,6 +278,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, com.ibm.fhi DateParmVal p = new DateParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); Timestamp t = generateTimestamp(instant.getValue().toInstant()); p.setValueDateStart(t); p.setValueDateEnd(t); @@ -277,6 +297,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, com.ibm.fhi NumberParmVal p = new NumberParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); BigDecimal value = new BigDecimal(integer.getValue()); p.setValueNumber(value); p.setValueNumberLow(value); @@ -294,12 +316,16 @@ public boolean visit(String elementName, int elementIndex, com.ibm.fhir.model.ty p.setResourceType(resourceType); p.setValueString(value.getValue()); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); result.add(p); } else if (TOKEN.equals(searchParamType)) { TokenParmVal p = new TokenParmVal(); p.setResourceType(resourceType); p.setValueCode(SearchUtil.normalizeForSearch(value.getValue())); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); result.add(p); } else { throw invalidComboException(searchParamType, value); @@ -321,12 +347,16 @@ public boolean visit(String elementName, int elementIndex, Uri uri) { TokenParmVal p = new TokenParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueCode(uri.getValue()); result.add(p); } else { StringParmVal p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(uri.getValue()); result.add(p); } @@ -350,6 +380,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Address add p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(aLine.getValue()); result.add(p); } @@ -357,6 +389,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Address add p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(address.getCity().getValue()); result.add(p); } @@ -364,6 +398,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Address add p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(address.getDistrict().getValue()); result.add(p); } @@ -371,6 +407,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Address add p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(address.getState().getValue()); result.add(p); } @@ -378,6 +416,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Address add p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(address.getCountry().getValue()); result.add(p); } @@ -385,6 +425,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Address add p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(address.getPostalCode().getValue()); result.add(p); } @@ -392,6 +434,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Address add p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(address.getText().getValue()); result.add(p); } @@ -411,6 +455,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, CodeableCon TokenParmVal p = new TokenParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode + SearchConstants.TEXT_MODIFIER_SUFFIX); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueCode(SearchUtil.normalizeForSearch(codeableConcept.getText().getValue())); result.add(p); } @@ -426,6 +472,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Coding codi TokenParmVal p = new TokenParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); setTokenValues(p, coding.getSystem(), coding.getCode().getValue()); result.add(p); if (coding.getDisplay() != null && coding.getDisplay().hasValue()) { @@ -433,6 +481,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Coding codi p = new TokenParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode + SearchConstants.TEXT_MODIFIER_SUFFIX); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueCode(SearchUtil.normalizeForSearch(coding.getDisplay().getValue())); result.add(p); } @@ -449,6 +499,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, ContactPoin TokenParmVal telecom = new TokenParmVal(); telecom.setResourceType(resourceType); telecom.setName(searchParamCode); + telecom.setUrl(searchParamUrl); + telecom.setVersion(searchParamVersion); telecom.setValueCode(contactPoint.getValue().getValue()); result.add(telecom); } @@ -466,6 +518,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, HumanName h p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(humanName.getFamily().getValue()); result.add(p); } @@ -473,6 +527,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, HumanName h p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(given.getValue()); result.add(p); } @@ -480,6 +536,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, HumanName h p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(prefix.getValue()); result.add(p); } @@ -487,6 +545,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, HumanName h p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(suffix.getValue()); result.add(p); } @@ -494,6 +554,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, HumanName h p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(humanName.getText().getValue()); result.add(p); } @@ -509,6 +571,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Money money QuantityParmVal p = new QuantityParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueNumber(money.getValue().getValue()); if (money.getCurrency() != null) { p.setValueCode(money.getCurrency().getValue()); @@ -530,6 +594,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Period peri DateParmVal p = new DateParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); if (period.getStart() == null || period.getStart().getValue() == null) { p.setValueDateStart(SMALLEST_TIMESTAMP); } else { @@ -564,6 +630,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Quantity qu QuantityParmVal p = new QuantityParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueNumber(value); p.setValueNumberLow(valueLow); p.setValueNumberHigh(valueHigh); @@ -581,6 +649,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Quantity qu QuantityParmVal p = new QuantityParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueNumber(value); p.setValueNumberLow(valueLow); p.setValueNumberHigh(valueHigh); @@ -594,6 +664,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Quantity qu QuantityParmVal p = new QuantityParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueNumber(value); p.setValueNumberLow(valueLow); p.setValueNumberHigh(valueHigh); @@ -612,6 +684,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Range range QuantityParmVal p = new QuantityParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); if (range.getLow() != null && range.getLow().getValue() != null && range.getLow().getValue().getValue() != null) { if (range.getLow().getSystem() != null) { @@ -656,6 +730,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Identifier TokenParmVal p = new TokenParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); setTokenValues(p, identifier.getSystem(), identifier.getValue().getValue()); result.add(p); if (identifier.getType() != null) { @@ -665,11 +741,15 @@ public boolean visit(java.lang.String elementName, int elementIndex, Identifier CompositeParmVal cp = new CompositeParmVal(); cp.setResourceType(resourceType); cp.setName(searchParamCode + SearchConstants.OF_TYPE_MODIFIER_SUFFIX); + cp.setUrl(searchParamUrl); + cp.setVersion(searchParamVersion); // type p = new TokenParmVal(); p.setResourceType(cp.getResourceType()); p.setName(SearchUtil.makeCompositeSubCode(cp.getName(), SearchConstants.OF_TYPE_MODIFIER_COMPONENT_TYPE)); + p.setUrl(cp.getUrl()); + p.setVersion(cp.getVersion()); setTokenValues(p, typeCoding.getSystem(), typeCoding.getCode().getValue()); cp.addComponent(p); @@ -677,6 +757,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Identifier p = new TokenParmVal(); p.setResourceType(cp.getResourceType()); p.setName(SearchUtil.makeCompositeSubCode(cp.getName(), SearchConstants.OF_TYPE_MODIFIER_COMPONENT_VALUE)); + p.setUrl(cp.getUrl()); + p.setVersion(cp.getVersion()); p.setValueCode(identifier.getValue().getValue()); cp.addComponent(p); @@ -705,6 +787,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Reference r p.setResourceType(resourceType); p.setRefValue(refValue); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); result.add(p); } Identifier identifier = reference.getIdentifier(); @@ -712,6 +796,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Reference r TokenParmVal p = new TokenParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode + SearchConstants.IDENTIFIER_MODIFIER_SUFFIX); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); setTokenValues(p, identifier.getSystem(), identifier.getValue().getValue()); result.add(p); } @@ -752,6 +838,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Location.Po LocationParmVal p = new LocationParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); // The following code ensures that the lat/lon is only added when there is a pair. boolean add = false; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ParameterHashUtil.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ParameterHashUtil.java new file mode 100644 index 00000000000..1f1b77ff26c --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ParameterHashUtil.java @@ -0,0 +1,87 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.util; + + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Base64.Encoder; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; + +import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValue; + +/** + * Utility methods for generating Base64-encoded SHA-256 hash for search parameters. + */ +public class ParameterHashUtil { + private static final Charset UTF_8 = StandardCharsets.UTF_8; + private static final String SHA_256 = "SHA-256"; + private final Encoder encoder; + private final MessageDigest digest; + + public ParameterHashUtil() { + encoder = Base64.getEncoder(); + try { + digest = MessageDigest.getInstance(SHA_256); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("MessageDigest not found: " + SHA_256, e); + } + } + + /** + * Gets a Base64-encoded SHA-256 hash for a list of ExtractedParameterValues. + * This is used to quickly determine if the search parameters changed + * from the existing values in the database, which can then be used to + * avoid deleting and re-inserting the search parameters into the database. + * @param parameters extracted search parameters + * @return the hash + */ + public String getParametersHash(List parameterValues) { + // Sort hashes to make it deterministic + List sortedList = new ArrayList<>(parameterValues.size()); + for (ExtractedParameterValue parameterValue : parameterValues) { + sortedList.add(Objects.toString(parameterValue.getHash(this), "")); + } + sortedList.sort(Comparator.comparing(String::toString)); + + StringBuilder sb = new StringBuilder("|"); + for (String hash : sortedList) { + sb.append(hash).append("|"); + } + + byte[] hashBytes = digest.digest(sb.toString().getBytes(UTF_8)); + return bytesToB64(hashBytes); + } + + /** + * Gets a Base64-encoded SHA-256 hash for a name-value pair. + * @param name the name + * @param value the value + * @return the hash + */ + public String getNameValueHash(String name, String value) { + StringBuilder sb = new StringBuilder("["); + sb.append(Objects.toString(name, "")).append("]=[").append(Objects.toString(value, "")).append("]"); + byte[] hashBytes = digest.digest(sb.toString().getBytes(UTF_8)); + return bytesToB64(hashBytes); + } + + /** + * Convert bytes to Base64-encoded string. + * @param bytes the bytes + * @return Base64-encoded string + */ + private String bytesToB64(byte[] bytes) { + return new String(encoder.encode(bytes)); + } +} diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterHashUtilTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterHashUtilTest.java new file mode 100644 index 00000000000..4ff988059c5 --- /dev/null +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterHashUtilTest.java @@ -0,0 +1,189 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.test.util; + + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; + +import java.math.BigDecimal; +import java.time.Instant; +import java.util.Arrays; + +import org.testng.annotations.Test; + +import com.ibm.fhir.persistence.jdbc.dto.CompositeParmVal; +import com.ibm.fhir.persistence.jdbc.dto.DateParmVal; +import com.ibm.fhir.persistence.jdbc.dto.LocationParmVal; +import com.ibm.fhir.persistence.jdbc.dto.NumberParmVal; +import com.ibm.fhir.persistence.jdbc.dto.QuantityParmVal; +import com.ibm.fhir.persistence.jdbc.dto.ReferenceParmVal; +import com.ibm.fhir.persistence.jdbc.dto.StringParmVal; +import com.ibm.fhir.persistence.jdbc.dto.TokenParmVal; +import com.ibm.fhir.persistence.jdbc.util.ExtractedSearchParameters; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; +import com.ibm.fhir.persistence.jdbc.util.type.NumberParmBehaviorUtil; +import com.ibm.fhir.search.date.DateTimeHandler; +import com.ibm.fhir.search.util.ReferenceValue; +import com.ibm.fhir.search.util.ReferenceValue.ReferenceType; + +/** + * Utility to do testing of the parameter hash utility. + */ +public class ParameterHashUtilTest { + + @Test + public void testEmptyExtractedParameters() throws Exception { + ParameterHashUtil util = new ParameterHashUtil(); + + ExtractedSearchParameters esp1 = new ExtractedSearchParameters(); + ExtractedSearchParameters esp2 = new ExtractedSearchParameters(); + + // Hashes not generated yet + assertNull(esp1.getHash()); + assertNull(esp2.getHash()); + + // Generate hashes + esp1.generateHash(util); + esp2.generateHash(util); + + // Check hashes + String hash1 = esp1.getHash(); + String hash2 = esp2.getHash(); + assertNotNull(hash1); + assertNotNull(hash2); + assertEquals(hash1, hash2); + } + + @Test + public void testExtractedParametersDifferentOrders() throws Exception { + ParameterHashUtil util = new ParameterHashUtil(); + Instant instant = Instant.now(); + + // Define some search parameter values + StringParmVal p1 = new StringParmVal(); + p1.setResourceType("Patient"); + p1.setName("code1"); + p1.setUrl("url1"); + p1.setVersion("version1"); + p1.setValueString("valueString1"); + + TokenParmVal p2 = new TokenParmVal(); + p2.setResourceType("Patient"); + p2.setName("code2"); + p2.setUrl("url2"); + p2.setVersion("version2"); + p2.setValueSystem("valueSystem2"); + p2.setValueCode("valueCode2"); + + ReferenceParmVal p3 = new ReferenceParmVal(); + p3.setResourceType("Patient"); + p3.setName("code3"); + p3.setUrl("url3"); + p3.setVersion("version3"); + p3.setRefValue(new ReferenceValue("Patient", "value3", ReferenceType.LOGICAL, 1)); + + QuantityParmVal p4 = new QuantityParmVal(); + p4.setResourceType("Patient"); + p4.setName("code4"); + p4.setUrl("url4"); + p4.setVersion("version4"); + p4.setValueNumber(new BigDecimal(4)); + p4.setValueNumberLow(new BigDecimal(3.9)); + p4.setValueNumberHigh(new BigDecimal(4.1)); + p4.setValueCode("valueCode4"); + p4.setValueSystem("valueSystem4"); + + NumberParmVal p5 = new NumberParmVal(); + p5.setResourceType("Patient"); + p5.setName("code5"); + p5.setUrl("url5"); + p5.setVersion("version5"); + BigDecimal value5 = new BigDecimal(5); + p5.setValueNumber(value5); + p5.setValueNumberLow(NumberParmBehaviorUtil.generateLowerBound(value5)); + p5.setValueNumberHigh(NumberParmBehaviorUtil.generateUpperBound(value5)); + + LocationParmVal p6 = new LocationParmVal(); + p6.setResourceType("Patient"); + p6.setName("code6"); + p6.setUrl("url6"); + p6.setVersion("version6"); + p6.setValueLatitude(6.6); + p6.setValueLongitude(60.6); + + DateParmVal p7 = new DateParmVal(); + p7.setResourceType("Patient"); + p7.setName("code7"); + p7.setUrl("url7"); + p7.setVersion("version7"); + p7.setValueDateStart(DateTimeHandler.generateTimestamp(instant)); + p7.setValueDateEnd(DateTimeHandler.generateTimestamp(instant.plusSeconds(1))); + + CompositeParmVal p8a = new CompositeParmVal(); + p8a.setResourceType("Patient"); + p8a.setName("code8"); + p8a.setUrl("url8"); + p8a.setVersion("version8"); + p8a.addComponent(p1); + p8a.addComponent(p5); + + CompositeParmVal p8b = new CompositeParmVal(); + p8b.setResourceType("Patient"); + p8b.setName("code8"); + p8b.setUrl("url8"); + p8b.setVersion("version8"); + p8b.addComponent(p5); + p8b.addComponent(p1); + + CompositeParmVal p8diff = new CompositeParmVal(); + p8diff.setResourceType("Patient"); + p8diff.setName("code8"); + p8diff.setUrl("url8"); + p8diff.setVersion("version8"); + p8diff.addComponent(p4); + p8diff.addComponent(p6); + + // Add the search parameters in different orders for esp1 and esp2, which still results in same hash, + // but for esp3 and esp4, the search parameters are different, so they should not match the others + ExtractedSearchParameters esp1 = new ExtractedSearchParameters(); + esp1.getParameters().addAll(Arrays.asList(p1, p2, p3, p4, p5, p6, p7, p8a)); + ExtractedSearchParameters esp2 = new ExtractedSearchParameters(); + esp2.getParameters().addAll(Arrays.asList(p8b, p6, p4, p2, p1, p3, p5, p7)); + ExtractedSearchParameters esp3 = new ExtractedSearchParameters(); + esp3.getParameters().addAll(Arrays.asList(p1, p2, p3, p4, p5, p6, p7, p8diff)); + ExtractedSearchParameters esp4 = new ExtractedSearchParameters(); + + // Hashes not generated yet + assertNull(esp1.getHash()); + assertNull(esp2.getHash()); + assertNull(esp3.getHash()); + assertNull(esp4.getHash()); + + // Generate hashes + esp1.generateHash(util); + esp2.generateHash(util); + esp3.generateHash(util); + esp4.generateHash(util); + + // Check hashes + String hash1 = esp1.getHash(); + String hash2 = esp2.getHash(); + String hash3 = esp3.getHash(); + String hash4 = esp4.getHash(); + assertNotNull(hash1); + assertNotNull(hash2); + assertNotNull(hash3); + assertNotNull(hash4); + assertEquals(hash1, hash2); + assertNotEquals(hash1, hash3); + assertNotEquals(hash1, hash4); + assertNotEquals(hash3, hash4); + } +} diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java index 212a6173a92..9c0cae5bfc1 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java @@ -6,6 +6,10 @@ package com.ibm.fhir.schema.control; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Optional; + /** * Tracks the incremental changes to the FHIR schema as it evolves. Incremental * changes to the schema should be recorded here to create a new version number @@ -15,21 +19,21 @@ public enum FhirSchemaVersion { // Make sure the vid values are unique...this cannot be done programmatically with an enum - V0001(1, "Initial version") - ,V0002(2, "Composite search value support") - ,V0003(3, "issue-1263 fhir_ref_sequence start with 20000") - ,V0004(4, "row_id sequence cache 20 to 1000") - ,V0005(5, "issue-1331 add index for resource.last_updated") - ,V0006(6, "issue-1366 normalized schema for storing resource references") - ,V0007(7, "issue-1273 add ref_version_id column to xxx_TOKEN_VALUES_V view") - ,V0008(8, "issue-1929 expose common_token_value_id in xxx_TOKEN_VALUES_V view") - ,V0009(9, "issue-1683 refactor composite values") - ,V0010(10, "issue-1958 add IS_DELETED flag to each xxx_LOGICAL_RESOURCES table") - ,V0011(11, "issue-2011 add LAST_UPDATED to each xxx_LOGICAL_RESOURCES table") - ,V0012(12, "issue-2109 add VERSION_ID to each xxx_LOGICAL_RESOURCES table") - ,V0013(13, "Add $erase operation for hard delete scenarios") - ,V0014(14, "whole-system search and canonical references") - ,V0015(15, "issue-2155 add parameter hash to bypass update during reindex") + V0001(1, "Initial version", true) + ,V0002(2, "Composite search value support", true) + ,V0003(3, "issue-1263 fhir_ref_sequence start with 20000", true) + ,V0004(4, "row_id sequence cache 20 to 1000", true) + ,V0005(5, "issue-1331 add index for resource.last_updated", true) + ,V0006(6, "issue-1366 normalized schema for storing resource references", true) + ,V0007(7, "issue-1273 add ref_version_id column to xxx_TOKEN_VALUES_V view", true) + ,V0008(8, "issue-1929 expose common_token_value_id in xxx_TOKEN_VALUES_V view", true) + ,V0009(9, "issue-1683 refactor composite values", true) + ,V0010(10, "issue-1958 add IS_DELETED flag to each xxx_LOGICAL_RESOURCES table", true) + ,V0011(11, "issue-2011 add LAST_UPDATED to each xxx_LOGICAL_RESOURCES table", true) + ,V0012(12, "issue-2109 add VERSION_ID to each xxx_LOGICAL_RESOURCES table", true) + ,V0013(13, "Add $erase operation for hard delete scenarios", true) + ,V0014(14, "whole-system search and canonical references", true) + ,V0015(15, "issue-2155 add parameter hash to bypass update during reindex", true) ; // The version number recorded in the VERSION_HISTORY @@ -38,14 +42,20 @@ public enum FhirSchemaVersion { // A meaningful description of the schema change private final String description; + // Version change affects parameter storage, which would require reindex of all resources + // even if the search parameters and extracted values for those resources did not change + private final boolean parameterStorageUpdated; + /** * Constructor for the enum - * @param vn - * @param description + * @param vn the version number + * @param description the description + * @param parameterStorageUpdated true if version change affects parameter storage, otherwise false */ - private FhirSchemaVersion(int vn, String description) { + private FhirSchemaVersion(int vn, String description, boolean parameterStorageUpdated) { this.vid = vn; this.description = description; + this.parameterStorageUpdated = parameterStorageUpdated; } /** @@ -64,4 +74,26 @@ public int vid() { public String getDescription() { return this.description; } + + /** + * Determines if the version change affects parameter storage + * @return + */ + public boolean isParameterStorageUpdated() { + return this.parameterStorageUpdated; + } + + /** + * Gets the latest version that included a parameter storage update, which + * would require all resources to reindex all search parameters, even if the + * search parameters and extracted values did not change. + * @return latest version that included a parameter storage update + */ + public static FhirSchemaVersion getLatestParameterStorageUpdate() { + Optional version = Arrays.stream(FhirSchemaVersion.values()) + .filter(k -> k.isParameterStorageUpdated()) + .sorted(Comparator.comparing(FhirSchemaVersion::vid).reversed()) + .findFirst(); + return version.isPresent() ? version.get() : null; + } }