Issue #2155 - Use hash to determine if search parameters need update

Signed-off-by: Troy Biesterfeld <tbieste@us.ibm.com>
LinuxForHealth · Jun 18, 2021 · 4e6e701 · 4e6e701
1 parent 3d47a09
commit 4e6e701
Show file tree

Hide file tree

Showing 19 changed files with 893 additions and 237 deletions.
diff --git a/...-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/ReindexResourceDAO.java b/...-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/ReindexResourceDAO.java
@@ -51,23 +51,23 @@ public class ReindexResourceDAO extends ResourceDAOImpl {
     // the is_deleted flag. Until it does, the queries will return deleted
     // resources, which can be skipped for reindex. (issue-2055)
     private static final String PICK_SINGLE_RESOURCE = ""
-            + "  SELECT lr.logical_resource_id, lr.resource_type_id, lr.logical_id, lr.reindex_txid "
+            + "  SELECT lr.logical_resource_id, lr.resource_type_id, lr.logical_id, lr.reindex_txid, lr.parameter_hash "
             + "    FROM logical_resources lr "
             + "   WHERE lr.resource_type_id = ? "
             + "     AND lr.logical_id = ? "
             + "     AND lr.reindex_tstamp < ? "
             ;
 
     private static final String PICK_SINGLE_RESOURCE_TYPE = ""
-            + "  SELECT lr.logical_resource_id, lr.resource_type_id, lr.logical_id, lr.reindex_txid "
+            + "  SELECT lr.logical_resource_id, lr.resource_type_id, lr.logical_id, lr.reindex_txid, lr.parameter_hash "
             + "    FROM logical_resources lr "
             + "   WHERE lr.resource_type_id = ? "
             + "     AND lr.reindex_tstamp < ? "
             + "OFFSET ? ROWS FETCH FIRST 1 ROWS ONLY "
             ;
 
     private static final String PICK_ANY_RESOURCE = ""
-            + "  SELECT lr.logical_resource_id, lr.resource_type_id, lr.logical_id, lr.reindex_txid "
+            + "  SELECT lr.logical_resource_id, lr.resource_type_id, lr.logical_id, lr.reindex_txid, lr.parameter_hash "
             + "    FROM logical_resources lr "
             + "   WHERE lr.reindex_tstamp < ? "
             + "OFFSET ? ROWS FETCH FIRST 1 ROWS ONLY "
@@ -111,6 +111,7 @@ public ReindexResourceDAO(Connection connection, IDatabaseTranslator translator,
      * Getter for the translator currently held by this DAO
      * @return
      */
+    @Override
     protected IDatabaseTranslator getTranslator() {
         return this.translator;
     }
@@ -169,7 +170,7 @@ protected ResourceIndexRecord getNextResource(SecureRandom random, Instant reind
                 }
                 ResultSet rs = stmt.executeQuery();
                 if (rs.next()) {
-                    result = new ResourceIndexRecord(rs.getLong(1), rs.getInt(2), rs.getString(3), rs.getLong(4));
+                    result = new ResourceIndexRecord(rs.getLong(1), rs.getInt(2), rs.getString(3), rs.getLong(4), rs.getString(5));
                 }
             } catch (SQLException x) {
                 logger.log(Level.SEVERE, select, x);
@@ -265,12 +266,13 @@ public ResourceIndexRecord getResourceToReindex(Instant reindexTstamp, Integer r
     /**
      * Reindex the resource by deleting existing parameters and replacing them with those passed in.
      * @param tablePrefix the table prefix
-     * @param parameters the extracted parameters
+     * @param parameters A collection of search parameters to be persisted along with the passed Resource
+     * @param parameterHashB64 Base64 encoded SHA-256 hash of parameters
      * @param logicalId the logical id
      * @param logicalResourceId the logical resource id
      * @throws Exception
      */
-    public void updateParameters(String tablePrefix, List<ExtractedParameterValue> parameters, String logicalId, long logicalResourceId) throws Exception {
+    public void updateParameters(String tablePrefix, List<ExtractedParameterValue> parameters, String parameterHashB64, String logicalId, long logicalResourceId) throws Exception {
 
         final String METHODNAME = "updateParameters() for " + tablePrefix + "/" + logicalId;
         logger.entering(CLASSNAME, METHODNAME);
@@ -295,7 +297,6 @@ public void updateParameters(String tablePrefix, List<ExtractedParameterValue> p
         deleteFromParameterTable(connection, "str_values", logicalResourceId);
         deleteFromParameterTable(connection, "date_values", logicalResourceId);
 
-
         if (parameters != null && !parameters.isEmpty()) {
             JDBCIdentityCache identityCache = new JDBCIdentityCacheImpl(getCache(), this, parameterDao, getResourceReferenceDAO());
             // Check if this is multitenant
@@ -310,6 +311,10 @@ identityCache, getResourceReferenceDAO(), getTransactionData())) {
                 throw translator.translate(x);
             }
         }
+
+        // Update the "parameters hash" in the LOGICAL_RESOURCES table
+        updateParametersHash(connection, logicalResourceId, parameterHashB64);
+
         logger.exiting(CLASSNAME, METHODNAME);
     }
 
@@ -335,4 +340,27 @@ protected void deleteFromParameterTable(Connection conn, String tableName, long
             throw translator.translate(x);
         }
     }
+
+    /**
+     * Updates the parameters hash in the LOGICAL_RESOURCES table.
+     * @param conn the connection
+     * @param logicalResourceId the logical resource ID
+     * @param hash the parameters hash
+     * @throws SQLException
+     */
+    protected void updateParametersHash(Connection conn, long logicalResourceId, String parameterHashB64) throws SQLException {
+        final String SQL = "UPDATE logical_resources SET parameter_hash = ? WHERE logical_resource_id = ?";
+        try (PreparedStatement stmt = conn.prepareStatement(SQL)) {
+            // bind parameters
+            stmt.setString(1, parameterHashB64);
+            stmt.setLong(2, logicalResourceId);
+            stmt.executeUpdate();
+            if (logger.isLoggable(Level.FINEST)) {
+                logger.finest("Update parameter_hash [" + parameterHashB64 + "] for logicalResourceId [" + logicalResourceId + "]");
+            }
+        } catch (SQLException x) {
+            logger.log(Level.SEVERE, SQL, x);
+            throw translator.translate(x);
+        }
+    }
 }
diff --git a/...istence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceIndexRecord.java b/...istence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceIndexRecord.java
@@ -30,11 +30,15 @@ public class ResourceIndexRecord {
     // Deletion flag for the resource. Set when we read the resource
     private boolean deleted;
 
-    public ResourceIndexRecord(long logicalResourceId, int resourceTypeId, String logicalId, long transactionId) {
+    // Base64-encoded SHA-256 hash of the search parameters
+    private String parameterHash;
+
+    public ResourceIndexRecord(long logicalResourceId, int resourceTypeId, String logicalId, long transactionId, String parameterHash) {
         this.logicalResourceId = logicalResourceId;
         this.resourceTypeId = resourceTypeId;
         this.logicalId = logicalId;
         this.transactionId = transactionId;
+        this.parameterHash = parameterHash;
     }
 
     /**
@@ -92,4 +96,20 @@ public boolean isDeleted() {
     public void setDeleted(boolean deleted) {
         this.deleted = deleted;
     }
+
+    /**
+     * Gets the Base64-encoded SHA-256 hash of the search parameters.
+     * @return the Base64-encoded SHA-256 hash
+     */
+    public String getParameterHash() {
+        return parameterHash;
+    }
+
+    /**
+     * Gets the Base64-encoded SHA-256 hash of the search parameters.
+     * @param parameterHash the Base64-encoded SHA-256 hash
+     */
+    public void setParameterHash(String parameterHash) {
+        this.parameterHash = parameterHash;
+    }
 }
diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/CompositeParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/CompositeParmVal.java
@@ -10,6 +10,7 @@
 import java.util.List;
 
 import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil;
 
 /**
  * This class defines the Data Transfer Object representing a row in the X_DATE_VALUES tables.
@@ -30,10 +31,16 @@ public CompositeParmVal() {
     /**
      * We know our type, so we can call the correct method on the visitor
      */
+    @Override
     public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException {
         visitor.visit(this);
     }
 
+    @Override
+    public String getHash(ParameterHashUtil parameterHashUtil) {
+        return parameterHashUtil.getNameValueHash(getHashHeader(), parameterHashUtil.getParametersHash(component));
+    }
+
     /**
      * @return get the list of components in this composite parameter
      */

diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/DateParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/DateParmVal.java
@@ -7,8 +7,10 @@
 package com.ibm.fhir.persistence.jdbc.dto;
 
 import java.sql.Timestamp;
+import java.util.Objects;
 
 import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil;
 
 /**
  * This class defines the Data Transfer Object representing a row in the X_DATE_VALUES tables.
@@ -18,9 +20,6 @@ public class DateParmVal extends ExtractedParameterValue {
     private Timestamp valueDateStart;
     private Timestamp valueDateEnd;
 
-    // The SearchParameter base type. If "Resource", then this is a Resource-level attribute
-    private String base;
-
     public enum TimeType {
         YEAR,
         YEAR_MONTH,
@@ -53,13 +52,22 @@ public void setValueDateEnd(Timestamp valueDateEnd) {
     /**
      * We know our type, so we can call the correct method on the visitor
      */
+    @Override
     public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException {
         visitor.visit(this);
     }
 
+    @Override
+    public String getHash(ParameterHashUtil parameterHashUtil) {
+        StringBuilder sb = new StringBuilder();
+        sb.append(Objects.toString(valueDateStart, ""));
+        sb.append("|").append(Objects.toString(valueDateEnd, ""));
+        return parameterHashUtil.getNameValueHash(getHashHeader(), sb.toString());
+    }
+
     @Override
     public String toString() {
         return "DateParmVal [resourceType=" + getResourceType() + ", name=" + getName()
-                + ", valueDateStart=" + valueDateStart + ", valueDateEnd=" + valueDateEnd + ", base=" + base + "]";
+                + ", valueDateStart=" + valueDateStart + ", valueDateEnd=" + valueDateEnd + "]";
     }
 }
diff --git a/...istence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ExtractedParameterValue.java b/...istence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ExtractedParameterValue.java
@@ -6,7 +6,11 @@
 
 package com.ibm.fhir.persistence.jdbc.dto;
 
+import java.util.Objects;
+
 import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil;
+import com.ibm.fhir.schema.control.FhirSchemaVersion;
 
 /**
  * A search parameter value extracted from a resource and ready to store / index for search
@@ -25,6 +29,10 @@ public abstract class ExtractedParameterValue {
     // The bass resource name
     private String base;
 
+    // URL and version of search parameter
+    private String url;
+    private String version;
+
     /**
      * Protected constructor
      */
@@ -93,4 +101,53 @@ public String getName() {
     public void setName(String name) {
         this.name = name;
     }
+
+    /**
+     * @return the url
+     */
+    public String getUrl() {
+        return url;
+    }
+
+    /**
+     * @param url the url to set
+     */
+    public void setUrl(String url) {
+        this.url = url;
+    }
+
+    /**
+     * @return the version
+     */
+    public String getVersion() {
+        return version;
+    }
+
+    /**
+     * @param version the version to set
+     */
+    public void setVersion(String version) {
+        this.version = version;
+    }
+
+    /**
+     * Gets the hash header.
+     * @return the hash header
+     */
+    protected String getHashHeader() {
+        StringBuilder sb = new StringBuilder(name);
+        sb.append(Objects.toString(FhirSchemaVersion.getLatestParameterStorageUpdate(), "")).append("|");
+        sb.append(Objects.toString(name, "")).append("|");
+        sb.append(Objects.toString(url, "")).append("|");
+        sb.append(Objects.toString(version, ""));
+        return sb.toString();
+    }
+
+    /**
+     * Gets the hash representation of the parameter.
+     * This should be generated from the search parameter (schemaVersion, code, url, version) and the extracted value.
+     * @param the parameter hash utility to use for generating hashes
+     * @return the hash
+     */
+    public abstract String getHash(ParameterHashUtil parameterHashUtil);
 }
diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/LocationParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/LocationParmVal.java
@@ -6,7 +6,10 @@
 
 package com.ibm.fhir.persistence.jdbc.dto;
 
+import java.util.Objects;
+
 import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil;
 
 /**
  * This class defines the Data Transfer Object representing a row in the X_LATLNG_VALUES tables.
@@ -42,7 +45,16 @@ public void setValueLatitude(Double valueLatitude) {
     /**
      * We know our type, so we can call the correct method on the visitor
      */
+    @Override
     public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException {
         visitor.visit(this);
     }
+
+    @Override
+    public String getHash(ParameterHashUtil parameterHashUtil) {
+        StringBuilder sb = new StringBuilder();
+        sb.append(Objects.toString(valueLongitude, ""));
+        sb.append("|").append(Objects.toString(valueLatitude, ""));
+        return parameterHashUtil.getNameValueHash(getHashHeader(), sb.toString());
+    }
 }
diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/NumberParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/NumberParmVal.java
@@ -7,8 +7,10 @@
 package com.ibm.fhir.persistence.jdbc.dto;
 
 import java.math.BigDecimal;
+import java.util.Objects;
 
 import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil;
 
 /**
  * This class defines the Data Transfer Object representing a row in the X_NUMBER_VALUES tables.
@@ -19,15 +21,13 @@ public class NumberParmVal extends ExtractedParameterValue {
     private BigDecimal valueNumberLow;
     private BigDecimal valueNumberHigh;
 
-    // The SearchParameter base type. If "Resource", then this is a Resource-level attribute
-    private String base;
-
     /**
      * Public constructor
      */
     public NumberParmVal() {
         super();
     }
+
     public BigDecimal getValueNumber() {
         return valueNumber;
     }
@@ -55,7 +55,17 @@ public void setValueNumberHigh(BigDecimal valueNumberHigh) {
     /**
      * We know our type, so we can call the correct method on the visitor
      */
+    @Override
     public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException {
         visitor.visit(this);
     }
+
+    @Override
+    public String getHash(ParameterHashUtil parameterHashUtil) {
+        StringBuilder sb = new StringBuilder();
+        sb.append(Objects.toString(valueNumber, ""));
+        sb.append("|").append(Objects.toString(valueNumberLow, ""));
+        sb.append("|").append(Objects.toString(valueNumberHigh, ""));
+        return parameterHashUtil.getNameValueHash(getHashHeader(), sb.toString());
+    }
 }
diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/QuantityParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/QuantityParmVal.java
@@ -7,9 +7,11 @@
 package com.ibm.fhir.persistence.jdbc.dto;
 
 import java.math.BigDecimal;
+import java.util.Objects;
 
 import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
 import com.ibm.fhir.persistence.jdbc.JDBCConstants;
+import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil;
 
 /**
  * This class defines the Data Transfer Object representing a row in the X_QUANTITY_VALUES tables.
@@ -83,4 +85,15 @@ public void setValueNumberHigh(BigDecimal valueNumberHigh) {
     public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException {
         visitor.visit(this);
     }
+
+    @Override
+    public String getHash(ParameterHashUtil parameterHashUtil) {
+        StringBuilder sb = new StringBuilder();
+        sb.append(Objects.toString(valueNumber, ""));
+        sb.append("|").append(Objects.toString(valueNumberLow, ""));
+        sb.append("|").append(Objects.toString(valueNumberHigh, ""));
+        sb.append("|").append(getValueSystem());
+        sb.append("|").append(getValueCode());
+        return parameterHashUtil.getNameValueHash(getHashHeader(), sb.toString());
+    }
 }