diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java index d981705d8a2..0bdb3ef6b35 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java @@ -68,6 +68,10 @@ import com.ibm.fhir.task.api.ITaskGroup; import com.ibm.fhir.task.core.service.TaskService; +/** + * The fhir-bucket application for loading data from COS into a FHIR server + * and tracking the returned ids along with response times. + */ public class Main { private static final Logger logger = Logger.getLogger(Main.class.getName()); private static final int DEFAULT_CONNECTION_POOL_SIZE = 10; @@ -191,6 +195,12 @@ public class Main { // How many reindex calls should we run in parallel private int reindexConcurrentRequests = 1; + // The number of patients to fetch into the buffer + private int patientBufferSize = 500000; + + // How many times should we cycle through the patient buffer before refilling + private int bufferRecycleCount = 1; + /** * Parse command line arguments * @param args @@ -209,6 +219,13 @@ public void parseArgs(String[] args) { case "--create-schema": this.createSchema = true; break; + case "--schema-name": + if (i < args.length + 1) { + this.schemaName = args[++i]; + } else { + throw new IllegalArgumentException("missing value for --schema-name"); + } + break; case "--cos-properties": if (i < args.length + 1) { loadCosProperties(args[++i]); @@ -356,6 +373,20 @@ public void parseArgs(String[] args) { throw new IllegalArgumentException("missing value for --max-resources-per-bundle"); } break; + case "--patient-buffer-size": + if (i < args.length + 1) { + this.patientBufferSize = Integer.parseInt(args[++i]); + } else { + throw new IllegalArgumentException("missing value for --patient-buffer-size"); + } + break; + case "--buffer-recycle-count": + if (i < args.length + 1) { + this.bufferRecycleCount = Integer.parseInt(args[++i]); + } else { + throw new IllegalArgumentException("missing value for --buffer-recycle-count"); + } + break; case "--incremental": this.incremental = true; break; @@ -653,10 +684,10 @@ public void bootstrapDb() { if (adapter.getTranslator().getType() == DbType.POSTGRESQL) { // Postgres doesn't support batched merges, so we go with a simpler UPSERT - MergeResourceTypesPostgres mrt = new MergeResourceTypesPostgres(resourceTypes); + MergeResourceTypesPostgres mrt = new MergeResourceTypesPostgres(schemaName, resourceTypes); adapter.runStatement(mrt); } else { - MergeResourceTypes mrt = new MergeResourceTypes(resourceTypes); + MergeResourceTypes mrt = new MergeResourceTypes(schemaName, resourceTypes); adapter.runStatement(mrt); } } catch (Exception x) { @@ -825,7 +856,7 @@ protected void scanAndLoad() { if (this.concurrentPayerRequests > 0 && fhirClient != null) { // set up the CMS payer thread to add some read-load to the system InteropScenario scenario = new InteropScenario(this.fhirClient); - cmsPayerWorkload = new InteropWorkload(dataAccess, scenario, concurrentPayerRequests, 500000); + cmsPayerWorkload = new InteropWorkload(dataAccess, scenario, concurrentPayerRequests, this.patientBufferSize, this.bufferRecycleCount); cmsPayerWorkload.init(); } diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/interop/InteropWorkload.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/interop/InteropWorkload.java index c0cddde56b0..a9c8cd32f02 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/interop/InteropWorkload.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/interop/InteropWorkload.java @@ -32,52 +32,62 @@ public class InteropWorkload { // The scenario we use to process each randomly picked patient private final IPatientScenario patientScenario; - + // the maximum number of requests we permit private final int maxConcurrentRequests; - + private final Lock lock = new ReentrantLock(); private final Condition capacityCondition = lock.newCondition(); - + private volatile int runningRequests; - + private volatile boolean running = true; // The thread running the main loop private Thread thread; - + // How many patients should we load into the buffer private final int patientBufferSize; - + + // How many times should we use the same set of patient ids? + private final int bufferRecycleCount; + // Access to the FHIRBATCH schema private final DataAccess dataAccess; // thread pool for processing requests private final ExecutorService pool = Executors.newCachedThreadPool(); - + // for picking random patient ids private final SecureRandom random = new SecureRandom(); - + private long statsResetTime = -1; private final AtomicInteger fhirRequests = new AtomicInteger(); private final AtomicLong fhirRequestTime = new AtomicLong(); private final AtomicInteger resourceCount = new AtomicInteger(); - + // how many nanoseconds between stats reports private static final long STATS_REPORT_TIME = 10L * 1000000000L; /** * Public constructor - * @param client + * @param dataAccess + * @param patientScenario * @param maxConcurrentRequests + * @param patientBufferSize + * @param bufferRecycleCount */ - public InteropWorkload(DataAccess dataAccess, IPatientScenario patientScenario, int maxConcurrentRequests, int patientBufferSize) { + public InteropWorkload(DataAccess dataAccess, IPatientScenario patientScenario, int maxConcurrentRequests, int patientBufferSize, int bufferRecycleCount) { + if (bufferRecycleCount < 1) { + throw new IllegalArgumentException("bufferRecycleCount must be >= 1"); + } this.dataAccess = dataAccess; this.patientScenario = patientScenario; this.maxConcurrentRequests = maxConcurrentRequests; this.patientBufferSize = patientBufferSize; + this.bufferRecycleCount = bufferRecycleCount; } - + /** * Start the main loop @@ -90,10 +100,10 @@ public void init() { thread = new Thread(() -> mainLoop()); thread.start(); } - + public void signalStop() { this.running = false; - + lock.lock(); try { // wake up the thread if it's waiting on the capacity condition @@ -101,17 +111,17 @@ public void signalStop() { } finally { lock.unlock(); } - - + + // try to break into any IO operation for a quicker exit if (thread != null) { thread.interrupt(); } - + // make sure the pool doesn't start new work pool.shutdown(); } - + /** * Wait until things are stopped */ @@ -119,20 +129,20 @@ public void waitForStop() { if (this.running) { signalStop(); } - + try { pool.awaitTermination(5, TimeUnit.SECONDS); } catch (InterruptedException x) { logger.warning("Wait for pool shutdown interrupted"); } } - + /** * The main loop in this object which starts when {@link #init()} is called * and will run until {@link #shutdown()}. */ protected void mainLoop() { - + // How many samples have we taken from the current buffer? int samples = 0; // The list of patientIds we process @@ -142,13 +152,13 @@ protected void mainLoop() { while (this.running) { try { - if (patientIdBuffer.isEmpty() || samples > patientIdBuffer.size()) { + if (patientIdBuffer.isEmpty() || samples > patientIdBuffer.size() * this.bufferRecycleCount) { // Refill the buffer of patient ids. There might be more available now patientIdBuffer.clear(); samples = 0; dataAccess.selectRandomPatientIds(patientIdBuffer, this.patientBufferSize); } - + // calculate how many requests we want to submit from the buffer, based // on the maxConcurrentRequests. int batchSize = 0; @@ -157,8 +167,8 @@ protected void mainLoop() { while (running && runningRequests == maxConcurrentRequests) { capacityCondition.await(5, TimeUnit.SECONDS); } - - // Submit as many requests as we have available. If we have a small + + // Submit as many requests as we have available. If we have a small // patient buffer, then patients are more likely to be picked more than once int freeCapacity = maxConcurrentRequests - runningRequests; batchSize = Math.min(patientIdBuffer.size(), freeCapacity); @@ -171,7 +181,7 @@ protected void mainLoop() { } finally { lock.unlock(); } - + // Submit a request for each allocated patient to the thread pool for (int i=0; i processPatientThr(patientId)); } - + long now = System.nanoTime(); if (now >= nextStatsReport) { // Time to report average throughput stats @@ -191,10 +201,10 @@ protected void mainLoop() { if (this.fhirRequests.get() > 0) { avgResponseTime = this.fhirRequestTime.get() / 1e9 / this.fhirRequests.get(); } - - logger.info(String.format("STATS: FHIR=%7.1f calls/s, rate=%7.1f resources/s, response time=%5.3f s", + + logger.info(String.format("STATS: FHIR=%7.1f calls/s, rate=%7.1f resources/s, response time=%5.3f s", avgCallPerSecond, avgResourcesPerSecond, avgResponseTime)); - + // Reset the stats for the next report window statsStartTime = now; nextStatsReport = now + STATS_REPORT_TIME; @@ -231,7 +241,7 @@ private void processPatientThr(String patientId) { if (logger.isLoggable(Level.FINE)) { logger.fine("Processing patient: '" + patientId + "'"); } - + patientScenario.process(patientId, fhirRequests, fhirRequestTime, resourceCount); } catch (Exception x) { logger.log(Level.SEVERE, "Processing patient '" + patientId + "'" , x); diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddBucketPath.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddBucketPath.java index 999ee83dc25..93be50bedf7 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddBucketPath.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddBucketPath.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -16,6 +16,7 @@ import com.ibm.fhir.database.utils.api.IDatabaseSupplier; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; import com.ibm.fhir.database.utils.model.DbType; /** @@ -25,18 +26,24 @@ public class AddBucketPath implements IDatabaseSupplier { private static final Logger logger = Logger.getLogger(RegisterLoaderInstance.class.getName()); + // The name of the schema holding the FHIRBUCKET tables + private final String schemaName; + // The name of the bucket private final String bucketName; - + // The path in which the item resides...basically the key up and including the last / private final String bucketPath; + /** * Public constructor + * @param schemaName * @param bucketName * @param bucketPath */ - public AddBucketPath(String bucketName, String bucketPath) { + public AddBucketPath(String schemaName, String bucketName, String bucketPath) { + this.schemaName = schemaName; this.bucketName = bucketName; this.bucketPath = bucketPath; } @@ -48,15 +55,16 @@ public Long run(IDatabaseTranslator translator, Connection c) { // MERGE and upsert-like tricks don't appear to work with Derby // when using autogenerated identity columns. So we have to // try the old-fashioned way and handle duplicate key - String dml; + final String bucketPaths = DataDefinitionUtil.getQualifiedName(schemaName, "bucket_paths"); + final String dml; if (translator.getType() == DbType.POSTGRESQL) { // For POSTGRES, if a statement fails it causes the whole transaction // to fail, so we need turn this into an UPSERT - dml = "INSERT INTO bucket_paths (bucket_name, bucket_path) VALUES (?,?) ON CONFLICT(bucket_name, bucket_path) DO NOTHING"; + dml = "INSERT INTO " + bucketPaths + "(bucket_name, bucket_path) VALUES (?,?) ON CONFLICT(bucket_name, bucket_path) DO NOTHING"; } else { - dml = "INSERT INTO bucket_paths (bucket_name, bucket_path) VALUES (?,?)"; + dml = "INSERT INTO " + bucketPaths + "(bucket_name, bucket_path) VALUES (?,?)"; } - + try (PreparedStatement ps = c.prepareStatement(dml, Statement.RETURN_GENERATED_KEYS)) { ps.setString(1, bucketName); ps.setString(2, bucketPath); @@ -77,11 +85,11 @@ public Long run(IDatabaseTranslator translator, Connection c) { throw translator.translate(x); } } - + // If we didn't create a new record, fetch the id of the existing record (we don't delete these // records, so no chance of a race condition if (bucketPathId == null) { - final String SQL = "SELECT bucket_path_id FROM bucket_paths WHERE bucket_name = ? AND bucket_path = ?"; + final String SQL = "SELECT bucket_path_id FROM " + bucketPaths + " WHERE bucket_name = ? AND bucket_path = ?"; try (PreparedStatement ps = c.prepareStatement(SQL)) { ps.setString(1, bucketName); ps.setString(2, bucketPath); @@ -96,7 +104,7 @@ public Long run(IDatabaseTranslator translator, Connection c) { throw translator.translate(x); } } - + return bucketPathId; } } \ No newline at end of file diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddResourceBundle.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddResourceBundle.java index 4358b97e2e1..5215d12650d 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddResourceBundle.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddResourceBundle.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -19,6 +19,7 @@ import com.ibm.fhir.bucket.api.ResourceBundleData; import com.ibm.fhir.database.utils.api.IDatabaseSupplier; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; import com.ibm.fhir.database.utils.model.DbType; /** @@ -27,15 +28,18 @@ */ public class AddResourceBundle implements IDatabaseSupplier { private static final Logger logger = Logger.getLogger(RegisterLoaderInstance.class.getName()); - + + // The schema holding the tables + private final String schemaName; + // The database id of the bucket_path private final long bucketPathId; // The name of the object (e.g. bundle file) within the bucket private final String objectName; - + private final long objectSize; - + // The type of file represented by this object private final FileType fileType; @@ -47,10 +51,12 @@ public class AddResourceBundle implements IDatabaseSupplier /** * Public constructor + * @param schemaName * @param bucketId * @param objectName */ - public AddResourceBundle(long bucketPathId, String objectName, long objectSize, FileType fileType, String eTag, Date lastModified) { + public AddResourceBundle(String schemaName, long bucketPathId, String objectName, long objectSize, FileType fileType, String eTag, Date lastModified) { + this.schemaName = schemaName; this.bucketPathId = bucketPathId; this.objectName = objectName; this.objectSize = objectSize; @@ -68,19 +74,20 @@ public ResourceBundleData run(IDatabaseTranslator translator, Connection c) { // try the old-fashioned way and handle duplicate key final String currentTimestamp = translator.currentTimestampString(); int version = 1; - String dml; + final String resourceBundles = DataDefinitionUtil.getQualifiedName(schemaName, "resource_bundles"); + final String dml; if (translator.getType() == DbType.POSTGRESQL) { // For PostgresSQL, make sure we don't break the current transaction // if the statement fails...annoying - dml = "INSERT INTO resource_bundles (" + dml = "INSERT INTO " + resourceBundles + "(" + "bucket_path_id, object_name, object_size, file_type, etag, last_modified, scan_tstamp, version) " + " VALUES (?, ?, ?, ?, ?, ?, " + currentTimestamp + ", ?) ON CONFLICT (bucket_path_id, object_name) DO NOTHING"; } else { - dml = "INSERT INTO resource_bundles (" + dml = "INSERT INTO " + resourceBundles + "(" + "bucket_path_id, object_name, object_size, file_type, etag, last_modified, scan_tstamp, version) " + " VALUES (?, ?, ?, ?, ?, ?, " + currentTimestamp + ", ?)"; } - + try (PreparedStatement ps = c.prepareStatement(dml, Statement.RETURN_GENERATED_KEYS)) { ps.setLong(1, bucketPathId); ps.setString(2, objectName); @@ -114,13 +121,13 @@ public ResourceBundleData run(IDatabaseTranslator translator, Connection c) { throw translator.translate(x); } } - + // If we didn't create a new record, fetch the old record before we update. It's important // to make this a select-for-update to avoid a possible race-condition if (result == null) { final String SQL = translator.addForUpdate("" + "SELECT resource_bundle_id, object_size, file_type, etag, last_modified, scan_tstamp, version " - + " FROM resource_bundles " + + " FROM " + resourceBundles + " WHERE bucket_path_id = ? " + " AND object_name = ?"); try (PreparedStatement ps = c.prepareStatement(SQL)) { @@ -144,14 +151,14 @@ public ResourceBundleData run(IDatabaseTranslator translator, Connection c) { + bucketPathId + ", " + objectName); throw translator.translate(x); } - + // If the current database record doesn't match what we've been passed // then we want to update it with the latest and bump the version number // so that we can see it has changed. if (!result.matches(this.objectSize, this.eTag, this.lastModified)) { - + final String UPD = "" - + "UPDATE resource_bundles " + + "UPDATE " + resourceBundles + " SET object_size = ?, " + " etag = ?, " + " last_modified = ?, " @@ -160,7 +167,7 @@ public ResourceBundleData run(IDatabaseTranslator translator, Connection c) { + " allocation_id = NULL, " // reset state so that this + " loader_instance_id = NULL " // file will be picked up + " WHERE resource_bundle_id = ?"; - + try (PreparedStatement ps = c.prepareStatement(UPD)) { ps.setLong(1, objectSize); ps.setString(2, eTag); @@ -175,7 +182,7 @@ public ResourceBundleData run(IDatabaseTranslator translator, Connection c) { } } } - + return result; } } \ No newline at end of file diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddResourceBundleErrors.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddResourceBundleErrors.java index 2a8c8606478..7f9336db92b 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddResourceBundleErrors.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddResourceBundleErrors.java @@ -1,11 +1,14 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2021 * * SPDX-License-Identifier: Apache-2.0 */ package com.ibm.fhir.bucket.persistence; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.ERROR_TEXT_LEN; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.HTTP_STATUS_TEXT_LEN; + import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -17,9 +20,9 @@ import java.util.logging.Logger; import com.ibm.fhir.bucket.api.ResourceBundleError; -import static com.ibm.fhir.bucket.persistence.SchemaConstants.*; import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; /** * DAO to encapsulate all the SQL/DML used to retrieve and persist data @@ -28,21 +31,28 @@ public class AddResourceBundleErrors implements IDatabaseStatement { private static final Logger logger = Logger.getLogger(RegisterLoaderInstance.class.getName()); + // The schema with the FHIRBUCKET tables + private final String schemaName; + // The list of resource types we want to add private final List errors; - + // The resource bundle where we hit the error private final long resourceBundleLoadId; - + // The SQL batch size private final int batchSize; - + /** * Public constructor - * @param resourceType + * @param schemaName + * @param resourceBundleLoadId + * @param errors + * @param batchSize */ - public AddResourceBundleErrors(long resourceBundleLoadId, Collection errors, + public AddResourceBundleErrors(String schemaName, long resourceBundleLoadId, Collection errors, int batchSize) { + this.schemaName = schemaName; this.resourceBundleLoadId = resourceBundleLoadId; this.errors = new ArrayList<>(errors); this.batchSize = batchSize; @@ -50,13 +60,14 @@ public AddResourceBundleErrors(long resourceBundleLoadId, Collection 0) { // final batch ps.executeBatch(); @@ -90,7 +101,7 @@ public void run(IDatabaseTranslator translator, Connection c) { throw translator.translate(x); } } - + /** * Convenience function to set a nullable int field * @param ps @@ -119,7 +130,7 @@ private void setField(PreparedStatement ps, int nbr, String value, int maxLen) t if (value != null && value.length() > maxLen) { value = value.substring(0, maxLen); } - + if (value != null) { ps.setString(nbr, value); } else { diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AllocateJobs.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AllocateJobs.java index ede57656721..c4141a4d5e1 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AllocateJobs.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AllocateJobs.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -21,6 +21,7 @@ import com.ibm.fhir.database.utils.api.DataAccessException; import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; /** * DAO to encapsulate all the SQL/DML used to retrieve and persist data @@ -31,26 +32,30 @@ public class AllocateJobs implements IDatabaseStatement { // The schema with all the database objects private final String schemaName; - + // The list of allocated jobs private final List jobList; - + // Limit allocations to the given file type private final FileType fileType; - + // The id of this loader instance being allocated the job private final long loaderInstanceId; // The number of jobs to allocate private final int free; - + // Limit job to matching bucket-path combinations private final Collection bucketPaths; /** * Public constructor - * @param bucketName - * @param bucketPath + * @param schemaName + * @param jobList + * @param fileType + * @param loaderInstanceId + * @param free + * @param bucketPaths */ public AllocateJobs(String schemaName, List jobList, FileType fileType, long loaderInstanceId, int free, Collection bucketPaths) { this.schemaName = schemaName; @@ -86,7 +91,7 @@ public void run(IDatabaseTranslator translator, Connection c) { if (bpBuilder.length() > 0) { bpBuilder.append(" OR "); } - + bpBuilder.append("bp.bucket_name = ? AND bp.bucket_path = ?"); } @@ -94,26 +99,28 @@ public void run(IDatabaseTranslator translator, Connection c) { if (bpBuilder.length() > 0) { bucketPathPredicate = " AND (" + bpBuilder.toString() + ")"; } - + // Mark the records we want to allocate using the unique allocationId we just obtained // Note the ORDER BY in the inner select is important to avoid deadlocks when running // concurrent loaders + final String resourceBundles = DataDefinitionUtil.getQualifiedName(schemaName, "resource_bundles"); + final String bucketPathsTbl = DataDefinitionUtil.getQualifiedName(schemaName, "bucket_paths"); final String currentTimestamp = translator.currentTimestampString(); final String MARK = "" - + " UPDATE resource_bundles rb " + + " UPDATE " + resourceBundles + " rb " + " SET allocation_id = ?, " + " loader_instance_id = ? " + " WHERE rb.resource_bundle_id IN ( " + " SELECT rbInner.resource_bundle_id " - + " FROM resource_bundles rbInner, " - + " bucket_paths bp " // bp is the table alias referenced in the bucketPathPredicate built above + + " FROM " + resourceBundles + " rbInner, " + + " " + bucketPathsTbl + " bp " // bp is the table alias referenced in the bucketPathPredicate built above + " WHERE rbInner.allocation_id IS NULL " + " AND rbInner.file_type = ? " + " AND bp.bucket_path_id = rbInner.bucket_path_id " + bucketPathPredicate + " ORDER BY rbInner.last_modified, rbInner.resource_bundle_id " + " FETCH FIRST ? ROWS ONLY)"; - + try (PreparedStatement ps = c.prepareStatement(MARK)) { int a = 1; ps.setLong(a++, allocationId); @@ -125,7 +132,7 @@ public void run(IDatabaseTranslator translator, Connection c) { ps.setString(a++, bp.getBucketName()); ps.setString(a++, bp.getPathPrefix()); } - + ps.setInt(a++, free); ps.executeUpdate(); } catch (SQLException x) { @@ -135,10 +142,11 @@ public void run(IDatabaseTranslator translator, Connection c) { // Create new RESOURCE_BUNDLE_LOAD records for each of the bundles // that have just been allocated + final String resourceBundleLoads = DataDefinitionUtil.getQualifiedName(schemaName, "resource_bundle_loads"); final String INS = "" - + " INSERT INTO resource_bundle_loads (resource_bundle_id, allocation_id, loader_instance_id, load_started, version) " + + " INSERT INTO " + resourceBundleLoads + "(resource_bundle_id, allocation_id, loader_instance_id, load_started, version) " + " SELECT rb.resource_bundle_id, rb.allocation_id, rb.loader_instance_id, " + currentTimestamp + ", rb.version " - + " FROM resource_bundles rb " + + " FROM " + resourceBundles + " rb " + " WHERE rb.allocation_id = ? " // allocation_id is assigned to us from a sequence, so no other loader will have it ; try (PreparedStatement ps = c.prepareStatement(INS)) { @@ -148,20 +156,20 @@ public void run(IDatabaseTranslator translator, Connection c) { logger.log(Level.SEVERE, INS, x); throw new DataAccessException("Insert allocated jobs failed"); } - + // Now fetch the records we just marked. Order by just provides consistent ordering final String FETCH = "" + "SELECT bl.resource_bundle_load_id, rb.resource_bundle_id, bp.bucket_name, bp.bucket_path, " + " rb.object_name, rb.object_size, rb.file_type, rb.version " - + " FROM resource_bundles rb, " - + " bucket_paths bp," - + " resource_bundle_loads bl " + + " FROM " + resourceBundles + " rb, " + + " " + bucketPathsTbl + " bp, " + + " " + resourceBundleLoads + " bl " + " WHERE rb.allocation_id = ? " + " AND bp.bucket_path_id = rb.bucket_path_id " + " AND bl.resource_bundle_id = rb.resource_bundle_id " + " AND bl.allocation_id = rb.allocation_id " + "ORDER BY rb.last_modified, rb.resource_bundle_id "; - + try (PreparedStatement ps = c.prepareStatement(FETCH)) { ps.setLong(1, allocationId); ResultSet rs = ps.executeQuery(); diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/ClearStaleAllocations.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/ClearStaleAllocations.java index 2693c944288..3ed74e58bc2 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/ClearStaleAllocations.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/ClearStaleAllocations.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -15,6 +15,7 @@ import com.ibm.fhir.database.utils.api.DataAccessException; import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; /** * DAO to encapsulate all the SQL/DML used to retrieve and persist data @@ -23,19 +24,27 @@ public class ClearStaleAllocations implements IDatabaseStatement { private static final Logger logger = Logger.getLogger(AllocateJobs.class.getName()); + // the schema holding the tables + private final String schemaName; + // The id of this loader instance so we can ignore self allocations private final long loaderInstanceId; // The number of MS before we consider a missed heartbeat private final long heartbeatTimeoutMs; - + // Number of seconds to wait before recycling a completed job private final int recycleSeconds; /** * Public constructor + * @param schemaName + * @param loaderInstanceId + * @param heartbeatTimeoutMs + * @param recycleSeconds */ - public ClearStaleAllocations(long loaderInstanceId, long heartbeatTimeoutMs, int recycleSeconds) { + public ClearStaleAllocations(String schemaName, long loaderInstanceId, long heartbeatTimeoutMs, int recycleSeconds) { + this.schemaName = schemaName; this.loaderInstanceId = loaderInstanceId; this.heartbeatTimeoutMs = heartbeatTimeoutMs; this.recycleSeconds = recycleSeconds; @@ -46,20 +55,21 @@ public void run(IDatabaseTranslator translator, Connection c) { // Firstly, mark as stopped any loader instances which are currently active // but haven't updated their heartbeat within the required time. + final String loaderInstances = DataDefinitionUtil.getQualifiedName(schemaName, "loader_instances"); final String currentTimestamp = translator.currentTimestampString(); final String MARK = "" - + "UPDATE loader_instances " + + "UPDATE " + loaderInstances + " SET status = 'STOPPED' " + " WHERE loader_instance_id != ? " + " AND status = 'RUNNING' " + " AND " + translator.timestampDiff(currentTimestamp, "heartbeat_tstamp", null) + " >= ?" ; - + try (PreparedStatement ps = c.prepareStatement(MARK)) { ps.setLong(1, loaderInstanceId); ps.setLong(2, heartbeatTimeoutMs / 1000); int affectedRows = ps.executeUpdate(); - + if (affectedRows > 0) { logger.info("Cleared RUNNING status record count: " + affectedRows); } @@ -68,20 +78,22 @@ public void run(IDatabaseTranslator translator, Connection c) { throw new DataAccessException("Mark stopped loader instances failed"); } - + // Clear out any allocations for resource bundles for which the - // assigned loader instance is considered dead but were not - // marked as complete. This will allow the bundles to be + // assigned loader instance is considered dead but were not + // marked as complete. This will allow the bundles to be // picked up again by another/new loader instance + final String resourceBundles = DataDefinitionUtil.getQualifiedName(schemaName, "resource_bundles"); + final String resourceBundleLoads = DataDefinitionUtil.getQualifiedName(schemaName, "resource_bundle_loads"); final String UPD = "" - + "UPDATE resource_bundles " + + "UPDATE " + resourceBundles + " SET allocation_id = NULL, " + " loader_instance_id = NULL " + " WHERE resource_bundle_id IN ( " + " SELECT rb.resource_bundle_id " - + " FROM resource_bundles rb, " - + " loader_instances li, " - + " resource_bundle_loads bl " + + " FROM " + resourceBundles + " rb, " + + " " + loaderInstances + " li, " + + " " + resourceBundleLoads + " bl " + " WHERE li.loader_instance_id = rb.loader_instance_id " + " AND rb.allocation_id IS NOT NULL " + " AND li.loader_instance_id != ? " @@ -91,11 +103,11 @@ public void run(IDatabaseTranslator translator, Connection c) { + " AND bl.resource_bundle_id = rb.resource_bundle_id " + " AND bl.load_completed IS NULL " + " )"; - + try (PreparedStatement ps = c.prepareStatement(UPD)) { ps.setLong(1, loaderInstanceId); int rowsAffected = ps.executeUpdate(); - + if (rowsAffected > 0 && logger.isLoggable(Level.FINE)) { logger.fine("Cleared resource_bundles allocation count: " + rowsAffected); } @@ -112,25 +124,25 @@ public void run(IDatabaseTranslator translator, Connection c) { final String current = translator.currentTimestampString(); final String diff = translator.timestampDiff(current, "bl.load_completed", null); final String RECYCLE = "" - + "UPDATE resource_bundles " + + "UPDATE " + resourceBundles + " SET loader_instance_id = NULL, " // deallocate so it gets picked up again + " allocation_id = NULL, " + " version = version + 1 " // pretend it's a new version of the file + " WHERE resource_bundle_id IN ( " + " SELECT rb.resource_bundle_id " - + " FROM resource_bundles rb, " - + " resource_bundle_loads bl " + + " FROM " + resourceBundles + " rb, " + + " " + resourceBundleLoads + " bl " + " WHERE bl.allocation_id = rb.allocation_id " // most recent allocation + " AND bl.loader_instance_id = rb.loader_instance_id " + " AND bl.resource_bundle_id = rb.resource_bundle_id " + " AND bl.load_completed IS NOT NULL " // just in case someone changes the diff expression + " AND " + diff + " >= ? " // more than seconds after the last load completed + " )"; - + try (PreparedStatement ps = c.prepareStatement(RECYCLE)) { ps.setInt(1, recycleSeconds); int rowsAffected = ps.executeUpdate(); - + if (rowsAffected > 0 && logger.isLoggable(Level.FINE)) { logger.fine("Recycled bundles count: " + rowsAffected); } diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/GetLastProcessedLineNumber.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/GetLastProcessedLineNumber.java index e7014d6b989..e42b2e15153 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/GetLastProcessedLineNumber.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/GetLastProcessedLineNumber.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -15,6 +15,7 @@ import com.ibm.fhir.database.utils.api.IDatabaseSupplier; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; /** * Finds the greater line number successfully processed for a particular resource bundle @@ -22,28 +23,36 @@ public class GetLastProcessedLineNumber implements IDatabaseSupplier { private static final Logger logger = Logger.getLogger(RegisterLoaderInstance.class.getName()); + // The schema holding the FHIRBUCKET tables + private final String schemaName; + // PK of the loader instance to update private final long resourceBundleId; - + // The version of file which generated the ids private final int version; - + /** * Public constructor - * @param loaderInstanceId + * @param schemaName + * @param resourceBundleId + * @param version */ - public GetLastProcessedLineNumber(long resourceBundleId, int version) { + public GetLastProcessedLineNumber(String schemaName, long resourceBundleId, int version) { + this.schemaName = schemaName; this.resourceBundleId = resourceBundleId; this.version = version; } @Override public Integer run(IDatabaseTranslator translator, Connection c) { - + + final String logicalResources = DataDefinitionUtil.getQualifiedName(schemaName, "logical_resources"); + final String resourceBundleLoads = DataDefinitionUtil.getQualifiedName(schemaName, "resource_bundle_loads"); final String SQL = "" + "SELECT max(lr.line_number) " - + " FROM logical_resources lr, " - + " resource_bundle_loads bl " + + " FROM " + logicalResources + " lr, " + + " " + resourceBundleLoads + " bl " + " WHERE bl.resource_bundle_id = ? " + " AND bl.version = ? " + " AND lr.resource_bundle_load_id = bl.resource_bundle_load_id "; diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/GetLogicalIds.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/GetLogicalIds.java index 7f83323ae27..d395e330cdc 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/GetLogicalIds.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/GetLogicalIds.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -16,6 +16,7 @@ import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; /** * Fetch a batch of roughly random patientIds. Should not be used for any @@ -24,20 +25,27 @@ public class GetLogicalIds implements IDatabaseStatement { private static final Logger logger = Logger.getLogger(GetLogicalIds.class.getName()); + // The schema holding the FHIRBUCKET tables + final String schemaName; + // The list to fill with logical ids final List logicalIds; - + // How many ids to fetch private final int maxCount; // The resource type private final String resourceType; - + /** * Public constructor - * @param loaderInstanceId + * @param schemaName + * @param logicalIds + * @param resourceType + * @param maxCount */ - public GetLogicalIds(List logicalIds, String resourceType, int maxCount) { + public GetLogicalIds(String schemaName, List logicalIds, String resourceType, int maxCount) { + this.schemaName = schemaName; this.logicalIds = logicalIds; this.resourceType = resourceType; this.maxCount = maxCount; @@ -47,12 +55,14 @@ public GetLogicalIds(List logicalIds, String resourceType, int maxCount) public void run(IDatabaseTranslator translator, Connection c) { // Fetch the list of patient ids up to the given max + final String logicalResources = DataDefinitionUtil.getQualifiedName(this.schemaName, "logical_resources"); + final String resourceTypes = DataDefinitionUtil.getQualifiedName(this.schemaName, "resource_types"); final String SQL = "" + "SELECT lr.logical_id " - + " FROM fhirbucket.logical_resources lr," - + " fhirbucket.resource_types rt " + + " FROM " + logicalResources + " lr," + + " " + resourceTypes + " rt " + " WHERE lr.resource_type_id = rt.resource_type_id " - + " AND rt.resource_type = ? " + + " AND rt.resource_type = ? " + "FETCH FIRST ? ROWS ONLY;"; try (PreparedStatement ps = c.prepareStatement(SQL)) { ps.setString(1, this.resourceType); diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/GetResourceRefsForBundleLine.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/GetResourceRefsForBundleLine.java index 3a0f0476dcd..a05b225b906 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/GetResourceRefsForBundleLine.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/GetResourceRefsForBundleLine.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -18,6 +18,7 @@ import com.ibm.fhir.bucket.api.ResourceRef; import com.ibm.fhir.database.utils.api.IDatabaseSupplier; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; /** * Fetches the list of resources which have been created from processing a given @@ -26,21 +27,25 @@ public class GetResourceRefsForBundleLine implements IDatabaseSupplier> { private static final Logger logger = Logger.getLogger(RegisterLoaderInstance.class.getName()); + private final String schemaName; + // PK of the loader instance to update private final long resourceBundleId; - + // The version of file which generated the ids private final int version; - + private final int lineNumber; - + /** * Public constructor + * @param schemaName * @param resourceBundleId * @param version * @param lineNumber */ - public GetResourceRefsForBundleLine(long resourceBundleId, int version, int lineNumber) { + public GetResourceRefsForBundleLine(String schemaName, long resourceBundleId, int version, int lineNumber) { + this.schemaName = schemaName; this.resourceBundleId = resourceBundleId; this.version = version; this.lineNumber = lineNumber; @@ -49,11 +54,14 @@ public GetResourceRefsForBundleLine(long resourceBundleId, int version, int line @Override public List run(IDatabaseTranslator translator, Connection c) { List result = new ArrayList<>(); + final String logicalResources = DataDefinitionUtil.getQualifiedName(schemaName, "logical_resources"); + final String resourceBundleLoads = DataDefinitionUtil.getQualifiedName(schemaName, "resource_bundle_loads"); + final String resourceTypes = DataDefinitionUtil.getQualifiedName(schemaName, "resource_types"); final String SQL = "" + "SELECT rt.resource_type, lr.logical_id " - + " FROM logical_resources lr, " - + " resource_bundle_loads bl," - + " resource_types rt " + + " FROM " + logicalResources + " lr, " + + " " + resourceBundleLoads + " bl," + + " " + resourceTypes + " rt " + " WHERE bl.resource_bundle_id = ? " + " AND bl.version = ? " + " AND lr.line_number = ? " @@ -73,7 +81,7 @@ public List run(IDatabaseTranslator translator, Connection c) { + resourceBundleId + ", " + version + ", " + lineNumber); throw translator.translate(x); } - + return result; } } \ No newline at end of file diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/LoaderInstanceHeartbeat.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/LoaderInstanceHeartbeat.java index 1ee3527b587..3efe902ddd2 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/LoaderInstanceHeartbeat.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/LoaderInstanceHeartbeat.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -14,6 +14,7 @@ import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; /** * Updates the last seen timestamp of the LOADER_INSTANCES record @@ -22,23 +23,29 @@ public class LoaderInstanceHeartbeat implements IDatabaseStatement { private static final Logger logger = Logger.getLogger(RegisterLoaderInstance.class.getName()); + // The schema holding the fhirbucket tables + private final String schemaName; + // PK of the loader instance to update private final long loaderInstanceId; - + /** * Public constructor + * @param schemaName * @param loaderInstanceId */ - public LoaderInstanceHeartbeat(long loaderInstanceId) { + public LoaderInstanceHeartbeat(String schemaName, long loaderInstanceId) { + this.schemaName = schemaName; this.loaderInstanceId = loaderInstanceId; } @Override public void run(IDatabaseTranslator translator, Connection c) { final String currentTimestamp = translator.currentTimestampString(); - + + final String loaderInstances = DataDefinitionUtil.getQualifiedName(schemaName, "loader_instances"); final String DML = "" - + "UPDATE loader_instances " + + "UPDATE " + loaderInstances + " SET heartbeat_tstamp = " + currentTimestamp + " WHERE loader_instance_id = ?"; try (PreparedStatement ps = c.prepareStatement(DML)) { diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/MarkBundleDone.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/MarkBundleDone.java index f4ab39141a8..398fe9fbb77 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/MarkBundleDone.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/MarkBundleDone.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -14,6 +14,7 @@ import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; /** * Updates the LOAD_COMPLETED timestamp of the resource_bundles record @@ -21,19 +22,26 @@ public class MarkBundleDone implements IDatabaseStatement { private static final Logger logger = Logger.getLogger(MarkBundleDone.class.getName()); + // The schema holding the FHIRBUCKET tables + private final String schemaName; + // PK of the resource_bundle_loads to update private final long resourceBundleLoadId; - + // How many records failed when processing this file/bundle private final int failureCount; - + private final int rowsProcessed; - + /** * Public constructor - * @param loaderInstanceId + * @param schemaName + * @param resourceBundleLoadId + * @param failureCount + * @param rowsProcessed */ - public MarkBundleDone(long resourceBundleLoadId, int failureCount, int rowsProcessed) { + public MarkBundleDone(String schemaName, long resourceBundleLoadId, int failureCount, int rowsProcessed) { + this.schemaName = schemaName; this.resourceBundleLoadId = resourceBundleLoadId; this.failureCount = failureCount; this.rowsProcessed = rowsProcessed; @@ -41,10 +49,11 @@ public MarkBundleDone(long resourceBundleLoadId, int failureCount, int rowsProce @Override public void run(IDatabaseTranslator translator, Connection c) { - + + final String resourceBundleLoads = DataDefinitionUtil.getQualifiedName(schemaName, "resource_bundle_loads"); final String currentTimestamp = translator.currentTimestampString(); final String DML = "" - + "UPDATE resource_bundle_loads " + + "UPDATE " + resourceBundleLoads + " SET load_completed = " + currentTimestamp + ", " + " failure_count = ?, " + " rows_processed = ? " diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/MergeResourceTypes.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/MergeResourceTypes.java index 4953800383f..fee467d304a 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/MergeResourceTypes.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/MergeResourceTypes.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -17,42 +17,48 @@ import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; import com.ibm.fhir.database.utils.model.DbType; /** * DAO to encapsulate all the SQL/DML used to retrieve and persist data - * in the schema. + * in the schema. * Supports: Db2 and Derby * Does not support: PostgreSQL */ public class MergeResourceTypes implements IDatabaseStatement { private static final Logger logger = Logger.getLogger(RegisterLoaderInstance.class.getName()); + private final String schemaName; + // The list of resource types we want to add private final List resourceTypes; - + /** * Public constructor - * @param resourceType + * @param schemaName + * @param resourceTypes */ - public MergeResourceTypes(Collection resourceTypes) { + public MergeResourceTypes(String schemaName, Collection resourceTypes) { + this.schemaName = schemaName; // copy the list for safety this.resourceTypes = new ArrayList(resourceTypes); } @Override public void run(IDatabaseTranslator translator, Connection c) { - + + final String tgtName = DataDefinitionUtil.getQualifiedName(schemaName, "resource_types"); final String dual = translator.dualTableName(); final String source = dual == null ? "(SELECT 1)" : dual; // Use a bulk merge approach to insert resource types not previously // loaded - final String merge = "MERGE INTO resource_types tgt " + final String merge = "MERGE INTO " + tgtName + " tgt " + " USING " + source + " src " + " ON tgt.resource_type = ? " + " WHEN NOT MATCHED THEN INSERT (resource_type) VALUES (?)"; - + try (PreparedStatement ps = c.prepareStatement(merge)) { // Assume the list is small enough to process in one batch for (String resourceType: resourceTypes) { diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/MergeResourceTypesPostgres.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/MergeResourceTypesPostgres.java index 8d090300f8d..04b1001e6cc 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/MergeResourceTypesPostgres.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/MergeResourceTypesPostgres.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -17,24 +17,28 @@ import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; /** * DAO to encapsulate all the SQL/DML used to retrieve and persist data - * in the schema. + * in the schema. * Supports only: PostgreSQL */ public class MergeResourceTypesPostgres implements IDatabaseStatement { private static final Logger logger = Logger.getLogger(RegisterLoaderInstance.class.getName()); + private final String schemaName; // The list of resource types we want to add private final List resourceTypes; - + /** * Public constructor + * @param schemaName * @param resourceTypes the list of resource types to merge into the RESOURCE_TYPES * table */ - public MergeResourceTypesPostgres(Collection resourceTypes) { + public MergeResourceTypesPostgres(String schemaName, Collection resourceTypes) { + this.schemaName = schemaName; // copy the list for safety this.resourceTypes = new ArrayList(resourceTypes); } @@ -43,11 +47,12 @@ public MergeResourceTypesPostgres(Collection resourceTypes) { public void run(IDatabaseTranslator translator, Connection c) { // UPSERT PostgreSQL style + final String tableName = DataDefinitionUtil.getQualifiedName(schemaName, "resource_types"); final String dml = "" - + " INSERT INTO resource_types (resource_type) " + + " INSERT INTO " + tableName + "(resource_type) " + " VALUES (?) " + " ON CONFLICT (resource_type) DO NOTHING"; - + try (PreparedStatement ps = c.prepareStatement(dml)) { for (String resourceType: resourceTypes) { ps.setString(1, resourceType); diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/RecordLogicalId.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/RecordLogicalId.java index b5ed68b73c9..f9b71bbc7a6 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/RecordLogicalId.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/RecordLogicalId.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -15,6 +15,7 @@ import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; import com.ibm.fhir.database.utils.model.DbType; /** @@ -24,6 +25,9 @@ public class RecordLogicalId implements IDatabaseStatement { private static final Logger logger = Logger.getLogger(RegisterLoaderInstance.class.getName()); + // the schema holding the fhirbucket tables + private final String schemaName; + // FK describing the type of the resource private final int resourceTypeId; @@ -35,16 +39,21 @@ public class RecordLogicalId implements IDatabaseStatement { // the line number of the resource (in an NDJSON file) private final int lineNumber; - + // Response time if this was an individual resource create (not part of a bundle) private final Integer responseTimeMs; /** * Public constructor - * @param bucketId - * @param objectName + * @param schemaName + * @param resourceTypeId + * @param logicalId + * @param resourceBundleLoadId + * @param lineNumber + * @param responseTimeMs */ - public RecordLogicalId(int resourceTypeId, String logicalId, long resourceBundleLoadId, int lineNumber, Integer responseTimeMs) { + public RecordLogicalId(String schemaName, int resourceTypeId, String logicalId, long resourceBundleLoadId, int lineNumber, Integer responseTimeMs) { + this.schemaName = schemaName; this.resourceTypeId = resourceTypeId; this.logicalId = logicalId; this.resourceBundleLoadId = resourceBundleLoadId; @@ -56,21 +65,22 @@ public RecordLogicalId(int resourceTypeId, String logicalId, long resourceBundle public void run(IDatabaseTranslator translator, Connection c) { final String currentTimestamp = translator.currentTimestampString(); - String dml; + final String logicalResources = DataDefinitionUtil.getQualifiedName(schemaName, "logical_resources"); + final String dml; if (translator.getType() == DbType.POSTGRESQL) { // Use UPSERT syntax for Postgres to avoid breaking the transaction when // a statement fails - dml = - "INSERT INTO logical_resources (" + dml = + "INSERT INTO " + logicalResources + "(" + " resource_type_id, logical_id, resource_bundle_load_id, line_number, response_time_ms, created_tstamp) " + " VALUES (?, ?, ?, ?, ?, " + currentTimestamp + ") ON CONFLICT (resource_type_id, logical_id) DO NOTHING"; } else { - dml = - "INSERT INTO logical_resources (" + dml = + "INSERT INTO " + logicalResources + "(" + " resource_type_id, logical_id, resource_bundle_load_id, line_number, response_time_ms, created_tstamp) " + " VALUES (?, ?, ?, ?, ?, " + currentTimestamp + ")"; } - + try (PreparedStatement ps = c.prepareStatement(dml)) { ps.setLong(1, resourceTypeId); ps.setString(2, logicalId); @@ -86,7 +96,7 @@ public void run(IDatabaseTranslator translator, Connection c) { if (translator.isDuplicate(x)) { // This resource has already been recorded, so we'll just warn in case something // is going wrong - logger.warning("Duplicate resource logical id: " + resourceTypeId + "/" + logicalId + logger.warning("Duplicate resource logical id: " + resourceTypeId + "/" + logicalId + " from " + resourceBundleLoadId + "#" + lineNumber); } else { // log this, but don't propagate values in the exception diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/RegisterLoaderInstance.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/RegisterLoaderInstance.java index ec1fb3ec8f2..32bf49cee2f 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/RegisterLoaderInstance.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/RegisterLoaderInstance.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -16,6 +16,7 @@ import com.ibm.fhir.database.utils.api.DataAccessException; import com.ibm.fhir.database.utils.api.IDatabaseSupplier; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; /** * DAO to encapsulate all the SQL/DML used to retrieve and persist data @@ -26,14 +27,24 @@ public class RegisterLoaderInstance implements IDatabaseSupplier { // unique key (UUID) representing this loader instance private final String loaderInstanceKey; - + // the host we are running on private final String host; - + // the process id, if we've been able to get it private final int pid; - - public RegisterLoaderInstance(String loaderInstanceKey, String host, int pid) { + + private final String schemaName; + + /** + * Public constructor + * @param schemaName + * @param loaderInstanceKey + * @param host + * @param pid + */ + public RegisterLoaderInstance(String schemaName, String loaderInstanceKey, String host, int pid) { + this.schemaName = schemaName; this.loaderInstanceKey = loaderInstanceKey; this.host = host; this.pid = pid; @@ -44,8 +55,9 @@ public Long run(IDatabaseTranslator translator, Connection c) { Long loaderInstanceId; // loader_instance_id is generated always as identity + final String tableName = DataDefinitionUtil.getQualifiedName(schemaName, "loader_instances"); final String currentTimestamp = translator.currentTimestampString(); - final String SQL = "INSERT INTO loader_instances (loader_instance_key, hostname, pid, heartbeat_tstamp, status) " + final String SQL = "INSERT INTO " + tableName + "(loader_instance_key, hostname, pid, heartbeat_tstamp, status) " + " VALUES (?, ?, ?, " + currentTimestamp + ", 'RUNNING')"; try (PreparedStatement ps = c.prepareStatement(SQL, 1)) { ps.setString(1, loaderInstanceKey); @@ -64,8 +76,8 @@ public Long run(IDatabaseTranslator translator, Connection c) { + loaderInstanceKey + ", " + host + ", " + pid); throw translator.translate(x); } - - + + return loaderInstanceId; } diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/ResourceTypesReader.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/ResourceTypesReader.java index 8868437ab45..fe7b074f04a 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/ResourceTypesReader.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/ResourceTypesReader.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -17,6 +17,7 @@ import com.ibm.fhir.database.utils.api.IDatabaseSupplier; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; /** * DAO to encapsulate all the SQL/DML used to retrieve and persist data @@ -25,18 +26,23 @@ public class ResourceTypesReader implements IDatabaseSupplier> { private static final Logger logger = Logger.getLogger(RegisterLoaderInstance.class.getName()); + private final String schemaName; + /** * Public constructor + * @param schemaName */ - public ResourceTypesReader() { + public ResourceTypesReader(String schemaName) { + this.schemaName = schemaName; } @Override public List run(IDatabaseTranslator translator, Connection c) { List result = new ArrayList<>(); - final String SQL = "SELECT resource_type_id, resource_type FROM resource_types"; - + final String tableName = DataDefinitionUtil.getQualifiedName(schemaName, "resource_types"); + final String SQL = "SELECT resource_type_id, resource_type FROM " + tableName; + try (PreparedStatement ps = c.prepareStatement(SQL)) { ResultSet rs = ps.executeQuery(); while (rs.next()) { @@ -49,7 +55,7 @@ public List run(IDatabaseTranslator translator, Connection c) { logger.log(Level.SEVERE, "Error reading resource types"); throw translator.translate(x); } - + return result; } } \ No newline at end of file diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/scanner/DataAccess.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/scanner/DataAccess.java index 0aab57cdae3..73d154c14b3 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/scanner/DataAccess.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/scanner/DataAccess.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -29,15 +29,14 @@ import com.ibm.fhir.bucket.persistence.AllocateJobs; import com.ibm.fhir.bucket.persistence.ClearStaleAllocations; import com.ibm.fhir.bucket.persistence.GetLastProcessedLineNumber; +import com.ibm.fhir.bucket.persistence.GetLogicalIds; import com.ibm.fhir.bucket.persistence.GetResourceRefsForBundleLine; import com.ibm.fhir.bucket.persistence.LoaderInstanceHeartbeat; import com.ibm.fhir.bucket.persistence.MarkBundleDone; import com.ibm.fhir.bucket.persistence.RecordLogicalId; -import com.ibm.fhir.bucket.persistence.RecordLogicalIdList; import com.ibm.fhir.bucket.persistence.RegisterLoaderInstance; import com.ibm.fhir.bucket.persistence.ResourceTypeRec; import com.ibm.fhir.bucket.persistence.ResourceTypesReader; -import com.ibm.fhir.bucket.persistence.GetLogicalIds; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.api.ITransaction; import com.ibm.fhir.database.utils.api.ITransactionProvider; @@ -51,28 +50,28 @@ public class DataAccess { // no heartbeats for 60 seconds means something has gone wrong private static final long HEARTBEAT_TIMEOUT_MS = 60000; - + // how many errors to insert per JDBC batch private int errorBatchSize = 10; - + // The adapter we use to execute database statements private final IDatabaseAdapter dbAdapter; - + // Simple transaction service for use outside of JEE private final ITransactionProvider transactionProvider; // Internal cache of resource types, which are created as part of schema deployment private final Map resourceTypeMap = new ConcurrentHashMap<>(); - + // The unique id string representing this instance of the loader private final String instanceId; - + // The id returned by the database when registering this loader instance private long loaderInstanceId; - + // the name of the schema holding all the tables private final String schemaName; - + /** * Public constructor * @param connectionPool @@ -83,7 +82,7 @@ public DataAccess(IDatabaseAdapter dbAdapter, ITransactionProvider txProvider, S this.dbAdapter = dbAdapter; this.transactionProvider = txProvider; this.schemaName = schemaName; - + // Generate a unique id string to represent this instance of the loader while it's running UUID uuid = UUID.randomUUID(); this.instanceId = uuid.toString(); @@ -95,12 +94,12 @@ public DataAccess(IDatabaseAdapter dbAdapter, ITransactionProvider txProvider, S public void init() { try (ITransaction tx = transactionProvider.getTransaction()) { try { - List resourceTypes = dbAdapter.runStatement(new ResourceTypesReader()); + List resourceTypes = dbAdapter.runStatement(new ResourceTypesReader(this.schemaName)); resourceTypes.stream().forEach(rt -> resourceTypeMap.put(rt.getResourceType(), rt.getResourceTypeId())); - + // Register this loader instance InetAddress addr = InetAddress.getLocalHost(); - RegisterLoaderInstance c1 = new RegisterLoaderInstance(instanceId, addr.getHostName(), -1); + RegisterLoaderInstance c1 = new RegisterLoaderInstance(this.schemaName, instanceId, addr.getHostName(), -1); this.loaderInstanceId = dbAdapter.runStatement(c1); } catch (UnknownHostException x) { logger.severe("FATAL ERROR. Failed to register instance"); @@ -132,19 +131,19 @@ public void registerBucketItem(CosItem item) { name = path; path = "/"; } - + if (name != null) { - AddBucketPath c1 = new AddBucketPath(item.getBucketName(), path); + AddBucketPath c1 = new AddBucketPath(this.schemaName, item.getBucketName(), path); Long bucketPathId = dbAdapter.runStatement(c1); - + // Now register the bundle using the bucket record we created/retrieved - AddResourceBundle c2 = new AddResourceBundle(bucketPathId, name, item.getSize(), item.getFileType(), + AddResourceBundle c2 = new AddResourceBundle(schemaName, bucketPathId, name, item.getSize(), item.getFileType(), item.geteTag(), item.getLastModified()); ResourceBundleData old = dbAdapter.runStatement(c2); if (old != null && !old.matches(item.getSize(), item.geteTag(), item.getLastModified())) { // log the fact that the item has been changed in COS and so we've updated our // record of it in the bucket database -> it will be processed again. - logger.info("COS item changed, " + item.toString() + logger.info("COS item changed, " + item.toString() + ", old={size=" + old.getObjectSize() + ", etag=" + old.geteTag() + ", lastModified=" + old.getLastModified() + "}" + ", new={size=" + item.getSize() + ", etag=" + item.geteTag() + ", lastModified=" + item.getLastModified() + "}" ); @@ -169,9 +168,9 @@ public void allocateJobs(List jobList, FileType fileType, int f try { // First business of the day is to check for liveness and clear // any allocations for instances we think are no longer active - ClearStaleAllocations liveness = new ClearStaleAllocations(loaderInstanceId, HEARTBEAT_TIMEOUT_MS, recycleSeconds); + ClearStaleAllocations liveness = new ClearStaleAllocations(schemaName, loaderInstanceId, HEARTBEAT_TIMEOUT_MS, recycleSeconds); dbAdapter.runStatement(liveness); - + AllocateJobs cmd = new AllocateJobs(schemaName, jobList, fileType, loaderInstanceId, free, bucketPaths); dbAdapter.runStatement(cmd); } catch (Exception x) { @@ -194,8 +193,8 @@ public void recordLogicalId(String resourceType, String logicalId, long resource // unlikely, unless the map hasn't been initialized properly throw new IllegalStateException("resourceType not found: " + resourceType); } - - RecordLogicalId cmd = new RecordLogicalId(resourceTypeId, logicalId, resourceBundleLoadId, lineNumber, responseTimeMs); + + RecordLogicalId cmd = new RecordLogicalId(schemaName, resourceTypeId, logicalId, resourceBundleLoadId, lineNumber, responseTimeMs); dbAdapter.runStatement(cmd); } catch (Exception x) { tx.setRollbackOnly(); @@ -211,7 +210,7 @@ public void recordLogicalId(String resourceType, String logicalId, long resource public void heartbeat() { try (ITransaction tx = transactionProvider.getTransaction()) { try { - LoaderInstanceHeartbeat heartbeat = new LoaderInstanceHeartbeat(this.loaderInstanceId); + LoaderInstanceHeartbeat heartbeat = new LoaderInstanceHeartbeat(this.schemaName, this.loaderInstanceId); dbAdapter.runStatement(heartbeat); } catch (Exception x) { tx.setRollbackOnly(); @@ -226,7 +225,7 @@ public void heartbeat() { public void markJobDone(BucketLoaderJob job) { try (ITransaction tx = transactionProvider.getTransaction()) { try { - MarkBundleDone c1 = new MarkBundleDone(job.getResourceBundleLoadId(), job.getFailureCount(), job.getCompletedCount()); + MarkBundleDone c1 = new MarkBundleDone(schemaName, job.getResourceBundleLoadId(), job.getFailureCount(), job.getCompletedCount()); dbAdapter.runStatement(c1); } catch (Exception x) { tx.setRollbackOnly(); @@ -252,17 +251,17 @@ public void recordLogicalIds(long resourceBundleLoadId, int lineNumber, List errors) { try (ITransaction tx = transactionProvider.getTransaction()) { try { - AddResourceBundleErrors cmd = new AddResourceBundleErrors(resourceBundleLoadId, errors, errorBatchSize); + AddResourceBundleErrors cmd = new AddResourceBundleErrors(schemaName, resourceBundleLoadId, errors, errorBatchSize); dbAdapter.runStatement(cmd); } catch (Exception x) { tx.setRollbackOnly(); @@ -297,7 +296,7 @@ public void recordErrors(long resourceBundleLoadId, int lineNumber, List getResourceRefsForLine(long resourceBundleId, int version, int lineNumber) { try (ITransaction tx = transactionProvider.getTransaction()) { try { - GetResourceRefsForBundleLine cmd = new GetResourceRefsForBundleLine(resourceBundleId, version, lineNumber); + GetResourceRefsForBundleLine cmd = new GetResourceRefsForBundleLine(this.schemaName, resourceBundleId, version, lineNumber); return dbAdapter.runStatement(cmd); } catch (Exception x) { tx.setRollbackOnly(); @@ -334,7 +333,7 @@ public void selectRandomPatientIds(List patientIds, int maxPatients) { try (ITransaction tx = transactionProvider.getTransaction()) { try { // Grab the list of patient logical ids - GetLogicalIds cmd = new GetLogicalIds(patientIds, Patient.class.getSimpleName(), maxPatients); + GetLogicalIds cmd = new GetLogicalIds(this.schemaName, patientIds, Patient.class.getSimpleName(), maxPatients); dbAdapter.runStatement(cmd); } catch (Exception x) { tx.setRollbackOnly(); diff --git a/fhir-bucket/src/test/java/com/ibm/fhir/bucket/persistence/test/FhirBucketSchemaTest.java b/fhir-bucket/src/test/java/com/ibm/fhir/bucket/persistence/test/FhirBucketSchemaTest.java index 132b7eb5915..a81754e3171 100644 --- a/fhir-bucket/src/test/java/com/ibm/fhir/bucket/persistence/test/FhirBucketSchemaTest.java +++ b/fhir-bucket/src/test/java/com/ibm/fhir/bucket/persistence/test/FhirBucketSchemaTest.java @@ -137,12 +137,12 @@ public void basicBucketSchemaTests() { DerbyAdapter adapter = new DerbyAdapter(connectionPool); try (ITransaction tx = transactionProvider.getTransaction()) { try { - RegisterLoaderInstance c1 = new RegisterLoaderInstance(uuid.toString(), "host", 1234); + RegisterLoaderInstance c1 = new RegisterLoaderInstance(DATA_SCHEMA_NAME, uuid.toString(), "host", 1234); Long lid = adapter.runStatement(c1); assertNotNull(lid); this.loaderInstanceId = lid; // save for future tests - AddBucketPath c2 = new AddBucketPath("bucket1", "/path/to/dir1/"); + AddBucketPath c2 = new AddBucketPath(DATA_SCHEMA_NAME, "bucket1", "/path/to/dir1/"); Long bucketId = adapter.runStatement(c2); assertNotNull(bucketId); @@ -151,13 +151,13 @@ public void basicBucketSchemaTests() { assertNotNull(id2); assertEquals(id2, bucketId); - AddBucketPath c3 = new AddBucketPath("bucket1", "/path/to/dir1/"); + AddBucketPath c3 = new AddBucketPath(DATA_SCHEMA_NAME, "bucket1", "/path/to/dir1/"); Long id3 = adapter.runStatement(c3); assertNotNull(id3); assertEquals(id3, id2); // Register a resource bundle under the first bucket path "bucket1:/path/to/dir1/" - AddResourceBundle c4 = new AddResourceBundle(bucketId, "patient1.json", 1024, FileType.JSON, "1234abcd", new Date()); + AddResourceBundle c4 = new AddResourceBundle(DATA_SCHEMA_NAME, bucketId, "patient1.json", 1024, FileType.JSON, "1234abcd", new Date()); ResourceBundleData id4 = adapter.runStatement(c4); assertNotNull(id4); @@ -167,7 +167,7 @@ public void basicBucketSchemaTests() { assertEquals(id4.getResourceBundleId(), id5.getResourceBundleId()); // Add a second resource bundle record - AddResourceBundle c5 = new AddResourceBundle(bucketId, "patient2.json", 1024, FileType.JSON, "1234abcd", new Date()); + AddResourceBundle c5 = new AddResourceBundle(DATA_SCHEMA_NAME, bucketId, "patient2.json", 1024, FileType.JSON, "1234abcd", new Date()); ResourceBundleData id6 = adapter.runStatement(c5); assertNotNull(id6); assertNotEquals(id6.getResourceBundleId(), id5.getResourceBundleId()); @@ -176,7 +176,7 @@ public void basicBucketSchemaTests() { Set resourceTypes = Arrays.stream(FHIRResourceType.Value.values()) .map(FHIRResourceType.Value::value) .collect(Collectors.toSet()); - MergeResourceTypes c6 = new MergeResourceTypes(resourceTypes); + MergeResourceTypes c6 = new MergeResourceTypes(DATA_SCHEMA_NAME, resourceTypes); adapter.runStatement(c6); @@ -193,7 +193,7 @@ public void readResourceTypesTest() { DerbyAdapter adapter = new DerbyAdapter(connectionPool); try (ITransaction tx = transactionProvider.getTransaction()) { try { - ResourceTypesReader c1 = new ResourceTypesReader(); + ResourceTypesReader c1 = new ResourceTypesReader(DATA_SCHEMA_NAME); List resourceTypes = adapter.runStatement(c1); // Check against our reference set of resources @@ -221,11 +221,11 @@ public void resourceBundlesTest() { try (ITransaction tx = transactionProvider.getTransaction()) { try { // Need a bucket_path so we can create a resource_bundle - AddBucketPath c1 = new AddBucketPath("bucket1", "/path/to/dir2/"); + AddBucketPath c1 = new AddBucketPath(DATA_SCHEMA_NAME, "bucket1", "/path/to/dir2/"); Long bucketPathId = adapter.runStatement(c1); // Test creation of resource bundles - AddResourceBundle c2 = new AddResourceBundle(bucketPathId, "patient1.json", 1024, FileType.JSON, "abcd123", new Date()); + AddResourceBundle c2 = new AddResourceBundle(DATA_SCHEMA_NAME, bucketPathId, "patient1.json", 1024, FileType.JSON, "abcd123", new Date()); ResourceBundleData resourceBundleData = adapter.runStatement(c2); assertNotNull(resourceBundleData); } catch (Throwable t) { @@ -261,7 +261,7 @@ public void allocateJobsTest() { // Remove any stale allocations. Give a fake loaderInstanceId because // we don't touch stuff that we own. Make the timeout negative to force // the timeout - ClearStaleAllocations c4 = new ClearStaleAllocations(loaderInstanceId + 1, -60000, -1); + ClearStaleAllocations c4 = new ClearStaleAllocations(DATA_SCHEMA_NAME, loaderInstanceId + 1, -60000, -1); adapter.runStatement(c4); // Now we should be able to see all 3 allocations be reassigned @@ -270,11 +270,11 @@ public void allocateJobsTest() { adapter.runStatement(c5); assertEquals(jobList.size(), 3); - MarkBundleDone c6 = new MarkBundleDone(jobList.get(0).getResourceBundleLoadId(), 0, 1); + MarkBundleDone c6 = new MarkBundleDone(DATA_SCHEMA_NAME, jobList.get(0).getResourceBundleLoadId(), 0, 1); adapter.runStatement(c6); // recycle completed jobs immediately - ClearStaleAllocations c7 = new ClearStaleAllocations(loaderInstanceId + 1, 100000, 0); + ClearStaleAllocations c7 = new ClearStaleAllocations(DATA_SCHEMA_NAME, loaderInstanceId + 1, 100000, 0); adapter.runStatement(c7); // Grab the job-list again. Should get 3 @@ -284,7 +284,7 @@ public void allocateJobsTest() { // With a job, we have a resource_bundle_loads record, so we can create some resources Map resourceTypeMap = new HashMap<>(); - List resourceTypes = adapter.runStatement(new ResourceTypesReader()); + List resourceTypes = adapter.runStatement(new ResourceTypesReader(DATA_SCHEMA_NAME)); resourceTypes.stream().forEach(rt -> resourceTypeMap.put(rt.getResourceType(), rt.getResourceTypeId())); final int patientTypeId = resourceTypeMap.get("Patient"); BucketLoaderJob job = jobList.get(0); @@ -297,11 +297,11 @@ public void allocateJobsTest() { // Single logical id upload final int lineNumber = LINE_COUNT; - RecordLogicalId c8 = new RecordLogicalId(patientTypeId, "patient-5", job.getResourceBundleLoadId(), lineNumber, 0); + RecordLogicalId c8 = new RecordLogicalId(DATA_SCHEMA_NAME, patientTypeId, "patient-5", job.getResourceBundleLoadId(), lineNumber, 0); adapter.runStatement(c8); // Check that the max line number for the given bundle - GetLastProcessedLineNumber c9 = new GetLastProcessedLineNumber(job.getResourceBundleId(), job.getVersion()); + GetLastProcessedLineNumber c9 = new GetLastProcessedLineNumber(DATA_SCHEMA_NAME, job.getResourceBundleId(), job.getVersion()); Integer lastLine = adapter.runStatement(c9); assertNotNull(lastLine); assertEquals(lastLine.intValue(), lineNumber); @@ -312,17 +312,17 @@ public void allocateJobsTest() { List errors = new ArrayList<>(); errors.add(new ResourceBundleError(0, "error1", null, null, null)); errors.add(new ResourceBundleError(1, "error2", 60000, 400, "timeout")); - AddResourceBundleErrors c10 = new AddResourceBundleErrors(job.getResourceBundleLoadId(), errors, 10); + AddResourceBundleErrors c10 = new AddResourceBundleErrors(DATA_SCHEMA_NAME, job.getResourceBundleLoadId(), errors, 10); adapter.runStatement(c10); // Fetch some ResourceRefs for a line we know we have loaded - GetResourceRefsForBundleLine c11 = new GetResourceRefsForBundleLine(job.getResourceBundleId(), job.getVersion(), lineNumber); + GetResourceRefsForBundleLine c11 = new GetResourceRefsForBundleLine(DATA_SCHEMA_NAME, job.getResourceBundleId(), job.getVersion(), lineNumber); List refs = adapter.runStatement(c11); assertNotNull(refs); assertEquals(refs.size(), 1); // And an empty list - GetResourceRefsForBundleLine c12 = new GetResourceRefsForBundleLine(job.getResourceBundleId(), job.getVersion(), lineNumber+1); + GetResourceRefsForBundleLine c12 = new GetResourceRefsForBundleLine(DATA_SCHEMA_NAME, job.getResourceBundleId(), job.getVersion(), lineNumber+1); refs = adapter.runStatement(c12); assertNotNull(refs); assertEquals(refs.size(), 0); @@ -335,7 +335,7 @@ public void allocateJobsTest() { adapter.runStatement(c13); // Make sure we can find both resources - GetResourceRefsForBundleLine c14 = new GetResourceRefsForBundleLine(job.getResourceBundleId(), job.getVersion(), lastLine+1); + GetResourceRefsForBundleLine c14 = new GetResourceRefsForBundleLine(DATA_SCHEMA_NAME, job.getResourceBundleId(), job.getVersion(), lastLine+1); refs = adapter.runStatement(c14); assertNotNull(refs); assertEquals(refs.size(), 2); diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java index 40afa08129f..05d5a5cee4e 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java @@ -404,4 +404,25 @@ public void dropVariable(String schemaName, String variableName) { final String ddl = "DROP VARIABLE " + nm; warnOnce(MessageKey.DROP_VARIABLE, "Not supported in PostgreSQL: " + ddl); } + + @Override + public void createOrReplaceFunction(String schemaName, String functionName, Supplier supplier) { + // For PostgreSQL, we need to drop the function first to avoid ending up + // with the same function name having different args (non-unique) which + // causes problems later on + final String objectName = DataDefinitionUtil.getQualifiedName(schemaName, functionName); + logger.info("Dropping current function " + objectName); + + final StringBuilder ddl = new StringBuilder() + .append("DROP FUNCTION IF EXISTS ") + .append(objectName); + + final String ddlString = ddl.toString(); + if (logger.isLoggable(Level.FINE)) { + logger.fine(ddlString); + } + runStatement(ddlString); + + super.createOrReplaceFunction(schemaName, functionName, supplier); + } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/Select.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/Select.java index dccecaea44d..f4fad77035d 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/Select.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/Select.java @@ -9,6 +9,8 @@ import static com.ibm.fhir.database.utils.query.SqlConstants.FROM; import static com.ibm.fhir.database.utils.query.SqlConstants.SELECT; import static com.ibm.fhir.database.utils.query.SqlConstants.SPACE; +import static com.ibm.fhir.database.utils.query.SqlConstants.UNION; +import static com.ibm.fhir.database.utils.query.SqlConstants.UNION_ALL; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.derby.DerbyTranslator; @@ -48,6 +50,12 @@ public class Select { // offset/limit for pagination private PaginationClause paginationClause; + // Another Select to UNION with this select. Optional + private Select union; + + // If true, the specified UNION is a UNION ALL + private boolean unionAll = false; + /** * Default constructor. Not a DISTINCT select. */ @@ -218,6 +226,11 @@ public String toString() { result.append(SPACE).append(this.paginationClause.toString()); } + if (this.union != null) { + result.append(SPACE).append(unionAll ? UNION_ALL : UNION).append(SPACE) + .append(this.union.toString()); + } + return result.toString(); } @@ -250,7 +263,8 @@ public String toDebugString() { * @return */ public T render(StatementRenderer renderer) { - return renderer.select(distinct, selectList, fromClause, whereClause, groupByClause, havingClause, orderByClause, paginationClause); + return renderer.select(distinct, selectList, fromClause, whereClause, groupByClause, havingClause, + orderByClause, paginationClause, unionAll, union); } /** @@ -319,4 +333,20 @@ public void addPagination(int offset, int rowsPerPage) { public OrderByClause getOrderByClause() { return this.orderByClause; } + + /** + * Set a select to UNION with this query. + */ + public void setUnion(Select union) { + this.union = union; + this.unionAll = false; + } + + /** + * Set a select to UNION ALL with this query. + */ + public void setUnionAll(Select unionAll) { + this.union = unionAll; + this.unionAll = true; + } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/SelectAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/SelectAdapter.java index 3237893122c..2e62d2b58f7 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/SelectAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/SelectAdapter.java @@ -123,6 +123,26 @@ public void pagination(int offset, int rowsPerPage) { select.addPagination(offset, rowsPerPage); } + /** + * Add a select via UNION + * + * @param unionSelect the select to be UNION'd to this select statement + * @return + */ + public void union(Select unionSelect) { + select.setUnion(unionSelect); + } + + /** + * Add a select via UNION ALL + * + * @param unionAllSelect the select to be UNION ALL'd to this select statement + * @return + */ + public void unionAll(Select unionAllSelect) { + select.setUnionAll(unionAllSelect); + } + /** * Get the statement we've been constructing * diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/SqlConstants.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/SqlConstants.java index ab850d172ac..66ff2f390d0 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/SqlConstants.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/SqlConstants.java @@ -25,4 +25,6 @@ public class SqlConstants { public static final String THEN = "THEN"; public static final String END = "END"; public static final String ORDER_BY = "ORDER BY"; + public static final String UNION = "UNION"; + public static final String UNION_ALL = "UNION ALL"; } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/expression/StatementRenderer.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/expression/StatementRenderer.java index 39deecb9b5b..c439173efe4 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/expression/StatementRenderer.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/expression/StatementRenderer.java @@ -14,6 +14,7 @@ import com.ibm.fhir.database.utils.query.HavingClause; import com.ibm.fhir.database.utils.query.OrderByClause; import com.ibm.fhir.database.utils.query.PaginationClause; +import com.ibm.fhir.database.utils.query.Select; import com.ibm.fhir.database.utils.query.SelectList; import com.ibm.fhir.database.utils.query.WhereClause; import com.ibm.fhir.database.utils.query.node.ExpNode; @@ -36,10 +37,12 @@ public interface StatementRenderer { * @param havingClause * @param orderByClause * @param paginationClause + * @param unionAll + * @param union * @return */ T select(boolean distinct, SelectList selectList, FromClause fromClause, WhereClause whereClause, GroupByClause groupByClause, HavingClause havingClause, - OrderByClause orderByClause, PaginationClause paginationClause); + OrderByClause orderByClause, PaginationClause paginationClause, boolean unionAll, Select union); /** * @param items diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/expression/StringStatementRenderer.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/expression/StringStatementRenderer.java index ed7fb5590f4..1c48a07c18c 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/expression/StringStatementRenderer.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/expression/StringStatementRenderer.java @@ -9,6 +9,8 @@ import static com.ibm.fhir.database.utils.query.SqlConstants.FROM; import static com.ibm.fhir.database.utils.query.SqlConstants.SELECT; import static com.ibm.fhir.database.utils.query.SqlConstants.SPACE; +import static com.ibm.fhir.database.utils.query.SqlConstants.UNION; +import static com.ibm.fhir.database.utils.query.SqlConstants.UNION_ALL; import static com.ibm.fhir.database.utils.query.SqlConstants.WHERE; import java.util.List; @@ -21,6 +23,7 @@ import com.ibm.fhir.database.utils.query.HavingClause; import com.ibm.fhir.database.utils.query.OrderByClause; import com.ibm.fhir.database.utils.query.PaginationClause; +import com.ibm.fhir.database.utils.query.Select; import com.ibm.fhir.database.utils.query.SelectList; import com.ibm.fhir.database.utils.query.WhereClause; import com.ibm.fhir.database.utils.query.node.BindMarkerNode; @@ -59,7 +62,7 @@ public StringStatementRenderer(IDatabaseTranslator translator, List resourceTypes = Arrays.asList("Patient", "Condition", "Observation"); + Select first = null; + Select previous = null; + + // Create a set of selects combined by UNION ALL + for (String resourceType : resourceTypes) { + // Create a simple select statement + Select select = Select.select("1") + .from(resourceType + "_TOKEN_VALUES_V", alias("param")) + .where("param", "PARAMETER_NAME_ID").eq(1274) + .build(); + + // Link to previous select via UNION ALL + if (previous != null) { + previous.setUnionAll(select); + } else { + first = select; + } + previous = select; + } + + // And make sure it renders to the correct string + final String SQL = "SELECT 1" + + " FROM Patient_TOKEN_VALUES_V AS param" + + " WHERE param.PARAMETER_NAME_ID = 1274" + + " UNION ALL" + + " SELECT 1" + + " FROM Condition_TOKEN_VALUES_V AS param" + + " WHERE param.PARAMETER_NAME_ID = 1274" + + " UNION ALL" + + " SELECT 1" + + " FROM Observation_TOKEN_VALUES_V AS param" + + " WHERE param.PARAMETER_NAME_ID = 1274"; + final List bindMarkers = new ArrayList<>(); + StringStatementRenderer renderer = new StringStatementRenderer(TRANSLATOR, bindMarkers, false); + assertEquals(first.render(renderer), SQL); + assertEquals(bindMarkers.size(), 0); + } + } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRPersistenceJDBCCache.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRPersistenceJDBCCache.java index a665255d4ea..c1ffaabcee5 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRPersistenceJDBCCache.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRPersistenceJDBCCache.java @@ -7,6 +7,7 @@ package com.ibm.fhir.persistence.jdbc; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.api.IIdNameCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; /** @@ -34,6 +35,12 @@ public interface FHIRPersistenceJDBCCache { */ INameIdCache getResourceTypeCache(); + /** + * Getter for the cache of resource type ids used to look up resource type name + * @return + */ + IIdNameCache getResourceTypeNameCache(); + /** * Getter for the cache of parameter names * @return diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/CommonTokenValuesCacheImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/CommonTokenValuesCacheImpl.java index f7bc6ac8412..4315489289c 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/CommonTokenValuesCacheImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/CommonTokenValuesCacheImpl.java @@ -13,6 +13,7 @@ import java.util.Map; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceProfileRec; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceTokenValueRec; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; @@ -29,22 +30,30 @@ public class CommonTokenValuesCacheImpl implements ICommonTokenValuesCache { private final ThreadLocal> commonTokenValues = new ThreadLocal<>(); - // The lru cache shared at the server level + // thread-local cache of canonicals + private final ThreadLocal> canonicalValues = new ThreadLocal<>(); + + // The lru code systems cache shared at the server level private final LRUCache codeSystemsCache; - // The lru cache shared at the server level + // The lru token values cache shared at the server level private final LRUCache tokenValuesCache; + // The lru canonical values cache shared at the server level + private final LRUCache canonicalValuesCache; /** * Public constructor - * @param sharedExternalSystemNameCacheSize + * @param codeSystemCacheSize + * @param tokenValueCacheSize + * @param canonicalCacheSize */ - public CommonTokenValuesCacheImpl(int codeSystemCacheSize, int tokenValueCacheSize) { + public CommonTokenValuesCacheImpl(int codeSystemCacheSize, int tokenValueCacheSize, int canonicalCacheSize) { // LRU cache for quick lookup of code-systems and token-values codeSystemsCache = new LRUCache<>(codeSystemCacheSize); tokenValuesCache = new LRUCache<>(tokenValueCacheSize); + canonicalValuesCache = new LRUCache<>(canonicalCacheSize); } /** @@ -73,6 +82,16 @@ public void updateSharedMaps() { valMap.clear(); } + LinkedHashMap canMap = canonicalValues.get(); + if (canMap != null) { + synchronized(this.canonicalValuesCache) { + canonicalValuesCache.update(canMap); + } + + // clear the thread-local cache + canMap.clear(); + } + } @Override @@ -198,6 +217,53 @@ public void resolveTokenValues(Collection tokenValues, } } + @Override + public void resolveCanonicalValues(Collection profileValues, + List misses) { + // Make one pass over the collection and resolve as much as we can in one go. Anything + // we can't resolve gets put into the corresponding missing lists. Worst case is two passes, when + // there's nothing in the local cache and we have to then look up everything in the shared cache + + // See what we have currently in our thread-local cache + LinkedHashMap canMap = canonicalValues.get(); + + List foundKeys = new ArrayList<>(profileValues.size()); // for updating LRU + List needToFind = new ArrayList<>(profileValues.size()); // for the canonical values we haven't yet found + for (ResourceProfileRec tv: profileValues) { + if (canMap != null) { + Integer id = canMap.get(tv.getCanonicalValue()); + if (id != null) { + foundKeys.add(tv.getCanonicalValue()); + tv.setCanonicalValueId(id); + } else { + // not found, so add to the cache miss list + needToFind.add(tv); + } + } else { + // no thread-local cache yet, so need to find them all + needToFind.add(tv); + } + } + + // If we still have keys to find, look them up in the shared cache (which we need to lock first) + if (needToFind.size() > 0) { + synchronized (this.canonicalValuesCache) { + for (ResourceProfileRec xr: needToFind) { + Integer id = canonicalValuesCache.get(xr.getCanonicalValue()); + if (id != null) { + xr.setCanonicalValueId(id); + + // Update the local cache with this value + addCanonicalValue(xr.getCanonicalValue(), id); + } else { + // cache miss so add this record to the miss list for further processing + misses.add(xr); + } + } + } + } + } + @Override public void addCodeSystem(String codeSystem, int id) { @@ -227,10 +293,25 @@ public void addTokenValue(CommonTokenValue key, long id) { map.put(key, id); } + @Override + public void addCanonicalValue(String url, int id) { + LinkedHashMap map = canonicalValues.get(); + + if (map == null) { + map = new LinkedHashMap<>(); + canonicalValues.set(map); + } + + // add the id to the thread-local cache. The shared cache is updated + // only if a call is made to #updateSharedMaps() + map.put(url, id); + } + @Override public void reset() { codeSystems.remove(); commonTokenValues.remove(); + canonicalValues.remove(); // clear the shared caches too synchronized (this.codeSystemsCache) { @@ -240,6 +321,10 @@ public void reset() { synchronized (this.tokenValuesCache) { this.tokenValuesCache.clear(); } + + synchronized (this.canonicalValuesCache) { + this.canonicalValuesCache.clear(); + } } @Override @@ -257,6 +342,12 @@ public void clearLocalMaps() { if (valMap != null) { valMap.clear(); } + + LinkedHashMap canMap = canonicalValues.get(); + + if (canMap != null) { + canMap.clear(); + } } @Override @@ -299,4 +390,25 @@ public Long getCommonTokenValueId(String codeSystem, String tokenValue) { return result; } + + @Override + public Integer getCanonicalId(String canonicalValue) { + Integer result; + + LinkedHashMap valMap = canonicalValues.get(); + result = valMap != null ? valMap.get(canonicalValue) : null; + if (result == null) { + // not found in the local cache, try the shared cache + synchronized (canonicalValuesCache) { + result = canonicalValuesCache.get(canonicalValue); + } + + if (result != null) { + // add to the local cache so we can find it again without locking + addCanonicalValue(canonicalValue, result); + } + } + + return result; + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheImpl.java index d7a6e3e1d2c..4e3a95a903a 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheImpl.java @@ -11,6 +11,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.api.IIdNameCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; /** @@ -21,6 +22,8 @@ public class FHIRPersistenceJDBCCacheImpl implements FHIRPersistenceJDBCCache { private final INameIdCache resourceTypeCache; + private final IIdNameCache resourceTypeNameCache; + private final INameIdCache parameterNameCache; private final ICommonTokenValuesCache resourceReferenceCache; @@ -31,11 +34,14 @@ public class FHIRPersistenceJDBCCacheImpl implements FHIRPersistenceJDBCCache { /** * Public constructor * @param resourceTypeCache + * @param resourceTypeNameCache * @param parameterNameCache * @param resourceReferenceCache */ - public FHIRPersistenceJDBCCacheImpl(INameIdCache resourceTypeCache, INameIdCache parameterNameCache, ICommonTokenValuesCache resourceReferenceCache) { + public FHIRPersistenceJDBCCacheImpl(INameIdCache resourceTypeCache, IIdNameCache resourceTypeNameCache, + INameIdCache parameterNameCache, ICommonTokenValuesCache resourceReferenceCache) { this.resourceTypeCache = resourceTypeCache; + this.resourceTypeNameCache = resourceTypeNameCache; this.parameterNameCache = parameterNameCache; this.resourceReferenceCache = resourceReferenceCache; } @@ -47,11 +53,22 @@ public ICommonTokenValuesCache getResourceReferenceCache() { return resourceReferenceCache; } + /** + * @return the resourceTypeCache + */ @Override public INameIdCache getResourceTypeCache() { return this.resourceTypeCache; } + /** + * @return the resourceTypeNameCache + */ + @Override + public IIdNameCache getResourceTypeNameCache() { + return this.resourceTypeNameCache; + } + /** * @return the parameterNameCache */ @@ -63,6 +80,7 @@ public INameIdCache getParameterNameCache() { public void transactionCommitted() { logger.fine("Transaction committed - updating cache shared maps"); resourceTypeCache.updateSharedMaps(); + resourceTypeNameCache.updateSharedMaps(); parameterNameCache.updateSharedMaps(); resourceReferenceCache.updateSharedMaps(); } @@ -71,6 +89,7 @@ public void transactionCommitted() { public void transactionRolledBack() { logger.fine("Transaction rolled back - clearing local maps"); resourceTypeCache.clearLocalMaps(); + resourceTypeNameCache.clearLocalMaps(); parameterNameCache.clearLocalMaps(); resourceReferenceCache.clearLocalMaps(); } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheUtil.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheUtil.java index d4746f705df..6f8afabee99 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheUtil.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheUtil.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -7,6 +7,7 @@ package com.ibm.fhir.persistence.jdbc.cache; import java.util.Map; +import java.util.stream.Collectors; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; @@ -23,9 +24,9 @@ public class FHIRPersistenceJDBCCacheUtil { * Factory function to create a new cache instance * @return */ - public static FHIRPersistenceJDBCCache create(int codeSystemCacheSize, int tokenValueCacheSize) { - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(codeSystemCacheSize, tokenValueCacheSize); - return new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + public static FHIRPersistenceJDBCCache create(int codeSystemCacheSize, int tokenValueCacheSize, int canonicalCacheSize) { + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(codeSystemCacheSize, tokenValueCacheSize, canonicalCacheSize); + return new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); } /** @@ -36,10 +37,13 @@ public static FHIRPersistenceJDBCCache create(int codeSystemCacheSize, int token public static void prefill(ResourceDAO resourceDAO, ParameterDAO parameterDAO, FHIRPersistenceJDBCCache cache) throws FHIRPersistenceException { Map resourceTypes = resourceDAO.readAllResourceTypeNames(); cache.getResourceTypeCache().prefill(resourceTypes); + + Map resourceTypeNames = resourceTypes.entrySet().stream().collect(Collectors.toMap(map -> map.getValue(), map -> map.getKey())); + cache.getResourceTypeNameCache().prefill(resourceTypeNames); Map parameterNames = parameterDAO.readAllSearchParameterNames(); cache.getParameterNameCache().prefill(parameterNames); - + Map codeSystems = parameterDAO.readAllCodeSystems(); cache.getResourceReferenceCache().prefillCodeSystems(codeSystems); } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCTenantCache.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCTenantCache.java index 6f0c09d3559..fd378abddd5 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCTenantCache.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCTenantCache.java @@ -73,7 +73,8 @@ protected FHIRPersistenceJDBCCache createCache(String cacheKey) { } else { int externalSystemCacheSize = pg.getIntProperty("externalSystemCacheSize", 1000); int externalValueCacheSize = pg.getIntProperty("externalValueCacheSize", 100000); - return FHIRPersistenceJDBCCacheUtil.create(externalSystemCacheSize, externalValueCacheSize); + int canonicalCacheSize = pg.getIntProperty("canonicalCacheSize", 1000); + return FHIRPersistenceJDBCCacheUtil.create(externalSystemCacheSize, externalValueCacheSize, canonicalCacheSize); } } catch (IllegalStateException ise) { throw ise; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/IdNameCache.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/IdNameCache.java new file mode 100644 index 00000000000..fbb9cd526f1 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/IdNameCache.java @@ -0,0 +1,88 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.cache; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import com.ibm.fhir.persistence.jdbc.dao.api.IIdNameCache; + + +/** + * @param the type of the key value held by the cache + */ +public class IdNameCache implements IIdNameCache { + + // The local cache + private final ThreadLocal> local = new ThreadLocal<>(); + + // The cache shared at the server level + private final ConcurrentHashMap shared = new ConcurrentHashMap<>(); + + /** + * Public constructor + */ + public IdNameCache() { + } + + @Override + public void updateSharedMaps() { + + Map localMap = local.get(); + if (localMap != null) { + // no need to synchronize because we can use a ConcurrentHashMap + shared.putAll(localMap); + localMap.clear(); + } + } + + @Override + public String getName(T key) { + String result = null; + Map localMap = local.get(); + if (localMap != null) { + result = localMap.get(key); + } + + if (result == null) { + result = shared.get(key); + } + return result; + } + + @Override + public void addEntry(T id, String name) { + Map localMap = local.get(); + if (localMap == null) { + localMap = new HashMap<>(); + local.set(localMap); + } + localMap.put(id, name); + } + + @Override + public void reset() { + local.remove(); + shared.clear(); + } + + @Override + public void clearLocalMaps() { + Map map = local.get(); + if (map != null) { + map.clear(); + } + } + + @Override + public void prefill(Map content) { + // as the given content is supposed to be already committed in the database, + // we can add it directly to the shared map + this.shared.putAll(content); + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/CreateTempTablesAction.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/CreateTempTablesAction.java index 7da36f36da3..7e474bc367b 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/CreateTempTablesAction.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/CreateTempTablesAction.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -13,6 +13,7 @@ import com.ibm.fhir.database.utils.common.JdbcTarget; import com.ibm.fhir.database.utils.derby.DerbyAdapter; import com.ibm.fhir.database.utils.model.DbType; +import com.ibm.fhir.persistence.jdbc.derby.CreateCanonicalValuesTmp; import com.ibm.fhir.persistence.jdbc.derby.CreateCodeSystemsTmp; import com.ibm.fhir.persistence.jdbc.derby.CreateCommonTokenValuesTmp; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; @@ -52,6 +53,7 @@ public void performOn(FHIRDbFlavor flavor, Connection connection) throws FHIRPer createCodeSystemsTmp(adapter); createCommonTokenValuesTmp(adapter); + createCanonicalValuesTmp(adapter); } // perform next action in the chain @@ -60,7 +62,7 @@ public void performOn(FHIRDbFlavor flavor, Connection connection) throws FHIRPer /** * Create the declared global temporary table COMMON_TOKEN_VALUES_TMP - * @param connection + * @param adapter * @throws FHIRPersistenceDBConnectException */ public void createCommonTokenValuesTmp(DerbyAdapter adapter) throws FHIRPersistenceDBConnectException { @@ -70,11 +72,21 @@ public void createCommonTokenValuesTmp(DerbyAdapter adapter) throws FHIRPersiste /** * Create the declared global temporary table CODE_SYSTEMS_TMP - * @param connection + * @param adapter * @throws FHIRPersistenceDBConnectException */ public void createCodeSystemsTmp(DerbyAdapter adapter) throws FHIRPersistenceDBConnectException { IDatabaseStatement cmd = new CreateCodeSystemsTmp(); adapter.runStatement(cmd); } + + /** + * Create the declared global temporary table COMMON_TOKEN_VALUES_TMP + * @param adapter + * @throws FHIRPersistenceDBConnectException + */ + public void createCanonicalValuesTmp(DerbyAdapter adapter) throws FHIRPersistenceDBConnectException { + IDatabaseStatement cmd = new CreateCanonicalValuesTmp(); + adapter.runStatement(cmd); + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/EraseResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/EraseResourceDAO.java index 61ab01a94b8..8645e74c835 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/EraseResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/EraseResourceDAO.java @@ -23,6 +23,7 @@ import com.ibm.fhir.persistence.jdbc.connection.FHIRDbFlavor; import com.ibm.fhir.persistence.jdbc.dao.api.IResourceReferenceDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceDAOImpl; +import com.ibm.fhir.persistence.jdbc.util.ParameterTableSupport; /** * EraseDAO is the data access layer of the erase operation which executes directly @@ -297,43 +298,10 @@ public void runInDao() throws SQLException { public void deleteFromAllParametersTables(String tablePrefix, long logicalResourceId) throws SQLException { final String method = "deleteFromAllParametersTables"; LOG.entering(CLASSNAME, method); - - // existing resource, so need to delete all its parameters - deleteFromParameterTable(tablePrefix + "_str_values", logicalResourceId); - deleteFromParameterTable(tablePrefix + "_number_values", logicalResourceId); - deleteFromParameterTable(tablePrefix + "_date_values", logicalResourceId); - deleteFromParameterTable(tablePrefix + "_latlng_values", logicalResourceId); - deleteFromParameterTable(tablePrefix + "_resource_token_refs", logicalResourceId); - deleteFromParameterTable(tablePrefix + "_quantity_values", logicalResourceId); - deleteFromParameterTable("str_values", logicalResourceId); - deleteFromParameterTable("date_values", logicalResourceId); - deleteFromParameterTable("resource_token_refs", logicalResourceId); + ParameterTableSupport.deleteFromParameterTables(getConnection(), tablePrefix, logicalResourceId); LOG.exiting(CLASSNAME, method); } - /** - * Delete all parameters for the given logical resource id from the parameters table - * - * @param tableName - * @param logicalResourceId - * @throws SQLException - */ - public void deleteFromParameterTable(String tableName, long logicalResourceId) throws SQLException { - final String DML = "DELETE FROM " + tableName + " WHERE logical_resource_id = ?"; - - try (PreparedStatement stmt = getConnection().prepareStatement(DML)) { - // bind parameters - stmt.setLong(1, logicalResourceId); - int deleted = stmt.executeUpdate(); - if (LOG.isLoggable(Level.FINEST)) { - LOG.finest("Deleted from [" + tableName + "] deleted [" + deleted + "] for logicalResourceId [" + logicalResourceId + "]"); - } - } catch (SQLException x) { - LOG.log(Level.SEVERE, DML, x); - throw translator.translate(x); - } - } - /** * processes the erase * diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/ReindexResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/ReindexResourceDAO.java index 1534e1d5f0b..af48042e009 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/ReindexResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/ReindexResourceDAO.java @@ -33,6 +33,7 @@ import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceDAOImpl; import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValue; import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; +import com.ibm.fhir.persistence.jdbc.util.ParameterTableSupport; /** * DAO used to contain the logic required to reindex a given resource @@ -277,14 +278,7 @@ public void updateParameters(String tablePrefix, List p // no need to close Connection connection = getConnection(); - - // existing resource, so need to delete all its parameters - deleteFromParameterTable(connection, tablePrefix + "_str_values", logicalResourceId); - deleteFromParameterTable(connection, tablePrefix + "_number_values", logicalResourceId); - deleteFromParameterTable(connection, tablePrefix + "_date_values", logicalResourceId); - deleteFromParameterTable(connection, tablePrefix + "_latlng_values", logicalResourceId); - deleteFromParameterTable(connection, tablePrefix + "_resource_token_refs", logicalResourceId); - deleteFromParameterTable(connection, tablePrefix + "_quantity_values", logicalResourceId); + ParameterTableSupport.deleteFromParameterTables(connection, tablePrefix, logicalResourceId); if (parameters != null && !parameters.isEmpty()) { JDBCIdentityCache identityCache = new JDBCIdentityCacheImpl(getCache(), this, parameterDao, getResourceReferenceDAO()); @@ -302,27 +296,4 @@ identityCache, getResourceReferenceDAO(), getTransactionData())) { } logger.exiting(CLASSNAME, METHODNAME); } - - /** - * Delete all parameters for the given resourceId from the parameters table - * @param conn - * @param tableName - * @param logicalResourceId - * @throws SQLException - */ - protected void deleteFromParameterTable(Connection conn, String tableName, long logicalResourceId) throws SQLException { - final String DML = "DELETE FROM " + tableName + " WHERE logical_resource_id = ?"; - - try (PreparedStatement stmt = conn.prepareStatement(DML)) { - // bind parameters - stmt.setLong(1, logicalResourceId); - int deleted = stmt.executeUpdate(); - if (logger.isLoggable(Level.FINEST)) { - logger.finest("Deleted from [" + tableName + "] deleted [" + deleted + "] for logicalResourceId [" + logicalResourceId + "]"); - } - } catch (SQLException x) { - logger.log(Level.SEVERE, DML, x); - throw translator.translate(x); - } - } } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ICommonTokenValuesCache.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ICommonTokenValuesCache.java index bda96adb14b..61294b0613d 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ICommonTokenValuesCache.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ICommonTokenValuesCache.java @@ -10,6 +10,7 @@ import java.util.List; import java.util.Map; +import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceProfileRec; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceTokenValueRec; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; @@ -46,11 +47,19 @@ void resolveCodeSystems(Collection tokenValues, * resolveCodeSystems to make sure we have code-system ids set for each * record. This also means that code-systems which don't yet exist must * be created before this method can be called (because we need the id) - * @param commonTokenValues + * @param tokenValues * @param misses the objects we couldn't find in the cache */ void resolveTokenValues(Collection tokenValues, - List system); + List misses); + + /** + * Look up the ids for the common canonical values in the cache + * @param profileValues the collection of profile values containing the canonical urls + * @param misses the objects we couldn't find in the cache + */ + void resolveCanonicalValues(Collection profileValues, + List misses); /** * Look up the id of the named codeSystem @@ -66,6 +75,13 @@ void resolveTokenValues(Collection tokenValues, */ void addCodeSystem(String codeSystem, int id); + /** + * Add the url-id mapping to the local cache + * @param url + * @param id + */ + void addCanonicalValue(String url, int id); + /** * Add the CommonTokenValue and id to the local cache * @param key @@ -99,4 +115,11 @@ void resolveTokenValues(Collection tokenValues, * @return */ Long getCommonTokenValueId(String codeSystem, String tokenValue); + + /** + * Get the cached database id for the given canonical url + * @param url + * @return + */ + Integer getCanonicalId(String url); } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/IIdNameCache.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/IIdNameCache.java new file mode 100644 index 00000000000..f0c4223af41 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/IIdNameCache.java @@ -0,0 +1,55 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.dao.api; + +import java.util.Map; + +/** + * Interface to a cache mapping an id of type T to a string. Supports + * thread-local caching to support temporary staging of values pending + * successful completion of a transaction. + * @param + */ +public interface IIdNameCache { + + /** + * Get the name for the given id + * @param id + * @return + */ + String getName(T id); + + /** + * Add the entry to the local cache + * @param id + * @param name + */ + void addEntry(T id, String name); + + /** + * Called after a transaction commit() to transfer all the staged (thread-local) data + * over to the shared cache. + */ + void updateSharedMaps(); + + /** + * Clear both local shared caches - useful for unit tests + */ + void reset(); + + /** + * Clear anything cached in thread-local (after transaction rollback, for example) + */ + void clearLocalMaps(); + + /** + * Prefill the shared map with the given content (must come data already + * committed in the database) + * @param content + */ + void prefill(Map content); +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/IResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/IResourceReferenceDAO.java index ead24d3fb46..eed925046b8 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/IResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/IResourceReferenceDAO.java @@ -10,6 +10,7 @@ import java.util.List; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceProfileRec; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceTokenValueRec; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValueResult; @@ -37,14 +38,20 @@ public interface IResourceReferenceDAO { * as necessary * @param resourceType * @param xrefs + * @param profileRecs + * @param tagRecs + * @param securityRecs */ - void addCommonTokenValues(String resourceType, Collection xrefs); + void addNormalizedValues(String resourceType, Collection xrefs, Collection profileRecs, Collection tagRecs, Collection securityRecs); /** * Persist the records, which may span multiple resource types * @param records + * @param profileRecs + * @param tagRecs + * @param securityRecs */ - void persist(Collection records); + void persist(Collection records, Collection profileRecs, Collection tagRecs, Collection securityRecs); /** * Find the database id for the given token value and system @@ -60,4 +67,11 @@ public interface IResourceReferenceDAO { * @return */ List readCommonTokenValueIdList(String tokenValue); + + /** + * Read the database canonical_id for the given value + * @param canonicalValue + * @return + */ + Integer readCanonicalId(String canonicalValue); } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/JDBCIdentityCache.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/JDBCIdentityCache.java index e257e6aff57..05c4625441b 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/JDBCIdentityCache.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/JDBCIdentityCache.java @@ -7,6 +7,7 @@ package com.ibm.fhir.persistence.jdbc.dao.api; import java.util.List; +import java.util.Set; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; @@ -25,6 +26,15 @@ public interface JDBCIdentityCache { */ Integer getResourceTypeId(String resourceType) throws FHIRPersistenceException; + /** + * Get the resource type name for the resourceTypeId. Reads from a cache or database + * if required. + * @param resourceTypeId + * @return + * @throws FHIRPersistenceException + */ + String getResourceTypeName(Integer resourceTypeId) throws FHIRPersistenceException; + /** * Get the database id for the named code-system. Creates new records if necessary * @param codeSystem @@ -33,6 +43,15 @@ public interface JDBCIdentityCache { */ Integer getCodeSystemId(String codeSystem) throws FHIRPersistenceException; + /** + * Get the database id for the given canonical value. Read only. If the value + * does not exist, -1 is returned. + * @param canonicalValue + * @return + * @throws FHIRPersistenceException + */ + Integer getCanonicalId(String canonicalValue) throws FHIRPersistenceException; + /** * Get the database id for the given parameter name. Creates new records if necessary. * @param parameterName @@ -60,4 +79,10 @@ public interface JDBCIdentityCache { * @return */ List getCommonTokenValueIdList(String tokenValue); + + /** + * Get the set of all resource type names. + * @return + */ + Set getResourceTypeNames() throws FHIRPersistenceException; } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceDAO.java index af9fc60eaf2..71defe160db 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceDAO.java @@ -179,6 +179,15 @@ List search(String sqlSelect) */ int searchCount(String sqlSelectCount) throws FHIRPersistenceDataAccessException, FHIRPersistenceDBConnectException; + /** + * Executes the whole-system filter search contained in the passed {@link Select}, using its encapsulated search string and bind variables. + * @param select - Contains a search query and (optionally) bind variables. + * @return Map> A map of FHIR resource type ID to list of logical resource IDs satisfying the passed search. + * @throws FHIRPersistenceDataAccessException + * @throws FHIRPersistenceDBConnectException + */ + Map> searchWholeSystem(Select select) throws FHIRPersistenceDataAccessException, FHIRPersistenceDBConnectException; + /** * Sets the current persistence context * @param context @@ -217,6 +226,7 @@ List search(String sqlSelect) * After insert, the generated primary key is acquired and set in the Resource object. * @param resource A Resource Data Transfer Object * @param parameters A collection of search parameters to be persisted along with the passed Resource + * @param parameterHashB64 Base64 encoded SHA-256 hash of parameters * @param parameterDao The Parameter DAO * @return Resource The Resource DTO * @throws FHIRPersistenceDataAccessException @@ -224,6 +234,6 @@ List search(String sqlSelect) * @throws FHIRPersistenceVersionIdMismatchException * @throws FHIRPersistenceException */ - Resource insert(Resource resource, List parameters, ParameterDAO parameterDao) + Resource insert(Resource resource, List parameters, String parameterHashB64, ParameterDAO parameterDao) throws FHIRPersistenceException; } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/JDBCIdentityCacheImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/JDBCIdentityCacheImpl.java index 51ecc941705..9ac7e578843 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/JDBCIdentityCacheImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/JDBCIdentityCacheImpl.java @@ -7,6 +7,8 @@ package com.ibm.fhir.persistence.jdbc.dao.impl; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; @@ -66,6 +68,31 @@ public Integer getResourceTypeId(String resourceType) throws FHIRPersistenceExce } cache.getResourceTypeCache().addEntry(resourceType, result); + cache.getResourceTypeNameCache().addEntry(result, resourceType); + } + return result; + } + + @Override + public String getResourceTypeName(Integer resourceTypeId) throws FHIRPersistenceException { + String result = cache.getResourceTypeNameCache().getName(resourceTypeId); + if (result == null) { + // try the database instead and just cache all results found in this cache + // and in the resource type cache as well + Map resourceMap = resourceDAO.readAllResourceTypeNames(); + for (Map.Entry entry : resourceMap.entrySet()) { + if (entry.getValue() == resourceTypeId) { + result = entry.getKey(); + } + cache.getResourceTypeNameCache().addEntry(entry.getValue(), entry.getKey()); + cache.getResourceTypeCache().addEntry(entry.getKey(), entry.getValue()); + } + + if (result == null) { + // likely a configuration error, caused by the schema being generated + // for a subset of all possible resource types + throw new FHIRPersistenceDataAccessException("Resource type ID not registered in database: '" + resourceTypeId + "'"); + } } return result; } @@ -94,6 +121,21 @@ public Integer getParameterNameId(String parameterName) throws FHIRPersistenceEx return result; } + @Override + public Integer getCanonicalId(String canonicalValue) throws FHIRPersistenceException { + Integer result = cache.getResourceReferenceCache().getCanonicalId(canonicalValue); + if (result == null) { + result = resourceReferenceDAO.readCanonicalId(canonicalValue); + if (result != null) { + cache.getResourceReferenceCache().addCanonicalValue(canonicalValue, result); + } else { + result = -1; + } + } + + return result; + } + @Override public Long getCommonTokenValueId(String codeSystem, String tokenValue) { Long result = cache.getResourceReferenceCache().getCommonTokenValueId(codeSystem, tokenValue); @@ -122,4 +164,9 @@ public Long getCommonTokenValueId(String codeSystem, String tokenValue) { public List getCommonTokenValueIdList(String tokenValue) { return resourceReferenceDAO.readCommonTokenValueIdList(tokenValue); } + + @Override + public Set getResourceTypeNames() throws FHIRPersistenceException { + return resourceDAO.readAllResourceTypeNames().keySet(); + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java index ebd13960e41..28a6f694dbc 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java @@ -7,6 +7,9 @@ package com.ibm.fhir.persistence.jdbc.dao.impl; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.UTC; +import static com.ibm.fhir.search.SearchConstants.PROFILE; +import static com.ibm.fhir.search.SearchConstants.SECURITY; +import static com.ibm.fhir.search.SearchConstants.TAG; import java.math.BigDecimal; import java.sql.Connection; @@ -35,6 +38,7 @@ import com.ibm.fhir.persistence.jdbc.dto.TokenParmVal; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; +import com.ibm.fhir.persistence.jdbc.util.CanonicalSupport; import com.ibm.fhir.schema.control.FhirSchemaConstants; import com.ibm.fhir.search.util.ReferenceValue; import com.ibm.fhir.search.util.ReferenceValue.ReferenceType; @@ -55,6 +59,9 @@ public class ParameterVisitorBatchDAO implements ExtractedParameterValueVisitor, // the max number of rows we accumulate for a given statement before we submit the batch private final int batchSize; + // Enable the feature to store whole system parameters + private final boolean storeWholeSystemParams = true; + // FK to the logical resource for the parameters being added private final long logicalResourceId; @@ -93,6 +100,15 @@ public class ParameterVisitorBatchDAO implements ExtractedParameterValueVisitor, // Collect a list of token values to process in one go private final List tokenValueRecs = new ArrayList<>(); + // Tags are now stored in their own tables + private final List tagTokenRecs = new ArrayList<>(); + + // Security params are now stored in their own tables + private final List securityTokenRecs = new ArrayList<>(); + + // Profiles are now stored in their own tables + private final List profileRecs = new ArrayList<>(); + // The table prefix (resourceType) private final String tablePrefix; @@ -157,19 +173,24 @@ public ParameterVisitorBatchDAO(Connection c, String adminSchemaName, String tab insertLocation = multitenant ? "INSERT INTO " + tablePrefix + "_latlng_values (mt_id, parameter_name_id, latitude_value, longitude_value, logical_resource_id, composite_id) VALUES (" + adminSchemaName + ".sv_tenant_id,?,?,?,?,?)" : "INSERT INTO " + tablePrefix + "_latlng_values (parameter_name_id, latitude_value, longitude_value, logical_resource_id, composite_id) VALUES (?,?,?,?,?)"; - // System level string attributes - String insertSystemString = multitenant ? - "INSERT INTO str_values (mt_id, parameter_name_id, str_value, str_value_lcase, logical_resource_id) VALUES (" + adminSchemaName + ".sv_tenant_id,?,?,?,?)" - : - "INSERT INTO str_values (parameter_name_id, str_value, str_value_lcase, logical_resource_id) VALUES (?,?,?,?)"; - systemStrings = c.prepareStatement(insertSystemString); - - // System level date attributes - String insertSystemDate = multitenant ? - "INSERT INTO date_values (mt_id, parameter_name_id, date_start, date_end, logical_resource_id) VALUES (" + adminSchemaName + ".sv_tenant_id,?,?,?,?)" - : - "INSERT INTO date_values (parameter_name_id, date_start, date_end, logical_resource_id) VALUES (?,?,?,?)"; - systemDates = c.prepareStatement(insertSystemDate); + if (storeWholeSystemParams) { + // System level string attributes + String insertSystemString = multitenant ? + "INSERT INTO str_values (mt_id, parameter_name_id, str_value, str_value_lcase, logical_resource_id) VALUES (" + adminSchemaName + ".sv_tenant_id,?,?,?,?)" + : + "INSERT INTO str_values (parameter_name_id, str_value, str_value_lcase, logical_resource_id) VALUES (?,?,?,?)"; + systemStrings = c.prepareStatement(insertSystemString); + + // System level date attributes + String insertSystemDate = multitenant ? + "INSERT INTO date_values (mt_id, parameter_name_id, date_start, date_end, logical_resource_id) VALUES (" + adminSchemaName + ".sv_tenant_id,?,?,?,?)" + : + "INSERT INTO date_values (parameter_name_id, date_start, date_end, logical_resource_id) VALUES (?,?,?,?)"; + systemDates = c.prepareStatement(insertSystemDate); + } else { + systemStrings = null; + systemDates = null; + } } /** @@ -195,6 +216,12 @@ public void visit(StringParmVal param) throws FHIRPersistenceException { String parameterName = param.getName(); String value = param.getValueString(); + if (PROFILE.equals(parameterName)) { + // profile canonicals are now stored in their own tables. + processProfile(param); + return; + } + while (value != null && value.getBytes().length > FhirSchemaConstants.MAX_SEARCH_STRING_BYTES) { // keep chopping the string in half until its byte representation fits inside // the VARCHAR @@ -203,7 +230,7 @@ public void visit(StringParmVal param) throws FHIRPersistenceException { try { int parameterNameId = getParameterNameId(parameterName); - if (isBase(param)) { + if (storeWholeSystem(param)) { if (logger.isLoggable(Level.FINE)) { logger.fine("systemStringValue: " + parameterName + "[" + parameterNameId + "], " + value); } @@ -224,19 +251,19 @@ public void visit(StringParmVal param) throws FHIRPersistenceException { systemStrings.executeBatch(); systemStringCount = 0; } - } else { - // standard resource property - if (logger.isLoggable(Level.FINE)) { - logger.fine("stringValue: " + parameterName + "[" + parameterNameId + "], " + value); - } + } - setStringParms(strings, parameterNameId, value); - strings.addBatch(); + // always store at the resource-specific level + if (logger.isLoggable(Level.FINE)) { + logger.fine("stringValue: " + parameterName + "[" + parameterNameId + "], " + value); + } - if (++stringCount == this.batchSize) { - strings.executeBatch(); - stringCount = 0; - } + setStringParms(strings, parameterNameId, value); + strings.addBatch(); + + if (++stringCount == this.batchSize) { + strings.executeBatch(); + stringCount = 0; } } catch (SQLException x) { throw new FHIRPersistenceDataAccessException(parameterName + "=" + value, x); @@ -256,6 +283,25 @@ private void setStringParms(PreparedStatement insert, int parameterNameId, Strin setCompositeId(insert, 5); } + /** + * Special case to store profile strings as canonical uri values in their own table + * @param param + * @throws FHIRPersistenceException + */ + private void processProfile(StringParmVal param) throws FHIRPersistenceException { + final String parameterName = param.getName(); + final int parameterNameId = getParameterNameId(parameterName); + final int resourceTypeId = identityCache.getResourceTypeId(param.getResourceType()); + + // Parse the parameter value to extract the URI|VERSION#FRAGMENT pieces + ResourceProfileRec rec = CanonicalSupport.makeResourceProfileRec(parameterNameId, param.getResourceType(), resourceTypeId, this.logicalResourceId, param.getValueString(), param.isWholeSystem()); + if (transactionData != null) { + transactionData.addValue(rec); + } else { + profileRecs.add(rec); + } + } + /** * Set the composite_id column value or null if required * @param ps the statement @@ -278,7 +324,7 @@ public void visit(NumberParmVal param) throws FHIRPersistenceException { BigDecimal valueHigh = param.getValueNumberHigh(); // System-level number search parameters are not supported - if (isBase(param)) { + if (storeWholeSystem(param)) { String msg = "System-level number search parameters are not supported: " + parameterName; logger.warning(msg); throw new IllegalArgumentException(msg); @@ -322,7 +368,7 @@ public void visit(DateParmVal param) throws FHIRPersistenceException { try { int parameterNameId = getParameterNameId(parameterName); - if (isBase(param)) { + if (storeWholeSystem(param)) { // store as a system level search param if (logger.isLoggable(Level.FINE)) { logger.fine("systemDateValue: " + parameterName + "[" + parameterNameId + "], " @@ -337,19 +383,20 @@ public void visit(DateParmVal param) throws FHIRPersistenceException { systemDates.executeBatch(); systemDateCount = 0; } - } else { - if (logger.isLoggable(Level.FINE)) { - logger.fine("dateValue: " + parameterName + "[" + parameterNameId + "], " - + "period: [" + dateStart + ", " + dateEnd + "]"); - } + } + + // always store the param at the resource-specific level + if (logger.isLoggable(Level.FINE)) { + logger.fine("dateValue: " + parameterName + "[" + parameterNameId + "], " + + "period: [" + dateStart + ", " + dateEnd + "]"); + } - setDateParms(dates, parameterNameId, dateStart, dateEnd); - dates.addBatch(); + setDateParms(dates, parameterNameId, dateStart, dateEnd); + dates.addBatch(); - if (++dateCount == this.batchSize) { - dates.executeBatch(); - dateCount = 0; - } + if (++dateCount == this.batchSize) { + dates.executeBatch(); + dateCount = 0; } } catch (SQLException x) { throw new FHIRPersistenceDataAccessException(parameterName + "={" + dateStart + ", " + dateEnd + "}", x); @@ -373,7 +420,7 @@ public void visit(TokenParmVal param) throws FHIRPersistenceException { try { int parameterNameId = getParameterNameId(parameterName); - boolean isSystemParam = isBase(param); + boolean isSystemParam = storeWholeSystem(param); if (logger.isLoggable(Level.FINE)) { logger.fine("tokenValue: " + parameterName + "[" + parameterNameId + "], " + codeSystem + ", " + tokenValue); @@ -387,10 +434,30 @@ public void visit(TokenParmVal param) throws FHIRPersistenceException { // Issue 1683, for composites we now also record the current composite id (can be null) ResourceTokenValueRec rec = new ResourceTokenValueRec(parameterNameId, param.getResourceType(), resourceTypeId, logicalResourceId, codeSystem, tokenValue, this.currentCompositeId, isSystemParam); - if (this.transactionData != null) { - this.transactionData.addValue(rec); + if (TAG.equals(parameterName)) { + // tag search params are often low-selectivity (many resources sharing the same value) so + // we put them into their own tables to allow better cardinality estimation by the query + // optimizer + if (this.transactionData != null) { + this.transactionData.addTagValue(rec); + } else { + this.tagTokenRecs.add(rec); + } + } else if (SECURITY.equals(parameterName)) { + // search search params are often low-selectivity (many resources sharing the same value) so + // we put them into their own tables to allow better cardinality estimation by the query + // optimizer + if (this.transactionData != null) { + this.transactionData.addSecurityValue(rec); + } else { + this.securityTokenRecs.add(rec); + } } else { - this.tokenValueRecs.add(rec); + if (this.transactionData != null) { + this.transactionData.addValue(rec); + } else { + this.tokenValueRecs.add(rec); + } } } catch (FHIRPersistenceDataAccessException x) { throw new FHIRPersistenceDataAccessException(parameterName + "=" + codeSystem + ":" + tokenValue, x); @@ -407,7 +474,7 @@ public void visit(QuantityParmVal param) throws FHIRPersistenceException { BigDecimal quantityHigh = param.getValueNumberHigh(); // System-level quantity search parameters are not supported - if (isBase(param)) { + if (storeWholeSystem(param)) { String msg = "System-level quantity search parameters are not supported: " + parameterName; logger.warning(msg); throw new IllegalArgumentException(msg); @@ -464,7 +531,7 @@ public void visit(LocationParmVal param) throws FHIRPersistenceException { double lng = param.getValueLongitude(); // System-level location search parameters are not supported - if (isBase(param)) { + if (storeWholeSystem(param)) { String msg = "System-level location search parameters are not supported: " + parameterName; logger.warning(msg); throw new IllegalArgumentException(msg); @@ -529,7 +596,6 @@ public void close() throws Exception { dateCount = 0; } - if (quantityCount > 0) { quantities.executeBatch(); quantityCount = 0; @@ -555,9 +621,9 @@ public void close() throws Exception { } } - // Process any tokens and references we've collected along the way - if (!tokenValueRecs.isEmpty()) { - this.resourceReferenceDAO.addCommonTokenValues(this.tablePrefix, tokenValueRecs); + if (this.transactionData == null) { + // Not using transaction data, so we need to process collected values right here + this.resourceReferenceDAO.addNormalizedValues(this.tablePrefix, tokenValueRecs, profileRecs, tagTokenRecs, securityTokenRecs); } closeStatement(strings); @@ -580,8 +646,13 @@ private void closeStatement(PreparedStatement ps) { } } - private boolean isBase(ExtractedParameterValue param) { - return "Resource".equals(param.getBase()); + /** + * Should we store this parameter also at the whole-system search level? + * @param param + * @return + */ + private boolean storeWholeSystem(ExtractedParameterValue param) { + return storeWholeSystemParams && param.isWholeSystem(); } @Override diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java index 92ce5ff4a10..37fe1e5de50 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java @@ -90,11 +90,11 @@ public class ResourceDAOImpl extends FHIRDbDAOImpl implements ResourceDAO { "LR.LOGICAL_ID = ? AND R.LOGICAL_RESOURCE_ID = LR.LOGICAL_RESOURCE_ID AND R.VERSION_ID = ?"; // @formatter:off - // 0 - // 1 2 3 4 5 6 7 8 + // 0 1 + // 1 2 3 4 5 6 7 8 9 0 // @formatter:on // Don't forget that we must account for IN and OUT parameters. - private static final String SQL_INSERT_WITH_PARAMETERS = "CALL %s.add_any_resource(?,?,?,?,?,?,?,?)"; + private static final String SQL_INSERT_WITH_PARAMETERS = "CALL %s.add_any_resource(?,?,?,?,?,?,?,?,?,?)"; // Read version history of the resource identified by its logical-id private static final String SQL_HISTORY = @@ -539,7 +539,7 @@ protected Integer getResourceTypeIdFromCaches(String resourceType) { } @Override - public Resource insert(Resource resource, List parameters, ParameterDAO parameterDao) + public Resource insert(Resource resource, List parameters, String parameterHashB64, ParameterDAO parameterDao) throws FHIRPersistenceException { final String METHODNAME = "insert(Resource, List"; log.entering(CLASSNAME, METHODNAME); @@ -585,15 +585,18 @@ public Resource insert(Resource resource, List paramete stmt.setTimestamp(4, lastUpdated, UTC); stmt.setString(5, resource.isDeleted() ? "Y": "N"); stmt.setInt(6, resource.getVersionId()); - stmt.registerOutParameter(7, Types.BIGINT); - stmt.registerOutParameter(8, Types.BIGINT); + stmt.setString(7, parameterHashB64); + stmt.registerOutParameter(8, Types.BIGINT); // logical_resource_id + stmt.registerOutParameter(9, Types.BIGINT); // resource_id + stmt.registerOutParameter(10, Types.VARCHAR); // current_hash stmt.execute(); long latestTime = System.nanoTime(); double dbCallDuration = (latestTime-dbCallStartTime)/1e6; - resource.setId(stmt.getLong(7)); - long versionedResourceRowId = stmt.getLong(8); + resource.setId(stmt.getLong(8)); + long versionedResourceRowId = stmt.getLong(9); + String currentHash = stmt.getString(10); if (large) { String largeStmtString = String.format(LARGE_BLOB, resource.getResourceType()); try (PreparedStatement ps = connection.prepareStatement(largeStmtString)) { @@ -613,8 +616,10 @@ public Resource insert(Resource resource, List paramete // Parameter time // TODO FHIR_ADMIN schema name needs to come from the configuration/context + // We can skip the parameter insert if we've been given parameterHashB64 and + // it matches the current value just returned by the stored procedure call long paramInsertStartTime = latestTime; - if (parameters != null) { + if (parameters != null && (parameterHashB64 == null || !parameterHashB64.equals(currentHash))) { JDBCIdentityCache identityCache = new JDBCIdentityCacheImpl(cache, this, parameterDao, getResourceReferenceDAO()); try (ParameterVisitorBatchDAO pvd = new ParameterVisitorBatchDAO(connection, "FHIR_ADMIN", resource.getResourceType(), true, resource.getId(), 100, identityCache, resourceReferenceDAO, this.transactionData)) { @@ -813,5 +818,58 @@ public List searchForIds(Select dataQuery) throws FHIRPersistenceDataAcces log.exiting(CLASSNAME, METHODNAME); } return resourceIds; - } + } + + /** + * Delete all parameters for the given resourceId from the named parameter value table + * @param conn + * @param tableName + * @param logicalResourceId + * @throws SQLException + */ + protected void deleteFromParameterTable(Connection conn, String tableName, long logicalResourceId) throws SQLException { + final String delStrValues = "DELETE FROM " + tableName + " WHERE logical_resource_id = ?"; + try (PreparedStatement stmt = conn.prepareStatement(delStrValues)) { + // bind parameters + stmt.setLong(1, logicalResourceId); + stmt.executeUpdate(); + } + } + + @Override + public Map> searchWholeSystem(Select wholeSystemQuery) throws FHIRPersistenceDataAccessException, + FHIRPersistenceDBConnectException { + final String METHODNAME = "searchWholeSystem"; + log.entering(CLASSNAME, METHODNAME); + + Map> resultMap = new HashMap<>(); + Connection connection = getConnection(); // do not close + ResultSet resultSet = null; + long dbCallStartTime; + double dbCallDuration; + + try (PreparedStatement stmt = QueryUtil.prepareSelect(connection, wholeSystemQuery, getTranslator())) { + dbCallStartTime = System.nanoTime(); + resultSet = stmt.executeQuery(); + dbCallDuration = (System.nanoTime() - dbCallStartTime) / 1e6; + if (log.isLoggable(Level.FINE)) { + log.fine("Successfully retrieved logical resource Ids [took " + dbCallDuration + " ms]"); + } + + // Transform the resultSet into a map of resource type IDs to logical resource IDs + while (resultSet.next()) { + Integer resourceTypeId = resultSet.getInt(1); + Long logicalResourceId = resultSet.getLong(2); + resultMap.computeIfAbsent(resourceTypeId, k -> new ArrayList<>()).add(logicalResourceId); + } + } catch (Throwable e) { + FHIRPersistenceDataAccessException fx = new FHIRPersistenceDataAccessException("Failure retrieving logical resource Ids"); + final String errMsg = "Failure retrieving logical resource Ids. SqlQueryData=" + wholeSystemQuery.toDebugString(); + throw severe(log, fx, errMsg, e); + } finally { + log.exiting(CLASSNAME, METHODNAME); + } + + return resultMap; + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceProfileRec.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceProfileRec.java new file mode 100644 index 00000000000..7e52329aa91 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceProfileRec.java @@ -0,0 +1,96 @@ +/* + * (C) Copyright IBM Corp. 2020, 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.dao.impl; + + +/** + * A DTO representing a mapping of a resource and token value. The + * record is used to drive the population of the CODE_SYSTEMS, COMMON_TOKEN_VALUES + * TOKEN_VALUES_MAP and _TOKEN_VALUES_MAP tables. + */ +public class ResourceProfileRec extends ResourceRefRec { + + // The external system name and its normalized database id (when we have it) + private final String canonicalValue; + private int canonicalValueId = -1; + + // The optional version value of the canonical uri + private final String version; + + // The optional fragment of the canonical uri + private final String fragment; + + // Is this a system-level search param? + private final boolean systemLevel; + + /** + * Public constructor + * @param parameterNameId + * @param resourceType + * @param resourceTypeId + * @param logicalResourceId + * @param canonicalValue + * @param version + * @param fragment + * @param systemLevel + */ + public ResourceProfileRec(int parameterNameId, String resourceType, long resourceTypeId, long logicalResourceId, + String canonicalValue, String version, String fragment, boolean systemLevel) { + super(parameterNameId, resourceType, resourceTypeId, logicalResourceId); + this.canonicalValue = canonicalValue; + this.systemLevel = systemLevel; + this.version = version; + this.fragment = fragment; + } + + /** + * @return the codeSystemValue + */ + public String getCanonicalValue() { + return canonicalValue; + } + + /** + * Getter for the database id + * @return + */ + public int getCanonicalValueId() { + return this.canonicalValueId; + } + + /** + * Sets the database id for the canonicalValue record. + * @param canonicalValueId to set + */ + public void setCanonicalValueId(int canonicalValueId) { + // because we're setting this, it can no longer be null + this.canonicalValueId = canonicalValueId; + } + + /** + * @return the systemLevel + */ + public boolean isSystemLevel() { + return systemLevel; + } + + /** + * Get the optional string version of the canonical uri + * @return + */ + public String getVersion() { + return this.version; + } + + /** + * Get the optional string fragment of the canonical uri + * @return + */ + public String getFragment() { + return this.fragment; + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java index 278be66496a..f08aad5a7f3 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java @@ -153,19 +153,33 @@ public CommonTokenValueResult readCommonTokenValueId(String codeSystem, String t } @Override - public void addCommonTokenValues(String resourceType, Collection xrefs) { - // Grab the ids for all the code-systems, and upsert any misses - List systemMisses = new ArrayList<>(); - cache.resolveCodeSystems(xrefs, systemMisses); - upsertCodeSystems(systemMisses); + public Integer readCanonicalId(String canonicalValue) { + Integer result; + final String SQL = "" + + "SELECT canonical_id " + + " FROM common_canonical_values " + + " WHERE url = ? "; + try (PreparedStatement ps = connection.prepareStatement(SQL)) { + ps.setString(1, canonicalValue); + ResultSet rs = ps.executeQuery(); + if (rs.next()) { + result = rs.getInt(1); + } else { + result = null; + } + } catch (SQLException x) { + logger.log(Level.SEVERE, SQL, x); + throw translator.translate(x); + } - // Now that all the code-systems ids are known, we can search the cache - // for all the token values, upserting anything new - List valueMisses = new ArrayList<>(); - cache.resolveTokenValues(xrefs, valueMisses); - upsertCommonTokenValues(valueMisses); + return result; + } - insertResourceTokenRefs(resourceType, xrefs); + @Override + public void addNormalizedValues(String resourceType, Collection xrefs, Collection profileRecs, Collection tagRecs, Collection securityRecs) { + // This method is only called when we're not using transaction data + logger.fine("Persist parameters for this resource - no transaction data available"); + persist(xrefs, profileRecs, tagRecs, securityRecs); } /** @@ -225,6 +239,58 @@ protected void insertResourceTokenRefs(String resourceType, Collection xrefs) { + // Now all the values should have ids assigned so we can go ahead and insert them + // as a batch + final String tableName = "RESOURCE_TOKEN_REFS"; + DataDefinitionUtil.assertValidName(tableName); + final String insert = "INSERT INTO " + tableName + "(" + + "parameter_name_id, logical_resource_id, common_token_value_id, ref_version_id) " + + "VALUES (?, ?, ?, ?)"; + try (PreparedStatement ps = connection.prepareStatement(insert)) { + int count = 0; + for (ResourceTokenValueRec xr: xrefs) { + if (xr.isSystemLevel()) { + ps.setInt(1, xr.getParameterNameId()); + ps.setLong(2, xr.getLogicalResourceId()); + + // common token value can be null + if (xr.getCommonTokenValueId() != null) { + ps.setLong(3, xr.getCommonTokenValueId()); + } else { + ps.setNull(3, Types.BIGINT); + } + + // version can be null + if (xr.getRefVersionId() != null) { + ps.setInt(4, xr.getRefVersionId()); + } else { + ps.setNull(4, Types.INTEGER); + } + + ps.addBatch(); + if (++count == BATCH_SIZE) { + ps.executeBatch(); + count = 0; + } + } + } + + if (count > 0) { + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, insert, x); + throw translator.translate(x); + } + } + + /** * Add all the systems we currently don't have in the database. If all target * databases handled MERGE properly this would be easy, but they don't so @@ -296,13 +362,313 @@ public void upsertCodeSystems(List systems) { } } + /** + * Add the missing values to the database (and get ids allocated) + * @param profileValues + */ + public void upsertCanonicalValues(List profileValues) { + if (profileValues.isEmpty()) { + return; + } + + // Unique list so we don't try and create the same name more than once + Set names = profileValues.stream().map(xr -> xr.getCanonicalValue()).collect(Collectors.toSet()); + StringBuilder paramList = new StringBuilder(); + StringBuilder inList = new StringBuilder(); + for (int i=0; i 0) { + paramList.append(", "); + inList.append(","); + } + paramList.append("(CAST(? AS VARCHAR(" + FhirSchemaConstants.CANONICAL_URL_BYTES + ")))"); + inList.append("?"); + } + + final String paramListStr = paramList.toString(); + doCanonicalValuesUpsert(paramListStr, names); + + + // Now grab the ids for the rows we just created. If we had a RETURNING implementation + // which worked reliably across all our database platforms, we wouldn't need this + // second query. + StringBuilder select = new StringBuilder(); + select.append("SELECT url, canonical_id FROM common_canonical_values WHERE url IN ("); + select.append(inList); + select.append(")"); + + Map idMap = new HashMap<>(); + try (PreparedStatement ps = connection.prepareStatement(select.toString())) { + // load a map with all the ids we need which we can then use to update the + // ExternalResourceReferenceRec objects + int a = 1; + for (String name: names) { + ps.setString(a++, name); + } + + ResultSet rs = ps.executeQuery(); + while (rs.next()) { + idMap.put(rs.getString(1), rs.getInt(2)); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, select.toString(), x); + throw translator.translate(x); + } + + // Now update the ids for all the matching systems in our list + for (ResourceProfileRec xr: profileValues) { + Integer id = idMap.get(xr.getCanonicalValue()); + if (id != null) { + xr.setCanonicalValueId(id); + + // Add this value to the (thread-local) cache + cache.addCanonicalValue(xr.getCanonicalValue(), id); + } else { + // Unlikely...but need to handle just in case + logger.severe("Record for common_canonical_value '" + xr.getCanonicalValue() + "' inserted but not found"); + throw new IllegalStateException("id deleted from database!"); + } + } + } + + protected void insertResourceProfiles(String resourceType, Collection profiles) { + // Now all the values should have ids assigned so we can go ahead and insert them + // as a batch + final String tableName = resourceType + "_PROFILES"; + DataDefinitionUtil.assertValidName(tableName); + final String insert = "INSERT INTO " + tableName + "(" + + "logical_resource_id, canonical_id, version, fragment) " + + "VALUES (?, ?, ?, ?)"; + try (PreparedStatement ps = connection.prepareStatement(insert)) { + int count = 0; + for (ResourceProfileRec xr: profiles) { + ps.setLong(1, xr.getLogicalResourceId()); + ps.setInt(2, xr.getCanonicalValueId()); + + // canonical version can be null + if (xr.getVersion() != null) { + ps.setString(3, xr.getVersion()); + } else { + ps.setNull(3, Types.VARCHAR); + } + + // canonical fragment can be null + if (xr.getFragment() != null) { + ps.setString(4, xr.getFragment()); + } else { + ps.setNull(4, Types.VARCHAR); + } + ps.addBatch(); + if (++count == BATCH_SIZE) { + ps.executeBatch(); + count = 0; + } + } + + if (count > 0) { + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, insert, x); + throw translator.translate(x); + } + } + + /** + * Insert PROFILE parameters + * @param resourceType + * @param profiles + */ + protected void insertSystemResourceProfiles(String resourceType, Collection profiles) { + final String tableName = "LOGICAL_RESOURCE_PROFILES"; + DataDefinitionUtil.assertValidName(tableName); + final String insert = "INSERT INTO " + tableName + "(" + + "logical_resource_id, canonical_id, version, fragment) " + + "VALUES (?, ?, ?, ?)"; + try (PreparedStatement ps = connection.prepareStatement(insert)) { + int count = 0; + for (ResourceProfileRec xr: profiles) { + ps.setLong(1, xr.getLogicalResourceId()); + ps.setInt(2, xr.getCanonicalValueId()); + + // canonical version can be null + if (xr.getVersion() != null) { + ps.setString(3, xr.getVersion()); + } else { + ps.setNull(3, Types.VARCHAR); + } + + // canonical fragment can be null + if (xr.getFragment() != null) { + ps.setString(4, xr.getFragment()); + } else { + ps.setNull(4, Types.VARCHAR); + } + ps.addBatch(); + if (++count == BATCH_SIZE) { + ps.executeBatch(); + count = 0; + } + } + + if (count > 0) { + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, insert, x); + throw translator.translate(x); + } + } + + /** + * Insert the tags referenced by the given collection of token value records + * @param resourceType + * @param xrefs + */ + protected void insertResourceTags(String resourceType, Collection xrefs) { + // Now all the values should have ids assigned so we can go ahead and insert them + // as a batch + final String tableName = resourceType + "_TAGS"; + DataDefinitionUtil.assertValidName(tableName); + final String insert = "INSERT INTO " + tableName + "(" + + "logical_resource_id, common_token_value_id) " + + "VALUES (?, ?)"; + try (PreparedStatement ps = connection.prepareStatement(insert)) { + int count = 0; + for (ResourceTokenValueRec xr: xrefs) { + ps.setLong(1, xr.getLogicalResourceId()); + ps.setLong(2, xr.getCommonTokenValueId()); + ps.addBatch(); + if (++count == BATCH_SIZE) { + ps.executeBatch(); + count = 0; + } + } + + if (count > 0) { + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, insert, x); + throw translator.translate(x); + } + } + + /** + * Insert _tag parameters to the whole-system LOGICAL_RESOURCE_TAGS table + * @param resourceType + * @param xrefs + */ + protected void insertSystemResourceTags(String resourceType, Collection xrefs) { + // Now all the values should have ids assigned so we can go ahead and insert them + // as a batch + final String tableName = "LOGICAL_RESOURCE_TAGS"; + DataDefinitionUtil.assertValidName(tableName); + final String insert = "INSERT INTO " + tableName + "(" + + "logical_resource_id, common_token_value_id) " + + "VALUES (?, ?)"; + try (PreparedStatement ps = connection.prepareStatement(insert)) { + int count = 0; + for (ResourceTokenValueRec xr: xrefs) { + ps.setLong(1, xr.getLogicalResourceId()); + ps.setLong(2, xr.getCommonTokenValueId()); + ps.addBatch(); + if (++count == BATCH_SIZE) { + ps.executeBatch(); + count = 0; + } + } + + if (count > 0) { + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, insert, x); + throw translator.translate(x); + } + } + + /** + * Insert _security parameters to the resource-specific xx_SECURITY table + * @param resourceType + * @param xrefs + */ + protected void insertResourceSecurity(String resourceType, Collection xrefs) { + // Now all the values should have ids assigned so we can go ahead and insert them + // as a batch + final String tableName = resourceType + "_SECURITY"; + DataDefinitionUtil.assertValidName(tableName); + final String insert = "INSERT INTO " + tableName + "(" + + "logical_resource_id, common_token_value_id) " + + "VALUES (?, ?)"; + try (PreparedStatement ps = connection.prepareStatement(insert)) { + int count = 0; + for (ResourceTokenValueRec xr: xrefs) { + ps.setLong(1, xr.getLogicalResourceId()); + ps.setLong(2, xr.getCommonTokenValueId()); + ps.addBatch(); + if (++count == BATCH_SIZE) { + ps.executeBatch(); + count = 0; + } + } + + if (count > 0) { + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, insert, x); + throw translator.translate(x); + } + } + + /** + * Insert _security parametes to the whole-system LOGICAL_REOURCE_SECURITY table + * @param resourceType + * @param xrefs + */ + protected void insertSystemResourceSecurity(String resourceType, Collection xrefs) { + // Now all the values should have ids assigned so we can go ahead and insert them + // as a batch + final String tableName = "LOGICAL_RESOURCE_SECURITY"; + DataDefinitionUtil.assertValidName(tableName); + final String insert = "INSERT INTO " + tableName + "(" + + "logical_resource_id, common_token_value_id) " + + "VALUES (?, ?)"; + try (PreparedStatement ps = connection.prepareStatement(insert)) { + int count = 0; + for (ResourceTokenValueRec xr: xrefs) { + ps.setLong(1, xr.getLogicalResourceId()); + ps.setLong(2, xr.getCommonTokenValueId()); + ps.addBatch(); + if (++count == BATCH_SIZE) { + ps.executeBatch(); + count = 0; + } + } + + if (count > 0) { + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, insert, x); + throw translator.translate(x); + } + } + /** * Insert any missing values into the code_systems table * @param paramList - * @param systems + * @param systemNames */ public abstract void doCodeSystemsUpsert(String paramList, Collection systemNames); + /** + * Insert any missing values into the common_canonical_values table + * @param paramList + * @param urls + */ + public abstract void doCanonicalValuesUpsert(String paramList, Collection urls); + /** * Add reference value records for each unique reference name in the given list * @param values @@ -403,19 +769,28 @@ public void upsertCommonTokenValues(List values) { protected abstract void doCommonTokenValuesUpsert(String paramList, Collection tokenValues); @Override - public void persist(Collection records) { + public void persist(Collection records, Collection profileRecs, Collection tagRecs, Collection securityRecs) { // Grab the ids for all the code-systems, and upsert any misses List systemMisses = new ArrayList<>(); cache.resolveCodeSystems(records, systemMisses); + cache.resolveCodeSystems(tagRecs, systemMisses); + cache.resolveCodeSystems(securityRecs, systemMisses); upsertCodeSystems(systemMisses); // Now that all the code-systems ids are known, we can search the cache // for all the token values, upserting anything new List valueMisses = new ArrayList<>(); cache.resolveTokenValues(records, valueMisses); + cache.resolveTokenValues(tagRecs, valueMisses); + cache.resolveTokenValues(securityRecs, valueMisses); upsertCommonTokenValues(valueMisses); - // Now split the records into groups based on resource type. + // Process all the common canonical values + List canonicalMisses = new ArrayList<>(); + cache.resolveCanonicalValues(profileRecs, canonicalMisses); + upsertCanonicalValues(canonicalMisses); + + // Now split the token-value records into groups based on resource type. Map> recordMap = new HashMap<>(); for (ResourceTokenValueRec rtv: records) { List list = recordMap.computeIfAbsent(rtv.getResourceType(), k -> { return new ArrayList<>(); }); @@ -424,6 +799,43 @@ public void persist(Collection records) { for (Map.Entry> entry: recordMap.entrySet()) { insertResourceTokenRefs(entry.getKey(), entry.getValue()); + insertSystemResourceTokenRefs(entry.getKey(), entry.getValue()); + } + + // Split profile values by resource type + Map> profileMap = new HashMap<>(); + for (ResourceProfileRec rtv: profileRecs) { + List list = profileMap.computeIfAbsent(rtv.getResourceType(), k -> { return new ArrayList<>(); }); + list.add(rtv); + } + + for (Map.Entry> entry: profileMap.entrySet()) { + insertResourceProfiles(entry.getKey(), entry.getValue()); + insertSystemResourceProfiles(entry.getKey(), entry.getValue()); + } + + // Split tag records by resource type + Map> tagMap = new HashMap<>(); + for (ResourceTokenValueRec rtv: tagRecs) { + List list = tagMap.computeIfAbsent(rtv.getResourceType(), k -> { return new ArrayList<>(); }); + list.add(rtv); + } + + for (Map.Entry> entry: tagMap.entrySet()) { + insertResourceTags(entry.getKey(), entry.getValue()); + insertSystemResourceTags(entry.getKey(), entry.getValue()); + } + + // Split security records by resource type + Map> securityMap = new HashMap<>(); + for (ResourceTokenValueRec rtv: securityRecs) { + List list = securityMap.computeIfAbsent(rtv.getResourceType(), k -> { return new ArrayList<>(); }); + list.add(rtv); + } + + for (Map.Entry> entry: securityMap.entrySet()) { + insertResourceSecurity(entry.getKey(), entry.getValue()); + insertSystemResourceSecurity(entry.getKey(), entry.getValue()); } } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java index 40fe93f5b42..bf3054fc930 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java @@ -19,6 +19,7 @@ import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceProfileRec; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceTokenValueRec; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; @@ -86,6 +87,43 @@ public void doCodeSystemsUpsert(String paramList, Collection systemNames } } + @Override + public void doCanonicalValuesUpsert(String paramList, Collection urls) { + // query is a negative outer join so we only pick the rows where + // the row "s" from the actual table doesn't exist. + + final List sortedNames = new ArrayList<>(urls); + sortedNames.sort((String left, String right) -> left.compareTo(right)); + + final String nextVal = getTranslator().nextValue(getSchemaName(), "fhir_ref_sequence"); + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO common_canonical_values (mt_id, canonical_id, url) "); + insert.append(" SELECT ").append(adminSchemaName).append(".sv_tenant_id, "); + insert.append(nextVal).append(", v.name "); + insert.append(" FROM (VALUES ").append(paramList).append(" ) AS v(name) "); + insert.append(" LEFT OUTER JOIN common_canonical_values s "); + insert.append(" ON s.url = v.name "); + insert.append(" WHERE s.url IS NULL "); + + // Note, we use PreparedStatement here on purpose. Partly because it's + // secure coding best practice, but also because many resources will have the + // same number of parameters, and hopefully we'll therefore share a small subset + // of statements for better performance. Although once the cache warms up, this + // shouldn't be called at all. + try (PreparedStatement ps = getConnection().prepareStatement(insert.toString())) { + // bind all the code_system_name values as parameters + int a = 1; + for (String name: sortedNames) { + ps.setString(a++, name); + } + + ps.executeUpdate(); + } catch (SQLException x) { + logger.log(Level.SEVERE, insert.toString(), x); + throw getTranslator().translate(x); + } + } + @Override protected void doCommonTokenValuesUpsert(String paramList, Collection tokenValues) { StringBuilder insert = new StringBuilder(); @@ -181,4 +219,255 @@ protected void insertResourceTokenRefs(String resourceType, Collection xrefs) { + // Now all the values should have ids assigned so we can go ahead and insert them + // as a batch + final String tableName = "RESOURCE_TOKEN_REFS"; + DataDefinitionUtil.assertValidName(tableName); + final String insert = "INSERT INTO " + tableName + "(" + + "mt_id, parameter_name_id, logical_resource_id, common_token_value_id, ref_version_id) " + + "VALUES (" + this.adminSchemaName + ".SV_TENANT_ID, ?, ?, ?, ?)"; + try (PreparedStatement ps = getConnection().prepareStatement(insert)) { + int count = 0; + for (ResourceTokenValueRec xr: xrefs) { + if (xr.isSystemLevel()) { + ps.setInt(1, xr.getParameterNameId()); + ps.setLong(2, xr.getLogicalResourceId()); + + // common token value can be null + if (xr.getCommonTokenValueId() != null) { + ps.setLong(3, xr.getCommonTokenValueId()); + } else { + ps.setNull(3, Types.BIGINT); + } + + // version can be null + if (xr.getRefVersionId() != null) { + ps.setInt(4, xr.getRefVersionId()); + } else { + ps.setNull(4, Types.INTEGER); + } + + ps.addBatch(); + if (++count == BATCH_SIZE) { + ps.executeBatch(); + count = 0; + } + } + } + + if (count > 0) { + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, insert, x); + throw getTranslator().translate(x); + } + } + + @Override + protected void insertResourceProfiles(String resourceType, Collection profiles) { + // Now all the values should have ids assigned so we can go ahead and insert them + // as a batch + final String tableName = resourceType + "_PROFILES"; + DataDefinitionUtil.assertValidName(tableName); + final String insert = "INSERT INTO " + tableName + "(" + + "mt_id, logical_resource_id, canonical_id, version, fragment) " + + "VALUES (" + this.adminSchemaName + ".SV_TENANT_ID, ?, ?, ?, ?)"; + try (PreparedStatement ps = getConnection().prepareStatement(insert)) { + int count = 0; + for (ResourceProfileRec xr: profiles) { + ps.setLong(1, xr.getLogicalResourceId()); + ps.setInt(2, xr.getCanonicalValueId()); + + // canonical version can be null + if (xr.getVersion() != null) { + ps.setString(3, xr.getVersion()); + } else { + ps.setNull(3, Types.VARCHAR); + } + + // canonical fragment can be null + if (xr.getFragment() != null) { + ps.setString(4, xr.getFragment()); + } else { + ps.setNull(4, Types.VARCHAR); + } + ps.addBatch(); + if (++count == BATCH_SIZE) { + ps.executeBatch(); + count = 0; + } + } + + if (count > 0) { + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, insert, x); + throw getTranslator().translate(x); + } + } + + @Override + protected void insertSystemResourceProfiles(String resourceType, Collection profiles) { + final String tableName = "LOGICAL_RESOURCE_PROFILES"; + DataDefinitionUtil.assertValidName(tableName); + final String insert = "INSERT INTO " + tableName + "(" + + "mt_id, logical_resource_id, canonical_id, version, fragment) " + + "VALUES (" + this.adminSchemaName + ".SV_TENANT_ID, ?, ?, ?, ?)"; + try (PreparedStatement ps = getConnection().prepareStatement(insert)) { + int count = 0; + for (ResourceProfileRec xr: profiles) { + ps.setLong(1, xr.getLogicalResourceId()); + ps.setInt(2, xr.getCanonicalValueId()); + + // canonical version can be null + if (xr.getVersion() != null) { + ps.setString(3, xr.getVersion()); + } else { + ps.setNull(3, Types.VARCHAR); + } + + // canonical fragment can be null + if (xr.getFragment() != null) { + ps.setString(4, xr.getFragment()); + } else { + ps.setNull(4, Types.VARCHAR); + } + ps.addBatch(); + if (++count == BATCH_SIZE) { + ps.executeBatch(); + count = 0; + } + } + + if (count > 0) { + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, insert, x); + throw getTranslator().translate(x); + } + } + + @Override + protected void insertResourceTags(String resourceType, Collection xrefs) { + // Now all the values should have ids assigned so we can go ahead and insert them + // as a batch + final String tableName = resourceType + "_TAGS"; + DataDefinitionUtil.assertValidName(tableName); + final String insert = "INSERT INTO " + tableName + "(" + + "mt_id, logical_resource_id, common_token_value_id) " + + "VALUES (" + this.adminSchemaName + ".SV_TENANT_ID, ?, ?)"; + try (PreparedStatement ps = getConnection().prepareStatement(insert)) { + int count = 0; + for (ResourceTokenValueRec xr: xrefs) { + ps.setLong(1, xr.getLogicalResourceId()); + ps.setLong(2, xr.getCommonTokenValueId()); + ps.addBatch(); + if (++count == BATCH_SIZE) { + ps.executeBatch(); + count = 0; + } + } + + if (count > 0) { + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, insert, x); + throw getTranslator().translate(x); + } + } + + @Override + protected void insertSystemResourceTags(String resourceType, Collection xrefs) { + // Now all the values should have ids assigned so we can go ahead and insert them + // as a batch + final String tableName = "LOGICAL_RESOURCE_TAGS"; + DataDefinitionUtil.assertValidName(tableName); + final String insert = "INSERT INTO " + tableName + "(" + + "mt_id, logical_resource_id, common_token_value_id) " + + "VALUES (" + this.adminSchemaName + ".SV_TENANT_ID, ?, ?)"; + try (PreparedStatement ps = getConnection().prepareStatement(insert)) { + int count = 0; + for (ResourceTokenValueRec xr: xrefs) { + ps.setLong(1, xr.getLogicalResourceId()); + ps.setLong(2, xr.getCommonTokenValueId()); + ps.addBatch(); + if (++count == BATCH_SIZE) { + ps.executeBatch(); + count = 0; + } + } + + if (count > 0) { + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, insert, x); + throw getTranslator().translate(x); + } + } + @Override + protected void insertResourceSecurity(String resourceType, Collection xrefs) { + // Now all the values should have ids assigned so we can go ahead and insert them + // as a batch + final String tableName = resourceType + "_SECURITY"; + DataDefinitionUtil.assertValidName(tableName); + final String insert = "INSERT INTO " + tableName + "(" + + "mt_id, logical_resource_id, common_token_value_id) " + + "VALUES (" + this.adminSchemaName + ".SV_TENANT_ID, ?, ?)"; + try (PreparedStatement ps = getConnection().prepareStatement(insert)) { + int count = 0; + for (ResourceTokenValueRec xr: xrefs) { + ps.setLong(1, xr.getLogicalResourceId()); + ps.setLong(2, xr.getCommonTokenValueId()); + ps.addBatch(); + if (++count == BATCH_SIZE) { + ps.executeBatch(); + count = 0; + } + } + + if (count > 0) { + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, insert, x); + throw getTranslator().translate(x); + } + } + + @Override + protected void insertSystemResourceSecurity(String resourceType, Collection xrefs) { + // Now all the values should have ids assigned so we can go ahead and insert them + // as a batch + final String tableName = "LOGICAL_RESOURCE_SECURITY"; + DataDefinitionUtil.assertValidName(tableName); + final String insert = "INSERT INTO " + tableName + "(" + + "mt_id, logical_resource_id, common_token_value_id) " + + "VALUES (" + this.adminSchemaName + ".SV_TENANT_ID, ?, ?)"; + try (PreparedStatement ps = getConnection().prepareStatement(insert)) { + int count = 0; + for (ResourceTokenValueRec xr: xrefs) { + ps.setLong(1, xr.getLogicalResourceId()); + ps.setLong(2, xr.getCommonTokenValueId()); + ps.addBatch(); + if (++count == BATCH_SIZE) { + ps.executeBatch(); + count = 0; + } + } + + if (count > 0) { + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, insert, x); + throw getTranslator().translate(x); + } + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/CreateCanonicalValuesTmp.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/CreateCanonicalValuesTmp.java new file mode 100644 index 00000000000..c46a2b60627 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/CreateCanonicalValuesTmp.java @@ -0,0 +1,62 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.derby; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.logging.Level; +import java.util.logging.Logger; + +import com.ibm.fhir.database.utils.api.IDatabaseStatement; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.schema.control.FhirSchemaConstants; + + +/** + * Create the CANONICAL_VALUES_TMP table + */ +public class CreateCanonicalValuesTmp implements IDatabaseStatement { + private static final Logger logger = Logger.getLogger(CreateCanonicalValuesTmp.class.getName()); + + @Override + public void run(IDatabaseTranslator translator, Connection c) { + + if (!isExists(c)) { + final String ddl = "" + + "DECLARE GLOBAL TEMPORARY TABLE canonical_values_tmp (" + + " url VARCHAR(" + FhirSchemaConstants.MAX_TOKEN_VALUE_BYTES + ")" + + ") NOT LOGGED"; + + try (Statement s = c.createStatement()) { + s.executeUpdate(ddl); + } catch (SQLException x) { + logger.log(Level.SEVERE, ddl, x); + throw translator.translate(x); + } + } + } + + /** + * Does the table currently exist + * @param c + * @return + */ + private boolean isExists(Connection c) { + boolean result = false; + + final String sql = "SELECT 1 FROM SESSION.canonical_values_tmp WHERE 1=0"; + try (Statement s = c.createStatement()) { + s.executeQuery(sql); + result = true; + } catch (SQLException x) { + // NOP + } + + return result; + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java index 7dd903c79d5..2325e8e28ff 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java @@ -40,6 +40,7 @@ import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceFKVException; import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; +import com.ibm.fhir.persistence.jdbc.util.ParameterTableSupport; import com.ibm.fhir.persistence.jdbc.util.ResourceTypesCache; /** @@ -82,6 +83,7 @@ public DerbyResourceDAO(Connection connection, String schemaName, FHIRDbFlavor f * by sql. * @param resource The FHIR Resource to be inserted. * @param parameters The Resource's search parameters to be inserted. + * @oaram parameterHashB64 * @param parameterDao * @return The Resource DTO * @throws FHIRPersistenceDataAccessException @@ -89,7 +91,7 @@ public DerbyResourceDAO(Connection connection, String schemaName, FHIRDbFlavor f * @throws FHIRPersistenceVersionIdMismatchException */ @Override - public Resource insert(Resource resource, List parameters, ParameterDAO parameterDao) + public Resource insert(Resource resource, List parameters, String parameterHashB64, ParameterDAO parameterDao) throws FHIRPersistenceException { final String METHODNAME = "insert"; logger.entering(CLASSNAME, METHODNAME); @@ -129,6 +131,7 @@ public Resource insert(Resource resource, List paramet resource.isDeleted(), sourceKey, resource.getVersionId(), + parameterHashB64, connection, parameterDao ); @@ -194,12 +197,15 @@ public Resource insert(Resource resource, List paramet * @param p_is_deleted * @param p_source_key * @param p_version - * + * @param p_parameterHashB64 + * @param conn + * @param parameterDao + * @param p_parameterHashB64 Base64 encoded parameter hash value * @return the resource_id for the entry we created * @throws Exception */ public long storeResource(String tablePrefix, List parameters, String p_logical_id, InputStream p_payload, Timestamp p_last_updated, boolean p_is_deleted, - String p_source_key, Integer p_version, Connection conn, ParameterDAO parameterDao) throws Exception { + String p_source_key, Integer p_version, String p_parameterHashB64, Connection conn, ParameterDAO parameterDao) throws Exception { final String METHODNAME = "storeResource() for " + tablePrefix + " resource"; logger.entering(CLASSNAME, METHODNAME); @@ -212,6 +218,10 @@ public long storeResource(String tablePrefix, List para boolean v_duplicate = false; int v_current_version; + // used to bypass param delete/insert if all param values are the same + String currentParameterHash = null; + boolean requireParameterUpdate = true; + String v_resource_type = tablePrefix; // Map the resource type name to the normalized id value in the database @@ -245,7 +255,7 @@ public long storeResource(String tablePrefix, List para if (logger.isLoggable(Level.FINEST)) { logger.finest("Getting LOGICAL_RESOURCES row lock for: " + v_resource_type + "/" + p_logical_id); } - final String SELECT_FOR_UPDATE = "SELECT logical_resource_id FROM logical_resources WHERE resource_type_id = ? AND logical_id = ? FOR UPDATE WITH RS"; + final String SELECT_FOR_UPDATE = "SELECT logical_resource_id, parameter_hash FROM logical_resources WHERE resource_type_id = ? AND logical_id = ? FOR UPDATE WITH RS"; try (PreparedStatement stmt = conn.prepareStatement(SELECT_FOR_UPDATE)) { stmt.setInt(1, v_resource_type_id); stmt.setString(2, p_logical_id); @@ -255,6 +265,7 @@ public long storeResource(String tablePrefix, List para logger.finest("Resource locked: " + v_resource_type + "/" + p_logical_id); } v_logical_resource_id = rs.getLong(1); + currentParameterHash = rs.getString(2); } else { if (logger.isLoggable(Level.FINEST)) { @@ -287,13 +298,16 @@ public long storeResource(String tablePrefix, List para if (logger.isLoggable(Level.FINEST)) { logger.finest("Creating new logical_resources row for: " + v_resource_type + "/" + p_logical_id); } - final String sql4 = "INSERT INTO logical_resources (logical_resource_id, resource_type_id, logical_id, reindex_tstamp) VALUES (?, ?, ?, ?)"; + final String sql4 = "INSERT INTO logical_resources (logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash) VALUES (?, ?, ?, ?, ?, ?, ?)"; try (PreparedStatement stmt = conn.prepareStatement(sql4)) { // bind parameters stmt.setLong(1, v_logical_resource_id); stmt.setInt(2, v_resource_type_id); stmt.setString(3, p_logical_id); stmt.setTimestamp(4, Timestamp.valueOf(DEFAULT_VALUE_REINDEX_TSTAMP), UTC); + stmt.setString(5, p_is_deleted ? "Y" : "N"); // from V0014 + stmt.setTimestamp(6, p_last_updated, UTC); // from V0014 + stmt.setString(7, p_parameterHashB64); // from V0015 stmt.executeUpdate(); if (logger.isLoggable(Level.FINEST)) { @@ -331,16 +345,15 @@ public long storeResource(String tablePrefix, List para logger.finest("Resource locked: " + v_resource_type + "/" + p_logical_id); } v_logical_resource_id = res.getLong(1); + currentParameterHash = res.getString(2); res.next(); - } - else { + } else { // Extremely unlikely as we should never delete logical resource records throw new IllegalStateException("Logical resource was deleted: " + tablePrefix + "/" + p_logical_id); } } } - } - else { + } else { v_new_resource = true; // Insert the resource-specific logical resource record. Remember that logical_id is denormalized @@ -400,18 +413,12 @@ public long storeResource(String tablePrefix, List para throw new SQLException("Concurrent update - mismatch of version in JSON", "99001"); } - // existing resource, so need to delete all its parameters - deleteFromParameterTable(conn, tablePrefix + "_str_values", v_logical_resource_id); - deleteFromParameterTable(conn, tablePrefix + "_number_values", v_logical_resource_id); - deleteFromParameterTable(conn, tablePrefix + "_date_values", v_logical_resource_id); - deleteFromParameterTable(conn, tablePrefix + "_latlng_values", v_logical_resource_id); - deleteFromParameterTable(conn, tablePrefix + "_resource_token_refs", v_logical_resource_id); - deleteFromParameterTable(conn, tablePrefix + "_quantity_values", v_logical_resource_id); - - // delete any system level parameters we have for this resource - deleteFromParameterTable(conn, "str_values", v_logical_resource_id); - deleteFromParameterTable(conn, "date_values", v_logical_resource_id); - deleteFromParameterTable(conn, "resource_token_refs", v_logical_resource_id); + // existing resource, so need to delete all its parameters unless they share + // an identical hash, in which case we can bypass the delete/insert + requireParameterUpdate = currentParameterHash == null || currentParameterHash.isEmpty() || !currentParameterHash.equals(p_parameterHashB64); + if (requireParameterUpdate) { + ParameterTableSupport.deleteFromParameterTables(conn, tablePrefix, v_logical_resource_id); + } } // Finally we get to the big resource data insert @@ -438,7 +445,7 @@ public long storeResource(String tablePrefix, List para if (logger.isLoggable(Level.FINEST)) { logger.finest("Updating " + tablePrefix + "_logical_resources: " + v_resource_type + "/" + p_logical_id); } - String sql4 = "UPDATE " + tablePrefix + "_logical_resources SET current_resource_id = ?, is_deleted = ?, last_updated = ?, version_id = ? WHERE logical_resource_id = ?"; + final String sql4 = "UPDATE " + tablePrefix + "_logical_resources SET current_resource_id = ?, is_deleted = ?, last_updated = ?, version_id = ? WHERE logical_resource_id = ?"; try (PreparedStatement stmt = conn.prepareStatement(sql4)) { // bind parameters stmt.setLong(1, v_resource_id); @@ -451,12 +458,27 @@ public long storeResource(String tablePrefix, List para logger.finest("Updated " + tablePrefix + "_logical_resources: " + v_resource_type + "/" + p_logical_id); } } + + // For schema V0014, now we also need to update the is_deleted and last_updated values + // in LOGICAL_RESOURCES to support whole-system search + final String sql4b = "UPDATE logical_resources SET is_deleted = ?, last_updated = ?, parameter_hash = ? WHERE logical_resource_id = ?"; + try (PreparedStatement stmt = conn.prepareStatement(sql4b)) { + // bind parameters + stmt.setString(1, p_is_deleted ? "Y" : "N"); + stmt.setTimestamp(2, p_last_updated, UTC); + stmt.setString(3, p_parameterHashB64); + stmt.setLong(4, v_logical_resource_id); + stmt.executeUpdate(); + if (logger.isLoggable(Level.FINEST)) { + logger.finest("Updated logical_resources: " + v_resource_type + "/" + p_logical_id); + } + } } // To keep things simple for the Derby use-case, we just use a visitor to // handle inserts of parameters directly in the resource parameter tables. // Note we don't get any parameters for the resource soft-delete operation - if (parameters != null) { + if (parameters != null && requireParameterUpdate) { // Derby doesn't support partitioned multi-tenancy, so we disable it on the DAO: if (logger.isLoggable(Level.FINEST)) { logger.finest("Storing parameters for: " + v_resource_type + "/" + p_logical_id); @@ -489,25 +511,6 @@ identityCache, getResourceReferenceDAO(), getTransactionData())) { return v_resource_id; } - - /** - * Delete all parameters for the given resourceId from the parameters table - * - * @param conn - * @param tableName - * @param logicalResourceId - * @throws SQLException - */ - protected void deleteFromParameterTable(Connection conn, String tableName, long logicalResourceId) throws SQLException { - final String delStrValues = "DELETE FROM " + tableName + " WHERE logical_resource_id = ?"; - try (PreparedStatement stmt = conn.prepareStatement(delStrValues)) { - // bind parameters - stmt.setLong(1, logicalResourceId); - stmt.executeUpdate(); - } - - } - /** * Read the id for the named type * @param resourceTypeName diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceReferenceDAO.java index 04ceadc51f4..db1d46a44cd 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceReferenceDAO.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -35,9 +35,9 @@ */ public class DerbyResourceReferenceDAO extends ResourceReferenceDAO { private static final Logger logger = Logger.getLogger(PostgresResourceReferenceDAO.class.getName()); - + private static final int BATCH_SIZE = 100; - + /** * Public constructor * @param t @@ -48,10 +48,10 @@ public class DerbyResourceReferenceDAO extends ResourceReferenceDAO { public DerbyResourceReferenceDAO(IDatabaseTranslator t, Connection c, String schemaName, ICommonTokenValuesCache cache) { super(t, c, schemaName, cache); } - + @Override public void doCodeSystemsUpsert(String paramList, Collection systemNames) { - + // We'll assume that the rows will be processed in the order they are // inserted, although there's not really a guarantee this is the case final List sortedNames = new ArrayList<>(systemNames); @@ -65,13 +65,13 @@ public void doCodeSystemsUpsert(String paramList, Collection systemNames for (String systemName: systemNames) { ps.setString(1, systemName); ps.addBatch(); - + if (++batchCount == BATCH_SIZE) { ps.executeBatch(); batchCount = 0; } } - + if (batchCount > 0) { ps.executeBatch(); } @@ -79,7 +79,7 @@ public void doCodeSystemsUpsert(String paramList, Collection systemNames logger.log(Level.SEVERE, insert.toString(), x); throw getTranslator().translate(x); } - + // Upsert values. Can't use an order by in this situation because // Derby doesn't like this when pulling values from the sequence, @@ -93,7 +93,59 @@ public void doCodeSystemsUpsert(String paramList, Collection systemNames upsert.append(" LEFT OUTER JOIN code_systems cs "); upsert.append(" ON cs.code_system_name = src.code_system_name "); upsert.append(" WHERE cs.code_system_name IS NULL "); - + + try (Statement s = getConnection().createStatement()) { + s.executeUpdate(upsert.toString()); + } catch (SQLException x) { + logger.log(Level.SEVERE, upsert.toString(), x); + throw getTranslator().translate(x); + } + } + + @Override + public void doCanonicalValuesUpsert(String paramList, Collection urls) { + + // We'll assume that the rows will be processed in the order they are + // inserted, although there's not really a guarantee this is the case + final List sortedNames = new ArrayList<>(urls); + sortedNames.sort((String left, String right) -> left.compareTo(right)); + + // Derby doesn't like really huge VALUES lists, so we instead need + // to go with a declared temporary table. + final String insert = "INSERT INTO SESSION.canonical_values_tmp (url) VALUES (?)"; + int batchCount = 0; + try (PreparedStatement ps = getConnection().prepareStatement(insert)) { + for (String url: urls) { + ps.setString(1, url); + ps.addBatch(); + + if (++batchCount == BATCH_SIZE) { + ps.executeBatch(); + batchCount = 0; + } + } + + if (batchCount > 0) { + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, insert.toString(), x); + throw getTranslator().translate(x); + } + + // Upsert values. Can't use an order by in this situation because + // Derby doesn't like this when pulling values from the sequence, + // which seems like a defect, because the values should only be + // evaluated after the join and where clauses. + final String nextVal = getTranslator().nextValue(getSchemaName(), "fhir_ref_sequence"); + StringBuilder upsert = new StringBuilder(); + upsert.append("INSERT INTO common_canonical_values (canonical_id, url) "); + upsert.append(" SELECT ").append(nextVal).append(", src.url "); + upsert.append(" FROM SESSION.canonical_values_tmp src "); + upsert.append(" LEFT OUTER JOIN common_canonical_values cs "); + upsert.append(" ON cs.url = src.url "); + upsert.append(" WHERE cs.url IS NULL "); + try (Statement s = getConnection().createStatement()) { s.executeUpdate(upsert.toString()); } catch (SQLException x) { @@ -104,7 +156,7 @@ public void doCodeSystemsUpsert(String paramList, Collection systemNames @Override protected void doCommonTokenValuesUpsert(String paramList, Collection tokenValues) { - + final String insert = "INSERT INTO SESSION.common_token_values_tmp(token_value, code_system_id) VALUES (?, ?)"; int batchCount = 0; try (PreparedStatement ps = getConnection().prepareStatement(insert)) { @@ -112,13 +164,13 @@ protected void doCommonTokenValuesUpsert(String paramList, Collection 0) { ps.executeBatch(); } @@ -138,7 +190,7 @@ protected void doCommonTokenValuesUpsert(String paramList, Collection values) { // Special case for Derby so we don't try and create monster SQL statements // resulting in a stack overflow when Derby attempts to parse it. - + // Unique list so we don't try and create the same name more than once. // Ignore any null token-values, because we don't want to (can't) store // them in our common token values table. Set tokenValues = values.stream().filter(x -> x.getTokenValue() != null).map(xr -> new CommonTokenValue(xr.getCodeSystemValueId(), xr.getTokenValue())).collect(Collectors.toSet()); - + if (tokenValues.isEmpty()) { // nothing to do return; @@ -173,7 +225,7 @@ public void upsertCommonTokenValues(List values) { select.append(" SESSION.common_token_values_tmp tmp "); select.append(" WHERE ctv.token_value = tmp.token_value "); select.append(" AND ctv.code_system_id = tmp.code_system_id "); - + Map idMap = new HashMap<>(); try (PreparedStatement ps = getConnection().prepareStatement(select.toString())) { ResultSet rs = ps.executeQuery(); @@ -185,7 +237,7 @@ public void upsertCommonTokenValues(List values) { } catch (SQLException x) { throw getTranslator().translate(x); } - + // Now update the ids for all the matching systems in our list for (ResourceTokenValueRec xr: values) { // ignore entries with null tokenValue elements - we don't store them in common_token_values @@ -194,7 +246,7 @@ public void upsertCommonTokenValues(List values) { Long id = idMap.get(key); if (id != null) { xr.setCommonTokenValueId(id); - + // update the thread-local cache with this id. The values aren't committed to the shared cache // until the transaction commits getCache().addTokenValue(key, id); diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/CanonicalSearchParam.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/CanonicalSearchParam.java new file mode 100644 index 00000000000..cc920717a8e --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/CanonicalSearchParam.java @@ -0,0 +1,31 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.domain; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.search.parameters.QueryParameter; + +/** + * A canonical search parameter + */ +public class CanonicalSearchParam extends SearchParam { + + /** + * Public constructor + * @param rootResourceType + * @param name + * @param queryParameter the search query parameter being wrapped + */ + public CanonicalSearchParam(String rootResourceType, String name, QueryParameter queryParameter) { + super(rootResourceType, name, queryParameter); + } + + @Override + public T visit(T queryData, SearchQueryVisitor visitor) throws FHIRPersistenceException { + return visitor.addCanonicalParam(queryData, getRootResourceType(), getQueryParameter()); + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/DomainSortParameter.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/DomainSortParameter.java index ed8b3e2405c..f771f982337 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/DomainSortParameter.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/DomainSortParameter.java @@ -26,6 +26,14 @@ public DomainSortParameter(SortParameter sp) { this.sortParameter = sp; } + /** + * Get the sort parameter. + * @return sort parameter + */ + public SortParameter getSortParameter() { + return this.sortParameter; + } + /** * Visitor to apply the sort parameter to the query builder represented by the visitor * @param queryData diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchDataQuery.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchDataQuery.java index aa7ef4d3d07..59eac64f512 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchDataQuery.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchDataQuery.java @@ -14,6 +14,12 @@ * this class fetches the data and handles pagination */ public class SearchDataQuery extends SearchQuery { + + // Is sorting required? + boolean addSorting = true; + + // Is pagination required? + boolean addPagination = true; /** * Public constructor @@ -23,6 +29,18 @@ public SearchDataQuery(String resourceType) { super(resourceType); } + /** + * Public constructor + * @param resourceType + * @param addSorting + * @param addPagination + */ + public SearchDataQuery(String resourceType, boolean addSorting, boolean addPagination) { + super(resourceType); + this.addSorting = addSorting; + this.addPagination = addPagination; + } + @Override public T getRoot(SearchQueryVisitor visitor) { return visitor.dataRoot(getRootResourceType()); @@ -38,8 +56,12 @@ public T visit(SearchQueryVisitor visitor) throws FHIRPersistenceExceptio query = visitor.joinResources(query); // now attach the requisite ordering and pagination clauses - query = visitor.addSorting(query, "LR"); - query = visitor.addPagination(query); + if (addSorting) { + query = visitor.addSorting(query, "LR"); + } + if (addPagination) { + query = visitor.addPagination(query); + } return query; } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java index 987feda3e9c..b52afc91944 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java @@ -14,6 +14,7 @@ import static com.ibm.fhir.persistence.jdbc.JDBCConstants.CODE_SYSTEM_ID; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.COMMON_TOKEN_VALUE_ID; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.DATE_START; +import static com.ibm.fhir.persistence.jdbc.JDBCConstants.DESCENDING; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.EQ; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.ESCAPE_PERCENT; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.ESCAPE_UNDERSCORE; @@ -22,6 +23,7 @@ import static com.ibm.fhir.persistence.jdbc.JDBCConstants.MAX; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.MIN; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.NUMBER_VALUE; +import static com.ibm.fhir.persistence.jdbc.JDBCConstants.PARAMETER_NAME_ID; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.PERCENT_WILDCARD; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.QUANTITY_VALUE; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.RIGHT_PAREN; @@ -29,6 +31,11 @@ import static com.ibm.fhir.persistence.jdbc.JDBCConstants.UNDERSCORE_WILDCARD; import static com.ibm.fhir.persistence.jdbc.JDBCConstants._LOGICAL_RESOURCES; import static com.ibm.fhir.persistence.jdbc.JDBCConstants._RESOURCES; +import static com.ibm.fhir.search.SearchConstants.ID; +import static com.ibm.fhir.search.SearchConstants.LAST_UPDATED; +import static com.ibm.fhir.search.SearchConstants.PROFILE; +import static com.ibm.fhir.search.SearchConstants.SECURITY; +import static com.ibm.fhir.search.SearchConstants.TAG; import java.util.ArrayList; import java.util.Arrays; @@ -52,10 +59,13 @@ import com.ibm.fhir.database.utils.query.expression.StringExpNodeVisitor; import com.ibm.fhir.database.utils.query.node.ExpNode; import com.ibm.fhir.model.resource.CodeSystem; +import com.ibm.fhir.model.resource.Resource; import com.ibm.fhir.model.type.Code; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.exception.FHIRPersistenceNotSupportedException; import com.ibm.fhir.persistence.jdbc.dao.api.JDBCIdentityCache; +import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceProfileRec; +import com.ibm.fhir.persistence.jdbc.util.CanonicalSupport; import com.ibm.fhir.persistence.jdbc.util.NewUriModifierUtil; import com.ibm.fhir.persistence.jdbc.util.QuerySegmentAggregator; import com.ibm.fhir.persistence.jdbc.util.SqlParameterEncoder; @@ -92,6 +102,7 @@ public class SearchQueryRenderer implements SearchQueryVisitor { private final static String STR_VALUE = "STR_VALUE"; private final static String STR_VALUE_LCASE = "STR_VALUE_LCASE"; + private final static String LOGICAL_RESOURCES = "LOGICAL_RESOURCES"; // A cache providing access to various database reference ids private final JDBCIdentityCache identityCache; @@ -133,7 +144,11 @@ protected int getNextAliasIndex() { * @return the table name */ protected String resourceLogicalResources(String resourceType) { - return resourceType + _LOGICAL_RESOURCES; + if (isWholeSystemSearch(resourceType)) { + return LOGICAL_RESOURCES; + } else { + return resourceType + _LOGICAL_RESOURCES; + } } /** @@ -185,6 +200,16 @@ protected int getCodeSystemId(String codeSystemName) throws FHIRPersistenceExcep return this.identityCache.getCodeSystemId(codeSystemName); } + /** + * Get the id for the given canonicalValue (cache lookup). + * @param canonicalValue + * @return the database id, or -1 if the value does not exist + * @throws FHIRPersistenceException + */ + protected int getCanonicalId(String canonicalValue) throws FHIRPersistenceException { + return this.identityCache.getCanonicalId(canonicalValue); + } + @Override public QueryData countRoot(String rootResourceType) { final int aliasIndex = 0; @@ -196,7 +221,7 @@ public QueryData countRoot(String rootResourceType) { // No need to join with xx_RESOURCES, because we only need to count // undeleted logical resources, not individual resource versions /* - SELECT COUNT(*) + SELECT COUNT(*) AS CNT FROM Patient_LOGICAL_RESOURCES AS LR0 WHERE LR0.IS_DELETED = 'N' AND EXISTS ( @@ -208,7 +233,7 @@ AND EXISTS ( WHERE LR1.IS_DELETED = 'N' AND LR1.LOGICAL_RESOURCE_ID = LR0.LOGICAL_RESOURCE_ID) */ - SelectAdapter select = Select.select("COUNT(*)"); + SelectAdapter select = Select.select().addColumn(null, "COUNT(*)", alias("CNT")); select.from(xxLogicalResources, alias(lrAliasName)) .where(lrAliasName, IS_DELETED).eq(string("N")); return new QueryData(select, lrAliasName, null, rootResourceType, 0); @@ -253,19 +278,20 @@ AND EXISTS ( @Override public QueryData getParameterBaseQuery(QueryData parent) { final int aliasIndex = getNextAliasIndex(); - final String xxLogicalResources = parent.getResourceType() + "_LOGICAL_RESOURCES"; + final String xxLogicalResources = resourceLogicalResources(parent.getResourceType()); final String lrAlias = "LR" + aliasIndex; final String parentLRAlias = parent.getLRAlias(); // SELECT 1 FROM xx_LOGICAL_RESOURCES LRn // INNER JOIN ... // INNER JOIN ... - // WHERE LRn.IS_DELETED = 'N' - // AND LRn.LOGICAL_RESOURCE_ID = LRp.LOGICAL_RESOURCE_ID + // WHERE LRn.LOGICAL_RESOURCE_ID = LRp.LOGICAL_RESOURCE_ID + // + // Note: IS_DELETED is not checked in this sub-query - if necessary, that is + // the responsibility of the parent query. SelectAdapter exists = Select.select("1"); exists.from(xxLogicalResources, alias(lrAlias)) - .where(lrAlias, "IS_DELETED").eq().literal("N") // TODO remove either from here or parent - .and(lrAlias, "LOGICAL_RESOURCE_ID").eq(parentLRAlias, "LOGICAL_RESOURCE_ID"); // correlate to parent query + .where(lrAlias, "LOGICAL_RESOURCE_ID").eq(parentLRAlias, "LOGICAL_RESOURCE_ID"); // correlate to parent query // Add this exists to the parent query parent.getQuery().from().where().and().exists(exists.build()); @@ -346,6 +372,112 @@ public QueryData sortRoot(String rootResourceType) { return new QueryData(select, lrAliasName, null, rootResourceType, 0); } + @Override + public QueryData wholeSystemFilterRoot() { + /* Final query should look like this: + SELECT LR0.RESOURCE_TYPE_ID, LR0.LOGICAL_RESOURCE_ID + FROM LOGICAL_RESOURCES AS LR0 + WHERE LR0.IS_DELETED = 'N' + AND EXISTS ( + SELECT 1 + FROM LOGICAL_RESOURCES AS LR1 + INNER JOIN RESOURCE_TOKEN_REFS AS P2 ON P2.LOGICAL_RESOURCE_ID = LR1.LOGICAL_RESOURCE_ID + AND P2.PARAMETER_NAME_ID = 1008 + AND ((P2.COMMON_TOKEN_VALUE_ID = 4)) + WHERE LR1.IS_DELETED = 'N' + AND LR1.LOGICAL_RESOURCE_ID = LR0.LOGICAL_RESOURCE_ID) + ORDER BY LR0.LOGICAL_RESOURCE_ID + FETCH FIRST 10 ROWS ONLY + */ + + final String lrAliasName = "LR0"; + + // The core data query joining together the logical resources table. Query + // parameters are bolted on as exists statements in the WHERE clause. + SelectAdapter select = Select.select("LR0.RESOURCE_TYPE_ID", "LR0.LOGICAL_RESOURCE_ID"); + select.from(LOGICAL_RESOURCES, alias(lrAliasName)) + .where(lrAliasName, IS_DELETED).eq().literal("N"); + return new QueryData(select, lrAliasName, null, Resource.class.getSimpleName(), 0); + } + + @Override + public QueryData wholeSystemDataRoot(String rootResourceType) { + /* Final query should look something like this: + SELECT R.RESOURCE_ID, R.LOGICAL_RESOURCE_ID, R.VERSION_ID, R.LAST_UPDATED, R.IS_DELETED, R.DATA, LR.LOGICAL_ID + FROM ( + SELECT LR.LOGICAL_RESOURCE_ID, LR.LOGICAL_ID, LR.CURRENT_RESOURCE_ID + FROM Patient_LOGICAL_RESOURCES AS LR + WHERE LR.IS_DELETED = 'N') AS LR + AND LR.LOGICAL_RESOURCE_ID IN (2,4,6,10,12,14,20,24,26,29)) AS LR + INNER JOIN Patient_RESOURCES AS R ON LR.CURRENT_RESOURCE_ID = R.RESOURCE_ID + ORDER BY LR.LOGICAL_RESOURCE_ID + FETCH FIRST 10 ROWS ONLY + */ + final String xxLogicalResources = resourceLogicalResources(rootResourceType); + final String lrAliasName = "LR"; + + // The core data query joining together the logical resources table. The final + // query is constructed when joinResources is called. + SelectAdapter select = Select.select("LR.LOGICAL_RESOURCE_ID", "LR.LOGICAL_ID", "LR.CURRENT_RESOURCE_ID"); + select.from(xxLogicalResources, alias(lrAliasName)) + .where(lrAliasName, IS_DELETED).eq().literal("N"); + return new QueryData(select, lrAliasName, null, rootResourceType, 0); + } + + @Override + public QueryData wrapWholeSystem(List queries, boolean isCountQuery) { + /* We need to either sum the counts of each individual count query or + aggregate the data of each individual data query. + Final query should look something like this for a count query: + SELECT SUM(CNT) + FROM ( + + UNION ALL + + UNION ALL + ... + UNION ALL + + ) AS COMBINED RESULTS + + Final query should look something like this for a data query: + SELECT RESOURCE_ID, LOGICAL_RESOURCE_ID, VERSION_ID, LAST_UPDATED, IS_DELETED, DATA, LOGICAL_ID + FROM ( + + UNION ALL + + UNION ALL + ... + UNION ALL + + ) AS COMBINED RESULTS + ORDER BY COMBINED RESULTS.LOGICAL_RESOURCE_ID + FETCH FIRST 10 ROWS ONLY + */ + SelectAdapter select; + if (isCountQuery) { + select = Select.select("SUM(CNT)"); + } else { + select = Select.select("RESOURCE_ID", "LOGICAL_RESOURCE_ID", "VERSION_ID", "LAST_UPDATED", "IS_DELETED", "DATA", "LOGICAL_ID"); + } + SelectAdapter first = null; + SelectAdapter previous = null; + for (QueryData query : queries) { + SelectAdapter subSelect = query.getQuery(); + if (previous == null) { + // Save head of UNION'd selects + first = subSelect; + } else { + // Add current select as union of previous + previous.unionAll(subSelect.getSelect()); + } + previous = subSelect; + } + select.from(first.getSelect(), alias("COMBINED_RESULTS")); + + return new QueryData(select, null, null, Resource.class.getSimpleName(), 0); + } + /** * Get the filter predicate for the given token query parameter. * @param queryParm the token query parameter @@ -415,18 +547,19 @@ protected WhereFragment getTokenFilter(QueryParameter queryParm, String paramAli } } else { // Traditional approach, using a join to xx_TOKEN_VALUES_V - + // Include code if present if (code != null) { where.col(paramAlias, TOKEN_VALUE).operator(operator); if (operator == Operator.LIKE) { - // Must escape special wildcard characters _ and % in the parameter value string. + // Must escape special wildcard characters _ and % in the parameter value string + // as well as the escape character itself. String textSearchString = normalizedCode + .replace("+", "++") .replace(PERCENT_WILDCARD, ESCAPE_PERCENT) - .replace(UNDERSCORE_WILDCARD, ESCAPE_UNDERSCORE) - .replace("+", "++")+ PERCENT_WILDCARD; + .replace(UNDERSCORE_WILDCARD, ESCAPE_UNDERSCORE) + PERCENT_WILDCARD; where.bind(SearchUtil.normalizeForSearch(textSearchString)).escape("+"); - + } else { where.bind(normalizedCode); } @@ -598,9 +731,11 @@ protected WhereFragment getStringFilter(QueryParameter queryParm, String paramAl multiple = true; } if (operator == Operator.LIKE) { - // Must escape special wildcard characters _ and % in the parameter value string. + // Must escape special wildcard characters _ and % in the parameter value string + // as well as the escape character itself. String tempSearchValue = SqlParameterEncoder.encode(value.getValueString() + .replace("+", "++") .replace(PERCENT_WILDCARD, ESCAPE_PERCENT) .replace(UNDERSCORE_WILDCARD, ESCAPE_UNDERSCORE)); @@ -641,6 +776,7 @@ protected WhereFragment getStringFilter(QueryParameter queryParm, String paramAl if (queryParm.getModifier() == Modifier.BELOW) { String tempSearchValue = SqlParameterEncoder.encode(value.getValueString() + .replace("+", "++") .replace(PERCENT_WILDCARD, ESCAPE_PERCENT) .replace(UNDERSCORE_WILDCARD, ESCAPE_UNDERSCORE)); @@ -704,6 +840,26 @@ public QueryData addSorting(QueryData queryData, String lrAlias) { return queryData; } + @Override + public QueryData addWholeSystemSorting(QueryData queryData, List sortParms, String lrAlias) { + if (sortParms == null || sortParms.isEmpty()) { + return addSorting(queryData, lrAlias); + } else { + for (DomainSortParameter sortParm : sortParms) { + // for whole-system searches, sort parameters can only be _id or _lastUpdated + StringBuilder expression = new StringBuilder(); + expression.append(ID.equals(sortParm.getSortParameter().getCode()) ? + DataDefinitionUtil.getQualifiedName(lrAlias, "LOGICAL_ID") : + DataDefinitionUtil.getQualifiedName(lrAlias, "LAST_UPDATED")) + .append(" ") + .append(Direction.DECREASING.equals(sortParm.getSortParameter().getDirection()) ? DESCENDING : ""); + + queryData.getQuery().from().orderBy(expression.toString()); + } + } + return queryData; + } + @Override public QueryData addPagination(QueryData queryData) { queryData.getQuery().pagination(rowOffset, rowsPerPage); @@ -719,31 +875,43 @@ public QueryData addPagination(QueryData queryData) { * @param paramType * @return */ - public String paramValuesTableName(String resourceType, Type paramType) { - StringBuilder name = new StringBuilder(resourceType); - switch (paramType) { + public String paramValuesTableName(String resourceType, QueryParameter queryParm) { + boolean wholeSystemSearch = isWholeSystemSearch(resourceType); + + StringBuilder name = new StringBuilder(wholeSystemSearch ? "" : resourceType + "_"); + switch (queryParm.getType()) { case URI: case STRING: - name.append("_STR_VALUES"); + if (PROFILE.equals(queryParm.getCode())) { + name.append(wholeSystemSearch ? "LOGICAL_RESOURCE_PROFILES" : "PROFILES"); + } else { + name.append("STR_VALUES"); + } break; case NUMBER: - name.append("_NUMBER_VALUES"); + name.append("NUMBER_VALUES"); break; case QUANTITY: - name.append("_QUANTITY_VALUES"); + name.append("QUANTITY_VALUES"); break; case DATE: - name.append("_DATE_VALUES"); + name.append("DATE_VALUES"); break; case SPECIAL: - name.append("_LATLNG_VALUES"); + name.append("LATLNG_VALUES"); break; case REFERENCE: case TOKEN: - name.append("_RESOURCE_TOKEN_REFS"); // bypass the xx_TOKEN_VALUES_V for performance reasons + if (TAG.equals(queryParm.getCode())) { + name.append(wholeSystemSearch ? "LOGICAL_RESOURCE_TAGS" : "TAGS"); + } else if (SECURITY.equals(queryParm.getCode())) { + name.append(wholeSystemSearch ? "LOGICAL_RESOURCE_SECURITY" : "SECURITY"); + } else { + name.append("RESOURCE_TOKEN_REFS"); // bypass the xx_TOKEN_VALUES_V for performance reasons + } break; case COMPOSITE: - name.append("_LOGICAL_RESOURCES"); + name.append("LOGICAL_RESOURCES"); break; } return name.toString(); @@ -802,7 +970,7 @@ protected WhereFragment getFilterPredicate(QueryData queryData, QueryParameter q final String code = queryParm.getCode(); final String parentAlias = queryData.getLRAlias(); - if ("_id".equals(code)) { + if (ID.equals(code)) { List values = queryParm.getValues().stream().map(p -> p.getValueCode()).collect(Collectors.toList()); if (values.size() == 1) { filter.col(parentAlias, "LOGICAL_ID").eq().bind(values.get(0)); @@ -812,7 +980,7 @@ protected WhereFragment getFilterPredicate(QueryData queryData, QueryParameter q } else { throw new FHIRPersistenceException("_id parameter value list is empty"); } - } else if ("_lastUpdated".equals(code)) { + } else if (LAST_UPDATED.equals(code)) { // Compute the _lastUpdated filter predicate for the given query parameter NewLastUpdatedParmBehaviorUtil util = new NewLastUpdatedParmBehaviorUtil(parentAlias); util.executeBehavior(filter, queryParm); @@ -824,7 +992,7 @@ protected WhereFragment getFilterPredicate(QueryData queryData, QueryParameter q // AND P3.PARAMETER_NAME_ID = 123 -- 'name parameter' // AND P3.STR_VALUE = 'Jones') -- 'name filter' final int aliasIndex = getNextAliasIndex(); - final String paramTable = paramValuesTableName(queryData.getResourceType(), queryParm.getType()); + final String paramTable = paramValuesTableName(queryData.getResourceType(), queryParm); final String paramAlias = getParamAlias(aliasIndex); SelectAdapter exists = Select.select("1"); exists.from(paramTable, alias(paramAlias)) @@ -1307,13 +1475,32 @@ public QueryData addRevIncludeFilter(QueryData queryData, InclusionParameter inc return queryData; } + @Override + public QueryData addWholeSystemDataFilter(QueryData queryData, String resourceType, List logicalResourceIds) throws FHIRPersistenceException { + // Build the IN clause for the logical resource IDs. + SelectAdapter select = queryData.getQuery(); + select.from().where().and("LR", "LOGICAL_RESOURCE_ID").inLiteralLong(logicalResourceIds); + + return queryData; + } + + @Override + public QueryData addWholeSystemResourceTypeFilter(QueryData queryData, List resourceTypeIds) throws FHIRPersistenceException { + // Build the IN clause for the resource type IDs. + List longResourceTypeIds = resourceTypeIds.stream().map(i -> i.longValue()).collect(Collectors.toList()); + SelectAdapter select = queryData.getQuery(); + select.from().where().and("LR0", "RESOURCE_TYPE_ID").inLiteralLong(longResourceTypeIds); + + return queryData; + } + @Override public QueryData addTokenParam(QueryData queryData, String resourceType, QueryParameter queryParm) throws FHIRPersistenceException { // Add a join to the query. The NOT/NOT_IN modifiers are trickier because // they need to be handled as a NOT EXISTS clause. final int aliasIndex = getNextAliasIndex(); final SelectAdapter query = queryData.getQuery(); - final String paramAlias = "P" + aliasIndex; + final String paramAlias = getParamAlias(aliasIndex); final String lrAlias = queryData.getLRAlias(); // join to LR at the same query level final ExpNode filter; filter = getTokenFilter(queryParm, paramAlias).getExpression(); @@ -1330,8 +1517,8 @@ public QueryData addTokenParam(QueryData queryData, String resourceType, QueryPa // Use a nested NOT EXISTS (...) instead of a simple join SelectAdapter exists = Select.select("1"); exists.from(xxTokenValues, alias(paramAlias)) - .where(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID") // correlate with the main query - .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(parameterName)); + .where(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID") // correlate with the main query + .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(parameterName)); // add the filter predicate to the exists where clause exists.from().where().and(filter); @@ -1347,13 +1534,102 @@ public QueryData addTokenParam(QueryData queryData, String resourceType, QueryPa return queryData; } + @Override + public QueryData addTagParam(QueryData queryData, String resourceType, QueryParameter queryParm) throws FHIRPersistenceException { + return addGlobalTokenParam(queryData, queryParm, + Modifier.TEXT.equals(queryParm.getModifier()) ? "RESOURCE_TOKEN_REFS" : + isWholeSystemSearch(resourceType) ? "LOGICAL_RESOURCE_TAGS" : resourceType + "_TAGS"); + } + + @Override + public QueryData addSecurityParam(QueryData queryData, String resourceType, QueryParameter queryParm) throws FHIRPersistenceException { + return addGlobalTokenParam(queryData, queryParm, + Modifier.TEXT.equals(queryParm.getModifier()) ? "RESOURCE_TOKEN_REFS" : + isWholeSystemSearch(resourceType) ? "LOGICAL_RESOURCE_SECURITY" : resourceType + "_SECURITY"); + } + + private QueryData addGlobalTokenParam(QueryData queryData, QueryParameter queryParm, String parameterTable) throws FHIRPersistenceException { + // Add a join to the query. The NOT/NOT_IN modifiers are trickier because + // they need to be handled as a NOT EXISTS clause. + final int aliasIndex = getNextAliasIndex(); + final SelectAdapter query = queryData.getQuery(); + final String paramAlias = getParamAlias(aliasIndex); + final String lrAlias = queryData.getLRAlias(); // join to LR at the same query level + + // Global tokens are stored in their own parameter table and therefore don't need a + // parameter_name_id but otherwise are just like any other token. + final ExpNode filter = getTokenFilter(queryParm, paramAlias).getExpression(); + + boolean tokenValueSearch = false; + String tokenValuesAlias = paramAlias; + if (Modifier.IN.equals(queryParm.getModifier()) || Modifier.NOT_IN.equals(queryParm.getModifier()) || + Modifier.ABOVE.equals(queryParm.getModifier()) || Modifier.BELOW.equals(queryParm.getModifier()) || + Modifier.TEXT.equals(queryParm.getModifier())) { + // For a search against a global search parameter where we need access to the search parameter value, we + // have to join the common token values table (which contains the parameter value) to the parameter table, + // which only contains a common_token_value_id. + // Since the filter is using paramAlias and the filter will get ANDed to the common token values join, we + // need to use paramAlias as the common token values alias and generate a new alias for the parameter table. + tokenValuesAlias = getParamAlias(getNextAliasIndex()); + tokenValueSearch = true; + } + + if (Modifier.NOT.equals(queryParm.getModifier()) || Modifier.NOT_IN.equals(queryParm.getModifier())) { + // Use a nested NOT EXISTS (...) instead of a simple join + SelectAdapter exists = Select.select("1"); + exists.from(parameterTable, alias(tokenValuesAlias)) + .where(tokenValuesAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID"); // correlate with the main query + if (tokenValueSearch) { + if (Modifier.TEXT.equals(queryParm.getModifier())) { + exists.from().where().and(tokenValuesAlias, "PARAMETER_NAME_ID") + .eq(getParameterNameId(queryParm.getCode() + SearchConstants.TEXT_MODIFIER_SUFFIX)); + } + // Join to common token values table and add filter predicate to the common token values join on clause + exists.from().innerJoin("COMMON_TOKEN_VALUES", alias(paramAlias), on(paramAlias, "COMMON_TOKEN_VALUE_ID") + .eq(tokenValuesAlias, "COMMON_TOKEN_VALUE_ID") + .and(filter)); + } else { + // Add the filter predicate to the exists where clause + exists.from().where().and(filter); + } + // Add as a not exists to the main query + query.from().where().and().notExists(exists.build()); + } else { + if (tokenValueSearch) { + if (Modifier.TEXT.equals(queryParm.getModifier())) { + query.from().where().and(tokenValuesAlias, "PARAMETER_NAME_ID") + .eq(getParameterNameId(queryParm.getCode() + SearchConstants.TEXT_MODIFIER_SUFFIX)); + } + // Join the xx_TAGS table and the common token values table to the exists + query.from().innerJoin(parameterTable, alias(tokenValuesAlias), on(tokenValuesAlias, "LOGICAL_RESOURCE_ID") + .eq(lrAlias, "LOGICAL_RESOURCE_ID")); + query.from().innerJoin("COMMON_TOKEN_VALUES", alias(paramAlias), on(paramAlias, "COMMON_TOKEN_VALUE_ID") + .eq(tokenValuesAlias, "COMMON_TOKEN_VALUE_ID") + .and(filter)); + } else { + // Attach the parameter table to the single parameter exists join + query.from().innerJoin(parameterTable, alias(tokenValuesAlias), on(tokenValuesAlias, "LOGICAL_RESOURCE_ID") + .eq(lrAlias, "LOGICAL_RESOURCE_ID") + .and(filter)); + } + } + + // We're not changing the level, so we return the same queryData we were given + return queryData; + } + @Override public QueryData addStringParam(QueryData queryData, String resourceType, QueryParameter queryParm) throws FHIRPersistenceException { // Join to the string parameter table // Attach an exists clause to filter the result based on the string query parameter definition final int aliasIndex = getNextAliasIndex(); final String lrAlias = queryData.getLRAlias(); - final String paramTableName = resourceType + "_STR_VALUES"; + final String paramTableName; + if (isWholeSystemSearch(resourceType)) { + paramTableName = "STR_VALUES"; + } else { + paramTableName = resourceType + "_STR_VALUES"; + } final String paramAlias = getParamAlias(aliasIndex); final String parameterName = queryParm.getCode(); @@ -1368,6 +1644,67 @@ public QueryData addStringParam(QueryData queryData, String resourceType, QueryP return queryData; } + @Override + public QueryData addCanonicalParam(QueryData queryData, String resourceType, QueryParameter queryParm) throws FHIRPersistenceException { + // Join to the canonical parameter table...which in this case means the xx_profiles table + // because current _profile is the only search parameter to be defined as a canonical + final int aliasIndex = getNextAliasIndex(); + final String lrAlias = queryData.getLRAlias(); + final String paramTableName; + if (isWholeSystemSearch(resourceType)) { + paramTableName = "LOGICAL_RESOURCE_PROFILES"; + } else { + paramTableName = resourceType + "_PROFILES"; + } + final String paramAlias = getParamAlias(aliasIndex); + final String parameterName = queryParm.getCode(); + if (!"_profile".equals(parameterName)) { + throw new FHIRPersistenceException("Only _profile is supported as a canonical"); + } + + // Build the filter predicate for the canonical values, handling the parse + WhereFragment whereFragment = new WhereFragment(); + whereFragment.leftParen(); + + boolean multiple = false; + for (QueryParameterValue value : queryParm.getValues()) { + // Concatenate multiple matches with an OR + if (multiple) { + whereFragment.or(); + } else { + multiple = true; + } + + // Reuse the same CanonicalSupport code used for param extraction to parse the search value + ResourceProfileRec rpc = CanonicalSupport.makeResourceProfileRec(-1, resourceType, -1, -1, value.getValueString(), false); + int canonicalId = getCanonicalId(rpc.getCanonicalValue()); + whereFragment.col(paramAlias, "CANONICAL_ID").eq(canonicalId); + + // TODO double-check semantics of ABOVE and BELOW in this context + if (rpc.getVersion() != null && !rpc.getVersion().isEmpty()) { + if (queryParm.getModifier() == Modifier.ABOVE) { + whereFragment.and(paramAlias, "VERSION").gte().bind(rpc.getVersion()); + } else if (queryParm.getModifier() == Modifier.BELOW) { + whereFragment.and(paramAlias, "VERSION").lt().bind(rpc.getVersion()); + } else { + whereFragment.and(paramAlias, "VERSION").eq().bind(rpc.getVersion()); + } + } + + if (rpc.getFragment() != null && !rpc.getFragment().isEmpty()) { + whereFragment.and(paramAlias, "FRAGMENT").eq().bind(rpc.getFragment()); + } + } + + whereFragment.rightParen(); + + SelectAdapter query = queryData.getQuery(); + query.from().innerJoin(paramTableName, alias(paramAlias), on(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID") + .and(whereFragment.getExpression())); + + return queryData; + } + @Override public QueryData addMissingParam(QueryData queryData, QueryParameter queryParm, boolean isMissing) throws FHIRPersistenceException { // note that there's no filter here to look for a specific value. We simply want to know @@ -1375,15 +1712,19 @@ public QueryData addMissingParam(QueryData queryData, QueryParameter queryParm, final String parameterName = queryParm.getCode(); final int aliasIndex = getNextAliasIndex(); final String resourceType = queryData.getResourceType(); - final String paramTableName = paramValuesTableName(resourceType, queryParm.getType()); + final String paramTableName = paramValuesTableName(resourceType, queryParm); final String lrAlias = queryData.getLRAlias(); - final String paramAlias = "P" + aliasIndex; + final String paramAlias = getParamAlias(aliasIndex); SelectAdapter exists = Select.select("1"); exists.from(paramTableName, alias(paramAlias)) - .where(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID") // correlate with the main query - .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(parameterName)) - ; + .where(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID"); // correlate with the main query + + // Do not need PARAMETER_NAME_ID clause for _profile, _tag, or _security parameters since they have + // their own tables. + if (!PROFILE.equals(parameterName) && !SECURITY.equals(parameterName) && !TAG.equals(parameterName)) { + exists.from().where().and(paramAlias, PARAMETER_NAME_ID).eq(getParameterNameId(parameterName)); + } // Add the exists to the where clause of the main query which already has a predicate // so we need to AND the exists @@ -1419,7 +1760,7 @@ public QueryData addChained(QueryData queryData, QueryParameter currentParm) thr final String targetResourceType = currentParm.getModifierResourceTypeName(); final String tokenValues = sourceResourceType + "_TOKEN_VALUES_V"; // because we need TOKEN_VALUE final String xxLogicalResources = targetResourceType + "_LOGICAL_RESOURCES"; - final String paramAlias = "P" + aliasIndex; + final String paramAlias = getParamAlias(aliasIndex); final String lrAlias = "LR" + aliasIndex; final String lrPrevAlias = queryData.getLRAlias(); final Integer codeSystemIdForTargetResourceType = getCodeSystemId(targetResourceType); @@ -1450,9 +1791,9 @@ public void addFilter(QueryData queryData, QueryParameter currentParm) throws FH final String code = currentParm.getCode(); final String lrAlias = queryData.getLRAlias(); - if ("_id".equals(code)) { + if (ID.equals(code)) { addIdFilter(queryData, currentParm); - } else if ("_lastUpdated".equals(code)) { + } else if (LAST_UPDATED.equals(code)) { // Compute the _lastUpdated filter predicate for the given query parameter NewLastUpdatedParmBehaviorUtil util = new NewLastUpdatedParmBehaviorUtil(lrAlias); WhereFragment filter = new WhereFragment(); @@ -1468,7 +1809,7 @@ public void addFilter(QueryData queryData, QueryParameter currentParm) throws FH // AND P3.PARAMETER_NAME_ID = 123 -- 'name parameter' // AND P3.STR_VALUE = 'Jones') -- 'name filter' final int aliasIndex = getNextAliasIndex(); - final String paramTable = paramValuesTableName(queryData.getResourceType(), currentParm.getType()); + final String paramTable = paramValuesTableName(queryData.getResourceType(), currentParm); final String paramAlias = getParamAlias(aliasIndex); WhereFragment pf = paramFilter(currentParm, paramAlias); @@ -1476,19 +1817,28 @@ public void addFilter(QueryData queryData, QueryParameter currentParm) throws FH // Needs to be handled as a NOT EXISTS correlated subquery SelectAdapter exists = Select.select("1"); exists.from(paramTable, alias(paramAlias)) - .where(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID") // correlate to parent query - .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(currentParm.getCode())) - .and(pf.getExpression()); + .where(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID"); // correlate to parent query + if (!PROFILE.equals(code) && !SECURITY.equals(code) && !TAG.equals(code)) { + exists.from().where().and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(currentParm.getCode())); + } + exists.from().where().and(pf.getExpression()); // Add the sub-query as a NOT EXISTS filter to the main query currentSubQuery.from().where().and().notExists(exists.build()); } else { // Filter the query by adding a join - currentSubQuery.from() - .innerJoin(paramTable, alias(paramAlias), - on(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID") - .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(currentParm.getCode())) - .and(pf.getExpression())); + if (!PROFILE.equals(code) && !SECURITY.equals(code) && !TAG.equals(code)) { + currentSubQuery.from() + .innerJoin(paramTable, alias(paramAlias), + on(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID") + .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(currentParm.getCode())) + .and(pf.getExpression())); + } else { + currentSubQuery.from() + .innerJoin(paramTable, alias(paramAlias), + on(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID") + .and(pf.getExpression())); + } } } } @@ -1512,7 +1862,7 @@ public QueryData addReverseChained(QueryData queryData, QueryParameter currentPa final String resourceTypeName = currentParm.getModifierResourceTypeName(); final String tokenValues = resourceTypeName + "_TOKEN_VALUES_V"; final String xxLogicalResources = resourceTypeName + "_LOGICAL_RESOURCES"; - final String paramAlias = "P" + aliasIndex; + final String paramAlias = getParamAlias(aliasIndex); final String lrAlias = "LR" + aliasIndex; final String lrPrevAlias = queryData.getLRAlias(); final Integer codeSystemIdForRefResourceType = getCodeSystemId(refResourceType); @@ -1542,7 +1892,7 @@ public QueryData addNumberParam(QueryData queryData, String resourceType, QueryP final int aliasIndex = getNextAliasIndex(); final SelectAdapter query = queryData.getQuery(); final String paramTableName = resourceType + "_NUMBER_VALUES"; - final String paramAlias = "P" + aliasIndex; + final String paramAlias = getParamAlias(aliasIndex); final String lrAlias = queryData.getLRAlias(); ExpNode filter = getNumberFilter(queryParm, paramAlias).getExpression(); @@ -1560,7 +1910,7 @@ public QueryData addQuantityParam(QueryData queryData, String resourceType, Quer final int aliasIndex = getNextAliasIndex(); final SelectAdapter query = queryData.getQuery(); final String paramTableName = resourceType + "_QUANTITY_VALUES"; - final String paramAlias = "P" + aliasIndex; + final String paramAlias = getParamAlias(aliasIndex); final String lrAlias = queryData.getLRAlias(); ExpNode filter = getQuantityFilter(queryParm, paramAlias).getExpression(); @@ -1578,7 +1928,7 @@ public QueryData addDateParam(QueryData queryData, String resourceType, QueryPar final int aliasIndex = getNextAliasIndex(); final SelectAdapter query = queryData.getQuery(); final String paramTableName = resourceType + "_DATE_VALUES"; - final String paramAlias = "P" + aliasIndex; + final String paramAlias = getParamAlias(aliasIndex); final String lrAlias = queryData.getLRAlias(); ExpNode filter = getDateFilter(queryParm, paramAlias).getExpression(); query.from().innerJoin(paramTableName, alias(paramAlias), on(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID") @@ -1594,7 +1944,7 @@ public QueryData addLocationParam(QueryData queryData, String resourceType, Quer final int aliasIndex = getNextAliasIndex(); final SelectAdapter query = queryData.getQuery(); final String paramTableName = resourceType + "_LATLNG_VALUES"; - final String paramAlias = "P" + aliasIndex; + final String paramAlias = getParamAlias(aliasIndex); final String lrAlias = queryData.getLRAlias(); ExpNode filter = getLocationFilter(queryParm, paramAlias).getExpression(); query.from().innerJoin(paramTableName, alias(paramAlias), on(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID") @@ -1608,7 +1958,7 @@ public QueryData addLocationParam(QueryData queryData, String resourceType, Quer public QueryData addReferenceParam(QueryData queryData, String resourceType, QueryParameter queryParm) throws FHIRPersistenceException { final int aliasIndex = getNextAliasIndex(); final SelectAdapter query = queryData.getQuery(); - final String paramAlias = "P" + aliasIndex; + final String paramAlias = getParamAlias(aliasIndex); final String lrAlias = queryData.getLRAlias(); // Grab the filter expression first. We can then inspect the expression to @@ -1702,7 +2052,7 @@ public QueryData addCompositeParam(QueryData queryData, QueryParameter queryParm private int addCompositeParamTable(SelectAdapter query, String resourceType, String lrAlias, QueryParameter component, int componentNum, int firstAliasIndex) throws FHIRPersistenceException { final int aliasIndex = getNextAliasIndex(); - String paramTableAlias = "P" + aliasIndex; + String paramTableAlias = getParamAlias(aliasIndex); String parameterName = component.getCode(); // Grab the parameter filter expression first so that we can see if it's safe to apply @@ -1724,7 +2074,7 @@ private int addCompositeParamTable(SelectAdapter query, String resourceType, Str } else { // also join to the first parameter table - final String firstTableAlias = "P" + firstAliasIndex; + final String firstTableAlias = getParamAlias(firstAliasIndex); query.from().innerJoin(valuesTable, alias(paramTableAlias), on(paramTableAlias, "LOGICAL_RESOURCE_ID").eq(firstTableAlias, "LOGICAL_RESOURCE_ID") .and(paramTableAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(parameterName)) @@ -1847,24 +2197,37 @@ public void addSortParam(QueryData queryData, String code, Type type, Direction addAggregateAndOrderByExpressions(queryData, code, type, direction, paramAlias); // Now add the parameter table as an outer join - final String paramTable = getSortParameterTableName(queryData.getResourceType(), type); + final String paramTable = getSortParameterTableName(queryData.getResourceType(), code, type); final String lrAlias = queryData.getLRAlias(); - query.from() - .leftOuterJoin(paramTable, alias(paramAlias), - on(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID") - .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(code))); + if (PROFILE.equals(code) || SECURITY.equals(code) || TAG.equals(code)) { + // For a sort by _tag, _profile, or _security we need to join the parameter-specific token + // table with the common token values table. + String parameterTableAlias = getParamAlias(getNextAliasIndex()); + query.from() + .leftOuterJoin(paramTable, alias(parameterTableAlias), + on(parameterTableAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID")); + query.from() + .innerJoin("COMMON_TOKEN_VALUES", alias(paramAlias), + on(paramAlias, "COMMON_TOKEN_VALUE_ID").eq(parameterTableAlias, "COMMON_TOKEN_VALUE_ID")); + } else { + query.from() + .leftOuterJoin(paramTable, alias(paramAlias), + on(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID") + .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(code))); + } } /** - * Returns the name of the database table corresponding to the type of the + * Returns the name of the database table corresponding to the code and type of the * passed sort parameter. * - * @param sortParm A valid SortParameter + * @param code A SortParameter code + * @param type A SortParameter type * @return String - A database table name * @throws FHIRPersistenceException */ - protected String getSortParameterTableName(String resourceType, Type type) throws FHIRPersistenceException { + protected String getSortParameterTableName(String resourceType, String code, Type type) throws FHIRPersistenceException { final String METHODNAME = "getSortParameterTableName"; logger.entering(CLASSNAME, METHODNAME); @@ -1874,14 +2237,24 @@ protected String getSortParameterTableName(String resourceType, Type type) throw switch (type) { case URI: case STRING: - sortParameterTableName.append("STR_VALUES"); + if (PROFILE.equals(code)) { + sortParameterTableName.append("PROFILES"); + } else { + sortParameterTableName.append("STR_VALUES"); + } break; case DATE: sortParameterTableName.append("DATE_VALUES"); break; case REFERENCE: case TOKEN: - sortParameterTableName.append("TOKEN_VALUES_V"); + if (TAG.equals(code)) { + sortParameterTableName.append("TAGS"); + } else if (SECURITY.equals(code)) { + sortParameterTableName.append("SECURITY"); + } else { + sortParameterTableName.append("TOKEN_VALUES_V"); + } break; case NUMBER: sortParameterTableName.append("NUMBER_VALUES"); @@ -1914,7 +2287,11 @@ private void addAggregateAndOrderByExpressions(QueryData queryData, String code, SelectAdapter query = queryData.getQuery(); List valueAttributeNames; - valueAttributeNames = this.getValueAttributeNames(type); + if (PROFILE.equals(code) || SECURITY.equals(code) || TAG.equals(code)) { + valueAttributeNames = Collections.singletonList(TOKEN_VALUE); + } else { + valueAttributeNames = this.getValueAttributeNames(type); + } for (String attributeName : valueAttributeNames) { StringBuilder expression = new StringBuilder(); final String dirExp; @@ -1985,4 +2362,13 @@ private List getValueAttributeNames(Type type) throws FHIRPersistenceExc logger.exiting(CLASSNAME, METHODNAME); return attributeNames; } + + /** + * Check if whole-system search. + * @param resourceType + * @return true if whole-system search, false otherwise + */ + private boolean isWholeSystemSearch(String resourceType) { + return Resource.class.getSimpleName().equals(resourceType); + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryVisitor.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryVisitor.java index e7a2f76d02d..d1306ac8aa9 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryVisitor.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryVisitor.java @@ -76,6 +76,27 @@ public interface SearchQueryVisitor { */ T sortRoot(String rootResourceType); + /** + * The root of the FHIR whole-system filter search query + * @return + */ + T wholeSystemFilterRoot(); + + /** + * The root of the FHIR whole-system data search query + * @param rootResourceType + * @return + */ + T wholeSystemDataRoot(String rootResourceType); + + /** + * The wrapper for whole-system search + * @param queries + * @param isCountQuery + * @return + */ + T wrapWholeSystem(List queries, boolean isCountQuery); + /** * Filter the query using the given parameter id and token value * @param query @@ -86,6 +107,26 @@ public interface SearchQueryVisitor { */ T addTokenParam(T query, String resourceType, QueryParameter queryParm) throws FHIRPersistenceException; + /** + * Filter the query using the given tag query parameter + * @param query + * @param resourceType + * @param queryParm + * @return + * @throws FHIRPersistenceException + */ + T addTagParam(T query, String resourceType, QueryParameter queryParm) throws FHIRPersistenceException; + + /** + * Filter the query using the given security query parameter + * @param query + * @param resourceType + * @param queryParm + * @return + * @throws FHIRPersistenceException + */ + T addSecurityParam(T query, String resourceType, QueryParameter queryParm) throws FHIRPersistenceException; + /** * Filter the query using the given string parameter * @param query @@ -95,6 +136,15 @@ public interface SearchQueryVisitor { */ T addStringParam(T query, String resourceType, QueryParameter queryParm) throws FHIRPersistenceException; + /** + * Filter the query using the given canonical parameter + * @param query + * @param resourceType + * @param queryParm + * @return + */ + T addCanonicalParam(T query, String resourceType, QueryParameter queryParm) throws FHIRPersistenceException; + /** * Filter the query using the given number parameter * @param queryData @@ -192,6 +242,15 @@ public interface SearchQueryVisitor { */ T addSorting(T query, String lrAlias); + /** + * Add sorting (order by) for whole-system search to the query + * @param query + * @param sortParms + * @param lrAlias + * @return + */ + T addWholeSystemSorting(T query, List sortParms, String lrAlias); + /** * Add pagination (LIMIT/OFFSET) to the query * @param query @@ -246,6 +305,21 @@ public interface SearchQueryVisitor { */ T addRevIncludeFilter(T query, InclusionParameter inclusionParm, List logicalResourceIds) throws FHIRPersistenceException; + /** + * @param query + * @param resourceType + * @param logicalResourceIds + * @return + */ + T addWholeSystemDataFilter(T query, String resourceType, List logicalResourceIds) throws FHIRPersistenceException; + + /** + * @param query + * @param resourceTypeIds + * @return + */ + T addWholeSystemResourceTypeFilter(T query, List resourceTypeIds) throws FHIRPersistenceException; + /** * Add the given sort parameter to the sort query * @param queryData diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchWholeSystemDataQuery.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchWholeSystemDataQuery.java new file mode 100644 index 00000000000..fda89c66c07 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchWholeSystemDataQuery.java @@ -0,0 +1,48 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.domain; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; + +/** + * Domain model of the FHIR search context representing the query used + * to perform the search operation in the database. The query built by + * this class fetches the resource data for whole system searches. + */ +public class SearchWholeSystemDataQuery extends SearchQuery { + + /** + * Public constructor + * @param resourceType + */ + public SearchWholeSystemDataQuery(String resourceType) { + super(resourceType); + } + + @Override + public T getRoot(SearchQueryVisitor visitor) { + return visitor.wholeSystemDataRoot(getRootResourceType()); + } + + @Override + public T visit(SearchQueryVisitor visitor) throws FHIRPersistenceException { + + // get the core data query + // NOTE: we don't call super.visit() here because it would add an + // unnecessary EXISTS clause to the query when processing parameters. + // We never have parameters so don't need that processing. + T query = getRoot(visitor); + + // Pre-process the whole-system data extension + visitExtensions(query, visitor); + + // Join the core logical resource selection to the resource versions table + query = visitor.joinResources(query); + + return query; + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchWholeSystemFilterQuery.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchWholeSystemFilterQuery.java new file mode 100644 index 00000000000..6719afb17ba --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchWholeSystemFilterQuery.java @@ -0,0 +1,58 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.domain; + +import java.util.ArrayList; +import java.util.List; + +import com.ibm.fhir.model.resource.Resource; +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; + +/** + * Domain model of the FHIR search context representing the query used + * to perform the search operation in the database. The query built by + * this class fetches the filter data for whole system searches, which + * is a set of logical resource id + resource type. + */ +public class SearchWholeSystemFilterQuery extends SearchQuery { + + // Sort parameters + final List sortParameters = new ArrayList<>(); + + /** + * Public constructor + */ + public SearchWholeSystemFilterQuery() { + super(Resource.class.getSimpleName()); + } + + /** + * Add the given sort parameter to the sortParameters list. + * @param dsp + */ + public void add(DomainSortParameter dsp) { + this.sortParameters.add(dsp); + } + + @Override + public T getRoot(SearchQueryVisitor visitor) { + return visitor.wholeSystemFilterRoot(); + } + + @Override + public T visit(SearchQueryVisitor visitor) throws FHIRPersistenceException { + + // get the core data query + T query = super.visit(visitor); + + // now attach the requisite ordering and pagination clauses + query = visitor.addWholeSystemSorting(query, sortParameters, "LR0"); + query = visitor.addPagination(query); + + return query; + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchWholeSystemQuery.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchWholeSystemQuery.java new file mode 100644 index 00000000000..2cdc1184b4a --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchWholeSystemQuery.java @@ -0,0 +1,81 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.domain; + +import java.util.ArrayList; +import java.util.List; + +import com.ibm.fhir.model.resource.Resource; +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; + +/** + * A domain model of the query used to retrieve the count or data + * for a particular whole-system search. + */ +public class SearchWholeSystemQuery extends SearchQuery { + + // Set of domain models for resource types used in this whole-system search + List domainModels; + + // Flag indicating if a count query or a data query + boolean isCountQuery; + + // Is pagination required? + boolean addPagination = true; + + // Sort parameters + final List sortParameters = new ArrayList<>(); + + /** + * Public constructor + * @param domainModels + * @param isCountQuery + * @param addPagination + */ + public SearchWholeSystemQuery(List domainModels, boolean isCountQuery, boolean addPagination) { + super(Resource.class.getSimpleName()); + this.domainModels = domainModels; + this.isCountQuery = isCountQuery; + this.addPagination = addPagination; + } + + /** + * Add the given sort parameter to the sortParameters list. + * @param dsp + */ + public void add(DomainSortParameter dsp) { + this.sortParameters.add(dsp); + } + + @Override + public T getRoot(SearchQueryVisitor visitor) { + return null; + } + + @Override + public T visit(SearchQueryVisitor visitor) throws FHIRPersistenceException { + + // Get the core data queries + List queries = new ArrayList<>(); + for (SearchQuery domainModel : domainModels) { + queries.add(domainModel.visit(visitor)); + } + + // Get the overall query + T query = visitor.wrapWholeSystem(queries, isCountQuery); + + // Add sorting and pagination + if (!isCountQuery) { + query = visitor.addWholeSystemSorting(query, sortParameters, "COMBINED_RESULTS"); + if (addPagination) { + query = visitor.addPagination(query); + } + } + + return query; + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SecuritySearchParam.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SecuritySearchParam.java new file mode 100644 index 00000000000..9d73b72ad63 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SecuritySearchParam.java @@ -0,0 +1,32 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.domain; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.search.parameters.QueryParameter; + +/** + * A token search parameter for the _security search parameter + */ +public class SecuritySearchParam extends SearchParam { + + /** + * Public constructor + * @param rootResourceName + * @param name + * @param queryParameter + */ + public SecuritySearchParam(String rootResourceName, String name, QueryParameter queryParameter) { + super(rootResourceName, name, queryParameter); + } + + @Override + public T visit(T queryData, SearchQueryVisitor visitor) throws FHIRPersistenceException { + QueryParameter queryParm = getQueryParameter(); + return visitor.addSecurityParam(queryData, getRootResourceType(), queryParm); + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/TagSearchParam.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/TagSearchParam.java new file mode 100644 index 00000000000..dce7a36257e --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/TagSearchParam.java @@ -0,0 +1,32 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.domain; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.search.parameters.QueryParameter; + +/** + * A token search parameter + */ +public class TagSearchParam extends SearchParam { + + /** + * Public constructor + * @param rootResourceName + * @param name + * @param queryParameter + */ + public TagSearchParam(String rootResourceName, String name, QueryParameter queryParameter) { + super(rootResourceName, name, queryParameter); + } + + @Override + public T visit(T queryData, SearchQueryVisitor visitor) throws FHIRPersistenceException { + QueryParameter queryParm = getQueryParameter(); + return visitor.addTagParam(queryData, getRootResourceType(), queryParm); + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/CompositeParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/CompositeParmVal.java index d9ffb05a954..549ddfb90cc 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/CompositeParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/CompositeParmVal.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2017,2019 + * (C) Copyright IBM Corp. 2017, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -12,38 +12,19 @@ import com.ibm.fhir.persistence.exception.FHIRPersistenceException; /** - * This class defines the Data Transfer Object representing a row in the X_DATE_VALUES tables. + * This class defines the Data Transfer Object representing a composite parameter. */ -public class CompositeParmVal implements ExtractedParameterValue { - - private String resourceType; - private String name; +public class CompositeParmVal extends ExtractedParameterValue { + private List component; - - // The SearchParameter base type. If "Resource", then this is a Resource-level attribute - private String base; + /** + * Public constructor + */ public CompositeParmVal() { - super(); component = new ArrayList<>(2); } - public void setName(String name) { - this.name = name; - } - - public String getName() { - return name; - } - - public String getResourceType() { - return resourceType; - } - - public void setResourceType(String resourceType) { - this.resourceType = resourceType; - } - /** * We know our type, so we can call the correct method on the visitor */ @@ -51,20 +32,6 @@ public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenc visitor.visit(this); } - /** - * @return the base - */ - public String getBase() { - return base; - } - - /** - * @param base the base to set - */ - public void setBase(String base) { - this.base = base; - } - /** * @return get the list of components in this composite parameter */ @@ -87,4 +54,4 @@ public void addComponent(ExtractedParameterValue... component) { this.component.add(value); } } -} +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/DateParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/DateParmVal.java index e82395f3a71..fc14de1b793 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/DateParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/DateParmVal.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2017,2019 + * (C) Copyright IBM Corp. 2017, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -13,16 +13,11 @@ /** * This class defines the Data Transfer Object representing a row in the X_DATE_VALUES tables. */ -public class DateParmVal implements ExtractedParameterValue { +public class DateParmVal extends ExtractedParameterValue { - private String resourceType; - private String name; private Timestamp valueDateStart; private Timestamp valueDateEnd; - // The SearchParameter base type. If "Resource", then this is a Resource-level attribute - private String base; - public enum TimeType { YEAR, YEAR_MONTH, @@ -52,22 +47,6 @@ public void setValueDateEnd(Timestamp valueDateEnd) { this.valueDateEnd = valueDateEnd; } - public void setName(String name) { - this.name = name; - } - - public String getName() { - return name; - } - - public String getResourceType() { - return resourceType; - } - - public void setResourceType(String resourceType) { - this.resourceType = resourceType; - } - /** * We know our type, so we can call the correct method on the visitor */ @@ -75,23 +54,9 @@ public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenc visitor.visit(this); } - /** - * @return the base - */ - public String getBase() { - return base; - } - - /** - * @param base the base to set - */ - public void setBase(String base) { - this.base = base; - } - @Override public String toString() { - return "DateParmVal [resourceType=" + resourceType + ", name=" + name - + ", valueDateStart=" + valueDateStart + ", valueDateEnd=" + valueDateEnd + ", base=" + base + "]"; + return "DateParmVal [resourceType=" + getResourceType() + ", name=" + getName() + + ", valueDateStart=" + valueDateStart + ", valueDateEnd=" + valueDateEnd + ", base=" + getBase() + "]"; } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ExtractedParameterValue.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ExtractedParameterValue.java index 61ab25875c1..b5ff89aa54a 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ExtractedParameterValue.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ExtractedParameterValue.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2017,2019 + * (C) Copyright IBM Corp. 2017, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -11,27 +11,86 @@ /** * A search parameter value extracted from a resource and ready to store / index for search */ -public interface ExtractedParameterValue { +public abstract class ExtractedParameterValue { - public void setName(String name); + // The name (code) of this parameter + private String name; - public String getName(); + // A subset of search params are also stored at the whole-system level + private boolean wholeSystem; - public String getResourceType(); - public void setResourceType(String resourceType); + // The resource type associated with this parameter + private String resourceType; + + // The base resource name + private String base; + + /** + * Protected constructor + */ + protected ExtractedParameterValue() { + } + + /** + * Getter for the parameter's resource type + * @return + */ + public String getResourceType() { + return this.resourceType; + } + + /** + * Setter for the parameter's resource type + * @param resourceType + */ + public void setResourceType(String resourceType) { + this.resourceType = resourceType; + } /** * We know our type, so we can call the correct method on the visitor */ - public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException; + public abstract void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException; /** * @return the base */ - public String getBase(); + public String getBase() { + return this.base; + } /** * @param base the base to set */ - public void setBase(String base); -} + public void setBase(String base) { + this.base = base; + } + + /** + * @return the wholeSystem + */ + public boolean isWholeSystem() { + return wholeSystem; + } + + /** + * @param wholeSystem the wholeSystem to set + */ + public void setWholeSystem(boolean wholeSystem) { + this.wholeSystem = wholeSystem; + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param name the name to set + */ + public void setName(String name) { + this.name = name; + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/LocationParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/LocationParmVal.java index 6b72db24dff..da67655aedc 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/LocationParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/LocationParmVal.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2017,2019 + * (C) Copyright IBM Corp. 2017, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -11,28 +11,18 @@ /** * This class defines the Data Transfer Object representing a row in the X_LATLNG_VALUES tables. */ -public class LocationParmVal implements ExtractedParameterValue { - - private String resourceType; - private String name; +public class LocationParmVal extends ExtractedParameterValue { + private Double valueLongitude; private Double valueLatitude; - - // The SearchParameter base type. If "Resource", then this is a Resource-level attribute - private String base; - + + /** + * Public constructor + */ public LocationParmVal() { super(); } - public void setName(String name) { - this.name = name; - } - - public String getName() { - return name; - } - public Double getValueLongitude() { return valueLongitude; } @@ -49,32 +39,10 @@ public void setValueLatitude(Double valueLatitude) { this.valueLatitude = valueLatitude; } - public String getResourceType() { - return resourceType; - } - - public void setResourceType(String resourceType) { - this.resourceType = resourceType; - } - /** * We know our type, so we can call the correct method on the visitor */ public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { visitor.visit(this); } - - /** - * @return the base - */ - public String getBase() { - return base; - } - - /** - * @param base the base to set - */ - public void setBase(String base) { - this.base = base; - } -} +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/NumberParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/NumberParmVal.java index c015816ab5a..4897bb48f3e 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/NumberParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/NumberParmVal.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2017,2019 + * (C) Copyright IBM Corp. 2017, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -13,29 +13,21 @@ /** * This class defines the Data Transfer Object representing a row in the X_NUMBER_VALUES tables. */ -public class NumberParmVal implements ExtractedParameterValue { - - private String resourceType; - private String name; +public class NumberParmVal extends ExtractedParameterValue { + private BigDecimal valueNumber; private BigDecimal valueNumberLow; private BigDecimal valueNumberHigh; - + // The SearchParameter base type. If "Resource", then this is a Resource-level attribute private String base; + /** + * Public constructor + */ public NumberParmVal() { super(); } - - public void setName(String name) { - this.name = name; - } - - public String getName() { - return name; - } - public BigDecimal getValueNumber() { return valueNumber; } @@ -60,32 +52,10 @@ public void setValueNumberHigh(BigDecimal valueNumberHigh) { this.valueNumberHigh = valueNumberHigh; } - public String getResourceType() { - return resourceType; - } - - public void setResourceType(String resourceType) { - this.resourceType = resourceType; - } - /** * We know our type, so we can call the correct method on the visitor */ public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { visitor.visit(this); } - - /** - * @return the base - */ - public String getBase() { - return base; - } - - /** - * @param base the base to set - */ - public void setBase(String base) { - this.base = base; - } -} +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/QuantityParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/QuantityParmVal.java index 372b5dc49d1..8098760e231 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/QuantityParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/QuantityParmVal.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2017,2021 + * (C) Copyright IBM Corp. 2017, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -14,33 +14,21 @@ /** * This class defines the Data Transfer Object representing a row in the X_QUANTITY_VALUES tables. */ -public class QuantityParmVal implements ExtractedParameterValue { +public class QuantityParmVal extends ExtractedParameterValue { - private String resourceType; - private String name; private BigDecimal valueNumber; private BigDecimal valueNumberLow; private BigDecimal valueNumberHigh; private String valueSystem; private String valueCode; - // The SearchParameter base type. If "Resource", then this is a Resource-level attribute - private String base; - + /** + * Public constructor + */ public QuantityParmVal() { super(); } - @Override - public void setName(String name) { - this.name = name; - } - - @Override - public String getName() { - return name; - } - public BigDecimal getValueNumber() { return valueNumber; } @@ -88,16 +76,6 @@ public void setValueNumberHigh(BigDecimal valueNumberHigh) { this.valueNumberHigh = valueNumberHigh; } - @Override - public String getResourceType() { - return resourceType; - } - - @Override - public void setResourceType(String resourceType) { - this.resourceType = resourceType; - } - /** * We know our type, so we can call the correct method on the visitor */ @@ -105,20 +83,4 @@ public void setResourceType(String resourceType) { public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { visitor.visit(this); } - - /** - * @return the base - */ - @Override - public String getBase() { - return base; - } - - /** - * @param base the base to set - */ - @Override - public void setBase(String base) { - this.base = base; - } -} +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ReferenceParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ReferenceParmVal.java index 0d07dc41d26..41a2d7a3e70 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ReferenceParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ReferenceParmVal.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2017,2020 + * (C) Copyright IBM Corp. 2017, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -13,16 +13,7 @@ /** * DTO representing external and local reference parameters */ -public class ReferenceParmVal implements ExtractedParameterValue { - - // The resource type name - private String resourceType; - - // The name of the parameter (key into PARAMETER_NAMES) - private String name; - - // The SearchParameter base type. If "Resource", then this is a Resource-level attribute - private String base; +public class ReferenceParmVal extends ExtractedParameterValue { // The value of the reference after it has been processed to determine target resource type, version etc. private ReferenceValue refValue; @@ -34,14 +25,6 @@ public ReferenceParmVal() { super(); } - public void setName(String name) { - this.name = name; - } - - public String getName() { - return name; - } - /** * Get the refValue * @return @@ -58,20 +41,6 @@ public void setRefValue(ReferenceValue refValue) { this.refValue = refValue; } - /** - * Get the reference type of the parameter (the origin, not the target of the reference) - */ - public String getResourceType() { - return resourceType; - } - - /** - * Set the reference type of the parameter (the origin, not the target of the reference) - */ - public void setResourceType(String resourceType) { - this.resourceType = resourceType; - } - public Type getType() { return Type.REFERENCE; } @@ -82,18 +51,4 @@ public Type getType() { public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { visitor.visit(this); } - - /** - * @return the base - */ - public String getBase() { - return base; - } - - /** - * @param base the base to set - */ - public void setBase(String base) { - this.base = base; - } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/StringParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/StringParmVal.java index b2f9dd832b6..db3ebebbaea 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/StringParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/StringParmVal.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2017,2019 + * (C) Copyright IBM Corp. 2017, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -11,27 +11,18 @@ /** * This class defines the Data Transfer Object representing a row in the X_STR_VALUES tables. */ -public class StringParmVal implements ExtractedParameterValue { - - private String resourceType; - private String name; +public class StringParmVal extends ExtractedParameterValue { + + // The string value of this extracted parameter private String valueString; - - // The SearchParameter base type. If "Resource", then this is a Resource-level attribute - private String base; + /** + * Public constructor + */ public StringParmVal() { super(); } - public void setName(String name) { - this.name = name; - } - - public String getName() { - return name; - } - public String getValueString() { return valueString; } @@ -40,32 +31,10 @@ public void setValueString(String valueString) { this.valueString = valueString; } - public String getResourceType() { - return resourceType; - } - - public void setResourceType(String resourceType) { - this.resourceType = resourceType; - } - /** * We know our type, so we can call the correct method on the visitor */ public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { visitor.visit(this); } - - /** - * @return the base - */ - public String getBase() { - return base; - } - - /** - * @param base the base to set - */ - public void setBase(String base) { - this.base = base; - } -} +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/TokenParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/TokenParmVal.java index 1c271b636fa..6090f67aaa7 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/TokenParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/TokenParmVal.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2017,2021 + * (C) Copyright IBM Corp. 2017, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -12,16 +12,14 @@ /** * This class defines the Data Transfer Object representing a row in the X_TOKEN_VALUES tables. */ -public class TokenParmVal implements ExtractedParameterValue { +public class TokenParmVal extends ExtractedParameterValue { - private String resourceType; - private String name; private String valueSystem; private String valueCode; - // The SearchParameter base type. If "Resource", then this is a Resource-level attribute - private String base; - + /** + * Public constructor + */ public TokenParmVal() { super(); } @@ -32,14 +30,6 @@ public String toString() { return getResourceType() + "[" + getName() + ", " + getValueSystem() + ", " + getValueCode() + "]"; } - public void setName(String name) { - this.name = name; - } - - public String getName() { - return name; - } - public String getValueSystem() { if (valueSystem == null) { return JDBCConstants.DEFAULT_TOKEN_SYSTEM; @@ -59,32 +49,10 @@ public void setValueCode(String valueCode) { this.valueCode = valueCode; } - public String getResourceType() { - return resourceType; - } - - public void setResourceType(String resourceType) { - this.resourceType = resourceType; - } - /** * We know our type, so we can call the correct method on the visitor */ public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { visitor.visit(this); } - - /** - * @return the base - */ - public String getBase() { - return base; - } - - /** - * @param base the base to set - */ - public void setBase(String base) { - this.base = base; - } -} +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/UriParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/UriParmVal.java index c6b5cbffa0f..fce102c7ed2 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/UriParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/UriParmVal.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2017,2019 + * (C) Copyright IBM Corp. 2017, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -8,27 +8,21 @@ import com.ibm.fhir.persistence.exception.FHIRPersistenceException; -public class UriParmVal implements ExtractedParameterValue { - - private String resourceType; - private String name; +/** + * Not used + */ +@Deprecated +public class UriParmVal extends ExtractedParameterValue { + private String valueString; - - // The SearchParameter base type. If "Resource", then this is a Resource-level attribute - private String base; + /** + * Public constructor + */ public UriParmVal() { super(); } - public void setName(String name) { - this.name = name; - } - - public String getName() { - return name; - } - public String getValueString() { return valueString; } @@ -37,32 +31,10 @@ public void setValueString(String valueString) { this.valueString = valueString; } - public String getResourceType() { - return resourceType; - } - - public void setResourceType(String resourceType) { - this.resourceType = resourceType; - } - /** * We know our type, so we can call the correct method on the visitor */ public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { // visitor.visit(this); } - - /** - * @return the base - */ - public String getBase() { - return base; - } - - /** - * @param base the base to set - */ - public void setBase(String base) { - this.base = base; - } -} +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java index 4e3a6851439..43e9aa2f138 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java @@ -126,6 +126,7 @@ import com.ibm.fhir.persistence.jdbc.dao.impl.FetchResourcePayloadsDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.JDBCIdentityCacheImpl; import com.ibm.fhir.persistence.jdbc.dao.impl.ParameterDAOImpl; +import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceProfileRec; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceTokenValueRec; import com.ibm.fhir.persistence.jdbc.dao.impl.TransactionDataImpl; @@ -385,8 +386,9 @@ public SingleResourceResult create(FHIRPersistenceContex ParameterDAO parameterDao = makeParameterDAO(connection); // Persist the Resource DTO. + final String parameterHashB64 = ""; resourceDao.setPersistenceContext(context); - resourceDao.insert(resourceDTO, this.extractSearchParameters(updatedResource, resourceDTO), parameterDao); + resourceDao.insert(resourceDTO, this.extractSearchParameters(updatedResource, resourceDTO), parameterHashB64, parameterDao); if (log.isLoggable(Level.FINE)) { log.fine("Persisted FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' id=" + resourceDTO.getId() + ", version=" + resourceDTO.getVersionId()); @@ -608,8 +610,9 @@ public SingleResourceResult update(FHIRPersistenceContex createResourceDTO(logicalId, newVersionNumber, lastUpdated, updatedResource); // Persist the Resource DTO. + final String parameterHashB64 = ""; resourceDao.setPersistenceContext(context); - resourceDao.insert(resourceDTO, this.extractSearchParameters(updatedResource, resourceDTO), parameterDao); + resourceDao.insert(resourceDTO, this.extractSearchParameters(updatedResource, resourceDTO), parameterHashB64, parameterDao); if (log.isLoggable(Level.FINE)) { log.fine("Persisted FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' id=" + resourceDTO.getId() + ", version=" + resourceDTO.getVersionId()); @@ -650,14 +653,11 @@ public SingleResourceResult update(FHIRPersistenceContex public MultiResourceResult search(FHIRPersistenceContext context, Class resourceType) throws FHIRPersistenceException { - // Fall back to the old search code for whole-system searches which are not yet supported - // by the new code. - if (isSystemLevelSearch(resourceType) || !this.optQueryBuilderEnabled) { - // New query builder doesn't support system-level search at this point, so route - // to the old way. + // Fall back to the old search code if new query builder has been disabled. + if (!this.optQueryBuilderEnabled) { return oldSearch(context, resourceType); } else { - // non-system-level search and the new query builder hasn't been disabled (it is enabled by default) + // new query builder hasn't been disabled (it is enabled by default) return newSearch(context, resourceType); } } @@ -752,7 +752,20 @@ public MultiResourceResult newSearch(FHIRPersistenceContext context, C // path than other sorted searches. Since _include and _revinclude are not supported // with system-level search, no special logic to handle it differently is needed here. List resourceDTOList; - if (searchContext.hasSortParameters() && !resourceType.equals(Resource.class)) { + if (isSystemLevelSearch(resourceType)) { + // If search parameters were specified other than those whose values get indexed + // in global values tables, then we will execute the old-style UNION'd query that + // was built. Otherwise, we need to execute the new whole-system filter query and + // then build and execute the new whole-system data query. + if (!allSearchParmsAreGlobal(searchContext.getSearchParameters())) { + resourceDTOList = resourceDao.search(query); + } else { + Map> resourceTypeIdToLogicalResourceIdMap = resourceDao.searchWholeSystem(query); + Select wholeSystemDataQuery = queryBuilder.buildWholeSystemDataQuery(searchContext, + resourceTypeIdToLogicalResourceIdMap); + resourceDTOList = resourceDao.search(wholeSystemDataQuery); + } + } else if (searchContext.hasSortParameters()) { resourceDTOList = this.buildSortedResourceDTOList(resourceDao, resourceType, resourceDao.searchForIds(query)); } else { resourceDTOList = resourceDao.search(query); @@ -1403,8 +1416,9 @@ public SingleResourceResult delete(FHIRPersistenceContex resourceDTO.setDeleted(true); // Persist the logically deleted Resource DTO. + final String parameterHashB64 = ""; resourceDao.setPersistenceContext(context); - resourceDao.insert(resourceDTO, null, null); + resourceDao.insert(resourceDTO, null, parameterHashB64, null); if (log.isLoggable(Level.FINE)) { log.fine("Persisted FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' id=" + resourceDTO.getId() @@ -1884,6 +1898,7 @@ private List extractSearchParameters(Resource fhirResou for (Entry> entry : map.entrySet()) { SearchParameter sp = entry.getKey(); code = sp.getCode().getValue(); + final boolean wholeSystemParam = isWholeSystem(sp); // As not to inject any other special handling logic, this is a simple inline check to see if // _id or _lastUpdated are used, and ignore those extracted values. @@ -1981,7 +1996,6 @@ private List extractSearchParameters(Resource fhirResou ExtractedParameterValue componentParam = parameters.get(0); // override the component parameter name with the composite parameter name componentParam.setName(SearchUtil.makeCompositeSubCode(code, componentParam.getName())); - componentParam.setBase(p.getBase()); p.addComponent(componentParam); } else if (node.isSystemValue()){ ExtractedParameterValue primitiveParam = processPrimitiveValue(node.asSystemValue()); @@ -2035,6 +2049,10 @@ private List extractSearchParameters(Resource fhirResou ExtractedParameterValue p = processPrimitiveValue(value.asSystemValue()); p.setName(code); p.setResourceType(fhirResource.getClass().getSimpleName()); + + if (wholeSystemParam) { + p.setWholeSystem(true); + } allParameters.add(p); if (log.isLoggable(Level.FINE)) { log.fine("Extracted Parameter '" + p.getName() + "' from Resource."); @@ -2067,6 +2085,9 @@ private List extractSearchParameters(Resource fhirResou // retrieve the list of parameters built from all the FHIRPathElementNode values List parameters = parameterBuilder.getResult(); for (ExtractedParameterValue p : parameters) { + if (wholeSystemParam) { + p.setWholeSystem(true); + } allParameters.add(p); if (log.isLoggable(Level.FINE)) { log.fine("Extracted Parameter '" + p.getName() + "' from Resource."); @@ -2085,6 +2106,24 @@ private List extractSearchParameters(Resource fhirResou return allParameters; } + /** + * Should we also store values for this {@link SearchParameter} in the special whole-system + * param tables (for more efficient whole-system search queries). + * @param sp + * @return + */ + private boolean isWholeSystem(SearchParameter sp) { + + // Strip off any :text suffix before we check to see if this is in the + // whole-system search parameter list + String parameterName = sp.getCode().getValue(); + if (parameterName.endsWith(SearchConstants.TEXT_MODIFIER_SUFFIX)) { + parameterName = parameterName.substring(0, parameterName.length() - SearchConstants.TEXT_MODIFIER_SUFFIX.length()); + } + + return SearchConstants.SYSTEM_LEVEL_GLOBAL_PARAMETER_NAMES.contains(parameterName); + } + /** * Augment the given allParameters list with ibm-internal parameters that represent relationships * between the fhirResource to its compartments. These parameter values are subsequently used @@ -2610,11 +2649,15 @@ private ParameterTransactionDataImpl createTransactionData(String datasourceId) * that have been accumulated during the transaction. This collection therefore * contains multiple resource types, which have to be processed separately. * @param records + * @param profileRecs + * @param tagRecs + * @param securityRecs + * @throws FHIRPersistenceException */ - public void persistResourceTokenValueRecords(Collection records) throws FHIRPersistenceException { + public void persistResourceTokenValueRecords(Collection records, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException { try (Connection connection = openConnection()) { IResourceReferenceDAO rrd = makeResourceReferenceDAO(connection); - rrd.persist(records); + rrd.persist(records, profileRecs, tagRecs, securityRecs); } catch(FHIRPersistenceFKVException e) { log.log(Level.SEVERE, "FK violation", e); throw e; @@ -2710,4 +2753,14 @@ public ResourceEraseRecord erase(EraseDTO eraseDto) throws FHIRPersistenceExcept return eraseRecord; } + + private boolean allSearchParmsAreGlobal(List queryParms) { + for (QueryParameter queryParm : queryParms) { + if (!SearchConstants.SYSTEM_LEVEL_GLOBAL_PARAMETER_NAMES.contains(queryParm.getCode())) { + return false; + } + } + return true; + } + } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/ParameterTransactionDataImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/ParameterTransactionDataImpl.java index 7e8b5fe8450..2e63763310a 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/ParameterTransactionDataImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/ParameterTransactionDataImpl.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -15,6 +15,7 @@ import javax.transaction.UserTransaction; import com.ibm.fhir.persistence.jdbc.TransactionData; +import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceProfileRec; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceTokenValueRec; /** @@ -26,7 +27,7 @@ public class ParameterTransactionDataImpl implements TransactionData { // The id/name of the datasource to which this data belongs private final String datasourceId; - + // remember the impl which was used to create us. This impl should simplify // access to the datasource we're associated with private final FHIRPersistenceJDBCImpl impl; @@ -36,7 +37,16 @@ public class ParameterTransactionDataImpl implements TransactionData { // Collect all the token values so we can submit once per transaction private final List tokenValueRecs = new ArrayList<>(); - + + // Collect all the profile values so we can submit once per transaction + private final List profileRecs = new ArrayList<>(); + + // Collect all the tag values so we can submit once per transaction + private final List tagRecs = new ArrayList<>(); + + // Collect all the security values so we can submit once per transaction + private final List securityRecs = new ArrayList<>(); + /** * Public constructor * @param datasourceId @@ -47,12 +57,12 @@ public ParameterTransactionDataImpl(String datasourceId, FHIRPersistenceJDBCImpl this.impl = impl; this.userTransaction = userTransaction; } - + @Override public void persist() { - + try { - impl.persistResourceTokenValueRecords(tokenValueRecs); + impl.persistResourceTokenValueRecords(tokenValueRecs, profileRecs, tagRecs, securityRecs); } catch (Throwable t) { logger.log(Level.SEVERE, "Failed persisting parameter transaction data. Marking transaction for rollback", t); try { @@ -70,4 +80,34 @@ public void persist() { public void addValue(ResourceTokenValueRec rec) { tokenValueRecs.add(rec); } -} + + /** + * Add the given profile parameter record to the list of records being accumulated in + * this transaction data. The records will be inserted to the database together at the + * end, just prior to the commit (see {@link #persist()} + * @param rec + */ + public void addValue(ResourceProfileRec rec) { + profileRecs.add(rec); + } + + /** + * Add the given tag parameter record to the list of records being accumulated in + * this transaction data. The records will be inserted to the database together at the + * end, just prior to the commit (see {@link #persist()} + * @param rec + */ + public void addTagValue(ResourceTokenValueRec rec) { + tagRecs.add(rec); + } + + /** + * Add the given security parameter record to the list of records being accumulated in + * the transaction data. The records will be inserted to the database together at the end, + * just prior to the commit (see {@link #persist()} + * @param rec + */ + public void addSecurityValue(ResourceTokenValueRec rec) { + securityRecs.add(rec); + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java index 1035d090c08..788bf630e17 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java @@ -43,17 +43,15 @@ import com.ibm.fhir.persistence.jdbc.util.ResourceTypesCache; /** - * Data access object for writing FHIR resources to an postgresql database. - * - * @implNote This class follows the logic of the DB2 stored procedure, but does so - * using a series of individual JDBC statements. + * Data access object for writing FHIR resources to an postgresql database using + * the stored procedure (or function, in this case) */ public class PostgresResourceDAO extends ResourceDAOImpl { private static final String CLASSNAME = PostgresResourceDAO.class.getSimpleName(); private static final Logger logger = Logger.getLogger(CLASSNAME); private static final String SQL_READ_RESOURCE_TYPE = "{CALL %s.add_resource_type(?, ?)}"; - private static final String SQL_INSERT_WITH_PARAMETERS = "{CALL %s.add_any_resource(?,?,?,?,?,?,?,?)}"; + private static final String SQL_INSERT_WITH_PARAMETERS = "{CALL %s.add_any_resource(?,?,?,?,?,?,?,?,?,?)}"; // DAO used to obtain sequence values from FHIR_REF_SEQUENCE private FhirRefSequenceDAO fhirRefSequenceDAO; @@ -67,16 +65,8 @@ public PostgresResourceDAO(Connection connection, String schemaName, FHIRDbFlavo super(connection, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi); } - /** - * Inserts the passed FHIR Resource and associated search parameters to a postgresql FHIR database. - * @param resource The FHIR Resource to be inserted. - * @param parameters The Resource's search parameters to be inserted. - * @param parameterDao - * @return The Resource DTO - * @throws FHIRPersistenceException - */ @Override - public Resource insert(Resource resource, List parameters, ParameterDAO parameterDao) + public Resource insert(Resource resource, List parameters, String parameterHashB64, ParameterDAO parameterDao) throws FHIRPersistenceException { final String METHODNAME = "insert(Resource, List"; logger.entering(CLASSNAME, METHODNAME); @@ -115,19 +105,23 @@ public Resource insert(Resource resource, List paramete stmt.setString(5, resource.isDeleted() ? "Y": "N"); stmt.setString(6, UUID.randomUUID().toString()); stmt.setInt(7, resource.getVersionId()); - stmt.registerOutParameter(8, Types.BIGINT); + stmt.setString(8, parameterHashB64); + stmt.registerOutParameter(9, Types.BIGINT); + stmt.registerOutParameter(10, Types.VARCHAR); // The old parameter_hash dbCallStartTime = System.nanoTime(); stmt.execute(); dbCallDuration = (System.nanoTime()-dbCallStartTime)/1e6; - resource.setId(stmt.getLong(8)); + resource.setId(stmt.getLong(9)); // Parameter time // To keep things simple for the postgresql use-case, we just use a visitor to // handle inserts of parameters directly in the resource parameter tables. // Note we don't get any parameters for the resource soft-delete operation - if (parameters != null) { + final String currentParameterHash = stmt.getString(10); + if (parameters != null && (parameterHashB64 == null || parameterHashB64.isEmpty() + || !parameterHashB64.equals(currentParameterHash))) { // postgresql doesn't support partitioned multi-tenancy, so we disable it on the DAO: JDBCIdentityCache identityCache = new JDBCIdentityCacheImpl(getCache(), this, parameterDao, getResourceReferenceDAO()); try (ParameterVisitorBatchDAO pvd = new ParameterVisitorBatchDAO(connection, null, resource.getResourceType(), false, resource.getId(), 100, @@ -184,6 +178,7 @@ protected void deleteFromParameterTable(Connection conn, String tableName, long /** * Read the id for the named type * @param resourceTypeName + * @param conn * @return the database id, or null if the named record is not found * @throws SQLException */ @@ -206,6 +201,7 @@ protected Integer getResourceTypeId(String resourceTypeName, Connection conn) th /** * stored-procedure-less implementation for managing the resource_types table * @param resourceTypeName + * @param conn * @throw SQLException */ public int getOrCreateResourceType(String resourceTypeName, Connection conn) throws SQLException { @@ -232,7 +228,6 @@ public int getOrCreateResourceType(String resourceTypeName, Connection conn) thr return result; } - @Override public Integer readResourceTypeId(String resourceType) throws FHIRPersistenceDBConnectException, FHIRPersistenceDataAccessException { final String METHODNAME = "readResourceTypeId"; @@ -267,5 +262,4 @@ public Integer readResourceTypeId(String resourceType) throws FHIRPersistenceDBC } return resourceTypeId; } - -} +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceNoProcDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceNoProcDAO.java index 31f245c9a5f..43d5e1d9ccc 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceNoProcDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceNoProcDAO.java @@ -41,6 +41,7 @@ import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceFKVException; import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; +import com.ibm.fhir.persistence.jdbc.util.ParameterTableSupport; import com.ibm.fhir.persistence.jdbc.util.ResourceTypesCache; /** @@ -71,20 +72,8 @@ public PostgresResourceNoProcDAO(Connection connection, String schemaName, FHIRD super(connection, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi); } - /** - * Inserts the passed FHIR Resource and associated search parameters to a Derby or PostgreSql FHIR database. - * The search parameters are stored first by calling the passed parameterDao. Then the Resource is stored - * by sql. - * @param resource The FHIR Resource to be inserted. - * @param parameters The Resource's search parameters to be inserted. - * @param parameterDao - * @return The Resource DTO - * @throws FHIRPersistenceDataAccessException - * @throws FHIRPersistenceDBConnectException - * @throws FHIRPersistenceVersionIdMismatchException - */ @Override - public Resource insert(Resource resource, List parameters, ParameterDAO parameterDao) + public Resource insert(Resource resource, List parameters, String parameterHashB64, ParameterDAO parameterDao) throws FHIRPersistenceException { final String METHODNAME = "insert"; logger.entering(CLASSNAME, METHODNAME); @@ -95,6 +84,7 @@ public Resource insert(Resource resource, List paramete boolean acquiredFromCache; long dbCallStartTime; double dbCallDuration; + boolean requireParameterUpdate = true; try { resourceTypeId = getResourceTypeIdFromCaches(resource.getResourceType()); @@ -124,6 +114,7 @@ public Resource insert(Resource resource, List paramete resource.isDeleted(), sourceKey, resource.getVersionId(), + parameterHashB64, connection, parameterDao ); @@ -180,7 +171,6 @@ public Resource insert(Resource resource, List paramete * * Note the execution flow aligns very closely with the DB2 stored procedure * implementation (fhir-persistence-schema/src/main/resources/add_any_resource.sql) - * * @param tablePrefix * @param parameters * @param p_logical_id @@ -188,13 +178,15 @@ public Resource insert(Resource resource, List paramete * @param p_last_updated * @param p_is_deleted * @param p_source_key - * @param p_version the intended version for this resource - * + * @param p_version + * @param parameterHashB64 + * @param conn + * @param parameterDao * @return the resource_id for the entry we created * @throws Exception */ public long storeResource(String tablePrefix, List parameters, String p_logical_id, InputStream p_payload, Timestamp p_last_updated, boolean p_is_deleted, - String p_source_key, Integer p_version, Connection conn, ParameterDAO parameterDao) throws Exception { + String p_source_key, Integer p_version, String parameterHashB64, Connection conn, ParameterDAO parameterDao) throws Exception { final String METHODNAME = "storeResource() for " + tablePrefix + " resource"; logger.entering(CLASSNAME, METHODNAME); @@ -207,6 +199,7 @@ public long storeResource(String tablePrefix, List para boolean v_not_found = false; boolean v_duplicate = false; int v_current_version = 0; + String currentHash = null; String v_resource_type = tablePrefix; @@ -218,13 +211,14 @@ public long storeResource(String tablePrefix, List para } // Get a lock at the system-wide logical resource level. Note the PostgreSQL-specific syntax - final String SELECT_FOR_UPDATE = "SELECT logical_resource_id FROM logical_resources WHERE resource_type_id = ? AND logical_id = ? FOR NO KEY UPDATE"; + final String SELECT_FOR_UPDATE = "SELECT logical_resource_id, parameter_hash FROM logical_resources WHERE resource_type_id = ? AND logical_id = ? FOR NO KEY UPDATE"; try (PreparedStatement stmt = conn.prepareStatement(SELECT_FOR_UPDATE)) { stmt.setInt(1, v_resource_type_id); stmt.setString(2, p_logical_id); ResultSet rs = stmt.executeQuery(); if (rs.next()) { v_logical_resource_id = rs.getLong(1); + currentHash = rs.getString(2); } else { v_not_found = true; @@ -248,7 +242,7 @@ public long storeResource(String tablePrefix, List para } // insert the system-wide logical resource record. - final String sql3 = "INSERT INTO logical_resources (logical_resource_id, resource_type_id, logical_id, reindex_tstamp) VALUES (?, ?, ?, ?) " + final String sql3 = "INSERT INTO logical_resources (logical_resource_id, resource_type_id, logical_id, reindex_tstamp, parameter_hash) VALUES (?, ?, ?, ?, ?) " + " ON CONFLICT DO NOTHING" + " RETURNING logical_resource_id"; try (PreparedStatement stmt = conn.prepareStatement(sql3)) { @@ -257,6 +251,7 @@ public long storeResource(String tablePrefix, List para stmt.setInt(2, v_resource_type_id); stmt.setString(3, p_logical_id); stmt.setTimestamp(4, Timestamp.valueOf(DEFAULT_VALUE_REINDEX_TSTAMP), UTC); + stmt.setString(5, parameterHashB64); stmt.execute(); ResultSet rs = stmt.getResultSet(); @@ -279,6 +274,7 @@ public long storeResource(String tablePrefix, List para ResultSet res = stmt.executeQuery(); if (res.next()) { v_logical_resource_id = res.getLong(1); + currentHash = res.getString(2); } else { // Extremely unlikely as we should never delete logical resource records @@ -331,15 +327,24 @@ public long storeResource(String tablePrefix, List para } // existing resource, so need to delete all its parameters - deleteFromParameterTable(conn, tablePrefix + "_str_values", v_logical_resource_id); - deleteFromParameterTable(conn, tablePrefix + "_number_values", v_logical_resource_id); - deleteFromParameterTable(conn, tablePrefix + "_date_values", v_logical_resource_id); - deleteFromParameterTable(conn, tablePrefix + "_latlng_values", v_logical_resource_id); - deleteFromParameterTable(conn, tablePrefix + "_resource_token_refs", v_logical_resource_id); // replaces _token_values - deleteFromParameterTable(conn, tablePrefix + "_quantity_values", v_logical_resource_id); - deleteFromParameterTable(conn, "str_values", v_logical_resource_id); - deleteFromParameterTable(conn, "date_values", v_logical_resource_id); - deleteFromParameterTable(conn, "resource_token_refs", v_logical_resource_id); + if (currentHash == null || currentHash.isEmpty() || !currentHash.equals(parameterHashB64)) { + ParameterTableSupport.deleteFromParameterTables(conn, tablePrefix, v_logical_resource_id); + } + + // For schema V0014, now we also need to update the is_deleted and last_updated values + // in LOGICAL_RESOURCES to support whole-system search + final String sql4b = "UPDATE logical_resources SET is_deleted = ?, last_updated = ?, parameter_hash = ? WHERE logical_resource_id = ?"; + try (PreparedStatement stmt = conn.prepareStatement(sql4b)) { + // bind parameters + stmt.setString(1, p_is_deleted ? "Y" : "N"); + stmt.setTimestamp(2, p_last_updated, UTC); + stmt.setString(3, parameterHashB64); + stmt.setLong(4, v_logical_resource_id); + stmt.executeUpdate(); + if (logger.isLoggable(Level.FINEST)) { + logger.finest("Updated logical_resources: " + v_resource_type + "/" + p_logical_id); + } + } } /** @@ -384,7 +389,7 @@ public long storeResource(String tablePrefix, List para } // Note we don't get any parameters for the resource soft-delete operation - if (parameters != null) { + if (parameters != null && (currentHash == null || currentHash.isEmpty() || !currentHash.equals(parameterHashB64))) { // PostgreSQL doesn't support partitioned multi-tenancy, so we disable it on the DAO: JDBCIdentityCache identityCache = new JDBCIdentityCacheImpl(getCache(), this, parameterDao, getResourceReferenceDAO()); try (ParameterVisitorBatchDAO pvd = new ParameterVisitorBatchDAO(conn, null, tablePrefix, false, v_logical_resource_id, 100, @@ -414,23 +419,6 @@ identityCache, getResourceReferenceDAO(), getTransactionData())) { return v_resource_id; } - /** - * Delete all parameters for the given resourceId from the parameters table - * - * @param conn - * @param tableName - * @param logicalResourceId - * @throws SQLException - */ - protected void deleteFromParameterTable(Connection conn, String tableName, long logicalResourceId) throws SQLException { - final String delStrValues = "DELETE FROM " + tableName + " WHERE logical_resource_id = ?"; - try (PreparedStatement stmt = conn.prepareStatement(delStrValues)) { - // bind parameters - stmt.setLong(1, logicalResourceId); - stmt.executeUpdate(); - } - } - /** * Read the id for the named type * @param resourceTypeName diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceReferenceDAO.java index 29c2ff077db..005c95e63da 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceReferenceDAO.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -27,7 +27,7 @@ */ public class PostgresResourceReferenceDAO extends ResourceReferenceDAO { private static final Logger logger = Logger.getLogger(PostgresResourceReferenceDAO.class.getName()); - + /** * Public constructor * @param t @@ -38,7 +38,7 @@ public class PostgresResourceReferenceDAO extends ResourceReferenceDAO { public PostgresResourceReferenceDAO(IDatabaseTranslator t, Connection c, String schemaName, ICommonTokenValuesCache cache) { super(t, c, schemaName, cache); } - + @Override public void doCodeSystemsUpsert(String paramList, Collection systemNames) { // query is a negative outer join so we only pick the rows where @@ -52,7 +52,7 @@ public void doCodeSystemsUpsert(String paramList, Collection systemNames insert.append(" FROM "); insert.append(" (VALUES ").append(paramList).append(" ) AS v(name) "); insert.append(" ON CONFLICT DO NOTHING "); - + // Note, we use PreparedStatement here on purpose. Partly because it's // secure coding best practice, but also because many resources will have the // same number of parameters, and hopefully we'll therefore share a small subset @@ -67,7 +67,42 @@ public void doCodeSystemsUpsert(String paramList, Collection systemNames for (String name: sortedNames) { ps.setString(a++, name); } - + + ps.executeUpdate(); + } catch (SQLException x) { + logger.log(Level.SEVERE, insert.toString(), x); + throw getTranslator().translate(x); + } + } + + @Override + public void doCanonicalValuesUpsert(String paramList, Collection urls) { + // Because of how PostgreSQL MVCC implementation, the insert from negative outer + // join pattern doesn't work...you still hit conflicts. The PostgreSQL pattern + // for upsert is ON CONFLICT DO NOTHING, which is what we use here: + final String nextVal = getTranslator().nextValue(getSchemaName(), "fhir_ref_sequence"); + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO common_canonical_values (canonical_id, url) "); + insert.append(" SELECT ").append(nextVal).append(", v.name "); + insert.append(" FROM "); + insert.append(" (VALUES ").append(paramList).append(" ) AS v(name) "); + insert.append(" ON CONFLICT DO NOTHING "); + + // Note, we use PreparedStatement here on purpose. Partly because it's + // secure coding best practice, but also because many resources will have the + // same number of parameters, and hopefully we'll therefore share a small subset + // of statements for better performance. Although once the cache warms up, this + // shouldn't be called at all. + final List sortedNames = new ArrayList<>(urls); + sortedNames.sort((String left, String right) -> left.compareTo(right)); + + try (PreparedStatement ps = getConnection().prepareStatement(insert.toString())) { + // bind all the code_system_name values as parameters + int a = 1; + for (String name: sortedNames) { + ps.setString(a++, name); + } + ps.executeUpdate(); } catch (SQLException x) { logger.log(Level.SEVERE, insert.toString(), x); @@ -88,7 +123,7 @@ protected void doCommonTokenValuesUpsert(String paramList, Collection findex && findex > -1) { + throw new FHIRPersistenceException("Invalid profile URI"); + } + + // Extract version if given + if (vindex > 0) { + if (findex > -1) { + version = paramValue.substring(vindex+1, findex); // everything after the | but before the # + } else { + version = paramValue.substring(vindex+1); // everything after the | + } + if (version.isEmpty()) { + version = null; + } + uri = paramValue.substring(0, vindex); // everything before the | + } + + // Extract fragment if given + if (findex > 0) { + fragment = paramValue.substring(findex+1); + if (fragment.isEmpty()) { + fragment = null; + } + + if (vindex < 0) { + // fragment but no version + uri = paramValue.substring(0, findex); // everything before the # + } + } + + return new ResourceProfileRec(parameterNameId, resourceType, resourceTypeId, logicalResourceId, uri, version, fragment, systemLevel); + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCParameterBuildingVisitor.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCParameterBuildingVisitor.java index 61d07be0abc..3fdb199680c 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCParameterBuildingVisitor.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCParameterBuildingVisitor.java @@ -104,13 +104,17 @@ public class JDBCParameterBuildingVisitor extends DefaultVisitor { */ private List result; + /** + * Public constructor + * @param resourceType + * @param searchParameter + */ public JDBCParameterBuildingVisitor(String resourceType, SearchParameter searchParameter) { super(false); this.resourceType = resourceType; this.searchParamCode = searchParameter.getCode().getValue(); this.searchParamType = searchParameter.getType(); - - result = new ArrayList<>(); + this.result = new ArrayList<>(); } /** diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCQueryBuilder.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCQueryBuilder.java index 56434333d7c..797b69d56a0 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCQueryBuilder.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCQueryBuilder.java @@ -20,6 +20,7 @@ import static com.ibm.fhir.persistence.jdbc.JDBCConstants.ESCAPE_UNDERSCORE; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.EXISTS; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.FROM; +import static com.ibm.fhir.persistence.jdbc.JDBCConstants.GTE; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.IN; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.IS_DELETED_NO; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.JOIN; @@ -27,6 +28,7 @@ import static com.ibm.fhir.persistence.jdbc.JDBCConstants.LIKE; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.LOGICAL_ID; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.LOGICAL_RESOURCE_ID; +import static com.ibm.fhir.persistence.jdbc.JDBCConstants.LT; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.NOT; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.ON; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.OR; @@ -46,6 +48,11 @@ import static com.ibm.fhir.persistence.jdbc.JDBCConstants._LOGICAL_RESOURCES; import static com.ibm.fhir.persistence.jdbc.JDBCConstants._RESOURCES; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.modifierOperatorMap; +import static com.ibm.fhir.search.SearchConstants.ID; +import static com.ibm.fhir.search.SearchConstants.LAST_UPDATED; +import static com.ibm.fhir.search.SearchConstants.PROFILE; +import static com.ibm.fhir.search.SearchConstants.SECURITY; +import static com.ibm.fhir.search.SearchConstants.TAG; import java.sql.Timestamp; import java.util.ArrayList; @@ -71,6 +78,7 @@ import com.ibm.fhir.persistence.jdbc.dao.api.JDBCIdentityCache; import com.ibm.fhir.persistence.jdbc.dao.api.ParameterDAO; import com.ibm.fhir.persistence.jdbc.dao.api.ResourceDAO; +import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceProfileRec; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.util.type.DateParmBehaviorUtil; @@ -231,7 +239,7 @@ private QuerySegmentAggregator buildQueryCommon(Class resourceType, FHIRSearc @Override public int compare(QueryParameter leftParameter, QueryParameter rightParameter) { int result = 0; - if (QuerySegmentAggregator.ID.equals(leftParameter.getCode())) { + if (ID.equals(leftParameter.getCode())) { result = -100; } else if (LastUpdatedParmBehaviorUtil.LAST_UPDATED.equals(leftParameter.getCode())) { result = -90; @@ -399,7 +407,11 @@ protected SqlQueryData buildQueryParm(Class resourceType, QueryParameter quer databaseQueryParm = this.processDateParm(resourceType, queryParm, paramTableAlias); break; case TOKEN: - databaseQueryParm = this.processTokenParm(resourceType, queryParm, paramTableAlias, logicalRsrcTableAlias, endOfChain); + if (TAG.equals(queryParm.getCode()) || SECURITY.equals(queryParm.getCode())) { + databaseQueryParm = this.processGlobalTokenParm(resourceType, queryParm, paramTableAlias, logicalRsrcTableAlias, endOfChain); + } else { + databaseQueryParm = this.processTokenParm(resourceType, queryParm, paramTableAlias, logicalRsrcTableAlias, endOfChain); + } break; case NUMBER: databaseQueryParm = this.processNumberParm(resourceType, queryParm, paramTableAlias); @@ -408,7 +420,11 @@ protected SqlQueryData buildQueryParm(Class resourceType, QueryParameter quer databaseQueryParm = this.processQuantityParm(resourceType, queryParm, paramTableAlias); break; case URI: - databaseQueryParm = this.processUriParm(queryParm, paramTableAlias); + if (PROFILE.equals(queryParm.getCode())) { + databaseQueryParm = this.processProfileParm(resourceType, queryParm, paramTableAlias); + } else { + databaseQueryParm = this.processUriParm(queryParm, paramTableAlias); + } break; case COMPOSITE: databaseQueryParm = this.processCompositeParm(resourceType, queryParm, paramTableAlias, logicalRsrcTableAlias); @@ -460,9 +476,11 @@ private SqlQueryData processStringParm(QueryParameter queryParm, String tableAli appendEscape = false; if (LIKE.equals(operator)) { - // Must escape special wildcard characters _ and % in the parameter value string. + // Must escape special wildcard characters _ and % in the parameter value string + // as well as the escape character itself. tempSearchValue = SqlParameterEncoder.encode(value.getValueString() + .replace("+", "++") .replace(PERCENT_WILDCARD, ESCAPE_PERCENT) .replace(UNDERSCORE_WILDCARD, ESCAPE_UNDERSCORE)); if (Modifier.CONTAINS.equals(queryParm.getModifier())) { @@ -801,11 +819,11 @@ protected SqlQueryData processChainedReferenceParm(QueryParameter queryParm) thr String code = currentParm.getCode(); SqlQueryData sqlQueryData; - if ("_id".equals(code)) { + if (ID.equals(code)) { // The code '_id' is only going to be the end of the change as it is a base element. // We know at this point this is an '_id' and at the tail of the parameter chain sqlQueryData = buildChainedIdClause(currentParm, chainedParmVar); - } else if ("_lastUpdated".equals(code)) { + } else if (LAST_UPDATED.equals(code)) { // Build the rest: (LAST_UPDATED ?) LastUpdatedParmBehaviorUtil util = new LastUpdatedParmBehaviorUtil(); StringBuilder lastUpdatedWhereClause = new StringBuilder(); @@ -919,7 +937,7 @@ private void appendInnerSelect(StringBuilder whereClauseSegment, QueryParameter whereClauseSegment.append(COMMA).append(resourceTypeName).append(_RESOURCES).append(SPACE).append(chainedResourceVar); // If we're dealing with anything other than id, then proceed to add the parameters table. - if (nextParameter != null && !"_id".equals(nextParameter.getCode())) { + if (nextParameter != null && !ID.equals(nextParameter.getCode())) { whereClauseSegment.append(COMMA) .append(QuerySegmentAggregator.tableName(resourceTypeName, nextParameter)).append(chainedParmVar); } @@ -934,7 +952,7 @@ private void appendInnerSelect(StringBuilder whereClauseSegment, QueryParameter .append(AND); // CP1.LOGICAL_RESOURCE_ID = CLR1.LOGICAL_RESOURCE_ID AND - if (nextParameter != null && !"_id".equals(nextParameter.getCode())) { + if (nextParameter != null && !ID.equals(nextParameter.getCode())) { whereClauseSegment.append(chainedParmTableAlias).append(LOGICAL_RESOURCE_ID).append(EQ) .append(chainedResourceTableAlias).append(LOGICAL_RESOURCE_ID) .append(AND); @@ -1217,7 +1235,7 @@ private SqlQueryData processTokenParm(Class resourceType, QueryParameter quer String tableAlias = paramTableAlias; String queryParmCode = queryParm.getCode(); - if (!QuerySegmentAggregator.ID.equals(queryParmCode)) { + if (!ID.equals(queryParmCode)) { // Append the suffix for :text modifier if (Modifier.TEXT.equals(queryParm.getModifier())) { @@ -1262,8 +1280,10 @@ private SqlQueryData processTokenParm(Class resourceType, QueryParameter quer if (LIKE.equals(operator)) { whereClauseSegment.append(tableAlias + DOT).append(TOKEN_VALUE).append(operator).append(BIND_VAR); - // Must escape special wildcard characters _ and % in the parameter value string. + // Must escape special wildcard characters _ and % in the parameter value string + // as well as the escape character itself. String textSearchString = SqlParameterEncoder.encode(value.getValueCode()) + .replace("+", "++") .replace(PERCENT_WILDCARD, ESCAPE_PERCENT) .replace(UNDERSCORE_WILDCARD, ESCAPE_UNDERSCORE) + PERCENT_WILDCARD; bindVariables.add(SearchUtil.normalizeForSearch(textSearchString)); @@ -1607,9 +1627,16 @@ private SqlQueryData processMissingParm(Class resourceType, QueryParameter qu String valuesTable = !ModelSupport.isAbstract(resourceType) ? QuerySegmentAggregator.tableName(resourceType.getSimpleName(), queryParm) : PARAMETER_TABLE_NAME_PLACEHOLDER; String subqueryTableAlias = endOfChain ? (paramTableAlias + "_param0") : paramTableAlias; whereClauseSegment.append("(SELECT 1 FROM " + valuesTable + AS + subqueryTableAlias + WHERE); - this.populateNameIdSubSegment(whereClauseSegment, queryParm.getCode(), subqueryTableAlias); - whereClauseSegment.append(AND).append(subqueryTableAlias).append(".LOGICAL_RESOURCE_ID = ").append(logicalRsrcTableAlias).append(".LOGICAL_RESOURCE_ID"); // correlate the [NOT] EXISTS subquery - whereClauseSegment.append(RIGHT_PAREN).append(RIGHT_PAREN); + if (!PROFILE.equals(queryParm.getCode()) && !SECURITY.equals(queryParm.getCode()) && !TAG.equals(queryParm.getCode())) { + this.populateNameIdSubSegment(whereClauseSegment, queryParm.getCode(), subqueryTableAlias); + whereClauseSegment.append(AND).append(subqueryTableAlias).append(".LOGICAL_RESOURCE_ID = ") + .append(logicalRsrcTableAlias).append(".LOGICAL_RESOURCE_ID"); // correlate the [NOT] EXISTS subquery + whereClauseSegment.append(RIGHT_PAREN); + } else { + whereClauseSegment.append(subqueryTableAlias).append(".LOGICAL_RESOURCE_ID = ") + .append(logicalRsrcTableAlias).append(".LOGICAL_RESOURCE_ID"); // correlate the [NOT] EXISTS subquery + } + whereClauseSegment.append(RIGHT_PAREN); } SqlQueryData queryData = new SqlQueryData(whereClauseSegment.toString(), bindVariables); @@ -1808,7 +1835,7 @@ protected SqlQueryData processReverseChainedReferenceParm(Class resourceType, } else if (parmIndex == lastParmIndex) { // This logic processes the LAST parameter in the chain. SqlQueryData sqlQueryData; - if ("_id".equals(currentParm.getCode())) { + if (ID.equals(currentParm.getCode())) { if (!chainedParmProcessed) { // Build this join: // @formatter:off @@ -1823,7 +1850,7 @@ protected SqlQueryData processReverseChainedReferenceParm(Class resourceType, } // Build the rest: CLRx.LOGICAL_ID IN (?) sqlQueryData = buildChainedIdClause(currentParm, chainedParmVar); - } else if ("_lastUpdated".equals(currentParm.getCode())) { + } else if (LAST_UPDATED.equals(currentParm.getCode())) { if (!chainedParmProcessed) { // Build this join: // @formatter:off @@ -2074,4 +2101,194 @@ private void populateCodesSubSegment(StringBuilder whereClauseSegment, Modifier log.exiting(CLASSNAME, METHODNAME); } + private SqlQueryData processProfileParm(Class resourceType, QueryParameter queryParm, String tableAlias) + throws FHIRPersistenceException { + final String METHODNAME = "processProfileParm"; + log.entering(CLASSNAME, METHODNAME, queryParm.toString()); + + // Join to the canonical parameter table...which in this case means the xx_profiles table + // because currently _profile is the only search parameter to be defined as a canonical + + StringBuilder whereClauseSegment = new StringBuilder(); + boolean parmValueProcessed = false; + SqlQueryData queryData; + List bindVariables = new ArrayList<>(); + + whereClauseSegment.append(LEFT_PAREN); + for (QueryParameterValue value : queryParm.getValues()) { + // If multiple values are present, we need to OR them together. + if (parmValueProcessed) { + whereClauseSegment.append(OR); + } + + // Reuse the same CanonicalSupport code used for param extraction to parse the search value + ResourceProfileRec rpc = CanonicalSupport.makeResourceProfileRec(-1, resourceType.getSimpleName(), -1, -1, + value.getValueString(), false); + int canonicalId = identityCache.getCanonicalId(rpc.getCanonicalValue()); + whereClauseSegment.append(tableAlias).append(DOT).append("CANONICAL_ID").append(EQ).append(canonicalId); + + // TODO double-check semantics of ABOVE and BELOW in this context + if (rpc.getVersion() != null && !rpc.getVersion().isEmpty()) { + whereClauseSegment.append(AND).append(tableAlias).append(DOT).append("VERSION"); + if (queryParm.getModifier() == Modifier.ABOVE) { + whereClauseSegment.append(GTE); + } else if (queryParm.getModifier() == Modifier.BELOW) { + whereClauseSegment.append(LT); + } else { + whereClauseSegment.append(EQ); + } + whereClauseSegment.append(BIND_VAR); + bindVariables.add(rpc.getVersion()); + } + + if (rpc.getFragment() != null && !rpc.getFragment().isEmpty()) { + whereClauseSegment.append(AND).append(tableAlias).append(DOT).append("FRAGMENT").append(EQ).append(BIND_VAR); + bindVariables.add(rpc.getFragment()); + } + + parmValueProcessed = true; + } + whereClauseSegment.append(RIGHT_PAREN); + + queryData = new SqlQueryData(whereClauseSegment.toString(), bindVariables); + log.exiting(CLASSNAME, METHODNAME); + return queryData; + } + + private SqlQueryData processGlobalTokenParm(Class resourceType, QueryParameter queryParm, String paramTableAlias, String logicalRsrcTableAlias, boolean endOfChain) throws FHIRPersistenceException { + final String METHODNAME = "processGlobalTokenParm"; + log.entering(CLASSNAME, METHODNAME, queryParm.toString()); + + StringBuilder whereClauseSegment = new StringBuilder(); + String operator = this.getOperator(queryParm, EQ); + boolean parmValueProcessed = false; + boolean appendEscape; + SqlQueryData queryData; + List bindVariables = new ArrayList<>(); + String tableAlias = paramTableAlias; + + // Only generate NOT EXISTS subquery if :not modifier is within chained query; + // when :not modifier is within non-chained query QuerySegmentAggregator.buildWhereClause generates the NOT EXISTS subquery + boolean surroundWithNotExistsSubquery = Modifier.NOT.equals(queryParm.getModifier()) && endOfChain; + if (surroundWithNotExistsSubquery) { + whereClauseSegment.append(NOT).append(EXISTS); + + // PARAMETER_TABLE_NAME_PLACEHOLDER is replaced by the actual table name for the resource type by QuerySegmentAggregator.buildWhereClause(...) + String valuesTable = !ModelSupport.isAbstract(resourceType) ? QuerySegmentAggregator.tableName(resourceType.getSimpleName(), queryParm) : PARAMETER_TABLE_NAME_PLACEHOLDER; + tableAlias = paramTableAlias + "_param0"; + whereClauseSegment.append("(SELECT 1 FROM " + valuesTable + AS + tableAlias + WHERE); + } + + whereClauseSegment.append(LEFT_PAREN); + for (QueryParameterValue value : queryParm.getValues()) { + appendEscape = false; + + // If multiple values are present, we need to OR them together. + if (parmValueProcessed) { + whereClauseSegment.append(OR); + } + + whereClauseSegment.append(LEFT_PAREN); + + if (Modifier.IN.equals(queryParm.getModifier()) || Modifier.NOT_IN.equals(queryParm.getModifier()) || + Modifier.ABOVE.equals(queryParm.getModifier()) || Modifier.BELOW.equals(queryParm.getModifier())) { + populateCodesSubSegment(whereClauseSegment, queryParm.getModifier(), value, tableAlias); + } else { + final String system = value.getValueSystem() != null && !value.getValueSystem().isEmpty() ? value.getValueSystem() : null; + final String code = value.getValueCode() != null ? value.getValueCode() : null; // empty code is a valid value + + // Determine code normalization based on code system case-sensitivity + String normalizedCode = null; + if (code != null) { + if (system != null) { + boolean codeSystemIsCaseSensitive = CodeSystemSupport.isCaseSensitive(system); + normalizedCode = SqlParameterEncoder.encode(codeSystemIsCaseSensitive ? + code : SearchUtil.normalizeForSearch(code)); + } else { + normalizedCode = SqlParameterEncoder.encode(SearchUtil.normalizeForSearch(code)); + } + } + + // Include code + if (EQ.equals(operator) && code != null) { + if (system == null || system.equals("*")) { + // Even though we don't have a system, we can still use a list of + // common_token_value_ids matching the value-code, allowing a similar optimization + Set ctvs = new HashSet<>(); + ctvs.addAll(identityCache.getCommonTokenValueIdList(SqlParameterEncoder.encode(code))); + ctvs.addAll(identityCache.getCommonTokenValueIdList(SqlParameterEncoder.encode(SearchUtil.normalizeForSearch(code)))); + List ctvList = new ArrayList<>(ctvs); + if (ctvs.isEmpty()) { + // use -1...resulting in no data + whereClauseSegment.append(tableAlias).append(DOT).append(COMMON_TOKEN_VALUE_ID).append(EQ) + .append(-1); + } else if (ctvs.size() == 1) { + whereClauseSegment.append(tableAlias).append(DOT).append(COMMON_TOKEN_VALUE_ID).append(EQ) + .append(ctvList.get(0)); + } else { + whereClauseSegment.append(tableAlias).append(DOT).append(COMMON_TOKEN_VALUE_ID).append(IN) + .append(LEFT_PAREN) + .append(ctvList.stream().map(c -> c.toString()).collect(Collectors.joining(","))) + .append(RIGHT_PAREN); + } + } else { + Long commonTokenValueId = getCommonTokenValueId(system, normalizedCode); + whereClauseSegment.append(tableAlias).append(DOT).append(COMMON_TOKEN_VALUE_ID).append(EQ) + .append(commonTokenValueId != null ? commonTokenValueId : -1); + } + } else { + // Traditional approach, using a join to xx_TOKEN_VALUES_V + + // Include code if present + if (code != null) { + whereClauseSegment.append(tableAlias).append(DOT).append(TOKEN_VALUE).append(operator).append(BIND_VAR); + if (LIKE.equals(operator)) { + // Must escape special wildcard characters _ and % in the parameter value string + // as well as the escape character itself. + String textSearchString = normalizedCode + .replace("+", "++") + .replace(PERCENT_WILDCARD, ESCAPE_PERCENT) + .replace(UNDERSCORE_WILDCARD, ESCAPE_UNDERSCORE) + PERCENT_WILDCARD; + bindVariables.add(SearchUtil.normalizeForSearch(textSearchString)); + appendEscape = true; + + } else { + bindVariables.add(normalizedCode); + } + } + + // Include system if present + if (system != null) { + if (code != null) { + whereClauseSegment.append(AND); + } + + // Filter on the code system for the given parameter + whereClauseSegment.append(tableAlias).append(DOT).append(CODE_SYSTEM_ID).append(EQ) + .append(nullCheck(identityCache.getCodeSystemId(system))); + } + } + } + + // Build this piece: ESCAPE '+' + if (appendEscape) { + whereClauseSegment.append(ESCAPE_EXPR); + } + + whereClauseSegment.append(RIGHT_PAREN); + parmValueProcessed = true; + } + + whereClauseSegment.append(RIGHT_PAREN); + + if (surroundWithNotExistsSubquery) { + whereClauseSegment.append(AND).append(tableAlias).append(".LOGICAL_RESOURCE_ID = ").append(logicalRsrcTableAlias).append(".LOGICAL_RESOURCE_ID"); + whereClauseSegment.append(RIGHT_PAREN); + } + queryData = new SqlQueryData(whereClauseSegment.toString(), bindVariables); + + log.exiting(CLASSNAME, METHODNAME); + return queryData; + } + } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/NewQueryBuilder.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/NewQueryBuilder.java index bdf83a29eb1..1c20036f707 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/NewQueryBuilder.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/NewQueryBuilder.java @@ -9,17 +9,27 @@ import static com.ibm.fhir.persistence.jdbc.JDBCConstants.EQ; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.LIKE; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.modifierOperatorMap; +import static com.ibm.fhir.search.SearchConstants.ID; +import static com.ibm.fhir.search.SearchConstants.LAST_UPDATED; +import static com.ibm.fhir.search.SearchConstants.PROFILE; +import static com.ibm.fhir.search.SearchConstants.SECURITY; +import static com.ibm.fhir.search.SearchConstants.TAG; +import java.util.ArrayList; import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.logging.Logger; import com.ibm.fhir.database.utils.query.Select; import com.ibm.fhir.model.resource.Location; +import com.ibm.fhir.model.resource.Resource; +import com.ibm.fhir.model.util.ModelSupport; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.exception.FHIRPersistenceNotSupportedException; import com.ibm.fhir.persistence.jdbc.connection.QueryHints; import com.ibm.fhir.persistence.jdbc.dao.api.JDBCIdentityCache; +import com.ibm.fhir.persistence.jdbc.domain.CanonicalSearchParam; import com.ibm.fhir.persistence.jdbc.domain.ChainedSearchParam; import com.ibm.fhir.persistence.jdbc.domain.CompositeSearchParam; import com.ibm.fhir.persistence.jdbc.domain.DateSearchParam; @@ -40,9 +50,13 @@ import com.ibm.fhir.persistence.jdbc.domain.SearchQuery; import com.ibm.fhir.persistence.jdbc.domain.SearchQueryRenderer; import com.ibm.fhir.persistence.jdbc.domain.SearchSortQuery; +import com.ibm.fhir.persistence.jdbc.domain.SearchWholeSystemDataQuery; +import com.ibm.fhir.persistence.jdbc.domain.SearchWholeSystemFilterQuery; +import com.ibm.fhir.persistence.jdbc.domain.SearchWholeSystemQuery; +import com.ibm.fhir.persistence.jdbc.domain.SecuritySearchParam; import com.ibm.fhir.persistence.jdbc.domain.StringSearchParam; +import com.ibm.fhir.persistence.jdbc.domain.TagSearchParam; import com.ibm.fhir.persistence.jdbc.domain.TokenSearchParam; -import com.ibm.fhir.persistence.jdbc.util.type.LastUpdatedParmBehaviorUtil; import com.ibm.fhir.search.SearchConstants; import com.ibm.fhir.search.SearchConstants.Modifier; import com.ibm.fhir.search.SearchConstants.Type; @@ -110,8 +124,6 @@ public class NewQueryBuilder { /** * Public constructor - * @param parameterDao - * @param resourceDao * @param queryHints * @param identityCache */ @@ -143,9 +155,43 @@ public Select buildCountQuery(Class resourceType, FHIRSearchContext searchCon log.entering(CLASSNAME, METHODNAME, new Object[] { resourceType.getSimpleName(), searchContext.getSearchParameters() }); - SearchCountQuery domainModel = new SearchCountQuery(resourceType.getSimpleName()); + final SearchQuery domainModel; + if (Resource.class.equals(resourceType)) { + // Whole-system search + if (allSearchParmsAreGlobal(searchContext.getSearchParameters())) { + // Can do query against global tables + domainModel = new SearchCountQuery(resourceType.getSimpleName()); + if (searchContext.getSearchResourceTypes() != null) { + // The _type parameter was specified, so we need to filter the + // query by resource type. Add an extension to the domain model + // with the specified resource type IDs. + addResourceTypeExtension(domainModel, searchContext.getSearchResourceTypes()); + } + } else { + // Not all search parameters are global (values indexed into global values tables). + // Need to do old-style UNION'd query against all resource types. + List resourceTypes = searchContext.getSearchResourceTypes(); + if (resourceTypes == null) { + // The _type parameter was not specified, so we need to generate a list + // of all supported resource types for our UNION query. + resourceTypes = new ArrayList<>(this.identityCache.getResourceTypeNames()); + resourceTypes.remove("Resource"); + resourceTypes.remove("DomainResource"); + } + // Create a domain model for each resource type + List subDomainModels = new ArrayList<>(); + for (String domainResourceType : resourceTypes) { + SearchQuery subDomainModel = new SearchCountQuery(domainResourceType); + buildModelCommon(subDomainModel, ModelSupport.getResourceType(domainResourceType), searchContext); + subDomainModels.add(subDomainModel); + } + // Create a wrapper whole-system search domain model + domainModel = new SearchWholeSystemQuery(subDomainModels, true, false); + } + } else { + domainModel = new SearchCountQuery(resourceType.getSimpleName()); + } buildModelCommon(domainModel, resourceType, searchContext); - Select result = renderQuery(domainModel, searchContext); log.exiting(CLASSNAME, METHODNAME); @@ -178,8 +224,48 @@ public Select buildQuery(Class resourceType, FHIRSearchContext searchContext) new Object[] { resourceType.getSimpleName(), searchContext.getSearchParameters() }); final SearchQuery domainModel; - - if (searchContext.hasSortParameters()) { + if (Resource.class.equals(resourceType)) { + // Whole-system search + if (allSearchParmsAreGlobal(searchContext.getSearchParameters())) { + // Can do a filter query against global tables + SearchWholeSystemFilterQuery wholeSystemFilterQuery = new SearchWholeSystemFilterQuery(); + for (SortParameter sp: searchContext.getSortParameters()) { + wholeSystemFilterQuery.add(new DomainSortParameter(sp)); + } + if (searchContext.getSearchResourceTypes() != null) { + // The _type parameter was specified, so we need to filter the + // query by resource type. Add an extension to the domain model + // with the specified resource type IDs. + addResourceTypeExtension(wholeSystemFilterQuery, searchContext.getSearchResourceTypes()); + } + domainModel = wholeSystemFilterQuery; + } else { + // Not all search parameters are global (values indexed into global values tables). + // Need to do old-style UNION'd query against all resource types. + List resourceTypes = searchContext.getSearchResourceTypes(); + if (resourceTypes == null) { + // The _type parameter was not specified, so we need to generate a list + // of all supported resource types for our UNION query. + resourceTypes = new ArrayList<>(this.identityCache.getResourceTypeNames()); + resourceTypes.remove("Resource"); + resourceTypes.remove("DomainResource"); + } + // Create a domain model for each resource type + List subDomainModels = new ArrayList<>(); + for (String domainResourceType : resourceTypes) { + SearchQuery subDomainModel = new SearchDataQuery(domainResourceType, false, false); + buildModelCommon(subDomainModel, ModelSupport.getResourceType(domainResourceType), searchContext); + subDomainModels.add(subDomainModel); + } + // Create a wrapper whole-system search domain model + SearchWholeSystemQuery wholeSystemAllDataQuery = + new SearchWholeSystemQuery(subDomainModels, false, true); + for (SortParameter sp: searchContext.getSortParameters()) { + wholeSystemAllDataQuery.add(new DomainSortParameter(sp)); + } + domainModel = wholeSystemAllDataQuery; + } + } else if (searchContext.hasSortParameters()) { // Special variant of the query which will sort based on the given sort params // and return a list of resource-ids which are then used to fetch the actual data // (matching the old query builder design...for now). @@ -269,6 +355,44 @@ private void buildIncludeModel(SearchQuery domainModel, Class resourceType, F } + /** + * Builds a query that returns resource data for the specified whole-system search. + * + * @param searchContext - the search context. + * @param resourceTypeIdToLogicalResourceIdMap - map of resource type Ids to logical resource Ids + * @return Select the query to fetch the specified list of resources + * @throws Exception + */ + public Select buildWholeSystemDataQuery(FHIRSearchContext searchContext, Map> resourceTypeIdToLogicalResourceIdMap) throws Exception { + final String METHODNAME = "buildWholeSystemDataQuery"; + log.entering(CLASSNAME, METHODNAME); + + // Create domain model for each resource type found by the filter query + List subDomainModels = new ArrayList<>(); + for (Integer resourceTypeId : resourceTypeIdToLogicalResourceIdMap.keySet()) { + String resourceType = identityCache.getResourceTypeName(resourceTypeId); + List logicalResourceIds = resourceTypeIdToLogicalResourceIdMap.get(resourceTypeId); + SearchQuery subDomainModel = new SearchWholeSystemDataQuery(resourceType); + subDomainModel.add(new WholeSystemDataExtension(resourceType, logicalResourceIds)); + buildModelCommon(subDomainModel, ModelSupport.getResourceType(resourceType), searchContext); + subDomainModels.add(subDomainModel); + } + // Create whole-system search domain model + SearchWholeSystemQuery wholeSystemAllDataQuery = + new SearchWholeSystemQuery(subDomainModels, false, false); + for (SortParameter sp: searchContext.getSortParameters()) { + wholeSystemAllDataQuery.add(new DomainSortParameter(sp)); + } + final SearchQuery domainModel = wholeSystemAllDataQuery; + + buildModelCommon(domainModel, Resource.class, searchContext); + Select result = renderQuery(domainModel, searchContext); + + log.exiting(CLASSNAME, METHODNAME); + return result; + } + /** * Contains logic common to the building of both 'count' resource queries and * 'data' resource queries. @@ -293,9 +417,9 @@ private void buildModelCommon(SearchQuery domainModel, Class resourceType, FH @Override public int compare(QueryParameter leftParameter, QueryParameter rightParameter) { int result = 0; - if (QuerySegmentAggregator.ID.equals(leftParameter.getCode())) { + if (ID.equals(leftParameter.getCode())) { result = -100; - } else if (LastUpdatedParmBehaviorUtil.LAST_UPDATED.equals(leftParameter.getCode())) { + } else if (LAST_UPDATED.equals(leftParameter.getCode())) { result = -90; } return result; @@ -337,9 +461,9 @@ private void processQueryParameter(SearchQuery domainModel, Class resourceTyp final String code = queryParm.getCode(); if (LocationUtil.isLocation(resourceType, queryParm)) { domainModel.add(new LocationSearchParam(resourceType.getSimpleName(), queryParm.getCode(), queryParm)); - } else if ("_id".equals(code)) { + } else if (ID.equals(code)) { domainModel.add(new IdSearchParam(resourceType.getSimpleName(), queryParm.getCode(), queryParm)); - } else if ("_lastUpdated".equals(code)) { + } else if (LAST_UPDATED.equals(code)) { domainModel.add(new LastUpdatedSearchParam(resourceType.getSimpleName(), queryParm.getCode(), queryParm)); } else { final Type type = queryParm.getType(); @@ -362,7 +486,13 @@ private void processQueryParameter(SearchQuery domainModel, Class resourceTyp domainModel.add(new DateSearchParam(resourceType.getSimpleName(), queryParm.getCode(), queryParm)); break; case TOKEN: - domainModel.add(new TokenSearchParam(resourceType.getSimpleName(), queryParm.getCode(), queryParm)); + if (TAG.equals(queryParm.getCode())) { + domainModel.add(new TagSearchParam(resourceType.getSimpleName(), queryParm.getCode(), queryParm)); + } else if (SECURITY.equals(queryParm.getCode())) { + domainModel.add(new SecuritySearchParam(resourceType.getSimpleName(), queryParm.getCode(), queryParm)); + } else { + domainModel.add(new TokenSearchParam(resourceType.getSimpleName(), queryParm.getCode(), queryParm)); + } break; case NUMBER: domainModel.add(new NumberSearchParam(resourceType.getSimpleName(), queryParm.getCode(), queryParm)); @@ -371,7 +501,11 @@ private void processQueryParameter(SearchQuery domainModel, Class resourceTyp domainModel.add(new QuantitySearchParam(resourceType.getSimpleName(), queryParm.getCode(), queryParm)); break; case URI: - domainModel.add(new StringSearchParam(resourceType.getSimpleName(), queryParm.getCode(), queryParm)); + if (PROFILE.equals(queryParm.getCode())) { + domainModel.add(new CanonicalSearchParam(resourceType.getSimpleName(), queryParm.getCode(), queryParm)); + } else { + domainModel.add(new StringSearchParam(resourceType.getSimpleName(), queryParm.getCode(), queryParm)); + } break; case COMPOSITE: domainModel.add(new CompositeSearchParam(resourceType.getSimpleName(), queryParm.getCode(), queryParm)); @@ -447,4 +581,21 @@ protected String getOperator(QueryParameter queryParm, String defaultOverride) { log.exiting(CLASSNAME, METHODNAME, operator); return operator; } + + private boolean allSearchParmsAreGlobal(List queryParms) { + for (QueryParameter queryParm : queryParms) { + if (!SearchConstants.SYSTEM_LEVEL_GLOBAL_PARAMETER_NAMES.contains(queryParm.getCode())) { + return false; + } + } + return true; + } + + private void addResourceTypeExtension(SearchQuery domainModel, List resourceTypes) throws FHIRPersistenceException { + List resourceTypeIds = new ArrayList<>(); + for (String resourceType : resourceTypes) { + resourceTypeIds.add(this.identityCache.getResourceTypeId(resourceType)); + } + domainModel.add(new WholeSystemResourceTypeExtension(resourceTypeIds)); + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ParameterTableSupport.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ParameterTableSupport.java new file mode 100644 index 00000000000..639c0ef15fd --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ParameterTableSupport.java @@ -0,0 +1,63 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.util; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; + +/** + * Support functions for managing the search parameter value tables + */ +public class ParameterTableSupport { + + /** + * Delete any current parameters from the whole-system and resource-specific parameter tables + * for the given resourcetype and logical_resource_id + * @param conn + * @param tablePrefix + * @param v_logical_resource_id + * @throws SQLException + */ + public static void deleteFromParameterTables(Connection conn, String tablePrefix, long v_logical_resource_id) throws SQLException { + deleteFromParameterTable(conn, tablePrefix + "_str_values", v_logical_resource_id); + deleteFromParameterTable(conn, tablePrefix + "_number_values", v_logical_resource_id); + deleteFromParameterTable(conn, tablePrefix + "_date_values", v_logical_resource_id); + deleteFromParameterTable(conn, tablePrefix + "_latlng_values", v_logical_resource_id); + deleteFromParameterTable(conn, tablePrefix + "_resource_token_refs", v_logical_resource_id); + deleteFromParameterTable(conn, tablePrefix + "_quantity_values", v_logical_resource_id); + deleteFromParameterTable(conn, tablePrefix + "_profiles", v_logical_resource_id); + deleteFromParameterTable(conn, tablePrefix + "_tags", v_logical_resource_id); + deleteFromParameterTable(conn, tablePrefix + "_security", v_logical_resource_id); + + // delete any system level parameters we have for this resource + deleteFromParameterTable(conn, "str_values", v_logical_resource_id); + deleteFromParameterTable(conn, "date_values", v_logical_resource_id); + deleteFromParameterTable(conn, "resource_token_refs", v_logical_resource_id); + deleteFromParameterTable(conn, "logical_resource_profiles", v_logical_resource_id); + deleteFromParameterTable(conn, "logical_resource_tags", v_logical_resource_id); + deleteFromParameterTable(conn, "logical_resource_security", v_logical_resource_id); + } + + /** + * Delete all parameters for the given resourceId from the parameters table + * + * @param conn + * @param tableName + * @param logicalResourceId + * @throws SQLException + */ + private static void deleteFromParameterTable(Connection conn, String tableName, long logicalResourceId) throws SQLException { + final String delStrValues = "DELETE FROM " + tableName + " WHERE logical_resource_id = ?"; + try (PreparedStatement stmt = conn.prepareStatement(delStrValues)) { + // bind parameters + stmt.setLong(1, logicalResourceId); + stmt.executeUpdate(); + } + + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/QuerySegmentAggregator.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/QuerySegmentAggregator.java index 3b1c6759854..10e71967a54 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/QuerySegmentAggregator.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/QuerySegmentAggregator.java @@ -24,7 +24,11 @@ import static com.ibm.fhir.persistence.jdbc.JDBCConstants.ROWS_ONLY; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.UNION; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.WHERE; -import static com.ibm.fhir.persistence.jdbc.util.type.LastUpdatedParmBehaviorUtil.LAST_UPDATED; +import static com.ibm.fhir.search.SearchConstants.ID; +import static com.ibm.fhir.search.SearchConstants.LAST_UPDATED; +import static com.ibm.fhir.search.SearchConstants.PROFILE; +import static com.ibm.fhir.search.SearchConstants.SECURITY; +import static com.ibm.fhir.search.SearchConstants.TAG; import java.util.ArrayList; import java.util.Arrays; @@ -73,7 +77,6 @@ public class QuerySegmentAggregator { protected static final String WHERE_CLAUSE_ROOT = "WHERE R.IS_DELETED = 'N'"; // Enables the SKIP_WHERE of WHERE clauses. - public static final String ID = "_id"; public static final String ID_COLUMN_NAME = "LOGICAL_ID "; protected static final Set SKIP_WHERE = new HashSet<>(Arrays.asList(ID, LAST_UPDATED)); @@ -610,14 +613,35 @@ protected void buildWhereClause(StringBuilder whereClause, String overrideType) missingOrNotModifierWhereClause.append(missingOrNotModifierWhereClause.length() == 0 ? WHERE : AND) .append(" NOT EXISTS (SELECT 1 FROM ") .append(valuesTable) + .append(AS); + if (TAG.equals(param.getCode()) || SECURITY.equals(param.getCode())) { + String valuesTableAlias = paramTableAlias + "_P"; + missingOrNotModifierWhereClause.append(valuesTableAlias) + .append(JOIN) + .append("COMMON_TOKEN_VALUES") .append(AS) .append(paramTableAlias) + .append(ON) + .append(paramTableAlias) + .append(".COMMON_TOKEN_VALUE_ID = ") + .append(valuesTableAlias) + .append(".COMMON_TOKEN_VALUE_ID") + .append(AND) + .append(paramTableFilter) + .append(WHERE) + .append("LR.LOGICAL_RESOURCE_ID = ") + .append(valuesTableAlias) + .append(".LOGICAL_RESOURCE_ID") + .append(RIGHT_PAREN); + } else { + missingOrNotModifierWhereClause.append(paramTableAlias) .append(WHERE) .append(paramTableFilter) .append(" AND LR.LOGICAL_RESOURCE_ID = ") .append(paramTableAlias) .append(".LOGICAL_RESOURCE_ID") .append(RIGHT_PAREN); + } } else { // Join a standard parameter table @@ -626,13 +650,33 @@ protected void buildWhereClause(StringBuilder whereClause, String overrideType) // AND param0.LOGICAL_RESOURCE_ID = LR.LOGICAL_RESOURCE_ID whereClause.append(JOIN) .append(valuesTable) - .append(AS) - .append(paramTableAlias) - .append(ON) - .append(paramTableFilter) - .append(" AND LR.LOGICAL_RESOURCE_ID = ") - .append(paramTableAlias) - .append(".LOGICAL_RESOURCE_ID"); + .append(AS); + if (TAG.equals(param.getCode()) || SECURITY.equals(param.getCode())) { + String valuesTableAlias = paramTableAlias + "_P"; + whereClause.append(valuesTableAlias) + .append(ON) + .append(" LR.LOGICAL_RESOURCE_ID = ") + .append(valuesTableAlias) + .append(".LOGICAL_RESOURCE_ID") + .append(JOIN) + .append("COMMON_TOKEN_VALUES") + .append(AS) + .append(paramTableAlias) + .append(ON) + .append(paramTableAlias) + .append(".COMMON_TOKEN_VALUE_ID = ") + .append(valuesTableAlias) + .append(".COMMON_TOKEN_VALUE_ID") + .append(AND) + .append(paramTableFilter); + } else { + whereClause.append(paramTableAlias) + .append(ON) + .append(paramTableFilter) + .append(" AND LR.LOGICAL_RESOURCE_ID = ") + .append(paramTableAlias) + .append(".LOGICAL_RESOURCE_ID"); + } } } } else { @@ -655,6 +699,12 @@ public static String tableName(String resourceType, QueryParameter param) { StringBuilder name = new StringBuilder(resourceType); switch (param.getType()) { case URI: + if (PROFILE.equals(param.getCode())) { + name.append("_PROFILES "); + } else { + name.append("_STR_VALUES "); + } + break; case STRING: case NUMBER: case QUANTITY: @@ -666,6 +716,14 @@ public static String tableName(String resourceType, QueryParameter param) { case TOKEN: if (param.isReverseChained()) { name.append("_LOGICAL_RESOURCES"); + } else if (TAG.equals(param.getCode()) && + (param.getModifier() == null || + !Modifier.TEXT.equals(param.getModifier()))) { + name.append("_TAGS "); + } else if (SECURITY.equals(param.getCode()) && + (param.getModifier() == null || + !Modifier.TEXT.equals(param.getModifier()))) { + name.append("_SECURITY "); } else { name.append("_TOKEN_VALUES_V "); // uses view to hide new issue #1366 schema } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/SortedQuerySegmentAggregator.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/SortedQuerySegmentAggregator.java index f880520ca93..dcdaef8bfad 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/SortedQuerySegmentAggregator.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/SortedQuerySegmentAggregator.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2017, 2020 + * (C) Copyright IBM Corp. 2017, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -24,8 +24,14 @@ import static com.ibm.fhir.persistence.jdbc.JDBCConstants.SPACE; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.STR_VALUE; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.TOKEN_VALUE; +import static com.ibm.fhir.search.SearchConstants.ID; +import static com.ibm.fhir.search.SearchConstants.LAST_UPDATED; +import static com.ibm.fhir.search.SearchConstants.PROFILE; +import static com.ibm.fhir.search.SearchConstants.SECURITY; +import static com.ibm.fhir.search.SearchConstants.TAG; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.logging.Logger; @@ -205,7 +211,11 @@ private String buildAggregateExpression(SortParameter sortParm, int sortParmInde StringBuilder expression = new StringBuilder(); List valueAttributeNames; - valueAttributeNames = this.getValueAttributeNames(sortParm); + if (PROFILE.equals(sortParm.getCode()) || SECURITY.equals(sortParm.getCode()) || TAG.equals(sortParm.getCode())) { + valueAttributeNames = Collections.singletonList(TOKEN_VALUE); + } else { + valueAttributeNames = this.getValueAttributeNames(sortParm); + } boolean nameProcessed = false; for (String attributeName : valueAttributeNames) { if (nameProcessed) { @@ -218,7 +228,7 @@ private String buildAggregateExpression(SortParameter sortParm, int sortParmInde } expression.append(LEFT_PAREN); - if (SearchConstants.LAST_UPDATED.equals(sortParm.getCode())) { + if (LAST_UPDATED.equals(sortParm.getCode())) { expression.append("R.LAST_UPDATED"); } else { expression.append(SORT_PARAMETER_ALIAS).append(sortParmIndex).append(DOT_CHAR); @@ -305,21 +315,36 @@ private String buildSortJoinClause() throws FHIRPersistenceException { // Build the LEFT OUTER JOINs needed to access the required sort parameters. int sortParmIndex = 1; for (SortParameter sortParm : this.sortParameters) { - if (!SearchConstants.LAST_UPDATED.equals(sortParm.getCode())) { - sortParameterNameId = ParameterNamesCache.getParameterNameId(sortParm.getCode()); - if (sortParameterNameId == null) { - // Only read...don't try and create the parameter name if it doesn't exist - sortParameterNameId = this.parameterDao.readParameterNameId(sortParm.getCode()); - if (sortParameterNameId != null) { - this.parameterDao.addParameterNamesCacheCandidate(sortParm.getCode(), sortParameterNameId); - } else { - sortParameterNameId = -1; // so we don't break the query syntax + if (!LAST_UPDATED.equals(sortParm.getCode())) { + if (PROFILE.equals(sortParm.getCode()) || SECURITY.equals(sortParm.getCode()) || TAG.equals(sortParm.getCode())) { + // For a sort by _tag or _profile or _security, we need to join the parameter-specific token + // table with the common token values table. + joinBuffer.append(" LEFT OUTER JOIN ").append(this.getSortParameterTableName(sortParm)).append(SPACE) + .append(SORT_PARAMETER_ALIAS).append(sortParmIndex).append("_P") + .append(ON) + .append(SORT_PARAMETER_ALIAS).append(sortParmIndex).append("_P") + .append(".LOGICAL_RESOURCE_ID = R.LOGICAL_RESOURCE_ID") + .append(" INNER JOIN ").append("COMMON_TOKEN_VALUES").append(SPACE) + .append(SORT_PARAMETER_ALIAS).append(sortParmIndex) + .append(ON) + .append(SORT_PARAMETER_ALIAS).append(sortParmIndex) + .append(".COMMON_TOKEN_VALUE_ID = ").append(SORT_PARAMETER_ALIAS).append(sortParmIndex) + .append("_P.COMMON_TOKEN_VALUE_ID").append(SPACE); + } else { + sortParameterNameId = ParameterNamesCache.getParameterNameId(sortParm.getCode()); + if (sortParameterNameId == null) { + // Only read...don't try and create the parameter name if it doesn't exist + sortParameterNameId = this.parameterDao.readParameterNameId(sortParm.getCode()); + if (sortParameterNameId != null) { + this.parameterDao.addParameterNamesCacheCandidate(sortParm.getCode(), sortParameterNameId); + } else { + sortParameterNameId = -1; // so we don't break the query syntax + } } - } - // Note...the PARAMETER_NAME_ID=xxx is provided as a literal because this helps - // the query optimizer significantly with index range scan cardinality estimation - joinBuffer.append(" LEFT OUTER JOIN ").append(this.getSortParameterTableName(sortParm)).append(SPACE) + // Note...the PARAMETER_NAME_ID=xxx is provided as a literal because this helps + // the query optimizer significantly with index range scan cardinality estimation + joinBuffer.append(" LEFT OUTER JOIN ").append(this.getSortParameterTableName(sortParm)).append(SPACE) .append(SORT_PARAMETER_ALIAS).append(sortParmIndex) .append(ON) .append(LEFT_PAREN) @@ -329,6 +354,7 @@ private String buildSortJoinClause() throws FHIRPersistenceException { .append(SORT_PARAMETER_ALIAS).append(sortParmIndex) .append(".LOGICAL_RESOURCE_ID = R.LOGICAL_RESOURCE_ID") .append(RIGHT_PAREN).append(SPACE); + } sortParmIndex++; } @@ -356,14 +382,24 @@ private String getSortParameterTableName(SortParameter sortParm) throws FHIRPers switch (sortParm.getType()) { case URI: case STRING: - sortParameterTableName.append("STR_VALUES"); + if (PROFILE.equals(sortParm.getCode())) { + sortParameterTableName.append("PROFILES"); + } else { + sortParameterTableName.append("STR_VALUES"); + } break; case DATE: sortParameterTableName.append("DATE_VALUES"); break; case REFERENCE: case TOKEN: - sortParameterTableName.append("TOKEN_VALUES_V"); + if (TAG.equals(sortParm.getCode())) { + sortParameterTableName.append("TAGS"); + } else if (SECURITY.equals(sortParm.getCode())) { + sortParameterTableName.append("SECURITY"); + } else { + sortParameterTableName.append("TOKEN_VALUES_V"); + } break; case NUMBER: sortParameterTableName.append("NUMBER_VALUES"); @@ -437,7 +473,7 @@ private String buildSysLvlOrderByClause() throws FHIRPersistenceException { String code = sortParm.getCode(); if (ID.equals(code)) { orderByBuffer.append(ID_COLUMN_NAME); - } else if (LastUpdatedParmBehaviorUtil.LAST_UPDATED.equals(code)) { + } else if (LAST_UPDATED.equals(code)) { orderByBuffer.append(LastUpdatedParmBehaviorUtil.LAST_UPDATED_COLUMN_NAME).append(SPACE); } else { throw new FHIRPersistenceNotSupportedException( diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/WholeSystemDataExtension.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/WholeSystemDataExtension.java new file mode 100644 index 00000000000..70a63bb5598 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/WholeSystemDataExtension.java @@ -0,0 +1,40 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.util; + +import java.util.List; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.jdbc.domain.SearchExtension; +import com.ibm.fhir.persistence.jdbc.domain.SearchQueryVisitor; + +/** + * A SearchExtension used to add whole-system data search filters to the + * whole-system data query. + */ +public class WholeSystemDataExtension implements SearchExtension { + // The resource type being processed + private final String resourceType; + + // The list of LOGICAL_RESOURCE_ID values to filter the query + private final List logicalResourceIds; + + /** + * Public constructor + * @param resourceType + * @param logicalResourcesIds + */ + public WholeSystemDataExtension(String resourceType, List logicalResourceIds) { + this.resourceType = resourceType; + this.logicalResourceIds = logicalResourceIds; + } + + @Override + public T visit(T query, SearchQueryVisitor visitor) throws FHIRPersistenceException { + return visitor.addWholeSystemDataFilter(query, resourceType, logicalResourceIds); + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/WholeSystemResourceTypeExtension.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/WholeSystemResourceTypeExtension.java new file mode 100644 index 00000000000..95603586353 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/WholeSystemResourceTypeExtension.java @@ -0,0 +1,36 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.util; + +import java.util.List; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.jdbc.domain.SearchExtension; +import com.ibm.fhir.persistence.jdbc.domain.SearchQueryVisitor; + +/** + * A SearchExtension used to add resource type id filters to the + * whole-system count and data filter queries when the _type parameter + * is specified. + */ +public class WholeSystemResourceTypeExtension implements SearchExtension { + // The list of RESOURCE_TYPE_ID values to filter the query + private final List resourceTypeIds; + + /** + * Public constructor + * @param resourceTypeIds + */ + public WholeSystemResourceTypeExtension(List resourceTypeIds) { + this.resourceTypeIds = resourceTypeIds; + } + + @Override + public T visit(T query, SearchQueryVisitor visitor) throws FHIRPersistenceException { + return visitor.addWholeSystemResourceTypeFilter(query, resourceTypeIds); + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/cache/test/ResourceReferenceCacheImplTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/cache/test/ResourceReferenceCacheImplTest.java index 45526a6d84f..1c53a815905 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/cache/test/ResourceReferenceCacheImplTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/cache/test/ResourceReferenceCacheImplTest.java @@ -26,7 +26,7 @@ public void testExternalSystemNames() { // A cache with a limited size of 3 code systems and 2 token values // For this test to work, we have to make sure we can always resolve // all the code systems, so don't make the cache size smaller than 3 - CommonTokenValuesCacheImpl impl = new CommonTokenValuesCacheImpl(3, 2); + CommonTokenValuesCacheImpl impl = new CommonTokenValuesCacheImpl(3, 2, 1); impl.addCodeSystem("sys1", 1); impl.addCodeSystem("sys2", 2); impl.addCodeSystem("sys3", 3); diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompartmentTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompartmentTest.java index c839284e5f1..6851fd931bd 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompartmentTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompartmentTest.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -15,6 +15,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; @@ -44,8 +45,8 @@ public void bootstrapDatabase() throws Exception { derbyInit = new DerbyInitializer(this.testProps); IConnectionProvider cp = derbyInit.getConnectionProvider(false); this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompositeTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompositeTest.java index ab89958d468..0dd3ad82480 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompositeTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompositeTest.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2018, 2020 + * (C) Copyright IBM Corp. 2018, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -15,6 +15,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; @@ -25,9 +26,9 @@ public class JDBCSearchCompositeTest extends AbstractSearchCompositeTest { private Properties testProps; - + private PoolConnectionProvider connectionPool; - + private FHIRPersistenceJDBCCache cache; public JDBCSearchCompositeTest() throws Exception { @@ -43,8 +44,8 @@ public void bootstrapDatabase() throws Exception { IConnectionProvider cp = derbyInit.getConnectionProvider(false); this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); } } @@ -55,7 +56,7 @@ public FHIRPersistence getPersistenceImpl() throws Exception { } return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); } - + @Override protected void shutdownPools() throws Exception { // Mark the pool as no longer in use. This allows the pool to check for diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchDateTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchDateTest.java index 2770fdef857..97545c59295 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchDateTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchDateTest.java @@ -15,6 +15,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; @@ -41,8 +42,8 @@ public void bootstrapDatabase() throws Exception { IConnectionProvider cp = derbyInit.getConnectionProvider(false); this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); } } @Override diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchIdLastUpdatedTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchIdLastUpdatedTest.java index 445d3f20e70..1d1e06e24eb 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchIdLastUpdatedTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchIdLastUpdatedTest.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -15,6 +15,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; @@ -23,9 +24,9 @@ public class JDBCSearchIdLastUpdatedTest extends AbstractSearchIdAndLastUpdatedTest { private Properties testProps; - + private PoolConnectionProvider connectionPool; - + private FHIRPersistenceJDBCCache cache; public JDBCSearchIdLastUpdatedTest() throws Exception { @@ -40,8 +41,8 @@ public void bootstrapDatabase() throws Exception { derbyInit = new DerbyInitializer(this.testProps); IConnectionProvider cp = derbyInit.getConnectionProvider(false); this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); } } @@ -52,7 +53,7 @@ public FHIRPersistence getPersistenceImpl() throws Exception { } return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); } - + @Override protected void shutdownPools() throws Exception { // Mark the pool as no longer in use. This allows the pool to check for diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNearTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNearTest.java index 142697d18a5..883fe5a5215 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNearTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNearTest.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -44,6 +44,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; @@ -95,8 +96,8 @@ public void startup() throws Exception { savedResource = TestUtil.readExampleResource("json/spec/location-example.json"); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); persistence = new FHIRPersistenceJDBCImpl(this.testProps, connectionPool, cache); SingleResourceResult result = diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNumberTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNumberTest.java index 3cf43b8eeaa..7375c5ac40d 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNumberTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNumberTest.java @@ -15,6 +15,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; @@ -40,8 +41,8 @@ public void bootstrapDatabase() throws Exception { derbyInit = new DerbyInitializer(this.testProps); IConnectionProvider cp = derbyInit.getConnectionProvider(false); this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchQuantityTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchQuantityTest.java index e6a0a55736c..5dcbacc7200 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchQuantityTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchQuantityTest.java @@ -15,6 +15,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; @@ -41,8 +42,8 @@ public void bootstrapDatabase() throws Exception { derbyInit = new DerbyInitializer(this.testProps); IConnectionProvider cp = derbyInit.getConnectionProvider(false); this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchReferenceTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchReferenceTest.java index 9a1e6115ee7..561a8bef57d 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchReferenceTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchReferenceTest.java @@ -15,6 +15,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; @@ -41,8 +42,8 @@ public void bootstrapDatabase() throws Exception { derbyInit = new DerbyInitializer(this.testProps); IConnectionProvider cp = derbyInit.getConnectionProvider(false); this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchStringTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchStringTest.java index 2583fd0b2ef..ff20f9f60d6 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchStringTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchStringTest.java @@ -15,6 +15,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; @@ -42,8 +43,8 @@ public void bootstrapDatabase() throws Exception { derbyInit = new DerbyInitializer(this.testProps); IConnectionProvider cp = derbyInit.getConnectionProvider(false); this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchTokenTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchTokenTest.java index 38c081df729..6d4ff1c3e4b 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchTokenTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchTokenTest.java @@ -15,6 +15,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; @@ -42,8 +43,8 @@ public void bootstrapDatabase() throws Exception { derbyInit = new DerbyInitializer(this.testProps); IConnectionProvider cp = derbyInit.getConnectionProvider(false); this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchURITest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchURITest.java index 6115fff146c..f8baa21d3bc 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchURITest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchURITest.java @@ -15,6 +15,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; @@ -40,8 +41,8 @@ public void bootstrapDatabase() throws Exception { derbyInit = new DerbyInitializer(this.testProps); IConnectionProvider cp = derbyInit.getConnectionProvider(false); this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCWholeSystemSearchTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCWholeSystemSearchTest.java index 66b7f2f9031..5db0249970c 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCWholeSystemSearchTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCWholeSystemSearchTest.java @@ -15,6 +15,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; @@ -41,8 +42,8 @@ public void bootstrapDatabase() throws Exception { derbyInit = new DerbyInitializer(this.testProps); IConnectionProvider cp = derbyInit.getConnectionProvider(false); this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCChangesTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCChangesTest.java index 014c5f42c7c..1c4fbd0d316 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCChangesTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCChangesTest.java @@ -18,6 +18,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; @@ -47,8 +48,8 @@ public void bootstrapDatabase() throws Exception { derbyInit = new DerbyInitializer(this.testProps); IConnectionProvider cp = derbyInit.getConnectionProvider(false); this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCCompartmentTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCCompartmentTest.java index c085c9d5cd9..32ce6244c31 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCCompartmentTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCCompartmentTest.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2017, 2019, 2020 + * (C) Copyright IBM Corp. 2017, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -15,6 +15,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; @@ -23,13 +24,13 @@ public class JDBCCompartmentTest extends AbstractCompartmentTest { - + private Properties testProps; - + private PoolConnectionProvider connectionPool; - + private FHIRPersistenceJDBCCache cache; - + public JDBCCompartmentTest() throws Exception { this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); } @@ -42,11 +43,11 @@ public void bootstrapDatabase() throws Exception { derbyInit = new DerbyInitializer(this.testProps); IConnectionProvider cp = derbyInit.getConnectionProvider(false); this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); } } - + @Override public FHIRPersistence getPersistenceImpl() throws Exception { if (this.connectionPool == null) { diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCDeleteTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCDeleteTest.java index 1d92290fe68..1256d3fb93e 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCDeleteTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCDeleteTest.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2017, 2019, 2020 + * (C) Copyright IBM Corp. 2017, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -15,6 +15,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; @@ -30,12 +31,12 @@ public class JDBCDeleteTest extends AbstractDeleteTest { // test properties private Properties testProps; - + // Connection pool used to provide connections for the FHIRPersistenceJDBCImpl private PoolConnectionProvider connectionPool; - + private FHIRPersistenceJDBCCache cache; - + public JDBCDeleteTest() throws Exception { this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); } @@ -48,11 +49,11 @@ public void bootstrapDatabase() throws Exception { derbyInit = new DerbyInitializer(this.testProps); IConnectionProvider cp = derbyInit.getConnectionProvider(false); this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); } } - + @Override public FHIRPersistence getPersistenceImpl() throws Exception { if (this.connectionPool == null) { @@ -60,7 +61,7 @@ public FHIRPersistence getPersistenceImpl() throws Exception { } return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); } - + @Override protected void shutdownPools() throws Exception { // Mark the pool as no longer in use. This allows the pool to check for diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCEraseTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCEraseTest.java index 3433f4e5cf2..e53c5efd85f 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCEraseTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCEraseTest.java @@ -18,6 +18,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; @@ -48,8 +49,8 @@ public void bootstrapDatabase() throws Exception { derbyInit = new DerbyInitializer(this.testProps); IConnectionProvider cp = derbyInit.getConnectionProvider(false); this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); // Have to inject the Cache Entry. ResourceTypesCache.putResourceTypeId("all~default", "Basic", 8); diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCExportTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCExportTest.java index 9c2506e473e..21188a3e259 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCExportTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCExportTest.java @@ -18,6 +18,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; @@ -47,8 +48,8 @@ public void bootstrapDatabase() throws Exception { derbyInit = new DerbyInitializer(this.testProps); IConnectionProvider cp = derbyInit.getConnectionProvider(false); this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCIncludeRevincludeTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCIncludeRevincludeTest.java index af9d5a8d134..b4b81324015 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCIncludeRevincludeTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCIncludeRevincludeTest.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2017, 2020 + * (C) Copyright IBM Corp. 2017, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -15,6 +15,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; @@ -26,7 +27,7 @@ public class JDBCIncludeRevincludeTest extends AbstractIncludeRevincludeTest { // The connection pool wrapping the Derby test database private PoolConnectionProvider connectionPool; - + private FHIRPersistenceJDBCCache cache; public JDBCIncludeRevincludeTest() throws Exception { @@ -41,11 +42,11 @@ public void bootstrapDatabase() throws Exception { derbyInit = new DerbyInitializer(this.testProps); IConnectionProvider cp = derbyInit.getConnectionProvider(false); this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); } } - + @Override public FHIRPersistence getPersistenceImpl() throws Exception { if (this.connectionPool == null) { @@ -53,7 +54,7 @@ public FHIRPersistence getPersistenceImpl() throws Exception { } return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); } - + @Override protected void shutdownPools() throws Exception { // Mark the pool as no longer in use. This allows the pool to check for diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCMultiResourceTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCMultiResourceTest.java index eb43e09e864..34eeb477f28 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCMultiResourceTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCMultiResourceTest.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2018, 2019, 2020 + * (C) Copyright IBM Corp. 2018, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -15,6 +15,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; @@ -23,14 +24,14 @@ public class JDBCMultiResourceTest extends AbstractMultiResourceTest { - + private Properties testProps; - + // The connection pool wrapping the Derby test database private PoolConnectionProvider connectionPool; - + private FHIRPersistenceJDBCCache cache; - + public JDBCMultiResourceTest() throws Exception { this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); } @@ -43,11 +44,11 @@ public void bootstrapDatabase() throws Exception { derbyInit = new DerbyInitializer(this.testProps); IConnectionProvider cp = derbyInit.getConnectionProvider(false); this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); } } - + @Override public FHIRPersistence getPersistenceImpl() throws Exception { if (this.connectionPool == null) { @@ -55,7 +56,7 @@ public FHIRPersistence getPersistenceImpl() throws Exception { } return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); } - + @Override protected void shutdownPools() throws Exception { // Mark the pool as no longer in use. This allows the pool to check for diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCPagingTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCPagingTest.java index 32c1c87bc78..b0b8d8f8003 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCPagingTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCPagingTest.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2017, 2019, 2020 + * (C) Copyright IBM Corp. 2017, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -9,7 +9,6 @@ import java.sql.Connection; import java.sql.SQLException; import java.util.Properties; -import java.util.logging.Logger; import com.ibm.fhir.database.utils.api.IConnectionProvider; import com.ibm.fhir.database.utils.derby.DerbyMaster; @@ -19,6 +18,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; @@ -27,13 +27,13 @@ public class JDBCPagingTest extends AbstractPagingTest { - + private Properties testProps; - + private PoolConnectionProvider connectionPool; - + private FHIRPersistenceJDBCCache cache; - + public JDBCPagingTest() throws Exception { this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); } @@ -46,11 +46,11 @@ public void bootstrapDatabase() throws Exception { derbyInit = new DerbyInitializer(this.testProps); IConnectionProvider cp = derbyInit.getConnectionProvider(false); this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); } } - + @Override public FHIRPersistence getPersistenceImpl() throws Exception { if (this.connectionPool == null) { @@ -67,7 +67,7 @@ protected void shutdownPools() throws Exception { this.connectionPool.close(); } } - + @Override protected void debugLocks() { // Exception running a query. Let's dump the lock table diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCReverseChainTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCReverseChainTest.java index ac1910a32cc..8ae2a2781fc 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCReverseChainTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCReverseChainTest.java @@ -15,6 +15,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; @@ -41,8 +42,8 @@ public void bootstrapDatabase() throws Exception { derbyInit = new DerbyInitializer(this.testProps); IConnectionProvider cp = derbyInit.getConnectionProvider(false); this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCSortTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCSortTest.java index 8bbce32a55c..5f332bc3364 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCSortTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCSortTest.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2017,2019, 2020 + * (C) Copyright IBM Corp. 2017, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -15,6 +15,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; @@ -23,13 +24,13 @@ public class JDBCSortTest extends AbstractSortTest { - + private Properties testProps; - + private PoolConnectionProvider connectionPool; - + private FHIRPersistenceJDBCCache cache; - + public JDBCSortTest() throws Exception { this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); } @@ -42,11 +43,11 @@ public void bootstrapDatabase() throws Exception { derbyInit = new DerbyInitializer(this.testProps); IConnectionProvider cp = derbyInit.getConnectionProvider(false); this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); } } - + @Override public FHIRPersistence getPersistenceImpl() throws Exception { if (this.connectionPool == null) { @@ -54,7 +55,7 @@ public FHIRPersistence getPersistenceImpl() throws Exception { } return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); } - + @Override protected void shutdownPools() throws Exception { // Mark the pool as no longer in use. This allows the pool to check for diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/erase/EraseTestMain.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/erase/EraseTestMain.java index 2d9508540b9..7376384ea3f 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/erase/EraseTestMain.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/erase/EraseTestMain.java @@ -28,6 +28,7 @@ import com.ibm.fhir.persistence.jdbc.connection.FHIRDbFlavorImpl; import com.ibm.fhir.persistence.jdbc.dao.EraseResourceDAO; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.api.IIdNameCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; import com.ibm.fhir.schema.app.util.CommonUtil; @@ -222,6 +223,11 @@ public void prefill(Map content) { return cache; } + @Override + public IIdNameCache getResourceTypeNameCache() { + return null; + } + @Override public INameIdCache getParameterNameCache() { return null; diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/Main.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/Main.java index 85bbf4e0d92..71c7fdc8664 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/Main.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/Main.java @@ -53,6 +53,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; @@ -309,7 +310,7 @@ protected void process() throws Exception { break; } } - + /** * Configure the property group to inject the tenantKey, which is the only attribute * required for this scenario @@ -321,12 +322,12 @@ protected void configure(TestFHIRConfigProvider configProvider) throws Exception final String dsPropertyName = FHIRConfiguration.PROPERTY_DATASOURCES + "/default"; // The bare necessities we need to provide to the persistence layer in this case - final String jsonString = " {" + - " \"tenantKey\": \"" + this.tenantKey + "\"," + - " \"type\": \"db2\"," + - " \"multitenant\": true" + + final String jsonString = " {" + + " \"tenantKey\": \"" + this.tenantKey + "\"," + + " \"type\": \"db2\"," + + " \"multitenant\": true" + "}"; - + try (JsonReader reader = JSON_READER_FACTORY.createReader(new ByteArrayInputStream(jsonString.getBytes(StandardCharsets.UTF_8)))) { JsonObject jsonObj = reader.readObject(); PropertyGroup pg = new PropertyGroup(jsonObj); @@ -350,9 +351,9 @@ protected void processDB2() throws Exception { ITransactionProvider transactionProvider = new SimpleTransactionProvider(connectionPool); TestFHIRConfigProvider configProvider = new TestFHIRConfigProvider(new DefaultFHIRConfigProvider()); configure(configProvider); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); - + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); + // Provide the credentials we need for accessing a multi-tenant schema (if enabled) // Must set this BEFORE we create our persistence object if (this.tenantName == null || tenantKey == null) { @@ -432,8 +433,8 @@ protected void processDerby() throws Exception { // IConnectionProvider implementation used by the persistence // layer to obtain connections. try (DerbyFhirDatabase database = new DerbyFhirDatabase()) { - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); persistence = new FHIRPersistenceJDBCImpl(this.configProps, database, cache); // create a custom list of operations to apply in order to each resource @@ -484,8 +485,8 @@ protected void processDerbyNetwork() throws Exception { PoolConnectionProvider connectionPool = new PoolConnectionProvider(cp, this.threads); ITransactionProvider transactionProvider = new SimpleTransactionProvider(connectionPool); FHIRConfigProvider configProvider = new DefaultFHIRConfigProvider(); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); // create a custom list of operations to apply in order to each resource DriverMetrics dm = new DriverMetrics(); @@ -539,8 +540,8 @@ protected void processPostgreSql() throws Exception { PoolConnectionProvider connectionPool = new PoolConnectionProvider(cp, this.threads); ITransactionProvider transactionProvider = new SimpleTransactionProvider(connectionPool); FHIRConfigProvider configProvider = new DefaultFHIRConfigProvider(); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); // create a custom list of operations to apply in order to each resource @@ -594,6 +595,11 @@ protected void processParse() throws Exception { driver.setValidator(new ValidationProcessor()); } + // If we're testing concurrency, pass in a thread pool + if (this.pool != null) { + driver.setPool(this.pool, this.maxInflight); + } + runDriver(driver); } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/R4JDBCExamplesTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/R4JDBCExamplesTest.java index 8d6fa0900b7..dcbaefb4083 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/R4JDBCExamplesTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/R4JDBCExamplesTest.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -28,6 +28,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; @@ -37,10 +38,10 @@ public class R4JDBCExamplesTest extends AbstractPersistenceTest { private Properties properties; - + // provides connections to a bootstrapped Derby database private IConnectionProvider derbyConnectionProvider; - + /** * Public constructor * @throws Exception @@ -51,15 +52,15 @@ public R4JDBCExamplesTest() throws Exception { @Test(groups = { "jdbc-seed" }, singleThreaded = true, priority = -1) public void perform() throws Exception { - + // Use connection pool and transaction provider to make sure the resource operations // of each resource are committed after the processing is finished, and because this // testng test process the samples one by one, so set the connection pool size to 1. PoolConnectionProvider connectionPool = new PoolConnectionProvider(derbyConnectionProvider, 1); ITransactionProvider transactionProvider = new SimpleTransactionProvider(connectionPool); FHIRConfigProvider configProvider = new DefaultFHIRConfigProvider(); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); - FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); List operations = new ArrayList<>(); operations.add(new CreateOperation()); diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/CanonicalSupportTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/CanonicalSupportTest.java new file mode 100644 index 00000000000..5d9a4734ab2 --- /dev/null +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/CanonicalSupportTest.java @@ -0,0 +1,223 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.test.util; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; + +import org.testng.annotations.Test; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceProfileRec; +import com.ibm.fhir.persistence.jdbc.util.CanonicalSupport; + +/** + * Unit test for {@link CanonicalSupport} + */ +public class CanonicalSupportTest { + + /** + * Test a canonical uri without version or fragment + * @throws FHIRPersistenceException + */ + @Test + public void simpleUriTest() throws FHIRPersistenceException { + final String paramValue = "https://example.org/foo/bar"; + final int parameterNameId = 7; + final String resourceType = "Patient"; + final int resourceTypeId = 17; + final long logicalResourceId = 42; + final boolean systemLevel = true; + ResourceProfileRec rec = CanonicalSupport.makeResourceProfileRec(parameterNameId, resourceType, resourceTypeId, logicalResourceId, paramValue, systemLevel); + + assertNotNull(rec); + assertEquals(rec.getLogicalResourceId(), logicalResourceId); + assertEquals(rec.getCanonicalValue(), paramValue); + assertEquals(rec.getParameterNameId(), parameterNameId); + assertEquals(rec.getResourceType(), resourceType); + assertEquals(rec.getResourceTypeId(), resourceTypeId); + assertNull(rec.getVersion()); + assertNull(rec.getFragment()); + } + + /** + * Test a canonical uri with version but no fragment + * @throws FHIRPersistenceException + */ + @Test + public void uriVersionTest() throws FHIRPersistenceException { + final String uri = "https://example.org/foo/bar"; + final String version = "1.0"; + final String paramValue = uri + "|" + version; + final int parameterNameId = 7; + final String resourceType = "Patient"; + final int resourceTypeId = 17; + final long logicalResourceId = 42; + final boolean systemLevel = true; + ResourceProfileRec rec = CanonicalSupport.makeResourceProfileRec(parameterNameId, resourceType, resourceTypeId, logicalResourceId, paramValue, systemLevel); + + assertNotNull(rec); + assertEquals(rec.getLogicalResourceId(), logicalResourceId); + assertEquals(rec.getCanonicalValue(), uri); + assertEquals(rec.getParameterNameId(), parameterNameId); + assertEquals(rec.getResourceType(), resourceType); + assertEquals(rec.getResourceTypeId(), resourceTypeId); + assertEquals(rec.getVersion(), version); + assertNull(rec.getFragment()); + } + + /** + * Test a canonical uri with version and fragment + * @throws FHIRPersistenceException + */ + @Test + public void uriVersionFragmentTest() throws FHIRPersistenceException { + final String uri = "https://example.org/foo/bar"; + final String version = "1.0"; + final String fragment = "F1"; + final String paramValue = uri + "|" + version + "#" + fragment; + final int parameterNameId = 7; + final String resourceType = "Patient"; + final int resourceTypeId = 17; + final long logicalResourceId = 42; + final boolean systemLevel = true; + ResourceProfileRec rec = CanonicalSupport.makeResourceProfileRec(parameterNameId, resourceType, resourceTypeId, logicalResourceId, paramValue, systemLevel); + + assertNotNull(rec); + assertEquals(rec.getLogicalResourceId(), logicalResourceId); + assertEquals(rec.getCanonicalValue(), uri); + assertEquals(rec.getParameterNameId(), parameterNameId); + assertEquals(rec.getResourceType(), resourceType); + assertEquals(rec.getResourceTypeId(), resourceTypeId); + assertEquals(rec.getVersion(), version); + assertEquals(rec.getFragment(), fragment); + } + + /** + * Test a canonical uri with fragment but no version + * @throws FHIRPersistenceException + */ + @Test + public void uriFragmentTest() throws FHIRPersistenceException { + final String uri = "https://example.org/foo/bar"; + final String fragment = "F1"; + final String paramValue = uri + "#" + fragment; + final int parameterNameId = 7; + final String resourceType = "Patient"; + final int resourceTypeId = 17; + final long logicalResourceId = 42; + final boolean systemLevel = true; + ResourceProfileRec rec = CanonicalSupport.makeResourceProfileRec(parameterNameId, resourceType, resourceTypeId, logicalResourceId, paramValue, systemLevel); + + assertNotNull(rec); + assertEquals(rec.getLogicalResourceId(), logicalResourceId); + assertEquals(rec.getCanonicalValue(), uri); + assertEquals(rec.getParameterNameId(), parameterNameId); + assertEquals(rec.getResourceType(), resourceType); + assertEquals(rec.getResourceTypeId(), resourceTypeId); + assertNull(rec.getVersion()); + assertEquals(rec.getFragment(), fragment); + } + + /** + * Test a canonical uri with empty fragment + * @throws FHIRPersistenceException + */ + @Test + public void uriEmptyFragmentTest() throws FHIRPersistenceException { + final String uri = "https://example.org/foo/bar"; + final String fragment = ""; + final String paramValue = uri + "#" + fragment; + final int parameterNameId = 7; + final String resourceType = "Patient"; + final int resourceTypeId = 17; + final long logicalResourceId = 42; + final boolean systemLevel = true; + ResourceProfileRec rec = CanonicalSupport.makeResourceProfileRec(parameterNameId, resourceType, resourceTypeId, logicalResourceId, paramValue, systemLevel); + + assertNotNull(rec); + assertEquals(rec.getLogicalResourceId(), logicalResourceId); + assertEquals(rec.getCanonicalValue(), uri); + assertEquals(rec.getParameterNameId(), parameterNameId); + assertEquals(rec.getResourceType(), resourceType); + assertEquals(rec.getResourceTypeId(), resourceTypeId); + assertNull(rec.getVersion()); + assertNull(rec.getFragment()); + } + + /** + * Test a canonical uri with empty version + * @throws FHIRPersistenceException + */ + @Test + public void uriEmptyVersionTest() throws FHIRPersistenceException { + final String uri = "https://example.org/foo/bar"; + final String version = ""; + final String paramValue = uri + "|" + version; + final int parameterNameId = 7; + final String resourceType = "Patient"; + final int resourceTypeId = 17; + final long logicalResourceId = 42; + final boolean systemLevel = true; + ResourceProfileRec rec = CanonicalSupport.makeResourceProfileRec(parameterNameId, resourceType, resourceTypeId, logicalResourceId, paramValue, systemLevel); + + assertNotNull(rec); + assertEquals(rec.getLogicalResourceId(), logicalResourceId); + assertEquals(rec.getCanonicalValue(), uri); + assertEquals(rec.getParameterNameId(), parameterNameId); + assertEquals(rec.getResourceType(), resourceType); + assertEquals(rec.getResourceTypeId(), resourceTypeId); + assertNull(rec.getVersion()); + assertNull(rec.getFragment()); + } + + /** + * Test a canonical uri with empty version and fragment + * @throws FHIRPersistenceException + */ + @Test + public void uriEmptyVersionEmptyFragmentTest() throws FHIRPersistenceException { + final String uri = "https://example.org/foo/bar"; + final String version = ""; + final String fragment = ""; + final String paramValue = uri + "|" + version + "#" + fragment; + final int parameterNameId = 7; + final String resourceType = "Patient"; + final int resourceTypeId = 17; + final long logicalResourceId = 42; + final boolean systemLevel = true; + ResourceProfileRec rec = CanonicalSupport.makeResourceProfileRec(parameterNameId, resourceType, resourceTypeId, logicalResourceId, paramValue, systemLevel); + + assertNotNull(rec); + assertEquals(rec.getLogicalResourceId(), logicalResourceId); + assertEquals(rec.getCanonicalValue(), uri); + assertEquals(rec.getParameterNameId(), parameterNameId); + assertEquals(rec.getResourceType(), resourceType); + assertEquals(rec.getResourceTypeId(), resourceTypeId); + assertNull(rec.getVersion()); + assertNull(rec.getFragment()); + } + + /** + * Test a canonical uri with invalid fragment/version order + * @throws FHIRPersistenceException + */ + @Test(expectedExceptions = FHIRPersistenceException.class) + public void uriInvalidFragmentVersionTest() throws FHIRPersistenceException { + final String uri = "https://example.org/foo/bar"; + final String version = "1.0"; + final String fragment = "F1"; + final String paramValue = uri + "#" + fragment + "|" + version; + final int parameterNameId = 7; + final String resourceType = "Patient"; + final int resourceTypeId = 17; + final long logicalResourceId = 42; + final boolean systemLevel = true; + CanonicalSupport.makeResourceProfileRec(parameterNameId, resourceType, resourceTypeId, logicalResourceId, paramValue, systemLevel); + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterCounter.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterCounter.java new file mode 100644 index 00000000000..17064043ea5 --- /dev/null +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterCounter.java @@ -0,0 +1,191 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.test.util; + +import static com.ibm.fhir.schema.app.util.CommonUtil.getDbAdapter; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.JdbcPropertyAdapter; +import com.ibm.fhir.database.utils.common.JdbcTarget; +import com.ibm.fhir.database.utils.db2.Db2Translator; +import com.ibm.fhir.database.utils.derby.DerbyTranslator; +import com.ibm.fhir.database.utils.model.DbType; +import com.ibm.fhir.database.utils.postgres.PostgresTranslator; +import com.ibm.fhir.model.type.code.FHIRResourceType; +import com.ibm.fhir.schema.app.util.CommonUtil; + +/** + * Utility to count the number of parameter values stored in + * a database + */ +public class ParameterCounter { + private static final Logger logger = Logger.getLogger(ParameterCounter.class.getName()); + private final Properties databaseProperties = new Properties(); + private String databasePropertiesFile = "db.properties"; + private DbType dbType; + private IDatabaseTranslator translator; + private String schemaName; + + public void parseArgs(String[] args) { + for (int i=0; i resourceTypes = new HashSet<>(Arrays.stream(FHIRResourceType.Value.values()) + .map(FHIRResourceType.Value::value) + .collect(Collectors.toSet())); + + final List paramTables = Arrays.asList("STR_VALUES", "NUMBER_VALUES", "DATE_VALUES", "RESOURCE_TOKEN_REFS", "QUANTITY_VALUES", "LATLNG_VALUES"); + + for (String pt: paramTables) { + int parameterCount = 0; + for (String resourceType: resourceTypes) { + logger.fine(() -> "Counting " + resourceType + "_" + pt); + ParameterCounterDAO dao = new ParameterCounterDAO(schemaName, resourceType, pt); + parameterCount += adapter.runStatement(dao); + } + + System.out.println(String.format("COUNT [%20s] = %d", pt, parameterCount)); + } + } + + /** + * Get a connection to the database configured in the db.properties file + * @return + */ + protected Connection createConnection() { + Properties connectionProperties = new Properties(); + JdbcPropertyAdapter adapter = CommonUtil.getPropertyAdapter(dbType, this.databaseProperties); + adapter.getExtraProperties(connectionProperties); + + String url = translator.getUrl(this.databaseProperties); + logger.info("Opening connection to: " + url); + Connection connection; + try { + connection = DriverManager.getConnection(url, connectionProperties); + connection.setAutoCommit(false); + } catch (SQLException x) { + throw translator.translate(x); + } + return connection; + } + + /** + * Main entry point + * @param args + */ + public static void main(String[] args) { + ParameterCounter m = new ParameterCounter(); + try { + m.parseArgs(args); + m.configure(); + m.process(); + } catch (Exception x) { + logger.log(Level.SEVERE, x.getMessage(), x); + } + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterCounterDAO.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterCounterDAO.java new file mode 100644 index 00000000000..eaac1d10178 --- /dev/null +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterCounterDAO.java @@ -0,0 +1,45 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.test.util; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import com.ibm.fhir.database.utils.api.IDatabaseSupplier; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; + +/** + * Simple DAO to count the number of rows in a search parameter table + */ +public class ParameterCounterDAO implements IDatabaseSupplier { + private final String schemaName; + private final String resourceType; + private final String parameterTable; + + public ParameterCounterDAO(String schemaName, String resourceType, String parameterTable) { + this.schemaName = schemaName; + this.resourceType = resourceType; + this.parameterTable = parameterTable; + } + + @Override + public Integer run(IDatabaseTranslator translator, Connection c) { + final String tableName = resourceType + "_" + parameterTable; + final String qTableName = DataDefinitionUtil.getQualifiedName(schemaName, tableName); + final String SQL = "SELECT COUNT(*) FROM " + qTableName; + try (Statement s = c.createStatement()) { + ResultSet rs = s.executeQuery(SQL); + rs.next(); + return rs.getInt(1); + } catch (SQLException x) { + throw translator.translate(x); + } + } +} \ No newline at end of file diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java index e945502d1ca..a4581d98ac1 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java @@ -77,6 +77,7 @@ import com.ibm.fhir.schema.control.EnableForeignKey; import com.ibm.fhir.schema.control.FhirSchemaConstants; import com.ibm.fhir.schema.control.FhirSchemaGenerator; +import com.ibm.fhir.schema.control.GetLogicalResourceNeedsV0014Migration; import com.ibm.fhir.schema.control.GetResourceChangeLogEmpty; import com.ibm.fhir.schema.control.GetResourceTypeList; import com.ibm.fhir.schema.control.GetTenantInfo; @@ -84,6 +85,7 @@ import com.ibm.fhir.schema.control.GetXXLogicalResourceNeedsMigration; import com.ibm.fhir.schema.control.InitializeLogicalResourceDenorms; import com.ibm.fhir.schema.control.JavaBatchSchemaGenerator; +import com.ibm.fhir.schema.control.MigrateV0014LogicalResourceIsDeletedLastUpdated; import com.ibm.fhir.schema.control.OAuthSchemaGenerator; import com.ibm.fhir.schema.control.PopulateParameterNames; import com.ibm.fhir.schema.control.PopulateResourceTypes; @@ -174,6 +176,7 @@ public class Main { private boolean dropDetached; private boolean deleteTenantMeta; private boolean revokeTenantKey; + private boolean revokeAllTenantKeys; // Tenant Key Output or Input File private String tenantKeyFileName; @@ -403,6 +406,9 @@ protected void updateSchema() { // perform any updates we need related to the V0010 schema change (IS_DELETED flag) applyDataMigrationForV0010(); + // V0014 IS_DELETED and LAST_UPDATED added to whole-system LOGICAL_RESOURCES + applyDataMigrationForV0014(); + // Log warning messages that unused tables will be removed in a future release. // TODO: This will no longer be needed after the tables are removed (https://github.com/IBM/FHIR/issues/713). logWarningMessagesForDeprecatedTables(); @@ -1391,7 +1397,7 @@ protected void parseArgs(String[] args) { throw new IllegalArgumentException("Missing value for argument at posn: " + i); } this.tenantName = args[i]; - this.revokeTenantKey = true; + this.revokeAllTenantKeys = true; break; case "--update-proc": this.updateProc = true; @@ -1659,7 +1665,23 @@ protected void applyDataMigrationForV0010() { } else { doMigrationForV0010(); } + } + + protected void applyDataMigrationForV0014() { + if (MULTITENANT_FEATURE_ENABLED.contains(dbType)) { + // Process each tenant one-by-one + List tenants = getTenantList(); + for (TenantInfo ti: tenants) { + // If no --schema-name override was specified, we process all tenants, otherwise we + // process only tenants which belong to the override schema name + if (!schema.isOverrideDataSchema() || schema.matchesDataSchema(ti.getTenantSchema())) { + dataMigrationForV0014(ti); + } + } + } else { + dataMigrationForV0014(); + } } /** @@ -1683,6 +1705,26 @@ private Set getResourceTypes() { return result; } + /** + * Get the list of resource types from the database. + * @param adapter + * @param schemaName + * @return + */ + private List getResourceTypesList(IDatabaseAdapter adapter, String schemaName) { + + try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { + try { + GetResourceTypeList cmd = new GetResourceTypeList(schemaName); + return adapter.runStatement(cmd); + } catch (DataAccessException x) { + // Something went wrong, so mark the transaction as failed + tx.setRollbackOnly(); + throw x; + } + } + } + /** * Migrate the IS_DELETED data for the given tenant * @param ti @@ -1746,6 +1788,76 @@ private void doMigrationForV0010() { } } + /** + * Migrate the LOGICAL_RESOURCE IS_DELETED and LAST_UPDATED data for the given tenant + * @param ti + */ + private void dataMigrationForV0014(TenantInfo ti) { + // Multi-tenant schema so we know this is Db2: + Db2Adapter adapter = new Db2Adapter(connectionPool); + + List resourceTypes = getResourceTypesList(adapter, schema.getSchemaName()); + + // Process each update in its own transaction so we don't over-stress the tx log space + for (ResourceType resourceType: resourceTypes) { + try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { + try { + SetTenantIdDb2 setTenantId = new SetTenantIdDb2(schema.getAdminSchemaName(), ti.getTenantId()); + adapter.runStatement(setTenantId); + + logger.info("V0014 Migration: Updating " + "LOGICAL_RESOURCES.IS_DELETED and LAST_UPDATED for " + resourceType.toString() + + " for tenant '" + ti.getTenantName() + "', schema '" + ti.getTenantSchema() + "'"); + + dataMigrationForV0014(adapter, ti.getTenantSchema(), resourceType); + } catch (DataAccessException x) { + // Something went wrong, so mark the transaction as failed + tx.setRollbackOnly(); + throw x; + } + } + } + } + + /** + * Migrate the LOGICAL_RESOURCE IS_DELETED and LAST_UPDATED data + */ + private void dataMigrationForV0014() { + IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + List resourceTypes = getResourceTypesList(adapter, schema.getSchemaName()); + + // Process each resource type in its own transaction to avoid pressure on the tx log + for (ResourceType resourceType: resourceTypes) { + try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { + try { + dataMigrationForV0014(adapter, schema.getSchemaName(), resourceType); + } catch (DataAccessException x) { + // Something went wrong, so mark the transaction as failed + tx.setRollbackOnly(); + throw x; + } + } + } + } + + /** + * only process tables which have not yet had their data migrated. The migration can't be + * done as part of the schema change because some tables need a REORG which + * has to be done after the transaction in which the alter table was performed. + * + * @param adapter + * @param schemaName + * @param resourceType + */ + private void dataMigrationForV0014(IDatabaseAdapter adapter, String schemaName, ResourceType resourceType) { + GetLogicalResourceNeedsV0014Migration needsMigrating = new GetLogicalResourceNeedsV0014Migration(schemaName, resourceType.getId()); + if (adapter.runStatement(needsMigrating)) { + logger.info("V0014 Migration: Updating LOGICAL_RESOURCES.IS_DELETED and LAST_UPDATED for schema '" + + schemaName + "' and resource type '" + resourceType.toString() + "'"); + MigrateV0014LogicalResourceIsDeletedLastUpdated cmd = new MigrateV0014LogicalResourceIsDeletedLastUpdated(schemaName, resourceType.getName(), resourceType.getId()); + adapter.runStatement(cmd); + } + } + /** * Backfill the RESOURCE_CHANGE_LOG table if it is empty */ @@ -1892,6 +2004,11 @@ protected void process() { dropTenant(); } else if (this.revokeTenantKey) { revokeTenantKey(); + } else if (this.revokeAllTenantKeys) { + if (this.tenantKey != null) { + throw new IllegalArgumentException("[ERROR] --tenant-key should not be specified together with --drop-all-tenant-keys"); + } + revokeTenantKey(); } if (this.grantTo != null) { diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java index e43c6c84194..b644117429e 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java @@ -6,8 +6,10 @@ package com.ibm.fhir.schema.control; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.CANONICAL_ID; import static com.ibm.fhir.schema.control.FhirSchemaConstants.CODE; import static com.ibm.fhir.schema.control.FhirSchemaConstants.CODE_SYSTEM_ID; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.COMMON_CANONICAL_VALUES; import static com.ibm.fhir.schema.control.FhirSchemaConstants.COMMON_TOKEN_VALUES; import static com.ibm.fhir.schema.control.FhirSchemaConstants.COMMON_TOKEN_VALUE_ID; import static com.ibm.fhir.schema.control.FhirSchemaConstants.COMPOSITE_ID; @@ -21,6 +23,8 @@ import static com.ibm.fhir.schema.control.FhirSchemaConstants.DATE_START; import static com.ibm.fhir.schema.control.FhirSchemaConstants.DATE_VALUE_DROPPED_COLUMN; import static com.ibm.fhir.schema.control.FhirSchemaConstants.FK; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.FRAGMENT; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.FRAGMENT_BYTES; import static com.ibm.fhir.schema.control.FhirSchemaConstants.IDX; import static com.ibm.fhir.schema.control.FhirSchemaConstants.IS_DELETED; import static com.ibm.fhir.schema.control.FhirSchemaConstants.ITEM_LOGICAL_ID; @@ -43,6 +47,7 @@ import static com.ibm.fhir.schema.control.FhirSchemaConstants.PATIENT_CURRENT_REFS; import static com.ibm.fhir.schema.control.FhirSchemaConstants.PATIENT_LOGICAL_RESOURCES; import static com.ibm.fhir.schema.control.FhirSchemaConstants.PK; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.PROFILES; import static com.ibm.fhir.schema.control.FhirSchemaConstants.QUANTITY_VALUE; import static com.ibm.fhir.schema.control.FhirSchemaConstants.QUANTITY_VALUE_HIGH; import static com.ibm.fhir.schema.control.FhirSchemaConstants.QUANTITY_VALUE_LOW; @@ -51,9 +56,13 @@ import static com.ibm.fhir.schema.control.FhirSchemaConstants.RESOURCE_TOKEN_REFS; import static com.ibm.fhir.schema.control.FhirSchemaConstants.RESOURCE_TYPES; import static com.ibm.fhir.schema.control.FhirSchemaConstants.RESOURCE_TYPE_ID; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.SECURITY; import static com.ibm.fhir.schema.control.FhirSchemaConstants.STR_VALUE; import static com.ibm.fhir.schema.control.FhirSchemaConstants.STR_VALUE_LCASE; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.TAGS; import static com.ibm.fhir.schema.control.FhirSchemaConstants.TOKEN_VALUES_V; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.VERSION; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.VERSION_BYTES; import static com.ibm.fhir.schema.control.FhirSchemaConstants.VERSION_ID; import java.util.ArrayList; @@ -159,6 +168,9 @@ public ObjectGroup addResourceType(String resourceTypeName) { // composites table removed by issue-1683 addResourceTokenRefs(group, tablePrefix); addTokenValuesView(group, tablePrefix); + addProfiles(group, tablePrefix); + addTags(group, tablePrefix); + addSecurity(group, tablePrefix); // group all the tables under one object so that we can perform everything within one // transaction. This helps to eliminate deadlocks when adding the FK constraints due to @@ -178,7 +190,7 @@ public void addLogicalResources(List group, String prefix) { // shares a common primary key (logical_resource_id) with the system-wide table // We also have a FK constraint pointing back to that table to try and keep // things sensible. - Table tbl = Table.builder(schemaName, tableName) + Table.Builder builder = Table.builder(schemaName, tableName) .setTenantColumnName(MT_ID) .setVersion(FhirSchemaVersion.V0012.vid()) // V0011: is_deleted and last_updated, V0012: version_id .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) @@ -216,9 +228,9 @@ public void addLogicalResources(List group, String prefix) { } return statements; - }) - .build(model); + }); + Table tbl = builder.build(model); group.add(tbl); model.addTable(tbl); @@ -508,6 +520,104 @@ public Table addResourceTokenRefs(List group, String prefix) { return tbl; } + /** + * Add the resource-specific profiles table which maps to the normalized URI + * values stored in COMMON_CANONICAL_VALUES + * @param group + * @param prefix + * @return + */ + public Table addProfiles(List group, String prefix) { + + final String tableName = prefix + "_" + PROFILES; + + // logical_resources (1) ---- (*) patient_resource_token_refs (*) ---- (0|1) common_token_values + Table tbl = Table.builder(schemaName, tableName) + .setVersion(FhirSchemaVersion.V0014.vid()) + .setTenantColumnName(MT_ID) + .addBigIntColumn( CANONICAL_ID, false) + .addBigIntColumn( LOGICAL_RESOURCE_ID, false) + .addVarcharColumn( VERSION, VERSION_BYTES, true) + .addVarcharColumn( FRAGMENT, FRAGMENT_BYTES, true) + .addIndex(IDX + tableName + "_TPLR", CANONICAL_ID, LOGICAL_RESOURCE_ID) + .addIndex(IDX + tableName + "_LRPT", LOGICAL_RESOURCE_ID, CANONICAL_ID) + .addForeignKeyConstraint(FK + tableName + "_TV", schemaName, COMMON_CANONICAL_VALUES, CANONICAL_ID) + .addForeignKeyConstraint(FK + tableName + "_LR", schemaName, LOGICAL_RESOURCES, LOGICAL_RESOURCE_ID) + .setTablespace(fhirTablespace) + .addPrivileges(resourceTablePrivileges) + .enableAccessControl(this.sessionVariable) + .build(model); + + group.add(tbl); + model.addTable(tbl); + + return tbl; + } + + /** + * Resource-specific tags. Tags are treated as tokens, but are separated out + * into their own table to avoid issues with cardinality estimation (because + * they are not very selective). + * @param group + * @param prefix + * @return + */ + public Table addTags(List group, String prefix) { + + final String tableName = prefix + "_" + TAGS; + + // logical_resources (1) ---- (*) patient_tags (*) ---- (0|1) common_token_values + Table tbl = Table.builder(schemaName, tableName) + .setVersion(FhirSchemaVersion.V0014.vid()) + .setTenantColumnName(MT_ID) + .addBigIntColumn(COMMON_TOKEN_VALUE_ID, false) + .addBigIntColumn( LOGICAL_RESOURCE_ID, false) + .addIndex(IDX + tableName + "_TPLR", COMMON_TOKEN_VALUE_ID, LOGICAL_RESOURCE_ID) + .addIndex(IDX + tableName + "_LRPT", LOGICAL_RESOURCE_ID, COMMON_TOKEN_VALUE_ID) + .addForeignKeyConstraint(FK + tableName + "_TV", schemaName, COMMON_TOKEN_VALUES, COMMON_TOKEN_VALUE_ID) + .addForeignKeyConstraint(FK + tableName + "_LR", schemaName, LOGICAL_RESOURCES, LOGICAL_RESOURCE_ID) + .setTablespace(fhirTablespace) + .addPrivileges(resourceTablePrivileges) + .enableAccessControl(this.sessionVariable) + .build(model); + + group.add(tbl); + model.addTable(tbl); + + return tbl; + } + + /** + * Add the common_token_values mapping table for security search parameters + * @param group + * @param prefix + * @return + */ + public Table addSecurity(List group, String prefix) { + + final String tableName = prefix + "_" + SECURITY; + + // logical_resources (1) ---- (*) patient_security (*) ---- (0|1) common_token_values + Table tbl = Table.builder(schemaName, tableName) + .setVersion(FhirSchemaVersion.V0016.vid()) + .setTenantColumnName(MT_ID) + .addBigIntColumn(COMMON_TOKEN_VALUE_ID, false) + .addBigIntColumn( LOGICAL_RESOURCE_ID, false) + .addIndex(IDX + tableName + "_TPLR", COMMON_TOKEN_VALUE_ID, LOGICAL_RESOURCE_ID) + .addIndex(IDX + tableName + "_LRPT", LOGICAL_RESOURCE_ID, COMMON_TOKEN_VALUE_ID) + .addForeignKeyConstraint(FK + tableName + "_TV", schemaName, COMMON_TOKEN_VALUES, COMMON_TOKEN_VALUE_ID) + .addForeignKeyConstraint(FK + tableName + "_LR", schemaName, LOGICAL_RESOURCES, LOGICAL_RESOURCE_ID) + .setTablespace(fhirTablespace) + .addPrivileges(resourceTablePrivileges) + .enableAccessControl(this.sessionVariable) + .build(model); + + group.add(tbl); + model.addTable(tbl); + + return tbl; + } + /** * View created over common_token_values and resource_token_refs to hide the * schema change (V0006 issue 1366) as much as possible from the search query @@ -556,7 +666,6 @@ public void addTokenValuesView(List group, String prefix) { group.add(view); } - /** * 
 CREATE TABLE device_date_values  (
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java
index 37ba79d112c..5674bd703e6 100644
--- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java
@@ -55,6 +55,16 @@ public class FhirSchemaConstants {
     public static final String TENANT_NAME = "TENANT_NAME";
     public static final String TENANT_STATUS = "TENANT_STATUS";
 
+    public static final String COMMON_CANONICAL_VALUES = "COMMON_CANONICAL_VALUES";
+    public static final String CANONICAL_ID = "CANONICAL_ID";
+    public static final String URL = "URL";
+    public static final String PROFILES = "PROFILES";
+    public static final String SECURITY = "SECURITY";
+    public static final String TAGS = "TAGS";
+    public static final int CANONICAL_URL_BYTES = 1024; // a reasonable value of our choosing
+    public static final int VERSION_BYTES = 16;
+    public static final int FRAGMENT_BYTES = 16;
+
     // R4 Logical Resources
     public static final String LOGICAL_RESOURCES = "LOGICAL_RESOURCES";
     public static final String REINDEX_TSTAMP = "REINDEX_TSTAMP";
@@ -89,12 +99,16 @@ public class FhirSchemaConstants {
     public static final String LOGICAL_ID = "LOGICAL_ID";
     public static final String LOGICAL_RESOURCE_ID = "LOGICAL_RESOURCE_ID";
     public static final String DATA = "DATA";
+    public static final String FRAGMENT = "FRAGMENT";
     public static final String RESOURCE_ID = "RESOURCE_ID";
     public static final String CURRENT_RESOURCE_ID = "CURRENT_RESOURCE_ID";
     public static final String CHANGE_TSTAMP = "CHANGE_TSTAMP";
     public static final String VERSION_ID = "VERSION_ID";
+    public static final String VERSION = "VERSION";
     public static final String IS_DELETED = "IS_DELETED";
     public static final String LAST_UPDATED = "LAST_UPDATED";
+    public static final String PARAMETER_HASH = "PARAMETER_HASH";
+    public static final int PARAMETER_HASH_BYTES = 44; // For SHA-256 encoded as Base64
     public static final String PARAMETER_NAME = "PARAMETER_NAME";
     public static final String PARAMETER_NAME_ID = "PARAMETER_NAME_ID";
     public static final String STR_VALUE = "STR_VALUE";
@@ -148,6 +162,11 @@ public class FhirSchemaConstants {
 //    public static final String EXTERNAL_SYSTEM_NAME = "EXTERNAL_SYSTEM_NAME";
 //    public static final String EXTERNAL_REFERENCES = "EXTERNAL_REFERENCES";
 
+    // Mapping table identifying the profiles associated with a particular resource
+    public static final String LOGICAL_RESOURCE_PROFILES = "LOGICAL_RESOURCE_PROFILES";
+    public static final String LOGICAL_RESOURCE_SECURITY = "LOGICAL_RESOURCE_SECURITY";
+    public static final String LOGICAL_RESOURCE_TAGS = "LOGICAL_RESOURCE_TAGS";
+
     // For V0006 (issue #1366) token_values become normalized to improve storage efficiency
     public static final String COMMON_TOKEN_VALUES = "COMMON_TOKEN_VALUES";
     public static final String COMMON_TOKEN_VALUE_ID = "COMMON_TOKEN_VALUE_ID";
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java
index 099874bc05d..202835de962 100644
--- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java
@@ -6,11 +6,14 @@
 
 package com.ibm.fhir.schema.control;
 
+import static com.ibm.fhir.schema.control.FhirSchemaConstants.CANONICAL_ID;
+import static com.ibm.fhir.schema.control.FhirSchemaConstants.CANONICAL_URL_BYTES;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.CHANGE_TSTAMP;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.CHANGE_TYPE;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.CODE_SYSTEMS;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.CODE_SYSTEM_ID;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.CODE_SYSTEM_NAME;
+import static com.ibm.fhir.schema.control.FhirSchemaConstants.COMMON_CANONICAL_VALUES;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.COMMON_TOKEN_VALUES;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.COMMON_TOKEN_VALUE_ID;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.COMPARTMENT_LOGICAL_RESOURCE_ID;
@@ -22,16 +25,24 @@
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.FHIR_REF_SEQUENCE;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.FHIR_SEQUENCE;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.FK;
+import static com.ibm.fhir.schema.control.FhirSchemaConstants.FRAGMENT;
+import static com.ibm.fhir.schema.control.FhirSchemaConstants.FRAGMENT_BYTES;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.IDX;
+import static com.ibm.fhir.schema.control.FhirSchemaConstants.IS_DELETED;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.LAST_UPDATED;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_ID;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_ID_BYTES;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCES;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_COMPARTMENTS;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_ID;
+import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_PROFILES;
+import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_SECURITY;
+import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_TAGS;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.MAX_SEARCH_STRING_BYTES;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.MAX_TOKEN_VALUE_BYTES;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.MT_ID;
+import static com.ibm.fhir.schema.control.FhirSchemaConstants.PARAMETER_HASH;
+import static com.ibm.fhir.schema.control.FhirSchemaConstants.PARAMETER_HASH_BYTES;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.PARAMETER_NAME;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.PARAMETER_NAMES;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.PARAMETER_NAME_ID;
@@ -56,6 +67,9 @@
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.TENANT_SEQUENCE;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.TENANT_STATUS;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.TOKEN_VALUE;
+import static com.ibm.fhir.schema.control.FhirSchemaConstants.URL;
+import static com.ibm.fhir.schema.control.FhirSchemaConstants.VERSION;
+import static com.ibm.fhir.schema.control.FhirSchemaConstants.VERSION_BYTES;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.VERSION_ID;
 
 import java.util.ArrayList;
@@ -370,7 +384,10 @@ public void buildSchema(PhysicalDataModel model) {
         addReferencesSequence(model);
         addLogicalResourceCompartments(model);
         addResourceChangeLog(model); // track changes for easier export
-
+        addCommonCanonicalValues(model);   // V0014
+        addLogicalResourceProfiles(model); // V0014
+        addLogicalResourceTags(model);     // V0014
+        addLogicalResourceSecurity(model); // V0016
 
         Table globalStrValues = addResourceStrValues(model); // for system-level _profile parameters
         Table globalDateValues = addResourceDateValues(model); // for system-level date parameters
@@ -491,8 +508,10 @@ public void buildDatabaseSpecificArtifactsPostgres(PhysicalDataModel model) {
      */
     public void addLogicalResources(PhysicalDataModel pdm) {
         final String tableName = LOGICAL_RESOURCES;
+        final String mtId = this.multitenant ? MT_ID : null;
 
         final String IDX_LOGICAL_RESOURCES_RITS = "IDX_" + LOGICAL_RESOURCES + "_RITS";
+        final String IDX_LOGICAL_RESOURCES_LUPD = "IDX_" + LOGICAL_RESOURCES + "_LUPD";
 
         Table tbl = Table.builder(schemaName, tableName)
                 .setTenantColumnName(MT_ID)
@@ -501,14 +520,18 @@ public void addLogicalResources(PhysicalDataModel pdm) {
                 .addVarcharColumn(LOGICAL_ID, LOGICAL_ID_BYTES, false)
                 .addTimestampColumn(REINDEX_TSTAMP, false, "CURRENT_TIMESTAMP") // new column for V0006
                 .addBigIntColumn(REINDEX_TXID, false, "0")                      // new column for V0006
+                .addTimestampColumn(LAST_UPDATED, true)                         // new column for V0014
+                .addCharColumn(IS_DELETED, 1, false, "'X'")
+                .addVarcharColumn(PARAMETER_HASH, PARAMETER_HASH_BYTES, true)           // new column for V0015
                 .addPrimaryKey(tableName + "_PK", LOGICAL_RESOURCE_ID)
                 .addUniqueIndex("UNQ_" + LOGICAL_RESOURCES, RESOURCE_TYPE_ID, LOGICAL_ID)
                 .addIndex(IDX_LOGICAL_RESOURCES_RITS, new OrderedColumnDef(REINDEX_TSTAMP, OrderedColumnDef.Direction.DESC, null))
+                .addIndex(IDX_LOGICAL_RESOURCES_LUPD, new OrderedColumnDef(LAST_UPDATED, OrderedColumnDef.Direction.ASC, null))
                 .setTablespace(fhirTablespace)
                 .addPrivileges(resourceTablePrivileges)
                 .addForeignKeyConstraint(FK + tableName + "_RTID", schemaName, RESOURCE_TYPES, RESOURCE_TYPE_ID)
                 .enableAccessControl(this.sessionVariable)
-                .setVersion(FhirSchemaVersion.V0009.vid())
+                .setVersion(FhirSchemaVersion.V0015.vid())
                 .addMigration(priorVersion -> {
                     List statements = new ArrayList<>();
                     if (priorVersion == FhirSchemaVersion.V0001.vid()) {
@@ -523,7 +546,6 @@ public void addLogicalResources(PhysicalDataModel pdm) {
 
                         // Add the new index on REINDEX_TSTAMP. This index is special because it's the
                         // first index in our schema to use DESC.
-                        final String mtId = this.multitenant ? MT_ID : null;
                         List indexCols = Arrays.asList(new OrderedColumnDef(REINDEX_TSTAMP, OrderedColumnDef.Direction.DESC, null));
                         statements.add(new CreateIndexStatement(schemaName, IDX_LOGICAL_RESOURCES_RITS, tableName, mtId, indexCols));
                     }
@@ -533,6 +555,34 @@ public void addLogicalResources(PhysicalDataModel pdm) {
                         // used
                         statements.add(new DropTable(schemaName, "TOKEN_VALUES"));
                     }
+
+                    if (priorVersion < FhirSchemaVersion.V0014.vid()) {
+                        // Add LAST_UPDATED and IS_DELETED to whole-system logical_resources
+                        List cols = ColumnDefBuilder.builder()
+                                .addTimestampColumn(LAST_UPDATED, true)
+                                .addCharColumn(IS_DELETED, 1, false, "'X'")
+                                .buildColumns();
+
+                        statements.add(new AddColumn(schemaName, tableName, cols.get(0)));
+                        statements.add(new AddColumn(schemaName, tableName, cols.get(1)));
+
+                        // New index on the LAST_UPDATED. We don't need to include resource-type. If
+                        // you know the resource type, you'll be querying the resource-specific
+                        // xx_logical_resources table instead
+                        List indexCols = Arrays.asList(new OrderedColumnDef(LAST_UPDATED, OrderedColumnDef.Direction.ASC, null));
+                        statements.add(new CreateIndexStatement(schemaName, IDX_LOGICAL_RESOURCES_LUPD, tableName, mtId, indexCols));
+                    }
+
+
+                    if (priorVersion < FhirSchemaVersion.V0015.vid()) {
+                        // Add PARAM_HASH logical_resources
+                        List cols = ColumnDefBuilder.builder()
+                                .addVarcharColumn(PARAMETER_HASH, PARAMETER_HASH_BYTES, true)
+                                .buildColumns();
+                        statements.add(new AddColumn(schemaName, tableName, cols.get(0)));
+                    }
+
+
                     return statements;
                 })
                 .build(pdm);
@@ -544,6 +594,140 @@ public void addLogicalResources(PhysicalDataModel pdm) {
         pdm.addObject(tbl);
     }
 
+    /**
+     * Create the COMMON_CANONICAL_VALUES table. Used from schema V0014 to normalize
+     * meta.profile search parameters (similar to common_token_values). Only the url
+     * is included by design. The (optional) version and fragment values are stored
+     * in the parameter mapping table (logical_resource_profiles) in order to support
+     * inequalities on version while still using a literal CANONICAL_ID = x predicate.
+     * These canonical ids are cached in the server, so search queries won't need to
+     * join to this table. The URL is typically a long string, so by normalizing and
+     * storing/indexing it once, we reduce space consumption.
+     * @param pdm
+     */
+    public void addCommonCanonicalValues(PhysicalDataModel pdm) {
+        final String tableName = COMMON_CANONICAL_VALUES;
+        final String unqCanonicalUrl = "UNQ_" + tableName + "_URL";
+        Table tbl = Table.builder(schemaName, tableName)
+                .setVersion(FhirSchemaVersion.V0014.vid())
+                .setTenantColumnName(MT_ID)
+                .addBigIntColumn(CANONICAL_ID, false)
+                .addVarcharColumn(URL, CANONICAL_URL_BYTES, false)
+                .addPrimaryKey(tableName + "_PK", CANONICAL_ID)
+                .addUniqueIndex(unqCanonicalUrl, URL)
+                .setTablespace(fhirTablespace)
+                .addPrivileges(resourceTablePrivileges)
+                .enableAccessControl(this.sessionVariable)
+                .build(pdm);
+
+        // TODO should not need to add as a table and an object. Get the table to add itself?
+        tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP);
+        this.procedureDependencies.add(tbl);
+        pdm.addTable(tbl);
+        pdm.addObject(tbl);
+    }
+
+    /**
+     * A single-parameter table supporting _profile search parameter values
+     * Add the LOGICAL_RESOURCE_PROFILES table to the given {@link PhysicalDataModel}.
+     * This table maps logical resources to meta.profile values stored as canonical URIs
+     * in COMMON_CANONICAL_VALUES. Canonical values can include optional version and fragment
+     * values as described here: https://www.hl7.org/fhir/datatypes.html#canonical
+     * @param pdm
+     * @return
+     */
+    public Table addLogicalResourceProfiles(PhysicalDataModel pdm) {
+
+        final String tableName = LOGICAL_RESOURCE_PROFILES;
+
+        // logical_resources (1) ---- (*) logical_resource_profiles (*) ---- (1) common_canonical_values
+        Table tbl = Table.builder(schemaName, tableName)
+                .setVersion(FhirSchemaVersion.V0014.vid()) // table created at version V0014
+                .setTenantColumnName(MT_ID)
+                .addBigIntColumn(         CANONICAL_ID,     false) // FK referencing COMMON_CANONICAL_VALUES
+                .addBigIntColumn(  LOGICAL_RESOURCE_ID,     false) // FK referencing LOGICAL_RESOURCES
+                .addVarcharColumn(             VERSION,  VERSION_BYTES, true)
+                .addVarcharColumn(            FRAGMENT, FRAGMENT_BYTES, true)
+                .addIndex(IDX + tableName + "_CCVLR", CANONICAL_ID, LOGICAL_RESOURCE_ID)
+                .addIndex(IDX + tableName + "_LRCCV", LOGICAL_RESOURCE_ID, CANONICAL_ID)
+                .addForeignKeyConstraint(FK + tableName + "_CCV", schemaName, COMMON_CANONICAL_VALUES, CANONICAL_ID)
+                .addForeignKeyConstraint(FK + tableName + "_LR", schemaName, LOGICAL_RESOURCES, LOGICAL_RESOURCE_ID)
+                .setTablespace(fhirTablespace)
+                .addPrivileges(resourceTablePrivileges)
+                .enableAccessControl(this.sessionVariable)
+                .build(pdm);
+
+        tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP);
+        this.procedureDependencies.add(tbl);
+        pdm.addTable(tbl);
+        pdm.addObject(tbl);
+
+        return tbl;
+    }
+
+    /**
+     * A single-parameter table supporting _tag search parameter values.
+     * Tags are tokens, but because they may not be very selective we use a
+     * separate table in order to avoid messing up cardinality estimates
+     * in the query optimizer.
+     * @param pdm
+     * @return
+     */
+    public Table addLogicalResourceTags(PhysicalDataModel pdm) {
+
+        final String tableName = LOGICAL_RESOURCE_TAGS;
+
+        // logical_resources (1) ---- (*) logical_resource_tags (*) ---- (1) common_token_values
+        Table tbl = Table.builder(schemaName, tableName)
+                .setVersion(FhirSchemaVersion.V0014.vid()) // table created at version V0014
+                .setTenantColumnName(MT_ID)
+                .addBigIntColumn(COMMON_TOKEN_VALUE_ID,    false) // FK referencing COMMON_CANONICAL_VALUES
+                .addBigIntColumn(  LOGICAL_RESOURCE_ID,    false) // FK referencing LOGICAL_RESOURCES
+                .addIndex(IDX + tableName + "_CCVLR", COMMON_TOKEN_VALUE_ID, LOGICAL_RESOURCE_ID)
+                .addIndex(IDX + tableName + "_LRCCV", LOGICAL_RESOURCE_ID, COMMON_TOKEN_VALUE_ID)
+                .addForeignKeyConstraint(FK + tableName + "_CTV", schemaName, COMMON_TOKEN_VALUES, COMMON_TOKEN_VALUE_ID)
+                .addForeignKeyConstraint(FK + tableName + "_LR", schemaName, LOGICAL_RESOURCES, LOGICAL_RESOURCE_ID)
+                .setTablespace(fhirTablespace)
+                .addPrivileges(resourceTablePrivileges)
+                .enableAccessControl(this.sessionVariable)
+                .build(pdm);
+
+        pdm.addTable(tbl);
+        pdm.addObject(tbl);
+
+        return tbl;
+    }
+
+    /**
+     * Add the dedicated common_token_values mapping table for security search parameters
+     * @param pdm
+     * @return
+     */
+    public Table addLogicalResourceSecurity(PhysicalDataModel pdm) {
+
+        final String tableName = LOGICAL_RESOURCE_SECURITY;
+
+        // logical_resources (1) ---- (*) logical_resource_security (*) ---- (1) common_token_values
+        Table tbl = Table.builder(schemaName, tableName)
+                .setVersion(FhirSchemaVersion.V0016.vid()) // table created at version V0016
+                .setTenantColumnName(MT_ID)
+                .addBigIntColumn(COMMON_TOKEN_VALUE_ID,    false) // FK referencing COMMON_CANONICAL_VALUES
+                .addBigIntColumn(  LOGICAL_RESOURCE_ID,    false) // FK referencing LOGICAL_RESOURCES
+                .addIndex(IDX + tableName + "_CCVLR", COMMON_TOKEN_VALUE_ID, LOGICAL_RESOURCE_ID)
+                .addIndex(IDX + tableName + "_LRCCV", LOGICAL_RESOURCE_ID, COMMON_TOKEN_VALUE_ID)
+                .addForeignKeyConstraint(FK + tableName + "_CTV", schemaName, COMMON_TOKEN_VALUES, COMMON_TOKEN_VALUE_ID)
+                .addForeignKeyConstraint(FK + tableName + "_LR", schemaName, LOGICAL_RESOURCES, LOGICAL_RESOURCE_ID)
+                .setTablespace(fhirTablespace)
+                .addPrivileges(resourceTablePrivileges)
+                .enableAccessControl(this.sessionVariable)
+                .build(pdm);
+
+        pdm.addTable(tbl);
+        pdm.addObject(tbl);
+
+        return tbl;
+    }
+
     /**
      * Add the resource_change_log table. This table supports tracking of every change made
      * to a resource at the global level, making it much easier to stream a list of changes
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java
index b3a6274c84d..8ff1644219b 100644
--- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java
@@ -28,6 +28,9 @@ public enum FhirSchemaVersion {
     ,V0011(11, "issue-2011 add LAST_UPDATED to each xxx_LOGICAL_RESOURCES table")
     ,V0012(12, "issue-2109 add VERSION_ID to each xxx_LOGICAL_RESOURCES table")
     ,V0013(13, "Add $erase operation for hard delete scenarios")
+    ,V0014(14, "whole-system search and canonical references")
+    ,V0015(15, "issue-2155 add parameter hash to bypass update during reindex")
+    ,V0016(16, "issue-1921 add dedicated common_token_values mapping table for searching on security labels")
     ;
 
     // The version number recorded in the VERSION_HISTORY
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/GetLogicalResourceNeedsV0014Migration.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/GetLogicalResourceNeedsV0014Migration.java
new file mode 100644
index 00000000000..a927d33dbc0
--- /dev/null
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/GetLogicalResourceNeedsV0014Migration.java
@@ -0,0 +1,59 @@
+/*
+ * (C) Copyright IBM Corp. 2021
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.ibm.fhir.schema.control;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import com.ibm.fhir.database.utils.api.IDatabaseSupplier;
+import com.ibm.fhir.database.utils.api.IDatabaseTranslator;
+import com.ibm.fhir.database.utils.common.DataDefinitionUtil;
+
+/**
+ * Checks the value of IS_DELETED from the first row found in LOGICAL_RESOURCES for
+ * the given resource type. If this value is "X", it means that the table needs to be
+ * migrated. If the table is empty, then obviously there's no need for migration.
+ */
+public class GetLogicalResourceNeedsV0014Migration implements IDatabaseSupplier {
+
+    // The FHIR data schema
+    private final String schemaName;
+
+    // The database RESOURCE_TYPES.RESOURCE_TYPE_ID value for the subset we want to look for
+    private final int resourceTypeId;
+
+    public GetLogicalResourceNeedsV0014Migration(String schemaName, int resourceTypeId) {
+        this.schemaName = schemaName;
+        this.resourceTypeId = resourceTypeId;
+    }
+
+    @Override
+    public Boolean run(IDatabaseTranslator translator, Connection c) {
+        // The LOGICAL_RESOURCES table has an index {RESOURCE_TYPE_ID, LOGICAL_ID}
+        // which we can leverage here to filter by resource type without needing
+        // to join to the XX_LOGICAL_RESOURCES table.
+        Boolean result = false;
+        final String tableName = DataDefinitionUtil.getQualifiedName(schemaName, "LOGICAL_RESOURCES");
+        final String SQL = "SELECT is_deleted "
+                + "  FROM " + tableName
+                + " WHERE resource_type_id = " + this.resourceTypeId
+                + " " + translator.limit("1");
+
+        try (Statement s = c.createStatement()) {
+            ResultSet rs = s.executeQuery(SQL);
+            if (rs.next() && "X".equals(rs.getString(1))) {
+                result = true;
+            }
+        } catch (SQLException x) {
+            throw translator.translate(x);
+        }
+
+        return result;
+    }
+}
\ No newline at end of file
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/GetResourceTypeList.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/GetResourceTypeList.java
index 79ca6b5fc0b..2e8af350aff 100644
--- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/GetResourceTypeList.java
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/GetResourceTypeList.java
@@ -1,5 +1,5 @@
 /*
- * (C) Copyright IBM Corp. 2019
+ * (C) Copyright IBM Corp. 2019, 2021
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -38,7 +38,7 @@ public List run(IDatabaseTranslator translator, Connection c) {
             ResultSet rs = s.executeQuery(SQL);
             while (rs.next()) {
                 ResourceType rt = new ResourceType();
-                rt.setId(rs.getLong(1));
+                rt.setId(rs.getInt(1));
                 rt.setName(rs.getString(2));
                 result.add(rt);
             }
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/GetResourceTypes.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/GetResourceTypes.java
index 9460580d98a..3a182233e5e 100644
--- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/GetResourceTypes.java
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/GetResourceTypes.java
@@ -1,5 +1,5 @@
 /*
- * (C) Copyright IBM Corp. 2019
+ * (C) Copyright IBM Corp. 2019, 2021
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -30,14 +30,14 @@ public GetResourceTypes(String schemaName, Consumer c) {
 
     @Override
     public void run(IDatabaseTranslator translator, Connection c) {
-        final String SQL = "SELECT resource_type_id, resource_type " 
+        final String SQL = "SELECT resource_type_id, resource_type "
                          + "  FROM " + schemaName + ".RESOURCE_TYPES";
 
         try (Statement s = c.createStatement()) {
             ResultSet rs = s.executeQuery(SQL);
             while (rs.next()) {
                 ResourceType rt = new ResourceType();
-                rt.setId(rs.getLong(1));
+                rt.setId(rs.getInt(1));
                 rt.setName(rs.getString(2));
                 consumer.accept(rt);
             }
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/InitializeLogicalResourceDenorms.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/InitializeLogicalResourceDenorms.java
index 569ea2c49e8..13c906b411e 100644
--- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/InitializeLogicalResourceDenorms.java
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/InitializeLogicalResourceDenorms.java
@@ -34,7 +34,6 @@ public class InitializeLogicalResourceDenorms implements IDatabaseStatement {
 
     /**
      * Public constructor
-     *
      * @param schemaName
      * @param tableName
      */
@@ -115,8 +114,8 @@ private void runForDerby(IDatabaseTranslator translator, Connection c) {
                 }
                 updateStatement.setString(1, isDeleted);
                 updateStatement.setTimestamp(2, lastUpdated, SchemaConstants.UTC);
-                updateStatement.setLong(3, logicalResourceId);
-                updateStatement.setInt(4, versionId);
+                updateStatement.setInt(3, versionId);
+                updateStatement.setLong(4, logicalResourceId);
                 updateStatement.addBatch();
 
                 if (++batchCount == 500) {
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0014LogicalResourceIsDeletedLastUpdated.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0014LogicalResourceIsDeletedLastUpdated.java
new file mode 100644
index 00000000000..6f614f45593
--- /dev/null
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0014LogicalResourceIsDeletedLastUpdated.java
@@ -0,0 +1,139 @@
+/*
+ * (C) Copyright IBM Corp. 2021
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.ibm.fhir.schema.control;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.Timestamp;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import com.ibm.fhir.database.utils.api.IDatabaseStatement;
+import com.ibm.fhir.database.utils.api.IDatabaseTranslator;
+import com.ibm.fhir.database.utils.common.DataDefinitionUtil;
+import com.ibm.fhir.database.utils.model.DbType;
+import com.ibm.fhir.database.utils.version.SchemaConstants;
+
+/**
+ * Run a correlated update statement to update the new V0014 columns in LOGICAL_RESOURCES from the
+ * corresponding values in the xx_LOGICAL_RESOURCES table (for a specific resource type)
+ * Note that for this to work for the multi-tenant (Db2) schema,
+ * the SV_TENANT_ID needs to be set first.
+ */
+public class MigrateV0014LogicalResourceIsDeletedLastUpdated implements IDatabaseStatement {
+    private static final Logger logger = Logger.getLogger(MigrateV0014LogicalResourceIsDeletedLastUpdated.class.getName());
+
+    // The FHIR data schema
+    private final String schemaName;
+
+    // The ressource type name for the subset of resources we want to process
+    private final String resourceTypeName;
+
+    // The database id for the resource type
+    private final int resourceTypeId;
+
+    /**
+     * Public constructor
+     * @param schemaName
+     * @param tableName
+     */
+    public MigrateV0014LogicalResourceIsDeletedLastUpdated(String schemaName, String resourceTypeName, int resourceTypeId) {
+        this.schemaName = schemaName;
+        this.resourceTypeName = resourceTypeName;
+        this.resourceTypeId = resourceTypeId;
+    }
+
+    @Override
+    public void run(IDatabaseTranslator translator, Connection c) {
+        if (translator.getType() == DbType.DERBY) {
+            runForDerby(translator, c);
+        } else {
+            runCorrelatedUpdate(translator, c);
+        }
+    }
+
+    /**
+     * Perform the update using a correlated update statement, which works for Db2
+     * and PostgresSQL
+     * @param translator
+     * @param c
+     */
+    private void runCorrelatedUpdate(IDatabaseTranslator translator, Connection c) {
+        // Correlated update to grab the IS_DELETED and LAST_UPDATED values from xxx_RESOURCES and use them to
+        // set XX_LOGICAL_RESOURCES.IS_DELETED and LAST_UPDATED for the current_resource_id.
+        final String srcTable = DataDefinitionUtil.getQualifiedName(schemaName, resourceTypeName + "_LOGICAL_RESOURCES");
+        final String tgtTable = DataDefinitionUtil.getQualifiedName(schemaName, "LOGICAL_RESOURCES");
+
+        final String DML = "UPDATE " + tgtTable + " tgt "
+                + " SET (is_deleted, last_updated) = (SELECT src.is_deleted, src.last_updated FROM " + srcTable + " src WHERE tgt.logical_resource_id = src.logical_resource_id)"
+                        + " WHERE tgt.resource_type_id = " + this.resourceTypeId;
+                ;
+
+        try (PreparedStatement ps = c.prepareStatement(DML)) {
+            ps.executeUpdate();
+        } catch (SQLException x) {
+            throw translator.translate(x);
+        }
+    }
+
+    /**
+     * Derby doesn't support correlated update statements, so we have to
+     * do this manually
+     * @param translator
+     * @param c
+     */
+    private void runForDerby(IDatabaseTranslator translator, Connection c) {
+        final String lrTable = DataDefinitionUtil.getQualifiedName(schemaName, resourceTypeName + "_LOGICAL_RESOURCES");
+        // Fetch the is_deleted and last_updated statement from the current version of each resource...
+        final String select = ""
+                + "SELECT lr.logical_resource_id, lr.is_deleted, lr.last_updated "
+                + "  FROM " + lrTable + " lr  "
+                + " WHERE lr.resource_type_id = " + this.resourceTypeId;
+
+        // ...and use it to set the new column values in the corresponding XXX_LOGICAL_RESOURCES table
+        final String update = "UPDATE logical_resources SET is_deleted = ?, last_updated = ? WHERE logical_resource_id = ?";
+
+        try (Statement selectStatement = c.createStatement();
+             PreparedStatement updateStatement = c.prepareStatement(update)) {
+            ResultSet rs = selectStatement.executeQuery(select);
+
+            int batchCount = 0;
+            while (rs.next()) {
+                long logicalResourceId = rs.getLong(1);
+                String isDeleted = rs.getString(2);
+                Timestamp lastUpdated = rs.getTimestamp(3, SchemaConstants.UTC);
+
+                if (logger.isLoggable(Level.FINEST)) {
+                    // log the update in a form which is useful for debugging
+                    logger.finest("UPDATE logical_resources "
+                        + "   SET          is_deleted = '" + isDeleted + "'"
+                        + ",             last_updated = '" + lastUpdated.toString() + "'"
+                        + " WHERE logical_resource_id = " + logicalResourceId);
+                }
+                updateStatement.setString(1, isDeleted);
+                updateStatement.setTimestamp(2, lastUpdated, SchemaConstants.UTC);
+                updateStatement.setLong(3, logicalResourceId);
+                updateStatement.addBatch();
+
+                if (++batchCount == 500) {
+                    updateStatement.executeBatch();
+                    batchCount = 0;
+                }
+            }
+
+            if (batchCount > 0) {
+                updateStatement.executeBatch();
+            }
+
+        } catch (SQLException x) {
+            throw translator.translate(x);
+        }
+    }
+}
\ No newline at end of file
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/model/ResourceType.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/model/ResourceType.java
index ce8fd002a6f..2e3cf5bfb87 100644
--- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/model/ResourceType.java
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/model/ResourceType.java
@@ -1,5 +1,5 @@
 /*
- * (C) Copyright IBM Corp. 2019
+ * (C) Copyright IBM Corp. 2019, 2021
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -11,7 +11,7 @@
  */
 public class ResourceType {
     // the database-assigned id
-    private long id;
+    private int id;
 
     // the name of the resource type
     private String name;
@@ -24,14 +24,14 @@ public String toString() {
     /**
      * @return the id
      */
-    public long getId() {
+    public int getId() {
         return id;
     }
 
     /**
      * @param id the id to set
      */
-    public void setId(long id) {
+    public void setId(int id) {
         this.id = id;
     }
 
@@ -48,4 +48,4 @@ public String getName() {
     public void setName(String name) {
         this.name = name;
     }
-}
+}
\ No newline at end of file
diff --git a/fhir-persistence-schema/src/main/resources/db2/add_any_resource.sql b/fhir-persistence-schema/src/main/resources/db2/add_any_resource.sql
index 69a0a6d1bdf..2805083bb3f 100644
--- a/fhir-persistence-schema/src/main/resources/db2/add_any_resource.sql
+++ b/fhir-persistence-schema/src/main/resources/db2/add_any_resource.sql
@@ -21,8 +21,10 @@
 --   p_last_updated the last_updated time given by the FHIR server
 --   p_is_deleted:  the soft delete flag
 --   p_version:     the intended version id for this resource
+--   p_parameter_hash_b64: Base64 encoded hash of parameter values
 --   o_logical_resource_id: output field returning the newly assigned logical_resource_id value
 --   o_resource_id: output field returning the newly assigned resource_id value
+--   o_current_parameter_hash: Base64 current parameter hash if existing resource
 -- Exceptions:
 --   SQLSTATE 99001: on version conflict (concurrency)
 --   SQLSTATE 99002: missing expected row (data integrity)
@@ -33,8 +35,10 @@
       IN p_last_updated               TIMESTAMP,
       IN p_is_deleted                      CHAR(  1),
       IN p_version                          INT,
+      IN p_parameter_hash_b64           VARCHAR(44 OCTETS),
       OUT o_logical_resource_id          BIGINT,
-      OUT o_resource_row_id              BIGINT
+      OUT o_resource_row_id              BIGINT,
+      OUT o_current_parameter_hash      VARCHAR(44 OCTETS)
     )
     LANGUAGE SQL
     MODIFIES SQL DATA
@@ -49,6 +53,7 @@ BEGIN
   DECLARE v_not_found               INT     DEFAULT 0;
   DECLARE v_duplicate               INT     DEFAULT 0;
   DECLARE v_current_version         INT     DEFAULT 0;
+  DECLARE v_require_params          INT     DEFAULT 1;
   DECLARE v_change_type            CHAR(1)  DEFAULT NULL;
   DECLARE c_duplicate CONDITION FOR SQLSTATE '23505';
   DECLARE stmt STATEMENT;
@@ -67,7 +72,7 @@ BEGIN
   
   -- FOR UPDATE WITH RS does not appear to work using a prepared statement and
   -- cursor, so we have to run this directly against the logical_resources table.
-  SELECT logical_resource_id INTO v_logical_resource_id
+  SELECT logical_resource_id, parameter_hash INTO v_logical_resource_id, o_current_parameter_hash
     FROM {{SCHEMA_NAME}}.logical_resources
    WHERE resource_type_id = v_resource_type_id AND logical_id = p_logical_id
      FOR UPDATE WITH RS
@@ -78,9 +83,9 @@ BEGIN
   THEN
     VALUES NEXT VALUE FOR {{SCHEMA_NAME}}.fhir_sequence INTO v_logical_resource_id;
     PREPARE stmt FROM
-       'INSERT INTO ' || v_schema_name || '.logical_resources (mt_id, logical_resource_id, resource_type_id, logical_id, reindex_tstamp) '
-    || '     VALUES (?, ?, ?, ?, ?)';
-    EXECUTE stmt USING {{ADMIN_SCHEMA_NAME}}.sv_tenant_id, v_logical_resource_id, v_resource_type_id, p_logical_id, '1970-01-01-00.00.00.0';
+       'INSERT INTO ' || v_schema_name || '.logical_resources (mt_id, logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash) '
+    || '     VALUES (?, ?, ?, ?, ?, ?, ?, ?)';
+    EXECUTE stmt USING {{ADMIN_SCHEMA_NAME}}.sv_tenant_id, v_logical_resource_id, v_resource_type_id, p_logical_id, '1970-01-01-00.00.00.0', p_is_deleted, p_last_updated, p_parameter_hash_b64;
 
     -- remember that we have a concurrent system...so there is a possibility
     -- that another thread snuck in before us and created the logical resource. This
@@ -89,7 +94,7 @@ BEGIN
     THEN
       -- row exists, so we just need to obtain a lock on it. Because logical resource records are
       -- never deleted, we don't need to worry about it disappearing again before we grab the row lock
-      SELECT logical_resource_id INTO v_logical_resource_id
+      SELECT logical_resource_id, parameter_hash INTO v_logical_resource_id, o_current_parameter_hash
         FROM {{SCHEMA_NAME}}.logical_resources
        WHERE resource_type_id = v_resource_type_id AND logical_id = p_logical_id
          FOR UPDATE WITH RS
@@ -128,27 +133,43 @@ BEGIN
     THEN
         SIGNAL SQLSTATE '99001' SET MESSAGE_TEXT = 'Concurrent update - mismatch of version in JSON';
     END IF;
-
-    -- existing resource, so need to delete all its parameters. 
-    -- TODO patch parameter sets instead of all delete/all insert.
-    PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_str_values          WHERE logical_resource_id = ?';
-    EXECUTE stmt USING v_logical_resource_id;
-    PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_number_values       WHERE logical_resource_id = ?';
-    EXECUTE stmt USING v_logical_resource_id;
-    PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_date_values         WHERE logical_resource_id = ?';
-    EXECUTE stmt USING v_logical_resource_id;
-    PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_latlng_values       WHERE logical_resource_id = ?';
-    EXECUTE stmt USING v_logical_resource_id;
-    PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_resource_token_refs WHERE logical_resource_id = ?';
-    EXECUTE stmt USING v_logical_resource_id;
-    PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_quantity_values     WHERE logical_resource_id = ?';
-    EXECUTE stmt USING v_logical_resource_id;
-    PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || 'str_values          WHERE logical_resource_id = ?';
-    EXECUTE stmt USING v_logical_resource_id;
-    PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || 'date_values         WHERE logical_resource_id = ?';
-    EXECUTE stmt USING v_logical_resource_id;
-    PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || 'resource_token_refs WHERE logical_resource_id = ?';
-    EXECUTE stmt USING v_logical_resource_id;
+    
+    -- check the current vs new parameter hash to see if we can bypass the delete/insert
+    IF o_current_parameter_hash IS NULL OR o_current_parameter_hash != p_parameter_hash_b64
+    THEN
+	    -- existing resource, so need to delete all its parameters. 
+	    -- TODO patch parameter sets instead of all delete/all insert.
+	    PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_str_values          WHERE logical_resource_id = ?';
+	    EXECUTE stmt USING v_logical_resource_id;
+	    PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_number_values       WHERE logical_resource_id = ?';
+	    EXECUTE stmt USING v_logical_resource_id;
+	    PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_date_values         WHERE logical_resource_id = ?';
+	    EXECUTE stmt USING v_logical_resource_id;
+	    PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_latlng_values       WHERE logical_resource_id = ?';
+	    EXECUTE stmt USING v_logical_resource_id;
+	    PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_resource_token_refs WHERE logical_resource_id = ?';
+	    EXECUTE stmt USING v_logical_resource_id;
+	    PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_quantity_values     WHERE logical_resource_id = ?';
+	    EXECUTE stmt USING v_logical_resource_id;
+	    PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_profiles            WHERE logical_resource_id = ?';
+	    EXECUTE stmt USING v_logical_resource_id;
+	    PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_tags                WHERE logical_resource_id = ?';
+	    EXECUTE stmt USING v_logical_resource_id;
+        PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_security            WHERE logical_resource_id = ?';
+        EXECUTE stmt USING v_logical_resource_id;
+	    PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || 'str_values          WHERE logical_resource_id = ?';
+	    EXECUTE stmt USING v_logical_resource_id;
+	    PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || 'date_values         WHERE logical_resource_id = ?';
+	    EXECUTE stmt USING v_logical_resource_id;
+	    PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || 'resource_token_refs WHERE logical_resource_id = ?';
+	    EXECUTE stmt USING v_logical_resource_id;
+	    PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || 'logical_resource_profiles WHERE logical_resource_id = ?';
+	    EXECUTE stmt USING v_logical_resource_id;
+	    PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || 'logical_resource_tags     WHERE logical_resource_id = ?';
+	    EXECUTE stmt USING v_logical_resource_id;
+        PREPARE stmt FROM 'DELETE FROM ' || v_schema_name || '.' || 'logical_resource_security WHERE logical_resource_id = ?';
+        EXECUTE stmt USING v_logical_resource_id;
+	END IF; -- end if parameter hash is different    
   END IF; -- end if existing resource
 
   PREPARE stmt FROM
@@ -162,6 +183,10 @@ BEGIN
     -- need to update them here.
     PREPARE stmt FROM 'UPDATE ' || v_schema_name || '.' || p_resource_type || '_logical_resources SET current_resource_id = ?, is_deleted = ?, last_updated = ?, version_id = ? WHERE logical_resource_id = ?';
     EXECUTE stmt USING v_resource_id, p_is_deleted, p_last_updated, p_version, v_logical_resource_id;
+
+    -- For V0014 we also store is_deleted and last_updated at the logical_resource level
+    PREPARE stmt FROM 'UPDATE ' || v_schema_name || '.logical_resources SET is_deleted = ?, last_updated = ?, parameter_hash = ? WHERE logical_resource_id = ?';
+    EXECUTE stmt USING p_is_deleted, p_last_updated, p_parameter_hash_b64, v_logical_resource_id;
   END IF;
   
   -- DB2 doesn't support user defined array types in dynamic SQL UNNEST/CAST statements,
diff --git a/fhir-persistence-schema/src/main/resources/db2/erase_resource.sql b/fhir-persistence-schema/src/main/resources/db2/erase_resource.sql
index eae1092a092..dc2c1ff61a2 100644
--- a/fhir-persistence-schema/src/main/resources/db2/erase_resource.sql
+++ b/fhir-persistence-schema/src/main/resources/db2/erase_resource.sql
@@ -4,8 +4,6 @@
 -- SPDX-License-Identifier: Apache-2.0
 -------------------------------------------------------------------------------
 
--- LOADED ON: {{DATE}}
-
 -- ----------------------------------------------------------------------------
 -- Procedure to remove a resource, history and parameters values
 -- 
@@ -70,7 +68,7 @@ BEGIN
     PREPARE d_stmt FROM 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_number_values       WHERE logical_resource_id = ?';
     EXECUTE d_stmt USING v_logical_resource_id;
 
-    PREPARE d_stmt FROM 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_date_values       WHERE logical_resource_id = ?';
+    PREPARE d_stmt FROM 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_date_values         WHERE logical_resource_id = ?';
     EXECUTE d_stmt USING v_logical_resource_id;
 
     PREPARE d_stmt FROM 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_latlng_values       WHERE logical_resource_id = ?';
@@ -82,13 +80,31 @@ BEGIN
     PREPARE d_stmt FROM 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_quantity_values     WHERE logical_resource_id = ?';
     EXECUTE d_stmt USING v_logical_resource_id;
 
-    PREPARE d_stmt FROM 'DELETE FROM {{SCHEMA_NAME}}.' || 'str_values          WHERE logical_resource_id = ?';
+    PREPARE d_stmt FROM 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_profiles            WHERE logical_resource_id = ?';
+    EXECUTE d_stmt USING v_logical_resource_id;
+
+    PREPARE d_stmt FROM 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_tags                WHERE logical_resource_id = ?';
+    EXECUTE d_stmt USING v_logical_resource_id;
+
+    PREPARE d_stmt FROM 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_security            WHERE logical_resource_id = ?';
+    EXECUTE d_stmt USING v_logical_resource_id;
+
+    PREPARE d_stmt FROM 'DELETE FROM {{SCHEMA_NAME}}.' || 'str_values                WHERE logical_resource_id = ?';
+    EXECUTE d_stmt USING v_logical_resource_id;
+
+    PREPARE d_stmt FROM 'DELETE FROM {{SCHEMA_NAME}}.' || 'date_values               WHERE logical_resource_id = ?';
+    EXECUTE d_stmt USING v_logical_resource_id;
+
+    PREPARE d_stmt FROM 'DELETE FROM {{SCHEMA_NAME}}.' || 'resource_token_refs       WHERE logical_resource_id = ?';
+    EXECUTE d_stmt USING v_logical_resource_id;
+
+    PREPARE d_stmt FROM 'DELETE FROM {{SCHEMA_NAME}}.' || 'logical_resource_profiles WHERE logical_resource_id = ?';
     EXECUTE d_stmt USING v_logical_resource_id;
 
-    PREPARE d_stmt FROM 'DELETE FROM {{SCHEMA_NAME}}.' || 'date_values         WHERE logical_resource_id = ?';
+    PREPARE d_stmt FROM 'DELETE FROM {{SCHEMA_NAME}}.' || 'logical_resource_tags     WHERE logical_resource_id = ?';
     EXECUTE d_stmt USING v_logical_resource_id;
 
-    PREPARE d_stmt FROM 'DELETE FROM {{SCHEMA_NAME}}.' || 'resource_token_refs WHERE logical_resource_id = ?';
+    PREPARE d_stmt FROM 'DELETE FROM {{SCHEMA_NAME}}.' || 'logical_resource_security WHERE logical_resource_id = ?';
     EXECUTE d_stmt USING v_logical_resource_id;
 
     -- Step 4: Delete from Logical Resources table 
diff --git a/fhir-persistence-schema/src/main/resources/postgres/add_any_resource.sql b/fhir-persistence-schema/src/main/resources/postgres/add_any_resource.sql
index 94000b04dab..22fc8bb7d0c 100644
--- a/fhir-persistence-schema/src/main/resources/postgres/add_any_resource.sql
+++ b/fhir-persistence-schema/src/main/resources/postgres/add_any_resource.sql
@@ -31,7 +31,9 @@
       IN p_is_deleted                       CHAR(  1),
       IN p_source_key                    VARCHAR( 64),
       IN p_version                           INT,
-      OUT o_logical_resource_id            BIGINT)
+      IN p_parameter_hash_b64            VARCHAR( 44),
+      OUT o_logical_resource_id           BIGINT,
+      OUT o_current_parameter_hash       VARCHAR( 44))
     LANGUAGE plpgsql
      AS $$
 
@@ -46,9 +48,9 @@
   v_duplicate               INT := 0;
   v_current_version         INT := 0;
   v_change_type            CHAR(1) := NULL;
-
-  -- Because we don't really update any existing key, so use NO KEY UPDATE to achieve better concurrence performance.
-  lock_cur CURSOR (t_resource_type_id INT, t_logical_id VARCHAR(255)) FOR SELECT logical_resource_id FROM {{SCHEMA_NAME}}.logical_resources WHERE resource_type_id = t_resource_type_id AND logical_id = t_logical_id FOR NO KEY UPDATE;
+  
+  -- Because we don't really update any existing key, so use NO KEY UPDATE to achieve better concurrence performance. 
+  lock_cur CURSOR (t_resource_type_id INT, t_logical_id VARCHAR(255)) FOR SELECT logical_resource_id, parameter_hash FROM {{SCHEMA_NAME}}.logical_resources WHERE resource_type_id = t_resource_type_id AND logical_id = t_logical_id FOR NO KEY UPDATE;
 
 BEGIN
   -- LOADED ON: {{DATE}}
@@ -61,7 +63,7 @@ BEGIN
 
   -- Get a lock at the system-wide logical resource level
   OPEN lock_cur(t_resource_type_id := v_resource_type_id, t_logical_id := p_logical_id);
-  FETCH lock_cur INTO v_logical_resource_id;
+  FETCH lock_cur INTO v_logical_resource_id, o_current_parameter_hash;
   CLOSE lock_cur;
   
   -- Create the resource if we don't have it already
@@ -71,13 +73,13 @@ BEGIN
     -- remember that we have a concurrent system...so there is a possibility
     -- that another thread snuck in before us and created the logical resource. This
     -- is easy to handle, just turn around and read it
-    INSERT INTO {{SCHEMA_NAME}}.logical_resources (logical_resource_id, resource_type_id, logical_id, reindex_tstamp)
-         VALUES (v_logical_resource_id, v_resource_type_id, p_logical_id, '1970-01-01') ON CONFLICT DO NOTHING;
+    INSERT INTO {{SCHEMA_NAME}}.logical_resources (logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash)
+         VALUES (v_logical_resource_id, v_resource_type_id, p_logical_id, '1970-01-01', p_is_deleted, p_last_updated, p_parameter_hash_b64) ON CONFLICT DO NOTHING;
        
       -- row exists, so we just need to obtain a lock on it. Because logical resource records are
       -- never deleted, we don't need to worry about it disappearing again before we grab the row lock
       OPEN lock_cur (t_resource_type_id := v_resource_type_id, t_logical_id := p_logical_id);
-      FETCH lock_cur INTO t_logical_resource_id;
+      FETCH lock_cur INTO t_logical_resource_id, o_current_parameter_hash;
       CLOSE lock_cur;
 
     IF v_logical_resource_id = t_logical_resource_id
@@ -114,29 +116,43 @@ BEGIN
     THEN
       RAISE 'Concurrent update - mismatch of version in JSON' USING ERRCODE = '99001';
     END IF;
-    
-    -- existing resource, so need to delete all its parameters. 
-    -- TODO patch parameter sets instead of all delete/all insert.
-    EXECUTE 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_str_values          WHERE logical_resource_id = $1'
-      USING v_logical_resource_id;
-    EXECUTE 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_number_values       WHERE logical_resource_id = $1'
-      USING v_logical_resource_id;
-    EXECUTE 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_date_values         WHERE logical_resource_id = $1'
-      USING v_logical_resource_id;
-    EXECUTE 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_latlng_values       WHERE logical_resource_id = $1'
-      USING v_logical_resource_id;
-    EXECUTE 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_resource_token_refs WHERE logical_resource_id = $1'
-      USING v_logical_resource_id;
-    EXECUTE 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_quantity_values     WHERE logical_resource_id = $1'
-      USING v_logical_resource_id;
-    EXECUTE 'DELETE FROM ' || v_schema_name || '.' || 'str_values           WHERE logical_resource_id = $1'
-      USING v_logical_resource_id;
-    EXECUTE 'DELETE FROM ' || v_schema_name || '.' || 'date_values          WHERE logical_resource_id = $1'
-      USING v_logical_resource_id;
-    EXECUTE 'DELETE FROM ' || v_schema_name || '.' || 'resource_token_refs  WHERE logical_resource_id = $1'
-      USING v_logical_resource_id;
-  END IF;
 
+    IF o_current_parameter_hash IS NULL OR p_parameter_hash_b64 != o_current_parameter_hash
+    THEN
+	    -- existing resource, so need to delete all its parameters. 
+	    -- TODO patch parameter sets instead of all delete/all insert.
+	    EXECUTE 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_str_values          WHERE logical_resource_id = $1'
+	      USING v_logical_resource_id;
+	    EXECUTE 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_number_values       WHERE logical_resource_id = $1'
+	      USING v_logical_resource_id;
+	    EXECUTE 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_date_values         WHERE logical_resource_id = $1'
+	      USING v_logical_resource_id;
+	    EXECUTE 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_latlng_values       WHERE logical_resource_id = $1'
+	      USING v_logical_resource_id;
+	    EXECUTE 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_resource_token_refs WHERE logical_resource_id = $1'
+	      USING v_logical_resource_id;
+	    EXECUTE 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_quantity_values     WHERE logical_resource_id = $1'
+	      USING v_logical_resource_id;
+	    EXECUTE 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_profiles            WHERE logical_resource_id = $1'
+	      USING v_logical_resource_id;
+	    EXECUTE 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_tags                WHERE logical_resource_id = $1'
+	      USING v_logical_resource_id;
+        EXECUTE 'DELETE FROM ' || v_schema_name || '.' || p_resource_type || '_security            WHERE logical_resource_id = $1'
+          USING v_logical_resource_id;
+	    EXECUTE 'DELETE FROM ' || v_schema_name || '.' || 'str_values                 WHERE logical_resource_id = $1'
+	      USING v_logical_resource_id;
+	    EXECUTE 'DELETE FROM ' || v_schema_name || '.' || 'date_values                WHERE logical_resource_id = $1'
+	      USING v_logical_resource_id;
+	    EXECUTE 'DELETE FROM ' || v_schema_name || '.' || 'resource_token_refs        WHERE logical_resource_id = $1'
+	      USING v_logical_resource_id;
+	    EXECUTE 'DELETE FROM ' || v_schema_name || '.' || 'logical_resource_profiles  WHERE logical_resource_id = $1'
+	      USING v_logical_resource_id;
+	    EXECUTE 'DELETE FROM ' || v_schema_name || '.' || 'logical_resource_tags      WHERE logical_resource_id = $1'
+	      USING v_logical_resource_id;
+        EXECUTE 'DELETE FROM ' || v_schema_name || '.' || 'logical_resource_security  WHERE logical_resource_id = $1'
+          USING v_logical_resource_id;
+	END IF; -- end if check parameter hash
+  END IF; -- end if existing resource
 
   EXECUTE
          'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_resources (resource_id, logical_resource_id, version_id, data, last_updated, is_deleted) '
@@ -150,6 +166,10 @@ BEGIN
     -- need to update them here.
     EXECUTE 'UPDATE ' || v_schema_name || '.' || p_resource_type || '_logical_resources SET current_resource_id = $1, is_deleted = $2, last_updated = $3, version_id = $4 WHERE logical_resource_id = $5'
       USING v_resource_id, p_is_deleted, p_last_updated, p_version, v_logical_resource_id;
+
+    -- For V0014 we now also store is_deleted and last_updated values at the whole-system logical_resources level
+    EXECUTE 'UPDATE ' || v_schema_name || '.logical_resources SET is_deleted = $1, last_updated = $2, parameter_hash = $3 WHERE logical_resource_id = $4'
+      USING p_is_deleted, p_last_updated, p_parameter_hash_b64, v_logical_resource_id;
   END IF;
 
   -- Finally, write a record to RESOURCE_CHANGE_LOG which records each event
diff --git a/fhir-persistence-schema/src/main/resources/postgres/erase_resource.sql b/fhir-persistence-schema/src/main/resources/postgres/erase_resource.sql
index 26bb1ea1584..f5e250c3362 100644
--- a/fhir-persistence-schema/src/main/resources/postgres/erase_resource.sql
+++ b/fhir-persistence-schema/src/main/resources/postgres/erase_resource.sql
@@ -25,7 +25,6 @@
   v_total               BIGINT := 0;
 
 BEGIN
-  -- LOADED ON: {{DATE}}
   v_schema_name := '{{SCHEMA_NAME}}';
 
   -- Prep 1: Get the v_resource_type_id
@@ -72,11 +71,23 @@ BEGIN
     USING v_logical_resource_id;
     EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_quantity_values     WHERE logical_resource_id = $1'
     USING v_logical_resource_id;
-    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.str_values           WHERE logical_resource_id = $1'
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_profiles            WHERE logical_resource_id = $1'
     USING v_logical_resource_id;
-    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.date_values          WHERE logical_resource_id = $1'
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_tags                WHERE logical_resource_id = $1'
     USING v_logical_resource_id;
-    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.resource_token_refs  WHERE logical_resource_id = $1'
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_security            WHERE logical_resource_id = $1'
+    USING v_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.str_values                 WHERE logical_resource_id = $1'
+    USING v_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.date_values                WHERE logical_resource_id = $1'
+    USING v_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.resource_token_refs        WHERE logical_resource_id = $1'
+    USING v_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_profiles  WHERE logical_resource_id = $1'
+    USING v_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_tags      WHERE logical_resource_id = $1'
+    USING v_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_security  WHERE logical_resource_id = $1'
     USING v_logical_resource_id;
 
     -- Step 4: Delete from Logical Resources table 
diff --git a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/search/test/AbstractWholeSystemSearchTest.java b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/search/test/AbstractWholeSystemSearchTest.java
index b7146545dc4..766456123e8 100644
--- a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/search/test/AbstractWholeSystemSearchTest.java
+++ b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/search/test/AbstractWholeSystemSearchTest.java
@@ -32,6 +32,7 @@
 import com.ibm.fhir.model.type.Code;
 import com.ibm.fhir.model.type.Coding;
 import com.ibm.fhir.model.type.Meta;
+import com.ibm.fhir.model.type.Reference;
 import com.ibm.fhir.model.type.Uri;
 
 public abstract class AbstractWholeSystemSearchTest extends AbstractPLSearchTest {
@@ -43,7 +44,12 @@ public abstract class AbstractWholeSystemSearchTest extends AbstractPLSearchTest
     protected final String TAG3 = UUID.randomUUID().toString();
     protected final String SECURITY_SYSTEM = "http://ibm.com/fhir/security";
     protected final String SECURITY = UUID.randomUUID().toString();
+    protected final String TAG_SYSTEM2 = "http://terminology.hl7.org/CodeSystem/v3-ActReason";
+    protected final String TAG4 = "HSYSADMIN";
+    protected final String TAG4TEXT = "someSearchText";
     protected final String PROFILE = "http://ibm.com/fhir/profile/" + UUID.randomUUID().toString();
+    protected final String AUTHOR = "Practitioner/" + UUID.randomUUID().toString();
+    protected final String SOURCE = "http://ibm.com/fhir/source";
 
     @Override
     protected void setTenant() throws Exception {
@@ -62,6 +68,13 @@ public Basic getBasicResource() throws Exception {
                 Coding.builder()
                         .system(Uri.of(TAG_SYSTEM))
                         .code(Code.of(TAG2)).build();
+        Coding tag3 =
+                Coding.builder()
+                        .system(Uri.of(TAG_SYSTEM2))
+                        .code(Code.of(TAG4))
+                        .display(com.ibm.fhir.model.type.String.of(TAG4TEXT))
+                        .build();
+
         Coding security =
                 Coding.builder()
                         .system(Uri.of(SECURITY_SYSTEM))
@@ -72,11 +85,16 @@ public Basic getBasicResource() throws Exception {
                         .tag(tag)
                         .tag(tag)
                         .tag(tag2)
+                        .tag(tag3)
                         .security(security)
                         .profile(Canonical.of(PROFILE))
+                        .source(Uri.of(SOURCE))
                         .build();
 
-        return basic.toBuilder().meta(meta).build();
+        return basic.toBuilder()
+                .meta(meta)
+                .author(Reference.builder().reference(com.ibm.fhir.model.type.String.of(AUTHOR)).build())
+                .build();
     }
 
     @Test
@@ -208,6 +226,62 @@ public void testSearchAllUsingSecurity() throws Exception {
         assertTrue(isResourceInResponse(savedResource, resources), "Expected resource not found in the response");
     }
 
+    @Test
+    public void testSearchAllUsingSource() throws Exception {
+        List resources = runQueryTest(Resource.class, "_source", SOURCE);
+        assertNotNull(resources);
+        assertEquals(resources.size(), 1, "Number of resources returned");
+        assertTrue(isResourceInResponse(savedResource, resources), "Expected resource not found in the response");
+    }
+
+    @Test
+    public void testSearchAllUsingTagModifierText() throws Exception {
+        List resources = runQueryTest(Resource.class, "_tag:text", "someSearch");
+        assertNotNull(resources);
+        assertEquals(resources.size(), 1, "Number of resources returned");
+        assertTrue(isResourceInResponse(savedResource, resources), "Expected resource not found in the response");
+    }
+
+    @Test
+    public void testSearchAllUsingTagModifierIn() throws Exception {
+        List resources = runQueryTest(Resource.class, "_tag:in", "http://terminology.hl7.org/ValueSet/v3-ActReason");
+        assertNotNull(resources);
+        assertEquals(resources.size(), 1, "Number of resources returned");
+        assertTrue(isResourceInResponse(savedResource, resources), "Expected resource not found in the response");
+    }
+
+    @Test
+    public void testSearchAllUsingTagModifierNotIn() throws Exception {
+        Map> queryParms = new HashMap>();
+        queryParms.put("_tag:not-in", Collections.singletonList("http://hl7.org/fhir/ValueSet/common-tags"));
+        queryParms.put("_type", Collections.singletonList("Basic"));
+
+        if (DEBUG) {
+            generateOutput(savedResource);
+        }
+
+        List resources = runQueryTest(Resource.class, queryParms);
+        assertNotNull(resources);
+        assertEquals(resources.size(), 1, "Number of resources returned");
+        assertTrue(isResourceInResponse(savedResource, resources), "Expected resource not found in the response");
+    }
+
+    @Test
+    public void testSearchAllUsingTagModifierAbove() throws Exception {
+        List resources = runQueryTest(Resource.class, "_tag:above", TAG_SYSTEM2 + "|LABELING");
+        assertNotNull(resources);
+        assertEquals(resources.size(), 1, "Number of resources returned");
+        assertTrue(isResourceInResponse(savedResource, resources), "Expected resource not found in the response");
+    }
+
+    @Test
+    public void testSearchAllUsingTagModifierBelow() throws Exception {
+        List resources = runQueryTest(Resource.class, "_tag:below", TAG_SYSTEM2 + "|HOPERAT");
+        assertNotNull(resources);
+        assertEquals(resources.size(), 1, "Number of resources returned");
+        assertTrue(isResourceInResponse(savedResource, resources), "Expected resource not found in the response");
+    }
+
     @Test
     public void testSearchAllUsingProfile() throws Exception {
         List resources = runQueryTest(Resource.class, "_profile", PROFILE);
@@ -216,6 +290,72 @@ public void testSearchAllUsingProfile() throws Exception {
         assertTrue(isResourceInResponse(savedResource, resources), "Expected resource not found in the response");
     }
 
+    @Test
+    public void testSearchAllUsingProfileAndTag() throws Exception {
+        Map> queryParms = new HashMap>();
+        queryParms.put("_tag", Collections.singletonList(TAG_SYSTEM + "|" + TAG));
+        queryParms.put("_profile", Collections.singletonList(PROFILE));
+
+        if (DEBUG) {
+            generateOutput(savedResource);
+        }
+
+        List resources = runQueryTest(Resource.class, queryParms);
+        assertNotNull(resources);
+        assertEquals(resources.size(), 1, "Number of resources returned");
+        assertTrue(isResourceInResponse(savedResource, resources), "Expected resource not found in the response");
+    }
+
+    @Test
+    public void testSearchAllUsingSecurityAndTag() throws Exception {
+        Map> queryParms = new HashMap>();
+        queryParms.put("_tag", Collections.singletonList(TAG_SYSTEM + "|" + TAG));
+        queryParms.put("_security", Collections.singletonList(SECURITY_SYSTEM + "|" + SECURITY));
+
+        if (DEBUG) {
+            generateOutput(savedResource);
+        }
+
+        List resources = runQueryTest(Resource.class, queryParms);
+        assertNotNull(resources);
+        assertEquals(resources.size(), 1, "Number of resources returned");
+        assertTrue(isResourceInResponse(savedResource, resources), "Expected resource not found in the response");
+    }
+
+    @Test
+    public void testSearchAllUsingProfileAndSecurityAndTag() throws Exception {
+        Map> queryParms = new HashMap>();
+        queryParms.put("_profile", Collections.singletonList(PROFILE));
+        queryParms.put("_tag", Collections.singletonList(TAG_SYSTEM + "|" + TAG));
+        queryParms.put("_security", Collections.singletonList(SECURITY_SYSTEM + "|" + SECURITY));
+
+        if (DEBUG) {
+            generateOutput(savedResource);
+        }
+
+        List resources = runQueryTest(Resource.class, queryParms);
+        assertNotNull(resources);
+        assertEquals(resources.size(), 1, "Number of resources returned");
+        assertTrue(isResourceInResponse(savedResource, resources), "Expected resource not found in the response");
+    }
+
+    @Test
+    public void testSearchAllUsingProfileAndSecurityAndSource() throws Exception {
+        Map> queryParms = new HashMap>();
+        queryParms.put("_profile", Collections.singletonList(PROFILE));
+        queryParms.put("_source", Collections.singletonList(SOURCE));
+        queryParms.put("_security", Collections.singletonList(SECURITY_SYSTEM + "|" + SECURITY));
+
+        if (DEBUG) {
+            generateOutput(savedResource);
+        }
+
+        List resources = runQueryTest(Resource.class, queryParms);
+        assertNotNull(resources);
+        assertEquals(resources.size(), 1, "Number of resources returned");
+        assertTrue(isResourceInResponse(savedResource, resources), "Expected resource not found in the response");
+    }
+
     @Test
     public void testSearchAllUsingElements() throws Exception {
         // This might fail if there are more than 1000 resources in the test db
@@ -309,4 +449,112 @@ public void testSearchAllUsingTagNot_NoResults() throws Exception {
         assertEquals(resources.size(), 0, "Number of resources returned");
     }
 
-}
+    @Test
+    public void testSearchAllUsingType() throws Exception {
+        List resources = runQueryTest(Resource.class, "_type", "Basic,EvidenceVariable,ServiceRequest");
+        assertNotNull(resources);
+        assertEquals(resources.size(), 1, "Number of resources returned");
+        assertTrue(isResourceInResponse(savedResource, resources), "Expected resource not found in the response");
+    }
+
+    @Test
+    public void testSearchAllUsingTypeAndProfile() throws Exception {
+        Map> queryParms = new HashMap>();
+        queryParms.put("_type", Collections.singletonList("Basic,Patient,Observation"));
+        queryParms.put("_profile", Collections.singletonList(PROFILE));
+        List resources = runQueryTest(Resource.class, queryParms);
+        assertNotNull(resources);
+        assertEquals(resources.size(), 1, "Number of resources returned");
+        assertTrue(isResourceInResponse(savedResource, resources), "Expected resource not found in the response");
+    }
+
+    @Test
+    public void testSearchAllUsingTypeAndSecurity() throws Exception {
+        Map> queryParms = new HashMap>();
+        queryParms.put("_type", Collections.singletonList("Basic,Patient,Observation"));
+        queryParms.put("_security", Collections.singletonList(SECURITY_SYSTEM + "|" + SECURITY));
+        List resources = runQueryTest(Resource.class, queryParms);
+        assertNotNull(resources);
+        assertEquals(resources.size(), 1, "Number of resources returned");
+        assertTrue(isResourceInResponse(savedResource, resources), "Expected resource not found in the response");
+    }
+
+    @Test
+    public void testSearchAllUsingTypeAndSource() throws Exception {
+        Map> queryParms = new HashMap>();
+        queryParms.put("_type", Collections.singletonList("Basic,Patient,Observation"));
+        queryParms.put("_source", Collections.singletonList(SOURCE));
+        List resources = runQueryTest(Resource.class, queryParms);
+        assertNotNull(resources);
+        assertEquals(resources.size(), 1, "Number of resources returned");
+        assertTrue(isResourceInResponse(savedResource, resources), "Expected resource not found in the response");
+    }
+
+    @Test
+    public void testSearchAllUsingTypeAndProfileAndSourceAndTag() throws Exception {
+        Map> queryParms = new HashMap>();
+        queryParms.put("_type", Collections.singletonList("Basic,Patient,Observation"));
+        queryParms.put("_profile", Collections.singletonList(PROFILE));
+        queryParms.put("_source", Collections.singletonList(SOURCE));
+        queryParms.put("_tag", Collections.singletonList(TAG_SYSTEM + "|" + TAG));
+        List resources = runQueryTest(Resource.class, queryParms);
+        assertNotNull(resources);
+        assertEquals(resources.size(), 1, "Number of resources returned");
+        assertTrue(isResourceInResponse(savedResource, resources), "Expected resource not found in the response");
+    }
+
+    @Test
+    public void testSearchAllUsingTypeAndNonGlobalSearchParm() throws Exception {
+        Map> queryParms = new HashMap>();
+        queryParms.put("_type", Collections.singletonList("Basic,DetectedIssue,DocumentReference"));
+        queryParms.put("author", Collections.singletonList(AUTHOR));
+        List resources = runQueryTest(Resource.class, queryParms);
+        assertNotNull(resources);
+        assertEquals(resources.size(), 1, "Number of resources returned");
+        assertTrue(isResourceInResponse(savedResource, resources), "Expected resource not found in the response");
+    }
+
+    @Test
+    public void testSearchAllUsingTypeAndNonGlobalSearchParmAndProfile() throws Exception {
+        Map> queryParms = new HashMap>();
+        queryParms.put("_type", Collections.singletonList("Basic,DetectedIssue,DocumentReference"));
+        queryParms.put("author", Collections.singletonList(AUTHOR));
+        queryParms.put("_profile", Collections.singletonList(PROFILE));
+        List resources = runQueryTest(Resource.class, queryParms);
+        assertNotNull(resources);
+        assertEquals(resources.size(), 1, "Number of resources returned");
+        assertTrue(isResourceInResponse(savedResource, resources), "Expected resource not found in the response");
+    }
+
+    @Test
+    public void testSearchAllUsingTypeAndNonGlobalSearchParmAndSecurity() throws Exception {
+        Map> queryParms = new HashMap>();
+        queryParms.put("_type", Collections.singletonList("Basic,DetectedIssue,DocumentReference"));
+        queryParms.put("author", Collections.singletonList(AUTHOR));
+        queryParms.put("_security", Collections.singletonList(SECURITY_SYSTEM + "|" + SECURITY));
+        List resources = runQueryTest(Resource.class, queryParms);
+        assertNotNull(resources);
+        assertEquals(resources.size(), 1, "Number of resources returned");
+        assertTrue(isResourceInResponse(savedResource, resources), "Expected resource not found in the response");
+    }
+
+    @Test
+    public void testSearchAllUsingTypeAndNonGlobalSearchParmAndSource() throws Exception {
+        Map> queryParms = new HashMap>();
+        queryParms.put("_type", Collections.singletonList("Basic,DetectedIssue,DocumentReference"));
+        queryParms.put("author", Collections.singletonList(AUTHOR));
+        queryParms.put("_source", Collections.singletonList(SOURCE));
+        List resources = runQueryTest(Resource.class, queryParms);
+        assertNotNull(resources);
+        assertEquals(resources.size(), 1, "Number of resources returned");
+        assertTrue(isResourceInResponse(savedResource, resources), "Expected resource not found in the response");
+    }
+
+    @Test
+    public void testSearchAllUsingTypeNoResults() throws Exception {
+        List resources = runQueryTest(Resource.class, "_type", "EvidenceVariable,ServiceRequest");
+        assertNotNull(resources);
+        assertEquals(resources.size(), 0, "Number of resources returned");
+    }
+
+}
\ No newline at end of file
diff --git a/fhir-search/src/main/java/com/ibm/fhir/search/SearchConstants.java b/fhir-search/src/main/java/com/ibm/fhir/search/SearchConstants.java
index ef50bc124d4..3bf00279063 100644
--- a/fhir-search/src/main/java/com/ibm/fhir/search/SearchConstants.java
+++ b/fhir-search/src/main/java/com/ibm/fhir/search/SearchConstants.java
@@ -95,6 +95,18 @@ private SearchConstants() {
     // _has
     public static final String HAS = "_has";
 
+    // _profile
+    public static final String PROFILE = "_profile";
+
+    // _tag
+    public static final String TAG = "_tag";
+
+    // _security
+    public static final String SECURITY = "_security";
+    
+    // _source
+    public static final String SOURCE = "_source";
+    
     public static final String IMPLICIT_SYSTEM_EXT_URL = FHIRConstants.EXT_BASE + "implicit-system";
 
     // Extracted search parameter suffix for :identifier modifier
@@ -124,6 +136,10 @@ private SearchConstants() {
     public static final Set SEARCH_SINGLETON_PARAMETER_NAMES =
             Collections.unmodifiableSet(new HashSet<>(Arrays.asList(SORT, COUNT, PAGE, SUMMARY, TOTAL, ELEMENTS, RESOURCE_TYPE)));
 
+    // Set of whole-system search parameters indexed in global parameter tables
+    public static final Set SYSTEM_LEVEL_GLOBAL_PARAMETER_NAMES =
+            Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ID, LAST_UPDATED, PROFILE, SECURITY, SOURCE, TAG)));
+    
     // Empty Query String
     public static final String EMPTY_QUERY_STRING = "";
 
diff --git a/fhir-server-test/src/test/java/com/ibm/fhir/server/test/SearchAllTest.java b/fhir-server-test/src/test/java/com/ibm/fhir/server/test/SearchAllTest.java
index cd49f0ce12a..cfacfaedfb7 100644
--- a/fhir-server-test/src/test/java/com/ibm/fhir/server/test/SearchAllTest.java
+++ b/fhir-server-test/src/test/java/com/ibm/fhir/server/test/SearchAllTest.java
@@ -50,6 +50,7 @@
 import com.ibm.fhir.model.type.Coding;
 import com.ibm.fhir.model.type.Instant;
 import com.ibm.fhir.model.type.Meta;
+import com.ibm.fhir.model.type.Uri;
 
 public class SearchAllTest extends FHIRServerTestBase {
 
@@ -88,6 +89,7 @@ public void testCreatePatient() throws Exception {
                                 .tag(tag)
                                 .tag(tag2)
                                 .profile(Canonical.of("http://ibm.com/fhir/profile/Profile"))
+                                .source(Uri.of("http://ibm.com/fhir/source/Source"))
                                 .build())
                         .build();
 
@@ -255,6 +257,40 @@ public void testSearchAllUsingSecurity() throws Exception {
         assertTrue(bundle.getEntry().size() >= 1);
     }
 
+    @Test(groups = { "server-search-all" }, dependsOnMethods = { "testCreatePatient" })
+    public void testSearchAllUsingSource() throws Exception {
+        FHIRParameters parameters = new FHIRParameters();
+        parameters.searchParam("_source", "http://ibm.com/fhir/source/Source");
+        FHIRResponse response = client.searchAll(parameters, false, headerTenant, headerDataStore);
+        assertResponse(response.getResponse(), Response.Status.OK.getStatusCode());
+        Bundle bundle = response.getResource(Bundle.class);
+
+        assertNotNull(bundle);
+        if (DEBUG_SEARCH) {
+            generateOutput(bundle);
+        }
+
+        assertTrue(bundle.getEntry().size() >= 1);
+    }
+
+    @Test(groups = { "server-search-all" }, dependsOnMethods = { "testCreatePatient" })
+    public void testSearchAllUsingProfileAndSecurityAndSource() throws Exception {
+        FHIRParameters parameters = new FHIRParameters();
+        parameters.searchParam("_profile", "http://ibm.com/fhir/profile/Profile");
+        parameters.searchParam("_security", "security");
+        parameters.searchParam("_source", "http://ibm.com/fhir/source/Source");
+        FHIRResponse response = client.searchAll(parameters, false, headerTenant, headerDataStore);
+        assertResponse(response.getResponse(), Response.Status.OK.getStatusCode());
+        Bundle bundle = response.getResource(Bundle.class);
+
+        assertNotNull(bundle);
+        if (DEBUG_SEARCH) {
+            generateOutput(bundle);
+        }
+
+        assertTrue(bundle.getEntry().size() >= 1);
+    }
+
     @Test(groups = { "server-search-all" }, dependsOnMethods = { "testCreatePatient" })
     public void testSearchAllUsingProfile() throws Exception {
         FHIRParameters parameters = new FHIRParameters();
@@ -561,6 +597,7 @@ public void testCreatePatientAndObservationWithUniqueTag() throws Exception {
                                 .security(security)
                                 .tag(uniqueTag)
                                 .profile(Canonical.of("http://ibm.com/fhir/profile/Profile"))
+                                .source(Uri.of("http://ibm.com/fhir/source/Source"))
                                 .build())
                         .build();
 
@@ -698,6 +735,60 @@ public void testSearchAll2UsingUniqueTag_TwoTypes_elements() throws Exception {
         assertTrue(bundle.getEntry().size() == 2);
     }
 
+    @Test(groups = { "server-search-all" }, dependsOnMethods = { "testCreatePatientAndObservationWithUniqueTag" })
+    public void testSearchAll2UsingUniqueTag_TwoTypes_profile() throws Exception {
+        FHIRParameters parameters = new FHIRParameters();
+        parameters.searchParam("_tag", strUniqueTag);
+        parameters.searchParam("_type", "Patient,Observation");
+        parameters.searchParam("_profile", "http://ibm.com/fhir/profile/Profile");
+        FHIRResponse response = client.searchAll(parameters, true, headerTenant, headerDataStore);
+        assertResponse(response.getResponse(), Response.Status.OK.getStatusCode());
+        Bundle bundle = response.getResource(Bundle.class);
+        assertNotNull(bundle);
+        assertTrue(bundle.getEntry().size() == 1);
+    }
+
+    @Test(groups = { "server-search-all" }, dependsOnMethods = { "testCreatePatientAndObservationWithUniqueTag" })
+    public void testSearchAll2UsingUniqueTag_TwoTypes_security() throws Exception {
+        FHIRParameters parameters = new FHIRParameters();
+        parameters.searchParam("_tag", strUniqueTag);
+        parameters.searchParam("_type", "Patient,Observation");
+        parameters.searchParam("_security", "security");
+        FHIRResponse response = client.searchAll(parameters, true, headerTenant, headerDataStore);
+        assertResponse(response.getResponse(), Response.Status.OK.getStatusCode());
+        Bundle bundle = response.getResource(Bundle.class);
+        assertNotNull(bundle);
+        assertTrue(bundle.getEntry().size() == 1);
+    }
+
+    @Test(groups = { "server-search-all" }, dependsOnMethods = { "testCreatePatientAndObservationWithUniqueTag" })
+    public void testSearchAll2UsingUniqueTag_TwoTypes_source() throws Exception {
+        FHIRParameters parameters = new FHIRParameters();
+        parameters.searchParam("_tag", strUniqueTag);
+        parameters.searchParam("_type", "Patient,Observation");
+        parameters.searchParam("_source", "http://ibm.com/fhir/source/Source");
+        FHIRResponse response = client.searchAll(parameters, true, headerTenant, headerDataStore);
+        assertResponse(response.getResponse(), Response.Status.OK.getStatusCode());
+        Bundle bundle = response.getResource(Bundle.class);
+        assertNotNull(bundle);
+        assertTrue(bundle.getEntry().size() == 1);
+    }
+
+    @Test(groups = { "server-search-all" }, dependsOnMethods = { "testCreatePatientAndObservationWithUniqueTag" })
+    public void testSearchAll2UsingUniqueTag_TwoTypes_profile_security_source() throws Exception {
+        FHIRParameters parameters = new FHIRParameters();
+        parameters.searchParam("_tag", strUniqueTag);
+        parameters.searchParam("_type", "Patient,Observation");
+        parameters.searchParam("_profile", "http://ibm.com/fhir/profile/Profile");
+        parameters.searchParam("_security", "security");
+        parameters.searchParam("_source", "http://ibm.com/fhir/source/Source");
+        FHIRResponse response = client.searchAll(parameters, true, headerTenant, headerDataStore);
+        assertResponse(response.getResponse(), Response.Status.OK.getStatusCode());
+        Bundle bundle = response.getResource(Bundle.class);
+        assertNotNull(bundle);
+        assertTrue(bundle.getEntry().size() == 1);
+    }
+
     @Test(groups = { "server-search-all" }, dependsOnMethods = { "testCreatePatientAndObservationWithUniqueTag" })
     public void testSearchAll2_TwoTypes_InvalidInclude() throws Exception {
         FHIRParameters parameters = new FHIRParameters();
diff --git a/fhir-server-test/src/test/java/com/ibm/fhir/server/test/SortingTest.java b/fhir-server-test/src/test/java/com/ibm/fhir/server/test/SortingTest.java
index a39135b41bf..8411d2ac696 100644
--- a/fhir-server-test/src/test/java/com/ibm/fhir/server/test/SortingTest.java
+++ b/fhir-server-test/src/test/java/com/ibm/fhir/server/test/SortingTest.java
@@ -1,5 +1,5 @@
 /*
- * (C) Copyright IBM Corp. 2017,2019
+ * (C) Copyright IBM Corp. 2017, 2021
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -28,6 +28,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
+import java.util.UUID;
 import java.util.stream.Collectors;
 
 import javax.ws.rs.client.Entity;
@@ -56,6 +57,14 @@
 import com.ibm.fhir.model.util.JsonSupport;
 
 public class SortingTest extends FHIRServerTestBase {
+    private static final String TAG_SYSTEM = "http://ibm.com/fhir/sort-test";
+    private static final String TAG_CODE = "sort-test-" + UUID.randomUUID().toString();
+    private static final String TAG_DISPLAY = TAG_CODE;
+    public static final Coding TAG = Coding.builder()
+            .system(Uri.of(TAG_SYSTEM))
+            .code(Code.of(TAG_CODE))
+            .display(string(TAG_DISPLAY))
+            .build();
 
     private static final boolean DEBUG_SEARCH = false;
 
@@ -78,6 +87,7 @@ public void testCreatePatient1() throws Exception {
 
         // Build a new Patient and then call the 'create' API.
         Patient patient = TestUtil.readLocalResource("Patient_JohnDoe.json");
+        patient = FHIRUtil.addTag(patient, TAG);
 
         patient = patient.toBuilder().gender(AdministrativeGender.MALE).build();
         Entity entity = Entity.entity(patient, FHIRMediaType.APPLICATION_FHIR_JSON);
@@ -101,6 +111,7 @@ public void testCreatePatient2() throws Exception {
 
         // Build a new Patient and then call the 'create' API.
         Patient patient = TestUtil.readLocalResource("Patient_DavidOrtiz.json");
+        patient = FHIRUtil.addTag(patient, TAG);
 
         patient = patient.toBuilder().gender(AdministrativeGender.MALE).build();
         Entity entity = Entity.entity(patient, FHIRMediaType.APPLICATION_FHIR_JSON);
@@ -124,6 +135,7 @@ public void testCreatePatient3() throws Exception {
 
         // Build a new Patient and then call the 'create' API.
         Patient patient = TestUtil.readLocalResource("patient-example-a.json");
+        patient = FHIRUtil.addTag(patient, TAG);
 
         Entity entity = Entity.entity(patient, FHIRMediaType.APPLICATION_FHIR_JSON);
         Response response = target.path("Patient").request().post(entity, Response.class);
@@ -146,6 +158,7 @@ public void testCreatePatient4() throws Exception {
 
         // Build a new Patient and then call the 'create' API.
         Patient patient = TestUtil.readLocalResource("patient-example-c.json");
+        patient = FHIRUtil.addTag(patient, TAG);
 
         Entity entity = Entity.entity(patient, FHIRMediaType.APPLICATION_FHIR_JSON);
         Response response = target.path("Patient").request().post(entity, Response.class);
@@ -168,6 +181,7 @@ public void testCreatePatient5() throws Exception {
 
         // Build a new Patient and then call the 'create' API.
         Patient patient = TestUtil.readLocalResource("patient-example-a1.json");
+        patient = FHIRUtil.addTag(patient, TAG);
 
         Entity entity = Entity.entity(patient, FHIRMediaType.APPLICATION_FHIR_JSON);
         Response response = target.path("Patient").request().post(entity, Response.class);
@@ -189,6 +203,7 @@ public void testCreateObservation1() throws Exception {
         WebTarget target = getWebTarget();
 
         Observation observation = TestUtil.buildPatientObservation(patientId, "Observation1.json");
+        observation = FHIRUtil.addTag(observation, TAG);
         Entity entity = Entity.entity(observation, FHIRMediaType.APPLICATION_FHIR_JSON);
         Response response = target.path("Observation").request().post(entity, Response.class);
         assertResponse(response, Response.Status.CREATED.getStatusCode());
@@ -212,6 +227,7 @@ public void testCreateObservation2() throws Exception {
         WebTarget target = getWebTarget();
 
         Observation observation = TestUtil.buildPatientObservation(patientId, "Observation2.json");
+        observation = FHIRUtil.addTag(observation, TAG);
         Entity entity = Entity.entity(observation, FHIRMediaType.APPLICATION_FHIR_JSON);
         Response response = target.path("Observation").request().post(entity, Response.class);
         assertResponse(response, Response.Status.CREATED.getStatusCode());
@@ -235,6 +251,7 @@ public void testCreateObservation3() throws Exception {
         WebTarget target = getWebTarget();
 
         Observation observation = TestUtil.buildPatientObservation(patientId, "Observation3.json");
+        observation = FHIRUtil.addTag(observation, TAG);
         Entity entity = Entity.entity(observation, FHIRMediaType.APPLICATION_FHIR_JSON);
         Response response = target.path("Observation").request().post(entity, Response.class);
         assertResponse(response, Response.Status.CREATED.getStatusCode());
@@ -258,6 +275,7 @@ public void testCreateObservation5() throws Exception {
         WebTarget target = getWebTarget();
 
         Observation observation = TestUtil.buildPatientObservation("1", "Observation5.json");
+        observation = FHIRUtil.addTag(observation, TAG);
         Entity entity = Entity.entity(observation, FHIRMediaType.APPLICATION_FHIR_JSON);
         Response response = target.path("Observation").request().post(entity, Response.class);
         assertResponse(response, Response.Status.CREATED.getStatusCode());
@@ -276,13 +294,14 @@ public void testCreateObservation5() throws Exception {
         TestUtil.assertResourceEquals(observation, responseObservation);
     }
 
-    // Patient?gender=male&_sort=family
+    // Patient?gender=male&_sort=family&_tag=TAG_DISPLAY
     @Test(groups = { "server-search" }, dependsOnMethods = { "testCreatePatient1",
             "testCreatePatient2", "testCreatePatient3", "testCreatePatient4", "testCreatePatient5" })
     public void testSortAscending() {
         WebTarget target = getWebTarget();
         Response response =
                 target.path("Patient").queryParam("gender", "male").queryParam("_count", "50")
+                .queryParam("_tag", TAG_CODE)
                 .queryParam("_sort", "family").request(FHIRMediaType.APPLICATION_FHIR_JSON).get();
         assertResponse(response, Response.Status.OK.getStatusCode());
         Bundle bundle = response.readEntity(Bundle.class);
@@ -317,6 +336,7 @@ public void testSortAscending_filter_elements() throws Exception {
         WebTarget target = getWebTarget();
         Response response =
                 target.path("Patient").queryParam("gender", "male").queryParam("_count", "50")
+                .queryParam("_tag", TAG_CODE)
                 .queryParam("_sort", "family").queryParam("_elements", "gender,name")
                 .request(FHIRMediaType.APPLICATION_FHIR_JSON).get();
         assertResponse(response, Response.Status.OK.getStatusCode());
@@ -390,6 +410,7 @@ public void testSortDescending() {
         WebTarget target = getWebTarget();
         Response response =
                 target.path("Patient").queryParam("gender", "male").queryParam("_count", "50")
+                .queryParam("_tag", TAG_CODE)
                 .queryParam("_sort", "-family").request(FHIRMediaType.APPLICATION_FHIR_JSON).get();
         assertResponse(response, Response.Status.OK.getStatusCode());
         Bundle bundle = response.readEntity(Bundle.class);
@@ -424,6 +445,7 @@ public void testSortTelecom() {
         WebTarget target = getWebTarget();
         Response response =
                 target.path("Patient").queryParam("gender", "male").queryParam("_count", "50")
+                .queryParam("_tag", TAG_CODE)
                 .queryParam("_sort", "telecom").request(FHIRMediaType.APPLICATION_FHIR_JSON).get();
         assertResponse(response, Response.Status.OK.getStatusCode());
         Bundle bundle = response.readEntity(Bundle.class);
@@ -458,6 +480,7 @@ public void testSortBirthDate() {
         WebTarget target = getWebTarget();
         Response response =
                 target.path("Patient").queryParam("gender", "male").queryParam("_count", "50")
+                .queryParam("_tag", TAG_CODE)
                 .queryParam("_sort", "-birthdate").request(FHIRMediaType.APPLICATION_FHIR_JSON).get();
         assertResponse(response, Response.Status.OK.getStatusCode());
         Bundle bundle = response.readEntity(Bundle.class);
@@ -480,6 +503,7 @@ public void testSortTwoParameters() {
         WebTarget target = getWebTarget();
         Response response =
                 target.path("Patient").queryParam("_count", "50")
+                .queryParam("_tag", TAG_CODE)
                 .queryParam("_sort", "-family,birthdate").request(FHIRMediaType.APPLICATION_FHIR_JSON).get();
         assertResponse(response, Response.Status.OK.getStatusCode());
         Bundle bundle = response.readEntity(Bundle.class);
@@ -534,6 +558,7 @@ public void testSortTwoParametersDescending() {
         WebTarget target = getWebTarget();
         Response response =
                 target.path("Patient").queryParam("_count", "50")
+                .queryParam("_tag", TAG_CODE)
                 .queryParam("_sort", "-family,-birthdate").request(FHIRMediaType.APPLICATION_FHIR_JSON).get();
         assertResponse(response, Response.Status.OK.getStatusCode());
         Bundle bundle = response.readEntity(Bundle.class);
@@ -586,6 +611,7 @@ public void testSortValueQuantityAscending() {
         // we do support coding instead of code for the pipe search.
         Response response =
                 target.path("Observation").queryParam("status", "final").queryParam("code", "55284-4")
+                .queryParam("_tag", TAG_CODE)
                 .queryParam("_count", "100").queryParam("_sort", "component-value-quantity")
                 .request(FHIRMediaType.APPLICATION_FHIR_JSON).get();
         assertResponse(response, Response.Status.OK.getStatusCode());
@@ -662,7 +688,7 @@ private void assertTrueNaturalOrdering(List sortedList) {
                 BigDecimal current = iter.next();
 
                 if (DEBUG_SEARCH) {
-                    System.out.println(prior + " " + current);
+                    System.out.println(prior + " " + current + " compare=" + (prior.compareTo(current) <= 0));
                 }
 
                 assertTrue(prior.compareTo(current) <= 0);
@@ -783,6 +809,7 @@ public void testSortAscending_filter_summary() throws Exception {
         WebTarget target = getWebTarget();
         Response response =
                 target.path("Patient").queryParam("gender", "male")
+                .queryParam("_tag", TAG_CODE)
                 .queryParam("_count", "50").queryParam("_sort", "family")
                 .queryParam("_summary", "true").request(FHIRMediaType.APPLICATION_FHIR_JSON).get();
         assertResponse(response, Response.Status.OK.getStatusCode());
@@ -857,6 +884,7 @@ public void testSortAscending_filter_elements_summary() throws Exception {
         // The _summary=true should be ignored.
         Response response =
                 target.path("Patient").queryParam("gender", "male").queryParam("_count", "50")
+                .queryParam("_tag", TAG_CODE)
                 .queryParam("_sort", "family").queryParam("_elements", "gender,name")
                 .queryParam("_summary", "true")
                 .request(FHIRMediaType.APPLICATION_FHIR_JSON).get();
@@ -932,8 +960,9 @@ public void testSortAscending_filter_summary_count() throws Exception {
         WebTarget target = getWebTarget();
         Response response =
                 target.path("Patient").queryParam("gender", "male").queryParam("_count", "50")
-                .queryParam("_sort", "family").queryParam("_summary", "count")
-                .request(FHIRMediaType.APPLICATION_FHIR_JSON).get();
+                    .queryParam("_tag", TAG_CODE)
+                    .queryParam("_sort", "family").queryParam("_summary", "count")
+                    .request(FHIRMediaType.APPLICATION_FHIR_JSON).get();
         assertResponse(response, Response.Status.OK.getStatusCode());
         Bundle bundle = response.readEntity(Bundle.class);
         assertNotNull(bundle);
diff --git a/fhir-server-test/src/test/java/com/ibm/fhir/server/test/operation/ReindexOperationPhase2Test.java b/fhir-server-test/src/test/java/com/ibm/fhir/server/test/operation/ReindexOperationPhase2Test.java
index b56ceaf63b6..16c3e5a85eb 100644
--- a/fhir-server-test/src/test/java/com/ibm/fhir/server/test/operation/ReindexOperationPhase2Test.java
+++ b/fhir-server-test/src/test/java/com/ibm/fhir/server/test/operation/ReindexOperationPhase2Test.java
@@ -144,27 +144,31 @@ public void testReindex_ChangedExpression_Phase2_Search_Type() throws IOExceptio
 
     @Test(groups = { "reindex" }, dependsOnMethods = {"testReindex_ChangedExpression_Phase2_Search_Type"})
     public void testReindexWithInstanceExists_Phase2() {
-        List parameters = new ArrayList<>();
-        parameters.add(Parameter.builder()
-            .name(string("resourceCount"))
-            .value(of(5))
-            .build());
+        if (runIt) {
+            List parameters = new ArrayList<>();
+            parameters.add(Parameter.builder()
+                .name(string("resourceCount"))
+                .value(of(5))
+                .build());
 
-        Parameters.Builder builder = Parameters.builder();
-        builder.id(UUID.randomUUID().toString());
-        builder.parameter(parameters);
-        Parameters ps = builder.build();
+            Parameters.Builder builder = Parameters.builder();
+            builder.id(UUID.randomUUID().toString());
+            builder.parameter(parameters);
+            Parameters ps = builder.build();
 
-        Entity entity = Entity.entity(ps, FHIRMediaType.APPLICATION_FHIR_JSON);
+            Entity entity = Entity.entity(ps, FHIRMediaType.APPLICATION_FHIR_JSON);
 
-        Response r = getWebTarget()
-                .path("/Patient/REIN-DEX-TEST-1/$reindex")
-                .request(FHIRMediaType.APPLICATION_FHIR_JSON)
-                .header("X-FHIR-TENANT-ID", "default")
-                .header("X-FHIR-DSID", "default")
-                .post(entity, Response.class);
+            Response r = getWebTarget()
+                    .path("/Patient/REIN-DEX-TEST-1/$reindex")
+                    .request(FHIRMediaType.APPLICATION_FHIR_JSON)
+                    .header("X-FHIR-TENANT-ID", "default")
+                    .header("X-FHIR-DSID", "default")
+                    .post(entity, Response.class);
 
-        assertEquals(r.getStatus(), Status.OK.getStatusCode());
+            assertEquals(r.getStatus(), Status.OK.getStatusCode());
+        } else {
+            System.out.println("Skipping Phase 2 of Reindex Operation Tests");
+        }
     }
 
     /**
diff --git a/fhir-server-test/src/test/java/com/ibm/fhir/server/test/operation/ReindexOperationTest.java b/fhir-server-test/src/test/java/com/ibm/fhir/server/test/operation/ReindexOperationTest.java
index 53495485730..626186a4465 100644
--- a/fhir-server-test/src/test/java/com/ibm/fhir/server/test/operation/ReindexOperationTest.java
+++ b/fhir-server-test/src/test/java/com/ibm/fhir/server/test/operation/ReindexOperationTest.java
@@ -14,17 +14,20 @@
 import java.time.ZonedDateTime;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Properties;
 import java.util.UUID;
 
 import javax.ws.rs.client.Entity;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.Response.Status;
 
+import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
 import com.ibm.fhir.core.FHIRMediaType;
 import com.ibm.fhir.model.resource.Parameters;
 import com.ibm.fhir.model.resource.Parameters.Parameter;
+import com.ibm.fhir.model.test.TestUtil;
 import com.ibm.fhir.server.test.FHIRServerTestBase;
 
 /**
@@ -32,9 +35,20 @@
  * These reindex test always run.
  */
 public class ReindexOperationTest extends FHIRServerTestBase {
+    private boolean runIt = true;
+
+    @BeforeClass(enabled = false)
+    public void setup() throws Exception {
+        Properties testProperties = TestUtil.readTestProperties("test.properties");
+        runIt = Boolean.parseBoolean(testProperties.getProperty("test.reindex.enabled", "false"));
+    }
 
     @Test(groups = { "reindex" })
     public void testReindex() {
+        if (!runIt) {
+            System.out.println("Skipping over $reindex IT test");
+            return;
+        }
         List parameters = new ArrayList<>();
         parameters.add(Parameter.builder()
             .name(string("resourceCount"))
@@ -60,6 +74,10 @@ public void testReindex() {
 
     @Test(groups = { "reindex" })
     public void testReindexWithType() {
+        if (!runIt) {
+            System.out.println("Skipping over $reindex IT test");
+            return;
+        }
         List parameters = new ArrayList<>();
         parameters.add(Parameter.builder()
             .name(string("resourceCount"))
@@ -138,6 +156,10 @@ public void testReindexWithInstanceDoesntExist() {
 
     @Test(groups = { "reindex" }, dependsOnMethods = {})
     public void testReindexFromTimestampInstant() {
+        if (!runIt) {
+            System.out.println("Skipping over $reindex IT test");
+            return;
+        }
         List parameters = new ArrayList<>();
         parameters.add(Parameter.builder()
             .name(string("resourceCount"))
@@ -167,6 +189,10 @@ public void testReindexFromTimestampInstant() {
 
     @Test(groups = { "reindex" }, dependsOnMethods = {})
     public void testReindexFromTimestampDayTimeFormat() {
+        if (!runIt) {
+            System.out.println("Skipping over $reindex IT test");
+            return;
+        }
         ZonedDateTime zdt = ZonedDateTime.now();
         String tstamp =
                 zdt.getYear()
@@ -245,6 +271,10 @@ public void testReindexUsingUnsupportedGet() {
 
     @Test(groups = { "reindex" }, dependsOnMethods = {})
     public void testReindexWithLogicalIdDoesntExist() {
+        if (!runIt) {
+            System.out.println("Skipping over $reindex IT test");
+            return;
+        }
         /*
          * Note, this still passes, and ends up reindexing the default number of resources.
          */
@@ -273,6 +303,10 @@ public void testReindexWithLogicalIdDoesntExist() {
 
     @Test(groups = { "reindex" }, dependsOnMethods = {})
     public void testReindexWithSpecificTypeLogicalId() {
+        if (!runIt) {
+            System.out.println("Skipping over $reindex IT test");
+            return;
+        }
         List parameters = new ArrayList<>();
         parameters.add(Parameter.builder()
             .name(string("resourceLogicalId"))
@@ -298,6 +332,10 @@ public void testReindexWithSpecificTypeLogicalId() {
 
     @Test(groups = { "reindex" }, dependsOnMethods = {})
     public void testReindexWithSpecificTypeMissingLogicalId() {
+        if (!runIt) {
+            System.out.println("Skipping over $reindex IT test");
+            return;
+        }
         List parameters = new ArrayList<>();
         parameters.add(Parameter.builder()
             .name(string("resourceLogicalId"))
@@ -348,6 +386,10 @@ public void testReindexWithSpecificTypeThatDoesntExist() {
 
     @Test(groups = { "reindex" }, dependsOnMethods = {})
     public void testReindexWithPatientResourceType() {
+        if (!runIt) {
+            System.out.println("Skipping over $reindex IT test");
+            return;
+        }
         List parameters = new ArrayList<>();
         parameters.add(Parameter.builder()
             .name(string("resourceLogicalId"))