Issues 1921, 1923, and 2499 - whole-system search updates and related changes

fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/Select.java

@@ -9,6 +9,8 @@
 import static com.ibm.fhir.database.utils.query.SqlConstants.FROM;
 import static com.ibm.fhir.database.utils.query.SqlConstants.SELECT;
 import static com.ibm.fhir.database.utils.query.SqlConstants.SPACE;
+import static com.ibm.fhir.database.utils.query.SqlConstants.UNION;
+import static com.ibm.fhir.database.utils.query.SqlConstants.UNION_ALL;
 
 import com.ibm.fhir.database.utils.api.IDatabaseTranslator;
 import com.ibm.fhir.database.utils.derby.DerbyTranslator;
@@ -48,6 +50,12 @@ public class Select {
     // offset/limit for pagination
     private PaginationClause paginationClause;
 
+    // Another Select to UNION with this select. Optional
+    private Select union;
+
+    // If true, the specified UNION is a UNION ALL
+    private boolean unionAll = false;
+
     /**
      * Default constructor. Not a DISTINCT select.
      */
@@ -218,6 +226,11 @@ public String toString() {
             result.append(SPACE).append(this.paginationClause.toString());
         }
 
+        if (this.union != null) {
+            result.append(SPACE).append(unionAll ? UNION_ALL : UNION).append(SPACE)
+                    .append(this.union.toString());
+        }
+
         return result.toString();
     }
 
@@ -250,7 +263,8 @@ public String toDebugString() {
      * @return
      */
     public <T> T render(StatementRenderer<T> renderer) {
-        return renderer.select(distinct, selectList, fromClause, whereClause, groupByClause, havingClause, orderByClause, paginationClause);
+        return renderer.select(distinct, selectList, fromClause, whereClause, groupByClause, havingClause,
+            orderByClause, paginationClause, unionAll, union);
     }
 
     /**
@@ -319,4 +333,20 @@ public void addPagination(int offset, int rowsPerPage) {
     public OrderByClause getOrderByClause() {
         return this.orderByClause;
     }
+
+    /**
+     * Set a select to UNION with this query.
+     */
+    public void setUnion(Select union) {
+        this.union = union;
+        this.unionAll = false;
+    }
+
+    /**
+     * Set a select to UNION ALL with this query.
+     */
+    public void setUnionAll(Select unionAll) {
+        this.union = unionAll;
+        this.unionAll = true;
+    }
 }

fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/SelectAdapter.java

@@ -123,6 +123,26 @@ public void pagination(int offset, int rowsPerPage) {
         select.addPagination(offset, rowsPerPage);
     }
 
+    /**
+     * Add a select via UNION
+     *
+     * @param unionSelect the select to be UNION'd to this select statement
+     * @return
+     */
+    public void union(Select unionSelect) {
+        select.setUnion(unionSelect);
+    }
+
+    /**
+     * Add a select via UNION ALL
+     *
+     * @param unionAllSelect the select to be UNION ALL'd to this select statement
+     * @return
+     */
+    public void unionAll(Select unionAllSelect) {
+        select.setUnionAll(unionAllSelect);
+    }
+
     /**
      * Get the statement we've been constructing
      *

fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/SqlConstants.java

@@ -25,4 +25,6 @@ public class SqlConstants {
     public static final String THEN = "THEN";
     public static final String END = "END";
     public static final String ORDER_BY = "ORDER BY";
+    public static final String UNION = "UNION";
+    public static final String UNION_ALL = "UNION ALL";
 }

fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/expression/StatementRenderer.java

@@ -14,6 +14,7 @@
 import com.ibm.fhir.database.utils.query.HavingClause;
 import com.ibm.fhir.database.utils.query.OrderByClause;
 import com.ibm.fhir.database.utils.query.PaginationClause;
+import com.ibm.fhir.database.utils.query.Select;
 import com.ibm.fhir.database.utils.query.SelectList;
 import com.ibm.fhir.database.utils.query.WhereClause;
 import com.ibm.fhir.database.utils.query.node.ExpNode;
@@ -36,10 +37,12 @@ public interface StatementRenderer<T> {
      * @param havingClause
      * @param orderByClause
      * @param paginationClause
+     * @param unionAll
+     * @param union
      * @return
      */
     T select(boolean distinct, SelectList selectList, FromClause fromClause, WhereClause whereClause, GroupByClause groupByClause, HavingClause havingClause,
-        OrderByClause orderByClause, PaginationClause paginationClause);
+        OrderByClause orderByClause, PaginationClause paginationClause, boolean unionAll, Select union);
 
     /**
      * @param items

fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/expression/StringStatementRenderer.java

@@ -9,6 +9,8 @@
 import static com.ibm.fhir.database.utils.query.SqlConstants.FROM;
 import static com.ibm.fhir.database.utils.query.SqlConstants.SELECT;
 import static com.ibm.fhir.database.utils.query.SqlConstants.SPACE;
+import static com.ibm.fhir.database.utils.query.SqlConstants.UNION;
+import static com.ibm.fhir.database.utils.query.SqlConstants.UNION_ALL;
 import static com.ibm.fhir.database.utils.query.SqlConstants.WHERE;
 
 import java.util.List;
@@ -21,6 +23,7 @@
 import com.ibm.fhir.database.utils.query.HavingClause;
 import com.ibm.fhir.database.utils.query.OrderByClause;
 import com.ibm.fhir.database.utils.query.PaginationClause;
+import com.ibm.fhir.database.utils.query.Select;
 import com.ibm.fhir.database.utils.query.SelectList;
 import com.ibm.fhir.database.utils.query.WhereClause;
 import com.ibm.fhir.database.utils.query.node.BindMarkerNode;
@@ -59,7 +62,7 @@ public StringStatementRenderer(IDatabaseTranslator translator, List<BindMarkerNo
 
     @Override
     public String select(boolean distinct, SelectList selectList, FromClause fromClause, WhereClause whereClause, GroupByClause groupByClause, HavingClause havingClause,
-        OrderByClause orderByClause, PaginationClause paginationClause) {
+        OrderByClause orderByClause, PaginationClause paginationClause, boolean unionAll, Select union) {
 
         StringExpNodeVisitor whereClauseRenderer = new StringExpNodeVisitor(this.translator, this.collectBindMarkersInto, this.pretty);
 
@@ -115,6 +118,14 @@ public String select(boolean distinct, SelectList selectList, FromClause fromCla
             result.append(SPACE).append(paginationClause.getSqlString(this.translator));
         }
 
+        if (union != null) {
+            if (this.pretty) {
+                result.append(NEWLINE).append("     "); // 5 spaces
+            }
+            result.append(SPACE).append(unionAll ? UNION_ALL : UNION)
+                .append(SPACE).append(union.render(this));
+        }
+
         return result.toString();
     }
 

fhir-database-utils/src/test/java/com/ibm/fhir/database/utils/query/SearchQueryTest.java

@@ -18,6 +18,7 @@
 import java.math.BigDecimal;
 import java.time.Instant;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
 import org.testng.annotations.Test;
@@ -420,4 +421,50 @@ public void bindBigDecimal(BigDecimal value) {
         };
         bindMarker.visit(v);
     }
+
+    /**
+     * Simple union all of select statements
+     */
+    @Test
+    public void unionAllTest() {
+
+        List<String> resourceTypes = Arrays.asList("Patient", "Condition", "Observation");
+        Select first = null;
+        Select previous = null;
+
+        // Create a set of selects combined by UNION ALL
+        for (String resourceType : resourceTypes) {
+            // Create a simple select statement
+            Select select = Select.select("1")
+                    .from(resourceType + "_TOKEN_VALUES_V", alias("param"))
+                    .where("param", "PARAMETER_NAME_ID").eq(1274)
+                    .build();
+
+            // Link to previous select via UNION ALL
+            if (previous != null) {
+                previous.setUnionAll(select);
+            } else {
+                first = select;
+            }
+            previous = select;
+        }
+
+        // And make sure it renders to the correct string
+        final String SQL = "SELECT 1"
+                + " FROM Patient_TOKEN_VALUES_V AS param"
+                + " WHERE param.PARAMETER_NAME_ID = 1274"
+                + " UNION ALL"
+                + " SELECT 1"
+                + " FROM Condition_TOKEN_VALUES_V AS param"
+                + " WHERE param.PARAMETER_NAME_ID = 1274"
+                + " UNION ALL"
+                + " SELECT 1"
+                + " FROM Observation_TOKEN_VALUES_V AS param"
+                + " WHERE param.PARAMETER_NAME_ID = 1274";
+        final List<BindMarkerNode> bindMarkers = new ArrayList<>();
+        StringStatementRenderer renderer = new StringStatementRenderer(TRANSLATOR, bindMarkers, false);
+        assertEquals(first.render(renderer), SQL);
+        assertEquals(bindMarkers.size(), 0);
+    }
+
 }

fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRPersistenceJDBCCache.java

@@ -7,6 +7,7 @@
 package com.ibm.fhir.persistence.jdbc;
 
 import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache;
+import com.ibm.fhir.persistence.jdbc.dao.api.IIdNameCache;
 import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache;
 
 /**
@@ -34,6 +35,12 @@ public interface FHIRPersistenceJDBCCache {
      */
     INameIdCache<Integer> getResourceTypeCache();
 
+    /**
+     * Getter for the cache of resource type ids used to look up resource type name
+     * @return
+     */
+    IIdNameCache<Integer> getResourceTypeNameCache();
+
     /**
      * Getter for the cache of parameter names
      * @return

fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/JDBCConstants.java

@@ -47,11 +47,6 @@ public class JDBCConstants {
     public static final String IS_DELETED_NO = "IS_DELETED = 'N'";
     public static final String IS_DELETED = "IS_DELETED";
 
-    // Special parameter names
-    public static final String PARAM_NAME_PROFILE = "_profile";
-    public static final String PARAM_NAME_SECURITY = "_security";
-    public static final String PARAM_NAME_TAG = "_tag";
-
     // Generic SQL query string constants
     public static final String DOT = ".";
     public static final char DOT_CHAR = '.';

fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheImpl.java

@@ -11,6 +11,7 @@
 
 import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache;
 import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache;
+import com.ibm.fhir.persistence.jdbc.dao.api.IIdNameCache;
 import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache;
 
 /**
@@ -21,6 +22,8 @@ public class FHIRPersistenceJDBCCacheImpl implements FHIRPersistenceJDBCCache {
 
     private final INameIdCache<Integer> resourceTypeCache;
 
+    private final IIdNameCache<Integer> resourceTypeNameCache;
+
     private final INameIdCache<Integer> parameterNameCache;
 
     private final ICommonTokenValuesCache resourceReferenceCache;
@@ -31,11 +34,14 @@ public class FHIRPersistenceJDBCCacheImpl implements FHIRPersistenceJDBCCache {
     /**
      * Public constructor
      * @param resourceTypeCache
+     * @param resourceTypeNameCache
      * @param parameterNameCache
      * @param resourceReferenceCache
      */
-    public FHIRPersistenceJDBCCacheImpl(INameIdCache<Integer> resourceTypeCache, INameIdCache<Integer> parameterNameCache, ICommonTokenValuesCache resourceReferenceCache) {
+    public FHIRPersistenceJDBCCacheImpl(INameIdCache<Integer> resourceTypeCache, IIdNameCache<Integer> resourceTypeNameCache,
+            INameIdCache<Integer> parameterNameCache, ICommonTokenValuesCache resourceReferenceCache) {
         this.resourceTypeCache = resourceTypeCache;
+        this.resourceTypeNameCache = resourceTypeNameCache;
         this.parameterNameCache = parameterNameCache;
         this.resourceReferenceCache = resourceReferenceCache;
     }
@@ -47,11 +53,22 @@ public ICommonTokenValuesCache getResourceReferenceCache() {
         return resourceReferenceCache;
     }
 
+    /**
+     * @return the resourceTypeCache
+     */
     @Override
     public INameIdCache<Integer> getResourceTypeCache() {
         return this.resourceTypeCache;
     }
 
+    /**
+     * @return the resourceTypeNameCache
+     */
+    @Override
+    public IIdNameCache<Integer> getResourceTypeNameCache() {
+        return this.resourceTypeNameCache;
+    }
+
     /**
      * @return the parameterNameCache
      */
@@ -63,6 +80,7 @@ public INameIdCache<Integer> getParameterNameCache() {
     public void transactionCommitted() {
         logger.fine("Transaction committed - updating cache shared maps");
         resourceTypeCache.updateSharedMaps();
+        resourceTypeNameCache.updateSharedMaps();
         parameterNameCache.updateSharedMaps();
         resourceReferenceCache.updateSharedMaps();
     }
@@ -71,6 +89,7 @@ public void transactionCommitted() {
     public void transactionRolledBack() {
         logger.fine("Transaction rolled back - clearing local maps");
         resourceTypeCache.clearLocalMaps();
+        resourceTypeNameCache.clearLocalMaps();
         parameterNameCache.clearLocalMaps();
         resourceReferenceCache.clearLocalMaps();
     }

fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheUtil.java

@@ -1,12 +1,13 @@
 /*
- * (C) Copyright IBM Corp. 2020
+ * (C) Copyright IBM Corp. 2020, 2021
  *
  * SPDX-License-Identifier: Apache-2.0
  */
 
 package com.ibm.fhir.persistence.jdbc.cache;
 
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
 import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache;
@@ -25,8 +26,8 @@ public class FHIRPersistenceJDBCCacheUtil {
      */
     public static FHIRPersistenceJDBCCache create(int codeSystemCacheSize, int tokenValueCacheSize, int canonicalCacheSize) {
         ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(codeSystemCacheSize, tokenValueCacheSize, canonicalCacheSize);
-        return new FHIRPersistenceJDBCCacheImpl(new NameIdCache<Integer>(), new NameIdCache<Integer>(), rrc);
-
+        return new FHIRPersistenceJDBCCacheImpl(new NameIdCache<Integer>(), new IdNameCache<Integer>(), new NameIdCache<Integer>(), rrc);
+        
     }
     /**
      * Prefill the cache with constants already committed in the database
@@ -36,6 +37,9 @@ public static FHIRPersistenceJDBCCache create(int codeSystemCacheSize, int token
     public static void prefill(ResourceDAO resourceDAO, ParameterDAO parameterDAO, FHIRPersistenceJDBCCache cache) throws FHIRPersistenceException {
         Map<String,Integer> resourceTypes = resourceDAO.readAllResourceTypeNames();
         cache.getResourceTypeCache().prefill(resourceTypes);
+
+        Map<Integer,String> resourceTypeNames = resourceTypes.entrySet().stream().collect(Collectors.toMap(map -> map.getValue(), map -> map.getKey()));
+        cache.getResourceTypeNameCache().prefill(resourceTypeNames);
 
         Map<String,Integer> parameterNames = parameterDAO.readAllSearchParameterNames();
         cache.getParameterNameCache().prefill(parameterNames);