Clickhouse implementation of study view

core/src/main/java/org/mskcc/cbio/portal/dao/ClickhouseJdbcDataSource.java

@@ -0,0 +1,18 @@
+package org.mskcc.cbio.portal.dao;
+
+import org.apache.commons.dbcp2.BasicDataSource;
+import org.mskcc.cbio.portal.util.GlobalProperties;
+
+public class ClickhouseJdbcDataSource extends BasicDataSource {
+
+ public ClickhouseJdbcDataSource() {
+ String connectionString = GlobalProperties.getProperty("db.clickhouse.connection_string");
+ String user = GlobalProperties.getProperty("db.clickhouse.user");
+ String password = GlobalProperties.getProperty("db.clickhouse.password");
+ this.setDriverClassName("com.clickhouse.jdbc.ClickHouseDriver");
+ this.setUsername(user);
+ this.setPassword(password);
+ this.setUrl(connectionString);
+ this.setJmxName("org.cbioportal:DataSource=clickhouse");
+ }
+}

core/src/test/resources/applicationContext-dao.xml

@@ -80,7 +80,7 @@
 
  <!-- scan for mappers and let them be autowired -->
  <bean class="org.mybatis.spring.mapper.MapperScannerConfigurer">
- <property name="basePackage" value="org.mskcc.cbio.portal.persistence,org.cbioportal.persistence.mybatis" />
+ <property name="basePackage" value="org.mskcc.cbio.portal.persistence,org.cbioportal.persistence.mybatis,org.cbioportal.persistence.mybatiscolumnstore" />
  </bean>
 
  <!-- enable component scanning (beware that this does not enable mapper scanning!) -->

docs/user-guide/faq.md

@@ -302,7 +302,7 @@ filterJson is set in the url hash string. Here are the allowed parameters and fo
  "value": "string"
  }]
  }],
- "geneFilters": [{
+ "studyViewGeneFilters": [{
  "geneQueries": [
  ["string"]
  ],

persistence/persistence-api/pom.xml

@@ -16,6 +16,10 @@
  <groupId>org.mskcc.cbio</groupId>
  <artifactId>model</artifactId>
  </dependency>
+ <dependency>
+ <groupId>org.mskcc.cbio</groupId>
+ <artifactId>webparam</artifactId>
+ </dependency>
  <dependency>
  <groupId>org.mskcc.cbio</groupId>
  <artifactId>utils</artifactId>
@@ -37,6 +41,10 @@
  <groupId>commons-io</groupId>
  <artifactId>commons-io</artifactId>
  </dependency>
+ <dependency>
+ <groupId>org.mskcc.cbio</groupId>
+ <artifactId>webparam</artifactId>
+ </dependency>
  </dependencies>
 
  <build>

persistence/persistence-api/src/main/java/org/cbioportal/persistence/DiscreteCopyNumberRepository.java

@@ -2,8 +2,8 @@
 
 import org.cbioportal.model.CopyNumberCountByGene;
 import org.cbioportal.model.DiscreteCopyNumberData;
-import org.cbioportal.model.GeneFilterQuery;
 import org.cbioportal.model.meta.BaseMeta;
+import org.cbioportal.webparam.GeneFilterQuery;
 import org.springframework.cache.annotation.Cacheable;
 
 import java.util.List;

persistence/persistence-api/src/main/java/org/cbioportal/persistence/MutationRepository.java

@@ -1,9 +1,9 @@
 package org.cbioportal.persistence;
 
-import org.cbioportal.model.GeneFilterQuery;
 import org.cbioportal.model.Mutation;
 import org.cbioportal.model.MutationCountByPosition;
 import org.cbioportal.model.meta.MutationMeta;
+import org.cbioportal.webparam.GeneFilterQuery;
 import org.springframework.cache.annotation.Cacheable;
 
 import java.util.List;

persistence/persistence-api/src/main/java/org/cbioportal/persistence/StructuralVariantRepository.java

@@ -2,9 +2,9 @@
 
 import java.util.List;
 
-import org.cbioportal.model.GeneFilterQuery;
 import org.cbioportal.model.StructuralVariant;
 import org.cbioportal.model.StructuralVariantQuery;
+import org.cbioportal.webparam.GeneFilterQuery;
 import org.springframework.cache.annotation.Cacheable;
 
 public interface StructuralVariantRepository {

persistence/persistence-api/src/main/java/org/cbioportal/persistence/StudyViewRepository.java

@@ -0,0 +1,31 @@
+package org.cbioportal.persistence;
+
+import org.cbioportal.model.AlterationCountByGene;
+import org.cbioportal.model.ClinicalData;
+import org.cbioportal.model.ClinicalDataCount;
+import org.cbioportal.model.Sample;
+import org.cbioportal.persistence.enums.ClinicalAttributeDataSource;
+import org.cbioportal.persistence.enums.ClinicalAttributeDataType;
+import org.cbioportal.webparam.CategorizedClinicalDataCountFilter;
+import org.cbioportal.webparam.StudyViewFilter;
+
+import java.util.List;
+
+public interface StudyViewRepository {
+ List<Sample> getFilteredSamplesFromColumnstore(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter);
+
+ List<ClinicalData> getSampleClinicalDataFromStudyViewFilter(StudyViewFilter studyViewFilter, List<String> attributeIds, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter);
+
+ List<ClinicalData> getPatientClinicalDataFromStudyViewFilter(StudyViewFilter studyViewFilter, List<String> attributeIds, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter);
+
+ List<AlterationCountByGene> getMutatedGenes(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter);
+
+ List<ClinicalDataCount> getClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, List<String> filteredAttributes);
+
+ List<ClinicalDataCount> getSampleClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, List<String> filteredAttributes);
+
+ List<ClinicalDataCount> getPatientClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, List<String> filteredAttributes);
+
+ List<String> getClinicalDataAttributeNames(ClinicalAttributeDataSource clinicalAttributeDataSource, ClinicalAttributeDataType dataType);
+
+}

persistence/persistence-api/src/main/java/org/cbioportal/persistence/enums/ClinicalAttributeDataSource.java

@@ -0,0 +1,15 @@
+package org.cbioportal.persistence.enums;
+
+public enum ClinicalAttributeDataSource {
+ PATIENT("PATIENT"),SAMPLE("SAMPLE");
+
+ private final String value;
+
+ ClinicalAttributeDataSource(String value) {
+ this.value = value;
+ }
+
+ public String getValue() {
+ return this.value;
+ }
+}

persistence/persistence-api/src/main/java/org/cbioportal/persistence/enums/ClinicalAttributeDataType.java

@@ -0,0 +1,16 @@
+package org.cbioportal.persistence.enums;
+
+public enum ClinicalAttributeDataType {
+ CATEGORICAL("CATEGORICAL"),
+ NUMERIC("NUMERIC");
+
+ private final String value;
+
+ ClinicalAttributeDataType(String value) {
+ this.value = value;
+ }
+
+ public String getValue() {
+ return this.value;
+ }
+}

persistence/persistence-connections/src/main/resources/applicationContext-persistenceConnections.xml

@@ -24,7 +24,7 @@
 
  <!-- scan for mappers and let them be autowired -->
  <bean class="org.mybatis.spring.mapper.MapperScannerConfigurer">
- <property name="basePackage" value="org.mskcc.cbio.portal.persistence,org.cbioportal.persistence.mybatis" />
+ <property name="basePackage" value="org.mskcc.cbio.portal.persistence,org.cbioportal.persistence.mybatis,org.cbioportal.persistence.mybatiscolumnstore" />
  </bean>
 
  <!-- enable component scanning (beware that this does not enable mapper scanning!) -->
@@ -42,11 +42,15 @@
  <bean id="transactionManager" class="org.springframework.jdbc.datasource.DataSourceTransactionManager">
  <property name="dataSource" ref="businessDataSource" />
  </bean>
+
+ <bean id="columnstoreTransactionManager" class="org.springframework.jdbc.datasource.DataSourceTransactionManager">
+ <property name="dataSource" ref="columnstoreDataSource" />
+ </bean>
 
  <bean id="customObjectFactory" class="org.cbioportal.persistence.mybatis.util.CustomMyBatisObjectFactory" />
 
  <!-- define the SqlSessionFactory -->
- <bean id="sqlSessionFactory" class="org.mybatis.spring.SqlSessionFactoryBean">
+ <bean id="sqlSessionFactory" class="org.mybatis.spring.SqlSessionFactoryBean" primary="true">
  <property name="dataSource" ref="businessDataSource" />
  <property name="typeAliasesPackage" value="org.mskcc.cbio.portal.model" />
  <property name="typeHandlersPackage" value="org.cbioportal.persistence.mybatis.typehandler" />
@@ -61,6 +65,19 @@
  <!-- having to autowire repositories into java classes that do not make direct use of them -->
  <property name="mapperLocations" value="classpath*:org/cbioportal/persistence/mybatis/**/*.xml" />
  </bean>
+ <bean id="columnstoreSqlSessionFactory" class="org.mybatis.spring.SqlSessionFactoryBean">
+ <property name="dataSource" ref="columnstoreDataSource" />
+ <property name="typeAliasesPackage" value="org.mskcc.cbio.portal.model" />
+ <property name="typeHandlersPackage" value="org.cbioportal.persistence.mybatis.typehandler" />
+ <property name="objectFactory" ref="customObjectFactory" />
+ <!-- mapper locations is set here to support interdependency between mappers without -->
+ <!-- having to autowire repositories into java classes that do not make direct use of them -->
+ <!-- <property name="mapperLocations" value="classpath*:org/cbioportal/persistence/mybatiscolumnstore/*.xml" />-->
+ </bean>
+ <bean id="studyViewMapper" class="org.mybatis.spring.mapper.MapperFactoryBean">
+ <property name="mapperInterface" value="org.cbioportal.persistence.mybatiscolumnstore.StudyViewMapper" />
+ <property name="sqlSessionFactory" ref="columnstoreSqlSessionFactory" />
+ </bean>
  </beans>
 
  <beans profile="dbcp">
@@ -72,6 +89,8 @@
  <list><ref bean="businessDataSource"/></list>
  </property>
  </bean>
+ <bean id="columnstoreDataSource" destroy-method="close"
+ class="org.mskcc.cbio.portal.dao.ClickhouseJdbcDataSource" />
  </beans>
 
  <beans profile="jndi">

persistence/persistence-mybatis/pom.xml

@@ -7,6 +7,18 @@
  PROJECT_VERSION env variable (see version.sh) -->
  <version>0-unknown-version-SNAPSHOT</version>
  </parent>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>9</source>
+ <target>9</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
  <modelVersion>4.0.0</modelVersion>
 
  <artifactId>persistence-mybatis</artifactId>
@@ -58,5 +70,9 @@
  <groupId>org.mockito</groupId>
  <artifactId>mockito-core</artifactId>
  </dependency>
+ <dependency>
+ <groupId>org.mskcc.cbio</groupId>
+ <artifactId>webparam</artifactId>
+ </dependency>
  </dependencies>
 </project>

persistence/persistence-mybatis/src/main/java/org/cbioportal/persistence/mybatis/DiscreteCopyNumberMapper.java

@@ -2,7 +2,7 @@
 
 import org.cbioportal.model.CopyNumberCountByGene;
 import org.cbioportal.model.DiscreteCopyNumberData;
-import org.cbioportal.model.GeneFilterQuery;
+import org.cbioportal.webparam.GeneFilterQuery;
 import org.cbioportal.model.meta.BaseMeta;
 
 import java.util.List;

persistence/persistence-mybatis/src/main/java/org/cbioportal/persistence/mybatis/DiscreteCopyNumberMyBatisRepository.java

@@ -2,7 +2,7 @@
 
 import org.cbioportal.model.CopyNumberCountByGene;
 import org.cbioportal.model.DiscreteCopyNumberData;
-import org.cbioportal.model.GeneFilterQuery;
+import org.cbioportal.webparam.GeneFilterQuery;
 import org.cbioportal.model.meta.BaseMeta;
 import org.cbioportal.persistence.DiscreteCopyNumberRepository;
 import org.springframework.beans.factory.annotation.Autowired;

persistence/persistence-mybatis/src/main/java/org/cbioportal/persistence/mybatis/MutationMapper.java

@@ -1,6 +1,6 @@
 package org.cbioportal.persistence.mybatis;
 
-import org.cbioportal.model.GeneFilterQuery;
+import org.cbioportal.webparam.GeneFilterQuery;
 import org.cbioportal.model.Mutation;
 import org.cbioportal.model.MutationCountByPosition;
 import org.cbioportal.model.meta.MutationMeta;

persistence/persistence-mybatis/src/main/java/org/cbioportal/persistence/mybatis/MutationMyBatisRepository.java

@@ -1,6 +1,6 @@
 package org.cbioportal.persistence.mybatis;
 
-import org.cbioportal.model.GeneFilterQuery;
+import org.cbioportal.webparam.GeneFilterQuery;
 import org.cbioportal.model.Mutation;
 import org.cbioportal.model.MutationCountByPosition;
 import org.cbioportal.model.meta.MutationMeta;

persistence/persistence-mybatis/src/main/java/org/cbioportal/persistence/mybatis/StructuralVariantMapper.java

@@ -25,9 +25,9 @@
 
 import java.util.List;
 
-import org.cbioportal.model.GeneFilterQuery;
 import org.cbioportal.model.StructuralVariant;
 import org.cbioportal.model.StructuralVariantQuery;
+import org.cbioportal.webparam.GeneFilterQuery;
 
 public interface StructuralVariantMapper {
 

persistence/persistence-mybatis/src/main/java/org/cbioportal/persistence/mybatis/StructuralVariantMyBatisRepository.java

@@ -23,7 +23,7 @@
 
 package org.cbioportal.persistence.mybatis;
 
-import org.cbioportal.model.GeneFilterQuery;
+import org.cbioportal.webparam.GeneFilterQuery;
 import org.cbioportal.model.StructuralVariant;
 import org.cbioportal.model.StructuralVariantQuery;
 import org.cbioportal.persistence.StructuralVariantRepository;

persistence/persistence-mybatis/src/main/java/org/cbioportal/persistence/mybatiscolumnstore/StudyViewMapper.java

@@ -0,0 +1,31 @@
+package org.cbioportal.persistence.mybatiscolumnstore;
+
+import org.cbioportal.model.AlterationCountByGene;
+import org.cbioportal.model.ClinicalData;
+import org.cbioportal.model.ClinicalDataCount;
+import org.cbioportal.model.Sample;
+import org.cbioportal.webparam.CategorizedClinicalDataCountFilter;
+import org.cbioportal.webparam.StudyViewFilter;
+
+import java.util.List;
+
+public interface StudyViewMapper {
+ List<Sample> getFilteredSamples(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, boolean applyPatientIdFilters);
+
+ List<AlterationCountByGene> getMutatedGenes(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, boolean applyPatientIdFilters);
+
+ List<ClinicalDataCount> getPatientClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter,
+ boolean applyPatientIdFilters, List<String> attributeIds, List<String> filteredAttributeValues);
+
+ List<ClinicalDataCount> getSampleClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter,
+ boolean applyPatientIdFilters, List<String> attributeIds, List<String> filteredAttributeValues );
+
+ List<ClinicalDataCount> getClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter,
+ boolean applyPatientIdFilters, List<String> attributeIds, List<String> filteredAttributeValues);
+
+ List<String> getClinicalAttributeNames(String tableName);
+
+ List<ClinicalData> getSampleClinicalDataFromStudyViewFilter(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, boolean applyPatientIdFilters, List<String> attributeIds);
+
+ List<ClinicalData> getPatientClinicalDataFromStudyViewFilter(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, boolean applyPatientIdFilters, List<String> attributeIds);
+}

persistence/persistence-mybatis/src/main/java/org/cbioportal/persistence/mybatiscolumnstore/StudyViewMyBatisRepository.java

@@ -0,0 +1,71 @@
+package org.cbioportal.persistence.mybatiscolumnstore;
+
+import org.cbioportal.model.AlterationCountByGene;
+import org.cbioportal.model.ClinicalData;
+import org.cbioportal.model.ClinicalDataCount;
+import org.cbioportal.model.Sample;
+import org.cbioportal.persistence.StudyViewRepository;
+import org.cbioportal.persistence.enums.ClinicalAttributeDataSource;
+import org.cbioportal.persistence.enums.ClinicalAttributeDataType;
+import org.cbioportal.webparam.CategorizedClinicalDataCountFilter;
+import org.cbioportal.webparam.StudyViewFilter;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Repository;
+
+import java.util.List;
+
+@Repository
+public class StudyViewMyBatisRepository implements StudyViewRepository {
+
+ private final List<String> FILTERED_CLINICAL_ATTR_VALUES = List.of("NA", "NAN", "N/A");
+ @Autowired
+ private StudyViewMapper studyViewMapper;
+
+ @Override
+ public List<Sample> getFilteredSamplesFromColumnstore(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) {
+
+ return studyViewMapper.getFilteredSamples(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter));
+ }
+
+ @Override
+ public List<AlterationCountByGene> getMutatedGenes(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) {
+ return studyViewMapper.getMutatedGenes(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter));
+ }
+
+ @Override
+ public List<ClinicalDataCount> getClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, List<String> filteredAttributes) {
+ return studyViewMapper.getClinicalDataCounts(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter),
+ filteredAttributes, FILTERED_CLINICAL_ATTR_VALUES );
+ }
+
+ @Override
+ public List<ClinicalDataCount> getSampleClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, List<String> filteredAttributes) {
+ return studyViewMapper.getSampleClinicalDataCounts(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter),
+ filteredAttributes, FILTERED_CLINICAL_ATTR_VALUES );
+ }
+
+ @Override
+ public List<ClinicalDataCount> getPatientClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, List<String> filteredAttributes) {
+ return studyViewMapper.getPatientClinicalDataCounts(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter),
+ filteredAttributes, FILTERED_CLINICAL_ATTR_VALUES);
+ }
+
+ @Override
+ public List<String> getClinicalDataAttributeNames(ClinicalAttributeDataSource clinicalAttributeDataSource, ClinicalAttributeDataType dataType) {
+ String tableName = clinicalAttributeDataSource.getValue().toLowerCase() + "_clinical_attribute_" + dataType.getValue().toLowerCase();
+ return studyViewMapper.getClinicalAttributeNames(tableName);
+ }
+
+ private boolean shouldApplyPatientIdFilters(CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) {
+ return categorizedClinicalDataCountFilter.getPatientCategoricalClinicalDataFilters() != null && !categorizedClinicalDataCountFilter.getPatientCategoricalClinicalDataFilters().isEmpty()
+ || categorizedClinicalDataCountFilter.getPatientNumericalClinicalDataFilters() != null && !categorizedClinicalDataCountFilter.getPatientNumericalClinicalDataFilters().isEmpty();
+ }
+
+ public List<ClinicalData> getSampleClinicalDataFromStudyViewFilter(StudyViewFilter studyViewFilter, List<String> attributeIds, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) {
+ return studyViewMapper.getSampleClinicalDataFromStudyViewFilter(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter), attributeIds);
+ }
+
+ public List<ClinicalData> getPatientClinicalDataFromStudyViewFilter(StudyViewFilter studyViewFilter, List<String> attributeIds, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) {
+ return studyViewMapper.getPatientClinicalDataFromStudyViewFilter(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter), attributeIds);
+ }
+}