Merge master into v5.3.0

core/src/main/scripts/importer/validateData.py

@@ -2941,7 +2941,6 @@ def checkLine(self, data):
                     self.logger.error(
                             'Value in DFS_STATUS column is not 0:DiseaseFree, '
                             '1:Recurred/Progressed, 1:Recurred, 1:Progressed',
-                            'DiseaseFree, Recurred/Progressed, Recurred or Progressed',
                             extra={'line_number': self.line_number,
                                    'column_number': col_index + 1,
                                    'cause': value})

core/src/test/resources/applicationContext-dao.xml

@@ -67,7 +67,16 @@
 		<property name="maxTotal" value="100" />
 		<property name="poolPreparedStatements" value="true" />
 	</bean>
-
+
+    <bean id="databaseIdProvider" class="org.apache.ibatis.mapping.VendorDatabaseIdProvider">
+        <property name="properties">
+            <props>
+                <prop key="MySQL">mysql</prop>
+                <prop key="H2">h2</prop>
+            </props>
+        </property>
+    </bean>
+
     <!-- define the SqlSessionFactory -->
     <bean id="sqlSessionFactory" class="org.mybatis.spring.SqlSessionFactoryBean">
         <property name="dataSource" ref="businessDataSource" />
@@ -76,6 +85,7 @@
         <!-- mapper locations is set here to support interdependency between mappers without -->
         <!-- having to autowire repositories into java classes that do not make direct uses of them -->
         <property name="mapperLocations" value="classpath*:org/cbioportal/persistence/mybatis/**/*.xml"/>
+        <property name="databaseIdProvider" ref="databaseIdProvider"/>
     </bean>
 
     <!-- scan for mappers and let them be autowired -->

docs/About-Us.md

@@ -61,10 +61,11 @@ The cBioPortal for Cancer Genomics was originally developed at [Memorial Sloan K
 
 ## Bilkent University
 * Ugur Dogrusoz
-* M Salih Altun
+* Yusuf Ziya Sengul
 
 ## Alumni
 * B Arman Aksoy
+* M Salih Altun
 * Istemi Bahceci
 * Caitlin Byrne
 * Hsiao-Wei Chen
@@ -81,10 +82,10 @@ The cBioPortal for Cancer Genomics was originally developed at [Memorial Sloan K
 * Michael Heuer
 * Anders Jacobsen
 * Karthik Kalletla
+* Peter Kok
 * Erik Larsson
 * Dong Li
 * Tamba Monrose
-* Peter Kok
 * Irina Pulyakhina
 * Pichai Raman
 * M Furkan Sahin

docs/deployment/customization/portal.properties-Reference.md

@@ -627,6 +627,30 @@ By default, the tree is expanded (property value is `false`).
 skin.geneset_hierarchy.collapse_by_default = true
 ```
 
+## Cross study expression and protein data
+
+
+By default we hide expression data for multi-study queries as they are 
+usually not normalized across studies. For the public cBioPortal for instance, only TCGA Pancan Atlas studies expression data has been normalized.  
+
+If you know the expression data in your instance is comparable, or is comparable for a subset of
+studies, you can configure a rule as follows. 
+
+The value of this property can be boolean (true|false) or a javascript function
+which executes at runtime and is passed the list of study objects being queried
+by the user and evaluates whether expression data can be safely displayed.
+
+```
+// a function that accepts the users's selected studies and
+// returns whether or not to allow expression data from the involved studies to be mixed
+enable_cross_study_expression = (selectedStudies)=>{ [your logic] return true|false }
+```
+
+```
+// boolean
+enable_cross_study_expression = true|false
+```
+
 ## Request Body Compression
 
 ### Background

model/src/main/java/org/cbioportal/model/Binnable.java

@@ -0,0 +1,13 @@
+package org.cbioportal.model;
+
+/**
+ * Data that can be binned, clinical or custom
+ */
+public interface Binnable {
+    String getAttrId();
+    String getAttrValue();
+    String getSampleId();
+    String getPatientId();
+    String getStudyId();
+    Boolean isPatientAttribute();
+}

model/src/main/java/org/cbioportal/model/ClinicalData.java

@@ -2,7 +2,7 @@
 
 import javax.validation.constraints.NotNull;
 
-public class ClinicalData extends UniqueKeyBase {
+public class ClinicalData extends UniqueKeyBase implements Binnable  {
 
     private Integer internalId;
     private String sampleId;
@@ -51,6 +51,13 @@ public String getAttrId() {
         return attrId;
     }
 
+    public Boolean isPatientAttribute() {
+        if(clinicalAttribute == null) {
+            return null;
+        }
+        return this.clinicalAttribute.getPatientAttribute();
+    }
+
     public void setAttrId(String attrId) {
         this.attrId = attrId;
     }

persistence/persistence-connections/src/main/resources/applicationContext-persistenceConnections.xml

@@ -44,7 +44,15 @@
         </bean>
 
         <bean id="customObjectFactory" class="org.cbioportal.persistence.mybatis.util.CustomMyBatisObjectFactory" />
-
+
+        <bean id="databaseIdProvider" class="org.apache.ibatis.mapping.VendorDatabaseIdProvider">
+            <property name="properties">
+                <props>
+                    <prop key="MySQL">mysql</prop>
+                    <prop key="H2">h2</prop>
+                </props>
+            </property>
+        </bean> 
         <!-- define the SqlSessionFactory -->
         <bean id="sqlSessionFactory" class="org.mybatis.spring.SqlSessionFactoryBean">
             <property name="dataSource" ref="businessDataSource" />
@@ -60,6 +68,7 @@
             <!-- mapper locations is set here to support interdependency between mappers without -->
             <!-- having to autowire repositories into java classes that do not make direct use of them -->
             <property name="mapperLocations" value="classpath*:org/cbioportal/persistence/mybatis/**/*.xml" />
+            <property name="databaseIdProvider" ref="databaseIdProvider"/>
         </bean>
     </beans>
 

persistence/persistence-mybatis/src/main/resources/org/cbioportal/persistence/mybatis/ClinicalEventMapper.xml

@@ -142,7 +142,14 @@
 
     <select id="getPatientsDistinctClinicalEventInStudies" resultType="org.cbioportal.model.ClinicalEvent">
         SELECT
-        clinical_event.CLINICAL_EVENT_ID AS clinicalEventId,
+        <choose>
+            <when test="_databaseId == 'mysql'">
+                ANY_VALUE(clinical_event.CLINICAL_EVENT_ID) AS clinicalEventId,
+            </when>
+            <when test="_databaseId == 'h2'">
+                clinical_event.CLINICAL_EVENT_ID AS clinicalEventId,
+            </when>
+        </choose>
         clinical_event.EVENT_TYPE AS eventType,
         patient.STABLE_ID AS patientId
         <include refid="from"/>

persistence/persistence-mybatis/src/test/resources/testContextDatabase.xml

@@ -45,10 +45,20 @@
         <property name="url" value="jdbc:h2:mem:;MODE=MySQL;DATABASE_TO_UPPER=false;INIT=RUNSCRIPT FROM 'classpath:cgds-test.sql'\;RUNSCRIPT FROM 'classpath:testSql.sql'"/>
     </bean>
 
+    <bean id="databaseIdProvider" class="org.apache.ibatis.mapping.VendorDatabaseIdProvider">
+        <property name="properties">
+            <props>
+                <prop key="MySQL">mysql</prop>
+                <prop key="H2">h2</prop>
+            </props>
+        </property>
+    </bean>
+
     <bean id="sqlSessionFactory" class="org.mybatis.spring.SqlSessionFactoryBean">
         <property name="dataSource" ref="dbcpDataSource"/>
         <property name="typeAliasesPackage" value="org.mskcc.cbio.portal.model"/>
         <property name="typeHandlersPackage" value="org.cbioportal.persistence.mybatis.typehandler"/>
+        <property name="databaseIdProvider" ref="databaseIdProvider"/>
     </bean>
 
     <bean id="transactionManager" class="org.springframework.jdbc.datasource.DataSourceTransactionManager">

pom.xml

@@ -276,7 +276,7 @@
   </profiles>
 
   <properties>
-    <frontend.version>v5.2.10</frontend.version>
+    <frontend.version>v5.2.11</frontend.version>
     <frontend.groupId>com.github.cbioportal</frontend.groupId>
     <spring.version>5.2.20.RELEASE</spring.version>
     <spring.integration.version>5.3.0.RELEASE</spring.integration.version>

portal/src/main/webapp/config_service.jsp

@@ -113,6 +113,7 @@
             "skin.home_page.unauthorized_studies_global_message",
             "skin.show_settings_menu",
             "skin.hide_logout_button",
+            "enable_cross_study_expression",
             "quick_search.enabled",
             "default_cross_cancer_study_session_id",
             "default_cross_cancer_study_list",

service/pom.xml

@@ -24,6 +24,29 @@
             <groupId>org.mskcc.cbio</groupId>
             <artifactId>permission-evaluator</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.github.cbioportal</groupId>
+            <artifactId>session-service</artifactId>
+            <classifier>model</classifier>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.logging.log4j</groupId>
+                    <artifactId>log4j-to-slf4j</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>ch.qos.logback</groupId>
+                    <artifactId>logback-classic</artifactId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>spring-boot-starter-web</artifactId>
+                    <groupId>org.springframework.boot</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>spring-boot-starter-data-mongodb</artifactId>
+                    <groupId>org.springframework.boot</groupId>
+                </exclusion>
+            </exclusions>
+        </dependency>
         <dependency>
             <groupId>org.springframework</groupId>
             <artifactId>spring-test</artifactId>
@@ -57,6 +80,10 @@
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-collections4</artifactId>
         </dependency>
+        <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>

service/src/main/java/org/cbioportal/service/AttributeByStudyService.java

@@ -0,0 +1,9 @@
+package org.cbioportal.service;
+
+import org.cbioportal.model.ClinicalAttribute;
+
+import java.util.List;
+
+public interface AttributeByStudyService {
+    List<ClinicalAttribute> getClinicalAttributesByStudyIdsAndAttributeIds(List<String> studyIds, List<String> attributeIds);
+}

service/src/main/java/org/cbioportal/service/ClinicalAttributeService.java

@@ -8,7 +8,7 @@
 
 import java.util.List;
 
-public interface ClinicalAttributeService {
+public interface ClinicalAttributeService extends AttributeByStudyService {
 
     List<ClinicalAttribute> getAllClinicalAttributes(String projection, Integer pageSize, Integer pageNumber,
                                                      String sortBy, String direction);
@@ -31,5 +31,4 @@ List<ClinicalAttribute> getAllClinicalAttributesInStudy(String studyId, String p
 
     List<ClinicalAttributeCount> getClinicalAttributeCountsBySampleListId(String sampleListId);
 
-    List<ClinicalAttribute> getClinicalAttributesByStudyIdsAndAttributeIds(List<String> studyIds, List<String> attributeIds);
 }

service/src/main/java/org/cbioportal/service/CustomDataService.java

@@ -0,0 +1,10 @@
+package org.cbioportal.service;
+
+import org.cbioportal.service.util.CustomDataSession;
+
+import java.util.List;
+import java.util.Map;
+
+public interface CustomDataService {
+    Map<String, CustomDataSession> getCustomDataSessions(List<String> attributes);
+}

service/src/main/java/org/cbioportal/service/impl/CustomDataServiceImpl.java

@@ -0,0 +1,58 @@
+package org.cbioportal.service.impl;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.cbioportal.service.CustomDataService;
+import org.cbioportal.service.util.CustomDataSession;
+import org.cbioportal.service.util.SessionServiceRequestHandler;
+import org.cbioportal.session_service.domain.SessionType;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
+
+@Service
+public class CustomDataServiceImpl implements CustomDataService {
+    @Autowired
+    private SessionServiceRequestHandler sessionServiceRequestHandler;
+
+    @Autowired
+    private ObjectMapper sessionServiceObjectMapper;
+
+    /**
+     * Retrieve CustomDataSession from session service for custom data attributes. 
+     * @param customAttributeIds - attribute id/hash of custom data used as session service key.
+     * @return Map of custom data attribute id to the CustomDataSession
+     */
+    @Override
+    public Map<String, CustomDataSession> getCustomDataSessions(List<String> customAttributeIds) {
+        Map<String, CompletableFuture<CustomDataSession>> postFuturesMap = customAttributeIds.stream()
+            .collect(Collectors.toMap(
+                attributeId -> attributeId,
+                attributeId -> CompletableFuture.supplyAsync(() -> {
+                    try {
+                        String customDataSessionJson = sessionServiceRequestHandler.getSessionDataJson(
+                            SessionType.custom_data,
+                            attributeId
+                        );
+                        return sessionServiceObjectMapper.readValue(customDataSessionJson, CustomDataSession.class);
+                    } catch (Exception e) {
+                        return null;
+                    }
+                })
+            ));
+
+        CompletableFuture.allOf(postFuturesMap.values().toArray(new CompletableFuture[postFuturesMap.size()])).join();
+
+        Map<String, CustomDataSession> customDataSessions = postFuturesMap.entrySet().stream()
+            .filter(entry -> entry.getValue().join() != null)
+            .collect(Collectors.toMap(
+                entry -> entry.getKey(),
+                entry -> entry.getValue().join()
+            ));
+
+        return customDataSessions;
+    }
+}

service/src/main/java/org/cbioportal/service/util/BinnableCustomDataValue.java

@@ -0,0 +1,49 @@
+package org.cbioportal.service.util;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import org.cbioportal.model.Binnable;
+
+import java.io.Serializable;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class BinnableCustomDataValue implements Binnable, Serializable {
+
+    private final CustomDataValue customDataValue;
+    private final String attrId;
+    private final Boolean patientAttribute;
+
+    public BinnableCustomDataValue(
+        CustomDataValue customDataValue,
+        String attributeId,
+        Boolean patientAttribute
+    ) {
+        this.customDataValue = customDataValue;
+        this.attrId = attributeId;
+        this.patientAttribute = patientAttribute;
+    }
+
+    public String getSampleId() {
+        return customDataValue.getSampleId();
+    }
+
+    public String getPatientId() {
+        return customDataValue.getPatientId();
+    }
+
+    public String getStudyId() {
+        return customDataValue.getStudyId();
+    }
+
+    public String getAttrId() {
+        return attrId;
+    }
+
+    public Boolean isPatientAttribute() {
+        return patientAttribute;
+    }
+
+    public String getAttrValue() {
+        return customDataValue.getValue();
+    }
+
+}