Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add samples field to custom filters to allow fetching samples without session #10998

Merged
merged 7 commits into from
Oct 2, 2024
Merged

Add samples field to custom filters to allow fetching samples without session #10998

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
faee857
add samples field and handling to custom filters
Sep 17, 2024
7fbf836
refactor custom data filter util methods
Sep 20, 2024
90bc25b
remove attributeId from customSampleIdentifiers
Sep 23, 2024
2cc0547
remove unneeded imports and additional clean up
Sep 23, 2024
18926f4
fix sonar issues
Sep 25, 2024
8cdd137
edit specs to embed assertResponse in each test
Sep 26, 2024
3ded818
Fix build error
fuzhaoyuan Oct 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
123 changes: 120 additions & 3 deletions src/main/java/org/cbioportal/web/columnar/util/CustomDataFilterUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.util.Objects;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;

@Component
public class CustomDataFilterUtil {
Expand All @@ -41,12 +42,28 @@ public List<CustomSampleIdentifier> extractCustomDataSamples(final StudyViewFilt
return null;
}

List<CustomSampleIdentifier> customSampleIdentifiers = new ArrayList<>();

if (CollectionUtils.isEmpty(studyViewFilter.getCustomDataFilters())) {
return null;
}

final List<CustomSampleIdentifier> customSamplesFromProperty = studyViewFilter.getCustomDataFilters().stream()
.flatMap(filter -> {
List<CustomSampleIdentifier> samples = filter.getSamples();
return (samples != null) ? samples.stream() : Stream.empty();
})
.toList();

if (!customSamplesFromProperty.isEmpty()) {
return extractCustomDataSamplesWithoutSession(studyViewFilter, customSamplesFromProperty);
}
else {
return extractCustomDataSamplesWithSession(studyViewFilter);
}
}

private List<CustomSampleIdentifier> extractCustomDataSamplesWithSession(final StudyViewFilter studyViewFilter) {
List<CustomSampleIdentifier> customSampleIdentifiers = new ArrayList<>();

final List<String> attributeIds = studyViewFilter.getCustomDataFilters().stream()
.map(ClinicalDataFilter::getAttributeId)
.collect(Collectors.toList());
Expand Down Expand Up @@ -74,7 +91,6 @@ public List<CustomSampleIdentifier> extractCustomDataSamples(final StudyViewFilt
CustomSampleIdentifier customSampleIdentifier = new CustomSampleIdentifier();
customSampleIdentifier.setStudyId(datum.getStudyId());
customSampleIdentifier.setSampleId(datum.getSampleId());
customSampleIdentifier.setAttributeId(customDataSession.getId());
customSampleIdentifiers.add(customSampleIdentifier);
customDataByStudySampleSession.put(datum.getStudyId(), datum.getSampleId(), customDataSession.getId(), value);
})
Expand Down Expand Up @@ -119,6 +135,74 @@ public List<CustomSampleIdentifier> extractCustomDataSamples(final StudyViewFilt

return filtered;
}

private List<CustomSampleIdentifier> extractCustomDataSamplesWithoutSession(final StudyViewFilter studyViewFilter, List<CustomSampleIdentifier> customSamplesFromProperty) {
List<CustomSampleIdentifier> customSampleIdentifiers = new ArrayList<>(customSamplesFromProperty);

List<ClinicalDataFilter> equalityFilters = new ArrayList<>();
List<ClinicalDataFilter> intervalFilters = new ArrayList<>();

studyViewFilter.getCustomDataFilters().forEach(filter -> {
if (filter.getDatatype()
.equals(CustomDatatype.STRING.name())
) {
equalityFilters.add(filter);
} else {
intervalFilters.add(filter);
}
});

MultiKeyMap<String, String> customDataByStudySampleName = new MultiKeyMap<>();

studyViewFilter.getCustomDataFilters().stream().forEach(filter ->
filter.getSamples().forEach(datum -> {
String value = datum.getValue().toUpperCase();
if (value.equals("NAN") || value.equals("N/A")) {
value = "NA";
}
customDataByStudySampleName.put(datum.getStudyId(), datum.getSampleId(), filter.getDisplayName(), value);
})
);

List<CustomSampleIdentifier> filtered = new ArrayList<>();
customSampleIdentifiers.forEach(customSampleIdentifier -> {
int equalityFilterCount = getFilteredCountByDataEqualityWithStudySampleNameMap(equalityFilters, customDataByStudySampleName,
customSampleIdentifier.getSampleId(), customSampleIdentifier.getStudyId(), false);
int intervalFilterCount = getFilteredCountByDataIntervalWithStudySampleNameMap(intervalFilters, customDataByStudySampleName,
customSampleIdentifier.getSampleId(), customSampleIdentifier.getStudyId());
if (equalityFilterCount == equalityFilters.size()
&& intervalFilterCount == intervalFilters.size()
) {
filtered.add(customSampleIdentifier);
}
else {
customSampleIdentifier.setIsFilteredOut(true);
filtered.add(customSampleIdentifier);
}
});
return filtered;
}

private Integer getFilteredCountByDataEqualityWithStudySampleNameMap(List<ClinicalDataFilter> attributes, MultiKeyMap<String, String> clinicalDataMap,
String entityId, String studyId, boolean negateFilters) {
Integer count = 0;
for (ClinicalDataFilter s : attributes) {
List<String> filteredValues = s.getValues()
.stream()
.map(DataFilterValue::getValue)
.collect(Collectors.toList());
filteredValues.replaceAll(String::toUpperCase);
if (clinicalDataMap.containsKey(studyId, entityId, s.getDisplayName())) {
String value = clinicalDataMap.get(studyId, entityId, s.getDisplayName());
if (negateFilters ^ filteredValues.contains(value)) {
count++;
}
} else if (negateFilters ^ filteredValues.contains("NA")) {
count++;
}
}
return count;
}

private <S> Integer getFilteredCountByDataInterval(List<ClinicalDataFilter> attributes, MultiKeyMap<String, S> clinicalDataMap,
String entityId, String studyId) {
Expand Down Expand Up @@ -157,6 +241,39 @@ else if (specialValues.contains(attrValue.toUpperCase())) {
return count;
}

private Integer getFilteredCountByDataIntervalWithStudySampleNameMap(List<ClinicalDataFilter> attributes, MultiKeyMap<String, String> clinicalDataMap,
String entityId, String studyId) {
int count = 0;

for (ClinicalDataFilter filter : attributes) {
if (clinicalDataMap.containsKey(studyId, entityId, filter.getDisplayName())) {
String attrValue = clinicalDataMap.get(studyId, entityId, filter.getDisplayName());
Range<BigDecimal> rangeValue = calculateRangeValueForAttr(attrValue);

// find range filters
List<Range<BigDecimal>> ranges = filter.getValues().stream()
.map(this::calculateRangeValueForFilter)
.filter(Objects::nonNull)
.toList();

// find special value filters
List<String> specialValues = filter.getValues().stream()
.filter(f -> f.getValue() != null)
.map(f -> f.getValue().toUpperCase())
.toList();

if ((rangeValue != null && ranges.stream().anyMatch(r -> r.encloses(rangeValue))) ||
(rangeValue == null && specialValues.contains(attrValue.toUpperCase()))) {
count++;
}
} else if (Boolean.TRUE.equals(containsNA(filter))) {
count++;
}
}

return count;
}

private Range<BigDecimal> calculateRangeValueForAttr(String attrValue) {
if (attrValue == null) {
return null;
Expand Down
27 changes: 27 additions & 0 deletions src/main/java/org/cbioportal/web/parameter/ClinicalDataFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@
public class ClinicalDataFilter extends DataFilter implements Serializable {

private String attributeId;
private List<CustomSampleIdentifier> samples;
private String datatype;
private String displayName;

public String getAttributeId() {
return attributeId;
Expand All @@ -15,4 +18,28 @@ public void setAttributeId(String attributeId) {
this.attributeId = attributeId;
}

public List<CustomSampleIdentifier> getSamples() {
return samples;
}

public void setSamples(List<CustomSampleIdentifier> samples) {
this.samples = samples;
}

public String getDatatype() {
return datatype;
}

public void setDatatype(String datatype) {
this.datatype = datatype;
}

public String getDisplayName() {
return displayName;
}

public void setDisplayName(String displayName) {
this.displayName = displayName;
}

}
10 changes: 5 additions & 5 deletions src/main/java/org/cbioportal/web/parameter/CustomSampleIdentifier.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
public class CustomSampleIdentifier extends SampleIdentifier implements Serializable {

private boolean isFilteredOut = false;
private String attributeId;
private String value;

public boolean getIsFilteredOut() {
return isFilteredOut;
Expand All @@ -15,11 +15,11 @@ public void setIsFilteredOut(boolean isFilteredOut) {
this.isFilteredOut = isFilteredOut;
}

public String getAttributeId() {
return attributeId;
public String getValue() {
return value;
}

public void setAttributeId(String attributeId) {
this.attributeId = attributeId;
public void setValue(String value) {
this.value = value;
}
}
6 changes: 2 additions & 4 deletions src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@
sample_unique_id IN (
'',
<foreach item="sampleIdentifier" collection="studyViewFilterHelper.customDataSamples" separator=",">
<if test="customDataFilter.attributeId == sampleIdentifier.getAttributeId() and !sampleIdentifier.getIsFilteredOut()">
<if test="!sampleIdentifier.getIsFilteredOut()">
'${sampleIdentifier.studyId}_${sampleIdentifier.sampleId}'
</if>
</foreach>
Expand All @@ -94,9 +94,7 @@
OR
sample_unique_id NOT IN (
<foreach item="sampleIdentifier" collection="studyViewFilterHelper.customDataSamples" separator=",">
<if test="customDataFilter.attributeId == sampleIdentifier.getAttributeId()">
'${sampleIdentifier.studyId}_${sampleIdentifier.sampleId}'
</if>
'${sampleIdentifier.studyId}_${sampleIdentifier.sampleId}'
</foreach>
)
</if>
Expand Down
14 changes: 11 additions & 3 deletions src/test/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapperTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,11 @@
import org.cbioportal.persistence.helper.AlterationFilterHelper;
import org.cbioportal.persistence.helper.StudyViewFilterHelper;
import org.cbioportal.persistence.mybatisclickhouse.config.MyBatisConfig;
import org.cbioportal.web.parameter.*;
import org.cbioportal.web.parameter.ClinicalDataFilter;
import org.cbioportal.web.parameter.CustomSampleIdentifier;
import org.cbioportal.web.parameter.DataFilter;
import org.cbioportal.web.parameter.DataFilterValue;
import org.cbioportal.web.parameter.StudyViewFilter;
import org.cbioportal.web.parameter.filter.AndedPatientTreatmentFilters;
import org.cbioportal.web.parameter.filter.AndedSampleTreatmentFilters;
import org.cbioportal.web.parameter.filter.OredPatientTreatmentFilters;
Expand All @@ -23,7 +27,12 @@
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringRunner;

import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
Expand Down Expand Up @@ -59,7 +68,6 @@ public void getFilteredSamples() {
assertEquals(0, filteredSamples1.size());

CustomSampleIdentifier customSampleIdentifier = new CustomSampleIdentifier();
customSampleIdentifier.setAttributeId("123");
customSampleIdentifier.setStudyId("acc_tcga");
customSampleIdentifier.setSampleId("tcga-a1-a0sb-01");
var filteredSamples2 = studyViewMapper.getFilteredSamples(StudyViewFilterHelper.build(studyViewFilter, null, Arrays.asList(customSampleIdentifier)));
Expand Down
Loading
Loading