From 6dee88d5ef390824cfe46568ada455270ef7bf79 Mon Sep 17 00:00:00 2001 From: wivern Date: Wed, 21 Feb 2024 19:09:41 +0300 Subject: [PATCH 01/15] ATL-10 Robust characterization analysis module (Table and Listings) --- pom.xml | 4 +- .../cohortcharacterization/CcController.java | 9 + .../cohortcharacterization/CcService.java | 6 + .../cohortcharacterization/CcServiceImpl.java | 212 +++++++++++++++++- ...GenerateCohortCharacterizationTasklet.java | 5 +- .../BaseCcDTOToCcEntityConverter.java | 12 +- .../converter/CcToCcDTOConverter.java | 2 +- .../domain/CcFeAnalysisEntity.java | 80 +++++++ .../domain/CohortCharacterizationEntity.java | 23 +- .../dto/AbstractTemporalResult.java | 94 ++++++++ .../dto/CcTemporalAnnualResult.java | 13 ++ .../dto/CcTemporalResult.java | 31 +++ .../report/PrevalenceItem.java | 18 ++ .../report/TemporalAnnualItem.java | 31 +++ .../report/TemporalItem.java | 40 ++++ .../repository/CcFeAnalysisRepository.java | 7 + ...nalysisEntityToFeAnalysisDTOConverter.java | 2 + ...isEntityToFeAnalysisShortDTOConverter.java | 23 ++ ...nalysisEntityToFeAnalysisDTOConverter.java | 2 + .../feanalysis/domain/FeAnalysisEntity.java | 22 ++ .../feanalysis/dto/FeAnalysisShortDTO.java | 38 ++++ ...132805__add_temporal_result_permission.sql | 10 + ...14154300__add_feature_temporal_options.sql | 83 +++++++ .../ddl/results/cohort_characterizations.sql | 38 ++++ .../sql/queryTemporalAnnualResults.sql | 21 ++ .../sql/queryTemporalResults.sql | 23 ++ 26 files changed, 824 insertions(+), 25 deletions(-) create mode 100644 src/main/java/org/ohdsi/webapi/cohortcharacterization/domain/CcFeAnalysisEntity.java create mode 100644 src/main/java/org/ohdsi/webapi/cohortcharacterization/dto/AbstractTemporalResult.java create mode 100644 src/main/java/org/ohdsi/webapi/cohortcharacterization/dto/CcTemporalAnnualResult.java create mode 100644 src/main/java/org/ohdsi/webapi/cohortcharacterization/dto/CcTemporalResult.java create mode 100644 src/main/java/org/ohdsi/webapi/cohortcharacterization/report/TemporalAnnualItem.java create mode 100644 src/main/java/org/ohdsi/webapi/cohortcharacterization/report/TemporalItem.java create mode 100644 src/main/java/org/ohdsi/webapi/cohortcharacterization/repository/CcFeAnalysisRepository.java create mode 100644 src/main/java/org/ohdsi/webapi/feanalysis/converter/CcFeAnalysisEntityToFeAnalysisShortDTOConverter.java create mode 100644 src/main/resources/db/migration/postgresql/V2.14.0.20240126132805__add_temporal_result_permission.sql create mode 100644 src/main/resources/db/migration/postgresql/V2.14.0.20240214154300__add_feature_temporal_options.sql create mode 100644 src/main/resources/resources/cohortcharacterizations/sql/queryTemporalAnnualResults.sql create mode 100644 src/main/resources/resources/cohortcharacterizations/sql/queryTemporalResults.sql diff --git a/pom.xml b/pom.xml index 962fac6b09..8de9c521aa 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ 1.12.0 2.1.3 0.4.0 - 3.2.0 + 3.3.0-SNAPSHOT 8.5.87 1.5 @@ -1157,7 +1157,7 @@ org.ohdsi SkeletonCohortCharacterization - 1.2.1 + 2.0.0-SNAPSHOT org.ohdsi diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcController.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcController.java index d32f478a8d..4586fab327 100644 --- a/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcController.java +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcController.java @@ -18,6 +18,7 @@ import org.ohdsi.webapi.cohortcharacterization.dto.CcPrevalenceStat; import org.ohdsi.webapi.cohortcharacterization.dto.CcResult; import org.ohdsi.webapi.cohortcharacterization.dto.CcShortDTO; +import org.ohdsi.webapi.cohortcharacterization.dto.CcTemporalResult; import org.ohdsi.webapi.cohortcharacterization.dto.CcVersionFullDTO; import org.ohdsi.webapi.cohortcharacterization.dto.CohortCharacterizationDTO; import org.ohdsi.webapi.cohortcharacterization.dto.ExportExecutionResultRequest; @@ -445,6 +446,14 @@ public List getGenerationsResults( return service.findResultAsList(generationId, thresholdLevel); } + @GET + @Path("/generation/{generationId}/temporalresult") + @Produces(MediaType.APPLICATION_JSON) + @Consumes(MediaType.APPLICATION_JSON) + public List getGenerationTemporalResults(@PathParam("generationId") final Long generationId) { + return service.findTemporalResultAsList(generationId); + } + @POST @Path("/generation/{generationId}/result") @Produces(MediaType.APPLICATION_JSON) diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcService.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcService.java index 3f2dd701bd..fb851a85a1 100644 --- a/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcService.java +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcService.java @@ -6,6 +6,7 @@ import org.ohdsi.webapi.cohortcharacterization.dto.CcPrevalenceStat; import org.ohdsi.webapi.cohortcharacterization.dto.CcResult; import org.ohdsi.webapi.cohortcharacterization.dto.CcShortDTO; +import org.ohdsi.webapi.cohortcharacterization.dto.CcTemporalResult; import org.ohdsi.webapi.cohortcharacterization.dto.CcVersionFullDTO; import org.ohdsi.webapi.cohortcharacterization.dto.CohortCharacterizationDTO; import org.ohdsi.webapi.cohortcharacterization.dto.ExecutionResultRequest; @@ -15,6 +16,8 @@ import org.ohdsi.webapi.cohortdefinition.event.CohortDefinitionChangedEvent; import org.ohdsi.webapi.feanalysis.event.FeAnalysisChangedEvent; import org.ohdsi.webapi.job.JobExecutionResource; +import org.ohdsi.webapi.shiro.annotations.CcGenerationId; +import org.ohdsi.webapi.shiro.annotations.DataSourceAccess; import org.ohdsi.webapi.tag.domain.HasTags; import org.ohdsi.webapi.tag.dto.TagNameListRequestDTO; import org.ohdsi.webapi.versioning.domain.CharacterizationVersion; @@ -64,6 +67,9 @@ public interface CcService extends HasTags { List findGenerationsByCcIdAndSource(Long id, String sourceKey); + @DataSourceAccess + List findTemporalResultAsList(@CcGenerationId Long generationId); + GenerationResults findResult(Long generationId, ExecutionResultRequest params); List findResultAsList(Long generationId, float thresholdLevel); diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcServiceImpl.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcServiceImpl.java index a10b3a8ff9..642c0d907a 100644 --- a/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcServiceImpl.java +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcServiceImpl.java @@ -17,16 +17,20 @@ import org.ohdsi.webapi.Constants; import org.ohdsi.webapi.JobInvalidator; import org.ohdsi.webapi.cohortcharacterization.converter.SerializedCcToCcConverter; +import org.ohdsi.webapi.cohortcharacterization.domain.CcFeAnalysisEntity; import org.ohdsi.webapi.cohortcharacterization.domain.CcGenerationEntity; import org.ohdsi.webapi.cohortcharacterization.domain.CcParamEntity; import org.ohdsi.webapi.cohortcharacterization.domain.CcStrataConceptSetEntity; import org.ohdsi.webapi.cohortcharacterization.domain.CcStrataEntity; import org.ohdsi.webapi.cohortcharacterization.domain.CohortCharacterizationEntity; +import org.ohdsi.webapi.cohortcharacterization.dto.AbstractTemporalResult; import org.ohdsi.webapi.cohortcharacterization.dto.CcDistributionStat; import org.ohdsi.webapi.cohortcharacterization.dto.CcExportDTO; import org.ohdsi.webapi.cohortcharacterization.dto.CcPrevalenceStat; import org.ohdsi.webapi.cohortcharacterization.dto.CcResult; import org.ohdsi.webapi.cohortcharacterization.dto.CcShortDTO; +import org.ohdsi.webapi.cohortcharacterization.dto.CcTemporalAnnualResult; +import org.ohdsi.webapi.cohortcharacterization.dto.CcTemporalResult; import org.ohdsi.webapi.cohortcharacterization.dto.CcVersionFullDTO; import org.ohdsi.webapi.cohortcharacterization.dto.CohortCharacterizationDTO; import org.ohdsi.webapi.cohortcharacterization.dto.ExecutionResultRequest; @@ -34,9 +38,13 @@ import org.ohdsi.webapi.cohortcharacterization.dto.GenerationResults; import org.ohdsi.webapi.cohortcharacterization.report.AnalysisItem; import org.ohdsi.webapi.cohortcharacterization.report.AnalysisResultItem; +import org.ohdsi.webapi.cohortcharacterization.report.PrevalenceItem; import org.ohdsi.webapi.cohortcharacterization.report.Report; +import org.ohdsi.webapi.cohortcharacterization.report.TemporalAnnualItem; +import org.ohdsi.webapi.cohortcharacterization.report.TemporalItem; import org.ohdsi.webapi.cohortcharacterization.repository.AnalysisGenerationInfoEntityRepository; import org.ohdsi.webapi.cohortcharacterization.repository.CcConceptSetRepository; +import org.ohdsi.webapi.cohortcharacterization.repository.CcFeAnalysisRepository; import org.ohdsi.webapi.cohortcharacterization.repository.CcGenerationEntityRepository; import org.ohdsi.webapi.cohortcharacterization.repository.CcParamRepository; import org.ohdsi.webapi.cohortcharacterization.repository.CcRepository; @@ -99,6 +107,7 @@ import org.springframework.core.env.Environment; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import org.springframework.jdbc.core.RowMapper; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -122,10 +131,14 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; +import java.util.function.Supplier; import java.util.stream.Collectors; +import static java.util.stream.Collectors.groupingBy; import static org.ohdsi.analysis.cohortcharacterization.design.CcResultType.DISTRIBUTION; import static org.ohdsi.analysis.cohortcharacterization.design.CcResultType.PREVALENCE; import static org.ohdsi.webapi.Constants.GENERATE_COHORT_CHARACTERIZATION; @@ -149,6 +162,8 @@ public class CcServiceImpl extends AbstractDaoService implements CcService, Gene private static final String[] PARAMETERS_COUNT = {"cohort_characterization_generation_id", "vocabulary_schema"}; private static final String[] PREVALENCE_STATS_PARAMS = {"cdm_database_schema", "cdm_results_schema", "cc_generation_id", "analysis_id", "cohort_id", "covariate_id"}; private final String QUERY_RESULTS = ResourceHelper.GetResourceAsString("/resources/cohortcharacterizations/sql/queryResults.sql"); + private final String QUERY_TEMPORAL_RESULTS = ResourceHelper.GetResourceAsString("/resources/cohortcharacterizations/sql/queryTemporalResults.sql"); + private final String QUERY_TEMPORAL_ANNUAL_RESULTS = ResourceHelper.GetResourceAsString("/resources/cohortcharacterizations/sql/queryTemporalAnnualResults.sql"); private final String QUERY_COUNT = ResourceHelper.GetResourceAsString("/resources/cohortcharacterizations/sql/queryCountWithoutThreshold.sql"); private final String DELETE_RESULTS = ResourceHelper.GetResourceAsString("/resources/cohortcharacterizations/sql/deleteResults.sql"); private final String DELETE_EXECUTION = ResourceHelper.GetResourceAsString("/resources/cohortcharacterizations/sql/deleteExecution.sql"); @@ -209,6 +224,7 @@ public class CcServiceImpl extends AbstractDaoService implements CcService, Gene private final JobInvalidator jobInvalidator; private final GenericConversionService genericConversionService; private final VocabularyService vocabularyService; + private final CcFeAnalysisRepository ccFeAnalysisRepository; private VersionService versionService; private PermissionService permissionService; @@ -238,7 +254,7 @@ public CcServiceImpl( final ApplicationEventPublisher eventPublisher, final JobInvalidator jobInvalidator, final VocabularyService vocabularyService, - final VersionService versionService, + CcFeAnalysisRepository ccFeAnalysisRepository, final VersionService versionService, final PermissionService permissionService, @Qualifier("conversionService") final GenericConversionService genericConversionService, Environment env) { @@ -260,6 +276,7 @@ public CcServiceImpl( this.eventPublisher = eventPublisher; this.jobInvalidator = jobInvalidator; this.vocabularyService = vocabularyService; + this.ccFeAnalysisRepository = ccFeAnalysisRepository; this.permissionService = permissionService; this.genericConversionService = genericConversionService; this.versionService = versionService; @@ -275,7 +292,7 @@ public CohortCharacterizationEntity createCc(final CohortCharacterizationEntity } private CohortCharacterizationEntity saveCc(final CohortCharacterizationEntity entity) { - CohortCharacterizationEntity savedEntity = repository.saveAndFlush(entity); + CohortCharacterizationEntity savedEntity = repository.save(entity); for(CcStrataEntity strata: entity.getStratas()){ strata.setCohortCharacterization(savedEntity); @@ -422,8 +439,10 @@ private void updateCohorts(final CohortCharacterizationEntity entity, final Coho } private void updateAnalyses(final CohortCharacterizationEntity entity, final CohortCharacterizationEntity foundEntity) { - foundEntity.getFeatureAnalyses().clear(); - foundEntity.getFeatureAnalyses().addAll(entity.getFeatureAnalyses()); + ccFeAnalysisRepository.delete(foundEntity.getCcFeatureAnalyses()); + foundEntity.getCcFeatureAnalyses().clear(); + foundEntity.getCcFeatureAnalyses().addAll(entity.getCcFeatureAnalyses()); + ccFeAnalysisRepository.save(foundEntity.getCcFeatureAnalyses()); } private List getLinksToDelete(final CohortCharacterizationEntity foundEntity, @@ -644,17 +663,29 @@ public List findAllIncompleteGenerations() { } protected List findResults(final Long generationId, ExecutionResultRequest params) { + return executeFindResults(generationId, params, QUERY_RESULTS, getGenerationResults()); + } + + private List findTemporalResults(final Long generationId, ExecutionResultRequest params) { + return executeFindResults(generationId, params, QUERY_TEMPORAL_RESULTS, getGenerationTemporalResult()); + } + + private List findTemporalAnnualResults(final Long generationId, ExecutionResultRequest params) { + return executeFindResults(generationId, params, QUERY_TEMPORAL_ANNUAL_RESULTS, getGenerationTemporalAnnualResult()); + } + + private List executeFindResults(final Long generationId, ExecutionResultRequest params, String query, RowMapper rowMapper) { final CcGenerationEntity generationEntity = ccGenerationRepository.findById(generationId) .orElseThrow(() -> new IllegalArgumentException(String.format(GENERATION_NOT_FOUND_ERROR, generationId))); final Source source = generationEntity.getSource(); String analysis = params.getAnalysisIds().stream().map(String::valueOf).collect(Collectors.joining(",")); String cohorts = params.getCohortIds().stream().map(String::valueOf).collect(Collectors.joining(",")); - String generationResults = sourceAwareSqlRender.renderSql(source.getSourceId(), QUERY_RESULTS, PARAMETERS_RESULTS_FILTERED, + String generationResults = sourceAwareSqlRender.renderSql(source.getSourceId(), query, PARAMETERS_RESULTS_FILTERED, new String[]{String.valueOf(generationId), String.valueOf(params.getThresholdValuePct()), analysis, cohorts, SourceUtils.getVocabularyQualifier(source)}); final String tempSchema = SourceUtils.getTempQualifier(source); String translatedSql = SqlTranslate.translateSql(generationResults, source.getSourceDialect(), SessionUtils.sessionId(), tempSchema); - return getGenerationResults(source, translatedSql); + return this.getSourceJdbcTemplate(source).query(translatedSql, rowMapper); } @Override @@ -720,7 +751,34 @@ public List findResultAsList(@CcGenerationId final Long generationId, .map(fa -> fa.getDomain().toString()).distinct().collect(Collectors.toList())); return findResults(generationId, params); } - + + @Override + @DataSourceAccess + public List findTemporalResultAsList(@CcGenerationId final Long generationId) { + ExecutionResultRequest params = getExecutionResultRequest(generationId); + return findTemporalResults(generationId, params); + } + + @DataSourceAccess + public List findTemporalAnnualResultAsList(@CcGenerationId final Long generationId) { + ExecutionResultRequest params = getExecutionResultRequest(generationId); + return findTemporalAnnualResults(generationId, params); + } + + private ExecutionResultRequest getExecutionResultRequest(Long generationId) { + ExecutionResultRequest params = new ExecutionResultRequest(); + CcGenerationEntity generationEntity = ccGenerationRepository.findById(generationId) + .orElseThrow(() -> new IllegalArgumentException(String.format(GENERATION_NOT_FOUND_ERROR, generationId))); + CohortCharacterizationEntity characterization = generationEntity.getCohortCharacterization(); + params.setCohortIds(characterization.getCohortDefinitions().stream() + .map(CohortDefinition::getId).collect(Collectors.toList())); + params.setAnalysisIds(characterization.getFeatureAnalyses().stream() + .map(this::mapFeatureAnalysisId).collect(Collectors.toList())); + params.setDomainIds(generationEntity.getCohortCharacterization().getFeatureAnalyses().stream() + .map(fa -> fa.getDomain().toString()).distinct().collect(Collectors.toList())); + return params; + } + @Override @DataSourceAccess public GenerationResults findResult(@CcGenerationId final Long generationId, ExecutionResultRequest params) { @@ -758,6 +816,7 @@ public GenerationResults findResult(@CcGenerationId final Long generationId, Exe .noneMatch(fe -> mapFeatureAnalysisId(fe).equals(s) && params.getDomainIds().contains(fe.getDomain().toString()))); List ccResults = findResults(generationId, params); + List ccTemporalResults = findTemporalResults(generationId, params); // create initial structure and fill with results Map analysisMap = new HashMap<>(); @@ -789,7 +848,56 @@ public GenerationResults findResult(@CcGenerationId final Long generationId, Exe .filter(def -> params.getCohortIds().contains(def.getId())) .collect(Collectors.toSet()); + //Temporal + Map prespecAnalysisIdMap = FeatureExtraction.getNameToPrespecAnalysis().values() + .stream().collect(Collectors.toMap(a -> a.analysisId, a -> a)); + List temporalResult = findTemporalResultAsList(generationId); + List mappedResult = temporalResult.stream().map(tr -> mapTemporalResult(featureAnalyses, prespecAnalysisIdMap, tr, CcTemporalResult::new, + (source, target) -> { + target.setStartDay(source.getStartDay()); + target.setEndDay(source.getEndDay()); + target.setTimeId(source.getTimeId()); + })) + .collect(Collectors.toList()); + Map>>> temporalByCohort = groupByResult(mappedResult); + + List temporalAnnualResult = findTemporalAnnualResultAsList(generationId); + List mappedAnnualResult = temporalAnnualResult.stream().map(tr -> mapTemporalResult(featureAnalyses, prespecAnalysisIdMap, tr, CcTemporalAnnualResult::new, + (source, target) -> { + target.setYear(source.getYear()); + })) + .collect(Collectors.toList()); + Map>>> annualByCohort = groupByResult(mappedAnnualResult); + List reports = prepareReportData(analysisMap, cohortDefs, featureAnalyses); + reports.forEach(r -> { + r.items.stream() + .filter(i -> Objects.equals(i.getFaType(), StandardFeatureAnalysisType.PRESET.toString())) + .map(PrevalenceItem.class::cast) + .forEach(item -> { + setTemporal(temporalByCohort, item, cov -> { + List temporalItems = cov.stream().map(temp -> { + TemporalItem ti = new TemporalItem(); + ti.setAvg(temp.getAvg()); + ti.setCount(temp.getCount()); + ti.setStartDay(temp.getStartDay()); + ti.setEndDay(temp.getEndDay()); + return ti; + }).collect(Collectors.toList()); + item.setTemporal(temporalItems); + }); + setTemporal(annualByCohort, item, cov -> { + List temporalAnnualItems = cov.stream().map(temp -> { + TemporalAnnualItem tai = new TemporalAnnualItem(); + tai.setYear(temp.getYear()); + tai.setAvg(temp.getAvg()); + tai.setCount(temp.getCount()); + return tai; + }).collect(Collectors.toList()); + item.setTemporalAnnual(temporalAnnualItems); + }); + }); + }); GenerationResults res = new GenerationResults(); res.setReports(reports); @@ -797,6 +905,45 @@ public GenerationResults findResult(@CcGenerationId final Long generationId, Exe return res; } + private static void setTemporal(Map>>> temporalByCohort, PrevalenceItem item, Consumer> setter) { + Optional.ofNullable(temporalByCohort.get(item.getCohortId())) + .flatMap(cr -> Optional.ofNullable(cr.get(item.getAnalysisId())) + .flatMap(ar -> Optional.ofNullable(ar.get(item.getCovariateId())))) + .ifPresent(setter); + } + + private static Map>>> groupByResult(List mappedAnnualResult) { + return mappedAnnualResult.stream() + .collect(groupingBy(T::getCohortId, groupingBy(T::getAnalysisId, groupingBy(T::getCovariateId)))); + } + + private static T mapTemporalResult( + Set featureAnalyses, + Map prespecAnalysisIdMap, + T source, + Supplier constructor, + BiConsumer initializer + ) { + T result = constructor.get(); + String analysisName = prespecAnalysisIdMap.get(source.getAnalysisId()).analysisName; + Integer analysisId = featureAnalyses.stream().filter(fa -> Objects.equals(fa.getRawDesign(), analysisName)) + .findFirst() + .map(FeAnalysisEntity::getId) + .orElseThrow(() -> new IllegalArgumentException(String.format("Preset analysis [%s} is not mapped to feature", analysisName))); + result.setAnalysisId(analysisId); + result.setCovariateId(source.getCovariateId()); + result.setAvg(source.getAvg()); + result.setCount(source.getCount()); + result.setAnalysisName(source.getAnalysisName()); + result.setCovariateName(source.getCovariateName()); + result.setStrataId(source.getStrataId()); + result.setCohortId(source.getCohortId()); + result.setStrataName(source.getStrataName()); + result.setConceptId(source.getConceptId()); + initializer.accept(source, result); + return result; + } + private Integer mapFeatureAnalysisId(FeAnalysisEntity feAnalysis) { if (feAnalysis.isPreset()) { @@ -1074,8 +1221,8 @@ public List listByTags(TagNameListRequestDTO requestDTO) { return listByTags(entities, names, CcShortDTO.class); } - private List getGenerationResults(final Source source, final String translatedSql) { - return this.getSourceJdbcTemplate(source).query(translatedSql, (rs, rowNum) -> { + private RowMapper getGenerationResults() { + return (rs, rowNum) -> { final String type = rs.getString("type"); if (StringUtils.equals(type, DISTRIBUTION.toString())) { final CcDistributionStat distributionStat = new CcDistributionStat(); @@ -1088,7 +1235,45 @@ private List getGenerationResults(final Source source, final String tr return prevalenceStat; } return null; - }); + }; + } + + private RowMapper getGenerationTemporalResult() { + return (rs, rowNum) -> { + CcTemporalResult result = new CcTemporalResult(); + result.setAnalysisName(rs.getString("analysis_name")); + result.setAvg(rs.getDouble("avg_value")); + result.setAnalysisId(rs.getInt("analysis_id")); + result.setCohortId(rs.getInt("cohort_definition_id")); + result.setConceptId(rs.getInt("concept_id")); + result.setCount(rs.getLong("count_value")); + result.setCovariateId(rs.getLong("covariate_id")); + result.setCovariateName(rs.getString("covariate_name")); + result.setStrataId(rs.getInt("strata_id")); + result.setEndDay(rs.getInt("end_day")); + result.setStartDay(rs.getInt("start_day")); + result.setStrataName(rs.getString("strata_name")); + result.setTimeId(rs.getInt("time_id")); + return result; + }; + } + + private RowMapper getGenerationTemporalAnnualResult() { + return (rs, rowNum) -> { + CcTemporalAnnualResult result = new CcTemporalAnnualResult(); + result.setAnalysisName(rs.getString("analysis_name")); + result.setAvg(rs.getDouble("avg_value")); + result.setAnalysisId(rs.getInt("analysis_id")); + result.setCohortId(rs.getInt("cohort_definition_id")); + result.setConceptId(rs.getInt("concept_id")); + result.setCount(rs.getLong("count_value")); + result.setCovariateId(rs.getLong("covariate_id")); + result.setCovariateName(rs.getString("covariate_name")); + result.setStrataId(rs.getInt("strata_id")); + result.setStrataName(rs.getString("strata_name")); + result.setYear(rs.getInt("event_year")); + return result; + }; } private void gatherForPrevalence(final CcPrevalenceStat stat, final ResultSet rs) throws SQLException { @@ -1165,7 +1350,12 @@ private List importAnalyses(final CohortCharacterizationEntity entity, } } - persistedEntity.setFeatureAnalyses(analysesSet); + persistedEntity.setFeatureAnalyses(analysesSet.stream().map(a -> { + CcFeAnalysisEntity feAnalysisEntity = new CcFeAnalysisEntity(); + feAnalysisEntity.setFeatureAnalysis(a); + feAnalysisEntity.setCohortCharacterization(entity); + return feAnalysisEntity; + }).collect(Collectors.toSet())); return savedAnalysesIds; } diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java index c7039e58f8..fad2b06435 100644 --- a/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java @@ -22,6 +22,7 @@ import org.ohdsi.sql.SqlSplit; import org.ohdsi.sql.SqlTranslate; import org.ohdsi.webapi.cohortcharacterization.converter.SerializedCcToCcConverter; +import org.ohdsi.webapi.cohortcharacterization.domain.CcFeAnalysisEntity; import org.ohdsi.webapi.cohortcharacterization.domain.CohortCharacterizationEntity; import org.ohdsi.webapi.cohortcharacterization.repository.AnalysisGenerationInfoEntityRepository; import org.ohdsi.webapi.common.generation.AnalysisTasklet; @@ -75,9 +76,11 @@ protected String[] prepareQueries(ChunkContext chunkContext, CancelableJdbcTempl final String cohortTable = jobParams.get(TARGET_TABLE).toString(); final String sessionId = jobParams.get(SESSION_ID).toString(); final String tempSchema = SourceUtils.getTempQualifier(source); + boolean includeAnnual = cohortCharacterization.getCcFeatureAnalyses().stream().anyMatch(CcFeAnalysisEntity::getIncludeAnnual); + boolean includeTemporal = cohortCharacterization.getCcFeatureAnalyses().stream().anyMatch(CcFeAnalysisEntity::getIncludeTemporal); CCQueryBuilder ccQueryBuilder = new CCQueryBuilder(cohortCharacterization, cohortTable, sessionId, SourceUtils.getCdmQualifier(source), SourceUtils.getResultsQualifier(source), - SourceUtils.getVocabularyQualifier(source), tempSchema, jobId); + SourceUtils.getVocabularyQualifier(source), tempSchema, jobId, includeAnnual, includeTemporal); String sql = ccQueryBuilder.build(); /* diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/converter/BaseCcDTOToCcEntityConverter.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/converter/BaseCcDTOToCcEntityConverter.java index 509a0c2c53..294478593d 100644 --- a/src/main/java/org/ohdsi/webapi/cohortcharacterization/converter/BaseCcDTOToCcEntityConverter.java +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/converter/BaseCcDTOToCcEntityConverter.java @@ -5,6 +5,7 @@ import org.ohdsi.analysis.CohortMetadata; import org.ohdsi.analysis.Utils; import org.ohdsi.analysis.cohortcharacterization.design.CcResultType; +import org.ohdsi.webapi.cohortcharacterization.domain.CcFeAnalysisEntity; import org.ohdsi.webapi.cohortcharacterization.domain.CcStrataConceptSetEntity; import org.ohdsi.webapi.cohortcharacterization.domain.CcParamEntity; import org.ohdsi.webapi.cohortcharacterization.domain.CcStrataEntity; @@ -51,7 +52,16 @@ public CohortCharacterizationEntity convert(T source) { fa.setStatType(CcResultType.PREVALENCE); } }); - cohortCharacterization.setFeatureAnalyses(converterUtils.convertSet(source.getFeatureAnalyses(), FeAnalysisEntity.class)); + cohortCharacterization.setFeatureAnalyses( + source.getFeatureAnalyses().stream().map(fa -> { + CcFeAnalysisEntity feAnalysisEntity = new CcFeAnalysisEntity(); + feAnalysisEntity.setFeatureAnalysis(conversionService.convert(fa, FeAnalysisEntity.class)); + feAnalysisEntity.setCohortCharacterization(cohortCharacterization); + feAnalysisEntity.setIncludeAnnual(fa.getIncludeAnnual()); + feAnalysisEntity.setIncludeTemporal(fa.getIncludeTemporal()); + return feAnalysisEntity; + }).collect(Collectors.toSet()) + ); cohortCharacterization.setParameters(converterUtils.convertSet(source.getParameters(), CcParamEntity.class)); cohortCharacterization.setStratas(converterUtils.convertSet(source.getStratas(), CcStrataEntity.class)); diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/converter/CcToCcDTOConverter.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/converter/CcToCcDTOConverter.java index c9aad19aaa..c8b8d34df0 100644 --- a/src/main/java/org/ohdsi/webapi/cohortcharacterization/converter/CcToCcDTOConverter.java +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/converter/CcToCcDTOConverter.java @@ -15,7 +15,7 @@ public CohortCharacterizationDTO convert(final CohortCharacterizationEntity sour final CohortCharacterizationDTO cohortCharacterizationDTO = super.convert(source); cohortCharacterizationDTO.setCohorts(converterUtils.convertSet(source.getCohortDefinitions(), CohortMetadataImplDTO.class)); - cohortCharacterizationDTO.setFeatureAnalyses(converterUtils.convertSet(source.getFeatureAnalyses(), FeAnalysisShortDTO.class)); + cohortCharacterizationDTO.setFeatureAnalyses(converterUtils.convertSet(source.getCcFeatureAnalyses(), FeAnalysisShortDTO.class)); return cohortCharacterizationDTO; } diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/domain/CcFeAnalysisEntity.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/domain/CcFeAnalysisEntity.java new file mode 100644 index 0000000000..d3a45fb5bd --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/domain/CcFeAnalysisEntity.java @@ -0,0 +1,80 @@ +package org.ohdsi.webapi.cohortcharacterization.domain; + +import org.hibernate.annotations.GenericGenerator; +import org.hibernate.annotations.Parameter; +import org.ohdsi.webapi.feanalysis.domain.FeAnalysisEntity; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; +import javax.persistence.JoinColumn; +import javax.persistence.ManyToOne; +import javax.persistence.Table; + +@Entity +@Table(name = "cc_analysis") +public class CcFeAnalysisEntity { + + @Id + @GenericGenerator( + name = "cc_analysis_generator", + strategy = "org.hibernate.id.enhanced.SequenceStyleGenerator", + parameters = { + @Parameter(name = "sequence_name", value = "cc_analysis_seq"), + @Parameter(name = "increment_size", value = "1") + } + ) + @GeneratedValue(generator = "cc_analysis_generator") + private Long id; + @ManyToOne(optional = false) + @JoinColumn(name = "cohort_characterization_id") + private CohortCharacterizationEntity cohortCharacterization; + @ManyToOne(optional = false) + @JoinColumn(name = "fe_analysis_id") + private FeAnalysisEntity featureAnalysis; + @Column(name = "include_annual") + private Boolean includeAnnual; + @Column(name = "include_temporal") + private Boolean includeTemporal; + + public CohortCharacterizationEntity getCohortCharacterization() { + return cohortCharacterization; + } + + public void setCohortCharacterization(CohortCharacterizationEntity cohortCharacterization) { + this.cohortCharacterization = cohortCharacterization; + } + + public FeAnalysisEntity getFeatureAnalysis() { + return featureAnalysis; + } + + public void setFeatureAnalysis(FeAnalysisEntity featureAnalysis) { + this.featureAnalysis = featureAnalysis; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Boolean getIncludeAnnual() { + return includeAnnual; + } + + public void setIncludeAnnual(Boolean includeAnnual) { + this.includeAnnual = includeAnnual; + } + + public Boolean getIncludeTemporal() { + return includeTemporal; + } + + public void setIncludeTemporal(Boolean includeTemporal) { + this.includeTemporal = includeTemporal; + } +} diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/domain/CohortCharacterizationEntity.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/domain/CohortCharacterizationEntity.java index c9aef23268..99a427d889 100644 --- a/src/main/java/org/ohdsi/webapi/cohortcharacterization/domain/CohortCharacterizationEntity.java +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/domain/CohortCharacterizationEntity.java @@ -1,6 +1,7 @@ package org.ohdsi.webapi.cohortcharacterization.domain; import java.util.*; +import java.util.stream.Collectors; import javax.persistence.CascadeType; import javax.persistence.Column; @@ -52,11 +53,9 @@ public class CohortCharacterizationEntity extends CommonEntityExt implemen inverseJoinColumns = @JoinColumn(name = "cohort_id", referencedColumnName = "id")) private Set cohortDefinitions = new HashSet<>(); - @ManyToMany(targetEntity = FeAnalysisEntity.class, fetch = FetchType.LAZY) - @JoinTable(name = "cc_analysis", - joinColumns = @JoinColumn(name = "cohort_characterization_id", referencedColumnName = "id"), - inverseJoinColumns = @JoinColumn(name = "fe_analysis_id", referencedColumnName = "id")) - private Set featureAnalyses = new HashSet<>(); + @OneToMany(orphanRemoval = true) + @JoinColumn(name = "cohort_characterization_id", insertable = false, updatable = false, nullable = false) + private Set featureAnalyses = new HashSet<>(); @OneToMany(mappedBy = "cohortCharacterization", fetch = FetchType.LAZY, targetEntity = CcParamEntity.class) private Set parameters = new HashSet<>(); @@ -89,6 +88,12 @@ public Set getCohorts() { @Override public Set getFeatureAnalyses() { + return featureAnalyses != null ? + featureAnalyses.stream().map(CcFeAnalysisEntity::getFeatureAnalysis).collect(Collectors.toSet()) : + Collections.emptySet(); + } + + public Set getCcFeatureAnalyses() { return featureAnalyses; } @@ -97,6 +102,10 @@ public Set getParameters() { return parameters; } + public void setFeatureAnalyses(Set featureAnalyses) { + this.featureAnalyses = featureAnalyses; + } + @Override public Long getId() { return id; @@ -126,10 +135,6 @@ public void setParameters(final Set parameters) { this.parameters = parameters; } - public void setFeatureAnalyses(final Set featureAnalyses) { - this.featureAnalyses = featureAnalyses; - } - public Set getCohortDefinitions() { return cohortDefinitions; } diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/dto/AbstractTemporalResult.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/dto/AbstractTemporalResult.java new file mode 100644 index 0000000000..383ee8514a --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/dto/AbstractTemporalResult.java @@ -0,0 +1,94 @@ +package org.ohdsi.webapi.cohortcharacterization.dto; + +public abstract class AbstractTemporalResult { + protected Integer analysisId; + protected String analysisName; + protected Long covariateId; + protected String covariateName; + protected Integer strataId; + protected String strataName; + protected Integer conceptId; + protected Integer cohortId; + protected Long count; + protected Double avg; + + public Integer getAnalysisId() { + return analysisId; + } + + public void setAnalysisId(Integer analysisId) { + this.analysisId = analysisId; + } + + public String getAnalysisName() { + return analysisName; + } + + public void setAnalysisName(String analysisName) { + this.analysisName = analysisName; + } + + public Long getCovariateId() { + return covariateId; + } + + public void setCovariateId(Long covariateId) { + this.covariateId = covariateId; + } + + public String getCovariateName() { + return covariateName; + } + + public void setCovariateName(String covariateName) { + this.covariateName = covariateName; + } + + public Integer getStrataId() { + return strataId; + } + + public void setStrataId(Integer strataId) { + this.strataId = strataId; + } + + public String getStrataName() { + return strataName; + } + + public void setStrataName(String strataName) { + this.strataName = strataName; + } + + public Integer getConceptId() { + return conceptId; + } + + public void setConceptId(Integer conceptId) { + this.conceptId = conceptId; + } + + public Long getCount() { + return count; + } + + public void setCount(Long count) { + this.count = count; + } + + public Double getAvg() { + return avg; + } + + public void setAvg(Double avg) { + this.avg = avg; + } + + public Integer getCohortId() { + return cohortId; + } + + public void setCohortId(Integer cohortId) { + this.cohortId = cohortId; + } +} diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/dto/CcTemporalAnnualResult.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/dto/CcTemporalAnnualResult.java new file mode 100644 index 0000000000..c40c5078ef --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/dto/CcTemporalAnnualResult.java @@ -0,0 +1,13 @@ +package org.ohdsi.webapi.cohortcharacterization.dto; + +public class CcTemporalAnnualResult extends AbstractTemporalResult{ + private Integer year; + + public Integer getYear() { + return year; + } + + public void setYear(Integer year) { + this.year = year; + } +} diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/dto/CcTemporalResult.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/dto/CcTemporalResult.java new file mode 100644 index 0000000000..ae5bf55754 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/dto/CcTemporalResult.java @@ -0,0 +1,31 @@ +package org.ohdsi.webapi.cohortcharacterization.dto; + +public class CcTemporalResult extends AbstractTemporalResult { + private Integer timeId; + private Integer startDay; + private Integer endDay; + + public Integer getTimeId() { + return timeId; + } + + public void setTimeId(Integer timeId) { + this.timeId = timeId; + } + + public Integer getStartDay() { + return startDay; + } + + public void setStartDay(Integer startDay) { + this.startDay = startDay; + } + + public Integer getEndDay() { + return endDay; + } + + public void setEndDay(Integer endDay) { + this.endDay = endDay; + } +} diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/report/PrevalenceItem.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/report/PrevalenceItem.java index bbe50ff98f..ce08d90ad7 100644 --- a/src/main/java/org/ohdsi/webapi/cohortcharacterization/report/PrevalenceItem.java +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/report/PrevalenceItem.java @@ -12,6 +12,8 @@ public class PrevalenceItem extends ExportItem { protected final Long count; protected final Double pct; protected final Double avg; + private List temporal; + private List temporalAnnual; public PrevalenceItem(CcPrevalenceStat prevalenceStat, String cohortName) { super(prevalenceStat); @@ -98,6 +100,22 @@ public String getCohortName() { return cohortName; } + public List getTemporal() { + return temporal; + } + + public void setTemporal(List temporal) { + this.temporal = temporal; + } + + public List getTemporalAnnual() { + return temporalAnnual; + } + + public void setTemporalAnnual(List temporalAnnual) { + this.temporalAnnual = temporalAnnual; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/report/TemporalAnnualItem.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/report/TemporalAnnualItem.java new file mode 100644 index 0000000000..901c66a106 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/report/TemporalAnnualItem.java @@ -0,0 +1,31 @@ +package org.ohdsi.webapi.cohortcharacterization.report; + +public class TemporalAnnualItem { + private Long count; + private Double avg; + private Integer year; + + public Long getCount() { + return count; + } + + public void setCount(Long count) { + this.count = count; + } + + public Double getAvg() { + return avg; + } + + public void setAvg(Double avg) { + this.avg = avg; + } + + public Integer getYear() { + return year; + } + + public void setYear(Integer year) { + this.year = year; + } +} diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/report/TemporalItem.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/report/TemporalItem.java new file mode 100644 index 0000000000..2286d2dc3c --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/report/TemporalItem.java @@ -0,0 +1,40 @@ +package org.ohdsi.webapi.cohortcharacterization.report; + +public class TemporalItem { + private Long count; + private Double avg; + private Integer startDay; + private Integer endDay; + + public Long getCount() { + return count; + } + + public void setCount(Long count) { + this.count = count; + } + + public Double getAvg() { + return avg; + } + + public void setAvg(Double avg) { + this.avg = avg; + } + + public Integer getStartDay() { + return startDay; + } + + public void setStartDay(Integer startDay) { + this.startDay = startDay; + } + + public Integer getEndDay() { + return endDay; + } + + public void setEndDay(Integer endDay) { + this.endDay = endDay; + } +} diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/repository/CcFeAnalysisRepository.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/repository/CcFeAnalysisRepository.java new file mode 100644 index 0000000000..b85fbe20a2 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/repository/CcFeAnalysisRepository.java @@ -0,0 +1,7 @@ +package org.ohdsi.webapi.cohortcharacterization.repository; + +import org.ohdsi.webapi.cohortcharacterization.domain.CcFeAnalysisEntity; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface CcFeAnalysisRepository extends JpaRepository { +} diff --git a/src/main/java/org/ohdsi/webapi/feanalysis/converter/BaseFeAnalysisEntityToFeAnalysisDTOConverter.java b/src/main/java/org/ohdsi/webapi/feanalysis/converter/BaseFeAnalysisEntityToFeAnalysisDTOConverter.java index daae37aa09..65e9144031 100644 --- a/src/main/java/org/ohdsi/webapi/feanalysis/converter/BaseFeAnalysisEntityToFeAnalysisDTOConverter.java +++ b/src/main/java/org/ohdsi/webapi/feanalysis/converter/BaseFeAnalysisEntityToFeAnalysisDTOConverter.java @@ -15,5 +15,7 @@ public void doConvert(FeAnalysisEntity source, T target) { target.setDomain(source.getDomain()); target.setDescription(source.getDescr()); target.setStatType(source.getStatType()); + target.setSupportsAnnual(source.getSupportsAnnual()); + target.setSupportsTemporal(source.getSupportsTemporal()); } } diff --git a/src/main/java/org/ohdsi/webapi/feanalysis/converter/CcFeAnalysisEntityToFeAnalysisShortDTOConverter.java b/src/main/java/org/ohdsi/webapi/feanalysis/converter/CcFeAnalysisEntityToFeAnalysisShortDTOConverter.java new file mode 100644 index 0000000000..c43a985aeb --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/feanalysis/converter/CcFeAnalysisEntityToFeAnalysisShortDTOConverter.java @@ -0,0 +1,23 @@ +package org.ohdsi.webapi.feanalysis.converter; + +import org.ohdsi.webapi.cohortcharacterization.domain.CcFeAnalysisEntity; +import org.ohdsi.webapi.converter.BaseConversionServiceAwareConverter; +import org.ohdsi.webapi.feanalysis.dto.FeAnalysisShortDTO; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.convert.support.GenericConversionService; +import org.springframework.stereotype.Component; + +@Component +public class CcFeAnalysisEntityToFeAnalysisShortDTOConverter extends BaseConversionServiceAwareConverter { + + @Autowired + private GenericConversionService conversionService; + + @Override + public FeAnalysisShortDTO convert(CcFeAnalysisEntity source) { + FeAnalysisShortDTO dto = conversionService.convert(source.getFeatureAnalysis(), FeAnalysisShortDTO.class); + dto.setIncludeAnnual(source.getIncludeAnnual()); + dto.setIncludeTemporal(source.getIncludeTemporal()); + return dto; + } +} diff --git a/src/main/java/org/ohdsi/webapi/feanalysis/converter/FeAnalysisEntityToFeAnalysisDTOConverter.java b/src/main/java/org/ohdsi/webapi/feanalysis/converter/FeAnalysisEntityToFeAnalysisDTOConverter.java index f39f12cd2f..6c44bb473c 100644 --- a/src/main/java/org/ohdsi/webapi/feanalysis/converter/FeAnalysisEntityToFeAnalysisDTOConverter.java +++ b/src/main/java/org/ohdsi/webapi/feanalysis/converter/FeAnalysisEntityToFeAnalysisDTOConverter.java @@ -24,6 +24,8 @@ public class FeAnalysisEntityToFeAnalysisDTOConverter extends BaseFeAnalysisEnti public FeAnalysisDTO convert(final FeAnalysisEntity source) { final FeAnalysisDTO dto = super.convert(source); dto.setDesign(convertDesignToJson(source)); + dto.setSupportsAnnual(source.getSupportsAnnual()); + dto.setSupportsTemporal(source.getSupportsTemporal()); if (CRITERIA_SET.equals(source.getType())){ FeAnalysisWithConceptSetDTO dtoWithConceptSet = (FeAnalysisWithConceptSetDTO) dto; FeAnalysisWithCriteriaEntity sourceWithCriteria = (FeAnalysisWithCriteriaEntity) source; diff --git a/src/main/java/org/ohdsi/webapi/feanalysis/domain/FeAnalysisEntity.java b/src/main/java/org/ohdsi/webapi/feanalysis/domain/FeAnalysisEntity.java index 868303e1ef..b644abaf6a 100644 --- a/src/main/java/org/ohdsi/webapi/feanalysis/domain/FeAnalysisEntity.java +++ b/src/main/java/org/ohdsi/webapi/feanalysis/domain/FeAnalysisEntity.java @@ -92,6 +92,12 @@ public FeAnalysisEntity(final FeAnalysisEntity entityForCopy) { @Enumerated(value = EnumType.STRING) private CcResultType statType; + @Column(name = "supports_annual", updatable = false, insertable = false) + private Boolean supportsAnnual; + + @Column(name = "supports_temporal", updatable = false, insertable = false) + private Boolean supportsTemporal; + @Override public Integer getId() { return id; @@ -207,5 +213,21 @@ public void setStatType(final CcResultType statType) { this.statType = statType; } + + public Boolean getSupportsAnnual() { + return supportsAnnual; + } + + public void setSupportsAnnual(Boolean supportsAnnual) { + this.supportsAnnual = supportsAnnual; + } + + public Boolean getSupportsTemporal() { + return supportsTemporal; + } + + public void setSupportsTemporal(Boolean supportsTemporal) { + this.supportsTemporal = supportsTemporal; + } } diff --git a/src/main/java/org/ohdsi/webapi/feanalysis/dto/FeAnalysisShortDTO.java b/src/main/java/org/ohdsi/webapi/feanalysis/dto/FeAnalysisShortDTO.java index fc0629a091..02bedcf4ac 100644 --- a/src/main/java/org/ohdsi/webapi/feanalysis/dto/FeAnalysisShortDTO.java +++ b/src/main/java/org/ohdsi/webapi/feanalysis/dto/FeAnalysisShortDTO.java @@ -10,6 +10,8 @@ public class FeAnalysisShortDTO extends CommonEntityDTO { @JsonProperty("description") protected String description; + protected Boolean supportsAnnual; + protected Boolean supportsTemporal; @JsonProperty("id") private Integer id; @JsonProperty("name") @@ -20,6 +22,10 @@ public class FeAnalysisShortDTO extends CommonEntityDTO { private StandardFeatureAnalysisDomain domain; @JsonProperty("statType") private CcResultType statType; + @JsonProperty("includeAnnual") + private Boolean includeAnnual; + @JsonProperty("includeTemporal") + private Boolean includeTemporal; public Integer getId() { @@ -76,4 +82,36 @@ public void setStatType(CcResultType statType) { this.statType = statType; } + + public Boolean getSupportsAnnual() { + return supportsAnnual; + } + + public void setSupportsAnnual(Boolean supportsAnnual) { + this.supportsAnnual = supportsAnnual; + } + + public Boolean getSupportsTemporal() { + return supportsTemporal; + } + + public void setSupportsTemporal(Boolean supportsTemporal) { + this.supportsTemporal = supportsTemporal; + } + + public Boolean getIncludeAnnual() { + return includeAnnual; + } + + public void setIncludeAnnual(Boolean includeAnnual) { + this.includeAnnual = includeAnnual; + } + + public Boolean getIncludeTemporal() { + return includeTemporal; + } + + public void setIncludeTemporal(Boolean includeTemporal) { + this.includeTemporal = includeTemporal; + } } diff --git a/src/main/resources/db/migration/postgresql/V2.14.0.20240126132805__add_temporal_result_permission.sql b/src/main/resources/db/migration/postgresql/V2.14.0.20240126132805__add_temporal_result_permission.sql new file mode 100644 index 0000000000..2832f61d77 --- /dev/null +++ b/src/main/resources/db/migration/postgresql/V2.14.0.20240126132805__add_temporal_result_permission.sql @@ -0,0 +1,10 @@ +INSERT INTO ${ohdsiSchema}.sec_permission(id, value, description) VALUES + (nextval('${ohdsiSchema}.sec_permission_id_seq'), 'cohort-characterization:generation:*:temporalresult:get', 'Get cohort characterization generation temporal results'); + +INSERT INTO ${ohdsiSchema}.sec_role_permission(role_id, permission_id) +SELECT sr.id, sp.id +FROM ${ohdsiSchema}.sec_permission SP, ${ohdsiSchema}.sec_role sr +WHERE sp."value" IN ( + 'cohort-characterization:generation:*:temporalresult:get' + ) + AND sr.name IN ('Atlas users'); \ No newline at end of file diff --git a/src/main/resources/db/migration/postgresql/V2.14.0.20240214154300__add_feature_temporal_options.sql b/src/main/resources/db/migration/postgresql/V2.14.0.20240214154300__add_feature_temporal_options.sql new file mode 100644 index 0000000000..b160f81d2e --- /dev/null +++ b/src/main/resources/db/migration/postgresql/V2.14.0.20240214154300__add_feature_temporal_options.sql @@ -0,0 +1,83 @@ +ALTER TABLE ${ohdsiSchema}.fe_analysis + ADD COLUMN IF NOT EXISTS supports_annual BOOLEAN default false, + ADD COLUMN IF NOT EXISTS supports_temporal BOOLEAN default false; + +UPDATE ${ohdsiSchema}.fe_analysis + SET supports_annual = TRUE +WHERE design in ( + 'ConditionOccurrenceAnyTimePrior', + 'ConditionOccurrenceLongTerm', + 'ConditionOccurrenceMediumTerm', + 'ConditionOccurrenceShortTerm', + 'ConditionOccurrencePrimaryInpatientAnyTimePrior', + 'ConditionOccurrencePrimaryInpatientLongTerm', + 'ConditionOccurrencePrimaryInpatientMediumTerm', + 'ConditionOccurrencePrimaryInpatientShortTerm', + 'ConditionEraAnyTimePrior', + 'ConditionEraLongTerm', + 'ConditionEraMediumTerm', + 'ConditionEraShortTerm', + 'ConditionEraOverlapping', + 'ConditionEraStartLongTerm', + 'ConditionEraStartMediumTerm', + 'ConditionEraStartShortTerm', + 'DrugExposureAnyTimePrior', + 'DrugExposureLongTerm', + 'DrugExposureMediumTerm', + 'DrugExposureShortTerm', + 'DrugEraAnyTimePrior', + 'DrugEraLongTerm', + 'DrugEraMediumTerm', + 'DrugEraShortTerm', + 'DrugEraOverlapping', + 'DrugEraStartLongTerm', + 'DrugEraStartMediumTerm', + 'DrugEraStartShortTerm', + 'ProcedureOccurrenceAnyTimePrior', + 'ProcedureOccurrenceLongTerm', + 'ProcedureOccurrenceMediumTerm', + 'ProcedureOccurrenceShortTerm', + 'DeviceExposureAnyTimePrior', + 'DeviceExposureLongTerm', + 'DeviceExposureMediumTerm', + 'DeviceExposureShortTerm', + 'MeasurementAnyTimePrior', + 'MeasurementLongTerm', + 'MeasurementMediumTerm', + 'MeasurementShortTerm' + 'ObservationAnyTimePrior', + 'ObservationLongTerm', + 'ObservationMediumTerm', + 'ObservationShortTerm' +); + +UPDATE ${ohdsiSchema}.fe_analysis + SET supports_temporal = TRUE +WHERE + design in ( + 'DemographicsGender', + 'DemographicsAge', + 'DemographicsAgeGroup', + 'DemographicsRace', + 'DemographicsEthnicity', + 'DemographicsIndexYear', + 'DemographicsIndexMonth', + 'DemographicsPriorObservationTime', + 'DemographicsPostObservationTime', + 'DemographicsTimeInCohort', + 'DemographicsIndexYearMonth' +); + +CREATE SEQUENCE IF NOT EXISTS ${ohdsiSchema}.cc_analysis_seq; + +ALTER TABLE ${ohdsiSchema}.cc_analysis + ADD COLUMN IF NOT EXISTS id BIGINT NOT NULL DEFAULT nextval('cc_analysis_seq'), + ADD COLUMN IF NOT EXISTS include_annual BOOLEAN DEFAULT false, + ADD COLUMN IF NOT EXISTS include_temporal BOOLEAN DEFAULT false; + +SELECT setval('${ohdsiSchema}.cc_analysis_seq', COALESCE(MAX(id) + 1, 1), false) FROM ${ohdsiSchema}.cc_analysis; + +ALTER TABLE ${ohdsiSchema}.cc_analysis + DROP CONSTRAINT cc_analysis_pkey; +ALTER TABLE ${ohdsiSchema}.cc_analysis + ADD CONSTRAINT cc_analysis_pkey PRIMARY KEY (id); diff --git a/src/main/resources/ddl/results/cohort_characterizations.sql b/src/main/resources/ddl/results/cohort_characterizations.sql index 077aa1f6eb..d5fcd4bb0d 100644 --- a/src/main/resources/ddl/results/cohort_characterizations.sql +++ b/src/main/resources/ddl/results/cohort_characterizations.sql @@ -26,4 +26,42 @@ CREATE TABLE @results_schema.cc_results aggregate_id INTEGER, aggregate_name VARCHAR(1000), missing_means_zero INTEGER +); + +IF OBJECT_ID('@results_schema.cc_temporal_results') IS NULL +CREATE TABLE @results_schema.cc_temporal_results( + type varchar(255), + fa_type varchar(255), + cc_generation_id bigint, + analysis_id integer, + analysis_name varchar(1000), + covariate_id bigint, + covariate_name varchar(1000), + strata_id bigint, + strata_name varchar(1000), + concept_id integer, + count_value bigint, + avg_value double precision, + cohort_definition_id bigint, + time_id integer, + start_day integer, + end_day integer +); + +IF OBJECT_ID('@results_schema.cc_temporal_annual_results') IS NULL + CREATE TABLE @results_schema.cc_temporal_annual_results( + type varchar(255), + fa_type varchar(255), + cc_generation_id bigint, + analysis_id integer, + analysis_name varchar(1000), + covariate_id bigint, + covariate_name varchar(1000), + strata_id bigint, + strata_name varchar(1000), + concept_id integer, + count_value bigint, + avg_value double precision, + cohort_definition_id bigint, + event_year integer ); \ No newline at end of file diff --git a/src/main/resources/resources/cohortcharacterizations/sql/queryTemporalAnnualResults.sql b/src/main/resources/resources/cohortcharacterizations/sql/queryTemporalAnnualResults.sql new file mode 100644 index 0000000000..10a57fa000 --- /dev/null +++ b/src/main/resources/resources/cohortcharacterizations/sql/queryTemporalAnnualResults.sql @@ -0,0 +1,21 @@ +select + r.type, + r.fa_type, + r.cc_generation_id, + r.analysis_id, + r.analysis_name, + r.covariate_id, + r.covariate_name, + c.concept_name, + r.concept_id, + r.count_value, + r.avg_value, + r.cohort_definition_id, + r.strata_id, + r.strata_name, + r.event_year +from @results_database_schema.cc_temporal_annual_results r + JOIN @vocabulary_schema.concept c on c.concept_id = r.concept_id +where r.cc_generation_id = @cohort_characterization_generation_id + and r.analysis_id in (@analysis_ids) + and r.cohort_definition_id in (@cohort_ids) diff --git a/src/main/resources/resources/cohortcharacterizations/sql/queryTemporalResults.sql b/src/main/resources/resources/cohortcharacterizations/sql/queryTemporalResults.sql new file mode 100644 index 0000000000..e3a7d8adb5 --- /dev/null +++ b/src/main/resources/resources/cohortcharacterizations/sql/queryTemporalResults.sql @@ -0,0 +1,23 @@ +select + r.type, + r.fa_type, + r.cc_generation_id, + r.analysis_id, + r.analysis_name, + r.covariate_id, + r.covariate_name, + c.concept_name, + r.concept_id, + r.count_value, + r.avg_value, + r.cohort_definition_id, + r.strata_id, + r.strata_name, + r.start_day, + r.end_day, + r.time_id +from @results_database_schema.cc_temporal_results r + JOIN @vocabulary_schema.concept c on c.concept_id = r.concept_id +where r.cc_generation_id = @cohort_characterization_generation_id + and r.analysis_id in (@analysis_ids) + and r.cohort_definition_id in (@cohort_ids) From ce6d28640a039c91442cb022dbc96805473582d4 Mon Sep 17 00:00:00 2001 From: git Date: Fri, 12 Apr 2024 17:13:11 +0300 Subject: [PATCH 02/15] Fixes NPE --- ...nalysisEntityToFeAnalysisDTOConverter.java | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/feanalysis/converter/FeAnalysisEntityToFeAnalysisDTOConverter.java b/src/main/java/org/ohdsi/webapi/feanalysis/converter/FeAnalysisEntityToFeAnalysisDTOConverter.java index 6c44bb473c..034796b232 100644 --- a/src/main/java/org/ohdsi/webapi/feanalysis/converter/FeAnalysisEntityToFeAnalysisDTOConverter.java +++ b/src/main/java/org/ohdsi/webapi/feanalysis/converter/FeAnalysisEntityToFeAnalysisDTOConverter.java @@ -10,6 +10,7 @@ import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.stream.Collectors; import static org.ohdsi.analysis.cohortcharacterization.design.StandardFeatureAnalysisType.CRITERIA_SET; @@ -36,26 +37,30 @@ public FeAnalysisDTO convert(final FeAnalysisEntity source) { @Override protected FeAnalysisDTO createResultObject(FeAnalysisEntity feAnalysisEntity) { - switch (feAnalysisEntity.getType()){ - case CRITERIA_SET: - return new FeAnalysisWithConceptSetDTO(); - default: - return new FeAnalysisDTO(); - } + return Optional.ofNullable(feAnalysisEntity.getType()).map(type -> { + switch (type) { + case CRITERIA_SET: + return new FeAnalysisWithConceptSetDTO(); + default: + return new FeAnalysisDTO(); + } + }).orElseGet(() -> new FeAnalysisDTO()); } private Object convertDesignToJson(final FeAnalysisEntity source) { - switch (source.getType()) { - case CRITERIA_SET: - FeAnalysisWithCriteriaEntity sourceWithCriteria = (FeAnalysisWithCriteriaEntity) source; - return sourceWithCriteria.getDesign() - .stream() - .map(this::convertCriteria) - .map(c -> (JsonNode)objectMapper.valueToTree(c)) - .collect(Collectors.toList()); - default: - return source.getDesign(); - } + return Optional.ofNullable(source.getType()).map(type -> { + switch (type) { + case CRITERIA_SET: + FeAnalysisWithCriteriaEntity sourceWithCriteria = (FeAnalysisWithCriteriaEntity) source; + return sourceWithCriteria.getDesign() + .stream() + .map(this::convertCriteria) + .map(c -> (JsonNode) objectMapper.valueToTree(c)) + .collect(Collectors.toList()); + default: + return source.getDesign(); + } + }).orElseGet(() -> source.getDesign()); } private BaseFeAnalysisCriteriaDTO convertCriteria(FeAnalysisCriteriaEntity criteriaEntity){ From 04cb84651b01e124fab3809d3b54b982e6d1745d Mon Sep 17 00:00:00 2001 From: alex-odysseus Date: Mon, 15 Apr 2024 21:05:53 +0200 Subject: [PATCH 03/15] To avoid NullPointerException when executing a Cohort Characterization It looks like only the 'annual' flag is currently transferred from the client which leads to storing NULL in the database --- .../GenerateCohortCharacterizationTasklet.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java index fad2b06435..54b2d3dabc 100644 --- a/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java @@ -77,10 +77,10 @@ protected String[] prepareQueries(ChunkContext chunkContext, CancelableJdbcTempl final String sessionId = jobParams.get(SESSION_ID).toString(); final String tempSchema = SourceUtils.getTempQualifier(source); boolean includeAnnual = cohortCharacterization.getCcFeatureAnalyses().stream().anyMatch(CcFeAnalysisEntity::getIncludeAnnual); - boolean includeTemporal = cohortCharacterization.getCcFeatureAnalyses().stream().anyMatch(CcFeAnalysisEntity::getIncludeTemporal); + // boolean includeTemporal = cohortCharacterization.getCcFeatureAnalyses().stream().anyMatch(CcFeAnalysisEntity::getIncludeTemporal); CCQueryBuilder ccQueryBuilder = new CCQueryBuilder(cohortCharacterization, cohortTable, sessionId, SourceUtils.getCdmQualifier(source), SourceUtils.getResultsQualifier(source), - SourceUtils.getVocabularyQualifier(source), tempSchema, jobId, includeAnnual, includeTemporal); + SourceUtils.getVocabularyQualifier(source), tempSchema, jobId, includeAnnual, false); String sql = ccQueryBuilder.build(); /* From 2029e69c274d1af6e45c52da6fd8d56dc76e61f6 Mon Sep 17 00:00:00 2001 From: git Date: Wed, 17 Apr 2024 14:11:17 +0300 Subject: [PATCH 04/15] Fixes characterization import --- .../webapi/cohortcharacterization/CcServiceImpl.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcServiceImpl.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcServiceImpl.java index 642c0d907a..5663403e9c 100644 --- a/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcServiceImpl.java +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcServiceImpl.java @@ -497,9 +497,9 @@ public CohortCharacterizationEntity importCc(final CohortCharacterizationEntity updateConceptSet(entity, persistedCohortCharacterization); importCohorts(entity, persistedCohortCharacterization); - List savedAnalysesIds = importAnalyses(entity, persistedCohortCharacterization); final CohortCharacterizationEntity savedEntity = saveCc(persistedCohortCharacterization); + List savedAnalysesIds = importAnalyses(entity, savedEntity); eventPublisher.publishEvent(new CcImportEvent(savedAnalysesIds)); @@ -1350,12 +1350,16 @@ private List importAnalyses(final CohortCharacterizationEntity entity, } } - persistedEntity.setFeatureAnalyses(analysesSet.stream().map(a -> { + persistedEntity.getCcFeatureAnalyses().clear(); + Set featureAnalyses = analysesSet.stream().map(a -> { CcFeAnalysisEntity feAnalysisEntity = new CcFeAnalysisEntity(); feAnalysisEntity.setFeatureAnalysis(a); - feAnalysisEntity.setCohortCharacterization(entity); + feAnalysisEntity.setCohortCharacterization(persistedEntity); return feAnalysisEntity; - }).collect(Collectors.toSet())); + }).collect(Collectors.toSet()); + ccFeAnalysisRepository.save(featureAnalyses); + + persistedEntity.getCcFeatureAnalyses().addAll(featureAnalyses); return savedAnalysesIds; } From f131aeb709364be4b8a1f49f6573c0d119178fae Mon Sep 17 00:00:00 2001 From: git Date: Wed, 24 Apr 2024 10:48:21 +0300 Subject: [PATCH 05/15] Fixes characterization create and copy --- .../ohdsi/webapi/cohortcharacterization/CcServiceImpl.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcServiceImpl.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcServiceImpl.java index 5663403e9c..b79dca6b29 100644 --- a/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcServiceImpl.java +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcServiceImpl.java @@ -304,6 +304,11 @@ private CohortCharacterizationEntity saveCc(final CohortCharacterizationEntity e paramRepository.save(param); } + for(CcFeAnalysisEntity analysis : entity.getCcFeatureAnalyses()) { + analysis.setCohortCharacterization(savedEntity); + ccFeAnalysisRepository.save(analysis); + } + entityManager.flush(); entityManager.refresh(savedEntity); From caa4628db074bc9a0b5edbf171f6a82bb2400cd1 Mon Sep 17 00:00:00 2001 From: git Date: Sat, 27 Apr 2024 10:18:02 +0300 Subject: [PATCH 06/15] Fixes potential NPE --- .../GenerateCohortCharacterizationTasklet.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java index 54b2d3dabc..de1ed9f757 100644 --- a/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java @@ -38,6 +38,7 @@ import java.sql.SQLException; import java.util.Map; +import java.util.Optional; import static org.ohdsi.webapi.Constants.Params.*; @@ -76,7 +77,8 @@ protected String[] prepareQueries(ChunkContext chunkContext, CancelableJdbcTempl final String cohortTable = jobParams.get(TARGET_TABLE).toString(); final String sessionId = jobParams.get(SESSION_ID).toString(); final String tempSchema = SourceUtils.getTempQualifier(source); - boolean includeAnnual = cohortCharacterization.getCcFeatureAnalyses().stream().anyMatch(CcFeAnalysisEntity::getIncludeAnnual); + boolean includeAnnual = cohortCharacterization.getCcFeatureAnalyses().stream() + .anyMatch(fe -> Optional.ofNullable(fe.getIncludeAnnual()).orElse(false)); // boolean includeTemporal = cohortCharacterization.getCcFeatureAnalyses().stream().anyMatch(CcFeAnalysisEntity::getIncludeTemporal); CCQueryBuilder ccQueryBuilder = new CCQueryBuilder(cohortCharacterization, cohortTable, sessionId, SourceUtils.getCdmQualifier(source), SourceUtils.getResultsQualifier(source), From 43a72f15c089709c14f26c21201a7a68ff7d46ff Mon Sep 17 00:00:00 2001 From: git Date: Sat, 27 Apr 2024 11:32:01 +0300 Subject: [PATCH 07/15] Adds i18n for supports annual prevalence column --- src/main/resources/i18n/messages_en.json | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/resources/i18n/messages_en.json b/src/main/resources/i18n/messages_en.json index be9e537c5c..601f5f8252 100644 --- a/src/main/resources/i18n/messages_en.json +++ b/src/main/resources/i18n/messages_en.json @@ -1477,6 +1477,7 @@ "stddev": "Std Dev", "strata": "Strata", "suggestedNegativeControl": "Suggested Negative Control", + "supportsAnnual": "Supports Annual", "tableQualifiers": "Table Qualifiers", "target": "Target", "targetCohortName": "Target Cohort Name", From 3d1fe59ab6c4ccbab369608b80d325051b507a17 Mon Sep 17 00:00:00 2001 From: git Date: Fri, 3 May 2024 11:44:53 +0300 Subject: [PATCH 08/15] Fixes FeAnalysis converter --- .../org/ohdsi/webapi/cohortcharacterization/CcController.java | 2 +- .../converter/BaseFeAnalysisDTOToFeAnalysisConverter.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcController.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcController.java index 4586fab327..3218cab36d 100644 --- a/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcController.java +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcController.java @@ -258,7 +258,7 @@ public CohortCharacterizationDTO update(@PathParam("id") final Long id, final Co final CohortCharacterizationEntity entity = conversionService.convert(dto, CohortCharacterizationEntity.class); entity.setId(id); final CohortCharacterizationEntity updatedEntity = service.updateCc(entity); - return conversionService.convert(updatedEntity, CohortCharacterizationDTO.class); + return convertCcToDto(updatedEntity); } /** diff --git a/src/main/java/org/ohdsi/webapi/feanalysis/converter/BaseFeAnalysisDTOToFeAnalysisConverter.java b/src/main/java/org/ohdsi/webapi/feanalysis/converter/BaseFeAnalysisDTOToFeAnalysisConverter.java index 694636c5b0..6063655470 100644 --- a/src/main/java/org/ohdsi/webapi/feanalysis/converter/BaseFeAnalysisDTOToFeAnalysisConverter.java +++ b/src/main/java/org/ohdsi/webapi/feanalysis/converter/BaseFeAnalysisDTOToFeAnalysisConverter.java @@ -18,6 +18,8 @@ public T convert(D source) { result.setName(StringUtils.trim(source.getName())); result.setType(source.getType()); result.setStatType(source.getStatType()); + result.setSupportsAnnual(source.getSupportsAnnual()); + result.setSupportsTemporal(source.getSupportsTemporal()); return result; } From 1c3f040cad50dd6d4768fd5a4121f2ddbe3196c6 Mon Sep 17 00:00:00 2001 From: git Date: Tue, 28 May 2024 15:39:25 +0300 Subject: [PATCH 09/15] Enables tempoeral CC results --- ...GenerateCohortCharacterizationTasklet.java | 4 +- .../domain/CcFeAnalysisEntity.java | 37 ++++++++++++++++++- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java index de1ed9f757..c5c24bf80f 100644 --- a/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java @@ -79,10 +79,10 @@ protected String[] prepareQueries(ChunkContext chunkContext, CancelableJdbcTempl final String tempSchema = SourceUtils.getTempQualifier(source); boolean includeAnnual = cohortCharacterization.getCcFeatureAnalyses().stream() .anyMatch(fe -> Optional.ofNullable(fe.getIncludeAnnual()).orElse(false)); - // boolean includeTemporal = cohortCharacterization.getCcFeatureAnalyses().stream().anyMatch(CcFeAnalysisEntity::getIncludeTemporal); + boolean includeTemporal = cohortCharacterization.getCcFeatureAnalyses().stream().anyMatch(CcFeAnalysisEntity::getIncludeTemporal); CCQueryBuilder ccQueryBuilder = new CCQueryBuilder(cohortCharacterization, cohortTable, sessionId, SourceUtils.getCdmQualifier(source), SourceUtils.getResultsQualifier(source), - SourceUtils.getVocabularyQualifier(source), tempSchema, jobId, includeAnnual, false); + SourceUtils.getVocabularyQualifier(source), tempSchema, jobId, includeAnnual, includeTemporal); String sql = ccQueryBuilder.build(); /* diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/domain/CcFeAnalysisEntity.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/domain/CcFeAnalysisEntity.java index d3a45fb5bd..1110d72ca0 100644 --- a/src/main/java/org/ohdsi/webapi/cohortcharacterization/domain/CcFeAnalysisEntity.java +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/domain/CcFeAnalysisEntity.java @@ -2,6 +2,9 @@ import org.hibernate.annotations.GenericGenerator; import org.hibernate.annotations.Parameter; +import org.ohdsi.analysis.cohortcharacterization.design.FeatureAnalysis; +import org.ohdsi.analysis.cohortcharacterization.design.FeatureAnalysisDomain; +import org.ohdsi.analysis.cohortcharacterization.design.FeatureAnalysisType; import org.ohdsi.webapi.feanalysis.domain.FeAnalysisEntity; import javax.persistence.Column; @@ -11,10 +14,13 @@ import javax.persistence.JoinColumn; import javax.persistence.ManyToOne; import javax.persistence.Table; +import java.util.Optional; +import java.util.function.Function; +import java.util.function.Supplier; @Entity @Table(name = "cc_analysis") -public class CcFeAnalysisEntity { +public class CcFeAnalysisEntity implements FeatureAnalysis { @Id @GenericGenerator( @@ -77,4 +83,33 @@ public Boolean getIncludeTemporal() { public void setIncludeTemporal(Boolean includeTemporal) { this.includeTemporal = includeTemporal; } + + private T mapFeatureAnalysis(Function getter) { + return Optional.ofNullable(featureAnalysis).map(getter).orElse(null); + } + + @Override + public FeatureAnalysisType getType() { + return mapFeatureAnalysis(FeatureAnalysis::getType); + } + + @Override + public String getName() { + return mapFeatureAnalysis(FeatureAnalysis::getName); + } + + @Override + public FeatureAnalysisDomain getDomain() { + return mapFeatureAnalysis(FeatureAnalysis::getDomain); + } + + @Override + public String getDescr() { + return mapFeatureAnalysis(FeatureAnalysis::getDescr); + } + + @Override + public Object getDesign() { + return mapFeatureAnalysis(FeatureAnalysis::getDesign); + } } From abe7b7f3b3d43de8025ee14ee9a6c2637e442d88 Mon Sep 17 00:00:00 2001 From: Oleg Kuzik Date: Thu, 11 Jul 2024 16:51:37 +0200 Subject: [PATCH 10/15] [ATL-10] Added an option to exclude creating comparative items for the covariates comparison feature --- .../cohortcharacterization/CcServiceImpl.java | 15 +++++++++++---- .../dto/ExecutionResultRequest.java | 10 ++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcServiceImpl.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcServiceImpl.java index b79dca6b29..c6b7aa7e4c 100644 --- a/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcServiceImpl.java +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcServiceImpl.java @@ -38,6 +38,8 @@ import org.ohdsi.webapi.cohortcharacterization.dto.GenerationResults; import org.ohdsi.webapi.cohortcharacterization.report.AnalysisItem; import org.ohdsi.webapi.cohortcharacterization.report.AnalysisResultItem; +import org.ohdsi.webapi.cohortcharacterization.report.ComparativeItem; +import org.ohdsi.webapi.cohortcharacterization.report.ExportItem; import org.ohdsi.webapi.cohortcharacterization.report.PrevalenceItem; import org.ohdsi.webapi.cohortcharacterization.report.Report; import org.ohdsi.webapi.cohortcharacterization.report.TemporalAnnualItem; @@ -874,11 +876,12 @@ public GenerationResults findResult(@CcGenerationId final Long generationId, Exe .collect(Collectors.toList()); Map>>> annualByCohort = groupByResult(mappedAnnualResult); - List reports = prepareReportData(analysisMap, cohortDefs, featureAnalyses); + List reports = prepareReportData(analysisMap, cohortDefs, featureAnalyses, params); reports.forEach(r -> { r.items.stream() .filter(i -> Objects.equals(i.getFaType(), StandardFeatureAnalysisType.PRESET.toString())) - .map(PrevalenceItem.class::cast) + .filter(o -> !r.isComparative) + .map(this::toPrevalenceItem) .forEach(item -> { setTemporal(temporalByCohort, item, cov -> { List temporalItems = cov.stream().map(temp -> { @@ -910,6 +913,10 @@ public GenerationResults findResult(@CcGenerationId final Long generationId, Exe return res; } + private PrevalenceItem toPrevalenceItem(ExportItem exportItem){ + return PrevalenceItem.class.cast(exportItem); + } + private static void setTemporal(Map>>> temporalByCohort, PrevalenceItem item, Consumer> setter) { Optional.ofNullable(temporalByCohort.get(item.getCohortId())) .flatMap(cr -> Optional.ofNullable(cr.get(item.getAnalysisId())) @@ -970,7 +977,7 @@ private String mapFeatureName(FeAnalysisEntity entity) { } private List prepareReportData(Map analysisMap, Set cohortDefs, - Set featureAnalyses) { + Set featureAnalyses, ExecutionResultRequest params) { // Create map to get cohort name by its id final Map definitionMap = cohortDefs.stream() .collect(Collectors.toMap(CohortDefinition::getId, Function.identity())); @@ -1006,7 +1013,7 @@ private List prepareReportData(Map analysisMap, S reports.add(simpleReport); // comparative mode - if (definitionMap.size() == 2) { + if (definitionMap.size() == 2 && !params.getExcludeComparativeResults()) { Iterator iter = definitionMap.values().iterator(); CohortDefinition firstCohortDef = iter.next(); CohortDefinition secondCohortDef = iter.next(); diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/dto/ExecutionResultRequest.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/dto/ExecutionResultRequest.java index 9fbea1dd89..22b4c16665 100644 --- a/src/main/java/org/ohdsi/webapi/cohortcharacterization/dto/ExecutionResultRequest.java +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/dto/ExecutionResultRequest.java @@ -27,6 +27,8 @@ public class ExecutionResultRequest { @JsonProperty("showEmptyResults") private Boolean isShowEmptyResults = false; + @JsonProperty("excludeComparativeResults") + private boolean excludeComparativeResults; public List getCohortIds() { if(cohortIds == null) { return Collections.emptyList(); @@ -87,4 +89,12 @@ public Boolean getShowEmptyResults() { public void setShowEmptyResults(Boolean showEmptyResults) { isShowEmptyResults = showEmptyResults; } + + public boolean getExcludeComparativeResults() { + return excludeComparativeResults; + } + + public void setExcludeComparativeResults(boolean excludeComparativeResults) { + this.excludeComparativeResults = excludeComparativeResults; + } } From d03fbd82a014cfcd9a18f1dfe7bd1b130eeac53c Mon Sep 17 00:00:00 2001 From: Alex Manoylenko Date: Tue, 23 Jul 2024 10:51:24 +0000 Subject: [PATCH 11/15] Fixing NullPointerException for the temporal Feature Analysis flag like it was done for the annual Feature Analysis flag --- .../GenerateCohortCharacterizationTasklet.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java index c5c24bf80f..0a91861219 100644 --- a/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java @@ -79,7 +79,8 @@ protected String[] prepareQueries(ChunkContext chunkContext, CancelableJdbcTempl final String tempSchema = SourceUtils.getTempQualifier(source); boolean includeAnnual = cohortCharacterization.getCcFeatureAnalyses().stream() .anyMatch(fe -> Optional.ofNullable(fe.getIncludeAnnual()).orElse(false)); - boolean includeTemporal = cohortCharacterization.getCcFeatureAnalyses().stream().anyMatch(CcFeAnalysisEntity::getIncludeTemporal); + boolean includeTemporal = cohortCharacterization.getCcFeatureAnalyses().stream() + .anyMatch(fe -> Optional.ofNullable(fe.getIncludeTemporal()).orElse(false)); CCQueryBuilder ccQueryBuilder = new CCQueryBuilder(cohortCharacterization, cohortTable, sessionId, SourceUtils.getCdmQualifier(source), SourceUtils.getResultsQualifier(source), SourceUtils.getVocabularyQualifier(source), tempSchema, jobId, includeAnnual, includeTemporal); From 674c03aba274986c6db0a7ebaac39fc159f4465a Mon Sep 17 00:00:00 2001 From: alex-odysseus Date: Wed, 28 Aug 2024 22:36:17 +0200 Subject: [PATCH 12/15] Referencing FeatureExtraction 3.7.0-SNAPSHOT where the temporal annual distribution functionality should be available --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d2f4ae3455..3263c4ffe8 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ 1.12.0 2.1.3 0.4.0 - 3.3.0-SNAPSHOT + 3.7.0-SNAPSHOT 8.5.87 1.5 From 9b64699e6f67d28cb03ddfd210ea0a444857efef Mon Sep 17 00:00:00 2001 From: oleg-odysseus Date: Thu, 11 Jul 2024 16:51:37 +0200 Subject: [PATCH 13/15] [ATL-10] Added an option to exclude creating comparative items for the covariates comparison feature --- .../cohortcharacterization/CcServiceImpl.java | 15 +++++++++++---- .../dto/ExecutionResultRequest.java | 10 ++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcServiceImpl.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcServiceImpl.java index b79dca6b29..c6b7aa7e4c 100644 --- a/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcServiceImpl.java +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/CcServiceImpl.java @@ -38,6 +38,8 @@ import org.ohdsi.webapi.cohortcharacterization.dto.GenerationResults; import org.ohdsi.webapi.cohortcharacterization.report.AnalysisItem; import org.ohdsi.webapi.cohortcharacterization.report.AnalysisResultItem; +import org.ohdsi.webapi.cohortcharacterization.report.ComparativeItem; +import org.ohdsi.webapi.cohortcharacterization.report.ExportItem; import org.ohdsi.webapi.cohortcharacterization.report.PrevalenceItem; import org.ohdsi.webapi.cohortcharacterization.report.Report; import org.ohdsi.webapi.cohortcharacterization.report.TemporalAnnualItem; @@ -874,11 +876,12 @@ public GenerationResults findResult(@CcGenerationId final Long generationId, Exe .collect(Collectors.toList()); Map>>> annualByCohort = groupByResult(mappedAnnualResult); - List reports = prepareReportData(analysisMap, cohortDefs, featureAnalyses); + List reports = prepareReportData(analysisMap, cohortDefs, featureAnalyses, params); reports.forEach(r -> { r.items.stream() .filter(i -> Objects.equals(i.getFaType(), StandardFeatureAnalysisType.PRESET.toString())) - .map(PrevalenceItem.class::cast) + .filter(o -> !r.isComparative) + .map(this::toPrevalenceItem) .forEach(item -> { setTemporal(temporalByCohort, item, cov -> { List temporalItems = cov.stream().map(temp -> { @@ -910,6 +913,10 @@ public GenerationResults findResult(@CcGenerationId final Long generationId, Exe return res; } + private PrevalenceItem toPrevalenceItem(ExportItem exportItem){ + return PrevalenceItem.class.cast(exportItem); + } + private static void setTemporal(Map>>> temporalByCohort, PrevalenceItem item, Consumer> setter) { Optional.ofNullable(temporalByCohort.get(item.getCohortId())) .flatMap(cr -> Optional.ofNullable(cr.get(item.getAnalysisId())) @@ -970,7 +977,7 @@ private String mapFeatureName(FeAnalysisEntity entity) { } private List prepareReportData(Map analysisMap, Set cohortDefs, - Set featureAnalyses) { + Set featureAnalyses, ExecutionResultRequest params) { // Create map to get cohort name by its id final Map definitionMap = cohortDefs.stream() .collect(Collectors.toMap(CohortDefinition::getId, Function.identity())); @@ -1006,7 +1013,7 @@ private List prepareReportData(Map analysisMap, S reports.add(simpleReport); // comparative mode - if (definitionMap.size() == 2) { + if (definitionMap.size() == 2 && !params.getExcludeComparativeResults()) { Iterator iter = definitionMap.values().iterator(); CohortDefinition firstCohortDef = iter.next(); CohortDefinition secondCohortDef = iter.next(); diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/dto/ExecutionResultRequest.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/dto/ExecutionResultRequest.java index 9fbea1dd89..22b4c16665 100644 --- a/src/main/java/org/ohdsi/webapi/cohortcharacterization/dto/ExecutionResultRequest.java +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/dto/ExecutionResultRequest.java @@ -27,6 +27,8 @@ public class ExecutionResultRequest { @JsonProperty("showEmptyResults") private Boolean isShowEmptyResults = false; + @JsonProperty("excludeComparativeResults") + private boolean excludeComparativeResults; public List getCohortIds() { if(cohortIds == null) { return Collections.emptyList(); @@ -87,4 +89,12 @@ public Boolean getShowEmptyResults() { public void setShowEmptyResults(Boolean showEmptyResults) { isShowEmptyResults = showEmptyResults; } + + public boolean getExcludeComparativeResults() { + return excludeComparativeResults; + } + + public void setExcludeComparativeResults(boolean excludeComparativeResults) { + this.excludeComparativeResults = excludeComparativeResults; + } } From accf3159ada7f90317d92c1f87d46436b930476f Mon Sep 17 00:00:00 2001 From: Alex Manoylenko Date: Tue, 23 Jul 2024 10:51:24 +0000 Subject: [PATCH 14/15] Fixing NullPointerException for the temporal Feature Analysis flag like it was done for the annual Feature Analysis flag --- .../GenerateCohortCharacterizationTasklet.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java b/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java index c5c24bf80f..0a91861219 100644 --- a/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java +++ b/src/main/java/org/ohdsi/webapi/cohortcharacterization/GenerateCohortCharacterizationTasklet.java @@ -79,7 +79,8 @@ protected String[] prepareQueries(ChunkContext chunkContext, CancelableJdbcTempl final String tempSchema = SourceUtils.getTempQualifier(source); boolean includeAnnual = cohortCharacterization.getCcFeatureAnalyses().stream() .anyMatch(fe -> Optional.ofNullable(fe.getIncludeAnnual()).orElse(false)); - boolean includeTemporal = cohortCharacterization.getCcFeatureAnalyses().stream().anyMatch(CcFeAnalysisEntity::getIncludeTemporal); + boolean includeTemporal = cohortCharacterization.getCcFeatureAnalyses().stream() + .anyMatch(fe -> Optional.ofNullable(fe.getIncludeTemporal()).orElse(false)); CCQueryBuilder ccQueryBuilder = new CCQueryBuilder(cohortCharacterization, cohortTable, sessionId, SourceUtils.getCdmQualifier(source), SourceUtils.getResultsQualifier(source), SourceUtils.getVocabularyQualifier(source), tempSchema, jobId, includeAnnual, includeTemporal); From 7f7f196ed1479693183292614489675d040feba7 Mon Sep 17 00:00:00 2001 From: oleg-odysseus Date: Tue, 23 Jul 2024 20:55:48 +0200 Subject: [PATCH 15/15] Fixed NullPointerException during saving of a cohort characterization with missing design --- .../converter/FeAnalysisEntityToFeAnalysisDTOConverter.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/ohdsi/webapi/feanalysis/converter/FeAnalysisEntityToFeAnalysisDTOConverter.java b/src/main/java/org/ohdsi/webapi/feanalysis/converter/FeAnalysisEntityToFeAnalysisDTOConverter.java index 034796b232..5343f8863d 100644 --- a/src/main/java/org/ohdsi/webapi/feanalysis/converter/FeAnalysisEntityToFeAnalysisDTOConverter.java +++ b/src/main/java/org/ohdsi/webapi/feanalysis/converter/FeAnalysisEntityToFeAnalysisDTOConverter.java @@ -2,12 +2,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.collections4.CollectionUtils; import org.ohdsi.analysis.cohortcharacterization.design.FeatureAnalysisAggregate; import org.ohdsi.webapi.feanalysis.domain.*; import org.ohdsi.webapi.feanalysis.dto.*; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -52,6 +54,9 @@ private Object convertDesignToJson(final FeAnalysisEntity source) { switch (type) { case CRITERIA_SET: FeAnalysisWithCriteriaEntity sourceWithCriteria = (FeAnalysisWithCriteriaEntity) source; + if (CollectionUtils.isEmpty(sourceWithCriteria.getDesign())) { + return Collections.emptyList(); + } return sourceWithCriteria.getDesign() .stream() .map(this::convertCriteria)