Merge branch 'release-2.8.x' into TASK-4684

.github/workflows/test-analysis.yml

@@ -24,7 +24,7 @@ on:
       SSH_TESTING_SERVER_PASSWORD:
         required: true
 env:
-  xb_version: "1.6.3"
+  xb_version: "1.6.4"
 jobs:
   analysis:
     name: Execute Sonar Analysis
@@ -77,7 +77,7 @@ jobs:
     needs: test
     strategy:
       matrix:
-        module: [ "opencga-app", "opencga-catalog", "opencga-client", "opencga-clinical", "opencga-core", "opencga-master", "opencga-server", "opencga-storage", "opencga-storage/opencga-storage-app", "opencga-storage/opencga-storage-benchmark", "opencga-storage/opencga-storage-core", "opencga-storage/opencga-storage-hadoop", "opencga-storage/opencga-storage-server", "opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core", "opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps" ]
+        module: ["opencga-app", "opencga-catalog", "opencga-client", "opencga-clinical", "opencga-core", "opencga-master", "opencga-server", "opencga-storage", "opencga-storage/opencga-storage-app", "opencga-storage/opencga-storage-benchmark", "opencga-storage/opencga-storage-core", "opencga-storage/opencga-storage-hadoop", "opencga-storage/opencga-storage-server", "opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core", "opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps"]
     steps:
       - name: Download result dir
         uses: actions/download-artifact@v3

opencga-analysis/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga</artifactId>
-        <version>2.8.3-SNAPSHOT</version>
+        <version>2.8.4-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/InterpretationAnalysis.java

@@ -79,8 +79,8 @@ protected InterpretationMethod getInterpretationMethod(String name) {
                 .setName(name)
                 .setDependencies(Collections.singletonList(new Software()
                         .setName("OpenCGA")
-                        .setVersion(GitRepositoryState.get().getBuildVersion())
-                        .setCommit(GitRepositoryState.get().getCommitId())));
+                        .setVersion(GitRepositoryState.getInstance().getBuildVersion())
+                        .setCommit(GitRepositoryState.getInstance().getCommitId())));
         return method;
     }
 
@@ -114,8 +114,8 @@ protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnaly
 
         // Interpretation method
         InterpretationMethod method = new InterpretationMethod(getId(), null, null,
-                Collections.singletonList(new Software().setName("OpenCGA").setVersion(GitRepositoryState.get().getBuildVersion())
-                        .setCommit(GitRepositoryState.get().getCommitId())));
+                Collections.singletonList(new Software().setName("OpenCGA").setVersion(GitRepositoryState.getInstance().getBuildVersion())
+                        .setCommit(GitRepositoryState.getInstance().getCommitId())));
 
         // Analyst
         ClinicalAnalyst analyst = clinicalInterpretationManager.getAnalyst(token);

opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java

@@ -137,8 +137,8 @@ protected void run() throws ToolException {
     protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnalysis) throws ToolException, StorageEngineException,
             CatalogException, IOException {
         // Interpretation method
-        InterpretationMethod method = new InterpretationMethod(getId(), GitRepositoryState.get().getBuildVersion(),
-                GitRepositoryState.get().getCommitId(), Collections.singletonList(
+        InterpretationMethod method = new InterpretationMethod(getId(), GitRepositoryState.getInstance().getBuildVersion(),
+                GitRepositoryState.getInstance().getCommitId(), Collections.singletonList(
                 new Software()
                         .setName("Exomiser")
                         .setRepository("Docker: " + ExomiserWrapperAnalysisExecutor.DOCKER_IMAGE_NAME)

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/circos/CircosLocalAnalysisExecutor.java

@@ -58,7 +58,7 @@
 public class CircosLocalAnalysisExecutor extends CircosAnalysisExecutor implements StorageToolExecutor {
 
     public final static String R_DOCKER_IMAGE = "opencb/opencga-ext-tools:"
-            + GitRepositoryState.get().getBuildVersion();
+            + GitRepositoryState.getInstance().getBuildVersion();
     private VariantStorageManager storageManager;
 
     private File snvsFile;

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/genomePlot/GenomePlotLocalAnalysisExecutor.java

@@ -61,7 +61,7 @@
 public class GenomePlotLocalAnalysisExecutor extends GenomePlotAnalysisExecutor implements StorageToolExecutor {
 
     public final static String R_DOCKER_IMAGE = "opencb/opencga-ext-tools:"
-            + GitRepositoryState.get().getBuildVersion();
+            + GitRepositoryState.getInstance().getBuildVersion();
 
     private GenomePlotConfig plotConfig;
 

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/hrdetect/HRDetectLocalAnalysisExecutor.java

@@ -16,31 +16,21 @@
 
 package org.opencb.opencga.analysis.variant.hrdetect;
 
-import htsjdk.samtools.reference.BlockCompressedIndexedFastaSequenceFile;
-import htsjdk.samtools.reference.FastaSequenceIndex;
-import htsjdk.samtools.reference.ReferenceSequence;
-import htsjdk.samtools.util.GZIIndex;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.opencb.biodata.models.variant.StudyEntry;
 import org.opencb.biodata.models.variant.Variant;
-import org.opencb.biodata.models.variant.avro.VariantType;
 import org.opencb.commons.datastore.core.Query;
 import org.opencb.commons.datastore.core.QueryOptions;
-import org.opencb.commons.datastore.core.QueryResultWriter;
 import org.opencb.commons.exec.Command;
 import org.opencb.commons.utils.DockerUtils;
-import org.opencb.opencga.analysis.ResourceUtils;
 import org.opencb.opencga.analysis.StorageToolExecutor;
 import org.opencb.opencga.catalog.exceptions.CatalogException;
 import org.opencb.opencga.core.common.GitRepositoryState;
 import org.opencb.opencga.core.exceptions.ToolException;
 import org.opencb.opencga.core.exceptions.ToolExecutorException;
-import org.opencb.opencga.core.response.OpenCGAResult;
-import org.opencb.opencga.core.response.VariantQueryResult;
 import org.opencb.opencga.core.tools.annotations.ToolExecutor;
 import org.opencb.opencga.core.tools.variant.HRDetectAnalysisExecutor;
-import org.opencb.opencga.core.tools.variant.MutationalSignatureAnalysisExecutor;
 import org.opencb.opencga.storage.core.exceptions.StorageEngineException;
 import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam;
 import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator;
@@ -53,14 +43,12 @@
 import java.nio.file.Paths;
 import java.util.*;
 
-import static org.opencb.opencga.analysis.variant.mutationalSignature.MutationalSignatureAnalysis.CATALOGUES_FILENAME_DEFAULT;
-
 @ToolExecutor(id="opencga-local", tool = HRDetectAnalysis.ID,
         framework = ToolExecutor.Framework.LOCAL, source = ToolExecutor.Source.STORAGE)
 public class HRDetectLocalAnalysisExecutor extends HRDetectAnalysisExecutor
         implements StorageToolExecutor {
 
-    public final static String R_DOCKER_IMAGE = "opencb/opencga-ext-tools:" + GitRepositoryState.get().getBuildVersion();
+    public final static String R_DOCKER_IMAGE = "opencb/opencga-ext-tools:" + GitRepositoryState.getInstance().getBuildVersion();
 
     private final static String CNV_FILENAME = "cnv.tsv";
     private final static String INDEL_FILENAME = "indel.vcf";

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java

@@ -49,6 +49,7 @@
 import org.opencb.opencga.catalog.managers.StudyManager;
 import org.opencb.opencga.core.api.ParamConstants;
 import org.opencb.opencga.core.cellbase.CellBaseValidator;
+import org.opencb.opencga.core.common.ExceptionUtils;
 import org.opencb.opencga.core.common.UriUtils;
 import org.opencb.opencga.core.config.storage.CellBaseConfiguration;
 import org.opencb.opencga.core.config.storage.SampleIndexConfiguration;
@@ -564,7 +565,10 @@ public OpenCGAResult<Job> setCellbaseConfiguration(String project, CellBaseConfi
                                                        String annotationSaveId, String token)
             throws CatalogException, StorageEngineException {
         StopWatch stopwatch = StopWatch.createStarted();
-        return secureOperationByProject("configureCellbase", project, new ObjectMap(), token, engine -> {
+        return secureOperationByProject("configureCellbase", project, new ObjectMap()
+                .append("cellbaseConfiguration", cellbaseConfiguration)
+                .append("annotate", annotate)
+                .append("annotationSaveId", annotationSaveId), token, engine -> {
             OpenCGAResult<Job> result = new OpenCGAResult<>();
             result.setResultType(Job.class.getCanonicalName());
             result.setResults(new ArrayList<>());
@@ -1225,7 +1229,7 @@ private <R> R secureTool(String toolId, boolean isOperation, ObjectMap params, S
             throw e;
         } catch (Exception e) {
             exception = e;
-            throw new StorageEngineException("Error executing operation " + toolId, e);
+            throw new StorageEngineException("Error executing operation '" + toolId + "' : " + e.getMessage(), e);
         } finally {
             if (result instanceof DataResult) {
                 auditAttributes.append("dbTime", ((DataResult) result).getTime());
@@ -1237,6 +1241,8 @@ private <R> R secureTool(String toolId, boolean isOperation, ObjectMap params, S
             if (exception != null) {
                 auditAttributes.append("errorType", exception.getClass());
                 auditAttributes.append("errorMessage", exception.getMessage());
+                auditAttributes.append("errorMessageFull", ExceptionUtils.prettyExceptionMessage(exception, false, true));
+                auditAttributes.append("exceptionStackTrace", ExceptionUtils.prettyExceptionStackTrace(exception));
                 status = new AuditRecord.Status(AuditRecord.Status.Result.ERROR,
                         new Error(-1, exception.getClass().getName(), exception.getMessage()));
             } else {

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java

@@ -508,6 +508,16 @@ protected boolean synchronizeFiles(StudyMetadata study, List<File> files, String
             }
             fileSamplesMap.put(fileMetadata.getName(), samples);
             allSamples.addAll(fileMetadata.getSamples());
+            if (samples.size() > 100) {
+                // Try to reuse value.
+                // If the file holds more than 100 samples, it's most likely this same set of samples is already present
+                for (Set<String> value : fileSamplesMap.values()) {
+                    if (value.equals(samples)) {
+                        fileSamplesMap.put(fileMetadata.getName(), value);
+                        break;
+                    }
+                }
+            }
         }
 
         if (!indexedFilesFromStorage.isEmpty()) {

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java

@@ -64,11 +64,12 @@
 public class MutationalSignatureLocalAnalysisExecutor extends MutationalSignatureAnalysisExecutor
         implements StorageToolExecutor {
 
-    public static final String R_DOCKER_IMAGE = "opencb/opencga-ext-tools:" + GitRepositoryState.get().getBuildVersion();
-
     private static final String SVCLASS = "SVCLASS";
     private static final String EXT_SVTYPE = "EXT_SVTYPE";
 
+    public final static String R_DOCKER_IMAGE = "opencb/opencga-ext-tools:"
+            + GitRepositoryState.getInstance().getBuildVersion();
+
     private Path opencgaHome;
 
     private Logger logger = LoggerFactory.getLogger(this.getClass());
@@ -591,7 +592,9 @@ private void computeSignatureFitting() throws IOException, ToolException, Catalo
         StringBuilder scriptParams = new StringBuilder("R CMD Rscript --vanilla ")
                 .append("/opt/opencga/signature.tools.lib/scripts/signatureFit")
                 .append(" --catalogues=/data/input/").append(cataloguesFile.getName())
-                .append(" --outdir=/data/output");
+                .append(" --outdir=/data/output")
+                .append(" --commonsigtier=T2");
+
         if (StringUtils.isNotEmpty(getFitMethod())) {
             scriptParams.append(" --fitmethod=").append(getFitMethod());
         }

opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java

@@ -32,7 +32,7 @@ public String getDockerImageName() {
     }
 
     public String getDockerImageVersion() {
-        return GitRepositoryState.get().getBuildVersion();
+        return GitRepositoryState.getInstance().getBuildVersion();
     }
 
     private Logger privateLogger = LoggerFactory.getLogger(DockerWrapperAnalysisExecutor.class);

opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaSolrExtenalResource.java

@@ -50,8 +50,8 @@ protected void before() throws Throwable {
         Path rootDir = Paths.get("target/test-data", "junit-rga-solr-" + TimeUtils.getTimeMillis());
         Files.createDirectories(rootDir);
 
-        String mainConfigSet = "opencga-rga-configset-" + GitRepositoryState.get().getBuildVersion();
-        String auxConfigSet = "opencga-rga-aux-configset-" + GitRepositoryState.get().getBuildVersion();
+        String mainConfigSet = "opencga-rga-configset-" + GitRepositoryState.getInstance().getBuildVersion();
+        String auxConfigSet = "opencga-rga-aux-configset-" + GitRepositoryState.getInstance().getBuildVersion();
         copyConfigSetConfiguration(mainConfigSet, "managed-schema", rootDir);
         copyConfigSetConfiguration(auxConfigSet, "aux-managed-schema", rootDir);
 

opencga-app/app/cloud/docker/opencga-ext-tools/Dockerfile

@@ -17,14 +17,15 @@ RUN apt-get update -y && DEBIAN_FRONTEND="noninteractive" TZ="Europe/London" apt
     ## Installation dependencies using R install.packages() is slower than apt-get but final size is 400GB smaller.
     R -e "install.packages(c('BiocManager', 'RCircos', 'nnls', 'ggplot2', 'jsonlite', 'optparse', 'knitr', 'configr', 'dplyr', 'rmarkdown', 'tidyr', 'httr', 'kinship2', 'limSolve'))" && \
     R -e "BiocManager::install('BiocStyle')" && \
+    R -e "BiocManager::install('BSgenome.Hsapiens.UCSC.hg38')" && \
     ## signature.tools.lib installation \
-    R -e 'install.packages(c("devtools", "getopt"), repos="https://www.stats.bris.ac.uk/R/")' && \
+    R -e 'install.packages(c("devtools", "getopt"), repos=c("http://cran.rstudio.com/", "https://www.stats.bris.ac.uk/R/"))' && \
     git clone https://github.com/Nik-Zainal-Group/signature.tools.lib.git /opt/opencga/signature.tools.lib
 
 WORKDIR /opt/opencga/signature.tools.lib
 
 RUN git fetch origin --tags && \
-    git checkout tags/v2.4.1 && \
+    git checkout tags/v2.4.2 && \
     sed -i '/Mmusculus/d' DESCRIPTION && \
     sed -i '/Cfamiliaris/d' DESCRIPTION && \
     sed -i '/1000genomes/d' DESCRIPTION && \

opencga-app/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga</artifactId>
-        <version>2.8.3-SNAPSHOT</version>
+        <version>2.8.4-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
@@ -306,7 +306,7 @@
                                     <arg value="${build.dir}/misc/git"/>
                                 </exec>
                                 <exec executable="cp">
-                                    <arg value="${project.basedir}/../opencga-core/target/generated-resources/git.properties"/>
+                                    <arg value="${project.basedir}/../opencga-core/target/generated-resources/org/opencb/opencga/core/git.properties"/>
                                     <arg value="${build.dir}/misc/git"/>
                                 </exec>
 

opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/AdminCliOptionsParser.java

@@ -800,8 +800,8 @@ public void printUsage() {
         if (parsedCommand.isEmpty()) {
             System.err.println("");
             System.err.println("Program:     OpenCGA Admin (OpenCB)");
-            System.err.println("Version:     " + GitRepositoryState.get().getBuildVersion());
-            System.err.println("Git commit:  " + GitRepositoryState.get().getCommitId());
+            System.err.println("Version:     " + GitRepositoryState.getInstance().getBuildVersion());
+            System.err.println("Git commit:  " + GitRepositoryState.getInstance().getCommitId());
             System.err.println("Description: Big Data platform for processing and analysing NGS data");
             System.err.println("");
             System.err.println("Usage:       opencga-admin.sh [-h|--help] [--version] <command> [options]");

opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/AdminMain.java

@@ -30,7 +30,7 @@
  */
 public class AdminMain {
 
-    public static final String VERSION = GitRepositoryState.get().getBuildVersion();
+    public static final String VERSION = GitRepositoryState.getInstance().getBuildVersion();
 
     public static void main(String[] args) {
 
@@ -71,8 +71,8 @@ public static void main(String[] args) {
         String parsedCommand = cliOptionsParser.getCommand();
         if (parsedCommand == null || parsedCommand.isEmpty()) {
             if (cliOptionsParser.getGeneralOptions().version) {
-                System.out.println("Version " + GitRepositoryState.get().getBuildVersion());
-                System.out.println("Git version: " + GitRepositoryState.get().getBranch() + " " + GitRepositoryState.get().getCommitId());
+                System.out.println("Version " + GitRepositoryState.getInstance().getBuildVersion());
+                System.out.println("Git version: " + GitRepositoryState.getInstance().getBranch() + " " + GitRepositoryState.getInstance().getCommitId());
                 System.exit(0);
             } else if (cliOptionsParser.getGeneralOptions().help) {
                 cliOptionsParser.printUsage();

opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/MigrationCommandExecutor.java

@@ -167,7 +167,7 @@ private String parseVersion(String version) {
 
     public static String getDefaultVersion() {
         String version;
-        version = GitRepositoryState.get().getBuildVersion();
+        version = GitRepositoryState.getInstance().getBuildVersion();
         // Remove extra information
         version = version.split("-")[0];
         return version;

opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalCliOptionsParser.java

@@ -468,8 +468,8 @@ public void printUsage() {
         if (parsedCommand.isEmpty()) {
             System.err.println("");
             System.err.println("Program:     OpenCGA Analysis (OpenCB)");
-            System.err.println("Version:     " + GitRepositoryState.get().getBuildVersion());
-            System.err.println("Git commit:  " + GitRepositoryState.get().getCommitId());
+            System.err.println("Version:     " + GitRepositoryState.getInstance().getBuildVersion());
+            System.err.println("Git commit:  " + GitRepositoryState.getInstance().getCommitId());
             System.err.println("Description: Big Data platform for processing and analysing NGS data");
             System.err.println("");
             System.err.println("Usage:       opencga-internal.sh [-h|--help] [--version] <command> [options]");

opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalMain.java

@@ -28,7 +28,7 @@
  */
 public class InternalMain {
 
-    public static final String VERSION = GitRepositoryState.get().getBuildVersion();
+    public static final String VERSION = GitRepositoryState.getInstance().getBuildVersion();
 
     public static void main(String[] args) {
         System.exit(privateMain(args));
@@ -48,8 +48,8 @@ public static int privateMain(String[] args) {
         String parsedCommand = cliOptionsParser.getCommand();
         if (parsedCommand == null || parsedCommand.isEmpty()) {
             if (cliOptionsParser.getGeneralOptions().version) {
-                System.out.println("Version " + GitRepositoryState.get().getBuildVersion());
-                System.out.println("Git version: " + GitRepositoryState.get().getBranch() + " " + GitRepositoryState.get().getCommitId());
+                System.out.println("Version " + GitRepositoryState.getInstance().getBuildVersion());
+                System.out.println("Git version: " + GitRepositoryState.getInstance().getBranch() + " " + GitRepositoryState.getInstance().getCommitId());
                 return 0;
             } else if (cliOptionsParser.getGeneralOptions().help) {
                 cliOptionsParser.printUsage();