Merge branch 'develop' into TASK-5914

.github/workflows/manual-deploy-docker.yml

@@ -45,7 +45,7 @@ jobs:
             ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} ${{ inputs.hadoop }}
           fi
       - name: Maven Build (skip tests)
-        run: mvn -T 2 clean install -DskipTests -P${{ inputs.hadoop }} -Dopencga.war.name=opencga -Dcheckstyle.skip
+        run: mvn -T 2 clean install -DskipTests -P${{ inputs.hadoop }} -Dopencga.war.name=opencga -Dcheckstyle.skip  --no-transfer-progress
       - uses: actions/upload-artifact@v4
         with:
           name: build-folder

.github/workflows/manual-deploy-ext-tools.yml

@@ -40,7 +40,7 @@ jobs:
             ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} ${{ inputs.hadoop }}
           fi
       - name: Maven Build (skip tests)
-        run: mvn -T 2 clean install -DskipTests
+        run: mvn -T 2 clean install -DskipTests  --no-transfer-progress
       - uses: actions/upload-artifact@v4
         with:
           name: build-folder

.github/workflows/manual-deploy-python-notebook.yml

@@ -0,0 +1,59 @@
+name: Reusable deploy Docker python-notebook
+run-name: 'Run deploy Docker python-notebook ${{ inputs.branch }} by @${{ github.actor }}'
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: "The branch, tag or SHA of the source code to build docker."
+        type: string
+        required: true
+      tag:
+        description: "The tag for the new docker."
+        type: string
+        required: true
+      hadoop:
+        type: string
+        description: 'Hadoop flavour. Any of: [hdp3.1, hdi5.1, emr6.1, emr6.13]'
+        required: false
+        default: "hdp3.1"
+      pyopencga_version:
+        type: string
+        description: 'PyOpenCGA version.'
+        required: true
+
+jobs:
+  build:
+    name: Build Java app
+    runs-on: ubuntu-22.04
+    outputs:
+      version: ${{ steps.get_project_version.outputs.version }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: '10'
+          ref: "${{ inputs.branch }}"
+      - name: Set up JDK 8
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '8'
+          cache: 'maven'
+      - name: Install dependencies branches
+        run: |
+          if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then
+            chmod +x ./.github/workflows/scripts/get_same_branch.sh
+            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} ${{ inputs.hadoop }}
+          fi
+      - name: Maven Build (skip tests)
+        run: mvn -T 2 clean install -DskipTests
+      - uses: actions/upload-artifact@v4
+        with:
+          name: build-folder
+          path: build
+
+  deploy-docker-python-notebook:
+    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+    needs: build
+    with:
+      cli: python3 ./build/cloud/docker/docker-build.py push --images python-notebook --tag ${{ inputs.tag }} --docker-build-args "--build-arg VERSION=${{ inputs.pyopencga_version }}"
+    secrets: inherit

.github/workflows/pull-request-approved.yml

@@ -26,6 +26,7 @@ jobs:
           echo "github.event.pull_request.head.ref: ${{ github.event.pull_request.head.ref }}"
           xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.base.ref }})
           echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY}
+          echo "xetabase_branch: ${xetabase_branch}"
           echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT
         env:
           ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}

.github/workflows/pull-request-merged.yml

@@ -7,18 +7,17 @@ on:
       - "release-*"
     types:
       - closed
-  workflow_dispatch:
 
 jobs:
-  build:
-    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
-    with:
-      maven_opts: -P hdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
-
   delete-docker:
-    uses: opencb/java-common-libs/.github/workflows/delete-docker-hub-workflow.yml@develop
-    needs: build
-    with:
-      cli: python3 ./build/cloud/docker/docker-build.py delete --images base --tag ${{ github.head_ref }}
-    secrets: inherit
+    name: Execute delete docker image
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: '10'
+      - name: delete docker images
+        run: |
+          echo "Deleting docker images"
+          python3 ./opencga-app/app/cloud/docker/docker-build.py delete --images base --tag ${{ github.head_ref }}
 

.github/workflows/release.yml

@@ -42,13 +42,20 @@ jobs:
   # Add the release job that depends on all the previous jobs
   release:
     uses: opencb/java-common-libs/.github/workflows/release-github-workflow.yml@develop
-    needs: [ build-hdp, deploy-docker-hdp, deploy-maven, deploy-python]
+    needs: [ build-hdp, deploy-docker-hdp, deploy-maven, deploy-python, deploy-docker-python-notebook]
     with:
       artifact: build-folder
       file: |
         opencga-client-${{ needs.build-hdp.outputs.version }}.tar.gz
         clients/R/opencgaR_${{ needs.build-hdp.outputs.version }}.tar.gz
 
+  deploy-docker-python-notebook:
+    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+    needs: [ build-hdp, deploy-python ]
+    with:
+      cli: python3 ./build/cloud/docker/docker-build.py push --images python-notebook --tag ${{ needs.build-hdp.outputs.version }} --docker-build-args "--build-arg VERSION=${{ needs.build-hdp.outputs.version }}"
+    secrets: inherit
+
   # Compile and deploy other hadoop flavours
   # Add the build-hdi and deploy-docker-hdi jobs
   build-hdi:

.github/workflows/scripts/get-xetabase-branch.sh

@@ -8,7 +8,7 @@ get_xetabase_branch() {
   # If the branch begins with 'TASK' and exists in the opencga-enterprise repository, I return it
   if [[ $input_branch == TASK* ]]; then
     if [ "$(git ls-remote "https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git" "$input_branch" )" ] ; then
-      echo $input_branch;
+      echo "$input_branch";
       return 0;
     fi
   fi

README.md

@@ -6,19 +6,13 @@ OpenCGA constitutes the big data analysis component of [OpenCB](http://www.openc
 ### Documentation
 You can find OpenCGA documentation and tutorials at: https://github.com/opencb/opencga/wiki.
 
-### Issues Tracking
-You can report bugs or request new features at [GitHub issue tracking](https://github.com/opencb/opencga/issues).
+### Feedback and Feature Requests
+Found a bug or have an idea for a new feature? Let us know at zettagenomics.com/academic.
 
 ### Release Notes and Roadmap
-Releases notes are available at [GitHub releases](https://github.com/opencb/opencga/releases).
-
-Roadmap is available at [GitHub milestones](https://github.com/opencb/opencga/milestones). You can report bugs or request new features at [GitHub issue tracking](https://github.com/opencb/opencga/issues).
-
-### Versioning
-OpenCGA is versioned following the rules from [Semantic versioning](https://semver.org/).
+Releases notes are available at https://zettagenomics.com/release-notes/
 
 ### Maintainers
-We recommend to contact OpenCGA developers by writing to OpenCB mailing list opencb@googlegroups.com. Current main developers and maintainers are:
 * Ignacio Medina (im411@cam.ac.uk) (_Founder and Project Leader_)
 * Jacobo Coll (jacobo.coll-moragon@genomicsengland.co.uk)
 * Pedro Furio (pedro.furio@genomicsengland.co.uk)
@@ -33,7 +27,7 @@ We recommend to contact OpenCGA developers by writing to OpenCB mailing list ope
 * Franscisco Salavert (fsalavert@cipf.es)
 
 ##### Contributing
-OpenCGA is an open-source and collaborative project. We appreciate any help and feedback from users, you can contribute in many different ways such as simple bug reporting and feature request. Dependending on your skills you are more than welcome to develop client tools, new features or even fixing bugs.
+OpenCGA is an open-source and collaborative project. We appreciate any help and feedback from users, you can contribute in many different ways such as simple bug reporting and feature request. Dependending on your skills you are more than welcome to develop client tools, new features, or even fix bugs.
 
 
 # How to build

opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java

@@ -23,7 +23,9 @@
 import org.opencb.opencga.core.exceptions.ToolException;
 import org.opencb.opencga.core.models.alignment.CoverageIndexParams;
 import org.opencb.opencga.core.models.common.Enums;
+import org.opencb.opencga.core.models.common.InternalStatus;
 import org.opencb.opencga.core.models.file.File;
+import org.opencb.opencga.core.models.file.FileInternalCoverageIndex;
 import org.opencb.opencga.core.tools.annotations.Tool;
 import org.opencb.opencga.core.tools.annotations.ToolParams;
 
@@ -40,7 +42,7 @@
 public class AlignmentCoverageAnalysis extends OpenCgaToolScopeStudy {
 
     public static final String ID = "coverage-index-run";
-    public static final String DESCRIPTION = "Compute the coverage from a given alignment file, e.g., create a "
+    public static final String DESCRIPTION = "Compute the coverage from a given BAM alignment file, e.g., create a "
             + AlignmentConstants.BIGWIG_EXTENSION + " file from a " + AlignmentConstants.BAM_EXTENSION + " file";
 
     @ToolParams
@@ -64,37 +66,36 @@ protected void check() throws Exception {
 
         //  Checking BAM file ID
         try {
-            bamCatalogFile = catalogManager.getFileManager().get(getStudy(), coverageParams.getBamFileId(), QueryOptions.empty(),
+            bamCatalogFile = catalogManager.getFileManager().get(getStudy(), coverageParams.getFileId(), QueryOptions.empty(),
                     getToken()).first();
             if (bamCatalogFile == null) {
-                throw new ToolException("Could not find BAM file from ID '" + coverageParams.getBamFileId() + "'");
+                throw new ToolException("Could not find BAM file from ID '" + coverageParams.getFileId() + "'");
             }
         } catch (Exception e) {
-            throw new ToolException("Could not get BAM file from ID " + coverageParams.getBamFileId());
+            throw new ToolException("Could not get BAM file from ID " + coverageParams.getFileId());
         }
 
         // Check if the input file is .bam
         if (!bamCatalogFile.getName().endsWith(AlignmentConstants.BAM_EXTENSION)) {
-            throw new ToolException("Invalid input alignment file '" + coverageParams.getBamFileId() + "' (" + bamCatalogFile.getName()
+            throw new ToolException("Invalid input alignment file '" + coverageParams.getFileId() + "' (" + bamCatalogFile.getName()
                     + "): it must be in BAM format");
         }
 
         // Getting BAI file
-        String baiFileId = coverageParams.getBaiFileId();
+        String baiFileId;
+        try {
+            baiFileId = bamCatalogFile.getInternal().getAlignment().getIndex().getFileId();
+        } catch (Exception e) {
+            throw new ToolException("Could not get internal alignment index file Id from BAM file ID '" + bamCatalogFile.getId() + "'");
+        }
         if (StringUtils.isEmpty(baiFileId)) {
-            // BAI file ID was not provided, looking for it
-            logger.info("BAI file ID was not provided, getting it from the internal alignment index of the BAM file ID {}",
-                    bamCatalogFile.getId());
-            try {
-                baiFileId = bamCatalogFile.getInternal().getAlignment().getIndex().getFileId();
-            } catch (Exception e) {
-                throw new ToolException("Could not get internal alignment index file Id from BAM file ID '" + bamCatalogFile.getId());
-            }
+            throw new ToolException("Could not find the alignment index file for the BAM file ID '" + bamCatalogFile.getId() + "'. Please,"
+                    + " create the alignment index file before computing the coverage");
         }
         try {
             baiCatalogFile = catalogManager.getFileManager().get(getStudy(), baiFileId, QueryOptions.empty(), getToken()).first();
             if (baiCatalogFile == null) {
-                throw new ToolException("Could not find BAI file from ID '" + coverageParams.getBaiFileId() + "'");
+                throw new ToolException("Could not find BAI file from ID '" + baiFileId + "'");
             }
         } catch (Exception e) {
             throw new ToolException("Could not get BAI file from file ID " + baiFileId);
@@ -115,6 +116,18 @@ protected void check() throws Exception {
             coverageParams.setWindowSize(Integer.parseInt(COVERAGE_WINDOW_SIZE_DEFAULT));
             logger.info("Window size is set to {}", coverageParams.getWindowSize());
         }
+
+        // Check overwrite
+        String bwFileId;
+        try {
+            bwFileId = bamCatalogFile.getInternal().getAlignment().getCoverage().getFileId();
+        } catch (Exception e) {
+            bwFileId = null;
+        }
+        if (StringUtils.isNotEmpty(bwFileId) && !coverageParams.isOverwrite()) {
+            throw new ToolException("Coverage file ID '" + bwFileId + "' already exists for file ID '" + bamCatalogFile.getId()
+                    + "'. To overwrite the BIGWIG file use the flag --overwrite");
+        }
     }
 
     @Override
@@ -192,7 +205,12 @@ protected void run() throws Exception {
             }
 
             // Link generated BIGWIG file and update samples info
-            AlignmentAnalysisUtils.linkAndUpdate(bamCatalogFile, bwPath, getJobId(), study, catalogManager, token);
+            File bwCatalogFile = AlignmentAnalysisUtils.linkAndUpdate(bamCatalogFile, bwPath, getJobId(), study, catalogManager, token);
+
+            // Update BAM file internal in order to set the coverage index (BIGWIG)
+            FileInternalCoverageIndex fileCoverageIndex = new FileInternalCoverageIndex(new InternalStatus(InternalStatus.READY),
+                    bwCatalogFile.getId(), "deeptools bamCoverage", coverageParams.getWindowSize());
+            catalogManager.getFileManager().updateFileInternalCoverageIndex(study, bamCatalogFile, fileCoverageIndex, token);
         });
     }
 }

opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentIndexOperation.java

@@ -19,15 +19,14 @@
 import org.apache.commons.lang3.StringUtils;
 import org.opencb.biodata.tools.alignment.BamManager;
 import org.opencb.commons.datastore.core.QueryOptions;
-import org.opencb.opencga.analysis.tools.OpenCgaTool;
 import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy;
 import org.opencb.opencga.catalog.exceptions.CatalogException;
 import org.opencb.opencga.core.exceptions.ToolException;
 import org.opencb.opencga.core.models.alignment.AlignmentIndexParams;
-import org.opencb.opencga.core.models.alignment.CoverageIndexParams;
 import org.opencb.opencga.core.models.common.Enums;
 import org.opencb.opencga.core.models.common.InternalStatus;
-import org.opencb.opencga.core.models.file.*;
+import org.opencb.opencga.core.models.file.File;
+import org.opencb.opencga.core.models.file.FileInternalAlignmentIndex;
 import org.opencb.opencga.core.response.OpenCGAResult;
 import org.opencb.opencga.core.tools.annotations.Tool;
 import org.opencb.opencga.core.tools.annotations.ToolParams;
@@ -40,7 +39,12 @@
 public class AlignmentIndexOperation extends OpenCgaToolScopeStudy {
 
     public static final String ID = "alignment-index-run";
-    public static final String DESCRIPTION = "Index a given alignment file, e.g., create a .bai file from a .bam file";
+    public static final String DESCRIPTION = "Index a given alignment file BAM/CRAM, e.g., create a " + AlignmentConstants.BAI_EXTENSION
+            + " file from a " + AlignmentConstants.BAM_EXTENSION + " file";
+
+    private String study;
+    private String inputFile;
+    private boolean overwrite = false;
 
     @ToolParams
     protected final AlignmentIndexParams indexParams = new AlignmentIndexParams();
@@ -81,6 +85,18 @@ protected void check() throws Exception {
             throw new ToolException("Invalid input alignment file '" + indexParams.getFileId() + "': it must be in BAM or CRAM format");
         }
 
+        // Check overwrite
+        String baiFileId;
+        try {
+            baiFileId = inputCatalogFile.getInternal().getAlignment().getIndex().getFileId();
+        } catch (Exception e) {
+            baiFileId = null;
+        }
+        if (StringUtils.isNotEmpty(baiFileId) && !overwrite) {
+            throw new ToolException("Alignment index file ID '" + baiFileId + "' already exists for file ID '" + inputCatalogFile.getId()
+                    + "'. To overwrite the alignment index file use the flag --overwrite");
+        }
+
         outputPath = getOutDir().resolve(filename + (filename.endsWith(AlignmentConstants.BAM_EXTENSION)
                 ? AlignmentConstants.BAI_EXTENSION : AlignmentConstants.CRAI_EXTENSION));
     }
@@ -131,4 +147,31 @@ protected void run() throws Exception {
             catalogManager.getFileManager().updateFileInternalAlignmentIndex(study, inputCatalogFile, fileAlignmentIndex, token);
         });
     }
+
+    public String getStudy() {
+        return study;
+    }
+
+    public AlignmentIndexOperation setStudy(String study) {
+        this.study = study;
+        return this;
+    }
+
+    public String getInputFile() {
+        return inputFile;
+    }
+
+    public AlignmentIndexOperation setInputFile(String inputFile) {
+        this.inputFile = inputFile;
+        return this;
+    }
+
+    public boolean isOverwrite() {
+        return overwrite;
+    }
+
+    public AlignmentIndexOperation setOverwrite(boolean overwrite) {
+        this.overwrite = overwrite;
+        return this;
+    }
 }

opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java

@@ -88,19 +88,6 @@ public AlignmentStorageManager(CatalogManager catalogManager, StorageEngineFacto
         initStatsMap();
     }
 
-//    //-------------------------------------------------------------------------
-//    // INDEX
-//    //-------------------------------------------------------------------------
-//
-//    public void index(String study, String inputFile, String outdir, String token) throws ToolException {
-//        ToolRunner toolRunner = new ToolRunner("", catalogManager, storageEngineFactory);
-//
-//        AlignmentIndexParams params = new AlignmentIndexParams();
-//        params.setFileId(inputFile);
-//        toolRunner.execute(AlignmentIndexOperation.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, study), Paths.get(outdir),
-//                jobId, token);
-//    }
-
     //-------------------------------------------------------------------------
     // QUERY
     //-------------------------------------------------------------------------