From b49926c534ff6473e4d0fdaeac4e5c77414dcb4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Fri, 27 Sep 2024 12:52:09 +0200 Subject: [PATCH 1/4] datastore: add a method to SolrManager to create a Solr client, #TASK-6981 --- .../commons/datastore/solr/SolrManager.java | 104 ++++++++++-------- 1 file changed, 58 insertions(+), 46 deletions(-) diff --git a/commons-datastore/commons-datastore-solr/src/main/java/org/opencb/commons/datastore/solr/SolrManager.java b/commons-datastore/commons-datastore-solr/src/main/java/org/opencb/commons/datastore/solr/SolrManager.java index d6258f88..ae6b974a 100644 --- a/commons-datastore/commons-datastore-solr/src/main/java/org/opencb/commons/datastore/solr/SolrManager.java +++ b/commons-datastore/commons-datastore-solr/src/main/java/org/opencb/commons/datastore/solr/SolrManager.java @@ -23,7 +23,10 @@ import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.impl.LBHttpSolrClient; -import org.apache.solr.client.solrj.request.*; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.client.solrj.request.CoreStatus; +import org.apache.solr.client.solrj.request.SolrPing; import org.apache.solr.client.solrj.response.SolrPingResponse; import org.apache.solr.common.SolrException; import org.slf4j.Logger; @@ -31,12 +34,13 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; public class SolrManager { - private String host; + private List hosts; private String mode; private SolrClient solrClient; @@ -65,51 +69,23 @@ public SolrManager(String host, String mode, int timeout) { * @param timeout Read timeout */ public SolrManager(List hosts, String mode, int timeout) { - hosts = hosts.stream().flatMap(s -> Arrays.stream(s.split(","))).collect(Collectors.toList()); - - this.host = String.join(",", hosts); + this.hosts = hosts.stream().flatMap(s -> Arrays.stream(s.split(","))).collect(Collectors.toList()); this.mode = mode; - - if (hosts.get(0).startsWith("http")) { - if (hosts.size() == 1) { - // Single HTTP endpoint. - this.solrClient = new HttpSolrClient.Builder(host).build(); - ((HttpSolrClient) this.solrClient).setRequestWriter(new BinaryRequestWriter()); - ((HttpSolrClient) this.solrClient).setSoTimeout(timeout); - } else { - // Use a LoadBalancer if there are multiple http hosts - this.solrClient = new LBHttpSolrClient.Builder().withBaseSolrUrls(hosts.toArray(new String[0])).build(); - - ((LBHttpSolrClient) this.solrClient).setRequestWriter(new BinaryRequestWriter()); - ((LBHttpSolrClient) this.solrClient).setSoTimeout(timeout); - } - } else { - // If the provided hosts are not http, assume zookeeper hosts like HOST:PORT - // This client will use Zookeeper to discover Solr endpoints for SolrCloud collections, and then use the - // LBHttpSolrClient to issue requests. - if (isCloud()) { - this.solrClient = new CloudSolrClient.Builder().withZkHost(hosts).build(); - - ((CloudSolrClient) this.solrClient).setRequestWriter(new BinaryRequestWriter()); - ((CloudSolrClient) this.solrClient).setSoTimeout(timeout); - } else { - throw new IllegalArgumentException("Can not initialize SolrManager from Zookeeper host not in Cloud mode"); - } - } + this.solrClient = newSolrClient(timeout); } public SolrManager(SolrClient solrClient, String host, String mode) { this.solrClient = solrClient; - this.host = host; + this.hosts = Collections.singletonList(host); this.mode = mode; } - @Deprecated - public SolrManager(SolrClient solrClient, String host, String mode, int timeout) { - this.solrClient = solrClient; - this.host = host; - this.mode = mode; - } +// @Deprecated +// public SolrManager(SolrClient solrClient, String host, String mode, int timeout) { +// this.solrClient = solrClient; +// this.hosts = Collectionshost; +// this.mode = mode; +// } public SolrCollection getCollection(String collection) throws SolrException { checkIsAlive(); @@ -188,7 +164,7 @@ public void create(String dbName, String configSet) throws SolrException { */ public void createCore(String coreName, String configSet) throws SolrException { try { - logger.debug("Creating core: host={}, core={}, configSet={}", host, coreName, configSet); + logger.debug("Creating core: host={}, core={}, configSet={}", StringUtils.join(",", hosts), coreName, configSet); CoreAdminRequest.Create request = new CoreAdminRequest.Create(); request.setCoreName(coreName); request.setConfigSet(configSet); @@ -210,7 +186,7 @@ public void createCore(String coreName, String configSet) throws SolrException { */ public void createCollection(String collectionName, String configSet) throws SolrException { logger.debug("Creating collection: host={}, collection={}, config={}, numShards={}, numReplicas={}", - host, collectionName, configSet, 1, 1); + StringUtils.join(",", hosts), collectionName, configSet, 1, 1); try { CollectionAdminRequest request = CollectionAdminRequest.createCollection(collectionName, configSet, 1, 1); request.process(solrClient); @@ -353,6 +329,42 @@ public void close() throws IOException { } } + public SolrClient newSolrClient(int timeout) { + SolrClient solrClient; + if (hosts.get(0).startsWith("http")) { + if (hosts.size() == 1) { + // Single HTTP endpoint. + solrClient = new HttpSolrClient.Builder(hosts.get(0)).build(); + ((HttpSolrClient) solrClient).setRequestWriter(new BinaryRequestWriter()); + ((HttpSolrClient) solrClient).setSoTimeout(timeout); + } else { + // Use a LoadBalancer if there are multiple http hosts + solrClient = new LBHttpSolrClient.Builder().withBaseSolrUrls(hosts.toArray(new String[0])).build(); + + ((LBHttpSolrClient) solrClient).setRequestWriter(new BinaryRequestWriter()); + ((LBHttpSolrClient) solrClient).setSoTimeout(timeout); + } + } else { + // If the provided hosts are not http, assume zookeeper hosts like HOST:PORT + // This client will use Zookeeper to discover Solr endpoints for SolrCloud collections, and then use the + // LBHttpSolrClient to issue requests. + if (isCloud()) { + solrClient = new CloudSolrClient.Builder().withZkHost(hosts).build(); + + ((CloudSolrClient) solrClient).setRequestWriter(new BinaryRequestWriter()); + ((CloudSolrClient) solrClient).setSoTimeout(timeout); + } else { + throw new IllegalArgumentException("Can not initialize SolrManager from Zookeeper host not in Cloud mode"); + } + } + + // Sanity check + if (solrClient == null) { + throw new IllegalArgumentException("Can not build a Solr client, please, check Solr configuration"); + } + return solrClient; + } + private boolean isCloud() { if (StringUtils.isEmpty(mode)) { logger.warn("Solr 'mode' is empty, setting default 'cloud'"); @@ -376,19 +388,19 @@ private boolean isCloud() { @Override public String toString() { final StringBuilder sb = new StringBuilder("SolrManager{"); - sb.append("host='").append(host).append('\''); + sb.append("hosts='").append(hosts).append('\''); sb.append(", mode='").append(mode).append('\''); sb.append(", solrClient=").append(solrClient); sb.append('}'); return sb.toString(); } - public String getHost() { - return host; + public List getHosts() { + return hosts; } - public SolrManager setHost(String host) { - this.host = host; + public SolrManager setHosts(List hosts) { + this.hosts = hosts; return this; } From 444088dbe59de9383fa6093ade20960886e726f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Mon, 30 Sep 2024 10:43:49 +0200 Subject: [PATCH 2/4] datastore: use final for solrClient, #TASK-6981 --- .../commons/datastore/solr/SolrManager.java | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/commons-datastore/commons-datastore-solr/src/main/java/org/opencb/commons/datastore/solr/SolrManager.java b/commons-datastore/commons-datastore-solr/src/main/java/org/opencb/commons/datastore/solr/SolrManager.java index ae6b974a..24a7fe8d 100644 --- a/commons-datastore/commons-datastore-solr/src/main/java/org/opencb/commons/datastore/solr/SolrManager.java +++ b/commons-datastore/commons-datastore-solr/src/main/java/org/opencb/commons/datastore/solr/SolrManager.java @@ -42,7 +42,7 @@ public class SolrManager { private List hosts; private String mode; - private SolrClient solrClient; + private final SolrClient solrClient; private final Logger logger = LoggerFactory.getLogger(SolrManager.class); @@ -80,13 +80,6 @@ public SolrManager(SolrClient solrClient, String host, String mode) { this.mode = mode; } -// @Deprecated -// public SolrManager(SolrClient solrClient, String host, String mode, int timeout) { -// this.solrClient = solrClient; -// this.hosts = Collectionshost; -// this.mode = mode; -// } - public SolrCollection getCollection(String collection) throws SolrException { checkIsAlive(); if (!exists(collection)) { @@ -358,10 +351,6 @@ public SolrClient newSolrClient(int timeout) { } } - // Sanity check - if (solrClient == null) { - throw new IllegalArgumentException("Can not build a Solr client, please, check Solr configuration"); - } return solrClient; } @@ -416,9 +405,4 @@ public SolrManager setMode(String mode) { public SolrClient getSolrClient() { return solrClient; } - - public SolrManager setSolrClient(SolrClient solrClient) { - this.solrClient = solrClient; - return this; - } } From 179f751b857555b16a02bafc4eb32563d818945a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Wed, 2 Oct 2024 12:33:18 +0200 Subject: [PATCH 3/4] datastore: use final in SolrManager members, #TASK-6981 --- .../commons/datastore/solr/SolrManager.java | 20 +++---------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/commons-datastore/commons-datastore-solr/src/main/java/org/opencb/commons/datastore/solr/SolrManager.java b/commons-datastore/commons-datastore-solr/src/main/java/org/opencb/commons/datastore/solr/SolrManager.java index 24a7fe8d..15f65bf0 100644 --- a/commons-datastore/commons-datastore-solr/src/main/java/org/opencb/commons/datastore/solr/SolrManager.java +++ b/commons-datastore/commons-datastore-solr/src/main/java/org/opencb/commons/datastore/solr/SolrManager.java @@ -40,8 +40,8 @@ public class SolrManager { - private List hosts; - private String mode; + private final List hosts; + private final String mode; private final SolrClient solrClient; private final Logger logger = LoggerFactory.getLogger(SolrManager.class); @@ -323,7 +323,7 @@ public void close() throws IOException { } public SolrClient newSolrClient(int timeout) { - SolrClient solrClient; + final SolrClient solrClient; if (hosts.get(0).startsWith("http")) { if (hosts.size() == 1) { // Single HTTP endpoint. @@ -355,10 +355,6 @@ public SolrClient newSolrClient(int timeout) { } private boolean isCloud() { - if (StringUtils.isEmpty(mode)) { - logger.warn("Solr 'mode' is empty, setting default 'cloud'"); - mode = "cloud"; - } switch (mode.toLowerCase()) { case "collection": case "cloud": { @@ -388,20 +384,10 @@ public List getHosts() { return hosts; } - public SolrManager setHosts(List hosts) { - this.hosts = hosts; - return this; - } - public String getMode() { return mode; } - public SolrManager setMode(String mode) { - this.mode = mode; - return this; - } - public SolrClient getSolrClient() { return solrClient; } From 51748f267c9e8faa04093b21caec128b996d6756 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Wed, 2 Oct 2024 14:17:50 +0200 Subject: [PATCH 4/4] cicd: Fix calculating opencga branch in pull request approve workflow #TASK-6981 --- .github/workflows/test-xetabase-workflow.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index ed802879..57445f9a 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -51,13 +51,18 @@ jobs: path: opencga-enterprise fetch-depth: "10" - id: get_opencga_branch - name: Get OpenCGA branch from 'pom.xml' property + name: Get OpenCGA branch run: | - pwd - ls -lrtha - ls -lrtha ./opencga-enterprise - chmod +x ./opencga-enterprise/.github/workflows/scripts/get-opencga-xetabase-branch.sh - opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/get-opencga-xetabase-branch.sh) + # If the task exists in the opencga repository, this is the branch to be tested + if [[ "${{ inputs.task }}" == TASK* ]]; then + if [ "$(git ls-remote "https://github.com/opencb/opencga.git" "${{ inputs.task }}" ] ; then + opencga_branch="${{ inputs.task }}"; + return 0; + fi + else + chmod +x ./opencga-enterprise/.github/workflows/scripts/get-opencga-xetabase-branch.sh + opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/get-opencga-xetabase-branch.sh) + fi echo "opencga_branch=${opencga_branch}" echo "opencga_branch=${opencga_branch}" >> $GITHUB_OUTPUT - name: Clone OpenCGA branch '${{ steps.get_opencga_branch.outputs.opencga_branch }}'